"Fossies" - the Fresh Open Source Software Archive

Member "neutron-14.0.3/neutron/tests/functional/agent/l3/test_ha_router.py" (22 Oct 2019, 23236 Bytes) of package /linux/misc/openstack/neutron-14.0.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_ha_router.py": 14.0.2_vs_14.0.3.

    1 # Copyright (c) 2014 Red Hat, Inc.
    2 # All Rights Reserved.
    3 #
    4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    5 #    not use this file except in compliance with the License. You may obtain
    6 #    a copy of the License at
    7 #
    8 #         http://www.apache.org/licenses/LICENSE-2.0
    9 #
   10 #    Unless required by applicable law or agreed to in writing, software
   11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   13 #    License for the specific language governing permissions and limitations
   14 #    under the License.
   15 
   16 import copy
   17 
   18 import mock
   19 from neutron_lib import constants
   20 import testtools
   21 
   22 from neutron.agent.common import ovs_lib
   23 from neutron.agent.l3 import agent as neutron_l3_agent
   24 from neutron.agent.linux import ip_lib
   25 from neutron.common import ipv6_utils
   26 from neutron.common import utils as common_utils
   27 from neutron.tests.common import l3_test_common
   28 from neutron.tests.common import net_helpers
   29 from neutron.tests.functional.agent.l3 import framework
   30 
   31 
   32 class L3HATestCase(framework.L3AgentTestFramework):
   33 
   34     def test_ha_router_update_floatingip_statuses(self):
   35         self._test_update_floatingip_statuses(
   36             self.generate_router_info(enable_ha=True))
   37 
   38     def test_keepalived_state_change_notification(self):
   39         enqueue_mock = mock.patch.object(
   40             self.agent, 'enqueue_state_change',
   41             side_effect=self.change_router_state).start()
   42         router_info = self.generate_router_info(enable_ha=True)
   43         router = self.manage_router(self.agent, router_info)
   44         common_utils.wait_until_true(lambda: router.ha_state == 'master')
   45 
   46         self.fail_ha_router(router)
   47         common_utils.wait_until_true(lambda: router.ha_state == 'backup')
   48 
   49         common_utils.wait_until_true(lambda: enqueue_mock.call_count == 3)
   50         calls = [args[0] for args in enqueue_mock.call_args_list]
   51         self.assertEqual((router.router_id, 'backup'), calls[0])
   52         self.assertEqual((router.router_id, 'master'), calls[1])
   53         self.assertEqual((router.router_id, 'backup'), calls[2])
   54 
   55     def _expected_rpc_report(self, expected):
   56         calls = (args[0][1] for args in
   57                  self.agent.plugin_rpc.update_ha_routers_states.call_args_list)
   58 
   59         # Get the last state reported for each router
   60         actual_router_states = {}
   61         for call in calls:
   62             for router_id, state in call.items():
   63                 actual_router_states[router_id] = state
   64 
   65         return actual_router_states == expected
   66 
   67     def test_keepalived_state_change_bulk_rpc(self):
   68         router_info = self.generate_router_info(enable_ha=True)
   69         router1 = self.manage_router(self.agent, router_info)
   70         self.fail_ha_router(router1)
   71         router_info = self.generate_router_info(enable_ha=True)
   72         router2 = self.manage_router(self.agent, router_info)
   73 
   74         common_utils.wait_until_true(lambda: router1.ha_state == 'backup')
   75         common_utils.wait_until_true(lambda: router2.ha_state == 'master')
   76         common_utils.wait_until_true(
   77             lambda: self._expected_rpc_report(
   78                 {router1.router_id: 'standby', router2.router_id: 'active'}))
   79 
   80     def test_ha_router_lifecycle(self):
   81         router_info = self._router_lifecycle(enable_ha=True)
   82         # ensure everything was cleaned up
   83         self._router_lifecycle(enable_ha=True, router_info=router_info)
   84 
   85     def test_conntrack_disassociate_fip_ha_router(self):
   86         self._test_conntrack_disassociate_fip(ha=True)
   87 
   88     def test_ipv6_ha_router_lifecycle(self):
   89         self._router_lifecycle(enable_ha=True,
   90                                ip_version=constants.IP_VERSION_6)
   91 
   92     def test_ipv6_ha_router_lifecycle_with_no_gw_subnet(self):
   93         self.agent.conf.set_override('ipv6_gateway',
   94                                      'fe80::f816:3eff:fe2e:1')
   95         self._router_lifecycle(enable_ha=True,
   96                                ip_version=constants.IP_VERSION_6,
   97                                v6_ext_gw_with_sub=False)
   98 
   99     def test_ipv6_ha_router_lifecycle_with_no_gw_subnet_for_router_advts(self):
  100         # Verify that router gw interface is configured to receive Router
  101         # Advts from upstream router when no external gateway is configured.
  102         self._router_lifecycle(enable_ha=True, dual_stack=True,
  103                                v6_ext_gw_with_sub=False)
  104 
  105     def _test_ipv6_router_advts_and_fwd_helper(self, state, enable_v6_gw,
  106                                                expected_ra,
  107                                                expected_forwarding):
  108         # Schedule router to l3 agent, and then add router gateway. Verify
  109         # that router gw interface is configured to receive Router Advts and
  110         # IPv6 forwarding is enabled.
  111         router_info = l3_test_common.prepare_router_data(
  112             enable_snat=True, enable_ha=True, dual_stack=True, enable_gw=False)
  113         router = self.manage_router(self.agent, router_info)
  114         common_utils.wait_until_true(lambda: router.ha_state == 'master')
  115         if state == 'backup':
  116             self.fail_ha_router(router)
  117             common_utils.wait_until_true(lambda: router.ha_state == 'backup')
  118         _ext_dev_name, ex_port = l3_test_common.prepare_ext_gw_test(
  119             mock.Mock(), router, dual_stack=enable_v6_gw)
  120         router_info['gw_port'] = ex_port
  121         router.process()
  122         self._assert_ipv6_accept_ra(router, expected_ra)
  123         # As router is going first to master and than to backup mode,
  124         # ipv6_forwarding should be enabled on "all" interface always after
  125         # that transition
  126         self._assert_ipv6_forwarding(router, expected_forwarding,
  127                                      True)
  128 
  129     @testtools.skipUnless(ipv6_utils.is_enabled_and_bind_by_default(),
  130                           "IPv6 is not enabled")
  131     def test_ipv6_router_advts_and_fwd_after_router_state_change_master(self):
  132         # Check that RA and forwarding are enabled when there's no IPv6
  133         # gateway.
  134         self._test_ipv6_router_advts_and_fwd_helper('master',
  135                                                     enable_v6_gw=False,
  136                                                     expected_ra=True,
  137                                                     expected_forwarding=True)
  138         # Check that RA is disabled and forwarding is enabled when an IPv6
  139         # gateway is configured.
  140         self._test_ipv6_router_advts_and_fwd_helper('master',
  141                                                     enable_v6_gw=True,
  142                                                     expected_ra=False,
  143                                                     expected_forwarding=True)
  144 
  145     @testtools.skipUnless(ipv6_utils.is_enabled_and_bind_by_default(),
  146                           "IPv6 is not enabled")
  147     def test_ipv6_router_advts_and_fwd_after_router_state_change_backup(self):
  148         # Check that both RA and forwarding are disabled on backup instances
  149         self._test_ipv6_router_advts_and_fwd_helper('backup',
  150                                                     enable_v6_gw=False,
  151                                                     expected_ra=False,
  152                                                     expected_forwarding=False)
  153         self._test_ipv6_router_advts_and_fwd_helper('backup',
  154                                                     enable_v6_gw=True,
  155                                                     expected_ra=False,
  156                                                     expected_forwarding=False)
  157 
  158     def test_keepalived_configuration(self):
  159         router_info = self.generate_router_info(enable_ha=True)
  160         router = self.manage_router(self.agent, router_info)
  161         expected = self.get_expected_keepalive_configuration(router)
  162 
  163         self.assertEqual(expected,
  164                          router.keepalived_manager.get_conf_on_disk())
  165 
  166         # Add a new FIP and change the GW IP address
  167         router.router = copy.deepcopy(router.router)
  168         existing_fip = '19.4.4.2'
  169         new_fip = '19.4.4.3'
  170         self._add_fip(router, new_fip)
  171         subnet_id = framework._uuid()
  172         fixed_ips = [{'ip_address': '19.4.4.10',
  173                       'prefixlen': 24,
  174                       'subnet_id': subnet_id}]
  175         subnets = [{'id': subnet_id,
  176                     'cidr': '19.4.4.0/24',
  177                     'gateway_ip': '19.4.4.5'}]
  178         router.router['gw_port']['subnets'] = subnets
  179         router.router['gw_port']['fixed_ips'] = fixed_ips
  180 
  181         router.process()
  182 
  183         # Get the updated configuration and assert that both FIPs are in,
  184         # and that the GW IP address was updated.
  185         new_config = router.keepalived_manager.config.get_config_str()
  186         old_gw = '0.0.0.0/0 via 19.4.4.1'
  187         new_gw = '0.0.0.0/0 via 19.4.4.5'
  188         old_external_device_ip = '19.4.4.4'
  189         new_external_device_ip = '19.4.4.10'
  190         self.assertIn(existing_fip, new_config)
  191         self.assertIn(new_fip, new_config)
  192         self.assertNotIn(old_gw, new_config)
  193         self.assertIn(new_gw, new_config)
  194         external_port = router.get_ex_gw_port()
  195         external_device_name = router.get_external_device_name(
  196             external_port['id'])
  197         self.assertNotIn('%s/24 dev %s' %
  198                          (old_external_device_ip, external_device_name),
  199                          new_config)
  200         self.assertIn('%s/24 dev %s' %
  201                       (new_external_device_ip, external_device_name),
  202                       new_config)
  203 
  204     def test_ha_router_conf_on_restarted_agent(self):
  205         router_info = self.generate_router_info(enable_ha=True)
  206         router1 = self.manage_router(self.agent, router_info)
  207         self._add_fip(router1, '192.168.111.12')
  208         restarted_agent = neutron_l3_agent.L3NATAgentWithStateReport(
  209             self.agent.host, self.agent.conf)
  210         self.manage_router(restarted_agent, router1.router)
  211         common_utils.wait_until_true(
  212             lambda: self.floating_ips_configured(router1))
  213         self.assertIn(
  214             router1._get_primary_vip(),
  215             self._get_addresses_on_device(
  216                 router1.ns_name,
  217                 router1.get_ha_device_name()))
  218 
  219     def test_ha_router_ipv6_radvd_status(self):
  220         router_info = self.generate_router_info(
  221             ip_version=constants.IP_VERSION_6, enable_ha=True)
  222         router1 = self.manage_router(self.agent, router_info)
  223         common_utils.wait_until_true(lambda: router1.ha_state == 'master')
  224         common_utils.wait_until_true(lambda: router1.radvd.enabled)
  225 
  226         def _check_lla_status(router, expected):
  227             internal_devices = router.router[constants.INTERFACE_KEY]
  228             for device in internal_devices:
  229                 lladdr = ip_lib.get_ipv6_lladdr(device['mac_address'])
  230                 exists = ip_lib.device_exists_with_ips_and_mac(
  231                     router.get_internal_device_name(device['id']), [lladdr],
  232                     device['mac_address'], router.ns_name)
  233                 self.assertEqual(expected, exists)
  234 
  235         _check_lla_status(router1, True)
  236 
  237         device_name = router1.get_ha_device_name()
  238         ha_device = ip_lib.IPDevice(device_name, namespace=router1.ns_name)
  239         ha_device.link.set_down()
  240 
  241         common_utils.wait_until_true(lambda: router1.ha_state == 'backup')
  242         common_utils.wait_until_true(
  243             lambda: not router1.radvd.enabled, timeout=10)
  244         _check_lla_status(router1, False)
  245 
  246     def test_ha_router_process_ipv6_subnets_to_existing_port(self):
  247         router_info = self.generate_router_info(enable_ha=True,
  248             ip_version=constants.IP_VERSION_6)
  249         router = self.manage_router(self.agent, router_info)
  250 
  251         def verify_ip_in_keepalived_config(router, iface):
  252             config = router.keepalived_manager.config.get_config_str()
  253             ip_cidrs = common_utils.fixed_ip_cidrs(iface['fixed_ips'])
  254             for ip_addr in ip_cidrs:
  255                 self.assertIn(ip_addr, config)
  256 
  257         interface_id = router.router[constants.INTERFACE_KEY][0]['id']
  258         slaac = constants.IPV6_SLAAC
  259         slaac_mode = {'ra_mode': slaac, 'address_mode': slaac}
  260 
  261         # Add a second IPv6 subnet to the router internal interface.
  262         self._add_internal_interface_by_subnet(router.router, count=1,
  263                 ip_version=constants.IP_VERSION_6,
  264                 ipv6_subnet_modes=[slaac_mode],
  265                 interface_id=interface_id)
  266         router.process()
  267         common_utils.wait_until_true(lambda: router.ha_state == 'master')
  268 
  269         # Verify that router internal interface is present and is configured
  270         # with IP address from both the subnets.
  271         internal_iface = router.router[constants.INTERFACE_KEY][0]
  272         self.assertEqual(2, len(internal_iface['fixed_ips']))
  273         self._assert_internal_devices(router)
  274 
  275         # Verify that keepalived config is properly updated.
  276         verify_ip_in_keepalived_config(router, internal_iface)
  277 
  278         # Remove one subnet from the router internal iface
  279         interfaces = copy.deepcopy(router.router.get(
  280             constants.INTERFACE_KEY, []))
  281         fixed_ips, subnets = [], []
  282         fixed_ips.append(interfaces[0]['fixed_ips'][0])
  283         subnets.append(interfaces[0]['subnets'][0])
  284         interfaces[0].update({'fixed_ips': fixed_ips, 'subnets': subnets})
  285         router.router[constants.INTERFACE_KEY] = interfaces
  286         router.process()
  287 
  288         # Verify that router internal interface has a single ipaddress
  289         internal_iface = router.router[constants.INTERFACE_KEY][0]
  290         self.assertEqual(1, len(internal_iface['fixed_ips']))
  291         self._assert_internal_devices(router)
  292 
  293         # Verify that keepalived config is properly updated.
  294         verify_ip_in_keepalived_config(router, internal_iface)
  295 
  296     def test_delete_external_gateway_on_standby_router(self):
  297         router_info = self.generate_router_info(enable_ha=True)
  298         router = self.manage_router(self.agent, router_info)
  299 
  300         self.fail_ha_router(router)
  301         common_utils.wait_until_true(lambda: router.ha_state == 'backup')
  302 
  303         # The purpose of the test is to simply make sure no exception is raised
  304         port = router.get_ex_gw_port()
  305         interface_name = router.get_external_device_name(port['id'])
  306         router.external_gateway_removed(port, interface_name)
  307 
  308     def test_removing_floatingip_immediately(self):
  309         router_info = self.generate_router_info(enable_ha=True)
  310         router = self.manage_router(self.agent, router_info)
  311         ex_gw_port = router.get_ex_gw_port()
  312         interface_name = router.get_external_device_interface_name(ex_gw_port)
  313         common_utils.wait_until_true(lambda: router.ha_state == 'master')
  314         self._add_fip(router, '172.168.1.20', fixed_address='10.0.0.3')
  315         router.process()
  316         router.router[constants.FLOATINGIP_KEY] = []
  317         # The purpose of the test is to simply make sure no exception is raised
  318         # Because router.process will consume the FloatingIpSetupException,
  319         # call the configure_fip_addresses directly here
  320         router.configure_fip_addresses(interface_name)
  321 
  322     def test_ha_port_status_update(self):
  323         router_info = self.generate_router_info(enable_ha=True)
  324         router_info[constants.HA_INTERFACE_KEY]['status'] = (
  325             constants.PORT_STATUS_DOWN)
  326         router1 = self.manage_router(self.agent, router_info)
  327         common_utils.wait_until_true(lambda: router1.ha_state == 'backup')
  328 
  329         router1.router[constants.HA_INTERFACE_KEY]['status'] = (
  330             constants.PORT_STATUS_ACTIVE)
  331         self.agent._process_updated_router(router1.router)
  332         common_utils.wait_until_true(lambda: router1.ha_state == 'master')
  333 
  334     def test_ha_router_namespace_has_ip_nonlocal_bind_disabled(self):
  335         router_info = self.generate_router_info(enable_ha=True)
  336         router = self.manage_router(self.agent, router_info)
  337         try:
  338             ip_nonlocal_bind_value = ip_lib.get_ip_nonlocal_bind(
  339                 router.router_namespace.name)
  340         except RuntimeError as rte:
  341             stat_message = 'cannot stat /proc/sys/net/ipv4/ip_nonlocal_bind'
  342             if stat_message in str(rte):
  343                 raise self.skipException(
  344                     "This kernel doesn't support %s in network namespaces." % (
  345                         ip_lib.IP_NONLOCAL_BIND))
  346             raise
  347         self.assertEqual(0, ip_nonlocal_bind_value)
  348 
  349     @testtools.skipUnless(ipv6_utils.is_enabled_and_bind_by_default(),
  350                           "IPv6 is not enabled")
  351     def test_ha_router_namespace_has_ipv6_forwarding_disabled(self):
  352         router_info = self.generate_router_info(enable_ha=True)
  353         router_info[constants.HA_INTERFACE_KEY]['status'] = (
  354             constants.PORT_STATUS_DOWN)
  355         router = self.manage_router(self.agent, router_info)
  356         external_port = router.get_ex_gw_port()
  357         external_device_name = router.get_external_device_name(
  358             external_port['id'])
  359 
  360         common_utils.wait_until_true(lambda: router.ha_state == 'backup')
  361         self._wait_until_ipv6_forwarding_has_state(
  362             router.ns_name, external_device_name, 0)
  363 
  364         router.router[constants.HA_INTERFACE_KEY]['status'] = (
  365             constants.PORT_STATUS_ACTIVE)
  366         self.agent._process_updated_router(router.router)
  367         common_utils.wait_until_true(lambda: router.ha_state == 'master')
  368         self._wait_until_ipv6_forwarding_has_state(
  369             router.ns_name, external_device_name, 1)
  370 
  371     @testtools.skipUnless(ipv6_utils.is_enabled_and_bind_by_default(),
  372                           "IPv6 is not enabled")
  373     def test_ha_router_without_gw_ipv6_forwarding_state(self):
  374         router_info = self.generate_router_info(
  375             enable_ha=True, enable_gw=False)
  376         router_info[constants.HA_INTERFACE_KEY]['status'] = (
  377             constants.PORT_STATUS_DOWN)
  378         router = self.manage_router(self.agent, router_info)
  379 
  380         common_utils.wait_until_true(lambda: router.ha_state == 'backup')
  381         self._wait_until_ipv6_forwarding_has_state(router.ns_name, 'all', 0)
  382 
  383         router.router[constants.HA_INTERFACE_KEY]['status'] = (
  384             constants.PORT_STATUS_ACTIVE)
  385         self.agent._process_updated_router(router.router)
  386         common_utils.wait_until_true(lambda: router.ha_state == 'master')
  387         self._wait_until_ipv6_forwarding_has_state(router.ns_name, 'all', 1)
  388 
  389 
  390 class L3HATestFailover(framework.L3AgentTestFramework):
  391 
  392     def setUp(self):
  393         super(L3HATestFailover, self).setUp()
  394         conf = self._configure_agent('agent2')
  395         self.failover_agent = neutron_l3_agent.L3NATAgentWithStateReport(
  396             'agent2', conf)
  397 
  398         br_int_1 = self._get_agent_ovs_integration_bridge(self.agent)
  399         br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent)
  400 
  401         veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports
  402         veth1.link.set_up()
  403         veth2.link.set_up()
  404         br_int_1.add_port(veth1.name)
  405         br_int_2.add_port(veth2.name)
  406 
  407     @staticmethod
  408     def fail_gw_router_port(router):
  409         # NOTE(slaweq): in HA failover tests there are two integration bridges
  410         # connected with veth pair to each other. To stop traffic from router's
  411         # namespace to gw ip (19.4.4.1) it needs to be blocked by openflow rule
  412         # as simple setting ovs_integration_bridge device DOWN will not be
  413         # enough because same IP address is also configured on
  414         # ovs_integration_bridge device from second router and it will still
  415         # respond to ping
  416         r_br = ovs_lib.OVSBridge(router.driver.conf.ovs_integration_bridge)
  417         external_port = router.get_ex_gw_port()
  418         for subnet in external_port['subnets']:
  419             r_br.add_flow(
  420                 proto='ip', nw_dst=subnet['gateway_ip'], actions='drop')
  421 
  422     @staticmethod
  423     def restore_gw_router_port(router):
  424         r_br = ovs_lib.OVSBridge(router.driver.conf.ovs_integration_bridge)
  425         external_port = router.get_ex_gw_port()
  426         for subnet in external_port['subnets']:
  427             r_br.delete_flows(proto='ip', nw_dst=subnet['gateway_ip'])
  428 
  429     def test_ha_router_failover(self):
  430         router1, router2 = self.create_ha_routers()
  431 
  432         master_router, slave_router = self._get_master_and_slave_routers(
  433             router1, router2)
  434 
  435         self._assert_ipv6_accept_ra(master_router, True)
  436         self._assert_ipv6_forwarding(master_router, True, True)
  437         self._assert_ipv6_accept_ra(slave_router, False)
  438         self._assert_ipv6_forwarding(slave_router, False, False)
  439 
  440         self.fail_ha_router(router1)
  441 
  442         # NOTE: passing slave_router as first argument, because we expect
  443         # that this router should be the master
  444         new_master, new_slave = self._get_master_and_slave_routers(
  445             slave_router, master_router)
  446 
  447         self.assertEqual(master_router, new_slave)
  448         self.assertEqual(slave_router, new_master)
  449         self._assert_ipv6_accept_ra(new_master, True)
  450         self._assert_ipv6_forwarding(new_master, True, True)
  451         self._assert_ipv6_accept_ra(new_slave, False)
  452         # after transition from master -> slave, 'all' IPv6 forwarding should
  453         # be enabled
  454         self._assert_ipv6_forwarding(new_slave, False, True)
  455 
  456     def test_ha_router_lost_gw_connection(self):
  457         self.agent.conf.set_override(
  458             'ha_vrrp_health_check_interval', 5)
  459         self.failover_agent.conf.set_override(
  460             'ha_vrrp_health_check_interval', 5)
  461 
  462         router1, router2 = self.create_ha_routers()
  463 
  464         master_router, slave_router = self._get_master_and_slave_routers(
  465             router1, router2)
  466 
  467         self.fail_gw_router_port(master_router)
  468 
  469         # NOTE: passing slave_router as first argument, because we expect
  470         # that this router should be the master
  471         new_master, new_slave = self._get_master_and_slave_routers(
  472             slave_router, master_router)
  473 
  474         self.assertEqual(master_router, new_slave)
  475         self.assertEqual(slave_router, new_master)
  476 
  477     def test_both_ha_router_lost_gw_connection(self):
  478         self.agent.conf.set_override(
  479             'ha_vrrp_health_check_interval', 5)
  480         self.failover_agent.conf.set_override(
  481             'ha_vrrp_health_check_interval', 5)
  482 
  483         router1, router2 = self.create_ha_routers()
  484 
  485         master_router, slave_router = self._get_master_and_slave_routers(
  486             router1, router2)
  487 
  488         self.fail_gw_router_port(master_router)
  489         self.fail_gw_router_port(slave_router)
  490 
  491         common_utils.wait_until_true(
  492             lambda: master_router.ha_state == 'master')
  493         common_utils.wait_until_true(
  494             lambda: slave_router.ha_state == 'master')
  495 
  496         self.restore_gw_router_port(master_router)
  497 
  498         new_master, new_slave = self._get_master_and_slave_routers(
  499             master_router, slave_router)
  500 
  501         self.assertEqual(master_router, new_master)
  502         self.assertEqual(slave_router, new_slave)
  503 
  504 
  505 class LinuxBridgeL3HATestCase(L3HATestCase):
  506     INTERFACE_DRIVER = 'neutron.agent.linux.interface.BridgeInterfaceDriver'