"Fossies" - the Fresh Open Source Software Archive

Member "neutron-14.0.3/neutron/tests/functional/agent/l3/framework.py" (22 Oct 2019, 31150 Bytes) of package /linux/misc/openstack/neutron-14.0.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "framework.py": 14.0.2_vs_14.0.3.

    1 # Copyright (c) 2014 Red Hat, Inc.
    2 # All Rights Reserved.
    3 #
    4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    5 #    not use this file except in compliance with the License. You may obtain
    6 #    a copy of the License at
    7 #
    8 #         http://www.apache.org/licenses/LICENSE-2.0
    9 #
   10 #    Unless required by applicable law or agreed to in writing, software
   11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   13 #    License for the specific language governing permissions and limitations
   14 #    under the License.
   15 
   16 import copy
   17 import functools
   18 import textwrap
   19 
   20 import mock
   21 import netaddr
   22 from neutron_lib import constants
   23 from oslo_config import cfg
   24 from oslo_log import log as logging
   25 from oslo_utils import uuidutils
   26 import testtools
   27 
   28 from neutron.agent.common import ovs_lib
   29 from neutron.agent.l3 import agent as neutron_l3_agent
   30 from neutron.agent.l3 import namespaces
   31 from neutron.agent.l3 import router_info as l3_router_info
   32 from neutron.agent import l3_agent as l3_agent_main
   33 from neutron.agent.linux import external_process
   34 from neutron.agent.linux import interface
   35 from neutron.agent.linux import ip_lib
   36 from neutron.agent.linux import keepalived
   37 from neutron.agent.metadata import driver as metadata_driver
   38 from neutron.common import constants as n_const
   39 from neutron.common import utils as common_utils
   40 from neutron.conf.agent import common as agent_config
   41 from neutron.conf import common as common_config
   42 from neutron.tests.common import l3_test_common
   43 from neutron.tests.common import net_helpers
   44 from neutron.tests.functional import base
   45 
   46 
   47 _uuid = uuidutils.generate_uuid
   48 
   49 OVS_INTERFACE_DRIVER = 'neutron.agent.linux.interface.OVSInterfaceDriver'
   50 
   51 
   52 def get_ovs_bridge(br_name):
   53     return ovs_lib.OVSBridge(br_name)
   54 
   55 
   56 class L3AgentTestFramework(base.BaseSudoTestCase):
   57     INTERFACE_DRIVER = OVS_INTERFACE_DRIVER
   58     NESTED_NAMESPACE_SEPARATOR = '@'
   59 
   60     def setUp(self):
   61         super(L3AgentTestFramework, self).setUp()
   62         self.mock_plugin_api = mock.patch(
   63             'neutron.agent.l3.agent.L3PluginApi').start().return_value
   64         mock.patch('neutron.agent.rpc.PluginReportStateAPI').start()
   65         self.conf = self._configure_agent('agent1')
   66         self.agent = neutron_l3_agent.L3NATAgentWithStateReport('agent1',
   67                                                                 self.conf)
   68 
   69     def _get_config_opts(self):
   70         config = cfg.ConfigOpts()
   71         config.register_opts(common_config.core_opts)
   72         config.register_opts(common_config.core_cli_opts)
   73         logging.register_options(config)
   74         agent_config.register_process_monitor_opts(config)
   75         agent_config.register_root_helper(config)
   76         return config
   77 
   78     def _configure_agent(self, host, agent_mode='dvr_snat'):
   79         conf = self._get_config_opts()
   80         l3_agent_main.register_opts(conf)
   81         conf.set_override('interface_driver', self.INTERFACE_DRIVER)
   82 
   83         br_int = self.useFixture(net_helpers.OVSBridgeFixture()).bridge
   84         conf.set_override('ovs_integration_bridge', br_int.br_name)
   85 
   86         temp_dir = self.get_new_temp_dir()
   87         get_temp_file_path = functools.partial(self.get_temp_file_path,
   88                                                root=temp_dir)
   89         conf.set_override('state_path', temp_dir.path)
   90         conf.set_override('log_file',
   91                           get_temp_file_path('log_file'))
   92         conf.set_override('metadata_proxy_socket',
   93                           get_temp_file_path('metadata_proxy'))
   94         conf.set_override('ha_confs_path',
   95                           get_temp_file_path('ha_confs'))
   96         conf.set_override('external_pids',
   97                           get_temp_file_path('external/pids'))
   98         conf.set_override('host', host)
   99         conf.set_override('agent_mode', agent_mode)
  100 
  101         return conf
  102 
  103     def _get_agent_ovs_integration_bridge(self, agent):
  104         return get_ovs_bridge(agent.conf.ovs_integration_bridge)
  105 
  106     def generate_router_info(self, enable_ha,
  107                              ip_version=constants.IP_VERSION_4,
  108                              extra_routes=True,
  109                              enable_fip=True, enable_snat=True,
  110                              num_internal_ports=1,
  111                              dual_stack=False, enable_gw=True,
  112                              v6_ext_gw_with_sub=True,
  113                              enable_pf_floating_ip=False,
  114                              qos_policy_id=None):
  115         if ip_version == constants.IP_VERSION_6 and not dual_stack:
  116             enable_snat = False
  117             enable_fip = False
  118             extra_routes = False
  119 
  120         return l3_test_common.prepare_router_data(ip_version=ip_version,
  121                                                   enable_snat=enable_snat,
  122                                                   num_internal_ports=(
  123                                                       num_internal_ports),
  124                                                   enable_floating_ip=(
  125                                                       enable_fip),
  126                                                   enable_ha=enable_ha,
  127                                                   extra_routes=extra_routes,
  128                                                   dual_stack=dual_stack,
  129                                                   enable_gw=enable_gw,
  130                                                   v6_ext_gw_with_sub=(
  131                                                       v6_ext_gw_with_sub),
  132                                                   enable_pf_floating_ip=(
  133                                                       enable_pf_floating_ip),
  134                                                   qos_policy_id=qos_policy_id)
  135 
  136     def change_router_state(self, router_id, state):
  137         ri = self.agent.router_info.get(router_id)
  138         if not ri:
  139             self.fail('Router %s is not present in the L3 agent' % router_id)
  140         ri.ha_state = state
  141 
  142     def _test_conntrack_disassociate_fip(self, ha):
  143         '''Test that conntrack immediately drops stateful connection
  144            that uses floating IP once it's disassociated.
  145         '''
  146         router_info = self.generate_router_info(enable_ha=ha)
  147         router = self.manage_router(self.agent, router_info)
  148 
  149         port = net_helpers.get_free_namespace_port(
  150             constants.PROTO_NAME_TCP, router.ns_name)
  151         client_address = '19.4.4.3'
  152         server_address = '35.4.0.4'
  153 
  154         def clean_fips(router):
  155             router.router[constants.FLOATINGIP_KEY] = []
  156 
  157         clean_fips(router)
  158         self._add_fip(router, client_address, fixed_address=server_address)
  159         router.process()
  160 
  161         router_ns = ip_lib.IPWrapper(namespace=router.ns_name)
  162         netcat = net_helpers.NetcatTester(
  163             router.ns_name, router.ns_name, client_address, port,
  164             protocol=net_helpers.NetcatTester.TCP)
  165         self.addCleanup(netcat.stop_processes)
  166 
  167         def assert_num_of_conntrack_rules(n):
  168             out = router_ns.netns.execute(["conntrack", "-L",
  169                                            "--orig-src", client_address])
  170             self.assertEqual(
  171                 n, len([line for line in out.strip().split('\n') if line]))
  172 
  173         if ha:
  174             common_utils.wait_until_true(lambda: router.ha_state == 'master')
  175 
  176         with self.assert_max_execution_time(100):
  177             assert_num_of_conntrack_rules(0)
  178 
  179             self.assertTrue(netcat.test_connectivity())
  180             assert_num_of_conntrack_rules(1)
  181 
  182             clean_fips(router)
  183             router.process()
  184             assert_num_of_conntrack_rules(0)
  185 
  186             with testtools.ExpectedException(RuntimeError):
  187                 netcat.test_connectivity()
  188 
  189     def _test_update_floatingip_statuses(self, router_info):
  190         router = self.manage_router(self.agent, router_info)
  191         rpc = self.agent.plugin_rpc.update_floatingip_statuses
  192         self.assertTrue(rpc.called)
  193 
  194         # Assert that every defined FIP is updated via RPC
  195         expected_fips = set([
  196             (fip['id'], constants.FLOATINGIP_STATUS_ACTIVE) for fip in
  197             router.router[constants.FLOATINGIP_KEY]])
  198         call = [args[0] for args in rpc.call_args_list][0]
  199         actual_fips = set(
  200             [(fip_id, status) for fip_id, status in call[2].items()])
  201         self.assertEqual(expected_fips, actual_fips)
  202 
  203     def _gateway_check(self, gateway_ip, external_device):
  204         expected_gateway = gateway_ip
  205         ip_vers = netaddr.IPAddress(expected_gateway).version
  206         existing_gateway = (external_device.route.get_gateway(
  207             ip_version=ip_vers).get('gateway'))
  208         self.assertEqual(expected_gateway, existing_gateway)
  209 
  210     def _assert_ha_device(self, router):
  211         def ha_router_dev_name_getter(not_used):
  212             return router.get_ha_device_name()
  213         self.assertTrue(self.device_exists_with_ips_and_mac(
  214             router.router[constants.HA_INTERFACE_KEY],
  215             ha_router_dev_name_getter, router.ns_name))
  216 
  217     def _assert_gateway(self, router, v6_ext_gw_with_sub=True):
  218         external_port = router.get_ex_gw_port()
  219         external_device_name = router.get_external_device_name(
  220             external_port['id'])
  221         external_device = ip_lib.IPDevice(external_device_name,
  222                                           namespace=router.ns_name)
  223         for subnet in external_port['subnets']:
  224             self._gateway_check(subnet['gateway_ip'], external_device)
  225         if not v6_ext_gw_with_sub:
  226             self._gateway_check(self.agent.conf.ipv6_gateway,
  227                                 external_device)
  228 
  229     def _check_external_device(self, router):
  230         external_port = router.get_ex_gw_port()
  231         return (self.device_exists_with_ips_and_mac(
  232             external_port, router.get_external_device_name,
  233             router.ns_name))
  234 
  235     def _assert_external_device(self, router):
  236         self.assertTrue(self._check_external_device(router))
  237 
  238     def _wait_until_ipv6_accept_ra_has_state(
  239             self, ns_name, device_name, enabled):
  240         ip_wrapper = ip_lib.IPWrapper(namespace=ns_name)
  241 
  242         def _ipv6_accept_ra_state():
  243             ra_state = ip_wrapper.netns.execute(['sysctl', '-b',
  244                 'net.ipv6.conf.%s.accept_ra' % device_name])
  245             return (
  246                 enabled == (int(ra_state) != n_const.ACCEPT_RA_DISABLED))
  247 
  248         common_utils.wait_until_true(_ipv6_accept_ra_state)
  249 
  250     def _assert_ipv6_accept_ra(self, router, enabled=True):
  251         external_port = router.get_ex_gw_port()
  252         external_device_name = router.get_external_device_name(
  253             external_port['id'])
  254 
  255         self._wait_until_ipv6_accept_ra_has_state(
  256             router.ns_name, external_device_name, enabled)
  257 
  258     def _wait_until_ipv6_forwarding_has_state(self, ns_name, dev_name, state):
  259 
  260         def _ipv6_forwarding_has_state():
  261             return ip_lib.get_ipv6_forwarding(
  262                 device=dev_name, namespace=ns_name) == state
  263 
  264         common_utils.wait_until_true(_ipv6_forwarding_has_state)
  265 
  266     def _assert_ipv6_forwarding(self, router, enabled=True, all_enabled=True):
  267         external_port = router.get_ex_gw_port()
  268         external_device_name = router.get_external_device_name(
  269             external_port['id'])
  270         self._wait_until_ipv6_forwarding_has_state(
  271             router.ns_name, external_device_name, int(enabled))
  272         self._wait_until_ipv6_forwarding_has_state(
  273             router.ns_name, 'all', int(all_enabled))
  274 
  275     def _router_lifecycle(self, enable_ha, ip_version=constants.IP_VERSION_4,
  276                           dual_stack=False, v6_ext_gw_with_sub=True,
  277                           router_info=None):
  278         router_info = router_info or self.generate_router_info(
  279             enable_ha, ip_version, dual_stack=dual_stack,
  280             v6_ext_gw_with_sub=(v6_ext_gw_with_sub))
  281         return_copy = copy.deepcopy(router_info)
  282         router = self.manage_router(self.agent, router_info)
  283 
  284         # Add multiple-IPv6-prefix internal router port
  285         slaac = constants.IPV6_SLAAC
  286         slaac_mode = {'ra_mode': slaac, 'address_mode': slaac}
  287         subnet_modes = [slaac_mode] * 2
  288         self._add_internal_interface_by_subnet(router.router,
  289             count=2, ip_version=constants.IP_VERSION_6,
  290             ipv6_subnet_modes=subnet_modes)
  291         router.process()
  292 
  293         if enable_ha:
  294             common_utils.wait_until_true(lambda: router.ha_state == 'master')
  295 
  296             # Keepalived notifies of a state transition when it starts,
  297             # not when it ends. Thus, we have to wait until keepalived finishes
  298             # configuring everything. We verify this by waiting until the last
  299             # device has an IP address.
  300             device = router.router[constants.INTERFACE_KEY][-1]
  301             device_exists = functools.partial(
  302                 self.device_exists_with_ips_and_mac,
  303                 device,
  304                 router.get_internal_device_name,
  305                 router.ns_name)
  306             common_utils.wait_until_true(device_exists)
  307 
  308         self.assertTrue(self._namespace_exists(router.ns_name))
  309         common_utils.wait_until_true(
  310             lambda: self._metadata_proxy_exists(self.agent.conf, router))
  311         self._assert_internal_devices(router)
  312         self._assert_external_device(router)
  313         if not (enable_ha and
  314                 (ip_version == constants.IP_VERSION_6 or dual_stack)):
  315             # Note(SridharG): enable the assert_gateway for IPv6 once
  316             # keepalived on Ubuntu14.04 (i.e., check-neutron-dsvm-functional
  317             # platform) is updated to 1.2.10 (or above).
  318             # For more details: https://review.openstack.org/#/c/151284/
  319             self._assert_gateway(router, v6_ext_gw_with_sub)
  320             self.assertTrue(self.floating_ips_configured(router))
  321             self._assert_snat_chains(router)
  322             self._assert_floating_ip_chains(router)
  323             self._assert_iptables_rules_converged(router)
  324             self._assert_extra_routes(router)
  325             if (ip_version == constants.IP_VERSION_6 or dual_stack):
  326                 ip_versions = [constants.IP_VERSION_4, constants.IP_VERSION_6]
  327             else:
  328                 ip_versions = [constants.IP_VERSION_4]
  329             self._assert_onlink_subnet_routes(router, ip_versions)
  330         self._assert_metadata_chains(router)
  331 
  332         # Verify router gateway interface is configured to receive Router Advts
  333         # when IPv6 is enabled and no IPv6 gateway is configured.
  334         if router.use_ipv6 and not v6_ext_gw_with_sub:
  335             if not self.agent.conf.ipv6_gateway:
  336                 self._assert_ipv6_accept_ra(router)
  337 
  338         if enable_ha:
  339             self._assert_ha_device(router)
  340             common_utils.wait_until_true(
  341                 lambda: router.keepalived_manager.get_process().active,
  342                 timeout=15)
  343 
  344         self._delete_router(self.agent, router.router_id)
  345 
  346         self._assert_interfaces_deleted_from_ovs()
  347         self._assert_router_does_not_exist(router)
  348         if enable_ha:
  349             common_utils.wait_until_true(
  350                 lambda: not router.keepalived_manager.get_process().active,
  351                 timeout=15)
  352         return return_copy
  353 
  354     def manage_router(self, agent, router):
  355         self.addCleanup(agent._safe_router_removed, router['id'])
  356 
  357         # NOTE(mangelajo): Neutron functional for l3 don't rely on openvswitch
  358         #                  agent tagging ports, and all ports remain untagged
  359         #                  during test execution.
  360         #                  Workaround related to lp#1767422 plugs new ports as
  361         #                  dead vlan (4095) to avoid issues, we need to remove
  362         #                  such tag during functional l3 testing.
  363         original_plug_new = interface.OVSInterfaceDriver.plug_new
  364 
  365         def new_ovs_plug(self, *args, **kwargs):
  366             original_plug_new(self, *args, **kwargs)
  367             bridge = (kwargs.get('bridge') or args[4] or
  368                       self.conf.ovs_integration_bridge)
  369             device_name = kwargs.get('device_name') or args[2]
  370             ovsbr = ovs_lib.OVSBridge(bridge)
  371             ovsbr.clear_db_attribute('Port', device_name, 'tag')
  372 
  373         with mock.patch(OVS_INTERFACE_DRIVER + '.plug_new', autospec=True) as (
  374                 ovs_plug):
  375             ovs_plug.side_effect = new_ovs_plug
  376             agent._process_added_router(router)
  377 
  378         return agent.router_info[router['id']]
  379 
  380     def _delete_router(self, agent, router_id):
  381         agent._safe_router_removed(router_id)
  382 
  383     def _add_fip(self, router, fip_address, fixed_address='10.0.0.2',
  384                  host=None, fixed_ip_address_scope=None):
  385         fip = {'id': _uuid(),
  386                'port_id': _uuid(),
  387                'floating_ip_address': fip_address,
  388                'fixed_ip_address': fixed_address,
  389                'host': host,
  390                'fixed_ip_address_scope': fixed_ip_address_scope}
  391         router.router[constants.FLOATINGIP_KEY].append(fip)
  392 
  393     def _add_internal_interface_by_subnet(self, router, count=1,
  394                                           ip_version=constants.IP_VERSION_4,
  395                                           ipv6_subnet_modes=None,
  396                                           interface_id=None):
  397         return l3_test_common.router_append_subnet(router, count,
  398                                                    ip_version,
  399                                                    ipv6_subnet_modes,
  400                                                    interface_id)
  401 
  402     def _namespace_exists(self, namespace):
  403         return ip_lib.network_namespace_exists(namespace)
  404 
  405     def _metadata_proxy_exists(self, conf, router):
  406         pm = external_process.ProcessManager(
  407             conf,
  408             router.router_id,
  409             router.ns_name,
  410             service=metadata_driver.HAPROXY_SERVICE)
  411         return pm.active
  412 
  413     def device_exists_with_ips_and_mac(self, expected_device, name_getter,
  414                                        namespace):
  415         ip_cidrs = common_utils.fixed_ip_cidrs(expected_device['fixed_ips'])
  416         return ip_lib.device_exists_with_ips_and_mac(
  417             name_getter(expected_device['id']), ip_cidrs,
  418             expected_device['mac_address'], namespace)
  419 
  420     @staticmethod
  421     def _port_first_ip_cidr(port):
  422         fixed_ip = port['fixed_ips'][0]
  423         return common_utils.ip_to_cidr(fixed_ip['ip_address'],
  424                                        fixed_ip['prefixlen'])
  425 
  426     def get_device_mtu(self, target_device, name_getter, namespace):
  427         device = ip_lib.IPDevice(name_getter(target_device), namespace)
  428         return device.link.mtu
  429 
  430     def get_expected_keepalive_configuration(self, router):
  431         ha_device_name = router.get_ha_device_name()
  432         external_port = router.get_ex_gw_port()
  433         ex_port_ipv6 = ip_lib.get_ipv6_lladdr(external_port['mac_address'])
  434         ex_device_name = router.get_external_device_name(
  435             external_port['id'])
  436         external_device_cidr = self._port_first_ip_cidr(external_port)
  437         internal_port = router.router[constants.INTERFACE_KEY][0]
  438         int_port_ipv6 = ip_lib.get_ipv6_lladdr(internal_port['mac_address'])
  439         internal_device_name = router.get_internal_device_name(
  440             internal_port['id'])
  441         internal_device_cidr = self._port_first_ip_cidr(internal_port)
  442         floating_ip_cidr = common_utils.ip_to_cidr(
  443             router.get_floating_ips()[0]['floating_ip_address'])
  444         default_gateway_ip = external_port['subnets'][0].get('gateway_ip')
  445         extra_subnet_cidr = external_port['extra_subnets'][0].get('cidr')
  446         return textwrap.dedent("""\
  447             global_defs {
  448                 notification_email_from %(email_from)s
  449                 router_id %(router_id)s
  450             }
  451             vrrp_instance VR_1 {
  452                 state BACKUP
  453                 interface %(ha_device_name)s
  454                 virtual_router_id 1
  455                 priority 50
  456                 garp_master_delay 60
  457                 nopreempt
  458                 advert_int 2
  459                 track_interface {
  460                     %(ha_device_name)s
  461                 }
  462                 virtual_ipaddress {
  463                     169.254.0.1/24 dev %(ha_device_name)s
  464                 }
  465                 virtual_ipaddress_excluded {
  466                     %(floating_ip_cidr)s dev %(ex_device_name)s
  467                     %(external_device_cidr)s dev %(ex_device_name)s
  468                     %(internal_device_cidr)s dev %(internal_device_name)s
  469                     %(ex_port_ipv6)s dev %(ex_device_name)s scope link
  470                     %(int_port_ipv6)s dev %(internal_device_name)s scope link
  471                 }
  472                 virtual_routes {
  473                     0.0.0.0/0 via %(default_gateway_ip)s dev %(ex_device_name)s
  474                     8.8.8.0/24 via 19.4.4.4
  475                     %(extra_subnet_cidr)s dev %(ex_device_name)s scope link
  476                 }
  477             }""") % {
  478             'email_from': keepalived.KEEPALIVED_EMAIL_FROM,
  479             'router_id': keepalived.KEEPALIVED_ROUTER_ID,
  480             'ha_device_name': ha_device_name,
  481             'ex_device_name': ex_device_name,
  482             'external_device_cidr': external_device_cidr,
  483             'internal_device_name': internal_device_name,
  484             'internal_device_cidr': internal_device_cidr,
  485             'floating_ip_cidr': floating_ip_cidr,
  486             'default_gateway_ip': default_gateway_ip,
  487             'int_port_ipv6': int_port_ipv6,
  488             'ex_port_ipv6': ex_port_ipv6,
  489             'extra_subnet_cidr': extra_subnet_cidr,
  490         }
  491 
  492     def _get_rule(self, iptables_manager, table, chain, predicate):
  493         rules = iptables_manager.get_chain(table, chain)
  494         result = next(rule for rule in rules if predicate(rule))
  495         return result
  496 
  497     def _assert_router_does_not_exist(self, router):
  498         # If the namespace assertion succeeds
  499         # then the devices and iptable rules have also been deleted,
  500         # so there's no need to check that explicitly.
  501         self.assertFalse(self._namespace_exists(router.ns_name))
  502         common_utils.wait_until_true(
  503             lambda: not self._metadata_proxy_exists(self.agent.conf, router),
  504             timeout=10)
  505 
  506     def _assert_snat_chains(self, router):
  507         self.assertFalse(router.iptables_manager.is_chain_empty(
  508             'nat', 'snat'))
  509         self.assertFalse(router.iptables_manager.is_chain_empty(
  510             'nat', 'POSTROUTING'))
  511 
  512     def _assert_floating_ip_chains(self, router, snat_bound_fip=False):
  513         if snat_bound_fip:
  514             self.assertFalse(router.snat_iptables_manager.is_chain_empty(
  515                 'nat', 'float-snat'))
  516 
  517         self.assertFalse(router.iptables_manager.is_chain_empty(
  518             'nat', 'float-snat'))
  519 
  520     def _assert_iptables_rules_converged(self, router):
  521         # if your code is failing on this line, it means you are not generating
  522         # your iptables rules in the same format that iptables-save returns
  523         # them. run iptables-save to see the format they should be in
  524         self.assertFalse(router.iptables_manager.apply())
  525 
  526     def _assert_metadata_chains(self, router):
  527         metadata_port_filter = lambda rule: (
  528             str(self.agent.conf.metadata_port) in rule.rule)
  529         self.assertTrue(self._get_rule(router.iptables_manager,
  530                                        'nat',
  531                                        'PREROUTING',
  532                                        metadata_port_filter))
  533         self.assertTrue(self._get_rule(router.iptables_manager,
  534                                        'filter',
  535                                        'INPUT',
  536                                        metadata_port_filter))
  537 
  538     def _assert_internal_devices(self, router):
  539         internal_devices = router.router[constants.INTERFACE_KEY]
  540         self.assertGreater(len(internal_devices), 0)
  541         for device in internal_devices:
  542             self.assertTrue(self.device_exists_with_ips_and_mac(
  543                 device, router.get_internal_device_name, router.ns_name))
  544 
  545     def _assert_extra_routes(self, router, namespace=None):
  546         if namespace is None:
  547             namespace = router.ns_name
  548         routes = ip_lib.get_routing_table(4, namespace=namespace)
  549         routes = [{'nexthop': route['nexthop'],
  550                    'destination': route['destination']} for route in routes]
  551 
  552         for extra_route in router.router['routes']:
  553             self.assertIn(extra_route, routes)
  554 
  555     def _assert_onlink_subnet_routes(
  556             self, router, ip_versions, namespace=None):
  557         ns_name = namespace or router.ns_name
  558         routes = []
  559         for ip_version in ip_versions:
  560             _routes = ip_lib.get_routing_table(ip_version,
  561                                                namespace=ns_name)
  562             routes.extend(_routes)
  563         routes = set(route['destination'] for route in routes)
  564         extra_subnets = router.get_ex_gw_port()['extra_subnets']
  565         for extra_subnet in (route['cidr'] for route in extra_subnets):
  566             self.assertIn(extra_subnet, routes)
  567 
  568     def _assert_interfaces_deleted_from_ovs(self):
  569         bridge = ovs_lib.OVSBridge(self.agent.conf.ovs_integration_bridge)
  570         self.assertFalse(bridge.get_port_name_list())
  571 
  572     def floating_ips_configured(self, router):
  573         floating_ips = router.router[constants.FLOATINGIP_KEY]
  574         external_port = router.get_ex_gw_port()
  575         return len(floating_ips) and all(
  576             ip_lib.device_exists_with_ips_and_mac(
  577                 router.get_external_device_name(external_port['id']),
  578                 ['%s/32' % fip['floating_ip_address']],
  579                 external_port['mac_address'],
  580                 namespace=router.ns_name) for fip in floating_ips)
  581 
  582     def _create_router(self, router_info, agent):
  583 
  584         ns_name = "%s%s%s" % (
  585             'qrouter-' + router_info['id'],
  586             self.NESTED_NAMESPACE_SEPARATOR, agent.host)
  587         ext_name = "qg-%s-%s" % (agent.host, _uuid()[-4:])
  588         int_name = "qr-%s-%s" % (agent.host, _uuid()[-4:])
  589 
  590         get_ns_name = mock.patch.object(
  591             namespaces.RouterNamespace, '_get_ns_name').start()
  592         get_ns_name.return_value = ns_name
  593         get_ext_name = mock.patch.object(l3_router_info.RouterInfo,
  594             'get_external_device_name').start()
  595         get_ext_name.return_value = ext_name
  596         get_int_name = mock.patch.object(l3_router_info.RouterInfo,
  597             'get_internal_device_name').start()
  598         get_int_name.return_value = int_name
  599 
  600         router = self.manage_router(agent, router_info)
  601 
  602         router_ext_name = mock.patch.object(router,
  603             'get_external_device_name').start()
  604         router_ext_name.return_value = get_ext_name.return_value
  605         router_int_name = mock.patch.object(router,
  606             'get_internal_device_name').start()
  607         router_int_name.return_value = get_int_name.return_value
  608 
  609         return router
  610 
  611     def create_ha_routers(self):
  612         router_info = self.generate_router_info(enable_ha=True)
  613         router1 = self._create_router(router_info, self.agent)
  614         self._add_fip(router1, '192.168.111.12')
  615 
  616         r1_br = ip_lib.IPDevice(router1.driver.conf.ovs_integration_bridge)
  617         r1_br.addr.add('19.4.4.1/24')
  618         r1_br.link.set_up()
  619 
  620         router_info_2 = copy.deepcopy(router_info)
  621         router_info_2[constants.HA_INTERFACE_KEY] = (
  622             l3_test_common.get_ha_interface(ip='169.254.192.2',
  623                                             mac='22:22:22:22:22:22'))
  624         router2 = self._create_router(router_info_2, self.failover_agent)
  625 
  626         r2_br = ip_lib.IPDevice(router2.driver.conf.ovs_integration_bridge)
  627         r2_br.addr.add('19.4.4.1/24')
  628         r2_br.link.set_up()
  629 
  630         return (router1, router2)
  631 
  632     def _get_master_and_slave_routers(self, router1, router2,
  633                                       check_external_device=True):
  634 
  635         try:
  636             common_utils.wait_until_true(
  637                 lambda: router1.ha_state == 'master')
  638             if check_external_device:
  639                 common_utils.wait_until_true(
  640                     lambda: self._check_external_device(router1))
  641             master_router = router1
  642             slave_router = router2
  643         except common_utils.WaitTimeout:
  644             common_utils.wait_until_true(
  645                 lambda: router2.ha_state == 'master')
  646             if check_external_device:
  647                 common_utils.wait_until_true(
  648                     lambda: self._check_external_device(router2))
  649             master_router = router2
  650             slave_router = router1
  651 
  652         common_utils.wait_until_true(
  653                 lambda: master_router.ha_state == 'master')
  654         if check_external_device:
  655             common_utils.wait_until_true(
  656                 lambda: self._check_external_device(master_router))
  657         common_utils.wait_until_true(
  658             lambda: slave_router.ha_state == 'backup')
  659         return master_router, slave_router
  660 
  661     def fail_ha_router(self, router):
  662         device_name = router.get_ha_device_name()
  663         ha_device = ip_lib.IPDevice(device_name, router.ha_namespace)
  664         ha_device.link.set_down()
  665 
  666     @staticmethod
  667     def fail_gw_router_port(router):
  668         r_br = ip_lib.IPDevice(router.driver.conf.ovs_integration_bridge)
  669         r_br.link.set_down()
  670 
  671     @staticmethod
  672     def restore_gw_router_port(router):
  673         r_br = ip_lib.IPDevice(router.driver.conf.ovs_integration_bridge)
  674         r_br.link.set_up()
  675 
  676     @classmethod
  677     def _get_addresses_on_device(cls, namespace, interface):
  678         return [address['cidr'] for address in
  679                 ip_lib.IPDevice(interface, namespace=namespace).addr.list()]
  680 
  681     def _assert_no_ip_addresses_on_interface(self, namespace, interface):
  682         self.assertEqual(
  683             [], self._get_addresses_on_device(namespace, interface))
  684 
  685     def _assert_ip_addresses_on_interface(self,
  686                                           namespace, interface, ip_addresses):
  687         for ip_address in ip_addresses:
  688             self._assert_ip_address_on_interface(namespace, interface,
  689                                                  ip_address)
  690 
  691     def _assert_ip_address_not_on_interface(
  692         self, namespace, interface, ip_address):
  693         self.assertNotIn(
  694             ip_address, self._get_addresses_on_device(namespace, interface))
  695 
  696     def _assert_ip_address_on_interface(self,
  697                                         namespace, interface, ip_address):
  698         self.assertIn(
  699             ip_address, self._get_addresses_on_device(namespace, interface))
  700 
  701     def _assert_ping_reply_from_expected_address(
  702         self, ping_result, expected_address):
  703         ping_results = ping_result.split('\n')
  704         self.assertGreater(
  705             len(ping_results), 1,
  706             "The result from ping should be multiple lines")
  707         self.assertIn(
  708             expected_address, ping_results[1],
  709             ("Expect to see %s in the reply of ping, but failed" %
  710              expected_address))