"Fossies" - the Fresh Open Source Software Archive

Member "neutron-14.0.3/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py" (22 Oct 2019, 20650 Bytes) of package /linux/misc/openstack/neutron-14.0.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "eswitch_manager.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 14.0.1_vs_14.0.2.

    1 # Copyright 2014 Mellanox Technologies, Ltd
    2 #
    3 # Licensed under the Apache License, Version 2.0 (the "License");
    4 # you may not use this file except in compliance with the License.
    5 # You may obtain a copy of the License at
    6 #
    7 #    http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 # Unless required by applicable law or agreed to in writing, software
   10 # distributed under the License is distributed on an "AS IS" BASIS,
   11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
   12 # implied.
   13 # See the License for the specific language governing permissions and
   14 # limitations under the License.
   15 
   16 import os
   17 import re
   18 
   19 from neutron_lib.utils import helpers
   20 from oslo_log import log as logging
   21 
   22 from neutron._i18n import _
   23 from neutron.agent.linux import ip_link_support
   24 from neutron.plugins.ml2.drivers.mech_sriov.agent.common \
   25     import exceptions as exc
   26 from neutron.plugins.ml2.drivers.mech_sriov.agent import pci_lib
   27 
   28 LOG = logging.getLogger(__name__)
   29 
   30 
   31 class PciOsWrapper(object):
   32     """OS wrapper for checking virtual functions"""
   33 
   34     DEVICE_PATH = "/sys/class/net/%s/device"
   35     PCI_PATH = "/sys/class/net/%s/device/virtfn%s/net"
   36     NUMVFS_PATH = "/sys/class/net/%s/device/sriov_numvfs"
   37     VIRTFN_FORMAT = r"^virtfn(?P<vf_index>\d+)"
   38     VIRTFN_REG_EX = re.compile(VIRTFN_FORMAT)
   39 
   40     @classmethod
   41     def scan_vf_devices(cls, dev_name):
   42         """Scan os directories to get VF devices
   43 
   44         @param dev_name: pf network device name
   45         @return: list of virtual functions
   46         """
   47         vf_list = []
   48         dev_path = cls.DEVICE_PATH % dev_name
   49         if not os.path.isdir(dev_path):
   50             LOG.error("Failed to get devices for %s", dev_name)
   51             raise exc.InvalidDeviceError(dev_name=dev_name,
   52                                          reason=_("Device not found"))
   53         file_list = os.listdir(dev_path)
   54         for file_name in file_list:
   55             pattern_match = cls.VIRTFN_REG_EX.match(file_name)
   56             if pattern_match:
   57                 vf_index = int(pattern_match.group("vf_index"))
   58                 file_path = os.path.join(dev_path, file_name)
   59                 if os.path.islink(file_path):
   60                     file_link = os.readlink(file_path)
   61                     pci_slot = os.path.basename(file_link)
   62                     vf_list.append((pci_slot, vf_index))
   63         return vf_list
   64 
   65     @classmethod
   66     def pf_device_exists(cls, dev_name):
   67         return os.path.isdir(cls.DEVICE_PATH % dev_name)
   68 
   69     @classmethod
   70     def is_assigned_vf(cls, dev_name, vf_index, ip_link_show_output):
   71         """Check if VF is assigned.
   72 
   73         Checks if a given vf index of a given device name is assigned
   74         by checking the relevant path in the system:
   75         VF is assigned if:
   76             Direct VF: PCI_PATH does not exist.
   77             Macvtap VF: macvtap@<vf interface> interface exists in ip link show
   78         @param dev_name: pf network device name
   79         @param vf_index: vf index
   80         @param ip_link_show_output: 'ip link show' output
   81         """
   82 
   83         if not cls.pf_device_exists(dev_name):
   84             # If the root PCI path does not exist, then the VF cannot
   85             # actually have been allocated and there is no way we can
   86             # manage it.
   87             return False
   88 
   89         path = cls.PCI_PATH % (dev_name, vf_index)
   90 
   91         try:
   92             ifname_list = os.listdir(path)
   93         except OSError:
   94             # PCI_PATH does not exist means that the DIRECT VF assigned
   95             return True
   96 
   97         # Note(moshele) kernel < 3.13 doesn't create symbolic link
   98         # for macvtap interface. Therefore we workaround it
   99         # by parsing ip link show and checking if macvtap interface exists
  100         for ifname in ifname_list:
  101             if pci_lib.PciDeviceIPWrapper.is_macvtap_assigned(
  102                     ifname, ip_link_show_output):
  103                 return True
  104         return False
  105 
  106     @classmethod
  107     def get_numvfs(cls, dev_name):
  108         """Get configured number of VFs on device
  109 
  110         @param dev_name: pf network device name
  111         @return: integer number of VFs or -1
  112         if sriov_numvfs file not found (device doesn't support this config)
  113         """
  114         try:
  115             with open(cls.NUMVFS_PATH % dev_name) as f:
  116                 numvfs = int(f.read())
  117                 LOG.debug("Number of VFs configured on device %s: %s",
  118                     dev_name, numvfs)
  119                 return numvfs
  120         except IOError:
  121             LOG.warning("Error reading sriov_numvfs file for device %s, "
  122                         "probably not supported by this device", dev_name)
  123             return -1
  124 
  125 
  126 class EmbSwitch(object):
  127     """Class to manage logical embedded switch entity.
  128 
  129     Embedded Switch object is logical entity representing all VFs
  130     connected to  same physical network
  131     Each physical network is mapped to PF network device interface,
  132     meaning all its VF, excluding the devices in exclude_device list.
  133     @ivar pci_slot_map: dictionary for mapping each pci slot to vf index
  134     @ivar pci_dev_wrapper: pci device wrapper
  135     """
  136 
  137     def __init__(self, dev_name, exclude_devices):
  138         """Constructor
  139 
  140         @param dev_name: network device name
  141         @param exclude_devices: list of pci slots to exclude
  142         """
  143         self.dev_name = dev_name
  144         self.pci_slot_map = {}
  145         self.scanned_pci_list = []
  146         self.pci_dev_wrapper = pci_lib.PciDeviceIPWrapper(dev_name)
  147 
  148         self._load_devices(exclude_devices)
  149 
  150     def _load_devices(self, exclude_devices):
  151         """Load devices from driver and filter if needed.
  152 
  153         @param exclude_devices: excluded devices mapping device_name: pci slots
  154         """
  155         self.scanned_pci_list = PciOsWrapper.scan_vf_devices(self.dev_name)
  156         for pci_slot, vf_index in self.scanned_pci_list:
  157             if pci_slot not in exclude_devices:
  158                 self.pci_slot_map[pci_slot] = vf_index
  159 
  160     def get_pci_slot_list(self):
  161         """Get list of VF addresses."""
  162         return self.pci_slot_map.keys()
  163 
  164     def get_assigned_devices_info(self):
  165         """Get assigned Virtual Functions mac and pci slot
  166         information and populates vf_to_pci_slot mappings
  167 
  168         @return: list of VF pair (mac address, pci slot)
  169         """
  170         vf_to_pci_slot_mapping = {}
  171         assigned_devices_info = []
  172         ls = self.pci_dev_wrapper.link_show()
  173         for pci_slot, vf_index in self.pci_slot_map.items():
  174             if not PciOsWrapper.is_assigned_vf(self.dev_name, vf_index, ls):
  175                 continue
  176             vf_to_pci_slot_mapping[vf_index] = pci_slot
  177         if vf_to_pci_slot_mapping:
  178             vf_to_mac_mapping = self.pci_dev_wrapper.get_assigned_macs(
  179                 list(vf_to_pci_slot_mapping.keys()))
  180             for vf_index, mac in vf_to_mac_mapping.items():
  181                 pci_slot = vf_to_pci_slot_mapping[vf_index]
  182                 assigned_devices_info.append((mac, pci_slot))
  183         return assigned_devices_info
  184 
  185     def get_device_state(self, pci_slot):
  186         """Get device state.
  187 
  188         @param pci_slot: Virtual Function address
  189         """
  190         vf_index = self._get_vf_index(pci_slot)
  191         return self.pci_dev_wrapper.get_vf_state(vf_index)
  192 
  193     def set_device_state(self, pci_slot, state, propagate_uplink_state):
  194         """Set device state.
  195 
  196         @param pci_slot: Virtual Function address
  197         @param state: link state
  198         """
  199         vf_index = self._get_vf_index(pci_slot)
  200         return self.pci_dev_wrapper.set_vf_state(vf_index, state,
  201                                                  auto=propagate_uplink_state)
  202 
  203     def set_device_rate(self, pci_slot, rate_type, rate_kbps):
  204         """Set device rate: rate (max_tx_rate), min_tx_rate
  205 
  206         @param pci_slot: Virtual Function address
  207         @param rate_type: device rate name type. Could be 'rate' and
  208                           'min_tx_rate'.
  209         @param rate_kbps: device rate in kbps
  210         """
  211         vf_index = self._get_vf_index(pci_slot)
  212         # NOTE(ralonsoh): ip link sets rate in Mbps therefore we need to
  213         # convert the rate_kbps value from kbps to Mbps.
  214         # Zero means to disable the rate so the lowest rate available is 1Mbps.
  215         # Floating numbers are not allowed
  216         if 0 < rate_kbps < 1000:
  217             rate_mbps = 1
  218         else:
  219             rate_mbps = helpers.round_val(rate_kbps / 1000.0)
  220 
  221         log_dict = {
  222             'rate_mbps': rate_mbps,
  223             'rate_kbps': rate_kbps,
  224             'vf_index': vf_index,
  225             'rate_type': rate_type
  226         }
  227         if rate_kbps % 1000 != 0:
  228             LOG.debug("'%(rate_type)s' for SR-IOV ports is counted in Mbps; "
  229                       "setting %(rate_mbps)s Mbps limit for port %(vf_index)s "
  230                       "instead of %(rate_kbps)s kbps",
  231                       log_dict)
  232         else:
  233             LOG.debug("Setting %(rate_mbps)s Mbps limit for port %(vf_index)s",
  234                       log_dict)
  235 
  236         return self.pci_dev_wrapper.set_vf_rate(vf_index, rate_type, rate_mbps)
  237 
  238     def _get_vf_index(self, pci_slot):
  239         vf_index = self.pci_slot_map.get(pci_slot)
  240         if vf_index is None:
  241             LOG.warning("Cannot find vf index for pci slot %s",
  242                         pci_slot)
  243             raise exc.InvalidPciSlotError(pci_slot=pci_slot)
  244         return vf_index
  245 
  246     def set_device_spoofcheck(self, pci_slot, enabled):
  247         """Set device spoofchecking
  248 
  249         @param pci_slot: Virtual Function address
  250         @param enabled: True to enable spoofcheck, False to disable
  251         """
  252         vf_index = self.pci_slot_map.get(pci_slot)
  253         if vf_index is None:
  254             raise exc.InvalidPciSlotError(pci_slot=pci_slot)
  255         return self.pci_dev_wrapper.set_vf_spoofcheck(vf_index, enabled)
  256 
  257     def get_pci_device(self, pci_slot):
  258         """Get mac address for given Virtual Function address
  259 
  260         @param pci_slot: pci slot
  261         @return: MAC address of virtual function
  262         """
  263         vf_index = self.pci_slot_map.get(pci_slot)
  264         mac = None
  265         if vf_index is not None:
  266             ls = self.pci_dev_wrapper.link_show()
  267             if PciOsWrapper.is_assigned_vf(self.dev_name, vf_index, ls):
  268                 macs = self.pci_dev_wrapper.get_assigned_macs([vf_index])
  269                 mac = macs.get(vf_index)
  270         return mac
  271 
  272 
  273 class ESwitchManager(object):
  274     """Manages logical Embedded Switch entities for physical network."""
  275 
  276     def __new__(cls):
  277         # make it a singleton
  278         if not hasattr(cls, '_instance'):
  279             cls._instance = super(ESwitchManager, cls).__new__(cls)
  280             cls.emb_switches_map = {}
  281             cls.pci_slot_map = {}
  282             cls.skipped_devices = set()
  283         return cls._instance
  284 
  285     def device_exists(self, device_mac, pci_slot):
  286         """Verify if device exists.
  287 
  288         Check if a device mac exists and matches the given VF pci slot
  289         @param device_mac: device mac
  290         @param pci_slot: VF address
  291         """
  292         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  293         if embedded_switch:
  294             return True
  295         return False
  296 
  297     def get_assigned_devices_info(self, phys_net=None):
  298         """Get all assigned devices.
  299 
  300         Get all assigned devices belongs to given embedded switch
  301         @param phys_net: physical network, if none get all assigned devices
  302         @return: set of assigned VFs (mac address, pci slot) pair
  303         """
  304         if phys_net:
  305             eswitch_objects = self.emb_switches_map.get(phys_net, set())
  306         else:
  307             eswitch_objects = set()
  308             for eswitch_list in self.emb_switches_map.values():
  309                 eswitch_objects |= set(eswitch_list)
  310         assigned_devices = set()
  311         for embedded_switch in eswitch_objects:
  312             for device in embedded_switch.get_assigned_devices_info():
  313                 assigned_devices.add(device)
  314         return assigned_devices
  315 
  316     def get_device_state(self, device_mac, pci_slot):
  317         """Get device state.
  318 
  319         Get the device state (up/enable, down/disable, or auto)
  320         @param device_mac: device mac
  321         @param pci_slot: VF PCI slot
  322         @return: device state (enable/disable/auto) None if failed
  323         """
  324         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  325         if embedded_switch:
  326             return embedded_switch.get_device_state(pci_slot)
  327         return pci_lib.LinkState.DISABLE
  328 
  329     def set_device_max_rate(self, device_mac, pci_slot, max_kbps):
  330         """Set device max rate
  331 
  332         Sets the device max rate in kbps
  333         @param device_mac: device mac
  334         @param pci_slot: pci slot
  335         @param max_kbps: device max rate in kbps
  336         """
  337         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  338         if embedded_switch:
  339             embedded_switch.set_device_rate(
  340                 pci_slot,
  341                 ip_link_support.IpLinkConstants.IP_LINK_CAPABILITY_RATE,
  342                 max_kbps)
  343 
  344     def set_device_min_tx_rate(self, device_mac, pci_slot, min_kbps):
  345         """Set device min_tx_rate
  346 
  347         Sets the device min_tx_rate in kbps
  348         @param device_mac: device mac
  349         @param pci_slot: pci slot
  350         @param max_kbps: device min_tx_rate in kbps
  351         """
  352         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  353         if embedded_switch:
  354             embedded_switch.set_device_rate(
  355                 pci_slot,
  356                 ip_link_support.IpLinkConstants.IP_LINK_CAPABILITY_MIN_TX_RATE,
  357                 min_kbps)
  358 
  359     def set_device_state(self, device_mac, pci_slot, admin_state_up,
  360                          propagate_uplink_state):
  361         """Set device state
  362 
  363         Sets the device state (up or down)
  364         @param device_mac: device mac
  365         @param pci_slot: pci slot
  366         @param admin_state_up: device admin state True/False
  367         @param propagate_uplink_state: follow uplink state True/False
  368         """
  369         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  370         if embedded_switch:
  371             embedded_switch.set_device_state(pci_slot,
  372                                              admin_state_up,
  373                                              propagate_uplink_state)
  374 
  375     def set_device_spoofcheck(self, device_mac, pci_slot, enabled):
  376         """Set device spoofcheck
  377 
  378         Sets device spoofchecking (enabled or disabled)
  379         @param device_mac: device mac
  380         @param pci_slot: pci slot
  381         @param enabled: device spoofchecking
  382         """
  383         embedded_switch = self._get_emb_eswitch(device_mac, pci_slot)
  384         if embedded_switch:
  385             embedded_switch.set_device_spoofcheck(pci_slot,
  386                                                   enabled)
  387 
  388     def _process_emb_switch_map(self, phys_net, dev_name, exclude_devices):
  389         """Process emb_switch_map
  390         @param phys_net: physical network
  391         @param dev_name: device name
  392         @param exclude_devices: PCI devices to ignore.
  393         """
  394         emb_switches = self.emb_switches_map.get(phys_net, [])
  395         for switch in emb_switches:
  396             if switch.dev_name == dev_name:
  397                 if not PciOsWrapper.pf_device_exists(dev_name):
  398                     # If the device is given to the VM as PCI-PT
  399                     # then delete the respective emb_switch from map
  400                     self.emb_switches_map.get(phys_net).remove(switch)
  401                 return
  402 
  403         # We don't know about this device at the moment, so add to the map.
  404         if PciOsWrapper.pf_device_exists(dev_name):
  405             self._create_emb_switch(
  406                 phys_net, dev_name,
  407                 exclude_devices.get(dev_name, set()))
  408 
  409     def discover_devices(self, device_mappings, exclude_devices):
  410         """Discover which Virtual functions to manage.
  411 
  412         Discover devices, and create embedded switch object for network device
  413         @param device_mappings: device mapping physical_network:device_name
  414         @param exclude_devices: excluded devices mapping device_name: pci slots
  415         """
  416         if exclude_devices is None:
  417             exclude_devices = {}
  418         for phys_net, dev_names in device_mappings.items():
  419             for dev_name in dev_names:
  420                 self._process_emb_switch_map(phys_net, dev_name,
  421                                              exclude_devices)
  422 
  423     def _create_emb_switch(self, phys_net, dev_name, exclude_devices):
  424         embedded_switch = EmbSwitch(dev_name, exclude_devices)
  425         numvfs = PciOsWrapper.get_numvfs(dev_name)
  426         if numvfs == 0:
  427             # numvfs might be 0 on pre-up state of a device
  428             # giving such devices one more chance to initialize
  429             if dev_name not in self.skipped_devices:
  430                 self.skipped_devices.add(dev_name)
  431                 LOG.info("Device %s has 0 VFs configured. Skipping "
  432                          "for now to let the device initialize", dev_name)
  433                 return
  434             else:
  435                 # looks like device indeed has 0 VFs configured
  436                 # it is probably used just as direct-physical
  437                 LOG.info("Device %s has 0 VFs configured", dev_name)
  438 
  439         numvfs_cur = len(embedded_switch.scanned_pci_list)
  440         if numvfs >= 0 and numvfs > numvfs_cur:
  441             LOG.info("Not all VFs were initialized on device %(device)s: "
  442                      "expected - %(expected)s, actual - %(actual)s. Skipping.",
  443                      {'device': dev_name, 'expected': numvfs,
  444                       'actual': numvfs_cur})
  445             self.skipped_devices.add(dev_name)
  446             return
  447 
  448         self.emb_switches_map.setdefault(phys_net, []).append(embedded_switch)
  449         for pci_slot in embedded_switch.get_pci_slot_list():
  450             self.pci_slot_map[pci_slot] = embedded_switch
  451         self.skipped_devices.discard(dev_name)
  452 
  453     def _get_emb_eswitch(self, device_mac, pci_slot):
  454         """Get embedded switch.
  455 
  456         Get embedded switch by pci slot and validate pci has device mac
  457         @param device_mac: device mac
  458         @param pci_slot: pci slot
  459         """
  460         embedded_switch = self.pci_slot_map.get(pci_slot)
  461         if embedded_switch:
  462             used_device_mac = embedded_switch.get_pci_device(pci_slot)
  463             if used_device_mac != device_mac:
  464                 LOG.warning("device pci mismatch: %(device_mac)s "
  465                             "- %(pci_slot)s",
  466                             {"device_mac": device_mac, "pci_slot": pci_slot})
  467                 embedded_switch = None
  468         return embedded_switch
  469 
  470     def clear_max_rate(self, pci_slot):
  471         """Clear the VF "rate" parameter
  472 
  473         Clear the "rate" configuration from VF by setting it to 0.
  474         @param pci_slot: VF PCI slot
  475         """
  476         self._clear_rate(
  477             pci_slot,
  478             ip_link_support.IpLinkConstants.IP_LINK_CAPABILITY_RATE)
  479 
  480     def clear_min_tx_rate(self, pci_slot):
  481         """Clear the VF "min_tx_rate" parameter
  482 
  483         Clear the "min_tx_rate" configuration from VF by setting it to 0.
  484         @param pci_slot: VF PCI slot
  485         """
  486         self._clear_rate(
  487             pci_slot,
  488             ip_link_support.IpLinkConstants.IP_LINK_CAPABILITY_MIN_TX_RATE)
  489 
  490     def _clear_rate(self, pci_slot, rate_type):
  491         """Clear the VF rate parameter specified in rate_type
  492 
  493         Clear the rate configuration from VF by setting it to 0.
  494         @param pci_slot: VF PCI slot
  495         @param rate_type: rate to clear ('rate', 'min_tx_rate')
  496         """
  497         # NOTE(Moshe Levi): we don't use the self._get_emb_eswitch here,
  498         # because when clearing the VF it may be not assigned. This happens
  499         # when libvirt releases the VF back to the hypervisor on delete VM.
  500         # Therefore we should just clear the VF rate according to pci_slot no
  501         # matter if VF is assigned or not.
  502         embedded_switch = self.pci_slot_map.get(pci_slot)
  503         if embedded_switch:
  504             # NOTE(Moshe Levi): check the pci_slot is not assigned to some
  505             # other port before resetting the rate.
  506             if embedded_switch.get_pci_device(pci_slot) is None:
  507                 embedded_switch.set_device_rate(pci_slot, rate_type, 0)
  508             else:
  509                 LOG.warning("VF with PCI slot %(pci_slot)s is already "
  510                             "assigned; skipping reset for '%(rate_type)s' "
  511                             "device configuration parameter",
  512                             {'pci_slot': pci_slot, 'rate_type': rate_type})
  513         else:
  514             LOG.error("PCI slot %(pci_slot)s has no mapping to Embedded "
  515                       "Switch; skipping", {'pci_slot': pci_slot})