"Fossies" - the Fresh Open Source Software Archive

Member "nova-22.0.1/nova/pci/manager.py" (19 Nov 2020, 17666 Bytes) of package /linux/misc/openstack/nova-22.0.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "manager.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 22.0.0_vs_22.0.1.

    1 # Copyright (c) 2013 Intel, Inc.
    2 # Copyright (c) 2013 OpenStack Foundation
    3 # All Rights Reserved.
    4 #
    5 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    6 #    not use this file except in compliance with the License. You may obtain
    7 #    a copy of the License at
    8 #
    9 #         http://www.apache.org/licenses/LICENSE-2.0
   10 #
   11 #    Unless required by applicable law or agreed to in writing, software
   12 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   13 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   14 #    License for the specific language governing permissions and limitations
   15 #    under the License.
   16 
   17 import collections
   18 
   19 from oslo_config import cfg
   20 from oslo_log import log as logging
   21 from oslo_serialization import jsonutils
   22 
   23 from nova import exception
   24 from nova import objects
   25 from nova.objects import fields
   26 from nova.pci import stats
   27 from nova.pci import whitelist
   28 
   29 CONF = cfg.CONF
   30 LOG = logging.getLogger(__name__)
   31 
   32 
   33 class PciDevTracker(object):
   34     """Manage pci devices in a compute node.
   35 
   36     This class fetches pci passthrough information from hypervisor
   37     and tracks the usage of these devices.
   38 
   39     It's called by compute node resource tracker to allocate and free
   40     devices to/from instances, and to update the available pci passthrough
   41     device information from the hypervisor periodically.
   42 
   43     The `pci_devs` attribute of this class is the in-memory "master copy" of
   44     all devices on each compute host, and all data changes that happen when
   45     claiming/allocating/freeing devices HAVE TO be made against instances
   46     contained in `pci_devs` list, because they are periodically flushed to the
   47     DB when the save() method is called.
   48 
   49     It is unsafe to fetch PciDevice objects elsewhere in the code for update
   50     purposes as those changes will end up being overwritten when the `pci_devs`
   51     are saved.
   52     """
   53 
   54     def __init__(self, context, node_id=None):
   55         """Create a pci device tracker.
   56 
   57         If a node_id is passed in, it will fetch pci devices information
   58         from database, otherwise, it will create an empty devices list
   59         and the resource tracker will update the node_id information later.
   60         """
   61 
   62         super(PciDevTracker, self).__init__()
   63         self.stale = {}
   64         self.node_id = node_id
   65         self.dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist)
   66         self.stats = stats.PciDeviceStats(dev_filter=self.dev_filter)
   67         self._context = context
   68         if node_id:
   69             self.pci_devs = objects.PciDeviceList.get_by_compute_node(
   70                     context, node_id)
   71         else:
   72             self.pci_devs = objects.PciDeviceList(objects=[])
   73         self._build_device_tree(self.pci_devs)
   74         self._initial_instance_usage()
   75 
   76     def _initial_instance_usage(self):
   77         self.allocations = collections.defaultdict(list)
   78         self.claims = collections.defaultdict(list)
   79         for dev in self.pci_devs:
   80             uuid = dev.instance_uuid
   81             if dev.status == fields.PciDeviceStatus.CLAIMED:
   82                 self.claims[uuid].append(dev)
   83             elif dev.status == fields.PciDeviceStatus.ALLOCATED:
   84                 self.allocations[uuid].append(dev)
   85             elif dev.status == fields.PciDeviceStatus.AVAILABLE:
   86                 self.stats.add_device(dev)
   87 
   88     def save(self, context):
   89         for dev in self.pci_devs:
   90             if dev.obj_what_changed():
   91                 with dev.obj_alternate_context(context):
   92                     dev.save()
   93                     if dev.status == fields.PciDeviceStatus.DELETED:
   94                         self.pci_devs.objects.remove(dev)
   95 
   96     @property
   97     def pci_stats(self):
   98         return self.stats
   99 
  100     def update_devices_from_hypervisor_resources(self, devices_json):
  101         """Sync the pci device tracker with hypervisor information.
  102 
  103         To support pci device hot plug, we sync with the hypervisor
  104         periodically, fetching all devices information from hypervisor,
  105         update the tracker and sync the DB information.
  106 
  107         Devices should not be hot-plugged when assigned to a guest,
  108         but possibly the hypervisor has no such guarantee. The best
  109         we can do is to give a warning if a device is changed
  110         or removed while assigned.
  111 
  112         :param devices_json: The JSON-ified string of device information
  113                              that is returned from the virt driver's
  114                              get_available_resource() call in the
  115                              pci_passthrough_devices key.
  116         """
  117 
  118         devices = []
  119         for dev in jsonutils.loads(devices_json):
  120             if self.dev_filter.device_assignable(dev):
  121                 devices.append(dev)
  122         self._set_hvdevs(devices)
  123 
  124     @staticmethod
  125     def _build_device_tree(all_devs):
  126         """Build a tree of devices that represents parent-child relationships.
  127 
  128         We need to have the relationships set up so that we can easily make
  129         all the necessary changes to parent/child devices without having to
  130         figure it out at each call site.
  131 
  132         This method just adds references to relevant instances already found
  133         in `pci_devs` to `child_devices` and `parent_device` fields of each
  134         one.
  135 
  136         Currently relationships are considered for SR-IOV PFs/VFs only.
  137         """
  138 
  139         # Ensures that devices are ordered in ASC so VFs will come
  140         # after their PFs.
  141         all_devs.sort(key=lambda x: x.address)
  142 
  143         parents = {}
  144         for dev in all_devs:
  145             if dev.status in (fields.PciDeviceStatus.REMOVED,
  146                               fields.PciDeviceStatus.DELETED):
  147                 # NOTE(ndipanov): Removed devs are pruned from
  148                 # self.pci_devs on save() so we need to make sure we
  149                 # are not looking at removed ones as we may build up
  150                 # the tree sooner than they are pruned.
  151                 continue
  152             if dev.dev_type == fields.PciDeviceType.SRIOV_PF:
  153                 dev.child_devices = []
  154                 parents[dev.address] = dev
  155             elif dev.dev_type == fields.PciDeviceType.SRIOV_VF:
  156                 dev.parent_device = parents.get(dev.parent_addr)
  157                 if dev.parent_device:
  158                     parents[dev.parent_addr].child_devices.append(dev)
  159 
  160     def _set_hvdevs(self, devices):
  161         exist_addrs = set([dev.address for dev in self.pci_devs])
  162         new_addrs = set([dev['address'] for dev in devices])
  163 
  164         for existed in self.pci_devs:
  165             if existed.address in exist_addrs - new_addrs:
  166                 # Remove previously tracked PCI devices that are either
  167                 # no longer reported by the hypervisor or have been removed
  168                 # from the pci whitelist.
  169                 try:
  170                     existed.remove()
  171                 except exception.PciDeviceInvalidStatus as e:
  172                     LOG.warning("Unable to remove device with %(status)s "
  173                                 "ownership %(instance_uuid)s because of "
  174                                 "%(pci_exception)s. "
  175                                 "Check your [pci]passthrough_whitelist "
  176                                 "configuration to make sure this allocated "
  177                                 "device is whitelisted. If you have removed "
  178                                 "the device from the whitelist intentionally "
  179                                 "or the device is no longer available on the "
  180                                 "host you will need to delete the server or "
  181                                 "migrate it to another host to silence this "
  182                                 "warning.",
  183                                 {'status': existed.status,
  184                                  'instance_uuid': existed.instance_uuid,
  185                                  'pci_exception': e.format_message()})
  186                     # NOTE(sean-k-mooney): the device may not be tracked for
  187                     # two reasons: first the device could have been removed
  188                     # from the host or second the whitelist could have been
  189                     # updated. While force removing may seam reasonable, if
  190                     # the device is allocated to a vm, force removing the
  191                     # device entry from the resource tracker can prevent the vm
  192                     # from rebooting. If the PCI device was removed due to an
  193                     # update to the PCI whitelist which was later reverted,
  194                     # removing the entry from the database and adding it back
  195                     # later may lead to the scheduler incorrectly selecting
  196                     # this host and the ResourceTracker assigning the PCI
  197                     # device to a second vm. To prevent this bug we skip
  198                     # deleting the device from the db in this iteration and
  199                     # will try again on the next sync.
  200                     continue
  201                 else:
  202                     # Note(yjiang5): no need to update stats if an assigned
  203                     # device is hot removed.
  204                     self.stats.remove_device(existed)
  205             else:
  206                 # Update tracked devices.
  207                 new_value = next((dev for dev in devices if
  208                     dev['address'] == existed.address))
  209                 new_value['compute_node_id'] = self.node_id
  210                 if existed.status in (fields.PciDeviceStatus.CLAIMED,
  211                                       fields.PciDeviceStatus.ALLOCATED):
  212                     # Pci properties may change while assigned because of
  213                     # hotplug or config changes. Although normally this should
  214                     # not happen.
  215 
  216                     # As the devices have been assigned to an instance,
  217                     # we defer the change till the instance is destroyed.
  218                     # We will not sync the new properties with database
  219                     # before that.
  220 
  221                     # TODO(yjiang5): Not sure if this is a right policy, but
  222                     # at least it avoids some confusion and, if needed,
  223                     # we can add more action like killing the instance
  224                     # by force in future.
  225                     self.stale[new_value['address']] = new_value
  226                 else:
  227                     existed.update_device(new_value)
  228                     self.stats.update_device(existed)
  229 
  230         # Track newly discovered devices.
  231         for dev in [dev for dev in devices if
  232                     dev['address'] in new_addrs - exist_addrs]:
  233             dev['compute_node_id'] = self.node_id
  234             dev_obj = objects.PciDevice.create(self._context, dev)
  235             self.pci_devs.objects.append(dev_obj)
  236             self.stats.add_device(dev_obj)
  237 
  238         self._build_device_tree(self.pci_devs)
  239 
  240     def _claim_instance(self, context, pci_requests, instance_numa_topology):
  241         instance_cells = None
  242         if instance_numa_topology:
  243             instance_cells = instance_numa_topology.cells
  244 
  245         devs = self.stats.consume_requests(pci_requests.requests,
  246                                            instance_cells)
  247         if not devs:
  248             return None
  249 
  250         instance_uuid = pci_requests.instance_uuid
  251         for dev in devs:
  252             dev.claim(instance_uuid)
  253         if instance_numa_topology and any(
  254                                         dev.numa_node is None for dev in devs):
  255             LOG.warning("Assigning a pci device without numa affinity to "
  256                         "instance %(instance)s which has numa topology",
  257                         {'instance': instance_uuid})
  258         return devs
  259 
  260     def _allocate_instance(self, instance, devs):
  261         for dev in devs:
  262             dev.allocate(instance)
  263 
  264     def allocate_instance(self, instance):
  265         devs = self.claims.pop(instance['uuid'], [])
  266         self._allocate_instance(instance, devs)
  267         if devs:
  268             self.allocations[instance['uuid']] += devs
  269 
  270     def claim_instance(self, context, pci_requests, instance_numa_topology):
  271         devs = []
  272         if self.pci_devs and pci_requests.requests:
  273             instance_uuid = pci_requests.instance_uuid
  274             devs = self._claim_instance(context, pci_requests,
  275                                         instance_numa_topology)
  276             if devs:
  277                 self.claims[instance_uuid] = devs
  278         return devs
  279 
  280     def free_device(self, dev, instance):
  281         """Free device from pci resource tracker
  282 
  283         :param dev: cloned pci device object that needs to be free
  284         :param instance: the instance that this pci device
  285                          is allocated to
  286         """
  287         for pci_dev in self.pci_devs:
  288             # Find the matching pci device in the pci resource tracker.
  289             # Once found, free it.
  290             if dev.id == pci_dev.id and dev.instance_uuid == instance['uuid']:
  291                 self._remove_device_from_pci_mapping(
  292                     instance['uuid'], pci_dev, self.allocations)
  293                 self._remove_device_from_pci_mapping(
  294                     instance['uuid'], pci_dev, self.claims)
  295                 self._free_device(pci_dev)
  296                 break
  297 
  298     def _remove_device_from_pci_mapping(
  299             self, instance_uuid, pci_device, pci_mapping):
  300         """Remove a PCI device from allocations or claims.
  301 
  302         If there are no more PCI devices, pop the uuid.
  303         """
  304         pci_devices = pci_mapping.get(instance_uuid, [])
  305         if pci_device in pci_devices:
  306             pci_devices.remove(pci_device)
  307             if len(pci_devices) == 0:
  308                 pci_mapping.pop(instance_uuid, None)
  309 
  310     def _free_device(self, dev, instance=None):
  311         freed_devs = dev.free(instance)
  312         stale = self.stale.pop(dev.address, None)
  313         if stale:
  314             dev.update_device(stale)
  315         for dev in freed_devs:
  316             self.stats.add_device(dev)
  317 
  318     def free_instance_allocations(self, context, instance):
  319         """Free devices that are in ALLOCATED state for instance.
  320 
  321         :param context: user request context (nova.context.RequestContext)
  322         :param instance: instance object
  323         """
  324         if self.allocations.pop(instance['uuid'], None):
  325             for dev in self.pci_devs:
  326                 if (dev.status == fields.PciDeviceStatus.ALLOCATED and
  327                         dev.instance_uuid == instance['uuid']):
  328                     self._free_device(dev)
  329 
  330     def free_instance_claims(self, context, instance):
  331         """Free devices that are in CLAIMED state for instance.
  332 
  333         :param context: user request context (nova.context.RequestContext)
  334         :param instance: instance object
  335         """
  336         if self.claims.pop(instance['uuid'], None):
  337             for dev in self.pci_devs:
  338                 if (dev.status == fields.PciDeviceStatus.CLAIMED and
  339                         dev.instance_uuid == instance['uuid']):
  340                     self._free_device(dev)
  341 
  342     def free_instance(self, context, instance):
  343         """Free devices that are in CLAIMED or ALLOCATED state for instance.
  344 
  345         :param context: user request context (nova.context.RequestContext)
  346         :param instance: instance object
  347         """
  348         # Note(yjiang5): When an instance is resized, the devices in the
  349         # destination node are claimed to the instance in prep_resize stage.
  350         # However, the instance contains only allocated devices
  351         # information, not the claimed one. So we can't use
  352         # instance['pci_devices'] to check the devices to be freed.
  353         self.free_instance_allocations(context, instance)
  354         self.free_instance_claims(context, instance)
  355 
  356     def update_pci_for_instance(self, context, instance, sign):
  357         """Update PCI usage information if devices are de/allocated.
  358         """
  359         if not self.pci_devs:
  360             return
  361 
  362         if sign == -1:
  363             self.free_instance(context, instance)
  364         if sign == 1:
  365             self.allocate_instance(instance)
  366 
  367     def clean_usage(self, instances, migrations, orphans):
  368         """Remove all usages for instances not passed in the parameter.
  369 
  370         The caller should hold the COMPUTE_RESOURCE_SEMAPHORE lock
  371         """
  372         existed = set(inst['uuid'] for inst in instances)
  373         existed |= set(mig['instance_uuid'] for mig in migrations)
  374         existed |= set(inst['uuid'] for inst in orphans)
  375 
  376         # need to copy keys, because the dict is modified in the loop body
  377         for uuid in list(self.claims):
  378             if uuid not in existed:
  379                 devs = self.claims.pop(uuid, [])
  380                 for dev in devs:
  381                     self._free_device(dev)
  382         # need to copy keys, because the dict is modified in the loop body
  383         for uuid in list(self.allocations):
  384             if uuid not in existed:
  385                 devs = self.allocations.pop(uuid, [])
  386                 for dev in devs:
  387                     self._free_device(dev)
  388 
  389 
  390 def get_instance_pci_devs(inst, request_id=None):
  391     """Get the devices allocated to one or all requests for an instance.
  392 
  393     - For generic PCI request, the request id is None.
  394     - For sr-iov networking, the request id is a valid uuid
  395     - There are a couple of cases where all the PCI devices allocated to an
  396       instance need to be returned. Refer to libvirt driver that handles
  397       soft_reboot and hard_boot of 'xen' instances.
  398     """
  399     pci_devices = inst.pci_devices
  400     if pci_devices is None:
  401         return []
  402     return [device for device in pci_devices if
  403                    device.request_id == request_id or request_id == 'all']