nova-22.0.1.tar.gz: .../virt/libvirt/host.py

"Fossies" - the Fresh Open Source Software Archive

Member "nova-22.0.1/nova/virt/libvirt/host.py" (19 Nov 2020, 52211 Bytes) of package /linux/misc/openstack/nova-22.0.1.tar.gz:

As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "host.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report:

22.0.0_vs_22.0.1.

1 # Copyright 2010 United States Government as represented by the 2 # Administrator of the National Aeronautics and Space Administration. 3 # All Rights Reserved. 4 # Copyright (c) 2010 Citrix Systems, Inc. 5 # Copyright (c) 2011 Piston Cloud Computing, Inc 6 # Copyright (c) 2012 University Of Minho 7 # (c) Copyright 2013 Hewlett-Packard Development Company, L.P. 8 # 9 # Licensed under the Apache License, Version 2.0 (the "License"); you may 10 # not use this file except in compliance with the License. You may obtain 11 # a copy of the License at 12 # 13 # http://www.apache.org/licenses/LICENSE-2.0 14 # 15 # Unless required by applicable law or agreed to in writing, software 16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 18 # License for the specific language governing permissions and limitations 19 # under the License. 20 21 """ 22 Manages information about the host OS and hypervisor. 23 24 This class encapsulates a connection to the libvirt 25 daemon and provides certain higher level APIs around 26 the raw libvirt API. These APIs are then used by all 27 the other libvirt related classes 28 """ 29 30 from collections import defaultdict 31 import inspect 32 import operator 33 import os 34 import queue 35 import socket 36 import threading 37 import typing as ty 38 39 from eventlet import greenio 40 from eventlet import greenthread 41 from eventlet import patcher 42 from eventlet import tpool 43 from oslo_log import log as logging 44 from oslo_utils import excutils 45 from oslo_utils import importutils 46 from oslo_utils import units 47 from oslo_utils import versionutils 48 49 from nova.compute import utils as compute_utils 50 import nova.conf 51 from nova import context as nova_context 52 from nova import exception 53 from nova.i18n import _ 54 from nova import rpc 55 from nova import utils 56 from nova.virt import event as virtevent 57 from nova.virt.libvirt import config as vconfig 58 from nova.virt.libvirt import guest as libvirt_guest 59 from nova.virt.libvirt import migration as libvirt_migrate 60 from nova.virt.libvirt import utils as libvirt_utils 61 62 if ty.TYPE_CHECKING: 63 import libvirt 64 else: 65 libvirt = None 66 67 LOG = logging.getLogger(__name__) 68 69 native_socket = patcher.original('socket') 70 native_threading = patcher.original("threading") 71 native_Queue = patcher.original("queue") 72 73 CONF = nova.conf.CONF 74 75 76 # This list is for libvirt hypervisor drivers that need special handling. 77 # This is *not* the complete list of supported hypervisor drivers. 78 HV_DRIVER_QEMU = "QEMU" 79 HV_DRIVER_XEN = "Xen" 80 81 SEV_KERNEL_PARAM_FILE = '/sys/module/kvm_amd/parameters/sev' 82 83 84 class Host(object): 85 86 def __init__(self, uri, read_only=False, 87 conn_event_handler=None, 88 lifecycle_event_handler=None): 89 90 global libvirt 91 if libvirt is None: 92 libvirt = importutils.import_module('libvirt') 93 94 self._uri = uri 95 self._read_only = read_only 96 self._initial_connection = True 97 self._conn_event_handler = conn_event_handler 98 self._conn_event_handler_queue: queue.Queue[ty.Callable] = ( 99 queue.Queue()) 100 self._lifecycle_event_handler = lifecycle_event_handler 101 self._caps = None 102 self._domain_caps = None 103 self._hostname = None 104 105 self._wrapped_conn = None 106 self._wrapped_conn_lock = threading.Lock() 107 self._event_queue: ty.Optional[queue.Queue[ty.Callable]] = None 108 109 self._events_delayed = {} 110 # Note(toabctl): During a reboot of a domain, STOPPED and 111 # STARTED events are sent. To prevent shutting 112 # down the domain during a reboot, delay the 113 # STOPPED lifecycle event some seconds. 114 self._lifecycle_delay = 15 115 116 self._initialized = False 117 self._libvirt_proxy_classes = self._get_libvirt_proxy_classes(libvirt) 118 self._libvirt_proxy = self._wrap_libvirt_proxy(libvirt) 119 120 # AMD SEV is conditional on support in the hardware, kernel, 121 # qemu, and libvirt. This is determined on demand and 122 # memoized by the supports_amd_sev property below. 123 self._supports_amd_sev = None 124 125 self._has_hyperthreading = None 126 127 @staticmethod 128 def _get_libvirt_proxy_classes(libvirt_module): 129 """Return a tuple for tpool.Proxy's autowrap argument containing all 130 public vir* classes defined by the libvirt module. 131 """ 132 133 # Get a list of (name, class) tuples of libvirt classes 134 classes = inspect.getmembers(libvirt_module, inspect.isclass) 135 136 # Return a list of just the vir* classes, filtering out libvirtError 137 # and any private globals pointing at private internal classes. 138 return tuple([cls[1] for cls in classes if cls[0].startswith("vir")]) 139 140 def _wrap_libvirt_proxy(self, obj): 141 """Return an object wrapped in a tpool.Proxy using autowrap appropriate 142 for the libvirt module. 143 """ 144 145 # libvirt is not pure python, so eventlet monkey patching doesn't work 146 # on it. Consequently long-running libvirt calls will not yield to 147 # eventlet's event loop, starving all other greenthreads until 148 # completion. eventlet's tpool.Proxy handles this situation for us by 149 # executing proxied calls in a native thread. 150 return tpool.Proxy(obj, autowrap=self._libvirt_proxy_classes) 151 152 def _native_thread(self): 153 """Receives async events coming in from libvirtd. 154 155 This is a native thread which runs the default 156 libvirt event loop implementation. This processes 157 any incoming async events from libvirtd and queues 158 them for later dispatch. This thread is only 159 permitted to use libvirt python APIs, and the 160 driver.queue_event method. In particular any use 161 of logging is forbidden, since it will confuse 162 eventlet's greenthread integration 163 """ 164 165 while True: 166 libvirt.virEventRunDefaultImpl() 167 168 def _dispatch_thread(self): 169 """Dispatches async events coming in from libvirtd. 170 171 This is a green thread which waits for events to 172 arrive from the libvirt event loop thread. This 173 then dispatches the events to the compute manager. 174 """ 175 176 while True: 177 self._dispatch_events() 178 179 def _conn_event_thread(self): 180 """Dispatches async connection events""" 181 # NOTE(mdbooth): This thread doesn't need to jump through the same 182 # hoops as _dispatch_thread because it doesn't interact directly 183 # with the libvirt native thread. 184 while True: 185 self._dispatch_conn_event() 186 187 def _dispatch_conn_event(self): 188 # NOTE(mdbooth): Splitting out this loop looks redundant, but it 189 # means we can easily dispatch events synchronously from tests and 190 # it isn't completely awful. 191 handler = self._conn_event_handler_queue.get() 192 try: 193 handler() 194 except Exception: 195 LOG.exception('Exception handling connection event') 196 finally: 197 self._conn_event_handler_queue.task_done() 198 199 @staticmethod 200 def _event_lifecycle_callback(conn, dom, event, detail, opaque): 201 """Receives lifecycle events from libvirt. 202 203 NB: this method is executing in a native thread, not 204 an eventlet coroutine. It can only invoke other libvirt 205 APIs, or use self._queue_event(). Any use of logging APIs 206 in particular is forbidden. 207 """ 208 209 self = opaque 210 211 uuid = dom.UUIDString() 212 transition = None 213 if event == libvirt.VIR_DOMAIN_EVENT_STOPPED: 214 transition = virtevent.EVENT_LIFECYCLE_STOPPED 215 elif event == libvirt.VIR_DOMAIN_EVENT_STARTED: 216 transition = virtevent.EVENT_LIFECYCLE_STARTED 217 elif event == libvirt.VIR_DOMAIN_EVENT_SUSPENDED: 218 if detail == libvirt.VIR_DOMAIN_EVENT_SUSPENDED_POSTCOPY: 219 transition = virtevent.EVENT_LIFECYCLE_POSTCOPY_STARTED 220 elif detail == libvirt.VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED: 221 # VIR_DOMAIN_EVENT_SUSPENDED_MIGRATED is also sent when live 222 # migration of the guest fails, so we cannot simply rely 223 # on the event itself but need to check if the job itself was 224 # successful. 225 # NOTE(mriedem): The job check logic here is copied from 226 # LibvirtDriver._live_migration_monitor. 227 guest = libvirt_guest.Guest(dom) 228 info = guest.get_job_info() 229 if info.type == libvirt.VIR_DOMAIN_JOB_NONE: 230 # Either still running, or failed or completed, 231 # lets untangle the mess. 232 info.type = libvirt_migrate.find_job_type( 233 guest, instance=None, logging_ok=False) 234 235 if info.type == libvirt.VIR_DOMAIN_JOB_COMPLETED: 236 transition = virtevent.EVENT_LIFECYCLE_MIGRATION_COMPLETED 237 else: 238 # Failed or some other status we don't know about, so just 239 # opt to report the guest is paused. 240 transition = virtevent.EVENT_LIFECYCLE_PAUSED 241 else: 242 transition = virtevent.EVENT_LIFECYCLE_PAUSED 243 elif event == libvirt.VIR_DOMAIN_EVENT_RESUMED: 244 transition = virtevent.EVENT_LIFECYCLE_RESUMED 245 246 if transition is not None: 247 self._queue_event(virtevent.LifecycleEvent(uuid, transition)) 248 249 def _close_callback(self, conn, reason, opaque): 250 close_info = {'conn': conn, 'reason': reason} 251 self._queue_event(close_info) 252 253 @staticmethod 254 def _test_connection(conn): 255 try: 256 conn.getLibVersion() 257 return True 258 except libvirt.libvirtError as e: 259 if (e.get_error_code() in (libvirt.VIR_ERR_SYSTEM_ERROR, 260 libvirt.VIR_ERR_INTERNAL_ERROR) and 261 e.get_error_domain() in (libvirt.VIR_FROM_REMOTE, 262 libvirt.VIR_FROM_RPC)): 263 LOG.debug('Connection to libvirt broke') 264 return False 265 raise 266 267 @staticmethod 268 def _connect_auth_cb(creds, opaque): 269 if len(creds) == 0: 270 return 0 271 raise exception.InternalError( 272 _("Can not handle authentication request for %d credentials") 273 % len(creds)) 274 275 def _connect(self, uri, read_only): 276 auth = [[libvirt.VIR_CRED_AUTHNAME, 277 libvirt.VIR_CRED_ECHOPROMPT, 278 libvirt.VIR_CRED_REALM, 279 libvirt.VIR_CRED_PASSPHRASE, 280 libvirt.VIR_CRED_NOECHOPROMPT, 281 libvirt.VIR_CRED_EXTERNAL], 282 Host._connect_auth_cb, 283 None] 284 285 flags = 0 286 if read_only: 287 flags = libvirt.VIR_CONNECT_RO 288 return self._libvirt_proxy.openAuth(uri, auth, flags) 289 290 def _queue_event(self, event): 291 """Puts an event on the queue for dispatch. 292 293 This method is called by the native event thread to 294 put events on the queue for later dispatch by the 295 green thread. Any use of logging APIs is forbidden. 296 """ 297 298 if self._event_queue is None: 299 return 300 301 # Queue the event... 302 self._event_queue.put(event) 303 304 # ...then wakeup the green thread to dispatch it 305 c = ' '.encode() 306 self._event_notify_send.write(c) 307 self._event_notify_send.flush() 308 309 def _dispatch_events(self): 310 """Wait for & dispatch events from native thread 311 312 Blocks until native thread indicates some events 313 are ready. Then dispatches all queued events. 314 """ 315 316 # Wait to be notified that there are some 317 # events pending 318 try: 319 _c = self._event_notify_recv.read(1) 320 assert _c 321 except ValueError: 322 return # will be raised when pipe is closed 323 324 # Process as many events as possible without 325 # blocking 326 last_close_event = None 327 # required for mypy 328 if self._event_queue is None: 329 return 330 while not self._event_queue.empty(): 331 try: 332 event_type = ty.Union[ 333 virtevent.LifecycleEvent, ty.Mapping[str, ty.Any]] 334 event: event_type = self._event_queue.get(block=False) 335 if isinstance(event, virtevent.LifecycleEvent): 336 # call possibly with delay 337 self._event_emit_delayed(event) 338 339 elif 'conn' in event and 'reason' in event: 340 last_close_event = event 341 except native_Queue.Empty: 342 pass 343 if last_close_event is None: 344 return 345 conn = last_close_event['conn'] 346 # get_new_connection may already have disabled the host, 347 # in which case _wrapped_conn is None. 348 with self._wrapped_conn_lock: 349 if conn == self._wrapped_conn: 350 reason = str(last_close_event['reason']) 351 msg = _("Connection to libvirt lost: %s") % reason 352 self._wrapped_conn = None 353 self._queue_conn_event_handler(False, msg) 354 355 def _event_emit_delayed(self, event): 356 """Emit events - possibly delayed.""" 357 def event_cleanup(gt, *args, **kwargs): 358 """Callback function for greenthread. Called 359 to cleanup the _events_delayed dictionary when an event 360 was called. 361 """ 362 event = args[0] 363 self._events_delayed.pop(event.uuid, None) 364 365 # Cleanup possible delayed stop events. 366 if event.uuid in self._events_delayed.keys(): 367 self._events_delayed[event.uuid].cancel() 368 self._events_delayed.pop(event.uuid, None) 369 LOG.debug("Removed pending event for %s due to " 370 "lifecycle event", event.uuid) 371 372 if event.transition == virtevent.EVENT_LIFECYCLE_STOPPED: 373 # Delay STOPPED event, as they may be followed by a STARTED 374 # event in case the instance is rebooting 375 id_ = greenthread.spawn_after(self._lifecycle_delay, 376 self._event_emit, event) 377 self._events_delayed[event.uuid] = id_ 378 # add callback to cleanup self._events_delayed dict after 379 # event was called 380 id_.link(event_cleanup, event) 381 else: 382 self._event_emit(event) 383 384 def _event_emit(self, event): 385 if self._lifecycle_event_handler is not None: 386 self._lifecycle_event_handler(event) 387 388 def _init_events_pipe(self): 389 """Create a self-pipe for the native thread to synchronize on. 390 391 This code is taken from the eventlet tpool module, under terms 392 of the Apache License v2.0. 393 """ 394 395 self._event_queue = native_Queue.Queue() 396 try: 397 rpipe, wpipe = os.pipe() 398 self._event_notify_send = greenio.GreenPipe(wpipe, 'wb', 0) 399 self._event_notify_recv = greenio.GreenPipe(rpipe, 'rb', 0) 400 except (ImportError, NotImplementedError): 401 # This is Windows compatibility -- use a socket instead 402 # of a pipe because pipes don't really exist on Windows. 403 sock = native_socket.socket(socket.AF_INET, socket.SOCK_STREAM) 404 sock.bind(('localhost', 0)) 405 sock.listen(50) 406 csock = native_socket.socket(socket.AF_INET, socket.SOCK_STREAM) 407 csock.connect(('localhost', sock.getsockname()[1])) 408 nsock, addr = sock.accept() 409 self._event_notify_send = nsock.makefile('wb', 0) 410 gsock = greenio.GreenSocket(csock) 411 self._event_notify_recv = gsock.makefile('rb', 0) 412 413 def _init_events(self): 414 """Initializes the libvirt events subsystem. 415 416 This requires running a native thread to provide the 417 libvirt event loop integration. This forwards events 418 to a green thread which does the actual dispatching. 419 """ 420 421 self._init_events_pipe() 422 423 LOG.debug("Starting native event thread") 424 self._event_thread = native_threading.Thread( 425 target=self._native_thread) 426 self._event_thread.setDaemon(True) 427 self._event_thread.start() 428 429 LOG.debug("Starting green dispatch thread") 430 utils.spawn(self._dispatch_thread) 431 432 def _get_new_connection(self): 433 # call with _wrapped_conn_lock held 434 LOG.debug('Connecting to libvirt: %s', self._uri) 435 436 # This will raise an exception on failure 437 wrapped_conn = self._connect(self._uri, self._read_only) 438 439 try: 440 LOG.debug("Registering for lifecycle events %s", self) 441 wrapped_conn.domainEventRegisterAny( 442 None, 443 libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE, 444 self._event_lifecycle_callback, 445 self) 446 except Exception as e: 447 LOG.warning("URI %(uri)s does not support events: %(error)s", 448 {'uri': self._uri, 'error': e}) 449 450 try: 451 LOG.debug("Registering for connection events: %s", str(self)) 452 wrapped_conn.registerCloseCallback(self._close_callback, None) 453 except libvirt.libvirtError as e: 454 LOG.warning("URI %(uri)s does not support connection" 455 " events: %(error)s", 456 {'uri': self._uri, 'error': e}) 457 458 return wrapped_conn 459 460 def _queue_conn_event_handler(self, *args, **kwargs): 461 if self._conn_event_handler is None: 462 return 463 464 def handler(): 465 return self._conn_event_handler(*args, **kwargs) 466 467 self._conn_event_handler_queue.put(handler) 468 469 def _get_connection(self): 470 # multiple concurrent connections are protected by _wrapped_conn_lock 471 with self._wrapped_conn_lock: 472 # Drop the existing connection if it is not usable 473 if (self._wrapped_conn is not None and 474 not self._test_connection(self._wrapped_conn)): 475 self._wrapped_conn = None 476 # Connection was previously up, and went down 477 self._queue_conn_event_handler( 478 False, _('Connection to libvirt lost')) 479 480 if self._wrapped_conn is None: 481 try: 482 # This will raise if it fails to get a connection 483 self._wrapped_conn = self._get_new_connection() 484 except Exception as ex: 485 with excutils.save_and_reraise_exception(): 486 # If we previously had a connection and it went down, 487 # we generated a down event for that above. 488 # We also want to generate a down event for an initial 489 # failure, which won't be handled above. 490 if self._initial_connection: 491 self._queue_conn_event_handler( 492 False, 493 _('Failed to connect to libvirt: %(msg)s') % 494 {'msg': ex}) 495 finally: 496 self._initial_connection = False 497 498 self._queue_conn_event_handler(True, None) 499 500 return self._wrapped_conn 501 502 def get_connection(self): 503 """Returns a connection to the hypervisor 504 505 This method should be used to create and return a well 506 configured connection to the hypervisor. 507 508 :returns: a libvirt.virConnect object 509 """ 510 try: 511 conn = self._get_connection() 512 except libvirt.libvirtError as ex: 513 LOG.exception("Connection to libvirt failed: %s", ex) 514 payload = {'ip': CONF.my_ip, 'method': '_connect', 'reason': ex} 515 ctxt = nova_context.get_admin_context() 516 rpc.get_notifier('compute').error(ctxt, 517 'compute.libvirt.error', 518 payload) 519 compute_utils.notify_about_libvirt_connect_error( 520 ctxt, ip=CONF.my_ip, exception=ex) 521 raise exception.HypervisorUnavailable() 522 523 return conn 524 525 @staticmethod 526 def _libvirt_error_handler(context, err): 527 # Just ignore instead of default outputting to stderr. 528 pass 529 530 def initialize(self): 531 if self._initialized: 532 return 533 534 # NOTE(dkliban): Error handler needs to be registered before libvirt 535 # connection is used for the first time. Otherwise, the 536 # handler does not get registered. 537 libvirt.registerErrorHandler(self._libvirt_error_handler, None) 538 libvirt.virEventRegisterDefaultImpl() 539 self._init_events() 540 541 LOG.debug("Starting connection event dispatch thread") 542 utils.spawn(self._conn_event_thread) 543 544 self._initialized = True 545 546 def _version_check(self, lv_ver=None, hv_ver=None, hv_type=None, 547 op=operator.lt): 548 """Check libvirt version, hypervisor version, and hypervisor type 549 550 :param hv_type: hypervisor driver from the top of this file. 551 """ 552 conn = self.get_connection() 553 try: 554 if lv_ver is not None: 555 libvirt_version = conn.getLibVersion() 556 if op(libvirt_version, 557 versionutils.convert_version_to_int(lv_ver)): 558 return False 559 560 if hv_ver is not None: 561 hypervisor_version = conn.getVersion() 562 if op(hypervisor_version, 563 versionutils.convert_version_to_int(hv_ver)): 564 return False 565 566 if hv_type is not None: 567 hypervisor_type = conn.getType() 568 if hypervisor_type != hv_type: 569 return False 570 571 return True 572 except Exception: 573 return False 574 575 def has_min_version(self, lv_ver=None, hv_ver=None, hv_type=None): 576 return self._version_check( 577 lv_ver=lv_ver, hv_ver=hv_ver, hv_type=hv_type, op=operator.lt) 578 579 def has_version(self, lv_ver=None, hv_ver=None, hv_type=None): 580 return self._version_check( 581 lv_ver=lv_ver, hv_ver=hv_ver, hv_type=hv_type, op=operator.ne) 582 583 def get_guest(self, instance): 584 """Retrieve libvirt guest object for an instance. 585 586 All libvirt error handling should be handled in this method and 587 relevant nova exceptions should be raised in response. 588 589 :param instance: a nova.objects.Instance object 590 591 :returns: a nova.virt.libvirt.Guest object 592 :raises exception.InstanceNotFound: The domain was not found 593 :raises exception.InternalError: A libvirt error occurred 594 """ 595 return libvirt_guest.Guest(self._get_domain(instance)) 596 597 def _get_domain(self, instance): 598 """Retrieve libvirt domain object for an instance. 599 600 All libvirt error handling should be handled in this method and 601 relevant nova exceptions should be raised in response. 602 603 :param instance: a nova.objects.Instance object 604 605 :returns: a libvirt.Domain object 606 :raises exception.InstanceNotFound: The domain was not found 607 :raises exception.InternalError: A libvirt error occurred 608 """ 609 try: 610 conn = self.get_connection() 611 return conn.lookupByUUIDString(instance.uuid) 612 except libvirt.libvirtError as ex: 613 error_code = ex.get_error_code() 614 if error_code == libvirt.VIR_ERR_NO_DOMAIN: 615 raise exception.InstanceNotFound(instance_id=instance.uuid) 616 617 msg = (_('Error from libvirt while looking up %(instance_name)s: ' 618 '[Error Code %(error_code)s] %(ex)s') % 619 {'instance_name': instance.name, 620 'error_code': error_code, 621 'ex': ex}) 622 raise exception.InternalError(msg) 623 624 def list_guests(self, only_running=True, only_guests=True): 625 """Get a list of Guest objects for nova instances 626 627 :param only_running: True to only return running instances 628 :param only_guests: True to filter out any host domain (eg Dom-0) 629 630 See method "list_instance_domains" for more information. 631 632 :returns: list of Guest objects 633 """ 634 return [libvirt_guest.Guest(dom) for dom in self.list_instance_domains( 635 only_running=only_running, only_guests=only_guests)] 636 637 def list_instance_domains(self, only_running=True, only_guests=True): 638 """Get a list of libvirt.Domain objects for nova instances 639 640 :param only_running: True to only return running instances 641 :param only_guests: True to filter out any host domain (eg Dom-0) 642 643 Query libvirt to a get a list of all libvirt.Domain objects 644 that correspond to nova instances. If the only_running parameter 645 is true this list will only include active domains, otherwise 646 inactive domains will be included too. If the only_guests parameter 647 is true the list will have any "host" domain (aka Xen Domain-0) 648 filtered out. 649 650 :returns: list of libvirt.Domain objects 651 """ 652 flags = libvirt.VIR_CONNECT_LIST_DOMAINS_ACTIVE 653 if not only_running: 654 flags = flags | libvirt.VIR_CONNECT_LIST_DOMAINS_INACTIVE 655 656 # listAllDomains() returns <list of virDomain>, not <virDomain>, so 657 # tpool.Proxy's autowrap won't catch it. We need to wrap the 658 # contents of the list we return. 659 alldoms = (self._wrap_libvirt_proxy(dom) 660 for dom in self.get_connection().listAllDomains(flags)) 661 662 doms = [] 663 for dom in alldoms: 664 if only_guests and dom.ID() == 0: 665 continue 666 doms.append(dom) 667 668 return doms 669 670 def get_online_cpus(self): 671 """Get the set of CPUs that are online on the host 672 673 :returns: set of online CPUs, raises libvirtError on error 674 """ 675 cpus, cpu_map, online = self.get_connection().getCPUMap() 676 677 online_cpus = set() 678 for cpu in range(cpus): 679 if cpu_map[cpu]: 680 online_cpus.add(cpu) 681 682 return online_cpus 683 684 def get_cpu_model_names(self): 685 """Get the cpu models based on host CPU arch 686 687 :returns: a list of cpu models which supported by the given CPU arch 688 """ 689 arch = self.get_capabilities().host.cpu.arch 690 return self.get_connection().getCPUModelNames(arch) 691 692 @staticmethod 693 def _log_host_capabilities(xmlstr): 694 # NOTE(mriedem): This looks a bit weird but we do this so we can stub 695 # out this method in unit/functional test runs since the xml string is 696 # big and it can cause subunit parsing to fail (see bug 1813147). 697 LOG.info("Libvirt host capabilities %s", xmlstr) 698 699 def get_capabilities(self): 700 """Returns the host capabilities information 701 702 Returns an instance of config.LibvirtConfigCaps representing 703 the capabilities of the host. 704 705 Note: The result is cached in the member attribute _caps. 706 707 :returns: a config.LibvirtConfigCaps object 708 """ 709 if not self._caps: 710 xmlstr = self.get_connection().getCapabilities() 711 self._log_host_capabilities(xmlstr) 712 self._caps = vconfig.LibvirtConfigCaps() 713 self._caps.parse_str(xmlstr) 714 # NOTE(mriedem): Don't attempt to get baseline CPU features 715 # if libvirt can't determine the host cpu model. 716 if (hasattr(libvirt, 717 'VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES') and 718 self._caps.host.cpu.model is not None): 719 try: 720 xml_str = self._caps.host.cpu.to_xml() 721 if isinstance(xml_str, bytes): 722 xml_str = xml_str.decode('utf-8') 723 features = self.get_connection().baselineCPU( 724 [xml_str], 725 libvirt.VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES) 726 if features: 727 cpu = vconfig.LibvirtConfigCPU() 728 cpu.parse_str(features) 729 self._caps.host.cpu.features = cpu.features 730 except libvirt.libvirtError as ex: 731 error_code = ex.get_error_code() 732 if error_code == libvirt.VIR_ERR_NO_SUPPORT: 733 LOG.warning("URI %(uri)s does not support full set" 734 " of host capabilities: %(error)s", 735 {'uri': self._uri, 'error': ex}) 736 else: 737 raise 738 return self._caps 739 740 def get_domain_capabilities(self): 741 """Returns the capabilities you can request when creating a 742 domain (VM) with that hypervisor, for various combinations of 743 architecture and machine type. 744 745 In this context the fuzzy word "hypervisor" implies QEMU 746 binary, libvirt itself and the host config. libvirt provides 747 this in order that callers can determine what the underlying 748 emulator and/or libvirt is capable of, prior to creating a domain 749 (for instance via virDomainCreateXML or virDomainDefineXML). 750 However nova needs to know the capabilities much earlier, when 751 the host's compute service is first initialised, in order that 752 placement decisions can be made across many compute hosts. 753 Therefore this is expected to be called during the init_host() 754 phase of the driver lifecycle rather than just before booting 755 an instance. 756 757 This causes an additional complication since the Python 758 binding for this libvirt API call requires the architecture 759 and machine type to be provided. So in order to gain a full 760 picture of the hypervisor's capabilities, technically we need 761 to call it with the right parameters, once for each 762 (architecture, machine_type) combination which we care about. 763 However the libvirt experts have advised us that in practice 764 the domain capabilities do not (yet, at least) vary enough 765 across machine types to justify the cost of calling 766 getDomainCapabilities() once for every single (architecture, 767 machine_type) combination. In particular, SEV support isn't 768 reported per-machine type, and since there are usually many 769 machine types, we heed the advice of the experts that it's 770 typically sufficient to call it once per host architecture: 771 772 https://bugzilla.redhat.com/show_bug.cgi?id=1683471#c7 773 774 However, that's not quite sufficient in the context of nova, 775 because SEV guests typically require a q35 machine type, as do 776 KVM/QEMU guests that want Secure Boot, whereas the current 777 default machine type for x86_64 is 'pc'. So we need results 778 from the getDomainCapabilities API for at least those two. 779 Fortunately we can take advantage of the results from the 780 getCapabilities API which marks selected machine types as 781 canonical, e.g.: 782 783 <machine canonical='pc-i440fx-2.11' maxCpus='255'>pc</machine> 784 <machine canonical='pc-q35-2.11' maxCpus='288'>q35</machine> 785 786 So for now, we call getDomainCapabilities for these canonical 787 machine types of each architecture, plus for the 788 architecture's default machine type, if that is not one of the 789 canonical types. 790 791 Future domain capabilities might report SEV in a more 792 fine-grained manner, and we also expect to use this method to 793 detect other features, such as for gracefully handling machine 794 types and potentially for detecting OVMF binaries. Therefore 795 we memoize the results of the API calls in a nested dict where 796 the top-level keys are architectures, and second-level keys 797 are machine types, in order to allow easy expansion later. 798 799 Whenever libvirt/QEMU are updated, cached domCapabilities 800 would get outdated (because QEMU will contain new features and 801 the capabilities will vary). However, this should not be a 802 problem here, because when libvirt/QEMU gets updated, the 803 nova-compute agent also needs restarting, at which point the 804 memoization will vanish because it's not persisted to disk. 805 806 Note: The result is cached in the member attribute 807 _domain_caps. 808 809 :returns: a nested dict of dicts which maps architectures to 810 machine types to instances of config.LibvirtConfigDomainCaps 811 representing the domain capabilities of the host for that arch 812 and machine type: 813 814 { arch: 815 { machine_type: LibvirtConfigDomainCaps } 816 } 817 """ 818 if self._domain_caps: 819 return self._domain_caps 820 821 domain_caps: ty.Dict = defaultdict(dict) 822 caps = self.get_capabilities() 823 virt_type = CONF.libvirt.virt_type 824 825 for guest in caps.guests: 826 arch = guest.arch 827 domain = guest.domains.get(virt_type, guest.default_domain) 828 829 for machine_type in self._get_machine_types(arch, domain): 830 # It is expected that if there are multiple <guest> 831 # elements, each will have a different architecture; 832 # for example, on x86 hosts one <guest> will contain 833 # <arch name='i686'> and one will contain <arch 834 # name='x86_64'>. But it doesn't hurt to add a safety 835 # net to avoid needlessly calling libvirt's API more 836 # times than we need. 837 if machine_type and machine_type in domain_caps[arch]: 838 continue 839 self._add_to_domain_capabilities(domain.emulator, arch, 840 domain_caps, machine_type, 841 virt_type) 842 843 # NOTE(aspiers): Use a temporary variable to update the 844 # instance variable atomically, otherwise if some API 845 # calls succeeded and then one failed, we might 846 # accidentally memoize a partial result. 847 self._domain_caps = domain_caps 848 849 return self._domain_caps 850 851 def _get_machine_types(self, arch, domain): 852 """Get the machine types for this architecture for which we need to 853 call getDomainCapabilities, i.e. the canonical machine types, 854 and the default machine type (if it's not one of the canonical 855 machine types). 856 857 See the docstring for get_domain_capabilities() for an explanation 858 of why we choose this set of machine types. 859 """ 860 # NOTE(aspiers): machine_type could be None here if nova 861 # doesn't have a default machine type for this architecture. 862 # See _add_to_domain_capabilities() below for how this is handled. 863 mtypes = set([libvirt_utils.get_default_machine_type(arch)]) 864 mtypes.update(domain.aliases.keys()) 865 LOG.debug("Getting domain capabilities for %(arch)s via " 866 "machine types: %(mtypes)s", 867 {'arch': arch, 'mtypes': mtypes}) 868 return mtypes 869 870 def _add_to_domain_capabilities(self, emulator_bin, arch, domain_caps, 871 machine_type, virt_type): 872 # NOTE(aspiers): machine_type could be None here if nova 873 # doesn't have a default machine type for this architecture. 874 # In that case we pass a machine_type of None to the libvirt 875 # API and rely on it choosing a sensible default which will be 876 # returned in the <machine> element. It could also be an 877 # alias like 'pc' rather than a full machine type. 878 # 879 # NOTE(kchamart): Prior to libvirt v4.7.0 libvirt picked its 880 # default machine type for x86, 'pc', as reported by QEMU's 881 # default. From libvirt v4.7.0 onwards, libvirt _explicitly_ 882 # declared the "preferred" default for x86 as 'pc' (and 883 # appropriate values for other architectures), and only uses 884 # QEMU's reported default (whatever that may be) if 'pc' does 885 # not exist. This was done "to isolate applications from 886 # hypervisor changes that may cause incompatibilities" -- 887 # i.e. if, or when, QEMU changes its default machine type to 888 # something else. Refer to this libvirt commit: 889 # 890 # https://libvirt.org/git/?p=libvirt.git;a=commit;h=26cfb1a3 891 try: 892 cap_obj = self._get_domain_capabilities( 893 emulator_bin=emulator_bin, arch=arch, 894 machine_type=machine_type, virt_type=virt_type) 895 except libvirt.libvirtError as ex: 896 # NOTE(sean-k-mooney): This can happen for several 897 # reasons, but one common example is if you have 898 # multiple QEMU emulators installed and you set 899 # virt-type=kvm. In this case any non-native emulator, 900 # e.g. AArch64 on an x86 host, will (correctly) raise 901 # an exception as KVM cannot be used to accelerate CPU 902 # instructions for non-native architectures. 903 error_code = ex.get_error_code() 904 LOG.debug( 905 "Error from libvirt when retrieving domain capabilities " 906 "for arch %(arch)s / virt_type %(virt_type)s / " 907 "machine_type %(mach_type)s: " 908 "[Error Code %(error_code)s]: %(exception)s", 909 {'arch': arch, 'virt_type': virt_type, 910 'mach_type': machine_type, 'error_code': error_code, 911 'exception': ex}) 912 # Remove archs added by default dict lookup when checking 913 # if the machine type has already been recoded. 914 if arch in domain_caps: 915 domain_caps.pop(arch) 916 return 917 918 # Register the domain caps using the expanded form of 919 # machine type returned by libvirt in the <machine> 920 # element (e.g. pc-i440fx-2.11) 921 if cap_obj.machine_type: 922 domain_caps[arch][cap_obj.machine_type] = cap_obj 923 else: 924 # NOTE(aspiers): In theory this should never happen, 925 # but better safe than sorry. 926 LOG.warning( 927 "libvirt getDomainCapabilities(" 928 "emulator_bin=%(emulator_bin)s, arch=%(arch)s, " 929 "machine_type=%(machine_type)s, virt_type=%(virt_type)s) " 930 "returned null <machine> type", 931 {'emulator_bin': emulator_bin, 'arch': arch, 932 'machine_type': machine_type, 'virt_type': virt_type} 933 ) 934 935 # And if we passed an alias, register the domain caps 936 # under that too. 937 if machine_type and machine_type != cap_obj.machine_type: 938 domain_caps[arch][machine_type] = cap_obj 939 cap_obj.machine_type_alias = machine_type 940 941 def _get_domain_capabilities(self, emulator_bin=None, arch=None, 942 machine_type=None, virt_type=None, flags=0): 943 xmlstr = self.get_connection().getDomainCapabilities( 944 emulator_bin, 945 arch, 946 machine_type, 947 virt_type, 948 flags 949 ) 950 LOG.debug("Libvirt host hypervisor capabilities for arch=%s and " 951 "machine_type=%s:\n%s", arch, machine_type, xmlstr) 952 caps = vconfig.LibvirtConfigDomainCaps() 953 caps.parse_str(xmlstr) 954 return caps 955 956 def get_driver_type(self): 957 """Get hypervisor type. 958 959 :returns: hypervisor type (ex. qemu) 960 961 """ 962 963 return self.get_connection().getType() 964 965 def get_version(self): 966 """Get hypervisor version. 967 968 :returns: hypervisor version (ex. 12003) 969 970 """ 971 972 return self.get_connection().getVersion() 973 974 def get_hostname(self): 975 """Returns the hostname of the hypervisor.""" 976 hostname = self.get_connection().getHostname() 977 if self._hostname is None: 978 self._hostname = hostname 979 elif hostname != self._hostname: 980 LOG.error('Hostname has changed from %(old)s ' 981 'to %(new)s. A restart is required to take effect.', 982 {'old': self._hostname, 'new': hostname}) 983 return self._hostname 984 985 def find_secret(self, usage_type, usage_id): 986 """Find a secret. 987 988 usage_type: one of 'iscsi', 'ceph', 'rbd' or 'volume' 989 usage_id: name of resource in secret 990 """ 991 if usage_type == 'iscsi': 992 usage_type_const = libvirt.VIR_SECRET_USAGE_TYPE_ISCSI 993 elif usage_type in ('rbd', 'ceph'): 994 usage_type_const = libvirt.VIR_SECRET_USAGE_TYPE_CEPH 995 elif usage_type == 'volume': 996 usage_type_const = libvirt.VIR_SECRET_USAGE_TYPE_VOLUME 997 else: 998 msg = _("Invalid usage_type: %s") 999 raise exception.InternalError(msg % usage_type) 1000 1001 try: 1002 conn = self.get_connection() 1003 return conn.secretLookupByUsage(usage_type_const, usage_id) 1004 except libvirt.libvirtError as e: 1005 if e.get_error_code() == libvirt.VIR_ERR_NO_SECRET: 1006 return None 1007 1008 def create_secret(self, usage_type, usage_id, password=None, uuid=None): 1009 """Create a secret. 1010 1011 :param usage_type: one of 'iscsi', 'ceph', 'rbd', 'volume', 'vtpm'. 1012 'rbd' will be converted to 'ceph'. 'vtpm' secrets 1013 are private and ephemeral; others are not. 1014 :param usage_id: name of resource in secret 1015 :param password: optional secret value to set 1016 :param uuid: optional UUID of the secret; else one is generated by 1017 libvirt 1018 """ 1019 secret_conf = vconfig.LibvirtConfigSecret() 1020 secret_conf.ephemeral = usage_type == 'vtpm' 1021 secret_conf.private = usage_type == 'vtpm' 1022 secret_conf.usage_id = usage_id 1023 secret_conf.uuid = uuid 1024 if usage_type in ('rbd', 'ceph'): 1025 secret_conf.usage_type = 'ceph' 1026 elif usage_type == 'iscsi': 1027 secret_conf.usage_type = 'iscsi' 1028 elif usage_type == 'volume': 1029 secret_conf.usage_type = 'volume' 1030 elif usage_type == 'vtpm': 1031 secret_conf.usage_type = 'vtpm' 1032 else: 1033 msg = _("Invalid usage_type: %s") 1034 raise exception.InternalError(msg % usage_type) 1035 1036 xml = secret_conf.to_xml() 1037 try: 1038 LOG.debug('Secret XML: %s', xml) 1039 conn = self.get_connection() 1040 secret = conn.secretDefineXML(xml) 1041 if password is not None: 1042 secret.setValue(password) 1043 return secret 1044 except libvirt.libvirtError: 1045 with excutils.save_and_reraise_exception(): 1046 LOG.error('Error defining a secret with XML: %s', xml) 1047 1048 def delete_secret(self, usage_type, usage_id): 1049 """Delete a secret. 1050 1051 :param usage_type: one of 'iscsi', 'ceph', 'rbd', 'volume' or 'vtpm' 1052 :param usage_id: name of resource in secret 1053 """ 1054 secret = self.find_secret(usage_type, usage_id) 1055 if secret is not None: 1056 secret.undefine() 1057 1058 def _get_hardware_info(self): 1059 """Returns hardware information about the Node. 1060 1061 Note that the memory size is reported in MiB instead of KiB. 1062 """ 1063 return self.get_connection().getInfo() 1064 1065 def get_memory_mb_total(self): 1066 """Get the total memory size(MB) of physical computer. 1067 1068 :returns: the total amount of memory(MB). 1069 """ 1070 if CONF.libvirt.file_backed_memory > 0: 1071 return CONF.libvirt.file_backed_memory 1072 else: 1073 return self._get_hardware_info()[1] 1074 1075 def _sum_domain_memory_mb(self, include_host=True): 1076 """Get the total memory consumed by guest domains 1077 1078 If include_host is True, subtract available host memory from guest 0 1079 to get real used memory within dom0 within xen 1080 """ 1081 used = 0 1082 for guest in self.list_guests(only_guests=False): 1083 try: 1084 # TODO(sahid): Use get_info... 1085 dom_mem = int(guest._get_domain_info()[2]) 1086 except libvirt.libvirtError as e: 1087 LOG.warning("couldn't obtain the memory from domain:" 1088 " %(uuid)s, exception: %(ex)s", 1089 {"uuid": guest.uuid, "ex": e}) 1090 continue 1091 if include_host and guest.id == 0: 1092 # Memory usage for the host domain (dom0 in xen) is the 1093 # reported memory minus available memory 1094 used += (dom_mem - self._get_avail_memory_kb()) 1095 else: 1096 used += dom_mem 1097 # Convert it to MB 1098 return used // units.Ki 1099 1100 @staticmethod 1101 def _get_avail_memory_kb(): 1102 with open('/proc/meminfo') as fp: 1103 m = fp.read().split() 1104 idx1 = m.index('MemFree:') 1105 idx2 = m.index('Buffers:') 1106 idx3 = m.index('Cached:') 1107 1108 avail = int(m[idx1 + 1]) + int(m[idx2 + 1]) + int(m[idx3 + 1]) 1109 1110 return avail 1111 1112 def get_memory_mb_used(self): 1113 """Get the used memory size(MB) of physical computer. 1114 1115 :returns: the total usage of memory(MB). 1116 """ 1117 if CONF.libvirt.virt_type == 'xen': 1118 # For xen, report the sum of all domains, with 1119 return self._sum_domain_memory_mb(include_host=True) 1120 elif CONF.libvirt.file_backed_memory > 0: 1121 # For file_backed_memory, report the total usage of guests, 1122 # ignoring host memory 1123 return self._sum_domain_memory_mb(include_host=False) 1124 else: 1125 return (self.get_memory_mb_total() - 1126 (self._get_avail_memory_kb() // units.Ki)) 1127 1128 def get_cpu_stats(self): 1129 """Returns the current CPU state of the host with frequency.""" 1130 stats = self.get_connection().getCPUStats( 1131 libvirt.VIR_NODE_CPU_STATS_ALL_CPUS, 0) 1132 # getInfo() returns various information about the host node 1133 # No. 3 is the expected CPU frequency. 1134 stats["frequency"] = self._get_hardware_info()[3] 1135 return stats 1136 1137 def write_instance_config(self, xml): 1138 """Defines a domain, but does not start it. 1139 1140 :param xml: XML domain definition of the guest. 1141 1142 :returns: an instance of Guest 1143 """ 1144 domain = self.get_connection().defineXML(xml) 1145 return libvirt_guest.Guest(domain) 1146 1147 def device_lookup_by_name(self, name): 1148 """Lookup a node device by its name. 1149 1150 1151 :returns: a virNodeDevice instance 1152 """ 1153 return self.get_connection().nodeDeviceLookupByName(name) 1154 1155 def list_pci_devices(self, flags=0): 1156 """Lookup pci devices. 1157 1158 :returns: a list of virNodeDevice instance 1159 """ 1160 return self._list_devices("pci", flags=flags) 1161 1162 def list_mdev_capable_devices(self, flags=0): 1163 """Lookup devices supporting mdev capabilities. 1164 1165 :returns: a list of virNodeDevice instance 1166 """ 1167 return self._list_devices("mdev_types", flags=flags) 1168 1169 def list_mediated_devices(self, flags=0): 1170 """Lookup mediated devices. 1171 1172 :returns: a list of virNodeDevice instance 1173 """ 1174 return self._list_devices("mdev", flags=flags) 1175 1176 def _list_devices(self, cap, flags=0): 1177 """Lookup devices. 1178 1179 :returns: a list of virNodeDevice instance 1180 """ 1181 try: 1182 return self.get_connection().listDevices(cap, flags) 1183 except libvirt.libvirtError as ex: 1184 error_code = ex.get_error_code() 1185 if error_code == libvirt.VIR_ERR_NO_SUPPORT: 1186 LOG.warning("URI %(uri)s does not support " 1187 "listDevices: %(error)s", 1188 {'uri': self._uri, 'error': ex}) 1189 return [] 1190 else: 1191 raise 1192 1193 def list_all_devices( 1194 self, flags: int = 0) -> ty.List['libvirt.virNodeDevice']: 1195 """Lookup devices. 1196 1197 :param flags: a bitmask of flags to filter the returned devices. 1198 :returns: a list of virNodeDevice xml strings. 1199 """ 1200 try: 1201 return self.get_connection().listAllDevices(flags) or [] 1202 except libvirt.libvirtError as ex: 1203 LOG.warning(ex) 1204 return [] 1205 1206 def compare_cpu(self, xmlDesc, flags=0): 1207 """Compares the given CPU description with the host CPU.""" 1208 return self.get_connection().compareCPU(xmlDesc, flags) 1209 1210 def is_cpu_control_policy_capable(self): 1211 """Returns whether kernel configuration CGROUP_SCHED is enabled 1212 1213 CONFIG_CGROUP_SCHED may be disabled in some kernel configs to 1214 improve scheduler latency. 1215 """ 1216 try: 1217 with open("/proc/self/mounts", "r") as fd: 1218 for line in fd.readlines(): 1219 # mount options and split options 1220 bits = line.split()[3].split(",") 1221 if "cpu" in bits: 1222 return True 1223 return False 1224 except IOError: 1225 return False 1226 1227 @property 1228 def has_hyperthreading(self): 1229 """Determine if host CPU has SMT, a.k.a. HyperThreading. 1230 1231 :return: True if the host has SMT enabled, else False. 1232 """ 1233 if self._has_hyperthreading is not None: 1234 return self._has_hyperthreading 1235 1236 self._has_hyperthreading = False 1237 1238 # we don't use '/capabilities/host/cpu/topology' since libvirt doesn't 1239 # guarantee the accuracy of this information 1240 for cell in self.get_capabilities().host.topology.cells: 1241 if any(len(cpu.siblings) > 1 for cpu in cell.cpus if cpu.siblings): 1242 self._has_hyperthreading = True 1243 break 1244 1245 return self._has_hyperthreading 1246 1247 def _kernel_supports_amd_sev(self): 1248 if not os.path.exists(SEV_KERNEL_PARAM_FILE): 1249 LOG.debug("%s does not exist", SEV_KERNEL_PARAM_FILE) 1250 return False 1251 1252 with open(SEV_KERNEL_PARAM_FILE) as f: 1253 contents = f.read() 1254 LOG.debug("%s contains [%s]", SEV_KERNEL_PARAM_FILE, contents) 1255 return contents == "1\n" 1256 1257 @property 1258 def supports_amd_sev(self): 1259 """Returns a boolean indicating whether AMD SEV (Secure Encrypted 1260 Virtualization) is supported. This is conditional on support 1261 in the hardware, kernel, qemu, and libvirt. 1262 1263 The result is memoized, since it is not expected to change 1264 during the lifetime of a running nova-compute service; if the 1265 hypervisor stack is changed or reconfigured in a way which 1266 would affect the support, nova-compute should be restarted 1267 anyway. 1268 """ 1269 if self._supports_amd_sev is None: 1270 self._set_amd_sev_support() 1271 return self._supports_amd_sev 1272 1273 def _set_amd_sev_support(self): 1274 self._supports_amd_sev = False 1275 1276 if not self._kernel_supports_amd_sev(): 1277 LOG.info("kernel doesn't support AMD SEV") 1278 self._supports_amd_sev = False 1279 return 1280 1281 domain_caps = self.get_domain_capabilities() 1282 for arch in domain_caps: 1283 for machine_type in domain_caps[arch]: 1284 LOG.debug("Checking SEV support for arch %s " 1285 "and machine type %s", arch, machine_type) 1286 for feature in domain_caps[arch][machine_type].features: 1287 feature_is_sev = isinstance( 1288 feature, vconfig.LibvirtConfigDomainCapsFeatureSev) 1289 if (feature_is_sev and feature.supported): 1290 LOG.info("AMD SEV support detected") 1291 self._supports_amd_sev = True 1292 return 1293 1294 LOG.debug("No AMD SEV support detected for any (arch, machine_type)")