"Fossies" - the Fresh Open Source Software Archive

Member "cinder-14.0.2/cinder/image/image_utils.py" (4 Oct 2019, 33689 Bytes) of package /linux/misc/openstack/cinder-14.0.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "image_utils.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 14.0.2_vs_15.0.0.

    1 # Copyright 2010 United States Government as represented by the
    2 # Administrator of the National Aeronautics and Space Administration.
    3 # All Rights Reserved.
    4 # Copyright (c) 2010 Citrix Systems, Inc.
    5 #
    6 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    7 #    not use this file except in compliance with the License. You may obtain
    8 #    a copy of the License at
    9 #
   10 #         http://www.apache.org/licenses/LICENSE-2.0
   11 #
   12 #    Unless required by applicable law or agreed to in writing, software
   13 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   14 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   15 #    License for the specific language governing permissions and limitations
   16 #    under the License.
   17 
   18 """
   19 Helper methods to deal with images.
   20 
   21 This is essentially a copy from nova.virt.images.py
   22 Some slight modifications, but at some point
   23 we should look at maybe pushing this up to Oslo
   24 """
   25 
   26 
   27 import contextlib
   28 import errno
   29 import math
   30 import os
   31 import re
   32 import tempfile
   33 
   34 import cryptography
   35 from cursive import exception as cursive_exception
   36 from cursive import signature_utils
   37 from eventlet import tpool
   38 from oslo_concurrency import processutils
   39 from oslo_config import cfg
   40 from oslo_log import log as logging
   41 from oslo_utils import fileutils
   42 from oslo_utils import imageutils
   43 from oslo_utils import timeutils
   44 from oslo_utils import units
   45 import psutil
   46 import six
   47 
   48 from cinder import exception
   49 from cinder.i18n import _
   50 from cinder import utils
   51 from cinder.volume import throttling
   52 from cinder.volume import utils as volume_utils
   53 
   54 LOG = logging.getLogger(__name__)
   55 
   56 image_opts = [
   57     cfg.StrOpt('image_conversion_dir',
   58                default='$state_path/conversion',
   59                help='Directory used for temporary storage '
   60                'during image conversion'), ]
   61 
   62 CONF = cfg.CONF
   63 CONF.register_opts(image_opts)
   64 
   65 QEMU_IMG_LIMITS = processutils.ProcessLimits(
   66     cpu_time=8,
   67     address_space=1 * units.Gi)
   68 
   69 
   70 QEMU_IMG_FORMAT_MAP = {
   71     # Convert formats of Glance images to how they are processed with qemu-img.
   72     'iso': 'raw',
   73     'vhd': 'vpc',
   74     'ploop': 'parallels',
   75 }
   76 QEMU_IMG_FORMAT_MAP_INV = {v: k for k, v in QEMU_IMG_FORMAT_MAP.items()}
   77 
   78 QEMU_IMG_VERSION = None
   79 QEMU_IMG_MIN_FORCE_SHARE_VERSION = [2, 10, 0]
   80 QEMU_IMG_MIN_CONVERT_LUKS_VERSION = '2.10'
   81 
   82 
   83 def fixup_disk_format(disk_format):
   84     """Return the format to be provided to qemu-img convert."""
   85 
   86     return QEMU_IMG_FORMAT_MAP.get(disk_format, disk_format)
   87 
   88 
   89 def from_qemu_img_disk_format(disk_format):
   90     """Return the conventional format derived from qemu-img format."""
   91 
   92     return QEMU_IMG_FORMAT_MAP_INV.get(disk_format, disk_format)
   93 
   94 
   95 def qemu_img_info(path, run_as_root=True, force_share=False):
   96     """Return an object containing the parsed output from qemu-img info."""
   97     cmd = ['env', 'LC_ALL=C', 'qemu-img', 'info']
   98     if force_share:
   99         if qemu_img_supports_force_share():
  100             cmd.append('--force-share')
  101         else:
  102             msg = _("qemu-img --force-share requested, but "
  103                     "qemu-img does not support this parameter")
  104             LOG.warning(msg)
  105     cmd.append(path)
  106 
  107     if os.name == 'nt':
  108         cmd = cmd[2:]
  109     out, _err = utils.execute(*cmd, run_as_root=run_as_root,
  110                               prlimit=QEMU_IMG_LIMITS)
  111     info = imageutils.QemuImgInfo(out)
  112 
  113     # From Cinder's point of view, any 'luks' formatted images
  114     # should be treated as 'raw'.
  115     if info.file_format == 'luks':
  116         info.file_format = 'raw'
  117 
  118     return info
  119 
  120 
  121 def get_qemu_img_version():
  122     """The qemu-img version will be cached until the process is restarted."""
  123 
  124     global QEMU_IMG_VERSION
  125     if QEMU_IMG_VERSION is not None:
  126         return QEMU_IMG_VERSION
  127 
  128     info = utils.execute('qemu-img', '--version', check_exit_code=False)[0]
  129     pattern = r"qemu-img version ([0-9\.]*)"
  130     version = re.match(pattern, info)
  131     if not version:
  132         LOG.warning("qemu-img is not installed.")
  133         return None
  134     QEMU_IMG_VERSION = _get_version_from_string(version.groups()[0])
  135     return QEMU_IMG_VERSION
  136 
  137 
  138 def qemu_img_supports_force_share():
  139     return get_qemu_img_version() > [2, 10, 0]
  140 
  141 
  142 def _get_qemu_convert_cmd(src, dest, out_format, src_format=None,
  143                           out_subformat=None, cache_mode=None,
  144                           prefix=None, cipher_spec=None, passphrase_file=None):
  145 
  146     if out_format == 'vhd':
  147         # qemu-img still uses the legacy vpc name
  148         out_format = 'vpc'
  149 
  150     cmd = ['qemu-img', 'convert', '-O', out_format]
  151 
  152     if prefix:
  153         cmd = list(prefix) + cmd
  154 
  155     if cache_mode:
  156         cmd += ('-t', cache_mode)
  157 
  158     if out_subformat:
  159         cmd += ('-o', 'subformat=%s' % out_subformat)
  160 
  161     # AMI images can be raw or qcow2 but qemu-img doesn't accept "ami" as
  162     # an image format, so we use automatic detection.
  163     # TODO(geguileo): This fixes unencrypted AMI image case, but we need to
  164     # fix the encrypted case.
  165 
  166     if (src_format or '').lower() not in ('', 'ami'):
  167         cmd += ('-f', src_format)  # prevent detection of format
  168 
  169     # NOTE(lyarwood): When converting to LUKS add the cipher spec if present
  170     # and create a secret for the passphrase, written to a temp file
  171     if out_format == 'luks':
  172         check_qemu_img_version(QEMU_IMG_MIN_CONVERT_LUKS_VERSION)
  173         if cipher_spec:
  174             cmd += ('-o', 'cipher-alg=%s,cipher-mode=%s,ivgen-alg=%s' %
  175                     (cipher_spec['cipher_alg'], cipher_spec['cipher_mode'],
  176                      cipher_spec['ivgen_alg']))
  177         cmd += ('--object',
  178                 'secret,id=luks_sec,format=raw,file=%s' % passphrase_file,
  179                 '-o', 'key-secret=luks_sec')
  180 
  181     cmd += [src, dest]
  182 
  183     return cmd
  184 
  185 
  186 def _get_version_from_string(version_string):
  187     return [int(x) for x in version_string.split('.')]
  188 
  189 
  190 def check_qemu_img_version(minimum_version):
  191     qemu_version = get_qemu_img_version()
  192     if (qemu_version is None
  193        or qemu_version < _get_version_from_string(minimum_version)):
  194         if qemu_version:
  195             current_version = '.'.join((str(element)
  196                                        for element in qemu_version))
  197         else:
  198             current_version = None
  199 
  200         _msg = _('qemu-img %(minimum_version)s or later is required by '
  201                  'this volume driver. Current qemu-img version: '
  202                  '%(current_version)s') % {'minimum_version': minimum_version,
  203                                            'current_version': current_version}
  204         raise exception.VolumeBackendAPIException(data=_msg)
  205 
  206 
  207 def _convert_image(prefix, source, dest, out_format,
  208                    out_subformat=None, src_format=None,
  209                    run_as_root=True, cipher_spec=None, passphrase_file=None):
  210     """Convert image to other format."""
  211 
  212     # Check whether O_DIRECT is supported and set '-t none' if it is
  213     # This is needed to ensure that all data hit the device before
  214     # it gets unmapped remotely from the host for some backends
  215     # Reference Bug: #1363016
  216 
  217     # NOTE(jdg): In the case of file devices qemu does the
  218     # flush properly and more efficiently than would be done
  219     # setting O_DIRECT, so check for that and skip the
  220     # setting for non BLK devs
  221     if (utils.is_blk_device(dest) and
  222             volume_utils.check_for_odirect_support(source,
  223                                                    dest,
  224                                                    'oflag=direct')):
  225         cache_mode = 'none'
  226     else:
  227         # use default
  228         cache_mode = None
  229 
  230     cmd = _get_qemu_convert_cmd(source, dest,
  231                                 out_format=out_format,
  232                                 src_format=src_format,
  233                                 out_subformat=out_subformat,
  234                                 cache_mode=cache_mode,
  235                                 prefix=prefix,
  236                                 cipher_spec=cipher_spec,
  237                                 passphrase_file=passphrase_file)
  238 
  239     start_time = timeutils.utcnow()
  240 
  241     # If there is not enough space on the conversion partition, include
  242     # the partitions's name in the error message.
  243     try:
  244         utils.execute(*cmd, run_as_root=run_as_root)
  245     except processutils.ProcessExecutionError as ex:
  246         if "No space left" in ex.stderr and CONF.image_conversion_dir in dest:
  247             conversion_dir = CONF.image_conversion_dir
  248             while not os.path.ismount(conversion_dir):
  249                 conversion_dir = os.path.dirname(conversion_dir)
  250 
  251             message = _("Insufficient free space on %(location)s for image "
  252                         "conversion.") % {'location': conversion_dir}
  253             LOG.error(message)
  254 
  255         raise
  256 
  257     duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
  258 
  259     # NOTE(jdg): use a default of 1, mostly for unit test, but in
  260     # some incredible event this is 0 (cirros image?) don't barf
  261     if duration < 1:
  262         duration = 1
  263     try:
  264         image_size = qemu_img_info(source,
  265                                    run_as_root=run_as_root).virtual_size
  266     except ValueError as e:
  267         msg = ("The image was successfully converted, but image size "
  268                "is unavailable. src %(src)s, dest %(dest)s. %(error)s")
  269         LOG.info(msg, {"src": source,
  270                        "dest": dest,
  271                        "error": e})
  272         return
  273 
  274     fsz_mb = image_size / units.Mi
  275     mbps = (fsz_mb / duration)
  276     msg = ("Image conversion details: src %(src)s, size %(sz).2f MB, "
  277            "duration %(duration).2f sec, destination %(dest)s")
  278     LOG.debug(msg, {"src": source,
  279                     "sz": fsz_mb,
  280                     "duration": duration,
  281                     "dest": dest})
  282 
  283     msg = "Converted %(sz).2f MB image at %(mbps).2f MB/s"
  284     LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
  285 
  286 
  287 def convert_image(source, dest, out_format, out_subformat=None,
  288                   src_format=None, run_as_root=True, throttle=None,
  289                   cipher_spec=None, passphrase_file=None):
  290     if not throttle:
  291         throttle = throttling.Throttle.get_default()
  292     with throttle.subcommand(source, dest) as throttle_cmd:
  293         _convert_image(tuple(throttle_cmd['prefix']),
  294                        source, dest,
  295                        out_format,
  296                        out_subformat=out_subformat,
  297                        src_format=src_format,
  298                        run_as_root=run_as_root,
  299                        cipher_spec=cipher_spec,
  300                        passphrase_file=passphrase_file)
  301 
  302 
  303 def resize_image(source, size, run_as_root=False):
  304     """Changes the virtual size of the image."""
  305     cmd = ('qemu-img', 'resize', source, '%sG' % size)
  306     utils.execute(*cmd, run_as_root=run_as_root)
  307 
  308 
  309 def _verify_image(img_file, verifier):
  310     # This methods must be called from a native thread, as the file I/O may
  311     # not yield to other greenthread in some cases, and since the update and
  312     # verify operations are CPU bound there would not be any yielding either,
  313     # which could lead to thread starvation.
  314     while True:
  315         chunk = img_file.read(1024)
  316         if not chunk:
  317             break
  318         verifier.update(chunk)
  319     verifier.verify()
  320 
  321 
  322 def verify_glance_image_signature(context, image_service, image_id, path):
  323     verifier = None
  324     image_meta = image_service.show(context, image_id)
  325     image_properties = image_meta.get('properties', {})
  326     img_signature = image_properties.get('img_signature')
  327     img_sig_hash_method = image_properties.get('img_signature_hash_method')
  328     img_sig_cert_uuid = image_properties.get('img_signature_certificate_uuid')
  329     img_sig_key_type = image_properties.get('img_signature_key_type')
  330     if all(m is None for m in [img_signature,
  331                                img_sig_cert_uuid,
  332                                img_sig_hash_method,
  333                                img_sig_key_type]):
  334         # NOTE(tommylikehu): We won't verify the image signature
  335         # if none of the signature metadata presents.
  336         return False
  337     if any(m is None for m in [img_signature,
  338                                img_sig_cert_uuid,
  339                                img_sig_hash_method,
  340                                img_sig_key_type]):
  341             LOG.error('Image signature metadata for image %s is '
  342                       'incomplete.', image_id)
  343             raise exception.InvalidSignatureImage(image_id=image_id)
  344 
  345     try:
  346         verifier = signature_utils.get_verifier(
  347             context=context,
  348             img_signature_certificate_uuid=img_sig_cert_uuid,
  349             img_signature_hash_method=img_sig_hash_method,
  350             img_signature=img_signature,
  351             img_signature_key_type=img_sig_key_type,
  352         )
  353     except cursive_exception.SignatureVerificationError:
  354         message = _('Failed to get verifier for image: %s') % image_id
  355         LOG.error(message)
  356         raise exception.ImageSignatureVerificationException(
  357             reason=message)
  358     if verifier:
  359         with fileutils.remove_path_on_error(path):
  360             with open(path, "rb") as tem_file:
  361                 try:
  362                     tpool.execute(_verify_image, tem_file, verifier)
  363                     LOG.info('Image signature verification succeeded '
  364                              'for image: %s', image_id)
  365                     return True
  366                 except cryptography.exceptions.InvalidSignature:
  367                     message = _('Image signature verification '
  368                                 'failed for image: %s') % image_id
  369                     LOG.error(message)
  370                     raise exception.ImageSignatureVerificationException(
  371                         reason=message)
  372                 except Exception as ex:
  373                     message = _('Failed to verify signature for '
  374                                 'image: %(image)s due to '
  375                                 'error: %(error)s ') % {'image': image_id,
  376                                                         'error':
  377                                                             six.text_type(ex)}
  378                     LOG.error(message)
  379                     raise exception.ImageSignatureVerificationException(
  380                         reason=message)
  381     return False
  382 
  383 
  384 def fetch(context, image_service, image_id, path, _user_id, _project_id):
  385     # TODO(vish): Improve context handling and add owner and auth data
  386     #             when it is added to glance.  Right now there is no
  387     #             auth checking in glance, so we assume that access was
  388     #             checked before we got here.
  389     start_time = timeutils.utcnow()
  390     with fileutils.remove_path_on_error(path):
  391         with open(path, "wb") as image_file:
  392             try:
  393                 image_service.download(context, image_id,
  394                                        tpool.Proxy(image_file))
  395             except IOError as e:
  396                 if e.errno == errno.ENOSPC:
  397                     params = {'path': os.path.dirname(path),
  398                               'image': image_id}
  399                     reason = _("No space left in image_conversion_dir "
  400                                "path (%(path)s) while fetching "
  401                                "image %(image)s.") % params
  402                     LOG.exception(reason)
  403                     raise exception.ImageTooBig(image_id=image_id,
  404                                                 reason=reason)
  405 
  406                 reason = ("IOError: %(errno)s %(strerror)s" %
  407                           {'errno': e.errno, 'strerror': e.strerror})
  408                 LOG.error(reason)
  409                 raise exception.ImageDownloadFailed(image_href=image_id,
  410                                                     reason=reason)
  411 
  412     duration = timeutils.delta_seconds(start_time, timeutils.utcnow())
  413 
  414     # NOTE(jdg): use a default of 1, mostly for unit test, but in
  415     # some incredible event this is 0 (cirros image?) don't barf
  416     if duration < 1:
  417         duration = 1
  418     fsz_mb = os.stat(image_file.name).st_size / units.Mi
  419     mbps = (fsz_mb / duration)
  420     msg = ("Image fetch details: dest %(dest)s, size %(sz).2f MB, "
  421            "duration %(duration).2f sec")
  422     LOG.debug(msg, {"dest": image_file.name,
  423                     "sz": fsz_mb,
  424                     "duration": duration})
  425     msg = "Image download %(sz).2f MB at %(mbps).2f MB/s"
  426     LOG.info(msg, {"sz": fsz_mb, "mbps": mbps})
  427 
  428 
  429 def get_qemu_data(image_id, has_meta, disk_format_raw, dest, run_as_root,
  430                   force_share=False):
  431     # We may be on a system that doesn't have qemu-img installed.  That
  432     # is ok if we are working with a RAW image.  This logic checks to see
  433     # if qemu-img is installed.  If not we make sure the image is RAW and
  434     # throw an exception if not.  Otherwise we stop before needing
  435     # qemu-img.  Systems with qemu-img will always progress through the
  436     # whole function.
  437     try:
  438         # Use the empty tmp file to make sure qemu_img_info works.
  439         data = qemu_img_info(dest,
  440                              run_as_root=run_as_root,
  441                              force_share=force_share)
  442     # There are a lot of cases that can cause a process execution
  443     # error, but until we do more work to separate out the various
  444     # cases we'll keep the general catch here
  445     except processutils.ProcessExecutionError:
  446         data = None
  447         if has_meta:
  448             if not disk_format_raw:
  449                 raise exception.ImageUnacceptable(
  450                     reason=_("qemu-img is not installed and image is of "
  451                              "type %s.  Only RAW images can be used if "
  452                              "qemu-img is not installed.") %
  453                     disk_format_raw,
  454                     image_id=image_id)
  455         else:
  456             raise exception.ImageUnacceptable(
  457                 reason=_("qemu-img is not installed and the disk "
  458                          "format is not specified.  Only RAW images "
  459                          "can be used if qemu-img is not installed."),
  460                 image_id=image_id)
  461     return data
  462 
  463 
  464 def fetch_verify_image(context, image_service, image_id, dest,
  465                        user_id=None, project_id=None, size=None,
  466                        run_as_root=True):
  467     fetch(context, image_service, image_id, dest,
  468           None, None)
  469     image_meta = image_service.show(context, image_id)
  470 
  471     with fileutils.remove_path_on_error(dest):
  472         has_meta = False if not image_meta else True
  473         try:
  474             format_raw = True if image_meta['disk_format'] == 'raw' else False
  475         except TypeError:
  476             format_raw = False
  477         data = get_qemu_data(image_id, has_meta, format_raw,
  478                              dest, run_as_root)
  479         # We can only really do verification of the image if we have
  480         # qemu data to use
  481         if data is not None:
  482             fmt = data.file_format
  483             if fmt is None:
  484                 raise exception.ImageUnacceptable(
  485                     reason=_("'qemu-img info' parsing failed."),
  486                     image_id=image_id)
  487 
  488             backing_file = data.backing_file
  489             if backing_file is not None:
  490                 raise exception.ImageUnacceptable(
  491                     image_id=image_id,
  492                     reason=(_("fmt=%(fmt)s backed by: %(backing_file)s") %
  493                             {'fmt': fmt, 'backing_file': backing_file}))
  494 
  495             # NOTE(xqueralt): If the image virtual size doesn't fit in the
  496             # requested volume there is no point on resizing it because it will
  497             # generate an unusable image.
  498             if size is not None:
  499                 check_virtual_size(data.virtual_size, size, image_id)
  500 
  501 
  502 def fetch_to_vhd(context, image_service,
  503                  image_id, dest, blocksize, volume_subformat=None,
  504                  user_id=None, project_id=None, run_as_root=True):
  505     fetch_to_volume_format(context, image_service, image_id, dest, 'vpc',
  506                            blocksize, volume_subformat=volume_subformat,
  507                            user_id=user_id, project_id=project_id,
  508                            run_as_root=run_as_root)
  509 
  510 
  511 def fetch_to_raw(context, image_service,
  512                  image_id, dest, blocksize,
  513                  user_id=None, project_id=None, size=None, run_as_root=True):
  514     fetch_to_volume_format(context, image_service, image_id, dest, 'raw',
  515                            blocksize, user_id=user_id, project_id=project_id,
  516                            size=size, run_as_root=run_as_root)
  517 
  518 
  519 def fetch_to_volume_format(context, image_service,
  520                            image_id, dest, volume_format, blocksize,
  521                            volume_subformat=None, user_id=None,
  522                            project_id=None, size=None, run_as_root=True):
  523     qemu_img = True
  524     image_meta = image_service.show(context, image_id)
  525 
  526     # NOTE(avishay): I'm not crazy about creating temp files which may be
  527     # large and cause disk full errors which would confuse users.
  528     # Unfortunately it seems that you can't pipe to 'qemu-img convert' because
  529     # it seeks. Maybe we can think of something for a future version.
  530     with temporary_file() as tmp:
  531         has_meta = False if not image_meta else True
  532         try:
  533             format_raw = True if image_meta['disk_format'] == 'raw' else False
  534         except TypeError:
  535             format_raw = False
  536         data = get_qemu_data(image_id, has_meta, format_raw,
  537                              tmp, run_as_root)
  538         if data is None:
  539             qemu_img = False
  540 
  541         tmp_images = TemporaryImages.for_image_service(image_service)
  542         tmp_image = tmp_images.get(context, image_id)
  543         if tmp_image:
  544             tmp = tmp_image
  545         else:
  546             fetch(context, image_service, image_id, tmp, user_id, project_id)
  547 
  548         if is_xenserver_format(image_meta):
  549             replace_xenserver_image_with_coalesced_vhd(tmp)
  550 
  551         if not qemu_img:
  552             # qemu-img is not installed but we do have a RAW image.  As a
  553             # result we only need to copy the image to the destination and then
  554             # return.
  555             LOG.debug('Copying image from %(tmp)s to volume %(dest)s - '
  556                       'size: %(size)s', {'tmp': tmp, 'dest': dest,
  557                                          'size': image_meta['size']})
  558             image_size_m = math.ceil(float(image_meta['size']) / units.Mi)
  559             volume_utils.copy_volume(tmp, dest, image_size_m, blocksize)
  560             return
  561 
  562         data = qemu_img_info(tmp, run_as_root=run_as_root)
  563 
  564         # NOTE(xqueralt): If the image virtual size doesn't fit in the
  565         # requested volume there is no point on resizing it because it will
  566         # generate an unusable image.
  567         if size is not None:
  568             check_virtual_size(data.virtual_size, size, image_id)
  569 
  570         fmt = data.file_format
  571         if fmt is None:
  572             raise exception.ImageUnacceptable(
  573                 reason=_("'qemu-img info' parsing failed."),
  574                 image_id=image_id)
  575 
  576         backing_file = data.backing_file
  577         if backing_file is not None:
  578             raise exception.ImageUnacceptable(
  579                 image_id=image_id,
  580                 reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
  581                 % {'fmt': fmt, 'backing_file': backing_file, })
  582 
  583         # NOTE(jdg): I'm using qemu-img convert to write
  584         # to the volume regardless if it *needs* conversion or not
  585         # TODO(avishay): We can speed this up by checking if the image is raw
  586         # and if so, writing directly to the device. However, we need to keep
  587         # check via 'qemu-img info' that what we copied was in fact a raw
  588         # image and not a different format with a backing file, which may be
  589         # malicious.
  590         LOG.debug("%s was %s, converting to %s ", image_id, fmt, volume_format)
  591         disk_format = fixup_disk_format(image_meta['disk_format'])
  592 
  593         convert_image(tmp, dest, volume_format,
  594                       out_subformat=volume_subformat,
  595                       src_format=disk_format,
  596                       run_as_root=run_as_root)
  597 
  598 
  599 def _validate_file_format(image_data, expected_format):
  600     if image_data.file_format == expected_format:
  601         return True
  602     elif image_data.file_format == 'vpc' and expected_format == 'vhd':
  603         # qemu-img still uses the legacy 'vpc' name for the vhd format.
  604         return True
  605     return False
  606 
  607 
  608 def upload_volume(context, image_service, image_meta, volume_path,
  609                   volume_format='raw', run_as_root=True):
  610     image_id = image_meta['id']
  611     if (image_meta['disk_format'] == volume_format):
  612         LOG.debug("%s was %s, no need to convert to %s",
  613                   image_id, volume_format, image_meta['disk_format'])
  614         if os.name == 'nt' or os.access(volume_path, os.R_OK):
  615             with open(volume_path, 'rb') as image_file:
  616                 image_service.update(context, image_id, {},
  617                                      tpool.Proxy(image_file))
  618         else:
  619             with utils.temporary_chown(volume_path):
  620                 with open(volume_path, 'rb') as image_file:
  621                     image_service.update(context, image_id, {},
  622                                          tpool.Proxy(image_file))
  623         return
  624 
  625     with temporary_file() as tmp:
  626         LOG.debug("%s was %s, converting to %s",
  627                   image_id, volume_format, image_meta['disk_format'])
  628 
  629         data = qemu_img_info(volume_path, run_as_root=run_as_root)
  630         backing_file = data.backing_file
  631         fmt = data.file_format
  632         if backing_file is not None:
  633             # Disallow backing files as a security measure.
  634             # This prevents a user from writing an image header into a raw
  635             # volume with a backing file pointing to data they wish to
  636             # access.
  637             raise exception.ImageUnacceptable(
  638                 image_id=image_id,
  639                 reason=_("fmt=%(fmt)s backed by:%(backing_file)s")
  640                 % {'fmt': fmt, 'backing_file': backing_file})
  641 
  642         out_format = fixup_disk_format(image_meta['disk_format'])
  643         convert_image(volume_path, tmp, out_format,
  644                       run_as_root=run_as_root)
  645 
  646         data = qemu_img_info(tmp, run_as_root=run_as_root)
  647         if data.file_format != out_format:
  648             raise exception.ImageUnacceptable(
  649                 image_id=image_id,
  650                 reason=_("Converted to %(f1)s, but format is now %(f2)s") %
  651                 {'f1': out_format, 'f2': data.file_format})
  652 
  653         with open(tmp, 'rb') as image_file:
  654             image_service.update(context, image_id, {},
  655                                  tpool.Proxy(image_file))
  656 
  657 
  658 def check_virtual_size(virtual_size, volume_size, image_id):
  659     virtual_size = int(math.ceil(float(virtual_size) / units.Gi))
  660 
  661     if virtual_size > volume_size:
  662         params = {'image_size': virtual_size,
  663                   'volume_size': volume_size}
  664         reason = _("Image virtual size is %(image_size)dGB"
  665                    " and doesn't fit in a volume of size"
  666                    " %(volume_size)dGB.") % params
  667         raise exception.ImageUnacceptable(image_id=image_id,
  668                                           reason=reason)
  669     return virtual_size
  670 
  671 
  672 def check_available_space(dest, image_size, image_id):
  673     # TODO(e0ne): replace psutil with shutil.disk_usage when we drop
  674     # Python 2.7 support.
  675     if not os.path.isdir(dest):
  676         dest = os.path.dirname(dest)
  677 
  678     free_space = psutil.disk_usage(dest).free
  679     if free_space <= image_size:
  680         msg = ('There is no space on %(dest_dir)s to convert image. '
  681                'Requested: %(image_size)s, available: %(free_space)s.'
  682                ) % {'dest_dir': dest,
  683                     'image_size': image_size,
  684                     'free_space': free_space}
  685         raise exception.ImageTooBig(image_id=image_id, reason=msg)
  686 
  687 
  688 def is_xenserver_format(image_meta):
  689     return (
  690         image_meta['disk_format'] == 'vhd'
  691         and image_meta['container_format'] == 'ovf'
  692     )
  693 
  694 
  695 def set_vhd_parent(vhd_path, parentpath):
  696     utils.execute('vhd-util', 'modify', '-n', vhd_path, '-p', parentpath)
  697 
  698 
  699 def extract_targz(archive_name, target):
  700     utils.execute('tar', '-xzf', archive_name, '-C', target)
  701 
  702 
  703 def fix_vhd_chain(vhd_chain):
  704     for child, parent in zip(vhd_chain[:-1], vhd_chain[1:]):
  705         set_vhd_parent(child, parent)
  706 
  707 
  708 def get_vhd_size(vhd_path):
  709     out, _err = utils.execute('vhd-util', 'query', '-n', vhd_path, '-v')
  710     return int(out)
  711 
  712 
  713 def resize_vhd(vhd_path, size, journal):
  714     utils.execute(
  715         'vhd-util', 'resize', '-n', vhd_path, '-s', '%d' % size, '-j', journal)
  716 
  717 
  718 def coalesce_vhd(vhd_path):
  719     utils.execute(
  720         'vhd-util', 'coalesce', '-n', vhd_path)
  721 
  722 
  723 def create_temporary_file(*args, **kwargs):
  724     fileutils.ensure_tree(CONF.image_conversion_dir)
  725 
  726     fd, tmp = tempfile.mkstemp(dir=CONF.image_conversion_dir, *args, **kwargs)
  727     os.close(fd)
  728     return tmp
  729 
  730 
  731 def cleanup_temporary_file(backend_name):
  732     temp_dir = CONF.image_conversion_dir
  733     if (not temp_dir or not os.path.exists(temp_dir)):
  734         LOG.debug("Configuration image_conversion_dir is None or the path "
  735                   "doesn't exist.")
  736         return
  737     try:
  738         # TODO(wanghao): Consider using os.scandir for better performance in
  739         # future when cinder only supports Python version 3.5+.
  740         files = os.listdir(CONF.image_conversion_dir)
  741         # NOTE(wanghao): For multi-backend case, if one backend was slow
  742         # starting but another backend is up and doing an image conversion,
  743         # init_host should only clean the tmp files which belongs to its
  744         # backend.
  745         for tmp_file in files:
  746             if tmp_file.endswith(backend_name):
  747                 path = os.path.join(temp_dir, tmp_file)
  748                 os.remove(path)
  749     except OSError as e:
  750         LOG.warning("Exception caught while clearing temporary image "
  751                     "files: %s", e)
  752 
  753 
  754 @contextlib.contextmanager
  755 def temporary_file(*args, **kwargs):
  756     tmp = None
  757     try:
  758         tmp = create_temporary_file(*args, **kwargs)
  759         yield tmp
  760     finally:
  761         if tmp:
  762             fileutils.delete_if_exists(tmp)
  763 
  764 
  765 def temporary_dir():
  766     fileutils.ensure_tree(CONF.image_conversion_dir)
  767 
  768     return utils.tempdir(dir=CONF.image_conversion_dir)
  769 
  770 
  771 def coalesce_chain(vhd_chain):
  772     for child, parent in zip(vhd_chain[:-1], vhd_chain[1:]):
  773         with temporary_dir() as directory_for_journal:
  774             size = get_vhd_size(child)
  775             journal_file = os.path.join(
  776                 directory_for_journal, 'vhd-util-resize-journal')
  777             resize_vhd(parent, size, journal_file)
  778             coalesce_vhd(child)
  779 
  780     return vhd_chain[-1]
  781 
  782 
  783 def discover_vhd_chain(directory):
  784     counter = 0
  785     chain = []
  786 
  787     while True:
  788         fpath = os.path.join(directory, '%d.vhd' % counter)
  789         if os.path.exists(fpath):
  790             chain.append(fpath)
  791         else:
  792             break
  793         counter += 1
  794 
  795     return chain
  796 
  797 
  798 def replace_xenserver_image_with_coalesced_vhd(image_file):
  799     with temporary_dir() as tempdir:
  800         extract_targz(image_file, tempdir)
  801         chain = discover_vhd_chain(tempdir)
  802         fix_vhd_chain(chain)
  803         coalesced = coalesce_chain(chain)
  804         fileutils.delete_if_exists(image_file)
  805         os.rename(coalesced, image_file)
  806 
  807 
  808 def decode_cipher(cipher_spec, key_size):
  809     """Decode a dm-crypt style cipher specification string
  810 
  811        The assumed format being cipher[:keycount]-chainmode-ivmode[:ivopts] as
  812        documented under linux/Documentation/device-mapper/dm-crypt.txt in the
  813        kernel source tree.
  814     """
  815     cipher_alg, cipher_mode, ivgen_alg = cipher_spec.split('-')
  816     cipher_alg = cipher_alg + '-' + str(key_size)
  817 
  818     return {'cipher_alg': cipher_alg,
  819             'cipher_mode': cipher_mode,
  820             'ivgen_alg': ivgen_alg}
  821 
  822 
  823 class TemporaryImages(object):
  824     """Manage temporarily downloaded images to avoid downloading it twice.
  825 
  826     In the 'with TemporaryImages.fetch(image_service, ctx, image_id) as tmp'
  827     clause, 'tmp' can be used as the downloaded image path. In addition,
  828     image_utils.fetch() will use the pre-fetched image by the TemporaryImages.
  829     This is useful to inspect image contents before conversion.
  830     """
  831 
  832     def __init__(self, image_service):
  833         self.temporary_images = {}
  834         self.image_service = image_service
  835         image_service.temp_images = self
  836 
  837     @staticmethod
  838     def for_image_service(image_service):
  839         instance = image_service.temp_images
  840         if instance:
  841             return instance
  842         return TemporaryImages(image_service)
  843 
  844     @classmethod
  845     @contextlib.contextmanager
  846     def fetch(cls, image_service, context, image_id, suffix=''):
  847         tmp_images = cls.for_image_service(image_service).temporary_images
  848         with temporary_file(suffix=suffix) as tmp:
  849             fetch_verify_image(context, image_service, image_id, tmp)
  850             user = context.user_id
  851             if not tmp_images.get(user):
  852                 tmp_images[user] = {}
  853             tmp_images[user][image_id] = tmp
  854             LOG.debug("Temporary image %(id)s is fetched for user %(user)s.",
  855                       {'id': image_id, 'user': user})
  856             yield tmp
  857             del tmp_images[user][image_id]
  858         LOG.debug("Temporary image %(id)s for user %(user)s is deleted.",
  859                   {'id': image_id, 'user': user})
  860 
  861     def get(self, context, image_id):
  862         user = context.user_id
  863         if not self.temporary_images.get(user):
  864             return None
  865         return self.temporary_images[user].get(image_id)