"Fossies" - the Fresh Open Source Software Archive

Member "freezer-10.0.0/freezer/openstack/restore.py" (14 Apr 2021, 11903 Bytes) of package /linux/misc/openstack/freezer-10.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "restore.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 9.0.0_vs_10.0.0.

    1 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
    2 #
    3 # Licensed under the Apache License, Version 2.0 (the "License");
    4 # you may not use this file except in compliance with the License.
    5 # You may obtain a copy of the License at
    6 #
    7 #     http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 # Unless required by applicable law or agreed to in writing, software
   10 # distributed under the License is distributed on an "AS IS" BASIS,
   11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   12 # See the License for the specific language governing permissions and
   13 # limitations under the License.
   14 
   15 
   16 """
   17 Freezer restore modes related functions
   18 """
   19 
   20 import os
   21 import shutil
   22 import tempfile
   23 import time
   24 
   25 from oslo_config import cfg
   26 from oslo_log import log
   27 from oslo_serialization import jsonutils as json
   28 
   29 from freezer.utils import utils
   30 
   31 CONF = cfg.CONF
   32 LOG = log.getLogger(__name__)
   33 
   34 
   35 class RestoreOs(object):
   36     def __init__(self, client_manager, container, storage):
   37         self.client_manager = client_manager
   38         self.container = container
   39         self.storage = storage
   40 
   41     def _get_backups(self, path, restore_from_timestamp):
   42         """
   43         :param path:
   44         :type path: str
   45         :param restore_from_timestamp:
   46         :type restore_from_timestamp: int
   47         :return:
   48         """
   49         if self.storage.type == "swift":
   50             swift = self.client_manager.get_swift()
   51             path = "{0}_segments/{1}/".format(self.container, path)
   52             container_name, path = self.get_storage_info(path)
   53             info, backups = swift.get_container(container_name, prefix=path)
   54             backups = sorted(
   55                 map(lambda x: int(x["name"].rsplit("/", 1)[-1]), backups))
   56         elif self.storage.type == "s3":
   57             bucket_name, path = self.get_storage_info(path)
   58             backups = self.storage.list_all_objects(
   59                 bucket_name=bucket_name,
   60                 prefix=path
   61             )
   62             backups = sorted(
   63                 map(lambda x: int(x['Key'].split("/", 2)[1]), backups))
   64         elif self.storage.type == "local":
   65             path = "{0}/{1}".format(self.container, path)
   66             backups = os.listdir(os.path.abspath(path))
   67         elif self.storage.type in ['ssh', 'ftp', 'ftps']:
   68             path = "{0}/{1}".format(self.container, path)
   69             backups = self.storage.listdir(path)
   70         else:
   71             msg = ("{} storage type is not supported at the moment."
   72                    " Try local, SWIFT, SSH(SFTP), FTP or FTPS ".
   73                    format(self.storage.type))
   74             LOG.error(msg)
   75             raise BaseException(msg)
   76         backups = list(filter(lambda x: int(x) >= restore_from_timestamp,
   77                               backups))
   78         if not backups:
   79             msg = "Cannot find backups for path: %s" % path
   80             LOG.error(msg)
   81             raise BaseException(msg)
   82         return backups[-1]
   83 
   84     def get_storage_info(self, path):
   85         storage_info = self.container.split('/', 1)
   86         container_name = storage_info[0]
   87         object_prefix = storage_info[1] if '/' in self.container else ''
   88         if object_prefix != '':
   89             path = "{0}/{1}".format(object_prefix, path)
   90         return container_name, path
   91 
   92     def _create_image(self, path, restore_from_timestamp):
   93         """
   94         :param path:
   95         :param restore_from_timestamp:
   96         :type restore_from_timestamp: int
   97         :return:
   98         """
   99         backup = self._get_backups(path, restore_from_timestamp)
  100         if self.storage.type == 'swift':
  101             swift = self.client_manager.get_swift()
  102             path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
  103             stream = swift.get_object(self.container,
  104                                       "{}/{}".format(path, backup),
  105                                       resp_chunk_size=10000000)
  106             length = int(stream[0]["x-object-meta-length"])
  107             data = utils.ReSizeStream(stream[1], length, 1)
  108             info = stream[0]
  109             image = self.client_manager.create_image(
  110                 name="restore_{}".format(path),
  111                 container_format="bare",
  112                 disk_format="raw",
  113                 data=data)
  114             return info, image
  115         elif self.storage.type == 's3':
  116             if self.storage.get_object_prefix() != '':
  117                 base_path = "{0}/{1}/{2}".format(
  118                     self.storage.get_object_prefix(),
  119                     path,
  120                     backup
  121                 )
  122             else:
  123                 base_path = "{0}/{1}".format(path, backup)
  124             image_file = "{0}/{1}".format(base_path, path)
  125             s3_object = self.storage.get_object(
  126                 bucket_name=self.storage.get_bucket_name(),
  127                 key=image_file
  128             )
  129             stream = utils.S3ResponseStream(data=s3_object['Body'],
  130                                             chunk_size=10000000)
  131             data = utils.ReSizeStream(
  132                 stream,
  133                 s3_object['ContentLength'],
  134                 1
  135             )
  136             metadata = "{0}/metadata".format(base_path)
  137             metadata_object = self.storage.get_object(
  138                 bucket_name=self.storage.get_bucket_name(),
  139                 key=metadata
  140             )
  141             info = json.loads(metadata_object['Body'])
  142 
  143             image = self.client_manager.create_image(
  144                 name="restore_{}".format(path),
  145                 container_format="bare",
  146                 disk_format="raw",
  147                 data=data)
  148             return info, image
  149         elif self.storage.type == 'local':
  150             image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
  151                                                   backup, path)
  152             metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
  153                                                           path, backup)
  154             try:
  155                 data = open(image_file, 'rb')
  156             except Exception:
  157                 msg = "Failed to open image file {}".format(image_file)
  158                 LOG.error(msg)
  159                 raise BaseException(msg)
  160             info = json.load(open(metadata_file, 'r'))
  161             image = self.client_manager.create_image(
  162                 name="restore_{}".format(path),
  163                 container_format="bare",
  164                 disk_format="raw",
  165                 data=data)
  166             return info, image
  167         elif self.storage.type == 'ssh':
  168             image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
  169                                                   backup, path)
  170             metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
  171                                                           path, backup)
  172             try:
  173                 data = self.storage.open(image_file, 'rb')
  174             except Exception:
  175                 msg = "Failed to open remote image file {}".format(image_file)
  176                 LOG.error(msg)
  177                 raise BaseException(msg)
  178             info = json.loads(self.storage.read_metadata_file(metadata_file))
  179             image = self.client_manager.create_image(
  180                 name="restore_{}".format(path),
  181                 container_format="bare",
  182                 disk_format="raw",
  183                 data=data)
  184             return info, image
  185         elif self.storage.type in ['ftp', 'ftps']:
  186             image_file = "{0}/{1}/{2}/{3}".format(self.container, path,
  187                                                   backup, path)
  188             metadata_file = "{0}/{1}/{2}/metadata".format(self.container,
  189                                                           path, backup)
  190             try:
  191                 tmpdir = tempfile.mkdtemp()
  192             except Exception:
  193                 LOG.error("Unable to create a tmp directory")
  194                 raise Exception("Unable to create a tmp directory")
  195             try:
  196                 data_image = utils.path_join(tmpdir, "data_image")
  197                 LOG.info('create image restore ftp storage')
  198                 self.storage.get_file(image_file, data_image)
  199                 data_meta = utils.path_join(tmpdir, "data_meta")
  200                 self.storage.get_file(metadata_file, data_meta)
  201                 data = open(data_image, 'rb')
  202                 info = json.load(open(data_meta, 'r'))
  203                 image = self.client_manager.create_image(
  204                     name="restore_{}".format(path),
  205                     container_format="bare",
  206                     disk_format="raw",
  207                     data=data)
  208                 return info, image
  209             finally:
  210                 shutil.rmtree(tmpdir)
  211         else:
  212             return {}
  213 
  214     def restore_cinder(self, volume_id=None,
  215                        backup_id=None,
  216                        restore_from_timestamp=None):
  217         """
  218         Restoring cinder backup using
  219         :param volume_id:
  220         :param backup_id:
  221         :param restore_from_timestamp:
  222         :return:
  223         """
  224         cinder = self.client_manager.get_cinder()
  225         search_opts = {
  226             'volume_id': volume_id,
  227             'status': 'available',
  228         }
  229         if not backup_id:
  230             backups = cinder.backups.list(search_opts=search_opts)
  231 
  232             def get_backups_from_timestamp(backups, restore_from_timestamp):
  233                 for backup in backups:
  234                     backup_created_date = backup.created_at.split('.')[0]
  235                     backup_created_timestamp = (
  236                         utils.date_to_timestamp(backup_created_date))
  237                     if backup_created_timestamp >= restore_from_timestamp:
  238                         yield backup
  239 
  240             backups_filter = get_backups_from_timestamp(backups,
  241                                                         restore_from_timestamp)
  242             if not backups_filter:
  243                 LOG.warning("no available backups for cinder volume,"
  244                             "restore newest backup")
  245                 backup = max(backups, key=lambda x: x.created_at)
  246             else:
  247                 backup = min(backups_filter, key=lambda x: x.created_at)
  248             backup_id = backup.id
  249         cinder.restores.restore(backup_id=backup_id)
  250 
  251     def restore_cinder_by_glance(self, volume_id, restore_from_timestamp):
  252         """
  253         1) Define swift directory
  254         2) Download and upload to glance
  255         3) Create volume from glance
  256         4) Delete
  257 
  258         :param restore_from_timestamp:
  259         :type restore_from_timestamp: int
  260         :param volume_id: - id of attached cinder volume
  261         """
  262         (info, image) = self._create_image(volume_id, restore_from_timestamp)
  263         length = int(info["x-object-meta-length"])
  264         gb = 1073741824
  265         size = length / gb
  266         if length % gb > 0:
  267             size += 1
  268         LOG.info("Creating volume from image")
  269         cinder_client = self.client_manager.get_cinder()
  270         volume = cinder_client.volumes.create(
  271             size,
  272             imageRef=image.id,
  273             name=info.get('volume_name',
  274                           CONF.get('backup_name',
  275                                    CONF.get('cinder_vol_id', None)
  276                                    )
  277                           )
  278         )
  279         while volume.status != "available":
  280             try:
  281                 LOG.info("Volume copy status: " + volume.status)
  282                 volume = cinder_client.volumes.get(volume.id)
  283                 if volume.status == "error":
  284                     raise Exception("Volume copy status: error")
  285                 time.sleep(5)
  286             except Exception as e:
  287                 LOG.exception(e)
  288                 if volume.status != "error":
  289                     LOG.warn("Exception getting volume status")
  290 
  291         LOG.info("Deleting temporary image {}".format(image.id))
  292         self.client_manager.get_glance().images.delete(image.id)