"Fossies" - the Fresh Open Source Software Archive

Member "glance-19.0.0/glance/db/sqlalchemy/metadata.py" (16 Oct 2019, 18024 Bytes) of package /linux/misc/openstack/glance-19.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "metadata.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 18.0.0_vs_19.0.0.

    1 # Copyright 2010 United States Government as represented by the
    2 # Administrator of the National Aeronautics and Space Administration.
    3 # All Rights Reserved.
    4 #
    5 # Copyright 2013 OpenStack Foundation
    6 # Copyright 2013 Intel Corporation
    7 #
    8 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    9 #    not use this file except in compliance with the License. You may obtain
   10 #    a copy of the License at
   11 #
   12 #         http://www.apache.org/licenses/LICENSE-2.0
   13 #
   14 #    Unless required by applicable law or agreed to in writing, software
   15 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   16 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   17 #    License for the specific language governing permissions and limitations
   18 #    under the License.
   19 
   20 import json
   21 import os
   22 from os.path import isfile
   23 from os.path import join
   24 import re
   25 
   26 from oslo_config import cfg
   27 from oslo_log import log as logging
   28 from oslo_utils import encodeutils
   29 import six
   30 import sqlalchemy
   31 from sqlalchemy import and_
   32 from sqlalchemy.schema import MetaData
   33 from sqlalchemy.sql import select
   34 
   35 from glance.common import timeutils
   36 from glance.i18n import _, _LE, _LI, _LW
   37 
   38 LOG = logging.getLogger(__name__)
   39 
   40 metadata_opts = [
   41     cfg.StrOpt('metadata_source_path',
   42                default='/etc/glance/metadefs/',
   43                help=_("""
   44 Absolute path to the directory where JSON metadefs files are stored.
   45 
   46 Glance Metadata Definitions ("metadefs") are served from the database,
   47 but are stored in files in the JSON format.  The files in this
   48 directory are used to initialize the metadefs in the database.
   49 Additionally, when metadefs are exported from the database, the files
   50 are written to this directory.
   51 
   52 NOTE: If you plan to export metadefs, make sure that this directory
   53 has write permissions set for the user being used to run the
   54 glance-api service.
   55 
   56 Possible values:
   57     * String value representing a valid absolute pathname
   58 
   59 Related options:
   60     * None
   61 
   62 """)),
   63 ]
   64 
   65 CONF = cfg.CONF
   66 CONF.register_opts(metadata_opts)
   67 
   68 
   69 def get_metadef_namespaces_table(meta):
   70     return sqlalchemy.Table('metadef_namespaces', meta, autoload=True)
   71 
   72 
   73 def get_metadef_resource_types_table(meta):
   74     return sqlalchemy.Table('metadef_resource_types', meta, autoload=True)
   75 
   76 
   77 def get_metadef_namespace_resource_types_table(meta):
   78     return sqlalchemy.Table('metadef_namespace_resource_types', meta,
   79                             autoload=True)
   80 
   81 
   82 def get_metadef_properties_table(meta):
   83     return sqlalchemy.Table('metadef_properties', meta, autoload=True)
   84 
   85 
   86 def get_metadef_objects_table(meta):
   87     return sqlalchemy.Table('metadef_objects', meta, autoload=True)
   88 
   89 
   90 def get_metadef_tags_table(meta):
   91     return sqlalchemy.Table('metadef_tags', meta, autoload=True)
   92 
   93 
   94 def _get_resource_type_id(meta, name):
   95     rt_table = get_metadef_resource_types_table(meta)
   96     resource_type = (
   97         select([rt_table.c.id]).
   98         where(rt_table.c.name == name).
   99         select_from(rt_table).
  100         execute().fetchone())
  101     if resource_type:
  102         return resource_type[0]
  103     return None
  104 
  105 
  106 def _get_resource_type(meta, resource_type_id):
  107     rt_table = get_metadef_resource_types_table(meta)
  108     return (
  109         rt_table.select().
  110         where(rt_table.c.id == resource_type_id).
  111         execute().fetchone())
  112 
  113 
  114 def _get_namespace_resource_types(meta, namespace_id):
  115     namespace_resource_types_table = (
  116         get_metadef_namespace_resource_types_table(meta))
  117     return (
  118         namespace_resource_types_table.select().
  119         where(namespace_resource_types_table.c.namespace_id == namespace_id).
  120         execute().fetchall())
  121 
  122 
  123 def _get_namespace_resource_type_by_ids(meta, namespace_id, rt_id):
  124     namespace_resource_types_table = (
  125         get_metadef_namespace_resource_types_table(meta))
  126     return (
  127         namespace_resource_types_table.select().
  128         where(and_(
  129             namespace_resource_types_table.c.namespace_id == namespace_id,
  130             namespace_resource_types_table.c.resource_type_id == rt_id)).
  131         execute().fetchone())
  132 
  133 
  134 def _get_properties(meta, namespace_id):
  135     properties_table = get_metadef_properties_table(meta)
  136     return (
  137         properties_table.select().
  138         where(properties_table.c.namespace_id == namespace_id).
  139         execute().fetchall())
  140 
  141 
  142 def _get_objects(meta, namespace_id):
  143     objects_table = get_metadef_objects_table(meta)
  144     return (
  145         objects_table.select().
  146         where(objects_table.c.namespace_id == namespace_id).
  147         execute().fetchall())
  148 
  149 
  150 def _get_tags(meta, namespace_id):
  151     tags_table = get_metadef_tags_table(meta)
  152     return (
  153         tags_table.select().
  154         where(tags_table.c.namespace_id == namespace_id).
  155         execute().fetchall())
  156 
  157 
  158 def _get_resource_id(table, namespace_id, resource_name):
  159     resource = (
  160         select([table.c.id]).
  161         where(and_(table.c.namespace_id == namespace_id,
  162                    table.c.name == resource_name)).
  163         select_from(table).
  164         execute().fetchone())
  165     if resource:
  166         return resource[0]
  167     return None
  168 
  169 
  170 def _clear_metadata(meta):
  171     metadef_tables = [get_metadef_properties_table(meta),
  172                       get_metadef_objects_table(meta),
  173                       get_metadef_tags_table(meta),
  174                       get_metadef_namespace_resource_types_table(meta),
  175                       get_metadef_namespaces_table(meta),
  176                       get_metadef_resource_types_table(meta)]
  177 
  178     for table in metadef_tables:
  179         table.delete().execute()
  180         LOG.info(_LI("Table %s has been cleared"), table)
  181 
  182 
  183 def _clear_namespace_metadata(meta, namespace_id):
  184     metadef_tables = [get_metadef_properties_table(meta),
  185                       get_metadef_objects_table(meta),
  186                       get_metadef_tags_table(meta),
  187                       get_metadef_namespace_resource_types_table(meta)]
  188     namespaces_table = get_metadef_namespaces_table(meta)
  189 
  190     for table in metadef_tables:
  191         table.delete().where(table.c.namespace_id == namespace_id).execute()
  192     namespaces_table.delete().where(
  193         namespaces_table.c.id == namespace_id).execute()
  194 
  195 
  196 def _populate_metadata(meta, metadata_path=None, merge=False,
  197                        prefer_new=False, overwrite=False):
  198     if not metadata_path:
  199         metadata_path = CONF.metadata_source_path
  200 
  201     try:
  202         if isfile(metadata_path):
  203             json_schema_files = [metadata_path]
  204         else:
  205             json_schema_files = [f for f in os.listdir(metadata_path)
  206                                  if isfile(join(metadata_path, f))
  207                                  and f.endswith('.json')]
  208     except OSError as e:
  209         LOG.error(encodeutils.exception_to_unicode(e))
  210         return
  211 
  212     if not json_schema_files:
  213         LOG.error(_LE("Json schema files not found in %s. Aborting."),
  214                   metadata_path)
  215         return
  216 
  217     namespaces_table = get_metadef_namespaces_table(meta)
  218     namespace_rt_table = get_metadef_namespace_resource_types_table(meta)
  219     objects_table = get_metadef_objects_table(meta)
  220     tags_table = get_metadef_tags_table(meta)
  221     properties_table = get_metadef_properties_table(meta)
  222     resource_types_table = get_metadef_resource_types_table(meta)
  223 
  224     for json_schema_file in json_schema_files:
  225         try:
  226             file = join(metadata_path, json_schema_file)
  227             with open(file) as json_file:
  228                 metadata = json.load(json_file)
  229         except Exception as e:
  230             LOG.error(_LE("Failed to parse json file %(file_path)s while "
  231                           "populating metadata due to: %(error_msg)s"),
  232                       {"file_path": file,
  233                        "error_msg": encodeutils.exception_to_unicode(e)})
  234             continue
  235 
  236         values = {
  237             'namespace': metadata.get('namespace'),
  238             'display_name': metadata.get('display_name'),
  239             'description': metadata.get('description'),
  240             'visibility': metadata.get('visibility'),
  241             'protected': metadata.get('protected'),
  242             'owner': metadata.get('owner', 'admin')
  243         }
  244 
  245         db_namespace = select(
  246             [namespaces_table.c.id]
  247         ).where(
  248             namespaces_table.c.namespace == values['namespace']
  249         ).select_from(
  250             namespaces_table
  251         ).execute().fetchone()
  252 
  253         if db_namespace and overwrite:
  254             LOG.info(_LI("Overwriting namespace %s"), values['namespace'])
  255             _clear_namespace_metadata(meta, db_namespace[0])
  256             db_namespace = None
  257 
  258         if not db_namespace:
  259             values.update({'created_at': timeutils.utcnow()})
  260             _insert_data_to_db(namespaces_table, values)
  261 
  262             db_namespace = select(
  263                 [namespaces_table.c.id]
  264             ).where(
  265                 namespaces_table.c.namespace == values['namespace']
  266             ).select_from(
  267                 namespaces_table
  268             ).execute().fetchone()
  269         elif not merge:
  270             LOG.info(_LI("Skipping namespace %s. It already exists in the "
  271                          "database."), values['namespace'])
  272             continue
  273         elif prefer_new:
  274             values.update({'updated_at': timeutils.utcnow()})
  275             _update_data_in_db(namespaces_table, values,
  276                                namespaces_table.c.id, db_namespace[0])
  277 
  278         namespace_id = db_namespace[0]
  279 
  280         for resource_type in metadata.get('resource_type_associations', []):
  281             rt_id = _get_resource_type_id(meta, resource_type['name'])
  282             if not rt_id:
  283                 val = {
  284                     'name': resource_type['name'],
  285                     'created_at': timeutils.utcnow(),
  286                     'protected': True
  287                 }
  288                 _insert_data_to_db(resource_types_table, val)
  289                 rt_id = _get_resource_type_id(meta, resource_type['name'])
  290             elif prefer_new:
  291                 val = {'updated_at': timeutils.utcnow()}
  292                 _update_data_in_db(resource_types_table, val,
  293                                    resource_types_table.c.id, rt_id)
  294 
  295             values = {
  296                 'namespace_id': namespace_id,
  297                 'resource_type_id': rt_id,
  298                 'properties_target': resource_type.get(
  299                     'properties_target'),
  300                 'prefix': resource_type.get('prefix')
  301             }
  302             namespace_resource_type = _get_namespace_resource_type_by_ids(
  303                 meta, namespace_id, rt_id)
  304             if not namespace_resource_type:
  305                 values.update({'created_at': timeutils.utcnow()})
  306                 _insert_data_to_db(namespace_rt_table, values)
  307             elif prefer_new:
  308                 values.update({'updated_at': timeutils.utcnow()})
  309                 _update_rt_association(namespace_rt_table, values,
  310                                        rt_id, namespace_id)
  311 
  312         for property, schema in six.iteritems(metadata.get('properties',
  313                                                            {})):
  314             values = {
  315                 'name': property,
  316                 'namespace_id': namespace_id,
  317                 'json_schema': json.dumps(schema)
  318             }
  319             property_id = _get_resource_id(properties_table,
  320                                            namespace_id, property)
  321             if not property_id:
  322                 values.update({'created_at': timeutils.utcnow()})
  323                 _insert_data_to_db(properties_table, values)
  324             elif prefer_new:
  325                 values.update({'updated_at': timeutils.utcnow()})
  326                 _update_data_in_db(properties_table, values,
  327                                    properties_table.c.id, property_id)
  328 
  329         for object in metadata.get('objects', []):
  330             values = {
  331                 'name': object['name'],
  332                 'description': object.get('description'),
  333                 'namespace_id': namespace_id,
  334                 'json_schema': json.dumps(
  335                     object.get('properties'))
  336             }
  337             object_id = _get_resource_id(objects_table, namespace_id,
  338                                          object['name'])
  339             if not object_id:
  340                 values.update({'created_at': timeutils.utcnow()})
  341                 _insert_data_to_db(objects_table, values)
  342             elif prefer_new:
  343                 values.update({'updated_at': timeutils.utcnow()})
  344                 _update_data_in_db(objects_table, values,
  345                                    objects_table.c.id, object_id)
  346 
  347         for tag in metadata.get('tags', []):
  348             values = {
  349                 'name': tag.get('name'),
  350                 'namespace_id': namespace_id,
  351             }
  352             tag_id = _get_resource_id(tags_table, namespace_id, tag['name'])
  353             if not tag_id:
  354                 values.update({'created_at': timeutils.utcnow()})
  355                 _insert_data_to_db(tags_table, values)
  356             elif prefer_new:
  357                 values.update({'updated_at': timeutils.utcnow()})
  358                 _update_data_in_db(tags_table, values,
  359                                    tags_table.c.id, tag_id)
  360 
  361         LOG.info(_LI("File %s loaded to database."), file)
  362 
  363     LOG.info(_LI("Metadata loading finished"))
  364 
  365 
  366 def _insert_data_to_db(table, values, log_exception=True):
  367     try:
  368         table.insert(values=values).execute()
  369     except sqlalchemy.exc.IntegrityError:
  370         if log_exception:
  371             LOG.warning(_LW("Duplicate entry for values: %s"), values)
  372 
  373 
  374 def _update_data_in_db(table, values, column, value):
  375     try:
  376         (table.update(values=values).
  377          where(column == value).execute())
  378     except sqlalchemy.exc.IntegrityError:
  379         LOG.warning(_LW("Duplicate entry for values: %s"), values)
  380 
  381 
  382 def _update_rt_association(table, values, rt_id, namespace_id):
  383     try:
  384         (table.update(values=values).
  385          where(and_(table.c.resource_type_id == rt_id,
  386                     table.c.namespace_id == namespace_id)).execute())
  387     except sqlalchemy.exc.IntegrityError:
  388         LOG.warning(_LW("Duplicate entry for values: %s"), values)
  389 
  390 
  391 def _export_data_to_file(meta, path):
  392     if not path:
  393         path = CONF.metadata_source_path
  394 
  395     namespace_table = get_metadef_namespaces_table(meta)
  396     namespaces = namespace_table.select().execute().fetchall()
  397 
  398     pattern = re.compile(r'[\W_]+', re.UNICODE)
  399 
  400     for id, namespace in enumerate(namespaces, start=1):
  401         namespace_id = namespace['id']
  402         namespace_file_name = pattern.sub('', namespace['display_name'])
  403 
  404         values = {
  405             'namespace': namespace['namespace'],
  406             'display_name': namespace['display_name'],
  407             'description': namespace['description'],
  408             'visibility': namespace['visibility'],
  409             'protected': namespace['protected'],
  410             'resource_type_associations': [],
  411             'properties': {},
  412             'objects': [],
  413             'tags': []
  414         }
  415 
  416         namespace_resource_types = _get_namespace_resource_types(meta,
  417                                                                  namespace_id)
  418         db_objects = _get_objects(meta, namespace_id)
  419         db_properties = _get_properties(meta, namespace_id)
  420         db_tags = _get_tags(meta, namespace_id)
  421 
  422         resource_types = []
  423         for namespace_resource_type in namespace_resource_types:
  424             resource_type = _get_resource_type(
  425                 meta, namespace_resource_type['resource_type_id'])
  426             resource_types.append({
  427                 'name': resource_type['name'],
  428                 'prefix': namespace_resource_type['prefix'],
  429                 'properties_target': namespace_resource_type[
  430                     'properties_target']
  431             })
  432         values.update({
  433             'resource_type_associations': resource_types
  434         })
  435 
  436         objects = []
  437         for object in db_objects:
  438             objects.append({
  439                 "name": object['name'],
  440                 "description": object['description'],
  441                 "properties": json.loads(object['json_schema'])
  442             })
  443         values.update({
  444             'objects': objects
  445         })
  446 
  447         properties = {}
  448         for property in db_properties:
  449             properties.update({
  450                 property['name']: json.loads(property['json_schema'])
  451             })
  452         values.update({
  453             'properties': properties
  454         })
  455 
  456         tags = []
  457         for tag in db_tags:
  458             tags.append({
  459                 "name": tag['name']
  460             })
  461         values.update({
  462             'tags': tags
  463         })
  464 
  465         try:
  466             file_name = ''.join([path, namespace_file_name, '.json'])
  467             if isfile(file_name):
  468                 LOG.info(_LI("Overwriting: %s"), file_name)
  469             with open(file_name, 'w') as json_file:
  470                 json_file.write(json.dumps(values))
  471         except Exception as e:
  472             LOG.exception(encodeutils.exception_to_unicode(e))
  473         LOG.info(_LI("Namespace %(namespace)s saved in %(file)s"), {
  474             'namespace': namespace_file_name, 'file': file_name})
  475 
  476 
  477 def db_load_metadefs(engine, metadata_path=None, merge=False,
  478                      prefer_new=False, overwrite=False):
  479     meta = MetaData()
  480     meta.bind = engine
  481 
  482     if not merge and (prefer_new or overwrite):
  483         LOG.error(_LE("To use --prefer_new or --overwrite you need to combine "
  484                       "of these options with --merge option."))
  485         return
  486 
  487     if prefer_new and overwrite and merge:
  488         LOG.error(_LE("Please provide no more than one option from this list: "
  489                       "--prefer_new, --overwrite"))
  490         return
  491 
  492     _populate_metadata(meta, metadata_path, merge, prefer_new, overwrite)
  493 
  494 
  495 def db_unload_metadefs(engine):
  496     meta = MetaData()
  497     meta.bind = engine
  498 
  499     _clear_metadata(meta)
  500 
  501 
  502 def db_export_metadefs(engine, metadata_path=None):
  503     meta = MetaData()
  504     meta.bind = engine
  505 
  506     _export_data_to_file(meta, metadata_path)