"Fossies" - the Fresh Open Source Software Archive

Member "manila-11.0.1/manila/db/sqlalchemy/api.py" (1 Feb 2021, 200187 Bytes) of package /linux/misc/openstack/manila-11.0.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "api.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 11.0.0_vs_11.0.1.

    1 # Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
    2 # Copyright 2010 United States Government as represented by the
    3 # Administrator of the National Aeronautics and Space Administration.
    4 # Copyright (c) 2014 Mirantis, Inc.
    5 # All Rights Reserved.
    6 #
    7 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    8 #    not use this file except in compliance with the License. You may obtain
    9 #    a copy of the License at
   10 #
   11 #         http://www.apache.org/licenses/LICENSE-2.0
   12 #
   13 #    Unless required by applicable law or agreed to in writing, software
   14 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   15 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   16 #    License for the specific language governing permissions and limitations
   17 #    under the License.
   18 
   19 """Implementation of SQLAlchemy backend."""
   20 
   21 import copy
   22 import datetime
   23 from functools import wraps
   24 import ipaddress
   25 import sys
   26 import warnings
   27 
   28 # NOTE(uglide): Required to override default oslo_db Query class
   29 import manila.db.sqlalchemy.query  # noqa
   30 
   31 from oslo_config import cfg
   32 from oslo_db import api as oslo_db_api
   33 from oslo_db import exception as db_exc
   34 from oslo_db import exception as db_exception
   35 from oslo_db import options as db_options
   36 from oslo_db.sqlalchemy import session
   37 from oslo_db.sqlalchemy import utils as db_utils
   38 from oslo_log import log
   39 from oslo_utils import excutils
   40 from oslo_utils import timeutils
   41 from oslo_utils import uuidutils
   42 import six
   43 from sqlalchemy import MetaData
   44 from sqlalchemy import or_
   45 from sqlalchemy.orm import joinedload
   46 from sqlalchemy.orm import subqueryload
   47 from sqlalchemy.sql.expression import literal
   48 from sqlalchemy.sql.expression import true
   49 from sqlalchemy.sql import func
   50 
   51 from manila.common import constants
   52 from manila.db.sqlalchemy import models
   53 from manila.db.sqlalchemy import utils
   54 from manila import exception
   55 from manila.i18n import _
   56 from manila import quota
   57 
   58 CONF = cfg.CONF
   59 
   60 LOG = log.getLogger(__name__)
   61 QUOTAS = quota.QUOTAS
   62 
   63 _DEFAULT_QUOTA_NAME = 'default'
   64 PER_PROJECT_QUOTAS = []
   65 
   66 _FACADE = None
   67 
   68 _DEFAULT_SQL_CONNECTION = 'sqlite://'
   69 db_options.set_defaults(cfg.CONF,
   70                         connection=_DEFAULT_SQL_CONNECTION)
   71 
   72 
   73 def _create_facade_lazily():
   74     global _FACADE
   75     if _FACADE is None:
   76         _FACADE = session.EngineFacade.from_config(cfg.CONF)
   77     return _FACADE
   78 
   79 
   80 def get_engine():
   81     facade = _create_facade_lazily()
   82     return facade.get_engine()
   83 
   84 
   85 def get_session(**kwargs):
   86     facade = _create_facade_lazily()
   87     return facade.get_session(**kwargs)
   88 
   89 
   90 def get_backend():
   91     """The backend is this module itself."""
   92 
   93     return sys.modules[__name__]
   94 
   95 
   96 def is_admin_context(context):
   97     """Indicates if the request context is an administrator."""
   98     if not context:
   99         warnings.warn(_('Use of empty request context is deprecated'),
  100                       DeprecationWarning)
  101         raise Exception('die')
  102     return context.is_admin
  103 
  104 
  105 def is_user_context(context):
  106     """Indicates if the request context is a normal user."""
  107     if not context:
  108         return False
  109     if context.is_admin:
  110         return False
  111     if not context.user_id or not context.project_id:
  112         return False
  113     return True
  114 
  115 
  116 def authorize_project_context(context, project_id):
  117     """Ensures a request has permission to access the given project."""
  118     if is_user_context(context):
  119         if not context.project_id:
  120             raise exception.NotAuthorized()
  121         elif context.project_id != project_id:
  122             raise exception.NotAuthorized()
  123 
  124 
  125 def authorize_user_context(context, user_id):
  126     """Ensures a request has permission to access the given user."""
  127     if is_user_context(context):
  128         if not context.user_id:
  129             raise exception.NotAuthorized()
  130         elif context.user_id != user_id:
  131             raise exception.NotAuthorized()
  132 
  133 
  134 def authorize_quota_class_context(context, class_name):
  135     """Ensures a request has permission to access the given quota class."""
  136     if is_user_context(context):
  137         if not context.quota_class:
  138             raise exception.NotAuthorized()
  139         elif context.quota_class != class_name:
  140             raise exception.NotAuthorized()
  141 
  142 
  143 def require_admin_context(f):
  144     """Decorator to require admin request context.
  145 
  146     The first argument to the wrapped function must be the context.
  147 
  148     """
  149     @wraps(f)
  150     def wrapper(*args, **kwargs):
  151         if not is_admin_context(args[0]):
  152             raise exception.AdminRequired()
  153         return f(*args, **kwargs)
  154     return wrapper
  155 
  156 
  157 def require_context(f):
  158     """Decorator to require *any* user or admin context.
  159 
  160     This does no authorization for user or project access matching, see
  161     :py:func:`authorize_project_context` and
  162     :py:func:`authorize_user_context`.
  163 
  164     The first argument to the wrapped function must be the context.
  165 
  166     """
  167     @wraps(f)
  168     def wrapper(*args, **kwargs):
  169         if not is_admin_context(args[0]) and not is_user_context(args[0]):
  170             raise exception.NotAuthorized()
  171         return f(*args, **kwargs)
  172     return wrapper
  173 
  174 
  175 def require_share_exists(f):
  176     """Decorator to require the specified share to exist.
  177 
  178     Requires the wrapped function to use context and share_id as
  179     their first two arguments.
  180     """
  181     @wraps(f)
  182     def wrapper(context, share_id, *args, **kwargs):
  183         share_get(context, share_id)
  184         return f(context, share_id, *args, **kwargs)
  185     wrapper.__name__ = f.__name__
  186     return wrapper
  187 
  188 
  189 def require_share_instance_exists(f):
  190     """Decorator to require the specified share instance to exist.
  191 
  192     Requires the wrapped function to use context and share_instance_id as
  193     their first two arguments.
  194     """
  195     @wraps(f)
  196     def wrapper(context, share_instance_id, *args, **kwargs):
  197         share_instance_get(context, share_instance_id)
  198         return f(context, share_instance_id, *args, **kwargs)
  199     wrapper.__name__ = f.__name__
  200     return wrapper
  201 
  202 
  203 def apply_sorting(model, query, sort_key, sort_dir):
  204     if sort_dir.lower() not in ('desc', 'asc'):
  205         msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' "
  206                 "and sort direction is '%(sort_dir)s'.") % {
  207                     "sort_key": sort_key, "sort_dir": sort_dir}
  208         raise exception.InvalidInput(reason=msg)
  209 
  210     # NOTE(maaoyu): We add the additional sort by ID in this case to
  211     # get deterministic results. Without the ordering by ID this could
  212     # lead to flapping return lists.
  213     sort_keys = [sort_key]
  214     if sort_key != 'id':
  215         sort_keys.append('id')
  216 
  217     for sort_key in sort_keys:
  218         sort_attr = getattr(model, sort_key)
  219         sort_method = getattr(sort_attr, sort_dir.lower())
  220         query = query.order_by(sort_method())
  221 
  222     return query
  223 
  224 
  225 def handle_db_data_error(f):
  226     def wrapper(*args, **kwargs):
  227         try:
  228             return f(*args, **kwargs)
  229         except db_exc.DBDataError:
  230             msg = _('Error writing field to database.')
  231             LOG.exception(msg)
  232             raise exception.Invalid(msg)
  233 
  234     return wrapper
  235 
  236 
  237 def model_query(context, model, *args, **kwargs):
  238     """Query helper that accounts for context's `read_deleted` field.
  239 
  240     :param context: context to query under
  241     :param model: model to query. Must be a subclass of ModelBase.
  242     :param session: if present, the session to use
  243     :param read_deleted: if present, overrides context's read_deleted field.
  244     :param project_only: if present and context is user-type, then restrict
  245             query to match the context's project_id.
  246     """
  247     session = kwargs.get('session') or get_session()
  248     read_deleted = kwargs.get('read_deleted') or context.read_deleted
  249     project_only = kwargs.get('project_only')
  250     kwargs = dict()
  251 
  252     if project_only and not context.is_admin:
  253         kwargs['project_id'] = context.project_id
  254     if read_deleted in ('no', 'n', False):
  255         kwargs['deleted'] = False
  256     elif read_deleted in ('yes', 'y', True):
  257         kwargs['deleted'] = True
  258 
  259     return db_utils.model_query(
  260         model=model, session=session, args=args, **kwargs)
  261 
  262 
  263 def exact_filter(query, model, filters, legal_keys,
  264                  created_at_key='created_at'):
  265     """Applies exact match filtering to a query.
  266 
  267     Returns the updated query.  Modifies filters argument to remove
  268     filters consumed.
  269 
  270     :param query: query to apply filters to
  271     :param model: model object the query applies to, for IN-style
  272                   filtering
  273     :param filters: dictionary of filters; values that are lists,
  274                     tuples, sets, or frozensets cause an 'IN' test to
  275                     be performed, while exact matching ('==' operator)
  276                     is used for other values
  277     :param legal_keys: list of keys to apply exact filtering to
  278     """
  279 
  280     filter_dict = {}
  281     created_at_attr = getattr(model, created_at_key, None)
  282     # Walk through all the keys
  283     for key in legal_keys:
  284         # Skip ones we're not filtering on
  285         if key not in filters:
  286             continue
  287 
  288         # OK, filtering on this key; what value do we search for?
  289         value = filters.pop(key)
  290 
  291         if key == 'created_since' and created_at_attr:
  292             # This is a reserved query parameter to indicate resources created
  293             # after a particular datetime
  294             value = timeutils.normalize_time(value)
  295             query = query.filter(created_at_attr.op('>=')(value))
  296         elif key == 'created_before' and created_at_attr:
  297             # This is a reserved query parameter to indicate resources created
  298             # before a particular datetime
  299             value = timeutils.normalize_time(value)
  300             query = query.filter(created_at_attr.op('<=')(value))
  301         elif isinstance(value, (list, tuple, set, frozenset)):
  302             # Looking for values in a list; apply to query directly
  303             column_attr = getattr(model, key)
  304             query = query.filter(column_attr.in_(value))
  305         else:
  306             # OK, simple exact match; save for later
  307             filter_dict[key] = value
  308 
  309     # Apply simple exact matches
  310     if filter_dict:
  311         query = query.filter_by(**filter_dict)
  312 
  313     return query
  314 
  315 
  316 def ensure_model_dict_has_id(model_dict):
  317     if not model_dict.get('id'):
  318         model_dict['id'] = uuidutils.generate_uuid()
  319     return model_dict
  320 
  321 
  322 def _sync_shares(context, project_id, user_id, session, share_type_id=None):
  323     (shares, gigs) = share_data_get_for_project(
  324         context, project_id, user_id, share_type_id=share_type_id,
  325         session=session)
  326     return {'shares': shares}
  327 
  328 
  329 def _sync_snapshots(context, project_id, user_id, session, share_type_id=None):
  330     (snapshots, gigs) = snapshot_data_get_for_project(
  331         context, project_id, user_id, share_type_id=share_type_id,
  332         session=session)
  333     return {'snapshots': snapshots}
  334 
  335 
  336 def _sync_gigabytes(context, project_id, user_id, session, share_type_id=None):
  337     _junk, share_gigs = share_data_get_for_project(
  338         context, project_id, user_id, share_type_id=share_type_id,
  339         session=session)
  340     return {"gigabytes": share_gigs}
  341 
  342 
  343 def _sync_snapshot_gigabytes(context, project_id, user_id, session,
  344                              share_type_id=None):
  345     _junk, snapshot_gigs = snapshot_data_get_for_project(
  346         context, project_id, user_id, share_type_id=share_type_id,
  347         session=session)
  348     return {"snapshot_gigabytes": snapshot_gigs}
  349 
  350 
  351 def _sync_share_networks(context, project_id, user_id, session,
  352                          share_type_id=None):
  353     share_networks_count = count_share_networks(
  354         context, project_id, user_id, share_type_id=share_type_id,
  355         session=session)
  356     return {'share_networks': share_networks_count}
  357 
  358 
  359 def _sync_share_groups(context, project_id, user_id, session,
  360                        share_type_id=None):
  361     share_groups_count = count_share_groups(
  362         context, project_id, user_id, share_type_id=share_type_id,
  363         session=session)
  364     return {'share_groups': share_groups_count}
  365 
  366 
  367 def _sync_share_group_snapshots(context, project_id, user_id, session,
  368                                 share_type_id=None):
  369     share_group_snapshots_count = count_share_group_snapshots(
  370         context, project_id, user_id, share_type_id=share_type_id,
  371         session=session)
  372     return {'share_group_snapshots': share_group_snapshots_count}
  373 
  374 
  375 def _sync_share_replicas(context, project_id, user_id, session,
  376                          share_type_id=None):
  377     share_replicas_count, _junk = share_replica_data_get_for_project(
  378         context, project_id, user_id, session, share_type_id=share_type_id)
  379     return {'share_replicas': share_replicas_count}
  380 
  381 
  382 def _sync_replica_gigabytes(context, project_id, user_id, session,
  383                             share_type_id=None):
  384     _junk, replica_gigs = share_replica_data_get_for_project(
  385         context, project_id, user_id, session, share_type_id=share_type_id)
  386     return {'replica_gigabytes': replica_gigs}
  387 
  388 
  389 QUOTA_SYNC_FUNCTIONS = {
  390     '_sync_shares': _sync_shares,
  391     '_sync_snapshots': _sync_snapshots,
  392     '_sync_gigabytes': _sync_gigabytes,
  393     '_sync_snapshot_gigabytes': _sync_snapshot_gigabytes,
  394     '_sync_share_networks': _sync_share_networks,
  395     '_sync_share_groups': _sync_share_groups,
  396     '_sync_share_group_snapshots': _sync_share_group_snapshots,
  397     '_sync_share_replicas': _sync_share_replicas,
  398     '_sync_replica_gigabytes': _sync_replica_gigabytes,
  399 }
  400 
  401 
  402 ###################
  403 
  404 @require_admin_context
  405 def share_resources_host_update(context, current_host, new_host):
  406     """Updates the 'host' attribute of resources"""
  407 
  408     resources = {
  409         'instances': models.ShareInstance,
  410         'servers': models.ShareServer,
  411         'groups': models.ShareGroup,
  412     }
  413     result = {}
  414 
  415     session = get_session()
  416     with session.begin():
  417         for res_name, res_model in resources.items():
  418             host_field = res_model.host
  419             query = model_query(
  420                 context, res_model, session=session, read_deleted="no",
  421             ).filter(host_field.like('{}%'.format(current_host)))
  422             count = query.update(
  423                 {host_field: func.replace(host_field, current_host, new_host)},
  424                 synchronize_session=False)
  425             result.update({res_name: count})
  426     return result
  427 
  428 
  429 ###################
  430 
  431 
  432 @require_admin_context
  433 def service_destroy(context, service_id):
  434     session = get_session()
  435     with session.begin():
  436         service_ref = service_get(context, service_id, session=session)
  437         service_ref.soft_delete(session)
  438 
  439 
  440 @require_admin_context
  441 def service_get(context, service_id, session=None):
  442     result = (model_query(
  443         context,
  444         models.Service,
  445         session=session).
  446         filter_by(id=service_id).
  447         first())
  448     if not result:
  449         raise exception.ServiceNotFound(service_id=service_id)
  450 
  451     return result
  452 
  453 
  454 @require_admin_context
  455 def service_get_all(context, disabled=None):
  456     query = model_query(context, models.Service)
  457 
  458     if disabled is not None:
  459         query = query.filter_by(disabled=disabled)
  460 
  461     return query.all()
  462 
  463 
  464 @require_admin_context
  465 def service_get_all_by_topic(context, topic):
  466     return (model_query(
  467         context, models.Service, read_deleted="no").
  468         filter_by(disabled=False).
  469         filter_by(topic=topic).
  470         all())
  471 
  472 
  473 @require_admin_context
  474 def service_get_by_host_and_topic(context, host, topic):
  475     result = (model_query(
  476         context, models.Service, read_deleted="no").
  477         filter_by(disabled=False).
  478         filter_by(host=host).
  479         filter_by(topic=topic).
  480         first())
  481     if not result:
  482         raise exception.ServiceNotFound(service_id=host)
  483     return result
  484 
  485 
  486 @require_admin_context
  487 def _service_get_all_topic_subquery(context, session, topic, subq, label):
  488     sort_value = getattr(subq.c, label)
  489     return (model_query(context, models.Service,
  490                         func.coalesce(sort_value, 0),
  491                         session=session, read_deleted="no").
  492             filter_by(topic=topic).
  493             filter_by(disabled=False).
  494             outerjoin((subq, models.Service.host == subq.c.host)).
  495             order_by(sort_value).
  496             all())
  497 
  498 
  499 @require_admin_context
  500 def service_get_all_share_sorted(context):
  501     session = get_session()
  502     with session.begin():
  503         topic = CONF.share_topic
  504         label = 'share_gigabytes'
  505         subq = (model_query(context, models.Share,
  506                             func.sum(models.Share.size).label(label),
  507                             session=session, read_deleted="no").
  508                 join(models.ShareInstance,
  509                      models.ShareInstance.share_id == models.Share.id).
  510                 group_by(models.ShareInstance.host).
  511                 subquery())
  512         return _service_get_all_topic_subquery(context,
  513                                                session,
  514                                                topic,
  515                                                subq,
  516                                                label)
  517 
  518 
  519 @require_admin_context
  520 def service_get_by_args(context, host, binary):
  521     result = (model_query(context, models.Service).
  522               filter_by(host=host).
  523               filter_by(binary=binary).
  524               first())
  525 
  526     if not result:
  527         raise exception.HostBinaryNotFound(host=host, binary=binary)
  528 
  529     return result
  530 
  531 
  532 @require_admin_context
  533 def service_create(context, values):
  534     session = get_session()
  535 
  536     _ensure_availability_zone_exists(context, values, session)
  537 
  538     service_ref = models.Service()
  539     service_ref.update(values)
  540     if not CONF.enable_new_services:
  541         service_ref.disabled = True
  542 
  543     with session.begin():
  544         service_ref.save(session)
  545         return service_ref
  546 
  547 
  548 @require_admin_context
  549 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
  550 def service_update(context, service_id, values):
  551     session = get_session()
  552 
  553     _ensure_availability_zone_exists(context, values, session, strict=False)
  554 
  555     with session.begin():
  556         service_ref = service_get(context, service_id, session=session)
  557         service_ref.update(values)
  558         service_ref.save(session=session)
  559 
  560 
  561 ###################
  562 
  563 
  564 @require_context
  565 def quota_get_all_by_project_and_user(context, project_id, user_id):
  566     authorize_project_context(context, project_id)
  567     user_quotas = model_query(
  568         context, models.ProjectUserQuota,
  569         models.ProjectUserQuota.resource,
  570         models.ProjectUserQuota.hard_limit,
  571     ).filter_by(
  572         project_id=project_id,
  573     ).filter_by(
  574         user_id=user_id,
  575     ).all()
  576 
  577     result = {'project_id': project_id, 'user_id': user_id}
  578     for u_quota in user_quotas:
  579         result[u_quota.resource] = u_quota.hard_limit
  580     return result
  581 
  582 
  583 @require_context
  584 def quota_get_all_by_project_and_share_type(context, project_id,
  585                                             share_type_id):
  586     authorize_project_context(context, project_id)
  587     share_type_quotas = model_query(
  588         context, models.ProjectShareTypeQuota,
  589         models.ProjectShareTypeQuota.resource,
  590         models.ProjectShareTypeQuota.hard_limit,
  591     ).filter_by(
  592         project_id=project_id,
  593     ).filter_by(
  594         share_type_id=share_type_id,
  595     ).all()
  596 
  597     result = {
  598         'project_id': project_id,
  599         'share_type_id': share_type_id,
  600     }
  601     for st_quota in share_type_quotas:
  602         result[st_quota.resource] = st_quota.hard_limit
  603     return result
  604 
  605 
  606 @require_context
  607 def quota_get_all_by_project(context, project_id):
  608     authorize_project_context(context, project_id)
  609     project_quotas = model_query(
  610         context, models.Quota, read_deleted="no",
  611     ).filter_by(
  612         project_id=project_id,
  613     ).all()
  614 
  615     result = {'project_id': project_id}
  616     for p_quota in project_quotas:
  617         result[p_quota.resource] = p_quota.hard_limit
  618     return result
  619 
  620 
  621 @require_context
  622 def quota_get_all(context, project_id):
  623     authorize_project_context(context, project_id)
  624 
  625     result = (model_query(context, models.ProjectUserQuota).
  626               filter_by(project_id=project_id).
  627               all())
  628 
  629     return result
  630 
  631 
  632 @require_admin_context
  633 def quota_create(context, project_id, resource, limit, user_id=None,
  634                  share_type_id=None):
  635     per_user = user_id and resource not in PER_PROJECT_QUOTAS
  636 
  637     if per_user:
  638         check = model_query(context, models.ProjectUserQuota).filter(
  639             models.ProjectUserQuota.project_id == project_id,
  640             models.ProjectUserQuota.user_id == user_id,
  641             models.ProjectUserQuota.resource == resource,
  642         ).all()
  643         quota_ref = models.ProjectUserQuota()
  644         quota_ref.user_id = user_id
  645     elif share_type_id:
  646         check = model_query(context, models.ProjectShareTypeQuota).filter(
  647             models.ProjectShareTypeQuota.project_id == project_id,
  648             models.ProjectShareTypeQuota.share_type_id == share_type_id,
  649             models.ProjectShareTypeQuota.resource == resource,
  650         ).all()
  651         quota_ref = models.ProjectShareTypeQuota()
  652         quota_ref.share_type_id = share_type_id
  653     else:
  654         check = model_query(context, models.Quota).filter(
  655             models.Quota.project_id == project_id,
  656             models.Quota.resource == resource,
  657         ).all()
  658         quota_ref = models.Quota()
  659     if check:
  660         raise exception.QuotaExists(project_id=project_id, resource=resource)
  661 
  662     quota_ref.project_id = project_id
  663     quota_ref.resource = resource
  664     quota_ref.hard_limit = limit
  665     session = get_session()
  666     with session.begin():
  667         quota_ref.save(session)
  668     return quota_ref
  669 
  670 
  671 @require_admin_context
  672 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
  673 def quota_update(context, project_id, resource, limit, user_id=None,
  674                  share_type_id=None):
  675     per_user = user_id and resource not in PER_PROJECT_QUOTAS
  676     if per_user:
  677         query = model_query(context, models.ProjectUserQuota).filter(
  678             models.ProjectUserQuota.project_id == project_id,
  679             models.ProjectUserQuota.user_id == user_id,
  680             models.ProjectUserQuota.resource == resource,
  681         )
  682     elif share_type_id:
  683         query = model_query(context, models.ProjectShareTypeQuota).filter(
  684             models.ProjectShareTypeQuota.project_id == project_id,
  685             models.ProjectShareTypeQuota.share_type_id == share_type_id,
  686             models.ProjectShareTypeQuota.resource == resource,
  687         )
  688     else:
  689         query = model_query(context, models.Quota).filter(
  690             models.Quota.project_id == project_id,
  691             models.Quota.resource == resource,
  692         )
  693 
  694     result = query.update({'hard_limit': limit})
  695     if not result:
  696         if per_user:
  697             raise exception.ProjectUserQuotaNotFound(
  698                 project_id=project_id, user_id=user_id)
  699         elif share_type_id:
  700             raise exception.ProjectShareTypeQuotaNotFound(
  701                 project_id=project_id, share_type=share_type_id)
  702         raise exception.ProjectQuotaNotFound(project_id=project_id)
  703 
  704 
  705 ###################
  706 
  707 
  708 @require_context
  709 def quota_class_get(context, class_name, resource, session=None):
  710     result = (model_query(context, models.QuotaClass, session=session,
  711                           read_deleted="no").
  712               filter_by(class_name=class_name).
  713               filter_by(resource=resource).
  714               first())
  715 
  716     if not result:
  717         raise exception.QuotaClassNotFound(class_name=class_name)
  718 
  719     return result
  720 
  721 
  722 @require_context
  723 def quota_class_get_default(context):
  724     rows = (model_query(context, models.QuotaClass, read_deleted="no").
  725             filter_by(class_name=_DEFAULT_QUOTA_NAME).
  726             all())
  727 
  728     result = {'class_name': _DEFAULT_QUOTA_NAME}
  729     for row in rows:
  730         result[row.resource] = row.hard_limit
  731 
  732     return result
  733 
  734 
  735 @require_context
  736 def quota_class_get_all_by_name(context, class_name):
  737     authorize_quota_class_context(context, class_name)
  738 
  739     rows = (model_query(context, models.QuotaClass, read_deleted="no").
  740             filter_by(class_name=class_name).
  741             all())
  742 
  743     result = {'class_name': class_name}
  744     for row in rows:
  745         result[row.resource] = row.hard_limit
  746 
  747     return result
  748 
  749 
  750 @require_admin_context
  751 def quota_class_create(context, class_name, resource, limit):
  752     quota_class_ref = models.QuotaClass()
  753     quota_class_ref.class_name = class_name
  754     quota_class_ref.resource = resource
  755     quota_class_ref.hard_limit = limit
  756     session = get_session()
  757     with session.begin():
  758         quota_class_ref.save(session)
  759     return quota_class_ref
  760 
  761 
  762 @require_admin_context
  763 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
  764 def quota_class_update(context, class_name, resource, limit):
  765     result = (model_query(context, models.QuotaClass, read_deleted="no").
  766               filter_by(class_name=class_name).
  767               filter_by(resource=resource).
  768               update({'hard_limit': limit}))
  769 
  770     if not result:
  771         raise exception.QuotaClassNotFound(class_name=class_name)
  772 
  773 
  774 ###################
  775 
  776 
  777 @require_context
  778 def quota_usage_get(context, project_id, resource, user_id=None,
  779                     share_type_id=None):
  780     query = (model_query(context, models.QuotaUsage, read_deleted="no").
  781              filter_by(project_id=project_id).
  782              filter_by(resource=resource))
  783     if user_id:
  784         if resource not in PER_PROJECT_QUOTAS:
  785             result = query.filter_by(user_id=user_id).first()
  786         else:
  787             result = query.filter_by(user_id=None).first()
  788     elif share_type_id:
  789         result = query.filter_by(queryshare_type_id=share_type_id).first()
  790     else:
  791         result = query.first()
  792 
  793     if not result:
  794         raise exception.QuotaUsageNotFound(project_id=project_id)
  795 
  796     return result
  797 
  798 
  799 def _quota_usage_get_all(context, project_id, user_id=None,
  800                          share_type_id=None):
  801     authorize_project_context(context, project_id)
  802     query = (model_query(context, models.QuotaUsage, read_deleted="no").
  803              filter_by(project_id=project_id))
  804     result = {'project_id': project_id}
  805     if user_id:
  806         query = query.filter(or_(models.QuotaUsage.user_id == user_id,
  807                                  models.QuotaUsage.user_id is None))
  808         result['user_id'] = user_id
  809     elif share_type_id:
  810         query = query.filter_by(share_type_id=share_type_id)
  811         result['share_type_id'] = share_type_id
  812     else:
  813         query = query.filter_by(share_type_id=None)
  814 
  815     rows = query.all()
  816     for row in rows:
  817         if row.resource in result:
  818             result[row.resource]['in_use'] += row.in_use
  819             result[row.resource]['reserved'] += row.reserved
  820         else:
  821             result[row.resource] = dict(in_use=row.in_use,
  822                                         reserved=row.reserved)
  823 
  824     return result
  825 
  826 
  827 @require_context
  828 def quota_usage_get_all_by_project(context, project_id):
  829     return _quota_usage_get_all(context, project_id)
  830 
  831 
  832 @require_context
  833 def quota_usage_get_all_by_project_and_user(context, project_id, user_id):
  834     return _quota_usage_get_all(context, project_id, user_id=user_id)
  835 
  836 
  837 @require_context
  838 def quota_usage_get_all_by_project_and_share_type(context, project_id,
  839                                                   share_type_id):
  840     return _quota_usage_get_all(
  841         context, project_id, share_type_id=share_type_id)
  842 
  843 
  844 def _quota_usage_create(context, project_id, user_id, resource, in_use,
  845                         reserved, until_refresh, share_type_id=None,
  846                         session=None):
  847     quota_usage_ref = models.QuotaUsage()
  848     if share_type_id:
  849         quota_usage_ref.share_type_id = share_type_id
  850     else:
  851         quota_usage_ref.user_id = user_id
  852     quota_usage_ref.project_id = project_id
  853     quota_usage_ref.resource = resource
  854     quota_usage_ref.in_use = in_use
  855     quota_usage_ref.reserved = reserved
  856     quota_usage_ref.until_refresh = until_refresh
  857     # updated_at is needed for judgement of max_age
  858     quota_usage_ref.updated_at = timeutils.utcnow()
  859 
  860     quota_usage_ref.save(session=session)
  861 
  862     return quota_usage_ref
  863 
  864 
  865 @require_admin_context
  866 def quota_usage_create(context, project_id, user_id, resource, in_use,
  867                        reserved, until_refresh, share_type_id=None):
  868     session = get_session()
  869     return _quota_usage_create(
  870         context, project_id, user_id, resource, in_use, reserved,
  871         until_refresh, share_type_id=share_type_id, session=session)
  872 
  873 
  874 @require_admin_context
  875 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
  876 def quota_usage_update(context, project_id, user_id, resource,
  877                        share_type_id=None, **kwargs):
  878     updates = {}
  879     for key in ('in_use', 'reserved', 'until_refresh'):
  880         if key in kwargs:
  881             updates[key] = kwargs[key]
  882 
  883     query = model_query(
  884         context, models.QuotaUsage, read_deleted="no",
  885     ).filter_by(project_id=project_id).filter_by(resource=resource)
  886     if share_type_id:
  887         query = query.filter_by(share_type_id=share_type_id)
  888     else:
  889         query = query.filter(or_(models.QuotaUsage.user_id == user_id,
  890                                  models.QuotaUsage.user_id is None))
  891     result = query.update(updates)
  892 
  893     if not result:
  894         raise exception.QuotaUsageNotFound(project_id=project_id)
  895 
  896 
  897 ###################
  898 
  899 
  900 def _reservation_create(context, uuid, usage, project_id, user_id, resource,
  901                         delta, expire, share_type_id=None, session=None):
  902     reservation_ref = models.Reservation()
  903     reservation_ref.uuid = uuid
  904     reservation_ref.usage_id = usage['id']
  905     reservation_ref.project_id = project_id
  906     if share_type_id:
  907         reservation_ref.share_type_id = share_type_id
  908     else:
  909         reservation_ref.user_id = user_id
  910     reservation_ref.resource = resource
  911     reservation_ref.delta = delta
  912     reservation_ref.expire = expire
  913     reservation_ref.save(session=session)
  914     return reservation_ref
  915 
  916 
  917 ###################
  918 
  919 
  920 # NOTE(johannes): The quota code uses SQL locking to ensure races don't
  921 # cause under or over counting of resources. To avoid deadlocks, this
  922 # code always acquires the lock on quota_usages before acquiring the lock
  923 # on reservations.
  924 
  925 def _get_share_type_quota_usages(context, session, project_id, share_type_id):
  926     rows = model_query(
  927         context, models.QuotaUsage, read_deleted="no", session=session,
  928     ).filter(
  929         models.QuotaUsage.project_id == project_id,
  930         models.QuotaUsage.share_type_id == share_type_id,
  931     ).with_lockmode('update').all()
  932     return {row.resource: row for row in rows}
  933 
  934 
  935 def _get_user_quota_usages(context, session, project_id, user_id):
  936     # Broken out for testability
  937     rows = (model_query(context, models.QuotaUsage,
  938                         read_deleted="no",
  939                         session=session).
  940             filter_by(project_id=project_id).
  941             filter(or_(models.QuotaUsage.user_id == user_id,
  942                        models.QuotaUsage.user_id is None)).
  943             with_lockmode('update').
  944             all())
  945     return {row.resource: row for row in rows}
  946 
  947 
  948 def _get_project_quota_usages(context, session, project_id):
  949     rows = (model_query(context, models.QuotaUsage,
  950                         read_deleted="no",
  951                         session=session).
  952             filter_by(project_id=project_id).
  953             filter(models.QuotaUsage.share_type_id is None).
  954             with_lockmode('update').
  955             all())
  956     result = dict()
  957     # Get the total count of in_use,reserved
  958     for row in rows:
  959         if row.resource in result:
  960             result[row.resource]['in_use'] += row.in_use
  961             result[row.resource]['reserved'] += row.reserved
  962             result[row.resource]['total'] += (row.in_use + row.reserved)
  963         else:
  964             result[row.resource] = dict(in_use=row.in_use,
  965                                         reserved=row.reserved,
  966                                         total=row.in_use + row.reserved)
  967     return result
  968 
  969 
  970 @require_context
  971 def quota_reserve(context, resources, project_quotas, user_quotas,
  972                   share_type_quotas, deltas, expire, until_refresh,
  973                   max_age, project_id=None, user_id=None, share_type_id=None,
  974                   overquota_allowed=False):
  975     user_reservations = _quota_reserve(
  976         context, resources, project_quotas, user_quotas,
  977         deltas, expire, until_refresh, max_age, project_id, user_id=user_id,
  978         overquota_allowed=overquota_allowed)
  979     if share_type_id:
  980         try:
  981             st_reservations = _quota_reserve(
  982                 context, resources, project_quotas, share_type_quotas,
  983                 deltas, expire, until_refresh, max_age, project_id,
  984                 share_type_id=share_type_id,
  985                 overquota_allowed=overquota_allowed)
  986         except exception.OverQuota:
  987             with excutils.save_and_reraise_exception():
  988                 # rollback previous reservations
  989                 reservation_rollback(
  990                     context, user_reservations,
  991                     project_id=project_id, user_id=user_id)
  992         return user_reservations + st_reservations
  993     return user_reservations
  994 
  995 
  996 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
  997 def _quota_reserve(context, resources, project_quotas, user_or_st_quotas,
  998                    deltas, expire, until_refresh,
  999                    max_age, project_id=None, user_id=None, share_type_id=None,
 1000                    overquota_allowed=False):
 1001     elevated = context.elevated()
 1002     session = get_session()
 1003     with session.begin():
 1004 
 1005         if project_id is None:
 1006             project_id = context.project_id
 1007         if share_type_id:
 1008             user_or_st_usages = _get_share_type_quota_usages(
 1009                 context, session, project_id, share_type_id)
 1010         else:
 1011             user_id = user_id if user_id else context.user_id
 1012             user_or_st_usages = _get_user_quota_usages(
 1013                 context, session, project_id, user_id)
 1014 
 1015         # Get the current usages
 1016         project_usages = _get_project_quota_usages(
 1017             context, session, project_id)
 1018 
 1019         # Handle usage refresh
 1020         work = set(deltas.keys())
 1021         while work:
 1022             resource = work.pop()
 1023 
 1024             # Do we need to refresh the usage?
 1025             refresh = False
 1026             if ((resource not in PER_PROJECT_QUOTAS) and
 1027                     (resource not in user_or_st_usages)):
 1028                 user_or_st_usages[resource] = _quota_usage_create(
 1029                     elevated,
 1030                     project_id,
 1031                     user_id,
 1032                     resource,
 1033                     0, 0,
 1034                     until_refresh or None,
 1035                     share_type_id=share_type_id,
 1036                     session=session)
 1037                 refresh = True
 1038             elif ((resource in PER_PROJECT_QUOTAS) and
 1039                     (resource not in user_or_st_usages)):
 1040                 user_or_st_usages[resource] = _quota_usage_create(
 1041                     elevated,
 1042                     project_id,
 1043                     None,
 1044                     resource,
 1045                     0, 0,
 1046                     until_refresh or None,
 1047                     share_type_id=share_type_id,
 1048                     session=session)
 1049                 refresh = True
 1050             elif user_or_st_usages[resource].in_use < 0:
 1051                 # Negative in_use count indicates a desync, so try to
 1052                 # heal from that...
 1053                 refresh = True
 1054             elif user_or_st_usages[resource].until_refresh is not None:
 1055                 user_or_st_usages[resource].until_refresh -= 1
 1056                 if user_or_st_usages[resource].until_refresh <= 0:
 1057                     refresh = True
 1058             elif max_age and (user_or_st_usages[resource].updated_at -
 1059                               timeutils.utcnow()).seconds >= max_age:
 1060                 refresh = True
 1061 
 1062             # OK, refresh the usage
 1063             if refresh:
 1064                 # Grab the sync routine
 1065                 sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync]
 1066 
 1067                 updates = sync(
 1068                     elevated, project_id, user_id,
 1069                     share_type_id=share_type_id, session=session)
 1070                 for res, in_use in updates.items():
 1071                     # Make sure we have a destination for the usage!
 1072                     if ((res not in PER_PROJECT_QUOTAS) and
 1073                             (res not in user_or_st_usages)):
 1074                         user_or_st_usages[res] = _quota_usage_create(
 1075                             elevated,
 1076                             project_id,
 1077                             user_id,
 1078                             res,
 1079                             0, 0,
 1080                             until_refresh or None,
 1081                             share_type_id=share_type_id,
 1082                             session=session)
 1083                     if ((res in PER_PROJECT_QUOTAS) and
 1084                             (res not in user_or_st_usages)):
 1085                         user_or_st_usages[res] = _quota_usage_create(
 1086                             elevated,
 1087                             project_id,
 1088                             None,
 1089                             res,
 1090                             0, 0,
 1091                             until_refresh or None,
 1092                             share_type_id=share_type_id,
 1093                             session=session)
 1094 
 1095                     if user_or_st_usages[res].in_use != in_use:
 1096                         LOG.debug(
 1097                             'quota_usages out of sync, updating. '
 1098                             'project_id: %(project_id)s, '
 1099                             'user_id: %(user_id)s, '
 1100                             'share_type_id: %(share_type_id)s, '
 1101                             'resource: %(res)s, '
 1102                             'tracked usage: %(tracked_use)s, '
 1103                             'actual usage: %(in_use)s',
 1104                             {'project_id': project_id,
 1105                              'user_id': user_id,
 1106                              'share_type_id': share_type_id,
 1107                              'res': res,
 1108                              'tracked_use': user_or_st_usages[res].in_use,
 1109                              'in_use': in_use})
 1110 
 1111                     # Update the usage
 1112                     user_or_st_usages[res].in_use = in_use
 1113                     user_or_st_usages[res].until_refresh = (
 1114                         until_refresh or None)
 1115 
 1116                     # Because more than one resource may be refreshed
 1117                     # by the call to the sync routine, and we don't
 1118                     # want to double-sync, we make sure all refreshed
 1119                     # resources are dropped from the work set.
 1120                     work.discard(res)
 1121 
 1122                     # NOTE(Vek): We make the assumption that the sync
 1123                     #            routine actually refreshes the
 1124                     #            resources that it is the sync routine
 1125                     #            for.  We don't check, because this is
 1126                     #            a best-effort mechanism.
 1127 
 1128         # Check for deltas that would go negative
 1129         unders = [res for res, delta in deltas.items()
 1130                   if delta < 0 and
 1131                   delta + user_or_st_usages[res].in_use < 0]
 1132 
 1133         # Now, let's check the quotas
 1134         # NOTE(Vek): We're only concerned about positive increments.
 1135         #            If a project has gone over quota, we want them to
 1136         #            be able to reduce their usage without any
 1137         #            problems.
 1138         for key, value in user_or_st_usages.items():
 1139             if key not in project_usages:
 1140                 project_usages[key] = value
 1141         overs = [res for res, delta in deltas.items()
 1142                  if user_or_st_quotas[res] >= 0 and delta >= 0 and
 1143                  (0 <= project_quotas[res] < delta +
 1144                   project_usages[res]['total'] or
 1145                   user_or_st_quotas[res] < delta +
 1146                   user_or_st_usages[res].total)]
 1147 
 1148         # NOTE(carloss): If OverQuota is allowed, there is no problem to exceed
 1149         # the quotas, so we reset the overs list and LOG it.
 1150         if overs and overquota_allowed:
 1151             msg = _("The service has identified one or more exceeded "
 1152                     "quotas. Please check the quotas for project "
 1153                     "%(project_id)s, user %(user_id)s and share type "
 1154                     "%(share_type_id)s, and adjust them if "
 1155                     "necessary.") % {
 1156                 "project_id": project_id,
 1157                 "user_id": user_id,
 1158                 "share_type_id": share_type_id
 1159             }
 1160             LOG.warning(msg)
 1161             overs = []
 1162 
 1163         # NOTE(Vek): The quota check needs to be in the transaction,
 1164         #            but the transaction doesn't fail just because
 1165         #            we're over quota, so the OverQuota raise is
 1166         #            outside the transaction.  If we did the raise
 1167         #            here, our usage updates would be discarded, but
 1168         #            they're not invalidated by being over-quota.
 1169 
 1170         # Create the reservations
 1171         if not overs:
 1172             reservations = []
 1173             for res, delta in deltas.items():
 1174                 reservation = _reservation_create(elevated,
 1175                                                   uuidutils.generate_uuid(),
 1176                                                   user_or_st_usages[res],
 1177                                                   project_id,
 1178                                                   user_id,
 1179                                                   res, delta, expire,
 1180                                                   share_type_id=share_type_id,
 1181                                                   session=session)
 1182                 reservations.append(reservation.uuid)
 1183 
 1184                 # Also update the reserved quantity
 1185                 # NOTE(Vek): Again, we are only concerned here about
 1186                 #            positive increments.  Here, though, we're
 1187                 #            worried about the following scenario:
 1188                 #
 1189                 #            1) User initiates resize down.
 1190                 #            2) User allocates a new instance.
 1191                 #            3) Resize down fails or is reverted.
 1192                 #            4) User is now over quota.
 1193                 #
 1194                 #            To prevent this, we only update the
 1195                 #            reserved value if the delta is positive.
 1196                 if delta > 0:
 1197                     user_or_st_usages[res].reserved += delta
 1198 
 1199         # Apply updates to the usages table
 1200         for usage_ref in user_or_st_usages.values():
 1201             session.add(usage_ref)
 1202 
 1203     if unders:
 1204         LOG.warning("Change will make usage less than 0 for the following "
 1205                     "resources: %s", unders)
 1206     if overs:
 1207         if project_quotas == user_or_st_quotas:
 1208             usages = project_usages
 1209         else:
 1210             usages = user_or_st_usages
 1211         usages = {k: dict(in_use=v['in_use'], reserved=v['reserved'])
 1212                   for k, v in usages.items()}
 1213         raise exception.OverQuota(
 1214             overs=sorted(overs), quotas=user_or_st_quotas, usages=usages)
 1215 
 1216     return reservations
 1217 
 1218 
 1219 def _quota_reservations_query(session, context, reservations):
 1220     """Return the relevant reservations."""
 1221 
 1222     # Get the listed reservations
 1223     return (model_query(context, models.Reservation,
 1224                         read_deleted="no",
 1225                         session=session).
 1226             filter(models.Reservation.uuid.in_(reservations)).
 1227             with_lockmode('update'))
 1228 
 1229 
 1230 @require_context
 1231 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 1232 def reservation_commit(context, reservations, project_id=None, user_id=None,
 1233                        share_type_id=None):
 1234     session = get_session()
 1235     with session.begin():
 1236         if share_type_id:
 1237             st_usages = _get_share_type_quota_usages(
 1238                 context, session, project_id, share_type_id)
 1239         else:
 1240             st_usages = {}
 1241         user_usages = _get_user_quota_usages(
 1242             context, session, project_id, user_id)
 1243 
 1244         reservation_query = _quota_reservations_query(
 1245             session, context, reservations)
 1246         for reservation in reservation_query.all():
 1247             if reservation['share_type_id']:
 1248                 usages = st_usages
 1249             else:
 1250                 usages = user_usages
 1251             usage = usages[reservation.resource]
 1252             if reservation.delta >= 0:
 1253                 usage.reserved -= reservation.delta
 1254             usage.in_use += reservation.delta
 1255         reservation_query.soft_delete(synchronize_session=False)
 1256 
 1257 
 1258 @require_context
 1259 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 1260 def reservation_rollback(context, reservations, project_id=None, user_id=None,
 1261                          share_type_id=None):
 1262     session = get_session()
 1263     with session.begin():
 1264         if share_type_id:
 1265             st_usages = _get_share_type_quota_usages(
 1266                 context, session, project_id, share_type_id)
 1267         else:
 1268             st_usages = {}
 1269         user_usages = _get_user_quota_usages(
 1270             context, session, project_id, user_id)
 1271 
 1272         reservation_query = _quota_reservations_query(
 1273             session, context, reservations)
 1274         for reservation in reservation_query.all():
 1275             if reservation['share_type_id']:
 1276                 usages = st_usages
 1277             else:
 1278                 usages = user_usages
 1279             usage = usages[reservation.resource]
 1280             if reservation.delta >= 0:
 1281                 usage.reserved -= reservation.delta
 1282         reservation_query.soft_delete(synchronize_session=False)
 1283 
 1284 
 1285 @require_admin_context
 1286 def quota_destroy_all_by_project_and_user(context, project_id, user_id):
 1287     session = get_session()
 1288     with session.begin():
 1289         (model_query(context, models.ProjectUserQuota, session=session,
 1290                      read_deleted="no").
 1291          filter_by(project_id=project_id).
 1292          filter_by(user_id=user_id).soft_delete(synchronize_session=False))
 1293 
 1294         (model_query(context, models.QuotaUsage,
 1295                      session=session, read_deleted="no").
 1296          filter_by(project_id=project_id).
 1297          filter_by(user_id=user_id).soft_delete(synchronize_session=False))
 1298 
 1299         (model_query(context, models.Reservation,
 1300                      session=session, read_deleted="no").
 1301          filter_by(project_id=project_id).
 1302          filter_by(user_id=user_id).soft_delete(synchronize_session=False))
 1303 
 1304 
 1305 @require_admin_context
 1306 def quota_destroy_all_by_share_type(context, share_type_id, project_id=None):
 1307     """Soft deletes all quotas, usages and reservations.
 1308 
 1309     :param context: request context for queries, updates and logging
 1310     :param share_type_id: ID of the share type to filter the quotas, usages
 1311         and reservations under.
 1312     :param project_id: ID of the project to filter the quotas, usages and
 1313         reservations under. If not provided, share type quotas for all
 1314         projects will be acted upon.
 1315     """
 1316     session = get_session()
 1317     with session.begin():
 1318         share_type_quotas = model_query(
 1319             context, models.ProjectShareTypeQuota, session=session,
 1320             read_deleted="no",
 1321         ).filter_by(share_type_id=share_type_id)
 1322 
 1323         share_type_quota_usages = model_query(
 1324             context, models.QuotaUsage, session=session, read_deleted="no",
 1325         ).filter_by(share_type_id=share_type_id)
 1326 
 1327         share_type_quota_reservations = model_query(
 1328             context, models.Reservation, session=session, read_deleted="no",
 1329         ).filter_by(share_type_id=share_type_id)
 1330 
 1331         if project_id is not None:
 1332             share_type_quotas = share_type_quotas.filter_by(
 1333                 project_id=project_id)
 1334             share_type_quota_usages = share_type_quota_usages.filter_by(
 1335                 project_id=project_id)
 1336             share_type_quota_reservations = (
 1337                 share_type_quota_reservations.filter_by(project_id=project_id))
 1338 
 1339         share_type_quotas.soft_delete(synchronize_session=False)
 1340         share_type_quota_usages.soft_delete(synchronize_session=False)
 1341         share_type_quota_reservations.soft_delete(synchronize_session=False)
 1342 
 1343 
 1344 @require_admin_context
 1345 def quota_destroy_all_by_project(context, project_id):
 1346     session = get_session()
 1347     with session.begin():
 1348         (model_query(context, models.Quota, session=session,
 1349                      read_deleted="no").
 1350          filter_by(project_id=project_id).
 1351          soft_delete(synchronize_session=False))
 1352 
 1353         (model_query(context, models.ProjectUserQuota, session=session,
 1354                      read_deleted="no").
 1355          filter_by(project_id=project_id).
 1356          soft_delete(synchronize_session=False))
 1357 
 1358         (model_query(context, models.QuotaUsage,
 1359                      session=session, read_deleted="no").
 1360          filter_by(project_id=project_id).
 1361          soft_delete(synchronize_session=False))
 1362 
 1363         (model_query(context, models.Reservation,
 1364                      session=session, read_deleted="no").
 1365          filter_by(project_id=project_id).
 1366          soft_delete(synchronize_session=False))
 1367 
 1368 
 1369 @require_admin_context
 1370 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 1371 def reservation_expire(context):
 1372     session = get_session()
 1373     with session.begin():
 1374         current_time = timeutils.utcnow()
 1375         reservation_query = (model_query(
 1376             context, models.Reservation,
 1377             session=session, read_deleted="no").
 1378             filter(models.Reservation.expire < current_time))
 1379 
 1380         for reservation in reservation_query.all():
 1381             if reservation.delta >= 0:
 1382                 quota_usage = model_query(context, models.QuotaUsage,
 1383                                           session=session,
 1384                                           read_deleted="no").filter(
 1385                     models.QuotaUsage.id == reservation.usage_id).first()
 1386                 quota_usage.reserved -= reservation.delta
 1387                 session.add(quota_usage)
 1388 
 1389         reservation_query.soft_delete(synchronize_session=False)
 1390 
 1391 
 1392 ################
 1393 
 1394 def _extract_subdict_by_fields(source_dict, fields):
 1395     dict_to_extract_from = copy.deepcopy(source_dict)
 1396     sub_dict = {}
 1397     for field in fields:
 1398         field_value = dict_to_extract_from.pop(field, None)
 1399         if field_value:
 1400             sub_dict.update({field: field_value})
 1401 
 1402     return sub_dict, dict_to_extract_from
 1403 
 1404 
 1405 def _extract_share_instance_values(values):
 1406     share_instance_model_fields = [
 1407         'status', 'host', 'scheduled_at', 'launched_at', 'terminated_at',
 1408         'share_server_id', 'share_network_id', 'availability_zone',
 1409         'replica_state', 'share_type_id', 'share_type', 'access_rules_status',
 1410     ]
 1411     share_instance_values, share_values = (
 1412         _extract_subdict_by_fields(values, share_instance_model_fields)
 1413     )
 1414     return share_instance_values, share_values
 1415 
 1416 
 1417 def _change_size_to_instance_size(snap_instance_values):
 1418     if 'size' in snap_instance_values:
 1419         snap_instance_values['instance_size'] = snap_instance_values['size']
 1420         snap_instance_values.pop('size')
 1421 
 1422 
 1423 def _extract_snapshot_instance_values(values):
 1424     fields = ['status', 'progress', 'provider_location']
 1425     snapshot_instance_values, snapshot_values = (
 1426         _extract_subdict_by_fields(values, fields)
 1427     )
 1428     return snapshot_instance_values, snapshot_values
 1429 
 1430 
 1431 ################
 1432 
 1433 
 1434 @require_context
 1435 def share_instance_create(context, share_id, values):
 1436     session = get_session()
 1437     with session.begin():
 1438         return _share_instance_create(context, share_id, values, session)
 1439 
 1440 
 1441 def _share_instance_create(context, share_id, values, session):
 1442     if not values.get('id'):
 1443         values['id'] = uuidutils.generate_uuid()
 1444     values.update({'share_id': share_id})
 1445 
 1446     share_instance_ref = models.ShareInstance()
 1447     share_instance_ref.update(values)
 1448     share_instance_ref.save(session=session)
 1449 
 1450     return share_instance_get(context, share_instance_ref['id'],
 1451                               session=session)
 1452 
 1453 
 1454 @require_context
 1455 def share_instance_update(context, share_instance_id, values,
 1456                           with_share_data=False):
 1457     session = get_session()
 1458     _ensure_availability_zone_exists(context, values, session, strict=False)
 1459     with session.begin():
 1460         instance_ref = _share_instance_update(
 1461             context, share_instance_id, values, session
 1462         )
 1463         if with_share_data:
 1464             parent_share = share_get(context, instance_ref['share_id'],
 1465                                      session=session)
 1466             instance_ref.set_share_data(parent_share)
 1467         return instance_ref
 1468 
 1469 
 1470 def share_and_snapshot_instances_status_update(
 1471         context, values, share_instance_ids=None, snapshot_instance_ids=None,
 1472         current_expected_status=None):
 1473     updated_share_instances = None
 1474     updated_snapshot_instances = None
 1475     session = get_session()
 1476     with session.begin():
 1477         if current_expected_status and share_instance_ids:
 1478             filters = {'instance_ids': share_instance_ids}
 1479             share_instances = share_instances_get_all(
 1480                 context, filters=filters, session=session)
 1481             all_instances_are_compliant = all(
 1482                 instance['status'] == current_expected_status
 1483                 for instance in share_instances)
 1484 
 1485             if not all_instances_are_compliant:
 1486                 msg = _('At least one of the shares is not in the %(status)s '
 1487                         'status.') % {
 1488                     'status': current_expected_status
 1489                 }
 1490                 raise exception.InvalidShareInstance(reason=msg)
 1491 
 1492         if current_expected_status and snapshot_instance_ids:
 1493             filters = {'instance_ids': snapshot_instance_ids}
 1494             snapshot_instances = share_snapshot_instance_get_all_with_filters(
 1495                 context, filters, session=session)
 1496             all_snap_instances_are_compliant = all(
 1497                 snap_instance['status'] == current_expected_status
 1498                 for snap_instance in snapshot_instances)
 1499             if not all_snap_instances_are_compliant:
 1500                 msg = _('At least one of the snapshots is not in the '
 1501                         '%(status)s status.') % {
 1502                     'status': current_expected_status
 1503                 }
 1504                 raise exception.InvalidShareSnapshotInstance(reason=msg)
 1505 
 1506         if share_instance_ids:
 1507             updated_share_instances = share_instances_status_update(
 1508                 context, share_instance_ids, values, session=session)
 1509 
 1510         if snapshot_instance_ids:
 1511             updated_snapshot_instances = (
 1512                 share_snapshot_instances_status_update(
 1513                     context, snapshot_instance_ids, values, session=session))
 1514 
 1515     return updated_share_instances, updated_snapshot_instances
 1516 
 1517 
 1518 @require_context
 1519 def share_instances_status_update(
 1520         context, share_instance_ids, values, session=None):
 1521     session = session or get_session()
 1522 
 1523     result = (
 1524         model_query(
 1525             context, models.ShareInstance, read_deleted="no",
 1526             session=session).filter(
 1527             models.ShareInstance.id.in_(share_instance_ids)).update(
 1528             values, synchronize_session=False))
 1529     return result
 1530 
 1531 
 1532 def _share_instance_update(context, share_instance_id, values, session):
 1533     share_instance_ref = share_instance_get(context, share_instance_id,
 1534                                             session=session)
 1535     share_instance_ref.update(values)
 1536     share_instance_ref.save(session=session)
 1537     return share_instance_ref
 1538 
 1539 
 1540 @require_context
 1541 def share_instance_get(context, share_instance_id, session=None,
 1542                        with_share_data=False):
 1543     if session is None:
 1544         session = get_session()
 1545     result = model_query(
 1546         context, models.ShareInstance, session=session,
 1547     ).filter_by(
 1548         id=share_instance_id,
 1549     ).options(
 1550         joinedload('export_locations'),
 1551         joinedload('share_type'),
 1552     ).first()
 1553     if result is None:
 1554         raise exception.NotFound()
 1555 
 1556     if with_share_data:
 1557         parent_share = share_get(context, result['share_id'], session=session)
 1558         result.set_share_data(parent_share)
 1559 
 1560     return result
 1561 
 1562 
 1563 @require_admin_context
 1564 def share_instances_get_all(context, filters=None, session=None):
 1565     session = session or get_session()
 1566     query = model_query(
 1567         context, models.ShareInstance, session=session, read_deleted="no",
 1568     ).options(
 1569         joinedload('export_locations'),
 1570     )
 1571 
 1572     filters = filters or {}
 1573 
 1574     export_location_id = filters.get('export_location_id')
 1575     export_location_path = filters.get('export_location_path')
 1576     if export_location_id or export_location_path:
 1577         query = query.join(
 1578             models.ShareInstanceExportLocations,
 1579             models.ShareInstanceExportLocations.share_instance_id ==
 1580             models.ShareInstance.id)
 1581         if export_location_path:
 1582             query = query.filter(
 1583                 models.ShareInstanceExportLocations.path ==
 1584                 export_location_path)
 1585         if export_location_id:
 1586             query = query.filter(
 1587                 models.ShareInstanceExportLocations.uuid ==
 1588                 export_location_id)
 1589 
 1590     instance_ids = filters.get('instance_ids')
 1591     if instance_ids:
 1592         query = query.filter(models.ShareInstance.id.in_(instance_ids))
 1593 
 1594     # TODO(gouthamr): This DB API method needs to be generalized for all
 1595     # share instance fields.
 1596     host = filters.get('host')
 1597     if host:
 1598         query = query.filter(
 1599             or_(models.ShareInstance.host == host,
 1600                 models.ShareInstance.host.like("{0}#%".format(host)))
 1601         )
 1602     share_server_id = filters.get('share_server_id')
 1603     if share_server_id:
 1604         query = query.filter(
 1605             models.ShareInstance.share_server_id == share_server_id)
 1606 
 1607     # Returns list of share instances that satisfy filters.
 1608     query = query.all()
 1609     return query
 1610 
 1611 
 1612 @require_context
 1613 def _update_share_instance_usages(context, share, instance_ref,
 1614                                   is_replica=False):
 1615     deltas = {}
 1616     no_instances_remain = len(share.instances) == 0
 1617     share_usages_to_release = {"shares": -1, "gigabytes": -share['size']}
 1618     replica_usages_to_release = {"share_replicas": -1,
 1619                                  "replica_gigabytes": -share['size']}
 1620 
 1621     if is_replica and no_instances_remain:
 1622         # A share that had a replication_type is being deleted, so there's
 1623         # need to update the share replica quotas and the share quotas
 1624         deltas.update(replica_usages_to_release)
 1625         deltas.update(share_usages_to_release)
 1626     elif is_replica:
 1627         # The user is deleting a share replica
 1628         deltas.update(replica_usages_to_release)
 1629     else:
 1630         # A share with no replication_type is being deleted
 1631         deltas.update(share_usages_to_release)
 1632 
 1633     reservations = None
 1634     try:
 1635         # we give the user_id of the share, to update
 1636         # the quota usage for the user, who created the share
 1637         reservations = QUOTAS.reserve(
 1638             context,
 1639             project_id=share['project_id'],
 1640             user_id=share['user_id'],
 1641             share_type_id=instance_ref['share_type_id'],
 1642             **deltas)
 1643         QUOTAS.commit(
 1644             context, reservations, project_id=share['project_id'],
 1645             user_id=share['user_id'],
 1646             share_type_id=instance_ref['share_type_id'])
 1647     except Exception:
 1648         resource_name = (
 1649             'share replica' if is_replica else 'share')
 1650         resource_id = instance_ref['id'] if is_replica else share['id']
 1651         msg = (_("Failed to update usages deleting %(resource_name)s "
 1652                  "'%(id)s'.") % {'id': resource_id,
 1653                                  "resource_name": resource_name})
 1654         LOG.exception(msg)
 1655         if reservations:
 1656             QUOTAS.rollback(
 1657                 context, reservations,
 1658                 share_type_id=instance_ref['share_type_id'])
 1659 
 1660 
 1661 @require_context
 1662 def share_instance_delete(context, instance_id, session=None,
 1663                           need_to_update_usages=False):
 1664     if session is None:
 1665         session = get_session()
 1666 
 1667     with session.begin():
 1668         share_export_locations_update(context, instance_id, [], delete=True)
 1669         instance_ref = share_instance_get(context, instance_id,
 1670                                           session=session)
 1671         is_replica = instance_ref['replica_state'] is not None
 1672         instance_ref.soft_delete(session=session, update_status=True)
 1673         share = share_get(context, instance_ref['share_id'], session=session)
 1674         if len(share.instances) == 0:
 1675             share_access_delete_all_by_share(context, share['id'])
 1676             session.query(models.ShareMetadata).filter_by(
 1677                 share_id=share['id']).soft_delete()
 1678             share.soft_delete(session=session)
 1679 
 1680         if need_to_update_usages:
 1681             _update_share_instance_usages(context, share, instance_ref,
 1682                                           is_replica=is_replica)
 1683 
 1684 
 1685 def _set_instances_share_data(context, instances, session):
 1686     if instances and not isinstance(instances, list):
 1687         instances = [instances]
 1688 
 1689     instances_with_share_data = []
 1690     for instance in instances:
 1691         try:
 1692             parent_share = share_get(context, instance['share_id'],
 1693                                      session=session)
 1694         except exception.NotFound:
 1695             continue
 1696         instance.set_share_data(parent_share)
 1697         instances_with_share_data.append(instance)
 1698     return instances_with_share_data
 1699 
 1700 
 1701 @require_admin_context
 1702 def share_instances_get_all_by_host(context, host, with_share_data=False,
 1703                                     status=None, session=None):
 1704     """Retrieves all share instances hosted on a host."""
 1705     session = session or get_session()
 1706     instances = (
 1707         model_query(context, models.ShareInstance).filter(
 1708             or_(
 1709                 models.ShareInstance.host == host,
 1710                 models.ShareInstance.host.like("{0}#%".format(host))
 1711             )
 1712         )
 1713     )
 1714     if status is not None:
 1715         instances = instances.filter(models.ShareInstance.status == status)
 1716     # Returns list of all instances that satisfy filters.
 1717     instances = instances.all()
 1718 
 1719     if with_share_data:
 1720         instances = _set_instances_share_data(context, instances, session)
 1721     return instances
 1722 
 1723 
 1724 @require_context
 1725 def share_instances_get_all_by_share_network(context, share_network_id):
 1726     """Returns list of share instances that belong to given share network."""
 1727     result = (
 1728         model_query(context, models.ShareInstance).filter(
 1729             models.ShareInstance.share_network_id == share_network_id,
 1730         ).all()
 1731     )
 1732     return result
 1733 
 1734 
 1735 @require_context
 1736 def share_instances_get_all_by_share_server(context, share_server_id,
 1737                                             with_share_data=False):
 1738     """Returns list of share instance with given share server."""
 1739     session = get_session()
 1740     result = (
 1741         model_query(context, models.ShareInstance).filter(
 1742             models.ShareInstance.share_server_id == share_server_id,
 1743         ).all()
 1744     )
 1745 
 1746     if with_share_data:
 1747         result = _set_instances_share_data(context, result, session)
 1748 
 1749     return result
 1750 
 1751 
 1752 @require_context
 1753 def share_instances_get_all_by_share(context, share_id):
 1754     """Returns list of share instances that belong to given share."""
 1755     result = (
 1756         model_query(context, models.ShareInstance).filter(
 1757             models.ShareInstance.share_id == share_id,
 1758         ).all()
 1759     )
 1760     return result
 1761 
 1762 
 1763 @require_context
 1764 def share_instances_get_all_by_share_group_id(context, share_group_id):
 1765     """Returns list of share instances that belong to given share group."""
 1766     result = (
 1767         model_query(context, models.Share).filter(
 1768             models.Share.share_group_id == share_group_id,
 1769         ).all()
 1770     )
 1771     instances = []
 1772     for share in result:
 1773         instance = share.instance
 1774         instance.set_share_data(share)
 1775         instances.append(instance)
 1776 
 1777     return instances
 1778 
 1779 
 1780 ################
 1781 
 1782 def _share_replica_get_with_filters(context, share_id=None, replica_id=None,
 1783                                     replica_state=None, status=None,
 1784                                     with_share_server=True, session=None):
 1785 
 1786     query = model_query(context, models.ShareInstance, session=session,
 1787                         read_deleted="no")
 1788 
 1789     if share_id is not None:
 1790         query = query.filter(models.ShareInstance.share_id == share_id)
 1791 
 1792     if replica_id is not None:
 1793         query = query.filter(models.ShareInstance.id == replica_id)
 1794 
 1795     if replica_state is not None:
 1796         query = query.filter(
 1797             models.ShareInstance.replica_state == replica_state)
 1798     else:
 1799         query = query.filter(models.ShareInstance.replica_state.isnot(None))
 1800 
 1801     if status is not None:
 1802         query = query.filter(models.ShareInstance.status == status)
 1803 
 1804     if with_share_server:
 1805         query = query.options(joinedload('share_server'))
 1806 
 1807     return query
 1808 
 1809 
 1810 def _set_replica_share_data(context, replicas, session):
 1811     if replicas and not isinstance(replicas, list):
 1812         replicas = [replicas]
 1813 
 1814     for replica in replicas:
 1815         parent_share = share_get(context, replica['share_id'], session=session)
 1816         replica.set_share_data(parent_share)
 1817 
 1818     return replicas
 1819 
 1820 
 1821 @require_context
 1822 def share_replicas_get_all(context, with_share_data=False,
 1823                            with_share_server=True, session=None):
 1824     """Returns replica instances for all available replicated shares."""
 1825     session = session or get_session()
 1826 
 1827     result = _share_replica_get_with_filters(
 1828         context, with_share_server=with_share_server, session=session).all()
 1829 
 1830     if with_share_data:
 1831         result = _set_replica_share_data(context, result, session)
 1832 
 1833     return result
 1834 
 1835 
 1836 @require_context
 1837 def share_replicas_get_all_by_share(context, share_id,
 1838                                     with_share_data=False,
 1839                                     with_share_server=False, session=None):
 1840     """Returns replica instances for a given share."""
 1841     session = session or get_session()
 1842 
 1843     result = _share_replica_get_with_filters(
 1844         context, with_share_server=with_share_server,
 1845         share_id=share_id, session=session).all()
 1846 
 1847     if with_share_data:
 1848         result = _set_replica_share_data(context, result, session)
 1849 
 1850     return result
 1851 
 1852 
 1853 @require_context
 1854 def share_replicas_get_available_active_replica(context, share_id,
 1855                                                 with_share_data=False,
 1856                                                 with_share_server=False,
 1857                                                 session=None):
 1858     """Returns an 'active' replica instance that is 'available'."""
 1859     session = session or get_session()
 1860 
 1861     result = _share_replica_get_with_filters(
 1862         context, with_share_server=with_share_server, share_id=share_id,
 1863         replica_state=constants.REPLICA_STATE_ACTIVE,
 1864         status=constants.STATUS_AVAILABLE, session=session).first()
 1865 
 1866     if result and with_share_data:
 1867         result = _set_replica_share_data(context, result, session)[0]
 1868 
 1869     return result
 1870 
 1871 
 1872 @require_context
 1873 def share_replica_get(context, replica_id, with_share_data=False,
 1874                       with_share_server=False, session=None):
 1875     """Returns summary of requested replica if available."""
 1876     session = session or get_session()
 1877 
 1878     result = _share_replica_get_with_filters(
 1879         context, with_share_server=with_share_server,
 1880         replica_id=replica_id, session=session).first()
 1881 
 1882     if result is None:
 1883         raise exception.ShareReplicaNotFound(replica_id=replica_id)
 1884 
 1885     if with_share_data:
 1886         result = _set_replica_share_data(context, result, session)[0]
 1887 
 1888     return result
 1889 
 1890 
 1891 @require_context
 1892 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 1893 def share_replica_update(context, share_replica_id, values,
 1894                          with_share_data=False, session=None):
 1895     """Updates a share replica with specified values."""
 1896     session = session or get_session()
 1897 
 1898     with session.begin():
 1899         _ensure_availability_zone_exists(context, values, session,
 1900                                          strict=False)
 1901         updated_share_replica = _share_instance_update(
 1902             context, share_replica_id, values, session=session)
 1903 
 1904         if with_share_data:
 1905             updated_share_replica = _set_replica_share_data(
 1906                 context, updated_share_replica, session)[0]
 1907 
 1908     return updated_share_replica
 1909 
 1910 
 1911 @require_context
 1912 def share_replica_delete(context, share_replica_id, session=None,
 1913                          need_to_update_usages=True):
 1914     """Deletes a share replica."""
 1915     session = session or get_session()
 1916 
 1917     share_instance_delete(context, share_replica_id, session=session,
 1918                           need_to_update_usages=need_to_update_usages)
 1919 
 1920 
 1921 ################
 1922 
 1923 
 1924 @require_context
 1925 def _share_get_query(context, session=None):
 1926     if session is None:
 1927         session = get_session()
 1928     return (model_query(context, models.Share, session=session).
 1929             options(joinedload('share_metadata')))
 1930 
 1931 
 1932 def _process_share_filters(query, filters, project_id=None, is_public=False):
 1933     if filters is None:
 1934         filters = {}
 1935 
 1936     share_filter_keys = ['share_group_id', 'snapshot_id']
 1937     instance_filter_keys = ['share_server_id', 'status', 'share_type_id',
 1938                             'host', 'share_network_id']
 1939     share_filters = {}
 1940     instance_filters = {}
 1941 
 1942     for k, v in filters.items():
 1943         share_filters.update({k: v}) if k in share_filter_keys else None
 1944         instance_filters.update({k: v}) if k in instance_filter_keys else None
 1945 
 1946     no_key = 'key_is_absent'
 1947 
 1948     def _filter_data(query, model, desired_filters):
 1949         for key, value in desired_filters.items():
 1950             filter_attr = getattr(model, key, no_key)
 1951             if filter_attr == no_key:
 1952                 pass
 1953             query = query.filter(filter_attr == value)
 1954         return query
 1955 
 1956     if share_filters:
 1957         query = _filter_data(query, models.Share, share_filters)
 1958     if instance_filters:
 1959         query = _filter_data(query, models.ShareInstance, instance_filters)
 1960 
 1961     if project_id:
 1962         if is_public:
 1963             query = query.filter(or_(models.Share.project_id == project_id,
 1964                                      models.Share.is_public))
 1965         else:
 1966             query = query.filter(models.Share.project_id == project_id)
 1967 
 1968     display_name = filters.get('display_name')
 1969     if display_name:
 1970         query = query.filter(
 1971             models.Share.display_name == display_name)
 1972     else:
 1973         display_name = filters.get('display_name~')
 1974         if display_name:
 1975             query = query.filter(models.Share.display_name.op('LIKE')(
 1976                 u'%' + display_name + u'%'))
 1977 
 1978     display_description = filters.get('display_description')
 1979     if display_description:
 1980         query = query.filter(
 1981             models.Share.display_description == display_description)
 1982     else:
 1983         display_description = filters.get('display_description~')
 1984         if display_description:
 1985             query = query.filter(models.Share.display_description.op('LIKE')(
 1986                 u'%' + display_description + u'%'))
 1987 
 1988     export_location_id = filters.pop('export_location_id', None)
 1989     export_location_path = filters.pop('export_location_path', None)
 1990     if export_location_id or export_location_path:
 1991         query = query.join(
 1992             models.ShareInstanceExportLocations,
 1993             models.ShareInstanceExportLocations.share_instance_id ==
 1994             models.ShareInstance.id)
 1995         if export_location_path:
 1996             query = query.filter(
 1997                 models.ShareInstanceExportLocations.path ==
 1998                 export_location_path)
 1999         if export_location_id:
 2000             query = query.filter(
 2001                 models.ShareInstanceExportLocations.uuid ==
 2002                 export_location_id)
 2003 
 2004     if 'metadata' in filters:
 2005         for k, v in filters['metadata'].items():
 2006             # pylint: disable=no-member
 2007             query = query.filter(
 2008                 or_(models.Share.share_metadata.any(
 2009                     key=k, value=v)))
 2010     if 'extra_specs' in filters:
 2011         query = query.join(
 2012             models.ShareTypeExtraSpecs,
 2013             models.ShareTypeExtraSpecs.share_type_id ==
 2014             models.ShareInstance.share_type_id)
 2015         for k, v in filters['extra_specs'].items():
 2016             query = query.filter(or_(models.ShareTypeExtraSpecs.key == k,
 2017                                      models.ShareTypeExtraSpecs.value == v))
 2018 
 2019     return query
 2020 
 2021 
 2022 def _metadata_refs(metadata_dict, meta_class):
 2023     metadata_refs = []
 2024     if metadata_dict:
 2025         for k, v in metadata_dict.items():
 2026             value = six.text_type(v) if isinstance(v, bool) else v
 2027 
 2028             metadata_ref = meta_class()
 2029             metadata_ref['key'] = k
 2030             metadata_ref['value'] = value
 2031             metadata_refs.append(metadata_ref)
 2032     return metadata_refs
 2033 
 2034 
 2035 @require_context
 2036 def share_create(context, share_values, create_share_instance=True):
 2037     values = copy.deepcopy(share_values)
 2038     values = ensure_model_dict_has_id(values)
 2039     values['share_metadata'] = _metadata_refs(values.get('metadata'),
 2040                                               models.ShareMetadata)
 2041     session = get_session()
 2042     share_ref = models.Share()
 2043     share_instance_values, share_values = _extract_share_instance_values(
 2044         values)
 2045     _ensure_availability_zone_exists(context, share_instance_values, session,
 2046                                      strict=False)
 2047     share_ref.update(share_values)
 2048 
 2049     with session.begin():
 2050         share_ref.save(session=session)
 2051 
 2052         if create_share_instance:
 2053             _share_instance_create(context, share_ref['id'],
 2054                                    share_instance_values, session=session)
 2055 
 2056         # NOTE(u_glide): Do so to prevent errors with relationships
 2057         return share_get(context, share_ref['id'], session=session)
 2058 
 2059 
 2060 @require_admin_context
 2061 def share_data_get_for_project(context, project_id, user_id,
 2062                                share_type_id=None, session=None):
 2063     query = (model_query(context, models.Share,
 2064                          func.count(models.Share.id),
 2065                          func.sum(models.Share.size),
 2066                          read_deleted="no",
 2067                          session=session).
 2068              filter_by(project_id=project_id))
 2069     if share_type_id:
 2070         query = query.join("instances").filter_by(share_type_id=share_type_id)
 2071     elif user_id:
 2072         query = query.filter_by(user_id=user_id)
 2073     result = query.first()
 2074     return (result[0] or 0, result[1] or 0)
 2075 
 2076 
 2077 @require_context
 2078 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 2079 def share_update(context, share_id, update_values):
 2080     session = get_session()
 2081     values = copy.deepcopy(update_values)
 2082 
 2083     share_instance_values, share_values = _extract_share_instance_values(
 2084         values)
 2085     _ensure_availability_zone_exists(context, share_instance_values, session,
 2086                                      strict=False)
 2087 
 2088     with session.begin():
 2089         share_ref = share_get(context, share_id, session=session)
 2090 
 2091         _share_instance_update(context, share_ref.instance['id'],
 2092                                share_instance_values, session=session)
 2093 
 2094         share_ref.update(share_values)
 2095         share_ref.save(session=session)
 2096         return share_ref
 2097 
 2098 
 2099 @require_context
 2100 def share_get(context, share_id, session=None):
 2101     result = _share_get_query(context, session).filter_by(id=share_id).first()
 2102 
 2103     if result is None:
 2104         raise exception.NotFound()
 2105 
 2106     return result
 2107 
 2108 
 2109 def _share_get_all_with_filters(context, project_id=None, share_server_id=None,
 2110                                 share_group_id=None, filters=None,
 2111                                 is_public=False, sort_key=None,
 2112                                 sort_dir=None):
 2113     """Returns sorted list of shares that satisfies filters.
 2114 
 2115     :param context: context to query under
 2116     :param project_id: project id that owns shares
 2117     :param share_server_id: share server that hosts shares
 2118     :param filters: dict of filters to specify share selection
 2119     :param is_public: public shares from other projects will be added
 2120                       to result if True
 2121     :param sort_key: key of models.Share to be used for sorting
 2122     :param sort_dir: desired direction of sorting, can be 'asc' and 'desc'
 2123     :returns: list -- models.Share
 2124     :raises: exception.InvalidInput
 2125     """
 2126     if filters is None:
 2127         filters = {}
 2128 
 2129     if not sort_key:
 2130         sort_key = 'created_at'
 2131     if not sort_dir:
 2132         sort_dir = 'desc'
 2133     query = (
 2134         _share_get_query(context).join(
 2135             models.ShareInstance,
 2136             models.ShareInstance.share_id == models.Share.id
 2137         )
 2138     )
 2139 
 2140     if share_group_id:
 2141         filters['share_group_id'] = share_group_id
 2142     if share_server_id:
 2143         filters['share_server_id'] = share_server_id
 2144 
 2145     query = _process_share_filters(
 2146         query, filters, project_id, is_public=is_public)
 2147 
 2148     try:
 2149         query = apply_sorting(models.Share, query, sort_key, sort_dir)
 2150     except AttributeError:
 2151         try:
 2152             query = apply_sorting(
 2153                 models.ShareInstance, query, sort_key, sort_dir)
 2154         except AttributeError:
 2155             msg = _("Wrong sorting key provided - '%s'.") % sort_key
 2156             raise exception.InvalidInput(reason=msg)
 2157 
 2158     if 'limit' in filters:
 2159         offset = filters.get('offset', 0)
 2160         query = query.limit(filters['limit']).offset(offset)
 2161 
 2162     # Returns list of shares that satisfy filters.
 2163     query = query.all()
 2164     return query
 2165 
 2166 
 2167 @require_admin_context
 2168 def share_get_all(context, filters=None, sort_key=None, sort_dir=None):
 2169     project_id = filters.pop('project_id', None) if filters else None
 2170     query = _share_get_all_with_filters(
 2171         context,
 2172         project_id=project_id,
 2173         filters=filters, sort_key=sort_key, sort_dir=sort_dir)
 2174 
 2175     return query
 2176 
 2177 
 2178 @require_context
 2179 def share_get_all_by_project(context, project_id, filters=None,
 2180                              is_public=False, sort_key=None, sort_dir=None):
 2181     """Returns list of shares with given project ID."""
 2182     query = _share_get_all_with_filters(
 2183         context, project_id=project_id, filters=filters, is_public=is_public,
 2184         sort_key=sort_key, sort_dir=sort_dir,
 2185     )
 2186     return query
 2187 
 2188 
 2189 @require_context
 2190 def share_get_all_by_share_group_id(context, share_group_id,
 2191                                     filters=None, sort_key=None,
 2192                                     sort_dir=None):
 2193     """Returns list of shares with given group ID."""
 2194     query = _share_get_all_with_filters(
 2195         context, share_group_id=share_group_id,
 2196         filters=filters, sort_key=sort_key, sort_dir=sort_dir,
 2197     )
 2198     return query
 2199 
 2200 
 2201 @require_context
 2202 def share_get_all_by_share_server(context, share_server_id, filters=None,
 2203                                   sort_key=None, sort_dir=None):
 2204     """Returns list of shares with given share server."""
 2205     query = _share_get_all_with_filters(
 2206         context, share_server_id=share_server_id, filters=filters,
 2207         sort_key=sort_key, sort_dir=sort_dir,
 2208     )
 2209     return query
 2210 
 2211 
 2212 @require_context
 2213 def share_delete(context, share_id):
 2214     session = get_session()
 2215 
 2216     with session.begin():
 2217         share_ref = share_get(context, share_id, session)
 2218 
 2219         if len(share_ref.instances) > 0:
 2220             msg = _("Share %(id)s has %(count)s share instances.") % {
 2221                 'id': share_id, 'count': len(share_ref.instances)}
 2222             raise exception.InvalidShare(msg)
 2223 
 2224         share_ref.soft_delete(session=session)
 2225 
 2226         (session.query(models.ShareMetadata).
 2227             filter_by(share_id=share_id).soft_delete())
 2228 
 2229 
 2230 ###################
 2231 
 2232 
 2233 def _share_access_get_query(context, session, values, read_deleted='no'):
 2234     """Get access record."""
 2235     query = (model_query(
 2236         context, models.ShareAccessMapping, session=session,
 2237         read_deleted=read_deleted).options(
 2238             joinedload('share_access_rules_metadata')))
 2239     return query.filter_by(**values)
 2240 
 2241 
 2242 def _share_instance_access_query(context, session, access_id=None,
 2243                                  instance_id=None):
 2244     filters = {'deleted': 'False'}
 2245 
 2246     if access_id is not None:
 2247         filters.update({'access_id': access_id})
 2248 
 2249     if instance_id is not None:
 2250         filters.update({'share_instance_id': instance_id})
 2251 
 2252     return model_query(context, models.ShareInstanceAccessMapping,
 2253                        session=session).filter_by(**filters)
 2254 
 2255 
 2256 def _share_access_metadata_get_item(context, access_id, key, session=None):
 2257     result = (_share_access_metadata_get_query(
 2258         context, access_id, session=session).filter_by(key=key).first())
 2259     if not result:
 2260         raise exception.ShareAccessMetadataNotFound(
 2261             metadata_key=key, access_id=access_id)
 2262     return result
 2263 
 2264 
 2265 def _share_access_metadata_get_query(context, access_id, session=None):
 2266     return (model_query(
 2267         context, models.ShareAccessRulesMetadata, session=session,
 2268         read_deleted="no").
 2269         filter_by(access_id=access_id).
 2270         options(joinedload('access')))
 2271 
 2272 
 2273 @require_context
 2274 def share_access_metadata_update(context, access_id, metadata):
 2275     session = get_session()
 2276 
 2277     with session.begin():
 2278         # Now update all existing items with new values, or create new meta
 2279         # objects
 2280         for meta_key, meta_value in metadata.items():
 2281 
 2282             # update the value whether it exists or not
 2283             item = {"value": meta_value}
 2284             try:
 2285                 meta_ref = _share_access_metadata_get_item(
 2286                     context, access_id, meta_key, session=session)
 2287             except exception.ShareAccessMetadataNotFound:
 2288                 meta_ref = models.ShareAccessRulesMetadata()
 2289                 item.update({"key": meta_key, "access_id": access_id})
 2290 
 2291             meta_ref.update(item)
 2292             meta_ref.save(session=session)
 2293 
 2294         return metadata
 2295 
 2296 
 2297 @require_context
 2298 def share_access_metadata_delete(context, access_id, key):
 2299     session = get_session()
 2300     with session.begin():
 2301         metadata = _share_access_metadata_get_item(
 2302             context, access_id, key, session=session)
 2303 
 2304         metadata.soft_delete(session)
 2305 
 2306 
 2307 @require_context
 2308 def share_access_create(context, values):
 2309     values = ensure_model_dict_has_id(values)
 2310     session = get_session()
 2311     with session.begin():
 2312         values['share_access_rules_metadata'] = (
 2313             _metadata_refs(values.get('metadata'),
 2314                            models.ShareAccessRulesMetadata))
 2315 
 2316         access_ref = models.ShareAccessMapping()
 2317         access_ref.update(values)
 2318         access_ref.save(session=session)
 2319 
 2320         parent_share = share_get(context, values['share_id'], session=session)
 2321 
 2322         for instance in parent_share.instances:
 2323             vals = {
 2324                 'share_instance_id': instance['id'],
 2325                 'access_id': access_ref['id'],
 2326             }
 2327 
 2328             _share_instance_access_create(vals, session)
 2329 
 2330     return share_access_get(context, access_ref['id'])
 2331 
 2332 
 2333 @require_context
 2334 def share_instance_access_create(context, values, share_instance_id):
 2335     values = ensure_model_dict_has_id(values)
 2336     session = get_session()
 2337     with session.begin():
 2338         access_list = _share_access_get_query(
 2339             context, session, {
 2340                 'share_id': values['share_id'],
 2341                 'access_type': values['access_type'],
 2342                 'access_to': values['access_to'],
 2343             }).all()
 2344         if len(access_list) > 0:
 2345             access_ref = access_list[0]
 2346         else:
 2347             access_ref = models.ShareAccessMapping()
 2348         access_ref.update(values)
 2349         access_ref.save(session=session)
 2350 
 2351         vals = {
 2352             'share_instance_id': share_instance_id,
 2353             'access_id': access_ref['id'],
 2354         }
 2355 
 2356         _share_instance_access_create(vals, session)
 2357 
 2358     return share_access_get(context, access_ref['id'])
 2359 
 2360 
 2361 @require_context
 2362 def share_instance_access_copy(context, share_id, instance_id, session=None):
 2363     """Copy access rules from share to share instance."""
 2364     session = session or get_session()
 2365 
 2366     share_access_rules = _share_access_get_query(
 2367         context, session, {'share_id': share_id}).all()
 2368 
 2369     for access_rule in share_access_rules:
 2370         values = {
 2371             'share_instance_id': instance_id,
 2372             'access_id': access_rule['id'],
 2373         }
 2374 
 2375         _share_instance_access_create(values, session)
 2376 
 2377     return share_access_rules
 2378 
 2379 
 2380 def _share_instance_access_create(values, session):
 2381     access_ref = models.ShareInstanceAccessMapping()
 2382     access_ref.update(ensure_model_dict_has_id(values))
 2383     access_ref.save(session=session)
 2384     return access_ref
 2385 
 2386 
 2387 @require_context
 2388 def share_access_get(context, access_id, session=None):
 2389     """Get access record."""
 2390     session = session or get_session()
 2391 
 2392     access = _share_access_get_query(
 2393         context, session, {'id': access_id}).first()
 2394     if access:
 2395         return access
 2396     else:
 2397         raise exception.NotFound()
 2398 
 2399 
 2400 @require_context
 2401 def share_instance_access_get(context, access_id, instance_id,
 2402                               with_share_access_data=True):
 2403     """Get access record."""
 2404     session = get_session()
 2405 
 2406     access = _share_instance_access_query(context, session, access_id,
 2407                                           instance_id).first()
 2408     if access is None:
 2409         raise exception.NotFound()
 2410 
 2411     if with_share_access_data:
 2412         access = _set_instances_share_access_data(context, access, session)[0]
 2413 
 2414     return access
 2415 
 2416 
 2417 @require_context
 2418 def share_access_get_all_for_share(context, share_id, filters=None,
 2419                                    session=None):
 2420     filters = filters or {}
 2421     session = session or get_session()
 2422     query = (_share_access_get_query(
 2423         context, session, {'share_id': share_id}).filter(
 2424         models.ShareAccessMapping.instance_mappings.any()))
 2425 
 2426     if 'metadata' in filters:
 2427         for k, v in filters['metadata'].items():
 2428             query = query.filter(
 2429                 or_(models.ShareAccessMapping.
 2430                     share_access_rules_metadata.any(key=k, value=v)))
 2431 
 2432     return query.all()
 2433 
 2434 
 2435 @require_context
 2436 def share_access_get_all_for_instance(context, instance_id, filters=None,
 2437                                       with_share_access_data=True,
 2438                                       session=None):
 2439     """Get all access rules related to a certain share instance."""
 2440     session = session or get_session()
 2441     filters = copy.deepcopy(filters) if filters else {}
 2442     filters.update({'share_instance_id': instance_id})
 2443     legal_filter_keys = ('id', 'share_instance_id', 'access_id', 'state')
 2444     query = _share_instance_access_query(context, session)
 2445 
 2446     query = exact_filter(
 2447         query, models.ShareInstanceAccessMapping, filters, legal_filter_keys)
 2448 
 2449     instance_accesses = query.all()
 2450 
 2451     if with_share_access_data:
 2452         instance_accesses = _set_instances_share_access_data(
 2453             context, instance_accesses, session)
 2454 
 2455     return instance_accesses
 2456 
 2457 
 2458 def _set_instances_share_access_data(context, instance_accesses, session):
 2459     if instance_accesses and not isinstance(instance_accesses, list):
 2460         instance_accesses = [instance_accesses]
 2461 
 2462     for instance_access in instance_accesses:
 2463         share_access = share_access_get(
 2464             context, instance_access['access_id'], session=session)
 2465         instance_access.set_share_access_data(share_access)
 2466 
 2467     return instance_accesses
 2468 
 2469 
 2470 def _set_instances_snapshot_access_data(context, instance_accesses, session):
 2471     if instance_accesses and not isinstance(instance_accesses, list):
 2472         instance_accesses = [instance_accesses]
 2473 
 2474     for instance_access in instance_accesses:
 2475         snapshot_access = share_snapshot_access_get(
 2476             context, instance_access['access_id'], session=session)
 2477         instance_access.set_snapshot_access_data(snapshot_access)
 2478 
 2479     return instance_accesses
 2480 
 2481 
 2482 @require_context
 2483 def share_access_get_all_by_type_and_access(context, share_id, access_type,
 2484                                             access):
 2485     session = get_session()
 2486     return _share_access_get_query(context, session,
 2487                                    {'share_id': share_id,
 2488                                     'access_type': access_type,
 2489                                     'access_to': access}).all()
 2490 
 2491 
 2492 @require_context
 2493 def share_access_check_for_existing_access(context, share_id, access_type,
 2494                                            access_to):
 2495     return _check_for_existing_access(
 2496         context, 'share', share_id, access_type, access_to)
 2497 
 2498 
 2499 def _check_for_existing_access(context, resource, resource_id, access_type,
 2500                                access_to):
 2501 
 2502     session = get_session()
 2503     if resource == 'share':
 2504         query_method = _share_access_get_query
 2505         access_to_field = models.ShareAccessMapping.access_to
 2506     else:
 2507         query_method = _share_snapshot_access_get_query
 2508         access_to_field = models.ShareSnapshotAccessMapping.access_to
 2509 
 2510     with session.begin():
 2511         if access_type == 'ip':
 2512             rules = query_method(
 2513                 context, session, {'%s_id' % resource: resource_id,
 2514                                    'access_type': access_type}).filter(
 2515                 access_to_field.startswith(access_to.split('/')[0])).all()
 2516 
 2517             matching_rules = [
 2518                 rule for rule in rules if
 2519                 ipaddress.ip_network(six.text_type(access_to)) ==
 2520                 ipaddress.ip_network(six.text_type(rule['access_to']))
 2521             ]
 2522             return len(matching_rules) > 0
 2523         else:
 2524             return query_method(
 2525                 context, session, {'%s_id' % resource: resource_id,
 2526                                    'access_type': access_type,
 2527                                    'access_to': access_to}).count() > 0
 2528 
 2529 
 2530 @require_context
 2531 def share_access_delete_all_by_share(context, share_id):
 2532     session = get_session()
 2533     with session.begin():
 2534         (session.query(models.ShareAccessMapping).
 2535             filter_by(share_id=share_id).soft_delete())
 2536 
 2537 
 2538 @require_context
 2539 def share_instance_access_delete(context, mapping_id):
 2540     session = get_session()
 2541     with session.begin():
 2542 
 2543         mapping = (session.query(models.ShareInstanceAccessMapping).
 2544                    filter_by(id=mapping_id).first())
 2545 
 2546         if not mapping:
 2547             exception.NotFound()
 2548 
 2549         mapping.soft_delete(session, update_status=True,
 2550                             status_field_name='state')
 2551 
 2552         other_mappings = _share_instance_access_query(
 2553             context, session, mapping['access_id']).all()
 2554 
 2555         # NOTE(u_glide): Remove access rule if all mappings were removed.
 2556         if len(other_mappings) == 0:
 2557             (session.query(models.ShareAccessRulesMetadata).filter_by(
 2558                 access_id=mapping['access_id']).soft_delete())
 2559 
 2560             (session.query(models.ShareAccessMapping).filter_by(
 2561                 id=mapping['access_id']).soft_delete())
 2562 
 2563 
 2564 @require_context
 2565 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 2566 def share_instance_access_update(context, access_id, instance_id, updates):
 2567     session = get_session()
 2568     share_access_fields = ('access_type', 'access_to', 'access_key',
 2569                            'access_level')
 2570 
 2571     share_access_map_updates, share_instance_access_map_updates = (
 2572         _extract_subdict_by_fields(updates, share_access_fields)
 2573     )
 2574 
 2575     with session.begin():
 2576         share_access = _share_access_get_query(
 2577             context, session, {'id': access_id}).first()
 2578         share_access.update(share_access_map_updates)
 2579         share_access.save(session=session)
 2580 
 2581         access = _share_instance_access_query(
 2582             context, session, access_id, instance_id).first()
 2583         access.update(share_instance_access_map_updates)
 2584         access.save(session=session)
 2585 
 2586         return access
 2587 
 2588 ###################
 2589 
 2590 
 2591 @require_context
 2592 def share_snapshot_instance_create(context, snapshot_id, values, session=None):
 2593     session = session or get_session()
 2594     values = copy.deepcopy(values)
 2595 
 2596     _change_size_to_instance_size(values)
 2597 
 2598     if not values.get('id'):
 2599         values['id'] = uuidutils.generate_uuid()
 2600     values.update({'snapshot_id': snapshot_id})
 2601 
 2602     instance_ref = models.ShareSnapshotInstance()
 2603     instance_ref.update(values)
 2604     instance_ref.save(session=session)
 2605 
 2606     return share_snapshot_instance_get(context, instance_ref['id'],
 2607                                        session=session)
 2608 
 2609 
 2610 @require_context
 2611 def share_snapshot_instance_update(context, instance_id, values):
 2612     session = get_session()
 2613     instance_ref = share_snapshot_instance_get(context, instance_id,
 2614                                                session=session)
 2615     _change_size_to_instance_size(values)
 2616 
 2617     # NOTE(u_glide): Ignore updates to custom properties
 2618     for extra_key in models.ShareSnapshotInstance._extra_keys:
 2619         if extra_key in values:
 2620             values.pop(extra_key)
 2621 
 2622     instance_ref.update(values)
 2623     instance_ref.save(session=session)
 2624     return instance_ref
 2625 
 2626 
 2627 @require_context
 2628 def share_snapshot_instance_delete(context, snapshot_instance_id,
 2629                                    session=None):
 2630     session = session or get_session()
 2631 
 2632     with session.begin():
 2633 
 2634         snapshot_instance_ref = share_snapshot_instance_get(
 2635             context, snapshot_instance_id, session=session)
 2636 
 2637         access_rules = share_snapshot_access_get_all_for_snapshot_instance(
 2638             context, snapshot_instance_id, session=session)
 2639         for rule in access_rules:
 2640             share_snapshot_instance_access_delete(
 2641                 context, rule['access_id'], snapshot_instance_id)
 2642 
 2643         for el in snapshot_instance_ref.export_locations:
 2644             share_snapshot_instance_export_location_delete(context, el['id'])
 2645 
 2646         snapshot_instance_ref.soft_delete(
 2647             session=session, update_status=True)
 2648         snapshot = share_snapshot_get(
 2649             context, snapshot_instance_ref['snapshot_id'], session=session)
 2650         if len(snapshot.instances) == 0:
 2651             snapshot.soft_delete(session=session)
 2652 
 2653 
 2654 @require_context
 2655 def share_snapshot_instance_get(context, snapshot_instance_id, session=None,
 2656                                 with_share_data=False):
 2657 
 2658     session = session or get_session()
 2659 
 2660     result = _share_snapshot_instance_get_with_filters(
 2661         context, instance_ids=[snapshot_instance_id], session=session).first()
 2662 
 2663     if result is None:
 2664         raise exception.ShareSnapshotInstanceNotFound(
 2665             instance_id=snapshot_instance_id)
 2666 
 2667     if with_share_data:
 2668         result = _set_share_snapshot_instance_data(context, result, session)[0]
 2669 
 2670     return result
 2671 
 2672 
 2673 @require_context
 2674 def share_snapshot_instance_get_all_with_filters(context, search_filters,
 2675                                                  with_share_data=False,
 2676                                                  session=None):
 2677     """Get snapshot instances filtered by known attrs, ignore unknown attrs.
 2678 
 2679     All filters accept list/tuples to filter on, along with simple values.
 2680     """
 2681     def listify(values):
 2682         if values:
 2683             if not isinstance(values, (list, tuple, set)):
 2684                 return values,
 2685             else:
 2686                 return values
 2687 
 2688     session = session or get_session()
 2689     _known_filters = ('instance_ids', 'snapshot_ids', 'share_instance_ids',
 2690                       'statuses')
 2691 
 2692     filters = {k: listify(search_filters.get(k)) for k in _known_filters}
 2693 
 2694     result = _share_snapshot_instance_get_with_filters(
 2695         context, session=session, **filters).all()
 2696 
 2697     if with_share_data:
 2698         result = _set_share_snapshot_instance_data(context, result, session)
 2699 
 2700     return result
 2701 
 2702 
 2703 def _share_snapshot_instance_get_with_filters(context, instance_ids=None,
 2704                                               snapshot_ids=None, statuses=None,
 2705                                               share_instance_ids=None,
 2706                                               session=None):
 2707 
 2708     query = model_query(context, models.ShareSnapshotInstance, session=session,
 2709                         read_deleted="no")
 2710 
 2711     if instance_ids is not None:
 2712         query = query.filter(
 2713             models.ShareSnapshotInstance.id.in_(instance_ids))
 2714 
 2715     if snapshot_ids is not None:
 2716         query = query.filter(
 2717             models.ShareSnapshotInstance.snapshot_id.in_(snapshot_ids))
 2718 
 2719     if share_instance_ids is not None:
 2720         query = query.filter(models.ShareSnapshotInstance.share_instance_id
 2721                              .in_(share_instance_ids))
 2722 
 2723     if statuses is not None:
 2724         query = query.filter(models.ShareSnapshotInstance.status.in_(statuses))
 2725 
 2726     query = query.options(joinedload('share_group_snapshot'))
 2727     return query
 2728 
 2729 
 2730 def _set_share_snapshot_instance_data(context, snapshot_instances, session):
 2731     if snapshot_instances and not isinstance(snapshot_instances, list):
 2732         snapshot_instances = [snapshot_instances]
 2733 
 2734     for snapshot_instance in snapshot_instances:
 2735         share_instance = share_instance_get(
 2736             context, snapshot_instance['share_instance_id'], session=session,
 2737             with_share_data=True)
 2738         snapshot_instance['share'] = share_instance
 2739 
 2740     return snapshot_instances
 2741 
 2742 
 2743 ###################
 2744 
 2745 
 2746 @require_context
 2747 def share_snapshot_create(context, create_values,
 2748                           create_snapshot_instance=True):
 2749     values = copy.deepcopy(create_values)
 2750     values = ensure_model_dict_has_id(values)
 2751 
 2752     snapshot_ref = models.ShareSnapshot()
 2753     snapshot_instance_values, snapshot_values = (
 2754         _extract_snapshot_instance_values(values)
 2755     )
 2756     share_ref = share_get(context, snapshot_values.get('share_id'))
 2757     snapshot_instance_values.update(
 2758         {'share_instance_id': share_ref.instance.id}
 2759     )
 2760 
 2761     snapshot_ref.update(snapshot_values)
 2762     session = get_session()
 2763     with session.begin():
 2764         snapshot_ref.save(session=session)
 2765 
 2766         if create_snapshot_instance:
 2767             share_snapshot_instance_create(
 2768                 context,
 2769                 snapshot_ref['id'],
 2770                 snapshot_instance_values,
 2771                 session=session
 2772             )
 2773         return share_snapshot_get(
 2774             context, snapshot_values['id'], session=session)
 2775 
 2776 
 2777 @require_admin_context
 2778 def snapshot_data_get_for_project(context, project_id, user_id,
 2779                                   share_type_id=None, session=None):
 2780     query = (model_query(context, models.ShareSnapshot,
 2781                          func.count(models.ShareSnapshot.id),
 2782                          func.sum(models.ShareSnapshot.size),
 2783                          read_deleted="no",
 2784                          session=session).
 2785              filter_by(project_id=project_id))
 2786 
 2787     if share_type_id:
 2788         query = query.join(
 2789             models.ShareInstance,
 2790             models.ShareInstance.share_id == models.ShareSnapshot.share_id,
 2791         ).filter_by(share_type_id=share_type_id)
 2792     elif user_id:
 2793         query = query.filter_by(user_id=user_id)
 2794     result = query.first()
 2795 
 2796     return result[0] or 0, result[1] or 0
 2797 
 2798 
 2799 @require_context
 2800 def share_snapshot_get(context, snapshot_id, session=None):
 2801     result = (model_query(context, models.ShareSnapshot, session=session,
 2802                           project_only=True).
 2803               filter_by(id=snapshot_id).
 2804               options(joinedload('share')).
 2805               options(joinedload('instances')).
 2806               first())
 2807 
 2808     if not result:
 2809         raise exception.ShareSnapshotNotFound(snapshot_id=snapshot_id)
 2810 
 2811     return result
 2812 
 2813 
 2814 def _share_snapshot_get_all_with_filters(context, project_id=None,
 2815                                          share_id=None, filters=None,
 2816                                          sort_key=None, sort_dir=None):
 2817     # Init data
 2818     sort_key = sort_key or 'share_id'
 2819     sort_dir = sort_dir or 'desc'
 2820     filters = filters or {}
 2821     query = model_query(context, models.ShareSnapshot)
 2822 
 2823     if project_id:
 2824         query = query.filter_by(project_id=project_id)
 2825     if share_id:
 2826         query = query.filter_by(share_id=share_id)
 2827     query = query.options(joinedload('share'))
 2828     query = query.options(joinedload('instances'))
 2829 
 2830     # Apply filters
 2831     if 'usage' in filters:
 2832         usage_filter_keys = ['any', 'used', 'unused']
 2833         if filters['usage'] == 'any':
 2834             pass
 2835         elif filters['usage'] == 'used':
 2836             query = query.filter(or_(models.Share.snapshot_id == (
 2837                 models.ShareSnapshot.id)))
 2838         elif filters['usage'] == 'unused':
 2839             query = query.filter(or_(models.Share.snapshot_id != (
 2840                 models.ShareSnapshot.id)))
 2841         else:
 2842             msg = _("Wrong 'usage' key provided - '%(key)s'. "
 2843                     "Expected keys are '%(ek)s'.") % {
 2844                         'key': filters['usage'],
 2845                         'ek': six.text_type(usage_filter_keys)}
 2846             raise exception.InvalidInput(reason=msg)
 2847 
 2848     # Apply sorting
 2849     try:
 2850         attr = getattr(models.ShareSnapshot, sort_key)
 2851     except AttributeError:
 2852         msg = _("Wrong sorting key provided - '%s'.") % sort_key
 2853         raise exception.InvalidInput(reason=msg)
 2854     if sort_dir.lower() == 'desc':
 2855         query = query.order_by(attr.desc())
 2856     elif sort_dir.lower() == 'asc':
 2857         query = query.order_by(attr.asc())
 2858     else:
 2859         msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' "
 2860                 "and sort direction is '%(sort_dir)s'.") % {
 2861                     "sort_key": sort_key, "sort_dir": sort_dir}
 2862         raise exception.InvalidInput(reason=msg)
 2863 
 2864     # Returns list of shares that satisfy filters
 2865     return query.all()
 2866 
 2867 
 2868 @require_admin_context
 2869 def share_snapshot_get_all(context, filters=None, sort_key=None,
 2870                            sort_dir=None):
 2871     return _share_snapshot_get_all_with_filters(
 2872         context, filters=filters, sort_key=sort_key, sort_dir=sort_dir,
 2873     )
 2874 
 2875 
 2876 @require_context
 2877 def share_snapshot_get_all_by_project(context, project_id, filters=None,
 2878                                       sort_key=None, sort_dir=None):
 2879     authorize_project_context(context, project_id)
 2880     return _share_snapshot_get_all_with_filters(
 2881         context, project_id=project_id,
 2882         filters=filters, sort_key=sort_key, sort_dir=sort_dir,
 2883     )
 2884 
 2885 
 2886 @require_context
 2887 def share_snapshot_get_all_for_share(context, share_id, filters=None,
 2888                                      sort_key=None, sort_dir=None):
 2889     return _share_snapshot_get_all_with_filters(
 2890         context, share_id=share_id,
 2891         filters=filters, sort_key=sort_key, sort_dir=sort_dir,
 2892     )
 2893 
 2894 
 2895 @require_context
 2896 def share_snapshot_get_latest_for_share(context, share_id):
 2897 
 2898     snapshots = _share_snapshot_get_all_with_filters(
 2899         context, share_id=share_id, sort_key='created_at', sort_dir='desc')
 2900     return snapshots[0] if snapshots else None
 2901 
 2902 
 2903 @require_context
 2904 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 2905 def share_snapshot_update(context, snapshot_id, values):
 2906     session = get_session()
 2907     with session.begin():
 2908         snapshot_ref = share_snapshot_get(context, snapshot_id,
 2909                                           session=session)
 2910 
 2911         instance_values, snapshot_values = (
 2912             _extract_snapshot_instance_values(values)
 2913         )
 2914 
 2915         if snapshot_values:
 2916             snapshot_ref.update(snapshot_values)
 2917             snapshot_ref.save(session=session)
 2918 
 2919         if instance_values:
 2920             snapshot_ref.instance.update(instance_values)
 2921             snapshot_ref.instance.save(session=session)
 2922 
 2923         return snapshot_ref
 2924 
 2925 
 2926 @require_context
 2927 def share_snapshot_instances_status_update(
 2928         context, snapshot_instance_ids, values, session=None):
 2929     session = session or get_session()
 2930 
 2931     result = (
 2932         model_query(
 2933             context, models.ShareSnapshotInstance,
 2934             read_deleted="no", session=session).filter(
 2935             models.ShareSnapshotInstance.id.in_(snapshot_instance_ids)
 2936             ).update(values, synchronize_session=False))
 2937 
 2938     return result
 2939 
 2940 #################################
 2941 
 2942 
 2943 @require_context
 2944 def share_snapshot_access_create(context, values):
 2945     values = ensure_model_dict_has_id(values)
 2946     session = get_session()
 2947     with session.begin():
 2948         access_ref = models.ShareSnapshotAccessMapping()
 2949         access_ref.update(values)
 2950         access_ref.save(session=session)
 2951 
 2952         snapshot = share_snapshot_get(context, values['share_snapshot_id'],
 2953                                       session=session)
 2954 
 2955         for instance in snapshot.instances:
 2956             vals = {
 2957                 'share_snapshot_instance_id': instance['id'],
 2958                 'access_id': access_ref['id'],
 2959             }
 2960 
 2961             _share_snapshot_instance_access_create(vals, session)
 2962 
 2963     return share_snapshot_access_get(context, access_ref['id'])
 2964 
 2965 
 2966 def _share_snapshot_access_get_query(context, session, filters,
 2967                                      read_deleted='no'):
 2968 
 2969     query = model_query(context, models.ShareSnapshotAccessMapping,
 2970                         session=session, read_deleted=read_deleted)
 2971     return query.filter_by(**filters)
 2972 
 2973 
 2974 def _share_snapshot_instance_access_get_query(context, session,
 2975                                               access_id=None,
 2976                                               share_snapshot_instance_id=None):
 2977     filters = {'deleted': 'False'}
 2978 
 2979     if access_id is not None:
 2980         filters.update({'access_id': access_id})
 2981 
 2982     if share_snapshot_instance_id is not None:
 2983         filters.update(
 2984             {'share_snapshot_instance_id': share_snapshot_instance_id})
 2985 
 2986     return model_query(context, models.ShareSnapshotInstanceAccessMapping,
 2987                        session=session).filter_by(**filters)
 2988 
 2989 
 2990 @require_context
 2991 def share_snapshot_instance_access_get_all(context, access_id, session):
 2992     rules = _share_snapshot_instance_access_get_query(
 2993         context, session, access_id=access_id).all()
 2994     return rules
 2995 
 2996 
 2997 @require_context
 2998 def share_snapshot_access_get(context, access_id, session=None):
 2999     session = session or get_session()
 3000 
 3001     access = _share_snapshot_access_get_query(
 3002         context, session, {'id': access_id}).first()
 3003 
 3004     if access:
 3005         return access
 3006     else:
 3007         raise exception.NotFound()
 3008 
 3009 
 3010 def _share_snapshot_instance_access_create(values, session):
 3011     access_ref = models.ShareSnapshotInstanceAccessMapping()
 3012     access_ref.update(ensure_model_dict_has_id(values))
 3013     access_ref.save(session=session)
 3014     return access_ref
 3015 
 3016 
 3017 @require_context
 3018 def share_snapshot_access_get_all_for_share_snapshot(context,
 3019                                                      share_snapshot_id,
 3020                                                      filters):
 3021     session = get_session()
 3022     filters['share_snapshot_id'] = share_snapshot_id
 3023     access_list = _share_snapshot_access_get_query(
 3024         context, session, filters).all()
 3025 
 3026     return access_list
 3027 
 3028 
 3029 @require_context
 3030 def share_snapshot_check_for_existing_access(context, share_snapshot_id,
 3031                                              access_type, access_to):
 3032     return _check_for_existing_access(
 3033         context, 'share_snapshot', share_snapshot_id, access_type, access_to)
 3034 
 3035 
 3036 @require_context
 3037 def share_snapshot_access_get_all_for_snapshot_instance(
 3038         context, snapshot_instance_id, filters=None,
 3039         with_snapshot_access_data=True, session=None):
 3040     """Get all access rules related to a certain snapshot instance."""
 3041     session = session or get_session()
 3042     filters = copy.deepcopy(filters) if filters else {}
 3043     filters.update({'share_snapshot_instance_id': snapshot_instance_id})
 3044 
 3045     query = _share_snapshot_instance_access_get_query(context, session)
 3046 
 3047     legal_filter_keys = (
 3048         'id', 'share_snapshot_instance_id', 'access_id', 'state')
 3049 
 3050     query = exact_filter(
 3051         query, models.ShareSnapshotInstanceAccessMapping, filters,
 3052         legal_filter_keys)
 3053 
 3054     instance_accesses = query.all()
 3055 
 3056     if with_snapshot_access_data:
 3057         instance_accesses = _set_instances_snapshot_access_data(
 3058             context, instance_accesses, session)
 3059 
 3060     return instance_accesses
 3061 
 3062 
 3063 @require_context
 3064 def share_snapshot_instance_access_update(
 3065         context, access_id, instance_id, updates):
 3066 
 3067     snapshot_access_fields = ('access_type', 'access_to')
 3068     snapshot_access_map_updates, share_instance_access_map_updates = (
 3069         _extract_subdict_by_fields(updates, snapshot_access_fields)
 3070     )
 3071 
 3072     session = get_session()
 3073     with session.begin():
 3074 
 3075         snapshot_access = _share_snapshot_access_get_query(
 3076             context, session, {'id': access_id}).first()
 3077         if not snapshot_access:
 3078             raise exception.NotFound()
 3079         snapshot_access.update(snapshot_access_map_updates)
 3080         snapshot_access.save(session=session)
 3081 
 3082         access = _share_snapshot_instance_access_get_query(
 3083             context, session, access_id=access_id,
 3084             share_snapshot_instance_id=instance_id).first()
 3085         if not access:
 3086             raise exception.NotFound()
 3087         access.update(share_instance_access_map_updates)
 3088         access.save(session=session)
 3089 
 3090         return access
 3091 
 3092 
 3093 @require_context
 3094 def share_snapshot_instance_access_get(
 3095         context, access_id, share_snapshot_instance_id,
 3096         with_snapshot_access_data=True):
 3097 
 3098     session = get_session()
 3099 
 3100     with session.begin():
 3101         access = _share_snapshot_instance_access_get_query(
 3102             context, session, access_id=access_id,
 3103             share_snapshot_instance_id=share_snapshot_instance_id).first()
 3104 
 3105         if access is None:
 3106             raise exception.NotFound()
 3107 
 3108         if with_snapshot_access_data:
 3109             return _set_instances_snapshot_access_data(
 3110                 context, access, session)[0]
 3111         else:
 3112             return access
 3113 
 3114 
 3115 @require_context
 3116 def share_snapshot_instance_access_delete(
 3117         context, access_id, snapshot_instance_id):
 3118     session = get_session()
 3119     with session.begin():
 3120 
 3121         rule = _share_snapshot_instance_access_get_query(
 3122             context, session, access_id=access_id,
 3123             share_snapshot_instance_id=snapshot_instance_id).first()
 3124 
 3125         if not rule:
 3126             exception.NotFound()
 3127 
 3128         rule.soft_delete(session, update_status=True,
 3129                          status_field_name='state')
 3130 
 3131         other_mappings = share_snapshot_instance_access_get_all(
 3132             context, rule['access_id'], session)
 3133 
 3134         if len(other_mappings) == 0:
 3135             (
 3136                 session.query(models.ShareSnapshotAccessMapping)
 3137                 .filter_by(id=rule['access_id'])
 3138                 .soft_delete(update_status=True, status_field_name='state')
 3139             )
 3140 
 3141 
 3142 @require_context
 3143 def share_snapshot_instance_export_location_create(context, values):
 3144 
 3145     values = ensure_model_dict_has_id(values)
 3146     session = get_session()
 3147     with session.begin():
 3148         ssiel = models.ShareSnapshotInstanceExportLocation()
 3149         ssiel.update(values)
 3150         ssiel.save(session=session)
 3151 
 3152         return ssiel
 3153 
 3154 
 3155 def _share_snapshot_instance_export_locations_get_query(context, session,
 3156                                                         values):
 3157     query = model_query(context, models.ShareSnapshotInstanceExportLocation,
 3158                         session=session)
 3159     return query.filter_by(**values)
 3160 
 3161 
 3162 @require_context
 3163 def share_snapshot_export_locations_get(context, snapshot_id):
 3164     session = get_session()
 3165     snapshot = share_snapshot_get(context, snapshot_id, session=session)
 3166     ins_ids = [ins['id'] for ins in snapshot.instances]
 3167     export_locations = _share_snapshot_instance_export_locations_get_query(
 3168         context, session, {}).filter(
 3169         models.ShareSnapshotInstanceExportLocation.
 3170             share_snapshot_instance_id.in_(ins_ids)).all()
 3171     return export_locations
 3172 
 3173 
 3174 @require_context
 3175 def share_snapshot_instance_export_locations_get_all(
 3176         context, share_snapshot_instance_id, session=None):
 3177 
 3178     if not session:
 3179         session = get_session()
 3180     export_locations = _share_snapshot_instance_export_locations_get_query(
 3181         context, session,
 3182         {'share_snapshot_instance_id': share_snapshot_instance_id}).all()
 3183     return export_locations
 3184 
 3185 
 3186 @require_context
 3187 def share_snapshot_instance_export_location_get(context, el_id):
 3188     session = get_session()
 3189 
 3190     export_location = _share_snapshot_instance_export_locations_get_query(
 3191         context, session, {'id': el_id}).first()
 3192 
 3193     if export_location:
 3194         return export_location
 3195     else:
 3196         raise exception.NotFound()
 3197 
 3198 
 3199 @require_context
 3200 def share_snapshot_instance_export_location_delete(context, el_id):
 3201     session = get_session()
 3202     with session.begin():
 3203 
 3204         el = _share_snapshot_instance_export_locations_get_query(
 3205             context, session, {'id': el_id}).first()
 3206 
 3207         if not el:
 3208             exception.NotFound()
 3209 
 3210         el.soft_delete(session=session)
 3211 
 3212 
 3213 @require_context
 3214 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3215 def share_snapshot_instance_export_locations_update(
 3216         context, share_snapshot_instance_id, export_locations, delete):
 3217     # NOTE(dviroel): Lets keep this backward compatibility for driver that
 3218     # may still return export_locations as string
 3219     if not isinstance(export_locations, (list, tuple, set)):
 3220         export_locations = (export_locations, )
 3221     export_locations_as_dicts = []
 3222     for el in export_locations:
 3223         export_location = el
 3224         if isinstance(el, six.string_types):
 3225             export_location = {
 3226                 "path": el,
 3227                 "is_admin_only": False,
 3228             }
 3229         elif not isinstance(export_location, dict):
 3230             raise exception.ManilaException(
 3231                 _("Wrong export location type '%s'.") % type(export_location))
 3232         export_locations_as_dicts.append(export_location)
 3233     export_locations = export_locations_as_dicts
 3234 
 3235     export_locations_paths = [el['path'] for el in export_locations]
 3236 
 3237     session = get_session()
 3238 
 3239     current_el_rows = share_snapshot_instance_export_locations_get_all(
 3240         context, share_snapshot_instance_id, session=session)
 3241 
 3242     def get_path_list_from_rows(rows):
 3243         return set([row['path'] for row in rows])
 3244 
 3245     current_el_paths = get_path_list_from_rows(current_el_rows)
 3246 
 3247     def create_indexed_time_dict(key_list):
 3248         base = timeutils.utcnow()
 3249         return {
 3250             # NOTE(u_glide): Incrementing timestamp by microseconds to make
 3251             # timestamp order match index order.
 3252             key: base + datetime.timedelta(microseconds=index)
 3253             for index, key in enumerate(key_list)
 3254         }
 3255 
 3256     indexed_update_time = create_indexed_time_dict(export_locations_paths)
 3257 
 3258     for el in current_el_rows:
 3259         if delete and el['path'] not in export_locations_paths:
 3260             el.soft_delete(session)
 3261         else:
 3262             updated_at = indexed_update_time[el['path']]
 3263             el.update({
 3264                 'updated_at': updated_at,
 3265             })
 3266             el.save(session=session)
 3267 
 3268     # Now add new export locations
 3269     for el in export_locations:
 3270         if el['path'] in current_el_paths:
 3271             # Already updated
 3272             continue
 3273 
 3274         location_ref = models.ShareSnapshotInstanceExportLocation()
 3275         location_ref.update({
 3276             'id': uuidutils.generate_uuid(),
 3277             'path': el['path'],
 3278             'share_snapshot_instance_id': share_snapshot_instance_id,
 3279             'updated_at': indexed_update_time[el['path']],
 3280             'is_admin_only': el.get('is_admin_only', False),
 3281         })
 3282         location_ref.save(session=session)
 3283 
 3284     return get_path_list_from_rows(
 3285         share_snapshot_instance_export_locations_get_all(
 3286             context, share_snapshot_instance_id, session=session))
 3287 
 3288 #################################
 3289 
 3290 
 3291 @require_context
 3292 @require_share_exists
 3293 def share_metadata_get(context, share_id):
 3294     return _share_metadata_get(context, share_id)
 3295 
 3296 
 3297 @require_context
 3298 @require_share_exists
 3299 def share_metadata_delete(context, share_id, key):
 3300     (_share_metadata_get_query(context, share_id).
 3301         filter_by(key=key).soft_delete())
 3302 
 3303 
 3304 @require_context
 3305 @require_share_exists
 3306 def share_metadata_update(context, share_id, metadata, delete):
 3307     return _share_metadata_update(context, share_id, metadata, delete)
 3308 
 3309 
 3310 def _share_metadata_get_query(context, share_id, session=None):
 3311     return (model_query(context, models.ShareMetadata, session=session,
 3312                         read_deleted="no").
 3313             filter_by(share_id=share_id).
 3314             options(joinedload('share')))
 3315 
 3316 
 3317 def _share_metadata_get(context, share_id, session=None):
 3318     rows = _share_metadata_get_query(context, share_id,
 3319                                      session=session).all()
 3320     result = {}
 3321     for row in rows:
 3322         result[row['key']] = row['value']
 3323 
 3324     return result
 3325 
 3326 
 3327 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3328 def _share_metadata_update(context, share_id, metadata, delete, session=None):
 3329     if not session:
 3330         session = get_session()
 3331 
 3332     with session.begin():
 3333         # Set existing metadata to deleted if delete argument is True
 3334         if delete:
 3335             original_metadata = _share_metadata_get(context, share_id,
 3336                                                     session=session)
 3337             for meta_key, meta_value in original_metadata.items():
 3338                 if meta_key not in metadata:
 3339                     meta_ref = _share_metadata_get_item(context, share_id,
 3340                                                         meta_key,
 3341                                                         session=session)
 3342                     meta_ref.soft_delete(session=session)
 3343 
 3344         meta_ref = None
 3345 
 3346         # Now update all existing items with new values, or create new meta
 3347         # objects
 3348         for meta_key, meta_value in metadata.items():
 3349 
 3350             # update the value whether it exists or not
 3351             item = {"value": meta_value}
 3352 
 3353             try:
 3354                 meta_ref = _share_metadata_get_item(context, share_id,
 3355                                                     meta_key,
 3356                                                     session=session)
 3357             except exception.ShareMetadataNotFound:
 3358                 meta_ref = models.ShareMetadata()
 3359                 item.update({"key": meta_key, "share_id": share_id})
 3360 
 3361             meta_ref.update(item)
 3362             meta_ref.save(session=session)
 3363 
 3364         return metadata
 3365 
 3366 
 3367 def _share_metadata_get_item(context, share_id, key, session=None):
 3368     result = (_share_metadata_get_query(context, share_id, session=session).
 3369               filter_by(key=key).
 3370               first())
 3371 
 3372     if not result:
 3373         raise exception.ShareMetadataNotFound(metadata_key=key,
 3374                                               share_id=share_id)
 3375     return result
 3376 
 3377 
 3378 ############################
 3379 # Export locations functions
 3380 ############################
 3381 
 3382 def _share_export_locations_get(context, share_instance_ids,
 3383                                 include_admin_only=True,
 3384                                 ignore_secondary_replicas=False, session=None):
 3385     session = session or get_session()
 3386 
 3387     if not isinstance(share_instance_ids, (set, list, tuple)):
 3388         share_instance_ids = (share_instance_ids, )
 3389 
 3390     query = model_query(
 3391         context,
 3392         models.ShareInstanceExportLocations,
 3393         session=session,
 3394         read_deleted="no",
 3395     ).filter(
 3396         models.ShareInstanceExportLocations.share_instance_id.in_(
 3397             share_instance_ids),
 3398     ).order_by(
 3399         "updated_at",
 3400     ).options(
 3401         joinedload("_el_metadata_bare"),
 3402     )
 3403 
 3404     if not include_admin_only:
 3405         query = query.filter_by(is_admin_only=False)
 3406 
 3407     if ignore_secondary_replicas:
 3408         replica_state_attr = models.ShareInstance.replica_state
 3409         query = query.join("share_instance").filter(
 3410             or_(replica_state_attr == None,  # noqa
 3411                 replica_state_attr == constants.REPLICA_STATE_ACTIVE))
 3412 
 3413     return query.all()
 3414 
 3415 
 3416 @require_context
 3417 @require_share_exists
 3418 def share_export_locations_get_by_share_id(context, share_id,
 3419                                            include_admin_only=True,
 3420                                            ignore_migration_destination=False,
 3421                                            ignore_secondary_replicas=False):
 3422     share = share_get(context, share_id)
 3423     if ignore_migration_destination:
 3424         ids = [instance.id for instance in share.instances
 3425                if instance['status'] != constants.STATUS_MIGRATING_TO]
 3426     else:
 3427         ids = [instance.id for instance in share.instances]
 3428     rows = _share_export_locations_get(
 3429         context, ids, include_admin_only=include_admin_only,
 3430         ignore_secondary_replicas=ignore_secondary_replicas)
 3431     return rows
 3432 
 3433 
 3434 @require_context
 3435 @require_share_instance_exists
 3436 def share_export_locations_get_by_share_instance_id(context,
 3437                                                     share_instance_id,
 3438                                                     include_admin_only=True):
 3439     rows = _share_export_locations_get(
 3440         context, [share_instance_id], include_admin_only=include_admin_only)
 3441     return rows
 3442 
 3443 
 3444 @require_context
 3445 @require_share_exists
 3446 def share_export_locations_get(context, share_id):
 3447     # NOTE(vponomaryov): this method is kept for compatibility with
 3448     # old approach. New one uses 'share_export_locations_get_by_share_id'.
 3449     # Which returns list of dicts instead of list of strings, as this one does.
 3450     share = share_get(context, share_id)
 3451     rows = _share_export_locations_get(
 3452         context, share.instance.id, context.is_admin)
 3453 
 3454     return [location['path'] for location in rows]
 3455 
 3456 
 3457 @require_context
 3458 def share_export_location_get_by_uuid(context, export_location_uuid,
 3459                                       ignore_secondary_replicas=False,
 3460                                       session=None):
 3461     session = session or get_session()
 3462 
 3463     query = model_query(
 3464         context,
 3465         models.ShareInstanceExportLocations,
 3466         session=session,
 3467         read_deleted="no",
 3468     ).filter_by(
 3469         uuid=export_location_uuid,
 3470     ).options(
 3471         joinedload("_el_metadata_bare"),
 3472     )
 3473 
 3474     if ignore_secondary_replicas:
 3475         replica_state_attr = models.ShareInstance.replica_state
 3476         query = query.join("share_instance").filter(
 3477             or_(replica_state_attr == None,  # noqa
 3478                 replica_state_attr == constants.REPLICA_STATE_ACTIVE))
 3479 
 3480     result = query.first()
 3481     if not result:
 3482         raise exception.ExportLocationNotFound(uuid=export_location_uuid)
 3483     return result
 3484 
 3485 
 3486 @require_context
 3487 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3488 def share_export_locations_update(context, share_instance_id, export_locations,
 3489                                   delete):
 3490     # NOTE(u_glide):
 3491     # Backward compatibility code for drivers,
 3492     # which return single export_location as string
 3493     if not isinstance(export_locations, (list, tuple, set)):
 3494         export_locations = (export_locations, )
 3495     export_locations_as_dicts = []
 3496     for el in export_locations:
 3497         # NOTE(vponomaryov): transform old export locations view to new one
 3498         export_location = el
 3499         if isinstance(el, six.string_types):
 3500             export_location = {
 3501                 "path": el,
 3502                 "is_admin_only": False,
 3503                 "metadata": {},
 3504             }
 3505         elif isinstance(export_location, dict):
 3506             if 'metadata' not in export_location:
 3507                 export_location['metadata'] = {}
 3508         else:
 3509             raise exception.ManilaException(
 3510                 _("Wrong export location type '%s'.") % type(export_location))
 3511         export_locations_as_dicts.append(export_location)
 3512     export_locations = export_locations_as_dicts
 3513 
 3514     export_locations_paths = [el['path'] for el in export_locations]
 3515 
 3516     session = get_session()
 3517 
 3518     current_el_rows = _share_export_locations_get(
 3519         context, share_instance_id, session=session)
 3520 
 3521     def get_path_list_from_rows(rows):
 3522         return set([row['path'] for row in rows])
 3523 
 3524     current_el_paths = get_path_list_from_rows(current_el_rows)
 3525 
 3526     def create_indexed_time_dict(key_list):
 3527         base = timeutils.utcnow()
 3528         return {
 3529             # NOTE(u_glide): Incrementing timestamp by microseconds to make
 3530             # timestamp order match index order.
 3531             key: base + datetime.timedelta(microseconds=index)
 3532             for index, key in enumerate(key_list)
 3533         }
 3534 
 3535     indexed_update_time = create_indexed_time_dict(export_locations_paths)
 3536 
 3537     for el in current_el_rows:
 3538         if delete and el['path'] not in export_locations_paths:
 3539             export_location_metadata_delete(context, el['uuid'])
 3540             el.soft_delete(session)
 3541         else:
 3542             updated_at = indexed_update_time[el['path']]
 3543             el.update({
 3544                 'updated_at': updated_at,
 3545                 'deleted': 0,
 3546             })
 3547             el.save(session=session)
 3548             if el['el_metadata']:
 3549                 export_location_metadata_update(
 3550                     context, el['uuid'], el['el_metadata'], session=session)
 3551 
 3552     # Now add new export locations
 3553     for el in export_locations:
 3554         if el['path'] in current_el_paths:
 3555             # Already updated
 3556             continue
 3557 
 3558         location_ref = models.ShareInstanceExportLocations()
 3559         location_ref.update({
 3560             'uuid': uuidutils.generate_uuid(),
 3561             'path': el['path'],
 3562             'share_instance_id': share_instance_id,
 3563             'updated_at': indexed_update_time[el['path']],
 3564             'deleted': 0,
 3565             'is_admin_only': el.get('is_admin_only', False),
 3566         })
 3567         location_ref.save(session=session)
 3568         if not el.get('metadata'):
 3569             continue
 3570         export_location_metadata_update(
 3571             context, location_ref['uuid'], el.get('metadata'), session=session)
 3572 
 3573     return get_path_list_from_rows(_share_export_locations_get(
 3574         context, share_instance_id, session=session))
 3575 
 3576 
 3577 #####################################
 3578 # Export locations metadata functions
 3579 #####################################
 3580 
 3581 def _export_location_metadata_get_query(context, export_location_uuid,
 3582                                         session=None):
 3583     session = session or get_session()
 3584     export_location_id = share_export_location_get_by_uuid(
 3585         context, export_location_uuid).id
 3586 
 3587     return model_query(
 3588         context, models.ShareInstanceExportLocationsMetadata, session=session,
 3589         read_deleted="no",
 3590     ).filter_by(
 3591         export_location_id=export_location_id,
 3592     )
 3593 
 3594 
 3595 @require_context
 3596 def export_location_metadata_get(context, export_location_uuid, session=None):
 3597     rows = _export_location_metadata_get_query(
 3598         context, export_location_uuid, session=session).all()
 3599     result = {}
 3600     for row in rows:
 3601         result[row["key"]] = row["value"]
 3602     return result
 3603 
 3604 
 3605 @require_context
 3606 def export_location_metadata_delete(context, export_location_uuid, keys=None):
 3607     session = get_session()
 3608     metadata = _export_location_metadata_get_query(
 3609         context, export_location_uuid, session=session,
 3610     )
 3611     # NOTE(vponomaryov): if keys is None then we delete all metadata.
 3612     if keys is not None:
 3613         keys = keys if isinstance(keys, (list, set, tuple)) else (keys, )
 3614         metadata = metadata.filter(
 3615             models.ShareInstanceExportLocationsMetadata.key.in_(keys))
 3616     metadata = metadata.all()
 3617     for meta_ref in metadata:
 3618         meta_ref.soft_delete(session=session)
 3619 
 3620 
 3621 @require_context
 3622 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3623 def export_location_metadata_update(context, export_location_uuid, metadata,
 3624                                     delete=False, session=None):
 3625     session = session or get_session()
 3626     if delete:
 3627         original_metadata = export_location_metadata_get(
 3628             context, export_location_uuid, session=session)
 3629         keys_for_deletion = set(original_metadata).difference(metadata)
 3630         if keys_for_deletion:
 3631             export_location_metadata_delete(
 3632                 context, export_location_uuid, keys=keys_for_deletion)
 3633 
 3634     el = share_export_location_get_by_uuid(context, export_location_uuid)
 3635     for meta_key, meta_value in metadata.items():
 3636         # NOTE(vponomaryov): we should use separate session
 3637         # for each meta_ref because of autoincrement of integer primary key
 3638         # that will not take effect using one session and we will rewrite,
 3639         # in that case, single record - first one added with this call.
 3640         session = get_session()
 3641 
 3642         if meta_value is None:
 3643             LOG.warning("%s should be properly defined in the driver.",
 3644                         meta_key)
 3645 
 3646         item = {"value": meta_value, "updated_at": timeutils.utcnow()}
 3647 
 3648         meta_ref = _export_location_metadata_get_query(
 3649             context, export_location_uuid, session=session,
 3650         ).filter_by(
 3651             key=meta_key,
 3652         ).first()
 3653 
 3654         if not meta_ref:
 3655             meta_ref = models.ShareInstanceExportLocationsMetadata()
 3656             item.update({
 3657                 "key": meta_key,
 3658                 "export_location_id": el.id,
 3659             })
 3660 
 3661         meta_ref.update(item)
 3662         meta_ref.save(session=session)
 3663 
 3664     return metadata
 3665 
 3666 
 3667 ###################################
 3668 
 3669 
 3670 @require_context
 3671 def security_service_create(context, values):
 3672     values = ensure_model_dict_has_id(values)
 3673 
 3674     security_service_ref = models.SecurityService()
 3675     security_service_ref.update(values)
 3676     session = get_session()
 3677 
 3678     with session.begin():
 3679         security_service_ref.save(session=session)
 3680 
 3681     return security_service_ref
 3682 
 3683 
 3684 @require_context
 3685 def security_service_delete(context, id):
 3686     session = get_session()
 3687     with session.begin():
 3688         security_service_ref = security_service_get(context,
 3689                                                     id,
 3690                                                     session=session)
 3691         security_service_ref.soft_delete(session)
 3692 
 3693 
 3694 @require_context
 3695 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3696 def security_service_update(context, id, values):
 3697     session = get_session()
 3698     with session.begin():
 3699         security_service_ref = security_service_get(context,
 3700                                                     id,
 3701                                                     session=session)
 3702         security_service_ref.update(values)
 3703         security_service_ref.save(session=session)
 3704         return security_service_ref
 3705 
 3706 
 3707 @require_context
 3708 def security_service_get(context, id, session=None):
 3709     result = (_security_service_get_query(context, session=session).
 3710               filter_by(id=id).first())
 3711 
 3712     if result is None:
 3713         raise exception.SecurityServiceNotFound(security_service_id=id)
 3714     return result
 3715 
 3716 
 3717 @require_context
 3718 def security_service_get_all(context):
 3719     return _security_service_get_query(context).all()
 3720 
 3721 
 3722 @require_context
 3723 def security_service_get_all_by_project(context, project_id):
 3724     return (_security_service_get_query(context).
 3725             filter_by(project_id=project_id).all())
 3726 
 3727 
 3728 def _security_service_get_query(context, session=None):
 3729     if session is None:
 3730         session = get_session()
 3731     return model_query(context, models.SecurityService, session=session)
 3732 
 3733 
 3734 ###################
 3735 
 3736 
 3737 def _network_get_query(context, session=None):
 3738     if session is None:
 3739         session = get_session()
 3740     return (model_query(context, models.ShareNetwork, session=session,
 3741                         project_only=True).
 3742             options(joinedload('share_instances'),
 3743                     joinedload('security_services'),
 3744                     subqueryload('share_network_subnets')))
 3745 
 3746 
 3747 @require_context
 3748 def share_network_create(context, values):
 3749     values = ensure_model_dict_has_id(values)
 3750 
 3751     network_ref = models.ShareNetwork()
 3752     network_ref.update(values)
 3753     session = get_session()
 3754     with session.begin():
 3755         network_ref.save(session=session)
 3756     return share_network_get(context, values['id'], session)
 3757 
 3758 
 3759 @require_context
 3760 def share_network_delete(context, id):
 3761     session = get_session()
 3762     with session.begin():
 3763         network_ref = share_network_get(context, id, session=session)
 3764         network_ref.soft_delete(session)
 3765 
 3766 
 3767 @require_context
 3768 def share_network_update(context, id, values):
 3769     session = get_session()
 3770     with session.begin():
 3771         network_ref = share_network_get(context, id, session=session)
 3772         network_ref.update(values)
 3773         network_ref.save(session=session)
 3774         return network_ref
 3775 
 3776 
 3777 @require_context
 3778 def share_network_get(context, id, session=None):
 3779     result = _network_get_query(context, session).filter_by(id=id).first()
 3780     if result is None:
 3781         raise exception.ShareNetworkNotFound(share_network_id=id)
 3782     return result
 3783 
 3784 
 3785 @require_context
 3786 def share_network_get_all(context):
 3787     return _network_get_query(context).all()
 3788 
 3789 
 3790 @require_context
 3791 def share_network_get_all_by_project(context, project_id):
 3792     return _network_get_query(context).filter_by(project_id=project_id).all()
 3793 
 3794 
 3795 @require_context
 3796 def share_network_get_all_by_security_service(context, security_service_id):
 3797     session = get_session()
 3798     return (model_query(context, models.ShareNetwork, session=session).
 3799             join(models.ShareNetworkSecurityServiceAssociation,
 3800             models.ShareNetwork.id ==
 3801             models.ShareNetworkSecurityServiceAssociation.share_network_id).
 3802             filter_by(security_service_id=security_service_id, deleted=0)
 3803             .all())
 3804 
 3805 
 3806 @require_context
 3807 def share_network_add_security_service(context, id, security_service_id):
 3808     session = get_session()
 3809 
 3810     with session.begin():
 3811         assoc_ref = (model_query(
 3812                      context,
 3813                      models.ShareNetworkSecurityServiceAssociation,
 3814                      session=session).
 3815                      filter_by(share_network_id=id).
 3816                      filter_by(
 3817                      security_service_id=security_service_id).first())
 3818 
 3819         if assoc_ref:
 3820             msg = "Already associated"
 3821             raise exception.ShareNetworkSecurityServiceAssociationError(
 3822                 share_network_id=id,
 3823                 security_service_id=security_service_id,
 3824                 reason=msg)
 3825 
 3826         share_nw_ref = share_network_get(context, id, session=session)
 3827         security_service_ref = security_service_get(context,
 3828                                                     security_service_id,
 3829                                                     session=session)
 3830         share_nw_ref.security_services += [security_service_ref]
 3831         share_nw_ref.save(session=session)
 3832 
 3833     return share_nw_ref
 3834 
 3835 
 3836 @require_context
 3837 def share_network_remove_security_service(context, id, security_service_id):
 3838     session = get_session()
 3839 
 3840     with session.begin():
 3841         share_nw_ref = share_network_get(context, id, session=session)
 3842         security_service_get(context, security_service_id, session=session)
 3843 
 3844         assoc_ref = (model_query(
 3845             context,
 3846             models.ShareNetworkSecurityServiceAssociation,
 3847             session=session).
 3848             filter_by(share_network_id=id).
 3849             filter_by(security_service_id=security_service_id).first())
 3850 
 3851         if assoc_ref:
 3852             assoc_ref.soft_delete(session)
 3853         else:
 3854             msg = "No association defined"
 3855             raise exception.ShareNetworkSecurityServiceDissociationError(
 3856                 share_network_id=id,
 3857                 security_service_id=security_service_id,
 3858                 reason=msg)
 3859 
 3860     return share_nw_ref
 3861 
 3862 
 3863 @require_context
 3864 def count_share_networks(context, project_id, user_id=None,
 3865                          share_type_id=None, session=None):
 3866     query = model_query(
 3867         context, models.ShareNetwork,
 3868         func.count(models.ShareNetwork.id),
 3869         read_deleted="no",
 3870         session=session).filter_by(project_id=project_id)
 3871     if share_type_id:
 3872         query = query.join("share_instances").filter_by(
 3873             share_type_id=share_type_id)
 3874     elif user_id is not None:
 3875         query = query.filter_by(user_id=user_id)
 3876     return query.first()[0]
 3877 
 3878 
 3879 ###################
 3880 
 3881 
 3882 @require_context
 3883 def _network_subnet_get_query(context, session=None):
 3884     if session is None:
 3885         session = get_session()
 3886     return (model_query(context, models.ShareNetworkSubnet, session=session).
 3887             options(joinedload('share_servers'), joinedload('share_network')))
 3888 
 3889 
 3890 @require_context
 3891 def share_network_subnet_create(context, values):
 3892     values = ensure_model_dict_has_id(values)
 3893 
 3894     network_subnet_ref = models.ShareNetworkSubnet()
 3895     network_subnet_ref.update(values)
 3896     session = get_session()
 3897     with session.begin():
 3898         network_subnet_ref.save(session=session)
 3899         return share_network_subnet_get(
 3900             context, network_subnet_ref['id'], session=session)
 3901 
 3902 
 3903 @require_context
 3904 def share_network_subnet_delete(context, network_subnet_id):
 3905 
 3906     session = get_session()
 3907     with session.begin():
 3908         network_subnet_ref = share_network_subnet_get(context,
 3909                                                       network_subnet_id,
 3910                                                       session=session)
 3911         network_subnet_ref.soft_delete(session=session, update_status=True)
 3912 
 3913 
 3914 @require_context
 3915 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 3916 def share_network_subnet_update(context, network_subnet_id, values):
 3917     session = get_session()
 3918     with session.begin():
 3919         network_subnet_ref = share_network_subnet_get(context,
 3920                                                       network_subnet_id,
 3921                                                       session=session)
 3922         network_subnet_ref.update(values)
 3923         network_subnet_ref.save(session=session)
 3924         return network_subnet_ref
 3925 
 3926 
 3927 @require_context
 3928 def share_network_subnet_get(context, network_subnet_id, session=None):
 3929     result = (_network_subnet_get_query(context, session)
 3930               .filter_by(id=network_subnet_id)
 3931               .first())
 3932     if result is None:
 3933         raise exception.ShareNetworkSubnetNotFound(
 3934             share_network_subnet_id=network_subnet_id)
 3935     return result
 3936 
 3937 
 3938 @require_context
 3939 def share_network_subnet_get_all(context):
 3940     return _network_subnet_get_query(context).all()
 3941 
 3942 
 3943 @require_context
 3944 def share_network_subnet_get_all_by_share_network(context, network_id):
 3945     return _network_subnet_get_query(context).filter_by(
 3946         share_network_id=network_id).all()
 3947 
 3948 
 3949 @require_context
 3950 def share_network_subnet_get_by_availability_zone_id(
 3951         context, share_network_id, availability_zone_id):
 3952     result = (_network_subnet_get_query(context).filter_by(
 3953         share_network_id=share_network_id,
 3954         availability_zone_id=availability_zone_id).first())
 3955     # If a specific subnet wasn't found, try get the default one
 3956     if availability_zone_id and not result:
 3957         return (_network_subnet_get_query(context).filter_by(
 3958             share_network_id=share_network_id,
 3959             availability_zone_id=None).first())
 3960     return result
 3961 
 3962 
 3963 @require_context
 3964 def share_network_subnet_get_default_subnet(context, share_network_id):
 3965     return share_network_subnet_get_by_availability_zone_id(
 3966         context, share_network_id, availability_zone_id=None)
 3967 
 3968 
 3969 ###################
 3970 
 3971 
 3972 def _server_get_query(context, session=None):
 3973     if session is None:
 3974         session = get_session()
 3975     return (model_query(context, models.ShareServer, session=session).
 3976             options(joinedload('share_instances'),
 3977                     joinedload('network_allocations'),
 3978                     joinedload('share_network_subnet')))
 3979 
 3980 
 3981 @require_context
 3982 def share_server_create(context, values):
 3983     values = ensure_model_dict_has_id(values)
 3984 
 3985     server_ref = models.ShareServer()
 3986     server_ref.update(values)
 3987     session = get_session()
 3988     with session.begin():
 3989         server_ref.save(session=session)
 3990         # NOTE(u_glide): Do so to prevent errors with relationships
 3991         return share_server_get(context, server_ref['id'], session=session)
 3992 
 3993 
 3994 @require_context
 3995 def share_server_delete(context, id):
 3996     session = get_session()
 3997     with session.begin():
 3998         server_ref = share_server_get(context, id, session=session)
 3999         share_server_backend_details_delete(context, id, session=session)
 4000         server_ref.soft_delete(session=session, update_status=True)
 4001 
 4002 
 4003 @require_context
 4004 def share_server_update(context, id, values):
 4005     session = get_session()
 4006     with session.begin():
 4007         server_ref = share_server_get(context, id, session=session)
 4008         server_ref.update(values)
 4009         server_ref.save(session=session)
 4010         return server_ref
 4011 
 4012 
 4013 @require_context
 4014 def share_server_get(context, server_id, session=None):
 4015     result = (_server_get_query(context, session).filter_by(id=server_id)
 4016               .first())
 4017     if result is None:
 4018         raise exception.ShareServerNotFound(share_server_id=server_id)
 4019     return result
 4020 
 4021 
 4022 @require_context
 4023 def share_server_search_by_identifier(context, identifier, session=None):
 4024 
 4025     identifier_field = models.ShareServer.identifier
 4026 
 4027     # try if given identifier is a substring of existing entry's identifier
 4028     result = (_server_get_query(context, session).filter(
 4029         identifier_field.like('%{}%'.format(identifier))).all())
 4030 
 4031     if not result:
 4032         # repeat it with underscores instead of hyphens
 4033         result = (_server_get_query(context, session).filter(
 4034             identifier_field.like('%{}%'.format(
 4035                 identifier.replace("-", "_")))).all())
 4036 
 4037     if not result:
 4038         # repeat it with hypens instead of underscores
 4039         result = (_server_get_query(context, session).filter(
 4040             identifier_field.like('%{}%'.format(
 4041                 identifier.replace("_", "-")))).all())
 4042 
 4043     if not result:
 4044         # try if an existing identifier is a substring of given identifier
 4045         result = (_server_get_query(context, session).filter(
 4046             literal(identifier).contains(identifier_field)).all())
 4047 
 4048     if not result:
 4049         # repeat it with underscores instead of hyphens
 4050         result = (_server_get_query(context, session).filter(
 4051             literal(identifier.replace("-", "_")).contains(
 4052                 identifier_field)).all())
 4053 
 4054     if not result:
 4055         # repeat it with hypens instead of underscores
 4056         result = (_server_get_query(context, session).filter(
 4057             literal(identifier.replace("_", "-")).contains(
 4058                 identifier_field)).all())
 4059 
 4060     if not result:
 4061         raise exception.ShareServerNotFound(share_server_id=identifier)
 4062 
 4063     return result
 4064 
 4065 
 4066 @require_context
 4067 def share_server_get_all_by_host_and_share_subnet_valid(context, host,
 4068                                                         share_subnet_id,
 4069                                                         session=None):
 4070     result = (_server_get_query(context, session).filter_by(host=host)
 4071               .filter_by(share_network_subnet_id=share_subnet_id)
 4072               .filter(models.ShareServer.status.in_(
 4073                       (constants.STATUS_CREATING,
 4074                        constants.STATUS_ACTIVE))).all())
 4075     if not result:
 4076         filters_description = ('share_network_subnet_id is "%(share_net_id)s",'
 4077                                ' host is "%(host)s" and status in'
 4078                                ' "%(status_cr)s" or "%(status_act)s"') % {
 4079             'share_net_id': share_subnet_id,
 4080             'host': host,
 4081             'status_cr': constants.STATUS_CREATING,
 4082             'status_act': constants.STATUS_ACTIVE,
 4083         }
 4084         raise exception.ShareServerNotFoundByFilters(
 4085             filters_description=filters_description)
 4086     return result
 4087 
 4088 
 4089 @require_context
 4090 def share_server_get_all(context):
 4091     return _server_get_query(context).all()
 4092 
 4093 
 4094 @require_context
 4095 def share_server_get_all_with_filters(context, filters):
 4096 
 4097     query = _server_get_query(context)
 4098 
 4099     if filters.get('host'):
 4100         query = query.filter_by(host=filters.get('host'))
 4101     if filters.get('status'):
 4102         query = query.filter_by(status=filters.get('status'))
 4103     if filters.get('source_share_server_id'):
 4104         query = query.filter_by(
 4105             source_share_server_id=filters.get('source_share_server_id'))
 4106 
 4107     return query.all()
 4108 
 4109 
 4110 @require_context
 4111 def share_server_get_all_by_host(context, host, filters=None):
 4112     if filters:
 4113         filters.update({'host': host})
 4114     else:
 4115         filters = {'host': host}
 4116     return share_server_get_all_with_filters(context, filters=filters)
 4117 
 4118 
 4119 @require_context
 4120 def share_server_get_all_unused_deletable(context, host, updated_before):
 4121     valid_server_status = (
 4122         constants.STATUS_INACTIVE,
 4123         constants.STATUS_ACTIVE,
 4124         constants.STATUS_ERROR,
 4125     )
 4126     result = (_server_get_query(context)
 4127               .filter_by(is_auto_deletable=True)
 4128               .filter_by(host=host)
 4129               .filter(~models.ShareServer.share_groups.any())
 4130               .filter(~models.ShareServer.share_instances.any())
 4131               .filter(models.ShareServer.status.in_(valid_server_status))
 4132               .filter(models.ShareServer.updated_at < updated_before).all())
 4133     return result
 4134 
 4135 
 4136 @require_context
 4137 def share_server_backend_details_set(context, share_server_id, server_details):
 4138     share_server_get(context, share_server_id)
 4139 
 4140     for meta_key, meta_value in server_details.items():
 4141         meta_ref = models.ShareServerBackendDetails()
 4142         meta_ref.update({
 4143             'key': meta_key,
 4144             'value': meta_value,
 4145             'share_server_id': share_server_id
 4146         })
 4147         session = get_session()
 4148         with session.begin():
 4149             meta_ref.save(session)
 4150     return server_details
 4151 
 4152 
 4153 @require_context
 4154 def share_server_backend_details_delete(context, share_server_id,
 4155                                         session=None):
 4156     if not session:
 4157         session = get_session()
 4158     share_server_details = (model_query(context,
 4159                                         models.ShareServerBackendDetails,
 4160                                         session=session)
 4161                             .filter_by(share_server_id=share_server_id).all())
 4162     for item in share_server_details:
 4163         item.soft_delete(session)
 4164 
 4165 
 4166 ###################
 4167 
 4168 def _driver_private_data_query(session, context, entity_id, key=None,
 4169                                read_deleted=False):
 4170     query = model_query(
 4171         context, models.DriverPrivateData, session=session,
 4172         read_deleted=read_deleted,
 4173     ).filter_by(
 4174         entity_uuid=entity_id,
 4175     )
 4176 
 4177     if isinstance(key, list):
 4178         return query.filter(models.DriverPrivateData.key.in_(key))
 4179     elif key is not None:
 4180         return query.filter_by(key=key)
 4181 
 4182     return query
 4183 
 4184 
 4185 @require_context
 4186 def driver_private_data_get(context, entity_id, key=None,
 4187                             default=None, session=None):
 4188     if not session:
 4189         session = get_session()
 4190 
 4191     query = _driver_private_data_query(session, context, entity_id, key)
 4192 
 4193     if key is None or isinstance(key, list):
 4194         return {item.key: item.value for item in query.all()}
 4195     else:
 4196         result = query.first()
 4197         return result["value"] if result is not None else default
 4198 
 4199 
 4200 @require_context
 4201 def driver_private_data_update(context, entity_id, details,
 4202                                delete_existing=False, session=None):
 4203     # NOTE(u_glide): following code modifies details dict, that's why we should
 4204     # copy it
 4205     new_details = copy.deepcopy(details)
 4206 
 4207     if not session:
 4208         session = get_session()
 4209 
 4210     with session.begin():
 4211         # Process existing data
 4212         original_data = session.query(models.DriverPrivateData).filter_by(
 4213             entity_uuid=entity_id).all()
 4214 
 4215         for data_ref in original_data:
 4216             in_new_details = data_ref['key'] in new_details
 4217 
 4218             if in_new_details:
 4219                 new_value = six.text_type(new_details.pop(data_ref['key']))
 4220                 data_ref.update({
 4221                     "value": new_value,
 4222                     "deleted": 0,
 4223                     "deleted_at": None
 4224                 })
 4225                 data_ref.save(session=session)
 4226             elif delete_existing and data_ref['deleted'] != 1:
 4227                 data_ref.update({
 4228                     "deleted": 1, "deleted_at": timeutils.utcnow()
 4229                 })
 4230                 data_ref.save(session=session)
 4231 
 4232         # Add new data
 4233         for key, value in new_details.items():
 4234             data_ref = models.DriverPrivateData()
 4235             data_ref.update({
 4236                 "entity_uuid": entity_id,
 4237                 "key": key,
 4238                 "value": six.text_type(value)
 4239             })
 4240             data_ref.save(session=session)
 4241 
 4242         return details
 4243 
 4244 
 4245 @require_context
 4246 def driver_private_data_delete(context, entity_id, key=None,
 4247                                session=None):
 4248     if not session:
 4249         session = get_session()
 4250 
 4251     with session.begin():
 4252         query = _driver_private_data_query(session, context,
 4253                                            entity_id, key)
 4254         query.update({"deleted": 1, "deleted_at": timeutils.utcnow()})
 4255 
 4256 
 4257 ###################
 4258 
 4259 
 4260 @require_context
 4261 def network_allocation_create(context, values):
 4262     values = ensure_model_dict_has_id(values)
 4263     alloc_ref = models.NetworkAllocation()
 4264     alloc_ref.update(values)
 4265     session = get_session()
 4266     with session.begin():
 4267         alloc_ref.save(session=session)
 4268     return alloc_ref
 4269 
 4270 
 4271 @require_context
 4272 def network_allocation_delete(context, id):
 4273     session = get_session()
 4274     with session.begin():
 4275         alloc_ref = network_allocation_get(context, id, session=session)
 4276         alloc_ref.soft_delete(session)
 4277 
 4278 
 4279 @require_context
 4280 def network_allocation_get(context, id, session=None, read_deleted="no"):
 4281     if session is None:
 4282         session = get_session()
 4283     result = (model_query(context, models.NetworkAllocation, session=session,
 4284                           read_deleted=read_deleted).
 4285               filter_by(id=id).first())
 4286     if result is None:
 4287         raise exception.NotFound()
 4288     return result
 4289 
 4290 
 4291 @require_context
 4292 def network_allocations_get_by_ip_address(context, ip_address):
 4293     session = get_session()
 4294     result = (model_query(context, models.NetworkAllocation, session=session).
 4295               filter_by(ip_address=ip_address).all())
 4296     return result or []
 4297 
 4298 
 4299 @require_context
 4300 def network_allocations_get_for_share_server(context, share_server_id,
 4301                                              session=None, label=None):
 4302     if session is None:
 4303         session = get_session()
 4304 
 4305     query = model_query(
 4306         context, models.NetworkAllocation, session=session,
 4307     ).filter_by(
 4308         share_server_id=share_server_id,
 4309     )
 4310     if label:
 4311         if label != 'admin':
 4312             query = query.filter(or_(
 4313                 # NOTE(vponomaryov): we treat None as alias for 'user'.
 4314                 models.NetworkAllocation.label == None,  # noqa
 4315                 models.NetworkAllocation.label == label,
 4316             ))
 4317         else:
 4318             query = query.filter(models.NetworkAllocation.label == label)
 4319 
 4320     result = query.all()
 4321     return result
 4322 
 4323 
 4324 @require_context
 4325 def network_allocation_update(context, id, values, read_deleted=None):
 4326     session = get_session()
 4327     with session.begin():
 4328         alloc_ref = network_allocation_get(context, id, session=session,
 4329                                            read_deleted=read_deleted)
 4330         alloc_ref.update(values)
 4331         alloc_ref.save(session=session)
 4332         return alloc_ref
 4333 
 4334 
 4335 ###################
 4336 
 4337 
 4338 def _dict_with_specs(inst_type_query, specs_key='extra_specs'):
 4339     """Convert type query result to dict with extra_spec and rate_limit.
 4340 
 4341     Takes a share [group] type query returned by sqlalchemy and returns it
 4342     as a dictionary, converting the extra/group specs entry from a list
 4343     of dicts:
 4344 
 4345     'extra_specs' : [{'key': 'k1', 'value': 'v1', ...}, ...]
 4346     'group_specs' : [{'key': 'k1', 'value': 'v1', ...}, ...]
 4347     to a single dict:
 4348     'extra_specs' : {'k1': 'v1'}
 4349     'group_specs' : {'k1': 'v1'}
 4350     """
 4351     inst_type_dict = dict(inst_type_query)
 4352     specs = {x['key']: x['value'] for x in inst_type_query[specs_key]}
 4353     inst_type_dict[specs_key] = specs
 4354     return inst_type_dict
 4355 
 4356 
 4357 @require_admin_context
 4358 def share_type_create(context, values, projects=None):
 4359     """Create a new share type.
 4360 
 4361     In order to pass in extra specs, the values dict should contain a
 4362     'extra_specs' key/value pair:
 4363     {'extra_specs' : {'k1': 'v1', 'k2': 'v2', ...}}
 4364     """
 4365     values = ensure_model_dict_has_id(values)
 4366 
 4367     projects = projects or []
 4368 
 4369     session = get_session()
 4370     with session.begin():
 4371         try:
 4372             values['extra_specs'] = _metadata_refs(values.get('extra_specs'),
 4373                                                    models.ShareTypeExtraSpecs)
 4374             share_type_ref = models.ShareTypes()
 4375             share_type_ref.update(values)
 4376             share_type_ref.save(session=session)
 4377         except db_exception.DBDuplicateEntry:
 4378             raise exception.ShareTypeExists(id=values['name'])
 4379         except Exception as e:
 4380             raise db_exception.DBError(e)
 4381 
 4382         for project in set(projects):
 4383             access_ref = models.ShareTypeProjects()
 4384             access_ref.update({"share_type_id": share_type_ref.id,
 4385                                "project_id": project})
 4386             access_ref.save(session=session)
 4387 
 4388         return share_type_ref
 4389 
 4390 
 4391 def _share_type_get_query(context, session=None, read_deleted=None,
 4392                           expected_fields=None):
 4393     expected_fields = expected_fields or []
 4394     query = (model_query(context,
 4395                          models.ShareTypes,
 4396                          session=session,
 4397                          read_deleted=read_deleted).
 4398              options(joinedload('extra_specs')))
 4399 
 4400     if 'projects' in expected_fields:
 4401         query = query.options(joinedload('projects'))
 4402 
 4403     if not context.is_admin:
 4404         the_filter = [models.ShareTypes.is_public == true()]
 4405         projects_attr = getattr(models.ShareTypes, 'projects')
 4406         the_filter.extend([
 4407             projects_attr.any(project_id=context.project_id)
 4408         ])
 4409         query = query.filter(or_(*the_filter))
 4410 
 4411     return query
 4412 
 4413 
 4414 @handle_db_data_error
 4415 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 4416 def _type_update(context, type_id, values, is_group):
 4417 
 4418     if values.get('name') is None:
 4419         values.pop('name', None)
 4420 
 4421     if is_group:
 4422         model = models.ShareGroupTypes
 4423         exists_exc = exception.ShareGroupTypeExists
 4424         exists_args = {'type_id': values.get('name')}
 4425     else:
 4426         model = models.ShareTypes
 4427         exists_exc = exception.ShareTypeExists
 4428         exists_args = {'id': values.get('name')}
 4429 
 4430     session = get_session()
 4431     with session.begin():
 4432         query = model_query(context, model, session=session)
 4433 
 4434         try:
 4435             result = query.filter_by(id=type_id).update(values)
 4436         except db_exception.DBDuplicateEntry:
 4437             # This exception only occurs if there's a non-deleted
 4438             # share/group type which has the same name as the name being
 4439             # updated.
 4440             raise exists_exc(**exists_args)
 4441 
 4442         if not result:
 4443             if is_group:
 4444                 raise exception.ShareGroupTypeNotFound(type_id=type_id)
 4445             else:
 4446                 raise exception.ShareTypeNotFound(share_type_id=type_id)
 4447 
 4448 
 4449 def share_type_update(context, share_type_id, values):
 4450     _type_update(context, share_type_id, values, is_group=False)
 4451 
 4452 
 4453 @require_context
 4454 def share_type_get_all(context, inactive=False, filters=None):
 4455     """Returns a dict describing all share_types with name as key."""
 4456     filters = filters or {}
 4457 
 4458     read_deleted = "yes" if inactive else "no"
 4459 
 4460     query = _share_type_get_query(context, read_deleted=read_deleted)
 4461 
 4462     if 'is_public' in filters and filters['is_public'] is not None:
 4463         the_filter = [models. ShareTypes.is_public == filters['is_public']]
 4464         if filters['is_public'] and context.project_id is not None:
 4465             projects_attr = getattr(models. ShareTypes, 'projects')
 4466             the_filter.extend([
 4467                 projects_attr.any(
 4468                     project_id=context.project_id, deleted=0)
 4469             ])
 4470         if len(the_filter) > 1:
 4471             query = query.filter(or_(*the_filter))
 4472         else:
 4473             query = query.filter(the_filter[0])
 4474 
 4475     rows = query.order_by("name").all()
 4476 
 4477     result = {}
 4478     for row in rows:
 4479         result[row['name']] = _dict_with_specs(row)
 4480 
 4481     return result
 4482 
 4483 
 4484 def _share_type_get_id_from_share_type_query(context, id, session=None):
 4485     return (model_query(
 4486             context, models.ShareTypes, read_deleted="no", session=session).
 4487             filter_by(id=id))
 4488 
 4489 
 4490 def _share_type_get_id_from_share_type(context, id, session=None):
 4491     result = _share_type_get_id_from_share_type_query(
 4492         context, id, session=session).first()
 4493     if not result:
 4494         raise exception.ShareTypeNotFound(share_type_id=id)
 4495     return result['id']
 4496 
 4497 
 4498 def _share_type_get(context, id, session=None, inactive=False,
 4499                     expected_fields=None):
 4500     expected_fields = expected_fields or []
 4501     read_deleted = "yes" if inactive else "no"
 4502     result = (_share_type_get_query(
 4503               context, session, read_deleted, expected_fields).
 4504               filter_by(id=id).
 4505               first())
 4506 
 4507     if not result:
 4508         # The only way that id could be None is if the default share type is
 4509         # not configured and no other share type was specified.
 4510         if id is None:
 4511             raise exception.DefaultShareTypeNotConfigured()
 4512         raise exception.ShareTypeNotFound(share_type_id=id)
 4513 
 4514     share_type = _dict_with_specs(result)
 4515 
 4516     if 'projects' in expected_fields:
 4517         share_type['projects'] = [p['project_id'] for p in result['projects']]
 4518 
 4519     return share_type
 4520 
 4521 
 4522 @require_context
 4523 def share_type_get(context, id, inactive=False, expected_fields=None):
 4524     """Return a dict describing specific share_type."""
 4525     return _share_type_get(context, id,
 4526                            session=None,
 4527                            inactive=inactive,
 4528                            expected_fields=expected_fields)
 4529 
 4530 
 4531 def _share_type_get_by_name(context, name, session=None):
 4532     result = (_share_type_get_query(context, session=session).
 4533               filter_by(name=name).
 4534               first())
 4535 
 4536     if not result:
 4537         raise exception.ShareTypeNotFoundByName(share_type_name=name)
 4538 
 4539     return _dict_with_specs(result)
 4540 
 4541 
 4542 @require_context
 4543 def share_type_get_by_name(context, name):
 4544     """Return a dict describing specific share_type."""
 4545     return _share_type_get_by_name(context, name)
 4546 
 4547 
 4548 @require_context
 4549 def share_type_get_by_name_or_id(context, name_or_id):
 4550     """Return a dict describing specific share_type using its name or ID.
 4551 
 4552     :returns: ShareType object or None if not found
 4553     """
 4554     try:
 4555         return _share_type_get(context, name_or_id)
 4556     except exception.ShareTypeNotFound:
 4557         try:
 4558             return _share_type_get_by_name(context, name_or_id)
 4559         except exception.ShareTypeNotFoundByName:
 4560             return None
 4561 
 4562 
 4563 @require_admin_context
 4564 def share_type_destroy(context, id):
 4565     session = get_session()
 4566     with session.begin():
 4567         _share_type_get(context, id, session)
 4568         shares_count = model_query(
 4569             context,
 4570             models.ShareInstance,
 4571             read_deleted="no",
 4572             session=session,
 4573         ).filter_by(share_type_id=id).count()
 4574         share_group_types_count = model_query(
 4575             context,
 4576             models.ShareGroupTypeShareTypeMapping,
 4577             read_deleted="no",
 4578             session=session,
 4579         ).filter_by(share_type_id=id).count()
 4580         if shares_count or share_group_types_count:
 4581             msg = ("Deletion of share type %(stype)s failed; it in use by "
 4582                    "%(shares)d shares and %(gtypes)d share group types")
 4583             msg_args = {'stype': id,
 4584                         'shares': shares_count,
 4585                         'gtypes': share_group_types_count}
 4586             LOG.error(msg, msg_args)
 4587             raise exception.ShareTypeInUse(share_type_id=id)
 4588 
 4589         model_query(
 4590             context, models.ShareTypeExtraSpecs, session=session
 4591         ).filter_by(
 4592             share_type_id=id
 4593         ).soft_delete()
 4594         model_query(
 4595             context, models.ShareTypeProjects, session=session
 4596         ).filter_by(
 4597             share_type_id=id,
 4598         ).soft_delete()
 4599         model_query(
 4600             context, models.ShareTypes, session=session
 4601         ).filter_by(
 4602             id=id
 4603         ).soft_delete()
 4604 
 4605     # Destroy any quotas, usages and reservations for the share type:
 4606     quota_destroy_all_by_share_type(context, id)
 4607 
 4608 
 4609 def _share_type_access_query(context, session=None):
 4610     return model_query(context, models.ShareTypeProjects, session=session,
 4611                        read_deleted="no")
 4612 
 4613 
 4614 @require_admin_context
 4615 def share_type_access_get_all(context, type_id):
 4616     share_type_id = _share_type_get_id_from_share_type(context, type_id)
 4617     return (_share_type_access_query(context).
 4618             filter_by(share_type_id=share_type_id).all())
 4619 
 4620 
 4621 @require_admin_context
 4622 def share_type_access_add(context, type_id, project_id):
 4623     """Add given tenant to the share type access list."""
 4624     share_type_id = _share_type_get_id_from_share_type(context, type_id)
 4625 
 4626     access_ref = models.ShareTypeProjects()
 4627     access_ref.update({"share_type_id": share_type_id,
 4628                        "project_id": project_id})
 4629 
 4630     session = get_session()
 4631     with session.begin():
 4632         try:
 4633             access_ref.save(session=session)
 4634         except db_exception.DBDuplicateEntry:
 4635             raise exception.ShareTypeAccessExists(share_type_id=type_id,
 4636                                                   project_id=project_id)
 4637         return access_ref
 4638 
 4639 
 4640 @require_admin_context
 4641 def share_type_access_remove(context, type_id, project_id):
 4642     """Remove given tenant from the share type access list."""
 4643     share_type_id = _share_type_get_id_from_share_type(context, type_id)
 4644 
 4645     count = (_share_type_access_query(context).
 4646              filter_by(share_type_id=share_type_id).
 4647              filter_by(project_id=project_id).
 4648              soft_delete(synchronize_session=False))
 4649     if count == 0:
 4650         raise exception.ShareTypeAccessNotFound(
 4651             share_type_id=type_id, project_id=project_id)
 4652 
 4653 ####################
 4654 
 4655 
 4656 def _share_type_extra_specs_query(context, share_type_id, session=None):
 4657     return (model_query(context, models.ShareTypeExtraSpecs, session=session,
 4658                         read_deleted="no").
 4659             filter_by(share_type_id=share_type_id).
 4660             options(joinedload('share_type')))
 4661 
 4662 
 4663 @require_context
 4664 def share_type_extra_specs_get(context, share_type_id):
 4665     rows = (_share_type_extra_specs_query(context, share_type_id).
 4666             all())
 4667 
 4668     result = {}
 4669     for row in rows:
 4670         result[row['key']] = row['value']
 4671 
 4672     return result
 4673 
 4674 
 4675 @require_context
 4676 def share_type_extra_specs_delete(context, share_type_id, key):
 4677     session = get_session()
 4678     with session.begin():
 4679         _share_type_extra_specs_get_item(context, share_type_id, key, session)
 4680         (_share_type_extra_specs_query(context, share_type_id, session).
 4681             filter_by(key=key).soft_delete())
 4682 
 4683 
 4684 def _share_type_extra_specs_get_item(context, share_type_id, key,
 4685                                      session=None):
 4686     result = _share_type_extra_specs_query(
 4687         context, share_type_id, session=session
 4688     ).filter_by(key=key).options(joinedload('share_type')).first()
 4689 
 4690     if not result:
 4691         raise exception.ShareTypeExtraSpecsNotFound(
 4692             extra_specs_key=key,
 4693             share_type_id=share_type_id)
 4694 
 4695     return result
 4696 
 4697 
 4698 @require_context
 4699 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 4700 def share_type_extra_specs_update_or_create(context, share_type_id, specs):
 4701     session = get_session()
 4702     with session.begin():
 4703         spec_ref = None
 4704         for key, value in specs.items():
 4705             try:
 4706                 spec_ref = _share_type_extra_specs_get_item(
 4707                     context, share_type_id, key, session)
 4708             except exception.ShareTypeExtraSpecsNotFound:
 4709                 spec_ref = models.ShareTypeExtraSpecs()
 4710             spec_ref.update({"key": key, "value": value,
 4711                              "share_type_id": share_type_id,
 4712                              "deleted": 0})
 4713             spec_ref.save(session=session)
 4714 
 4715         return specs
 4716 
 4717 
 4718 def _ensure_availability_zone_exists(context, values, session, strict=True):
 4719     az_name = values.pop('availability_zone', None)
 4720 
 4721     if strict and not az_name:
 4722         msg = _("Values dict should have 'availability_zone' field.")
 4723         raise ValueError(msg)
 4724     elif not az_name:
 4725         return
 4726 
 4727     if uuidutils.is_uuid_like(az_name):
 4728         az_ref = availability_zone_get(context, az_name, session=session)
 4729     else:
 4730         az_ref = availability_zone_create_if_not_exist(
 4731             context, az_name, session=session)
 4732 
 4733     values.update({'availability_zone_id': az_ref['id']})
 4734 
 4735 
 4736 @require_context
 4737 def availability_zone_get(context, id_or_name, session=None):
 4738     if session is None:
 4739         session = get_session()
 4740 
 4741     query = model_query(context, models.AvailabilityZone, session=session)
 4742 
 4743     if uuidutils.is_uuid_like(id_or_name):
 4744         query = query.filter_by(id=id_or_name)
 4745     else:
 4746         query = query.filter_by(name=id_or_name)
 4747 
 4748     result = query.first()
 4749 
 4750     if not result:
 4751         raise exception.AvailabilityZoneNotFound(id=id_or_name)
 4752 
 4753     return result
 4754 
 4755 
 4756 @require_context
 4757 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 4758 def availability_zone_create_if_not_exist(context, name, session=None):
 4759     if session is None:
 4760         session = get_session()
 4761 
 4762     az = models.AvailabilityZone()
 4763     az.update({'id': uuidutils.generate_uuid(), 'name': name})
 4764     try:
 4765         with session.begin():
 4766             az.save(session)
 4767     # NOTE(u_glide): Do not catch specific exception here, because it depends
 4768     # on concrete backend used by SqlAlchemy
 4769     except Exception:
 4770         return availability_zone_get(context, name, session=session)
 4771     return az
 4772 
 4773 
 4774 @require_context
 4775 def availability_zone_get_all(context):
 4776     session = get_session()
 4777 
 4778     enabled_services = model_query(
 4779         context, models.Service,
 4780         models.Service.availability_zone_id,
 4781         session=session,
 4782         read_deleted="no"
 4783     ).filter_by(disabled=False).distinct()
 4784 
 4785     return model_query(context, models.AvailabilityZone, session=session,
 4786                        read_deleted="no").filter(
 4787         models.AvailabilityZone.id.in_(enabled_services)
 4788     ).all()
 4789 
 4790 
 4791 @require_admin_context
 4792 def purge_deleted_records(context, age_in_days):
 4793     """Purge soft-deleted records older than(and equal) age from tables."""
 4794 
 4795     if age_in_days < 0:
 4796         msg = _('Must supply a non-negative value for "age_in_days".')
 4797         LOG.error(msg)
 4798         raise exception.InvalidParameterValue(msg)
 4799 
 4800     metadata = MetaData()
 4801     metadata.reflect(get_engine())
 4802     session = get_session()
 4803     session.begin()
 4804     deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days)
 4805 
 4806     for table in reversed(metadata.sorted_tables):
 4807         if 'deleted' in table.columns.keys():
 4808             try:
 4809                 mds = [m for m in models.__dict__.values() if
 4810                        (hasattr(m, '__tablename__') and
 4811                         m.__tablename__ == six.text_type(table))]
 4812                 if len(mds) > 0:
 4813                     # collect all soft-deleted records
 4814                     with session.begin_nested():
 4815                         model = mds[0]
 4816                         s_deleted_records = session.query(model).filter(
 4817                             model.deleted_at <= deleted_age)
 4818                     deleted_count = 0
 4819                     # delete records one by one,
 4820                     # skip the records which has FK constraints
 4821                     for record in s_deleted_records:
 4822                         try:
 4823                             with session.begin_nested():
 4824                                 session.delete(record)
 4825                                 deleted_count += 1
 4826                         except db_exc.DBError:
 4827                             LOG.warning(
 4828                                 ("Deleting soft-deleted resource %s "
 4829                                  "failed, skipping."), record)
 4830                     if deleted_count != 0:
 4831                         LOG.info("Deleted %(count)s records in "
 4832                                  "table %(table)s.",
 4833                                  {'count': deleted_count, 'table': table})
 4834             except db_exc.DBError:
 4835                 LOG.warning("Querying table %s's soft-deleted records "
 4836                             "failed, skipping.", table)
 4837     session.commit()
 4838 
 4839 
 4840 ####################
 4841 
 4842 
 4843 def _share_group_get(context, share_group_id, session=None):
 4844     session = session or get_session()
 4845     result = (model_query(context, models.ShareGroup,
 4846                           session=session,
 4847                           project_only=True,
 4848                           read_deleted='no').
 4849               filter_by(id=share_group_id).
 4850               options(joinedload('share_types')).
 4851               first())
 4852 
 4853     if not result:
 4854         raise exception.ShareGroupNotFound(share_group_id=share_group_id)
 4855 
 4856     return result
 4857 
 4858 
 4859 @require_context
 4860 def share_group_get(context, share_group_id, session=None):
 4861     return _share_group_get(context, share_group_id, session=session)
 4862 
 4863 
 4864 def _share_group_get_all(context, project_id=None, share_server_id=None,
 4865                          host=None, detailed=True, filters=None,
 4866                          sort_key=None, sort_dir=None, session=None):
 4867     session = session or get_session()
 4868     sort_key = sort_key or 'created_at'
 4869     sort_dir = sort_dir or 'desc'
 4870 
 4871     query = model_query(
 4872         context, models.ShareGroup, session=session, read_deleted='no')
 4873 
 4874     # Apply filters
 4875     if not filters:
 4876         filters = {}
 4877     no_key = 'key_is_absent'
 4878     for k, v in filters.items():
 4879         temp_k = k.rstrip('~') if k in constants.LIKE_FILTER else k
 4880         filter_attr = getattr(models.ShareGroup, temp_k, no_key)
 4881 
 4882         if filter_attr == no_key:
 4883             msg = _("Share groups cannot be filtered using '%s' key.")
 4884             raise exception.InvalidInput(reason=msg % k)
 4885 
 4886         if k in constants.LIKE_FILTER:
 4887             query = query.filter(filter_attr.op('LIKE')(u'%' + v + u'%'))
 4888         else:
 4889             query = query.filter(filter_attr == v)
 4890 
 4891     if project_id:
 4892         query = query.filter(
 4893             models.ShareGroup.project_id == project_id)
 4894     if host:
 4895         query = query.filter(
 4896             models.ShareGroup.host == host)
 4897     if share_server_id:
 4898         query = query.filter(
 4899             models.ShareGroup.share_server_id == share_server_id)
 4900 
 4901     try:
 4902         query = apply_sorting(models.ShareGroup, query, sort_key, sort_dir)
 4903     except AttributeError:
 4904         msg = _("Wrong sorting key provided - '%s'.") % sort_key
 4905         raise exception.InvalidInput(reason=msg)
 4906 
 4907     if detailed:
 4908         return query.options(joinedload('share_types')).all()
 4909     else:
 4910         query = query.with_entities(
 4911             models.ShareGroup.id, models.ShareGroup.name)
 4912         values = []
 4913         for sg_id, sg_name in query.all():
 4914             values.append({"id": sg_id, "name": sg_name})
 4915         return values
 4916 
 4917 
 4918 @require_admin_context
 4919 def share_group_get_all(context, detailed=True, filters=None, sort_key=None,
 4920                         sort_dir=None):
 4921     return _share_group_get_all(
 4922         context, detailed=detailed, filters=filters,
 4923         sort_key=sort_key, sort_dir=sort_dir)
 4924 
 4925 
 4926 @require_admin_context
 4927 def share_group_get_all_by_host(context, host, detailed=True):
 4928     return _share_group_get_all(context, host=host, detailed=detailed)
 4929 
 4930 
 4931 @require_context
 4932 def share_group_get_all_by_project(context, project_id, detailed=True,
 4933                                    filters=None, sort_key=None, sort_dir=None):
 4934     authorize_project_context(context, project_id)
 4935     return _share_group_get_all(
 4936         context, project_id=project_id, detailed=detailed, filters=filters,
 4937         sort_key=sort_key, sort_dir=sort_dir)
 4938 
 4939 
 4940 @require_context
 4941 def share_group_get_all_by_share_server(context, share_server_id, filters=None,
 4942                                         sort_key=None, sort_dir=None):
 4943     return _share_group_get_all(
 4944         context, share_server_id=share_server_id, filters=filters,
 4945         sort_key=sort_key, sort_dir=sort_dir)
 4946 
 4947 
 4948 @require_context
 4949 def share_group_create(context, values):
 4950     share_group = models.ShareGroup()
 4951     if not values.get('id'):
 4952         values['id'] = six.text_type(uuidutils.generate_uuid())
 4953 
 4954     mappings = []
 4955     for item in values.get('share_types') or []:
 4956         mapping = models.ShareGroupShareTypeMapping()
 4957         mapping['id'] = six.text_type(uuidutils.generate_uuid())
 4958         mapping['share_type_id'] = item
 4959         mapping['share_group_id'] = values['id']
 4960         mappings.append(mapping)
 4961 
 4962     values['share_types'] = mappings
 4963 
 4964     session = get_session()
 4965     with session.begin():
 4966         share_group.update(values)
 4967         session.add(share_group)
 4968 
 4969         return _share_group_get(context, values['id'], session=session)
 4970 
 4971 
 4972 @require_context
 4973 def share_group_update(context, share_group_id, values):
 4974     session = get_session()
 4975     with session.begin():
 4976         share_group_ref = _share_group_get(
 4977             context, share_group_id, session=session)
 4978         share_group_ref.update(values)
 4979         share_group_ref.save(session=session)
 4980         return share_group_ref
 4981 
 4982 
 4983 @require_admin_context
 4984 def share_group_destroy(context, share_group_id):
 4985     session = get_session()
 4986     with session.begin():
 4987         share_group_ref = _share_group_get(
 4988             context, share_group_id, session=session)
 4989         share_group_ref.soft_delete(session)
 4990         session.query(models.ShareGroupShareTypeMapping).filter_by(
 4991             share_group_id=share_group_ref['id']).soft_delete()
 4992 
 4993 
 4994 @require_context
 4995 def count_shares_in_share_group(context, share_group_id, session=None):
 4996     session = session or get_session()
 4997     return (model_query(context, models.Share, session=session,
 4998                         project_only=True, read_deleted="no").
 4999             filter_by(share_group_id=share_group_id).
 5000             count())
 5001 
 5002 
 5003 @require_context
 5004 def get_all_shares_by_share_group(context, share_group_id, session=None):
 5005     session = session or get_session()
 5006     return (model_query(
 5007             context, models.Share, session=session,
 5008             project_only=True, read_deleted="no").
 5009             filter_by(share_group_id=share_group_id).
 5010             all())
 5011 
 5012 
 5013 @require_context
 5014 def count_share_groups(context, project_id, user_id=None,
 5015                        share_type_id=None, session=None):
 5016     query = model_query(
 5017         context, models.ShareGroup,
 5018         func.count(models.ShareGroup.id),
 5019         read_deleted="no",
 5020         session=session).filter_by(project_id=project_id)
 5021     if share_type_id:
 5022         query = query.join("share_group_share_type_mappings").filter_by(
 5023             share_type_id=share_type_id)
 5024     elif user_id is not None:
 5025         query = query.filter_by(user_id=user_id)
 5026     return query.first()[0]
 5027 
 5028 
 5029 @require_context
 5030 def count_share_group_snapshots(context, project_id, user_id=None,
 5031                                 share_type_id=None, session=None):
 5032     query = model_query(
 5033         context, models.ShareGroupSnapshot,
 5034         func.count(models.ShareGroupSnapshot.id),
 5035         read_deleted="no",
 5036         session=session).filter_by(project_id=project_id)
 5037     if share_type_id:
 5038         query = query.join(
 5039             "share_group"
 5040         ).join(
 5041             "share_group_share_type_mappings"
 5042         ).filter_by(share_type_id=share_type_id)
 5043     elif user_id is not None:
 5044         query = query.filter_by(user_id=user_id)
 5045     return query.first()[0]
 5046 
 5047 
 5048 @require_context
 5049 def share_replica_data_get_for_project(context, project_id, user_id=None,
 5050                                        session=None, share_type_id=None):
 5051     session = session or get_session()
 5052     query = model_query(
 5053         context, models.ShareInstance,
 5054         func.count(models.ShareInstance.id),
 5055         func.sum(models.Share.size),
 5056         read_deleted="no",
 5057         session=session).join(
 5058         models.Share,
 5059         models.ShareInstance.share_id == models.Share.id).filter(
 5060         models.Share.project_id == project_id).filter(
 5061         models.ShareInstance.replica_state.isnot(None))
 5062 
 5063     if share_type_id:
 5064         query = query.filter(
 5065             models.ShareInstance.share_type_id == share_type_id)
 5066     elif user_id:
 5067         query = query.filter(models.Share.user_id == user_id)
 5068 
 5069     result = query.first()
 5070     return result[0] or 0, result[1] or 0
 5071 
 5072 
 5073 @require_context
 5074 def count_share_group_snapshots_in_share_group(context, share_group_id,
 5075                                                session=None):
 5076     session = session or get_session()
 5077     return model_query(
 5078         context, models.ShareGroupSnapshot, session=session,
 5079         project_only=True, read_deleted="no",
 5080     ).filter_by(
 5081         share_group_id=share_group_id,
 5082     ).count()
 5083 
 5084 
 5085 @require_context
 5086 def count_share_groups_in_share_network(context, share_network_id,
 5087                                         session=None):
 5088     session = session or get_session()
 5089     return (model_query(
 5090             context, models.ShareGroup, session=session,
 5091             project_only=True, read_deleted="no").
 5092             filter_by(share_network_id=share_network_id).
 5093             count())
 5094 
 5095 
 5096 @require_context
 5097 def count_share_group_snapshot_members_in_share(context, share_id,
 5098                                                 session=None):
 5099     session = session or get_session()
 5100     return model_query(
 5101         context, models.ShareSnapshotInstance, session=session,
 5102         project_only=True, read_deleted="no",
 5103     ).join(
 5104         models.ShareInstance,
 5105         models.ShareInstance.id == (
 5106             models.ShareSnapshotInstance.share_instance_id),
 5107     ).filter(
 5108         models.ShareInstance.share_id == share_id,
 5109     ).count()
 5110 
 5111 
 5112 @require_context
 5113 def _share_group_snapshot_get(context, share_group_snapshot_id, session=None):
 5114     session = session or get_session()
 5115     result = model_query(
 5116         context, models.ShareGroupSnapshot, session=session,
 5117         project_only=True, read_deleted='no',
 5118     ).options(
 5119         joinedload('share_group'),
 5120         joinedload('share_group_snapshot_members'),
 5121     ).filter_by(
 5122         id=share_group_snapshot_id,
 5123     ).first()
 5124 
 5125     if not result:
 5126         raise exception.ShareGroupSnapshotNotFound(
 5127             share_group_snapshot_id=share_group_snapshot_id)
 5128 
 5129     return result
 5130 
 5131 
 5132 def _share_group_snapshot_get_all(
 5133         context, project_id=None, detailed=True, filters=None,
 5134         sort_key=None, sort_dir=None, session=None):
 5135     session = session or get_session()
 5136     if not sort_key:
 5137         sort_key = 'created_at'
 5138     if not sort_dir:
 5139         sort_dir = 'desc'
 5140 
 5141     query = model_query(
 5142         context, models.ShareGroupSnapshot, session=session, read_deleted='no',
 5143     ).options(
 5144         joinedload('share_group'),
 5145         joinedload('share_group_snapshot_members'),
 5146     )
 5147 
 5148     # Apply filters
 5149     if not filters:
 5150         filters = {}
 5151     no_key = 'key_is_absent'
 5152     for k, v in filters.items():
 5153         filter_attr = getattr(models.ShareGroupSnapshot, k, no_key)
 5154         if filter_attr == no_key:
 5155             msg = _("Share group snapshots cannot be filtered using '%s' key.")
 5156             raise exception.InvalidInput(reason=msg % k)
 5157         query = query.filter(filter_attr == v)
 5158 
 5159     if project_id:
 5160         query = query.filter(
 5161             models.ShareGroupSnapshot.project_id == project_id)
 5162 
 5163     try:
 5164         query = apply_sorting(
 5165             models.ShareGroupSnapshot, query, sort_key, sort_dir)
 5166     except AttributeError:
 5167         msg = _("Wrong sorting key provided - '%s'.") % sort_key
 5168         raise exception.InvalidInput(reason=msg)
 5169 
 5170     if detailed:
 5171         return query.all()
 5172     else:
 5173         query = query.with_entities(models.ShareGroupSnapshot.id,
 5174                                     models.ShareGroupSnapshot.name)
 5175         values = []
 5176         for sgs_id, sgs_name in query.all():
 5177             values.append({"id": sgs_id, "name": sgs_name})
 5178         return values
 5179 
 5180 
 5181 @require_context
 5182 def share_group_snapshot_get(context, share_group_snapshot_id, session=None):
 5183     session = session or get_session()
 5184     return _share_group_snapshot_get(
 5185         context, share_group_snapshot_id, session=session)
 5186 
 5187 
 5188 @require_admin_context
 5189 def share_group_snapshot_get_all(
 5190         context, detailed=True, filters=None, sort_key=None, sort_dir=None):
 5191     return _share_group_snapshot_get_all(
 5192         context, filters=filters, detailed=detailed,
 5193         sort_key=sort_key, sort_dir=sort_dir)
 5194 
 5195 
 5196 @require_context
 5197 def share_group_snapshot_get_all_by_project(
 5198         context, project_id, detailed=True, filters=None,
 5199         sort_key=None, sort_dir=None):
 5200     authorize_project_context(context, project_id)
 5201     return _share_group_snapshot_get_all(
 5202         context, project_id=project_id, filters=filters, detailed=detailed,
 5203         sort_key=sort_key, sort_dir=sort_dir,
 5204     )
 5205 
 5206 
 5207 @require_context
 5208 def share_group_snapshot_create(context, values):
 5209     share_group_snapshot = models.ShareGroupSnapshot()
 5210     if not values.get('id'):
 5211         values['id'] = six.text_type(uuidutils.generate_uuid())
 5212 
 5213     session = get_session()
 5214     with session.begin():
 5215         share_group_snapshot.update(values)
 5216         session.add(share_group_snapshot)
 5217 
 5218         return _share_group_snapshot_get(
 5219             context, values['id'], session=session)
 5220 
 5221 
 5222 @require_context
 5223 def share_group_snapshot_update(context, share_group_snapshot_id, values):
 5224     session = get_session()
 5225     with session.begin():
 5226         share_group_ref = _share_group_snapshot_get(
 5227             context, share_group_snapshot_id, session=session)
 5228         share_group_ref.update(values)
 5229         share_group_ref.save(session=session)
 5230         return share_group_ref
 5231 
 5232 
 5233 @require_admin_context
 5234 def share_group_snapshot_destroy(context, share_group_snapshot_id):
 5235     session = get_session()
 5236     with session.begin():
 5237         share_group_snap_ref = _share_group_snapshot_get(
 5238             context, share_group_snapshot_id, session=session)
 5239         share_group_snap_ref.soft_delete(session)
 5240         session.query(models.ShareSnapshotInstance).filter_by(
 5241             share_group_snapshot_id=share_group_snapshot_id).soft_delete()
 5242 
 5243 
 5244 @require_context
 5245 def share_group_snapshot_members_get_all(context, share_group_snapshot_id,
 5246                                          session=None):
 5247     session = session or get_session()
 5248     query = model_query(
 5249         context, models.ShareSnapshotInstance, session=session,
 5250         read_deleted='no',
 5251     ).filter_by(share_group_snapshot_id=share_group_snapshot_id)
 5252     return query.all()
 5253 
 5254 
 5255 @require_context
 5256 def share_group_snapshot_member_get(context, member_id, session=None):
 5257     result = model_query(
 5258         context, models.ShareSnapshotInstance, session=session,
 5259         project_only=True, read_deleted='no',
 5260     ).filter_by(id=member_id).first()
 5261     if not result:
 5262         raise exception.ShareGroupSnapshotMemberNotFound(member_id=member_id)
 5263     return result
 5264 
 5265 
 5266 @require_context
 5267 def share_group_snapshot_member_create(context, values):
 5268     member = models.ShareSnapshotInstance()
 5269     if not values.get('id'):
 5270         values['id'] = six.text_type(uuidutils.generate_uuid())
 5271 
 5272     _change_size_to_instance_size(values)
 5273 
 5274     session = get_session()
 5275     with session.begin():
 5276         member.update(values)
 5277         session.add(member)
 5278 
 5279         return share_group_snapshot_member_get(
 5280             context, values['id'], session=session)
 5281 
 5282 
 5283 @require_context
 5284 def share_group_snapshot_member_update(context, member_id, values):
 5285     session = get_session()
 5286     _change_size_to_instance_size(values)
 5287     with session.begin():
 5288         member = share_group_snapshot_member_get(
 5289             context, member_id, session=session)
 5290         member.update(values)
 5291         session.add(member)
 5292         return share_group_snapshot_member_get(
 5293             context, member_id, session=session)
 5294 
 5295 
 5296 ####################
 5297 
 5298 
 5299 @require_admin_context
 5300 def share_group_type_create(context, values, projects=None):
 5301     """Create a new share group type.
 5302 
 5303     In order to pass in group specs, the values dict should contain a
 5304     'group_specs' key/value pair:
 5305     {'group_specs' : {'k1': 'v1', 'k2': 'v2', ...}}
 5306     """
 5307     values = ensure_model_dict_has_id(values)
 5308 
 5309     projects = projects or []
 5310 
 5311     session = get_session()
 5312     with session.begin():
 5313         try:
 5314             values['group_specs'] = _metadata_refs(
 5315                 values.get('group_specs'), models.ShareGroupTypeSpecs)
 5316             mappings = []
 5317             for item in values.get('share_types', []):
 5318                 share_type = share_type_get_by_name_or_id(context, item)
 5319                 if not share_type:
 5320                     raise exception.ShareTypeDoesNotExist(share_type=item)
 5321                 mapping = models.ShareGroupTypeShareTypeMapping()
 5322                 mapping['id'] = six.text_type(uuidutils.generate_uuid())
 5323                 mapping['share_type_id'] = share_type['id']
 5324                 mapping['share_group_type_id'] = values['id']
 5325                 mappings.append(mapping)
 5326 
 5327             values['share_types'] = mappings
 5328             share_group_type_ref = models.ShareGroupTypes()
 5329             share_group_type_ref.update(values)
 5330             share_group_type_ref.save(session=session)
 5331         except db_exception.DBDuplicateEntry:
 5332             raise exception.ShareGroupTypeExists(type_id=values['name'])
 5333         except exception.ShareTypeDoesNotExist:
 5334             raise
 5335         except Exception as e:
 5336             raise db_exception.DBError(e)
 5337 
 5338         for project in set(projects):
 5339             access_ref = models.ShareGroupTypeProjects()
 5340             access_ref.update({"share_group_type_id": share_group_type_ref.id,
 5341                                "project_id": project})
 5342             access_ref.save(session=session)
 5343 
 5344         return share_group_type_ref
 5345 
 5346 
 5347 def _share_group_type_get_query(context, session=None, read_deleted=None,
 5348                                 expected_fields=None):
 5349     expected_fields = expected_fields or []
 5350     query = model_query(
 5351         context, models.ShareGroupTypes, session=session,
 5352         read_deleted=read_deleted
 5353     ).options(
 5354         joinedload('group_specs'),
 5355         joinedload('share_types'),
 5356     )
 5357 
 5358     if 'projects' in expected_fields:
 5359         query = query.options(joinedload('projects'))
 5360 
 5361     if not context.is_admin:
 5362         the_filter = [models.ShareGroupTypes.is_public == true()]
 5363         projects_attr = getattr(models.ShareGroupTypes, 'projects')
 5364         the_filter.extend([
 5365             projects_attr.any(project_id=context.project_id)
 5366         ])
 5367         query = query.filter(or_(*the_filter))
 5368 
 5369     return query
 5370 
 5371 
 5372 @require_context
 5373 def share_group_type_get_all(context, inactive=False, filters=None):
 5374     """Returns a dict describing all share group types with name as key."""
 5375     filters = filters or {}
 5376     read_deleted = "yes" if inactive else "no"
 5377     query = _share_group_type_get_query(context, read_deleted=read_deleted)
 5378 
 5379     if 'is_public' in filters and filters['is_public'] is not None:
 5380         the_filter = [models.ShareGroupTypes