manila-8.1.4.tar.gz: .../db/sqlalchemy/api.py

"Fossies" - the Fresh Open Source Software Archive

Member "manila-8.1.4/manila/db/sqlalchemy/api.py" (19 Nov 2020, 179746 Bytes) of package /linux/misc/openstack/manila-8.1.4.tar.gz:

As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "api.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report:

8.1.3_vs_8.1.4.

1 # Copyright (c) 2011 X.commerce, a business unit of eBay Inc. 2 # Copyright 2010 United States Government as represented by the 3 # Administrator of the National Aeronautics and Space Administration. 4 # Copyright (c) 2014 Mirantis, Inc. 5 # All Rights Reserved. 6 # 7 # Licensed under the Apache License, Version 2.0 (the "License"); you may 8 # not use this file except in compliance with the License. You may obtain 9 # a copy of the License at 10 # 11 # http://www.apache.org/licenses/LICENSE-2.0 12 # 13 # Unless required by applicable law or agreed to in writing, software 14 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 15 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 16 # License for the specific language governing permissions and limitations 17 # under the License. 18 19 """Implementation of SQLAlchemy backend.""" 20 21 import copy 22 import datetime 23 from functools import wraps 24 import ipaddress 25 import sys 26 import warnings 27 28 # NOTE(uglide): Required to override default oslo_db Query class 29 import manila.db.sqlalchemy.query # noqa 30 31 from oslo_config import cfg 32 from oslo_db import api as oslo_db_api 33 from oslo_db import exception as db_exc 34 from oslo_db import exception as db_exception 35 from oslo_db import options as db_options 36 from oslo_db.sqlalchemy import session 37 from oslo_db.sqlalchemy import utils as db_utils 38 from oslo_log import log 39 from oslo_utils import excutils 40 from oslo_utils import timeutils 41 from oslo_utils import uuidutils 42 import six 43 from sqlalchemy import MetaData 44 from sqlalchemy import or_ 45 from sqlalchemy.orm import joinedload 46 from sqlalchemy.sql.expression import literal 47 from sqlalchemy.sql.expression import true 48 from sqlalchemy.sql import func 49 50 from manila.common import constants 51 from manila.db.sqlalchemy import models 52 from manila import exception 53 from manila.i18n import _ 54 from manila import quota 55 56 CONF = cfg.CONF 57 58 LOG = log.getLogger(__name__) 59 QUOTAS = quota.QUOTAS 60 61 _DEFAULT_QUOTA_NAME = 'default' 62 PER_PROJECT_QUOTAS = [] 63 64 _FACADE = None 65 66 _DEFAULT_SQL_CONNECTION = 'sqlite://' 67 db_options.set_defaults(cfg.CONF, 68 connection=_DEFAULT_SQL_CONNECTION) 69 70 71 def _create_facade_lazily(): 72 global _FACADE 73 if _FACADE is None: 74 _FACADE = session.EngineFacade.from_config(cfg.CONF) 75 return _FACADE 76 77 78 def get_engine(): 79 facade = _create_facade_lazily() 80 return facade.get_engine() 81 82 83 def get_session(**kwargs): 84 facade = _create_facade_lazily() 85 return facade.get_session(**kwargs) 86 87 88 def get_backend(): 89 """The backend is this module itself.""" 90 91 return sys.modules[__name__] 92 93 94 def is_admin_context(context): 95 """Indicates if the request context is an administrator.""" 96 if not context: 97 warnings.warn(_('Use of empty request context is deprecated'), 98 DeprecationWarning) 99 raise Exception('die') 100 return context.is_admin 101 102 103 def is_user_context(context): 104 """Indicates if the request context is a normal user.""" 105 if not context: 106 return False 107 if context.is_admin: 108 return False 109 if not context.user_id or not context.project_id: 110 return False 111 return True 112 113 114 def authorize_project_context(context, project_id): 115 """Ensures a request has permission to access the given project.""" 116 if is_user_context(context): 117 if not context.project_id: 118 raise exception.NotAuthorized() 119 elif context.project_id != project_id: 120 raise exception.NotAuthorized() 121 122 123 def authorize_user_context(context, user_id): 124 """Ensures a request has permission to access the given user.""" 125 if is_user_context(context): 126 if not context.user_id: 127 raise exception.NotAuthorized() 128 elif context.user_id != user_id: 129 raise exception.NotAuthorized() 130 131 132 def authorize_quota_class_context(context, class_name): 133 """Ensures a request has permission to access the given quota class.""" 134 if is_user_context(context): 135 if not context.quota_class: 136 raise exception.NotAuthorized() 137 elif context.quota_class != class_name: 138 raise exception.NotAuthorized() 139 140 141 def require_admin_context(f): 142 """Decorator to require admin request context. 143 144 The first argument to the wrapped function must be the context. 145 146 """ 147 @wraps(f) 148 def wrapper(*args, **kwargs): 149 if not is_admin_context(args[0]): 150 raise exception.AdminRequired() 151 return f(*args, **kwargs) 152 return wrapper 153 154 155 def require_context(f): 156 """Decorator to require *any* user or admin context. 157 158 This does no authorization for user or project access matching, see 159 :py:func:`authorize_project_context` and 160 :py:func:`authorize_user_context`. 161 162 The first argument to the wrapped function must be the context. 163 164 """ 165 @wraps(f) 166 def wrapper(*args, **kwargs): 167 if not is_admin_context(args[0]) and not is_user_context(args[0]): 168 raise exception.NotAuthorized() 169 return f(*args, **kwargs) 170 return wrapper 171 172 173 def require_share_exists(f): 174 """Decorator to require the specified share to exist. 175 176 Requires the wrapped function to use context and share_id as 177 their first two arguments. 178 """ 179 @wraps(f) 180 def wrapper(context, share_id, *args, **kwargs): 181 share_get(context, share_id) 182 return f(context, share_id, *args, **kwargs) 183 wrapper.__name__ = f.__name__ 184 return wrapper 185 186 187 def require_share_instance_exists(f): 188 """Decorator to require the specified share instance to exist. 189 190 Requires the wrapped function to use context and share_instance_id as 191 their first two arguments. 192 """ 193 @wraps(f) 194 def wrapper(context, share_instance_id, *args, **kwargs): 195 share_instance_get(context, share_instance_id) 196 return f(context, share_instance_id, *args, **kwargs) 197 wrapper.__name__ = f.__name__ 198 return wrapper 199 200 201 def apply_sorting(model, query, sort_key, sort_dir): 202 if sort_dir.lower() not in ('desc', 'asc'): 203 msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' " 204 "and sort direction is '%(sort_dir)s'.") % { 205 "sort_key": sort_key, "sort_dir": sort_dir} 206 raise exception.InvalidInput(reason=msg) 207 208 sort_attr = getattr(model, sort_key) 209 sort_method = getattr(sort_attr, sort_dir.lower()) 210 return query.order_by(sort_method()) 211 212 213 def model_query(context, model, *args, **kwargs): 214 """Query helper that accounts for context's `read_deleted` field. 215 216 :param context: context to query under 217 :param model: model to query. Must be a subclass of ModelBase. 218 :param session: if present, the session to use 219 :param read_deleted: if present, overrides context's read_deleted field. 220 :param project_only: if present and context is user-type, then restrict 221 query to match the context's project_id. 222 """ 223 session = kwargs.get('session') or get_session() 224 read_deleted = kwargs.get('read_deleted') or context.read_deleted 225 project_only = kwargs.get('project_only') 226 kwargs = dict() 227 228 if project_only and not context.is_admin: 229 kwargs['project_id'] = context.project_id 230 if read_deleted in ('no', 'n', False): 231 kwargs['deleted'] = False 232 elif read_deleted in ('yes', 'y', True): 233 kwargs['deleted'] = True 234 235 return db_utils.model_query( 236 model=model, session=session, args=args, **kwargs) 237 238 239 def exact_filter(query, model, filters, legal_keys): 240 """Applies exact match filtering to a query. 241 242 Returns the updated query. Modifies filters argument to remove 243 filters consumed. 244 245 :param query: query to apply filters to 246 :param model: model object the query applies to, for IN-style 247 filtering 248 :param filters: dictionary of filters; values that are lists, 249 tuples, sets, or frozensets cause an 'IN' test to 250 be performed, while exact matching ('==' operator) 251 is used for other values 252 :param legal_keys: list of keys to apply exact filtering to 253 """ 254 255 filter_dict = {} 256 257 # Walk through all the keys 258 for key in legal_keys: 259 # Skip ones we're not filtering on 260 if key not in filters: 261 continue 262 263 # OK, filtering on this key; what value do we search for? 264 value = filters.pop(key) 265 266 if isinstance(value, (list, tuple, set, frozenset)): 267 # Looking for values in a list; apply to query directly 268 column_attr = getattr(model, key) 269 query = query.filter(column_attr.in_(value)) 270 else: 271 # OK, simple exact match; save for later 272 filter_dict[key] = value 273 274 # Apply simple exact matches 275 if filter_dict: 276 query = query.filter_by(**filter_dict) 277 278 return query 279 280 281 def ensure_model_dict_has_id(model_dict): 282 if not model_dict.get('id'): 283 model_dict['id'] = uuidutils.generate_uuid() 284 return model_dict 285 286 287 def _sync_shares(context, project_id, user_id, session, share_type_id=None): 288 (shares, gigs) = share_data_get_for_project( 289 context, project_id, user_id, share_type_id=share_type_id, 290 session=session) 291 return {'shares': shares} 292 293 294 def _sync_snapshots(context, project_id, user_id, session, share_type_id=None): 295 (snapshots, gigs) = snapshot_data_get_for_project( 296 context, project_id, user_id, share_type_id=share_type_id, 297 session=session) 298 return {'snapshots': snapshots} 299 300 301 def _sync_gigabytes(context, project_id, user_id, session, share_type_id=None): 302 _junk, share_gigs = share_data_get_for_project( 303 context, project_id, user_id, share_type_id=share_type_id, 304 session=session) 305 return {"gigabytes": share_gigs} 306 307 308 def _sync_snapshot_gigabytes(context, project_id, user_id, session, 309 share_type_id=None): 310 _junk, snapshot_gigs = snapshot_data_get_for_project( 311 context, project_id, user_id, share_type_id=share_type_id, 312 session=session) 313 return {"snapshot_gigabytes": snapshot_gigs} 314 315 316 def _sync_share_networks(context, project_id, user_id, session, 317 share_type_id=None): 318 share_networks_count = count_share_networks( 319 context, project_id, user_id, share_type_id=share_type_id, 320 session=session) 321 return {'share_networks': share_networks_count} 322 323 324 def _sync_share_groups(context, project_id, user_id, session, 325 share_type_id=None): 326 share_groups_count = count_share_groups( 327 context, project_id, user_id, share_type_id=share_type_id, 328 session=session) 329 return {'share_groups': share_groups_count} 330 331 332 def _sync_share_group_snapshots(context, project_id, user_id, session, 333 share_type_id=None): 334 share_group_snapshots_count = count_share_group_snapshots( 335 context, project_id, user_id, share_type_id=share_type_id, 336 session=session) 337 return {'share_group_snapshots': share_group_snapshots_count} 338 339 340 QUOTA_SYNC_FUNCTIONS = { 341 '_sync_shares': _sync_shares, 342 '_sync_snapshots': _sync_snapshots, 343 '_sync_gigabytes': _sync_gigabytes, 344 '_sync_snapshot_gigabytes': _sync_snapshot_gigabytes, 345 '_sync_share_networks': _sync_share_networks, 346 '_sync_share_groups': _sync_share_groups, 347 '_sync_share_group_snapshots': _sync_share_group_snapshots, 348 } 349 350 351 ################### 352 353 354 @require_admin_context 355 def service_destroy(context, service_id): 356 session = get_session() 357 with session.begin(): 358 service_ref = service_get(context, service_id, session=session) 359 service_ref.soft_delete(session) 360 361 362 @require_admin_context 363 def service_get(context, service_id, session=None): 364 result = (model_query( 365 context, 366 models.Service, 367 session=session). 368 filter_by(id=service_id). 369 first()) 370 if not result: 371 raise exception.ServiceNotFound(service_id=service_id) 372 373 return result 374 375 376 @require_admin_context 377 def service_get_all(context, disabled=None): 378 query = model_query(context, models.Service) 379 380 if disabled is not None: 381 query = query.filter_by(disabled=disabled) 382 383 return query.all() 384 385 386 @require_admin_context 387 def service_get_all_by_topic(context, topic): 388 return (model_query( 389 context, models.Service, read_deleted="no"). 390 filter_by(disabled=False). 391 filter_by(topic=topic). 392 all()) 393 394 395 @require_admin_context 396 def service_get_by_host_and_topic(context, host, topic): 397 result = (model_query( 398 context, models.Service, read_deleted="no"). 399 filter_by(disabled=False). 400 filter_by(host=host). 401 filter_by(topic=topic). 402 first()) 403 if not result: 404 raise exception.ServiceNotFound(service_id=host) 405 return result 406 407 408 @require_admin_context 409 def _service_get_all_topic_subquery(context, session, topic, subq, label): 410 sort_value = getattr(subq.c, label) 411 return (model_query(context, models.Service, 412 func.coalesce(sort_value, 0), 413 session=session, read_deleted="no"). 414 filter_by(topic=topic). 415 filter_by(disabled=False). 416 outerjoin((subq, models.Service.host == subq.c.host)). 417 order_by(sort_value). 418 all()) 419 420 421 @require_admin_context 422 def service_get_all_share_sorted(context): 423 session = get_session() 424 with session.begin(): 425 topic = CONF.share_topic 426 label = 'share_gigabytes' 427 subq = (model_query(context, models.Share, 428 func.sum(models.Share.size).label(label), 429 session=session, read_deleted="no"). 430 join(models.ShareInstance, 431 models.ShareInstance.share_id == models.Share.id). 432 group_by(models.ShareInstance.host). 433 subquery()) 434 return _service_get_all_topic_subquery(context, 435 session, 436 topic, 437 subq, 438 label) 439 440 441 @require_admin_context 442 def service_get_by_args(context, host, binary): 443 result = (model_query(context, models.Service). 444 filter_by(host=host). 445 filter_by(binary=binary). 446 first()) 447 448 if not result: 449 raise exception.HostBinaryNotFound(host=host, binary=binary) 450 451 return result 452 453 454 @require_admin_context 455 def service_create(context, values): 456 session = get_session() 457 458 _ensure_availability_zone_exists(context, values, session) 459 460 service_ref = models.Service() 461 service_ref.update(values) 462 if not CONF.enable_new_services: 463 service_ref.disabled = True 464 465 with session.begin(): 466 service_ref.save(session) 467 return service_ref 468 469 470 @require_admin_context 471 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 472 def service_update(context, service_id, values): 473 session = get_session() 474 475 _ensure_availability_zone_exists(context, values, session, strict=False) 476 477 with session.begin(): 478 service_ref = service_get(context, service_id, session=session) 479 service_ref.update(values) 480 service_ref.save(session=session) 481 482 483 ################### 484 485 486 @require_context 487 def quota_get_all_by_project_and_user(context, project_id, user_id): 488 authorize_project_context(context, project_id) 489 user_quotas = model_query( 490 context, models.ProjectUserQuota, 491 models.ProjectUserQuota.resource, 492 models.ProjectUserQuota.hard_limit, 493 ).filter_by( 494 project_id=project_id, 495 ).filter_by( 496 user_id=user_id, 497 ).all() 498 499 result = {'project_id': project_id, 'user_id': user_id} 500 for u_quota in user_quotas: 501 result[u_quota.resource] = u_quota.hard_limit 502 return result 503 504 505 @require_context 506 def quota_get_all_by_project_and_share_type(context, project_id, 507 share_type_id): 508 authorize_project_context(context, project_id) 509 share_type_quotas = model_query( 510 context, models.ProjectShareTypeQuota, 511 models.ProjectShareTypeQuota.resource, 512 models.ProjectShareTypeQuota.hard_limit, 513 ).filter_by( 514 project_id=project_id, 515 ).filter_by( 516 share_type_id=share_type_id, 517 ).all() 518 519 result = { 520 'project_id': project_id, 521 'share_type_id': share_type_id, 522 } 523 for st_quota in share_type_quotas: 524 result[st_quota.resource] = st_quota.hard_limit 525 return result 526 527 528 @require_context 529 def quota_get_all_by_project(context, project_id): 530 authorize_project_context(context, project_id) 531 project_quotas = model_query( 532 context, models.Quota, read_deleted="no", 533 ).filter_by( 534 project_id=project_id, 535 ).all() 536 537 result = {'project_id': project_id} 538 for p_quota in project_quotas: 539 result[p_quota.resource] = p_quota.hard_limit 540 return result 541 542 543 @require_context 544 def quota_get_all(context, project_id): 545 authorize_project_context(context, project_id) 546 547 result = (model_query(context, models.ProjectUserQuota). 548 filter_by(project_id=project_id). 549 all()) 550 551 return result 552 553 554 @require_admin_context 555 def quota_create(context, project_id, resource, limit, user_id=None, 556 share_type_id=None): 557 per_user = user_id and resource not in PER_PROJECT_QUOTAS 558 559 if per_user: 560 check = model_query(context, models.ProjectUserQuota).filter( 561 models.ProjectUserQuota.project_id == project_id, 562 models.ProjectUserQuota.user_id == user_id, 563 models.ProjectUserQuota.resource == resource, 564 ).all() 565 quota_ref = models.ProjectUserQuota() 566 quota_ref.user_id = user_id 567 elif share_type_id: 568 check = model_query(context, models.ProjectShareTypeQuota).filter( 569 models.ProjectShareTypeQuota.project_id == project_id, 570 models.ProjectShareTypeQuota.share_type_id == share_type_id, 571 models.ProjectShareTypeQuota.resource == resource, 572 ).all() 573 quota_ref = models.ProjectShareTypeQuota() 574 quota_ref.share_type_id = share_type_id 575 else: 576 check = model_query(context, models.Quota).filter( 577 models.Quota.project_id == project_id, 578 models.Quota.resource == resource, 579 ).all() 580 quota_ref = models.Quota() 581 if check: 582 raise exception.QuotaExists(project_id=project_id, resource=resource) 583 584 quota_ref.project_id = project_id 585 quota_ref.resource = resource 586 quota_ref.hard_limit = limit 587 session = get_session() 588 with session.begin(): 589 quota_ref.save(session) 590 return quota_ref 591 592 593 @require_admin_context 594 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 595 def quota_update(context, project_id, resource, limit, user_id=None, 596 share_type_id=None): 597 per_user = user_id and resource not in PER_PROJECT_QUOTAS 598 if per_user: 599 query = model_query(context, models.ProjectUserQuota).filter( 600 models.ProjectUserQuota.project_id == project_id, 601 models.ProjectUserQuota.user_id == user_id, 602 models.ProjectUserQuota.resource == resource, 603 ) 604 elif share_type_id: 605 query = model_query(context, models.ProjectShareTypeQuota).filter( 606 models.ProjectShareTypeQuota.project_id == project_id, 607 models.ProjectShareTypeQuota.share_type_id == share_type_id, 608 models.ProjectShareTypeQuota.resource == resource, 609 ) 610 else: 611 query = model_query(context, models.Quota).filter( 612 models.Quota.project_id == project_id, 613 models.Quota.resource == resource, 614 ) 615 616 result = query.update({'hard_limit': limit}) 617 if not result: 618 if per_user: 619 raise exception.ProjectUserQuotaNotFound( 620 project_id=project_id, user_id=user_id) 621 elif share_type_id: 622 raise exception.ProjectShareTypeQuotaNotFound( 623 project_id=project_id, share_type=share_type_id) 624 raise exception.ProjectQuotaNotFound(project_id=project_id) 625 626 627 ################### 628 629 630 @require_context 631 def quota_class_get(context, class_name, resource, session=None): 632 result = (model_query(context, models.QuotaClass, session=session, 633 read_deleted="no"). 634 filter_by(class_name=class_name). 635 filter_by(resource=resource). 636 first()) 637 638 if not result: 639 raise exception.QuotaClassNotFound(class_name=class_name) 640 641 return result 642 643 644 @require_context 645 def quota_class_get_default(context): 646 rows = (model_query(context, models.QuotaClass, read_deleted="no"). 647 filter_by(class_name=_DEFAULT_QUOTA_NAME). 648 all()) 649 650 result = {'class_name': _DEFAULT_QUOTA_NAME} 651 for row in rows: 652 result[row.resource] = row.hard_limit 653 654 return result 655 656 657 @require_context 658 def quota_class_get_all_by_name(context, class_name): 659 authorize_quota_class_context(context, class_name) 660 661 rows = (model_query(context, models.QuotaClass, read_deleted="no"). 662 filter_by(class_name=class_name). 663 all()) 664 665 result = {'class_name': class_name} 666 for row in rows: 667 result[row.resource] = row.hard_limit 668 669 return result 670 671 672 @require_admin_context 673 def quota_class_create(context, class_name, resource, limit): 674 quota_class_ref = models.QuotaClass() 675 quota_class_ref.class_name = class_name 676 quota_class_ref.resource = resource 677 quota_class_ref.hard_limit = limit 678 session = get_session() 679 with session.begin(): 680 quota_class_ref.save(session) 681 return quota_class_ref 682 683 684 @require_admin_context 685 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 686 def quota_class_update(context, class_name, resource, limit): 687 result = (model_query(context, models.QuotaClass, read_deleted="no"). 688 filter_by(class_name=class_name). 689 filter_by(resource=resource). 690 update({'hard_limit': limit})) 691 692 if not result: 693 raise exception.QuotaClassNotFound(class_name=class_name) 694 695 696 ################### 697 698 699 @require_context 700 def quota_usage_get(context, project_id, resource, user_id=None, 701 share_type_id=None): 702 query = (model_query(context, models.QuotaUsage, read_deleted="no"). 703 filter_by(project_id=project_id). 704 filter_by(resource=resource)) 705 if user_id: 706 if resource not in PER_PROJECT_QUOTAS: 707 result = query.filter_by(user_id=user_id).first() 708 else: 709 result = query.filter_by(user_id=None).first() 710 elif share_type_id: 711 result = query.filter_by(queryshare_type_id=share_type_id).first() 712 else: 713 result = query.first() 714 715 if not result: 716 raise exception.QuotaUsageNotFound(project_id=project_id) 717 718 return result 719 720 721 def _quota_usage_get_all(context, project_id, user_id=None, 722 share_type_id=None): 723 authorize_project_context(context, project_id) 724 query = (model_query(context, models.QuotaUsage, read_deleted="no"). 725 filter_by(project_id=project_id)) 726 result = {'project_id': project_id} 727 if user_id: 728 query = query.filter(or_(models.QuotaUsage.user_id == user_id, 729 models.QuotaUsage.user_id is None)) 730 result['user_id'] = user_id 731 elif share_type_id: 732 query = query.filter_by(share_type_id=share_type_id) 733 result['share_type_id'] = share_type_id 734 else: 735 query = query.filter_by(share_type_id=None) 736 737 rows = query.all() 738 for row in rows: 739 if row.resource in result: 740 result[row.resource]['in_use'] += row.in_use 741 result[row.resource]['reserved'] += row.reserved 742 else: 743 result[row.resource] = dict(in_use=row.in_use, 744 reserved=row.reserved) 745 746 return result 747 748 749 @require_context 750 def quota_usage_get_all_by_project(context, project_id): 751 return _quota_usage_get_all(context, project_id) 752 753 754 @require_context 755 def quota_usage_get_all_by_project_and_user(context, project_id, user_id): 756 return _quota_usage_get_all(context, project_id, user_id=user_id) 757 758 759 @require_context 760 def quota_usage_get_all_by_project_and_share_type(context, project_id, 761 share_type_id): 762 return _quota_usage_get_all( 763 context, project_id, share_type_id=share_type_id) 764 765 766 def _quota_usage_create(context, project_id, user_id, resource, in_use, 767 reserved, until_refresh, share_type_id=None, 768 session=None): 769 quota_usage_ref = models.QuotaUsage() 770 if share_type_id: 771 quota_usage_ref.share_type_id = share_type_id 772 else: 773 quota_usage_ref.user_id = user_id 774 quota_usage_ref.project_id = project_id 775 quota_usage_ref.resource = resource 776 quota_usage_ref.in_use = in_use 777 quota_usage_ref.reserved = reserved 778 quota_usage_ref.until_refresh = until_refresh 779 # updated_at is needed for judgement of max_age 780 quota_usage_ref.updated_at = timeutils.utcnow() 781 782 quota_usage_ref.save(session=session) 783 784 return quota_usage_ref 785 786 787 @require_admin_context 788 def quota_usage_create(context, project_id, user_id, resource, in_use, 789 reserved, until_refresh, share_type_id=None): 790 session = get_session() 791 return _quota_usage_create( 792 context, project_id, user_id, resource, in_use, reserved, 793 until_refresh, share_type_id=share_type_id, session=session) 794 795 796 @require_admin_context 797 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 798 def quota_usage_update(context, project_id, user_id, resource, 799 share_type_id=None, **kwargs): 800 updates = {} 801 for key in ('in_use', 'reserved', 'until_refresh'): 802 if key in kwargs: 803 updates[key] = kwargs[key] 804 805 query = model_query( 806 context, models.QuotaUsage, read_deleted="no", 807 ).filter_by(project_id=project_id).filter_by(resource=resource) 808 if share_type_id: 809 query = query.filter_by(share_type_id=share_type_id) 810 else: 811 query = query.filter(or_(models.QuotaUsage.user_id == user_id, 812 models.QuotaUsage.user_id is None)) 813 result = query.update(updates) 814 815 if not result: 816 raise exception.QuotaUsageNotFound(project_id=project_id) 817 818 819 ################### 820 821 822 def _reservation_create(context, uuid, usage, project_id, user_id, resource, 823 delta, expire, share_type_id=None, session=None): 824 reservation_ref = models.Reservation() 825 reservation_ref.uuid = uuid 826 reservation_ref.usage_id = usage['id'] 827 reservation_ref.project_id = project_id 828 if share_type_id: 829 reservation_ref.share_type_id = share_type_id 830 else: 831 reservation_ref.user_id = user_id 832 reservation_ref.resource = resource 833 reservation_ref.delta = delta 834 reservation_ref.expire = expire 835 reservation_ref.save(session=session) 836 return reservation_ref 837 838 839 ################### 840 841 842 # NOTE(johannes): The quota code uses SQL locking to ensure races don't 843 # cause under or over counting of resources. To avoid deadlocks, this 844 # code always acquires the lock on quota_usages before acquiring the lock 845 # on reservations. 846 847 def _get_share_type_quota_usages(context, session, project_id, share_type_id): 848 rows = model_query( 849 context, models.QuotaUsage, read_deleted="no", session=session, 850 ).filter( 851 models.QuotaUsage.project_id == project_id, 852 models.QuotaUsage.share_type_id == share_type_id, 853 ).with_lockmode('update').all() 854 return {row.resource: row for row in rows} 855 856 857 def _get_user_quota_usages(context, session, project_id, user_id): 858 # Broken out for testability 859 rows = (model_query(context, models.QuotaUsage, 860 read_deleted="no", 861 session=session). 862 filter_by(project_id=project_id). 863 filter(or_(models.QuotaUsage.user_id == user_id, 864 models.QuotaUsage.user_id is None)). 865 with_lockmode('update'). 866 all()) 867 return {row.resource: row for row in rows} 868 869 870 def _get_project_quota_usages(context, session, project_id): 871 rows = (model_query(context, models.QuotaUsage, 872 read_deleted="no", 873 session=session). 874 filter_by(project_id=project_id). 875 filter(models.QuotaUsage.share_type_id is None). 876 with_lockmode('update'). 877 all()) 878 result = dict() 879 # Get the total count of in_use,reserved 880 for row in rows: 881 if row.resource in result: 882 result[row.resource]['in_use'] += row.in_use 883 result[row.resource]['reserved'] += row.reserved 884 result[row.resource]['total'] += (row.in_use + row.reserved) 885 else: 886 result[row.resource] = dict(in_use=row.in_use, 887 reserved=row.reserved, 888 total=row.in_use + row.reserved) 889 return result 890 891 892 @require_context 893 def quota_reserve(context, resources, project_quotas, user_quotas, 894 share_type_quotas, deltas, expire, until_refresh, 895 max_age, project_id=None, user_id=None, share_type_id=None): 896 user_reservations = _quota_reserve( 897 context, resources, project_quotas, user_quotas, 898 deltas, expire, until_refresh, max_age, project_id, user_id=user_id) 899 if share_type_id: 900 try: 901 st_reservations = _quota_reserve( 902 context, resources, project_quotas, share_type_quotas, 903 deltas, expire, until_refresh, max_age, project_id, 904 share_type_id=share_type_id) 905 except exception.OverQuota: 906 with excutils.save_and_reraise_exception(): 907 # rollback previous reservations 908 reservation_rollback( 909 context, user_reservations, 910 project_id=project_id, user_id=user_id) 911 return user_reservations + st_reservations 912 return user_reservations 913 914 915 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 916 def _quota_reserve(context, resources, project_quotas, user_or_st_quotas, 917 deltas, expire, until_refresh, 918 max_age, project_id=None, user_id=None, share_type_id=None): 919 elevated = context.elevated() 920 session = get_session() 921 with session.begin(): 922 923 if project_id is None: 924 project_id = context.project_id 925 if share_type_id: 926 user_or_st_usages = _get_share_type_quota_usages( 927 context, session, project_id, share_type_id) 928 else: 929 user_id = user_id if user_id else context.user_id 930 user_or_st_usages = _get_user_quota_usages( 931 context, session, project_id, user_id) 932 933 # Get the current usages 934 project_usages = _get_project_quota_usages( 935 context, session, project_id) 936 937 # Handle usage refresh 938 work = set(deltas.keys()) 939 while work: 940 resource = work.pop() 941 942 # Do we need to refresh the usage? 943 refresh = False 944 if ((resource not in PER_PROJECT_QUOTAS) and 945 (resource not in user_or_st_usages)): 946 user_or_st_usages[resource] = _quota_usage_create( 947 elevated, 948 project_id, 949 user_id, 950 resource, 951 0, 0, 952 until_refresh or None, 953 share_type_id=share_type_id, 954 session=session) 955 refresh = True 956 elif ((resource in PER_PROJECT_QUOTAS) and 957 (resource not in user_or_st_usages)): 958 user_or_st_usages[resource] = _quota_usage_create( 959 elevated, 960 project_id, 961 None, 962 resource, 963 0, 0, 964 until_refresh or None, 965 share_type_id=share_type_id, 966 session=session) 967 refresh = True 968 elif user_or_st_usages[resource].in_use < 0: 969 # Negative in_use count indicates a desync, so try to 970 # heal from that... 971 refresh = True 972 elif user_or_st_usages[resource].until_refresh is not None: 973 user_or_st_usages[resource].until_refresh -= 1 974 if user_or_st_usages[resource].until_refresh <= 0: 975 refresh = True 976 elif max_age and (user_or_st_usages[resource].updated_at - 977 timeutils.utcnow()).seconds >= max_age: 978 refresh = True 979 980 # OK, refresh the usage 981 if refresh: 982 # Grab the sync routine 983 sync = QUOTA_SYNC_FUNCTIONS[resources[resource].sync] 984 985 updates = sync( 986 elevated, project_id, user_id, 987 share_type_id=share_type_id, session=session) 988 for res, in_use in updates.items(): 989 # Make sure we have a destination for the usage! 990 if ((res not in PER_PROJECT_QUOTAS) and 991 (res not in user_or_st_usages)): 992 user_or_st_usages[res] = _quota_usage_create( 993 elevated, 994 project_id, 995 user_id, 996 res, 997 0, 0, 998 until_refresh or None, 999 share_type_id=share_type_id, 1000 session=session) 1001 if ((res in PER_PROJECT_QUOTAS) and 1002 (res not in user_or_st_usages)): 1003 user_or_st_usages[res] = _quota_usage_create( 1004 elevated, 1005 project_id, 1006 None, 1007 res, 1008 0, 0, 1009 until_refresh or None, 1010 share_type_id=share_type_id, 1011 session=session) 1012 1013 if user_or_st_usages[res].in_use != in_use: 1014 LOG.debug( 1015 'quota_usages out of sync, updating. ' 1016 'project_id: %(project_id)s, ' 1017 'user_id: %(user_id)s, ' 1018 'share_type_id: %(share_type_id)s, ' 1019 'resource: %(res)s, ' 1020 'tracked usage: %(tracked_use)s, ' 1021 'actual usage: %(in_use)s', 1022 {'project_id': project_id, 1023 'user_id': user_id, 1024 'share_type_id': share_type_id, 1025 'res': res, 1026 'tracked_use': user_or_st_usages[res].in_use, 1027 'in_use': in_use}) 1028 1029 # Update the usage 1030 user_or_st_usages[res].in_use = in_use 1031 user_or_st_usages[res].until_refresh = ( 1032 until_refresh or None) 1033 1034 # Because more than one resource may be refreshed 1035 # by the call to the sync routine, and we don't 1036 # want to double-sync, we make sure all refreshed 1037 # resources are dropped from the work set. 1038 work.discard(res) 1039 1040 # NOTE(Vek): We make the assumption that the sync 1041 # routine actually refreshes the 1042 # resources that it is the sync routine 1043 # for. We don't check, because this is 1044 # a best-effort mechanism. 1045 1046 # Check for deltas that would go negative 1047 unders = [res for res, delta in deltas.items() 1048 if delta < 0 and 1049 delta + user_or_st_usages[res].in_use < 0] 1050 1051 # Now, let's check the quotas 1052 # NOTE(Vek): We're only concerned about positive increments. 1053 # If a project has gone over quota, we want them to 1054 # be able to reduce their usage without any 1055 # problems. 1056 for key, value in user_or_st_usages.items(): 1057 if key not in project_usages: 1058 project_usages[key] = value 1059 overs = [res for res, delta in deltas.items() 1060 if user_or_st_quotas[res] >= 0 and delta >= 0 and 1061 (0 <= project_quotas[res] < delta + 1062 project_usages[res]['total'] or 1063 user_or_st_quotas[res] < delta + 1064 user_or_st_usages[res].total)] 1065 1066 # NOTE(Vek): The quota check needs to be in the transaction, 1067 # but the transaction doesn't fail just because 1068 # we're over quota, so the OverQuota raise is 1069 # outside the transaction. If we did the raise 1070 # here, our usage updates would be discarded, but 1071 # they're not invalidated by being over-quota. 1072 1073 # Create the reservations 1074 if not overs: 1075 reservations = [] 1076 for res, delta in deltas.items(): 1077 reservation = _reservation_create(elevated, 1078 uuidutils.generate_uuid(), 1079 user_or_st_usages[res], 1080 project_id, 1081 user_id, 1082 res, delta, expire, 1083 share_type_id=share_type_id, 1084 session=session) 1085 reservations.append(reservation.uuid) 1086 1087 # Also update the reserved quantity 1088 # NOTE(Vek): Again, we are only concerned here about 1089 # positive increments. Here, though, we're 1090 # worried about the following scenario: 1091 # 1092 # 1) User initiates resize down. 1093 # 2) User allocates a new instance. 1094 # 3) Resize down fails or is reverted. 1095 # 4) User is now over quota. 1096 # 1097 # To prevent this, we only update the 1098 # reserved value if the delta is positive. 1099 if delta > 0: 1100 user_or_st_usages[res].reserved += delta 1101 1102 # Apply updates to the usages table 1103 for usage_ref in user_or_st_usages.values(): 1104 session.add(usage_ref) 1105 1106 if unders: 1107 LOG.warning("Change will make usage less than 0 for the following " 1108 "resources: %s", unders) 1109 if overs: 1110 if project_quotas == user_or_st_quotas: 1111 usages = project_usages 1112 else: 1113 usages = user_or_st_usages 1114 usages = {k: dict(in_use=v['in_use'], reserved=v['reserved']) 1115 for k, v in usages.items()} 1116 raise exception.OverQuota( 1117 overs=sorted(overs), quotas=user_or_st_quotas, usages=usages) 1118 1119 return reservations 1120 1121 1122 def _quota_reservations_query(session, context, reservations): 1123 """Return the relevant reservations.""" 1124 1125 # Get the listed reservations 1126 return (model_query(context, models.Reservation, 1127 read_deleted="no", 1128 session=session). 1129 filter(models.Reservation.uuid.in_(reservations)). 1130 with_lockmode('update')) 1131 1132 1133 @require_context 1134 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 1135 def reservation_commit(context, reservations, project_id=None, user_id=None, 1136 share_type_id=None): 1137 session = get_session() 1138 with session.begin(): 1139 if share_type_id: 1140 st_usages = _get_share_type_quota_usages( 1141 context, session, project_id, share_type_id) 1142 else: 1143 st_usages = {} 1144 user_usages = _get_user_quota_usages( 1145 context, session, project_id, user_id) 1146 1147 reservation_query = _quota_reservations_query( 1148 session, context, reservations) 1149 for reservation in reservation_query.all(): 1150 if reservation['share_type_id']: 1151 usages = st_usages 1152 else: 1153 usages = user_usages 1154 usage = usages[reservation.resource] 1155 if reservation.delta >= 0: 1156 usage.reserved -= reservation.delta 1157 usage.in_use += reservation.delta 1158 reservation_query.soft_delete(synchronize_session=False) 1159 1160 1161 @require_context 1162 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 1163 def reservation_rollback(context, reservations, project_id=None, user_id=None, 1164 share_type_id=None): 1165 session = get_session() 1166 with session.begin(): 1167 if share_type_id: 1168 st_usages = _get_share_type_quota_usages( 1169 context, session, project_id, share_type_id) 1170 else: 1171 st_usages = {} 1172 user_usages = _get_user_quota_usages( 1173 context, session, project_id, user_id) 1174 1175 reservation_query = _quota_reservations_query( 1176 session, context, reservations) 1177 for reservation in reservation_query.all(): 1178 if reservation['share_type_id']: 1179 usages = st_usages 1180 else: 1181 usages = user_usages 1182 usage = usages[reservation.resource] 1183 if reservation.delta >= 0: 1184 usage.reserved -= reservation.delta 1185 reservation_query.soft_delete(synchronize_session=False) 1186 1187 1188 @require_admin_context 1189 def quota_destroy_all_by_project_and_user(context, project_id, user_id): 1190 session = get_session() 1191 with session.begin(): 1192 (model_query(context, models.ProjectUserQuota, session=session, 1193 read_deleted="no"). 1194 filter_by(project_id=project_id). 1195 filter_by(user_id=user_id).soft_delete(synchronize_session=False)) 1196 1197 (model_query(context, models.QuotaUsage, 1198 session=session, read_deleted="no"). 1199 filter_by(project_id=project_id). 1200 filter_by(user_id=user_id).soft_delete(synchronize_session=False)) 1201 1202 (model_query(context, models.Reservation, 1203 session=session, read_deleted="no"). 1204 filter_by(project_id=project_id). 1205 filter_by(user_id=user_id).soft_delete(synchronize_session=False)) 1206 1207 1208 @require_admin_context 1209 def quota_destroy_all_by_share_type(context, share_type_id, project_id=None): 1210 """Soft deletes all quotas, usages and reservations. 1211 1212 :param context: request context for queries, updates and logging 1213 :param share_type_id: ID of the share type to filter the quotas, usages 1214 and reservations under. 1215 :param project_id: ID of the project to filter the quotas, usages and 1216 reservations under. If not provided, share type quotas for all 1217 projects will be acted upon. 1218 """ 1219 session = get_session() 1220 with session.begin(): 1221 share_type_quotas = model_query( 1222 context, models.ProjectShareTypeQuota, session=session, 1223 read_deleted="no", 1224 ).filter_by(share_type_id=share_type_id) 1225 1226 share_type_quota_usages = model_query( 1227 context, models.QuotaUsage, session=session, read_deleted="no", 1228 ).filter_by(share_type_id=share_type_id) 1229 1230 share_type_quota_reservations = model_query( 1231 context, models.Reservation, session=session, read_deleted="no", 1232 ).filter_by(share_type_id=share_type_id) 1233 1234 if project_id is not None: 1235 share_type_quotas = share_type_quotas.filter_by( 1236 project_id=project_id) 1237 share_type_quota_usages = share_type_quota_usages.filter_by( 1238 project_id=project_id) 1239 share_type_quota_reservations = ( 1240 share_type_quota_reservations.filter_by(project_id=project_id)) 1241 1242 share_type_quotas.soft_delete(synchronize_session=False) 1243 share_type_quota_usages.soft_delete(synchronize_session=False) 1244 share_type_quota_reservations.soft_delete(synchronize_session=False) 1245 1246 1247 @require_admin_context 1248 def quota_destroy_all_by_project(context, project_id): 1249 session = get_session() 1250 with session.begin(): 1251 (model_query(context, models.Quota, session=session, 1252 read_deleted="no"). 1253 filter_by(project_id=project_id). 1254 soft_delete(synchronize_session=False)) 1255 1256 (model_query(context, models.ProjectUserQuota, session=session, 1257 read_deleted="no"). 1258 filter_by(project_id=project_id). 1259 soft_delete(synchronize_session=False)) 1260 1261 (model_query(context, models.QuotaUsage, 1262 session=session, read_deleted="no"). 1263 filter_by(project_id=project_id). 1264 soft_delete(synchronize_session=False)) 1265 1266 (model_query(context, models.Reservation, 1267 session=session, read_deleted="no"). 1268 filter_by(project_id=project_id). 1269 soft_delete(synchronize_session=False)) 1270 1271 1272 @require_admin_context 1273 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 1274 def reservation_expire(context): 1275 session = get_session() 1276 with session.begin(): 1277 current_time = timeutils.utcnow() 1278 reservation_query = (model_query( 1279 context, models.Reservation, 1280 session=session, read_deleted="no"). 1281 filter(models.Reservation.expire < current_time)) 1282 1283 for reservation in reservation_query.all(): 1284 if reservation.delta >= 0: 1285 quota_usage = model_query(context, models.QuotaUsage, 1286 session=session, 1287 read_deleted="no").filter( 1288 models.QuotaUsage.id == reservation.usage_id).first() 1289 quota_usage.reserved -= reservation.delta 1290 session.add(quota_usage) 1291 1292 reservation_query.soft_delete(synchronize_session=False) 1293 1294 1295 ################ 1296 1297 def _extract_subdict_by_fields(source_dict, fields): 1298 dict_to_extract_from = copy.deepcopy(source_dict) 1299 sub_dict = {} 1300 for field in fields: 1301 field_value = dict_to_extract_from.pop(field, None) 1302 if field_value: 1303 sub_dict.update({field: field_value}) 1304 1305 return sub_dict, dict_to_extract_from 1306 1307 1308 def _extract_share_instance_values(values): 1309 share_instance_model_fields = [ 1310 'status', 'host', 'scheduled_at', 'launched_at', 'terminated_at', 1311 'share_server_id', 'share_network_id', 'availability_zone', 1312 'replica_state', 'share_type_id', 'share_type', 'access_rules_status', 1313 ] 1314 share_instance_values, share_values = ( 1315 _extract_subdict_by_fields(values, share_instance_model_fields) 1316 ) 1317 return share_instance_values, share_values 1318 1319 1320 def _change_size_to_instance_size(snap_instance_values): 1321 if 'size' in snap_instance_values: 1322 snap_instance_values['instance_size'] = snap_instance_values['size'] 1323 snap_instance_values.pop('size') 1324 1325 1326 def _extract_snapshot_instance_values(values): 1327 fields = ['status', 'progress', 'provider_location'] 1328 snapshot_instance_values, snapshot_values = ( 1329 _extract_subdict_by_fields(values, fields) 1330 ) 1331 return snapshot_instance_values, snapshot_values 1332 1333 1334 ################ 1335 1336 1337 @require_context 1338 def share_instance_create(context, share_id, values): 1339 session = get_session() 1340 with session.begin(): 1341 return _share_instance_create(context, share_id, values, session) 1342 1343 1344 def _share_instance_create(context, share_id, values, session): 1345 if not values.get('id'): 1346 values['id'] = uuidutils.generate_uuid() 1347 values.update({'share_id': share_id}) 1348 1349 share_instance_ref = models.ShareInstance() 1350 share_instance_ref.update(values) 1351 share_instance_ref.save(session=session) 1352 1353 return share_instance_get(context, share_instance_ref['id'], 1354 session=session) 1355 1356 1357 @require_admin_context 1358 def share_instances_host_update(context, current_host, new_host): 1359 session = get_session() 1360 host_field = models.ShareInstance.host 1361 with session.begin(): 1362 query = model_query( 1363 context, models.ShareInstance, session=session, read_deleted="no", 1364 ).filter(host_field.like('{}%'.format(current_host))) 1365 result = query.update( 1366 {host_field: func.replace(host_field, current_host, new_host)}, 1367 synchronize_session=False) 1368 return result 1369 1370 1371 @require_context 1372 def share_instance_update(context, share_instance_id, values, 1373 with_share_data=False): 1374 session = get_session() 1375 _ensure_availability_zone_exists(context, values, session, strict=False) 1376 with session.begin(): 1377 instance_ref = _share_instance_update( 1378 context, share_instance_id, values, session 1379 ) 1380 if with_share_data: 1381 parent_share = share_get(context, instance_ref['share_id'], 1382 session=session) 1383 instance_ref.set_share_data(parent_share) 1384 return instance_ref 1385 1386 1387 def _share_instance_update(context, share_instance_id, values, session): 1388 share_instance_ref = share_instance_get(context, share_instance_id, 1389 session=session) 1390 share_instance_ref.update(values) 1391 share_instance_ref.save(session=session) 1392 return share_instance_ref 1393 1394 1395 @require_context 1396 def share_instance_get(context, share_instance_id, session=None, 1397 with_share_data=False): 1398 if session is None: 1399 session = get_session() 1400 result = model_query( 1401 context, models.ShareInstance, session=session, 1402 ).filter_by( 1403 id=share_instance_id, 1404 ).options( 1405 joinedload('export_locations'), 1406 joinedload('share_type'), 1407 ).first() 1408 if result is None: 1409 raise exception.NotFound() 1410 1411 if with_share_data: 1412 parent_share = share_get(context, result['share_id'], session=session) 1413 result.set_share_data(parent_share) 1414 1415 return result 1416 1417 1418 @require_admin_context 1419 def share_instances_get_all(context, filters=None): 1420 session = get_session() 1421 query = model_query( 1422 context, models.ShareInstance, session=session, read_deleted="no", 1423 ).options( 1424 joinedload('export_locations'), 1425 ) 1426 1427 filters = filters or {} 1428 1429 export_location_id = filters.get('export_location_id') 1430 export_location_path = filters.get('export_location_path') 1431 if export_location_id or export_location_path: 1432 query = query.join( 1433 models.ShareInstanceExportLocations, 1434 models.ShareInstanceExportLocations.share_instance_id == 1435 models.ShareInstance.id) 1436 if export_location_path: 1437 query = query.filter( 1438 models.ShareInstanceExportLocations.path == 1439 export_location_path) 1440 if export_location_id: 1441 query = query.filter( 1442 models.ShareInstanceExportLocations.uuid == 1443 export_location_id) 1444 1445 # Returns list of share instances that satisfy filters. 1446 query = query.all() 1447 return query 1448 1449 1450 @require_context 1451 def share_instance_delete(context, instance_id, session=None, 1452 need_to_update_usages=False): 1453 if session is None: 1454 session = get_session() 1455 1456 with session.begin(): 1457 share_export_locations_update(context, instance_id, [], delete=True) 1458 instance_ref = share_instance_get(context, instance_id, 1459 session=session) 1460 instance_ref.soft_delete(session=session, update_status=True) 1461 share = share_get(context, instance_ref['share_id'], session=session) 1462 if len(share.instances) == 0: 1463 share_access_delete_all_by_share(context, share['id']) 1464 session.query(models.ShareMetadata).filter_by( 1465 share_id=share['id']).soft_delete() 1466 share.soft_delete(session=session) 1467 1468 if need_to_update_usages: 1469 reservations = None 1470 try: 1471 # we give the user_id of the share, to update 1472 # the quota usage for the user, who created the share 1473 reservations = QUOTAS.reserve( 1474 context, 1475 project_id=share['project_id'], 1476 shares=-1, 1477 gigabytes=-share['size'], 1478 user_id=share['user_id'], 1479 share_type_id=instance_ref['share_type_id']) 1480 QUOTAS.commit( 1481 context, reservations, project_id=share['project_id'], 1482 user_id=share['user_id'], 1483 share_type_id=instance_ref['share_type_id']) 1484 except Exception: 1485 LOG.exception( 1486 "Failed to update usages deleting share '%s'.", 1487 share["id"]) 1488 if reservations: 1489 QUOTAS.rollback( 1490 context, reservations, 1491 share_type_id=instance_ref['share_type_id']) 1492 1493 1494 def _set_instances_share_data(context, instances, session): 1495 if instances and not isinstance(instances, list): 1496 instances = [instances] 1497 1498 instances_with_share_data = [] 1499 for instance in instances: 1500 try: 1501 parent_share = share_get(context, instance['share_id'], 1502 session=session) 1503 except exception.NotFound: 1504 continue 1505 instance.set_share_data(parent_share) 1506 instances_with_share_data.append(instance) 1507 return instances_with_share_data 1508 1509 1510 @require_admin_context 1511 def share_instances_get_all_by_host(context, host, with_share_data=False, 1512 session=None): 1513 """Retrieves all share instances hosted on a host.""" 1514 session = session or get_session() 1515 instances = ( 1516 model_query(context, models.ShareInstance).filter( 1517 or_( 1518 models.ShareInstance.host == host, 1519 models.ShareInstance.host.like("{0}#%".format(host)) 1520 ) 1521 ).all() 1522 ) 1523 1524 if with_share_data: 1525 instances = _set_instances_share_data(context, instances, session) 1526 return instances 1527 1528 1529 @require_context 1530 def share_instances_get_all_by_share_network(context, share_network_id): 1531 """Returns list of share instances that belong to given share network.""" 1532 result = ( 1533 model_query(context, models.ShareInstance).filter( 1534 models.ShareInstance.share_network_id == share_network_id, 1535 ).all() 1536 ) 1537 return result 1538 1539 1540 @require_context 1541 def share_instances_get_all_by_share_server(context, share_server_id): 1542 """Returns list of share instance with given share server.""" 1543 result = ( 1544 model_query(context, models.ShareInstance).filter( 1545 models.ShareInstance.share_server_id == share_server_id, 1546 ).all() 1547 ) 1548 return result 1549 1550 1551 @require_context 1552 def share_instances_get_all_by_share(context, share_id): 1553 """Returns list of share instances that belong to given share.""" 1554 result = ( 1555 model_query(context, models.ShareInstance).filter( 1556 models.ShareInstance.share_id == share_id, 1557 ).all() 1558 ) 1559 return result 1560 1561 1562 @require_context 1563 def share_instances_get_all_by_share_group_id(context, share_group_id): 1564 """Returns list of share instances that belong to given share group.""" 1565 result = ( 1566 model_query(context, models.Share).filter( 1567 models.Share.share_group_id == share_group_id, 1568 ).all() 1569 ) 1570 instances = [] 1571 for share in result: 1572 instance = share.instance 1573 instance.set_share_data(share) 1574 instances.append(instance) 1575 1576 return instances 1577 1578 1579 ################ 1580 1581 def _share_replica_get_with_filters(context, share_id=None, replica_id=None, 1582 replica_state=None, status=None, 1583 with_share_server=True, session=None): 1584 1585 query = model_query(context, models.ShareInstance, session=session, 1586 read_deleted="no") 1587 1588 if share_id is not None: 1589 query = query.filter(models.ShareInstance.share_id == share_id) 1590 1591 if replica_id is not None: 1592 query = query.filter(models.ShareInstance.id == replica_id) 1593 1594 if replica_state is not None: 1595 query = query.filter( 1596 models.ShareInstance.replica_state == replica_state) 1597 else: 1598 query = query.filter(models.ShareInstance.replica_state.isnot(None)) 1599 1600 if status is not None: 1601 query = query.filter(models.ShareInstance.status == status) 1602 1603 if with_share_server: 1604 query = query.options(joinedload('share_server')) 1605 1606 return query 1607 1608 1609 def _set_replica_share_data(context, replicas, session): 1610 if replicas and not isinstance(replicas, list): 1611 replicas = [replicas] 1612 1613 for replica in replicas: 1614 parent_share = share_get(context, replica['share_id'], session=session) 1615 replica.set_share_data(parent_share) 1616 1617 return replicas 1618 1619 1620 @require_context 1621 def share_replicas_get_all(context, with_share_data=False, 1622 with_share_server=True, session=None): 1623 """Returns replica instances for all available replicated shares.""" 1624 session = session or get_session() 1625 1626 result = _share_replica_get_with_filters( 1627 context, with_share_server=with_share_server, session=session).all() 1628 1629 if with_share_data: 1630 result = _set_replica_share_data(context, result, session) 1631 1632 return result 1633 1634 1635 @require_context 1636 def share_replicas_get_all_by_share(context, share_id, 1637 with_share_data=False, 1638 with_share_server=False, session=None): 1639 """Returns replica instances for a given share.""" 1640 session = session or get_session() 1641 1642 result = _share_replica_get_with_filters( 1643 context, with_share_server=with_share_server, 1644 share_id=share_id, session=session).all() 1645 1646 if with_share_data: 1647 result = _set_replica_share_data(context, result, session) 1648 1649 return result 1650 1651 1652 @require_context 1653 def share_replicas_get_available_active_replica(context, share_id, 1654 with_share_data=False, 1655 with_share_server=False, 1656 session=None): 1657 """Returns an 'active' replica instance that is 'available'.""" 1658 session = session or get_session() 1659 1660 result = _share_replica_get_with_filters( 1661 context, with_share_server=with_share_server, share_id=share_id, 1662 replica_state=constants.REPLICA_STATE_ACTIVE, 1663 status=constants.STATUS_AVAILABLE, session=session).first() 1664 1665 if result and with_share_data: 1666 result = _set_replica_share_data(context, result, session)[0] 1667 1668 return result 1669 1670 1671 @require_context 1672 def share_replica_get(context, replica_id, with_share_data=False, 1673 with_share_server=False, session=None): 1674 """Returns summary of requested replica if available.""" 1675 session = session or get_session() 1676 1677 result = _share_replica_get_with_filters( 1678 context, with_share_server=with_share_server, 1679 replica_id=replica_id, session=session).first() 1680 1681 if result is None: 1682 raise exception.ShareReplicaNotFound(replica_id=replica_id) 1683 1684 if with_share_data: 1685 result = _set_replica_share_data(context, result, session)[0] 1686 1687 return result 1688 1689 1690 @require_context 1691 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 1692 def share_replica_update(context, share_replica_id, values, 1693 with_share_data=False, session=None): 1694 """Updates a share replica with specified values.""" 1695 session = session or get_session() 1696 1697 with session.begin(): 1698 _ensure_availability_zone_exists(context, values, session, 1699 strict=False) 1700 updated_share_replica = _share_instance_update( 1701 context, share_replica_id, values, session=session) 1702 1703 if with_share_data: 1704 updated_share_replica = _set_replica_share_data( 1705 context, updated_share_replica, session)[0] 1706 1707 return updated_share_replica 1708 1709 1710 @require_context 1711 def share_replica_delete(context, share_replica_id, session=None): 1712 """Deletes a share replica.""" 1713 session = session or get_session() 1714 1715 share_instance_delete(context, share_replica_id, session=session) 1716 1717 1718 ################ 1719 1720 1721 def _share_get_query(context, session=None): 1722 if session is None: 1723 session = get_session() 1724 return (model_query(context, models.Share, session=session). 1725 options(joinedload('share_metadata'))) 1726 1727 1728 def _metadata_refs(metadata_dict, meta_class): 1729 metadata_refs = [] 1730 if metadata_dict: 1731 for k, v in metadata_dict.items(): 1732 value = six.text_type(v) if isinstance(v, bool) else v 1733 1734 metadata_ref = meta_class() 1735 metadata_ref['key'] = k 1736 metadata_ref['value'] = value 1737 metadata_refs.append(metadata_ref) 1738 return metadata_refs 1739 1740 1741 @require_context 1742 def share_create(context, share_values, create_share_instance=True): 1743 values = copy.deepcopy(share_values) 1744 values = ensure_model_dict_has_id(values) 1745 values['share_metadata'] = _metadata_refs(values.get('metadata'), 1746 models.ShareMetadata) 1747 session = get_session() 1748 share_ref = models.Share() 1749 share_instance_values, share_values = _extract_share_instance_values( 1750 values) 1751 _ensure_availability_zone_exists(context, share_instance_values, session, 1752 strict=False) 1753 share_ref.update(share_values) 1754 1755 with session.begin(): 1756 share_ref.save(session=session) 1757 1758 if create_share_instance: 1759 _share_instance_create(context, share_ref['id'], 1760 share_instance_values, session=session) 1761 1762 # NOTE(u_glide): Do so to prevent errors with relationships 1763 return share_get(context, share_ref['id'], session=session) 1764 1765 1766 @require_admin_context 1767 def share_data_get_for_project(context, project_id, user_id, 1768 share_type_id=None, session=None): 1769 query = (model_query(context, models.Share, 1770 func.count(models.Share.id), 1771 func.sum(models.Share.size), 1772 read_deleted="no", 1773 session=session). 1774 filter_by(project_id=project_id)) 1775 if share_type_id: 1776 query = query.join("instances").filter_by(share_type_id=share_type_id) 1777 elif user_id: 1778 query = query.filter_by(user_id=user_id) 1779 result = query.first() 1780 return (result[0] or 0, result[1] or 0) 1781 1782 1783 @require_context 1784 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 1785 def share_update(context, share_id, update_values): 1786 session = get_session() 1787 values = copy.deepcopy(update_values) 1788 1789 share_instance_values, share_values = _extract_share_instance_values( 1790 values) 1791 _ensure_availability_zone_exists(context, share_instance_values, session, 1792 strict=False) 1793 1794 with session.begin(): 1795 share_ref = share_get(context, share_id, session=session) 1796 1797 _share_instance_update(context, share_ref.instance['id'], 1798 share_instance_values, session=session) 1799 1800 share_ref.update(share_values) 1801 share_ref.save(session=session) 1802 return share_ref 1803 1804 1805 @require_context 1806 def share_get(context, share_id, session=None): 1807 result = _share_get_query(context, session).filter_by(id=share_id).first() 1808 1809 if result is None: 1810 raise exception.NotFound() 1811 1812 return result 1813 1814 1815 def _share_get_all_with_filters(context, project_id=None, share_server_id=None, 1816 share_group_id=None, filters=None, 1817 is_public=False, sort_key=None, 1818 sort_dir=None): 1819 """Returns sorted list of shares that satisfies filters. 1820 1821 :param context: context to query under 1822 :param project_id: project id that owns shares 1823 :param share_server_id: share server that hosts shares 1824 :param filters: dict of filters to specify share selection 1825 :param is_public: public shares from other projects will be added 1826 to result if True 1827 :param sort_key: key of models.Share to be used for sorting 1828 :param sort_dir: desired direction of sorting, can be 'asc' and 'desc' 1829 :returns: list -- models.Share 1830 :raises: exception.InvalidInput 1831 """ 1832 if not sort_key: 1833 sort_key = 'created_at' 1834 if not sort_dir: 1835 sort_dir = 'desc' 1836 query = ( 1837 _share_get_query(context).join( 1838 models.ShareInstance, 1839 models.ShareInstance.share_id == models.Share.id 1840 ) 1841 ) 1842 1843 if project_id: 1844 if is_public: 1845 query = query.filter(or_(models.Share.project_id == project_id, 1846 models.Share.is_public)) 1847 else: 1848 query = query.filter(models.Share.project_id == project_id) 1849 if share_server_id: 1850 query = query.filter( 1851 models.ShareInstance.share_server_id == share_server_id) 1852 1853 if share_group_id: 1854 query = query.filter( 1855 models.Share.share_group_id == share_group_id) 1856 1857 # Apply filters 1858 if not filters: 1859 filters = {} 1860 1861 export_location_id = filters.get('export_location_id') 1862 export_location_path = filters.get('export_location_path') 1863 if export_location_id or export_location_path: 1864 query = query.join( 1865 models.ShareInstanceExportLocations, 1866 models.ShareInstanceExportLocations.share_instance_id == 1867 models.ShareInstance.id) 1868 if export_location_path: 1869 query = query.filter( 1870 models.ShareInstanceExportLocations.path == 1871 export_location_path) 1872 if export_location_id: 1873 query = query.filter( 1874 models.ShareInstanceExportLocations.uuid == 1875 export_location_id) 1876 1877 if 'metadata' in filters: 1878 for k, v in filters['metadata'].items(): 1879 # pylint: disable=no-member 1880 query = query.filter( 1881 or_(models.Share.share_metadata.any( 1882 key=k, value=v))) 1883 if 'extra_specs' in filters: 1884 query = query.join( 1885 models.ShareTypeExtraSpecs, 1886 models.ShareTypeExtraSpecs.share_type_id == 1887 models.ShareInstance.share_type_id) 1888 for k, v in filters['extra_specs'].items(): 1889 query = query.filter(or_(models.ShareTypeExtraSpecs.key == k, 1890 models.ShareTypeExtraSpecs.value == v)) 1891 1892 try: 1893 query = apply_sorting(models.Share, query, sort_key, sort_dir) 1894 except AttributeError: 1895 try: 1896 query = apply_sorting( 1897 models.ShareInstance, query, sort_key, sort_dir) 1898 except AttributeError: 1899 msg = _("Wrong sorting key provided - '%s'.") % sort_key 1900 raise exception.InvalidInput(reason=msg) 1901 1902 if 'limit' in filters: 1903 offset = filters.get('offset', 0) 1904 query = query.limit(filters['limit']).offset(offset) 1905 1906 # Returns list of shares that satisfy filters. 1907 query = query.all() 1908 return query 1909 1910 1911 @require_admin_context 1912 def share_get_all(context, filters=None, sort_key=None, sort_dir=None): 1913 query = _share_get_all_with_filters( 1914 context, filters=filters, sort_key=sort_key, sort_dir=sort_dir) 1915 return query 1916 1917 1918 @require_context 1919 def share_get_all_by_project(context, project_id, filters=None, 1920 is_public=False, sort_key=None, sort_dir=None): 1921 """Returns list of shares with given project ID.""" 1922 query = _share_get_all_with_filters( 1923 context, project_id=project_id, filters=filters, is_public=is_public, 1924 sort_key=sort_key, sort_dir=sort_dir, 1925 ) 1926 return query 1927 1928 1929 @require_context 1930 def share_get_all_by_share_group_id(context, share_group_id, 1931 filters=None, sort_key=None, 1932 sort_dir=None): 1933 """Returns list of shares with given group ID.""" 1934 query = _share_get_all_with_filters( 1935 context, share_group_id=share_group_id, 1936 filters=filters, sort_key=sort_key, sort_dir=sort_dir, 1937 ) 1938 return query 1939 1940 1941 @require_context 1942 def share_get_all_by_share_server(context, share_server_id, filters=None, 1943 sort_key=None, sort_dir=None): 1944 """Returns list of shares with given share server.""" 1945 query = _share_get_all_with_filters( 1946 context, share_server_id=share_server_id, filters=filters, 1947 sort_key=sort_key, sort_dir=sort_dir, 1948 ) 1949 return query 1950 1951 1952 @require_context 1953 def share_delete(context, share_id): 1954 session = get_session() 1955 1956 with session.begin(): 1957 share_ref = share_get(context, share_id, session) 1958 1959 if len(share_ref.instances) > 0: 1960 msg = _("Share %(id)s has %(count)s share instances.") % { 1961 'id': share_id, 'count': len(share_ref.instances)} 1962 raise exception.InvalidShare(msg) 1963 1964 share_ref.soft_delete(session=session) 1965 1966 (session.query(models.ShareMetadata). 1967 filter_by(share_id=share_id).soft_delete()) 1968 1969 1970 ################### 1971 1972 1973 def _share_access_get_query(context, session, values, read_deleted='no'): 1974 """Get access record.""" 1975 query = (model_query( 1976 context, models.ShareAccessMapping, session=session, 1977 read_deleted=read_deleted).options( 1978 joinedload('share_access_rules_metadata'))) 1979 return query.filter_by(**values) 1980 1981 1982 def _share_instance_access_query(context, session, access_id=None, 1983 instance_id=None): 1984 filters = {'deleted': 'False'} 1985 1986 if access_id is not None: 1987 filters.update({'access_id': access_id}) 1988 1989 if instance_id is not None: 1990 filters.update({'share_instance_id': instance_id}) 1991 1992 return model_query(context, models.ShareInstanceAccessMapping, 1993 session=session).filter_by(**filters) 1994 1995 1996 def _share_access_metadata_get_item(context, access_id, key, session=None): 1997 result = (_share_access_metadata_get_query( 1998 context, access_id, session=session).filter_by(key=key).first()) 1999 if not result: 2000 raise exception.ShareAccessMetadataNotFound( 2001 metadata_key=key, access_id=access_id) 2002 return result 2003 2004 2005 def _share_access_metadata_get_query(context, access_id, session=None): 2006 return (model_query( 2007 context, models.ShareAccessRulesMetadata, session=session, 2008 read_deleted="no"). 2009 filter_by(access_id=access_id). 2010 options(joinedload('access'))) 2011 2012 2013 @require_context 2014 def share_access_metadata_update(context, access_id, metadata): 2015 session = get_session() 2016 2017 with session.begin(): 2018 # Now update all existing items with new values, or create new meta 2019 # objects 2020 for meta_key, meta_value in metadata.items(): 2021 2022 # update the value whether it exists or not 2023 item = {"value": meta_value} 2024 try: 2025 meta_ref = _share_access_metadata_get_item( 2026 context, access_id, meta_key, session=session) 2027 except exception.ShareAccessMetadataNotFound: 2028 meta_ref = models.ShareAccessRulesMetadata() 2029 item.update({"key": meta_key, "access_id": access_id}) 2030 2031 meta_ref.update(item) 2032 meta_ref.save(session=session) 2033 2034 return metadata 2035 2036 2037 @require_context 2038 def share_access_metadata_delete(context, access_id, key): 2039 session = get_session() 2040 with session.begin(): 2041 metadata = _share_access_metadata_get_item( 2042 context, access_id, key, session=session) 2043 2044 metadata.soft_delete(session) 2045 2046 2047 @require_context 2048 def share_access_create(context, values): 2049 values = ensure_model_dict_has_id(values) 2050 session = get_session() 2051 with session.begin(): 2052 values['share_access_rules_metadata'] = ( 2053 _metadata_refs(values.get('metadata'), 2054 models.ShareAccessRulesMetadata)) 2055 2056 access_ref = models.ShareAccessMapping() 2057 access_ref.update(values) 2058 access_ref.save(session=session) 2059 2060 parent_share = share_get(context, values['share_id'], session=session) 2061 2062 for instance in parent_share.instances: 2063 vals = { 2064 'share_instance_id': instance['id'], 2065 'access_id': access_ref['id'], 2066 } 2067 2068 _share_instance_access_create(vals, session) 2069 2070 return share_access_get(context, access_ref['id']) 2071 2072 2073 @require_context 2074 def share_instance_access_create(context, values, share_instance_id): 2075 values = ensure_model_dict_has_id(values) 2076 session = get_session() 2077 with session.begin(): 2078 access_list = _share_access_get_query( 2079 context, session, { 2080 'share_id': values['share_id'], 2081 'access_type': values['access_type'], 2082 'access_to': values['access_to'], 2083 }).all() 2084 if len(access_list) > 0: 2085 access_ref = access_list[0] 2086 else: 2087 access_ref = models.ShareAccessMapping() 2088 access_ref.update(values) 2089 access_ref.save(session=session) 2090 2091 vals = { 2092 'share_instance_id': share_instance_id, 2093 'access_id': access_ref['id'], 2094 } 2095 2096 _share_instance_access_create(vals, session) 2097 2098 return share_access_get(context, access_ref['id']) 2099 2100 2101 @require_context 2102 def share_instance_access_copy(context, share_id, instance_id, session=None): 2103 """Copy access rules from share to share instance.""" 2104 session = session or get_session() 2105 2106 share_access_rules = _share_access_get_query( 2107 context, session, {'share_id': share_id}).all() 2108 2109 for access_rule in share_access_rules: 2110 values = { 2111 'share_instance_id': instance_id, 2112 'access_id': access_rule['id'], 2113 } 2114 2115 _share_instance_access_create(values, session) 2116 2117 return share_access_rules 2118 2119 2120 def _share_instance_access_create(values, session): 2121 access_ref = models.ShareInstanceAccessMapping() 2122 access_ref.update(ensure_model_dict_has_id(values)) 2123 access_ref.save(session=session) 2124 return access_ref 2125 2126 2127 @require_context 2128 def share_access_get(context, access_id, session=None): 2129 """Get access record.""" 2130 session = session or get_session() 2131 2132 access = _share_access_get_query( 2133 context, session, {'id': access_id}).first() 2134 if access: 2135 return access 2136 else: 2137 raise exception.NotFound() 2138 2139 2140 @require_context 2141 def share_instance_access_get(context, access_id, instance_id, 2142 with_share_access_data=True): 2143 """Get access record.""" 2144 session = get_session() 2145 2146 access = _share_instance_access_query(context, session, access_id, 2147 instance_id).first() 2148 if access is None: 2149 raise exception.NotFound() 2150 2151 if with_share_access_data: 2152 access = _set_instances_share_access_data(context, access, session)[0] 2153 2154 return access 2155 2156 2157 @require_context 2158 def share_access_get_all_for_share(context, share_id, filters=None, 2159 session=None): 2160 filters = filters or {} 2161 session = session or get_session() 2162 query = (_share_access_get_query( 2163 context, session, {'share_id': share_id}).filter( 2164 models.ShareAccessMapping.instance_mappings.any())) 2165 2166 if 'metadata' in filters: 2167 for k, v in filters['metadata'].items(): 2168 query = query.filter( 2169 or_(models.ShareAccessMapping. 2170 share_access_rules_metadata.any(key=k, value=v))) 2171 2172 return query.all() 2173 2174 2175 @require_context 2176 def share_access_get_all_for_instance(context, instance_id, filters=None, 2177 with_share_access_data=True, 2178 session=None): 2179 """Get all access rules related to a certain share instance.""" 2180 session = session or get_session() 2181 filters = copy.deepcopy(filters) if filters else {} 2182 filters.update({'share_instance_id': instance_id}) 2183 legal_filter_keys = ('id', 'share_instance_id', 'access_id', 'state') 2184 query = _share_instance_access_query(context, session) 2185 2186 query = exact_filter( 2187 query, models.ShareInstanceAccessMapping, filters, legal_filter_keys) 2188 2189 instance_accesses = query.all() 2190 2191 if with_share_access_data: 2192 instance_accesses = _set_instances_share_access_data( 2193 context, instance_accesses, session) 2194 2195 return instance_accesses 2196 2197 2198 def _set_instances_share_access_data(context, instance_accesses, session): 2199 if instance_accesses and not isinstance(instance_accesses, list): 2200 instance_accesses = [instance_accesses] 2201 2202 for instance_access in instance_accesses: 2203 share_access = share_access_get( 2204 context, instance_access['access_id'], session=session) 2205 instance_access.set_share_access_data(share_access) 2206 2207 return instance_accesses 2208 2209 2210 def _set_instances_snapshot_access_data(context, instance_accesses, session): 2211 if instance_accesses and not isinstance(instance_accesses, list): 2212 instance_accesses = [instance_accesses] 2213 2214 for instance_access in instance_accesses: 2215 snapshot_access = share_snapshot_access_get( 2216 context, instance_access['access_id'], session=session) 2217 instance_access.set_snapshot_access_data(snapshot_access) 2218 2219 return instance_accesses 2220 2221 2222 @require_context 2223 def share_access_get_all_by_type_and_access(context, share_id, access_type, 2224 access): 2225 session = get_session() 2226 return _share_access_get_query(context, session, 2227 {'share_id': share_id, 2228 'access_type': access_type, 2229 'access_to': access}).all() 2230 2231 2232 @require_context 2233 def share_access_check_for_existing_access(context, share_id, access_type, 2234 access_to): 2235 return _check_for_existing_access( 2236 context, 'share', share_id, access_type, access_to) 2237 2238 2239 def _check_for_existing_access(context, resource, resource_id, access_type, 2240 access_to): 2241 2242 session = get_session() 2243 if resource == 'share': 2244 query_method = _share_access_get_query 2245 access_to_field = models.ShareAccessMapping.access_to 2246 else: 2247 query_method = _share_snapshot_access_get_query 2248 access_to_field = models.ShareSnapshotAccessMapping.access_to 2249 2250 with session.begin(): 2251 if access_type == 'ip': 2252 rules = query_method( 2253 context, session, {'%s_id' % resource: resource_id, 2254 'access_type': access_type}).filter( 2255 access_to_field.startswith(access_to.split('/')[0])).all() 2256 2257 matching_rules = [ 2258 rule for rule in rules if 2259 ipaddress.ip_network(six.text_type(access_to)) == 2260 ipaddress.ip_network(six.text_type(rule['access_to'])) 2261 ] 2262 return len(matching_rules) > 0 2263 else: 2264 return query_method( 2265 context, session, {'%s_id' % resource: resource_id, 2266 'access_type': access_type, 2267 'access_to': access_to}).count() > 0 2268 2269 2270 @require_context 2271 def share_access_delete_all_by_share(context, share_id): 2272 session = get_session() 2273 with session.begin(): 2274 (session.query(models.ShareAccessMapping). 2275 filter_by(share_id=share_id).soft_delete()) 2276 2277 2278 @require_context 2279 def share_instance_access_delete(context, mapping_id): 2280 session = get_session() 2281 with session.begin(): 2282 2283 mapping = (session.query(models.ShareInstanceAccessMapping). 2284 filter_by(id=mapping_id).first()) 2285 2286 if not mapping: 2287 exception.NotFound() 2288 2289 mapping.soft_delete(session, update_status=True, 2290 status_field_name='state') 2291 2292 other_mappings = _share_instance_access_query( 2293 context, session, mapping['access_id']).all() 2294 2295 # NOTE(u_glide): Remove access rule if all mappings were removed. 2296 if len(other_mappings) == 0: 2297 (session.query(models.ShareAccessRulesMetadata).filter_by( 2298 access_id=mapping['access_id']).soft_delete()) 2299 2300 (session.query(models.ShareAccessMapping).filter_by( 2301 id=mapping['access_id']).soft_delete()) 2302 2303 2304 @require_context 2305 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 2306 def share_instance_access_update(context, access_id, instance_id, updates): 2307 session = get_session() 2308 share_access_fields = ('access_type', 'access_to', 'access_key', 2309 'access_level') 2310 2311 share_access_map_updates, share_instance_access_map_updates = ( 2312 _extract_subdict_by_fields(updates, share_access_fields) 2313 ) 2314 2315 with session.begin(): 2316 share_access = _share_access_get_query( 2317 context, session, {'id': access_id}).first() 2318 share_access.update(share_access_map_updates) 2319 share_access.save(session=session) 2320 2321 access = _share_instance_access_query( 2322 context, session, access_id, instance_id).first() 2323 access.update(share_instance_access_map_updates) 2324 access.save(session=session) 2325 2326 return access 2327 2328 ################### 2329 2330 2331 @require_context 2332 def share_snapshot_instance_create(context, snapshot_id, values, session=None): 2333 session = session or get_session() 2334 values = copy.deepcopy(values) 2335 2336 _change_size_to_instance_size(values) 2337 2338 if not values.get('id'): 2339 values['id'] = uuidutils.generate_uuid() 2340 values.update({'snapshot_id': snapshot_id}) 2341 2342 instance_ref = models.ShareSnapshotInstance() 2343 instance_ref.update(values) 2344 instance_ref.save(session=session) 2345 2346 return share_snapshot_instance_get(context, instance_ref['id'], 2347 session=session) 2348 2349 2350 @require_context 2351 def share_snapshot_instance_update(context, instance_id, values): 2352 session = get_session() 2353 instance_ref = share_snapshot_instance_get(context, instance_id, 2354 session=session) 2355 _change_size_to_instance_size(values) 2356 2357 # NOTE(u_glide): Ignore updates to custom properties 2358 for extra_key in models.ShareSnapshotInstance._extra_keys: 2359 if extra_key in values: 2360 values.pop(extra_key) 2361 2362 instance_ref.update(values) 2363 instance_ref.save(session=session) 2364 return instance_ref 2365 2366 2367 @require_context 2368 def share_snapshot_instance_delete(context, snapshot_instance_id, 2369 session=None): 2370 session = session or get_session() 2371 2372 with session.begin(): 2373 2374 snapshot_instance_ref = share_snapshot_instance_get( 2375 context, snapshot_instance_id, session=session) 2376 2377 access_rules = share_snapshot_access_get_all_for_snapshot_instance( 2378 context, snapshot_instance_id, session=session) 2379 for rule in access_rules: 2380 share_snapshot_instance_access_delete( 2381 context, rule['access_id'], snapshot_instance_id) 2382 2383 for el in snapshot_instance_ref.export_locations: 2384 share_snapshot_instance_export_location_delete(context, el['id']) 2385 2386 snapshot_instance_ref.soft_delete( 2387 session=session, update_status=True) 2388 snapshot = share_snapshot_get( 2389 context, snapshot_instance_ref['snapshot_id'], session=session) 2390 if len(snapshot.instances) == 0: 2391 snapshot.soft_delete(session=session) 2392 2393 2394 @require_context 2395 def share_snapshot_instance_get(context, snapshot_instance_id, session=None, 2396 with_share_data=False): 2397 2398 session = session or get_session() 2399 2400 result = _share_snapshot_instance_get_with_filters( 2401 context, instance_ids=[snapshot_instance_id], session=session).first() 2402 2403 if result is None: 2404 raise exception.ShareSnapshotInstanceNotFound( 2405 instance_id=snapshot_instance_id) 2406 2407 if with_share_data: 2408 result = _set_share_snapshot_instance_data(context, result, session)[0] 2409 2410 return result 2411 2412 2413 @require_context 2414 def share_snapshot_instance_get_all_with_filters(context, search_filters, 2415 with_share_data=False, 2416 session=None): 2417 """Get snapshot instances filtered by known attrs, ignore unknown attrs. 2418 2419 All filters accept list/tuples to filter on, along with simple values. 2420 """ 2421 def listify(values): 2422 if values: 2423 if not isinstance(values, (list, tuple, set)): 2424 return values, 2425 else: 2426 return values 2427 2428 session = session or get_session() 2429 _known_filters = ('instance_ids', 'snapshot_ids', 'share_instance_ids', 2430 'statuses') 2431 2432 filters = {k: listify(search_filters.get(k)) for k in _known_filters} 2433 2434 result = _share_snapshot_instance_get_with_filters( 2435 context, session=session, **filters).all() 2436 2437 if with_share_data: 2438 result = _set_share_snapshot_instance_data(context, result, session) 2439 2440 return result 2441 2442 2443 def _share_snapshot_instance_get_with_filters(context, instance_ids=None, 2444 snapshot_ids=None, statuses=None, 2445 share_instance_ids=None, 2446 session=None): 2447 2448 query = model_query(context, models.ShareSnapshotInstance, session=session, 2449 read_deleted="no") 2450 2451 if instance_ids is not None: 2452 query = query.filter( 2453 models.ShareSnapshotInstance.id.in_(instance_ids)) 2454 2455 if snapshot_ids is not None: 2456 query = query.filter( 2457 models.ShareSnapshotInstance.snapshot_id.in_(snapshot_ids)) 2458 2459 if share_instance_ids is not None: 2460 query = query.filter(models.ShareSnapshotInstance.share_instance_id 2461 .in_(share_instance_ids)) 2462 2463 if statuses is not None: 2464 query = query.filter(models.ShareSnapshotInstance.status.in_(statuses)) 2465 2466 query = query.options(joinedload('share_group_snapshot')) 2467 return query 2468 2469 2470 def _set_share_snapshot_instance_data(context, snapshot_instances, session): 2471 if snapshot_instances and not isinstance(snapshot_instances, list): 2472 snapshot_instances = [snapshot_instances] 2473 2474 for snapshot_instance in snapshot_instances: 2475 share_instance = share_instance_get( 2476 context, snapshot_instance['share_instance_id'], session=session, 2477 with_share_data=True) 2478 snapshot_instance['share'] = share_instance 2479 2480 return snapshot_instances 2481 2482 2483 ################### 2484 2485 2486 @require_context 2487 def share_snapshot_create(context, create_values, 2488 create_snapshot_instance=True): 2489 values = copy.deepcopy(create_values) 2490 values = ensure_model_dict_has_id(values) 2491 2492 snapshot_ref = models.ShareSnapshot() 2493 snapshot_instance_values, snapshot_values = ( 2494 _extract_snapshot_instance_values(values) 2495 ) 2496 share_ref = share_get(context, snapshot_values.get('share_id')) 2497 snapshot_instance_values.update( 2498 {'share_instance_id': share_ref.instance.id} 2499 ) 2500 2501 snapshot_ref.update(snapshot_values) 2502 session = get_session() 2503 with session.begin(): 2504 snapshot_ref.save(session=session) 2505 2506 if create_snapshot_instance: 2507 share_snapshot_instance_create( 2508 context, 2509 snapshot_ref['id'], 2510 snapshot_instance_values, 2511 session=session 2512 ) 2513 return share_snapshot_get( 2514 context, snapshot_values['id'], session=session) 2515 2516 2517 @require_admin_context 2518 def snapshot_data_get_for_project(context, project_id, user_id, 2519 share_type_id=None, session=None): 2520 query = (model_query(context, models.ShareSnapshot, 2521 func.count(models.ShareSnapshot.id), 2522 func.sum(models.ShareSnapshot.size), 2523 read_deleted="no", 2524 session=session). 2525 filter_by(project_id=project_id)) 2526 2527 if share_type_id: 2528 query = query.join( 2529 models.ShareInstance, 2530 models.ShareInstance.share_id == models.ShareSnapshot.share_id, 2531 ).filter_by(share_type_id=share_type_id) 2532 elif user_id: 2533 query = query.filter_by(user_id=user_id) 2534 result = query.first() 2535 2536 return (result[0] or 0, result[1] or 0) 2537 2538 2539 @require_context 2540 def share_snapshot_get(context, snapshot_id, session=None): 2541 result = (model_query(context, models.ShareSnapshot, session=session, 2542 project_only=True). 2543 filter_by(id=snapshot_id). 2544 options(joinedload('share')). 2545 options(joinedload('instances')). 2546 first()) 2547 2548 if not result: 2549 raise exception.ShareSnapshotNotFound(snapshot_id=snapshot_id) 2550 2551 return result 2552 2553 2554 def _share_snapshot_get_all_with_filters(context, project_id=None, 2555 share_id=None, filters=None, 2556 sort_key=None, sort_dir=None): 2557 # Init data 2558 sort_key = sort_key or 'share_id' 2559 sort_dir = sort_dir or 'desc' 2560 filters = filters or {} 2561 query = model_query(context, models.ShareSnapshot) 2562 2563 if project_id: 2564 query = query.filter_by(project_id=project_id) 2565 if share_id: 2566 query = query.filter_by(share_id=share_id) 2567 query = query.options(joinedload('share')) 2568 query = query.options(joinedload('instances')) 2569 2570 # Apply filters 2571 if 'usage' in filters: 2572 usage_filter_keys = ['any', 'used', 'unused'] 2573 if filters['usage'] == 'any': 2574 pass 2575 elif filters['usage'] == 'used': 2576 query = query.filter(or_(models.Share.snapshot_id == ( 2577 models.ShareSnapshot.id))) 2578 elif filters['usage'] == 'unused': 2579 query = query.filter(or_(models.Share.snapshot_id != ( 2580 models.ShareSnapshot.id))) 2581 else: 2582 msg = _("Wrong 'usage' key provided - '%(key)s'. " 2583 "Expected keys are '%(ek)s'.") % { 2584 'key': filters['usage'], 2585 'ek': six.text_type(usage_filter_keys)} 2586 raise exception.InvalidInput(reason=msg) 2587 2588 # Apply sorting 2589 try: 2590 attr = getattr(models.ShareSnapshot, sort_key) 2591 except AttributeError: 2592 msg = _("Wrong sorting key provided - '%s'.") % sort_key 2593 raise exception.InvalidInput(reason=msg) 2594 if sort_dir.lower() == 'desc': 2595 query = query.order_by(attr.desc()) 2596 elif sort_dir.lower() == 'asc': 2597 query = query.order_by(attr.asc()) 2598 else: 2599 msg = _("Wrong sorting data provided: sort key is '%(sort_key)s' " 2600 "and sort direction is '%(sort_dir)s'.") % { 2601 "sort_key": sort_key, "sort_dir": sort_dir} 2602 raise exception.InvalidInput(reason=msg) 2603 2604 # Returns list of shares that satisfy filters 2605 return query.all() 2606 2607 2608 @require_admin_context 2609 def share_snapshot_get_all(context, filters=None, sort_key=None, 2610 sort_dir=None): 2611 return _share_snapshot_get_all_with_filters( 2612 context, filters=filters, sort_key=sort_key, sort_dir=sort_dir, 2613 ) 2614 2615 2616 @require_context 2617 def share_snapshot_get_all_by_project(context, project_id, filters=None, 2618 sort_key=None, sort_dir=None): 2619 authorize_project_context(context, project_id) 2620 return _share_snapshot_get_all_with_filters( 2621 context, project_id=project_id, 2622 filters=filters, sort_key=sort_key, sort_dir=sort_dir, 2623 ) 2624 2625 2626 @require_context 2627 def share_snapshot_get_all_for_share(context, share_id, filters=None, 2628 sort_key=None, sort_dir=None): 2629 return _share_snapshot_get_all_with_filters( 2630 context, share_id=share_id, 2631 filters=filters, sort_key=sort_key, sort_dir=sort_dir, 2632 ) 2633 2634 2635 @require_context 2636 def share_snapshot_get_latest_for_share(context, share_id): 2637 2638 snapshots = _share_snapshot_get_all_with_filters( 2639 context, share_id=share_id, sort_key='created_at', sort_dir='desc') 2640 return snapshots[0] if snapshots else None 2641 2642 2643 @require_context 2644 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 2645 def share_snapshot_update(context, snapshot_id, values): 2646 session = get_session() 2647 with session.begin(): 2648 snapshot_ref = share_snapshot_get(context, snapshot_id, 2649 session=session) 2650 2651 instance_values, snapshot_values = ( 2652 _extract_snapshot_instance_values(values) 2653 ) 2654 2655 if snapshot_values: 2656 snapshot_ref.update(snapshot_values) 2657 snapshot_ref.save(session=session) 2658 2659 if instance_values: 2660 snapshot_ref.instance.update(instance_values) 2661 snapshot_ref.instance.save(session=session) 2662 2663 return snapshot_ref 2664 2665 ################################# 2666 2667 2668 @require_context 2669 def share_snapshot_access_create(context, values): 2670 values = ensure_model_dict_has_id(values) 2671 session = get_session() 2672 with session.begin(): 2673 access_ref = models.ShareSnapshotAccessMapping() 2674 access_ref.update(values) 2675 access_ref.save(session=session) 2676 2677 snapshot = share_snapshot_get(context, values['share_snapshot_id'], 2678 session=session) 2679 2680 for instance in snapshot.instances: 2681 vals = { 2682 'share_snapshot_instance_id': instance['id'], 2683 'access_id': access_ref['id'], 2684 } 2685 2686 _share_snapshot_instance_access_create(vals, session) 2687 2688 return share_snapshot_access_get(context, access_ref['id']) 2689 2690 2691 def _share_snapshot_access_get_query(context, session, filters, 2692 read_deleted='no'): 2693 2694 query = model_query(context, models.ShareSnapshotAccessMapping, 2695 session=session, read_deleted=read_deleted) 2696 return query.filter_by(**filters) 2697 2698 2699 def _share_snapshot_instance_access_get_query(context, session, 2700 access_id=None, 2701 share_snapshot_instance_id=None): 2702 filters = {'deleted': 'False'} 2703 2704 if access_id is not None: 2705 filters.update({'access_id': access_id}) 2706 2707 if share_snapshot_instance_id is not None: 2708 filters.update( 2709 {'share_snapshot_instance_id': share_snapshot_instance_id}) 2710 2711 return model_query(context, models.ShareSnapshotInstanceAccessMapping, 2712 session=session).filter_by(**filters) 2713 2714 2715 @require_context 2716 def share_snapshot_instance_access_get_all(context, access_id, session): 2717 rules = _share_snapshot_instance_access_get_query( 2718 context, session, access_id=access_id).all() 2719 return rules 2720 2721 2722 @require_context 2723 def share_snapshot_access_get(context, access_id, session=None): 2724 session = session or get_session() 2725 2726 access = _share_snapshot_access_get_query( 2727 context, session, {'id': access_id}).first() 2728 2729 if access: 2730 return access 2731 else: 2732 raise exception.NotFound() 2733 2734 2735 def _share_snapshot_instance_access_create(values, session): 2736 access_ref = models.ShareSnapshotInstanceAccessMapping() 2737 access_ref.update(ensure_model_dict_has_id(values)) 2738 access_ref.save(session=session) 2739 return access_ref 2740 2741 2742 @require_context 2743 def share_snapshot_access_get_all_for_share_snapshot(context, 2744 share_snapshot_id, 2745 filters): 2746 session = get_session() 2747 filters['share_snapshot_id'] = share_snapshot_id 2748 access_list = _share_snapshot_access_get_query( 2749 context, session, filters).all() 2750 2751 return access_list 2752 2753 2754 @require_context 2755 def share_snapshot_check_for_existing_access(context, share_snapshot_id, 2756 access_type, access_to): 2757 return _check_for_existing_access( 2758 context, 'share_snapshot', share_snapshot_id, access_type, access_to) 2759 2760 2761 @require_context 2762 def share_snapshot_access_get_all_for_snapshot_instance( 2763 context, snapshot_instance_id, filters=None, 2764 with_snapshot_access_data=True, session=None): 2765 """Get all access rules related to a certain snapshot instance.""" 2766 session = session or get_session() 2767 filters = copy.deepcopy(filters) if filters else {} 2768 filters.update({'share_snapshot_instance_id': snapshot_instance_id}) 2769 2770 query = _share_snapshot_instance_access_get_query(context, session) 2771 2772 legal_filter_keys = ( 2773 'id', 'share_snapshot_instance_id', 'access_id', 'state') 2774 2775 query = exact_filter( 2776 query, models.ShareSnapshotInstanceAccessMapping, filters, 2777 legal_filter_keys) 2778 2779 instance_accesses = query.all() 2780 2781 if with_snapshot_access_data: 2782 instance_accesses = _set_instances_snapshot_access_data( 2783 context, instance_accesses, session) 2784 2785 return instance_accesses 2786 2787 2788 @require_context 2789 def share_snapshot_instance_access_update( 2790 context, access_id, instance_id, updates): 2791 2792 snapshot_access_fields = ('access_type', 'access_to') 2793 snapshot_access_map_updates, share_instance_access_map_updates = ( 2794 _extract_subdict_by_fields(updates, snapshot_access_fields) 2795 ) 2796 2797 session = get_session() 2798 with session.begin(): 2799 2800 snapshot_access = _share_snapshot_access_get_query( 2801 context, session, {'id': access_id}).first() 2802 if not snapshot_access: 2803 raise exception.NotFound() 2804 snapshot_access.update(snapshot_access_map_updates) 2805 snapshot_access.save(session=session) 2806 2807 access = _share_snapshot_instance_access_get_query( 2808 context, session, access_id=access_id, 2809 share_snapshot_instance_id=instance_id).first() 2810 if not access: 2811 raise exception.NotFound() 2812 access.update(share_instance_access_map_updates) 2813 access.save(session=session) 2814 2815 return access 2816 2817 2818 @require_context 2819 def share_snapshot_instance_access_get( 2820 context, access_id, share_snapshot_instance_id, 2821 with_snapshot_access_data=True): 2822 2823 session = get_session() 2824 2825 with session.begin(): 2826 access = _share_snapshot_instance_access_get_query( 2827 context, session, access_id=access_id, 2828 share_snapshot_instance_id=share_snapshot_instance_id).first() 2829 2830 if access is None: 2831 raise exception.NotFound() 2832 2833 if with_snapshot_access_data: 2834 return _set_instances_snapshot_access_data( 2835 context, access, session)[0] 2836 else: 2837 return access 2838 2839 2840 @require_context 2841 def share_snapshot_instance_access_delete( 2842 context, access_id, snapshot_instance_id): 2843 session = get_session() 2844 with session.begin(): 2845 2846 rule = _share_snapshot_instance_access_get_query( 2847 context, session, access_id=access_id, 2848 share_snapshot_instance_id=snapshot_instance_id).first() 2849 2850 if not rule: 2851 exception.NotFound() 2852 2853 rule.soft_delete(session, update_status=True, 2854 status_field_name='state') 2855 2856 other_mappings = share_snapshot_instance_access_get_all( 2857 context, rule['access_id'], session) 2858 2859 if len(other_mappings) == 0: 2860 ( 2861 session.query(models.ShareSnapshotAccessMapping) 2862 .filter_by(id=rule['access_id']) 2863 .soft_delete(update_status=True, status_field_name='state') 2864 ) 2865 2866 2867 @require_context 2868 def share_snapshot_instance_export_location_create(context, values): 2869 2870 values = ensure_model_dict_has_id(values) 2871 session = get_session() 2872 with session.begin(): 2873 ssiel = models.ShareSnapshotInstanceExportLocation() 2874 ssiel.update(values) 2875 ssiel.save(session=session) 2876 2877 return ssiel 2878 2879 2880 def _share_snapshot_instance_export_locations_get_query(context, session, 2881 values): 2882 query = model_query(context, models.ShareSnapshotInstanceExportLocation, 2883 session=session) 2884 return query.filter_by(**values) 2885 2886 2887 @require_context 2888 def share_snapshot_export_locations_get(context, snapshot_id): 2889 session = get_session() 2890 snapshot = share_snapshot_get(context, snapshot_id, session=session) 2891 ins_ids = [ins['id'] for ins in snapshot.instances] 2892 export_locations = _share_snapshot_instance_export_locations_get_query( 2893 context, session, {}).filter( 2894 models.ShareSnapshotInstanceExportLocation. 2895 share_snapshot_instance_id.in_(ins_ids)).all() 2896 return export_locations 2897 2898 2899 @require_context 2900 def share_snapshot_instance_export_locations_get_all( 2901 context, share_snapshot_instance_id): 2902 2903 session = get_session() 2904 export_locations = _share_snapshot_instance_export_locations_get_query( 2905 context, session, 2906 {'share_snapshot_instance_id': share_snapshot_instance_id}).all() 2907 return export_locations 2908 2909 2910 @require_context 2911 def share_snapshot_instance_export_location_get(context, el_id): 2912 session = get_session() 2913 2914 export_location = _share_snapshot_instance_export_locations_get_query( 2915 context, session, {'id': el_id}).first() 2916 2917 if export_location: 2918 return export_location 2919 else: 2920 raise exception.NotFound() 2921 2922 2923 @require_context 2924 def share_snapshot_instance_export_location_delete(context, el_id): 2925 session = get_session() 2926 with session.begin(): 2927 2928 el = _share_snapshot_instance_export_locations_get_query( 2929 context, session, {'id': el_id}).first() 2930 2931 if not el: 2932 exception.NotFound() 2933 2934 el.soft_delete(session=session) 2935 2936 ################################# 2937 2938 2939 @require_context 2940 @require_share_exists 2941 def share_metadata_get(context, share_id): 2942 return _share_metadata_get(context, share_id) 2943 2944 2945 @require_context 2946 @require_share_exists 2947 def share_metadata_delete(context, share_id, key): 2948 (_share_metadata_get_query(context, share_id). 2949 filter_by(key=key).soft_delete()) 2950 2951 2952 @require_context 2953 @require_share_exists 2954 def share_metadata_update(context, share_id, metadata, delete): 2955 return _share_metadata_update(context, share_id, metadata, delete) 2956 2957 2958 def _share_metadata_get_query(context, share_id, session=None): 2959 return (model_query(context, models.ShareMetadata, session=session, 2960 read_deleted="no"). 2961 filter_by(share_id=share_id). 2962 options(joinedload('share'))) 2963 2964 2965 def _share_metadata_get(context, share_id, session=None): 2966 rows = _share_metadata_get_query(context, share_id, 2967 session=session).all() 2968 result = {} 2969 for row in rows: 2970 result[row['key']] = row['value'] 2971 2972 return result 2973 2974 2975 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 2976 def _share_metadata_update(context, share_id, metadata, delete, session=None): 2977 if not session: 2978 session = get_session() 2979 2980 with session.begin(): 2981 # Set existing metadata to deleted if delete argument is True 2982 if delete: 2983 original_metadata = _share_metadata_get(context, share_id, 2984 session=session) 2985 for meta_key, meta_value in original_metadata.items(): 2986 if meta_key not in metadata: 2987 meta_ref = _share_metadata_get_item(context, share_id, 2988 meta_key, 2989 session=session) 2990 meta_ref.soft_delete(session=session) 2991 2992 meta_ref = None 2993 2994 # Now update all existing items with new values, or create new meta 2995 # objects 2996 for meta_key, meta_value in metadata.items(): 2997 2998 # update the value whether it exists or not 2999 item = {"value": meta_value} 3000 3001 try: 3002 meta_ref = _share_metadata_get_item(context, share_id, 3003 meta_key, 3004 session=session) 3005 except exception.ShareMetadataNotFound: 3006 meta_ref = models.ShareMetadata() 3007 item.update({"key": meta_key, "share_id": share_id}) 3008 3009 meta_ref.update(item) 3010 meta_ref.save(session=session) 3011 3012 return metadata 3013 3014 3015 def _share_metadata_get_item(context, share_id, key, session=None): 3016 result = (_share_metadata_get_query(context, share_id, session=session). 3017 filter_by(key=key). 3018 first()) 3019 3020 if not result: 3021 raise exception.ShareMetadataNotFound(metadata_key=key, 3022 share_id=share_id) 3023 return result 3024 3025 3026 ############################ 3027 # Export locations functions 3028 ############################ 3029 3030 def _share_export_locations_get(context, share_instance_ids, 3031 include_admin_only=True, 3032 ignore_secondary_replicas=False, session=None): 3033 session = session or get_session() 3034 3035 if not isinstance(share_instance_ids, (set, list, tuple)): 3036 share_instance_ids = (share_instance_ids, ) 3037 3038 query = model_query( 3039 context, 3040 models.ShareInstanceExportLocations, 3041 session=session, 3042 read_deleted="no", 3043 ).filter( 3044 models.ShareInstanceExportLocations.share_instance_id.in_( 3045 share_instance_ids), 3046 ).order_by( 3047 "updated_at", 3048 ).options( 3049 joinedload("_el_metadata_bare"), 3050 ) 3051 3052 if not include_admin_only: 3053 query = query.filter_by(is_admin_only=False) 3054 3055 if ignore_secondary_replicas: 3056 replica_state_attr = models.ShareInstance.replica_state 3057 query = query.join("share_instance").filter( 3058 or_(replica_state_attr == None, # noqa 3059 replica_state_attr == constants.REPLICA_STATE_ACTIVE)) 3060 3061 return query.all() 3062 3063 3064 @require_context 3065 @require_share_exists 3066 def share_export_locations_get_by_share_id(context, share_id, 3067 include_admin_only=True, 3068 ignore_migration_destination=False, 3069 ignore_secondary_replicas=False): 3070 share = share_get(context, share_id) 3071 if ignore_migration_destination: 3072 ids = [instance.id for instance in share.instances 3073 if instance['status'] != constants.STATUS_MIGRATING_TO] 3074 else: 3075 ids = [instance.id for instance in share.instances] 3076 rows = _share_export_locations_get( 3077 context, ids, include_admin_only=include_admin_only, 3078 ignore_secondary_replicas=ignore_secondary_replicas) 3079 return rows 3080 3081 3082 @require_context 3083 @require_share_instance_exists 3084 def share_export_locations_get_by_share_instance_id(context, 3085 share_instance_id, 3086 include_admin_only=True): 3087 rows = _share_export_locations_get( 3088 context, [share_instance_id], include_admin_only=include_admin_only) 3089 return rows 3090 3091 3092 @require_context 3093 @require_share_exists 3094 def share_export_locations_get(context, share_id): 3095 # NOTE(vponomaryov): this method is kept for compatibility with 3096 # old approach. New one uses 'share_export_locations_get_by_share_id'. 3097 # Which returns list of dicts instead of list of strings, as this one does. 3098 share = share_get(context, share_id) 3099 rows = _share_export_locations_get( 3100 context, share.instance.id, context.is_admin) 3101 3102 return [location['path'] for location in rows] 3103 3104 3105 @require_context 3106 def share_export_location_get_by_uuid(context, export_location_uuid, 3107 ignore_secondary_replicas=False, 3108 session=None): 3109 session = session or get_session() 3110 3111 query = model_query( 3112 context, 3113 models.ShareInstanceExportLocations, 3114 session=session, 3115 read_deleted="no", 3116 ).filter_by( 3117 uuid=export_location_uuid, 3118 ).options( 3119 joinedload("_el_metadata_bare"), 3120 ) 3121 3122 if ignore_secondary_replicas: 3123 replica_state_attr = models.ShareInstance.replica_state 3124 query = query.join("share_instance").filter( 3125 or_(replica_state_attr == None, # noqa 3126 replica_state_attr == constants.REPLICA_STATE_ACTIVE)) 3127 3128 result = query.first() 3129 if not result: 3130 raise exception.ExportLocationNotFound(uuid=export_location_uuid) 3131 return result 3132 3133 3134 @require_context 3135 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 3136 def share_export_locations_update(context, share_instance_id, export_locations, 3137 delete): 3138 # NOTE(u_glide): 3139 # Backward compatibility code for drivers, 3140 # which return single export_location as string 3141 if not isinstance(export_locations, (list, tuple, set)): 3142 export_locations = (export_locations, ) 3143 export_locations_as_dicts = [] 3144 for el in export_locations: 3145 # NOTE(vponomaryov): transform old export locations view to new one 3146 export_location = el 3147 if isinstance(el, six.string_types): 3148 export_location = { 3149 "path": el, 3150 "is_admin_only": False, 3151 "metadata": {}, 3152 } 3153 elif isinstance(export_location, dict): 3154 if 'metadata' not in export_location: 3155 export_location['metadata'] = {} 3156 else: 3157 raise exception.ManilaException( 3158 _("Wrong export location type '%s'.") % type(export_location)) 3159 export_locations_as_dicts.append(export_location) 3160 export_locations = export_locations_as_dicts 3161 3162 export_locations_paths = [el['path'] for el in export_locations] 3163 3164 session = get_session() 3165 3166 current_el_rows = _share_export_locations_get( 3167 context, share_instance_id, session=session) 3168 3169 def get_path_list_from_rows(rows): 3170 return set([l['path'] for l in rows]) 3171 3172 current_el_paths = get_path_list_from_rows(current_el_rows) 3173 3174 def create_indexed_time_dict(key_list): 3175 base = timeutils.utcnow() 3176 return { 3177 # NOTE(u_glide): Incrementing timestamp by microseconds to make 3178 # timestamp order match index order. 3179 key: base + datetime.timedelta(microseconds=index) 3180 for index, key in enumerate(key_list) 3181 } 3182 3183 indexed_update_time = create_indexed_time_dict(export_locations_paths) 3184 3185 for el in current_el_rows: 3186 if delete and el['path'] not in export_locations_paths: 3187 export_location_metadata_delete(context, el['uuid']) 3188 el.soft_delete(session) 3189 else: 3190 updated_at = indexed_update_time[el['path']] 3191 el.update({ 3192 'updated_at': updated_at, 3193 'deleted': 0, 3194 }) 3195 el.save(session=session) 3196 if el['el_metadata']: 3197 export_location_metadata_update( 3198 context, el['uuid'], el['el_metadata'], session=session) 3199 3200 # Now add new export locations 3201 for el in export_locations: 3202 if el['path'] in current_el_paths: 3203 # Already updated 3204 continue 3205 3206 location_ref = models.ShareInstanceExportLocations() 3207 location_ref.update({ 3208 'uuid': uuidutils.generate_uuid(), 3209 'path': el['path'], 3210 'share_instance_id': share_instance_id, 3211 'updated_at': indexed_update_time[el['path']], 3212 'deleted': 0, 3213 'is_admin_only': el.get('is_admin_only', False), 3214 }) 3215 location_ref.save(session=session) 3216 if not el.get('metadata'): 3217 continue 3218 export_location_metadata_update( 3219 context, location_ref['uuid'], el.get('metadata'), session=session) 3220 3221 return get_path_list_from_rows(_share_export_locations_get( 3222 context, share_instance_id, session=session)) 3223 3224 3225 ##################################### 3226 # Export locations metadata functions 3227 ##################################### 3228 3229 def _export_location_metadata_get_query(context, export_location_uuid, 3230 session=None): 3231 session = session or get_session() 3232 export_location_id = share_export_location_get_by_uuid( 3233 context, export_location_uuid).id 3234 3235 return model_query( 3236 context, models.ShareInstanceExportLocationsMetadata, session=session, 3237 read_deleted="no", 3238 ).filter_by( 3239 export_location_id=export_location_id, 3240 ) 3241 3242 3243 @require_context 3244 def export_location_metadata_get(context, export_location_uuid, session=None): 3245 rows = _export_location_metadata_get_query( 3246 context, export_location_uuid, session=session).all() 3247 result = {} 3248 for row in rows: 3249 result[row["key"]] = row["value"] 3250 return result 3251 3252 3253 @require_context 3254 def export_location_metadata_delete(context, export_location_uuid, keys=None): 3255 session = get_session() 3256 metadata = _export_location_metadata_get_query( 3257 context, export_location_uuid, session=session, 3258 ) 3259 # NOTE(vponomaryov): if keys is None then we delete all metadata. 3260 if keys is not None: 3261 keys = keys if isinstance(keys, (list, set, tuple)) else (keys, ) 3262 metadata = metadata.filter( 3263 models.ShareInstanceExportLocationsMetadata.key.in_(keys)) 3264 metadata = metadata.all() 3265 for meta_ref in metadata: 3266 meta_ref.soft_delete(session=session) 3267 3268 3269 @require_context 3270 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 3271 def export_location_metadata_update(context, export_location_uuid, metadata, 3272 delete=False, session=None): 3273 session = session or get_session() 3274 if delete: 3275 original_metadata = export_location_metadata_get( 3276 context, export_location_uuid, session=session) 3277 keys_for_deletion = set(original_metadata).difference(metadata) 3278 if keys_for_deletion: 3279 export_location_metadata_delete( 3280 context, export_location_uuid, keys=keys_for_deletion) 3281 3282 el = share_export_location_get_by_uuid(context, export_location_uuid) 3283 for meta_key, meta_value in metadata.items(): 3284 # NOTE(vponomaryov): we should use separate session 3285 # for each meta_ref because of autoincrement of integer primary key 3286 # that will not take effect using one session and we will rewrite, 3287 # in that case, single record - first one added with this call. 3288 session = get_session() 3289 3290 if meta_value is None: 3291 LOG.warning("%s should be properly defined in the driver.", 3292 meta_key) 3293 3294 item = {"value": meta_value, "updated_at": timeutils.utcnow()} 3295 3296 meta_ref = _export_location_metadata_get_query( 3297 context, export_location_uuid, session=session, 3298 ).filter_by( 3299 key=meta_key, 3300 ).first() 3301 3302 if not meta_ref: 3303 meta_ref = models.ShareInstanceExportLocationsMetadata() 3304 item.update({ 3305 "key": meta_key, 3306 "export_location_id": el.id, 3307 }) 3308 3309 meta_ref.update(item) 3310 meta_ref.save(session=session) 3311 3312 return metadata 3313 3314 3315 ################################### 3316 3317 3318 @require_context 3319 def security_service_create(context, values): 3320 values = ensure_model_dict_has_id(values) 3321 3322 security_service_ref = models.SecurityService() 3323 security_service_ref.update(values) 3324 session = get_session() 3325 3326 with session.begin(): 3327 security_service_ref.save(session=session) 3328 3329 return security_service_ref 3330 3331 3332 @require_context 3333 def security_service_delete(context, id): 3334 session = get_session() 3335 with session.begin(): 3336 security_service_ref = security_service_get(context, 3337 id, 3338 session=session) 3339 security_service_ref.soft_delete(session) 3340 3341 3342 @require_context 3343 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 3344 def security_service_update(context, id, values): 3345 session = get_session() 3346 with session.begin(): 3347 security_service_ref = security_service_get(context, 3348 id, 3349 session=session) 3350 security_service_ref.update(values) 3351 security_service_ref.save(session=session) 3352 return security_service_ref 3353 3354 3355 @require_context 3356 def security_service_get(context, id, session=None): 3357 result = (_security_service_get_query(context, session=session). 3358 filter_by(id=id).first()) 3359 3360 if result is None: 3361 raise exception.SecurityServiceNotFound(security_service_id=id) 3362 return result 3363 3364 3365 @require_context 3366 def security_service_get_all(context): 3367 return _security_service_get_query(context).all() 3368 3369 3370 @require_context 3371 def security_service_get_all_by_project(context, project_id): 3372 return (_security_service_get_query(context). 3373 filter_by(project_id=project_id).all()) 3374 3375 3376 def _security_service_get_query(context, session=None): 3377 if session is None: 3378 session = get_session() 3379 return model_query(context, models.SecurityService, session=session) 3380 3381 3382 ################### 3383 3384 3385 def _network_get_query(context, session=None): 3386 if session is None: 3387 session = get_session() 3388 return (model_query(context, models.ShareNetwork, session=session, 3389 project_only=True). 3390 options(joinedload('share_instances'), 3391 joinedload('security_services'), 3392 joinedload('share_servers'))) 3393 3394 3395 @require_context 3396 def share_network_create(context, values): 3397 values = ensure_model_dict_has_id(values) 3398 3399 network_ref = models.ShareNetwork() 3400 network_ref.update(values) 3401 session = get_session() 3402 with session.begin(): 3403 network_ref.save(session=session) 3404 return share_network_get(context, values['id'], session) 3405 3406 3407 @require_context 3408 def share_network_delete(context, id): 3409 session = get_session() 3410 with session.begin(): 3411 network_ref = share_network_get(context, id, session=session) 3412 network_ref.soft_delete(session) 3413 3414 3415 @require_context 3416 def share_network_update(context, id, values): 3417 session = get_session() 3418 with session.begin(): 3419 network_ref = share_network_get(context, id, session=session) 3420 network_ref.update(values) 3421 network_ref.save(session=session) 3422 return network_ref 3423 3424 3425 @require_context 3426 def share_network_get(context, id, session=None): 3427 result = _network_get_query(context, session).filter_by(id=id).first() 3428 if result is None: 3429 raise exception.ShareNetworkNotFound(share_network_id=id) 3430 return result 3431 3432 3433 @require_context 3434 def share_network_get_all(context): 3435 return _network_get_query(context).all() 3436 3437 3438 @require_context 3439 def share_network_get_all_by_project(context, project_id): 3440 return _network_get_query(context).filter_by(project_id=project_id).all() 3441 3442 3443 @require_context 3444 def share_network_get_all_by_security_service(context, security_service_id): 3445 session = get_session() 3446 return (model_query(context, models.ShareNetwork, session=session). 3447 join(models.ShareNetworkSecurityServiceAssociation, 3448 models.ShareNetwork.id == 3449 models.ShareNetworkSecurityServiceAssociation.share_network_id). 3450 filter_by(security_service_id=security_service_id, deleted=0). 3451 options(joinedload('share_servers')).all()) 3452 3453 3454 @require_context 3455 def share_network_add_security_service(context, id, security_service_id): 3456 session = get_session() 3457 3458 with session.begin(): 3459 assoc_ref = (model_query( 3460 context, 3461 models.ShareNetworkSecurityServiceAssociation, 3462 session=session). 3463 filter_by(share_network_id=id). 3464 filter_by( 3465 security_service_id=security_service_id).first()) 3466 3467 if assoc_ref: 3468 msg = "Already associated" 3469 raise exception.ShareNetworkSecurityServiceAssociationError( 3470 share_network_id=id, 3471 security_service_id=security_service_id, 3472 reason=msg) 3473 3474 share_nw_ref = share_network_get(context, id, session=session) 3475 security_service_ref = security_service_get(context, 3476 security_service_id, 3477 session=session) 3478 share_nw_ref.security_services += [security_service_ref] 3479 share_nw_ref.save(session=session) 3480 3481 return share_nw_ref 3482 3483 3484 @require_context 3485 def share_network_remove_security_service(context, id, security_service_id): 3486 session = get_session() 3487 3488 with session.begin(): 3489 share_nw_ref = share_network_get(context, id, session=session) 3490 security_service_get(context, security_service_id, session=session) 3491 3492 assoc_ref = (model_query( 3493 context, 3494 models.ShareNetworkSecurityServiceAssociation, 3495 session=session). 3496 filter_by(share_network_id=id). 3497 filter_by(security_service_id=security_service_id).first()) 3498 3499 if assoc_ref: 3500 assoc_ref.soft_delete(session) 3501 else: 3502 msg = "No association defined" 3503 raise exception.ShareNetworkSecurityServiceDissociationError( 3504 share_network_id=id, 3505 security_service_id=security_service_id, 3506 reason=msg) 3507 3508 return share_nw_ref 3509 3510 3511 @require_context 3512 def count_share_networks(context, project_id, user_id=None, 3513 share_type_id=None, session=None): 3514 query = model_query( 3515 context, models.ShareNetwork, 3516 func.count(models.ShareNetwork.id), 3517 read_deleted="no", 3518 session=session).filter_by(project_id=project_id) 3519 if share_type_id: 3520 query = query.join("share_instances").filter_by( 3521 share_type_id=share_type_id) 3522 elif user_id is not None: 3523 query = query.filter_by(user_id=user_id) 3524 return query.first()[0] 3525 3526 3527 ################### 3528 3529 3530 def _server_get_query(context, session=None): 3531 if session is None: 3532 session = get_session() 3533 return (model_query(context, models.ShareServer, session=session). 3534 options(joinedload('share_instances'), 3535 joinedload('network_allocations'), 3536 joinedload('share_network'))) 3537 3538 3539 @require_context 3540 def share_server_create(context, values): 3541 values = ensure_model_dict_has_id(values) 3542 3543 server_ref = models.ShareServer() 3544 server_ref.update(values) 3545 session = get_session() 3546 with session.begin(): 3547 server_ref.save(session=session) 3548 # NOTE(u_glide): Do so to prevent errors with relationships 3549 return share_server_get(context, server_ref['id'], session=session) 3550 3551 3552 @require_context 3553 def share_server_delete(context, id): 3554 session = get_session() 3555 with session.begin(): 3556 server_ref = share_server_get(context, id, session=session) 3557 share_server_backend_details_delete(context, id, session=session) 3558 server_ref.soft_delete(session=session, update_status=True) 3559 3560 3561 @require_context 3562 def share_server_update(context, id, values): 3563 session = get_session() 3564 with session.begin(): 3565 server_ref = share_server_get(context, id, session=session) 3566 server_ref.update(values) 3567 server_ref.save(session=session) 3568 return server_ref 3569 3570 3571 @require_context 3572 def share_server_get(context, server_id, session=None): 3573 result = (_server_get_query(context, session).filter_by(id=server_id) 3574 .first()) 3575 if result is None: 3576 raise exception.ShareServerNotFound(share_server_id=server_id) 3577 return result 3578 3579 3580 @require_context 3581 def share_server_search_by_identifier(context, identifier, session=None): 3582 3583 identifier_field = models.ShareServer.identifier 3584 3585 # try if given identifier is a substring of existing entry's identifier 3586 result = (_server_get_query(context, session).filter( 3587 identifier_field.like('%{}%'.format(identifier))).all()) 3588 3589 if not result: 3590 # repeat it with underscores instead of hyphens 3591 result = (_server_get_query(context, session).filter( 3592 identifier_field.like('%{}%'.format( 3593 identifier.replace("-", "_")))).all()) 3594 3595 if not result: 3596 # repeat it with hypens instead of underscores 3597 result = (_server_get_query(context, session).filter( 3598 identifier_field.like('%{}%'.format( 3599 identifier.replace("_", "-")))).all()) 3600 3601 if not result: 3602 # try if an existing identifier is a substring of given identifier 3603 result = (_server_get_query(context, session).filter( 3604 literal(identifier).contains(identifier_field)).all()) 3605 3606 if not result: 3607 # repeat it with underscores instead of hyphens 3608 result = (_server_get_query(context, session).filter( 3609 literal(identifier.replace("-", "_")).contains( 3610 identifier_field)).all()) 3611 3612 if not result: 3613 # repeat it with hypens instead of underscores 3614 result = (_server_get_query(context, session).filter( 3615 literal(identifier.replace("_", "-")).contains( 3616 identifier_field)).all()) 3617 3618 if not result: 3619 raise exception.ShareServerNotFound(share_server_id=identifier) 3620 3621 return result 3622 3623 3624 @require_context 3625 def share_server_get_all_by_host_and_share_net_valid(context, host, 3626 share_net_id, 3627 session=None): 3628 result = (_server_get_query(context, session).filter_by(host=host) 3629 .filter_by(share_network_id=share_net_id) 3630 .filter(models.ShareServer.status.in_( 3631 (constants.STATUS_CREATING, 3632 constants.STATUS_ACTIVE))).all()) 3633 if not result: 3634 filters_description = ('share_network_id is "%(share_net_id)s",' 3635 ' host is "%(host)s" and status in' 3636 ' "%(status_cr)s" or "%(status_act)s"') % { 3637 'share_net_id': share_net_id, 3638 'host': host, 3639 'status_cr': constants.STATUS_CREATING, 3640 'status_act': constants.STATUS_ACTIVE, 3641 } 3642 raise exception.ShareServerNotFoundByFilters( 3643 filters_description=filters_description) 3644 return result 3645 3646 3647 @require_context 3648 def share_server_get_all(context): 3649 return _server_get_query(context).all() 3650 3651 3652 @require_context 3653 def share_server_get_all_by_host(context, host): 3654 return _server_get_query(context).filter_by(host=host).all() 3655 3656 3657 @require_context 3658 def share_server_get_all_unused_deletable(context, host, updated_before): 3659 valid_server_status = ( 3660 constants.STATUS_INACTIVE, 3661 constants.STATUS_ACTIVE, 3662 constants.STATUS_ERROR, 3663 ) 3664 result = (_server_get_query(context) 3665 .filter_by(is_auto_deletable=True) 3666 .filter_by(host=host) 3667 .filter(~models.ShareServer.share_groups.any()) 3668 .filter(~models.ShareServer.share_instances.any()) 3669 .filter(models.ShareServer.status.in_(valid_server_status)) 3670 .filter(models.ShareServer.updated_at < updated_before).all()) 3671 return result 3672 3673 3674 @require_context 3675 def share_server_backend_details_set(context, share_server_id, server_details): 3676 share_server_get(context, share_server_id) 3677 3678 for meta_key, meta_value in server_details.items(): 3679 meta_ref = models.ShareServerBackendDetails() 3680 meta_ref.update({ 3681 'key': meta_key, 3682 'value': meta_value, 3683 'share_server_id': share_server_id 3684 }) 3685 session = get_session() 3686 with session.begin(): 3687 meta_ref.save(session) 3688 return server_details 3689 3690 3691 @require_context 3692 def share_server_backend_details_delete(context, share_server_id, 3693 session=None): 3694 if not session: 3695 session = get_session() 3696 share_server_details = (model_query(context, 3697 models.ShareServerBackendDetails, 3698 session=session) 3699 .filter_by(share_server_id=share_server_id).all()) 3700 for item in share_server_details: 3701 item.soft_delete(session) 3702 3703 3704 ################### 3705 3706 def _driver_private_data_query(session, context, entity_id, key=None, 3707 read_deleted=False): 3708 query = model_query( 3709 context, models.DriverPrivateData, session=session, 3710 read_deleted=read_deleted, 3711 ).filter_by( 3712 entity_uuid=entity_id, 3713 ) 3714 3715 if isinstance(key, list): 3716 return query.filter(models.DriverPrivateData.key.in_(key)) 3717 elif key is not None: 3718 return query.filter_by(key=key) 3719 3720 return query 3721 3722 3723 @require_context 3724 def driver_private_data_get(context, entity_id, key=None, 3725 default=None, session=None): 3726 if not session: 3727 session = get_session() 3728 3729 query = _driver_private_data_query(session, context, entity_id, key) 3730 3731 if key is None or isinstance(key, list): 3732 return {item.key: item.value for item in query.all()} 3733 else: 3734 result = query.first() 3735 return result["value"] if result is not None else default 3736 3737 3738 @require_context 3739 def driver_private_data_update(context, entity_id, details, 3740 delete_existing=False, session=None): 3741 # NOTE(u_glide): following code modifies details dict, that's why we should 3742 # copy it 3743 new_details = copy.deepcopy(details) 3744 3745 if not session: 3746 session = get_session() 3747 3748 with session.begin(): 3749 # Process existing data 3750 original_data = session.query(models.DriverPrivateData).filter_by( 3751 entity_uuid=entity_id).all() 3752 3753 for data_ref in original_data: 3754 in_new_details = data_ref['key'] in new_details 3755 3756 if in_new_details: 3757 new_value = six.text_type(new_details.pop(data_ref['key'])) 3758 data_ref.update({ 3759 "value": new_value, 3760 "deleted": 0, 3761 "deleted_at": None 3762 }) 3763 data_ref.save(session=session) 3764 elif delete_existing and data_ref['deleted'] != 1: 3765 data_ref.update({ 3766 "deleted": 1, "deleted_at": timeutils.utcnow() 3767 }) 3768 data_ref.save(session=session) 3769 3770 # Add new data 3771 for key, value in new_details.items(): 3772 data_ref = models.DriverPrivateData() 3773 data_ref.update({ 3774 "entity_uuid": entity_id, 3775 "key": key, 3776 "value": six.text_type(value) 3777 }) 3778 data_ref.save(session=session) 3779 3780 return details 3781 3782 3783 @require_context 3784 def driver_private_data_delete(context, entity_id, key=None, 3785 session=None): 3786 if not session: 3787 session = get_session() 3788 3789 with session.begin(): 3790 query = _driver_private_data_query(session, context, 3791 entity_id, key) 3792 query.update({"deleted": 1, "deleted_at": timeutils.utcnow()}) 3793 3794 3795 ################### 3796 3797 3798 @require_context 3799 def network_allocation_create(context, values): 3800 values = ensure_model_dict_has_id(values) 3801 alloc_ref = models.NetworkAllocation() 3802 alloc_ref.update(values) 3803 session = get_session() 3804 with session.begin(): 3805 alloc_ref.save(session=session) 3806 return alloc_ref 3807 3808 3809 @require_context 3810 def network_allocation_delete(context, id): 3811 session = get_session() 3812 with session.begin(): 3813 alloc_ref = network_allocation_get(context, id, session=session) 3814 alloc_ref.soft_delete(session) 3815 3816 3817 @require_context 3818 def network_allocation_get(context, id, session=None, read_deleted="no"): 3819 if session is None: 3820 session = get_session() 3821 result = (model_query(context, models.NetworkAllocation, session=session, 3822 read_deleted=read_deleted). 3823 filter_by(id=id).first()) 3824 if result is None: 3825 raise exception.NotFound() 3826 return result 3827 3828 3829 @require_context 3830 def network_allocations_get_by_ip_address(context, ip_address): 3831 session = get_session() 3832 result = (model_query(context, models.NetworkAllocation, session=session). 3833 filter_by(ip_address=ip_address).all()) 3834 return result or [] 3835 3836 3837 @require_context 3838 def network_allocations_get_for_share_server(context, share_server_id, 3839 session=None, label=None): 3840 if session is None: 3841 session = get_session() 3842 3843 query = model_query( 3844 context, models.NetworkAllocation, session=session, 3845 ).filter_by( 3846 share_server_id=share_server_id, 3847 ) 3848 if label: 3849 if label != 'admin': 3850 query = query.filter(or_( 3851 # NOTE(vponomaryov): we treat None as alias for 'user'. 3852 models.NetworkAllocation.label == None, # noqa 3853 models.NetworkAllocation.label == label, 3854 )) 3855 else: 3856 query = query.filter(models.NetworkAllocation.label == label) 3857 3858 result = query.all() 3859 return result 3860 3861 3862 @require_context 3863 def network_allocation_update(context, id, values, read_deleted=None): 3864 session = get_session() 3865 with session.begin(): 3866 alloc_ref = network_allocation_get(context, id, session=session, 3867 read_deleted=read_deleted) 3868 alloc_ref.update(values) 3869 alloc_ref.save(session=session) 3870 return alloc_ref 3871 3872 3873 ################### 3874 3875 3876 def _dict_with_specs(inst_type_query, specs_key='extra_specs'): 3877 """Convert type query result to dict with extra_spec and rate_limit. 3878 3879 Takes a share [group] type query returned by sqlalchemy and returns it 3880 as a dictionary, converting the extra/group specs entry from a list 3881 of dicts: 3882 3883 'extra_specs' : [{'key': 'k1', 'value': 'v1', ...}, ...] 3884 'group_specs' : [{'key': 'k1', 'value': 'v1', ...}, ...] 3885 to a single dict: 3886 'extra_specs' : {'k1': 'v1'} 3887 'group_specs' : {'k1': 'v1'} 3888 """ 3889 inst_type_dict = dict(inst_type_query) 3890 specs = {x['key']: x['value'] for x in inst_type_query[specs_key]} 3891 inst_type_dict[specs_key] = specs 3892 return inst_type_dict 3893 3894 3895 @require_admin_context 3896 def share_type_create(context, values, projects=None): 3897 """Create a new share type. 3898 3899 In order to pass in extra specs, the values dict should contain a 3900 'extra_specs' key/value pair: 3901 {'extra_specs' : {'k1': 'v1', 'k2': 'v2', ...}} 3902 """ 3903 values = ensure_model_dict_has_id(values) 3904 3905 projects = projects or [] 3906 3907 session = get_session() 3908 with session.begin(): 3909 try: 3910 values['extra_specs'] = _metadata_refs(values.get('extra_specs'), 3911 models.ShareTypeExtraSpecs) 3912 share_type_ref = models.ShareTypes() 3913 share_type_ref.update(values) 3914 share_type_ref.save(session=session) 3915 except db_exception.DBDuplicateEntry: 3916 raise exception.ShareTypeExists(id=values['name']) 3917 except Exception as e: 3918 raise db_exception.DBError(e) 3919 3920 for project in set(projects): 3921 access_ref = models.ShareTypeProjects() 3922 access_ref.update({"share_type_id": share_type_ref.id, 3923 "project_id": project}) 3924 access_ref.save(session=session) 3925 3926 return share_type_ref 3927 3928 3929 def _share_type_get_query(context, session=None, read_deleted=None, 3930 expected_fields=None): 3931 expected_fields = expected_fields or [] 3932 query = (model_query(context, 3933 models.ShareTypes, 3934 session=session, 3935 read_deleted=read_deleted). 3936 options(joinedload('extra_specs'))) 3937 3938 if 'projects' in expected_fields: 3939 query = query.options(joinedload('projects')) 3940 3941 if not context.is_admin: 3942 the_filter = [models.ShareTypes.is_public == true()] 3943 projects_attr = getattr(models.ShareTypes, 'projects') 3944 the_filter.extend([ 3945 projects_attr.any(project_id=context.project_id) 3946 ]) 3947 query = query.filter(or_(*the_filter)) 3948 3949 return query 3950 3951 3952 @require_context 3953 def share_type_get_all(context, inactive=False, filters=None): 3954 """Returns a dict describing all share_types with name as key.""" 3955 filters = filters or {} 3956 3957 read_deleted = "yes" if inactive else "no" 3958 3959 query = _share_type_get_query(context, read_deleted=read_deleted) 3960 3961 if 'is_public' in filters and filters['is_public'] is not None: 3962 the_filter = [models. ShareTypes.is_public == filters['is_public']] 3963 if filters['is_public'] and context.project_id is not None: 3964 projects_attr = getattr(models. ShareTypes, 'projects') 3965 the_filter.extend([ 3966 projects_attr.any( 3967 project_id=context.project_id, deleted=0) 3968 ]) 3969 if len(the_filter) > 1: 3970 query = query.filter(or_(*the_filter)) 3971 else: 3972 query = query.filter(the_filter[0]) 3973 3974 rows = query.order_by("name").all() 3975 3976 result = {} 3977 for row in rows: 3978 result[row['name']] = _dict_with_specs(row) 3979 3980 return result 3981 3982 3983 def _share_type_get_id_from_share_type_query(context, id, session=None): 3984 return (model_query( 3985 context, models.ShareTypes, read_deleted="no", session=session). 3986 filter_by(id=id)) 3987 3988 3989 def _share_type_get_id_from_share_type(context, id, session=None): 3990 result = _share_type_get_id_from_share_type_query( 3991 context, id, session=session).first() 3992 if not result: 3993 raise exception.ShareTypeNotFound(share_type_id=id) 3994 return result['id'] 3995 3996 3997 def _share_type_get(context, id, session=None, inactive=False, 3998 expected_fields=None): 3999 expected_fields = expected_fields or [] 4000 read_deleted = "yes" if inactive else "no" 4001 result = (_share_type_get_query( 4002 context, session, read_deleted, expected_fields). 4003 filter_by(id=id). 4004 first()) 4005 4006 if not result: 4007 # The only way that id could be None is if the default share type is 4008 # not configured and no other share type was specified. 4009 if id is None: 4010 raise exception.DefaultShareTypeNotConfigured() 4011 raise exception.ShareTypeNotFound(share_type_id=id) 4012 4013 share_type = _dict_with_specs(result) 4014 4015 if 'projects' in expected_fields: 4016 share_type['projects'] = [p['project_id'] for p in result['projects']] 4017 4018 return share_type 4019 4020 4021 @require_context 4022 def share_type_get(context, id, inactive=False, expected_fields=None): 4023 """Return a dict describing specific share_type.""" 4024 return _share_type_get(context, id, 4025 session=None, 4026 inactive=inactive, 4027 expected_fields=expected_fields) 4028 4029 4030 def _share_type_get_by_name(context, name, session=None): 4031 result = (_share_type_get_query(context, session=session). 4032 filter_by(name=name). 4033 first()) 4034 4035 if not result: 4036 raise exception.ShareTypeNotFoundByName(share_type_name=name) 4037 4038 return _dict_with_specs(result) 4039 4040 4041 @require_context 4042 def share_type_get_by_name(context, name): 4043 """Return a dict describing specific share_type.""" 4044 return _share_type_get_by_name(context, name) 4045 4046 4047 @require_context 4048 def share_type_get_by_name_or_id(context, name_or_id): 4049 """Return a dict describing specific share_type using its name or ID. 4050 4051 :returns: ShareType object or None if not found 4052 """ 4053 try: 4054 return _share_type_get(context, name_or_id) 4055 except exception.ShareTypeNotFound: 4056 try: 4057 return _share_type_get_by_name(context, name_or_id) 4058 except exception.ShareTypeNotFoundByName: 4059 return None 4060 4061 4062 @require_admin_context 4063 def share_type_destroy(context, id): 4064 session = get_session() 4065 with session.begin(): 4066 _share_type_get(context, id, session) 4067 shares_count = model_query( 4068 context, 4069 models.ShareInstance, 4070 read_deleted="no", 4071 session=session, 4072 ).filter_by(share_type_id=id).count() 4073 share_group_types_count = model_query( 4074 context, 4075 models.ShareGroupTypeShareTypeMapping, 4076 read_deleted="no", 4077 session=session, 4078 ).filter_by(share_type_id=id).count() 4079 if shares_count or share_group_types_count: 4080 msg = ("Deletion of share type %(stype)s failed; it in use by " 4081 "%(shares)d shares and %(gtypes)d share group types") 4082 msg_args = {'stype': id, 4083 'shares': shares_count, 4084 'gtypes': share_group_types_count} 4085 LOG.error(msg, msg_args) 4086 raise exception.ShareTypeInUse(share_type_id=id) 4087 4088 model_query( 4089 context, models.ShareTypeExtraSpecs, session=session 4090 ).filter_by( 4091 share_type_id=id 4092 ).soft_delete() 4093 model_query( 4094 context, models.ShareTypeProjects, session=session 4095 ).filter_by( 4096 share_type_id=id, 4097 ).soft_delete() 4098 model_query( 4099 context, models.ShareTypes, session=session 4100 ).filter_by( 4101 id=id 4102 ).soft_delete() 4103 4104 # Destroy any quotas, usages and reservations for the share type: 4105 quota_destroy_all_by_share_type(context, id) 4106 4107 4108 def _share_type_access_query(context, session=None): 4109 return model_query(context, models.ShareTypeProjects, session=session, 4110 read_deleted="no") 4111 4112 4113 @require_admin_context 4114 def share_type_access_get_all(context, type_id): 4115 share_type_id = _share_type_get_id_from_share_type(context, type_id) 4116 return (_share_type_access_query(context). 4117 filter_by(share_type_id=share_type_id).all()) 4118 4119 4120 @require_admin_context 4121 def share_type_access_add(context, type_id, project_id): 4122 """Add given tenant to the share type access list.""" 4123 share_type_id = _share_type_get_id_from_share_type(context, type_id) 4124 4125 access_ref = models.ShareTypeProjects() 4126 access_ref.update({"share_type_id": share_type_id, 4127 "project_id": project_id}) 4128 4129 session = get_session() 4130 with session.begin(): 4131 try: 4132 access_ref.save(session=session) 4133 except db_exception.DBDuplicateEntry: 4134 raise exception.ShareTypeAccessExists(share_type_id=type_id, 4135 project_id=project_id) 4136 return access_ref 4137 4138 4139 @require_admin_context 4140 def share_type_access_remove(context, type_id, project_id): 4141 """Remove given tenant from the share type access list.""" 4142 share_type_id = _share_type_get_id_from_share_type(context, type_id) 4143 4144 count = (_share_type_access_query(context). 4145 filter_by(share_type_id=share_type_id). 4146 filter_by(project_id=project_id). 4147 soft_delete(synchronize_session=False)) 4148 if count == 0: 4149 raise exception.ShareTypeAccessNotFound( 4150 share_type_id=type_id, project_id=project_id) 4151 4152 #################### 4153 4154 4155 def _share_type_extra_specs_query(context, share_type_id, session=None): 4156 return (model_query(context, models.ShareTypeExtraSpecs, session=session, 4157 read_deleted="no"). 4158 filter_by(share_type_id=share_type_id). 4159 options(joinedload('share_type'))) 4160 4161 4162 @require_context 4163 def share_type_extra_specs_get(context, share_type_id): 4164 rows = (_share_type_extra_specs_query(context, share_type_id). 4165 all()) 4166 4167 result = {} 4168 for row in rows: 4169 result[row['key']] = row['value'] 4170 4171 return result 4172 4173 4174 @require_context 4175 def share_type_extra_specs_delete(context, share_type_id, key): 4176 session = get_session() 4177 with session.begin(): 4178 _share_type_extra_specs_get_item(context, share_type_id, key, session) 4179 (_share_type_extra_specs_query(context, share_type_id, session). 4180 filter_by(key=key).soft_delete()) 4181 4182 4183 def _share_type_extra_specs_get_item(context, share_type_id, key, 4184 session=None): 4185 result = _share_type_extra_specs_query( 4186 context, share_type_id, session=session 4187 ).filter_by(key=key).options(joinedload('share_type')).first() 4188 4189 if not result: 4190 raise exception.ShareTypeExtraSpecsNotFound( 4191 extra_specs_key=key, 4192 share_type_id=share_type_id) 4193 4194 return result 4195 4196 4197 @require_context 4198 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 4199 def share_type_extra_specs_update_or_create(context, share_type_id, specs): 4200 session = get_session() 4201 with session.begin(): 4202 spec_ref = None 4203 for key, value in specs.items(): 4204 try: 4205 spec_ref = _share_type_extra_specs_get_item( 4206 context, share_type_id, key, session) 4207 except exception.ShareTypeExtraSpecsNotFound: 4208 spec_ref = models.ShareTypeExtraSpecs() 4209 spec_ref.update({"key": key, "value": value, 4210 "share_type_id": share_type_id, 4211 "deleted": 0}) 4212 spec_ref.save(session=session) 4213 4214 return specs 4215 4216 4217 def _ensure_availability_zone_exists(context, values, session, strict=True): 4218 az_name = values.pop('availability_zone', None) 4219 4220 if strict and not az_name: 4221 msg = _("Values dict should have 'availability_zone' field.") 4222 raise ValueError(msg) 4223 elif not az_name: 4224 return 4225 4226 if uuidutils.is_uuid_like(az_name): 4227 az_ref = availability_zone_get(context, az_name, session=session) 4228 else: 4229 az_ref = availability_zone_create_if_not_exist( 4230 context, az_name, session=session) 4231 4232 values.update({'availability_zone_id': az_ref['id']}) 4233 4234 4235 @require_context 4236 def availability_zone_get(context, id_or_name, session=None): 4237 if session is None: 4238 session = get_session() 4239 4240 query = model_query(context, models.AvailabilityZone, session=session) 4241 4242 if uuidutils.is_uuid_like(id_or_name): 4243 query = query.filter_by(id=id_or_name) 4244 else: 4245 query = query.filter_by(name=id_or_name) 4246 4247 result = query.first() 4248 4249 if not result: 4250 raise exception.AvailabilityZoneNotFound(id=id_or_name) 4251 4252 return result 4253 4254 4255 @require_context 4256 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 4257 def availability_zone_create_if_not_exist(context, name, session=None): 4258 if session is None: 4259 session = get_session() 4260 4261 az = models.AvailabilityZone() 4262 az.update({'id': uuidutils.generate_uuid(), 'name': name}) 4263 try: 4264 with session.begin(): 4265 az.save(session) 4266 # NOTE(u_glide): Do not catch specific exception here, because it depends 4267 # on concrete backend used by SqlAlchemy 4268 except Exception: 4269 return availability_zone_get(context, name, session=session) 4270 return az 4271 4272 4273 @require_context 4274 def availability_zone_get_all(context): 4275 session = get_session() 4276 4277 enabled_services = model_query( 4278 context, models.Service, 4279 models.Service.availability_zone_id, 4280 session=session, 4281 read_deleted="no" 4282 ).filter_by(disabled=False).distinct() 4283 4284 return model_query(context, models.AvailabilityZone, session=session, 4285 read_deleted="no").filter( 4286 models.AvailabilityZone.id.in_(enabled_services) 4287 ).all() 4288 4289 4290 @require_admin_context 4291 def purge_deleted_records(context, age_in_days): 4292 """Purge soft-deleted records older than(and equal) age from tables.""" 4293 4294 if age_in_days < 0: 4295 msg = _('Must supply a non-negative value for "age_in_days".') 4296 LOG.error(msg) 4297 raise exception.InvalidParameterValue(msg) 4298 4299 metadata = MetaData() 4300 metadata.reflect(get_engine()) 4301 session = get_session() 4302 session.begin() 4303 deleted_age = timeutils.utcnow() - datetime.timedelta(days=age_in_days) 4304 4305 for table in reversed(metadata.sorted_tables): 4306 if 'deleted' in table.columns.keys(): 4307 try: 4308 mds = [m for m in models.__dict__.values() if 4309 (hasattr(m, '__tablename__') and 4310 m.__tablename__ == six.text_type(table))] 4311 if len(mds) > 0: 4312 # collect all soft-deleted records 4313 with session.begin_nested(): 4314 model = mds[0] 4315 s_deleted_records = session.query(model).filter( 4316 model.deleted_at <= deleted_age) 4317 deleted_count = 0 4318 # delete records one by one, 4319 # skip the records which has FK constraints 4320 for record in s_deleted_records: 4321 try: 4322 with session.begin_nested(): 4323 session.delete(record) 4324 deleted_count += 1 4325 except db_exc.DBError: 4326 LOG.warning( 4327 ("Deleting soft-deleted resource %s " 4328 "failed, skipping."), record) 4329 if deleted_count != 0: 4330 LOG.info("Deleted %(count)s records in " 4331 "table %(table)s.", 4332 {'count': deleted_count, 'table': table}) 4333 except db_exc.DBError: 4334 LOG.warning("Querying table %s's soft-deleted records " 4335 "failed, skipping.", table) 4336 session.commit() 4337 4338 4339 #################### 4340 4341 4342 def _share_group_get(context, share_group_id, session=None): 4343 session = session or get_session() 4344 result = (model_query(context, models.ShareGroup, 4345 session=session, 4346 project_only=True, 4347 read_deleted='no'). 4348 filter_by(id=share_group_id). 4349 options(joinedload('share_types')). 4350 first()) 4351 4352 if not result: 4353 raise exception.ShareGroupNotFound(share_group_id=share_group_id) 4354 4355 return result 4356 4357 4358 @require_context 4359 def share_group_get(context, share_group_id, session=None): 4360 return _share_group_get(context, share_group_id, session=session) 4361 4362 4363 def _share_group_get_all(context, project_id=None, share_server_id=None, 4364 host=None, detailed=True, filters=None, 4365 sort_key=None, sort_dir=None, session=None): 4366 session = session or get_session() 4367 sort_key = sort_key or 'created_at' 4368 sort_dir = sort_dir or 'desc' 4369 4370 query = model_query( 4371 context, models.ShareGroup, session=session, read_deleted='no') 4372 4373 # Apply filters 4374 if not filters: 4375 filters = {} 4376 no_key = 'key_is_absent' 4377 for k, v in filters.items(): 4378 temp_k = k.rstrip('~') if k in constants.LIKE_FILTER else k 4379 filter_attr = getattr(models.ShareGroup, temp_k, no_key) 4380 4381 if filter_attr == no_key: 4382 msg = _("Share groups cannot be filtered using '%s' key.") 4383 raise exception.InvalidInput(reason=msg % k) 4384 4385 if k in constants.LIKE_FILTER: 4386 query = query.filter(filter_attr.op('LIKE')(u'%' + v + u'%')) 4387 else: 4388 query = query.filter(filter_attr == v) 4389 4390 if project_id: 4391 query = query.filter( 4392 models.ShareGroup.project_id == project_id) 4393 if host: 4394 query = query.filter( 4395 models.ShareGroup.host == host) 4396 if share_server_id: 4397 query = query.filter( 4398 models.ShareGroup.share_server_id == share_server_id) 4399 4400 try: 4401 query = apply_sorting(models.ShareGroup, query, sort_key, sort_dir) 4402 except AttributeError: 4403 msg = _("Wrong sorting key provided - '%s'.") % sort_key 4404 raise exception.InvalidInput(reason=msg) 4405 4406 if detailed: 4407 return query.options(joinedload('share_types')).all() 4408 else: 4409 query = query.with_entities( 4410 models.ShareGroup.id, models.ShareGroup.name) 4411 values = [] 4412 for sg_id, sg_name in query.all(): 4413 values.append({"id": sg_id, "name": sg_name}) 4414 return values 4415 4416 4417 @require_admin_context 4418 def share_group_get_all(context, detailed=True, filters=None, sort_key=None, 4419 sort_dir=None): 4420 return _share_group_get_all( 4421 context, detailed=detailed, filters=filters, 4422 sort_key=sort_key, sort_dir=sort_dir) 4423 4424 4425 @require_admin_context 4426 def share_group_get_all_by_host(context, host, detailed=True): 4427 return _share_group_get_all(context, host=host, detailed=detailed) 4428 4429 4430 @require_context 4431 def share_group_get_all_by_project(context, project_id, detailed=True, 4432 filters=None, sort_key=None, sort_dir=None): 4433 authorize_project_context(context, project_id) 4434 return _share_group_get_all( 4435 context, project_id=project_id, detailed=detailed, filters=filters, 4436 sort_key=sort_key, sort_dir=sort_dir) 4437 4438 4439 @require_context 4440 def share_group_get_all_by_share_server(context, share_server_id, filters=None, 4441 sort_key=None, sort_dir=None): 4442 return _share_group_get_all( 4443 context, share_server_id=share_server_id, filters=filters, 4444 sort_key=sort_key, sort_dir=sort_dir) 4445 4446 4447 @require_context 4448 def share_group_create(context, values): 4449 share_group = models.ShareGroup() 4450 if not values.get('id'): 4451 values['id'] = six.text_type(uuidutils.generate_uuid()) 4452 4453 mappings = [] 4454 for item in values.get('share_types') or []: 4455 mapping = models.ShareGroupShareTypeMapping() 4456 mapping['id'] = six.text_type(uuidutils.generate_uuid()) 4457 mapping['share_type_id'] = item 4458 mapping['share_group_id'] = values['id'] 4459 mappings.append(mapping) 4460 4461 values['share_types'] = mappings 4462 4463 session = get_session() 4464 with session.begin(): 4465 share_group.update(values) 4466 session.add(share_group) 4467 4468 return _share_group_get(context, values['id'], session=session) 4469 4470 4471 @require_context 4472 def share_group_update(context, share_group_id, values): 4473 session = get_session() 4474 with session.begin(): 4475 share_group_ref = _share_group_get( 4476 context, share_group_id, session=session) 4477 share_group_ref.update(values) 4478 share_group_ref.save(session=session) 4479 return share_group_ref 4480 4481 4482 @require_admin_context 4483 def share_group_destroy(context, share_group_id): 4484 session = get_session() 4485 with session.begin(): 4486 share_group_ref = _share_group_get( 4487 context, share_group_id, session=session) 4488 share_group_ref.soft_delete(session) 4489 session.query(models.ShareGroupShareTypeMapping).filter_by( 4490 share_group_id=share_group_ref['id']).soft_delete() 4491 4492 4493 @require_context 4494 def count_shares_in_share_group(context, share_group_id, session=None): 4495 session = session or get_session() 4496 return (model_query(context, models.Share, session=session, 4497 project_only=True, read_deleted="no"). 4498 filter_by(share_group_id=share_group_id). 4499 count()) 4500 4501 4502 @require_context 4503 def get_all_shares_by_share_group(context, share_group_id, session=None): 4504 session = session or get_session() 4505 return (model_query( 4506 context, models.Share, session=session, 4507 project_only=True, read_deleted="no"). 4508 filter_by(share_group_id=share_group_id). 4509 all()) 4510 4511 4512 @require_context 4513 def count_share_groups(context, project_id, user_id=None, 4514 share_type_id=None, session=None): 4515 query = model_query( 4516 context, models.ShareGroup, 4517 func.count(models.ShareGroup.id), 4518 read_deleted="no", 4519 session=session).filter_by(project_id=project_id) 4520 if share_type_id: 4521 query = query.join("share_group_share_type_mappings").filter_by( 4522 share_type_id=share_type_id) 4523 elif user_id is not None: 4524 query = query.filter_by(user_id=user_id) 4525 return query.first()[0] 4526 4527 4528 @require_context 4529 def count_share_group_snapshots(context, project_id, user_id=None, 4530 share_type_id=None, session=None): 4531 query = model_query( 4532 context, models.ShareGroupSnapshot, 4533 func.count(models.ShareGroupSnapshot.id), 4534 read_deleted="no", 4535 session=session).filter_by(project_id=project_id) 4536 if share_type_id: 4537 query = query.join( 4538 "share_group" 4539 ).join( 4540 "share_group_share_type_mappings" 4541 ).filter_by(share_type_id=share_type_id) 4542 elif user_id is not None: 4543 query = query.filter_by(user_id=user_id) 4544 return query.first()[0] 4545 4546 4547 @require_context 4548 def count_share_group_snapshots_in_share_group(context, share_group_id, 4549 session=None): 4550 session = session or get_session() 4551 return model_query( 4552 context, models.ShareGroupSnapshot, session=session, 4553 project_only=True, read_deleted="no", 4554 ).filter_by( 4555 share_group_id=share_group_id, 4556 ).count() 4557 4558 4559 @require_context 4560 def count_share_groups_in_share_network(context, share_network_id, 4561 session=None): 4562 session = session or get_session() 4563 return (model_query( 4564 context, models.ShareGroup, session=session, 4565 project_only=True, read_deleted="no"). 4566 filter_by(share_network_id=share_network_id). 4567 count()) 4568 4569 4570 @require_context 4571 def count_share_group_snapshot_members_in_share(context, share_id, 4572 session=None): 4573 session = session or get_session() 4574 return model_query( 4575 context, models.ShareSnapshotInstance, session=session, 4576 project_only=True, read_deleted="no", 4577 ).join( 4578 models.ShareInstance, 4579 models.ShareInstance.id == ( 4580 models.ShareSnapshotInstance.share_instance_id), 4581 ).filter( 4582 models.ShareInstance.share_id == share_id, 4583 ).count() 4584 4585 4586 @require_context 4587 def _share_group_snapshot_get(context, share_group_snapshot_id, session=None): 4588 session = session or get_session() 4589 result = model_query( 4590 context, models.ShareGroupSnapshot, session=session, 4591 project_only=True, read_deleted='no', 4592 ).options( 4593 joinedload('share_group'), 4594 joinedload('share_group_snapshot_members'), 4595 ).filter_by( 4596 id=share_group_snapshot_id, 4597 ).first() 4598 4599 if not result: 4600 raise exception.ShareGroupSnapshotNotFound( 4601 share_group_snapshot_id=share_group_snapshot_id) 4602 4603 return result 4604 4605 4606 def _share_group_snapshot_get_all( 4607 context, project_id=None, detailed=True, filters=None, 4608 sort_key=None, sort_dir=None, session=None): 4609 session = session or get_session() 4610 if not sort_key: 4611 sort_key = 'created_at' 4612 if not sort_dir: 4613 sort_dir = 'desc' 4614 4615 query = model_query( 4616 context, models.ShareGroupSnapshot, session=session, read_deleted='no', 4617 ).options( 4618 joinedload('share_group'), 4619 joinedload('share_group_snapshot_members'), 4620 ) 4621 4622 # Apply filters 4623 if not filters: 4624 filters = {} 4625 no_key = 'key_is_absent' 4626 for k, v in filters.items(): 4627 filter_attr = getattr(models.ShareGroupSnapshot, k, no_key) 4628 if filter_attr == no_key: 4629 msg = _("Share group snapshots cannot be filtered using '%s' key.") 4630 raise exception.InvalidInput(reason=msg % k) 4631 query = query.filter(filter_attr == v) 4632 4633 if project_id: 4634 query = query.filter( 4635 models.ShareGroupSnapshot.project_id == project_id) 4636 4637 try: 4638 query = apply_sorting( 4639 models.ShareGroupSnapshot, query, sort_key, sort_dir) 4640 except AttributeError: 4641 msg = _("Wrong sorting key provided - '%s'.") % sort_key 4642 raise exception.InvalidInput(reason=msg) 4643 4644 if detailed: 4645 return query.all() 4646 else: 4647 query = query.with_entities(models.ShareGroupSnapshot.id, 4648 models.ShareGroupSnapshot.name) 4649 values = [] 4650 for sgs_id, sgs_name in query.all(): 4651 values.append({"id": sgs_id, "name": sgs_name}) 4652 return values 4653 4654 4655 @require_context 4656 def share_group_snapshot_get(context, share_group_snapshot_id, session=None): 4657 session = session or get_session() 4658 return _share_group_snapshot_get( 4659 context, share_group_snapshot_id, session=session) 4660 4661 4662 @require_admin_context 4663 def share_group_snapshot_get_all( 4664 context, detailed=True, filters=None, sort_key=None, sort_dir=None): 4665 return _share_group_snapshot_get_all( 4666 context, filters=filters, detailed=detailed, 4667 sort_key=sort_key, sort_dir=sort_dir) 4668 4669 4670 @require_context 4671 def share_group_snapshot_get_all_by_project( 4672 context, project_id, detailed=True, filters=None, 4673 sort_key=None, sort_dir=None): 4674 authorize_project_context(context, project_id) 4675 return _share_group_snapshot_get_all( 4676 context, project_id=project_id, filters=filters, detailed=detailed, 4677 sort_key=sort_key, sort_dir=sort_dir, 4678 ) 4679 4680 4681 @require_context 4682 def share_group_snapshot_create(context, values): 4683 share_group_snapshot = models.ShareGroupSnapshot() 4684 if not values.get('id'): 4685 values['id'] = six.text_type(uuidutils.generate_uuid()) 4686 4687 session = get_session() 4688 with session.begin(): 4689 share_group_snapshot.update(values) 4690 session.add(share_group_snapshot) 4691 4692 return _share_group_snapshot_get( 4693 context, values['id'], session=session) 4694 4695 4696 @require_context 4697 def share_group_snapshot_update(context, share_group_snapshot_id, values): 4698 session = get_session() 4699 with session.begin(): 4700 share_group_ref = _share_group_snapshot_get( 4701 context, share_group_snapshot_id, session=session) 4702 share_group_ref.update(values) 4703 share_group_ref.save(session=session) 4704 return share_group_ref 4705 4706 4707 @require_admin_context 4708 def share_group_snapshot_destroy(context, share_group_snapshot_id): 4709 session = get_session() 4710 with session.begin(): 4711 share_group_snap_ref = _share_group_snapshot_get( 4712 context, share_group_snapshot_id, session=session) 4713 share_group_snap_ref.soft_delete(session) 4714 session.query(models.ShareSnapshotInstance).filter_by( 4715 share_group_snapshot_id=share_group_snapshot_id).soft_delete() 4716 4717 4718 @require_context 4719 def share_group_snapshot_members_get_all(context, share_group_snapshot_id, 4720 session=None): 4721 session = session or get_session() 4722 query = model_query( 4723 context, models.ShareSnapshotInstance, session=session, 4724 read_deleted='no', 4725 ).filter_by(share_group_snapshot_id=share_group_snapshot_id) 4726 return query.all() 4727 4728 4729 @require_context 4730 def share_group_snapshot_member_get(context, member_id, session=None): 4731 result = model_query( 4732 context, models.ShareSnapshotInstance, session=session, 4733 project_only=True, read_deleted='no', 4734 ).filter_by(id=member_id).first() 4735 if not result: 4736 raise exception.ShareGroupSnapshotMemberNotFound(member_id=member_id) 4737 return result 4738 4739 4740 @require_context 4741 def share_group_snapshot_member_create(context, values): 4742 member = models.ShareSnapshotInstance() 4743 if not values.get('id'): 4744 values['id'] = six.text_type(uuidutils.generate_uuid()) 4745 4746 _change_size_to_instance_size(values) 4747 4748 session = get_session() 4749 with session.begin(): 4750 member.update(values) 4751 session.add(member) 4752 4753 return share_group_snapshot_member_get( 4754 context, values['id'], session=session) 4755 4756 4757 @require_context 4758 def share_group_snapshot_member_update(context, member_id, values): 4759 session = get_session() 4760 _change_size_to_instance_size(values) 4761 with session.begin(): 4762 member = share_group_snapshot_member_get( 4763 context, member_id, session=session) 4764 member.update(values) 4765 session.add(member) 4766 return share_group_snapshot_member_get( 4767 context, member_id, session=session) 4768 4769 4770 #################### 4771 4772 4773 @require_admin_context 4774 def share_group_type_create(context, values, projects=None): 4775 """Create a new share group type. 4776 4777 In order to pass in group specs, the values dict should contain a 4778 'group_specs' key/value pair: 4779 {'group_specs' : {'k1': 'v1', 'k2': 'v2', ...}} 4780 """ 4781 values = ensure_model_dict_has_id(values) 4782 4783 projects = projects or [] 4784 4785 session = get_session() 4786 with session.begin(): 4787 try: 4788 values['group_specs'] = _metadata_refs( 4789 values.get('group_specs'), models.ShareGroupTypeSpecs) 4790 mappings = [] 4791 for item in values.get('share_types', []): 4792 share_type = share_type_get_by_name_or_id(context, item) 4793 if not share_type: 4794 raise exception.ShareTypeDoesNotExist(share_type=item) 4795 mapping = models.ShareGroupTypeShareTypeMapping() 4796 mapping['id'] = six.text_type(uuidutils.generate_uuid()) 4797 mapping['share_type_id'] = share_type['id'] 4798 mapping['share_group_type_id'] = values['id'] 4799 mappings.append(mapping) 4800 4801 values['share_types'] = mappings 4802 share_group_type_ref = models.ShareGroupTypes() 4803 share_group_type_ref.update(values) 4804 share_group_type_ref.save(session=session) 4805 except db_exception.DBDuplicateEntry: 4806 raise exception.ShareGroupTypeExists(type_id=values['name']) 4807 except exception.ShareTypeDoesNotExist: 4808 raise 4809 except Exception as e: 4810 raise db_exception.DBError(e) 4811 4812 for project in set(projects): 4813 access_ref = models.ShareGroupTypeProjects() 4814 access_ref.update({"share_group_type_id": share_group_type_ref.id, 4815 "project_id": project}) 4816 access_ref.save(session=session) 4817 4818 return share_group_type_ref 4819 4820 4821 def _share_group_type_get_query(context, session=None, read_deleted=None, 4822 expected_fields=None): 4823 expected_fields = expected_fields or [] 4824 query = model_query( 4825 context, models.ShareGroupTypes, session=session, 4826 read_deleted=read_deleted 4827 ).options( 4828 joinedload('group_specs'), 4829 joinedload('share_types'), 4830 ) 4831 4832 if 'projects' in expected_fields: 4833 query = query.options(joinedload('projects')) 4834 4835 if not context.is_admin: 4836 the_filter = [models.ShareGroupTypes.is_public == true()] 4837 projects_attr = getattr(models.ShareGroupTypes, 'projects') 4838 the_filter.extend([ 4839 projects_attr.any(project_id=context.project_id) 4840 ]) 4841 query = query.filter(or_(*the_filter)) 4842 4843 return query 4844 4845 4846 @require_context 4847 def share_group_type_get_all(context, inactive=False, filters=None): 4848 """Returns a dict describing all share group types with name as key.""" 4849 filters = filters or {} 4850 read_deleted = "yes" if inactive else "no" 4851 query = _share_group_type_get_query(context, read_deleted=read_deleted) 4852 4853 if 'is_public' in filters and filters['is_public'] is not None: 4854 the_filter = [models.ShareGroupTypes.is_public == filters['is_public']] 4855 if filters['is_public'] and context.project_id is not None: 4856 projects_attr = getattr(models. ShareGroupTypes, 'projects') 4857 the_filter.extend([ 4858 projects_attr.any( 4859 project_id=context.project_id, deleted=0) 4860 ]) 4861 if len(the_filter) > 1: 4862 query = query.filter(or_(*the_filter)) 4863 else: 4864 query = query.filter(the_filter[0]) 4865 4866 rows = query.order_by("name").all() 4867 4868 result = {} 4869 for row in rows: 4870 result[row['name']] = _dict_with_specs(row, 'group_specs') 4871 4872 return result 4873 4874 4875 def _share_group_type_get_id_from_share_group_type_query(context, type_id, 4876 session=None): 4877 return model_query( 4878 context, models.ShareGroupTypes, read_deleted="no", session=session, 4879 ).filter_by(id=type_id) 4880 4881 4882 def _share_group_type_get_id_from_share_group_type(context, type_id, 4883 session=None): 4884 result = _share_group_type_get_id_from_share_group_type_query( 4885 context, type_id, session=session).first() 4886 if not result: 4887 raise exception.ShareGroupTypeNotFound(type_id=type_id) 4888 return result['id'] 4889 4890 4891 @require_context 4892 def _share_group_type_get(context, type_id, session=None, inactive=False, 4893 expected_fields=None): 4894 expected_fields = expected_fields or [] 4895 read_deleted = "yes" if inactive else "no" 4896 result = _share_group_type_get_query( 4897 context, session, read_deleted, expected_fields, 4898 ).filter_by(id=type_id).first() 4899 4900 if not result: 4901 raise exception.ShareGroupTypeNotFound(type_id=type_id) 4902 4903 share_group_type = _dict_with_specs(result, 'group_specs') 4904 4905 if 'projects' in expected_fields: 4906 share_group_type['projects'] = [ 4907 p['project_id'] for p in result['projects']] 4908 4909 return share_group_type 4910 4911 4912 @require_context 4913 def share_group_type_get(context, type_id, inactive=False, 4914 expected_fields=None): 4915 """Return a dict describing specific share group type.""" 4916 return _share_group_type_get( 4917 context, type_id, session=None, inactive=inactive, 4918 expected_fields=expected_fields) 4919 4920 4921 @require_context 4922 def _share_group_type_get_by_name(context, name, session=None): 4923 result = model_query( 4924 context, models.ShareGroupTypes, session=session, 4925 ).options( 4926 joinedload('group_specs'), 4927 joinedload('share_types'), 4928 ).filter_by( 4929 name=name, 4930 ).first() 4931 if not result: 4932 raise exception.ShareGroupTypeNotFoundByName(type_name=name) 4933 return _dict_with_specs(result, 'group_specs') 4934 4935 4936 @require_context 4937 def share_group_type_get_by_name(context, name): 4938 """Return a dict describing specific share group type.""" 4939 return _share_group_type_get_by_name(context, name) 4940 4941 4942 @require_admin_context 4943 def share_group_type_destroy(context, type_id): 4944 session = get_session() 4945 with session.begin(): 4946 _share_group_type_get(context, type_id, session) 4947 results = model_query( 4948 context, models.ShareGroup, session=session, read_deleted="no", 4949 ).filter_by( 4950 share_group_type_id=type_id, 4951 ).count() 4952 if results: 4953 LOG.error('Share group type %s deletion failed, it in use.', 4954 type_id) 4955 raise exception.ShareGroupTypeInUse(type_id=type_id) 4956 model_query( 4957 context, models.ShareGroupTypeSpecs, session=session, 4958 ).filter_by( 4959 share_group_type_id=type_id, 4960 ).soft_delete() 4961 model_query( 4962 context, models.ShareGroupTypeShareTypeMapping, session=session 4963 ).filter_by( 4964 share_group_type_id=type_id, 4965 ).soft_delete() 4966 model_query( 4967 context, models.ShareGroupTypeProjects, session=session 4968 ).filter_by( 4969 share_group_type_id=type_id, 4970 ).soft_delete() 4971 model_query( 4972 context, models.ShareGroupTypes, session=session 4973 ).filter_by( 4974 id=type_id, 4975 ).soft_delete() 4976 4977 4978 def _share_group_type_access_query(context, session=None): 4979 return model_query(context, models.ShareGroupTypeProjects, session=session, 4980 read_deleted="no") 4981 4982 4983 @require_admin_context 4984 def share_group_type_access_get_all(context, type_id): 4985 share_group_type_id = _share_group_type_get_id_from_share_group_type( 4986 context, type_id) 4987 return _share_group_type_access_query(context).filter_by( 4988 share_group_type_id=share_group_type_id, 4989 ).all() 4990 4991 4992 @require_admin_context 4993 def share_group_type_access_add(context, type_id, project_id): 4994 """Add given tenant to the share group type access list.""" 4995 share_group_type_id = _share_group_type_get_id_from_share_group_type( 4996 context, type_id) 4997 access_ref = models.ShareGroupTypeProjects() 4998 access_ref.update({"share_group_type_id": share_group_type_id, 4999 "project_id": project_id}) 5000 session = get_session() 5001 with session.begin(): 5002 try: 5003 access_ref.save(session=session) 5004 except db_exception.DBDuplicateEntry: 5005 raise exception.ShareGroupTypeAccessExists( 5006 type_id=share_group_type_id, project_id=project_id) 5007 return access_ref 5008 5009 5010 @require_admin_context 5011 def share_group_type_access_remove(context, type_id, project_id): 5012 """Remove given tenant from the share group type access list.""" 5013 share_group_type_id = _share_group_type_get_id_from_share_group_type( 5014 context, type_id) 5015 count = _share_group_type_access_query(context).filter_by( 5016 share_group_type_id=share_group_type_id, 5017 ).filter_by( 5018 project_id=project_id, 5019 ).soft_delete( 5020 synchronize_session=False, 5021 ) 5022 if count == 0: 5023 raise exception.ShareGroupTypeAccessNotFound( 5024 type_id=share_group_type_id, project_id=project_id) 5025 5026 5027 def _share_group_type_specs_query(context, type_id, session=None): 5028 return model_query( 5029 context, models.ShareGroupTypeSpecs, session=session, read_deleted="no" 5030 ).filter_by( 5031 share_group_type_id=type_id, 5032 ).options( 5033 joinedload('share_group_type'), 5034 ) 5035 5036 5037 @require_context 5038 def share_group_type_specs_get(context, type_id): 5039 rows = _share_group_type_specs_query(context, type_id).all() 5040 result = {} 5041 for row in rows: 5042 result[row['key']] = row['value'] 5043 return result 5044 5045 5046 @require_context 5047 def share_group_type_specs_delete(context, type_id, key): 5048 session = get_session() 5049 with session.begin(): 5050 _share_group_type_specs_get_item(context, type_id, key, session) 5051 _share_group_type_specs_query( 5052 context, type_id, session, 5053 ).filter_by( 5054 key=key, 5055 ).soft_delete() 5056 5057 5058 @require_context 5059 def _share_group_type_specs_get_item(context, type_id, key, session=None): 5060 result = _share_group_type_specs_query( 5061 context, type_id, session=session, 5062 ).filter_by( 5063 key=key, 5064 ).options( 5065 joinedload('share_group_type'), 5066 ).first() 5067 5068 if not result: 5069 raise exception.ShareGroupTypeSpecsNotFound( 5070 specs_key=key, type_id=type_id) 5071 5072 return result 5073 5074 5075 @require_context 5076 @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) 5077 def share_group_type_specs_update_or_create(context, type_id, specs): 5078 session = get_session() 5079 with session.begin(): 5080 spec_ref = None 5081 for key, value in specs.items(): 5082 try: 5083 spec_ref = _share_group_type_specs_get_item( 5084 context, type_id, key, session) 5085 except exception.ShareGroupTypeSpecsNotFound: 5086 spec_ref = models.ShareGroupTypeSpecs() 5087 spec_ref.update({"key": key, "value": value, 5088 "share_group_type_id": type_id, "deleted": 0}) 5089 spec_ref.save(session=session) 5090 5091 return specs 5092 5093 5094 ############################### 5095 5096 5097 @require_context 5098 def message_get(context, message_id): 5099 query = model_query(context, 5100 models.Message, 5101 read_deleted="no", 5102 project_only="yes") 5103 result = query.filter_by(id=message_id).first() 5104 if not result: 5105 raise exception.MessageNotFound(message_id=message_id) 5106 return result 5107 5108 5109 @require_context 5110 def message_get_all(context, filters=None, sort_key='created_at', 5111 sort_dir='asc'): 5112 messages = models.Message 5113 query = model_query(context, 5114 messages, 5115 read_deleted="no", 5116 project_only="yes") 5117 5118 legal_filter_keys = ('request_id', 'resource_type', 'resource_id', 5119 'action_id', 'detail_id', 'message_level') 5120 5121 if not filters: 5122 filters = {} 5123 5124 query = exact_filter(query, messages, filters, legal_filter_keys) 5125 try: 5126 query = apply_sorting(messages, query, sort_key, sort_dir) 5127 except AttributeError: 5128 msg = _("Wrong sorting key provided - '%s'.") % sort_key 5129 raise exception.InvalidInput(reason=msg) 5130 5131 return query.all() 5132 5133 5134 @require_context 5135 def message_create(context, message_values): 5136 values = copy.deepcopy(message_values) 5137 message_ref = models.Message() 5138 if not values.get('id'): 5139 values['id'] = uuidutils.generate_uuid() 5140 message_ref.update(values) 5141 5142 session = get_session() 5143 with session.begin(): 5144 session.add(message_ref) 5145 5146 return message_get(context, message_ref['id']) 5147 5148 5149 @require_context 5150 def message_destroy(context, message): 5151 session = get_session() 5152 with session.begin(): 5153 (model_query(context, models.Message, session=session). 5154 filter_by(id=message.get('id')).soft_delete()) 5155 5156 5157 @require_admin_context 5158 def cleanup_expired_messages(context): 5159 session = get_session() 5160 now = timeutils.utcnow() 5161 with session.begin(): 5162 return session.query(models.Message).filter( 5163 models.Message.expires_at < now).delete() 5164 5165 5166 @require_context 5167 def backend_info_get(context, host): 5168 """Get hash info for given host.""" 5169 session = get_session() 5170 5171 result = _backend_info_query(session, context, host) 5172 5173 return result 5174 5175 5176 @require_context 5177 def backend_info_create(context, host, value): 5178 session = get_session() 5179 with session.begin(): 5180 info_ref = models.BackendInfo() 5181 info_ref.update({"host": host, 5182 "info_hash": value}) 5183 info_ref.save(session) 5184 return info_ref 5185 5186 5187 @require_context 5188 def backend_info_update(context, host, value=None, delete_existing=False): 5189 """Remove backend info for host name.""" 5190 session = get_session() 5191 5192 with session.begin(): 5193 info_ref = _backend_info_query(session, context, host) 5194 if info_ref: 5195 if value: 5196 info_ref.update({"info_hash": value}) 5197 elif delete_existing and info_ref['deleted'] != 1: 5198 info_ref.update({"deleted": 1, 5199 "deleted_at": timeutils.utcnow()}) 5200 else: 5201 info_ref = models.BackendInfo() 5202 info_ref.update({"host": host, 5203 "info_hash": value}) 5204 info_ref.save(session) 5205 return info_ref 5206 5207 5208 def _backend_info_query(session, context, host, read_deleted=False): 5209 result = model_query( 5210 context, models.BackendInfo, session=session, 5211 read_deleted=read_deleted, 5212 ).filter_by( 5213 host=host, 5214 ).first() 5215 5216 return result