"Fossies" - the Fresh Open Source Software Archive

Member "octavia-8.0.0/octavia/api/common/pagination.py" (14 Apr 2021, 17048 Bytes) of package /linux/misc/openstack/octavia-8.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file.

    1 #    Copyright 2016 Intel Corporation
    2 #
    3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    4 #    not use this file except in compliance with the License. You may obtain
    5 #    a copy of the License at
    6 #
    7 #         http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 #    Unless required by applicable law or agreed to in writing, software
   10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   12 #    License for the specific language governing permissions and limitations
   13 #    under the License.
   14 
   15 import copy
   16 import itertools
   17 
   18 from oslo_log import log as logging
   19 from pecan import request
   20 import sqlalchemy
   21 from sqlalchemy.orm import aliased
   22 import sqlalchemy.sql as sa_sql
   23 
   24 from octavia.api.common import types
   25 from octavia.common.config import cfg
   26 from octavia.common import constants
   27 from octavia.common import exceptions
   28 from octavia.db import base_models
   29 from octavia.db import models
   30 
   31 CONF = cfg.CONF
   32 LOG = logging.getLogger(__name__)
   33 
   34 
   35 class PaginationHelper(object):
   36     """Class helping to interact with pagination functionality
   37 
   38     Pass this class to `db.repositories` to apply it on query
   39     """
   40     _auxiliary_arguments = ('limit', 'marker',
   41                             'sort', 'sort_key', 'sort_dir',
   42                             'fields', 'page_reverse',
   43                             )
   44 
   45     def __init__(self, params, sort_dir=constants.DEFAULT_SORT_DIR):
   46         """Pagination Helper takes params and a default sort direction
   47 
   48         :param params: Contains the following:
   49                        limit: maximum number of items to return
   50                        marker: the last item of the previous page; we return
   51                                the next results after this value.
   52                        sort: array of attr by which results should be sorted
   53         :param sort_dir: default direction to sort (asc, desc)
   54         """
   55         self.marker = params.get('marker')
   56         self.sort_dir = self._validate_sort_dir(sort_dir)
   57         self.limit = self._parse_limit(params)
   58         self.sort_keys = self._parse_sort_keys(params)
   59         self.params = params
   60         self.filters = None
   61         self.page_reverse = params.get('page_reverse', 'False')
   62 
   63     @staticmethod
   64     def _parse_limit(params):
   65         if CONF.api_settings.pagination_max_limit == 'infinite':
   66             page_max_limit = None
   67         else:
   68             page_max_limit = int(CONF.api_settings.pagination_max_limit)
   69         limit = params.get('limit', page_max_limit)
   70         try:
   71             # Deal with limit being a string or int meaning 'Unlimited'
   72             if limit == 'infinite' or int(limit) < 1:
   73                 limit = None
   74             # If we don't have a max, just use whatever limit is specified
   75             elif page_max_limit is None:
   76                 limit = int(limit)
   77             # Otherwise, we need to compare against the max
   78             else:
   79                 limit = min(int(limit), page_max_limit)
   80         except ValueError as e:
   81             raise exceptions.InvalidLimit(key=limit) from e
   82         return limit
   83 
   84     def _parse_sort_keys(self, params):
   85         sort_keys_dirs = []
   86         sort = params.get('sort')
   87         sort_keys = params.get('sort_key')
   88         if sort:
   89             for sort_dir_key in sort.split(","):
   90                 comps = sort_dir_key.split(":")
   91                 if len(comps) == 1:  # Use default sort order
   92                     sort_keys_dirs.append((comps[0], self.sort_dir))
   93                 elif len(comps) == 2:
   94                     sort_keys_dirs.append(
   95                         (comps[0], self._validate_sort_dir(comps[1])))
   96                 else:
   97                     raise exceptions.InvalidSortKey(key=comps)
   98         elif sort_keys:
   99             sort_keys = sort_keys.split(',')
  100             sort_dirs = params.get('sort_dir')
  101             if not sort_dirs:
  102                 sort_dirs = [self.sort_dir] * len(sort_keys)
  103             else:
  104                 sort_dirs = sort_dirs.split(',')
  105 
  106             if len(sort_dirs) < len(sort_keys):
  107                 sort_dirs += [self.sort_dir] * (len(sort_keys) -
  108                                                 len(sort_dirs))
  109             for sk, sd in zip(sort_keys, sort_dirs):
  110                 sort_keys_dirs.append((sk, self._validate_sort_dir(sd)))
  111 
  112         return sort_keys_dirs
  113 
  114     def _parse_marker(self, session, model):
  115         return session.query(model).filter_by(id=self.marker).one_or_none()
  116 
  117     @staticmethod
  118     def _get_default_column_value(column_type):
  119         """Return the default value of the columns from DB table
  120 
  121         In postgreDB case, if no right default values are being set, an
  122         psycopg2.DataError will be thrown.
  123         """
  124         type_schema = {
  125             'datetime': None,
  126             'big_integer': 0,
  127             'integer': 0,
  128             'string': ''
  129         }
  130 
  131         if isinstance(column_type, sa_sql.type_api.Variant):
  132             return PaginationHelper._get_default_column_value(column_type.impl)
  133 
  134         return type_schema[column_type.__visit_name__]
  135 
  136     @staticmethod
  137     def _validate_sort_dir(sort_dir):
  138         sort_dir = sort_dir.lower()
  139         if sort_dir not in constants.ALLOWED_SORT_DIR:
  140             raise exceptions.InvalidSortDirection(key=sort_dir)
  141         return sort_dir
  142 
  143     def _make_links(self, model_list):
  144         if CONF.api_settings.api_base_uri:
  145             path_url = "{api_base_url}{path}".format(
  146                 api_base_url=CONF.api_settings.api_base_uri.rstrip('/'),
  147                 path=request.path)
  148         else:
  149             path_url = request.path_url
  150         links = []
  151         if model_list:
  152             prev_attr = ["limit={}".format(self.limit)]
  153             if self.params.get('sort'):
  154                 prev_attr.append("sort={}".format(self.params.get('sort')))
  155             if self.params.get('sort_key'):
  156                 prev_attr.append("sort_key={}".format(
  157                     self.params.get('sort_key')))
  158             next_attr = copy.copy(prev_attr)
  159             if self.marker:
  160                 prev_attr.append("marker={}".format(model_list[0].get('id')))
  161                 prev_attr.append("page_reverse=True")
  162                 prev_link = {
  163                     "rel": "previous",
  164                     "href": "{url}?{params}".format(
  165                         url=path_url,
  166                         params="&".join(prev_attr))
  167                 }
  168                 links.append(prev_link)
  169             # TODO(rm_work) Do we need to know when there are more vs exact?
  170             # We safely know if we have a full page, but it might include the
  171             # last element or it might not, it is unclear
  172             if len(model_list) >= self.limit:
  173                 next_attr.append("marker={}".format(model_list[-1].get('id')))
  174                 next_link = {
  175                     "rel": "next",
  176                     "href": "{url}?{params}".format(
  177                         url=path_url,
  178                         params="&".join(next_attr))
  179                 }
  180                 links.append(next_link)
  181         links = [types.PageType(**link) for link in links]
  182         return links
  183 
  184     def _apply_tags_filtering(self, params, model, query):
  185         if not getattr(model, "_tags", None):
  186             return query
  187 
  188         if 'tags' in params:
  189             tags = params.pop('tags')
  190 
  191             for tag in tags:
  192                 # This requires a multi-join to the tags table,
  193                 # so me must use aliases for each one.
  194                 tag_alias = aliased(base_models.Tags)
  195                 query = query.join(tag_alias, model._tags)
  196                 query = query.filter(tag_alias.tag == tag)
  197 
  198         if 'tags-any' in params:
  199             tags = params.pop('tags-any')
  200             tag_alias = aliased(base_models.Tags)
  201             query = query.join(tag_alias, model._tags)
  202             query = query.filter(tag_alias.tag.in_(tags))
  203 
  204         if 'not-tags' in params:
  205             tags = params.pop('not-tags')
  206             subq = query.session.query(model.id)
  207             for tag in tags:
  208                 tag_alias = aliased(base_models.Tags)
  209                 subq = subq.join(tag_alias, model._tags)
  210                 subq = subq.filter(tag_alias.tag == tag)
  211 
  212             query = query.filter(~model.id.in_(subq))
  213 
  214         if 'not-tags-any' in params:
  215             tags = params.pop('not-tags-any')
  216             query = query.filter(
  217                 ~model._tags.any(base_models.Tags.tag.in_(tags)))
  218 
  219         return query
  220 
  221     @staticmethod
  222     def _prepare_tags_list(param):
  223         """Split comma separated tags and return a flat list of tags."""
  224         if not isinstance(param, list):
  225             param = [param]
  226         return list(itertools.chain.from_iterable(
  227             tag.split(',') for tag in param))
  228 
  229     def apply(self, query, model, enforce_valid_params=True):
  230         """Returns a query with sorting / pagination criteria added.
  231 
  232         Pagination works by requiring a unique sort_key specified by sort_keys.
  233         (If sort_keys is not unique, then we risk looping through values.)
  234         We use the last row in the previous page as the pagination 'marker'.
  235         So we must return values that follow the passed marker in the order.
  236         With a single-valued sort_key, this would be easy: sort_key > X.
  237         With a compound-values sort_key, (k1, k2, k3) we must do this to repeat
  238         the lexicographical ordering:
  239         (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3)
  240         We also have to cope with different sort_directions.
  241         Typically, the id of the last row is used as the client-facing
  242         pagination marker, then the actual marker object must be fetched from
  243         the db and passed in to us as marker.
  244         :param query: the query object to which we should add
  245         paging/sorting/filtering
  246         :param model: the ORM model class
  247         :param enforce_valid_params: check for invalid enteries in self.params
  248 
  249         :rtype: sqlalchemy.orm.query.Query
  250         :returns: The query with sorting/pagination/filtering added.
  251         """
  252 
  253         # Add filtering
  254         if CONF.api_settings.allow_filtering:
  255             # Exclude (valid) arguments that are not used for data filtering
  256             filter_params = {k: v for k, v in self.params.items()
  257                              if k not in self._auxiliary_arguments}
  258 
  259             secondary_query_filter = filter_params.pop(
  260                 "project_id", None) if (model == models.Amphora) else None
  261 
  262             # Tranlate arguments from API standard to data model's field name
  263             filter_params = (
  264                 model.__v2_wsme__.translate_dict_keys_to_data_model(
  265                     filter_params)
  266             )
  267             if 'loadbalancer_id' in filter_params:
  268                 filter_params['load_balancer_id'] = filter_params.pop(
  269                     'loadbalancer_id')
  270 
  271             # Pop the 'tags' related parameters off before handling the
  272             # other filters. Then apply the 'tags' filters after the
  273             # other filters have been applied.
  274             tag_params = {}
  275             if 'tags' in filter_params:
  276                 tag_params['tags'] = self._prepare_tags_list(
  277                     filter_params.pop('tags'))
  278             if 'tags-any' in filter_params:
  279                 tag_params['tags-any'] = self._prepare_tags_list(
  280                     filter_params.pop('tags-any'))
  281             if 'not-tags' in filter_params:
  282                 tag_params['not-tags'] = self._prepare_tags_list(
  283                     filter_params.pop('not-tags'))
  284             if 'not-tags-any' in filter_params:
  285                 tag_params['not-tags-any'] = self._prepare_tags_list(
  286                     filter_params.pop('not-tags-any'))
  287 
  288             # Drop invalid arguments
  289             self.filters = {k: v for (k, v) in filter_params.items()
  290                             if k in vars(model.__data_model__())}
  291 
  292             if enforce_valid_params and (
  293                 len(self.filters) < len(filter_params)
  294             ):
  295                 raise exceptions.InvalidFilterArgument()
  296 
  297             query = model.apply_filter(query, model, self.filters)
  298             if secondary_query_filter is not None:
  299                 query = query.filter(model.load_balancer.has(
  300                     project_id=secondary_query_filter))
  301 
  302             # Apply tags filtering for the models which support tags.
  303             query = self._apply_tags_filtering(tag_params, model, query)
  304 
  305         # Add sorting
  306         if CONF.api_settings.allow_sorting:
  307             # Add default sort keys (if they are OK for the model)
  308             keys_only = [k[0] for k in self.sort_keys]
  309             for key in constants.DEFAULT_SORT_KEYS:
  310                 if key not in keys_only and hasattr(model, key):
  311                     self.sort_keys.append((key, self.sort_dir))
  312 
  313             for current_sort_key, current_sort_dir in self.sort_keys:
  314                 # Translate sort_key from API standard to data model's name
  315                 current_sort_key = (
  316                     model.__v2_wsme__.translate_key_to_data_model(
  317                         current_sort_key))
  318                 sort_dir_func = {
  319                     constants.ASC: sqlalchemy.asc,
  320                     constants.DESC: sqlalchemy.desc,
  321                 }[current_sort_dir]
  322 
  323                 try:
  324                     # The translated object may be a nested parameter
  325                     # such as vip.ip_address, so handle that case by
  326                     # joining with the nested table.
  327                     if '.' in current_sort_key:
  328                         parent, child = current_sort_key.split('.')
  329                         parent_obj = getattr(model, parent)
  330                         query = query.join(parent_obj)
  331                         sort_key_attr = child
  332                     else:
  333                         sort_key_attr = getattr(model, current_sort_key)
  334                 except AttributeError as e:
  335                     raise exceptions.InvalidSortKey(
  336                         key=current_sort_key) from e
  337                 query = query.order_by(sort_dir_func(sort_key_attr))
  338 
  339         # Add pagination
  340         if CONF.api_settings.allow_pagination:
  341             default = ''  # Default to an empty string if NULL
  342             if self.marker is not None:
  343                 marker_object = self._parse_marker(query.session, model)
  344                 if not marker_object:
  345                     raise exceptions.InvalidMarker(key=self.marker)
  346                 marker_values = []
  347                 for sort_key, _ in self.sort_keys:
  348                     v = getattr(marker_object, sort_key)
  349                     if v is None:
  350                         v = default
  351                     marker_values.append(v)
  352 
  353                 # Build up an array of sort criteria as in the docstring
  354                 criteria_list = []
  355                 for i in range(len(self.sort_keys)):
  356                     crit_attrs = []
  357                     for j in range(i):
  358                         model_attr = getattr(model, self.sort_keys[j][0])
  359                         default = PaginationHelper._get_default_column_value(
  360                             model_attr.property.columns[0].type)
  361                         attr = sa_sql.expression.case(
  362                             [(model_attr.isnot(None),
  363                               model_attr), ], else_=default)
  364                         crit_attrs.append((attr == marker_values[j]))
  365 
  366                     model_attr = getattr(model, self.sort_keys[i][0])
  367                     default = PaginationHelper._get_default_column_value(
  368                         model_attr.property.columns[0].type)
  369                     attr = sa_sql.expression.case(
  370                         [(model_attr.isnot(None),
  371                           model_attr), ], else_=default)
  372                     this_sort_dir = self.sort_keys[i][1]
  373                     if this_sort_dir == constants.DESC:
  374                         if self.page_reverse == "True":
  375                             crit_attrs.append((attr > marker_values[i]))
  376                         else:
  377                             crit_attrs.append((attr < marker_values[i]))
  378                     elif this_sort_dir == constants.ASC:
  379                         if self.page_reverse == "True":
  380                             crit_attrs.append((attr < marker_values[i]))
  381                         else:
  382                             crit_attrs.append((attr > marker_values[i]))
  383                     else:
  384                         raise exceptions.InvalidSortDirection(
  385                             key=this_sort_dir)
  386 
  387                     criteria = sa_sql.and_(*crit_attrs)
  388                     criteria_list.append(criteria)
  389 
  390                 f = sa_sql.or_(*criteria_list)
  391                 query = query.filter(f)
  392 
  393             if self.limit is not None:
  394                 query = query.limit(self.limit)
  395 
  396         model_list = query.all()
  397 
  398         links = None
  399         if CONF.api_settings.allow_pagination:
  400             links = self._make_links(model_list)
  401 
  402         return model_list, links