"Fossies" - the Fresh Open Source Software Archive

Member "barbican-12.0.0/barbican/model/clean.py" (14 Apr 2021, 15284 Bytes) of package /linux/misc/openstack/barbican-12.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "clean.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 11.0.0_vs_12.0.0.

    1 # Copyright (c) 2016 IBM
    2 #
    3 # Licensed under the Apache License, Version 2.0 (the "License");
    4 # you may not use this file except in compliance with the License.
    5 # You may obtain a copy of the License at
    6 #
    7 #    http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 # Unless required by applicable law or agreed to in writing, software
   10 # distributed under the License is distributed on an "AS IS" BASIS,
   11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
   12 # implied.
   13 # See the License for the specific language governing permissions and
   14 # limitations under the License.
   15 
   16 from barbican.common import config
   17 from barbican.model import models
   18 from barbican.model import repositories as repo
   19 from oslo_log import log
   20 from oslo_utils import timeutils
   21 
   22 from sqlalchemy import sql as sa_sql
   23 
   24 import datetime
   25 
   26 # Import and configure logging.
   27 CONF = config.CONF
   28 log.setup(CONF, 'barbican')
   29 LOG = log.getLogger(__name__)
   30 
   31 
   32 def cleanup_unassociated_projects():
   33     """Clean up unassociated projects.
   34 
   35     This looks for projects that have no children entries on the dependent
   36     tables and removes them.
   37     """
   38     LOG.debug("Cleaning up unassociated projects")
   39     session = repo.get_session()
   40     project_children_tables = [models.Order,
   41                                models.KEKDatum,
   42                                models.SecretConsumerMetadatum,
   43                                models.Secret,
   44                                models.ContainerConsumerMetadatum,
   45                                models.Container,
   46                                models.PreferredCertificateAuthority,
   47                                models.CertificateAuthority,
   48                                models.ProjectCertificateAuthority,
   49                                models.ProjectQuotas]
   50     children_names = map(lambda child: child.__name__, project_children_tables)
   51     LOG.debug("Children tables for Project table being checked: %s",
   52               str(children_names))
   53     sub_query = session.query(models.Project.id)
   54     for model in project_children_tables:
   55         sub_query = sub_query.outerjoin(model,
   56                                         models.Project.id == model.project_id)
   57         sub_query = sub_query.filter(model.id == None)  # noqa
   58     sub_query = sub_query.subquery()
   59     sub_query = sa_sql.select([sub_query])
   60     query = session.query(models.Project)
   61     query = query.filter(models.Project.id.in_(sub_query))
   62     delete_count = query.delete(synchronize_session='fetch')
   63     LOG.info("Cleaned up %(delete_count)s entries for "
   64              "%(project_name)s",
   65              {'delete_count': str(delete_count),
   66               'project_name': models.Project.__name__})
   67     return delete_count
   68 
   69 
   70 def cleanup_parent_with_no_child(parent_model, child_model,
   71                                  threshold_date=None):
   72     """Clean up soft deletions in parent that do not have references in child.
   73 
   74     Before running this function, the child table should be cleaned of
   75     soft deletions. This function left outer joins the parent and child
   76     tables and finds the parent entries that do not have a foreign key
   77     reference in the child table. Then the results are filtered by soft
   78     deletions and are cleaned up.
   79 
   80     :param parent_model: table class for parent
   81     :param child_model: table class for child which restricts parent deletion
   82     :param threshold_date: soft deletions older than this date will be removed
   83     :returns: total number of entries removed from database
   84     """
   85     LOG.debug("Cleaning soft deletes for %(parent_name)s without "
   86               "a child in %(child_name)s" %
   87               {'parent_name': parent_model.__name__,
   88                'child_name': child_model.__name__})
   89     session = repo.get_session()
   90     sub_query = session.query(parent_model.id)
   91     sub_query = sub_query.outerjoin(child_model)
   92     sub_query = sub_query.filter(child_model.id == None)  # noqa
   93     sub_query = sub_query.subquery()
   94     sub_query = sa_sql.select([sub_query])
   95     query = session.query(parent_model)
   96     query = query.filter(parent_model.id.in_(sub_query))
   97     query = query.filter(parent_model.deleted)
   98     if threshold_date:
   99         query = query.filter(parent_model.deleted_at <= threshold_date)
  100     delete_count = query.delete(synchronize_session='fetch')
  101     LOG.info("Cleaned up %(delete_count)s entries for %(parent_name)s "
  102              "with no children in %(child_name)s",
  103              {'delete_count': delete_count,
  104               'parent_name': parent_model.__name__,
  105               'child_name': child_model.__name__})
  106     return delete_count
  107 
  108 
  109 def cleanup_softdeletes(model, threshold_date=None):
  110     """Remove soft deletions from a table.
  111 
  112     :param model: table class to remove soft deletions
  113     :param threshold_date: soft deletions older than this date will be removed
  114     :returns: total number of entries removed from the database
  115     """
  116     LOG.debug("Cleaning soft deletes: %s", model.__name__)
  117     session = repo.get_session()
  118     query = session.query(model)
  119     query = query.filter_by(deleted=True)
  120     if threshold_date:
  121         query = query.filter(model.deleted_at <= threshold_date)
  122     delete_count = query.delete()
  123     LOG.info("Cleaned up %(delete_count)s entries for %(model_name)s",
  124              {'delete_count': delete_count,
  125               'model_name': model.__name__})
  126     return delete_count
  127 
  128 
  129 def cleanup_all(threshold_date=None):
  130     """Clean up the main soft deletable resources.
  131 
  132     This function contains an order of calls to
  133     clean up the soft-deletable resources.
  134 
  135     :param threshold_date: soft deletions older than this date will be removed
  136     :returns: total number of entries removed from the database
  137     """
  138     LOG.debug("Cleaning up soft deletions where deletion date"
  139               " is older than %s", str(threshold_date))
  140     total = 0
  141     total += cleanup_softdeletes(models.TransportKey,
  142                                  threshold_date=threshold_date)
  143 
  144     total += cleanup_softdeletes(models.OrderBarbicanMetadatum,
  145                                  threshold_date=threshold_date)
  146     total += cleanup_softdeletes(models.OrderRetryTask,
  147                                  threshold_date=threshold_date)
  148     total += cleanup_softdeletes(models.OrderPluginMetadatum,
  149                                  threshold_date=threshold_date)
  150     total += cleanup_parent_with_no_child(models.Order, models.OrderRetryTask,
  151                                           threshold_date=threshold_date)
  152 
  153     total += cleanup_softdeletes(models.EncryptedDatum,
  154                                  threshold_date=threshold_date)
  155     total += cleanup_softdeletes(models.SecretUserMetadatum,
  156                                  threshold_date=threshold_date)
  157     total += cleanup_softdeletes(models.SecretStoreMetadatum,
  158                                  threshold_date=threshold_date)
  159     total += cleanup_softdeletes(models.ContainerSecret,
  160                                  threshold_date=threshold_date)
  161 
  162     total += cleanup_softdeletes(models.SecretConsumerMetadatum,
  163                                  threshold_date=threshold_date)
  164     total += cleanup_parent_with_no_child(models.Secret, models.Order,
  165                                           threshold_date=threshold_date)
  166 
  167     total += cleanup_softdeletes(models.ContainerConsumerMetadatum,
  168                                  threshold_date=threshold_date)
  169     total += cleanup_parent_with_no_child(models.Container, models.Order,
  170                                           threshold_date=threshold_date)
  171     total += cleanup_softdeletes(models.KEKDatum,
  172                                  threshold_date=threshold_date)
  173 
  174     # TODO(edtubill) Clean up projects that were soft deleted by
  175     # the keystone listener
  176 
  177     LOG.info("Cleaned up %s soft deleted entries", total)
  178     return total
  179 
  180 
  181 def _soft_delete_expired_secrets(threshold_date):
  182     """Soft delete expired secrets.
  183 
  184     :param threshold_date: secrets that have expired past this date
  185                            will be soft deleted
  186     :returns: total number of secrets that were soft deleted
  187     """
  188     current_time = timeutils.utcnow()
  189     session = repo.get_session()
  190     query = session.query(models.Secret.id)
  191     query = query.filter(~models.Secret.deleted)
  192     query = query.filter(
  193         models.Secret.expiration <= threshold_date
  194     )
  195     update_count = query.update(
  196         {
  197             models.Secret.deleted: True,
  198             models.Secret.deleted_at: current_time
  199         },
  200         synchronize_session='fetch')
  201     return update_count
  202 
  203 
  204 def _hard_delete_acls_for_soft_deleted_secrets():
  205     """Remove acl entries for secrets that have been soft deleted.
  206 
  207     Removes entries in SecretACL and SecretACLUser which are for secrets
  208     that have been soft deleted.
  209     """
  210     session = repo.get_session()
  211     acl_user_sub_query = session.query(models.SecretACLUser.id)
  212     acl_user_sub_query = acl_user_sub_query.join(models.SecretACL)
  213     acl_user_sub_query = acl_user_sub_query.join(models.Secret)
  214     acl_user_sub_query = acl_user_sub_query.filter(models.Secret.deleted)
  215     acl_user_sub_query = acl_user_sub_query.subquery()
  216     acl_user_sub_query = sa_sql.select([acl_user_sub_query])
  217 
  218     acl_user_query = session.query(models.SecretACLUser)
  219     acl_user_query = acl_user_query.filter(
  220         models.SecretACLUser.id.in_(acl_user_sub_query))
  221     acl_total = acl_user_query.delete(synchronize_session='fetch')
  222 
  223     acl_sub_query = session.query(models.SecretACL.id)
  224     acl_sub_query = acl_sub_query.join(models.Secret)
  225     acl_sub_query = acl_sub_query.filter(models.Secret.deleted)
  226     acl_sub_query = acl_sub_query.subquery()
  227     acl_sub_query = sa_sql.select([acl_sub_query])
  228 
  229     acl_query = session.query(models.SecretACL)
  230     acl_query = acl_query.filter(
  231         models.SecretACL.id.in_(acl_sub_query))
  232     acl_total += acl_query.delete(synchronize_session='fetch')
  233     return acl_total
  234 
  235 
  236 def _soft_delete_expired_secret_children(threshold_date):
  237     """Soft delete the children tables of expired secrets.
  238 
  239     Soft deletes the children tables  and hard deletes the ACL children
  240     tables of the expired secrets.
  241     :param threshold_date: threshold date for secret expiration
  242     :returns: returns a pair for number of soft delete children and deleted
  243               ACLs
  244     """
  245     current_time = timeutils.utcnow()
  246 
  247     secret_children = [models.SecretStoreMetadatum,
  248                        models.SecretUserMetadatum,
  249                        models.EncryptedDatum,
  250                        models.ContainerSecret]
  251     children_names = map(lambda child: child.__name__, secret_children)
  252     LOG.debug("Children tables for Secret table being checked: %s",
  253               str(children_names))
  254     session = repo.get_session()
  255     update_count = 0
  256 
  257     for table in secret_children:
  258         # Go through children and soft delete them
  259         sub_query = session.query(table.id)
  260         sub_query = sub_query.join(models.Secret)
  261         sub_query = sub_query.filter(
  262             models.Secret.expiration <= threshold_date
  263         )
  264         sub_query = sub_query.subquery()
  265         sub_query = sa_sql.select([sub_query])
  266         query = session.query(table)
  267         query = query.filter(table.id.in_(sub_query))
  268         current_update_count = query.update(
  269             {
  270                 table.deleted: True,
  271                 table.deleted_at: current_time
  272             },
  273             synchronize_session='fetch')
  274         update_count += current_update_count
  275 
  276     session.flush()
  277     acl_total = _hard_delete_acls_for_soft_deleted_secrets()
  278     return update_count, acl_total
  279 
  280 
  281 def soft_delete_expired_secrets(threshold_date):
  282     """Soft deletes secrets that are past expiration date.
  283 
  284     The expired secrets and its children are marked for deletion.
  285     ACLs are soft deleted and then purged from the database.
  286 
  287     :param threshold_date: secrets that have expired past this date
  288                            will be soft deleted
  289     :returns: the sum of soft deleted entries and hard deleted acl entries
  290     """
  291     # Note: sqllite does not support multiple table updates so
  292     # several db updates are used instead
  293     LOG.debug('Soft deleting expired secrets older than: %s',
  294               str(threshold_date))
  295     update_count = _soft_delete_expired_secrets(threshold_date)
  296 
  297     children_count, acl_total = _soft_delete_expired_secret_children(
  298         threshold_date)
  299     update_count += children_count
  300     LOG.info("Soft deleted %(update_count)s entries due to secret "
  301              "expiration and %(acl_total)s secret acl entries "
  302              "were removed from the database",
  303              {'update_count': update_count,
  304               'acl_total': acl_total})
  305     return update_count + acl_total
  306 
  307 
  308 def clean_command(sql_url, min_num_days, do_clean_unassociated_projects,
  309                   do_soft_delete_expired_secrets, verbose, log_file):
  310     """Clean command to clean up the database.
  311 
  312     :param sql_url: sql connection string to connect to a database
  313     :param min_num_days: clean up soft deletions older than this date
  314     :param do_clean_unassociated_projects: If True, clean up
  315                                            unassociated projects
  316     :param do_soft_delete_expired_secrets: If True, soft delete secrets
  317                                            that have expired
  318     :param verbose: If True, log and print more information
  319     :param log_file: If set, override the log_file configured
  320     """
  321     if verbose:
  322         # The verbose flag prints out log events to the screen, otherwise
  323         # the log events will only go to the log file
  324         CONF.set_override('debug', True)
  325 
  326     if log_file:
  327         CONF.set_override('log_file', log_file)
  328 
  329     LOG.info("Cleaning up soft deletions in the barbican database")
  330     log.setup(CONF, 'barbican')
  331 
  332     cleanup_total = 0
  333     current_time = timeutils.utcnow()
  334     stop_watch = timeutils.StopWatch()
  335     stop_watch.start()
  336     try:
  337         if sql_url:
  338             CONF.set_override('sql_connection', sql_url)
  339         repo.setup_database_engine_and_factory()
  340 
  341         if do_clean_unassociated_projects:
  342             cleanup_total += cleanup_unassociated_projects()
  343 
  344         if do_soft_delete_expired_secrets:
  345             cleanup_total += soft_delete_expired_secrets(
  346                 threshold_date=current_time)
  347 
  348         threshold_date = None
  349         if min_num_days >= 0:
  350             threshold_date = current_time - datetime.timedelta(
  351                 days=min_num_days)
  352         else:
  353             threshold_date = current_time
  354         cleanup_total += cleanup_all(threshold_date=threshold_date)
  355         repo.commit()
  356 
  357     except Exception as ex:
  358         LOG.exception('Failed to clean up soft deletions in database.')
  359         repo.rollback()
  360         cleanup_total = 0  # rollback happened, no entries affected
  361         raise ex
  362     finally:
  363         stop_watch.stop()
  364         elapsed_time = stop_watch.elapsed()
  365         if verbose:
  366             CONF.clear_override('debug')
  367 
  368         if log_file:
  369             CONF.clear_override('log_file')
  370         repo.clear()
  371 
  372         if sql_url:
  373             CONF.clear_override('sql_connection')
  374 
  375         log.setup(CONF, 'barbican')  # reset the overrides
  376 
  377         LOG.info("Cleaning of database affected %s entries", cleanup_total)
  378         LOG.info('DB clean up finished in %s seconds', elapsed_time)