"Fossies" - the Fresh Open Source Software Archive

Member "keystone-17.0.0/keystone/tests/unit/test_sql_upgrade.py" (13 May 2020, 139037 Bytes) of package /linux/misc/openstack/keystone-17.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_sql_upgrade.py": 16.0.1_vs_17.0.0.

    1 # Copyright 2012 OpenStack Foundation
    2 #
    3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
    4 # not use this file except in compliance with the License. You may obtain
    5 # a copy of the License at
    6 #
    7 #      http://www.apache.org/licenses/LICENSE-2.0
    8 #
    9 # Unless required by applicable law or agreed to in writing, software
   10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   12 # License for the specific language governing permissions and limitations
   13 # under the License.
   14 """
   15 Test for SQL migration extensions.
   16 
   17 To run these tests against a live database:
   18 
   19 1. Set up a blank, live database.
   20 2. Export database information to environment variable
   21    ``OS_TEST_DBAPI_ADMIN_CONNECTION``. For example::
   22 
   23     export OS_TEST_DBAPI_ADMIN_CONNECTION=postgresql://localhost/postgres?host=
   24     /var/folders/7k/pwdhb_mj2cv4zyr0kyrlzjx40000gq/T/tmpMGqN8C&port=9824
   25 
   26 3. Run the tests using::
   27 
   28     tox -e py27 -- keystone.tests.unit.test_sql_upgrade
   29 
   30 For further information, see `oslo.db documentation
   31 <https://docs.openstack.org/oslo.db/latest/contributor/index.html#how-to-run-unit-tests>`_.
   32 
   33 WARNING::
   34 
   35     Your database will be wiped.
   36 
   37     Do not do this against a database with valuable data as
   38     all data will be lost.
   39 """
   40 
   41 import datetime
   42 import glob
   43 import json
   44 import os
   45 from unittest import mock
   46 import uuid
   47 
   48 import fixtures
   49 import migrate
   50 from migrate.versioning import repository
   51 from migrate.versioning import script
   52 from oslo_db import exception as db_exception
   53 from oslo_db.sqlalchemy import enginefacade
   54 from oslo_db.sqlalchemy import test_fixtures as db_fixtures
   55 from oslo_log import log
   56 from oslo_serialization import jsonutils
   57 from oslotest import base as test_base
   58 import pytz
   59 import sqlalchemy.exc
   60 from sqlalchemy import inspect
   61 from testtools import matchers
   62 
   63 from keystone.cmd import cli
   64 from keystone.common import sql
   65 from keystone.common.sql import upgrades
   66 from keystone.credential.providers import fernet as credential_fernet
   67 from keystone.resource.backends import base as resource_base
   68 from keystone.tests import unit
   69 from keystone.tests.unit import default_fixtures
   70 from keystone.tests.unit import ksfixtures
   71 from keystone.tests.unit.ksfixtures import database
   72 
   73 
   74 # NOTE(morganfainberg): This should be updated when each DB migration collapse
   75 # is done to mirror the expected structure of the DB in the format of
   76 # { <DB_TABLE_NAME>: [<COLUMN>, <COLUMN>, ...], ... }
   77 INITIAL_TABLE_STRUCTURE = {
   78     'credential': [
   79         'id', 'user_id', 'project_id', 'blob', 'type', 'extra',
   80     ],
   81     'domain': [
   82         'id', 'name', 'enabled', 'extra',
   83     ],
   84     'endpoint': [
   85         'id', 'legacy_endpoint_id', 'interface', 'region_id', 'service_id',
   86         'url', 'enabled', 'extra',
   87     ],
   88     'group': [
   89         'id', 'domain_id', 'name', 'description', 'extra',
   90     ],
   91     'policy': [
   92         'id', 'type', 'blob', 'extra',
   93     ],
   94     'project': [
   95         'id', 'name', 'extra', 'description', 'enabled', 'domain_id',
   96         'parent_id',
   97     ],
   98     'role': [
   99         'id', 'name', 'extra',
  100     ],
  101     'service': [
  102         'id', 'type', 'extra', 'enabled',
  103     ],
  104     'token': [
  105         'id', 'expires', 'extra', 'valid', 'trust_id', 'user_id',
  106     ],
  107     'trust': [
  108         'id', 'trustor_user_id', 'trustee_user_id', 'project_id',
  109         'impersonation', 'deleted_at', 'expires_at', 'remaining_uses', 'extra',
  110     ],
  111     'trust_role': [
  112         'trust_id', 'role_id',
  113     ],
  114     'user': [
  115         'id', 'name', 'extra', 'password', 'enabled', 'domain_id',
  116         'default_project_id',
  117     ],
  118     'user_group_membership': [
  119         'user_id', 'group_id',
  120     ],
  121     'region': [
  122         'id', 'description', 'parent_region_id', 'extra',
  123     ],
  124     'assignment': [
  125         'type', 'actor_id', 'target_id', 'role_id', 'inherited',
  126     ],
  127     'id_mapping': [
  128         'public_id', 'domain_id', 'local_id', 'entity_type',
  129     ],
  130     'whitelisted_config': [
  131         'domain_id', 'group', 'option', 'value',
  132     ],
  133     'sensitive_config': [
  134         'domain_id', 'group', 'option', 'value',
  135     ],
  136 }
  137 
  138 LEGACY_REPO = 'migrate_repo'
  139 EXPAND_REPO = 'expand_repo'
  140 DATA_MIGRATION_REPO = 'data_migration_repo'
  141 CONTRACT_REPO = 'contract_repo'
  142 
  143 
  144 # Test upgrades.get_init_version separately to ensure it works before
  145 # using in the SqlUpgrade tests.
  146 class SqlUpgradeGetInitVersionTests(unit.TestCase):
  147     @mock.patch.object(repository, 'Repository')
  148     def test_get_init_version_no_path(self, repo):
  149         migrate_versions = mock.MagicMock()
  150         # make a version list starting with zero. `get_init_version` will
  151         # return None for this value.
  152         migrate_versions.versions.versions = list(range(0, 5))
  153         repo.return_value = migrate_versions
  154 
  155         # os.path.isdir() is called by `find_repo()`. Mock it to avoid
  156         # an exception.
  157         with mock.patch('os.path.isdir', return_value=True):
  158             # since 0 is the smallest version expect None
  159             version = upgrades.get_init_version()
  160             self.assertIsNone(version)
  161 
  162         # check that the default path was used as the first argument to the
  163         # first invocation of repo. Cannot match the full path because it is
  164         # based on where the test is run.
  165         param = repo.call_args_list[0][0][0]
  166         self.assertTrue(param.endswith('/sql/' + LEGACY_REPO))
  167 
  168     @mock.patch.object(repository, 'Repository')
  169     def test_get_init_version_with_path_initial_version_0(self, repo):
  170         migrate_versions = mock.MagicMock()
  171         # make a version list starting with zero. `get_init_version` will
  172         # return None for this value.
  173         migrate_versions.versions.versions = list(range(0, 5))
  174         repo.return_value = migrate_versions
  175 
  176         # os.path.isdir() is called by `find_repo()`. Mock it to avoid
  177         # an exception.
  178         with mock.patch('os.path.isdir', return_value=True):
  179             path = '/keystone/' + LEGACY_REPO + '/'
  180 
  181             # since 0 is the smallest version expect None
  182             version = upgrades.get_init_version(abs_path=path)
  183             self.assertIsNone(version)
  184 
  185     @mock.patch.object(repository, 'Repository')
  186     def test_get_init_version_with_path(self, repo):
  187         initial_version = 10
  188 
  189         migrate_versions = mock.MagicMock()
  190         migrate_versions.versions.versions = list(range(initial_version + 1,
  191                                                         initial_version + 5))
  192         repo.return_value = migrate_versions
  193 
  194         # os.path.isdir() is called by `find_repo()`. Mock it to avoid
  195         # an exception.
  196         with mock.patch('os.path.isdir', return_value=True):
  197             path = '/keystone/' + LEGACY_REPO + '/'
  198 
  199             version = upgrades.get_init_version(abs_path=path)
  200             self.assertEqual(initial_version, version)
  201 
  202 
  203 class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin,
  204                      test_base.BaseTestCase):
  205     def setUp(self):
  206         super(SqlMigrateBase, self).setUp()
  207         self.engine = enginefacade.writer.get_engine()
  208         self.sessionmaker = enginefacade.writer.get_sessionmaker()
  209 
  210         # NOTE(dstanek): Clear out sqlalchemy-migrate's script cache to allow
  211         # us to have multiple repos (expand, migrate, contract) where the
  212         # modules have the same name (001_awesome.py).
  213         self.addCleanup(script.PythonScript.clear)
  214 
  215         # NOTE(dstanek): SQLAlchemy's migrate makes some assumptions in the
  216         # SQLite driver about the lack of foreign key enforcement.
  217         database.initialize_sql_session(self.engine.url,
  218                                         enforce_sqlite_fks=False)
  219 
  220         # Override keystone's context manager to be oslo.db's global context
  221         # manager.
  222         sql.core._TESTING_USE_GLOBAL_CONTEXT_MANAGER = True
  223         self.addCleanup(setattr,
  224                         sql.core, '_TESTING_USE_GLOBAL_CONTEXT_MANAGER', False)
  225         self.addCleanup(sql.cleanup)
  226 
  227         self.repos = {
  228             LEGACY_REPO: upgrades.Repository(self.engine, LEGACY_REPO),
  229             EXPAND_REPO: upgrades.Repository(self.engine, EXPAND_REPO),
  230             DATA_MIGRATION_REPO: upgrades.Repository(
  231                 self.engine, DATA_MIGRATION_REPO),
  232             CONTRACT_REPO: upgrades.Repository(self.engine, CONTRACT_REPO)}
  233 
  234     def upgrade(self, *args, **kwargs):
  235         """Upgrade the legacy migration repository."""
  236         self.repos[LEGACY_REPO].upgrade(*args, **kwargs)
  237 
  238     def expand(self, *args, **kwargs):
  239         """Expand database schema."""
  240         self.repos[EXPAND_REPO].upgrade(*args, **kwargs)
  241 
  242     def migrate(self, *args, **kwargs):
  243         """Migrate data."""
  244         self.repos[DATA_MIGRATION_REPO].upgrade(*args, **kwargs)
  245 
  246     def contract(self, *args, **kwargs):
  247         """Contract database schema."""
  248         self.repos[CONTRACT_REPO].upgrade(*args, **kwargs)
  249 
  250     @property
  251     def metadata(self):
  252         """A collection of tables and their associated schemas."""
  253         return sqlalchemy.MetaData(self.engine)
  254 
  255     def load_table(self, name):
  256         table = sqlalchemy.Table(name,
  257                                  self.metadata,
  258                                  autoload=True)
  259         return table
  260 
  261     def assertTableExists(self, table_name):
  262         try:
  263             self.load_table(table_name)
  264         except sqlalchemy.exc.NoSuchTableError:
  265             raise AssertionError('Table "%s" does not exist' % table_name)
  266 
  267     def assertTableDoesNotExist(self, table_name):
  268         """Assert that a given table exists cannot be selected by name."""
  269         # Switch to a different metadata otherwise you might still
  270         # detect renamed or dropped tables
  271         try:
  272             sqlalchemy.Table(table_name, self.metadata, autoload=True)
  273         except sqlalchemy.exc.NoSuchTableError:
  274             pass
  275         else:
  276             raise AssertionError('Table "%s" already exists' % table_name)
  277 
  278     def calc_table_row_count(self, table_name):
  279         """Return the number of rows in the table."""
  280         t = sqlalchemy.Table(table_name, self.metadata, autoload=True)
  281         session = self.sessionmaker()
  282         row_count = session.query(
  283             sqlalchemy.func.count('*')).select_from(t).scalar()
  284         return row_count
  285 
  286     def assertTableCountsMatch(self, table1_name, table2_name):
  287         table1_count = self.calc_table_row_count(table1_name)
  288         table2_count = self.calc_table_row_count(table2_name)
  289         if table1_count != table2_count:
  290             raise AssertionError('Table counts do not match: {0} ({1}), {2} '
  291                                  '({3})'.format(table1_name, table1_count,
  292                                                 table2_name, table2_count))
  293 
  294     def assertTableColumns(self, table_name, expected_cols):
  295         """Assert that the table contains the expected set of columns."""
  296         table = self.load_table(table_name)
  297         actual_cols = [col.name for col in table.columns]
  298         # Check if the columns are equal, but allow for a different order,
  299         # which might occur after an upgrade followed by a downgrade
  300         self.assertItemsEqual(expected_cols, actual_cols,
  301                               '%s table' % table_name)
  302 
  303     def insert_dict(self, session, table_name, d, table=None):
  304         """Naively inserts key-value pairs into a table, given a dictionary."""
  305         if table is None:
  306             this_table = sqlalchemy.Table(table_name, self.metadata,
  307                                           autoload=True)
  308         else:
  309             this_table = table
  310         insert = this_table.insert().values(**d)
  311         session.execute(insert)
  312 
  313     def does_pk_exist(self, table, pk_column):
  314         """Check whether a column is primary key on a table."""
  315         inspector = inspect(self.engine)
  316         pk_columns = inspector.get_pk_constraint(table)['constrained_columns']
  317 
  318         return pk_column in pk_columns
  319 
  320     def does_fk_exist(self, table, fk_column):
  321         inspector = inspect(self.engine)
  322         for fk in inspector.get_foreign_keys(table):
  323             if fk_column in fk['constrained_columns']:
  324                 return True
  325         return False
  326 
  327     def does_constraint_exist(self, table_name, constraint_name):
  328         table = sqlalchemy.Table(table_name, self.metadata, autoload=True)
  329         return constraint_name in [con.name for con in table.constraints]
  330 
  331     def does_index_exist(self, table_name, index_name):
  332         table = sqlalchemy.Table(table_name, self.metadata, autoload=True)
  333         return index_name in [idx.name for idx in table.indexes]
  334 
  335     def does_unique_constraint_exist(self, table_name, column_names):
  336         inspector = inspect(self.engine)
  337         constraints = inspector.get_unique_constraints(table_name)
  338         for c in constraints:
  339             if (len(c['column_names']) == 1 and
  340                     column_names in c['column_names']):
  341                 return True
  342             if (len(c['column_names'])) > 1 and isinstance(column_names, list):
  343                 return set(c['column_names']) == set(column_names)
  344         return False
  345 
  346 
  347 class SqlLegacyRepoUpgradeTests(SqlMigrateBase):
  348     def test_blank_db_to_start(self):
  349         self.assertTableDoesNotExist('user')
  350 
  351     def test_start_version_db_init_version(self):
  352         self.assertEqual(
  353             self.repos[LEGACY_REPO].min_version,
  354             self.repos[LEGACY_REPO].version,
  355             'DB is not at version %s' % (
  356                 self.repos[LEGACY_REPO].min_version)
  357         )
  358 
  359     def test_upgrade_add_initial_tables(self):
  360         self.upgrade(self.repos[LEGACY_REPO].min_version + 1)
  361         self.check_initial_table_structure()
  362 
  363     def check_initial_table_structure(self):
  364         for table in INITIAL_TABLE_STRUCTURE:
  365             self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table])
  366 
  367     def test_kilo_squash(self):
  368         self.upgrade(67)
  369 
  370         # In 053 the size of ID and parent region ID columns were changed
  371         table = sqlalchemy.Table('region', self.metadata, autoload=True)
  372         self.assertEqual(255, table.c.id.type.length)
  373         self.assertEqual(255, table.c.parent_region_id.type.length)
  374         table = sqlalchemy.Table('endpoint', self.metadata, autoload=True)
  375         self.assertEqual(255, table.c.region_id.type.length)
  376 
  377         # In 054 an index was created for the actor_id of the assignment table
  378         table = sqlalchemy.Table('assignment', self.metadata, autoload=True)
  379         index_data = [(idx.name, list(idx.columns.keys()))
  380                       for idx in table.indexes]
  381         self.assertIn(('ix_actor_id', ['actor_id']), index_data)
  382 
  383         # In 055 indexes were created for user and trust IDs in the token table
  384         table = sqlalchemy.Table('token', self.metadata, autoload=True)
  385         index_data = [(idx.name, list(idx.columns.keys()))
  386                       for idx in table.indexes]
  387         self.assertIn(('ix_token_user_id', ['user_id']), index_data)
  388         self.assertIn(('ix_token_trust_id', ['trust_id']), index_data)
  389 
  390         # In 062 the role ID foreign key was removed from the assignment table
  391         if self.engine.name == "mysql":
  392             self.assertFalse(self.does_fk_exist('assignment', 'role_id'))
  393 
  394         # In 064 the domain ID FK was removed from the group and user tables
  395         if self.engine.name != 'sqlite':
  396             # sqlite does not support FK deletions (or enforcement)
  397             self.assertFalse(self.does_fk_exist('group', 'domain_id'))
  398             self.assertFalse(self.does_fk_exist('user', 'domain_id'))
  399 
  400         # In 067 the role ID index was removed from the assignment table
  401         if self.engine.name == "mysql":
  402             self.assertFalse(self.does_index_exist('assignment',
  403                                                    'assignment_role_id_fkey'))
  404 
  405     def test_insert_assignment_inherited_pk(self):
  406         ASSIGNMENT_TABLE_NAME = 'assignment'
  407         INHERITED_COLUMN_NAME = 'inherited'
  408         ROLE_TABLE_NAME = 'role'
  409 
  410         self.upgrade(72)
  411 
  412         # Check that the 'inherited' column is not part of the PK
  413         self.assertFalse(self.does_pk_exist(ASSIGNMENT_TABLE_NAME,
  414                                             INHERITED_COLUMN_NAME))
  415 
  416         session = self.sessionmaker()
  417 
  418         role = {'id': uuid.uuid4().hex,
  419                 'name': uuid.uuid4().hex}
  420         self.insert_dict(session, ROLE_TABLE_NAME, role)
  421 
  422         # Create both inherited and noninherited role assignments
  423         inherited = {'type': 'UserProject',
  424                      'actor_id': uuid.uuid4().hex,
  425                      'target_id': uuid.uuid4().hex,
  426                      'role_id': role['id'],
  427                      'inherited': True}
  428 
  429         noninherited = inherited.copy()
  430         noninherited['inherited'] = False
  431 
  432         # Create another inherited role assignment as a spoiler
  433         spoiler = inherited.copy()
  434         spoiler['actor_id'] = uuid.uuid4().hex
  435 
  436         self.insert_dict(session, ASSIGNMENT_TABLE_NAME, inherited)
  437         self.insert_dict(session, ASSIGNMENT_TABLE_NAME, spoiler)
  438 
  439         # Since 'inherited' is not part of the PK, we can't insert noninherited
  440         self.assertRaises(db_exception.DBDuplicateEntry,
  441                           self.insert_dict,
  442                           session,
  443                           ASSIGNMENT_TABLE_NAME,
  444                           noninherited)
  445 
  446         session.close()
  447 
  448         self.upgrade(73)
  449 
  450         session = self.sessionmaker()
  451 
  452         # Check that the 'inherited' column is now part of the PK
  453         self.assertTrue(self.does_pk_exist(ASSIGNMENT_TABLE_NAME,
  454                                            INHERITED_COLUMN_NAME))
  455 
  456         # The noninherited role assignment can now be inserted
  457         self.insert_dict(session, ASSIGNMENT_TABLE_NAME, noninherited)
  458 
  459         assignment_table = sqlalchemy.Table(ASSIGNMENT_TABLE_NAME,
  460                                             self.metadata,
  461                                             autoload=True)
  462 
  463         assignments = session.query(assignment_table).all()
  464         for assignment in (inherited, spoiler, noninherited):
  465             self.assertIn((assignment['type'], assignment['actor_id'],
  466                            assignment['target_id'], assignment['role_id'],
  467                            assignment['inherited']),
  468                           assignments)
  469 
  470     def test_endpoint_policy_upgrade(self):
  471         self.assertTableDoesNotExist('policy_association')
  472         self.upgrade(81)
  473         self.assertTableColumns('policy_association',
  474                                 ['id', 'policy_id', 'endpoint_id',
  475                                  'service_id', 'region_id'])
  476 
  477     @mock.patch.object(upgrades, 'get_db_version', return_value=1)
  478     def test_endpoint_policy_already_migrated(self, mock_ep):
  479 
  480         # By setting the return value to 1, the migration has already been
  481         # run, and there's no need to create the table again
  482 
  483         self.upgrade(81)
  484 
  485         mock_ep.assert_called_once_with(extension='endpoint_policy',
  486                                         engine=mock.ANY)
  487 
  488         # It won't exist because we are mocking it, but we can verify
  489         # that 081 did not create the table
  490         self.assertTableDoesNotExist('policy_association')
  491 
  492     def test_create_federation_tables(self):
  493         self.identity_provider = 'identity_provider'
  494         self.federation_protocol = 'federation_protocol'
  495         self.service_provider = 'service_provider'
  496         self.mapping = 'mapping'
  497         self.remote_ids = 'idp_remote_ids'
  498 
  499         self.assertTableDoesNotExist(self.identity_provider)
  500         self.assertTableDoesNotExist(self.federation_protocol)
  501         self.assertTableDoesNotExist(self.service_provider)
  502         self.assertTableDoesNotExist(self.mapping)
  503         self.assertTableDoesNotExist(self.remote_ids)
  504 
  505         self.upgrade(82)
  506         self.assertTableColumns(self.identity_provider,
  507                                 ['id', 'description', 'enabled'])
  508 
  509         self.assertTableColumns(self.federation_protocol,
  510                                 ['id', 'idp_id', 'mapping_id'])
  511 
  512         self.assertTableColumns(self.mapping,
  513                                 ['id', 'rules'])
  514 
  515         self.assertTableColumns(self.service_provider,
  516                                 ['id', 'description', 'enabled', 'auth_url',
  517                                  'relay_state_prefix', 'sp_url'])
  518 
  519         self.assertTableColumns(self.remote_ids, ['idp_id', 'remote_id'])
  520 
  521         federation_protocol = sqlalchemy.Table(self.federation_protocol,
  522                                                self.metadata,
  523                                                autoload=True)
  524         self.assertFalse(federation_protocol.c.mapping_id.nullable)
  525 
  526         sp_table = sqlalchemy.Table(self.service_provider,
  527                                     self.metadata,
  528                                     autoload=True)
  529         self.assertFalse(sp_table.c.auth_url.nullable)
  530         self.assertFalse(sp_table.c.sp_url.nullable)
  531 
  532     @mock.patch.object(upgrades, 'get_db_version', return_value=8)
  533     def test_federation_already_migrated(self, mock_federation):
  534 
  535         # By setting the return value to 8, the migration has already been
  536         # run, and there's no need to create the table again.
  537         self.upgrade(82)
  538 
  539         mock_federation.assert_any_call(extension='federation',
  540                                         engine=mock.ANY)
  541 
  542         # It won't exist because we are mocking it, but we can verify
  543         # that 082 did not create the table.
  544         self.assertTableDoesNotExist('identity_provider')
  545         self.assertTableDoesNotExist('federation_protocol')
  546         self.assertTableDoesNotExist('mapping')
  547         self.assertTableDoesNotExist('service_provider')
  548         self.assertTableDoesNotExist('idp_remote_ids')
  549 
  550     def test_create_oauth_tables(self):
  551         consumer = 'consumer'
  552         request_token = 'request_token'
  553         access_token = 'access_token'
  554         self.assertTableDoesNotExist(consumer)
  555         self.assertTableDoesNotExist(request_token)
  556         self.assertTableDoesNotExist(access_token)
  557         self.upgrade(83)
  558         self.assertTableColumns(consumer,
  559                                 ['id',
  560                                  'description',
  561                                  'secret',
  562                                  'extra'])
  563         self.assertTableColumns(request_token,
  564                                 ['id',
  565                                  'request_secret',
  566                                  'verifier',
  567                                  'authorizing_user_id',
  568                                  'requested_project_id',
  569                                  'role_ids',
  570                                  'consumer_id',
  571                                  'expires_at'])
  572         self.assertTableColumns(access_token,
  573                                 ['id',
  574                                  'access_secret',
  575                                  'authorizing_user_id',
  576                                  'project_id',
  577                                  'role_ids',
  578                                  'consumer_id',
  579                                  'expires_at'])
  580 
  581     @mock.patch.object(upgrades, 'get_db_version', return_value=5)
  582     def test_oauth1_already_migrated(self, mock_oauth1):
  583 
  584         # By setting the return value to 5, the migration has already been
  585         # run, and there's no need to create the table again.
  586         self.upgrade(83)
  587 
  588         mock_oauth1.assert_any_call(extension='oauth1', engine=mock.ANY)
  589 
  590         # It won't exist because we are mocking it, but we can verify
  591         # that 083 did not create the table.
  592         self.assertTableDoesNotExist('consumer')
  593         self.assertTableDoesNotExist('request_token')
  594         self.assertTableDoesNotExist('access_token')
  595 
  596     def test_create_revoke_table(self):
  597         self.assertTableDoesNotExist('revocation_event')
  598         self.upgrade(84)
  599         self.assertTableColumns('revocation_event',
  600                                 ['id', 'domain_id', 'project_id', 'user_id',
  601                                  'role_id', 'trust_id', 'consumer_id',
  602                                  'access_token_id', 'issued_before',
  603                                  'expires_at', 'revoked_at',
  604                                  'audit_chain_id', 'audit_id'])
  605 
  606     @mock.patch.object(upgrades, 'get_db_version', return_value=2)
  607     def test_revoke_already_migrated(self, mock_revoke):
  608 
  609         # By setting the return value to 2, the migration has already been
  610         # run, and there's no need to create the table again.
  611         self.upgrade(84)
  612 
  613         mock_revoke.assert_any_call(extension='revoke', engine=mock.ANY)
  614 
  615         # It won't exist because we are mocking it, but we can verify
  616         # that 084 did not create the table.
  617         self.assertTableDoesNotExist('revocation_event')
  618 
  619     def test_project_is_domain_upgrade(self):
  620         self.upgrade(74)
  621         self.assertTableColumns('project',
  622                                 ['id', 'name', 'extra', 'description',
  623                                  'enabled', 'domain_id', 'parent_id',
  624                                  'is_domain'])
  625 
  626     def test_implied_roles_upgrade(self):
  627         self.upgrade(87)
  628         self.assertTableColumns('implied_role',
  629                                 ['prior_role_id', 'implied_role_id'])
  630         self.assertTrue(self.does_fk_exist('implied_role', 'prior_role_id'))
  631         self.assertTrue(self.does_fk_exist('implied_role', 'implied_role_id'))
  632 
  633     def test_add_config_registration(self):
  634         config_registration = 'config_register'
  635         self.upgrade(74)
  636         self.assertTableDoesNotExist(config_registration)
  637         self.upgrade(75)
  638         self.assertTableColumns(config_registration, ['type', 'domain_id'])
  639 
  640     def test_endpoint_filter_upgrade(self):
  641         def assert_tables_columns_exist():
  642             self.assertTableColumns('project_endpoint',
  643                                     ['endpoint_id', 'project_id'])
  644             self.assertTableColumns('endpoint_group',
  645                                     ['id', 'name', 'description', 'filters'])
  646             self.assertTableColumns('project_endpoint_group',
  647                                     ['endpoint_group_id', 'project_id'])
  648 
  649         self.assertTableDoesNotExist('project_endpoint')
  650         self.upgrade(85)
  651         assert_tables_columns_exist()
  652 
  653     @mock.patch.object(upgrades, 'get_db_version', return_value=2)
  654     def test_endpoint_filter_already_migrated(self, mock_endpoint_filter):
  655 
  656         # By setting the return value to 2, the migration has already been
  657         # run, and there's no need to create the table again.
  658         self.upgrade(85)
  659 
  660         mock_endpoint_filter.assert_any_call(extension='endpoint_filter',
  661                                              engine=mock.ANY)
  662 
  663         # It won't exist because we are mocking it, but we can verify
  664         # that 085 did not create the table.
  665         self.assertTableDoesNotExist('project_endpoint')
  666         self.assertTableDoesNotExist('endpoint_group')
  667         self.assertTableDoesNotExist('project_endpoint_group')
  668 
  669     def test_add_trust_unique_constraint_upgrade(self):
  670         self.upgrade(86)
  671         inspector = inspect(self.engine)
  672         constraints = inspector.get_unique_constraints('trust')
  673         constraint_names = [constraint['name'] for constraint in constraints]
  674         self.assertIn('duplicate_trust_constraint', constraint_names)
  675 
  676     def test_add_domain_specific_roles(self):
  677         """Check database upgraded successfully for domain specific roles.
  678 
  679         The following items need to be checked:
  680 
  681         - The domain_id column has been added
  682         - That it has been added to the uniqueness constraints
  683         - Existing roles have their domain_id columns set to the specific
  684           string of '<<null>>'
  685 
  686         """
  687         NULL_DOMAIN_ID = '<<null>>'
  688 
  689         self.upgrade(87)
  690         session = self.sessionmaker()
  691         role_table = sqlalchemy.Table('role', self.metadata, autoload=True)
  692         # Add a role before we upgrade, so we can check that its new domain_id
  693         # attribute is handled correctly
  694         role_id = uuid.uuid4().hex
  695         self.insert_dict(session, 'role',
  696                          {'id': role_id, 'name': uuid.uuid4().hex})
  697         session.close()
  698 
  699         self.upgrade(88)
  700 
  701         session = self.sessionmaker()
  702         self.assertTableColumns('role', ['id', 'name', 'domain_id', 'extra'])
  703         # Check the domain_id has been added to the uniqueness constraint
  704         inspector = inspect(self.engine)
  705         constraints = inspector.get_unique_constraints('role')
  706         constraint_columns = [
  707             constraint['column_names'] for constraint in constraints
  708             if constraint['name'] == 'ixu_role_name_domain_id']
  709         self.assertIn('domain_id', constraint_columns[0])
  710 
  711         # Now check our role has its domain_id attribute set correctly
  712         role_table = sqlalchemy.Table('role', self.metadata, autoload=True)
  713         cols = [role_table.c.domain_id]
  714         filter = role_table.c.id == role_id
  715         statement = sqlalchemy.select(cols).where(filter)
  716         role_entry = session.execute(statement).fetchone()
  717         self.assertEqual(NULL_DOMAIN_ID, role_entry[0])
  718 
  719     def test_add_root_of_all_domains(self):
  720         NULL_DOMAIN_ID = '<<keystone.domain.root>>'
  721         self.upgrade(89)
  722         session = self.sessionmaker()
  723 
  724         domain_table = sqlalchemy.Table(
  725             'domain', self.metadata, autoload=True)
  726         query = session.query(domain_table).filter_by(id=NULL_DOMAIN_ID)
  727         domain_from_db = query.one()
  728         self.assertIn(NULL_DOMAIN_ID, domain_from_db)
  729 
  730         project_table = sqlalchemy.Table(
  731             'project', self.metadata, autoload=True)
  732         query = session.query(project_table).filter_by(id=NULL_DOMAIN_ID)
  733         project_from_db = query.one()
  734         self.assertIn(NULL_DOMAIN_ID, project_from_db)
  735 
  736         session.close()
  737 
  738     def test_add_local_user_and_password_tables(self):
  739         local_user_table = 'local_user'
  740         password_table = 'password'
  741         self.upgrade(89)
  742         self.assertTableDoesNotExist(local_user_table)
  743         self.assertTableDoesNotExist(password_table)
  744         self.upgrade(90)
  745         self.assertTableColumns(local_user_table,
  746                                 ['id',
  747                                  'user_id',
  748                                  'domain_id',
  749                                  'name'])
  750         self.assertTableColumns(password_table,
  751                                 ['id',
  752                                  'local_user_id',
  753                                  'password'])
  754 
  755     def test_migrate_data_to_local_user_and_password_tables(self):
  756         def get_expected_users():
  757             expected_users = []
  758             for test_user in default_fixtures.USERS:
  759                 user = {}
  760                 user['id'] = uuid.uuid4().hex
  761                 user['name'] = test_user['name']
  762                 user['domain_id'] = test_user['domain_id']
  763                 user['password'] = test_user['password']
  764                 user['enabled'] = True
  765                 user['extra'] = json.dumps(uuid.uuid4().hex)
  766                 user['default_project_id'] = uuid.uuid4().hex
  767                 expected_users.append(user)
  768             return expected_users
  769 
  770         def add_users_to_db(expected_users, user_table):
  771             for user in expected_users:
  772                 ins = user_table.insert().values(
  773                     {'id': user['id'],
  774                      'name': user['name'],
  775                      'domain_id': user['domain_id'],
  776                      'password': user['password'],
  777                      'enabled': user['enabled'],
  778                      'extra': user['extra'],
  779                      'default_project_id': user['default_project_id']})
  780                 ins.execute()
  781 
  782         def get_users_from_db(user_table, local_user_table, password_table):
  783             sel = (
  784                 sqlalchemy.select([user_table.c.id,
  785                                    user_table.c.enabled,
  786                                    user_table.c.extra,
  787                                    user_table.c.default_project_id,
  788                                    local_user_table.c.name,
  789                                    local_user_table.c.domain_id,
  790                                    password_table.c.password])
  791                 .select_from(user_table.join(local_user_table,
  792                                              user_table.c.id ==
  793                                              local_user_table.c.user_id)
  794                                        .join(password_table,
  795                                              local_user_table.c.id ==
  796                                              password_table.c.local_user_id))
  797             )
  798             user_rows = sel.execute()
  799             users = []
  800             for row in user_rows:
  801                 users.append(
  802                     {'id': row['id'],
  803                      'name': row['name'],
  804                      'domain_id': row['domain_id'],
  805                      'password': row['password'],
  806                      'enabled': row['enabled'],
  807                      'extra': row['extra'],
  808                      'default_project_id': row['default_project_id']})
  809             return users
  810 
  811         user_table_name = 'user'
  812         local_user_table_name = 'local_user'
  813         password_table_name = 'password'
  814 
  815         # populate current user table
  816         self.upgrade(90)
  817         user_table = sqlalchemy.Table(
  818             user_table_name, self.metadata, autoload=True)
  819         expected_users = get_expected_users()
  820         add_users_to_db(expected_users, user_table)
  821 
  822         # upgrade to migration and test
  823         self.upgrade(91)
  824         self.assertTableCountsMatch(user_table_name, local_user_table_name)
  825         self.assertTableCountsMatch(local_user_table_name, password_table_name)
  826         user_table = sqlalchemy.Table(
  827             user_table_name, self.metadata, autoload=True)
  828         local_user_table = sqlalchemy.Table(
  829             local_user_table_name, self.metadata, autoload=True)
  830         password_table = sqlalchemy.Table(
  831             password_table_name, self.metadata, autoload=True)
  832         actual_users = get_users_from_db(user_table, local_user_table,
  833                                          password_table)
  834         self.assertItemsEqual(expected_users, actual_users)
  835 
  836     def test_migrate_user_with_null_password_to_password_tables(self):
  837         USER_TABLE_NAME = 'user'
  838         LOCAL_USER_TABLE_NAME = 'local_user'
  839         PASSWORD_TABLE_NAME = 'password'
  840         self.upgrade(90)
  841         user_ref = unit.new_user_ref(uuid.uuid4().hex)
  842         user_ref.pop('password')
  843         # pop extra attribute which doesn't recognized by SQL expression
  844         # layer.
  845         user_ref.pop('email')
  846         session = self.sessionmaker()
  847         self.insert_dict(session, USER_TABLE_NAME, user_ref)
  848         self.upgrade(91)
  849         # migration should be successful.
  850         self.assertTableCountsMatch(USER_TABLE_NAME, LOCAL_USER_TABLE_NAME)
  851         # no new entry was added to the password table because the
  852         # user doesn't have a password.
  853         rows = self.calc_table_row_count(PASSWORD_TABLE_NAME)
  854         self.assertEqual(0, rows)
  855 
  856     def test_migrate_user_skip_user_already_exist_in_local_user(self):
  857         USER_TABLE_NAME = 'user'
  858         LOCAL_USER_TABLE_NAME = 'local_user'
  859         self.upgrade(90)
  860         user1_ref = unit.new_user_ref(uuid.uuid4().hex)
  861         # pop extra attribute which doesn't recognized by SQL expression
  862         # layer.
  863         user1_ref.pop('email')
  864         user2_ref = unit.new_user_ref(uuid.uuid4().hex)
  865         user2_ref.pop('email')
  866         session = self.sessionmaker()
  867         self.insert_dict(session, USER_TABLE_NAME, user1_ref)
  868         self.insert_dict(session, USER_TABLE_NAME, user2_ref)
  869         user_id = user1_ref.pop('id')
  870         user_name = user1_ref.pop('name')
  871         domain_id = user1_ref.pop('domain_id')
  872         local_user_ref = {'user_id': user_id, 'name': user_name,
  873                           'domain_id': domain_id}
  874         self.insert_dict(session, LOCAL_USER_TABLE_NAME, local_user_ref)
  875         self.upgrade(91)
  876         # migration should be successful and user2_ref has been migrated to
  877         # `local_user` table.
  878         self.assertTableCountsMatch(USER_TABLE_NAME, LOCAL_USER_TABLE_NAME)
  879 
  880     def test_implied_roles_fk_on_delete_cascade(self):
  881         if self.engine.name == 'sqlite':
  882             self.skipTest('sqlite backend does not support foreign keys')
  883 
  884         self.upgrade(92)
  885 
  886         session = self.sessionmaker()
  887 
  888         ROLE_TABLE_NAME = 'role'
  889         role_table = sqlalchemy.Table(ROLE_TABLE_NAME, self.metadata,
  890                                       autoload=True)
  891         IMPLIED_ROLE_TABLE_NAME = 'implied_role'
  892         implied_role_table = sqlalchemy.Table(
  893             IMPLIED_ROLE_TABLE_NAME, self.metadata, autoload=True)
  894 
  895         def _create_three_roles():
  896             id_list = []
  897             for _ in range(3):
  898                 new_role_fields = {
  899                     'id': uuid.uuid4().hex,
  900                     'name': uuid.uuid4().hex,
  901                 }
  902                 self.insert_dict(session, ROLE_TABLE_NAME, new_role_fields,
  903                                  table=role_table)
  904                 id_list.append(new_role_fields['id'])
  905             return id_list
  906 
  907         role_id_list = _create_three_roles()
  908         implied_role_fields = {
  909             'prior_role_id': role_id_list[0],
  910             'implied_role_id': role_id_list[1],
  911         }
  912         self.insert_dict(session, IMPLIED_ROLE_TABLE_NAME, implied_role_fields,
  913                          table=implied_role_table)
  914 
  915         implied_role_fields = {
  916             'prior_role_id': role_id_list[0],
  917             'implied_role_id': role_id_list[2],
  918         }
  919         self.insert_dict(session, IMPLIED_ROLE_TABLE_NAME, implied_role_fields,
  920                          table=implied_role_table)
  921 
  922         # assert that there are two roles implied by role 0.
  923         implied_roles = session.query(implied_role_table).filter_by(
  924             prior_role_id=role_id_list[0]).all()
  925         self.assertThat(implied_roles, matchers.HasLength(2))
  926 
  927         session.execute(
  928             role_table.delete().where(role_table.c.id == role_id_list[0]))
  929         # assert the cascade deletion is effective.
  930         implied_roles = session.query(implied_role_table).filter_by(
  931             prior_role_id=role_id_list[0]).all()
  932         self.assertThat(implied_roles, matchers.HasLength(0))
  933 
  934     def test_domain_as_project_upgrade(self):
  935         self.skipTest('Domain as Project Upgrade Test is no longer needed and '
  936                       'unfortunately broken by the resource options code.')
  937 
  938         def _populate_domain_and_project_tables(session):
  939             # Three domains, with various different attributes
  940             self.domains = [{'id': uuid.uuid4().hex,
  941                              'name': uuid.uuid4().hex,
  942                              'enabled': True,
  943                              'extra': {'description': uuid.uuid4().hex,
  944                                        'another_attribute': True}},
  945                             {'id': uuid.uuid4().hex,
  946                              'name': uuid.uuid4().hex,
  947                              'enabled': True,
  948                              'extra': {'description': uuid.uuid4().hex}},
  949                             {'id': uuid.uuid4().hex,
  950                              'name': uuid.uuid4().hex,
  951                              'enabled': False}]
  952             # Four projects, two top level, two children
  953             self.projects = []
  954             self.projects.append(unit.new_project_ref(
  955                 domain_id=self.domains[0]['id'],
  956                 parent_id=None))
  957             self.projects.append(unit.new_project_ref(
  958                 domain_id=self.domains[0]['id'],
  959                 parent_id=self.projects[0]['id']))
  960             self.projects.append(unit.new_project_ref(
  961                 domain_id=self.domains[1]['id'],
  962                 parent_id=None))
  963             self.projects.append(unit.new_project_ref(
  964                 domain_id=self.domains[1]['id'],
  965                 parent_id=self.projects[2]['id']))
  966 
  967             for domain in self.domains:
  968                 this_domain = domain.copy()
  969                 if 'extra' in this_domain:
  970                     this_domain['extra'] = json.dumps(this_domain['extra'])
  971                 self.insert_dict(session, 'domain', this_domain)
  972             for project in self.projects:
  973                 # Tags are done via relationship, not column
  974                 project.pop('tags', None)
  975                 self.insert_dict(session, 'project', project)
  976 
  977         def _check_projects(projects):
  978 
  979             def _assert_domain_matches_project(project):
  980                 for domain in self.domains:
  981                     if project.id == domain['id']:
  982                         self.assertEqual(domain['name'], project.name)
  983                         self.assertEqual(domain['enabled'], project.enabled)
  984                         if domain['id'] == self.domains[0]['id']:
  985                             self.assertEqual(domain['extra']['description'],
  986                                              project.description)
  987                             self.assertEqual({'another_attribute': True},
  988                                              json.loads(project.extra))
  989                         elif domain['id'] == self.domains[1]['id']:
  990                             self.assertEqual(domain['extra']['description'],
  991                                              project.description)
  992                             self.assertEqual({}, json.loads(project.extra))
  993 
  994             # We had domains 3 we created, which should now be projects acting
  995             # as domains, To this we add the 4 original projects, plus the root
  996             # of all domains row.
  997             self.assertEqual(8, projects.count())
  998 
  999             project_ids = []
 1000             for project in projects:
 1001                 if project.is_domain:
 1002                     self.assertEqual(NULL_DOMAIN_ID, project.domain_id)
 1003                     self.assertIsNone(project.parent_id)
 1004                 else:
 1005                     self.assertIsNotNone(project.domain_id)
 1006                     self.assertIsNotNone(project.parent_id)
 1007                 project_ids.append(project.id)
 1008 
 1009             for domain in self.domains:
 1010                 self.assertIn(domain['id'], project_ids)
 1011             for project in self.projects:
 1012                 self.assertIn(project['id'], project_ids)
 1013 
 1014             # Now check the attributes of the domains came across OK
 1015             for project in projects:
 1016                 _assert_domain_matches_project(project)
 1017 
 1018         NULL_DOMAIN_ID = '<<keystone.domain.root>>'
 1019         self.upgrade(92)
 1020 
 1021         session = self.sessionmaker()
 1022 
 1023         _populate_domain_and_project_tables(session)
 1024 
 1025         self.upgrade(93)
 1026         proj_table = sqlalchemy.Table('project', self.metadata, autoload=True)
 1027 
 1028         projects = session.query(proj_table)
 1029         _check_projects(projects)
 1030 
 1031     def test_add_federated_user_table(self):
 1032         federated_user_table = 'federated_user'
 1033         self.upgrade(93)
 1034         self.assertTableDoesNotExist(federated_user_table)
 1035         self.upgrade(94)
 1036         self.assertTableColumns(federated_user_table,
 1037                                 ['id',
 1038                                  'user_id',
 1039                                  'idp_id',
 1040                                  'protocol_id',
 1041                                  'unique_id',
 1042                                  'display_name'])
 1043 
 1044     def test_add_int_pkey_to_revocation_event_table(self):
 1045         REVOCATION_EVENT_TABLE_NAME = 'revocation_event'
 1046         self.upgrade(94)
 1047         revocation_event_table = sqlalchemy.Table(REVOCATION_EVENT_TABLE_NAME,
 1048                                                   self.metadata, autoload=True)
 1049         # assert id column is a string (before)
 1050         self.assertEqual('VARCHAR(64)', str(revocation_event_table.c.id.type))
 1051         self.upgrade(95)
 1052         revocation_event_table = sqlalchemy.Table(REVOCATION_EVENT_TABLE_NAME,
 1053                                                   self.metadata, autoload=True)
 1054         # assert id column is an integer (after)
 1055         self.assertIsInstance(revocation_event_table.c.id.type, sql.Integer)
 1056 
 1057     def _add_unique_constraint_to_role_name(self,
 1058                                             constraint_name='ixu_role_name'):
 1059         role_table = sqlalchemy.Table('role', self.metadata, autoload=True)
 1060         migrate.UniqueConstraint(role_table.c.name,
 1061                                  name=constraint_name).create()
 1062 
 1063     def _drop_unique_constraint_to_role_name(self,
 1064                                              constraint_name='ixu_role_name'):
 1065         role_table = sqlalchemy.Table('role', self.metadata, autoload=True)
 1066         migrate.UniqueConstraint(role_table.c.name,
 1067                                  name=constraint_name).drop()
 1068 
 1069     def _add_unique_constraint_to_user_name_domainid(
 1070             self,
 1071             constraint_name='ixu_role_name'):
 1072         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 1073         migrate.UniqueConstraint(user_table.c.name, user_table.c.domain_id,
 1074                                  name=constraint_name).create()
 1075 
 1076     def _add_name_domain_id_columns_to_user(self):
 1077         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 1078         column_name = sqlalchemy.Column('name', sql.String(255))
 1079         column_domain_id = sqlalchemy.Column('domain_id', sql.String(64))
 1080         user_table.create_column(column_name)
 1081         user_table.create_column(column_domain_id)
 1082 
 1083     def _drop_unique_constraint_to_user_name_domainid(
 1084             self,
 1085             constraint_name='ixu_user_name_domain_id'):
 1086         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 1087         migrate.UniqueConstraint(user_table.c.name, user_table.c.domain_id,
 1088                                  name=constraint_name).drop()
 1089 
 1090     def test_migration_88_drops_unique_constraint(self):
 1091         self.upgrade(87)
 1092         if self.engine.name == 'mysql':
 1093             self.assertTrue(self.does_index_exist('role', 'ixu_role_name'))
 1094         else:
 1095             self.assertTrue(self.does_constraint_exist('role',
 1096                                                        'ixu_role_name'))
 1097         self.upgrade(88)
 1098         if self.engine.name == 'mysql':
 1099             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1100         else:
 1101             self.assertFalse(self.does_constraint_exist('role',
 1102                                                         'ixu_role_name'))
 1103 
 1104     def test_migration_88_inconsistent_constraint_name(self):
 1105         self.upgrade(87)
 1106         self._drop_unique_constraint_to_role_name()
 1107 
 1108         constraint_name = uuid.uuid4().hex
 1109         self._add_unique_constraint_to_role_name(
 1110             constraint_name=constraint_name)
 1111 
 1112         if self.engine.name == 'mysql':
 1113             self.assertTrue(self.does_index_exist('role', constraint_name))
 1114             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1115         else:
 1116             self.assertTrue(self.does_constraint_exist('role',
 1117                                                        constraint_name))
 1118             self.assertFalse(self.does_constraint_exist('role',
 1119                                                         'ixu_role_name'))
 1120 
 1121         self.upgrade(88)
 1122         if self.engine.name == 'mysql':
 1123             self.assertFalse(self.does_index_exist('role', constraint_name))
 1124             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1125         else:
 1126             self.assertFalse(self.does_constraint_exist('role',
 1127                                                         constraint_name))
 1128             self.assertFalse(self.does_constraint_exist('role',
 1129                                                         'ixu_role_name'))
 1130 
 1131     def test_migration_91_drops_unique_constraint(self):
 1132         self.upgrade(90)
 1133         if self.engine.name == 'mysql':
 1134             self.assertTrue(self.does_index_exist('user',
 1135                                                   'ixu_user_name_domain_id'))
 1136         else:
 1137             self.assertTrue(self.does_constraint_exist(
 1138                 'user',
 1139                 'ixu_user_name_domain_id'))
 1140         self.upgrade(91)
 1141         if self.engine.name == 'mysql':
 1142             self.assertFalse(self.does_index_exist(
 1143                 'user',
 1144                 'ixu_user_name_domain_id'))
 1145         else:
 1146             self.assertFalse(self.does_constraint_exist(
 1147                 'user',
 1148                 'ixu_user_name_domain_id'))
 1149 
 1150     def test_migration_91_inconsistent_constraint_name(self):
 1151         self.upgrade(90)
 1152         self._drop_unique_constraint_to_user_name_domainid()
 1153 
 1154         constraint_name = uuid.uuid4().hex
 1155         self._add_unique_constraint_to_user_name_domainid(
 1156             constraint_name=constraint_name)
 1157 
 1158         if self.engine.name == 'mysql':
 1159             self.assertTrue(self.does_index_exist('user', constraint_name))
 1160             self.assertFalse(self.does_index_exist(
 1161                 'user',
 1162                 'ixu_user_name_domain_id'))
 1163         else:
 1164             self.assertTrue(self.does_constraint_exist('user',
 1165                                                        constraint_name))
 1166             self.assertFalse(self.does_constraint_exist(
 1167                 'user',
 1168                 'ixu_user_name_domain_id'))
 1169 
 1170         self.upgrade(91)
 1171         if self.engine.name == 'mysql':
 1172             self.assertFalse(self.does_index_exist('user', constraint_name))
 1173             self.assertFalse(self.does_index_exist(
 1174                 'user',
 1175                 'ixu_user_name_domain_id'))
 1176         else:
 1177             self.assertFalse(self.does_constraint_exist('user',
 1178                                                         constraint_name))
 1179             self.assertFalse(self.does_constraint_exist(
 1180                 'user',
 1181                 'ixu_user_name_domain_id'))
 1182 
 1183     def test_migration_96(self):
 1184         self.upgrade(95)
 1185         if self.engine.name == 'mysql':
 1186             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1187         else:
 1188             self.assertFalse(self.does_constraint_exist('role',
 1189                                                         'ixu_role_name'))
 1190 
 1191         self.upgrade(96)
 1192         if self.engine.name == 'mysql':
 1193             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1194         else:
 1195             self.assertFalse(self.does_constraint_exist('role',
 1196                                                         'ixu_role_name'))
 1197 
 1198     def test_migration_96_constraint_exists(self):
 1199         self.upgrade(95)
 1200         self._add_unique_constraint_to_role_name()
 1201 
 1202         if self.engine.name == 'mysql':
 1203             self.assertTrue(self.does_index_exist('role', 'ixu_role_name'))
 1204         else:
 1205             self.assertTrue(self.does_constraint_exist('role',
 1206                                                        'ixu_role_name'))
 1207 
 1208         self.upgrade(96)
 1209         if self.engine.name == 'mysql':
 1210             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1211         else:
 1212             self.assertFalse(self.does_constraint_exist('role',
 1213                                                         'ixu_role_name'))
 1214 
 1215     def test_migration_97(self):
 1216         self.upgrade(96)
 1217         if self.engine.name == 'mysql':
 1218             self.assertFalse(self.does_index_exist(
 1219                 'user',
 1220                 'ixu_user_name_domain_id'))
 1221         else:
 1222             self.assertFalse(self.does_constraint_exist(
 1223                 'user',
 1224                 'ixu_user_name_domain_id'))
 1225 
 1226         self.upgrade(97)
 1227         if self.engine.name == 'mysql':
 1228             self.assertFalse(self.does_index_exist(
 1229                 'user',
 1230                 'ixu_user_name_domain_id'))
 1231         else:
 1232             self.assertFalse(self.does_constraint_exist(
 1233                 'user',
 1234                 'ixu_user_name_domain_id'))
 1235 
 1236     def test_migration_97_constraint_exists(self):
 1237         self.upgrade(96)
 1238         self._add_name_domain_id_columns_to_user()
 1239         self._add_unique_constraint_to_user_name_domainid(
 1240             constraint_name='ixu_user_name_domain_id')
 1241 
 1242         if self.engine.name == 'mysql':
 1243             self.assertTrue(self.does_index_exist(
 1244                 'user',
 1245                 'ixu_user_name_domain_id'))
 1246         else:
 1247             self.assertTrue(self.does_constraint_exist(
 1248                 'user',
 1249                 'ixu_user_name_domain_id'))
 1250 
 1251         self.upgrade(97)
 1252         if self.engine.name == 'mysql':
 1253             self.assertFalse(self.does_index_exist(
 1254                 'user',
 1255                 'ixu_user_name_domain_id'))
 1256         else:
 1257             self.assertFalse(self.does_constraint_exist(
 1258                 'user',
 1259                 'ixu_user_name_domain_id'))
 1260 
 1261     def test_migration_97_inconsistent_constraint_exists(self):
 1262         self.upgrade(96)
 1263         constraint_name = uuid.uuid4().hex
 1264         self._add_name_domain_id_columns_to_user()
 1265         self._add_unique_constraint_to_user_name_domainid(
 1266             constraint_name=constraint_name)
 1267 
 1268         if self.engine.name == 'mysql':
 1269             self.assertTrue(self.does_index_exist('user', constraint_name))
 1270         else:
 1271             self.assertTrue(self.does_constraint_exist('user',
 1272                                                        constraint_name))
 1273 
 1274         self.upgrade(97)
 1275         if self.engine.name == 'mysql':
 1276             self.assertFalse(self.does_index_exist('user', constraint_name))
 1277         else:
 1278             self.assertFalse(self.does_constraint_exist('user',
 1279                                                         constraint_name))
 1280 
 1281     def test_migration_101(self):
 1282         self.upgrade(100)
 1283         if self.engine.name == 'mysql':
 1284             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1285         else:
 1286             self.assertFalse(self.does_constraint_exist('role',
 1287                                                         'ixu_role_name'))
 1288         self.upgrade(101)
 1289         if self.engine.name == 'mysql':
 1290             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1291         else:
 1292             self.assertFalse(self.does_constraint_exist('role',
 1293                                                         'ixu_role_name'))
 1294 
 1295     def test_migration_101_constraint_exists(self):
 1296         self.upgrade(100)
 1297         self._add_unique_constraint_to_role_name()
 1298 
 1299         if self.engine.name == 'mysql':
 1300             self.assertTrue(self.does_index_exist('role', 'ixu_role_name'))
 1301         else:
 1302             self.assertTrue(self.does_constraint_exist('role',
 1303                                                        'ixu_role_name'))
 1304         self.upgrade(101)
 1305         if self.engine.name == 'mysql':
 1306             self.assertFalse(self.does_index_exist('role', 'ixu_role_name'))
 1307         else:
 1308             self.assertFalse(self.does_constraint_exist('role',
 1309                                                         'ixu_role_name'))
 1310 
 1311     def test_drop_domain_table(self):
 1312         self.upgrade(101)
 1313         self.assertTableExists('domain')
 1314         self.upgrade(102)
 1315         self.assertTableDoesNotExist('domain')
 1316 
 1317     def test_add_nonlocal_user_table(self):
 1318         nonlocal_user_table = 'nonlocal_user'
 1319         self.upgrade(102)
 1320         self.assertTableDoesNotExist(nonlocal_user_table)
 1321         self.upgrade(103)
 1322         self.assertTableColumns(nonlocal_user_table,
 1323                                 ['domain_id',
 1324                                  'name',
 1325                                  'user_id'])
 1326 
 1327     def test_migration_104(self):
 1328         self.upgrade(103)
 1329         if self.engine.name == 'mysql':
 1330             self.assertFalse(self.does_index_exist(
 1331                 'user',
 1332                 'ixu_user_name_domain_id'))
 1333         else:
 1334             self.assertFalse(self.does_constraint_exist(
 1335                 'user',
 1336                 'ixu_user_name_domain_id'))
 1337 
 1338         self.upgrade(104)
 1339         if self.engine.name == 'mysql':
 1340             self.assertFalse(self.does_index_exist(
 1341                 'user',
 1342                 'ixu_user_name_domain_id'))
 1343         else:
 1344             self.assertFalse(self.does_constraint_exist(
 1345                 'user',
 1346                 'ixu_user_name_domain_id'))
 1347 
 1348     def test_migration_104_constraint_exists(self):
 1349         self.upgrade(103)
 1350         self._add_name_domain_id_columns_to_user()
 1351         self._add_unique_constraint_to_user_name_domainid(
 1352             constraint_name='ixu_user_name_domain_id')
 1353 
 1354         if self.engine.name == 'mysql':
 1355             self.assertTrue(self.does_index_exist(
 1356                 'user',
 1357                 'ixu_user_name_domain_id'))
 1358         else:
 1359             self.assertTrue(self.does_constraint_exist(
 1360                 'user',
 1361                 'ixu_user_name_domain_id'))
 1362 
 1363         self.upgrade(104)
 1364         if self.engine.name == 'mysql':
 1365             self.assertFalse(self.does_index_exist(
 1366                 'user',
 1367                 'ixu_user_name_domain_id'))
 1368         else:
 1369             self.assertFalse(self.does_constraint_exist(
 1370                 'user',
 1371                 'ixu_user_name_domain_id'))
 1372 
 1373     def test_migration_104_inconsistent_constraint_exists(self):
 1374         self.upgrade(103)
 1375         constraint_name = uuid.uuid4().hex
 1376         self._add_name_domain_id_columns_to_user()
 1377         self._add_unique_constraint_to_user_name_domainid(
 1378             constraint_name=constraint_name)
 1379 
 1380         if self.engine.name == 'mysql':
 1381             self.assertTrue(self.does_index_exist('user', constraint_name))
 1382         else:
 1383             self.assertTrue(self.does_constraint_exist('user',
 1384                                                        constraint_name))
 1385 
 1386         self.upgrade(104)
 1387         if self.engine.name == 'mysql':
 1388             self.assertFalse(self.does_index_exist('user', constraint_name))
 1389         else:
 1390             self.assertFalse(self.does_constraint_exist('user',
 1391                                                         constraint_name))
 1392 
 1393     def test_migration_105_add_password_date_columns(self):
 1394         def add_user_model_record(session):
 1395             # add a user
 1396             user = {'id': uuid.uuid4().hex}
 1397             self.insert_dict(session, 'user', user)
 1398             # add a local user
 1399             local_user = {
 1400                 'id': 1,
 1401                 'user_id': user['id'],
 1402                 'domain_id': 'default',
 1403                 'name': uuid.uuid4().hex
 1404             }
 1405             self.insert_dict(session, 'local_user', local_user)
 1406             # add a password
 1407             password = {
 1408                 'local_user_id': local_user['id'],
 1409                 'password': uuid.uuid4().hex
 1410             }
 1411             self.insert_dict(session, 'password', password)
 1412         self.upgrade(104)
 1413         session = self.sessionmaker()
 1414         password_name = 'password'
 1415         # columns before
 1416         self.assertTableColumns(password_name,
 1417                                 ['id',
 1418                                  'local_user_id',
 1419                                  'password'])
 1420         # add record and verify table count is greater than zero
 1421         add_user_model_record(session)
 1422         password_table = sqlalchemy.Table(password_name, self.metadata,
 1423                                           autoload=True)
 1424         cnt = session.query(password_table).count()
 1425         self.assertGreater(cnt, 0)
 1426         self.upgrade(105)
 1427         # columns after
 1428         self.assertTableColumns(password_name,
 1429                                 ['id',
 1430                                  'local_user_id',
 1431                                  'password',
 1432                                  'created_at',
 1433                                  'expires_at'])
 1434         password_table = sqlalchemy.Table(password_name, self.metadata,
 1435                                           autoload=True)
 1436         # verify created_at is not null
 1437         null_created_at_cnt = (
 1438             session.query(password_table).filter_by(created_at=None).count())
 1439         self.assertEqual(null_created_at_cnt, 0)
 1440         # verify expires_at is null
 1441         null_expires_at_cnt = (
 1442             session.query(password_table).filter_by(expires_at=None).count())
 1443         self.assertGreater(null_expires_at_cnt, 0)
 1444 
 1445     def test_migration_106_allow_password_column_to_be_nullable(self):
 1446         password_table_name = 'password'
 1447         self.upgrade(105)
 1448         password_table = sqlalchemy.Table(password_table_name, self.metadata,
 1449                                           autoload=True)
 1450         self.assertFalse(password_table.c.password.nullable)
 1451         self.upgrade(106)
 1452         password_table = sqlalchemy.Table(password_table_name, self.metadata,
 1453                                           autoload=True)
 1454         self.assertTrue(password_table.c.password.nullable)
 1455 
 1456     def test_migration_107_add_user_date_columns(self):
 1457         user_table = 'user'
 1458         self.upgrade(106)
 1459         self.assertTableColumns(user_table,
 1460                                 ['id',
 1461                                  'extra',
 1462                                  'enabled',
 1463                                  'default_project_id'])
 1464         self.upgrade(107)
 1465         self.assertTableColumns(user_table,
 1466                                 ['id',
 1467                                  'extra',
 1468                                  'enabled',
 1469                                  'default_project_id',
 1470                                  'created_at',
 1471                                  'last_active_at'])
 1472 
 1473     def test_migration_108_add_failed_auth_columns(self):
 1474         self.upgrade(107)
 1475         table_name = 'local_user'
 1476         self.assertTableColumns(table_name,
 1477                                 ['id',
 1478                                  'user_id',
 1479                                  'domain_id',
 1480                                  'name'])
 1481         self.upgrade(108)
 1482         self.assertTableColumns(table_name,
 1483                                 ['id',
 1484                                  'user_id',
 1485                                  'domain_id',
 1486                                  'name',
 1487                                  'failed_auth_count',
 1488                                  'failed_auth_at'])
 1489 
 1490     def test_migration_109_add_password_self_service_column(self):
 1491         password_table = 'password'
 1492         self.upgrade(108)
 1493         self.assertTableColumns(password_table,
 1494                                 ['id',
 1495                                  'local_user_id',
 1496                                  'password',
 1497                                  'created_at',
 1498                                  'expires_at'])
 1499         self.upgrade(109)
 1500         self.assertTableColumns(password_table,
 1501                                 ['id',
 1502                                  'local_user_id',
 1503                                  'password',
 1504                                  'created_at',
 1505                                  'expires_at',
 1506                                  'self_service'])
 1507 
 1508 
 1509 class MySQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
 1510     FIXTURE = db_fixtures.MySQLOpportunisticFixture
 1511 
 1512 
 1513 class PostgreSQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests):
 1514     FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
 1515 
 1516 
 1517 class SqlExpandSchemaUpgradeTests(SqlMigrateBase):
 1518 
 1519     def setUp(self):
 1520         # Make sure the main repo is fully upgraded for this release since the
 1521         # expand phase is only run after such an upgrade
 1522         super(SqlExpandSchemaUpgradeTests, self).setUp()
 1523         self.upgrade()
 1524 
 1525     def test_start_version_db_init_version(self):
 1526         self.assertEqual(
 1527             self.repos[EXPAND_REPO].min_version,
 1528             self.repos[EXPAND_REPO].version)
 1529 
 1530 
 1531 class MySQLOpportunisticExpandSchemaUpgradeTestCase(
 1532         SqlExpandSchemaUpgradeTests):
 1533     FIXTURE = db_fixtures.MySQLOpportunisticFixture
 1534 
 1535 
 1536 class PostgreSQLOpportunisticExpandSchemaUpgradeTestCase(
 1537         SqlExpandSchemaUpgradeTests):
 1538     FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
 1539 
 1540 
 1541 class SqlDataMigrationUpgradeTests(SqlMigrateBase):
 1542 
 1543     def setUp(self):
 1544         # Make sure the legacy and expand repos are fully upgraded, since the
 1545         # data migration phase is only run after these are upgraded
 1546         super(SqlDataMigrationUpgradeTests, self).setUp()
 1547         self.upgrade()
 1548         self.expand()
 1549 
 1550     def test_start_version_db_init_version(self):
 1551         self.assertEqual(
 1552             self.repos[DATA_MIGRATION_REPO].min_version,
 1553             self.repos[DATA_MIGRATION_REPO].version)
 1554 
 1555 
 1556 class MySQLOpportunisticDataMigrationUpgradeTestCase(
 1557         SqlDataMigrationUpgradeTests):
 1558     FIXTURE = db_fixtures.MySQLOpportunisticFixture
 1559 
 1560 
 1561 class PostgreSQLOpportunisticDataMigrationUpgradeTestCase(
 1562         SqlDataMigrationUpgradeTests):
 1563     FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
 1564 
 1565 
 1566 class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase):
 1567 
 1568     def setUp(self):
 1569         # Make sure the legacy, expand and data migration repos are fully
 1570         # upgraded, since the contract phase is only run after these are
 1571         # upgraded.
 1572         super(SqlContractSchemaUpgradeTests, self).setUp()
 1573         self.useFixture(
 1574             ksfixtures.KeyRepository(
 1575                 self.config_fixture,
 1576                 'credential',
 1577                 credential_fernet.MAX_ACTIVE_KEYS
 1578             )
 1579         )
 1580         self.upgrade()
 1581         self.expand()
 1582         self.migrate()
 1583 
 1584     def test_start_version_db_init_version(self):
 1585         self.assertEqual(
 1586             self.repos[CONTRACT_REPO].min_version,
 1587             self.repos[CONTRACT_REPO].version)
 1588 
 1589 
 1590 class MySQLOpportunisticContractSchemaUpgradeTestCase(
 1591         SqlContractSchemaUpgradeTests):
 1592     FIXTURE = db_fixtures.MySQLOpportunisticFixture
 1593 
 1594 
 1595 class PostgreSQLOpportunisticContractSchemaUpgradeTestCase(
 1596         SqlContractSchemaUpgradeTests):
 1597     FIXTURE = db_fixtures.PostgresqlOpportunisticFixture
 1598 
 1599 
 1600 class VersionTests(SqlMigrateBase):
 1601     def test_core_initial(self):
 1602         """Get the version before migrated, it's the initial DB version."""
 1603         self.assertEqual(
 1604             self.repos[LEGACY_REPO].min_version,
 1605             self.repos[LEGACY_REPO].version)
 1606 
 1607     def test_core_max(self):
 1608         """When get the version after upgrading, it's the new version."""
 1609         self.upgrade()
 1610         self.assertEqual(
 1611             self.repos[LEGACY_REPO].max_version,
 1612             self.repos[LEGACY_REPO].version)
 1613 
 1614     def test_assert_not_schema_downgrade(self):
 1615         self.upgrade()
 1616         self.assertRaises(
 1617             db_exception.DBMigrationError,
 1618             upgrades._sync_common_repo,
 1619             self.repos[LEGACY_REPO].max_version - 1)
 1620 
 1621     def test_these_are_not_the_migrations_you_are_looking_for(self):
 1622         """Keystone has shifted to rolling upgrades.
 1623 
 1624         New database migrations should no longer land in the legacy migration
 1625         repository. Instead, new database migrations should be divided into
 1626         three discrete steps: schema expansion, data migration, and schema
 1627         contraction. These migrations live in a new set of database migration
 1628         repositories, called ``expand_repo``, ``data_migration_repo``, and
 1629         ``contract_repo``.
 1630 
 1631         For more information, see "Database Migrations" here:
 1632 
 1633             https://docs.openstack.org/keystone/latest/contributor/database-migrations.html
 1634 
 1635         """
 1636         # Note to reviewers: this version number should never change.
 1637         self.assertEqual(109, self.repos[LEGACY_REPO].max_version)
 1638 
 1639     def test_migrate_repos_stay_in_lockstep(self):
 1640         """Rolling upgrade repositories should always stay in lockstep.
 1641 
 1642         By maintaining a single "latest" version number in each of the three
 1643         migration repositories (expand, data migrate, and contract), we can
 1644         trivially prevent operators from "doing the wrong thing", such as
 1645         running upgrades operations out of order (for example, you should not
 1646         be able to run data migration 5 until schema expansion 5 has been run).
 1647 
 1648         For example, even if your rolling upgrade task *only* involves adding a
 1649         new column with a reasonable default, and doesn't require any triggers,
 1650         data migration, etc, you still need to create "empty" upgrade steps in
 1651         the data migration and contract repositories with the same version
 1652         number as the expansion.
 1653 
 1654         For more information, see "Database Migrations" here:
 1655 
 1656             https://docs.openstack.org/keystone/latest/contributor/database-migrations.html
 1657 
 1658         """
 1659         # Transitive comparison: expand == data migration == contract
 1660         self.assertEqual(
 1661             self.repos[EXPAND_REPO].max_version,
 1662             self.repos[DATA_MIGRATION_REPO].max_version)
 1663         self.assertEqual(
 1664             self.repos[DATA_MIGRATION_REPO].max_version,
 1665             self.repos[CONTRACT_REPO].max_version)
 1666 
 1667     def test_migrate_repos_file_names_have_prefix(self):
 1668         """Migration files should be unique to avoid caching errors.
 1669 
 1670         This test enforces migration files to include a prefix (expand,
 1671         migrate, contract) in order to keep them unique. Here is the required
 1672         format: [version]_[prefix]_[description]. For example:
 1673         001_expand_add_created_column.py
 1674 
 1675         """
 1676         versions_path = '/versions'
 1677         # test for expand prefix, e.g. 001_expand_new_fk_constraint.py
 1678         expand_list = glob.glob(
 1679             self.repos[EXPAND_REPO].repo_path + versions_path + '/*.py')
 1680         self.assertRepoFileNamePrefix(expand_list, 'expand')
 1681         # test for migrate prefix, e.g. 001_migrate_new_fk_constraint.py
 1682         repo_path = self.repos[DATA_MIGRATION_REPO].repo_path
 1683         migrate_list = glob.glob(repo_path + versions_path + '/*.py')
 1684         self.assertRepoFileNamePrefix(migrate_list, 'migrate')
 1685         # test for contract prefix, e.g. 001_contract_new_fk_constraint.py
 1686         contract_list = glob.glob(
 1687             self.repos[CONTRACT_REPO].repo_path + versions_path + '/*.py')
 1688         self.assertRepoFileNamePrefix(contract_list, 'contract')
 1689 
 1690     def assertRepoFileNamePrefix(self, repo_list, prefix):
 1691         if len(repo_list) > 1:
 1692             # grab the file name for the max version
 1693             file_name = os.path.basename(sorted(repo_list)[-2])
 1694             # pattern for the prefix standard, ignoring placeholder, init files
 1695             pattern = (
 1696                 '^[0-9]{3,}_PREFIX_|^[0-9]{3,}_placeholder.py|^__init__.py')
 1697             pattern = pattern.replace('PREFIX', prefix)
 1698             msg = 'Missing required prefix %s in $file_name' % prefix
 1699             self.assertRegex(file_name, pattern, msg)
 1700 
 1701 
 1702 class MigrationValidation(SqlMigrateBase, unit.TestCase):
 1703     """Test validation of database between database phases."""
 1704 
 1705     def _set_db_sync_command_versions(self):
 1706         self.expand(1)
 1707         self.migrate(1)
 1708         self.contract(1)
 1709         self.assertEqual(upgrades.get_db_version('expand_repo'), 1)
 1710         self.assertEqual(upgrades.get_db_version('data_migration_repo'), 1)
 1711         self.assertEqual(upgrades.get_db_version('contract_repo'), 1)
 1712 
 1713     def test_running_db_sync_expand_without_up_to_date_legacy_fails(self):
 1714         # Set Legacy version and then test that running expand fails if Legacy
 1715         # isn't at the latest version.
 1716         self.upgrade(67)
 1717         latest_version = self.repos[EXPAND_REPO].max_version
 1718         self.assertRaises(
 1719             db_exception.DBMigrationError,
 1720             self.expand,
 1721             latest_version,
 1722             "You are attempting to upgrade migrate ahead of expand")
 1723 
 1724     def test_running_db_sync_migrate_ahead_of_expand_fails(self):
 1725         self.upgrade()
 1726         self._set_db_sync_command_versions()
 1727         self.assertRaises(
 1728             db_exception.DBMigrationError,
 1729             self.migrate,
 1730             2,
 1731             "You are attempting to upgrade migrate ahead of expand")
 1732 
 1733     def test_running_db_sync_contract_ahead_of_migrate_fails(self):
 1734         self.upgrade()
 1735         self._set_db_sync_command_versions()
 1736         self.assertRaises(
 1737             db_exception.DBMigrationError,
 1738             self.contract,
 1739             2,
 1740             "You are attempting to upgrade contract ahead of migrate")
 1741 
 1742 
 1743 class FullMigration(SqlMigrateBase, unit.TestCase):
 1744     """Test complete orchestration between all database phases."""
 1745 
 1746     def setUp(self):
 1747         super(FullMigration, self).setUp()
 1748         # Upgrade the legacy repository
 1749         self.upgrade()
 1750 
 1751     def test_db_sync_check(self):
 1752         checker = cli.DbSync()
 1753         latest_version = self.repos[EXPAND_REPO].max_version
 1754 
 1755         # If the expand repository doesn't exist yet, then we need to make sure
 1756         # we advertise that `--expand` must be run first.
 1757         log_info = self.useFixture(fixtures.FakeLogger(level=log.INFO))
 1758         status = checker.check_db_sync_status()
 1759         self.assertIn("keystone-manage db_sync --expand", log_info.output)
 1760         self.assertEqual(status, 2)
 1761 
 1762         # Assert the correct message is printed when expand is the first step
 1763         # that needs to run
 1764         self.expand(1)
 1765         log_info = self.useFixture(fixtures.FakeLogger(level=log.INFO))
 1766         status = checker.check_db_sync_status()
 1767         self.assertIn("keystone-manage db_sync --expand", log_info.output)
 1768         self.assertEqual(status, 2)
 1769 
 1770         # Assert the correct message is printed when expand is farther than
 1771         # migrate
 1772         self.expand(latest_version)
 1773         log_info = self.useFixture(fixtures.FakeLogger(level=log.INFO))
 1774         status = checker.check_db_sync_status()
 1775         self.assertIn("keystone-manage db_sync --migrate", log_info.output)
 1776         self.assertEqual(status, 3)
 1777 
 1778         # Assert the correct message is printed when migrate is farther than
 1779         # contract
 1780         self.migrate(latest_version)
 1781         log_info = self.useFixture(fixtures.FakeLogger(level=log.INFO))
 1782         status = checker.check_db_sync_status()
 1783         self.assertIn("keystone-manage db_sync --contract", log_info.output)
 1784         self.assertEqual(status, 4)
 1785 
 1786         # Assert the correct message gets printed when all commands are on
 1787         # the same version
 1788         self.contract(latest_version)
 1789         log_info = self.useFixture(fixtures.FakeLogger(level=log.INFO))
 1790         status = checker.check_db_sync_status()
 1791         self.assertIn("All db_sync commands are upgraded", log_info.output)
 1792         self.assertEqual(status, 0)
 1793 
 1794     def test_out_of_sync_db_migration_fails(self):
 1795         # We shouldn't allow for operators to accidentally run migration out of
 1796         # order. This test ensures we fail if we attempt to upgrade the
 1797         # contract repository ahead of the expand or migrate repositories.
 1798         self.expand(3)
 1799         self.migrate(3)
 1800         self.assertRaises(db_exception.DBMigrationError, self.contract, 4)
 1801 
 1802     def test_migration_002_password_created_at_not_nullable(self):
 1803         # upgrade each repository to 001
 1804         self.expand(1)
 1805         self.migrate(1)
 1806         self.contract(1)
 1807 
 1808         password = sqlalchemy.Table('password', self.metadata, autoload=True)
 1809         self.assertTrue(password.c.created_at.nullable)
 1810         # upgrade each repository to 002
 1811         self.expand(2)
 1812         self.migrate(2)
 1813         self.contract(2)
 1814         password = sqlalchemy.Table('password', self.metadata, autoload=True)
 1815         if self.engine.name != 'sqlite':
 1816             self.assertFalse(password.c.created_at.nullable)
 1817 
 1818     def test_migration_003_migrate_unencrypted_credentials(self):
 1819         self.useFixture(
 1820             ksfixtures.KeyRepository(
 1821                 self.config_fixture,
 1822                 'credential',
 1823                 credential_fernet.MAX_ACTIVE_KEYS
 1824             )
 1825         )
 1826 
 1827         session = self.sessionmaker()
 1828         credential_table_name = 'credential'
 1829 
 1830         # upgrade each repository to 002
 1831         self.expand(2)
 1832         self.migrate(2)
 1833         self.contract(2)
 1834 
 1835         # populate the credential table with some sample credentials
 1836         credentials = list()
 1837         for i in range(5):
 1838             credential = {'id': uuid.uuid4().hex,
 1839                           'blob': uuid.uuid4().hex,
 1840                           'user_id': uuid.uuid4().hex,
 1841                           'type': 'cert'}
 1842             credentials.append(credential)
 1843             self.insert_dict(session, credential_table_name, credential)
 1844 
 1845         # verify the current schema
 1846         self.assertTableColumns(
 1847             credential_table_name,
 1848             ['id', 'user_id', 'project_id', 'type', 'blob', 'extra']
 1849         )
 1850 
 1851         # upgrade expand repo to 003 to add new columns
 1852         self.expand(3)
 1853 
 1854         # verify encrypted_blob and key_hash columns have been added and verify
 1855         # the original blob column is still there
 1856         self.assertTableColumns(
 1857             credential_table_name,
 1858             ['id', 'user_id', 'project_id', 'type', 'blob', 'extra',
 1859              'key_hash', 'encrypted_blob']
 1860         )
 1861 
 1862         # verify triggers by making sure we can't write to the credential table
 1863         credential = {'id': uuid.uuid4().hex,
 1864                       'blob': uuid.uuid4().hex,
 1865                       'user_id': uuid.uuid4().hex,
 1866                       'type': 'cert'}
 1867         self.assertRaises(db_exception.DBError,
 1868                           self.insert_dict,
 1869                           session,
 1870                           credential_table_name,
 1871                           credential)
 1872 
 1873         # upgrade migrate repo to 003 to migrate existing credentials
 1874         self.migrate(3)
 1875 
 1876         # make sure we've actually updated the credential with the
 1877         # encrypted blob and the corresponding key hash
 1878         credential_table = sqlalchemy.Table(
 1879             credential_table_name,
 1880             self.metadata,
 1881             autoload=True
 1882         )
 1883         for credential in credentials:
 1884             filter = credential_table.c.id == credential['id']
 1885             cols = [credential_table.c.key_hash, credential_table.c.blob,
 1886                     credential_table.c.encrypted_blob]
 1887             q = sqlalchemy.select(cols).where(filter)
 1888             result = session.execute(q).fetchone()
 1889 
 1890             self.assertIsNotNone(result.encrypted_blob)
 1891             self.assertIsNotNone(result.key_hash)
 1892             # verify the original blob column is still populated
 1893             self.assertEqual(result.blob, credential['blob'])
 1894 
 1895         # verify we can't make any writes to the credential table
 1896         credential = {'id': uuid.uuid4().hex,
 1897                       'blob': uuid.uuid4().hex,
 1898                       'user_id': uuid.uuid4().hex,
 1899                       'key_hash': uuid.uuid4().hex,
 1900                       'type': 'cert'}
 1901         self.assertRaises(db_exception.DBError,
 1902                           self.insert_dict,
 1903                           session,
 1904                           credential_table_name,
 1905                           credential)
 1906 
 1907         # upgrade contract repo to 003 to remove triggers and blob column
 1908         self.contract(3)
 1909 
 1910         # verify the new schema doesn't have a blob column anymore
 1911         self.assertTableColumns(
 1912             credential_table_name,
 1913             ['id', 'user_id', 'project_id', 'type', 'extra', 'key_hash',
 1914              'encrypted_blob']
 1915         )
 1916 
 1917         # verify that the triggers are gone by writing to the database
 1918         credential = {'id': uuid.uuid4().hex,
 1919                       'encrypted_blob': uuid.uuid4().hex,
 1920                       'key_hash': uuid.uuid4().hex,
 1921                       'user_id': uuid.uuid4().hex,
 1922                       'type': 'cert'}
 1923         self.insert_dict(session, credential_table_name, credential)
 1924 
 1925     def test_migration_004_reset_password_created_at(self):
 1926         # upgrade each repository to 003 and test
 1927         self.expand(3)
 1928         self.migrate(3)
 1929         self.contract(3)
 1930         password = sqlalchemy.Table('password', self.metadata, autoload=True)
 1931         # postgresql returns 'TIMESTAMP WITHOUT TIME ZONE'
 1932         self.assertTrue(
 1933             str(password.c.created_at.type).startswith('TIMESTAMP'))
 1934         # upgrade each repository to 004 and test
 1935         self.expand(4)
 1936         self.migrate(4)
 1937         self.contract(4)
 1938         password = sqlalchemy.Table('password', self.metadata, autoload=True)
 1939         # type would still be TIMESTAMP with postgresql
 1940         if self.engine.name == 'postgresql':
 1941             self.assertTrue(
 1942                 str(password.c.created_at.type).startswith('TIMESTAMP'))
 1943         else:
 1944             self.assertEqual('DATETIME', str(password.c.created_at.type))
 1945         self.assertFalse(password.c.created_at.nullable)
 1946 
 1947     def test_migration_010_add_revocation_event_indexes(self):
 1948         self.expand(9)
 1949         self.migrate(9)
 1950         self.contract(9)
 1951         self.assertFalse(self.does_index_exist(
 1952             'revocation_event',
 1953             'ix_revocation_event_issued_before'))
 1954         self.assertFalse(self.does_index_exist(
 1955             'revocation_event',
 1956             'ix_revocation_event_project_id_issued_before'))
 1957         self.assertFalse(self.does_index_exist(
 1958             'revocation_event',
 1959             'ix_revocation_event_user_id_issued_before'))
 1960         self.assertFalse(self.does_index_exist(
 1961             'revocation_event',
 1962             'ix_revocation_event_audit_id_issued_before'))
 1963         self.expand(10)
 1964         self.migrate(10)
 1965         self.contract(10)
 1966         self.assertTrue(self.does_index_exist(
 1967             'revocation_event',
 1968             'ix_revocation_event_issued_before'))
 1969         self.assertTrue(self.does_index_exist(
 1970             'revocation_event',
 1971             'ix_revocation_event_project_id_issued_before'))
 1972         self.assertTrue(self.does_index_exist(
 1973             'revocation_event',
 1974             'ix_revocation_event_user_id_issued_before'))
 1975         self.assertTrue(self.does_index_exist(
 1976             'revocation_event',
 1977             'ix_revocation_event_audit_id_issued_before'))
 1978 
 1979     def test_migration_011_user_id_unique_for_nonlocal_user(self):
 1980         table_name = 'nonlocal_user'
 1981         column = 'user_id'
 1982         self.expand(10)
 1983         self.migrate(10)
 1984         self.contract(10)
 1985         self.assertFalse(self.does_unique_constraint_exist(table_name, column))
 1986         self.expand(11)
 1987         self.migrate(11)
 1988         self.contract(11)
 1989         self.assertTrue(self.does_unique_constraint_exist(table_name, column))
 1990 
 1991     def test_migration_012_add_domain_id_to_idp(self):
 1992         def _create_domain():
 1993             domain_id = uuid.uuid4().hex
 1994             domain = {
 1995                 'id': domain_id,
 1996                 'name': domain_id,
 1997                 'enabled': True,
 1998                 'description': uuid.uuid4().hex,
 1999                 'domain_id': resource_base.NULL_DOMAIN_ID,
 2000                 'is_domain': True,
 2001                 'parent_id': None,
 2002                 'extra': '{}'
 2003             }
 2004             self.insert_dict(session, 'project', domain)
 2005             return domain_id
 2006 
 2007         def _get_new_idp(domain_id):
 2008             new_idp = {'id': uuid.uuid4().hex,
 2009                        'domain_id': domain_id,
 2010                        'enabled': True,
 2011                        'description': uuid.uuid4().hex}
 2012             return new_idp
 2013 
 2014         session = self.sessionmaker()
 2015         idp_name = 'identity_provider'
 2016         self.expand(11)
 2017         self.migrate(11)
 2018         self.contract(11)
 2019         self.assertTableColumns(idp_name,
 2020                                 ['id',
 2021                                  'enabled',
 2022                                  'description'])
 2023         # add some data
 2024         for i in range(5):
 2025             idp = {'id': uuid.uuid4().hex,
 2026                    'enabled': True,
 2027                    'description': uuid.uuid4().hex}
 2028             self.insert_dict(session, idp_name, idp)
 2029 
 2030         # upgrade
 2031         self.expand(12)
 2032         self.assertTableColumns(idp_name,
 2033                                 ['id',
 2034                                  'domain_id',
 2035                                  'enabled',
 2036                                  'description'])
 2037 
 2038         # confirm we cannot insert an idp during expand
 2039         domain_id = _create_domain()
 2040         new_idp = _get_new_idp(domain_id)
 2041         self.assertRaises(db_exception.DBError, self.insert_dict, session,
 2042                           idp_name, new_idp)
 2043 
 2044         # confirm we cannot insert an idp during migrate
 2045         self.migrate(12)
 2046         self.assertRaises(db_exception.DBError, self.insert_dict, session,
 2047                           idp_name, new_idp)
 2048 
 2049         # confirm we can insert a new idp after contract
 2050         self.contract(12)
 2051         self.insert_dict(session, idp_name, new_idp)
 2052 
 2053         # confirm domain_id column is not null
 2054         idp_table = sqlalchemy.Table(idp_name, self.metadata, autoload=True)
 2055         self.assertFalse(idp_table.c.domain_id.nullable)
 2056 
 2057     def test_migration_013_protocol_cascade_delete_for_federated_user(self):
 2058         if self.engine.name == 'sqlite':
 2059             self.skipTest('sqlite backend does not support foreign keys')
 2060 
 2061         self.expand(12)
 2062         self.migrate(12)
 2063         self.contract(12)
 2064 
 2065         # This test requires a bit of setup to properly work, first we create
 2066         # an identity provider, mapping and a protocol. Then, we create a
 2067         # federated user and delete the protocol. We expect the federated user
 2068         # to be deleted as well.
 2069 
 2070         session = self.sessionmaker()
 2071 
 2072         def _create_protocol():
 2073             domain = {
 2074                 'id': uuid.uuid4().hex,
 2075                 'name': uuid.uuid4().hex,
 2076                 'domain_id': resource_base.NULL_DOMAIN_ID,
 2077                 'is_domain': True,
 2078                 'parent_id': None
 2079             }
 2080             self.insert_dict(session, 'project', domain)
 2081 
 2082             idp = {'id': uuid.uuid4().hex, 'enabled': True,
 2083                    'domain_id': domain['id']}
 2084             self.insert_dict(session, 'identity_provider', idp)
 2085 
 2086             mapping = {'id': uuid.uuid4().hex, 'rules': json.dumps([])}
 2087             self.insert_dict(session, 'mapping', mapping)
 2088 
 2089             protocol = {'id': uuid.uuid4().hex, 'idp_id': idp['id'],
 2090                         'mapping_id': mapping['id']}
 2091             protocol_table = sqlalchemy.Table(
 2092                 'federation_protocol', self.metadata, autoload=True)
 2093             self.insert_dict(session, 'federation_protocol', protocol,
 2094                              table=protocol_table)
 2095 
 2096             return protocol, protocol_table
 2097 
 2098         def _create_federated_user(idp_id, protocol_id):
 2099             user = {'id': uuid.uuid4().hex}
 2100             self.insert_dict(session, 'user', user)
 2101 
 2102             # NOTE(rodrigods): do not set the ID, the engine will do that
 2103             # for us and we won't need it later.
 2104             federated_user = {
 2105                 'user_id': user['id'], 'idp_id': idp_id,
 2106                 'protocol_id': protocol_id, 'unique_id': uuid.uuid4().hex}
 2107             federated_table = sqlalchemy.Table(
 2108                 'federated_user', self.metadata, autoload=True)
 2109             self.insert_dict(session, 'federated_user', federated_user,
 2110                              table=federated_table)
 2111 
 2112             return federated_user, federated_table
 2113 
 2114         protocol, protocol_table = _create_protocol()
 2115         federated_user, federated_table = _create_federated_user(
 2116             protocol['idp_id'], protocol['id'])
 2117 
 2118         # before updating the foreign key, we won't be able to delete the
 2119         # protocol
 2120         self.assertRaises(db_exception.DBError,
 2121                           session.execute,
 2122                           protocol_table.delete().where(
 2123                               protocol_table.c.id == protocol['id']))
 2124 
 2125         self.expand(13)
 2126         self.migrate(13)
 2127         self.contract(13)
 2128 
 2129         # now we are able to delete the protocol
 2130         session.execute(
 2131             protocol_table.delete().where(
 2132                 protocol_table.c.id == protocol['id']))
 2133 
 2134         # assert the cascade deletion worked
 2135         federated_users = session.query(federated_table).filter_by(
 2136             protocol_id=federated_user['protocol_id']).all()
 2137         self.assertThat(federated_users, matchers.HasLength(0))
 2138 
 2139     def test_migration_014_add_domain_id_to_user_table(self):
 2140         def create_domain():
 2141             table = sqlalchemy.Table('project', self.metadata, autoload=True)
 2142             domain_id = uuid.uuid4().hex
 2143             domain = {
 2144                 'id': domain_id,
 2145                 'name': domain_id,
 2146                 'enabled': True,
 2147                 'description': uuid.uuid4().hex,
 2148                 'domain_id': resource_base.NULL_DOMAIN_ID,
 2149                 'is_domain': True,
 2150                 'parent_id': None,
 2151                 'extra': '{}'
 2152             }
 2153             table.insert().values(domain).execute()
 2154             return domain_id
 2155 
 2156         def create_user(table):
 2157             user_id = uuid.uuid4().hex
 2158             user = {'id': user_id, 'enabled': True}
 2159             table.insert().values(user).execute()
 2160             return user_id
 2161 
 2162         # insert local_user or nonlocal_user
 2163         def create_child_user(table, user_id, domain_id):
 2164             child_user = {
 2165                 'user_id': user_id,
 2166                 'domain_id': domain_id,
 2167                 'name': uuid.uuid4().hex
 2168             }
 2169             table.insert().values(child_user).execute()
 2170 
 2171         # update local_user or nonlocal_user
 2172         def update_child_user(table, user_id, new_domain_id):
 2173             table.update().where(table.c.user_id == user_id).values(
 2174                 domain_id=new_domain_id).execute()
 2175 
 2176         def assertUserDomain(user_id, domain_id):
 2177             user = sqlalchemy.Table('user', self.metadata, autoload=True)
 2178             cols = [user.c.domain_id]
 2179             filter = user.c.id == user_id
 2180             sel = sqlalchemy.select(cols).where(filter)
 2181             domains = sel.execute().fetchone()
 2182             self.assertEqual(domain_id, domains[0])
 2183 
 2184         user_table_name = 'user'
 2185         self.expand(13)
 2186         self.migrate(13)
 2187         self.contract(13)
 2188         self.assertTableColumns(
 2189             user_table_name, ['id', 'extra', 'enabled', 'default_project_id',
 2190                               'created_at', 'last_active_at'])
 2191         self.expand(14)
 2192         self.assertTableColumns(
 2193             user_table_name, ['id', 'extra', 'enabled', 'default_project_id',
 2194                               'created_at', 'last_active_at', 'domain_id'])
 2195         user_table = sqlalchemy.Table(user_table_name, self.metadata,
 2196                                       autoload=True)
 2197         local_user_table = sqlalchemy.Table('local_user', self.metadata,
 2198                                             autoload=True)
 2199         nonlocal_user_table = sqlalchemy.Table('nonlocal_user', self.metadata,
 2200                                                autoload=True)
 2201 
 2202         # add users before migrate to test that the user.domain_id gets updated
 2203         # after migrate
 2204         user_ids = []
 2205         expected_domain_id = create_domain()
 2206         user_id = create_user(user_table)
 2207         create_child_user(local_user_table, user_id, expected_domain_id)
 2208         user_ids.append(user_id)
 2209         user_id = create_user(user_table)
 2210         create_child_user(nonlocal_user_table, user_id, expected_domain_id)
 2211         user_ids.append(user_id)
 2212 
 2213         self.migrate(14)
 2214         # test local_user insert trigger updates user.domain_id
 2215         user_id = create_user(user_table)
 2216         domain_id = create_domain()
 2217         create_child_user(local_user_table, user_id, domain_id)
 2218         assertUserDomain(user_id, domain_id)
 2219 
 2220         # test local_user update trigger updates user.domain_id
 2221         new_domain_id = create_domain()
 2222         update_child_user(local_user_table, user_id, new_domain_id)
 2223         assertUserDomain(user_id, new_domain_id)
 2224 
 2225         # test nonlocal_user insert trigger updates user.domain_id
 2226         user_id = create_user(user_table)
 2227         create_child_user(nonlocal_user_table, user_id, domain_id)
 2228         assertUserDomain(user_id, domain_id)
 2229 
 2230         # test nonlocal_user update trigger updates user.domain_id
 2231         update_child_user(nonlocal_user_table, user_id, new_domain_id)
 2232         assertUserDomain(user_id, new_domain_id)
 2233 
 2234         self.contract(14)
 2235         # test migrate updated the user.domain_id
 2236         for user_id in user_ids:
 2237             assertUserDomain(user_id, expected_domain_id)
 2238 
 2239         # test unique and fk constraints
 2240         if self.engine.name == 'mysql':
 2241             self.assertTrue(
 2242                 self.does_index_exist('user', 'ixu_user_id_domain_id'))
 2243         else:
 2244             self.assertTrue(
 2245                 self.does_constraint_exist('user', 'ixu_user_id_domain_id'))
 2246         self.assertTrue(self.does_fk_exist('local_user', 'user_id'))
 2247         self.assertTrue(self.does_fk_exist('local_user', 'domain_id'))
 2248         self.assertTrue(self.does_fk_exist('nonlocal_user', 'user_id'))
 2249         self.assertTrue(self.does_fk_exist('nonlocal_user', 'domain_id'))
 2250 
 2251     def test_migration_015_update_federated_user_domain(self):
 2252         def create_domain():
 2253             table = sqlalchemy.Table('project', self.metadata, autoload=True)
 2254             domain_id = uuid.uuid4().hex
 2255             domain = {
 2256                 'id': domain_id,
 2257                 'name': domain_id,
 2258                 'enabled': True,
 2259                 'description': uuid.uuid4().hex,
 2260                 'domain_id': resource_base.NULL_DOMAIN_ID,
 2261                 'is_domain': True,
 2262                 'parent_id': None,
 2263                 'extra': '{}'
 2264             }
 2265             table.insert().values(domain).execute()
 2266             return domain_id
 2267 
 2268         def create_idp(domain_id):
 2269             table = sqlalchemy.Table('identity_provider', self.metadata,
 2270                                      autoload=True)
 2271             idp_id = uuid.uuid4().hex
 2272             idp = {
 2273                 'id': idp_id,
 2274                 'domain_id': domain_id,
 2275                 'enabled': True,
 2276                 'description': uuid.uuid4().hex
 2277             }
 2278             table.insert().values(idp).execute()
 2279             return idp_id
 2280 
 2281         def create_protocol(idp_id):
 2282             table = sqlalchemy.Table('federation_protocol', self.metadata,
 2283                                      autoload=True)
 2284             protocol_id = uuid.uuid4().hex
 2285             protocol = {
 2286                 'id': protocol_id,
 2287                 'idp_id': idp_id,
 2288                 'mapping_id': uuid.uuid4().hex
 2289             }
 2290             table.insert().values(protocol).execute()
 2291             return protocol_id
 2292 
 2293         def create_user():
 2294             table = sqlalchemy.Table('user', self.metadata, autoload=True)
 2295             user_id = uuid.uuid4().hex
 2296             user = {'id': user_id, 'enabled': True}
 2297             table.insert().values(user).execute()
 2298             return user_id
 2299 
 2300         def create_federated_user(user_id, idp_id, protocol_id):
 2301             table = sqlalchemy.Table('federated_user', self.metadata,
 2302                                      autoload=True)
 2303             federated_user = {
 2304                 'user_id': user_id,
 2305                 'idp_id': idp_id,
 2306                 'protocol_id': protocol_id,
 2307                 'unique_id': uuid.uuid4().hex,
 2308                 'display_name': uuid.uuid4().hex
 2309             }
 2310             table.insert().values(federated_user).execute()
 2311 
 2312         def assertUserDomain(user_id, domain_id):
 2313             table = sqlalchemy.Table('user', self.metadata, autoload=True)
 2314             where = table.c.id == user_id
 2315             stmt = sqlalchemy.select([table.c.domain_id]).where(where)
 2316             domains = stmt.execute().fetchone()
 2317             self.assertEqual(domain_id, domains[0])
 2318 
 2319         def assertUserDomainIsNone(user_id):
 2320             table = sqlalchemy.Table('user', self.metadata, autoload=True)
 2321             where = table.c.id == user_id
 2322             stmt = sqlalchemy.select([table.c.domain_id]).where(where)
 2323             domains = stmt.execute().fetchone()
 2324             self.assertIsNone(domains[0])
 2325 
 2326         self.expand(14)
 2327         self.migrate(14)
 2328         self.contract(14)
 2329 
 2330         domain_id = create_domain()
 2331         idp_id = create_idp(domain_id)
 2332         protocol_id = create_protocol(idp_id)
 2333 
 2334         # create user before expand to test data migration
 2335         user_id_before_expand = create_user()
 2336         create_federated_user(user_id_before_expand, idp_id, protocol_id)
 2337         assertUserDomainIsNone(user_id_before_expand)
 2338 
 2339         self.expand(15)
 2340         # create user before migrate to test insert trigger
 2341         user_id_before_migrate = create_user()
 2342         create_federated_user(user_id_before_migrate, idp_id, protocol_id)
 2343         assertUserDomain(user_id_before_migrate, domain_id)
 2344 
 2345         self.migrate(15)
 2346         # test insert trigger after migrate
 2347         user_id = create_user()
 2348         create_federated_user(user_id, idp_id, protocol_id)
 2349         assertUserDomain(user_id, domain_id)
 2350 
 2351         self.contract(15)
 2352         # test migrate updated the user.domain_id
 2353         assertUserDomain(user_id_before_expand, domain_id)
 2354 
 2355         # verify that the user.domain_id is now not nullable
 2356         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 2357         self.assertFalse(user_table.c.domain_id.nullable)
 2358 
 2359     def test_migration_016_add_user_options(self):
 2360         self.expand(15)
 2361         self.migrate(15)
 2362         self.contract(15)
 2363 
 2364         user_option = 'user_option'
 2365         self.assertTableDoesNotExist(user_option)
 2366         self.expand(16)
 2367         self.migrate(16)
 2368         self.contract(16)
 2369         self.assertTableColumns(user_option,
 2370                                 ['user_id', 'option_id', 'option_value'])
 2371 
 2372     def test_migration_024_add_created_expires_at_int_columns_password(self):
 2373 
 2374         self.expand(23)
 2375         self.migrate(23)
 2376         self.contract(23)
 2377 
 2378         password_table_name = 'password'
 2379 
 2380         self.assertTableColumns(
 2381             password_table_name,
 2382             ['id', 'local_user_id', 'password', 'password_hash', 'created_at',
 2383              'expires_at', 'self_service']
 2384         )
 2385 
 2386         self.expand(24)
 2387 
 2388         self.assertTableColumns(
 2389             password_table_name,
 2390             ['id', 'local_user_id', 'password', 'password_hash', 'created_at',
 2391              'expires_at', 'created_at_int', 'expires_at_int', 'self_service']
 2392         )
 2393 
 2394         # Create User and Local User
 2395         project_table = sqlalchemy.Table('project', self.metadata,
 2396                                          autoload=True)
 2397         domain_data = {'id': '_domain', 'domain_id': '_domain',
 2398                        'enabled': True, 'name': '_domain', 'is_domain': True}
 2399         project_table.insert().values(domain_data).execute()
 2400         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 2401         user_id = uuid.uuid4().hex
 2402         user = {'id': user_id, 'enabled': True, 'domain_id': domain_data['id']}
 2403         user_table.insert().values(user).execute()
 2404         local_user_table = sqlalchemy.Table('local_user', self.metadata,
 2405                                             autoload=True)
 2406         local_user = {
 2407             'id': 1, 'user_id': user_id, 'domain_id': user['domain_id'],
 2408             'name': 'name'}
 2409 
 2410         local_user_table.insert().values(local_user).execute()
 2411 
 2412         password_table = sqlalchemy.Table('password',
 2413                                           self.metadata, autoload=True)
 2414         password_data = {
 2415             'local_user_id': local_user['id'],
 2416             'created_at': datetime.datetime.utcnow(),
 2417             'expires_at': datetime.datetime.utcnow()}
 2418         password_table.insert().values(password_data).execute()
 2419 
 2420         self.migrate(24)
 2421         self.contract(24)
 2422         passwords = list(password_table.select().execute())
 2423 
 2424         epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
 2425 
 2426         for p in passwords:
 2427             c = (p.created_at.replace(tzinfo=pytz.UTC) - epoch).total_seconds()
 2428             e = (p.expires_at.replace(tzinfo=pytz.UTC) - epoch).total_seconds()
 2429             self.assertEqual(p.created_at_int, int(c * 1000000))
 2430             self.assertEqual(p.expires_at_int, int(e * 1000000))
 2431 
 2432         # Test contract phase and ensure data can not be null
 2433         self.contract(24)
 2434         meta = sqlalchemy.MetaData(self.engine)
 2435         pw_table = sqlalchemy.Table('password', meta, autoload=True)
 2436         self.assertFalse(pw_table.c.created_at_int.nullable)
 2437 
 2438     def test_migration_30_expand_add_project_tags_table(self):
 2439         self.expand(29)
 2440         self.migrate(29)
 2441         self.contract(29)
 2442 
 2443         table_name = 'project_tag'
 2444         self.assertTableDoesNotExist(table_name)
 2445 
 2446         self.expand(30)
 2447         self.migrate(30)
 2448         self.contract(30)
 2449 
 2450         self.assertTableExists(table_name)
 2451         self.assertTableColumns(
 2452             table_name,
 2453             ['project_id', 'name'])
 2454 
 2455     def test_migration_030_project_tags_works_correctly_after_migration(self):
 2456         if self.engine.name == 'sqlite':
 2457             self.skipTest('sqlite backend does not support foreign keys')
 2458 
 2459         self.expand(30)
 2460         self.migrate(30)
 2461         self.contract(30)
 2462 
 2463         project_table = sqlalchemy.Table(
 2464             'project', self.metadata, autoload=True)
 2465         tag_table = sqlalchemy.Table(
 2466             'project_tag', self.metadata, autoload=True)
 2467 
 2468         session = self.sessionmaker()
 2469         project_id = uuid.uuid4().hex
 2470 
 2471         project = {
 2472             'id': project_id,
 2473             'name': uuid.uuid4().hex,
 2474             'enabled': True,
 2475             'domain_id': resource_base.NULL_DOMAIN_ID,
 2476             'is_domain': False
 2477         }
 2478 
 2479         tag = {
 2480             'project_id': project_id,
 2481             'name': uuid.uuid4().hex
 2482         }
 2483 
 2484         self.insert_dict(session, 'project', project)
 2485         self.insert_dict(session, 'project_tag', tag)
 2486 
 2487         tags_query = session.query(tag_table).filter_by(
 2488             project_id=project_id).all()
 2489         self.assertThat(tags_query, matchers.HasLength(1))
 2490 
 2491         # Adding duplicate tags should cause error.
 2492         self.assertRaises(db_exception.DBDuplicateEntry,
 2493                           self.insert_dict,
 2494                           session, 'project_tag', tag)
 2495 
 2496         session.execute(
 2497             project_table.delete().where(project_table.c.id == project_id)
 2498         )
 2499 
 2500         tags_query = session.query(tag_table).filter_by(
 2501             project_id=project_id).all()
 2502         self.assertThat(tags_query, matchers.HasLength(0))
 2503 
 2504         session.close()
 2505 
 2506     def test_migration_031_adds_system_assignment_table(self):
 2507         self.expand(30)
 2508         self.migrate(30)
 2509         self.contract(30)
 2510 
 2511         system_assignment_table_name = 'system_assignment'
 2512         self.assertTableDoesNotExist(system_assignment_table_name)
 2513 
 2514         self.expand(31)
 2515         self.migrate(31)
 2516         self.contract(31)
 2517 
 2518         self.assertTableExists(system_assignment_table_name)
 2519         self.assertTableColumns(
 2520             system_assignment_table_name,
 2521             ['type', 'actor_id', 'target_id', 'role_id', 'inherited']
 2522         )
 2523 
 2524         system_assignment_table = sqlalchemy.Table(
 2525             system_assignment_table_name, self.metadata, autoload=True
 2526         )
 2527 
 2528         system_user = {
 2529             'type': 'UserSystem',
 2530             'target_id': uuid.uuid4().hex,
 2531             'actor_id': uuid.uuid4().hex,
 2532             'role_id': uuid.uuid4().hex,
 2533             'inherited': False
 2534         }
 2535         system_assignment_table.insert().values(system_user).execute()
 2536 
 2537         system_group = {
 2538             'type': 'GroupSystem',
 2539             'target_id': uuid.uuid4().hex,
 2540             'actor_id': uuid.uuid4().hex,
 2541             'role_id': uuid.uuid4().hex,
 2542             'inherited': False
 2543         }
 2544         system_assignment_table.insert().values(system_group).execute()
 2545 
 2546     def test_migration_032_add_expires_at_int_column_trust(self):
 2547 
 2548         self.expand(31)
 2549         self.migrate(31)
 2550         self.contract(31)
 2551 
 2552         trust_table_name = 'trust'
 2553 
 2554         self.assertTableColumns(
 2555             trust_table_name,
 2556             ['id', 'trustor_user_id', 'trustee_user_id', 'project_id',
 2557              'impersonation', 'deleted_at', 'expires_at', 'remaining_uses',
 2558              'extra'],
 2559         )
 2560 
 2561         self.expand(32)
 2562 
 2563         self.assertTableColumns(
 2564             trust_table_name,
 2565             ['id', 'trustor_user_id', 'trustee_user_id', 'project_id',
 2566              'impersonation', 'deleted_at', 'expires_at', 'expires_at_int',
 2567              'remaining_uses', 'extra'],
 2568         )
 2569 
 2570         # Create Trust
 2571         trust_table = sqlalchemy.Table('trust', self.metadata,
 2572                                        autoload=True)
 2573         trust_1_data = {
 2574             'id': uuid.uuid4().hex,
 2575             'trustor_user_id': uuid.uuid4().hex,
 2576             'trustee_user_id': uuid.uuid4().hex,
 2577             'project_id': uuid.uuid4().hex,
 2578             'impersonation': False,
 2579             'expires_at': datetime.datetime.utcnow()
 2580         }
 2581         trust_2_data = {
 2582             'id': uuid.uuid4().hex,
 2583             'trustor_user_id': uuid.uuid4().hex,
 2584             'trustee_user_id': uuid.uuid4().hex,
 2585             'project_id': uuid.uuid4().hex,
 2586             'impersonation': False,
 2587             'expires_at': None
 2588         }
 2589         trust_table.insert().values(trust_1_data).execute()
 2590         trust_table.insert().values(trust_2_data).execute()
 2591 
 2592         self.migrate(32)
 2593         self.contract(32)
 2594         trusts = list(trust_table.select().execute())
 2595 
 2596         epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
 2597 
 2598         for t in trusts:
 2599             if t.expires_at:
 2600                 e = t.expires_at.replace(tzinfo=pytz.UTC) - epoch
 2601                 e = e.total_seconds()
 2602                 self.assertEqual(t.expires_at_int, int(e * 1000000))
 2603 
 2604     def test_migration_033_adds_limits_table(self):
 2605         self.expand(32)
 2606         self.migrate(32)
 2607         self.contract(32)
 2608 
 2609         registered_limit_table_name = 'registered_limit'
 2610         limit_table_name = 'limit'
 2611         self.assertTableDoesNotExist(registered_limit_table_name)
 2612         self.assertTableDoesNotExist(limit_table_name)
 2613 
 2614         self.expand(33)
 2615         self.migrate(33)
 2616         self.contract(33)
 2617 
 2618         self.assertTableExists(registered_limit_table_name)
 2619         self.assertTableColumns(
 2620             registered_limit_table_name,
 2621             ['id', 'service_id', 'resource_name', 'region_id', 'default_limit']
 2622         )
 2623         self.assertTableExists(limit_table_name)
 2624         self.assertTableColumns(
 2625             limit_table_name,
 2626             ['id', 'project_id', 'service_id', 'resource_name', 'region_id',
 2627              'resource_limit']
 2628         )
 2629 
 2630         session = self.sessionmaker()
 2631         service_id = uuid.uuid4().hex
 2632         service = {
 2633             'id': service_id,
 2634             'type': 'compute',
 2635             'enabled': True
 2636         }
 2637         region = {
 2638             'id': 'RegionOne',
 2639             'description': 'test'
 2640         }
 2641         project_id = uuid.uuid4().hex
 2642         project = {
 2643             'id': project_id,
 2644             'name': 'nova',
 2645             'enabled': True,
 2646             'domain_id': resource_base.NULL_DOMAIN_ID,
 2647             'is_domain': False
 2648         }
 2649         self.insert_dict(session, 'service', service)
 2650         self.insert_dict(session, 'region', region)
 2651         self.insert_dict(session, 'project', project)
 2652 
 2653         # Insert one registered limit
 2654         registered_limit_table = sqlalchemy.Table(
 2655             registered_limit_table_name, self.metadata, autoload=True)
 2656         registered_limit = {
 2657             'id': uuid.uuid4().hex,
 2658             'service_id': service_id,
 2659             'region_id': 'RegionOne',
 2660             'resource_name': 'cores',
 2661             'default_limit': 10
 2662         }
 2663         registered_limit_table.insert().values(registered_limit).execute()
 2664 
 2665         # It will raise error if insert another one with same service_id,
 2666         # region_id and resource name.
 2667         registered_limit['id'] = uuid.uuid4().hex
 2668         registered_limit['default_limit'] = 20
 2669         self.assertRaises(db_exception.DBDuplicateEntry,
 2670                           registered_limit_table.insert().values(
 2671                               registered_limit).execute)
 2672 
 2673         # Insert one without region_id
 2674         registered_limit_without_region = {
 2675             'id': uuid.uuid4().hex,
 2676             'service_id': service_id,
 2677             'resource_name': 'cores',
 2678             'default_limit': 10
 2679         }
 2680         registered_limit_table.insert().values(
 2681             registered_limit_without_region).execute()
 2682 
 2683         # It will not raise error if insert another one with same service_id
 2684         # and resource_name but the region_id is None. Because that
 2685         # UniqueConstraint doesn't work if one of the columns is None. This
 2686         # should be controlled at the Manager layer to forbid this behavior.
 2687         registered_limit_without_region['id'] = uuid.uuid4().hex
 2688         registered_limit_table.insert().values(
 2689             registered_limit_without_region).execute()
 2690 
 2691         # Insert one limit
 2692         limit_table = sqlalchemy.Table(
 2693             limit_table_name, self.metadata, autoload=True)
 2694         limit = {
 2695             'id': uuid.uuid4().hex,
 2696             'project_id': project_id,
 2697             'service_id': service_id,
 2698             'region_id': 'RegionOne',
 2699             'resource_name': 'cores',
 2700             'resource_limit': 5
 2701         }
 2702         limit_table.insert().values(limit).execute()
 2703 
 2704         # Insert another one with the same project_id, service_id, region_id
 2705         # and resource_name, then raise error.
 2706         limit['id'] = uuid.uuid4().hex
 2707         limit['resource_limit'] = 10
 2708         self.assertRaises(db_exception.DBDuplicateEntry,
 2709                           limit_table.insert().values(limit).execute)
 2710 
 2711         # Insert one without region_id
 2712         limit_without_region = {
 2713             'id': uuid.uuid4().hex,
 2714             'project_id': project_id,
 2715             'service_id': service_id,
 2716             'resource_name': 'cores',
 2717             'resource_limit': 5
 2718         }
 2719         limit_table.insert().values(limit_without_region).execute()
 2720 
 2721     def test_migration_034_adds_application_credential_table(self):
 2722         self.expand(33)
 2723         self.migrate(33)
 2724         self.contract(33)
 2725 
 2726         application_credential_table_name = 'application_credential'
 2727         self.assertTableDoesNotExist(application_credential_table_name)
 2728         application_credential_role_table_name = 'application_credential_role'
 2729         self.assertTableDoesNotExist(application_credential_role_table_name)
 2730 
 2731         self.expand(34)
 2732         self.migrate(34)
 2733         self.contract(34)
 2734 
 2735         self.assertTableExists(application_credential_table_name)
 2736         self.assertTableColumns(
 2737             application_credential_table_name,
 2738             ['internal_id', 'id', 'name', 'secret_hash',
 2739              'description', 'user_id', 'project_id', 'expires_at',
 2740              'allow_application_credential_creation']
 2741         )
 2742         if self.engine.name == 'mysql':
 2743             self.assertTrue(self.does_index_exist(
 2744                 'application_credential', 'duplicate_app_cred_constraint'))
 2745         else:
 2746             self.assertTrue(self.does_constraint_exist(
 2747                 'application_credential', 'duplicate_app_cred_constraint'))
 2748         self.assertTableExists(application_credential_role_table_name)
 2749         self.assertTableColumns(
 2750             application_credential_role_table_name,
 2751             ['application_credential_id', 'role_id']
 2752         )
 2753 
 2754         app_cred_table = sqlalchemy.Table(
 2755             application_credential_table_name, self.metadata, autoload=True
 2756         )
 2757         app_cred_role_table = sqlalchemy.Table(
 2758             application_credential_role_table_name,
 2759             self.metadata, autoload=True
 2760         )
 2761         self.assertTrue(self.does_fk_exist('application_credential_role',
 2762                                            'application_credential_id'))
 2763 
 2764         expires_at = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
 2765         epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
 2766         expires_at_int = (expires_at - epoch).total_seconds()
 2767         app_cred = {
 2768             'internal_id': 1,
 2769             'id': uuid.uuid4().hex,
 2770             'name': uuid.uuid4().hex,
 2771             'secret_hash': uuid.uuid4().hex,
 2772             'description': uuid.uuid4().hex,
 2773             'user_id': uuid.uuid4().hex,
 2774             'project_id': uuid.uuid4().hex,
 2775             'expires_at': expires_at_int,
 2776             'allow_application_credential_creation': False
 2777         }
 2778         app_cred_table.insert().values(app_cred).execute()
 2779 
 2780         # Exercise unique constraint
 2781         dup_app_cred = {
 2782             'internal_id': 2,
 2783             'id': uuid.uuid4().hex,
 2784             'name': app_cred['name'],
 2785             'secret_hash': uuid.uuid4().hex,
 2786             'user_id': app_cred['user_id'],
 2787             'project_id': uuid.uuid4().hex
 2788         }
 2789         insert = app_cred_table.insert().values(dup_app_cred)
 2790         self.assertRaises(db_exception.DBDuplicateEntry,
 2791                           insert.execute)
 2792 
 2793         role_rel = {
 2794             'application_credential_id': app_cred['internal_id'],
 2795             'role_id': uuid.uuid4().hex
 2796         }
 2797         app_cred_role_table.insert().values(role_rel).execute()
 2798 
 2799         # Exercise role table primary keys
 2800         insert = app_cred_role_table.insert().values(role_rel)
 2801         self.assertRaises(db_exception.DBDuplicateEntry, insert.execute)
 2802 
 2803     def test_migration_035_add_system_column_to_credential_table(self):
 2804         self.expand(34)
 2805         self.migrate(34)
 2806         self.contract(34)
 2807 
 2808         application_credential_table_name = 'application_credential'
 2809         self.assertTableExists(application_credential_table_name)
 2810         self.assertTableColumns(
 2811             application_credential_table_name,
 2812             ['internal_id', 'id', 'name', 'secret_hash',
 2813              'description', 'user_id', 'project_id', 'expires_at',
 2814              'allow_application_credential_creation']
 2815         )
 2816 
 2817         self.expand(35)
 2818         self.migrate(35)
 2819         self.contract(35)
 2820 
 2821         self.assertTableColumns(
 2822             application_credential_table_name,
 2823             ['internal_id', 'id', 'name', 'secret_hash',
 2824              'description', 'user_id', 'project_id', 'system', 'expires_at',
 2825              'allow_application_credential_creation']
 2826         )
 2827 
 2828         application_credential_table = sqlalchemy.Table(
 2829             application_credential_table_name, self.metadata, autoload=True
 2830         )
 2831 
 2832         # Test that we can insert an application credential without project_id
 2833         # defined.
 2834         expires_at = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
 2835         epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)
 2836         expires_at_int = (expires_at - epoch).total_seconds()
 2837         app_cred = {
 2838             'internal_id': 1,
 2839             'id': uuid.uuid4().hex,
 2840             'name': uuid.uuid4().hex,
 2841             'secret_hash': uuid.uuid4().hex,
 2842             'description': uuid.uuid4().hex,
 2843             'user_id': uuid.uuid4().hex,
 2844             'system': uuid.uuid4().hex,
 2845             'expires_at': expires_at_int,
 2846             'allow_application_credential_creation': False
 2847         }
 2848         application_credential_table.insert().values(app_cred).execute()
 2849 
 2850         # Test that we can insert an application credential with a project_id
 2851         # and without system defined.
 2852         app_cred = {
 2853             'internal_id': 2,
 2854             'id': uuid.uuid4().hex,
 2855             'name': uuid.uuid4().hex,
 2856             'secret_hash': uuid.uuid4().hex,
 2857             'description': uuid.uuid4().hex,
 2858             'user_id': uuid.uuid4().hex,
 2859             'project_id': uuid.uuid4().hex,
 2860             'expires_at': expires_at_int,
 2861             'allow_application_credential_creation': False
 2862         }
 2863         application_credential_table.insert().values(app_cred).execute()
 2864 
 2865         # Test that we can create an application credential without a project
 2866         # or a system defined. Technically, project_id and system should be
 2867         # mutually exclusive, which will be handled by the application and not
 2868         # the data layer.
 2869         app_cred = {
 2870             'internal_id': 3,
 2871             'id': uuid.uuid4().hex,
 2872             'name': uuid.uuid4().hex,
 2873             'secret_hash': uuid.uuid4().hex,
 2874             'description': uuid.uuid4().hex,
 2875             'user_id': uuid.uuid4().hex,
 2876             'expires_at': expires_at_int,
 2877             'allow_application_credential_creation': False
 2878         }
 2879         application_credential_table.insert().values(app_cred).execute()
 2880 
 2881     def test_migration_036_rename_application_credentials_column(self):
 2882         self.expand(35)
 2883         self.migrate(35)
 2884         self.contract(35)
 2885 
 2886         application_credential_table_name = 'application_credential'
 2887         application_credential_role_table_name = 'application_credential_role'
 2888 
 2889         self.expand(36)
 2890         self.migrate(36)
 2891         self.contract(36)
 2892 
 2893         self.assertTableColumns(
 2894             application_credential_table_name,
 2895             ['internal_id', 'id', 'name', 'secret_hash',
 2896              'description', 'user_id', 'project_id', 'system', 'expires_at',
 2897              'unrestricted']
 2898         )
 2899 
 2900         application_credential_table = sqlalchemy.Table(
 2901             application_credential_table_name, self.metadata, autoload=True
 2902         )
 2903         app_cred_role_table = sqlalchemy.Table(
 2904             application_credential_role_table_name,
 2905             self.metadata, autoload=True
 2906         )
 2907 
 2908         # Test that the new column works
 2909         app_cred = {
 2910             'internal_id': 1,
 2911             'id': uuid.uuid4().hex,
 2912             'name': uuid.uuid4().hex,
 2913             'secret_hash': uuid.uuid4().hex,
 2914             'description': uuid.uuid4().hex,
 2915             'user_id': uuid.uuid4().hex,
 2916             'system': uuid.uuid4().hex,
 2917             'expires_at': None,
 2918             'unrestricted': False
 2919         }
 2920         application_credential_table.insert().values(app_cred).execute()
 2921         role_rel = {
 2922             'application_credential_id': app_cred['internal_id'],
 2923             'role_id': uuid.uuid4().hex
 2924         }
 2925         app_cred_role_table.insert().values(role_rel).execute()
 2926 
 2927     def test_migration_037_remove_service_and_region_fk_for_registered_limit(
 2928             self):
 2929         self.expand(37)
 2930         self.migrate(37)
 2931         self.contract(37)
 2932 
 2933         registered_limit_table_name = 'registered_limit'
 2934         registered_limit_table = sqlalchemy.Table(registered_limit_table_name,
 2935                                                   self.metadata, autoload=True)
 2936         self.assertEqual(set([]), registered_limit_table.foreign_keys)
 2937 
 2938     def test_migration_045_add_description_to_limit(self):
 2939 
 2940         self.expand(44)
 2941         self.migrate(44)
 2942         self.contract(44)
 2943 
 2944         registered_limit_table_name = 'registered_limit'
 2945         limit_table_name = 'limit'
 2946 
 2947         self.assertTableExists(registered_limit_table_name)
 2948         self.assertTableExists(limit_table_name)
 2949         self.assertTableColumns(
 2950             registered_limit_table_name,
 2951             ['id', 'service_id', 'region_id', 'resource_name', 'default_limit']
 2952         )
 2953         self.assertTableColumns(
 2954             limit_table_name,
 2955             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 2956              'resource_limit']
 2957         )
 2958 
 2959         self.expand(45)
 2960         self.migrate(45)
 2961         self.contract(45)
 2962 
 2963         registered_limit_table = sqlalchemy.Table(registered_limit_table_name,
 2964                                                   self.metadata, autoload=True)
 2965         limit_table = sqlalchemy.Table(limit_table_name,
 2966                                        self.metadata, autoload=True)
 2967         self.assertTableColumns(
 2968             registered_limit_table_name,
 2969             ['id', 'service_id', 'region_id', 'resource_name', 'default_limit',
 2970              'description']
 2971         )
 2972         self.assertTableColumns(
 2973             limit_table_name,
 2974             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 2975              'resource_limit', 'description']
 2976         )
 2977 
 2978         session = self.sessionmaker()
 2979         service_id = uuid.uuid4().hex
 2980         service = {
 2981             'id': service_id,
 2982             'type': 'compute',
 2983             'enabled': True
 2984         }
 2985         region = {
 2986             'id': 'RegionOne',
 2987             'description': 'test'
 2988         }
 2989         project_id = uuid.uuid4().hex
 2990         project = {
 2991             'id': project_id,
 2992             'name': 'nova',
 2993             'enabled': True,
 2994             'domain_id': resource_base.NULL_DOMAIN_ID,
 2995             'is_domain': False
 2996         }
 2997         self.insert_dict(session, 'service', service)
 2998         self.insert_dict(session, 'region', region)
 2999         self.insert_dict(session, 'project', project)
 3000 
 3001         # with description
 3002         registered_limit = {
 3003             'id': uuid.uuid4().hex,
 3004             'service_id': service_id,
 3005             'region_id': 'RegionOne',
 3006             'resource_name': 'cores',
 3007             'default_limit': 10,
 3008             'description': 'this is a description'
 3009         }
 3010         registered_limit_table.insert().values(registered_limit).execute()
 3011 
 3012         # without description
 3013         limit = {
 3014             'id': uuid.uuid4().hex,
 3015             'project_id': project_id,
 3016             'service_id': service_id,
 3017             'region_id': 'RegionOne',
 3018             'resource_name': 'cores',
 3019             'resource_limit': 5
 3020         }
 3021         limit_table.insert().values(limit).execute()
 3022 
 3023     def test_migration_046_copies_data_from_password_to_password_hash(self):
 3024         self.expand(46)
 3025         self.migrate(45)
 3026         self.contract(45)
 3027         # Create User and Local User
 3028         project_table = sqlalchemy.Table('project', self.metadata,
 3029                                          autoload=True)
 3030         domain_data = {'id': '_domain', 'domain_id': '_domain',
 3031                        'enabled': True, 'name': '_domain', 'is_domain': True}
 3032         project_table.insert().values(domain_data).execute()
 3033         user_table = sqlalchemy.Table('user', self.metadata, autoload=True)
 3034         user_id = uuid.uuid4().hex
 3035         user = {'id': user_id, 'enabled': True, 'domain_id': domain_data['id']}
 3036         user_table.insert().values(user).execute()
 3037         local_user_table = sqlalchemy.Table('local_user', self.metadata,
 3038                                             autoload=True)
 3039         local_user = {
 3040             'id': 1, 'user_id': user_id, 'domain_id': user['domain_id'],
 3041             'name': 'name'}
 3042 
 3043         local_user_table.insert().values(local_user).execute()
 3044 
 3045         password_table = sqlalchemy.Table('password',
 3046                                           self.metadata, autoload=True)
 3047         password_data = {
 3048             'local_user_id': local_user['id'],
 3049             'created_at': datetime.datetime.utcnow(),
 3050             'expires_at': datetime.datetime.utcnow(),
 3051             'password': uuid.uuid4().hex}
 3052         password_data1 = {
 3053             'local_user_id': local_user['id'],
 3054             'created_at': datetime.datetime.utcnow(),
 3055             'expires_at': datetime.datetime.utcnow(),
 3056             'password_hash': uuid.uuid4().hex}
 3057         password_data2 = {
 3058             'local_user_id': local_user['id'],
 3059             'created_at': datetime.datetime.utcnow(),
 3060             'expires_at': datetime.datetime.utcnow(),
 3061             'password': uuid.uuid4().hex,
 3062             'password_hash': uuid.uuid4().hex}
 3063         password_table.insert().values(password_data).execute()
 3064         password_table.insert().values(password_data1).execute()
 3065         password_table.insert().values(password_data2).execute()
 3066         self.migrate(46)
 3067         passwords = list(password_table.select().execute())
 3068         for p in passwords:
 3069             if p.password == password_data['password']:
 3070                 self.assertEqual(p.password_hash, p.password)
 3071                 self.assertIsNotNone(p.password)
 3072                 self.assertIsNotNone(p.password_hash)
 3073             elif p.password_hash == password_data1['password_hash']:
 3074                 self.assertIsNone(p.password)
 3075                 self.assertIsNotNone(p.password_hash)
 3076             elif p.password_hash == password_data2['password_hash']:
 3077                 self.assertIsNotNone(p.password)
 3078                 self.assertIsNotNone(p.password_hash)
 3079                 self.assertNotEqual(p.password, p.password_hash)
 3080             else:
 3081                 raise ValueError('Too Many Passwords Found')
 3082 
 3083     def test_migration_047_add_auto_increment_pk_column_to_unified_limit(self):
 3084         self.expand(46)
 3085         self.migrate(46)
 3086         self.contract(46)
 3087         registered_limit_table_name = 'registered_limit'
 3088         limit_table_name = 'limit'
 3089         self.assertTableColumns(
 3090             registered_limit_table_name,
 3091             ['id', 'service_id', 'region_id', 'resource_name', 'default_limit',
 3092              'description']
 3093         )
 3094         self.assertTableColumns(
 3095             limit_table_name,
 3096             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3097              'resource_limit', 'description']
 3098         )
 3099         self.assertTrue(self.does_pk_exist('registered_limit', 'id'))
 3100         self.assertTrue(self.does_pk_exist('limit', 'id'))
 3101         self.assertTrue(self.does_fk_exist('limit', 'project_id'))
 3102 
 3103         self.expand(47)
 3104         self.migrate(47)
 3105         self.contract(47)
 3106         self.assertTableColumns(
 3107             registered_limit_table_name,
 3108             ['id', 'service_id', 'region_id', 'resource_name', 'default_limit',
 3109              'description', 'internal_id']
 3110         )
 3111         self.assertTableColumns(
 3112             limit_table_name,
 3113             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3114              'resource_limit', 'description', 'internal_id']
 3115         )
 3116         self.assertFalse(self.does_pk_exist('registered_limit', 'id'))
 3117         self.assertTrue(self.does_pk_exist('registered_limit', 'internal_id'))
 3118         self.assertFalse(self.does_pk_exist('limit', 'id'))
 3119         self.assertTrue(self.does_pk_exist('limit', 'internal_id'))
 3120         limit_table = sqlalchemy.Table(limit_table_name,
 3121                                        self.metadata, autoload=True)
 3122         self.assertEqual(set([]), limit_table.foreign_keys)
 3123 
 3124     def test_migration_048_add_registered_limit_id_column_for_limit(self):
 3125         self.expand(47)
 3126         self.migrate(47)
 3127         self.contract(47)
 3128 
 3129         limit_table_name = 'limit'
 3130         self.assertTableColumns(
 3131             limit_table_name,
 3132             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3133              'resource_limit', 'description', 'internal_id']
 3134         )
 3135 
 3136         self.expand(48)
 3137         self.migrate(48)
 3138         self.contract(48)
 3139 
 3140         self.assertTableColumns(
 3141             limit_table_name,
 3142             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3143              'resource_limit', 'description', 'internal_id',
 3144              'registered_limit_id']
 3145         )
 3146         self.assertTrue(self.does_fk_exist('limit', 'registered_limit_id'))
 3147 
 3148     def test_migration_053_adds_description_to_role(self):
 3149         self.expand(52)
 3150         self.migrate(52)
 3151         self.contract(52)
 3152 
 3153         role_table_name = 'role'
 3154         self.assertTableColumns(
 3155             role_table_name,
 3156             ['id', 'name', 'domain_id', 'extra']
 3157         )
 3158 
 3159         self.expand(53)
 3160         self.migrate(53)
 3161         self.contract(53)
 3162 
 3163         self.assertTableColumns(
 3164             role_table_name,
 3165             ['id', 'name', 'domain_id', 'extra', 'description']
 3166         )
 3167 
 3168         role_table = sqlalchemy.Table(
 3169             role_table_name, self.metadata, autoload=True
 3170         )
 3171 
 3172         role = {
 3173             'id': uuid.uuid4().hex,
 3174             'name': "test",
 3175             'domain_id': resource_base.NULL_DOMAIN_ID,
 3176             'description': "This is a string"
 3177         }
 3178         role_table.insert().values(role).execute()
 3179 
 3180         role_without_description = {
 3181             'id': uuid.uuid4().hex,
 3182             'name': "test1",
 3183             'domain_id': resource_base.NULL_DOMAIN_ID
 3184         }
 3185         role_table.insert().values(role_without_description).execute()
 3186 
 3187     def test_migration_054_drop_old_password_column(self):
 3188         self.expand(53)
 3189         self.migrate(53)
 3190         self.contract(53)
 3191 
 3192         password_table = 'password'
 3193         self.assertTableColumns(
 3194             password_table,
 3195             ['id', 'local_user_id', 'password', 'password_hash',
 3196              'self_service', 'created_at_int', 'created_at', 'expires_at_int',
 3197              'expires_at']
 3198         )
 3199 
 3200         self.expand(54)
 3201         self.migrate(54)
 3202         self.contract(54)
 3203 
 3204         self.assertTableColumns(
 3205             password_table,
 3206             ['id', 'local_user_id', 'password_hash', 'self_service',
 3207              'created_at_int', 'created_at', 'expires_at_int', 'expires_at']
 3208         )
 3209 
 3210     def test_migration_055_add_domain_to_limit(self):
 3211         self.expand(54)
 3212         self.migrate(54)
 3213         self.contract(54)
 3214 
 3215         limit_table_name = 'limit'
 3216         limit_table = sqlalchemy.Table(limit_table_name, self.metadata,
 3217                                        autoload=True)
 3218         self.assertFalse(hasattr(limit_table.c, 'domain_id'))
 3219 
 3220         self.expand(55)
 3221         self.migrate(55)
 3222         self.contract(55)
 3223 
 3224         self.assertTableColumns(
 3225             limit_table_name,
 3226             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3227              'resource_limit', 'description', 'internal_id',
 3228              'registered_limit_id', 'domain_id'])
 3229         self.assertTrue(limit_table.c.project_id.nullable)
 3230 
 3231     def test_migration_056_add_application_credential_access_rules(self):
 3232         self.expand(55)
 3233         self.migrate(55)
 3234         self.contract(55)
 3235 
 3236         self.assertTableDoesNotExist('access_rule')
 3237         self.assertTableDoesNotExist('application_credential_access_rule')
 3238 
 3239         self.expand(56)
 3240         self.migrate(56)
 3241         self.contract(56)
 3242 
 3243         self.assertTableExists('access_rule')
 3244         self.assertTableExists('application_credential_access_rule')
 3245         self.assertTableColumns(
 3246             'access_rule',
 3247             ['id', 'service', 'path', 'method']
 3248         )
 3249         self.assertTableColumns(
 3250             'application_credential_access_rule',
 3251             ['application_credential_id', 'access_rule_id']
 3252         )
 3253         self.assertTrue(self.does_fk_exist(
 3254             'application_credential_access_rule', 'application_credential_id'))
 3255         self.assertTrue(self.does_fk_exist(
 3256             'application_credential_access_rule', 'access_rule_id'))
 3257 
 3258         app_cred_table = sqlalchemy.Table(
 3259             'application_credential', self.metadata, autoload=True
 3260         )
 3261         access_rule_table = sqlalchemy.Table(
 3262             'access_rule', self.metadata, autoload=True
 3263         )
 3264         app_cred_access_rule_table = sqlalchemy.Table(
 3265             'application_credential_access_rule',
 3266             self.metadata, autoload=True
 3267         )
 3268         app_cred = {
 3269             'internal_id': 1,
 3270             'id': uuid.uuid4().hex,
 3271             'name': uuid.uuid4().hex,
 3272             'secret_hash': uuid.uuid4().hex,
 3273             'user_id': uuid.uuid4().hex,
 3274             'project_id': uuid.uuid4().hex
 3275         }
 3276         app_cred_table.insert().values(app_cred).execute()
 3277         access_rule = {
 3278             'id': 1,
 3279             'service': uuid.uuid4().hex,
 3280             'path': '/v2.1/servers',
 3281             'method': 'GET'
 3282         }
 3283         access_rule_table.insert().values(access_rule).execute()
 3284         app_cred_access_rule_rel = {
 3285             'application_credential_id': app_cred['internal_id'],
 3286             'access_rule_id': access_rule['id']
 3287         }
 3288         app_cred_access_rule_table.insert().values(
 3289             app_cred_access_rule_rel).execute()
 3290 
 3291     def test_migration_062_add_trust_redelegation(self):
 3292         # ensure initial schema
 3293         self.expand(61)
 3294         self.migrate(61)
 3295         self.contract(61)
 3296         self.assertTableColumns('trust', ['id',
 3297                                           'trustor_user_id',
 3298                                           'trustee_user_id',
 3299                                           'project_id',
 3300                                           'impersonation',
 3301                                           'expires_at',
 3302                                           'expires_at_int',
 3303                                           'remaining_uses',
 3304                                           'deleted_at',
 3305                                           'extra'])
 3306 
 3307         # fixture
 3308         trust = {
 3309             'id': uuid.uuid4().hex,
 3310             'trustor_user_id': uuid.uuid4().hex,
 3311             'trustee_user_id': uuid.uuid4().hex,
 3312             'project_id': uuid.uuid4().hex,
 3313             'impersonation': True,
 3314             'expires_at': datetime.datetime.now(),
 3315             'remaining_uses': 10,
 3316             'deleted_at': datetime.datetime.now(),
 3317             'redelegated_trust_id': uuid.uuid4().hex,
 3318             'redelegation_count': 3,
 3319             'other': uuid.uuid4().hex
 3320         }
 3321         old_trust = trust.copy()
 3322         old_extra = {
 3323             'redelegated_trust_id': old_trust.pop('redelegated_trust_id'),
 3324             'redelegation_count': old_trust.pop('redelegation_count'),
 3325             'other': old_trust.pop('other')
 3326         }
 3327         old_trust['extra'] = jsonutils.dumps(old_extra)
 3328         # load fixture
 3329         session = self.sessionmaker()
 3330         self.insert_dict(session, 'trust', old_trust)
 3331 
 3332         # ensure redelegation data is in extra
 3333         stored_trust = list(
 3334             session.execute(self.load_table('trust').select())
 3335         )[0]
 3336         self.assertDictEqual({
 3337             'redelegated_trust_id': trust['redelegated_trust_id'],
 3338             'redelegation_count': trust['redelegation_count'],
 3339             'other': trust['other']},
 3340             jsonutils.loads(stored_trust.extra))
 3341 
 3342         # upgrade and ensure expected schema
 3343         self.expand(62)
 3344         self.migrate(62)
 3345         self.contract(62)
 3346         self.assertTableColumns('trust', ['id',
 3347                                           'trustor_user_id',
 3348                                           'trustee_user_id',
 3349                                           'project_id',
 3350                                           'impersonation',
 3351                                           'expires_at',
 3352                                           'expires_at_int',
 3353                                           'remaining_uses',
 3354                                           'deleted_at',
 3355                                           'redelegated_trust_id',
 3356                                           'redelegation_count',
 3357                                           'extra'])
 3358 
 3359         trust_table = sqlalchemy.Table('trust', self.metadata, autoload=True)
 3360         self.assertTrue(trust_table.c.redelegated_trust_id.nullable)
 3361         self.assertTrue(trust_table.c.redelegation_count.nullable)
 3362 
 3363         # test target data layout
 3364         upgraded_trust = list(
 3365             session.execute(self.load_table('trust').select())
 3366         )[0]
 3367         self.assertDictEqual({'other': trust['other']},
 3368                              jsonutils.loads(upgraded_trust.extra))
 3369         self.assertEqual(trust['redelegated_trust_id'],
 3370                          upgraded_trust.redelegated_trust_id)
 3371         self.assertEqual(trust['redelegation_count'],
 3372                          upgraded_trust.redelegation_count)
 3373 
 3374     def test_migration_063_drop_limit_columns(self):
 3375         self.expand(62)
 3376         self.migrate(62)
 3377         self.contract(62)
 3378 
 3379         limit_table = 'limit'
 3380         self.assertTableColumns(
 3381             limit_table,
 3382             ['id', 'project_id', 'service_id', 'region_id', 'resource_name',
 3383              'resource_limit', 'description', 'internal_id',
 3384              'registered_limit_id', 'domain_id'])
 3385 
 3386         self.expand(63)
 3387         self.migrate(63)
 3388         self.contract(63)
 3389 
 3390         self.assertTableColumns(
 3391             limit_table,
 3392             ['id', 'project_id', 'resource_limit', 'description',
 3393              'internal_id', 'registered_limit_id', 'domain_id'])
 3394 
 3395     def test_migration_064_add_remote_id_attribute_federation_protocol(self):
 3396         self.expand(63)
 3397         self.migrate(63)
 3398         self.contract(63)
 3399 
 3400         federation_protocol_table_name = 'federation_protocol'
 3401         self.assertTableColumns(
 3402             federation_protocol_table_name,
 3403             ['id', 'idp_id', 'mapping_id']
 3404         )
 3405 
 3406         self.expand(64)
 3407         self.migrate(64)
 3408         self.contract(64)
 3409 
 3410         self.assertTableColumns(
 3411             federation_protocol_table_name,
 3412             ['id', 'idp_id', 'mapping_id', 'remote_id_attribute']
 3413         )
 3414 
 3415     def test_migration_065_add_user_external_id_to_access_rule(self):
 3416         self.expand(64)
 3417         self.migrate(64)
 3418         self.contract(64)
 3419 
 3420         self.assertTableColumns(
 3421             'access_rule',
 3422             ['id', 'service', 'path', 'method']
 3423         )
 3424 
 3425         self.expand(65)
 3426         self.migrate(65)
 3427         self.contract(65)
 3428 
 3429         self.assertTableColumns(
 3430             'access_rule',
 3431             ['id', 'external_id', 'user_id', 'service', 'path', 'method']
 3432         )
 3433         self.assertTrue(self.does_index_exist('access_rule', 'external_id'))
 3434         self.assertTrue(self.does_index_exist('access_rule', 'user_id'))
 3435         self.assertTrue(self.does_unique_constraint_exist(
 3436             'access_rule', 'external_id'))
 3437         self.assertTrue(self.does_unique_constraint_exist(
 3438             'access_rule', ['user_id', 'service', 'path', 'method']))
 3439 
 3440     def test_migration_066_add_role_and_prject_options_tables(self):
 3441         self.expand(65)
 3442         self.migrate(65)
 3443         self.contract(65)
 3444 
 3445         role_option = 'role_option'
 3446         project_option = 'project_option'
 3447         self.assertTableDoesNotExist(role_option)
 3448         self.assertTableDoesNotExist(project_option)
 3449 
 3450         self.expand(66)
 3451         self.migrate(66)
 3452         self.contract(66)
 3453 
 3454         self.assertTableColumns(
 3455             project_option,
 3456             ['project_id', 'option_id', 'option_value'])
 3457 
 3458         self.assertTableColumns(
 3459             role_option,
 3460             ['role_id', 'option_id', 'option_value'])
 3461 
 3462     def test_migration_072_drop_domain_id_fk(self):
 3463         self.expand(71)
 3464         self.migrate(71)
 3465         self.contract(71)
 3466 
 3467         self.assertTrue(self.does_fk_exist('user', 'domain_id'))
 3468         self.assertTrue(self.does_fk_exist('identity_provider', 'domain_id'))
 3469 
 3470         self.expand(72)
 3471         self.migrate(72)
 3472         self.contract(72)
 3473 
 3474         self.assertFalse(self.does_fk_exist('user', 'domain_id'))
 3475         self.assertFalse(self.does_fk_exist('identity_provider', 'domain_id'))
 3476 
 3477     def test_migration_073_contract_expiring_group_membership(self):
 3478         self.expand(72)
 3479         self.migrate(72)
 3480         self.contract(72)
 3481 
 3482         membership_table = 'expiring_user_group_membership'
 3483         self.assertTableDoesNotExist(membership_table)
 3484 
 3485         idp_table = 'identity_provider'
 3486         self.assertTableColumns(
 3487             idp_table,
 3488             ['id', 'domain_id', 'enabled', 'description'])
 3489 
 3490         self.expand(73)
 3491         self.migrate(73)
 3492         self.contract(73)
 3493 
 3494         self.assertTableColumns(
 3495             membership_table,
 3496             ['user_id', 'group_id', 'idp_id', 'last_verified'])
 3497         self.assertTableColumns(
 3498             idp_table,
 3499             ['id', 'domain_id', 'enabled', 'description',
 3500              'authorization_ttl'])
 3501 
 3502 
 3503 class MySQLOpportunisticFullMigration(FullMigration):
 3504     FIXTURE = db_fixtures.MySQLOpportunisticFixture
 3505 
 3506 
 3507 class PostgreSQLOpportunisticFullMigration(FullMigration):
 3508     FIXTURE = db_fixtures.PostgresqlOpportunisticFixture