"Fossies" - the Fresh Open Source Software Archive

Member "cinder-14.0.2/cinder/test.py" (4 Oct 2019, 23044 Bytes) of package /linux/misc/openstack/cinder-14.0.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "test.py": 14.0.2_vs_15.0.0.

    1 # Copyright 2010 United States Government as represented by the
    2 # Administrator of the National Aeronautics and Space Administration.
    3 # All Rights Reserved.
    4 #
    5 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
    6 #    not use this file except in compliance with the License. You may obtain
    7 #    a copy of the License at
    8 #
    9 #         http://www.apache.org/licenses/LICENSE-2.0
   10 #
   11 #    Unless required by applicable law or agreed to in writing, software
   12 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   13 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
   14 #    License for the specific language governing permissions and limitations
   15 #    under the License.
   16 
   17 """Base classes for our unit tests.
   18 
   19 Allows overriding of CONF for use of fakes, and some black magic for
   20 inline callbacks.
   21 
   22 """
   23 import copy
   24 import logging
   25 import os
   26 import sys
   27 import uuid
   28 
   29 from eventlet import tpool
   30 import fixtures
   31 from keystonemiddleware import auth_token
   32 import mock
   33 from oslo_concurrency import lockutils
   34 from oslo_config import fixture as config_fixture
   35 from oslo_log.fixture import logging_error as log_fixture
   36 import oslo_messaging
   37 from oslo_messaging import conffixture as messaging_conffixture
   38 from oslo_serialization import jsonutils
   39 from oslo_utils import strutils
   40 from oslo_utils import timeutils
   41 import six
   42 import testtools
   43 
   44 from cinder.api import common as api_common
   45 from cinder.common import config
   46 from cinder import context
   47 from cinder import coordination
   48 from cinder.db import migration
   49 from cinder.db.sqlalchemy import api as sqla_api
   50 from cinder import exception
   51 from cinder import i18n
   52 from cinder.objects import base as objects_base
   53 from cinder import rpc
   54 from cinder import service
   55 from cinder.tests import fixtures as cinder_fixtures
   56 from cinder.tests.unit import conf_fixture
   57 from cinder.tests.unit import fake_notifier
   58 from cinder.volume import utils
   59 
   60 
   61 CONF = config.CONF
   62 
   63 _DB_CACHE = None
   64 SESSION_CONFIGURED = False
   65 
   66 
   67 class TestingException(Exception):
   68     pass
   69 
   70 
   71 class CinderExceptionReraiseFormatError(object):
   72     real_log_exception = exception.CinderException._log_exception
   73 
   74     @classmethod
   75     def patch(cls):
   76         exception.CinderException._log_exception = cls._wrap_log_exception
   77 
   78     @staticmethod
   79     def _wrap_log_exception(self):
   80         exc_info = sys.exc_info()
   81         CinderExceptionReraiseFormatError.real_log_exception(self)
   82         six.reraise(*exc_info)
   83 
   84 
   85 # NOTE(melwitt) This needs to be done at import time in order to also catch
   86 # CinderException format errors that are in mock decorators. In these cases,
   87 # the errors will be raised during test listing, before tests actually run.
   88 CinderExceptionReraiseFormatError.patch()
   89 
   90 
   91 class Database(fixtures.Fixture):
   92 
   93     def __init__(self, db_api, db_migrate, sql_connection):
   94         # NOTE(lhx_): oslo_db.enginefacade is configured in tests the same
   95         # way as it's done for any other services that uses the db
   96         global SESSION_CONFIGURED
   97         if not SESSION_CONFIGURED:
   98             sqla_api.configure(CONF)
   99             SESSION_CONFIGURED = True
  100         self.sql_connection = sql_connection
  101 
  102         # Suppress logging for test runs
  103         migrate_logger = logging.getLogger('migrate')
  104         migrate_logger.setLevel(logging.WARNING)
  105 
  106         self.engine = db_api.get_engine()
  107         self.engine.dispose()
  108         conn = self.engine.connect()
  109         db_migrate.db_sync()
  110         self._DB = "".join(line for line in conn.connection.iterdump())
  111         self.engine.dispose()
  112 
  113     def setUp(self):
  114         super(Database, self).setUp()
  115 
  116         conn = self.engine.connect()
  117         conn.connection.executescript(self._DB)
  118         self.addCleanup(self.engine.dispose)
  119 
  120 
  121 class TestCase(testtools.TestCase):
  122     """Test case base class for all unit tests."""
  123 
  124     POLICY_PATH = 'cinder/tests/unit/policy.json'
  125     RESOURCE_FILTER_PATH = 'etc/cinder/resource_filters.json'
  126     MOCK_WORKER = True
  127     MOCK_TOOZ = True
  128 
  129     def __init__(self, *args, **kwargs):
  130         super(TestCase, self).__init__(*args, **kwargs)
  131 
  132         # Suppress some log messages during test runs
  133         castellan_logger = logging.getLogger('castellan')
  134         castellan_logger.setLevel(logging.ERROR)
  135         stevedore_logger = logging.getLogger('stevedore')
  136         stevedore_logger.setLevel(logging.ERROR)
  137 
  138     def _get_joined_notifier(self, *args, **kwargs):
  139         # We create a new fake notifier but we join the notifications with
  140         # the default notifier
  141         notifier = fake_notifier.get_fake_notifier(*args, **kwargs)
  142         notifier.notifications = self.notifier.notifications
  143         return notifier
  144 
  145     def _reset_filter_file(self):
  146         self.override_config('resource_query_filters_file',
  147                              os.path.join(
  148                                  os.path.abspath(
  149                                      os.path.join(
  150                                          os.path.dirname(__file__),
  151                                          '..',
  152                                      )
  153                                  ),
  154                                  self.RESOURCE_FILTER_PATH))
  155         api_common._FILTERS_COLLECTION = None
  156 
  157     def setUp(self):
  158         """Run before each test method to initialize test environment."""
  159         super(TestCase, self).setUp()
  160 
  161         # Create default notifier
  162         self.notifier = fake_notifier.get_fake_notifier()
  163 
  164         # Mock rpc get notifier with fake notifier method that joins all
  165         # notifications with the default notifier
  166         self.patch('cinder.rpc.get_notifier',
  167                    side_effect=self._get_joined_notifier)
  168 
  169         if self.MOCK_WORKER:
  170             # Mock worker creation for all tests that don't care about it
  171             clean_path = 'cinder.objects.cleanable.CinderCleanableObject.%s'
  172             for method in ('create_worker', 'set_worker', 'unset_worker'):
  173                 self.patch(clean_path % method, return_value=None)
  174 
  175         if self.MOCK_TOOZ:
  176             self.patch('cinder.coordination.Coordinator.start')
  177             self.patch('cinder.coordination.Coordinator.stop')
  178             self.patch('cinder.coordination.Coordinator.get_lock')
  179 
  180         # Unit tests do not need to use lazy gettext
  181         i18n.enable_lazy(False)
  182 
  183         test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
  184         try:
  185             test_timeout = int(test_timeout)
  186         except ValueError:
  187             # If timeout value is invalid do not set a timeout.
  188             test_timeout = 0
  189         if test_timeout > 0:
  190             self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
  191         self.useFixture(fixtures.NestedTempfile())
  192         self.useFixture(fixtures.TempHomeDir())
  193 
  194         environ_enabled = (lambda var_name:
  195                            strutils.bool_from_string(os.environ.get(var_name)))
  196         if environ_enabled('OS_STDOUT_CAPTURE'):
  197             stdout = self.useFixture(fixtures.StringStream('stdout')).stream
  198             self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
  199         if environ_enabled('OS_STDERR_CAPTURE'):
  200             stderr = self.useFixture(fixtures.StringStream('stderr')).stream
  201             self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
  202 
  203         self.useFixture(log_fixture.get_logging_handle_error_fixture())
  204         self.useFixture(cinder_fixtures.StandardLogging())
  205 
  206         rpc.add_extra_exmods("cinder.tests.unit")
  207         self.addCleanup(rpc.clear_extra_exmods)
  208         self.addCleanup(rpc.cleanup)
  209 
  210         self.messaging_conf = messaging_conffixture.ConfFixture(CONF)
  211         self.messaging_conf.transport_url = 'fake:/'
  212         self.messaging_conf.response_timeout = 15
  213         self.useFixture(self.messaging_conf)
  214 
  215         # Load oslo_messaging_notifications config group so we can set an
  216         # override to prevent notifications from being ignored due to the
  217         # short-circuit mechanism.
  218         oslo_messaging.get_notification_transport(CONF)
  219         #  We need to use a valid driver for the notifications, so we use test.
  220         self.override_config('driver', ['test'],
  221                              group='oslo_messaging_notifications')
  222         rpc.init(CONF)
  223 
  224         # NOTE(geguileo): This is required because _determine_obj_version_cap
  225         # and _determine_rpc_version_cap functions in cinder.rpc.RPCAPI cache
  226         # versions in LAST_RPC_VERSIONS and LAST_OBJ_VERSIONS so we may have
  227         # weird interactions between tests if we don't clear them before each
  228         # test.
  229         rpc.LAST_OBJ_VERSIONS = {}
  230         rpc.LAST_RPC_VERSIONS = {}
  231 
  232         # Init AuthProtocol to register some base options first, such as
  233         # auth_url.
  234         auth_token.AuthProtocol('fake_app', {'auth_type': 'password',
  235                                              'auth_url': 'fake_url'})
  236 
  237         conf_fixture.set_defaults(CONF)
  238         CONF([], default_config_files=[])
  239 
  240         # NOTE(vish): We need a better method for creating fixtures for tests
  241         #             now that we have some required db setup for the system
  242         #             to work properly.
  243         self.start = timeutils.utcnow()
  244 
  245         CONF.set_default('connection', 'sqlite://', 'database')
  246         CONF.set_default('sqlite_synchronous', False, 'database')
  247 
  248         global _DB_CACHE
  249         if not _DB_CACHE:
  250             _DB_CACHE = Database(sqla_api, migration,
  251                                  sql_connection=CONF.database.connection)
  252         self.useFixture(_DB_CACHE)
  253 
  254         # NOTE(blk-u): WarningsFixture must be after the Database fixture
  255         # because sqlalchemy-migrate messes with the warnings filters.
  256         self.useFixture(cinder_fixtures.WarningsFixture())
  257 
  258         # NOTE(danms): Make sure to reset us back to non-remote objects
  259         # for each test to avoid interactions. Also, backup the object
  260         # registry.
  261         objects_base.CinderObject.indirection_api = None
  262         self._base_test_obj_backup = copy.copy(
  263             objects_base.CinderObjectRegistry._registry._obj_classes)
  264         self.addCleanup(self._restore_obj_registry)
  265 
  266         self.addCleanup(CONF.reset)
  267         self.addCleanup(self._common_cleanup)
  268         self.injected = []
  269         self._services = []
  270 
  271         fake_notifier.mock_notifier(self)
  272 
  273         # This will be cleaned up by the NestedTempfile fixture
  274         lock_path = self.useFixture(fixtures.TempDir()).path
  275         self.fixture = self.useFixture(
  276             config_fixture.Config(lockutils.CONF))
  277         self.fixture.config(lock_path=lock_path,
  278                             group='oslo_concurrency')
  279         lockutils.set_defaults(lock_path)
  280         self.override_config('policy_file',
  281                              os.path.join(
  282                                  os.path.abspath(
  283                                      os.path.join(
  284                                          os.path.dirname(__file__),
  285                                          '..',
  286                                      )
  287                                  ),
  288                                  self.POLICY_PATH),
  289                              group='oslo_policy')
  290         self.override_config('resource_query_filters_file',
  291                              os.path.join(
  292                                  os.path.abspath(
  293                                      os.path.join(
  294                                          os.path.dirname(__file__),
  295                                          '..',
  296                                      )
  297                                  ),
  298                                  self.RESOURCE_FILTER_PATH))
  299         self._disable_osprofiler()
  300 
  301         # NOTE(geguileo): This is required because common get_by_id method in
  302         # cinder.db.sqlalchemy.api caches get methods and if we use a mocked
  303         # get method in one test it would carry on to the next test.  So we
  304         # clear out the cache.
  305         sqla_api._GET_METHODS = {}
  306 
  307         self.override_config('backend_url', 'file://' + lock_path,
  308                              group='coordination')
  309         coordination.COORDINATOR.start()
  310         self.addCleanup(coordination.COORDINATOR.stop)
  311 
  312         if six.PY3:
  313             # TODO(smcginnis) Python 3 deprecates assertRaisesRegexp to
  314             # assertRaisesRegex, but Python 2 does not have the new name. This
  315             # can be removed once we stop supporting py2 or the new name is
  316             # added.
  317             self.assertRaisesRegexp = self.assertRaisesRegex
  318 
  319         # Ensure we have the default tpool size value and we don't carry
  320         # threads from other test runs.
  321         tpool.killall()
  322         tpool._nthreads = 20
  323 
  324         # NOTE(mikal): make sure we don't load a privsep helper accidentally
  325         self.useFixture(cinder_fixtures.PrivsepNoHelperFixture())
  326 
  327     def _restore_obj_registry(self):
  328         objects_base.CinderObjectRegistry._registry._obj_classes = \
  329             self._base_test_obj_backup
  330 
  331     def _disable_osprofiler(self):
  332         """Disable osprofiler.
  333 
  334         osprofiler should not run for unit tests.
  335         """
  336 
  337         side_effect = lambda value: value
  338         mock_decorator = mock.MagicMock(side_effect=side_effect)
  339         p = mock.patch("osprofiler.profiler.trace_cls",
  340                        return_value=mock_decorator)
  341         p.start()
  342 
  343     def _common_cleanup(self):
  344         """Runs after each test method to tear down test environment."""
  345 
  346         # Stop any timers
  347         for x in self.injected:
  348             try:
  349                 x.stop()
  350             except AssertionError:
  351                 pass
  352 
  353         # Kill any services
  354         for x in self._services:
  355             try:
  356                 x.kill()
  357             except Exception:
  358                 pass
  359 
  360         # Delete attributes that don't start with _ so they don't pin
  361         # memory around unnecessarily for the duration of the test
  362         # suite
  363         for key in [k for k in self.__dict__ if k[0] != '_']:
  364             del self.__dict__[key]
  365 
  366     def override_config(self, name, override, group=None):
  367         """Cleanly override CONF variables."""
  368         CONF.set_override(name, override, group)
  369         self.addCleanup(CONF.clear_override, name, group)
  370 
  371     def flags(self, **kw):
  372         """Override CONF variables for a test."""
  373         group = kw.pop('group', None)
  374         for k, v in kw.items():
  375             self.override_config(k, v, group)
  376 
  377     def start_service(self, name, host=None, **kwargs):
  378         host = host if host else uuid.uuid4().hex
  379         kwargs.setdefault('host', host)
  380         kwargs.setdefault('binary', 'cinder-%s' % name)
  381         svc = service.Service.create(**kwargs)
  382         svc.start()
  383         self._services.append(svc)
  384         return svc
  385 
  386     def mock_object(self, obj, attr_name, *args, **kwargs):
  387         """Use python mock to mock an object attribute
  388 
  389         Mocks the specified objects attribute with the given value.
  390         Automatically performs 'addCleanup' for the mock.
  391 
  392         """
  393         patcher = mock.patch.object(obj, attr_name, *args, **kwargs)
  394         result = patcher.start()
  395         self.addCleanup(patcher.stop)
  396         return result
  397 
  398     def patch(self, path, *args, **kwargs):
  399         """Use python mock to mock a path with automatic cleanup."""
  400         patcher = mock.patch(path, *args, **kwargs)
  401         result = patcher.start()
  402         self.addCleanup(patcher.stop)
  403         return result
  404 
  405     # Useful assertions
  406     def assert_notify_called(self, mock_notify, calls, any_order=False):
  407         if any_order is True:
  408             for c in calls:
  409                 # mock_notify.call_args_list = [
  410                 #     mock.call('INFO', 'volume.retype', ...),
  411                 #     mock.call('WARN', 'cinder.fire', ...)]
  412                 # m = mock_notify.call_args_list
  413                 # m[0] = Call
  414                 # m[0][0] = tuple('INFO', <context>, 'volume.retype', ...)
  415                 if not any(m for m in mock_notify.call_args_list
  416                            if (m[0][0] == c[0]     # 'INFO'
  417                                and
  418                                m[0][2] == c[1])):  # 'volume.retype'
  419                     raise AssertionError("notify call not found: %s" % c)
  420             return
  421 
  422         for i in range(0, len(calls)):
  423             mock_call = mock_notify.call_args_list[i]
  424             call = calls[i]
  425 
  426             posargs = mock_call[0]
  427 
  428             self.assertEqual(call[0], posargs[0])
  429             self.assertEqual(call[1], posargs[2])
  430 
  431     def assertTrue(self, x, *args, **kwargs):
  432         """Assert that value is True.
  433 
  434         If original behavior is required we will need to do:
  435             assertTrue(bool(result))
  436         """
  437         # assertTrue uses msg but assertIs uses message keyword argument
  438         args = list(args)
  439         msg = kwargs.pop('msg', args.pop(0) if args else '')
  440         kwargs.setdefault('message', msg)
  441         self.assertIs(True, x, *args, **kwargs)
  442 
  443     def assertFalse(self, x, *args, **kwargs):
  444         """Assert that value is False.
  445 
  446         If original behavior is required we will need to do:
  447             assertFalse(bool(result))
  448         """
  449         # assertTrue uses msg but assertIs uses message keyword argument
  450         args = list(args)
  451         msg = kwargs.pop('msg', args.pop(0) if args else '')
  452         kwargs.setdefault('message', msg)
  453         self.assertIs(False, x, *args, **kwargs)
  454 
  455     def stub_out(self, old, new):
  456         """Replace a function for the duration of the test.
  457 
  458         Use the monkey patch fixture to replace a function for the
  459         duration of a test. Useful when you want to provide fake
  460         methods instead of mocks during testing.
  461         This should be used instead of self.stubs.Set (which is based
  462         on mox) going forward.
  463         """
  464         self.useFixture(fixtures.MonkeyPatch(old, new))
  465 
  466 
  467 class ModelsObjectComparatorMixin(object):
  468     def _dict_from_object(self, obj, ignored_keys):
  469         if ignored_keys is None:
  470             ignored_keys = []
  471         obj = jsonutils.to_primitive(obj)  # Convert to dict first.
  472         items = obj.items()
  473         return {k: v for k, v in items
  474                 if k not in ignored_keys}
  475 
  476     def _assertEqualObjects(self, obj1, obj2, ignored_keys=None):
  477         obj1 = self._dict_from_object(obj1, ignored_keys)
  478         obj2 = self._dict_from_object(obj2, ignored_keys)
  479 
  480         self.assertEqual(
  481             len(obj1), len(obj2),
  482             "Keys mismatch: %s" % six.text_type(
  483                 set(obj1.keys()) ^ set(obj2.keys())))
  484         for key, value in obj1.items():
  485             self.assertEqual(value, obj2[key])
  486 
  487     def _assertEqualListsOfObjects(self, objs1, objs2, ignored_keys=None,
  488                                    msg=None):
  489         obj_to_dict = lambda o: self._dict_from_object(o, ignored_keys)
  490         objs1 = map(obj_to_dict, objs1)
  491         objs2 = list(map(obj_to_dict, objs2))
  492         # We don't care about the order of the lists, as long as they are in
  493         for obj1 in objs1:
  494             self.assertIn(obj1, objs2)
  495             objs2.remove(obj1)
  496         self.assertEqual([], objs2)
  497 
  498     def _assertEqualListsOfPrimitivesAsSets(self, primitives1, primitives2):
  499         self.assertEqual(len(primitives1), len(primitives2))
  500         for primitive in primitives1:
  501             self.assertIn(primitive, primitives2)
  502 
  503         for primitive in primitives2:
  504             self.assertIn(primitive, primitives1)
  505 
  506 
  507 class RPCAPITestCase(TestCase, ModelsObjectComparatorMixin):
  508     def setUp(self):
  509         super(RPCAPITestCase, self).setUp()
  510         self.context = context.get_admin_context()
  511         self.rpcapi = None
  512         self.base_version = '2.0'
  513 
  514     def _test_rpc_api(self, method, rpc_method, server=None, fanout=False,
  515                       version=None, expected_method=None,
  516                       expected_kwargs_diff=None, retval=None,
  517                       expected_retval=None, **kwargs):
  518         """Runs a test against RPC API method.
  519 
  520         :param method: Name of RPC API method.
  521         :param rpc_method: Expected RPC message type (cast or call).
  522         :param server: Expected hostname.
  523         :param fanout: True if expected call/cast should be fanout.
  524         :param version: Expected autocalculated RPC API version.
  525         :param expected_method: Expected RPC method name.
  526         :param expected_kwargs_diff: Map of expected changes between keyword
  527                                      arguments passed into the method and sent
  528                                      over RPC.
  529         :param retval: Value returned by RPC call/cast.
  530         :param expected_retval: Expected RPC API response (if different than
  531                                 retval).
  532         :param kwargs: Parameters passed into the RPC API method.
  533         """
  534 
  535         rpcapi = self.rpcapi()
  536         expected_kwargs_diff = expected_kwargs_diff or {}
  537         version = version or self.base_version
  538         topic = None
  539         if server is not None:
  540             backend = utils.extract_host(server)
  541             server = utils.extract_host(server, 'host')
  542             topic = 'cinder-volume.%s' % backend
  543 
  544         if expected_method is None:
  545             expected_method = method
  546 
  547         if expected_retval is None:
  548             expected_retval = retval
  549 
  550         target = {
  551             "server": server,
  552             "fanout": fanout,
  553             "version": version,
  554             "topic": topic,
  555         }
  556 
  557         # Initially we expect that we'll pass same arguments to RPC API method
  558         # and RPC call/cast...
  559         expected_msg = copy.deepcopy(kwargs)
  560         # ... but here we're taking exceptions into account.
  561         expected_msg.update(expected_kwargs_diff)
  562 
  563         def _fake_prepare_method(*args, **kwds):
  564             # This is checking if target will be properly created.
  565             for kwd in kwds:
  566                 self.assertEqual(target[kwd], kwds[kwd])
  567             return rpcapi.client
  568 
  569         def _fake_rpc_method(*args, **kwargs):
  570             # This checks if positional arguments passed to RPC method match.
  571             self.assertEqual((self.context, expected_method), args)
  572 
  573             # This checks if keyword arguments passed to RPC method match.
  574             for kwarg, value in kwargs.items():
  575                 # Getting possible changes into account.
  576                 if isinstance(value, objects_base.CinderObject):
  577                     # We need to compare objects differently.
  578                     self._assertEqualObjects(expected_msg[kwarg], value)
  579                 else:
  580                     self.assertEqual(expected_msg[kwarg], value)
  581 
  582             # Returning fake value we're supposed to return.
  583             if retval:
  584                 return retval
  585 
  586         # Enable mocks that will check everything and run RPC method.
  587         with mock.patch.object(rpcapi.client, "prepare",
  588                                side_effect=_fake_prepare_method):
  589             with mock.patch.object(rpcapi.client, rpc_method,
  590                                    side_effect=_fake_rpc_method):
  591                 real_retval = getattr(rpcapi, method)(self.context, **kwargs)
  592                 self.assertEqual(expected_retval, real_retval)