"Fossies" - the Fresh Open Source Software Archive

Member "salt-3002.2/tests/unit/test_loader.py" (18 Nov 2020, 54372 Bytes) of package /linux/misc/salt-3002.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_loader.py": 3002.1_vs_3002.2.

    1 """
    2     unit.loader
    3     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    4 
    5     Test Salt's loader
    6 """
    7 
    8 import collections
    9 import compileall
   10 import copy
   11 import imp
   12 import inspect
   13 import logging
   14 import os
   15 import shutil
   16 import sys
   17 import tempfile
   18 import textwrap
   19 
   20 import salt.config
   21 import salt.loader
   22 import salt.utils.files
   23 import salt.utils.stringutils
   24 from tests.support.case import ModuleCase
   25 from tests.support.helpers import slowTest
   26 from tests.support.mock import MagicMock, patch
   27 from tests.support.runtests import RUNTIME_VARS
   28 from tests.support.unit import TestCase
   29 
   30 log = logging.getLogger(__name__)
   31 
   32 
   33 def remove_bytecode(module_path):
   34     paths = [module_path + "c"]
   35     if hasattr(imp, "get_tag"):
   36         modname, ext = os.path.splitext(module_path.split(os.sep)[-1])
   37         paths.append(
   38             os.path.join(
   39                 os.path.dirname(module_path),
   40                 "__pycache__",
   41                 "{}.{}.pyc".format(modname, imp.get_tag()),
   42             )
   43         )
   44     for path in paths:
   45         if os.path.exists(path):
   46             os.unlink(path)
   47 
   48 
   49 loader_template = """
   50 import os
   51 from salt.utils.decorators import depends
   52 
   53 @depends('os')
   54 def loaded():
   55     return True
   56 
   57 @depends('non_existantmodulename')
   58 def not_loaded():
   59     return True
   60 """
   61 
   62 
   63 class LazyLoaderTest(TestCase):
   64     """
   65     Test the loader
   66     """
   67 
   68     module_name = "lazyloadertest"
   69 
   70     @classmethod
   71     def setUpClass(cls):
   72         cls.opts = salt.config.minion_config(None)
   73         cls.opts["grains"] = salt.loader.grains(cls.opts)
   74         if not os.path.isdir(RUNTIME_VARS.TMP):
   75             os.makedirs(RUNTIME_VARS.TMP)
   76         cls.utils = salt.loader.utils(cls.opts)
   77         cls.proxy = salt.loader.proxy(cls.opts)
   78         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
   79 
   80     def setUp(self):
   81         # Setup the module
   82         self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
   83         self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True)
   84         self.module_file = os.path.join(
   85             self.module_dir, "{}.py".format(self.module_name)
   86         )
   87         with salt.utils.files.fopen(self.module_file, "w") as fh:
   88             fh.write(salt.utils.stringutils.to_str(loader_template))
   89             fh.flush()
   90             os.fsync(fh.fileno())
   91 
   92         # Invoke the loader
   93         self.loader = salt.loader.LazyLoader(
   94             [self.module_dir],
   95             copy.deepcopy(self.opts),
   96             pack={
   97                 "__utils__": self.utils,
   98                 "__salt__": self.funcs,
   99                 "__proxy__": self.proxy,
  100             },
  101             tag="module",
  102         )
  103 
  104     def tearDown(self):
  105         del self.module_dir
  106         del self.module_file
  107         del self.loader
  108 
  109     @classmethod
  110     def tearDownClass(cls):
  111         del cls.opts
  112         del cls.funcs
  113         del cls.utils
  114         del cls.proxy
  115 
  116     @slowTest
  117     def test_depends(self):
  118         """
  119         Test that the depends decorator works properly
  120         """
  121         # Make sure depends correctly allowed a function to load. If this
  122         # results in a KeyError, the decorator is broken.
  123         self.assertTrue(inspect.isfunction(self.loader[self.module_name + ".loaded"]))
  124         # Make sure depends correctly kept a function from loading
  125         self.assertTrue(self.module_name + ".not_loaded" not in self.loader)
  126 
  127 
  128 loader_template_module = """
  129 import my_utils
  130 
  131 def run():
  132     return my_utils.run()
  133 """
  134 
  135 loader_template_utils = """
  136 def run():
  137     return True
  138 """
  139 
  140 
  141 class LazyLoaderUtilsTest(TestCase):
  142     """
  143     Test the loader
  144     """
  145 
  146     module_name = "lazyloaderutilstest"
  147     utils_name = "my_utils"
  148 
  149     @classmethod
  150     def setUpClass(cls):
  151         cls.opts = salt.config.minion_config(None)
  152         cls.opts["grains"] = salt.loader.grains(cls.opts)
  153         if not os.path.isdir(RUNTIME_VARS.TMP):
  154             os.makedirs(RUNTIME_VARS.TMP)
  155 
  156     def setUp(self):
  157         # Setup the module
  158         self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  159         self.module_file = os.path.join(
  160             self.module_dir, "{}.py".format(self.module_name)
  161         )
  162         with salt.utils.files.fopen(self.module_file, "w") as fh:
  163             fh.write(salt.utils.stringutils.to_str(loader_template_module))
  164             fh.flush()
  165             os.fsync(fh.fileno())
  166 
  167         self.utils_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  168         self.utils_file = os.path.join(self.utils_dir, "{}.py".format(self.utils_name))
  169         with salt.utils.files.fopen(self.utils_file, "w") as fh:
  170             fh.write(salt.utils.stringutils.to_str(loader_template_utils))
  171             fh.flush()
  172             os.fsync(fh.fileno())
  173 
  174     def tearDown(self):
  175         shutil.rmtree(self.module_dir)
  176         if os.path.isdir(self.module_dir):
  177             shutil.rmtree(self.module_dir)
  178         shutil.rmtree(self.utils_dir)
  179         if os.path.isdir(self.utils_dir):
  180             shutil.rmtree(self.utils_dir)
  181         del self.module_dir
  182         del self.module_file
  183         del self.utils_dir
  184         del self.utils_file
  185 
  186         if self.module_name in sys.modules:
  187             del sys.modules[self.module_name]
  188         if self.utils_name in sys.modules:
  189             del sys.modules[self.utils_name]
  190 
  191     @classmethod
  192     def tearDownClass(cls):
  193         del cls.opts
  194 
  195     def test_utils_found(self):
  196         """
  197         Test that the extra module directory is available for imports
  198         """
  199         loader = salt.loader.LazyLoader(
  200             [self.module_dir],
  201             copy.deepcopy(self.opts),
  202             tag="module",
  203             extra_module_dirs=[self.utils_dir],
  204         )
  205         self.assertTrue(inspect.isfunction(loader[self.module_name + ".run"]))
  206         self.assertTrue(loader[self.module_name + ".run"]())
  207 
  208     def test_utils_not_found(self):
  209         """
  210         Test that the extra module directory is not available for imports
  211         """
  212         loader = salt.loader.LazyLoader(
  213             [self.module_dir], copy.deepcopy(self.opts), tag="module"
  214         )
  215         self.assertTrue(self.module_name + ".run" not in loader)
  216 
  217 
  218 class LazyLoaderVirtualEnabledTest(TestCase):
  219     """
  220     Test the base loader of salt.
  221     """
  222 
  223     @classmethod
  224     def setUpClass(cls):
  225         cls.opts = salt.config.minion_config(None)
  226         cls.opts["disable_modules"] = ["pillar"]
  227         cls.opts["grains"] = salt.loader.grains(cls.opts)
  228         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
  229         cls.proxy = salt.loader.proxy(cls.opts)
  230         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
  231 
  232     def setUp(self):
  233         self.loader = salt.loader.LazyLoader(
  234             salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"),
  235             copy.deepcopy(self.opts),
  236             pack={
  237                 "__utils__": self.utils,
  238                 "__salt__": self.funcs,
  239                 "__proxy__": self.proxy,
  240             },
  241             tag="module",
  242         )
  243 
  244     def tearDown(self):
  245         del self.loader
  246 
  247     @classmethod
  248     def tearDownClass(cls):
  249         del cls.opts
  250         del cls.funcs
  251         del cls.utils
  252         del cls.proxy
  253 
  254     @slowTest
  255     def test_basic(self):
  256         """
  257         Ensure that it only loads stuff when needed
  258         """
  259         # make sure it starts empty
  260         self.assertEqual(self.loader._dict, {})
  261         # get something, and make sure its a func
  262         self.assertTrue(inspect.isfunction(self.loader["test.ping"]))
  263 
  264         # make sure we only loaded "test" functions
  265         for key, val in self.loader._dict.items():
  266             self.assertEqual(key.split(".", 1)[0], "test")
  267 
  268         # make sure the depends thing worked (double check of the depends testing,
  269         # since the loader does the calling magically
  270         self.assertFalse("test.missing_func" in self.loader._dict)
  271 
  272     def test_badkey(self):
  273         with self.assertRaises(KeyError):
  274             self.loader[None]  # pylint: disable=W0104
  275 
  276         with self.assertRaises(KeyError):
  277             self.loader[1]  # pylint: disable=W0104
  278 
  279     @slowTest
  280     def test_disable(self):
  281         self.assertNotIn("pillar.items", self.loader)
  282 
  283     @slowTest
  284     def test_len_load(self):
  285         """
  286         Since LazyLoader is a MutableMapping, if someone asks for len() we have
  287         to load all
  288         """
  289         self.assertEqual(self.loader._dict, {})
  290         len(self.loader)  # force a load all
  291         self.assertNotEqual(self.loader._dict, {})
  292 
  293     @slowTest
  294     def test_iter_load(self):
  295         """
  296         Since LazyLoader is a MutableMapping, if someone asks to iterate we have
  297         to load all
  298         """
  299         self.assertEqual(self.loader._dict, {})
  300         # force a load all
  301         for key, func in self.loader.items():
  302             break
  303         self.assertNotEqual(self.loader._dict, {})
  304 
  305     def test_context(self):
  306         """
  307         Make sure context is shared across modules
  308         """
  309         # make sure it starts empty
  310         self.assertEqual(self.loader._dict, {})
  311         # get something, and make sure its a func
  312         func = self.loader["test.ping"]
  313         with patch.dict(func.__globals__["__context__"], {"foo": "bar"}):
  314             self.assertEqual(
  315                 self.loader["test.echo"].__globals__["__context__"]["foo"], "bar"
  316             )
  317             self.assertEqual(
  318                 self.loader["grains.get"].__globals__["__context__"]["foo"], "bar"
  319             )
  320 
  321     def test_globals(self):
  322         func_globals = self.loader["test.ping"].__globals__
  323         self.assertEqual(func_globals["__grains__"], self.opts.get("grains", {}))
  324         self.assertEqual(func_globals["__pillar__"], self.opts.get("pillar", {}))
  325         # the opts passed into modules is at least a subset of the whole opts
  326         for key, val in func_globals["__opts__"].items():
  327             if (
  328                 key in salt.config.DEFAULT_MASTER_OPTS
  329                 and key not in salt.config.DEFAULT_MINION_OPTS
  330             ):
  331                 # We loaded the minion opts, but somewhere in the code, the master options got pulled in
  332                 # Let's just not check for equality since the option won't even exist in the loaded
  333                 # minion options
  334                 continue
  335             if (
  336                 key not in salt.config.DEFAULT_MASTER_OPTS
  337                 and key not in salt.config.DEFAULT_MINION_OPTS
  338             ):
  339                 # This isn't even a default configuration setting, lets carry on
  340                 continue
  341             self.assertEqual(self.opts[key], val)
  342 
  343     def test_pack(self):
  344         self.loader.pack["__foo__"] = "bar"
  345         func_globals = self.loader["test.ping"].__globals__
  346         self.assertEqual(func_globals["__foo__"], "bar")
  347 
  348     @slowTest
  349     def test_virtual(self):
  350         self.assertNotIn("test_virtual.ping", self.loader)
  351 
  352 
  353 class LazyLoaderVirtualDisabledTest(TestCase):
  354     """
  355     Test the loader of salt without __virtual__
  356     """
  357 
  358     @classmethod
  359     def setUpClass(cls):
  360         cls.opts = salt.config.minion_config(None)
  361         cls.opts["grains"] = salt.loader.grains(cls.opts)
  362         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
  363         cls.proxy = salt.loader.proxy(cls.opts)
  364         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
  365 
  366     def setUp(self):
  367         self.loader = salt.loader.LazyLoader(
  368             salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"),
  369             copy.deepcopy(self.opts),
  370             tag="module",
  371             pack={
  372                 "__utils__": self.utils,
  373                 "__salt__": self.funcs,
  374                 "__proxy__": self.proxy,
  375             },
  376             virtual_enable=False,
  377         )
  378 
  379     def tearDown(self):
  380         del self.loader
  381 
  382     @classmethod
  383     def tearDownClass(cls):
  384         del cls.opts
  385         del cls.utils
  386         del cls.funcs
  387         del cls.proxy
  388 
  389     @slowTest
  390     def test_virtual(self):
  391         self.assertTrue(inspect.isfunction(self.loader["test_virtual.ping"]))
  392 
  393 
  394 class LazyLoaderWhitelistTest(TestCase):
  395     """
  396     Test the loader of salt with a whitelist
  397     """
  398 
  399     @classmethod
  400     def setUpClass(cls):
  401         cls.opts = salt.config.minion_config(None)
  402         cls.opts["grains"] = salt.loader.grains(cls.opts)
  403         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
  404         cls.proxy = salt.loader.proxy(cls.opts)
  405         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
  406 
  407     def setUp(self):
  408         self.loader = salt.loader.LazyLoader(
  409             salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"),
  410             copy.deepcopy(self.opts),
  411             tag="module",
  412             pack={
  413                 "__utils__": self.utils,
  414                 "__salt__": self.funcs,
  415                 "__proxy__": self.proxy,
  416             },
  417             whitelist=["test", "pillar"],
  418         )
  419 
  420     def tearDown(self):
  421         del self.loader
  422 
  423     @classmethod
  424     def tearDownClass(cls):
  425         del cls.opts
  426         del cls.funcs
  427         del cls.utils
  428         del cls.proxy
  429 
  430     @slowTest
  431     def test_whitelist(self):
  432         self.assertTrue(inspect.isfunction(self.loader["test.ping"]))
  433         self.assertTrue(inspect.isfunction(self.loader["pillar.get"]))
  434 
  435         self.assertNotIn("grains.get", self.loader)
  436 
  437 
  438 class LazyLoaderGrainsBlacklistTest(TestCase):
  439     """
  440     Test the loader of grains with a blacklist
  441     """
  442 
  443     def setUp(self):
  444         self.opts = salt.config.minion_config(None)
  445 
  446     def tearDown(self):
  447         del self.opts
  448 
  449     @slowTest
  450     def test_whitelist(self):
  451         opts = copy.deepcopy(self.opts)
  452         opts["grains_blacklist"] = ["master", "os*", "ipv[46]"]
  453 
  454         grains = salt.loader.grains(opts)
  455         self.assertNotIn("master", grains)
  456         self.assertNotIn("os", {g[:2] for g in list(grains)})
  457         self.assertNotIn("ipv4", grains)
  458         self.assertNotIn("ipv6", grains)
  459 
  460 
  461 class LazyLoaderSingleItem(TestCase):
  462     """
  463     Test loading a single item via the _load() function
  464     """
  465 
  466     @classmethod
  467     def setUpClass(cls):
  468         cls.opts = salt.config.minion_config(None)
  469         cls.opts["grains"] = salt.loader.grains(cls.opts)
  470         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
  471         cls.proxy = salt.loader.proxy(cls.opts)
  472         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
  473 
  474     @classmethod
  475     def tearDownClass(cls):
  476         del cls.opts
  477         del cls.funcs
  478         del cls.utils
  479         del cls.proxy
  480 
  481     def setUp(self):
  482         self.loader = salt.loader.LazyLoader(
  483             salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"),
  484             copy.deepcopy(self.opts),
  485             pack={
  486                 "__utils__": self.utils,
  487                 "__salt__": self.funcs,
  488                 "__proxy__": self.proxy,
  489             },
  490             tag="module",
  491         )
  492 
  493     def tearDown(self):
  494         del self.loader
  495 
  496     def test_single_item_no_dot(self):
  497         """
  498         Checks that a KeyError is raised when the function key does not contain a '.'
  499         """
  500         key = "testing_no_dot"
  501         expected = "The key '{}' should contain a '.'".format(key)
  502         with self.assertRaises(KeyError) as err:
  503             inspect.isfunction(self.loader["testing_no_dot"])
  504 
  505         result = err.exception.args[0]
  506         assert result == expected, result
  507 
  508 
  509 module_template = """
  510 __load__ = ['test', 'test_alias']
  511 __func_alias__ = dict(test_alias='working_alias')
  512 from salt.utils.decorators import depends
  513 
  514 def test():
  515     return {count}
  516 
  517 def test_alias():
  518     return True
  519 
  520 def test2():
  521     return True
  522 
  523 @depends('non_existantmodulename')
  524 def test3():
  525     return True
  526 
  527 @depends('non_existantmodulename', fallback_function=test)
  528 def test4():
  529     return True
  530 """
  531 
  532 
  533 class LazyLoaderReloadingTest(TestCase):
  534     """
  535     Test the loader of salt with changing modules
  536     """
  537 
  538     module_name = "loadertest"
  539     module_key = "loadertest.test"
  540 
  541     @classmethod
  542     def setUpClass(cls):
  543         cls.opts = salt.config.minion_config(None)
  544         cls.opts["grains"] = salt.loader.grains(cls.opts)
  545         if not os.path.isdir(RUNTIME_VARS.TMP):
  546             os.makedirs(RUNTIME_VARS.TMP)
  547 
  548     def setUp(self):
  549         self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  550         self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True)
  551 
  552         self.count = 0
  553         opts = copy.deepcopy(self.opts)
  554         dirs = salt.loader._module_dirs(opts, "modules", "module")
  555         dirs.append(self.tmp_dir)
  556         self.utils = salt.loader.utils(opts)
  557         self.proxy = salt.loader.proxy(opts)
  558         self.minion_mods = salt.loader.minion_mods(opts)
  559         self.loader = salt.loader.LazyLoader(
  560             dirs,
  561             opts,
  562             tag="module",
  563             pack={
  564                 "__utils__": self.utils,
  565                 "__proxy__": self.proxy,
  566                 "__salt__": self.minion_mods,
  567             },
  568         )
  569 
  570     def tearDown(self):
  571         for attrname in ("tmp_dir", "utils", "proxy", "loader", "minion_mods", "utils"):
  572             try:
  573                 delattr(self, attrname)
  574             except AttributeError:
  575                 continue
  576 
  577     @classmethod
  578     def tearDownClass(cls):
  579         del cls.opts
  580 
  581     def update_module(self):
  582         self.count += 1
  583         with salt.utils.files.fopen(self.module_path, "wb") as fh:
  584             fh.write(
  585                 salt.utils.stringutils.to_bytes(
  586                     module_template.format(count=self.count)
  587                 )
  588             )
  589             fh.flush()
  590             os.fsync(fh.fileno())  # flush to disk
  591 
  592         # pyc files don't like it when we change the original quickly
  593         # since the header bytes only contain the timestamp (granularity of seconds)
  594         # TODO: don't write them? Is *much* slower on re-load (~3x)
  595         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
  596         remove_bytecode(self.module_path)
  597 
  598     def rm_module(self):
  599         os.unlink(self.module_path)
  600         remove_bytecode(self.module_path)
  601 
  602     @property
  603     def module_path(self):
  604         return os.path.join(self.tmp_dir, "{}.py".format(self.module_name))
  605 
  606     @slowTest
  607     def test_alias(self):
  608         """
  609         Make sure that you can access alias-d modules
  610         """
  611         # ensure it doesn't exist
  612         self.assertNotIn(self.module_key, self.loader)
  613 
  614         self.update_module()
  615         self.assertNotIn("{}.test_alias".format(self.module_name), self.loader)
  616         self.assertTrue(
  617             inspect.isfunction(self.loader["{}.working_alias".format(self.module_name)])
  618         )
  619 
  620     @slowTest
  621     def test_clear(self):
  622         self.assertTrue(inspect.isfunction(self.loader["test.ping"]))
  623         self.update_module()  # write out out custom module
  624         self.loader.clear()  # clear the loader dict
  625 
  626         # force a load of our module
  627         self.assertTrue(inspect.isfunction(self.loader[self.module_key]))
  628 
  629         # make sure we only loaded our custom module
  630         # which means that we did correctly refresh the file mapping
  631         for k, v in self.loader._dict.items():
  632             self.assertTrue(k.startswith(self.module_name))
  633 
  634     @slowTest
  635     def test_load(self):
  636         # ensure it doesn't exist
  637         self.assertNotIn(self.module_key, self.loader)
  638 
  639         self.update_module()
  640         self.assertTrue(inspect.isfunction(self.loader[self.module_key]))
  641 
  642     @slowTest
  643     def test__load__(self):
  644         """
  645         If a module specifies __load__ we should only load/expose those modules
  646         """
  647         self.update_module()
  648 
  649         # ensure it doesn't exist
  650         self.assertNotIn(self.module_key + "2", self.loader)
  651 
  652     @slowTest
  653     def test__load__and_depends(self):
  654         """
  655         If a module specifies __load__ we should only load/expose those modules
  656         """
  657         self.update_module()
  658         # ensure it doesn't exist
  659         self.assertNotIn(self.module_key + "3", self.loader)
  660         self.assertNotIn(self.module_key + "4", self.loader)
  661 
  662     @slowTest
  663     def test_reload(self):
  664         # ensure it doesn't exist
  665         self.assertNotIn(self.module_key, self.loader)
  666 
  667         # make sure it updates correctly
  668         for x in range(1, 3):
  669             self.update_module()
  670             self.loader.clear()
  671             self.assertEqual(self.loader[self.module_key](), self.count)
  672 
  673         self.rm_module()
  674         # make sure that even if we remove the module, its still loaded until a clear
  675         self.assertEqual(self.loader[self.module_key](), self.count)
  676         self.loader.clear()
  677         self.assertNotIn(self.module_key, self.loader)
  678 
  679     def test_wrong_bytecode(self):
  680         """
  681         Checks to make sure we don't even try to load .pyc files that are for a different Python
  682         This should pass (the load should fail) all the time because we don't run Salt on Py 3.4 anymore
  683         """
  684         test_module_name = "test_module.cpython-34"
  685         filemap_save = copy.deepcopy(self.loader.file_mapping)
  686         self.loader.file_mapping = {
  687             test_module_name: (
  688                 "/temp/path/does/not/matter/here/__pycache__/"
  689                 + test_module_name
  690                 + ".pyc",
  691                 ".pyc",
  692                 0,
  693             )
  694         }
  695 
  696         self.assertFalse(self.loader._load_module(test_module_name))
  697 
  698         self.loader.file_mapping = copy.deepcopy(filemap_save)
  699 
  700 
  701 virtual_aliases = ("loadertest2", "loadertest3")
  702 virtual_alias_module_template = """
  703 __virtual_aliases__ = {}
  704 
  705 def test():
  706     return True
  707 """.format(
  708     virtual_aliases
  709 )
  710 
  711 
  712 class LazyLoaderVirtualAliasTest(TestCase):
  713     """
  714     Test the loader of salt with changing modules
  715     """
  716 
  717     module_name = "loadertest"
  718 
  719     @classmethod
  720     def setUpClass(cls):
  721         cls.opts = salt.config.minion_config(None)
  722         cls.opts["grains"] = salt.loader.grains(cls.opts)
  723         if not os.path.isdir(RUNTIME_VARS.TMP):
  724             os.makedirs(RUNTIME_VARS.TMP)
  725 
  726     def setUp(self):
  727         self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  728         opts = copy.deepcopy(self.opts)
  729         dirs = salt.loader._module_dirs(opts, "modules", "module")
  730         dirs.append(self.tmp_dir)
  731         self.utils = salt.loader.utils(opts)
  732         self.proxy = salt.loader.proxy(opts)
  733         self.minion_mods = salt.loader.minion_mods(opts)
  734         self.loader = salt.loader.LazyLoader(
  735             dirs,
  736             opts,
  737             tag="module",
  738             pack={
  739                 "__utils__": self.utils,
  740                 "__proxy__": self.proxy,
  741                 "__salt__": self.minion_mods,
  742             },
  743         )
  744 
  745     def tearDown(self):
  746         del self.tmp_dir
  747         del self.utils
  748         del self.proxy
  749         del self.minion_mods
  750         del self.loader
  751 
  752     @classmethod
  753     def tearDownClass(cls):
  754         del cls.opts
  755 
  756     def update_module(self):
  757         with salt.utils.files.fopen(self.module_path, "wb") as fh:
  758             fh.write(salt.utils.stringutils.to_bytes(virtual_alias_module_template))
  759             fh.flush()
  760             os.fsync(fh.fileno())  # flush to disk
  761 
  762         # pyc files don't like it when we change the original quickly
  763         # since the header bytes only contain the timestamp (granularity of seconds)
  764         # TODO: don't write them? Is *much* slower on re-load (~3x)
  765         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
  766         remove_bytecode(self.module_path)
  767 
  768     @property
  769     def module_path(self):
  770         return os.path.join(self.tmp_dir, "{}.py".format(self.module_name))
  771 
  772     @slowTest
  773     def test_virtual_alias(self):
  774         """
  775         Test the __virtual_alias__ feature
  776         """
  777         self.update_module()
  778 
  779         mod_names = [self.module_name] + list(virtual_aliases)
  780         for mod_name in mod_names:
  781             func_name = ".".join((mod_name, "test"))
  782             log.debug("Running %s (dict attribute)", func_name)
  783             self.assertTrue(self.loader[func_name]())
  784             log.debug("Running %s (loader attribute)", func_name)
  785             self.assertTrue(getattr(self.loader, mod_name).test())
  786 
  787 
  788 submodule_template = """
  789 from __future__ import absolute_import
  790 
  791 import {0}.lib
  792 
  793 def test():
  794     return ({count}, {0}.lib.test())
  795 """
  796 
  797 submodule_lib_template = """
  798 def test():
  799     return {count}
  800 """
  801 
  802 
  803 class LazyLoaderSubmodReloadingTest(TestCase):
  804     """
  805     Test the loader of salt with changing modules
  806     """
  807 
  808     module_name = "loadertestsubmod"
  809     module_key = "loadertestsubmod.test"
  810 
  811     @classmethod
  812     def setUpClass(cls):
  813         cls.opts = salt.config.minion_config(None)
  814         cls.opts["grains"] = salt.loader.grains(cls.opts)
  815         if not os.path.isdir(RUNTIME_VARS.TMP):
  816             os.makedirs(RUNTIME_VARS.TMP)
  817 
  818     def setUp(self):
  819         self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  820         self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True)
  821         os.makedirs(self.module_dir)
  822 
  823         self.count = 0
  824         self.lib_count = 0
  825 
  826         opts = copy.deepcopy(self.opts)
  827         dirs = salt.loader._module_dirs(opts, "modules", "module")
  828         dirs.append(self.tmp_dir)
  829         self.utils = salt.loader.utils(opts)
  830         self.proxy = salt.loader.proxy(opts)
  831         self.minion_mods = salt.loader.minion_mods(opts)
  832         self.loader = salt.loader.LazyLoader(
  833             dirs,
  834             opts,
  835             tag="module",
  836             pack={
  837                 "__utils__": self.utils,
  838                 "__proxy__": self.proxy,
  839                 "__salt__": self.minion_mods,
  840             },
  841         )
  842 
  843     def tearDown(self):
  844         del self.tmp_dir
  845         del self.utils
  846         del self.proxy
  847         del self.minion_mods
  848         del self.loader
  849 
  850     @classmethod
  851     def tearDownClass(cls):
  852         del cls.opts
  853 
  854     def update_module(self):
  855         self.count += 1
  856         with salt.utils.files.fopen(self.module_path, "wb") as fh:
  857             fh.write(
  858                 salt.utils.stringutils.to_bytes(
  859                     submodule_template.format(self.module_name, count=self.count)
  860                 )
  861             )
  862             fh.flush()
  863             os.fsync(fh.fileno())  # flush to disk
  864 
  865         # pyc files don't like it when we change the original quickly
  866         # since the header bytes only contain the timestamp (granularity of seconds)
  867         # TODO: don't write them? Is *much* slower on re-load (~3x)
  868         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
  869         remove_bytecode(self.module_path)
  870 
  871     def rm_module(self):
  872         os.unlink(self.module_path)
  873         remove_bytecode(self.module_path)
  874 
  875     def update_lib(self):
  876         self.lib_count += 1
  877         for modname in list(sys.modules):
  878             if modname.startswith(self.module_name):
  879                 del sys.modules[modname]
  880         with salt.utils.files.fopen(self.lib_path, "wb") as fh:
  881             fh.write(
  882                 salt.utils.stringutils.to_bytes(
  883                     submodule_lib_template.format(count=self.lib_count)
  884                 )
  885             )
  886             fh.flush()
  887             os.fsync(fh.fileno())  # flush to disk
  888 
  889         # pyc files don't like it when we change the original quickly
  890         # since the header bytes only contain the timestamp (granularity of seconds)
  891         # TODO: don't write them? Is *much* slower on re-load (~3x)
  892         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
  893         remove_bytecode(self.lib_path)
  894 
  895     def rm_lib(self):
  896         for modname in list(sys.modules):
  897             if modname.startswith(self.module_name):
  898                 del sys.modules[modname]
  899         os.unlink(self.lib_path)
  900         remove_bytecode(self.lib_path)
  901 
  902     @property
  903     def module_dir(self):
  904         return os.path.join(self.tmp_dir, self.module_name)
  905 
  906     @property
  907     def module_path(self):
  908         return os.path.join(self.module_dir, "__init__.py")
  909 
  910     @property
  911     def lib_path(self):
  912         return os.path.join(self.module_dir, "lib.py")
  913 
  914     @slowTest
  915     def test_basic(self):
  916         # ensure it doesn't exist
  917         self.assertNotIn(self.module_key, self.loader)
  918 
  919         self.update_module()
  920         self.update_lib()
  921         self.loader.clear()
  922         self.assertIn(self.module_key, self.loader)
  923 
  924     @slowTest
  925     def test_reload(self):
  926         # ensure it doesn't exist
  927         self.assertNotIn(self.module_key, self.loader)
  928 
  929         # update both the module and the lib
  930         for x in range(1, 3):
  931             self.update_lib()
  932             self.update_module()
  933             self.loader.clear()
  934             self.assertNotIn(self.module_key, self.loader._dict)
  935             self.assertIn(self.module_key, self.loader)
  936             self.assertEqual(
  937                 self.loader[self.module_key](), (self.count, self.lib_count)
  938             )
  939 
  940         # update just the module
  941         for x in range(1, 3):
  942             self.update_module()
  943             self.loader.clear()
  944             self.assertNotIn(self.module_key, self.loader._dict)
  945             self.assertIn(self.module_key, self.loader)
  946             self.assertEqual(
  947                 self.loader[self.module_key](), (self.count, self.lib_count)
  948             )
  949 
  950         # update just the lib
  951         for x in range(1, 3):
  952             self.update_lib()
  953             self.loader.clear()
  954             self.assertNotIn(self.module_key, self.loader._dict)
  955             self.assertIn(self.module_key, self.loader)
  956             self.assertEqual(
  957                 self.loader[self.module_key](), (self.count, self.lib_count)
  958             )
  959 
  960         self.rm_module()
  961         # make sure that even if we remove the module, its still loaded until a clear
  962         self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
  963         self.loader.clear()
  964         self.assertNotIn(self.module_key, self.loader)
  965 
  966     @slowTest
  967     def test_reload_missing_lib(self):
  968         # ensure it doesn't exist
  969         self.assertNotIn(self.module_key, self.loader)
  970 
  971         # update both the module and the lib
  972         self.update_module()
  973         self.update_lib()
  974         self.loader.clear()
  975         self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count))
  976 
  977         # remove the lib, this means we should fail to load the module next time
  978         self.rm_lib()
  979         self.loader.clear()
  980         self.assertNotIn(self.module_key, self.loader)
  981 
  982 
  983 mod_template = """
  984 def test():
  985     return ({val})
  986 """
  987 
  988 
  989 class LazyLoaderModulePackageTest(TestCase):
  990     """
  991     Test the loader of salt with changing modules
  992     """
  993 
  994     module_name = "loadertestmodpkg"
  995     module_key = "loadertestmodpkg.test"
  996 
  997     @classmethod
  998     def setUpClass(cls):
  999         cls.opts = salt.config.minion_config(None)
 1000         cls.opts["grains"] = salt.loader.grains(cls.opts)
 1001         if not os.path.isdir(RUNTIME_VARS.TMP):
 1002             os.makedirs(RUNTIME_VARS.TMP)
 1003         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
 1004         cls.proxy = salt.loader.proxy(cls.opts)
 1005         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
 1006 
 1007     def setUp(self):
 1008         self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
 1009         self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True)
 1010 
 1011         dirs = salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module")
 1012         dirs.append(self.tmp_dir)
 1013         self.loader = salt.loader.LazyLoader(
 1014             dirs,
 1015             copy.deepcopy(self.opts),
 1016             pack={
 1017                 "__utils__": self.utils,
 1018                 "__salt__": self.funcs,
 1019                 "__proxy__": self.proxy,
 1020             },
 1021             tag="module",
 1022         )
 1023 
 1024     def tearDown(self):
 1025         del self.tmp_dir
 1026         del self.loader
 1027 
 1028     @classmethod
 1029     def tearDownClass(cls):
 1030         del cls.opts
 1031         del cls.funcs
 1032         del cls.utils
 1033         del cls.proxy
 1034 
 1035     def update_pyfile(self, pyfile, contents):
 1036         dirname = os.path.dirname(pyfile)
 1037         if not os.path.exists(dirname):
 1038             os.makedirs(dirname)
 1039         with salt.utils.files.fopen(pyfile, "wb") as fh:
 1040             fh.write(salt.utils.stringutils.to_bytes(contents))
 1041             fh.flush()
 1042             os.fsync(fh.fileno())  # flush to disk
 1043 
 1044         # pyc files don't like it when we change the original quickly
 1045         # since the header bytes only contain the timestamp (granularity of seconds)
 1046         # TODO: don't write them? Is *much* slower on re-load (~3x)
 1047         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
 1048         remove_bytecode(pyfile)
 1049 
 1050     def rm_pyfile(self, pyfile):
 1051         os.unlink(pyfile)
 1052         remove_bytecode(pyfile)
 1053 
 1054     def update_module(self, relative_path, contents):
 1055         self.update_pyfile(os.path.join(self.tmp_dir, relative_path), contents)
 1056 
 1057     def rm_module(self, relative_path):
 1058         self.rm_pyfile(os.path.join(self.tmp_dir, relative_path))
 1059 
 1060     @slowTest
 1061     def test_module(self):
 1062         # ensure it doesn't exist
 1063         self.assertNotIn("foo", self.loader)
 1064         self.assertNotIn("foo.test", self.loader)
 1065         self.update_module("foo.py", mod_template.format(val=1))
 1066         self.loader.clear()
 1067         self.assertIn("foo.test", self.loader)
 1068         self.assertEqual(self.loader["foo.test"](), 1)
 1069 
 1070     @slowTest
 1071     def test_package(self):
 1072         # ensure it doesn't exist
 1073         self.assertNotIn("foo", self.loader)
 1074         self.assertNotIn("foo.test", self.loader)
 1075         self.update_module("foo/__init__.py", mod_template.format(val=2))
 1076         self.loader.clear()
 1077         self.assertIn("foo.test", self.loader)
 1078         self.assertEqual(self.loader["foo.test"](), 2)
 1079 
 1080     @slowTest
 1081     def test_module_package_collision(self):
 1082         # ensure it doesn't exist
 1083         self.assertNotIn("foo", self.loader)
 1084         self.assertNotIn("foo.test", self.loader)
 1085         self.update_module("foo.py", mod_template.format(val=3))
 1086         self.loader.clear()
 1087         self.assertIn("foo.test", self.loader)
 1088         self.assertEqual(self.loader["foo.test"](), 3)
 1089 
 1090         self.update_module("foo/__init__.py", mod_template.format(val=4))
 1091         self.loader.clear()
 1092         self.assertIn("foo.test", self.loader)
 1093         self.assertEqual(self.loader["foo.test"](), 4)
 1094 
 1095 
 1096 deep_init_base = """
 1097 from __future__ import absolute_import
 1098 import {0}.top_lib
 1099 import {0}.top_lib.mid_lib
 1100 import {0}.top_lib.mid_lib.bot_lib
 1101 
 1102 def top():
 1103     return {0}.top_lib.test()
 1104 
 1105 def mid():
 1106     return {0}.top_lib.mid_lib.test()
 1107 
 1108 def bot():
 1109     return {0}.top_lib.mid_lib.bot_lib.test()
 1110 """
 1111 
 1112 
 1113 class LazyLoaderDeepSubmodReloadingTest(TestCase):
 1114     module_name = "loadertestsubmoddeep"
 1115     libs = ("top_lib", "mid_lib", "bot_lib")
 1116 
 1117     @classmethod
 1118     def setUpClass(cls):
 1119         cls.opts = salt.config.minion_config(None)
 1120         cls.opts["grains"] = salt.loader.grains(cls.opts)
 1121         if not os.path.isdir(RUNTIME_VARS.TMP):
 1122             os.makedirs(RUNTIME_VARS.TMP)
 1123 
 1124     def setUp(self):
 1125         self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
 1126         self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True)
 1127         os.makedirs(self.module_dir)
 1128 
 1129         self.lib_count = collections.defaultdict(int)  # mapping of path -> count
 1130 
 1131         # bootstrap libs
 1132         with salt.utils.files.fopen(
 1133             os.path.join(self.module_dir, "__init__.py"), "w"
 1134         ) as fh:
 1135             # No .decode() needed here as deep_init_base is defined as str and
 1136             # not bytes.
 1137             fh.write(
 1138                 salt.utils.stringutils.to_str(deep_init_base.format(self.module_name))
 1139             )
 1140             fh.flush()
 1141             os.fsync(fh.fileno())  # flush to disk
 1142 
 1143         self.lib_paths = {}
 1144         dir_path = self.module_dir
 1145         for lib_name in self.libs:
 1146             dir_path = os.path.join(dir_path, lib_name)
 1147             self.lib_paths[lib_name] = dir_path
 1148             os.makedirs(dir_path)
 1149             self.update_lib(lib_name)
 1150 
 1151         opts = copy.deepcopy(self.opts)
 1152         dirs = salt.loader._module_dirs(opts, "modules", "module")
 1153         dirs.append(self.tmp_dir)
 1154         self.utils = salt.loader.utils(opts)
 1155         self.proxy = salt.loader.proxy(opts)
 1156         self.minion_mods = salt.loader.minion_mods(opts)
 1157         self.loader = salt.loader.LazyLoader(
 1158             dirs,
 1159             copy.deepcopy(opts),
 1160             tag="module",
 1161             pack={
 1162                 "__utils__": self.utils,
 1163                 "__proxy__": self.proxy,
 1164                 "__salt__": self.minion_mods,
 1165             },
 1166         )
 1167         self.assertIn("{}.top".format(self.module_name), self.loader)
 1168 
 1169     def tearDown(self):
 1170         del self.tmp_dir
 1171         del self.lib_paths
 1172         del self.utils
 1173         del self.proxy
 1174         del self.minion_mods
 1175         del self.loader
 1176         del self.lib_count
 1177 
 1178     @classmethod
 1179     def tearDownClass(cls):
 1180         del cls.opts
 1181 
 1182     @property
 1183     def module_dir(self):
 1184         return os.path.join(self.tmp_dir, self.module_name)
 1185 
 1186     def update_lib(self, lib_name):
 1187         for modname in list(sys.modules):
 1188             if modname.startswith(self.module_name):
 1189                 del sys.modules[modname]
 1190         path = os.path.join(self.lib_paths[lib_name], "__init__.py")
 1191         self.lib_count[lib_name] += 1
 1192         with salt.utils.files.fopen(path, "wb") as fh:
 1193             fh.write(
 1194                 salt.utils.stringutils.to_bytes(
 1195                     submodule_lib_template.format(count=self.lib_count[lib_name])
 1196                 )
 1197             )
 1198             fh.flush()
 1199             os.fsync(fh.fileno())  # flush to disk
 1200 
 1201         # pyc files don't like it when we change the original quickly
 1202         # since the header bytes only contain the timestamp (granularity of seconds)
 1203         # TODO: don't write them? Is *much* slower on re-load (~3x)
 1204         # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode
 1205         remove_bytecode(path)
 1206 
 1207     @slowTest
 1208     def test_basic(self):
 1209         self.assertIn("{}.top".format(self.module_name), self.loader)
 1210 
 1211     def _verify_libs(self):
 1212         for lib in self.libs:
 1213             self.assertEqual(
 1214                 self.loader[
 1215                     "{}.{}".format(self.module_name, lib.replace("_lib", ""))
 1216                 ](),
 1217                 self.lib_count[lib],
 1218             )
 1219 
 1220     @slowTest
 1221     def test_reload(self):
 1222         """
 1223         Make sure that we can reload all libraries of arbitrary depth
 1224         """
 1225         self._verify_libs()
 1226 
 1227         # update them all
 1228         for lib in self.libs:
 1229             for x in range(5):
 1230                 self.update_lib(lib)
 1231                 self.loader.clear()
 1232                 self._verify_libs()
 1233 
 1234 
 1235 class LoaderMultipleGlobalTest(ModuleCase):
 1236     """
 1237     Tests when using multiple lazyloaders
 1238     """
 1239 
 1240     def setUp(self):
 1241         opts = salt.config.minion_config(None)
 1242         self.loader1 = salt.loader.LazyLoader(
 1243             salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"),
 1244             copy.deepcopy(opts),
 1245             pack={},
 1246             tag="module",
 1247             loaded_base_name="salt.loader1",
 1248         )
 1249         self.loader2 = salt.loader.LazyLoader(
 1250             salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"),
 1251             copy.deepcopy(opts),
 1252             pack={},
 1253             tag="module",
 1254             loaded_base_name="salt.loader2",
 1255         )
 1256 
 1257     def tearDown(self):
 1258         del self.loader1
 1259         del self.loader2
 1260 
 1261     def test_loader_globals(self):
 1262         """
 1263         Test to ensure loaders do not edit
 1264         each others loader's namespace
 1265         """
 1266         self.loader1.pack["__foo__"] = "bar1"
 1267         func1 = self.loader1["test.ping"]
 1268 
 1269         self.loader2.pack["__foo__"] = "bar2"
 1270         func2 = self.loader2["test.ping"]
 1271 
 1272         assert func1.__globals__["__foo__"] == "bar1"
 1273         assert func2.__globals__["__foo__"] == "bar2"
 1274 
 1275 
 1276 class LoaderCleanupTest(ModuleCase):
 1277     """
 1278     Tests the loader cleanup procedures
 1279     """
 1280 
 1281     def setUp(self):
 1282         opts = salt.config.minion_config(None)
 1283         self.loader1 = salt.loader.LazyLoader(
 1284             salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"),
 1285             copy.deepcopy(opts),
 1286             pack={},
 1287             tag="module",
 1288             loaded_base_name="salt.test",
 1289         )
 1290 
 1291     def tearDown(self):
 1292         del self.loader1
 1293 
 1294     def test_loader_clean_modules(self):
 1295         loaded_base_name = self.loader1.loaded_base_name
 1296         self.loader1.clean_modules()
 1297 
 1298         for name in list(sys.modules):
 1299             if name.startswith(loaded_base_name):
 1300                 self.fail(
 1301                     "Found a real module reference in sys.modules matching {!r}".format(
 1302                         loaded_base_name
 1303                     )
 1304                 )
 1305                 break
 1306 
 1307 
 1308 class LoaderGlobalsTest(ModuleCase):
 1309     """
 1310     Test all of the globals that the loader is responsible for adding to modules
 1311 
 1312     This shouldn't be done here, but should rather be done per module type (in the cases where they are used)
 1313     so they can check ALL globals that they have (or should have) access to.
 1314 
 1315     This is intended as a shorter term way of testing these so we don't break the loader
 1316     """
 1317 
 1318     def _verify_globals(self, mod_dict):
 1319         """
 1320         Verify that the globals listed in the doc string (from the test) are in these modules
 1321         """
 1322         # find the globals
 1323         global_vars = {}
 1324         for val in mod_dict.values():
 1325             # only find salty globals
 1326             if val.__module__.startswith("salt.loaded"):
 1327                 if hasattr(val, "__globals__"):
 1328                     if hasattr(val, "__wrapped__") or "__wrapped__" in val.__globals__:
 1329                         global_vars[val.__module__] = sys.modules[
 1330                             val.__module__
 1331                         ].__dict__
 1332                     else:
 1333                         global_vars[val.__module__] = val.__globals__
 1334 
 1335         # if we couldn't find any, then we have no modules -- so something is broken
 1336         self.assertNotEqual(global_vars, {}, msg="No modules were loaded.")
 1337 
 1338         # get the names of the globals you should have
 1339         func_name = inspect.stack()[1][3]
 1340         names = next(
 1341             iter(salt.utils.yaml.safe_load(getattr(self, func_name).__doc__).values())
 1342         )
 1343 
 1344         # Now, test each module!
 1345         for item in global_vars.values():
 1346             for name in names:
 1347                 self.assertIn(name, list(item.keys()))
 1348 
 1349     def test_auth(self):
 1350         """
 1351         Test that auth mods have:
 1352             - __pillar__
 1353             - __grains__
 1354             - __salt__
 1355             - __context__
 1356         """
 1357         self._verify_globals(salt.loader.auth(self.master_opts))
 1358 
 1359     def test_runners(self):
 1360         """
 1361         Test that runners have:
 1362             - __pillar__
 1363             - __salt__
 1364             - __opts__
 1365             - __grains__
 1366             - __context__
 1367         """
 1368         self._verify_globals(salt.loader.runner(self.master_opts))
 1369 
 1370     def test_returners(self):
 1371         """
 1372         Test that returners have:
 1373             - __salt__
 1374             - __opts__
 1375             - __pillar__
 1376             - __grains__
 1377             - __context__
 1378         """
 1379         self._verify_globals(salt.loader.returners(self.master_opts, {}))
 1380 
 1381     def test_pillars(self):
 1382         """
 1383         Test that pillars have:
 1384             - __salt__
 1385             - __opts__
 1386             - __pillar__
 1387             - __grains__
 1388             - __context__
 1389         """
 1390         self._verify_globals(salt.loader.pillars(self.master_opts, {}))
 1391 
 1392     def test_tops(self):
 1393         """
 1394         Test that tops have: []
 1395         """
 1396         self._verify_globals(salt.loader.tops(self.master_opts))
 1397 
 1398     def test_outputters(self):
 1399         """
 1400         Test that outputters have:
 1401             - __opts__
 1402             - __pillar__
 1403             - __grains__
 1404             - __context__
 1405         """
 1406         self._verify_globals(salt.loader.outputters(self.master_opts))
 1407 
 1408     def test_serializers(self):
 1409         """
 1410         Test that serializers have: []
 1411         """
 1412         self._verify_globals(salt.loader.serializers(self.master_opts))
 1413 
 1414     @slowTest
 1415     def test_states(self):
 1416         """
 1417         Test that states have:
 1418             - __pillar__
 1419             - __salt__
 1420             - __opts__
 1421             - __grains__
 1422             - __context__
 1423         """
 1424         opts = salt.config.minion_config(None)
 1425         opts["grains"] = salt.loader.grains(opts)
 1426         utils = salt.loader.utils(opts)
 1427         proxy = salt.loader.proxy(opts)
 1428         funcs = salt.loader.minion_mods(opts, utils=utils, proxy=proxy)
 1429         self._verify_globals(salt.loader.states(opts, funcs, utils, {}, proxy=proxy))
 1430 
 1431     def test_renderers(self):
 1432         """
 1433         Test that renderers have:
 1434             - __salt__    # Execution functions (i.e. __salt__['test.echo']('foo'))
 1435             - __grains__  # Grains (i.e. __grains__['os'])
 1436             - __pillar__  # Pillar data (i.e. __pillar__['foo'])
 1437             - __opts__    # Minion configuration options
 1438             - __context__ # Context dict shared amongst all modules of the same type
 1439         """
 1440         self._verify_globals(salt.loader.render(self.master_opts, {}))
 1441 
 1442 
 1443 class RawModTest(TestCase):
 1444     """
 1445     Test the interface of raw_mod
 1446     """
 1447 
 1448     def setUp(self):
 1449         self.opts = salt.config.minion_config(None)
 1450 
 1451     def tearDown(self):
 1452         del self.opts
 1453 
 1454     @slowTest
 1455     def test_basic(self):
 1456         testmod = salt.loader.raw_mod(self.opts, "test", None)
 1457         for k, v in testmod.items():
 1458             self.assertEqual(k.split(".")[0], "test")
 1459 
 1460     def test_bad_name(self):
 1461         testmod = salt.loader.raw_mod(self.opts, "module_we_do_not_have", None)
 1462         self.assertEqual(testmod, {})
 1463 
 1464 
 1465 class NetworkUtilsTestCase(ModuleCase):
 1466     def test_is_private(self):
 1467         mod = salt.loader.raw_mod(self.minion_opts, "network", None)
 1468         self.assertTrue(mod["network.is_private"]("10.0.0.1"), True)
 1469 
 1470     def test_is_loopback(self):
 1471         mod = salt.loader.raw_mod(self.minion_opts, "network", None)
 1472         self.assertTrue(mod["network.is_loopback"]("127.0.0.1"), True)
 1473 
 1474 
 1475 class LazyLoaderOptimizationOrderTest(TestCase):
 1476     """
 1477     Test the optimization order priority in the loader (PY3)
 1478     """
 1479 
 1480     module_name = "lazyloadertest"
 1481     module_content = textwrap.dedent(
 1482         """\
 1483         # -*- coding: utf-8 -*-
 1484         from __future__ import absolute_import
 1485 
 1486         def test():
 1487             return True
 1488         """
 1489     )
 1490 
 1491     @classmethod
 1492     def setUpClass(cls):
 1493         cls.opts = salt.config.minion_config(None)
 1494         cls.opts["grains"] = salt.loader.grains(cls.opts)
 1495         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
 1496         cls.proxy = salt.loader.proxy(cls.opts)
 1497         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
 1498 
 1499     @classmethod
 1500     def tearDownClass(cls):
 1501         del cls.opts
 1502         del cls.funcs
 1503         del cls.utils
 1504         del cls.proxy
 1505 
 1506     def setUp(self):
 1507         # Setup the module
 1508         self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
 1509         self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True)
 1510         self.module_file = os.path.join(
 1511             self.module_dir, "{}.py".format(self.module_name)
 1512         )
 1513 
 1514     def tearDown(self):
 1515         try:
 1516             delattr(self, "loader")
 1517         except AttributeError:
 1518             pass
 1519 
 1520     def _get_loader(self, order=None):
 1521         opts = copy.deepcopy(self.opts)
 1522         if order is not None:
 1523             opts["optimization_order"] = order
 1524         # Return a loader
 1525         return salt.loader.LazyLoader(
 1526             [self.module_dir],
 1527             opts,
 1528             pack={
 1529                 "__utils__": self.utils,
 1530                 "__salt__": self.funcs,
 1531                 "__proxy__": self.proxy,
 1532             },
 1533             tag="module",
 1534         )
 1535 
 1536     def _get_module_filename(self):
 1537         # The act of referencing the loader entry forces the module to be
 1538         # loaded by the LazyDict.
 1539         mod_fullname = self.loader[next(iter(self.loader))].__module__
 1540         return sys.modules[mod_fullname].__file__
 1541 
 1542     def _expected(self, optimize=0):
 1543         return "lazyloadertest.cpython-{}{}{}.pyc".format(
 1544             sys.version_info[0],
 1545             sys.version_info[1],
 1546             "" if not optimize else ".opt-{}".format(optimize),
 1547         )
 1548 
 1549     def _write_module_file(self):
 1550         with salt.utils.files.fopen(self.module_file, "w") as fh:
 1551             fh.write(self.module_content)
 1552             fh.flush()
 1553             os.fsync(fh.fileno())
 1554 
 1555     def _byte_compile(self):
 1556         compileall.compile_file(self.module_file, quiet=1, optimize=0)
 1557         compileall.compile_file(self.module_file, quiet=1, optimize=1)
 1558         compileall.compile_file(self.module_file, quiet=1, optimize=2)
 1559 
 1560     def _test_optimization_order(self, order):
 1561         self._write_module_file()
 1562         self._byte_compile()
 1563 
 1564         # Clean up the original file so that we can be assured we're only
 1565         # loading the byte-compiled files(s).
 1566         os.remove(self.module_file)
 1567 
 1568         self.loader = self._get_loader(order)
 1569         filename = self._get_module_filename()
 1570         basename = os.path.basename(filename)
 1571         assert basename == self._expected(order[0]), basename
 1572 
 1573         # Remove the file and make a new loader. We should now load the
 1574         # byte-compiled file with an optimization level matching the 2nd
 1575         # element of the order list.
 1576         os.remove(filename)
 1577         self.loader = self._get_loader(order)
 1578         filename = self._get_module_filename()
 1579         basename = os.path.basename(filename)
 1580         assert basename == self._expected(order[1]), basename
 1581 
 1582         # Remove the file and make a new loader. We should now load the
 1583         # byte-compiled file with an optimization level matching the 3rd
 1584         # element of the order list.
 1585         os.remove(filename)
 1586         self.loader = self._get_loader(order)
 1587         filename = self._get_module_filename()
 1588         basename = os.path.basename(filename)
 1589         assert basename == self._expected(order[2]), basename
 1590 
 1591     def test_optimization_order(self):
 1592         """
 1593         Test the optimization_order config param
 1594         """
 1595         self._test_optimization_order([0, 1, 2])
 1596         self._test_optimization_order([0, 2, 1])
 1597         self._test_optimization_order([1, 2, 0])
 1598         self._test_optimization_order([1, 0, 2])
 1599         self._test_optimization_order([2, 0, 1])
 1600         self._test_optimization_order([2, 1, 0])
 1601 
 1602     def test_load_source_file(self):
 1603         """
 1604         Make sure that .py files are preferred over .pyc files
 1605         """
 1606         self._write_module_file()
 1607         self._byte_compile()
 1608         self.loader = self._get_loader()
 1609         filename = self._get_module_filename()
 1610         basename = os.path.basename(filename)
 1611         expected = "lazyloadertest.py"
 1612         assert basename == expected, basename
 1613 
 1614 
 1615 class LoaderLoadCachedGrainsTest(TestCase):
 1616     """
 1617     Test how the loader works with cached grains
 1618     """
 1619 
 1620     @classmethod
 1621     def setUpClass(cls):
 1622         cls.opts = salt.config.minion_config(None)
 1623         if not os.path.isdir(RUNTIME_VARS.TMP):
 1624             os.makedirs(RUNTIME_VARS.TMP)
 1625 
 1626     def setUp(self):
 1627         self.cache_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
 1628         self.addCleanup(shutil.rmtree, self.cache_dir, ignore_errors=True)
 1629 
 1630         self.opts["cachedir"] = self.cache_dir
 1631         self.opts["grains_cache"] = True
 1632         self.opts["grains"] = salt.loader.grains(self.opts)
 1633 
 1634     @slowTest
 1635     def test_osrelease_info_has_correct_type(self):
 1636         """
 1637         Make sure osrelease_info is tuple after caching
 1638         """
 1639         grains = salt.loader.grains(self.opts)
 1640         osrelease_info = grains["osrelease_info"]
 1641         assert isinstance(osrelease_info, tuple), osrelease_info
 1642 
 1643 
 1644 class LazyLoaderRefreshFileMappingTest(TestCase):
 1645     """
 1646     Test that _refresh_file_mapping is called using acquiring LazyLoader._lock
 1647     """
 1648 
 1649     @classmethod
 1650     def setUpClass(cls):
 1651         cls.opts = salt.config.minion_config(None)
 1652         cls.opts["grains"] = salt.loader.grains(cls.opts)
 1653         cls.utils = salt.loader.utils(copy.deepcopy(cls.opts))
 1654         cls.proxy = salt.loader.proxy(cls.opts)
 1655         cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy)
 1656 
 1657     def setUp(self):
 1658         class LazyLoaderMock(salt.loader.LazyLoader):
 1659             pass
 1660 
 1661         self.LOADER_CLASS = LazyLoaderMock
 1662 
 1663     def __init_loader(self):
 1664         return self.LOADER_CLASS(
 1665             salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"),
 1666             copy.deepcopy(self.opts),
 1667             tag="module",
 1668             pack={
 1669                 "__utils__": self.utils,
 1670                 "__salt__": self.funcs,
 1671                 "__proxy__": self.proxy,
 1672             },
 1673         )
 1674 
 1675     @classmethod
 1676     def tearDownClass(cls):
 1677         del cls.opts
 1678         del cls.utils
 1679         del cls.funcs
 1680         del cls.proxy
 1681 
 1682     def test_lazyloader_refresh_file_mapping_called_with_lock_at___init__(self):
 1683         func_mock = MagicMock()
 1684         lock_mock = MagicMock()
 1685         lock_mock.__enter__ = MagicMock()
 1686         self.LOADER_CLASS._refresh_file_mapping = func_mock
 1687         with patch("threading.RLock", MagicMock(return_value=lock_mock)):
 1688             loader = self.__init_loader()
 1689         lock_mock.__enter__.assert_called()
 1690         func_mock.assert_called()
 1691         assert len(func_mock.call_args_list) == len(lock_mock.__enter__.call_args_list)
 1692         del loader
 1693 
 1694     def test_lazyloader_zip_modules(self):
 1695         self.opts["enable_zip_modules"] = True
 1696         try:
 1697             loader = self.__init_loader()
 1698             assert ".zip" in loader.suffix_map
 1699             assert ".zip" in loader.suffix_order
 1700         finally:
 1701             self.opts["enable_zip_modules"] = False
 1702         loader = self.__init_loader()
 1703         assert ".zip" not in loader.suffix_map
 1704         assert ".zip" not in loader.suffix_order
 1705 
 1706     def test_lazyloader_pyx_modules(self):
 1707         self.opts["cython_enable"] = True
 1708         try:
 1709             loader = self.__init_loader()
 1710             # Don't assert if the current environment has no pyximport
 1711             if salt.loader.pyximport is not None:
 1712                 assert ".pyx" in loader.suffix_map
 1713                 assert ".pyx" in loader.suffix_order
 1714         finally:
 1715             self.opts["cython_enable"] = False
 1716         loader = self.__init_loader()
 1717         assert ".pyx" not in loader.suffix_map
 1718         assert ".pyx" not in loader.suffix_order