salt-3002.2.tar.gz: .../unit/test_loader.py

"Fossies" - the Fresh Open Source Software Archive

Member "salt-3002.2/tests/unit/test_loader.py" (18 Nov 2020, 54372 Bytes) of package /linux/misc/salt-3002.2.tar.gz:

As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "test_loader.py":

3002.1_vs_3002.2.

1 """ 2 unit.loader 3 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 5 Test Salt's loader 6 """ 7 8 import collections 9 import compileall 10 import copy 11 import imp 12 import inspect 13 import logging 14 import os 15 import shutil 16 import sys 17 import tempfile 18 import textwrap 19 20 import salt.config 21 import salt.loader 22 import salt.utils.files 23 import salt.utils.stringutils 24 from tests.support.case import ModuleCase 25 from tests.support.helpers import slowTest 26 from tests.support.mock import MagicMock, patch 27 from tests.support.runtests import RUNTIME_VARS 28 from tests.support.unit import TestCase 29 30 log = logging.getLogger(__name__) 31 32 33 def remove_bytecode(module_path): 34 paths = [module_path + "c"] 35 if hasattr(imp, "get_tag"): 36 modname, ext = os.path.splitext(module_path.split(os.sep)[-1]) 37 paths.append( 38 os.path.join( 39 os.path.dirname(module_path), 40 "__pycache__", 41 "{}.{}.pyc".format(modname, imp.get_tag()), 42 ) 43 ) 44 for path in paths: 45 if os.path.exists(path): 46 os.unlink(path) 47 48 49 loader_template = """ 50 import os 51 from salt.utils.decorators import depends 52 53 @depends('os') 54 def loaded(): 55 return True 56 57 @depends('non_existantmodulename') 58 def not_loaded(): 59 return True 60 """ 61 62 63 class LazyLoaderTest(TestCase): 64 """ 65 Test the loader 66 """ 67 68 module_name = "lazyloadertest" 69 70 @classmethod 71 def setUpClass(cls): 72 cls.opts = salt.config.minion_config(None) 73 cls.opts["grains"] = salt.loader.grains(cls.opts) 74 if not os.path.isdir(RUNTIME_VARS.TMP): 75 os.makedirs(RUNTIME_VARS.TMP) 76 cls.utils = salt.loader.utils(cls.opts) 77 cls.proxy = salt.loader.proxy(cls.opts) 78 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 79 80 def setUp(self): 81 # Setup the module 82 self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 83 self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True) 84 self.module_file = os.path.join( 85 self.module_dir, "{}.py".format(self.module_name) 86 ) 87 with salt.utils.files.fopen(self.module_file, "w") as fh: 88 fh.write(salt.utils.stringutils.to_str(loader_template)) 89 fh.flush() 90 os.fsync(fh.fileno()) 91 92 # Invoke the loader 93 self.loader = salt.loader.LazyLoader( 94 [self.module_dir], 95 copy.deepcopy(self.opts), 96 pack={ 97 "__utils__": self.utils, 98 "__salt__": self.funcs, 99 "__proxy__": self.proxy, 100 }, 101 tag="module", 102 ) 103 104 def tearDown(self): 105 del self.module_dir 106 del self.module_file 107 del self.loader 108 109 @classmethod 110 def tearDownClass(cls): 111 del cls.opts 112 del cls.funcs 113 del cls.utils 114 del cls.proxy 115 116 @slowTest 117 def test_depends(self): 118 """ 119 Test that the depends decorator works properly 120 """ 121 # Make sure depends correctly allowed a function to load. If this 122 # results in a KeyError, the decorator is broken. 123 self.assertTrue(inspect.isfunction(self.loader[self.module_name + ".loaded"])) 124 # Make sure depends correctly kept a function from loading 125 self.assertTrue(self.module_name + ".not_loaded" not in self.loader) 126 127 128 loader_template_module = """ 129 import my_utils 130 131 def run(): 132 return my_utils.run() 133 """ 134 135 loader_template_utils = """ 136 def run(): 137 return True 138 """ 139 140 141 class LazyLoaderUtilsTest(TestCase): 142 """ 143 Test the loader 144 """ 145 146 module_name = "lazyloaderutilstest" 147 utils_name = "my_utils" 148 149 @classmethod 150 def setUpClass(cls): 151 cls.opts = salt.config.minion_config(None) 152 cls.opts["grains"] = salt.loader.grains(cls.opts) 153 if not os.path.isdir(RUNTIME_VARS.TMP): 154 os.makedirs(RUNTIME_VARS.TMP) 155 156 def setUp(self): 157 # Setup the module 158 self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 159 self.module_file = os.path.join( 160 self.module_dir, "{}.py".format(self.module_name) 161 ) 162 with salt.utils.files.fopen(self.module_file, "w") as fh: 163 fh.write(salt.utils.stringutils.to_str(loader_template_module)) 164 fh.flush() 165 os.fsync(fh.fileno()) 166 167 self.utils_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 168 self.utils_file = os.path.join(self.utils_dir, "{}.py".format(self.utils_name)) 169 with salt.utils.files.fopen(self.utils_file, "w") as fh: 170 fh.write(salt.utils.stringutils.to_str(loader_template_utils)) 171 fh.flush() 172 os.fsync(fh.fileno()) 173 174 def tearDown(self): 175 shutil.rmtree(self.module_dir) 176 if os.path.isdir(self.module_dir): 177 shutil.rmtree(self.module_dir) 178 shutil.rmtree(self.utils_dir) 179 if os.path.isdir(self.utils_dir): 180 shutil.rmtree(self.utils_dir) 181 del self.module_dir 182 del self.module_file 183 del self.utils_dir 184 del self.utils_file 185 186 if self.module_name in sys.modules: 187 del sys.modules[self.module_name] 188 if self.utils_name in sys.modules: 189 del sys.modules[self.utils_name] 190 191 @classmethod 192 def tearDownClass(cls): 193 del cls.opts 194 195 def test_utils_found(self): 196 """ 197 Test that the extra module directory is available for imports 198 """ 199 loader = salt.loader.LazyLoader( 200 [self.module_dir], 201 copy.deepcopy(self.opts), 202 tag="module", 203 extra_module_dirs=[self.utils_dir], 204 ) 205 self.assertTrue(inspect.isfunction(loader[self.module_name + ".run"])) 206 self.assertTrue(loader[self.module_name + ".run"]()) 207 208 def test_utils_not_found(self): 209 """ 210 Test that the extra module directory is not available for imports 211 """ 212 loader = salt.loader.LazyLoader( 213 [self.module_dir], copy.deepcopy(self.opts), tag="module" 214 ) 215 self.assertTrue(self.module_name + ".run" not in loader) 216 217 218 class LazyLoaderVirtualEnabledTest(TestCase): 219 """ 220 Test the base loader of salt. 221 """ 222 223 @classmethod 224 def setUpClass(cls): 225 cls.opts = salt.config.minion_config(None) 226 cls.opts["disable_modules"] = ["pillar"] 227 cls.opts["grains"] = salt.loader.grains(cls.opts) 228 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 229 cls.proxy = salt.loader.proxy(cls.opts) 230 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 231 232 def setUp(self): 233 self.loader = salt.loader.LazyLoader( 234 salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"), 235 copy.deepcopy(self.opts), 236 pack={ 237 "__utils__": self.utils, 238 "__salt__": self.funcs, 239 "__proxy__": self.proxy, 240 }, 241 tag="module", 242 ) 243 244 def tearDown(self): 245 del self.loader 246 247 @classmethod 248 def tearDownClass(cls): 249 del cls.opts 250 del cls.funcs 251 del cls.utils 252 del cls.proxy 253 254 @slowTest 255 def test_basic(self): 256 """ 257 Ensure that it only loads stuff when needed 258 """ 259 # make sure it starts empty 260 self.assertEqual(self.loader._dict, {}) 261 # get something, and make sure its a func 262 self.assertTrue(inspect.isfunction(self.loader["test.ping"])) 263 264 # make sure we only loaded "test" functions 265 for key, val in self.loader._dict.items(): 266 self.assertEqual(key.split(".", 1)[0], "test") 267 268 # make sure the depends thing worked (double check of the depends testing, 269 # since the loader does the calling magically 270 self.assertFalse("test.missing_func" in self.loader._dict) 271 272 def test_badkey(self): 273 with self.assertRaises(KeyError): 274 self.loader[None] # pylint: disable=W0104 275 276 with self.assertRaises(KeyError): 277 self.loader[1] # pylint: disable=W0104 278 279 @slowTest 280 def test_disable(self): 281 self.assertNotIn("pillar.items", self.loader) 282 283 @slowTest 284 def test_len_load(self): 285 """ 286 Since LazyLoader is a MutableMapping, if someone asks for len() we have 287 to load all 288 """ 289 self.assertEqual(self.loader._dict, {}) 290 len(self.loader) # force a load all 291 self.assertNotEqual(self.loader._dict, {}) 292 293 @slowTest 294 def test_iter_load(self): 295 """ 296 Since LazyLoader is a MutableMapping, if someone asks to iterate we have 297 to load all 298 """ 299 self.assertEqual(self.loader._dict, {}) 300 # force a load all 301 for key, func in self.loader.items(): 302 break 303 self.assertNotEqual(self.loader._dict, {}) 304 305 def test_context(self): 306 """ 307 Make sure context is shared across modules 308 """ 309 # make sure it starts empty 310 self.assertEqual(self.loader._dict, {}) 311 # get something, and make sure its a func 312 func = self.loader["test.ping"] 313 with patch.dict(func.__globals__["__context__"], {"foo": "bar"}): 314 self.assertEqual( 315 self.loader["test.echo"].__globals__["__context__"]["foo"], "bar" 316 ) 317 self.assertEqual( 318 self.loader["grains.get"].__globals__["__context__"]["foo"], "bar" 319 ) 320 321 def test_globals(self): 322 func_globals = self.loader["test.ping"].__globals__ 323 self.assertEqual(func_globals["__grains__"], self.opts.get("grains", {})) 324 self.assertEqual(func_globals["__pillar__"], self.opts.get("pillar", {})) 325 # the opts passed into modules is at least a subset of the whole opts 326 for key, val in func_globals["__opts__"].items(): 327 if ( 328 key in salt.config.DEFAULT_MASTER_OPTS 329 and key not in salt.config.DEFAULT_MINION_OPTS 330 ): 331 # We loaded the minion opts, but somewhere in the code, the master options got pulled in 332 # Let's just not check for equality since the option won't even exist in the loaded 333 # minion options 334 continue 335 if ( 336 key not in salt.config.DEFAULT_MASTER_OPTS 337 and key not in salt.config.DEFAULT_MINION_OPTS 338 ): 339 # This isn't even a default configuration setting, lets carry on 340 continue 341 self.assertEqual(self.opts[key], val) 342 343 def test_pack(self): 344 self.loader.pack["__foo__"] = "bar" 345 func_globals = self.loader["test.ping"].__globals__ 346 self.assertEqual(func_globals["__foo__"], "bar") 347 348 @slowTest 349 def test_virtual(self): 350 self.assertNotIn("test_virtual.ping", self.loader) 351 352 353 class LazyLoaderVirtualDisabledTest(TestCase): 354 """ 355 Test the loader of salt without __virtual__ 356 """ 357 358 @classmethod 359 def setUpClass(cls): 360 cls.opts = salt.config.minion_config(None) 361 cls.opts["grains"] = salt.loader.grains(cls.opts) 362 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 363 cls.proxy = salt.loader.proxy(cls.opts) 364 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 365 366 def setUp(self): 367 self.loader = salt.loader.LazyLoader( 368 salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"), 369 copy.deepcopy(self.opts), 370 tag="module", 371 pack={ 372 "__utils__": self.utils, 373 "__salt__": self.funcs, 374 "__proxy__": self.proxy, 375 }, 376 virtual_enable=False, 377 ) 378 379 def tearDown(self): 380 del self.loader 381 382 @classmethod 383 def tearDownClass(cls): 384 del cls.opts 385 del cls.utils 386 del cls.funcs 387 del cls.proxy 388 389 @slowTest 390 def test_virtual(self): 391 self.assertTrue(inspect.isfunction(self.loader["test_virtual.ping"])) 392 393 394 class LazyLoaderWhitelistTest(TestCase): 395 """ 396 Test the loader of salt with a whitelist 397 """ 398 399 @classmethod 400 def setUpClass(cls): 401 cls.opts = salt.config.minion_config(None) 402 cls.opts["grains"] = salt.loader.grains(cls.opts) 403 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 404 cls.proxy = salt.loader.proxy(cls.opts) 405 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 406 407 def setUp(self): 408 self.loader = salt.loader.LazyLoader( 409 salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"), 410 copy.deepcopy(self.opts), 411 tag="module", 412 pack={ 413 "__utils__": self.utils, 414 "__salt__": self.funcs, 415 "__proxy__": self.proxy, 416 }, 417 whitelist=["test", "pillar"], 418 ) 419 420 def tearDown(self): 421 del self.loader 422 423 @classmethod 424 def tearDownClass(cls): 425 del cls.opts 426 del cls.funcs 427 del cls.utils 428 del cls.proxy 429 430 @slowTest 431 def test_whitelist(self): 432 self.assertTrue(inspect.isfunction(self.loader["test.ping"])) 433 self.assertTrue(inspect.isfunction(self.loader["pillar.get"])) 434 435 self.assertNotIn("grains.get", self.loader) 436 437 438 class LazyLoaderGrainsBlacklistTest(TestCase): 439 """ 440 Test the loader of grains with a blacklist 441 """ 442 443 def setUp(self): 444 self.opts = salt.config.minion_config(None) 445 446 def tearDown(self): 447 del self.opts 448 449 @slowTest 450 def test_whitelist(self): 451 opts = copy.deepcopy(self.opts) 452 opts["grains_blacklist"] = ["master", "os*", "ipv[46]"] 453 454 grains = salt.loader.grains(opts) 455 self.assertNotIn("master", grains) 456 self.assertNotIn("os", {g[:2] for g in list(grains)}) 457 self.assertNotIn("ipv4", grains) 458 self.assertNotIn("ipv6", grains) 459 460 461 class LazyLoaderSingleItem(TestCase): 462 """ 463 Test loading a single item via the _load() function 464 """ 465 466 @classmethod 467 def setUpClass(cls): 468 cls.opts = salt.config.minion_config(None) 469 cls.opts["grains"] = salt.loader.grains(cls.opts) 470 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 471 cls.proxy = salt.loader.proxy(cls.opts) 472 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 473 474 @classmethod 475 def tearDownClass(cls): 476 del cls.opts 477 del cls.funcs 478 del cls.utils 479 del cls.proxy 480 481 def setUp(self): 482 self.loader = salt.loader.LazyLoader( 483 salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"), 484 copy.deepcopy(self.opts), 485 pack={ 486 "__utils__": self.utils, 487 "__salt__": self.funcs, 488 "__proxy__": self.proxy, 489 }, 490 tag="module", 491 ) 492 493 def tearDown(self): 494 del self.loader 495 496 def test_single_item_no_dot(self): 497 """ 498 Checks that a KeyError is raised when the function key does not contain a '.' 499 """ 500 key = "testing_no_dot" 501 expected = "The key '{}' should contain a '.'".format(key) 502 with self.assertRaises(KeyError) as err: 503 inspect.isfunction(self.loader["testing_no_dot"]) 504 505 result = err.exception.args[0] 506 assert result == expected, result 507 508 509 module_template = """ 510 __load__ = ['test', 'test_alias'] 511 __func_alias__ = dict(test_alias='working_alias') 512 from salt.utils.decorators import depends 513 514 def test(): 515 return {count} 516 517 def test_alias(): 518 return True 519 520 def test2(): 521 return True 522 523 @depends('non_existantmodulename') 524 def test3(): 525 return True 526 527 @depends('non_existantmodulename', fallback_function=test) 528 def test4(): 529 return True 530 """ 531 532 533 class LazyLoaderReloadingTest(TestCase): 534 """ 535 Test the loader of salt with changing modules 536 """ 537 538 module_name = "loadertest" 539 module_key = "loadertest.test" 540 541 @classmethod 542 def setUpClass(cls): 543 cls.opts = salt.config.minion_config(None) 544 cls.opts["grains"] = salt.loader.grains(cls.opts) 545 if not os.path.isdir(RUNTIME_VARS.TMP): 546 os.makedirs(RUNTIME_VARS.TMP) 547 548 def setUp(self): 549 self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 550 self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) 551 552 self.count = 0 553 opts = copy.deepcopy(self.opts) 554 dirs = salt.loader._module_dirs(opts, "modules", "module") 555 dirs.append(self.tmp_dir) 556 self.utils = salt.loader.utils(opts) 557 self.proxy = salt.loader.proxy(opts) 558 self.minion_mods = salt.loader.minion_mods(opts) 559 self.loader = salt.loader.LazyLoader( 560 dirs, 561 opts, 562 tag="module", 563 pack={ 564 "__utils__": self.utils, 565 "__proxy__": self.proxy, 566 "__salt__": self.minion_mods, 567 }, 568 ) 569 570 def tearDown(self): 571 for attrname in ("tmp_dir", "utils", "proxy", "loader", "minion_mods", "utils"): 572 try: 573 delattr(self, attrname) 574 except AttributeError: 575 continue 576 577 @classmethod 578 def tearDownClass(cls): 579 del cls.opts 580 581 def update_module(self): 582 self.count += 1 583 with salt.utils.files.fopen(self.module_path, "wb") as fh: 584 fh.write( 585 salt.utils.stringutils.to_bytes( 586 module_template.format(count=self.count) 587 ) 588 ) 589 fh.flush() 590 os.fsync(fh.fileno()) # flush to disk 591 592 # pyc files don't like it when we change the original quickly 593 # since the header bytes only contain the timestamp (granularity of seconds) 594 # TODO: don't write them? Is *much* slower on re-load (~3x) 595 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 596 remove_bytecode(self.module_path) 597 598 def rm_module(self): 599 os.unlink(self.module_path) 600 remove_bytecode(self.module_path) 601 602 @property 603 def module_path(self): 604 return os.path.join(self.tmp_dir, "{}.py".format(self.module_name)) 605 606 @slowTest 607 def test_alias(self): 608 """ 609 Make sure that you can access alias-d modules 610 """ 611 # ensure it doesn't exist 612 self.assertNotIn(self.module_key, self.loader) 613 614 self.update_module() 615 self.assertNotIn("{}.test_alias".format(self.module_name), self.loader) 616 self.assertTrue( 617 inspect.isfunction(self.loader["{}.working_alias".format(self.module_name)]) 618 ) 619 620 @slowTest 621 def test_clear(self): 622 self.assertTrue(inspect.isfunction(self.loader["test.ping"])) 623 self.update_module() # write out out custom module 624 self.loader.clear() # clear the loader dict 625 626 # force a load of our module 627 self.assertTrue(inspect.isfunction(self.loader[self.module_key])) 628 629 # make sure we only loaded our custom module 630 # which means that we did correctly refresh the file mapping 631 for k, v in self.loader._dict.items(): 632 self.assertTrue(k.startswith(self.module_name)) 633 634 @slowTest 635 def test_load(self): 636 # ensure it doesn't exist 637 self.assertNotIn(self.module_key, self.loader) 638 639 self.update_module() 640 self.assertTrue(inspect.isfunction(self.loader[self.module_key])) 641 642 @slowTest 643 def test__load__(self): 644 """ 645 If a module specifies __load__ we should only load/expose those modules 646 """ 647 self.update_module() 648 649 # ensure it doesn't exist 650 self.assertNotIn(self.module_key + "2", self.loader) 651 652 @slowTest 653 def test__load__and_depends(self): 654 """ 655 If a module specifies __load__ we should only load/expose those modules 656 """ 657 self.update_module() 658 # ensure it doesn't exist 659 self.assertNotIn(self.module_key + "3", self.loader) 660 self.assertNotIn(self.module_key + "4", self.loader) 661 662 @slowTest 663 def test_reload(self): 664 # ensure it doesn't exist 665 self.assertNotIn(self.module_key, self.loader) 666 667 # make sure it updates correctly 668 for x in range(1, 3): 669 self.update_module() 670 self.loader.clear() 671 self.assertEqual(self.loader[self.module_key](), self.count) 672 673 self.rm_module() 674 # make sure that even if we remove the module, its still loaded until a clear 675 self.assertEqual(self.loader[self.module_key](), self.count) 676 self.loader.clear() 677 self.assertNotIn(self.module_key, self.loader) 678 679 def test_wrong_bytecode(self): 680 """ 681 Checks to make sure we don't even try to load .pyc files that are for a different Python 682 This should pass (the load should fail) all the time because we don't run Salt on Py 3.4 anymore 683 """ 684 test_module_name = "test_module.cpython-34" 685 filemap_save = copy.deepcopy(self.loader.file_mapping) 686 self.loader.file_mapping = { 687 test_module_name: ( 688 "/temp/path/does/not/matter/here/__pycache__/" 689 + test_module_name 690 + ".pyc", 691 ".pyc", 692 0, 693 ) 694 } 695 696 self.assertFalse(self.loader._load_module(test_module_name)) 697 698 self.loader.file_mapping = copy.deepcopy(filemap_save) 699 700 701 virtual_aliases = ("loadertest2", "loadertest3") 702 virtual_alias_module_template = """ 703 __virtual_aliases__ = {} 704 705 def test(): 706 return True 707 """.format( 708 virtual_aliases 709 ) 710 711 712 class LazyLoaderVirtualAliasTest(TestCase): 713 """ 714 Test the loader of salt with changing modules 715 """ 716 717 module_name = "loadertest" 718 719 @classmethod 720 def setUpClass(cls): 721 cls.opts = salt.config.minion_config(None) 722 cls.opts["grains"] = salt.loader.grains(cls.opts) 723 if not os.path.isdir(RUNTIME_VARS.TMP): 724 os.makedirs(RUNTIME_VARS.TMP) 725 726 def setUp(self): 727 self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 728 opts = copy.deepcopy(self.opts) 729 dirs = salt.loader._module_dirs(opts, "modules", "module") 730 dirs.append(self.tmp_dir) 731 self.utils = salt.loader.utils(opts) 732 self.proxy = salt.loader.proxy(opts) 733 self.minion_mods = salt.loader.minion_mods(opts) 734 self.loader = salt.loader.LazyLoader( 735 dirs, 736 opts, 737 tag="module", 738 pack={ 739 "__utils__": self.utils, 740 "__proxy__": self.proxy, 741 "__salt__": self.minion_mods, 742 }, 743 ) 744 745 def tearDown(self): 746 del self.tmp_dir 747 del self.utils 748 del self.proxy 749 del self.minion_mods 750 del self.loader 751 752 @classmethod 753 def tearDownClass(cls): 754 del cls.opts 755 756 def update_module(self): 757 with salt.utils.files.fopen(self.module_path, "wb") as fh: 758 fh.write(salt.utils.stringutils.to_bytes(virtual_alias_module_template)) 759 fh.flush() 760 os.fsync(fh.fileno()) # flush to disk 761 762 # pyc files don't like it when we change the original quickly 763 # since the header bytes only contain the timestamp (granularity of seconds) 764 # TODO: don't write them? Is *much* slower on re-load (~3x) 765 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 766 remove_bytecode(self.module_path) 767 768 @property 769 def module_path(self): 770 return os.path.join(self.tmp_dir, "{}.py".format(self.module_name)) 771 772 @slowTest 773 def test_virtual_alias(self): 774 """ 775 Test the __virtual_alias__ feature 776 """ 777 self.update_module() 778 779 mod_names = [self.module_name] + list(virtual_aliases) 780 for mod_name in mod_names: 781 func_name = ".".join((mod_name, "test")) 782 log.debug("Running %s (dict attribute)", func_name) 783 self.assertTrue(self.loader[func_name]()) 784 log.debug("Running %s (loader attribute)", func_name) 785 self.assertTrue(getattr(self.loader, mod_name).test()) 786 787 788 submodule_template = """ 789 from __future__ import absolute_import 790 791 import {0}.lib 792 793 def test(): 794 return ({count}, {0}.lib.test()) 795 """ 796 797 submodule_lib_template = """ 798 def test(): 799 return {count} 800 """ 801 802 803 class LazyLoaderSubmodReloadingTest(TestCase): 804 """ 805 Test the loader of salt with changing modules 806 """ 807 808 module_name = "loadertestsubmod" 809 module_key = "loadertestsubmod.test" 810 811 @classmethod 812 def setUpClass(cls): 813 cls.opts = salt.config.minion_config(None) 814 cls.opts["grains"] = salt.loader.grains(cls.opts) 815 if not os.path.isdir(RUNTIME_VARS.TMP): 816 os.makedirs(RUNTIME_VARS.TMP) 817 818 def setUp(self): 819 self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 820 self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) 821 os.makedirs(self.module_dir) 822 823 self.count = 0 824 self.lib_count = 0 825 826 opts = copy.deepcopy(self.opts) 827 dirs = salt.loader._module_dirs(opts, "modules", "module") 828 dirs.append(self.tmp_dir) 829 self.utils = salt.loader.utils(opts) 830 self.proxy = salt.loader.proxy(opts) 831 self.minion_mods = salt.loader.minion_mods(opts) 832 self.loader = salt.loader.LazyLoader( 833 dirs, 834 opts, 835 tag="module", 836 pack={ 837 "__utils__": self.utils, 838 "__proxy__": self.proxy, 839 "__salt__": self.minion_mods, 840 }, 841 ) 842 843 def tearDown(self): 844 del self.tmp_dir 845 del self.utils 846 del self.proxy 847 del self.minion_mods 848 del self.loader 849 850 @classmethod 851 def tearDownClass(cls): 852 del cls.opts 853 854 def update_module(self): 855 self.count += 1 856 with salt.utils.files.fopen(self.module_path, "wb") as fh: 857 fh.write( 858 salt.utils.stringutils.to_bytes( 859 submodule_template.format(self.module_name, count=self.count) 860 ) 861 ) 862 fh.flush() 863 os.fsync(fh.fileno()) # flush to disk 864 865 # pyc files don't like it when we change the original quickly 866 # since the header bytes only contain the timestamp (granularity of seconds) 867 # TODO: don't write them? Is *much* slower on re-load (~3x) 868 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 869 remove_bytecode(self.module_path) 870 871 def rm_module(self): 872 os.unlink(self.module_path) 873 remove_bytecode(self.module_path) 874 875 def update_lib(self): 876 self.lib_count += 1 877 for modname in list(sys.modules): 878 if modname.startswith(self.module_name): 879 del sys.modules[modname] 880 with salt.utils.files.fopen(self.lib_path, "wb") as fh: 881 fh.write( 882 salt.utils.stringutils.to_bytes( 883 submodule_lib_template.format(count=self.lib_count) 884 ) 885 ) 886 fh.flush() 887 os.fsync(fh.fileno()) # flush to disk 888 889 # pyc files don't like it when we change the original quickly 890 # since the header bytes only contain the timestamp (granularity of seconds) 891 # TODO: don't write them? Is *much* slower on re-load (~3x) 892 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 893 remove_bytecode(self.lib_path) 894 895 def rm_lib(self): 896 for modname in list(sys.modules): 897 if modname.startswith(self.module_name): 898 del sys.modules[modname] 899 os.unlink(self.lib_path) 900 remove_bytecode(self.lib_path) 901 902 @property 903 def module_dir(self): 904 return os.path.join(self.tmp_dir, self.module_name) 905 906 @property 907 def module_path(self): 908 return os.path.join(self.module_dir, "__init__.py") 909 910 @property 911 def lib_path(self): 912 return os.path.join(self.module_dir, "lib.py") 913 914 @slowTest 915 def test_basic(self): 916 # ensure it doesn't exist 917 self.assertNotIn(self.module_key, self.loader) 918 919 self.update_module() 920 self.update_lib() 921 self.loader.clear() 922 self.assertIn(self.module_key, self.loader) 923 924 @slowTest 925 def test_reload(self): 926 # ensure it doesn't exist 927 self.assertNotIn(self.module_key, self.loader) 928 929 # update both the module and the lib 930 for x in range(1, 3): 931 self.update_lib() 932 self.update_module() 933 self.loader.clear() 934 self.assertNotIn(self.module_key, self.loader._dict) 935 self.assertIn(self.module_key, self.loader) 936 self.assertEqual( 937 self.loader[self.module_key](), (self.count, self.lib_count) 938 ) 939 940 # update just the module 941 for x in range(1, 3): 942 self.update_module() 943 self.loader.clear() 944 self.assertNotIn(self.module_key, self.loader._dict) 945 self.assertIn(self.module_key, self.loader) 946 self.assertEqual( 947 self.loader[self.module_key](), (self.count, self.lib_count) 948 ) 949 950 # update just the lib 951 for x in range(1, 3): 952 self.update_lib() 953 self.loader.clear() 954 self.assertNotIn(self.module_key, self.loader._dict) 955 self.assertIn(self.module_key, self.loader) 956 self.assertEqual( 957 self.loader[self.module_key](), (self.count, self.lib_count) 958 ) 959 960 self.rm_module() 961 # make sure that even if we remove the module, its still loaded until a clear 962 self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) 963 self.loader.clear() 964 self.assertNotIn(self.module_key, self.loader) 965 966 @slowTest 967 def test_reload_missing_lib(self): 968 # ensure it doesn't exist 969 self.assertNotIn(self.module_key, self.loader) 970 971 # update both the module and the lib 972 self.update_module() 973 self.update_lib() 974 self.loader.clear() 975 self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) 976 977 # remove the lib, this means we should fail to load the module next time 978 self.rm_lib() 979 self.loader.clear() 980 self.assertNotIn(self.module_key, self.loader) 981 982 983 mod_template = """ 984 def test(): 985 return ({val}) 986 """ 987 988 989 class LazyLoaderModulePackageTest(TestCase): 990 """ 991 Test the loader of salt with changing modules 992 """ 993 994 module_name = "loadertestmodpkg" 995 module_key = "loadertestmodpkg.test" 996 997 @classmethod 998 def setUpClass(cls): 999 cls.opts = salt.config.minion_config(None) 1000 cls.opts["grains"] = salt.loader.grains(cls.opts) 1001 if not os.path.isdir(RUNTIME_VARS.TMP): 1002 os.makedirs(RUNTIME_VARS.TMP) 1003 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 1004 cls.proxy = salt.loader.proxy(cls.opts) 1005 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 1006 1007 def setUp(self): 1008 self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 1009 self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) 1010 1011 dirs = salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module") 1012 dirs.append(self.tmp_dir) 1013 self.loader = salt.loader.LazyLoader( 1014 dirs, 1015 copy.deepcopy(self.opts), 1016 pack={ 1017 "__utils__": self.utils, 1018 "__salt__": self.funcs, 1019 "__proxy__": self.proxy, 1020 }, 1021 tag="module", 1022 ) 1023 1024 def tearDown(self): 1025 del self.tmp_dir 1026 del self.loader 1027 1028 @classmethod 1029 def tearDownClass(cls): 1030 del cls.opts 1031 del cls.funcs 1032 del cls.utils 1033 del cls.proxy 1034 1035 def update_pyfile(self, pyfile, contents): 1036 dirname = os.path.dirname(pyfile) 1037 if not os.path.exists(dirname): 1038 os.makedirs(dirname) 1039 with salt.utils.files.fopen(pyfile, "wb") as fh: 1040 fh.write(salt.utils.stringutils.to_bytes(contents)) 1041 fh.flush() 1042 os.fsync(fh.fileno()) # flush to disk 1043 1044 # pyc files don't like it when we change the original quickly 1045 # since the header bytes only contain the timestamp (granularity of seconds) 1046 # TODO: don't write them? Is *much* slower on re-load (~3x) 1047 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 1048 remove_bytecode(pyfile) 1049 1050 def rm_pyfile(self, pyfile): 1051 os.unlink(pyfile) 1052 remove_bytecode(pyfile) 1053 1054 def update_module(self, relative_path, contents): 1055 self.update_pyfile(os.path.join(self.tmp_dir, relative_path), contents) 1056 1057 def rm_module(self, relative_path): 1058 self.rm_pyfile(os.path.join(self.tmp_dir, relative_path)) 1059 1060 @slowTest 1061 def test_module(self): 1062 # ensure it doesn't exist 1063 self.assertNotIn("foo", self.loader) 1064 self.assertNotIn("foo.test", self.loader) 1065 self.update_module("foo.py", mod_template.format(val=1)) 1066 self.loader.clear() 1067 self.assertIn("foo.test", self.loader) 1068 self.assertEqual(self.loader["foo.test"](), 1) 1069 1070 @slowTest 1071 def test_package(self): 1072 # ensure it doesn't exist 1073 self.assertNotIn("foo", self.loader) 1074 self.assertNotIn("foo.test", self.loader) 1075 self.update_module("foo/__init__.py", mod_template.format(val=2)) 1076 self.loader.clear() 1077 self.assertIn("foo.test", self.loader) 1078 self.assertEqual(self.loader["foo.test"](), 2) 1079 1080 @slowTest 1081 def test_module_package_collision(self): 1082 # ensure it doesn't exist 1083 self.assertNotIn("foo", self.loader) 1084 self.assertNotIn("foo.test", self.loader) 1085 self.update_module("foo.py", mod_template.format(val=3)) 1086 self.loader.clear() 1087 self.assertIn("foo.test", self.loader) 1088 self.assertEqual(self.loader["foo.test"](), 3) 1089 1090 self.update_module("foo/__init__.py", mod_template.format(val=4)) 1091 self.loader.clear() 1092 self.assertIn("foo.test", self.loader) 1093 self.assertEqual(self.loader["foo.test"](), 4) 1094 1095 1096 deep_init_base = """ 1097 from __future__ import absolute_import 1098 import {0}.top_lib 1099 import {0}.top_lib.mid_lib 1100 import {0}.top_lib.mid_lib.bot_lib 1101 1102 def top(): 1103 return {0}.top_lib.test() 1104 1105 def mid(): 1106 return {0}.top_lib.mid_lib.test() 1107 1108 def bot(): 1109 return {0}.top_lib.mid_lib.bot_lib.test() 1110 """ 1111 1112 1113 class LazyLoaderDeepSubmodReloadingTest(TestCase): 1114 module_name = "loadertestsubmoddeep" 1115 libs = ("top_lib", "mid_lib", "bot_lib") 1116 1117 @classmethod 1118 def setUpClass(cls): 1119 cls.opts = salt.config.minion_config(None) 1120 cls.opts["grains"] = salt.loader.grains(cls.opts) 1121 if not os.path.isdir(RUNTIME_VARS.TMP): 1122 os.makedirs(RUNTIME_VARS.TMP) 1123 1124 def setUp(self): 1125 self.tmp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 1126 self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) 1127 os.makedirs(self.module_dir) 1128 1129 self.lib_count = collections.defaultdict(int) # mapping of path -> count 1130 1131 # bootstrap libs 1132 with salt.utils.files.fopen( 1133 os.path.join(self.module_dir, "__init__.py"), "w" 1134 ) as fh: 1135 # No .decode() needed here as deep_init_base is defined as str and 1136 # not bytes. 1137 fh.write( 1138 salt.utils.stringutils.to_str(deep_init_base.format(self.module_name)) 1139 ) 1140 fh.flush() 1141 os.fsync(fh.fileno()) # flush to disk 1142 1143 self.lib_paths = {} 1144 dir_path = self.module_dir 1145 for lib_name in self.libs: 1146 dir_path = os.path.join(dir_path, lib_name) 1147 self.lib_paths[lib_name] = dir_path 1148 os.makedirs(dir_path) 1149 self.update_lib(lib_name) 1150 1151 opts = copy.deepcopy(self.opts) 1152 dirs = salt.loader._module_dirs(opts, "modules", "module") 1153 dirs.append(self.tmp_dir) 1154 self.utils = salt.loader.utils(opts) 1155 self.proxy = salt.loader.proxy(opts) 1156 self.minion_mods = salt.loader.minion_mods(opts) 1157 self.loader = salt.loader.LazyLoader( 1158 dirs, 1159 copy.deepcopy(opts), 1160 tag="module", 1161 pack={ 1162 "__utils__": self.utils, 1163 "__proxy__": self.proxy, 1164 "__salt__": self.minion_mods, 1165 }, 1166 ) 1167 self.assertIn("{}.top".format(self.module_name), self.loader) 1168 1169 def tearDown(self): 1170 del self.tmp_dir 1171 del self.lib_paths 1172 del self.utils 1173 del self.proxy 1174 del self.minion_mods 1175 del self.loader 1176 del self.lib_count 1177 1178 @classmethod 1179 def tearDownClass(cls): 1180 del cls.opts 1181 1182 @property 1183 def module_dir(self): 1184 return os.path.join(self.tmp_dir, self.module_name) 1185 1186 def update_lib(self, lib_name): 1187 for modname in list(sys.modules): 1188 if modname.startswith(self.module_name): 1189 del sys.modules[modname] 1190 path = os.path.join(self.lib_paths[lib_name], "__init__.py") 1191 self.lib_count[lib_name] += 1 1192 with salt.utils.files.fopen(path, "wb") as fh: 1193 fh.write( 1194 salt.utils.stringutils.to_bytes( 1195 submodule_lib_template.format(count=self.lib_count[lib_name]) 1196 ) 1197 ) 1198 fh.flush() 1199 os.fsync(fh.fileno()) # flush to disk 1200 1201 # pyc files don't like it when we change the original quickly 1202 # since the header bytes only contain the timestamp (granularity of seconds) 1203 # TODO: don't write them? Is *much* slower on re-load (~3x) 1204 # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode 1205 remove_bytecode(path) 1206 1207 @slowTest 1208 def test_basic(self): 1209 self.assertIn("{}.top".format(self.module_name), self.loader) 1210 1211 def _verify_libs(self): 1212 for lib in self.libs: 1213 self.assertEqual( 1214 self.loader[ 1215 "{}.{}".format(self.module_name, lib.replace("_lib", "")) 1216 ](), 1217 self.lib_count[lib], 1218 ) 1219 1220 @slowTest 1221 def test_reload(self): 1222 """ 1223 Make sure that we can reload all libraries of arbitrary depth 1224 """ 1225 self._verify_libs() 1226 1227 # update them all 1228 for lib in self.libs: 1229 for x in range(5): 1230 self.update_lib(lib) 1231 self.loader.clear() 1232 self._verify_libs() 1233 1234 1235 class LoaderMultipleGlobalTest(ModuleCase): 1236 """ 1237 Tests when using multiple lazyloaders 1238 """ 1239 1240 def setUp(self): 1241 opts = salt.config.minion_config(None) 1242 self.loader1 = salt.loader.LazyLoader( 1243 salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"), 1244 copy.deepcopy(opts), 1245 pack={}, 1246 tag="module", 1247 loaded_base_name="salt.loader1", 1248 ) 1249 self.loader2 = salt.loader.LazyLoader( 1250 salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"), 1251 copy.deepcopy(opts), 1252 pack={}, 1253 tag="module", 1254 loaded_base_name="salt.loader2", 1255 ) 1256 1257 def tearDown(self): 1258 del self.loader1 1259 del self.loader2 1260 1261 def test_loader_globals(self): 1262 """ 1263 Test to ensure loaders do not edit 1264 each others loader's namespace 1265 """ 1266 self.loader1.pack["__foo__"] = "bar1" 1267 func1 = self.loader1["test.ping"] 1268 1269 self.loader2.pack["__foo__"] = "bar2" 1270 func2 = self.loader2["test.ping"] 1271 1272 assert func1.__globals__["__foo__"] == "bar1" 1273 assert func2.__globals__["__foo__"] == "bar2" 1274 1275 1276 class LoaderCleanupTest(ModuleCase): 1277 """ 1278 Tests the loader cleanup procedures 1279 """ 1280 1281 def setUp(self): 1282 opts = salt.config.minion_config(None) 1283 self.loader1 = salt.loader.LazyLoader( 1284 salt.loader._module_dirs(copy.deepcopy(opts), "modules", "module"), 1285 copy.deepcopy(opts), 1286 pack={}, 1287 tag="module", 1288 loaded_base_name="salt.test", 1289 ) 1290 1291 def tearDown(self): 1292 del self.loader1 1293 1294 def test_loader_clean_modules(self): 1295 loaded_base_name = self.loader1.loaded_base_name 1296 self.loader1.clean_modules() 1297 1298 for name in list(sys.modules): 1299 if name.startswith(loaded_base_name): 1300 self.fail( 1301 "Found a real module reference in sys.modules matching {!r}".format( 1302 loaded_base_name 1303 ) 1304 ) 1305 break 1306 1307 1308 class LoaderGlobalsTest(ModuleCase): 1309 """ 1310 Test all of the globals that the loader is responsible for adding to modules 1311 1312 This shouldn't be done here, but should rather be done per module type (in the cases where they are used) 1313 so they can check ALL globals that they have (or should have) access to. 1314 1315 This is intended as a shorter term way of testing these so we don't break the loader 1316 """ 1317 1318 def _verify_globals(self, mod_dict): 1319 """ 1320 Verify that the globals listed in the doc string (from the test) are in these modules 1321 """ 1322 # find the globals 1323 global_vars = {} 1324 for val in mod_dict.values(): 1325 # only find salty globals 1326 if val.__module__.startswith("salt.loaded"): 1327 if hasattr(val, "__globals__"): 1328 if hasattr(val, "__wrapped__") or "__wrapped__" in val.__globals__: 1329 global_vars[val.__module__] = sys.modules[ 1330 val.__module__ 1331 ].__dict__ 1332 else: 1333 global_vars[val.__module__] = val.__globals__ 1334 1335 # if we couldn't find any, then we have no modules -- so something is broken 1336 self.assertNotEqual(global_vars, {}, msg="No modules were loaded.") 1337 1338 # get the names of the globals you should have 1339 func_name = inspect.stack()[1][3] 1340 names = next( 1341 iter(salt.utils.yaml.safe_load(getattr(self, func_name).__doc__).values()) 1342 ) 1343 1344 # Now, test each module! 1345 for item in global_vars.values(): 1346 for name in names: 1347 self.assertIn(name, list(item.keys())) 1348 1349 def test_auth(self): 1350 """ 1351 Test that auth mods have: 1352 - __pillar__ 1353 - __grains__ 1354 - __salt__ 1355 - __context__ 1356 """ 1357 self._verify_globals(salt.loader.auth(self.master_opts)) 1358 1359 def test_runners(self): 1360 """ 1361 Test that runners have: 1362 - __pillar__ 1363 - __salt__ 1364 - __opts__ 1365 - __grains__ 1366 - __context__ 1367 """ 1368 self._verify_globals(salt.loader.runner(self.master_opts)) 1369 1370 def test_returners(self): 1371 """ 1372 Test that returners have: 1373 - __salt__ 1374 - __opts__ 1375 - __pillar__ 1376 - __grains__ 1377 - __context__ 1378 """ 1379 self._verify_globals(salt.loader.returners(self.master_opts, {})) 1380 1381 def test_pillars(self): 1382 """ 1383 Test that pillars have: 1384 - __salt__ 1385 - __opts__ 1386 - __pillar__ 1387 - __grains__ 1388 - __context__ 1389 """ 1390 self._verify_globals(salt.loader.pillars(self.master_opts, {})) 1391 1392 def test_tops(self): 1393 """ 1394 Test that tops have: [] 1395 """ 1396 self._verify_globals(salt.loader.tops(self.master_opts)) 1397 1398 def test_outputters(self): 1399 """ 1400 Test that outputters have: 1401 - __opts__ 1402 - __pillar__ 1403 - __grains__ 1404 - __context__ 1405 """ 1406 self._verify_globals(salt.loader.outputters(self.master_opts)) 1407 1408 def test_serializers(self): 1409 """ 1410 Test that serializers have: [] 1411 """ 1412 self._verify_globals(salt.loader.serializers(self.master_opts)) 1413 1414 @slowTest 1415 def test_states(self): 1416 """ 1417 Test that states have: 1418 - __pillar__ 1419 - __salt__ 1420 - __opts__ 1421 - __grains__ 1422 - __context__ 1423 """ 1424 opts = salt.config.minion_config(None) 1425 opts["grains"] = salt.loader.grains(opts) 1426 utils = salt.loader.utils(opts) 1427 proxy = salt.loader.proxy(opts) 1428 funcs = salt.loader.minion_mods(opts, utils=utils, proxy=proxy) 1429 self._verify_globals(salt.loader.states(opts, funcs, utils, {}, proxy=proxy)) 1430 1431 def test_renderers(self): 1432 """ 1433 Test that renderers have: 1434 - __salt__ # Execution functions (i.e. __salt__['test.echo']('foo')) 1435 - __grains__ # Grains (i.e. __grains__['os']) 1436 - __pillar__ # Pillar data (i.e. __pillar__['foo']) 1437 - __opts__ # Minion configuration options 1438 - __context__ # Context dict shared amongst all modules of the same type 1439 """ 1440 self._verify_globals(salt.loader.render(self.master_opts, {})) 1441 1442 1443 class RawModTest(TestCase): 1444 """ 1445 Test the interface of raw_mod 1446 """ 1447 1448 def setUp(self): 1449 self.opts = salt.config.minion_config(None) 1450 1451 def tearDown(self): 1452 del self.opts 1453 1454 @slowTest 1455 def test_basic(self): 1456 testmod = salt.loader.raw_mod(self.opts, "test", None) 1457 for k, v in testmod.items(): 1458 self.assertEqual(k.split(".")[0], "test") 1459 1460 def test_bad_name(self): 1461 testmod = salt.loader.raw_mod(self.opts, "module_we_do_not_have", None) 1462 self.assertEqual(testmod, {}) 1463 1464 1465 class NetworkUtilsTestCase(ModuleCase): 1466 def test_is_private(self): 1467 mod = salt.loader.raw_mod(self.minion_opts, "network", None) 1468 self.assertTrue(mod["network.is_private"]("10.0.0.1"), True) 1469 1470 def test_is_loopback(self): 1471 mod = salt.loader.raw_mod(self.minion_opts, "network", None) 1472 self.assertTrue(mod["network.is_loopback"]("127.0.0.1"), True) 1473 1474 1475 class LazyLoaderOptimizationOrderTest(TestCase): 1476 """ 1477 Test the optimization order priority in the loader (PY3) 1478 """ 1479 1480 module_name = "lazyloadertest" 1481 module_content = textwrap.dedent( 1482 """\ 1483 # -*- coding: utf-8 -*- 1484 from __future__ import absolute_import 1485 1486 def test(): 1487 return True 1488 """ 1489 ) 1490 1491 @classmethod 1492 def setUpClass(cls): 1493 cls.opts = salt.config.minion_config(None) 1494 cls.opts["grains"] = salt.loader.grains(cls.opts) 1495 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 1496 cls.proxy = salt.loader.proxy(cls.opts) 1497 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 1498 1499 @classmethod 1500 def tearDownClass(cls): 1501 del cls.opts 1502 del cls.funcs 1503 del cls.utils 1504 del cls.proxy 1505 1506 def setUp(self): 1507 # Setup the module 1508 self.module_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 1509 self.addCleanup(shutil.rmtree, self.module_dir, ignore_errors=True) 1510 self.module_file = os.path.join( 1511 self.module_dir, "{}.py".format(self.module_name) 1512 ) 1513 1514 def tearDown(self): 1515 try: 1516 delattr(self, "loader") 1517 except AttributeError: 1518 pass 1519 1520 def _get_loader(self, order=None): 1521 opts = copy.deepcopy(self.opts) 1522 if order is not None: 1523 opts["optimization_order"] = order 1524 # Return a loader 1525 return salt.loader.LazyLoader( 1526 [self.module_dir], 1527 opts, 1528 pack={ 1529 "__utils__": self.utils, 1530 "__salt__": self.funcs, 1531 "__proxy__": self.proxy, 1532 }, 1533 tag="module", 1534 ) 1535 1536 def _get_module_filename(self): 1537 # The act of referencing the loader entry forces the module to be 1538 # loaded by the LazyDict. 1539 mod_fullname = self.loader[next(iter(self.loader))].__module__ 1540 return sys.modules[mod_fullname].__file__ 1541 1542 def _expected(self, optimize=0): 1543 return "lazyloadertest.cpython-{}{}{}.pyc".format( 1544 sys.version_info[0], 1545 sys.version_info[1], 1546 "" if not optimize else ".opt-{}".format(optimize), 1547 ) 1548 1549 def _write_module_file(self): 1550 with salt.utils.files.fopen(self.module_file, "w") as fh: 1551 fh.write(self.module_content) 1552 fh.flush() 1553 os.fsync(fh.fileno()) 1554 1555 def _byte_compile(self): 1556 compileall.compile_file(self.module_file, quiet=1, optimize=0) 1557 compileall.compile_file(self.module_file, quiet=1, optimize=1) 1558 compileall.compile_file(self.module_file, quiet=1, optimize=2) 1559 1560 def _test_optimization_order(self, order): 1561 self._write_module_file() 1562 self._byte_compile() 1563 1564 # Clean up the original file so that we can be assured we're only 1565 # loading the byte-compiled files(s). 1566 os.remove(self.module_file) 1567 1568 self.loader = self._get_loader(order) 1569 filename = self._get_module_filename() 1570 basename = os.path.basename(filename) 1571 assert basename == self._expected(order[0]), basename 1572 1573 # Remove the file and make a new loader. We should now load the 1574 # byte-compiled file with an optimization level matching the 2nd 1575 # element of the order list. 1576 os.remove(filename) 1577 self.loader = self._get_loader(order) 1578 filename = self._get_module_filename() 1579 basename = os.path.basename(filename) 1580 assert basename == self._expected(order[1]), basename 1581 1582 # Remove the file and make a new loader. We should now load the 1583 # byte-compiled file with an optimization level matching the 3rd 1584 # element of the order list. 1585 os.remove(filename) 1586 self.loader = self._get_loader(order) 1587 filename = self._get_module_filename() 1588 basename = os.path.basename(filename) 1589 assert basename == self._expected(order[2]), basename 1590 1591 def test_optimization_order(self): 1592 """ 1593 Test the optimization_order config param 1594 """ 1595 self._test_optimization_order([0, 1, 2]) 1596 self._test_optimization_order([0, 2, 1]) 1597 self._test_optimization_order([1, 2, 0]) 1598 self._test_optimization_order([1, 0, 2]) 1599 self._test_optimization_order([2, 0, 1]) 1600 self._test_optimization_order([2, 1, 0]) 1601 1602 def test_load_source_file(self): 1603 """ 1604 Make sure that .py files are preferred over .pyc files 1605 """ 1606 self._write_module_file() 1607 self._byte_compile() 1608 self.loader = self._get_loader() 1609 filename = self._get_module_filename() 1610 basename = os.path.basename(filename) 1611 expected = "lazyloadertest.py" 1612 assert basename == expected, basename 1613 1614 1615 class LoaderLoadCachedGrainsTest(TestCase): 1616 """ 1617 Test how the loader works with cached grains 1618 """ 1619 1620 @classmethod 1621 def setUpClass(cls): 1622 cls.opts = salt.config.minion_config(None) 1623 if not os.path.isdir(RUNTIME_VARS.TMP): 1624 os.makedirs(RUNTIME_VARS.TMP) 1625 1626 def setUp(self): 1627 self.cache_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) 1628 self.addCleanup(shutil.rmtree, self.cache_dir, ignore_errors=True) 1629 1630 self.opts["cachedir"] = self.cache_dir 1631 self.opts["grains_cache"] = True 1632 self.opts["grains"] = salt.loader.grains(self.opts) 1633 1634 @slowTest 1635 def test_osrelease_info_has_correct_type(self): 1636 """ 1637 Make sure osrelease_info is tuple after caching 1638 """ 1639 grains = salt.loader.grains(self.opts) 1640 osrelease_info = grains["osrelease_info"] 1641 assert isinstance(osrelease_info, tuple), osrelease_info 1642 1643 1644 class LazyLoaderRefreshFileMappingTest(TestCase): 1645 """ 1646 Test that _refresh_file_mapping is called using acquiring LazyLoader._lock 1647 """ 1648 1649 @classmethod 1650 def setUpClass(cls): 1651 cls.opts = salt.config.minion_config(None) 1652 cls.opts["grains"] = salt.loader.grains(cls.opts) 1653 cls.utils = salt.loader.utils(copy.deepcopy(cls.opts)) 1654 cls.proxy = salt.loader.proxy(cls.opts) 1655 cls.funcs = salt.loader.minion_mods(cls.opts, utils=cls.utils, proxy=cls.proxy) 1656 1657 def setUp(self): 1658 class LazyLoaderMock(salt.loader.LazyLoader): 1659 pass 1660 1661 self.LOADER_CLASS = LazyLoaderMock 1662 1663 def __init_loader(self): 1664 return self.LOADER_CLASS( 1665 salt.loader._module_dirs(copy.deepcopy(self.opts), "modules", "module"), 1666 copy.deepcopy(self.opts), 1667 tag="module", 1668 pack={ 1669 "__utils__": self.utils, 1670 "__salt__": self.funcs, 1671 "__proxy__": self.proxy, 1672 }, 1673 ) 1674 1675 @classmethod 1676 def tearDownClass(cls): 1677 del cls.opts 1678 del cls.utils 1679 del cls.funcs 1680 del cls.proxy 1681 1682 def test_lazyloader_refresh_file_mapping_called_with_lock_at___init__(self): 1683 func_mock = MagicMock() 1684 lock_mock = MagicMock() 1685 lock_mock.__enter__ = MagicMock() 1686 self.LOADER_CLASS._refresh_file_mapping = func_mock 1687 with patch("threading.RLock", MagicMock(return_value=lock_mock)): 1688 loader = self.__init_loader() 1689 lock_mock.__enter__.assert_called() 1690 func_mock.assert_called() 1691 assert len(func_mock.call_args_list) == len(lock_mock.__enter__.call_args_list) 1692 del loader 1693 1694 def test_lazyloader_zip_modules(self): 1695 self.opts["enable_zip_modules"] = True 1696 try: 1697 loader = self.__init_loader() 1698 assert ".zip" in loader.suffix_map 1699 assert ".zip" in loader.suffix_order 1700 finally: 1701 self.opts["enable_zip_modules"] = False 1702 loader = self.__init_loader() 1703 assert ".zip" not in loader.suffix_map 1704 assert ".zip" not in loader.suffix_order 1705 1706 def test_lazyloader_pyx_modules(self): 1707 self.opts["cython_enable"] = True 1708 try: 1709 loader = self.__init_loader() 1710 # Don't assert if the current environment has no pyximport 1711 if salt.loader.pyximport is not None: 1712 assert ".pyx" in loader.suffix_map 1713 assert ".pyx" in loader.suffix_order 1714 finally: 1715 self.opts["cython_enable"] = False 1716 loader = self.__init_loader() 1717 assert ".pyx" not in loader.suffix_map 1718 assert ".pyx" not in loader.suffix_order