"Fossies" - the Fresh Open Source Software Archive 
Member "salt-3002.2/salt/loader.py" (18 Nov 2020, 74411 Bytes) of package /linux/misc/salt-3002.2.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "loader.py" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
3002.1_vs_3002.2.
1 """
2 The Salt loader is the core to Salt's plugin system, the loader scans
3 directories for python loadable code and organizes the code into the
4 plugin interfaces used by Salt.
5 """
6
7 import functools
8 import importlib.machinery # pylint: disable=no-name-in-module,import-error
9 import importlib.util # pylint: disable=no-name-in-module,import-error
10 import inspect
11 import logging
12 import os
13 import re
14 import sys
15 import tempfile
16 import threading
17 import time
18 import traceback
19 import types
20 from collections.abc import MutableMapping
21 from zipimport import zipimporter
22
23 import salt.config
24 import salt.defaults.events
25 import salt.defaults.exitcodes
26 import salt.syspaths
27 import salt.utils.args
28 import salt.utils.context
29 import salt.utils.data
30 import salt.utils.dictupdate
31 import salt.utils.event
32 import salt.utils.files
33 import salt.utils.lazy
34 import salt.utils.odict
35 import salt.utils.platform
36 import salt.utils.stringutils
37 import salt.utils.versions
38 from salt.exceptions import LoaderError
39 from salt.ext import six
40 from salt.ext.six.moves import reload_module
41 from salt.template import check_render_pipe_str
42 from salt.utils.decorators import Depends
43
44 try:
45 import pkg_resources
46
47 HAS_PKG_RESOURCES = True
48 except ImportError:
49 HAS_PKG_RESOURCES = False
50
51 log = logging.getLogger(__name__)
52
53 SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR)
54 LOADED_BASE_NAME = "salt.loaded"
55
56 # pylint: disable=no-member
57 MODULE_KIND_SOURCE = 1
58 MODULE_KIND_COMPILED = 2
59 MODULE_KIND_EXTENSION = 3
60 MODULE_KIND_PKG_DIRECTORY = 5
61 SUFFIXES = []
62 for suffix in importlib.machinery.EXTENSION_SUFFIXES:
63 SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION))
64 for suffix in importlib.machinery.SOURCE_SUFFIXES:
65 SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE))
66 for suffix in importlib.machinery.BYTECODE_SUFFIXES:
67 SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED))
68 MODULE_KIND_MAP = {
69 MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader,
70 MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader,
71 MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader,
72 }
73 # pylint: enable=no-member
74
75 PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_info[:2]))
76
77 # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
78 # which simplifies code readability, it adds some unsupported functions into
79 # the driver's module scope.
80 # We list un-supported functions here. These will be removed from the loaded.
81 # TODO: remove the need for this cross-module code. Maybe use NotImplemented
82 LIBCLOUD_FUNCS_NOT_SUPPORTED = (
83 "parallels.avail_sizes",
84 "parallels.avail_locations",
85 "proxmox.avail_sizes",
86 )
87
88 # Will be set to pyximport module at runtime if cython is enabled in config.
89 pyximport = None
90
91
92 def static_loader(
93 opts,
94 ext_type,
95 tag,
96 pack=None,
97 int_type=None,
98 ext_dirs=True,
99 ext_type_dirs=None,
100 base_path=None,
101 filter_name=None,
102 ):
103 funcs = LazyLoader(
104 _module_dirs(
105 opts, ext_type, tag, int_type, ext_dirs, ext_type_dirs, base_path,
106 ),
107 opts,
108 tag=tag,
109 pack=pack,
110 )
111 ret = {}
112 funcs._load_all()
113 if filter_name:
114 funcs = FilterDictWrapper(funcs, filter_name)
115 for key in funcs:
116 ret[key] = funcs[key]
117 return ret
118
119
120 def _format_entrypoint_target(ep):
121 """
122 Makes a string describing the target of an EntryPoint object.
123
124 Base strongly on EntryPoint.__str__().
125 """
126 s = ep.module_name
127 if ep.attrs:
128 s += ":" + ".".join(ep.attrs)
129 return s
130
131
132 def _module_dirs(
133 opts,
134 ext_type,
135 tag=None,
136 int_type=None,
137 ext_dirs=True,
138 ext_type_dirs=None,
139 base_path=None,
140 ):
141 if tag is None:
142 tag = ext_type
143 sys_types = os.path.join(base_path or SALT_BASE_PATH, int_type or ext_type)
144 ext_types = os.path.join(opts["extension_modules"], ext_type)
145
146 ext_type_types = []
147 if ext_dirs:
148 if ext_type_dirs is None:
149 ext_type_dirs = "{}_dirs".format(tag)
150 if ext_type_dirs in opts:
151 ext_type_types.extend(opts[ext_type_dirs])
152 if HAS_PKG_RESOURCES and ext_type_dirs:
153 for entry_point in pkg_resources.iter_entry_points(
154 "salt.loader", ext_type_dirs
155 ):
156 try:
157 loaded_entry_point = entry_point.load()
158 for path in loaded_entry_point():
159 ext_type_types.append(path)
160 except Exception as exc: # pylint: disable=broad-except
161 log.error(
162 "Error getting module directories from %s: %s",
163 _format_entrypoint_target(entry_point),
164 exc,
165 )
166 log.debug(
167 "Full backtrace for module directories error", exc_info=True
168 )
169
170 cli_module_dirs = []
171 # The dirs can be any module dir, or a in-tree _{ext_type} dir
172 for _dir in opts.get("module_dirs", []):
173 # Prepend to the list to match cli argument ordering
174 maybe_dir = os.path.join(_dir, ext_type)
175 if os.path.isdir(maybe_dir):
176 cli_module_dirs.insert(0, maybe_dir)
177 continue
178
179 maybe_dir = os.path.join(_dir, "_{}".format(ext_type))
180 if os.path.isdir(maybe_dir):
181 cli_module_dirs.insert(0, maybe_dir)
182
183 return cli_module_dirs + ext_type_types + [ext_types, sys_types]
184
185
186 def minion_mods(
187 opts,
188 context=None,
189 utils=None,
190 whitelist=None,
191 initial_load=False,
192 loaded_base_name=None,
193 notify=False,
194 static_modules=None,
195 proxy=None,
196 ):
197 """
198 Load execution modules
199
200 Returns a dictionary of execution modules appropriate for the current
201 system by evaluating the __virtual__() function in each module.
202
203 :param dict opts: The Salt options dictionary
204
205 :param dict context: A Salt context that should be made present inside
206 generated modules in __context__
207
208 :param dict utils: Utility functions which should be made available to
209 Salt modules in __utils__. See `utils_dirs` in
210 salt.config for additional information about
211 configuration.
212
213 :param list whitelist: A list of modules which should be whitelisted.
214 :param bool initial_load: Deprecated flag! Unused.
215 :param str loaded_base_name: A string marker for the loaded base name.
216 :param bool notify: Flag indicating that an event should be fired upon
217 completion of module loading.
218
219 .. code-block:: python
220
221 import salt.config
222 import salt.loader
223
224 __opts__ = salt.config.minion_config('/etc/salt/minion')
225 __grains__ = salt.loader.grains(__opts__)
226 __opts__['grains'] = __grains__
227 __utils__ = salt.loader.utils(__opts__)
228 __salt__ = salt.loader.minion_mods(__opts__, utils=__utils__)
229 __salt__['test.ping']()
230 """
231 # TODO Publish documentation for module whitelisting
232 if not whitelist:
233 whitelist = opts.get("whitelist_modules", None)
234 ret = LazyLoader(
235 _module_dirs(opts, "modules", "module"),
236 opts,
237 tag="module",
238 pack={"__context__": context, "__utils__": utils, "__proxy__": proxy},
239 whitelist=whitelist,
240 loaded_base_name=loaded_base_name,
241 static_modules=static_modules,
242 extra_module_dirs=utils.module_dirs if utils else None,
243 )
244
245 ret.pack["__salt__"] = ret
246
247 # Load any provider overrides from the configuration file providers option
248 # Note: Providers can be pkg, service, user or group - not to be confused
249 # with cloud providers.
250 providers = opts.get("providers", False)
251 if providers and isinstance(providers, dict):
252 for mod in providers:
253 # sometimes providers opts is not to diverge modules but
254 # for other configuration
255 try:
256 funcs = raw_mod(opts, providers[mod], ret)
257 except TypeError:
258 break
259 else:
260 if funcs:
261 for func in funcs:
262 f_key = "{}{}".format(mod, func[func.rindex(".") :])
263 ret[f_key] = funcs[func]
264
265 if notify:
266 with salt.utils.event.get_event("minion", opts=opts, listen=False) as evt:
267 evt.fire_event(
268 {"complete": True}, tag=salt.defaults.events.MINION_MOD_REFRESH_COMPLETE
269 )
270
271 return ret
272
273
274 def raw_mod(opts, name, functions, mod="modules"):
275 """
276 Returns a single module loaded raw and bypassing the __virtual__ function
277
278 .. code-block:: python
279
280 import salt.config
281 import salt.loader
282
283 __opts__ = salt.config.minion_config('/etc/salt/minion')
284 testmod = salt.loader.raw_mod(__opts__, 'test', None)
285 testmod['test.ping']()
286 """
287 loader = LazyLoader(
288 _module_dirs(opts, mod, "module"),
289 opts,
290 tag="rawmodule",
291 virtual_enable=False,
292 pack={"__salt__": functions},
293 )
294 # if we don't have the module, return an empty dict
295 if name not in loader.file_mapping:
296 return {}
297
298 loader._load_module(name) # load a single module (the one passed in)
299 return dict(loader._dict) # return a copy of *just* the funcs for `name`
300
301
302 def metaproxy(opts, loaded_base_name=None):
303 """
304 Return functions used in the meta proxy
305 """
306
307 return LazyLoader(
308 _module_dirs(opts, "metaproxy"),
309 opts,
310 tag="metaproxy",
311 loaded_base_name=loaded_base_name,
312 )
313
314
315 def matchers(opts):
316 """
317 Return the matcher services plugins
318 """
319 return LazyLoader(_module_dirs(opts, "matchers"), opts, tag="matchers")
320
321
322 def engines(opts, functions, runners, utils, proxy=None):
323 """
324 Return the master services plugins
325 """
326 pack = {
327 "__salt__": functions,
328 "__runners__": runners,
329 "__proxy__": proxy,
330 "__utils__": utils,
331 }
332 return LazyLoader(
333 _module_dirs(opts, "engines"),
334 opts,
335 tag="engines",
336 pack=pack,
337 extra_module_dirs=utils.module_dirs if utils else None,
338 )
339
340
341 def proxy(
342 opts, functions=None, returners=None, whitelist=None, utils=None, context=None
343 ):
344 """
345 Returns the proxy module for this salt-proxy-minion
346 """
347 ret = LazyLoader(
348 _module_dirs(opts, "proxy"),
349 opts,
350 tag="proxy",
351 pack={
352 "__salt__": functions,
353 "__ret__": returners,
354 "__utils__": utils,
355 "__context__": context,
356 },
357 extra_module_dirs=utils.module_dirs if utils else None,
358 )
359
360 ret.pack["__proxy__"] = ret
361
362 return ret
363
364
365 def returners(opts, functions, whitelist=None, context=None, proxy=None):
366 """
367 Returns the returner modules
368 """
369 return LazyLoader(
370 _module_dirs(opts, "returners", "returner"),
371 opts,
372 tag="returner",
373 whitelist=whitelist,
374 pack={"__salt__": functions, "__context__": context, "__proxy__": proxy or {}},
375 )
376
377
378 def utils(opts, whitelist=None, context=None, proxy=proxy):
379 """
380 Returns the utility modules
381 """
382 return LazyLoader(
383 _module_dirs(opts, "utils", ext_type_dirs="utils_dirs"),
384 opts,
385 tag="utils",
386 whitelist=whitelist,
387 pack={"__context__": context, "__proxy__": proxy or {}},
388 )
389
390
391 def pillars(opts, functions, context=None):
392 """
393 Returns the pillars modules
394 """
395 _utils = utils(opts)
396 ret = LazyLoader(
397 _module_dirs(opts, "pillar"),
398 opts,
399 tag="pillar",
400 pack={"__salt__": functions, "__context__": context, "__utils__": _utils},
401 extra_module_dirs=_utils.module_dirs,
402 )
403 ret.pack["__ext_pillar__"] = ret
404 return FilterDictWrapper(ret, ".ext_pillar")
405
406
407 def tops(opts):
408 """
409 Returns the tops modules
410 """
411 if "master_tops" not in opts:
412 return {}
413 whitelist = list(opts["master_tops"].keys())
414 ret = LazyLoader(
415 _module_dirs(opts, "tops", "top"), opts, tag="top", whitelist=whitelist,
416 )
417 return FilterDictWrapper(ret, ".top")
418
419
420 def wheels(opts, whitelist=None, context=None):
421 """
422 Returns the wheels modules
423 """
424 if context is None:
425 context = {}
426 return LazyLoader(
427 _module_dirs(opts, "wheel"),
428 opts,
429 tag="wheel",
430 whitelist=whitelist,
431 pack={"__context__": context},
432 )
433
434
435 def outputters(opts):
436 """
437 Returns the outputters modules
438
439 :param dict opts: The Salt options dictionary
440 :returns: LazyLoader instance, with only outputters present in the keyspace
441 """
442 ret = LazyLoader(
443 _module_dirs(opts, "output", ext_type_dirs="outputter_dirs"),
444 opts,
445 tag="output",
446 )
447 wrapped_ret = FilterDictWrapper(ret, ".output")
448 # TODO: this name seems terrible... __salt__ should always be execution mods
449 ret.pack["__salt__"] = wrapped_ret
450 return wrapped_ret
451
452
453 def serializers(opts):
454 """
455 Returns the serializers modules
456 :param dict opts: The Salt options dictionary
457 :returns: LazyLoader instance, with only serializers present in the keyspace
458 """
459 return LazyLoader(_module_dirs(opts, "serializers"), opts, tag="serializers",)
460
461
462 def eauth_tokens(opts):
463 """
464 Returns the tokens modules
465 :param dict opts: The Salt options dictionary
466 :returns: LazyLoader instance, with only token backends present in the keyspace
467 """
468 return LazyLoader(_module_dirs(opts, "tokens"), opts, tag="tokens",)
469
470
471 def auth(opts, whitelist=None):
472 """
473 Returns the auth modules
474
475 :param dict opts: The Salt options dictionary
476 :returns: LazyLoader
477 """
478 return LazyLoader(
479 _module_dirs(opts, "auth"),
480 opts,
481 tag="auth",
482 whitelist=whitelist,
483 pack={"__salt__": minion_mods(opts)},
484 )
485
486
487 def fileserver(opts, backends):
488 """
489 Returns the file server modules
490 """
491 _utils = utils(opts)
492
493 if backends is not None:
494 if not isinstance(backends, list):
495 backends = [backends]
496 # Make sure that the VCS backends work either with git or gitfs, hg or
497 # hgfs, etc.
498 vcs_re = re.compile("^(git|svn|hg)")
499 fs_re = re.compile("fs$")
500 vcs = []
501 non_vcs = []
502 for back in [fs_re.sub("", x) for x in backends]:
503 if vcs_re.match(back):
504 vcs.extend((back, back + "fs"))
505 else:
506 non_vcs.append(back)
507 backends = vcs + non_vcs
508
509 return LazyLoader(
510 _module_dirs(opts, "fileserver"),
511 opts,
512 tag="fileserver",
513 whitelist=backends,
514 pack={"__utils__": _utils},
515 extra_module_dirs=_utils.module_dirs,
516 )
517
518
519 def roster(opts, runner=None, utils=None, whitelist=None):
520 """
521 Returns the roster modules
522 """
523 return LazyLoader(
524 _module_dirs(opts, "roster"),
525 opts,
526 tag="roster",
527 whitelist=whitelist,
528 pack={"__runner__": runner, "__utils__": utils},
529 extra_module_dirs=utils.module_dirs if utils else None,
530 )
531
532
533 def thorium(opts, functions, runners):
534 """
535 Load the thorium runtime modules
536 """
537 pack = {"__salt__": functions, "__runner__": runners, "__context__": {}}
538 ret = LazyLoader(_module_dirs(opts, "thorium"), opts, tag="thorium", pack=pack)
539 ret.pack["__thorium__"] = ret
540 return ret
541
542
543 def states(
544 opts, functions, utils, serializers, whitelist=None, proxy=None, context=None
545 ):
546 """
547 Returns the state modules
548
549 :param dict opts: The Salt options dictionary
550 :param dict functions: A dictionary of minion modules, with module names as
551 keys and funcs as values.
552
553 .. code-block:: python
554
555 import salt.config
556 import salt.loader
557
558 __opts__ = salt.config.minion_config('/etc/salt/minion')
559 statemods = salt.loader.states(__opts__, None, None)
560 """
561 if context is None:
562 context = {}
563
564 ret = LazyLoader(
565 _module_dirs(opts, "states"),
566 opts,
567 tag="states",
568 pack={"__salt__": functions, "__proxy__": proxy or {}},
569 whitelist=whitelist,
570 extra_module_dirs=utils.module_dirs if utils else None,
571 )
572 ret.pack["__states__"] = ret
573 ret.pack["__utils__"] = utils
574 ret.pack["__serializers__"] = serializers
575 ret.pack["__context__"] = context
576 return ret
577
578
579 def beacons(opts, functions, context=None, proxy=None):
580 """
581 Load the beacon modules
582
583 :param dict opts: The Salt options dictionary
584 :param dict functions: A dictionary of minion modules, with module names as
585 keys and funcs as values.
586 """
587 return LazyLoader(
588 _module_dirs(opts, "beacons"),
589 opts,
590 tag="beacons",
591 pack={"__context__": context, "__salt__": functions, "__proxy__": proxy or {}},
592 virtual_funcs=[],
593 )
594
595
596 def log_handlers(opts):
597 """
598 Returns the custom logging handler modules
599
600 :param dict opts: The Salt options dictionary
601 """
602 ret = LazyLoader(
603 _module_dirs(
604 opts,
605 "log_handlers",
606 int_type="handlers",
607 base_path=os.path.join(SALT_BASE_PATH, "log"),
608 ),
609 opts,
610 tag="log_handlers",
611 )
612 return FilterDictWrapper(ret, ".setup_handlers")
613
614
615 def ssh_wrapper(opts, functions=None, context=None):
616 """
617 Returns the custom logging handler modules
618 """
619 return LazyLoader(
620 _module_dirs(
621 opts,
622 "wrapper",
623 base_path=os.path.join(SALT_BASE_PATH, os.path.join("client", "ssh")),
624 ),
625 opts,
626 tag="wrapper",
627 pack={
628 "__salt__": functions,
629 "__grains__": opts.get("grains", {}),
630 "__pillar__": opts.get("pillar", {}),
631 "__context__": context,
632 },
633 )
634
635
636 def render(opts, functions, states=None, proxy=None, context=None):
637 """
638 Returns the render modules
639 """
640 if context is None:
641 context = {}
642
643 pack = {
644 "__salt__": functions,
645 "__grains__": opts.get("grains", {}),
646 "__context__": context,
647 }
648
649 if states:
650 pack["__states__"] = states
651 pack["__proxy__"] = proxy or {}
652 ret = LazyLoader(
653 _module_dirs(opts, "renderers", "render", ext_type_dirs="render_dirs",),
654 opts,
655 tag="render",
656 pack=pack,
657 )
658 rend = FilterDictWrapper(ret, ".render")
659
660 if not check_render_pipe_str(
661 opts["renderer"], rend, opts["renderer_blacklist"], opts["renderer_whitelist"]
662 ):
663 err = (
664 "The renderer {} is unavailable, this error is often because "
665 "the needed software is unavailable".format(opts["renderer"])
666 )
667 log.critical(err)
668 raise LoaderError(err)
669 return rend
670
671
672 def grain_funcs(opts, proxy=None, context=None):
673 """
674 Returns the grain functions
675
676 .. code-block:: python
677
678 import salt.config
679 import salt.loader
680
681 __opts__ = salt.config.minion_config('/etc/salt/minion')
682 grainfuncs = salt.loader.grain_funcs(__opts__)
683 """
684 _utils = utils(opts, proxy=proxy)
685 pack = {"__utils__": utils(opts, proxy=proxy), "__context__": context}
686
687 ret = LazyLoader(
688 _module_dirs(opts, "grains", "grain", ext_type_dirs="grains_dirs",),
689 opts,
690 tag="grains",
691 extra_module_dirs=_utils.module_dirs,
692 pack=pack,
693 )
694 ret.pack["__utils__"] = _utils
695 return ret
696
697
698 def _format_cached_grains(cached_grains):
699 """
700 Returns cached grains with fixed types, like tuples.
701 """
702 if cached_grains.get("osrelease_info"):
703 osrelease_info = cached_grains["osrelease_info"]
704 if isinstance(osrelease_info, list):
705 cached_grains["osrelease_info"] = tuple(osrelease_info)
706 return cached_grains
707
708
709 def _load_cached_grains(opts, cfn):
710 """
711 Returns the grains cached in cfn, or None if the cache is too old or is
712 corrupted.
713 """
714 if not os.path.isfile(cfn):
715 log.debug("Grains cache file does not exist.")
716 return None
717
718 grains_cache_age = int(time.time() - os.path.getmtime(cfn))
719 if grains_cache_age > opts.get("grains_cache_expiration", 300):
720 log.debug(
721 "Grains cache last modified %s seconds ago and cache "
722 "expiration is set to %s. Grains cache expired. "
723 "Refreshing.",
724 grains_cache_age,
725 opts.get("grains_cache_expiration", 300),
726 )
727 return None
728
729 if opts.get("refresh_grains_cache", False):
730 log.debug("refresh_grains_cache requested, Refreshing.")
731 return None
732
733 log.debug("Retrieving grains from cache")
734 try:
735 serial = salt.payload.Serial(opts)
736 with salt.utils.files.fopen(cfn, "rb") as fp_:
737 cached_grains = salt.utils.data.decode(
738 serial.load(fp_), preserve_tuples=True
739 )
740 if not cached_grains:
741 log.debug("Cached grains are empty, cache might be corrupted. Refreshing.")
742 return None
743
744 return _format_cached_grains(cached_grains)
745 except OSError:
746 return None
747
748
749 def grains(opts, force_refresh=False, proxy=None, context=None):
750 """
751 Return the functions for the dynamic grains and the values for the static
752 grains.
753
754 Since grains are computed early in the startup process, grains functions
755 do not have __salt__ or __proxy__ available. At proxy-minion startup,
756 this function is called with the proxymodule LazyLoader object so grains
757 functions can communicate with their controlled device.
758
759 .. code-block:: python
760
761 import salt.config
762 import salt.loader
763
764 __opts__ = salt.config.minion_config('/etc/salt/minion')
765 __grains__ = salt.loader.grains(__opts__)
766 print __grains__['id']
767 """
768 # Need to re-import salt.config, somehow it got lost when a minion is starting
769 import salt.config
770
771 # if we have no grains, lets try loading from disk (TODO: move to decorator?)
772 cfn = os.path.join(opts["cachedir"], "grains.cache.p")
773 if not force_refresh and opts.get("grains_cache", False):
774 cached_grains = _load_cached_grains(opts, cfn)
775 if cached_grains:
776 return cached_grains
777 else:
778 log.debug("Grains refresh requested. Refreshing grains.")
779
780 if opts.get("skip_grains", False):
781 return {}
782 grains_deep_merge = opts.get("grains_deep_merge", False) is True
783 if "conf_file" in opts:
784 pre_opts = {}
785 pre_opts.update(
786 salt.config.load_config(
787 opts["conf_file"],
788 "SALT_MINION_CONFIG",
789 salt.config.DEFAULT_MINION_OPTS["conf_file"],
790 )
791 )
792 default_include = pre_opts.get("default_include", opts["default_include"])
793 include = pre_opts.get("include", [])
794 pre_opts.update(
795 salt.config.include_config(
796 default_include, opts["conf_file"], verbose=False
797 )
798 )
799 pre_opts.update(
800 salt.config.include_config(include, opts["conf_file"], verbose=True)
801 )
802 if "grains" in pre_opts:
803 opts["grains"] = pre_opts["grains"]
804 else:
805 opts["grains"] = {}
806 else:
807 opts["grains"] = {}
808
809 grains_data = {}
810 blist = opts.get("grains_blacklist", [])
811 funcs = grain_funcs(opts, proxy=proxy, context=context or {})
812 if force_refresh: # if we refresh, lets reload grain modules
813 funcs.clear()
814 # Run core grains
815 for key in funcs:
816 if not key.startswith("core."):
817 continue
818 log.trace("Loading %s grain", key)
819 ret = funcs[key]()
820 if not isinstance(ret, dict):
821 continue
822 if blist:
823 for key in list(ret):
824 for block in blist:
825 if salt.utils.stringutils.expr_match(key, block):
826 del ret[key]
827 log.trace("Filtering %s grain", key)
828 if not ret:
829 continue
830 if grains_deep_merge:
831 salt.utils.dictupdate.update(grains_data, ret)
832 else:
833 grains_data.update(ret)
834
835 # Run the rest of the grains
836 for key in funcs:
837 if key.startswith("core.") or key == "_errors":
838 continue
839 try:
840 # Grains are loaded too early to take advantage of the injected
841 # __proxy__ variable. Pass an instance of that LazyLoader
842 # here instead to grains functions if the grains functions take
843 # one parameter. Then the grains can have access to the
844 # proxymodule for retrieving information from the connected
845 # device.
846 log.trace("Loading %s grain", key)
847 parameters = salt.utils.args.get_function_argspec(funcs[key]).args
848 kwargs = {}
849 if "proxy" in parameters:
850 kwargs["proxy"] = proxy
851 if "grains" in parameters:
852 kwargs["grains"] = grains_data
853 ret = funcs[key](**kwargs)
854 except Exception: # pylint: disable=broad-except
855 if salt.utils.platform.is_proxy():
856 log.info(
857 "The following CRITICAL message may not be an error; the proxy may not be completely established yet."
858 )
859 log.critical(
860 "Failed to load grains defined in grain file %s in "
861 "function %s, error:\n",
862 key,
863 funcs[key],
864 exc_info=True,
865 )
866 continue
867 if not isinstance(ret, dict):
868 continue
869 if blist:
870 for key in list(ret):
871 for block in blist:
872 if salt.utils.stringutils.expr_match(key, block):
873 del ret[key]
874 log.trace("Filtering %s grain", key)
875 if not ret:
876 continue
877 if grains_deep_merge:
878 salt.utils.dictupdate.update(grains_data, ret)
879 else:
880 grains_data.update(ret)
881
882 if opts.get("proxy_merge_grains_in_module", True) and proxy:
883 try:
884 proxytype = proxy.opts["proxy"]["proxytype"]
885 if proxytype + ".grains" in proxy:
886 if (
887 proxytype + ".initialized" in proxy
888 and proxy[proxytype + ".initialized"]()
889 ):
890 try:
891 proxytype = proxy.opts["proxy"]["proxytype"]
892 ret = proxy[proxytype + ".grains"]()
893 if grains_deep_merge:
894 salt.utils.dictupdate.update(grains_data, ret)
895 else:
896 grains_data.update(ret)
897 except Exception: # pylint: disable=broad-except
898 log.critical(
899 "Failed to run proxy's grains function!", exc_info=True
900 )
901 except KeyError:
902 pass
903
904 grains_data.update(opts["grains"])
905 # Write cache if enabled
906 if opts.get("grains_cache", False):
907 with salt.utils.files.set_umask(0o077):
908 try:
909 if salt.utils.platform.is_windows():
910 # Late import
911 import salt.modules.cmdmod
912
913 # Make sure cache file isn't read-only
914 salt.modules.cmdmod._run_quiet('attrib -R "{}"'.format(cfn))
915 with salt.utils.files.fopen(cfn, "w+b") as fp_:
916 try:
917 serial = salt.payload.Serial(opts)
918 serial.dump(grains_data, fp_)
919 except TypeError as e:
920 log.error("Failed to serialize grains cache: %s", e)
921 raise # re-throw for cleanup
922 except Exception as e: # pylint: disable=broad-except
923 log.error("Unable to write to grains cache file %s: %s", cfn, e)
924 # Based on the original exception, the file may or may not have been
925 # created. If it was, we will remove it now, as the exception means
926 # the serialized data is not to be trusted, no matter what the
927 # exception is.
928 if os.path.isfile(cfn):
929 os.unlink(cfn)
930
931 if grains_deep_merge:
932 salt.utils.dictupdate.update(grains_data, opts["grains"])
933 else:
934 grains_data.update(opts["grains"])
935 return salt.utils.data.decode(grains_data, preserve_tuples=True)
936
937
938 # TODO: get rid of? Does anyone use this? You should use raw() instead
939 def call(fun, **kwargs):
940 """
941 Directly call a function inside a loader directory
942 """
943 args = kwargs.get("args", [])
944 dirs = kwargs.get("dirs", [])
945
946 funcs = LazyLoader(
947 [os.path.join(SALT_BASE_PATH, "modules")] + dirs,
948 None,
949 tag="modules",
950 virtual_enable=False,
951 )
952 return funcs[fun](*args)
953
954
955 def runner(opts, utils=None, context=None, whitelist=None):
956 """
957 Directly call a function inside a loader directory
958 """
959 if utils is None:
960 utils = {}
961 if context is None:
962 context = {}
963 ret = LazyLoader(
964 _module_dirs(opts, "runners", "runner", ext_type_dirs="runner_dirs"),
965 opts,
966 tag="runners",
967 pack={"__utils__": utils, "__context__": context},
968 whitelist=whitelist,
969 extra_module_dirs=utils.module_dirs if utils else None,
970 )
971 # TODO: change from __salt__ to something else, we overload __salt__ too much
972 ret.pack["__salt__"] = ret
973 return ret
974
975
976 def queues(opts):
977 """
978 Directly call a function inside a loader directory
979 """
980 return LazyLoader(
981 _module_dirs(opts, "queues", "queue", ext_type_dirs="queue_dirs"),
982 opts,
983 tag="queues",
984 )
985
986
987 def sdb(opts, functions=None, whitelist=None, utils=None):
988 """
989 Make a very small database call
990 """
991 if utils is None:
992 utils = {}
993
994 return LazyLoader(
995 _module_dirs(opts, "sdb"),
996 opts,
997 tag="sdb",
998 pack={
999 "__sdb__": functions,
1000 "__opts__": opts,
1001 "__utils__": utils,
1002 "__salt__": minion_mods(opts, utils=utils),
1003 },
1004 whitelist=whitelist,
1005 extra_module_dirs=utils.module_dirs if utils else None,
1006 )
1007
1008
1009 def pkgdb(opts):
1010 """
1011 Return modules for SPM's package database
1012
1013 .. versionadded:: 2015.8.0
1014 """
1015 return LazyLoader(
1016 _module_dirs(opts, "pkgdb", base_path=os.path.join(SALT_BASE_PATH, "spm")),
1017 opts,
1018 tag="pkgdb",
1019 )
1020
1021
1022 def pkgfiles(opts):
1023 """
1024 Return modules for SPM's file handling
1025
1026 .. versionadded:: 2015.8.0
1027 """
1028 return LazyLoader(
1029 _module_dirs(opts, "pkgfiles", base_path=os.path.join(SALT_BASE_PATH, "spm")),
1030 opts,
1031 tag="pkgfiles",
1032 )
1033
1034
1035 def clouds(opts):
1036 """
1037 Return the cloud functions
1038 """
1039 _utils = salt.loader.utils(opts)
1040 # Let's bring __active_provider_name__, defaulting to None, to all cloud
1041 # drivers. This will get temporarily updated/overridden with a context
1042 # manager when needed.
1043 functions = LazyLoader(
1044 _module_dirs(
1045 opts,
1046 "clouds",
1047 "cloud",
1048 base_path=os.path.join(SALT_BASE_PATH, "cloud"),
1049 int_type="clouds",
1050 ),
1051 opts,
1052 tag="clouds",
1053 pack={"__utils__": _utils, "__active_provider_name__": None},
1054 extra_module_dirs=_utils.module_dirs,
1055 )
1056 for funcname in LIBCLOUD_FUNCS_NOT_SUPPORTED:
1057 log.trace(
1058 "'%s' has been marked as not supported. Removing from the "
1059 "list of supported cloud functions",
1060 funcname,
1061 )
1062 functions.pop(funcname, None)
1063 return functions
1064
1065
1066 def netapi(opts):
1067 """
1068 Return the network api functions
1069 """
1070 return LazyLoader(_module_dirs(opts, "netapi"), opts, tag="netapi",)
1071
1072
1073 def executors(opts, functions=None, context=None, proxy=None):
1074 """
1075 Returns the executor modules
1076 """
1077 executors = LazyLoader(
1078 _module_dirs(opts, "executors", "executor"),
1079 opts,
1080 tag="executor",
1081 pack={
1082 "__salt__": functions,
1083 "__context__": context or {},
1084 "__proxy__": proxy or {},
1085 },
1086 )
1087 executors.pack["__executors__"] = executors
1088 return executors
1089
1090
1091 def cache(opts, serial):
1092 """
1093 Returns the returner modules
1094 """
1095 return LazyLoader(
1096 _module_dirs(opts, "cache", "cache"),
1097 opts,
1098 tag="cache",
1099 pack={"__opts__": opts, "__context__": {"serial": serial}},
1100 )
1101
1102
1103 def _generate_module(name):
1104 if name in sys.modules:
1105 return
1106
1107 code = "'''Salt loaded {} parent module'''".format(name.split(".")[-1])
1108 # ModuleType can't accept a unicode type on PY2
1109 module = types.ModuleType(str(name)) # future lint: disable=blacklisted-function
1110 exec(code, module.__dict__)
1111 sys.modules[name] = module
1112
1113
1114 def _mod_type(module_path):
1115 if module_path.startswith(SALT_BASE_PATH):
1116 return "int"
1117 return "ext"
1118
1119
1120 # TODO: move somewhere else?
1121 class FilterDictWrapper(MutableMapping):
1122 """
1123 Create a dict which wraps another dict with a specific key suffix on get
1124
1125 This is to replace "filter_load"
1126 """
1127
1128 def __init__(self, d, suffix):
1129 self._dict = d
1130 self.suffix = suffix
1131
1132 def __setitem__(self, key, val):
1133 self._dict[key] = val
1134
1135 def __delitem__(self, key):
1136 del self._dict[key]
1137
1138 def __getitem__(self, key):
1139 return self._dict[key + self.suffix]
1140
1141 def __len__(self):
1142 return len(self._dict)
1143
1144 def __iter__(self):
1145 for key in self._dict:
1146 if key.endswith(self.suffix):
1147 yield key.replace(self.suffix, "")
1148
1149
1150 class LazyLoader(salt.utils.lazy.LazyDict):
1151 """
1152 A pseduo-dictionary which has a set of keys which are the
1153 name of the module and function, delimited by a dot. When
1154 the value of the key is accessed, the function is then loaded
1155 from disk and into memory.
1156
1157 .. note::
1158
1159 Iterating over keys will cause all modules to be loaded.
1160
1161 :param list module_dirs: A list of directories on disk to search for modules
1162 :param dict opts: The salt options dictionary.
1163 :param str tag: The tag for the type of module to load
1164 :param func mod_type_check: A function which can be used to verify files
1165 :param dict pack: A dictionary of function to be packed into modules as they are loaded
1166 :param list whitelist: A list of modules to whitelist
1167 :param bool virtual_enable: Whether or not to respect the __virtual__ function when loading modules.
1168 :param str virtual_funcs: The name of additional functions in the module to call to verify its functionality.
1169 If not true, the module will not load.
1170 :param list extra_module_dirs: A list of directories that will be able to import from
1171 :returns: A LazyLoader object which functions as a dictionary. Keys are 'module.function' and values
1172 are function references themselves which are loaded on-demand.
1173 # TODO:
1174 - move modules_max_memory into here
1175 - singletons (per tag)
1176 """
1177
1178 mod_dict_class = salt.utils.odict.OrderedDict
1179
1180 def __init__(
1181 self,
1182 module_dirs,
1183 opts=None,
1184 tag="module",
1185 loaded_base_name=None,
1186 mod_type_check=None,
1187 pack=None,
1188 whitelist=None,
1189 virtual_enable=True,
1190 static_modules=None,
1191 proxy=None,
1192 virtual_funcs=None,
1193 extra_module_dirs=None,
1194 ): # pylint: disable=W0231
1195 """
1196 In pack, if any of the values are None they will be replaced with an
1197 empty context-specific dict
1198 """
1199
1200 self.inject_globals = {}
1201 self.pack = {} if pack is None else pack
1202 if opts is None:
1203 opts = {}
1204 threadsafety = not opts.get("multiprocessing")
1205 self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety)
1206 self.opts = self.__prep_mod_opts(opts)
1207
1208 self.module_dirs = module_dirs
1209 self.tag = tag
1210 self._gc_finalizer = None
1211 if loaded_base_name and loaded_base_name != LOADED_BASE_NAME:
1212 self.loaded_base_name = loaded_base_name
1213 else:
1214 self.loaded_base_name = LOADED_BASE_NAME
1215 self.mod_type_check = mod_type_check or _mod_type
1216
1217 if "__context__" not in self.pack:
1218 self.pack["__context__"] = None
1219
1220 for k, v in self.pack.items():
1221 if v is None: # if the value of a pack is None, lets make an empty dict
1222 self.context_dict.setdefault(k, {})
1223 self.pack[k] = salt.utils.context.NamespacedDictWrapper(
1224 self.context_dict, k
1225 )
1226
1227 self.whitelist = whitelist
1228 self.virtual_enable = virtual_enable
1229 self.initial_load = True
1230
1231 # names of modules that we don't have (errors, __virtual__, etc.)
1232 self.missing_modules = {} # mapping of name -> error
1233 self.loaded_modules = {} # mapping of module_name -> dict_of_functions
1234 self.loaded_files = set() # TODO: just remove them from file_mapping?
1235 self.static_modules = static_modules if static_modules else []
1236
1237 if virtual_funcs is None:
1238 virtual_funcs = []
1239 self.virtual_funcs = virtual_funcs
1240
1241 self.extra_module_dirs = extra_module_dirs if extra_module_dirs else []
1242 self._clean_module_dirs = []
1243
1244 self.disabled = set(
1245 self.opts.get(
1246 "disable_{}{}".format(self.tag, "" if self.tag[-1] == "s" else "s"), [],
1247 )
1248 )
1249
1250 # A map of suffix to description for imp
1251 self.suffix_map = {}
1252 # A list to determine precedence of extensions
1253 # Prefer packages (directories) over modules (single files)!
1254 self.suffix_order = [""]
1255 for (suffix, mode, kind) in SUFFIXES:
1256 self.suffix_map[suffix] = (suffix, mode, kind)
1257 self.suffix_order.append(suffix)
1258
1259 self._lock = threading.RLock()
1260 with self._lock:
1261 self._refresh_file_mapping()
1262
1263 super().__init__() # late init the lazy loader
1264 # create all of the import namespaces
1265 _generate_module("{}.int".format(self.loaded_base_name))
1266 _generate_module("{}.int.{}".format(self.loaded_base_name, tag))
1267 _generate_module("{}.ext".format(self.loaded_base_name))
1268 _generate_module("{}.ext.{}".format(self.loaded_base_name, tag))
1269
1270 def clean_modules(self):
1271 """
1272 Clean modules
1273 """
1274 for name in list(sys.modules):
1275 if name.startswith(self.loaded_base_name):
1276 del sys.modules[name]
1277
1278 def __getitem__(self, item):
1279 """
1280 Override the __getitem__ in order to decorate the returned function if we need
1281 to last-minute inject globals
1282 """
1283 func = super().__getitem__(item)
1284 if self.inject_globals:
1285 return global_injector_decorator(self.inject_globals)(func)
1286 else:
1287 return func
1288
1289 def __getattr__(self, mod_name):
1290 """
1291 Allow for "direct" attribute access-- this allows jinja templates to
1292 access things like `salt.test.ping()`
1293 """
1294 if mod_name in ("__getstate__", "__setstate__"):
1295 return object.__getattribute__(self, mod_name)
1296
1297 # if we have an attribute named that, lets return it.
1298 try:
1299 return object.__getattr__(self, mod_name) # pylint: disable=no-member
1300 except AttributeError:
1301 pass
1302
1303 # otherwise we assume its jinja template access
1304 if mod_name not in self.loaded_modules and not self.loaded:
1305 for name in self._iter_files(mod_name):
1306 if name in self.loaded_files:
1307 continue
1308 # if we got what we wanted, we are done
1309 if self._load_module(name) and mod_name in self.loaded_modules:
1310 break
1311 if mod_name in self.loaded_modules:
1312 return self.loaded_modules[mod_name]
1313 else:
1314 raise AttributeError(mod_name)
1315
1316 def missing_fun_string(self, function_name):
1317 """
1318 Return the error string for a missing function.
1319
1320 This can range from "not available' to "__virtual__" returned False
1321 """
1322 mod_name = function_name.split(".")[0]
1323 if mod_name in self.loaded_modules:
1324 return "'{}' is not available.".format(function_name)
1325 else:
1326 try:
1327 reason = self.missing_modules[mod_name]
1328 except KeyError:
1329 return "'{}' is not available.".format(function_name)
1330 else:
1331 if reason is not None:
1332 return "'{}' __virtual__ returned False: {}".format(
1333 mod_name, reason
1334 )
1335 else:
1336 return "'{}' __virtual__ returned False".format(mod_name)
1337
1338 def _refresh_file_mapping(self):
1339 """
1340 refresh the mapping of the FS on disk
1341 """
1342 # map of suffix to description for imp
1343 if (
1344 self.opts.get("cython_enable", True) is True
1345 and ".pyx" not in self.suffix_map
1346 ):
1347 try:
1348 global pyximport
1349 pyximport = __import__("pyximport") # pylint: disable=import-error
1350 pyximport.install()
1351 # add to suffix_map so file_mapping will pick it up
1352 self.suffix_map[".pyx"] = tuple()
1353 if ".pyx" not in self.suffix_order:
1354 self.suffix_order.append(".pyx")
1355 except ImportError:
1356 log.info(
1357 "Cython is enabled in the options but not present "
1358 "in the system path. Skipping Cython modules."
1359 )
1360 # Allow for zipimport of modules
1361 if (
1362 self.opts.get("enable_zip_modules", True) is True
1363 and ".zip" not in self.suffix_map
1364 ):
1365 self.suffix_map[".zip"] = tuple()
1366 if ".zip" not in self.suffix_order:
1367 self.suffix_order.append(".zip")
1368 # allow for module dirs
1369 self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY)
1370
1371 # create mapping of filename (without suffix) to (path, suffix)
1372 # The files are added in order of priority, so order *must* be retained.
1373 self.file_mapping = salt.utils.odict.OrderedDict()
1374
1375 opt_match = []
1376
1377 def _replace_pre_ext(obj):
1378 """
1379 Hack so we can get the optimization level that we replaced (if
1380 any) out of the re.sub call below. We use a list here because
1381 it is a persistent data structure that we will be able to
1382 access after re.sub is called.
1383 """
1384 opt_match.append(obj)
1385 return ""
1386
1387 for mod_dir in self.module_dirs:
1388 try:
1389 # Make sure we have a sorted listdir in order to have
1390 # expectable override results
1391 files = sorted(x for x in os.listdir(mod_dir) if x != "__pycache__")
1392 except OSError:
1393 continue # Next mod_dir
1394 try:
1395 pycache_files = [
1396 os.path.join("__pycache__", x)
1397 for x in sorted(os.listdir(os.path.join(mod_dir, "__pycache__")))
1398 ]
1399 except OSError:
1400 pass
1401 else:
1402 files.extend(pycache_files)
1403
1404 for filename in files:
1405 try:
1406 dirname, basename = os.path.split(filename)
1407 if basename.startswith("_"):
1408 # skip private modules
1409 # log messages omitted for obviousness
1410 continue # Next filename
1411 f_noext, ext = os.path.splitext(basename)
1412 f_noext = PY3_PRE_EXT.sub(_replace_pre_ext, f_noext)
1413 try:
1414 opt_level = int(opt_match.pop().group(1).rsplit("-", 1)[-1])
1415 except (AttributeError, IndexError, ValueError):
1416 # No regex match or no optimization level matched
1417 opt_level = 0
1418 try:
1419 opt_index = self.opts["optimization_order"].index(opt_level)
1420 except KeyError:
1421 log.trace(
1422 "Disallowed optimization level %d for module "
1423 "name '%s', skipping. Add %d to the "
1424 "'optimization_order' config option if you "
1425 "do not want to ignore this optimization "
1426 "level.",
1427 opt_level,
1428 f_noext,
1429 opt_level,
1430 )
1431 continue
1432
1433 # make sure it is a suffix we support
1434 if ext not in self.suffix_map:
1435 continue # Next filename
1436 if f_noext in self.disabled:
1437 log.trace(
1438 "Skipping %s, it is disabled by configuration", filename
1439 )
1440 continue # Next filename
1441 fpath = os.path.join(mod_dir, filename)
1442 # if its a directory, lets allow us to load that
1443 if ext == "":
1444 # is there something __init__?
1445 subfiles = os.listdir(fpath)
1446 for suffix in self.suffix_order:
1447 if "" == suffix:
1448 continue # Next suffix (__init__ must have a suffix)
1449 init_file = "__init__{}".format(suffix)
1450 if init_file in subfiles:
1451 break
1452 else:
1453 continue # Next filename
1454
1455 try:
1456 curr_ext = self.file_mapping[f_noext][1]
1457 curr_opt_index = self.file_mapping[f_noext][2]
1458 except KeyError:
1459 pass
1460 else:
1461 if "" in (curr_ext, ext) and curr_ext != ext:
1462 log.error(
1463 "Module/package collision: '%s' and '%s'",
1464 fpath,
1465 self.file_mapping[f_noext][0],
1466 )
1467
1468 if six.PY3 and ext == ".pyc" and curr_ext == ".pyc":
1469 # Check the optimization level
1470 if opt_index >= curr_opt_index:
1471 # Module name match, but a higher-priority
1472 # optimization level was already matched, skipping.
1473 continue
1474 elif not curr_ext or self.suffix_order.index(
1475 ext
1476 ) >= self.suffix_order.index(curr_ext):
1477 # Match found but a higher-priorty match already
1478 # exists, so skip this.
1479 continue
1480
1481 if six.PY3 and not dirname and ext == ".pyc":
1482 # On Python 3, we should only load .pyc files from the
1483 # __pycache__ subdirectory (i.e. when dirname is not an
1484 # empty string).
1485 continue
1486
1487 # Made it this far - add it
1488 self.file_mapping[f_noext] = (fpath, ext, opt_index)
1489
1490 except OSError:
1491 continue
1492 for smod in self.static_modules:
1493 f_noext = smod.split(".")[-1]
1494 self.file_mapping[f_noext] = (smod, ".o", 0)
1495
1496 def clear(self):
1497 """
1498 Clear the dict
1499 """
1500 with self._lock:
1501 super().clear() # clear the lazy loader
1502 self.loaded_files = set()
1503 self.missing_modules = {}
1504 self.loaded_modules = {}
1505 # if we have been loaded before, lets clear the file mapping since
1506 # we obviously want a re-do
1507 if hasattr(self, "opts"):
1508 self._refresh_file_mapping()
1509 self.initial_load = False
1510
1511 def __prep_mod_opts(self, opts):
1512 """
1513 Strip out of the opts any logger instance
1514 """
1515 if "__grains__" not in self.pack:
1516 self.context_dict["grains"] = opts.get("grains", {})
1517 self.pack["__grains__"] = salt.utils.context.NamespacedDictWrapper(
1518 self.context_dict, "grains"
1519 )
1520
1521 if "__pillar__" not in self.pack:
1522 self.context_dict["pillar"] = opts.get("pillar", {})
1523 self.pack["__pillar__"] = salt.utils.context.NamespacedDictWrapper(
1524 self.context_dict, "pillar"
1525 )
1526
1527 mod_opts = {}
1528 for key, val in list(opts.items()):
1529 if key == "logger":
1530 continue
1531 mod_opts[key] = val
1532 return mod_opts
1533
1534 def _iter_files(self, mod_name):
1535 """
1536 Iterate over all file_mapping files in order of closeness to mod_name
1537 """
1538 # do we have an exact match?
1539 if mod_name in self.file_mapping:
1540 yield mod_name
1541
1542 # do we have a partial match?
1543 for k in self.file_mapping:
1544 if mod_name in k:
1545 yield k
1546
1547 # anyone else? Bueller?
1548 for k in self.file_mapping:
1549 if mod_name not in k:
1550 yield k
1551
1552 def _reload_submodules(self, mod):
1553 submodules = (
1554 getattr(mod, sname)
1555 for sname in dir(mod)
1556 if isinstance(getattr(mod, sname), mod.__class__)
1557 )
1558
1559 # reload only custom "sub"modules
1560 for submodule in submodules:
1561 # it is a submodule if the name is in a namespace under mod
1562 if submodule.__name__.startswith(mod.__name__ + "."):
1563 reload_module(submodule)
1564 self._reload_submodules(submodule)
1565
1566 def __populate_sys_path(self):
1567 for directory in self.extra_module_dirs:
1568 if directory not in sys.path:
1569 sys.path.append(directory)
1570 self._clean_module_dirs.append(directory)
1571
1572 def __clean_sys_path(self):
1573 invalidate_path_importer_cache = False
1574 for directory in self._clean_module_dirs:
1575 if directory in sys.path:
1576 sys.path.remove(directory)
1577 invalidate_path_importer_cache = True
1578 self._clean_module_dirs = []
1579
1580 # Be sure that sys.path_importer_cache do not contains any
1581 # invalid FileFinder references
1582 importlib.invalidate_caches()
1583
1584 # Because we are mangling with importlib, we can find from
1585 # time to time an invalidation issue with
1586 # sys.path_importer_cache, that requires the removal of
1587 # FileFinder that remain None for the extra_module_dirs
1588 if invalidate_path_importer_cache:
1589 for directory in self.extra_module_dirs:
1590 if (
1591 directory in sys.path_importer_cache
1592 and sys.path_importer_cache[directory] is None
1593 ):
1594 del sys.path_importer_cache[directory]
1595
1596 def _load_module(self, name):
1597 mod = None
1598 fpath, suffix = self.file_mapping[name][:2]
1599 # if the fpath has `.cpython-3x` in it, but the running Py version
1600 # is 3.y, the following will cause us to return immediately and we won't try to import this .pyc.
1601 # This is for the unusual case where several Python versions share a single
1602 # source tree and drop their .pycs in the same __pycache__ folder.
1603 # If we were to load a .pyc for another Py version it's not a big problem
1604 # but the log will get spammed with "Bad Magic Number" messages that
1605 # can be very misleading if the user is debugging another problem.
1606 try:
1607 (implementation_tag, cache_tag_ver) = sys.implementation.cache_tag.split(
1608 "-"
1609 )
1610 if cache_tag_ver not in fpath and implementation_tag in fpath:
1611 log.trace(
1612 "Trying to load %s on %s, returning False.",
1613 fpath,
1614 sys.implementation.cache_tag,
1615 )
1616 return False
1617 except AttributeError:
1618 # Most likely Py 2.7 or some other Python version we don't really support
1619 pass
1620
1621 self.loaded_files.add(name)
1622 fpath_dirname = os.path.dirname(fpath)
1623 try:
1624 self.__populate_sys_path()
1625 sys.path.append(fpath_dirname)
1626 if suffix == ".pyx":
1627 mod = pyximport.load_module(name, fpath, tempfile.gettempdir())
1628 elif suffix == ".o":
1629 top_mod = __import__(fpath, globals(), locals(), [])
1630 comps = fpath.split(".")
1631 if len(comps) < 2:
1632 mod = top_mod
1633 else:
1634 mod = top_mod
1635 for subname in comps[1:]:
1636 mod = getattr(mod, subname)
1637 elif suffix == ".zip":
1638 mod = zipimporter(fpath).load_module(name)
1639 else:
1640 desc = self.suffix_map[suffix]
1641 # if it is a directory, we don't open a file
1642 try:
1643 mod_namespace = ".".join(
1644 (
1645 self.loaded_base_name,
1646 self.mod_type_check(fpath),
1647 self.tag,
1648 name,
1649 )
1650 )
1651 except TypeError:
1652 mod_namespace = "{}.{}.{}.{}".format(
1653 self.loaded_base_name,
1654 self.mod_type_check(fpath),
1655 self.tag,
1656 name,
1657 )
1658 if suffix == "":
1659 # pylint: disable=no-member
1660 # Package directory, look for __init__
1661 loader_details = [
1662 (
1663 importlib.machinery.SourceFileLoader,
1664 importlib.machinery.SOURCE_SUFFIXES,
1665 ),
1666 (
1667 importlib.machinery.SourcelessFileLoader,
1668 importlib.machinery.BYTECODE_SUFFIXES,
1669 ),
1670 (
1671 importlib.machinery.ExtensionFileLoader,
1672 importlib.machinery.EXTENSION_SUFFIXES,
1673 ),
1674 ]
1675 file_finder = importlib.machinery.FileFinder(
1676 fpath_dirname, *loader_details
1677 )
1678 spec = file_finder.find_spec(mod_namespace)
1679 if spec is None:
1680 raise ImportError()
1681 # TODO: Get rid of load_module in favor of
1682 # exec_module below. load_module is deprecated, but
1683 # loading using exec_module has been causing odd things
1684 # with the magic dunders we pack into the loaded
1685 # modules, most notably with salt-ssh's __opts__.
1686 mod = spec.loader.load_module()
1687 # mod = importlib.util.module_from_spec(spec)
1688 # spec.loader.exec_module(mod)
1689 # pylint: enable=no-member
1690 sys.modules[mod_namespace] = mod
1691 # reload all submodules if necessary
1692 if not self.initial_load:
1693 self._reload_submodules(mod)
1694 else:
1695 # pylint: disable=no-member
1696 loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath)
1697 spec = importlib.util.spec_from_file_location(
1698 mod_namespace, fpath, loader=loader
1699 )
1700 if spec is None:
1701 raise ImportError()
1702 # TODO: Get rid of load_module in favor of
1703 # exec_module below. load_module is deprecated, but
1704 # loading using exec_module has been causing odd things
1705 # with the magic dunders we pack into the loaded
1706 # modules, most notably with salt-ssh's __opts__.
1707 mod = spec.loader.load_module()
1708 # mod = importlib.util.module_from_spec(spec)
1709 # spec.loader.exec_module(mod)
1710 # pylint: enable=no-member
1711 sys.modules[mod_namespace] = mod
1712 except OSError:
1713 raise
1714 except ImportError as exc:
1715 if "magic number" in str(exc):
1716 error_msg = "Failed to import {} {}. Bad magic number. If migrating from Python2 to Python3, remove all .pyc files and try again.".format(
1717 self.tag, name
1718 )
1719 log.warning(error_msg)
1720 self.missing_modules[name] = error_msg
1721 log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True)
1722 self.missing_modules[name] = exc
1723 return False
1724 except Exception as error: # pylint: disable=broad-except
1725 log.error(
1726 "Failed to import %s %s, this is due most likely to a "
1727 "syntax error:\n",
1728 self.tag,
1729 name,
1730 exc_info=True,
1731 )
1732 self.missing_modules[name] = error
1733 return False
1734 except SystemExit as error:
1735 try:
1736 fn_, _, caller, _ = traceback.extract_tb(sys.exc_info()[2])[-1]
1737 except Exception: # pylint: disable=broad-except
1738 pass
1739 else:
1740 tgt_fn = os.path.join("salt", "utils", "process.py")
1741 if fn_.endswith(tgt_fn) and "_handle_signals" in caller:
1742 # Race conditon, SIGTERM or SIGINT received while loader
1743 # was in process of loading a module. Call sys.exit to
1744 # ensure that the process is killed.
1745 sys.exit(salt.defaults.exitcodes.EX_OK)
1746 log.error(
1747 "Failed to import %s %s as the module called exit()\n",
1748 self.tag,
1749 name,
1750 exc_info=True,
1751 )
1752 self.missing_modules[name] = error
1753 return False
1754 finally:
1755 sys.path.remove(fpath_dirname)
1756 self.__clean_sys_path()
1757
1758 if hasattr(mod, "__opts__"):
1759 mod.__opts__.update(self.opts)
1760 else:
1761 mod.__opts__ = self.opts
1762
1763 # pack whatever other globals we were asked to
1764 for p_name, p_value in self.pack.items():
1765 setattr(mod, p_name, p_value)
1766
1767 module_name = mod.__name__.rsplit(".", 1)[-1]
1768
1769 # Call a module's initialization method if it exists
1770 module_init = getattr(mod, "__init__", None)
1771 if inspect.isfunction(module_init):
1772 try:
1773 module_init(self.opts)
1774 except TypeError as e:
1775 log.error(e)
1776 except Exception: # pylint: disable=broad-except
1777 err_string = "__init__ failed"
1778 log.debug(
1779 "Error loading %s.%s: %s",
1780 self.tag,
1781 module_name,
1782 err_string,
1783 exc_info=True,
1784 )
1785 self.missing_modules[module_name] = err_string
1786 self.missing_modules[name] = err_string
1787 return False
1788
1789 # if virtual modules are enabled, we need to look for the
1790 # __virtual__() function inside that module and run it.
1791 if self.virtual_enable:
1792 virtual_funcs_to_process = ["__virtual__"] + self.virtual_funcs
1793 for virtual_func in virtual_funcs_to_process:
1794 (
1795 virtual_ret,
1796 module_name,
1797 virtual_err,
1798 virtual_aliases,
1799 ) = self._process_virtual(mod, module_name, virtual_func)
1800 if virtual_err is not None:
1801 log.trace(
1802 "Error loading %s.%s: %s", self.tag, module_name, virtual_err
1803 )
1804
1805 # if _process_virtual returned a non-True value then we are
1806 # supposed to not process this module
1807 if virtual_ret is not True and module_name not in self.missing_modules:
1808 # If a module has information about why it could not be loaded, record it
1809 self.missing_modules[module_name] = virtual_err
1810 self.missing_modules[name] = virtual_err
1811 return False
1812 else:
1813 virtual_aliases = ()
1814
1815 # If this is a proxy minion then MOST modules cannot work. Therefore, require that
1816 # any module that does work with salt-proxy-minion define __proxyenabled__ as a list
1817 # containing the names of the proxy types that the module supports.
1818 #
1819 # Render modules and state modules are OK though
1820 if "proxy" in self.opts:
1821 if self.tag in ["grains", "proxy"]:
1822 if not hasattr(mod, "__proxyenabled__") or (
1823 self.opts["proxy"]["proxytype"] not in mod.__proxyenabled__
1824 and "*" not in mod.__proxyenabled__
1825 ):
1826 err_string = "not a proxy_minion enabled module"
1827 self.missing_modules[module_name] = err_string
1828 self.missing_modules[name] = err_string
1829 return False
1830
1831 if getattr(mod, "__load__", False) is not False:
1832 log.info(
1833 "The functions from module '%s' are being loaded from the "
1834 "provided __load__ attribute",
1835 module_name,
1836 )
1837
1838 # If we had another module by the same virtual name, we should put any
1839 # new functions under the existing dictionary.
1840 mod_names = [module_name] + list(virtual_aliases)
1841 mod_dict = {
1842 x: self.loaded_modules.get(x, self.mod_dict_class()) for x in mod_names
1843 }
1844
1845 for attr in getattr(mod, "__load__", dir(mod)):
1846 if attr.startswith("_"):
1847 # private functions are skipped
1848 continue
1849 func = getattr(mod, attr)
1850 if not inspect.isfunction(func) and not isinstance(func, functools.partial):
1851 # Not a function!? Skip it!!!
1852 continue
1853 # Let's get the function name.
1854 # If the module has the __func_alias__ attribute, it must be a
1855 # dictionary mapping in the form of(key -> value):
1856 # <real-func-name> -> <desired-func-name>
1857 #
1858 # It default's of course to the found callable attribute name
1859 # if no alias is defined.
1860 funcname = getattr(mod, "__func_alias__", {}).get(attr, attr)
1861 for tgt_mod in mod_names:
1862 try:
1863 full_funcname = ".".join((tgt_mod, funcname))
1864 except TypeError:
1865 full_funcname = "{}.{}".format(tgt_mod, funcname)
1866 # Save many references for lookups
1867 # Careful not to overwrite existing (higher priority) functions
1868 if full_funcname not in self._dict:
1869 self._dict[full_funcname] = func
1870 if funcname not in mod_dict[tgt_mod]:
1871 setattr(mod_dict[tgt_mod], funcname, func)
1872 mod_dict[tgt_mod][funcname] = func
1873 self._apply_outputter(func, mod)
1874
1875 # enforce depends
1876 try:
1877 Depends.enforce_dependencies(self._dict, self.tag, name)
1878 except RuntimeError as exc:
1879 log.info(
1880 "Depends.enforce_dependencies() failed for the following " "reason: %s",
1881 exc,
1882 )
1883
1884 for tgt_mod in mod_names:
1885 self.loaded_modules[tgt_mod] = mod_dict[tgt_mod]
1886 return True
1887
1888 def _load(self, key):
1889 """
1890 Load a single item if you have it
1891 """
1892 # if the key doesn't have a '.' then it isn't valid for this mod dict
1893 if not isinstance(key, str):
1894 raise KeyError("The key must be a string.")
1895 if "." not in key:
1896 raise KeyError("The key '{}' should contain a '.'".format(key))
1897 mod_name, _ = key.split(".", 1)
1898 with self._lock:
1899 # It is possible that the key is in the dictionary after
1900 # acquiring the lock due to another thread loading it.
1901 if mod_name in self.missing_modules or key in self._dict:
1902 return True
1903 # if the modulename isn't in the whitelist, don't bother
1904 if self.whitelist and mod_name not in self.whitelist:
1905 log.error(
1906 "Failed to load function %s because its module (%s) is "
1907 "not in the whitelist: %s",
1908 key,
1909 mod_name,
1910 self.whitelist,
1911 )
1912 raise KeyError(key)
1913
1914 def _inner_load(mod_name):
1915 for name in self._iter_files(mod_name):
1916 if name in self.loaded_files:
1917 continue
1918 # if we got what we wanted, we are done
1919 if self._load_module(name) and key in self._dict:
1920 return True
1921 return False
1922
1923 # try to load the module
1924 ret = None
1925 reloaded = False
1926 # re-scan up to once, IOErrors or a failed load cause re-scans of the
1927 # filesystem
1928 while True:
1929 try:
1930 ret = _inner_load(mod_name)
1931 if not reloaded and ret is not True:
1932 self._refresh_file_mapping()
1933 reloaded = True
1934 continue
1935 break
1936 except OSError:
1937 if not reloaded:
1938 self._refresh_file_mapping()
1939 reloaded = True
1940 continue
1941
1942 return ret
1943
1944 def _load_all(self):
1945 """
1946 Load all of them
1947 """
1948 with self._lock:
1949 for name in self.file_mapping:
1950 if name in self.loaded_files or name in self.missing_modules:
1951 continue
1952 self._load_module(name)
1953
1954 self.loaded = True
1955
1956 def reload_modules(self):
1957 with self._lock:
1958 self.loaded_files = set()
1959 self._load_all()
1960
1961 def _apply_outputter(self, func, mod):
1962 """
1963 Apply the __outputter__ variable to the functions
1964 """
1965 if hasattr(mod, "__outputter__"):
1966 outp = mod.__outputter__
1967 if func.__name__ in outp:
1968 func.__outputter__ = outp[func.__name__]
1969
1970 def _process_virtual(self, mod, module_name, virtual_func="__virtual__"):
1971 """
1972 Given a loaded module and its default name determine its virtual name
1973
1974 This function returns a tuple. The first value will be either True or
1975 False and will indicate if the module should be loaded or not (i.e. if
1976 it threw and exception while processing its __virtual__ function). The
1977 second value is the determined virtual name, which may be the same as
1978 the value provided.
1979
1980 The default name can be calculated as follows::
1981
1982 module_name = mod.__name__.rsplit('.', 1)[-1]
1983 """
1984
1985 # The __virtual__ function will return either a True or False value.
1986 # If it returns a True value it can also set a module level attribute
1987 # named __virtualname__ with the name that the module should be
1988 # referred to as.
1989 #
1990 # This allows us to have things like the pkg module working on all
1991 # platforms under the name 'pkg'. It also allows for modules like
1992 # augeas_cfg to be referred to as 'augeas', which would otherwise have
1993 # namespace collisions. And finally it allows modules to return False
1994 # if they are not intended to run on the given platform or are missing
1995 # dependencies.
1996 virtual_aliases = getattr(mod, "__virtual_aliases__", tuple())
1997 try:
1998 error_reason = None
1999 if hasattr(mod, "__virtual__") and inspect.isfunction(mod.__virtual__):
2000 try:
2001 start = time.time()
2002 virtual = getattr(mod, virtual_func)()
2003 if isinstance(virtual, tuple):
2004 error_reason = virtual[1]
2005 virtual = virtual[0]
2006 if self.opts.get("virtual_timer", False):
2007 end = time.time() - start
2008 msg = "Virtual function took {} seconds for {}".format(
2009 end, module_name
2010 )
2011 log.warning(msg)
2012 except Exception as exc: # pylint: disable=broad-except
2013 error_reason = (
2014 "Exception raised when processing __virtual__ function"
2015 " for {}. Module will not be loaded: {}".format(
2016 mod.__name__, exc
2017 )
2018 )
2019 log.error(error_reason, exc_info_on_loglevel=logging.DEBUG)
2020 virtual = None
2021 # Get the module's virtual name
2022 virtualname = getattr(mod, "__virtualname__", virtual)
2023 if not virtual:
2024 # if __virtual__() evaluates to False then the module
2025 # wasn't meant for this platform or it's not supposed to
2026 # load for some other reason.
2027
2028 # Some modules might accidentally return None and are
2029 # improperly loaded
2030 if virtual is None:
2031 log.warning(
2032 "%s.__virtual__() is wrongly returning `None`. "
2033 "It should either return `True`, `False` or a new "
2034 "name. If you're the developer of the module "
2035 "'%s', please fix this.",
2036 mod.__name__,
2037 module_name,
2038 )
2039
2040 return (False, module_name, error_reason, virtual_aliases)
2041
2042 # At this point, __virtual__ did not return a
2043 # boolean value, let's check for deprecated usage
2044 # or module renames
2045 if virtual is not True and module_name != virtual:
2046 # The module is renaming itself. Updating the module name
2047 # with the new name
2048 log.trace("Loaded %s as virtual %s", module_name, virtual)
2049
2050 if virtualname != virtual:
2051 # The __virtualname__ attribute does not match what's
2052 # being returned by the __virtual__() function. This
2053 # should be considered an error.
2054 log.error(
2055 "The module '%s' is showing some bad usage. Its "
2056 "__virtualname__ attribute is set to '%s' yet the "
2057 "__virtual__() function is returning '%s'. These "
2058 "values should match!",
2059 mod.__name__,
2060 virtualname,
2061 virtual,
2062 )
2063
2064 module_name = virtualname
2065
2066 # If the __virtual__ function returns True and __virtualname__
2067 # is set then use it
2068 elif virtual is True and virtualname != module_name:
2069 if virtualname is not True:
2070 module_name = virtualname
2071
2072 except KeyError:
2073 # Key errors come out of the virtual function when passing
2074 # in incomplete grains sets, these can be safely ignored
2075 # and logged to debug, still, it includes the traceback to
2076 # help debugging.
2077 log.debug("KeyError when loading %s", module_name, exc_info=True)
2078
2079 except Exception: # pylint: disable=broad-except
2080 # If the module throws an exception during __virtual__()
2081 # then log the information and continue to the next.
2082 log.error(
2083 "Failed to read the virtual function for %s: %s",
2084 self.tag,
2085 module_name,
2086 exc_info=True,
2087 )
2088 return (False, module_name, error_reason, virtual_aliases)
2089
2090 return (True, module_name, None, virtual_aliases)
2091
2092
2093 def global_injector_decorator(inject_globals):
2094 """
2095 Decorator used by the LazyLoader to inject globals into a function at
2096 execute time.
2097
2098 globals
2099 Dictionary with global variables to inject
2100 """
2101
2102 def inner_decorator(f):
2103 @functools.wraps(f)
2104 def wrapper(*args, **kwargs):
2105 with salt.utils.context.func_globals_inject(f, **inject_globals):
2106 return f(*args, **kwargs)
2107
2108 return wrapper
2109
2110 return inner_decorator