"Fossies" - the Fresh Open Source Software Archive

Member "salt-3002.2/salt/loader.py" (18 Nov 2020, 74411 Bytes) of package /linux/misc/salt-3002.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "loader.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 3002.1_vs_3002.2.

    1 """
    2 The Salt loader is the core to Salt's plugin system, the loader scans
    3 directories for python loadable code and organizes the code into the
    4 plugin interfaces used by Salt.
    5 """
    6 
    7 import functools
    8 import importlib.machinery  # pylint: disable=no-name-in-module,import-error
    9 import importlib.util  # pylint: disable=no-name-in-module,import-error
   10 import inspect
   11 import logging
   12 import os
   13 import re
   14 import sys
   15 import tempfile
   16 import threading
   17 import time
   18 import traceback
   19 import types
   20 from collections.abc import MutableMapping
   21 from zipimport import zipimporter
   22 
   23 import salt.config
   24 import salt.defaults.events
   25 import salt.defaults.exitcodes
   26 import salt.syspaths
   27 import salt.utils.args
   28 import salt.utils.context
   29 import salt.utils.data
   30 import salt.utils.dictupdate
   31 import salt.utils.event
   32 import salt.utils.files
   33 import salt.utils.lazy
   34 import salt.utils.odict
   35 import salt.utils.platform
   36 import salt.utils.stringutils
   37 import salt.utils.versions
   38 from salt.exceptions import LoaderError
   39 from salt.ext import six
   40 from salt.ext.six.moves import reload_module
   41 from salt.template import check_render_pipe_str
   42 from salt.utils.decorators import Depends
   43 
   44 try:
   45     import pkg_resources
   46 
   47     HAS_PKG_RESOURCES = True
   48 except ImportError:
   49     HAS_PKG_RESOURCES = False
   50 
   51 log = logging.getLogger(__name__)
   52 
   53 SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR)
   54 LOADED_BASE_NAME = "salt.loaded"
   55 
   56 # pylint: disable=no-member
   57 MODULE_KIND_SOURCE = 1
   58 MODULE_KIND_COMPILED = 2
   59 MODULE_KIND_EXTENSION = 3
   60 MODULE_KIND_PKG_DIRECTORY = 5
   61 SUFFIXES = []
   62 for suffix in importlib.machinery.EXTENSION_SUFFIXES:
   63     SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION))
   64 for suffix in importlib.machinery.SOURCE_SUFFIXES:
   65     SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE))
   66 for suffix in importlib.machinery.BYTECODE_SUFFIXES:
   67     SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED))
   68 MODULE_KIND_MAP = {
   69     MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader,
   70     MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader,
   71     MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader,
   72 }
   73 # pylint: enable=no-member
   74 
   75 PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_info[:2]))
   76 
   77 # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
   78 # which simplifies code readability, it adds some unsupported functions into
   79 # the driver's module scope.
   80 # We list un-supported functions here. These will be removed from the loaded.
   81 #  TODO:  remove the need for this cross-module code. Maybe use NotImplemented
   82 LIBCLOUD_FUNCS_NOT_SUPPORTED = (
   83     "parallels.avail_sizes",
   84     "parallels.avail_locations",
   85     "proxmox.avail_sizes",
   86 )
   87 
   88 # Will be set to pyximport module at runtime if cython is enabled in config.
   89 pyximport = None
   90 
   91 
   92 def static_loader(
   93     opts,
   94     ext_type,
   95     tag,
   96     pack=None,
   97     int_type=None,
   98     ext_dirs=True,
   99     ext_type_dirs=None,
  100     base_path=None,
  101     filter_name=None,
  102 ):
  103     funcs = LazyLoader(
  104         _module_dirs(
  105             opts, ext_type, tag, int_type, ext_dirs, ext_type_dirs, base_path,
  106         ),
  107         opts,
  108         tag=tag,
  109         pack=pack,
  110     )
  111     ret = {}
  112     funcs._load_all()
  113     if filter_name:
  114         funcs = FilterDictWrapper(funcs, filter_name)
  115     for key in funcs:
  116         ret[key] = funcs[key]
  117     return ret
  118 
  119 
  120 def _format_entrypoint_target(ep):
  121     """
  122     Makes a string describing the target of an EntryPoint object.
  123 
  124     Base strongly on EntryPoint.__str__().
  125     """
  126     s = ep.module_name
  127     if ep.attrs:
  128         s += ":" + ".".join(ep.attrs)
  129     return s
  130 
  131 
  132 def _module_dirs(
  133     opts,
  134     ext_type,
  135     tag=None,
  136     int_type=None,
  137     ext_dirs=True,
  138     ext_type_dirs=None,
  139     base_path=None,
  140 ):
  141     if tag is None:
  142         tag = ext_type
  143     sys_types = os.path.join(base_path or SALT_BASE_PATH, int_type or ext_type)
  144     ext_types = os.path.join(opts["extension_modules"], ext_type)
  145 
  146     ext_type_types = []
  147     if ext_dirs:
  148         if ext_type_dirs is None:
  149             ext_type_dirs = "{}_dirs".format(tag)
  150         if ext_type_dirs in opts:
  151             ext_type_types.extend(opts[ext_type_dirs])
  152         if HAS_PKG_RESOURCES and ext_type_dirs:
  153             for entry_point in pkg_resources.iter_entry_points(
  154                 "salt.loader", ext_type_dirs
  155             ):
  156                 try:
  157                     loaded_entry_point = entry_point.load()
  158                     for path in loaded_entry_point():
  159                         ext_type_types.append(path)
  160                 except Exception as exc:  # pylint: disable=broad-except
  161                     log.error(
  162                         "Error getting module directories from %s: %s",
  163                         _format_entrypoint_target(entry_point),
  164                         exc,
  165                     )
  166                     log.debug(
  167                         "Full backtrace for module directories error", exc_info=True
  168                     )
  169 
  170     cli_module_dirs = []
  171     # The dirs can be any module dir, or a in-tree _{ext_type} dir
  172     for _dir in opts.get("module_dirs", []):
  173         # Prepend to the list to match cli argument ordering
  174         maybe_dir = os.path.join(_dir, ext_type)
  175         if os.path.isdir(maybe_dir):
  176             cli_module_dirs.insert(0, maybe_dir)
  177             continue
  178 
  179         maybe_dir = os.path.join(_dir, "_{}".format(ext_type))
  180         if os.path.isdir(maybe_dir):
  181             cli_module_dirs.insert(0, maybe_dir)
  182 
  183     return cli_module_dirs + ext_type_types + [ext_types, sys_types]
  184 
  185 
  186 def minion_mods(
  187     opts,
  188     context=None,
  189     utils=None,
  190     whitelist=None,
  191     initial_load=False,
  192     loaded_base_name=None,
  193     notify=False,
  194     static_modules=None,
  195     proxy=None,
  196 ):
  197     """
  198     Load execution modules
  199 
  200     Returns a dictionary of execution modules appropriate for the current
  201     system by evaluating the __virtual__() function in each module.
  202 
  203     :param dict opts: The Salt options dictionary
  204 
  205     :param dict context: A Salt context that should be made present inside
  206                             generated modules in __context__
  207 
  208     :param dict utils: Utility functions which should be made available to
  209                             Salt modules in __utils__. See `utils_dirs` in
  210                             salt.config for additional information about
  211                             configuration.
  212 
  213     :param list whitelist: A list of modules which should be whitelisted.
  214     :param bool initial_load: Deprecated flag! Unused.
  215     :param str loaded_base_name: A string marker for the loaded base name.
  216     :param bool notify: Flag indicating that an event should be fired upon
  217                         completion of module loading.
  218 
  219     .. code-block:: python
  220 
  221         import salt.config
  222         import salt.loader
  223 
  224         __opts__ = salt.config.minion_config('/etc/salt/minion')
  225         __grains__ = salt.loader.grains(__opts__)
  226         __opts__['grains'] = __grains__
  227         __utils__ = salt.loader.utils(__opts__)
  228         __salt__ = salt.loader.minion_mods(__opts__, utils=__utils__)
  229         __salt__['test.ping']()
  230     """
  231     # TODO Publish documentation for module whitelisting
  232     if not whitelist:
  233         whitelist = opts.get("whitelist_modules", None)
  234     ret = LazyLoader(
  235         _module_dirs(opts, "modules", "module"),
  236         opts,
  237         tag="module",
  238         pack={"__context__": context, "__utils__": utils, "__proxy__": proxy},
  239         whitelist=whitelist,
  240         loaded_base_name=loaded_base_name,
  241         static_modules=static_modules,
  242         extra_module_dirs=utils.module_dirs if utils else None,
  243     )
  244 
  245     ret.pack["__salt__"] = ret
  246 
  247     # Load any provider overrides from the configuration file providers option
  248     #  Note: Providers can be pkg, service, user or group - not to be confused
  249     #        with cloud providers.
  250     providers = opts.get("providers", False)
  251     if providers and isinstance(providers, dict):
  252         for mod in providers:
  253             # sometimes providers opts is not to diverge modules but
  254             # for other configuration
  255             try:
  256                 funcs = raw_mod(opts, providers[mod], ret)
  257             except TypeError:
  258                 break
  259             else:
  260                 if funcs:
  261                     for func in funcs:
  262                         f_key = "{}{}".format(mod, func[func.rindex(".") :])
  263                         ret[f_key] = funcs[func]
  264 
  265     if notify:
  266         with salt.utils.event.get_event("minion", opts=opts, listen=False) as evt:
  267             evt.fire_event(
  268                 {"complete": True}, tag=salt.defaults.events.MINION_MOD_REFRESH_COMPLETE
  269             )
  270 
  271     return ret
  272 
  273 
  274 def raw_mod(opts, name, functions, mod="modules"):
  275     """
  276     Returns a single module loaded raw and bypassing the __virtual__ function
  277 
  278     .. code-block:: python
  279 
  280         import salt.config
  281         import salt.loader
  282 
  283         __opts__ = salt.config.minion_config('/etc/salt/minion')
  284         testmod = salt.loader.raw_mod(__opts__, 'test', None)
  285         testmod['test.ping']()
  286     """
  287     loader = LazyLoader(
  288         _module_dirs(opts, mod, "module"),
  289         opts,
  290         tag="rawmodule",
  291         virtual_enable=False,
  292         pack={"__salt__": functions},
  293     )
  294     # if we don't have the module, return an empty dict
  295     if name not in loader.file_mapping:
  296         return {}
  297 
  298     loader._load_module(name)  # load a single module (the one passed in)
  299     return dict(loader._dict)  # return a copy of *just* the funcs for `name`
  300 
  301 
  302 def metaproxy(opts, loaded_base_name=None):
  303     """
  304     Return functions used in the meta proxy
  305     """
  306 
  307     return LazyLoader(
  308         _module_dirs(opts, "metaproxy"),
  309         opts,
  310         tag="metaproxy",
  311         loaded_base_name=loaded_base_name,
  312     )
  313 
  314 
  315 def matchers(opts):
  316     """
  317     Return the matcher services plugins
  318     """
  319     return LazyLoader(_module_dirs(opts, "matchers"), opts, tag="matchers")
  320 
  321 
  322 def engines(opts, functions, runners, utils, proxy=None):
  323     """
  324     Return the master services plugins
  325     """
  326     pack = {
  327         "__salt__": functions,
  328         "__runners__": runners,
  329         "__proxy__": proxy,
  330         "__utils__": utils,
  331     }
  332     return LazyLoader(
  333         _module_dirs(opts, "engines"),
  334         opts,
  335         tag="engines",
  336         pack=pack,
  337         extra_module_dirs=utils.module_dirs if utils else None,
  338     )
  339 
  340 
  341 def proxy(
  342     opts, functions=None, returners=None, whitelist=None, utils=None, context=None
  343 ):
  344     """
  345     Returns the proxy module for this salt-proxy-minion
  346     """
  347     ret = LazyLoader(
  348         _module_dirs(opts, "proxy"),
  349         opts,
  350         tag="proxy",
  351         pack={
  352             "__salt__": functions,
  353             "__ret__": returners,
  354             "__utils__": utils,
  355             "__context__": context,
  356         },
  357         extra_module_dirs=utils.module_dirs if utils else None,
  358     )
  359 
  360     ret.pack["__proxy__"] = ret
  361 
  362     return ret
  363 
  364 
  365 def returners(opts, functions, whitelist=None, context=None, proxy=None):
  366     """
  367     Returns the returner modules
  368     """
  369     return LazyLoader(
  370         _module_dirs(opts, "returners", "returner"),
  371         opts,
  372         tag="returner",
  373         whitelist=whitelist,
  374         pack={"__salt__": functions, "__context__": context, "__proxy__": proxy or {}},
  375     )
  376 
  377 
  378 def utils(opts, whitelist=None, context=None, proxy=proxy):
  379     """
  380     Returns the utility modules
  381     """
  382     return LazyLoader(
  383         _module_dirs(opts, "utils", ext_type_dirs="utils_dirs"),
  384         opts,
  385         tag="utils",
  386         whitelist=whitelist,
  387         pack={"__context__": context, "__proxy__": proxy or {}},
  388     )
  389 
  390 
  391 def pillars(opts, functions, context=None):
  392     """
  393     Returns the pillars modules
  394     """
  395     _utils = utils(opts)
  396     ret = LazyLoader(
  397         _module_dirs(opts, "pillar"),
  398         opts,
  399         tag="pillar",
  400         pack={"__salt__": functions, "__context__": context, "__utils__": _utils},
  401         extra_module_dirs=_utils.module_dirs,
  402     )
  403     ret.pack["__ext_pillar__"] = ret
  404     return FilterDictWrapper(ret, ".ext_pillar")
  405 
  406 
  407 def tops(opts):
  408     """
  409     Returns the tops modules
  410     """
  411     if "master_tops" not in opts:
  412         return {}
  413     whitelist = list(opts["master_tops"].keys())
  414     ret = LazyLoader(
  415         _module_dirs(opts, "tops", "top"), opts, tag="top", whitelist=whitelist,
  416     )
  417     return FilterDictWrapper(ret, ".top")
  418 
  419 
  420 def wheels(opts, whitelist=None, context=None):
  421     """
  422     Returns the wheels modules
  423     """
  424     if context is None:
  425         context = {}
  426     return LazyLoader(
  427         _module_dirs(opts, "wheel"),
  428         opts,
  429         tag="wheel",
  430         whitelist=whitelist,
  431         pack={"__context__": context},
  432     )
  433 
  434 
  435 def outputters(opts):
  436     """
  437     Returns the outputters modules
  438 
  439     :param dict opts: The Salt options dictionary
  440     :returns: LazyLoader instance, with only outputters present in the keyspace
  441     """
  442     ret = LazyLoader(
  443         _module_dirs(opts, "output", ext_type_dirs="outputter_dirs"),
  444         opts,
  445         tag="output",
  446     )
  447     wrapped_ret = FilterDictWrapper(ret, ".output")
  448     # TODO: this name seems terrible... __salt__ should always be execution mods
  449     ret.pack["__salt__"] = wrapped_ret
  450     return wrapped_ret
  451 
  452 
  453 def serializers(opts):
  454     """
  455     Returns the serializers modules
  456     :param dict opts: The Salt options dictionary
  457     :returns: LazyLoader instance, with only serializers present in the keyspace
  458     """
  459     return LazyLoader(_module_dirs(opts, "serializers"), opts, tag="serializers",)
  460 
  461 
  462 def eauth_tokens(opts):
  463     """
  464     Returns the tokens modules
  465     :param dict opts: The Salt options dictionary
  466     :returns: LazyLoader instance, with only token backends present in the keyspace
  467     """
  468     return LazyLoader(_module_dirs(opts, "tokens"), opts, tag="tokens",)
  469 
  470 
  471 def auth(opts, whitelist=None):
  472     """
  473     Returns the auth modules
  474 
  475     :param dict opts: The Salt options dictionary
  476     :returns: LazyLoader
  477     """
  478     return LazyLoader(
  479         _module_dirs(opts, "auth"),
  480         opts,
  481         tag="auth",
  482         whitelist=whitelist,
  483         pack={"__salt__": minion_mods(opts)},
  484     )
  485 
  486 
  487 def fileserver(opts, backends):
  488     """
  489     Returns the file server modules
  490     """
  491     _utils = utils(opts)
  492 
  493     if backends is not None:
  494         if not isinstance(backends, list):
  495             backends = [backends]
  496         # Make sure that the VCS backends work either with git or gitfs, hg or
  497         # hgfs, etc.
  498         vcs_re = re.compile("^(git|svn|hg)")
  499         fs_re = re.compile("fs$")
  500         vcs = []
  501         non_vcs = []
  502         for back in [fs_re.sub("", x) for x in backends]:
  503             if vcs_re.match(back):
  504                 vcs.extend((back, back + "fs"))
  505             else:
  506                 non_vcs.append(back)
  507         backends = vcs + non_vcs
  508 
  509     return LazyLoader(
  510         _module_dirs(opts, "fileserver"),
  511         opts,
  512         tag="fileserver",
  513         whitelist=backends,
  514         pack={"__utils__": _utils},
  515         extra_module_dirs=_utils.module_dirs,
  516     )
  517 
  518 
  519 def roster(opts, runner=None, utils=None, whitelist=None):
  520     """
  521     Returns the roster modules
  522     """
  523     return LazyLoader(
  524         _module_dirs(opts, "roster"),
  525         opts,
  526         tag="roster",
  527         whitelist=whitelist,
  528         pack={"__runner__": runner, "__utils__": utils},
  529         extra_module_dirs=utils.module_dirs if utils else None,
  530     )
  531 
  532 
  533 def thorium(opts, functions, runners):
  534     """
  535     Load the thorium runtime modules
  536     """
  537     pack = {"__salt__": functions, "__runner__": runners, "__context__": {}}
  538     ret = LazyLoader(_module_dirs(opts, "thorium"), opts, tag="thorium", pack=pack)
  539     ret.pack["__thorium__"] = ret
  540     return ret
  541 
  542 
  543 def states(
  544     opts, functions, utils, serializers, whitelist=None, proxy=None, context=None
  545 ):
  546     """
  547     Returns the state modules
  548 
  549     :param dict opts: The Salt options dictionary
  550     :param dict functions: A dictionary of minion modules, with module names as
  551                             keys and funcs as values.
  552 
  553     .. code-block:: python
  554 
  555         import salt.config
  556         import salt.loader
  557 
  558         __opts__ = salt.config.minion_config('/etc/salt/minion')
  559         statemods = salt.loader.states(__opts__, None, None)
  560     """
  561     if context is None:
  562         context = {}
  563 
  564     ret = LazyLoader(
  565         _module_dirs(opts, "states"),
  566         opts,
  567         tag="states",
  568         pack={"__salt__": functions, "__proxy__": proxy or {}},
  569         whitelist=whitelist,
  570         extra_module_dirs=utils.module_dirs if utils else None,
  571     )
  572     ret.pack["__states__"] = ret
  573     ret.pack["__utils__"] = utils
  574     ret.pack["__serializers__"] = serializers
  575     ret.pack["__context__"] = context
  576     return ret
  577 
  578 
  579 def beacons(opts, functions, context=None, proxy=None):
  580     """
  581     Load the beacon modules
  582 
  583     :param dict opts: The Salt options dictionary
  584     :param dict functions: A dictionary of minion modules, with module names as
  585                             keys and funcs as values.
  586     """
  587     return LazyLoader(
  588         _module_dirs(opts, "beacons"),
  589         opts,
  590         tag="beacons",
  591         pack={"__context__": context, "__salt__": functions, "__proxy__": proxy or {}},
  592         virtual_funcs=[],
  593     )
  594 
  595 
  596 def log_handlers(opts):
  597     """
  598     Returns the custom logging handler modules
  599 
  600     :param dict opts: The Salt options dictionary
  601     """
  602     ret = LazyLoader(
  603         _module_dirs(
  604             opts,
  605             "log_handlers",
  606             int_type="handlers",
  607             base_path=os.path.join(SALT_BASE_PATH, "log"),
  608         ),
  609         opts,
  610         tag="log_handlers",
  611     )
  612     return FilterDictWrapper(ret, ".setup_handlers")
  613 
  614 
  615 def ssh_wrapper(opts, functions=None, context=None):
  616     """
  617     Returns the custom logging handler modules
  618     """
  619     return LazyLoader(
  620         _module_dirs(
  621             opts,
  622             "wrapper",
  623             base_path=os.path.join(SALT_BASE_PATH, os.path.join("client", "ssh")),
  624         ),
  625         opts,
  626         tag="wrapper",
  627         pack={
  628             "__salt__": functions,
  629             "__grains__": opts.get("grains", {}),
  630             "__pillar__": opts.get("pillar", {}),
  631             "__context__": context,
  632         },
  633     )
  634 
  635 
  636 def render(opts, functions, states=None, proxy=None, context=None):
  637     """
  638     Returns the render modules
  639     """
  640     if context is None:
  641         context = {}
  642 
  643     pack = {
  644         "__salt__": functions,
  645         "__grains__": opts.get("grains", {}),
  646         "__context__": context,
  647     }
  648 
  649     if states:
  650         pack["__states__"] = states
  651     pack["__proxy__"] = proxy or {}
  652     ret = LazyLoader(
  653         _module_dirs(opts, "renderers", "render", ext_type_dirs="render_dirs",),
  654         opts,
  655         tag="render",
  656         pack=pack,
  657     )
  658     rend = FilterDictWrapper(ret, ".render")
  659 
  660     if not check_render_pipe_str(
  661         opts["renderer"], rend, opts["renderer_blacklist"], opts["renderer_whitelist"]
  662     ):
  663         err = (
  664             "The renderer {} is unavailable, this error is often because "
  665             "the needed software is unavailable".format(opts["renderer"])
  666         )
  667         log.critical(err)
  668         raise LoaderError(err)
  669     return rend
  670 
  671 
  672 def grain_funcs(opts, proxy=None, context=None):
  673     """
  674     Returns the grain functions
  675 
  676       .. code-block:: python
  677 
  678           import salt.config
  679           import salt.loader
  680 
  681           __opts__ = salt.config.minion_config('/etc/salt/minion')
  682           grainfuncs = salt.loader.grain_funcs(__opts__)
  683     """
  684     _utils = utils(opts, proxy=proxy)
  685     pack = {"__utils__": utils(opts, proxy=proxy), "__context__": context}
  686 
  687     ret = LazyLoader(
  688         _module_dirs(opts, "grains", "grain", ext_type_dirs="grains_dirs",),
  689         opts,
  690         tag="grains",
  691         extra_module_dirs=_utils.module_dirs,
  692         pack=pack,
  693     )
  694     ret.pack["__utils__"] = _utils
  695     return ret
  696 
  697 
  698 def _format_cached_grains(cached_grains):
  699     """
  700     Returns cached grains with fixed types, like tuples.
  701     """
  702     if cached_grains.get("osrelease_info"):
  703         osrelease_info = cached_grains["osrelease_info"]
  704         if isinstance(osrelease_info, list):
  705             cached_grains["osrelease_info"] = tuple(osrelease_info)
  706     return cached_grains
  707 
  708 
  709 def _load_cached_grains(opts, cfn):
  710     """
  711     Returns the grains cached in cfn, or None if the cache is too old or is
  712     corrupted.
  713     """
  714     if not os.path.isfile(cfn):
  715         log.debug("Grains cache file does not exist.")
  716         return None
  717 
  718     grains_cache_age = int(time.time() - os.path.getmtime(cfn))
  719     if grains_cache_age > opts.get("grains_cache_expiration", 300):
  720         log.debug(
  721             "Grains cache last modified %s seconds ago and cache "
  722             "expiration is set to %s. Grains cache expired. "
  723             "Refreshing.",
  724             grains_cache_age,
  725             opts.get("grains_cache_expiration", 300),
  726         )
  727         return None
  728 
  729     if opts.get("refresh_grains_cache", False):
  730         log.debug("refresh_grains_cache requested, Refreshing.")
  731         return None
  732 
  733     log.debug("Retrieving grains from cache")
  734     try:
  735         serial = salt.payload.Serial(opts)
  736         with salt.utils.files.fopen(cfn, "rb") as fp_:
  737             cached_grains = salt.utils.data.decode(
  738                 serial.load(fp_), preserve_tuples=True
  739             )
  740         if not cached_grains:
  741             log.debug("Cached grains are empty, cache might be corrupted. Refreshing.")
  742             return None
  743 
  744         return _format_cached_grains(cached_grains)
  745     except OSError:
  746         return None
  747 
  748 
  749 def grains(opts, force_refresh=False, proxy=None, context=None):
  750     """
  751     Return the functions for the dynamic grains and the values for the static
  752     grains.
  753 
  754     Since grains are computed early in the startup process, grains functions
  755     do not have __salt__ or __proxy__ available.  At proxy-minion startup,
  756     this function is called with the proxymodule LazyLoader object so grains
  757     functions can communicate with their controlled device.
  758 
  759     .. code-block:: python
  760 
  761         import salt.config
  762         import salt.loader
  763 
  764         __opts__ = salt.config.minion_config('/etc/salt/minion')
  765         __grains__ = salt.loader.grains(__opts__)
  766         print __grains__['id']
  767     """
  768     # Need to re-import salt.config, somehow it got lost when a minion is starting
  769     import salt.config
  770 
  771     # if we have no grains, lets try loading from disk (TODO: move to decorator?)
  772     cfn = os.path.join(opts["cachedir"], "grains.cache.p")
  773     if not force_refresh and opts.get("grains_cache", False):
  774         cached_grains = _load_cached_grains(opts, cfn)
  775         if cached_grains:
  776             return cached_grains
  777     else:
  778         log.debug("Grains refresh requested. Refreshing grains.")
  779 
  780     if opts.get("skip_grains", False):
  781         return {}
  782     grains_deep_merge = opts.get("grains_deep_merge", False) is True
  783     if "conf_file" in opts:
  784         pre_opts = {}
  785         pre_opts.update(
  786             salt.config.load_config(
  787                 opts["conf_file"],
  788                 "SALT_MINION_CONFIG",
  789                 salt.config.DEFAULT_MINION_OPTS["conf_file"],
  790             )
  791         )
  792         default_include = pre_opts.get("default_include", opts["default_include"])
  793         include = pre_opts.get("include", [])
  794         pre_opts.update(
  795             salt.config.include_config(
  796                 default_include, opts["conf_file"], verbose=False
  797             )
  798         )
  799         pre_opts.update(
  800             salt.config.include_config(include, opts["conf_file"], verbose=True)
  801         )
  802         if "grains" in pre_opts:
  803             opts["grains"] = pre_opts["grains"]
  804         else:
  805             opts["grains"] = {}
  806     else:
  807         opts["grains"] = {}
  808 
  809     grains_data = {}
  810     blist = opts.get("grains_blacklist", [])
  811     funcs = grain_funcs(opts, proxy=proxy, context=context or {})
  812     if force_refresh:  # if we refresh, lets reload grain modules
  813         funcs.clear()
  814     # Run core grains
  815     for key in funcs:
  816         if not key.startswith("core."):
  817             continue
  818         log.trace("Loading %s grain", key)
  819         ret = funcs[key]()
  820         if not isinstance(ret, dict):
  821             continue
  822         if blist:
  823             for key in list(ret):
  824                 for block in blist:
  825                     if salt.utils.stringutils.expr_match(key, block):
  826                         del ret[key]
  827                         log.trace("Filtering %s grain", key)
  828             if not ret:
  829                 continue
  830         if grains_deep_merge:
  831             salt.utils.dictupdate.update(grains_data, ret)
  832         else:
  833             grains_data.update(ret)
  834 
  835     # Run the rest of the grains
  836     for key in funcs:
  837         if key.startswith("core.") or key == "_errors":
  838             continue
  839         try:
  840             # Grains are loaded too early to take advantage of the injected
  841             # __proxy__ variable.  Pass an instance of that LazyLoader
  842             # here instead to grains functions if the grains functions take
  843             # one parameter.  Then the grains can have access to the
  844             # proxymodule for retrieving information from the connected
  845             # device.
  846             log.trace("Loading %s grain", key)
  847             parameters = salt.utils.args.get_function_argspec(funcs[key]).args
  848             kwargs = {}
  849             if "proxy" in parameters:
  850                 kwargs["proxy"] = proxy
  851             if "grains" in parameters:
  852                 kwargs["grains"] = grains_data
  853             ret = funcs[key](**kwargs)
  854         except Exception:  # pylint: disable=broad-except
  855             if salt.utils.platform.is_proxy():
  856                 log.info(
  857                     "The following CRITICAL message may not be an error; the proxy may not be completely established yet."
  858                 )
  859             log.critical(
  860                 "Failed to load grains defined in grain file %s in "
  861                 "function %s, error:\n",
  862                 key,
  863                 funcs[key],
  864                 exc_info=True,
  865             )
  866             continue
  867         if not isinstance(ret, dict):
  868             continue
  869         if blist:
  870             for key in list(ret):
  871                 for block in blist:
  872                     if salt.utils.stringutils.expr_match(key, block):
  873                         del ret[key]
  874                         log.trace("Filtering %s grain", key)
  875             if not ret:
  876                 continue
  877         if grains_deep_merge:
  878             salt.utils.dictupdate.update(grains_data, ret)
  879         else:
  880             grains_data.update(ret)
  881 
  882     if opts.get("proxy_merge_grains_in_module", True) and proxy:
  883         try:
  884             proxytype = proxy.opts["proxy"]["proxytype"]
  885             if proxytype + ".grains" in proxy:
  886                 if (
  887                     proxytype + ".initialized" in proxy
  888                     and proxy[proxytype + ".initialized"]()
  889                 ):
  890                     try:
  891                         proxytype = proxy.opts["proxy"]["proxytype"]
  892                         ret = proxy[proxytype + ".grains"]()
  893                         if grains_deep_merge:
  894                             salt.utils.dictupdate.update(grains_data, ret)
  895                         else:
  896                             grains_data.update(ret)
  897                     except Exception:  # pylint: disable=broad-except
  898                         log.critical(
  899                             "Failed to run proxy's grains function!", exc_info=True
  900                         )
  901         except KeyError:
  902             pass
  903 
  904     grains_data.update(opts["grains"])
  905     # Write cache if enabled
  906     if opts.get("grains_cache", False):
  907         with salt.utils.files.set_umask(0o077):
  908             try:
  909                 if salt.utils.platform.is_windows():
  910                     # Late import
  911                     import salt.modules.cmdmod
  912 
  913                     # Make sure cache file isn't read-only
  914                     salt.modules.cmdmod._run_quiet('attrib -R "{}"'.format(cfn))
  915                 with salt.utils.files.fopen(cfn, "w+b") as fp_:
  916                     try:
  917                         serial = salt.payload.Serial(opts)
  918                         serial.dump(grains_data, fp_)
  919                     except TypeError as e:
  920                         log.error("Failed to serialize grains cache: %s", e)
  921                         raise  # re-throw for cleanup
  922             except Exception as e:  # pylint: disable=broad-except
  923                 log.error("Unable to write to grains cache file %s: %s", cfn, e)
  924                 # Based on the original exception, the file may or may not have been
  925                 # created. If it was, we will remove it now, as the exception means
  926                 # the serialized data is not to be trusted, no matter what the
  927                 # exception is.
  928                 if os.path.isfile(cfn):
  929                     os.unlink(cfn)
  930 
  931     if grains_deep_merge:
  932         salt.utils.dictupdate.update(grains_data, opts["grains"])
  933     else:
  934         grains_data.update(opts["grains"])
  935     return salt.utils.data.decode(grains_data, preserve_tuples=True)
  936 
  937 
  938 # TODO: get rid of? Does anyone use this? You should use raw() instead
  939 def call(fun, **kwargs):
  940     """
  941     Directly call a function inside a loader directory
  942     """
  943     args = kwargs.get("args", [])
  944     dirs = kwargs.get("dirs", [])
  945 
  946     funcs = LazyLoader(
  947         [os.path.join(SALT_BASE_PATH, "modules")] + dirs,
  948         None,
  949         tag="modules",
  950         virtual_enable=False,
  951     )
  952     return funcs[fun](*args)
  953 
  954 
  955 def runner(opts, utils=None, context=None, whitelist=None):
  956     """
  957     Directly call a function inside a loader directory
  958     """
  959     if utils is None:
  960         utils = {}
  961     if context is None:
  962         context = {}
  963     ret = LazyLoader(
  964         _module_dirs(opts, "runners", "runner", ext_type_dirs="runner_dirs"),
  965         opts,
  966         tag="runners",
  967         pack={"__utils__": utils, "__context__": context},
  968         whitelist=whitelist,
  969         extra_module_dirs=utils.module_dirs if utils else None,
  970     )
  971     # TODO: change from __salt__ to something else, we overload __salt__ too much
  972     ret.pack["__salt__"] = ret
  973     return ret
  974 
  975 
  976 def queues(opts):
  977     """
  978     Directly call a function inside a loader directory
  979     """
  980     return LazyLoader(
  981         _module_dirs(opts, "queues", "queue", ext_type_dirs="queue_dirs"),
  982         opts,
  983         tag="queues",
  984     )
  985 
  986 
  987 def sdb(opts, functions=None, whitelist=None, utils=None):
  988     """
  989     Make a very small database call
  990     """
  991     if utils is None:
  992         utils = {}
  993 
  994     return LazyLoader(
  995         _module_dirs(opts, "sdb"),
  996         opts,
  997         tag="sdb",
  998         pack={
  999             "__sdb__": functions,
 1000             "__opts__": opts,
 1001             "__utils__": utils,
 1002             "__salt__": minion_mods(opts, utils=utils),
 1003         },
 1004         whitelist=whitelist,
 1005         extra_module_dirs=utils.module_dirs if utils else None,
 1006     )
 1007 
 1008 
 1009 def pkgdb(opts):
 1010     """
 1011     Return modules for SPM's package database
 1012 
 1013     .. versionadded:: 2015.8.0
 1014     """
 1015     return LazyLoader(
 1016         _module_dirs(opts, "pkgdb", base_path=os.path.join(SALT_BASE_PATH, "spm")),
 1017         opts,
 1018         tag="pkgdb",
 1019     )
 1020 
 1021 
 1022 def pkgfiles(opts):
 1023     """
 1024     Return modules for SPM's file handling
 1025 
 1026     .. versionadded:: 2015.8.0
 1027     """
 1028     return LazyLoader(
 1029         _module_dirs(opts, "pkgfiles", base_path=os.path.join(SALT_BASE_PATH, "spm")),
 1030         opts,
 1031         tag="pkgfiles",
 1032     )
 1033 
 1034 
 1035 def clouds(opts):
 1036     """
 1037     Return the cloud functions
 1038     """
 1039     _utils = salt.loader.utils(opts)
 1040     # Let's bring __active_provider_name__, defaulting to None, to all cloud
 1041     # drivers. This will get temporarily updated/overridden with a context
 1042     # manager when needed.
 1043     functions = LazyLoader(
 1044         _module_dirs(
 1045             opts,
 1046             "clouds",
 1047             "cloud",
 1048             base_path=os.path.join(SALT_BASE_PATH, "cloud"),
 1049             int_type="clouds",
 1050         ),
 1051         opts,
 1052         tag="clouds",
 1053         pack={"__utils__": _utils, "__active_provider_name__": None},
 1054         extra_module_dirs=_utils.module_dirs,
 1055     )
 1056     for funcname in LIBCLOUD_FUNCS_NOT_SUPPORTED:
 1057         log.trace(
 1058             "'%s' has been marked as not supported. Removing from the "
 1059             "list of supported cloud functions",
 1060             funcname,
 1061         )
 1062         functions.pop(funcname, None)
 1063     return functions
 1064 
 1065 
 1066 def netapi(opts):
 1067     """
 1068     Return the network api functions
 1069     """
 1070     return LazyLoader(_module_dirs(opts, "netapi"), opts, tag="netapi",)
 1071 
 1072 
 1073 def executors(opts, functions=None, context=None, proxy=None):
 1074     """
 1075     Returns the executor modules
 1076     """
 1077     executors = LazyLoader(
 1078         _module_dirs(opts, "executors", "executor"),
 1079         opts,
 1080         tag="executor",
 1081         pack={
 1082             "__salt__": functions,
 1083             "__context__": context or {},
 1084             "__proxy__": proxy or {},
 1085         },
 1086     )
 1087     executors.pack["__executors__"] = executors
 1088     return executors
 1089 
 1090 
 1091 def cache(opts, serial):
 1092     """
 1093     Returns the returner modules
 1094     """
 1095     return LazyLoader(
 1096         _module_dirs(opts, "cache", "cache"),
 1097         opts,
 1098         tag="cache",
 1099         pack={"__opts__": opts, "__context__": {"serial": serial}},
 1100     )
 1101 
 1102 
 1103 def _generate_module(name):
 1104     if name in sys.modules:
 1105         return
 1106 
 1107     code = "'''Salt loaded {} parent module'''".format(name.split(".")[-1])
 1108     # ModuleType can't accept a unicode type on PY2
 1109     module = types.ModuleType(str(name))  # future lint: disable=blacklisted-function
 1110     exec(code, module.__dict__)
 1111     sys.modules[name] = module
 1112 
 1113 
 1114 def _mod_type(module_path):
 1115     if module_path.startswith(SALT_BASE_PATH):
 1116         return "int"
 1117     return "ext"
 1118 
 1119 
 1120 # TODO: move somewhere else?
 1121 class FilterDictWrapper(MutableMapping):
 1122     """
 1123     Create a dict which wraps another dict with a specific key suffix on get
 1124 
 1125     This is to replace "filter_load"
 1126     """
 1127 
 1128     def __init__(self, d, suffix):
 1129         self._dict = d
 1130         self.suffix = suffix
 1131 
 1132     def __setitem__(self, key, val):
 1133         self._dict[key] = val
 1134 
 1135     def __delitem__(self, key):
 1136         del self._dict[key]
 1137 
 1138     def __getitem__(self, key):
 1139         return self._dict[key + self.suffix]
 1140 
 1141     def __len__(self):
 1142         return len(self._dict)
 1143 
 1144     def __iter__(self):
 1145         for key in self._dict:
 1146             if key.endswith(self.suffix):
 1147                 yield key.replace(self.suffix, "")
 1148 
 1149 
 1150 class LazyLoader(salt.utils.lazy.LazyDict):
 1151     """
 1152     A pseduo-dictionary which has a set of keys which are the
 1153     name of the module and function, delimited by a dot. When
 1154     the value of the key is accessed, the function is then loaded
 1155     from disk and into memory.
 1156 
 1157     .. note::
 1158 
 1159         Iterating over keys will cause all modules to be loaded.
 1160 
 1161     :param list module_dirs: A list of directories on disk to search for modules
 1162     :param dict opts: The salt options dictionary.
 1163     :param str tag: The tag for the type of module to load
 1164     :param func mod_type_check: A function which can be used to verify files
 1165     :param dict pack: A dictionary of function to be packed into modules as they are loaded
 1166     :param list whitelist: A list of modules to whitelist
 1167     :param bool virtual_enable: Whether or not to respect the __virtual__ function when loading modules.
 1168     :param str virtual_funcs: The name of additional functions in the module to call to verify its functionality.
 1169                                 If not true, the module will not load.
 1170     :param list extra_module_dirs: A list of directories that will be able to import from
 1171     :returns: A LazyLoader object which functions as a dictionary. Keys are 'module.function' and values
 1172     are function references themselves which are loaded on-demand.
 1173     # TODO:
 1174         - move modules_max_memory into here
 1175         - singletons (per tag)
 1176     """
 1177 
 1178     mod_dict_class = salt.utils.odict.OrderedDict
 1179 
 1180     def __init__(
 1181         self,
 1182         module_dirs,
 1183         opts=None,
 1184         tag="module",
 1185         loaded_base_name=None,
 1186         mod_type_check=None,
 1187         pack=None,
 1188         whitelist=None,
 1189         virtual_enable=True,
 1190         static_modules=None,
 1191         proxy=None,
 1192         virtual_funcs=None,
 1193         extra_module_dirs=None,
 1194     ):  # pylint: disable=W0231
 1195         """
 1196         In pack, if any of the values are None they will be replaced with an
 1197         empty context-specific dict
 1198         """
 1199 
 1200         self.inject_globals = {}
 1201         self.pack = {} if pack is None else pack
 1202         if opts is None:
 1203             opts = {}
 1204         threadsafety = not opts.get("multiprocessing")
 1205         self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety)
 1206         self.opts = self.__prep_mod_opts(opts)
 1207 
 1208         self.module_dirs = module_dirs
 1209         self.tag = tag
 1210         self._gc_finalizer = None
 1211         if loaded_base_name and loaded_base_name != LOADED_BASE_NAME:
 1212             self.loaded_base_name = loaded_base_name
 1213         else:
 1214             self.loaded_base_name = LOADED_BASE_NAME
 1215         self.mod_type_check = mod_type_check or _mod_type
 1216 
 1217         if "__context__" not in self.pack:
 1218             self.pack["__context__"] = None
 1219 
 1220         for k, v in self.pack.items():
 1221             if v is None:  # if the value of a pack is None, lets make an empty dict
 1222                 self.context_dict.setdefault(k, {})
 1223                 self.pack[k] = salt.utils.context.NamespacedDictWrapper(
 1224                     self.context_dict, k
 1225                 )
 1226 
 1227         self.whitelist = whitelist
 1228         self.virtual_enable = virtual_enable
 1229         self.initial_load = True
 1230 
 1231         # names of modules that we don't have (errors, __virtual__, etc.)
 1232         self.missing_modules = {}  # mapping of name -> error
 1233         self.loaded_modules = {}  # mapping of module_name -> dict_of_functions
 1234         self.loaded_files = set()  # TODO: just remove them from file_mapping?
 1235         self.static_modules = static_modules if static_modules else []
 1236 
 1237         if virtual_funcs is None:
 1238             virtual_funcs = []
 1239         self.virtual_funcs = virtual_funcs
 1240 
 1241         self.extra_module_dirs = extra_module_dirs if extra_module_dirs else []
 1242         self._clean_module_dirs = []
 1243 
 1244         self.disabled = set(
 1245             self.opts.get(
 1246                 "disable_{}{}".format(self.tag, "" if self.tag[-1] == "s" else "s"), [],
 1247             )
 1248         )
 1249 
 1250         # A map of suffix to description for imp
 1251         self.suffix_map = {}
 1252         # A list to determine precedence of extensions
 1253         # Prefer packages (directories) over modules (single files)!
 1254         self.suffix_order = [""]
 1255         for (suffix, mode, kind) in SUFFIXES:
 1256             self.suffix_map[suffix] = (suffix, mode, kind)
 1257             self.suffix_order.append(suffix)
 1258 
 1259         self._lock = threading.RLock()
 1260         with self._lock:
 1261             self._refresh_file_mapping()
 1262 
 1263         super().__init__()  # late init the lazy loader
 1264         # create all of the import namespaces
 1265         _generate_module("{}.int".format(self.loaded_base_name))
 1266         _generate_module("{}.int.{}".format(self.loaded_base_name, tag))
 1267         _generate_module("{}.ext".format(self.loaded_base_name))
 1268         _generate_module("{}.ext.{}".format(self.loaded_base_name, tag))
 1269 
 1270     def clean_modules(self):
 1271         """
 1272         Clean modules
 1273         """
 1274         for name in list(sys.modules):
 1275             if name.startswith(self.loaded_base_name):
 1276                 del sys.modules[name]
 1277 
 1278     def __getitem__(self, item):
 1279         """
 1280         Override the __getitem__ in order to decorate the returned function if we need
 1281         to last-minute inject globals
 1282         """
 1283         func = super().__getitem__(item)
 1284         if self.inject_globals:
 1285             return global_injector_decorator(self.inject_globals)(func)
 1286         else:
 1287             return func
 1288 
 1289     def __getattr__(self, mod_name):
 1290         """
 1291         Allow for "direct" attribute access-- this allows jinja templates to
 1292         access things like `salt.test.ping()`
 1293         """
 1294         if mod_name in ("__getstate__", "__setstate__"):
 1295             return object.__getattribute__(self, mod_name)
 1296 
 1297         # if we have an attribute named that, lets return it.
 1298         try:
 1299             return object.__getattr__(self, mod_name)  # pylint: disable=no-member
 1300         except AttributeError:
 1301             pass
 1302 
 1303         # otherwise we assume its jinja template access
 1304         if mod_name not in self.loaded_modules and not self.loaded:
 1305             for name in self._iter_files(mod_name):
 1306                 if name in self.loaded_files:
 1307                     continue
 1308                 # if we got what we wanted, we are done
 1309                 if self._load_module(name) and mod_name in self.loaded_modules:
 1310                     break
 1311         if mod_name in self.loaded_modules:
 1312             return self.loaded_modules[mod_name]
 1313         else:
 1314             raise AttributeError(mod_name)
 1315 
 1316     def missing_fun_string(self, function_name):
 1317         """
 1318         Return the error string for a missing function.
 1319 
 1320         This can range from "not available' to "__virtual__" returned False
 1321         """
 1322         mod_name = function_name.split(".")[0]
 1323         if mod_name in self.loaded_modules:
 1324             return "'{}' is not available.".format(function_name)
 1325         else:
 1326             try:
 1327                 reason = self.missing_modules[mod_name]
 1328             except KeyError:
 1329                 return "'{}' is not available.".format(function_name)
 1330             else:
 1331                 if reason is not None:
 1332                     return "'{}' __virtual__ returned False: {}".format(
 1333                         mod_name, reason
 1334                     )
 1335                 else:
 1336                     return "'{}' __virtual__ returned False".format(mod_name)
 1337 
 1338     def _refresh_file_mapping(self):
 1339         """
 1340         refresh the mapping of the FS on disk
 1341         """
 1342         # map of suffix to description for imp
 1343         if (
 1344             self.opts.get("cython_enable", True) is True
 1345             and ".pyx" not in self.suffix_map
 1346         ):
 1347             try:
 1348                 global pyximport
 1349                 pyximport = __import__("pyximport")  # pylint: disable=import-error
 1350                 pyximport.install()
 1351                 # add to suffix_map so file_mapping will pick it up
 1352                 self.suffix_map[".pyx"] = tuple()
 1353                 if ".pyx" not in self.suffix_order:
 1354                     self.suffix_order.append(".pyx")
 1355             except ImportError:
 1356                 log.info(
 1357                     "Cython is enabled in the options but not present "
 1358                     "in the system path. Skipping Cython modules."
 1359                 )
 1360         # Allow for zipimport of modules
 1361         if (
 1362             self.opts.get("enable_zip_modules", True) is True
 1363             and ".zip" not in self.suffix_map
 1364         ):
 1365             self.suffix_map[".zip"] = tuple()
 1366             if ".zip" not in self.suffix_order:
 1367                 self.suffix_order.append(".zip")
 1368         # allow for module dirs
 1369         self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY)
 1370 
 1371         # create mapping of filename (without suffix) to (path, suffix)
 1372         # The files are added in order of priority, so order *must* be retained.
 1373         self.file_mapping = salt.utils.odict.OrderedDict()
 1374 
 1375         opt_match = []
 1376 
 1377         def _replace_pre_ext(obj):
 1378             """
 1379             Hack so we can get the optimization level that we replaced (if
 1380             any) out of the re.sub call below. We use a list here because
 1381             it is a persistent data structure that we will be able to
 1382             access after re.sub is called.
 1383             """
 1384             opt_match.append(obj)
 1385             return ""
 1386 
 1387         for mod_dir in self.module_dirs:
 1388             try:
 1389                 # Make sure we have a sorted listdir in order to have
 1390                 # expectable override results
 1391                 files = sorted(x for x in os.listdir(mod_dir) if x != "__pycache__")
 1392             except OSError:
 1393                 continue  # Next mod_dir
 1394             try:
 1395                 pycache_files = [
 1396                     os.path.join("__pycache__", x)
 1397                     for x in sorted(os.listdir(os.path.join(mod_dir, "__pycache__")))
 1398                 ]
 1399             except OSError:
 1400                 pass
 1401             else:
 1402                 files.extend(pycache_files)
 1403 
 1404             for filename in files:
 1405                 try:
 1406                     dirname, basename = os.path.split(filename)
 1407                     if basename.startswith("_"):
 1408                         # skip private modules
 1409                         # log messages omitted for obviousness
 1410                         continue  # Next filename
 1411                     f_noext, ext = os.path.splitext(basename)
 1412                     f_noext = PY3_PRE_EXT.sub(_replace_pre_ext, f_noext)
 1413                     try:
 1414                         opt_level = int(opt_match.pop().group(1).rsplit("-", 1)[-1])
 1415                     except (AttributeError, IndexError, ValueError):
 1416                         # No regex match or no optimization level matched
 1417                         opt_level = 0
 1418                     try:
 1419                         opt_index = self.opts["optimization_order"].index(opt_level)
 1420                     except KeyError:
 1421                         log.trace(
 1422                             "Disallowed optimization level %d for module "
 1423                             "name '%s', skipping. Add %d to the "
 1424                             "'optimization_order' config option if you "
 1425                             "do not want to ignore this optimization "
 1426                             "level.",
 1427                             opt_level,
 1428                             f_noext,
 1429                             opt_level,
 1430                         )
 1431                         continue
 1432 
 1433                     # make sure it is a suffix we support
 1434                     if ext not in self.suffix_map:
 1435                         continue  # Next filename
 1436                     if f_noext in self.disabled:
 1437                         log.trace(
 1438                             "Skipping %s, it is disabled by configuration", filename
 1439                         )
 1440                         continue  # Next filename
 1441                     fpath = os.path.join(mod_dir, filename)
 1442                     # if its a directory, lets allow us to load that
 1443                     if ext == "":
 1444                         # is there something __init__?
 1445                         subfiles = os.listdir(fpath)
 1446                         for suffix in self.suffix_order:
 1447                             if "" == suffix:
 1448                                 continue  # Next suffix (__init__ must have a suffix)
 1449                             init_file = "__init__{}".format(suffix)
 1450                             if init_file in subfiles:
 1451                                 break
 1452                         else:
 1453                             continue  # Next filename
 1454 
 1455                     try:
 1456                         curr_ext = self.file_mapping[f_noext][1]
 1457                         curr_opt_index = self.file_mapping[f_noext][2]
 1458                     except KeyError:
 1459                         pass
 1460                     else:
 1461                         if "" in (curr_ext, ext) and curr_ext != ext:
 1462                             log.error(
 1463                                 "Module/package collision: '%s' and '%s'",
 1464                                 fpath,
 1465                                 self.file_mapping[f_noext][0],
 1466                             )
 1467 
 1468                         if six.PY3 and ext == ".pyc" and curr_ext == ".pyc":
 1469                             # Check the optimization level
 1470                             if opt_index >= curr_opt_index:
 1471                                 # Module name match, but a higher-priority
 1472                                 # optimization level was already matched, skipping.
 1473                                 continue
 1474                         elif not curr_ext or self.suffix_order.index(
 1475                             ext
 1476                         ) >= self.suffix_order.index(curr_ext):
 1477                             # Match found but a higher-priorty match already
 1478                             # exists, so skip this.
 1479                             continue
 1480 
 1481                     if six.PY3 and not dirname and ext == ".pyc":
 1482                         # On Python 3, we should only load .pyc files from the
 1483                         # __pycache__ subdirectory (i.e. when dirname is not an
 1484                         # empty string).
 1485                         continue
 1486 
 1487                     # Made it this far - add it
 1488                     self.file_mapping[f_noext] = (fpath, ext, opt_index)
 1489 
 1490                 except OSError:
 1491                     continue
 1492         for smod in self.static_modules:
 1493             f_noext = smod.split(".")[-1]
 1494             self.file_mapping[f_noext] = (smod, ".o", 0)
 1495 
 1496     def clear(self):
 1497         """
 1498         Clear the dict
 1499         """
 1500         with self._lock:
 1501             super().clear()  # clear the lazy loader
 1502             self.loaded_files = set()
 1503             self.missing_modules = {}
 1504             self.loaded_modules = {}
 1505             # if we have been loaded before, lets clear the file mapping since
 1506             # we obviously want a re-do
 1507             if hasattr(self, "opts"):
 1508                 self._refresh_file_mapping()
 1509             self.initial_load = False
 1510 
 1511     def __prep_mod_opts(self, opts):
 1512         """
 1513         Strip out of the opts any logger instance
 1514         """
 1515         if "__grains__" not in self.pack:
 1516             self.context_dict["grains"] = opts.get("grains", {})
 1517             self.pack["__grains__"] = salt.utils.context.NamespacedDictWrapper(
 1518                 self.context_dict, "grains"
 1519             )
 1520 
 1521         if "__pillar__" not in self.pack:
 1522             self.context_dict["pillar"] = opts.get("pillar", {})
 1523             self.pack["__pillar__"] = salt.utils.context.NamespacedDictWrapper(
 1524                 self.context_dict, "pillar"
 1525             )
 1526 
 1527         mod_opts = {}
 1528         for key, val in list(opts.items()):
 1529             if key == "logger":
 1530                 continue
 1531             mod_opts[key] = val
 1532         return mod_opts
 1533 
 1534     def _iter_files(self, mod_name):
 1535         """
 1536         Iterate over all file_mapping files in order of closeness to mod_name
 1537         """
 1538         # do we have an exact match?
 1539         if mod_name in self.file_mapping:
 1540             yield mod_name
 1541 
 1542         # do we have a partial match?
 1543         for k in self.file_mapping:
 1544             if mod_name in k:
 1545                 yield k
 1546 
 1547         # anyone else? Bueller?
 1548         for k in self.file_mapping:
 1549             if mod_name not in k:
 1550                 yield k
 1551 
 1552     def _reload_submodules(self, mod):
 1553         submodules = (
 1554             getattr(mod, sname)
 1555             for sname in dir(mod)
 1556             if isinstance(getattr(mod, sname), mod.__class__)
 1557         )
 1558 
 1559         # reload only custom "sub"modules
 1560         for submodule in submodules:
 1561             # it is a submodule if the name is in a namespace under mod
 1562             if submodule.__name__.startswith(mod.__name__ + "."):
 1563                 reload_module(submodule)
 1564                 self._reload_submodules(submodule)
 1565 
 1566     def __populate_sys_path(self):
 1567         for directory in self.extra_module_dirs:
 1568             if directory not in sys.path:
 1569                 sys.path.append(directory)
 1570                 self._clean_module_dirs.append(directory)
 1571 
 1572     def __clean_sys_path(self):
 1573         invalidate_path_importer_cache = False
 1574         for directory in self._clean_module_dirs:
 1575             if directory in sys.path:
 1576                 sys.path.remove(directory)
 1577                 invalidate_path_importer_cache = True
 1578         self._clean_module_dirs = []
 1579 
 1580         # Be sure that sys.path_importer_cache do not contains any
 1581         # invalid FileFinder references
 1582         importlib.invalidate_caches()
 1583 
 1584         # Because we are mangling with importlib, we can find from
 1585         # time to time an invalidation issue with
 1586         # sys.path_importer_cache, that requires the removal of
 1587         # FileFinder that remain None for the extra_module_dirs
 1588         if invalidate_path_importer_cache:
 1589             for directory in self.extra_module_dirs:
 1590                 if (
 1591                     directory in sys.path_importer_cache
 1592                     and sys.path_importer_cache[directory] is None
 1593                 ):
 1594                     del sys.path_importer_cache[directory]
 1595 
 1596     def _load_module(self, name):
 1597         mod = None
 1598         fpath, suffix = self.file_mapping[name][:2]
 1599         # if the fpath has `.cpython-3x` in it, but the running Py version
 1600         # is 3.y, the following will cause us to return immediately and we won't try to import this .pyc.
 1601         # This is for the unusual case where several Python versions share a single
 1602         # source tree and drop their .pycs in the same __pycache__ folder.
 1603         # If we were to load a .pyc for another Py version it's not a big problem
 1604         # but the log will get spammed with "Bad Magic Number" messages that
 1605         # can be very misleading if the user is debugging another problem.
 1606         try:
 1607             (implementation_tag, cache_tag_ver) = sys.implementation.cache_tag.split(
 1608                 "-"
 1609             )
 1610             if cache_tag_ver not in fpath and implementation_tag in fpath:
 1611                 log.trace(
 1612                     "Trying to load %s on %s, returning False.",
 1613                     fpath,
 1614                     sys.implementation.cache_tag,
 1615                 )
 1616                 return False
 1617         except AttributeError:
 1618             # Most likely Py 2.7 or some other Python version we don't really support
 1619             pass
 1620 
 1621         self.loaded_files.add(name)
 1622         fpath_dirname = os.path.dirname(fpath)
 1623         try:
 1624             self.__populate_sys_path()
 1625             sys.path.append(fpath_dirname)
 1626             if suffix == ".pyx":
 1627                 mod = pyximport.load_module(name, fpath, tempfile.gettempdir())
 1628             elif suffix == ".o":
 1629                 top_mod = __import__(fpath, globals(), locals(), [])
 1630                 comps = fpath.split(".")
 1631                 if len(comps) < 2:
 1632                     mod = top_mod
 1633                 else:
 1634                     mod = top_mod
 1635                     for subname in comps[1:]:
 1636                         mod = getattr(mod, subname)
 1637             elif suffix == ".zip":
 1638                 mod = zipimporter(fpath).load_module(name)
 1639             else:
 1640                 desc = self.suffix_map[suffix]
 1641                 # if it is a directory, we don't open a file
 1642                 try:
 1643                     mod_namespace = ".".join(
 1644                         (
 1645                             self.loaded_base_name,
 1646                             self.mod_type_check(fpath),
 1647                             self.tag,
 1648                             name,
 1649                         )
 1650                     )
 1651                 except TypeError:
 1652                     mod_namespace = "{}.{}.{}.{}".format(
 1653                         self.loaded_base_name,
 1654                         self.mod_type_check(fpath),
 1655                         self.tag,
 1656                         name,
 1657                     )
 1658                 if suffix == "":
 1659                     # pylint: disable=no-member
 1660                     # Package directory, look for __init__
 1661                     loader_details = [
 1662                         (
 1663                             importlib.machinery.SourceFileLoader,
 1664                             importlib.machinery.SOURCE_SUFFIXES,
 1665                         ),
 1666                         (
 1667                             importlib.machinery.SourcelessFileLoader,
 1668                             importlib.machinery.BYTECODE_SUFFIXES,
 1669                         ),
 1670                         (
 1671                             importlib.machinery.ExtensionFileLoader,
 1672                             importlib.machinery.EXTENSION_SUFFIXES,
 1673                         ),
 1674                     ]
 1675                     file_finder = importlib.machinery.FileFinder(
 1676                         fpath_dirname, *loader_details
 1677                     )
 1678                     spec = file_finder.find_spec(mod_namespace)
 1679                     if spec is None:
 1680                         raise ImportError()
 1681                     # TODO: Get rid of load_module in favor of
 1682                     # exec_module below. load_module is deprecated, but
 1683                     # loading using exec_module has been causing odd things
 1684                     # with the magic dunders we pack into the loaded
 1685                     # modules, most notably with salt-ssh's __opts__.
 1686                     mod = spec.loader.load_module()
 1687                     # mod = importlib.util.module_from_spec(spec)
 1688                     # spec.loader.exec_module(mod)
 1689                     # pylint: enable=no-member
 1690                     sys.modules[mod_namespace] = mod
 1691                     # reload all submodules if necessary
 1692                     if not self.initial_load:
 1693                         self._reload_submodules(mod)
 1694                 else:
 1695                     # pylint: disable=no-member
 1696                     loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath)
 1697                     spec = importlib.util.spec_from_file_location(
 1698                         mod_namespace, fpath, loader=loader
 1699                     )
 1700                     if spec is None:
 1701                         raise ImportError()
 1702                     # TODO: Get rid of load_module in favor of
 1703                     # exec_module below. load_module is deprecated, but
 1704                     # loading using exec_module has been causing odd things
 1705                     # with the magic dunders we pack into the loaded
 1706                     # modules, most notably with salt-ssh's __opts__.
 1707                     mod = spec.loader.load_module()
 1708                     # mod = importlib.util.module_from_spec(spec)
 1709                     # spec.loader.exec_module(mod)
 1710                     # pylint: enable=no-member
 1711                     sys.modules[mod_namespace] = mod
 1712         except OSError:
 1713             raise
 1714         except ImportError as exc:
 1715             if "magic number" in str(exc):
 1716                 error_msg = "Failed to import {} {}. Bad magic number. If migrating from Python2 to Python3, remove all .pyc files and try again.".format(
 1717                     self.tag, name
 1718                 )
 1719                 log.warning(error_msg)
 1720                 self.missing_modules[name] = error_msg
 1721             log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True)
 1722             self.missing_modules[name] = exc
 1723             return False
 1724         except Exception as error:  # pylint: disable=broad-except
 1725             log.error(
 1726                 "Failed to import %s %s, this is due most likely to a "
 1727                 "syntax error:\n",
 1728                 self.tag,
 1729                 name,
 1730                 exc_info=True,
 1731             )
 1732             self.missing_modules[name] = error
 1733             return False
 1734         except SystemExit as error:
 1735             try:
 1736                 fn_, _, caller, _ = traceback.extract_tb(sys.exc_info()[2])[-1]
 1737             except Exception:  # pylint: disable=broad-except
 1738                 pass
 1739             else:
 1740                 tgt_fn = os.path.join("salt", "utils", "process.py")
 1741                 if fn_.endswith(tgt_fn) and "_handle_signals" in caller:
 1742                     # Race conditon, SIGTERM or SIGINT received while loader
 1743                     # was in process of loading a module. Call sys.exit to
 1744                     # ensure that the process is killed.
 1745                     sys.exit(salt.defaults.exitcodes.EX_OK)
 1746             log.error(
 1747                 "Failed to import %s %s as the module called exit()\n",
 1748                 self.tag,
 1749                 name,
 1750                 exc_info=True,
 1751             )
 1752             self.missing_modules[name] = error
 1753             return False
 1754         finally:
 1755             sys.path.remove(fpath_dirname)
 1756             self.__clean_sys_path()
 1757 
 1758         if hasattr(mod, "__opts__"):
 1759             mod.__opts__.update(self.opts)
 1760         else:
 1761             mod.__opts__ = self.opts
 1762 
 1763         # pack whatever other globals we were asked to
 1764         for p_name, p_value in self.pack.items():
 1765             setattr(mod, p_name, p_value)
 1766 
 1767         module_name = mod.__name__.rsplit(".", 1)[-1]
 1768 
 1769         # Call a module's initialization method if it exists
 1770         module_init = getattr(mod, "__init__", None)
 1771         if inspect.isfunction(module_init):
 1772             try:
 1773                 module_init(self.opts)
 1774             except TypeError as e:
 1775                 log.error(e)
 1776             except Exception:  # pylint: disable=broad-except
 1777                 err_string = "__init__ failed"
 1778                 log.debug(
 1779                     "Error loading %s.%s: %s",
 1780                     self.tag,
 1781                     module_name,
 1782                     err_string,
 1783                     exc_info=True,
 1784                 )
 1785                 self.missing_modules[module_name] = err_string
 1786                 self.missing_modules[name] = err_string
 1787                 return False
 1788 
 1789         # if virtual modules are enabled, we need to look for the
 1790         # __virtual__() function inside that module and run it.
 1791         if self.virtual_enable:
 1792             virtual_funcs_to_process = ["__virtual__"] + self.virtual_funcs
 1793             for virtual_func in virtual_funcs_to_process:
 1794                 (
 1795                     virtual_ret,
 1796                     module_name,
 1797                     virtual_err,
 1798                     virtual_aliases,
 1799                 ) = self._process_virtual(mod, module_name, virtual_func)
 1800                 if virtual_err is not None:
 1801                     log.trace(
 1802                         "Error loading %s.%s: %s", self.tag, module_name, virtual_err
 1803                     )
 1804 
 1805                 # if _process_virtual returned a non-True value then we are
 1806                 # supposed to not process this module
 1807                 if virtual_ret is not True and module_name not in self.missing_modules:
 1808                     # If a module has information about why it could not be loaded, record it
 1809                     self.missing_modules[module_name] = virtual_err
 1810                     self.missing_modules[name] = virtual_err
 1811                     return False
 1812         else:
 1813             virtual_aliases = ()
 1814 
 1815         # If this is a proxy minion then MOST modules cannot work. Therefore, require that
 1816         # any module that does work with salt-proxy-minion define __proxyenabled__ as a list
 1817         # containing the names of the proxy types that the module supports.
 1818         #
 1819         # Render modules and state modules are OK though
 1820         if "proxy" in self.opts:
 1821             if self.tag in ["grains", "proxy"]:
 1822                 if not hasattr(mod, "__proxyenabled__") or (
 1823                     self.opts["proxy"]["proxytype"] not in mod.__proxyenabled__
 1824                     and "*" not in mod.__proxyenabled__
 1825                 ):
 1826                     err_string = "not a proxy_minion enabled module"
 1827                     self.missing_modules[module_name] = err_string
 1828                     self.missing_modules[name] = err_string
 1829                     return False
 1830 
 1831         if getattr(mod, "__load__", False) is not False:
 1832             log.info(
 1833                 "The functions from module '%s' are being loaded from the "
 1834                 "provided __load__ attribute",
 1835                 module_name,
 1836             )
 1837 
 1838         # If we had another module by the same virtual name, we should put any
 1839         # new functions under the existing dictionary.
 1840         mod_names = [module_name] + list(virtual_aliases)
 1841         mod_dict = {
 1842             x: self.loaded_modules.get(x, self.mod_dict_class()) for x in mod_names
 1843         }
 1844 
 1845         for attr in getattr(mod, "__load__", dir(mod)):
 1846             if attr.startswith("_"):
 1847                 # private functions are skipped
 1848                 continue
 1849             func = getattr(mod, attr)
 1850             if not inspect.isfunction(func) and not isinstance(func, functools.partial):
 1851                 # Not a function!? Skip it!!!
 1852                 continue
 1853             # Let's get the function name.
 1854             # If the module has the __func_alias__ attribute, it must be a
 1855             # dictionary mapping in the form of(key -> value):
 1856             #   <real-func-name> -> <desired-func-name>
 1857             #
 1858             # It default's of course to the found callable attribute name
 1859             # if no alias is defined.
 1860             funcname = getattr(mod, "__func_alias__", {}).get(attr, attr)
 1861             for tgt_mod in mod_names:
 1862                 try:
 1863                     full_funcname = ".".join((tgt_mod, funcname))
 1864                 except TypeError:
 1865                     full_funcname = "{}.{}".format(tgt_mod, funcname)
 1866                 # Save many references for lookups
 1867                 # Careful not to overwrite existing (higher priority) functions
 1868                 if full_funcname not in self._dict:
 1869                     self._dict[full_funcname] = func
 1870                 if funcname not in mod_dict[tgt_mod]:
 1871                     setattr(mod_dict[tgt_mod], funcname, func)
 1872                     mod_dict[tgt_mod][funcname] = func
 1873                     self._apply_outputter(func, mod)
 1874 
 1875         # enforce depends
 1876         try:
 1877             Depends.enforce_dependencies(self._dict, self.tag, name)
 1878         except RuntimeError as exc:
 1879             log.info(
 1880                 "Depends.enforce_dependencies() failed for the following " "reason: %s",
 1881                 exc,
 1882             )
 1883 
 1884         for tgt_mod in mod_names:
 1885             self.loaded_modules[tgt_mod] = mod_dict[tgt_mod]
 1886         return True
 1887 
 1888     def _load(self, key):
 1889         """
 1890         Load a single item if you have it
 1891         """
 1892         # if the key doesn't have a '.' then it isn't valid for this mod dict
 1893         if not isinstance(key, str):
 1894             raise KeyError("The key must be a string.")
 1895         if "." not in key:
 1896             raise KeyError("The key '{}' should contain a '.'".format(key))
 1897         mod_name, _ = key.split(".", 1)
 1898         with self._lock:
 1899             # It is possible that the key is in the dictionary after
 1900             # acquiring the lock due to another thread loading it.
 1901             if mod_name in self.missing_modules or key in self._dict:
 1902                 return True
 1903             # if the modulename isn't in the whitelist, don't bother
 1904             if self.whitelist and mod_name not in self.whitelist:
 1905                 log.error(
 1906                     "Failed to load function %s because its module (%s) is "
 1907                     "not in the whitelist: %s",
 1908                     key,
 1909                     mod_name,
 1910                     self.whitelist,
 1911                 )
 1912                 raise KeyError(key)
 1913 
 1914             def _inner_load(mod_name):
 1915                 for name in self._iter_files(mod_name):
 1916                     if name in self.loaded_files:
 1917                         continue
 1918                     # if we got what we wanted, we are done
 1919                     if self._load_module(name) and key in self._dict:
 1920                         return True
 1921                 return False
 1922 
 1923             # try to load the module
 1924             ret = None
 1925             reloaded = False
 1926             # re-scan up to once, IOErrors or a failed load cause re-scans of the
 1927             # filesystem
 1928             while True:
 1929                 try:
 1930                     ret = _inner_load(mod_name)
 1931                     if not reloaded and ret is not True:
 1932                         self._refresh_file_mapping()
 1933                         reloaded = True
 1934                         continue
 1935                     break
 1936                 except OSError:
 1937                     if not reloaded:
 1938                         self._refresh_file_mapping()
 1939                         reloaded = True
 1940                     continue
 1941 
 1942         return ret
 1943 
 1944     def _load_all(self):
 1945         """
 1946         Load all of them
 1947         """
 1948         with self._lock:
 1949             for name in self.file_mapping:
 1950                 if name in self.loaded_files or name in self.missing_modules:
 1951                     continue
 1952                 self._load_module(name)
 1953 
 1954             self.loaded = True
 1955 
 1956     def reload_modules(self):
 1957         with self._lock:
 1958             self.loaded_files = set()
 1959             self._load_all()
 1960 
 1961     def _apply_outputter(self, func, mod):
 1962         """
 1963         Apply the __outputter__ variable to the functions
 1964         """
 1965         if hasattr(mod, "__outputter__"):
 1966             outp = mod.__outputter__
 1967             if func.__name__ in outp:
 1968                 func.__outputter__ = outp[func.__name__]
 1969 
 1970     def _process_virtual(self, mod, module_name, virtual_func="__virtual__"):
 1971         """
 1972         Given a loaded module and its default name determine its virtual name
 1973 
 1974         This function returns a tuple. The first value will be either True or
 1975         False and will indicate if the module should be loaded or not (i.e. if
 1976         it threw and exception while processing its __virtual__ function). The
 1977         second value is the determined virtual name, which may be the same as
 1978         the value provided.
 1979 
 1980         The default name can be calculated as follows::
 1981 
 1982             module_name = mod.__name__.rsplit('.', 1)[-1]
 1983         """
 1984 
 1985         # The __virtual__ function will return either a True or False value.
 1986         # If it returns a True value it can also set a module level attribute
 1987         # named __virtualname__ with the name that the module should be
 1988         # referred to as.
 1989         #
 1990         # This allows us to have things like the pkg module working on all
 1991         # platforms under the name 'pkg'. It also allows for modules like
 1992         # augeas_cfg to be referred to as 'augeas', which would otherwise have
 1993         # namespace collisions. And finally it allows modules to return False
 1994         # if they are not intended to run on the given platform or are missing
 1995         # dependencies.
 1996         virtual_aliases = getattr(mod, "__virtual_aliases__", tuple())
 1997         try:
 1998             error_reason = None
 1999             if hasattr(mod, "__virtual__") and inspect.isfunction(mod.__virtual__):
 2000                 try:
 2001                     start = time.time()
 2002                     virtual = getattr(mod, virtual_func)()
 2003                     if isinstance(virtual, tuple):
 2004                         error_reason = virtual[1]
 2005                         virtual = virtual[0]
 2006                     if self.opts.get("virtual_timer", False):
 2007                         end = time.time() - start
 2008                         msg = "Virtual function took {} seconds for {}".format(
 2009                             end, module_name
 2010                         )
 2011                         log.warning(msg)
 2012                 except Exception as exc:  # pylint: disable=broad-except
 2013                     error_reason = (
 2014                         "Exception raised when processing __virtual__ function"
 2015                         " for {}. Module will not be loaded: {}".format(
 2016                             mod.__name__, exc
 2017                         )
 2018                     )
 2019                     log.error(error_reason, exc_info_on_loglevel=logging.DEBUG)
 2020                     virtual = None
 2021                 # Get the module's virtual name
 2022                 virtualname = getattr(mod, "__virtualname__", virtual)
 2023                 if not virtual:
 2024                     # if __virtual__() evaluates to False then the module
 2025                     # wasn't meant for this platform or it's not supposed to
 2026                     # load for some other reason.
 2027 
 2028                     # Some modules might accidentally return None and are
 2029                     # improperly loaded
 2030                     if virtual is None:
 2031                         log.warning(
 2032                             "%s.__virtual__() is wrongly returning `None`. "
 2033                             "It should either return `True`, `False` or a new "
 2034                             "name. If you're the developer of the module "
 2035                             "'%s', please fix this.",
 2036                             mod.__name__,
 2037                             module_name,
 2038                         )
 2039 
 2040                     return (False, module_name, error_reason, virtual_aliases)
 2041 
 2042                 # At this point, __virtual__ did not return a
 2043                 # boolean value, let's check for deprecated usage
 2044                 # or module renames
 2045                 if virtual is not True and module_name != virtual:
 2046                     # The module is renaming itself. Updating the module name
 2047                     # with the new name
 2048                     log.trace("Loaded %s as virtual %s", module_name, virtual)
 2049 
 2050                     if virtualname != virtual:
 2051                         # The __virtualname__ attribute does not match what's
 2052                         # being returned by the __virtual__() function. This
 2053                         # should be considered an error.
 2054                         log.error(
 2055                             "The module '%s' is showing some bad usage. Its "
 2056                             "__virtualname__ attribute is set to '%s' yet the "
 2057                             "__virtual__() function is returning '%s'. These "
 2058                             "values should match!",
 2059                             mod.__name__,
 2060                             virtualname,
 2061                             virtual,
 2062                         )
 2063 
 2064                     module_name = virtualname
 2065 
 2066                 # If the __virtual__ function returns True and __virtualname__
 2067                 # is set then use it
 2068                 elif virtual is True and virtualname != module_name:
 2069                     if virtualname is not True:
 2070                         module_name = virtualname
 2071 
 2072         except KeyError:
 2073             # Key errors come out of the virtual function when passing
 2074             # in incomplete grains sets, these can be safely ignored
 2075             # and logged to debug, still, it includes the traceback to
 2076             # help debugging.
 2077             log.debug("KeyError when loading %s", module_name, exc_info=True)
 2078 
 2079         except Exception:  # pylint: disable=broad-except
 2080             # If the module throws an exception during __virtual__()
 2081             # then log the information and continue to the next.
 2082             log.error(
 2083                 "Failed to read the virtual function for %s: %s",
 2084                 self.tag,
 2085                 module_name,
 2086                 exc_info=True,
 2087             )
 2088             return (False, module_name, error_reason, virtual_aliases)
 2089 
 2090         return (True, module_name, None, virtual_aliases)
 2091 
 2092 
 2093 def global_injector_decorator(inject_globals):
 2094     """
 2095     Decorator used by the LazyLoader to inject globals into a function at
 2096     execute time.
 2097 
 2098     globals
 2099         Dictionary with global variables to inject
 2100     """
 2101 
 2102     def inner_decorator(f):
 2103         @functools.wraps(f)
 2104         def wrapper(*args, **kwargs):
 2105             with salt.utils.context.func_globals_inject(f, **inject_globals):
 2106                 return f(*args, **kwargs)
 2107 
 2108         return wrapper
 2109 
 2110     return inner_decorator