is a systems management software for data center automation, cloud orchestration, server provisioning, configuration management and more. Community version.
| loader.py (salt-3002.1) | : | loader.py (salt-3002.2) |
|---|
| """ | | """ |
| The Salt loader is the core to Salt's plugin system, the loader scans | | The Salt loader is the core to Salt's plugin system, the loader scans |
| directories for python loadable code and organizes the code into the | | directories for python loadable code and organizes the code into the |
| plugin interfaces used by Salt. | | plugin interfaces used by Salt. |
| """ | | """ |
| | |
| import functools | | import functools |
|
| | import importlib.machinery # pylint: disable=no-name-in-module,import-error |
| | import importlib.util # pylint: disable=no-name-in-module,import-error |
| import inspect | | import inspect |
| import logging | | import logging |
| import os | | import os |
| import re | | import re |
| import sys | | import sys |
| import tempfile | | import tempfile |
| import threading | | import threading |
| import time | | import time |
| import traceback | | import traceback |
| import types | | import types |
|
| import weakref | | |
| from collections.abc import MutableMapping | | from collections.abc import MutableMapping |
| from zipimport import zipimporter | | from zipimport import zipimporter |
| | |
| import salt.config | | import salt.config |
| import salt.defaults.events | | import salt.defaults.events |
| import salt.defaults.exitcodes | | import salt.defaults.exitcodes |
| import salt.syspaths | | import salt.syspaths |
| import salt.utils.args | | import salt.utils.args |
| import salt.utils.context | | import salt.utils.context |
| import salt.utils.data | | import salt.utils.data |
| | |
| skipping to change at line 43 | | skipping to change at line 44 |
|---|
| import salt.utils.odict | | import salt.utils.odict |
| import salt.utils.platform | | import salt.utils.platform |
| import salt.utils.stringutils | | import salt.utils.stringutils |
| import salt.utils.versions | | import salt.utils.versions |
| from salt.exceptions import LoaderError | | from salt.exceptions import LoaderError |
| from salt.ext import six | | from salt.ext import six |
| from salt.ext.six.moves import reload_module | | from salt.ext.six.moves import reload_module |
| from salt.template import check_render_pipe_str | | from salt.template import check_render_pipe_str |
| from salt.utils.decorators import Depends | | from salt.utils.decorators import Depends |
| | |
|
| if sys.version_info[:2] >= (3, 5): | | |
| import importlib.machinery # pylint: disable=no-name-in-module,import-error | | |
| import importlib.util # pylint: disable=no-name-in-module,import-error | | |
| | |
| USE_IMPORTLIB = True | | |
| else: | | |
| import imp | | |
| | |
| USE_IMPORTLIB = False | | |
| | | |
| try: | | try: |
| import pkg_resources | | import pkg_resources |
| | |
| HAS_PKG_RESOURCES = True | | HAS_PKG_RESOURCES = True |
| except ImportError: | | except ImportError: |
| HAS_PKG_RESOURCES = False | | HAS_PKG_RESOURCES = False |
| | |
| log = logging.getLogger(__name__) | | log = logging.getLogger(__name__) |
| | |
| SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR) | | SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR) |
| LOADED_BASE_NAME = "salt.loaded" | | LOADED_BASE_NAME = "salt.loaded" |
| | |
|
| if USE_IMPORTLIB: | | # pylint: disable=no-member |
| # pylint: disable=no-member | | MODULE_KIND_SOURCE = 1 |
| MODULE_KIND_SOURCE = 1 | | MODULE_KIND_COMPILED = 2 |
| MODULE_KIND_COMPILED = 2 | | MODULE_KIND_EXTENSION = 3 |
| MODULE_KIND_EXTENSION = 3 | | MODULE_KIND_PKG_DIRECTORY = 5 |
| MODULE_KIND_PKG_DIRECTORY = 5 | | SUFFIXES = [] |
| SUFFIXES = [] | | for suffix in importlib.machinery.EXTENSION_SUFFIXES: |
| for suffix in importlib.machinery.EXTENSION_SUFFIXES: | | SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION)) |
| SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION)) | | for suffix in importlib.machinery.SOURCE_SUFFIXES: |
| for suffix in importlib.machinery.SOURCE_SUFFIXES: | | SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE)) |
| SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE)) | | for suffix in importlib.machinery.BYTECODE_SUFFIXES: |
| for suffix in importlib.machinery.BYTECODE_SUFFIXES: | | SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED)) |
| SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED)) | | MODULE_KIND_MAP = { |
| MODULE_KIND_MAP = { | | MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader, |
| MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader, | | MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader, |
| MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader, | | MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader, |
| MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader, | | } |
| } | | # pylint: enable=no-member |
| # pylint: enable=no-member | | |
| else: | | |
| SUFFIXES = imp.get_suffixes() | | |
| | |
| PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_inf
o[:2])) | | PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_inf
o[:2])) |
| | |
| # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *` | | # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *` |
| # which simplifies code readability, it adds some unsupported functions into | | # which simplifies code readability, it adds some unsupported functions into |
| # the driver's module scope. | | # the driver's module scope. |
| # We list un-supported functions here. These will be removed from the loaded. | | # We list un-supported functions here. These will be removed from the loaded. |
| # TODO: remove the need for this cross-module code. Maybe use NotImplemented | | # TODO: remove the need for this cross-module code. Maybe use NotImplemented |
| LIBCLOUD_FUNCS_NOT_SUPPORTED = ( | | LIBCLOUD_FUNCS_NOT_SUPPORTED = ( |
| "parallels.avail_sizes", | | "parallels.avail_sizes", |
| | |
| skipping to change at line 1089 | | skipping to change at line 1077 |
|---|
| # ModuleType can't accept a unicode type on PY2 | | # ModuleType can't accept a unicode type on PY2 |
| module = types.ModuleType(str(name)) # future lint: disable=blacklisted-fun
ction | | module = types.ModuleType(str(name)) # future lint: disable=blacklisted-fun
ction |
| exec(code, module.__dict__) | | exec(code, module.__dict__) |
| sys.modules[name] = module | | sys.modules[name] = module |
| | |
| def _mod_type(module_path): | | def _mod_type(module_path): |
| if module_path.startswith(SALT_BASE_PATH): | | if module_path.startswith(SALT_BASE_PATH): |
| return "int" | | return "int" |
| return "ext" | | return "ext" |
| | |
|
| def _cleanup_module_namespace(loaded_base_name, delete_from_sys_modules=False): | | |
| """ | | |
| Clean module namespace. | | |
| If ``delete_from_sys_modules`` is ``False``, then the module instance in ``s | | |
| ys.modules`` | | |
| will only be set to ``None``, when ``True``, it's actually ``del``elted. | | |
| | |
| The reason for this two stage cleanup procedure is because this function mig | | |
| ht | | |
| get called during the GC collection cycle and trigger https://bugs.python.or | | |
| g/issue40327 | | |
| | |
| We seem to specially trigger this during the CI test runs with ``coverage.py | | |
| `` tracking | | |
| the code usage: | | |
| | |
| Traceback (most recent call last): | | |
| File "/urs/lib64/python3.6/site-packages/coverage/multiproc.py", line | | |
| 37, in _bootstrap | | |
| cov.start() | | |
| File "/urs/lib64/python3.6/site-packages/coverage/control.py", line 52 | | |
| 7, in start | | |
| self._init_for_start() | | |
| File "/urs/lib64/python3.6/site-packages/coverage/control.py", line 45 | | |
| 5, in _init_for_start | | |
| concurrency=concurrency, | | |
| File "/urs/lib64/python3.6/site-packages/coverage/collector.py", line | | |
| 111, in __init__ | | |
| self.origin = short_stack() | | |
| File "/urs/lib64/python3.6/site-packages/coverage/debug.py", line 157, | | |
| in short_stack | | |
| stack = inspect.stack()[limit:skip:-1] | | |
| File "/usr/lib64/python3.6/inspect.py", line 1501, in stack | | |
| return getouterframes(sys._getframe(1), context) | | |
| File "/usr/lib64/python3.6/inspect.py", line 1478, in getouterframes | | |
| frameinfo = (frame,) + getframeinfo(frame, context) | | |
| File "/usr/lib64/python3.6/inspect.py", line 1452, in getframeinfo | | |
| lines, lnum = findsource(frame) | | |
| File "/usr/lib64/python3.6/inspect.py", line 780, in findsource | | |
| module = getmodule(object, file) | | |
| File "/usr/lib64/python3.6/inspect.py", line 732, in getmodule | | |
| for modname, module in list(sys.modules.items()): | | |
| RuntimeError: dictionary changed size during iteration | | |
| """ | | |
| for name in list(sys.modules): | | |
| if name.startswith(loaded_base_name): | | |
| if delete_from_sys_modules: | | |
| del sys.modules[name] | | |
| else: | | |
| mod = sys.modules[name] | | |
| sys.modules[name] = None | | |
| del mod | | |
| | | |
| # TODO: move somewhere else? | | # TODO: move somewhere else? |
| class FilterDictWrapper(MutableMapping): | | class FilterDictWrapper(MutableMapping): |
| """ | | """ |
| Create a dict which wraps another dict with a specific key suffix on get | | Create a dict which wraps another dict with a specific key suffix on get |
| | |
| This is to replace "filter_load" | | This is to replace "filter_load" |
| """ | | """ |
| | |
| def __init__(self, d, suffix): | | def __init__(self, d, suffix): |
| self._dict = d | | self._dict = d |
| | |
| skipping to change at line 1223 | | skipping to change at line 1167 |
|---|
| self.pack = {} if pack is None else pack | | self.pack = {} if pack is None else pack |
| if opts is None: | | if opts is None: |
| opts = {} | | opts = {} |
| threadsafety = not opts.get("multiprocessing") | | threadsafety = not opts.get("multiprocessing") |
| self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafe
ty) | | self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafe
ty) |
| self.opts = self.__prep_mod_opts(opts) | | self.opts = self.__prep_mod_opts(opts) |
| | |
| self.module_dirs = module_dirs | | self.module_dirs = module_dirs |
| self.tag = tag | | self.tag = tag |
| self._gc_finalizer = None | | self._gc_finalizer = None |
|
| if loaded_base_name: | | if loaded_base_name and loaded_base_name != LOADED_BASE_NAME: |
| self.loaded_base_name = loaded_base_name | | self.loaded_base_name = loaded_base_name |
| else: | | else: |
|
| self.loaded_base_name = "{}_{}".format(LOADED_BASE_NAME, id(self)) | | self.loaded_base_name = LOADED_BASE_NAME |
| # Remove any modules matching self.loaded_base_name that have been s | | |
| et to None previously | | |
| self.clean_modules() | | |
| # Make sure that, when this module is about to be GC'ed, we at least | | |
| set any modules in | | |
| # sys.modules which match self.loaded_base_name to None to reduce me | | |
| mory usage over time. | | |
| # ATTENTION: Do not replace the '_cleanup_module_namespace' function | | |
| on the call below with | | |
| # self.clean_modules as that WILL prevent this loader obj | | |
| ect from being garbage | | |
| # collected and the finalizer running. | | |
| self._gc_finalizer = weakref.finalize( | | |
| self, _cleanup_module_namespace, "{}".format(self.loaded_base_na | | |
| me) | | |
| ) | | |
| # This finalizer does not need to run when the process is exiting | | |
| self._gc_finalizer.atexit = False | | |
| self.mod_type_check = mod_type_check or _mod_type | | self.mod_type_check = mod_type_check or _mod_type |
| | |
| if "__context__" not in self.pack: | | if "__context__" not in self.pack: |
| self.pack["__context__"] = None | | self.pack["__context__"] = None |
| | |
| for k, v in self.pack.items(): | | for k, v in self.pack.items(): |
| if v is None: # if the value of a pack is None, lets make an empty
dict | | if v is None: # if the value of a pack is None, lets make an empty
dict |
| self.context_dict.setdefault(k, {}) | | self.context_dict.setdefault(k, {}) |
| self.pack[k] = salt.utils.context.NamespacedDictWrapper( | | self.pack[k] = salt.utils.context.NamespacedDictWrapper( |
| self.context_dict, k | | self.context_dict, k |
| | |
| skipping to change at line 1298 | | skipping to change at line 1230 |
|---|
| # create all of the import namespaces | | # create all of the import namespaces |
| _generate_module("{}.int".format(self.loaded_base_name)) | | _generate_module("{}.int".format(self.loaded_base_name)) |
| _generate_module("{}.int.{}".format(self.loaded_base_name, tag)) | | _generate_module("{}.int.{}".format(self.loaded_base_name, tag)) |
| _generate_module("{}.ext".format(self.loaded_base_name)) | | _generate_module("{}.ext".format(self.loaded_base_name)) |
| _generate_module("{}.ext.{}".format(self.loaded_base_name, tag)) | | _generate_module("{}.ext.{}".format(self.loaded_base_name, tag)) |
| | |
| def clean_modules(self): | | def clean_modules(self): |
| """ | | """ |
| Clean modules | | Clean modules |
| """ | | """ |
|
| if self._gc_finalizer is not None and self._gc_finalizer.alive: | | for name in list(sys.modules): |
| # Prevent the weakref.finalizer instance from running, there's no po | | if name.startswith(self.loaded_base_name): |
| int after the next call. | | del sys.modules[name] |
| self._gc_finalizer.detach() | | |
| _cleanup_module_namespace(self.loaded_base_name, delete_from_sys_modules | | |
| =True) | | |
| | |
| def __getitem__(self, item): | | def __getitem__(self, item): |
| """ | | """ |
| Override the __getitem__ in order to decorate the returned function if w
e need | | Override the __getitem__ in order to decorate the returned function if w
e need |
| to last-minute inject globals | | to last-minute inject globals |
| """ | | """ |
| func = super().__getitem__(item) | | func = super().__getitem__(item) |
| if self.inject_globals: | | if self.inject_globals: |
| return global_injector_decorator(self.inject_globals)(func) | | return global_injector_decorator(self.inject_globals)(func) |
| else: | | else: |
| | |
| skipping to change at line 1394 | | skipping to change at line 1325 |
|---|
| ) | | ) |
| # Allow for zipimport of modules | | # Allow for zipimport of modules |
| if ( | | if ( |
| self.opts.get("enable_zip_modules", True) is True | | self.opts.get("enable_zip_modules", True) is True |
| and ".zip" not in self.suffix_map | | and ".zip" not in self.suffix_map |
| ): | | ): |
| self.suffix_map[".zip"] = tuple() | | self.suffix_map[".zip"] = tuple() |
| if ".zip" not in self.suffix_order: | | if ".zip" not in self.suffix_order: |
| self.suffix_order.append(".zip") | | self.suffix_order.append(".zip") |
| # allow for module dirs | | # allow for module dirs |
|
| if USE_IMPORTLIB: | | self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY) |
| self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY) | | |
| else: | | |
| self.suffix_map[""] = ("", "", imp.PKG_DIRECTORY) | | |
| | |
| # create mapping of filename (without suffix) to (path, suffix) | | # create mapping of filename (without suffix) to (path, suffix) |
| # The files are added in order of priority, so order *must* be retained. | | # The files are added in order of priority, so order *must* be retained. |
| self.file_mapping = salt.utils.odict.OrderedDict() | | self.file_mapping = salt.utils.odict.OrderedDict() |
| | |
| opt_match = [] | | opt_match = [] |
| | |
| def _replace_pre_ext(obj): | | def _replace_pre_ext(obj): |
| """ | | """ |
| Hack so we can get the optimization level that we replaced (if | | Hack so we can get the optimization level that we replaced (if |
| | |
| skipping to change at line 1610 | | skipping to change at line 1538 |
|---|
| def __clean_sys_path(self): | | def __clean_sys_path(self): |
| invalidate_path_importer_cache = False | | invalidate_path_importer_cache = False |
| for directory in self._clean_module_dirs: | | for directory in self._clean_module_dirs: |
| if directory in sys.path: | | if directory in sys.path: |
| sys.path.remove(directory) | | sys.path.remove(directory) |
| invalidate_path_importer_cache = True | | invalidate_path_importer_cache = True |
| self._clean_module_dirs = [] | | self._clean_module_dirs = [] |
| | |
| # Be sure that sys.path_importer_cache do not contains any | | # Be sure that sys.path_importer_cache do not contains any |
| # invalid FileFinder references | | # invalid FileFinder references |
|
| if USE_IMPORTLIB: | | importlib.invalidate_caches() |
| importlib.invalidate_caches() | | |
| | |
| # Because we are mangling with importlib, we can find from | | # Because we are mangling with importlib, we can find from |
| # time to time an invalidation issue with | | # time to time an invalidation issue with |
| # sys.path_importer_cache, that requires the removal of | | # sys.path_importer_cache, that requires the removal of |
| # FileFinder that remain None for the extra_module_dirs | | # FileFinder that remain None for the extra_module_dirs |
| if invalidate_path_importer_cache: | | if invalidate_path_importer_cache: |
| for directory in self.extra_module_dirs: | | for directory in self.extra_module_dirs: |
| if ( | | if ( |
| directory in sys.path_importer_cache | | directory in sys.path_importer_cache |
| and sys.path_importer_cache[directory] is None | | and sys.path_importer_cache[directory] is None |
| | |
| skipping to change at line 1688 | | skipping to change at line 1615 |
|---|
| ) | | ) |
| ) | | ) |
| except TypeError: | | except TypeError: |
| mod_namespace = "{}.{}.{}.{}".format( | | mod_namespace = "{}.{}.{}.{}".format( |
| self.loaded_base_name, | | self.loaded_base_name, |
| self.mod_type_check(fpath), | | self.mod_type_check(fpath), |
| self.tag, | | self.tag, |
| name, | | name, |
| ) | | ) |
| if suffix == "": | | if suffix == "": |
|
| if USE_IMPORTLIB: | | # pylint: disable=no-member |
| # pylint: disable=no-member | | # Package directory, look for __init__ |
| # Package directory, look for __init__ | | loader_details = [ |
| loader_details = [ | | ( |
| ( | | importlib.machinery.SourceFileLoader, |
| importlib.machinery.SourceFileLoader, | | importlib.machinery.SOURCE_SUFFIXES, |
| importlib.machinery.SOURCE_SUFFIXES, | | ), |
| ), | | ( |
| ( | | importlib.machinery.SourcelessFileLoader, |
| importlib.machinery.SourcelessFileLoader, | | importlib.machinery.BYTECODE_SUFFIXES, |
| importlib.machinery.BYTECODE_SUFFIXES, | | ), |
| ), | | ( |
| ( | | importlib.machinery.ExtensionFileLoader, |
| importlib.machinery.ExtensionFileLoader, | | importlib.machinery.EXTENSION_SUFFIXES, |
| importlib.machinery.EXTENSION_SUFFIXES, | | ), |
| ), | | ] |
| ] | | file_finder = importlib.machinery.FileFinder( |
| file_finder = importlib.machinery.FileFinder( | | fpath_dirname, *loader_details |
| fpath_dirname, *loader_details | | ) |
| ) | | spec = file_finder.find_spec(mod_namespace) |
| spec = file_finder.find_spec(mod_namespace) | | if spec is None: |
| if spec is None: | | raise ImportError() |
| raise ImportError() | | # TODO: Get rid of load_module in favor of |
| # TODO: Get rid of load_module in favor of | | # exec_module below. load_module is deprecated, but |
| # exec_module below. load_module is deprecated, but | | # loading using exec_module has been causing odd things |
| # loading using exec_module has been causing odd things | | # with the magic dunders we pack into the loaded |
| # with the magic dunders we pack into the loaded | | # modules, most notably with salt-ssh's __opts__. |
| # modules, most notably with salt-ssh's __opts__. | | mod = spec.loader.load_module() |
| mod = spec.loader.load_module() | | # mod = importlib.util.module_from_spec(spec) |
| # mod = importlib.util.module_from_spec(spec) | | # spec.loader.exec_module(mod) |
| # spec.loader.exec_module(mod) | | # pylint: enable=no-member |
| # pylint: enable=no-member | | sys.modules[mod_namespace] = mod |
| sys.modules[mod_namespace] = mod | | |
| else: | | |
| mod = imp.load_module(mod_namespace, None, fpath, desc) | | |
| # reload all submodules if necessary | | # reload all submodules if necessary |
| if not self.initial_load: | | if not self.initial_load: |
| self._reload_submodules(mod) | | self._reload_submodules(mod) |
| else: | | else: |
|
| if USE_IMPORTLIB: | | # pylint: disable=no-member |
| # pylint: disable=no-member | | loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath) |
| loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath) | | spec = importlib.util.spec_from_file_location( |
| spec = importlib.util.spec_from_file_location( | | mod_namespace, fpath, loader=loader |
| mod_namespace, fpath, loader=loader | | ) |
| ) | | if spec is None: |
| if spec is None: | | raise ImportError() |
| raise ImportError() | | # TODO: Get rid of load_module in favor of |
| # TODO: Get rid of load_module in favor of | | # exec_module below. load_module is deprecated, but |
| # exec_module below. load_module is deprecated, but | | # loading using exec_module has been causing odd things |
| # loading using exec_module has been causing odd things | | # with the magic dunders we pack into the loaded |
| # with the magic dunders we pack into the loaded | | # modules, most notably with salt-ssh's __opts__. |
| # modules, most notably with salt-ssh's __opts__. | | mod = spec.loader.load_module() |
| mod = spec.loader.load_module() | | # mod = importlib.util.module_from_spec(spec) |
| # mod = importlib.util.module_from_spec(spec) | | # spec.loader.exec_module(mod) |
| # spec.loader.exec_module(mod) | | # pylint: enable=no-member |
| # pylint: enable=no-member | | sys.modules[mod_namespace] = mod |
| sys.modules[mod_namespace] = mod | | |
| else: | | |
| with salt.utils.files.fopen(fpath, desc[1]) as fn_: | | |
| mod = imp.load_module(mod_namespace, fn_, fpath, des | | |
| c) | | |
| except OSError: | | except OSError: |
| raise | | raise |
| except ImportError as exc: | | except ImportError as exc: |
| if "magic number" in str(exc): | | if "magic number" in str(exc): |
| error_msg = "Failed to import {} {}. Bad magic number. If migrat
ing from Python2 to Python3, remove all .pyc files and try again.".format( | | error_msg = "Failed to import {} {}. Bad magic number. If migrat
ing from Python2 to Python3, remove all .pyc files and try again.".format( |
| self.tag, name | | self.tag, name |
| ) | | ) |
| log.warning(error_msg) | | log.warning(error_msg) |
| self.missing_modules[name] = error_msg | | self.missing_modules[name] = error_msg |
| log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True
) | | log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True
) |
| | | |
| End of changes. 12 change blocks. |
|---|
| 174 lines changed or deleted | | 76 lines changed or added |
|---|
"Fossies" - the Fresh Open Source Software Archive 