"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "salt/loader.py" between
salt-3002.1.tar.gz and salt-3002.2.tar.gz

About: SaltStack is a systems management software for data center automation, cloud orchestration, server provisioning, configuration management and more. Community version.

loader.py  (salt-3002.1):loader.py  (salt-3002.2)
""" """
The Salt loader is the core to Salt's plugin system, the loader scans The Salt loader is the core to Salt's plugin system, the loader scans
directories for python loadable code and organizes the code into the directories for python loadable code and organizes the code into the
plugin interfaces used by Salt. plugin interfaces used by Salt.
""" """
import functools import functools
import importlib.machinery # pylint: disable=no-name-in-module,import-error
import importlib.util # pylint: disable=no-name-in-module,import-error
import inspect import inspect
import logging import logging
import os import os
import re import re
import sys import sys
import tempfile import tempfile
import threading import threading
import time import time
import traceback import traceback
import types import types
import weakref
from collections.abc import MutableMapping from collections.abc import MutableMapping
from zipimport import zipimporter from zipimport import zipimporter
import salt.config import salt.config
import salt.defaults.events import salt.defaults.events
import salt.defaults.exitcodes import salt.defaults.exitcodes
import salt.syspaths import salt.syspaths
import salt.utils.args import salt.utils.args
import salt.utils.context import salt.utils.context
import salt.utils.data import salt.utils.data
skipping to change at line 43 skipping to change at line 44
import salt.utils.odict import salt.utils.odict
import salt.utils.platform import salt.utils.platform
import salt.utils.stringutils import salt.utils.stringutils
import salt.utils.versions import salt.utils.versions
from salt.exceptions import LoaderError from salt.exceptions import LoaderError
from salt.ext import six from salt.ext import six
from salt.ext.six.moves import reload_module from salt.ext.six.moves import reload_module
from salt.template import check_render_pipe_str from salt.template import check_render_pipe_str
from salt.utils.decorators import Depends from salt.utils.decorators import Depends
if sys.version_info[:2] >= (3, 5):
import importlib.machinery # pylint: disable=no-name-in-module,import-error
import importlib.util # pylint: disable=no-name-in-module,import-error
USE_IMPORTLIB = True
else:
import imp
USE_IMPORTLIB = False
try: try:
import pkg_resources import pkg_resources
HAS_PKG_RESOURCES = True HAS_PKG_RESOURCES = True
except ImportError: except ImportError:
HAS_PKG_RESOURCES = False HAS_PKG_RESOURCES = False
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR) SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR)
LOADED_BASE_NAME = "salt.loaded" LOADED_BASE_NAME = "salt.loaded"
if USE_IMPORTLIB: # pylint: disable=no-member
# pylint: disable=no-member MODULE_KIND_SOURCE = 1
MODULE_KIND_SOURCE = 1 MODULE_KIND_COMPILED = 2
MODULE_KIND_COMPILED = 2 MODULE_KIND_EXTENSION = 3
MODULE_KIND_EXTENSION = 3 MODULE_KIND_PKG_DIRECTORY = 5
MODULE_KIND_PKG_DIRECTORY = 5 SUFFIXES = []
SUFFIXES = [] for suffix in importlib.machinery.EXTENSION_SUFFIXES:
for suffix in importlib.machinery.EXTENSION_SUFFIXES: SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION))
SUFFIXES.append((suffix, "rb", MODULE_KIND_EXTENSION)) for suffix in importlib.machinery.SOURCE_SUFFIXES:
for suffix in importlib.machinery.SOURCE_SUFFIXES: SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE))
SUFFIXES.append((suffix, "rb", MODULE_KIND_SOURCE)) for suffix in importlib.machinery.BYTECODE_SUFFIXES:
for suffix in importlib.machinery.BYTECODE_SUFFIXES: SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED))
SUFFIXES.append((suffix, "rb", MODULE_KIND_COMPILED)) MODULE_KIND_MAP = {
MODULE_KIND_MAP = { MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader,
MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader, MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader,
MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader, MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader,
MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader, }
} # pylint: enable=no-member
# pylint: enable=no-member
else:
SUFFIXES = imp.get_suffixes()
PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_inf o[:2])) PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_inf o[:2]))
# Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *` # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
# which simplifies code readability, it adds some unsupported functions into # which simplifies code readability, it adds some unsupported functions into
# the driver's module scope. # the driver's module scope.
# We list un-supported functions here. These will be removed from the loaded. # We list un-supported functions here. These will be removed from the loaded.
# TODO: remove the need for this cross-module code. Maybe use NotImplemented # TODO: remove the need for this cross-module code. Maybe use NotImplemented
LIBCLOUD_FUNCS_NOT_SUPPORTED = ( LIBCLOUD_FUNCS_NOT_SUPPORTED = (
"parallels.avail_sizes", "parallels.avail_sizes",
skipping to change at line 1089 skipping to change at line 1077
# ModuleType can't accept a unicode type on PY2 # ModuleType can't accept a unicode type on PY2
module = types.ModuleType(str(name)) # future lint: disable=blacklisted-fun ction module = types.ModuleType(str(name)) # future lint: disable=blacklisted-fun ction
exec(code, module.__dict__) exec(code, module.__dict__)
sys.modules[name] = module sys.modules[name] = module
def _mod_type(module_path): def _mod_type(module_path):
if module_path.startswith(SALT_BASE_PATH): if module_path.startswith(SALT_BASE_PATH):
return "int" return "int"
return "ext" return "ext"
def _cleanup_module_namespace(loaded_base_name, delete_from_sys_modules=False):
"""
Clean module namespace.
If ``delete_from_sys_modules`` is ``False``, then the module instance in ``s
ys.modules``
will only be set to ``None``, when ``True``, it's actually ``del``elted.
The reason for this two stage cleanup procedure is because this function mig
ht
get called during the GC collection cycle and trigger https://bugs.python.or
g/issue40327
We seem to specially trigger this during the CI test runs with ``coverage.py
`` tracking
the code usage:
Traceback (most recent call last):
File "/urs/lib64/python3.6/site-packages/coverage/multiproc.py", line
37, in _bootstrap
cov.start()
File "/urs/lib64/python3.6/site-packages/coverage/control.py", line 52
7, in start
self._init_for_start()
File "/urs/lib64/python3.6/site-packages/coverage/control.py", line 45
5, in _init_for_start
concurrency=concurrency,
File "/urs/lib64/python3.6/site-packages/coverage/collector.py", line
111, in __init__
self.origin = short_stack()
File "/urs/lib64/python3.6/site-packages/coverage/debug.py", line 157,
in short_stack
stack = inspect.stack()[limit:skip:-1]
File "/usr/lib64/python3.6/inspect.py", line 1501, in stack
return getouterframes(sys._getframe(1), context)
File "/usr/lib64/python3.6/inspect.py", line 1478, in getouterframes
frameinfo = (frame,) + getframeinfo(frame, context)
File "/usr/lib64/python3.6/inspect.py", line 1452, in getframeinfo
lines, lnum = findsource(frame)
File "/usr/lib64/python3.6/inspect.py", line 780, in findsource
module = getmodule(object, file)
File "/usr/lib64/python3.6/inspect.py", line 732, in getmodule
for modname, module in list(sys.modules.items()):
RuntimeError: dictionary changed size during iteration
"""
for name in list(sys.modules):
if name.startswith(loaded_base_name):
if delete_from_sys_modules:
del sys.modules[name]
else:
mod = sys.modules[name]
sys.modules[name] = None
del mod
# TODO: move somewhere else? # TODO: move somewhere else?
class FilterDictWrapper(MutableMapping): class FilterDictWrapper(MutableMapping):
""" """
Create a dict which wraps another dict with a specific key suffix on get Create a dict which wraps another dict with a specific key suffix on get
This is to replace "filter_load" This is to replace "filter_load"
""" """
def __init__(self, d, suffix): def __init__(self, d, suffix):
self._dict = d self._dict = d
skipping to change at line 1223 skipping to change at line 1167
self.pack = {} if pack is None else pack self.pack = {} if pack is None else pack
if opts is None: if opts is None:
opts = {} opts = {}
threadsafety = not opts.get("multiprocessing") threadsafety = not opts.get("multiprocessing")
self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafe ty) self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafe ty)
self.opts = self.__prep_mod_opts(opts) self.opts = self.__prep_mod_opts(opts)
self.module_dirs = module_dirs self.module_dirs = module_dirs
self.tag = tag self.tag = tag
self._gc_finalizer = None self._gc_finalizer = None
if loaded_base_name: if loaded_base_name and loaded_base_name != LOADED_BASE_NAME:
self.loaded_base_name = loaded_base_name self.loaded_base_name = loaded_base_name
else: else:
self.loaded_base_name = "{}_{}".format(LOADED_BASE_NAME, id(self)) self.loaded_base_name = LOADED_BASE_NAME
# Remove any modules matching self.loaded_base_name that have been s
et to None previously
self.clean_modules()
# Make sure that, when this module is about to be GC'ed, we at least
set any modules in
# sys.modules which match self.loaded_base_name to None to reduce me
mory usage over time.
# ATTENTION: Do not replace the '_cleanup_module_namespace' function
on the call below with
# self.clean_modules as that WILL prevent this loader obj
ect from being garbage
# collected and the finalizer running.
self._gc_finalizer = weakref.finalize(
self, _cleanup_module_namespace, "{}".format(self.loaded_base_na
me)
)
# This finalizer does not need to run when the process is exiting
self._gc_finalizer.atexit = False
self.mod_type_check = mod_type_check or _mod_type self.mod_type_check = mod_type_check or _mod_type
if "__context__" not in self.pack: if "__context__" not in self.pack:
self.pack["__context__"] = None self.pack["__context__"] = None
for k, v in self.pack.items(): for k, v in self.pack.items():
if v is None: # if the value of a pack is None, lets make an empty dict if v is None: # if the value of a pack is None, lets make an empty dict
self.context_dict.setdefault(k, {}) self.context_dict.setdefault(k, {})
self.pack[k] = salt.utils.context.NamespacedDictWrapper( self.pack[k] = salt.utils.context.NamespacedDictWrapper(
self.context_dict, k self.context_dict, k
skipping to change at line 1298 skipping to change at line 1230
# create all of the import namespaces # create all of the import namespaces
_generate_module("{}.int".format(self.loaded_base_name)) _generate_module("{}.int".format(self.loaded_base_name))
_generate_module("{}.int.{}".format(self.loaded_base_name, tag)) _generate_module("{}.int.{}".format(self.loaded_base_name, tag))
_generate_module("{}.ext".format(self.loaded_base_name)) _generate_module("{}.ext".format(self.loaded_base_name))
_generate_module("{}.ext.{}".format(self.loaded_base_name, tag)) _generate_module("{}.ext.{}".format(self.loaded_base_name, tag))
def clean_modules(self): def clean_modules(self):
""" """
Clean modules Clean modules
""" """
if self._gc_finalizer is not None and self._gc_finalizer.alive: for name in list(sys.modules):
# Prevent the weakref.finalizer instance from running, there's no po if name.startswith(self.loaded_base_name):
int after the next call. del sys.modules[name]
self._gc_finalizer.detach()
_cleanup_module_namespace(self.loaded_base_name, delete_from_sys_modules
=True)
def __getitem__(self, item): def __getitem__(self, item):
""" """
Override the __getitem__ in order to decorate the returned function if w e need Override the __getitem__ in order to decorate the returned function if w e need
to last-minute inject globals to last-minute inject globals
""" """
func = super().__getitem__(item) func = super().__getitem__(item)
if self.inject_globals: if self.inject_globals:
return global_injector_decorator(self.inject_globals)(func) return global_injector_decorator(self.inject_globals)(func)
else: else:
skipping to change at line 1394 skipping to change at line 1325
) )
# Allow for zipimport of modules # Allow for zipimport of modules
if ( if (
self.opts.get("enable_zip_modules", True) is True self.opts.get("enable_zip_modules", True) is True
and ".zip" not in self.suffix_map and ".zip" not in self.suffix_map
): ):
self.suffix_map[".zip"] = tuple() self.suffix_map[".zip"] = tuple()
if ".zip" not in self.suffix_order: if ".zip" not in self.suffix_order:
self.suffix_order.append(".zip") self.suffix_order.append(".zip")
# allow for module dirs # allow for module dirs
if USE_IMPORTLIB: self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY)
self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY)
else:
self.suffix_map[""] = ("", "", imp.PKG_DIRECTORY)
# create mapping of filename (without suffix) to (path, suffix) # create mapping of filename (without suffix) to (path, suffix)
# The files are added in order of priority, so order *must* be retained. # The files are added in order of priority, so order *must* be retained.
self.file_mapping = salt.utils.odict.OrderedDict() self.file_mapping = salt.utils.odict.OrderedDict()
opt_match = [] opt_match = []
def _replace_pre_ext(obj): def _replace_pre_ext(obj):
""" """
Hack so we can get the optimization level that we replaced (if Hack so we can get the optimization level that we replaced (if
skipping to change at line 1610 skipping to change at line 1538
def __clean_sys_path(self): def __clean_sys_path(self):
invalidate_path_importer_cache = False invalidate_path_importer_cache = False
for directory in self._clean_module_dirs: for directory in self._clean_module_dirs:
if directory in sys.path: if directory in sys.path:
sys.path.remove(directory) sys.path.remove(directory)
invalidate_path_importer_cache = True invalidate_path_importer_cache = True
self._clean_module_dirs = [] self._clean_module_dirs = []
# Be sure that sys.path_importer_cache do not contains any # Be sure that sys.path_importer_cache do not contains any
# invalid FileFinder references # invalid FileFinder references
if USE_IMPORTLIB: importlib.invalidate_caches()
importlib.invalidate_caches()
# Because we are mangling with importlib, we can find from # Because we are mangling with importlib, we can find from
# time to time an invalidation issue with # time to time an invalidation issue with
# sys.path_importer_cache, that requires the removal of # sys.path_importer_cache, that requires the removal of
# FileFinder that remain None for the extra_module_dirs # FileFinder that remain None for the extra_module_dirs
if invalidate_path_importer_cache: if invalidate_path_importer_cache:
for directory in self.extra_module_dirs: for directory in self.extra_module_dirs:
if ( if (
directory in sys.path_importer_cache directory in sys.path_importer_cache
and sys.path_importer_cache[directory] is None and sys.path_importer_cache[directory] is None
skipping to change at line 1688 skipping to change at line 1615
) )
) )
except TypeError: except TypeError:
mod_namespace = "{}.{}.{}.{}".format( mod_namespace = "{}.{}.{}.{}".format(
self.loaded_base_name, self.loaded_base_name,
self.mod_type_check(fpath), self.mod_type_check(fpath),
self.tag, self.tag,
name, name,
) )
if suffix == "": if suffix == "":
if USE_IMPORTLIB: # pylint: disable=no-member
# pylint: disable=no-member # Package directory, look for __init__
# Package directory, look for __init__ loader_details = [
loader_details = [ (
( importlib.machinery.SourceFileLoader,
importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES,
importlib.machinery.SOURCE_SUFFIXES, ),
), (
( importlib.machinery.SourcelessFileLoader,
importlib.machinery.SourcelessFileLoader, importlib.machinery.BYTECODE_SUFFIXES,
importlib.machinery.BYTECODE_SUFFIXES, ),
), (
( importlib.machinery.ExtensionFileLoader,
importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES,
importlib.machinery.EXTENSION_SUFFIXES, ),
), ]
] file_finder = importlib.machinery.FileFinder(
file_finder = importlib.machinery.FileFinder( fpath_dirname, *loader_details
fpath_dirname, *loader_details )
) spec = file_finder.find_spec(mod_namespace)
spec = file_finder.find_spec(mod_namespace) if spec is None:
if spec is None: raise ImportError()
raise ImportError() # TODO: Get rid of load_module in favor of
# TODO: Get rid of load_module in favor of # exec_module below. load_module is deprecated, but
# exec_module below. load_module is deprecated, but # loading using exec_module has been causing odd things
# loading using exec_module has been causing odd things # with the magic dunders we pack into the loaded
# with the magic dunders we pack into the loaded # modules, most notably with salt-ssh's __opts__.
# modules, most notably with salt-ssh's __opts__. mod = spec.loader.load_module()
mod = spec.loader.load_module() # mod = importlib.util.module_from_spec(spec)
# mod = importlib.util.module_from_spec(spec) # spec.loader.exec_module(mod)
# spec.loader.exec_module(mod) # pylint: enable=no-member
# pylint: enable=no-member sys.modules[mod_namespace] = mod
sys.modules[mod_namespace] = mod
else:
mod = imp.load_module(mod_namespace, None, fpath, desc)
# reload all submodules if necessary # reload all submodules if necessary
if not self.initial_load: if not self.initial_load:
self._reload_submodules(mod) self._reload_submodules(mod)
else: else:
if USE_IMPORTLIB: # pylint: disable=no-member
# pylint: disable=no-member loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath)
loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath) spec = importlib.util.spec_from_file_location(
spec = importlib.util.spec_from_file_location( mod_namespace, fpath, loader=loader
mod_namespace, fpath, loader=loader )
) if spec is None:
if spec is None: raise ImportError()
raise ImportError() # TODO: Get rid of load_module in favor of
# TODO: Get rid of load_module in favor of # exec_module below. load_module is deprecated, but
# exec_module below. load_module is deprecated, but # loading using exec_module has been causing odd things
# loading using exec_module has been causing odd things # with the magic dunders we pack into the loaded
# with the magic dunders we pack into the loaded # modules, most notably with salt-ssh's __opts__.
# modules, most notably with salt-ssh's __opts__. mod = spec.loader.load_module()
mod = spec.loader.load_module() # mod = importlib.util.module_from_spec(spec)
# mod = importlib.util.module_from_spec(spec) # spec.loader.exec_module(mod)
# spec.loader.exec_module(mod) # pylint: enable=no-member
# pylint: enable=no-member sys.modules[mod_namespace] = mod
sys.modules[mod_namespace] = mod
else:
with salt.utils.files.fopen(fpath, desc[1]) as fn_:
mod = imp.load_module(mod_namespace, fn_, fpath, des
c)
except OSError: except OSError:
raise raise
except ImportError as exc: except ImportError as exc:
if "magic number" in str(exc): if "magic number" in str(exc):
error_msg = "Failed to import {} {}. Bad magic number. If migrat ing from Python2 to Python3, remove all .pyc files and try again.".format( error_msg = "Failed to import {} {}. Bad magic number. If migrat ing from Python2 to Python3, remove all .pyc files and try again.".format(
self.tag, name self.tag, name
) )
log.warning(error_msg) log.warning(error_msg)
self.missing_modules[name] = error_msg self.missing_modules[name] = error_msg
log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True ) log.debug("Failed to import %s %s:\n", self.tag, name, exc_info=True )
 End of changes. 12 change blocks. 
174 lines changed or deleted 76 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)