"Fossies" - the Fresh Open Source Software Archive

Member "salt-3002.2/salt/state.py" (18 Nov 2020, 195163 Bytes) of package /linux/misc/salt-3002.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "state.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 3002.1_vs_3002.2.

    1 """
    2 The State Compiler is used to execute states in Salt. A state is unlike
    3 an execution module in that instead of just executing a command, it
    4 ensures that a certain state is present on the system.
    5 
    6 The data sent to the state calls is as follows:
    7     { 'state': '<state module name>',
    8       'fun': '<state function name>',
    9       'name': '<the name argument passed to all states>'
   10       'argn': '<arbitrary argument, can have many of these>'
   11       }
   12 """
   13 
   14 
   15 import copy
   16 import datetime
   17 import fnmatch
   18 import logging
   19 import os
   20 import random
   21 import re
   22 import site
   23 import sys
   24 import time
   25 import traceback
   26 
   27 import salt.fileclient
   28 import salt.loader
   29 import salt.minion
   30 import salt.pillar
   31 import salt.syspaths as syspaths
   32 import salt.transport.client
   33 import salt.utils.args
   34 import salt.utils.crypt
   35 import salt.utils.data
   36 import salt.utils.decorators.state
   37 import salt.utils.dictupdate
   38 import salt.utils.event
   39 import salt.utils.files
   40 import salt.utils.hashutils
   41 import salt.utils.immutabletypes as immutabletypes
   42 import salt.utils.msgpack
   43 import salt.utils.platform
   44 import salt.utils.process
   45 import salt.utils.url
   46 
   47 # Explicit late import to avoid circular import. DO NOT MOVE THIS.
   48 import salt.utils.yamlloader as yamlloader
   49 from salt.exceptions import CommandExecutionError, SaltRenderError, SaltReqTimeoutError
   50 
   51 # pylint: disable=import-error,no-name-in-module,redefined-builtin
   52 from salt.ext.six.moves import map, range, reload_module
   53 from salt.serializers.msgpack import deserialize as msgpack_deserialize
   54 from salt.serializers.msgpack import serialize as msgpack_serialize
   55 from salt.template import compile_template, compile_template_str
   56 from salt.utils.odict import DefaultOrderedDict, OrderedDict
   57 
   58 # pylint: enable=import-error,no-name-in-module,redefined-builtin
   59 
   60 log = logging.getLogger(__name__)
   61 
   62 
   63 # These are keywords passed to state module functions which are to be used
   64 # by salt in this state module and not on the actual state module function
   65 STATE_REQUISITE_KEYWORDS = frozenset(
   66     [
   67         "onchanges",
   68         "onchanges_any",
   69         "onfail",
   70         "onfail_any",
   71         "onfail_all",
   72         "onfail_stop",
   73         "prereq",
   74         "prerequired",
   75         "watch",
   76         "watch_any",
   77         "require",
   78         "require_any",
   79         "listen",
   80     ]
   81 )
   82 STATE_REQUISITE_IN_KEYWORDS = frozenset(
   83     ["onchanges_in", "onfail_in", "prereq_in", "watch_in", "require_in", "listen_in"]
   84 )
   85 STATE_RUNTIME_KEYWORDS = frozenset(
   86     [
   87         "fun",
   88         "state",
   89         "check_cmd",
   90         "failhard",
   91         "onlyif",
   92         "unless",
   93         "creates",
   94         "retry",
   95         "order",
   96         "parallel",
   97         "prereq",
   98         "prereq_in",
   99         "prerequired",
  100         "reload_modules",
  101         "reload_grains",
  102         "reload_pillar",
  103         "runas",
  104         "runas_password",
  105         "fire_event",
  106         "saltenv",
  107         "use",
  108         "use_in",
  109         "__env__",
  110         "__sls__",
  111         "__id__",
  112         "__orchestration_jid__",
  113         "__pub_user",
  114         "__pub_arg",
  115         "__pub_jid",
  116         "__pub_fun",
  117         "__pub_tgt",
  118         "__pub_ret",
  119         "__pub_pid",
  120         "__pub_tgt_type",
  121         "__prereq__",
  122         "__prerequired__",
  123     ]
  124 )
  125 
  126 STATE_INTERNAL_KEYWORDS = STATE_REQUISITE_KEYWORDS.union(
  127     STATE_REQUISITE_IN_KEYWORDS
  128 ).union(STATE_RUNTIME_KEYWORDS)
  129 
  130 
  131 def _odict_hashable(self):
  132     return id(self)
  133 
  134 
  135 OrderedDict.__hash__ = _odict_hashable
  136 
  137 
  138 def split_low_tag(tag):
  139     """
  140     Take a low tag and split it back into the low dict that it came from
  141     """
  142     state, id_, name, fun = tag.split("_|-")
  143 
  144     return {"state": state, "__id__": id_, "name": name, "fun": fun}
  145 
  146 
  147 def _gen_tag(low):
  148     """
  149     Generate the running dict tag string from the low data structure
  150     """
  151     return "{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}".format(low)
  152 
  153 
  154 def _clean_tag(tag):
  155     """
  156     Make tag name safe for filenames
  157     """
  158     return salt.utils.files.safe_filename_leaf(tag)
  159 
  160 
  161 def _l_tag(name, id_):
  162     low = {
  163         "name": "listen_{}".format(name),
  164         "__id__": "listen_{}".format(id_),
  165         "state": "Listen_Error",
  166         "fun": "Listen_Error",
  167     }
  168     return _gen_tag(low)
  169 
  170 
  171 def _calculate_fake_duration():
  172     """
  173     Generate a NULL duration for when states do not run
  174     but we want the results to be consistent.
  175     """
  176     utc_start_time = datetime.datetime.utcnow()
  177     local_start_time = utc_start_time - (
  178         datetime.datetime.utcnow() - datetime.datetime.now()
  179     )
  180     utc_finish_time = datetime.datetime.utcnow()
  181     start_time = local_start_time.time().isoformat()
  182     delta = utc_finish_time - utc_start_time
  183     # duration in milliseconds.microseconds
  184     duration = (delta.seconds * 1000000 + delta.microseconds) / 1000.0
  185 
  186     return start_time, duration
  187 
  188 
  189 def get_accumulator_dir(cachedir):
  190     """
  191     Return the directory that accumulator data is stored in, creating it if it
  192     doesn't exist.
  193     """
  194     fn_ = os.path.join(cachedir, "accumulator")
  195     if not os.path.isdir(fn_):
  196         # accumulator_dir is not present, create it
  197         os.makedirs(fn_)
  198     return fn_
  199 
  200 
  201 def trim_req(req):
  202     """
  203     Trim any function off of a requisite
  204     """
  205     reqfirst = next(iter(req))
  206     if "." in reqfirst:
  207         return {reqfirst.split(".")[0]: req[reqfirst]}
  208     return req
  209 
  210 
  211 def state_args(id_, state, high):
  212     """
  213     Return a set of the arguments passed to the named state
  214     """
  215     args = set()
  216     if id_ not in high:
  217         return args
  218     if state not in high[id_]:
  219         return args
  220     for item in high[id_][state]:
  221         if not isinstance(item, dict):
  222             continue
  223         if len(item) != 1:
  224             continue
  225         args.add(next(iter(item)))
  226     return args
  227 
  228 
  229 def find_name(name, state, high):
  230     """
  231     Scan high data for the id referencing the given name and return a list of (IDs, state) tuples that match
  232 
  233     Note: if `state` is sls, then we are looking for all IDs that match the given SLS
  234     """
  235     ext_id = []
  236     if name in high:
  237         ext_id.append((name, state))
  238     # if we are requiring an entire SLS, then we need to add ourselves to everything in that SLS
  239     elif state == "sls":
  240         for nid, item in high.items():
  241             if item["__sls__"] == name:
  242                 ext_id.append((nid, next(iter(item))))
  243     # otherwise we are requiring a single state, lets find it
  244     else:
  245         # We need to scan for the name
  246         for nid in high:
  247             if state in high[nid]:
  248                 if isinstance(high[nid][state], list):
  249                     for arg in high[nid][state]:
  250                         if not isinstance(arg, dict):
  251                             continue
  252                         if len(arg) != 1:
  253                             continue
  254                         if arg[next(iter(arg))] == name:
  255                             ext_id.append((nid, state))
  256     return ext_id
  257 
  258 
  259 def find_sls_ids(sls, high):
  260     """
  261     Scan for all ids in the given sls and return them in a dict; {name: state}
  262     """
  263     ret = []
  264     for nid, item in high.items():
  265         try:
  266             sls_tgt = item["__sls__"]
  267         except TypeError:
  268             if nid != "__exclude__":
  269                 log.error(
  270                     "Invalid non-dict item '%s' in high data. Value: %r", nid, item
  271                 )
  272             continue
  273         else:
  274             if sls_tgt == sls:
  275                 for st_ in item:
  276                     if st_.startswith("__"):
  277                         continue
  278                     ret.append((nid, st_))
  279     return ret
  280 
  281 
  282 def format_log(ret):
  283     """
  284     Format the state into a log message
  285     """
  286     msg = ""
  287     if isinstance(ret, dict):
  288         # Looks like the ret may be a valid state return
  289         if "changes" in ret:
  290             # Yep, looks like a valid state return
  291             chg = ret["changes"]
  292             if not chg:
  293                 if ret["comment"]:
  294                     msg = ret["comment"]
  295                 else:
  296                     msg = "No changes made for {0[name]}".format(ret)
  297             elif isinstance(chg, dict):
  298                 if "diff" in chg:
  299                     if isinstance(chg["diff"], str):
  300                         msg = "File changed:\n{}".format(chg["diff"])
  301                 if all([isinstance(x, dict) for x in chg.values()]):
  302                     if all([("old" in x and "new" in x) for x in chg.values()]):
  303                         msg = "Made the following changes:\n"
  304                         for pkg in chg:
  305                             old = chg[pkg]["old"]
  306                             if not old and old not in (False, None):
  307                                 old = "absent"
  308                             new = chg[pkg]["new"]
  309                             if not new and new not in (False, None):
  310                                 new = "absent"
  311                             # This must be able to handle unicode as some package names contain
  312                             # non-ascii characters like "Français" or "Español". See Issue #33605.
  313                             msg += "'{}' changed from '{}' to '{}'\n".format(
  314                                 pkg, old, new
  315                             )
  316             if not msg:
  317                 msg = str(ret["changes"])
  318             if ret["result"] is True or ret["result"] is None:
  319                 log.info(msg)
  320             else:
  321                 log.error(msg)
  322     else:
  323         # catch unhandled data
  324         log.info(str(ret))
  325 
  326 
  327 def master_compile(master_opts, minion_opts, grains, id_, saltenv):
  328     """
  329     Compile the master side low state data, and build the hidden state file
  330     """
  331     st_ = MasterHighState(master_opts, minion_opts, grains, id_, saltenv)
  332     return st_.compile_highstate()
  333 
  334 
  335 def ishashable(obj):
  336     try:
  337         hash(obj)
  338     except TypeError:
  339         return False
  340     return True
  341 
  342 
  343 def mock_ret(cdata):
  344     """
  345     Returns a mocked return dict with information about the run, without
  346     executing the state function
  347     """
  348     # As this is expanded it should be sent into the execution module
  349     # layer or it should be turned into a standalone loader system
  350     if cdata["args"]:
  351         name = cdata["args"][0]
  352     else:
  353         name = cdata["kwargs"]["name"]
  354     return {
  355         "name": name,
  356         "comment": "Not called, mocked",
  357         "changes": {},
  358         "result": True,
  359     }
  360 
  361 
  362 class StateError(Exception):
  363     """
  364     Custom exception class.
  365     """
  366 
  367 
  368 class Compiler:
  369     """
  370     Class used to compile and manage the High Data structure
  371     """
  372 
  373     def __init__(self, opts, renderers):
  374         self.opts = opts
  375         self.rend = renderers
  376 
  377     def render_template(self, template, **kwargs):
  378         """
  379         Enforce the states in a template
  380         """
  381         high = compile_template(
  382             template,
  383             self.rend,
  384             self.opts["renderer"],
  385             self.opts["renderer_blacklist"],
  386             self.opts["renderer_whitelist"],
  387             **kwargs
  388         )
  389         if not high:
  390             return high
  391         return self.pad_funcs(high)
  392 
  393     def pad_funcs(self, high):
  394         """
  395         Turns dot delimited function refs into function strings
  396         """
  397         for name in high:
  398             if not isinstance(high[name], dict):
  399                 if isinstance(high[name], str):
  400                     # Is this is a short state? It needs to be padded!
  401                     if "." in high[name]:
  402                         comps = high[name].split(".")
  403                         if len(comps) >= 2:
  404                             # Merge the comps
  405                             comps[1] = ".".join(comps[1 : len(comps)])
  406                         high[name] = {
  407                             # '__sls__': template,
  408                             # '__env__': None,
  409                             comps[0]: [comps[1]]
  410                         }
  411                         continue
  412                     continue
  413             skeys = set()
  414             for key in sorted(high[name]):
  415                 if key.startswith("_"):
  416                     continue
  417                 if not isinstance(high[name][key], list):
  418                     continue
  419                 if "." in key:
  420                     comps = key.split(".")
  421                     if len(comps) >= 2:
  422                         # Merge the comps
  423                         comps[1] = ".".join(comps[1 : len(comps)])
  424                     # Salt doesn't support state files such as:
  425                     #
  426                     # /etc/redis/redis.conf:
  427                     #   file.managed:
  428                     #     - user: redis
  429                     #     - group: redis
  430                     #     - mode: 644
  431                     #   file.comment:
  432                     #     - regex: ^requirepass
  433                     if comps[0] in skeys:
  434                         continue
  435                     high[name][comps[0]] = high[name].pop(key)
  436                     high[name][comps[0]].append(comps[1])
  437                     skeys.add(comps[0])
  438                     continue
  439                 skeys.add(key)
  440         return high
  441 
  442     def verify_high(self, high):
  443         """
  444         Verify that the high data is viable and follows the data structure
  445         """
  446         errors = []
  447         if not isinstance(high, dict):
  448             errors.append("High data is not a dictionary and is invalid")
  449         reqs = OrderedDict()
  450         for name, body in high.items():
  451             if name.startswith("__"):
  452                 continue
  453             if not isinstance(name, str):
  454                 errors.append(
  455                     "ID '{}' in SLS '{}' is not formed as a string, but "
  456                     "is a {}".format(name, body["__sls__"], type(name).__name__)
  457                 )
  458             if not isinstance(body, dict):
  459                 err = "The type {} in {} is not formatted as a dictionary".format(
  460                     name, body
  461                 )
  462                 errors.append(err)
  463                 continue
  464             for state in body:
  465                 if state.startswith("__"):
  466                     continue
  467                 if not isinstance(body[state], list):
  468                     errors.append(
  469                         "State '{}' in SLS '{}' is not formed as a list".format(
  470                             name, body["__sls__"]
  471                         )
  472                     )
  473                 else:
  474                     fun = 0
  475                     if "." in state:
  476                         fun += 1
  477                     for arg in body[state]:
  478                         if isinstance(arg, str):
  479                             fun += 1
  480                             if " " in arg.strip():
  481                                 errors.append(
  482                                     (
  483                                         'The function "{}" in state '
  484                                         '"{}" in SLS "{}" has '
  485                                         "whitespace, a function with whitespace is "
  486                                         "not supported, perhaps this is an argument "
  487                                         'that is missing a ":"'
  488                                     ).format(arg, name, body["__sls__"])
  489                                 )
  490                         elif isinstance(arg, dict):
  491                             # The arg is a dict, if the arg is require or
  492                             # watch, it must be a list.
  493                             #
  494                             # Add the requires to the reqs dict and check them
  495                             # all for recursive requisites.
  496                             argfirst = next(iter(arg))
  497                             if argfirst in ("require", "watch", "prereq", "onchanges"):
  498                                 if not isinstance(arg[argfirst], list):
  499                                     errors.append(
  500                                         (
  501                                             "The {}"
  502                                             " statement in state '{}' in SLS '{}' "
  503                                             "needs to be formed as a list"
  504                                         ).format(argfirst, name, body["__sls__"])
  505                                     )
  506                                 # It is a list, verify that the members of the
  507                                 # list are all single key dicts.
  508                                 else:
  509                                     reqs[name] = {"state": state}
  510                                     for req in arg[argfirst]:
  511                                         if isinstance(req, str):
  512                                             req = {"id": req}
  513                                         if not isinstance(req, dict):
  514                                             err = (
  515                                                 "Requisite declaration {}"
  516                                                 " in SLS {} is not formed as a"
  517                                                 " single key dictionary"
  518                                             ).format(req, body["__sls__"])
  519                                             errors.append(err)
  520                                             continue
  521                                         req_key = next(iter(req))
  522                                         req_val = req[req_key]
  523                                         if "." in req_key:
  524                                             errors.append(
  525                                                 "Invalid requisite type '{}' "
  526                                                 "in state '{}', in SLS "
  527                                                 "'{}'. Requisite types must "
  528                                                 "not contain dots, did you "
  529                                                 "mean '{}'?".format(
  530                                                     req_key,
  531                                                     name,
  532                                                     body["__sls__"],
  533                                                     req_key[: req_key.find(".")],
  534                                                 )
  535                                             )
  536                                         if not ishashable(req_val):
  537                                             errors.append(
  538                                                 (
  539                                                     'Illegal requisite "{}", '
  540                                                     "is SLS {}\n"
  541                                                 ).format(
  542                                                     str(req_val), body["__sls__"],
  543                                                 )
  544                                             )
  545                                             continue
  546 
  547                                         # Check for global recursive requisites
  548                                         reqs[name][req_val] = req_key
  549                                         # I am going beyond 80 chars on
  550                                         # purpose, this is just too much
  551                                         # of a pain to deal with otherwise
  552                                         if req_val in reqs:
  553                                             if name in reqs[req_val]:
  554                                                 if reqs[req_val][name] == state:
  555                                                     if (
  556                                                         reqs[req_val]["state"]
  557                                                         == reqs[name][req_val]
  558                                                     ):
  559                                                         err = (
  560                                                             "A recursive "
  561                                                             "requisite was found, SLS "
  562                                                             '"{}" ID "{}" ID "{}"'
  563                                                         ).format(
  564                                                             body["__sls__"],
  565                                                             name,
  566                                                             req_val,
  567                                                         )
  568                                                         errors.append(err)
  569                                 # Make sure that there is only one key in the
  570                                 # dict
  571                                 if len(list(arg)) != 1:
  572                                     errors.append(
  573                                         (
  574                                             "Multiple dictionaries "
  575                                             "defined in argument of state '{}' in SLS"
  576                                             " '{}'"
  577                                         ).format(name, body["__sls__"])
  578                                     )
  579                     if not fun:
  580                         if state == "require" or state == "watch":
  581                             continue
  582                         errors.append(
  583                             (
  584                                 "No function declared in state '{}' in" " SLS '{}'"
  585                             ).format(state, body["__sls__"])
  586                         )
  587                     elif fun > 1:
  588                         errors.append(
  589                             "Too many functions declared in state '{}' in "
  590                             "SLS '{}'".format(state, body["__sls__"])
  591                         )
  592         return errors
  593 
  594     def order_chunks(self, chunks):
  595         """
  596         Sort the chunk list verifying that the chunks follow the order
  597         specified in the order options.
  598         """
  599         cap = 1
  600         for chunk in chunks:
  601             if "order" in chunk:
  602                 if not isinstance(chunk["order"], int):
  603                     continue
  604 
  605                 chunk_order = chunk["order"]
  606                 if chunk_order > cap - 1 and chunk_order > 0:
  607                     cap = chunk_order + 100
  608         for chunk in chunks:
  609             if "order" not in chunk:
  610                 chunk["order"] = cap
  611                 continue
  612 
  613             if not isinstance(chunk["order"], (int, float)):
  614                 if chunk["order"] == "last":
  615                     chunk["order"] = cap + 1000000
  616                 elif chunk["order"] == "first":
  617                     chunk["order"] = 0
  618                 else:
  619                     chunk["order"] = cap
  620             if "name_order" in chunk:
  621                 chunk["order"] = chunk["order"] + chunk.pop("name_order") / 10000.0
  622             if chunk["order"] < 0:
  623                 chunk["order"] = cap + 1000000 + chunk["order"]
  624             chunk["name"] = salt.utils.data.decode(chunk["name"])
  625         chunks.sort(
  626             key=lambda chunk: (
  627                 chunk["order"],
  628                 "{0[state]}{0[name]}{0[fun]}".format(chunk),
  629             )
  630         )
  631         return chunks
  632 
  633     def compile_high_data(self, high):
  634         """
  635         "Compile" the high data as it is retrieved from the CLI or YAML into
  636         the individual state executor structures
  637         """
  638         chunks = []
  639         for name, body in high.items():
  640             if name.startswith("__"):
  641                 continue
  642             for state, run in body.items():
  643                 funcs = set()
  644                 names = []
  645                 if state.startswith("__"):
  646                     continue
  647                 chunk = {"state": state, "name": name}
  648                 if "__sls__" in body:
  649                     chunk["__sls__"] = body["__sls__"]
  650                 if "__env__" in body:
  651                     chunk["__env__"] = body["__env__"]
  652                 chunk["__id__"] = name
  653                 for arg in run:
  654                     if isinstance(arg, str):
  655                         funcs.add(arg)
  656                         continue
  657                     if isinstance(arg, dict):
  658                         for key, val in arg.items():
  659                             if key == "names":
  660                                 for _name in val:
  661                                     if _name not in names:
  662                                         names.append(_name)
  663                                 continue
  664                             else:
  665                                 chunk.update(arg)
  666                 if names:
  667                     name_order = 1
  668                     for entry in names:
  669                         live = copy.deepcopy(chunk)
  670                         if isinstance(entry, dict):
  671                             low_name = next(iter(entry.keys()))
  672                             live["name"] = low_name
  673                             list(map(live.update, entry[low_name]))
  674                         else:
  675                             live["name"] = entry
  676                         live["name_order"] = name_order
  677                         name_order = name_order + 1
  678                         for fun in funcs:
  679                             live["fun"] = fun
  680                             chunks.append(live)
  681                 else:
  682                     live = copy.deepcopy(chunk)
  683                     for fun in funcs:
  684                         live["fun"] = fun
  685                         chunks.append(live)
  686         chunks = self.order_chunks(chunks)
  687         return chunks
  688 
  689     def apply_exclude(self, high):
  690         """
  691         Read in the __exclude__ list and remove all excluded objects from the
  692         high data
  693         """
  694         if "__exclude__" not in high:
  695             return high
  696         ex_sls = set()
  697         ex_id = set()
  698         exclude = high.pop("__exclude__")
  699         for exc in exclude:
  700             if isinstance(exc, str):
  701                 # The exclude statement is a string, assume it is an sls
  702                 ex_sls.add(exc)
  703             if isinstance(exc, dict):
  704                 # Explicitly declared exclude
  705                 if len(exc) != 1:
  706                     continue
  707                 key = next(iter(exc.keys()))
  708                 if key == "sls":
  709                     ex_sls.add(exc["sls"])
  710                 elif key == "id":
  711                     ex_id.add(exc["id"])
  712         # Now the excludes have been simplified, use them
  713         if ex_sls:
  714             # There are sls excludes, find the associtaed ids
  715             for name, body in high.items():
  716                 if name.startswith("__"):
  717                     continue
  718                 if body.get("__sls__", "") in ex_sls:
  719                     ex_id.add(name)
  720         for id_ in ex_id:
  721             if id_ in high:
  722                 high.pop(id_)
  723         return high
  724 
  725 
  726 class State:
  727     """
  728     Class used to execute salt states
  729     """
  730 
  731     def __init__(
  732         self,
  733         opts,
  734         pillar_override=None,
  735         jid=None,
  736         pillar_enc=None,
  737         proxy=None,
  738         context=None,
  739         mocked=False,
  740         loader="states",
  741         initial_pillar=None,
  742     ):
  743         self.states_loader = loader
  744         if "grains" not in opts:
  745             opts["grains"] = salt.loader.grains(opts)
  746         self.opts = opts
  747         self.proxy = proxy
  748         self._pillar_override = pillar_override
  749         if pillar_enc is not None:
  750             try:
  751                 pillar_enc = pillar_enc.lower()
  752             except AttributeError:
  753                 pillar_enc = str(pillar_enc).lower()
  754         self._pillar_enc = pillar_enc
  755         log.debug("Gathering pillar data for state run")
  756         if initial_pillar and not self._pillar_override:
  757             self.opts["pillar"] = initial_pillar
  758         else:
  759             # Compile pillar data
  760             self.opts["pillar"] = self._gather_pillar()
  761             # Reapply overrides on top of compiled pillar
  762             if self._pillar_override:
  763                 self.opts["pillar"] = salt.utils.dictupdate.merge(
  764                     self.opts["pillar"],
  765                     self._pillar_override,
  766                     self.opts.get("pillar_source_merging_strategy", "smart"),
  767                     self.opts.get("renderer", "yaml"),
  768                     self.opts.get("pillar_merge_lists", False),
  769                 )
  770         log.debug("Finished gathering pillar data for state run")
  771         self.state_con = context or {}
  772         self.load_modules()
  773         self.active = set()
  774         self.mod_init = set()
  775         self.pre = {}
  776         self.__run_num = 0
  777         self.jid = jid
  778         self.instance_id = str(id(self))
  779         self.inject_globals = {}
  780         self.mocked = mocked
  781 
  782     def _gather_pillar(self):
  783         """
  784         Whenever a state run starts, gather the pillar data fresh
  785         """
  786         if self._pillar_override:
  787             if self._pillar_enc:
  788                 try:
  789                     self._pillar_override = salt.utils.crypt.decrypt(
  790                         self._pillar_override,
  791                         self._pillar_enc,
  792                         translate_newlines=True,
  793                         renderers=getattr(self, "rend", None),
  794                         opts=self.opts,
  795                         valid_rend=self.opts["decrypt_pillar_renderers"],
  796                     )
  797                 except Exception as exc:  # pylint: disable=broad-except
  798                     log.error("Failed to decrypt pillar override: %s", exc)
  799 
  800             if isinstance(self._pillar_override, str):
  801                 # This can happen if an entire pillar dictionary was passed as
  802                 # a single encrypted string. The override will have been
  803                 # decrypted above, and should now be a stringified dictionary.
  804                 # Use the YAML loader to convert that to a Python dictionary.
  805                 try:
  806                     self._pillar_override = yamlloader.load(
  807                         self._pillar_override, Loader=yamlloader.SaltYamlSafeLoader
  808                     )
  809                 except Exception as exc:  # pylint: disable=broad-except
  810                     log.error("Failed to load CLI pillar override")
  811                     log.exception(exc)
  812 
  813             if not isinstance(self._pillar_override, dict):
  814                 log.error("Pillar override was not passed as a dictionary")
  815                 self._pillar_override = None
  816 
  817         pillar = salt.pillar.get_pillar(
  818             self.opts,
  819             self.opts["grains"],
  820             self.opts["id"],
  821             self.opts["saltenv"],
  822             pillar_override=self._pillar_override,
  823             pillarenv=self.opts.get("pillarenv"),
  824         )
  825         return pillar.compile_pillar()
  826 
  827     def _mod_init(self, low):
  828         """
  829         Check the module initialization function, if this is the first run
  830         of a state package that has a mod_init function, then execute the
  831         mod_init function in the state module.
  832         """
  833         # ensure that the module is loaded
  834         try:
  835             self.states[
  836                 "{}.{}".format(low["state"], low["fun"])
  837             ]  # pylint: disable=W0106
  838         except KeyError:
  839             return
  840         minit = "{}.mod_init".format(low["state"])
  841         if low["state"] not in self.mod_init:
  842             if minit in self.states._dict:
  843                 mret = self.states[minit](low)
  844                 if not mret:
  845                     return
  846                 self.mod_init.add(low["state"])
  847 
  848     def _mod_aggregate(self, low, running, chunks):
  849         """
  850         Execute the aggregation systems to runtime modify the low chunk
  851         """
  852         agg_opt = self.functions["config.option"]("state_aggregate")
  853         if "aggregate" in low:
  854             agg_opt = low["aggregate"]
  855         if agg_opt is True:
  856             agg_opt = [low["state"]]
  857         elif not isinstance(agg_opt, list):
  858             return low
  859         if low["state"] in agg_opt and not low.get("__agg__"):
  860             agg_fun = "{}.mod_aggregate".format(low["state"])
  861             if agg_fun in self.states:
  862                 try:
  863                     low = self.states[agg_fun](low, chunks, running)
  864                     low["__agg__"] = True
  865                 except TypeError:
  866                     log.error("Failed to execute aggregate for state %s", low["state"])
  867         return low
  868 
  869     def _run_check(self, low_data):
  870         """
  871         Check that unless doesn't return 0, and that onlyif returns a 0.
  872         """
  873         ret = {"result": False, "comment": []}
  874         cmd_opts = {}
  875 
  876         # Set arguments from cmd.run state as appropriate
  877         POSSIBLE_CMD_ARGS = (
  878             "cwd",
  879             "root",
  880             "runas",
  881             "env",
  882             "prepend_path",
  883             "umask",
  884             "timeout",
  885             "success_retcodes",
  886         )
  887         for run_cmd_arg in POSSIBLE_CMD_ARGS:
  888             cmd_opts[run_cmd_arg] = low_data.get(run_cmd_arg)
  889 
  890         if "shell" in low_data:
  891             cmd_opts["shell"] = low_data["shell"]
  892         elif "shell" in self.opts["grains"]:
  893             cmd_opts["shell"] = self.opts["grains"].get("shell")
  894 
  895         if "onlyif" in low_data:
  896             _ret = self._run_check_onlyif(low_data, cmd_opts)
  897             ret["result"] = _ret["result"]
  898             ret["comment"].append(_ret["comment"])
  899             if "skip_watch" in _ret:
  900                 ret["skip_watch"] = _ret["skip_watch"]
  901 
  902         if "unless" in low_data:
  903             _ret = self._run_check_unless(low_data, cmd_opts)
  904             # If either result is True, the returned result should be True
  905             ret["result"] = _ret["result"] or ret["result"]
  906             ret["comment"].append(_ret["comment"])
  907             if "skip_watch" in _ret:
  908                 # If either result is True, the returned result should be True
  909                 ret["skip_watch"] = _ret["skip_watch"] or ret["skip_watch"]
  910 
  911         if "creates" in low_data:
  912             _ret = self._run_check_creates(low_data)
  913             ret["result"] = _ret["result"] or ret["result"]
  914             ret["comment"].append(_ret["comment"])
  915             if "skip_watch" in _ret:
  916                 # If either result is True, the returned result should be True
  917                 ret["skip_watch"] = _ret["skip_watch"] or ret["skip_watch"]
  918 
  919         return ret
  920 
  921     def _run_check_function(self, entry):
  922         """Format slot args and run unless/onlyif function."""
  923         fun = entry.pop("fun")
  924         args = entry.pop("args") if "args" in entry else []
  925         cdata = {"args": args, "kwargs": entry}
  926         self.format_slots(cdata)
  927         return self.functions[fun](*cdata["args"], **cdata["kwargs"])
  928 
  929     def _run_check_onlyif(self, low_data, cmd_opts):
  930         """
  931         Check that unless doesn't return 0, and that onlyif returns a 0.
  932         """
  933         ret = {"result": False}
  934 
  935         if not isinstance(low_data["onlyif"], list):
  936             low_data_onlyif = [low_data["onlyif"]]
  937         else:
  938             low_data_onlyif = low_data["onlyif"]
  939 
  940         def _check_cmd(cmd):
  941             if cmd != 0 and ret["result"] is False:
  942                 ret.update(
  943                     {
  944                         "comment": "onlyif condition is false",
  945                         "skip_watch": True,
  946                         "result": True,
  947                     }
  948                 )
  949             elif cmd == 0:
  950                 ret.update({"comment": "onlyif condition is true", "result": False})
  951 
  952         for entry in low_data_onlyif:
  953             if isinstance(entry, str):
  954                 try:
  955                     cmd = self.functions["cmd.retcode"](
  956                         entry, ignore_retcode=True, python_shell=True, **cmd_opts
  957                     )
  958                 except CommandExecutionError:
  959                     # Command failed, notify onlyif to skip running the item
  960                     cmd = 100
  961                 log.debug("Last command return code: %s", cmd)
  962                 _check_cmd(cmd)
  963             elif isinstance(entry, dict):
  964                 if "fun" not in entry:
  965                     ret["comment"] = "no `fun` argument in onlyif: {}".format(entry)
  966                     log.warning(ret["comment"])
  967                     return ret
  968 
  969                 get_return = entry.pop("get_return", None)
  970                 result = self._run_check_function(entry)
  971                 if get_return:
  972                     result = salt.utils.data.traverse_dict_and_list(result, get_return)
  973                 if self.state_con.get("retcode", 0):
  974                     _check_cmd(self.state_con["retcode"])
  975                 elif not result:
  976                     ret.update(
  977                         {
  978                             "comment": "onlyif condition is false",
  979                             "skip_watch": True,
  980                             "result": True,
  981                         }
  982                     )
  983                 else:
  984                     ret.update({"comment": "onlyif condition is true", "result": False})
  985 
  986             else:
  987                 ret.update(
  988                     {
  989                         "comment": "onlyif execution failed, bad type passed",
  990                         "result": False,
  991                     }
  992                 )
  993         return ret
  994 
  995     def _run_check_unless(self, low_data, cmd_opts):
  996         """
  997         Check that unless doesn't return 0, and that onlyif returns a 0.
  998         """
  999         ret = {"result": False}
 1000 
 1001         if not isinstance(low_data["unless"], list):
 1002             low_data_unless = [low_data["unless"]]
 1003         else:
 1004             low_data_unless = low_data["unless"]
 1005 
 1006         def _check_cmd(cmd):
 1007             if cmd == 0 and ret["result"] is False:
 1008                 ret.update(
 1009                     {
 1010                         "comment": "unless condition is true",
 1011                         "skip_watch": True,
 1012                         "result": True,
 1013                     }
 1014                 )
 1015             elif cmd != 0:
 1016                 ret.update({"comment": "unless condition is false", "result": False})
 1017 
 1018         for entry in low_data_unless:
 1019             if isinstance(entry, str):
 1020                 try:
 1021                     cmd = self.functions["cmd.retcode"](
 1022                         entry, ignore_retcode=True, python_shell=True, **cmd_opts
 1023                     )
 1024                     log.debug("Last command return code: %s", cmd)
 1025                 except CommandExecutionError:
 1026                     # Command failed, so notify unless to skip the item
 1027                     cmd = 0
 1028                 _check_cmd(cmd)
 1029             elif isinstance(entry, dict):
 1030                 if "fun" not in entry:
 1031                     ret["comment"] = "no `fun` argument in unless: {}".format(entry)
 1032                     log.warning(ret["comment"])
 1033                     return ret
 1034 
 1035                 get_return = entry.pop("get_return", None)
 1036                 result = self._run_check_function(entry)
 1037                 if get_return:
 1038                     result = salt.utils.data.traverse_dict_and_list(result, get_return)
 1039                 if self.state_con.get("retcode", 0):
 1040                     _check_cmd(self.state_con["retcode"])
 1041                 elif result:
 1042                     ret.update(
 1043                         {
 1044                             "comment": "unless condition is true",
 1045                             "skip_watch": True,
 1046                             "result": True,
 1047                         }
 1048                     )
 1049                 else:
 1050                     ret.update(
 1051                         {"comment": "unless condition is false", "result": False}
 1052                     )
 1053             else:
 1054                 ret.update(
 1055                     {
 1056                         "comment": "unless condition is false, bad type passed",
 1057                         "result": False,
 1058                     }
 1059                 )
 1060 
 1061         # No reason to stop, return ret
 1062         return ret
 1063 
 1064     def _run_check_cmd(self, low_data):
 1065         """
 1066         Alter the way a successful state run is determined
 1067         """
 1068         ret = {"result": False}
 1069         cmd_opts = {}
 1070         if "shell" in self.opts["grains"]:
 1071             cmd_opts["shell"] = self.opts["grains"].get("shell")
 1072         for entry in low_data["check_cmd"]:
 1073             cmd = self.functions["cmd.retcode"](
 1074                 entry, ignore_retcode=True, python_shell=True, **cmd_opts
 1075             )
 1076             log.debug("Last command return code: %s", cmd)
 1077             if cmd == 0 and ret["result"] is False:
 1078                 ret.update(
 1079                     {
 1080                         "comment": "check_cmd determined the state succeeded",
 1081                         "result": True,
 1082                     }
 1083                 )
 1084             elif cmd != 0:
 1085                 ret.update(
 1086                     {
 1087                         "comment": "check_cmd determined the state failed",
 1088                         "result": False,
 1089                     }
 1090                 )
 1091                 return ret
 1092         return ret
 1093 
 1094     def _run_check_creates(self, low_data):
 1095         """
 1096         Check that listed files exist
 1097         """
 1098         ret = {"result": False}
 1099 
 1100         if isinstance(low_data["creates"], str) and os.path.exists(low_data["creates"]):
 1101             ret["comment"] = "{} exists".format(low_data["creates"])
 1102             ret["result"] = True
 1103             ret["skip_watch"] = True
 1104         elif isinstance(low_data["creates"], list) and all(
 1105             [os.path.exists(path) for path in low_data["creates"]]
 1106         ):
 1107             ret["comment"] = "All files in creates exist"
 1108             ret["result"] = True
 1109             ret["skip_watch"] = True
 1110         else:
 1111             ret["comment"] = "Creates files not found"
 1112             ret["result"] = False
 1113 
 1114         return ret
 1115 
 1116     def reset_run_num(self):
 1117         """
 1118         Rest the run_num value to 0
 1119         """
 1120         self.__run_num = 0
 1121 
 1122     def _load_states(self):
 1123         """
 1124         Read the state loader value and loadup the correct states subsystem
 1125         """
 1126         if self.states_loader == "thorium":
 1127             self.states = salt.loader.thorium(
 1128                 self.opts, self.functions, {}
 1129             )  # TODO: Add runners, proxy?
 1130         else:
 1131             self.states = salt.loader.states(
 1132                 self.opts,
 1133                 self.functions,
 1134                 self.utils,
 1135                 self.serializers,
 1136                 context=self.state_con,
 1137                 proxy=self.proxy,
 1138             )
 1139 
 1140     def load_modules(self, data=None, proxy=None):
 1141         """
 1142         Load the modules into the state
 1143         """
 1144         log.info("Loading fresh modules for state activity")
 1145         self.utils = salt.loader.utils(self.opts)
 1146         self.functions = salt.loader.minion_mods(
 1147             self.opts, self.state_con, utils=self.utils, proxy=self.proxy
 1148         )
 1149         if isinstance(data, dict):
 1150             if data.get("provider", False):
 1151                 if isinstance(data["provider"], str):
 1152                     providers = [{data["state"]: data["provider"]}]
 1153                 elif isinstance(data["provider"], list):
 1154                     providers = data["provider"]
 1155                 else:
 1156                     providers = {}
 1157                 for provider in providers:
 1158                     for mod in provider:
 1159                         funcs = salt.loader.raw_mod(
 1160                             self.opts, provider[mod], self.functions
 1161                         )
 1162                         if funcs:
 1163                             for func in funcs:
 1164                                 f_key = "{}{}".format(mod, func[func.rindex(".") :])
 1165                                 self.functions[f_key] = funcs[func]
 1166         self.serializers = salt.loader.serializers(self.opts)
 1167         self._load_states()
 1168         self.rend = salt.loader.render(
 1169             self.opts,
 1170             self.functions,
 1171             states=self.states,
 1172             proxy=self.proxy,
 1173             context=self.state_con,
 1174         )
 1175 
 1176     def module_refresh(self):
 1177         """
 1178         Refresh all the modules
 1179         """
 1180         log.debug("Refreshing modules...")
 1181         if self.opts["grains"].get("os") != "MacOS":
 1182             # In case a package has been installed into the current python
 1183             # process 'site-packages', the 'site' module needs to be reloaded in
 1184             # order for the newly installed package to be importable.
 1185             try:
 1186                 reload_module(site)
 1187             except RuntimeError:
 1188                 log.error(
 1189                     "Error encountered during module reload. Modules were not reloaded."
 1190                 )
 1191             except TypeError:
 1192                 log.error(
 1193                     "Error encountered during module reload. Modules were not reloaded."
 1194                 )
 1195         self.load_modules()
 1196         if not self.opts.get("local", False) and self.opts.get("multiprocessing", True):
 1197             self.functions["saltutil.refresh_modules"]()
 1198 
 1199     def check_refresh(self, data, ret):
 1200         """
 1201         Check to see if the modules for this state instance need to be updated,
 1202         only update if the state is a file or a package and if it changed
 1203         something. If the file function is managed check to see if the file is a
 1204         possible module type, e.g. a python, pyx, or .so. Always refresh if the
 1205         function is recurse, since that can lay down anything.
 1206         """
 1207         _reload_modules = False
 1208         if data.get("reload_grains", False):
 1209             log.debug("Refreshing grains...")
 1210             self.opts["grains"] = salt.loader.grains(self.opts)
 1211             _reload_modules = True
 1212 
 1213         if data.get("reload_pillar", False):
 1214             log.debug("Refreshing pillar...")
 1215             self.opts["pillar"] = self._gather_pillar()
 1216             _reload_modules = True
 1217 
 1218         if not ret["changes"]:
 1219             if data.get("force_reload_modules", False):
 1220                 self.module_refresh()
 1221             return
 1222 
 1223         if data.get("reload_modules", False) or _reload_modules:
 1224             # User explicitly requests a reload
 1225             self.module_refresh()
 1226             return
 1227 
 1228         if data["state"] == "file":
 1229             if data["fun"] == "managed":
 1230                 if data["name"].endswith((".py", ".pyx", ".pyo", ".pyc", ".so")):
 1231                     self.module_refresh()
 1232             elif data["fun"] == "recurse":
 1233                 self.module_refresh()
 1234             elif data["fun"] == "symlink":
 1235                 if "bin" in data["name"]:
 1236                     self.module_refresh()
 1237         elif data["state"] in ("pkg", "ports", "pip"):
 1238             self.module_refresh()
 1239 
 1240     def verify_data(self, data):
 1241         """
 1242         Verify the data, return an error statement if something is wrong
 1243         """
 1244         errors = []
 1245         if "state" not in data:
 1246             errors.append('Missing "state" data')
 1247         if "fun" not in data:
 1248             errors.append('Missing "fun" data')
 1249         if "name" not in data:
 1250             errors.append('Missing "name" data')
 1251         if data["name"] and not isinstance(data["name"], str):
 1252             errors.append(
 1253                 "ID '{}' {}is not formed as a string, but is a {}".format(
 1254                     data["name"],
 1255                     "in SLS '{}' ".format(data["__sls__"]) if "__sls__" in data else "",
 1256                     type(data["name"]).__name__,
 1257                 )
 1258             )
 1259         if errors:
 1260             return errors
 1261         full = data["state"] + "." + data["fun"]
 1262         if full not in self.states:
 1263             if "__sls__" in data:
 1264                 errors.append(
 1265                     "State '{}' was not found in SLS '{}'".format(full, data["__sls__"])
 1266                 )
 1267                 reason = self.states.missing_fun_string(full)
 1268                 if reason:
 1269                     errors.append("Reason: {}".format(reason))
 1270             else:
 1271                 errors.append("Specified state '{}' was not found".format(full))
 1272         else:
 1273             # First verify that the parameters are met
 1274             aspec = salt.utils.args.get_function_argspec(self.states[full])
 1275             arglen = 0
 1276             deflen = 0
 1277             if isinstance(aspec.args, list):
 1278                 arglen = len(aspec.args)
 1279             if isinstance(aspec.defaults, tuple):
 1280                 deflen = len(aspec.defaults)
 1281             for ind in range(arglen - deflen):
 1282                 if aspec.args[ind] not in data:
 1283                     errors.append(
 1284                         "Missing parameter {} for state {}".format(
 1285                             aspec.args[ind], full
 1286                         )
 1287                     )
 1288         # If this chunk has a recursive require, then it will cause a
 1289         # recursive loop when executing, check for it
 1290         reqdec = ""
 1291         if "require" in data:
 1292             reqdec = "require"
 1293         if "watch" in data:
 1294             # Check to see if the service has a mod_watch function, if it does
 1295             # not, then just require
 1296             # to just require extend the require statement with the contents
 1297             # of watch so that the mod_watch function is not called and the
 1298             # requisite capability is still used
 1299             if "{}.mod_watch".format(data["state"]) not in self.states:
 1300                 if "require" in data:
 1301                     data["require"].extend(data.pop("watch"))
 1302                 else:
 1303                     data["require"] = data.pop("watch")
 1304                 reqdec = "require"
 1305             else:
 1306                 reqdec = "watch"
 1307         if reqdec:
 1308             for req in data[reqdec]:
 1309                 reqfirst = next(iter(req))
 1310                 if data["state"] == reqfirst:
 1311                     if fnmatch.fnmatch(data["name"], req[reqfirst]) or fnmatch.fnmatch(
 1312                         data["__id__"], req[reqfirst]
 1313                     ):
 1314                         err = (
 1315                             "Recursive require detected in SLS {} for"
 1316                             " require {} in ID {}"
 1317                         ).format(data["__sls__"], req, data["__id__"])
 1318                         errors.append(err)
 1319         return errors
 1320 
 1321     def verify_high(self, high):
 1322         """
 1323         Verify that the high data is viable and follows the data structure
 1324         """
 1325         errors = []
 1326         if not isinstance(high, dict):
 1327             errors.append("High data is not a dictionary and is invalid")
 1328         reqs = OrderedDict()
 1329         for name, body in high.items():
 1330             try:
 1331                 if name.startswith("__"):
 1332                     continue
 1333             except AttributeError:
 1334                 pass
 1335             if not isinstance(name, str):
 1336                 errors.append(
 1337                     "ID '{}' in SLS '{}' is not formed as a string, but "
 1338                     "is a {}. It may need to be quoted.".format(
 1339                         name, body["__sls__"], type(name).__name__
 1340                     )
 1341                 )
 1342             if not isinstance(body, dict):
 1343                 err = "The type {} in {} is not formatted as a dictionary".format(
 1344                     name, body
 1345                 )
 1346                 errors.append(err)
 1347                 continue
 1348             for state in body:
 1349                 if state.startswith("__"):
 1350                     continue
 1351                 if body[state] is None:
 1352                     errors.append(
 1353                         "ID '{}' in SLS '{}' contains a short declaration "
 1354                         "({}) with a trailing colon. When not passing any "
 1355                         "arguments to a state, the colon must be omitted.".format(
 1356                             name, body["__sls__"], state
 1357                         )
 1358                     )
 1359                     continue
 1360                 if not isinstance(body[state], list):
 1361                     errors.append(
 1362                         "State '{}' in SLS '{}' is not formed as a list".format(
 1363                             name, body["__sls__"]
 1364                         )
 1365                     )
 1366                 else:
 1367                     fun = 0
 1368                     if "." in state:
 1369                         fun += 1
 1370                     for arg in body[state]:
 1371                         if isinstance(arg, str):
 1372                             fun += 1
 1373                             if " " in arg.strip():
 1374                                 errors.append(
 1375                                     (
 1376                                         'The function "{}" in state '
 1377                                         '"{}" in SLS "{}" has '
 1378                                         "whitespace, a function with whitespace is "
 1379                                         "not supported, perhaps this is an argument "
 1380                                         'that is missing a ":"'
 1381                                     ).format(arg, name, body["__sls__"])
 1382                                 )
 1383                         elif isinstance(arg, dict):
 1384                             # The arg is a dict, if the arg is require or
 1385                             # watch, it must be a list.
 1386                             #
 1387                             # Add the requires to the reqs dict and check them
 1388                             # all for recursive requisites.
 1389                             argfirst = next(iter(arg))
 1390                             if argfirst == "names":
 1391                                 if not isinstance(arg[argfirst], list):
 1392                                     errors.append(
 1393                                         "The 'names' argument in state "
 1394                                         "'{}' in SLS '{}' needs to be "
 1395                                         "formed as a list".format(name, body["__sls__"])
 1396                                     )
 1397                             if argfirst in ("require", "watch", "prereq", "onchanges"):
 1398                                 if not isinstance(arg[argfirst], list):
 1399                                     errors.append(
 1400                                         "The {} statement in state '{}' in "
 1401                                         "SLS '{}' needs to be formed as a "
 1402                                         "list".format(argfirst, name, body["__sls__"])
 1403                                     )
 1404                                 # It is a list, verify that the members of the
 1405                                 # list are all single key dicts.
 1406                                 else:
 1407                                     reqs[name] = OrderedDict(state=state)
 1408                                     for req in arg[argfirst]:
 1409                                         if isinstance(req, str):
 1410                                             req = {"id": req}
 1411                                         if not isinstance(req, dict):
 1412                                             err = (
 1413                                                 "Requisite declaration {}"
 1414                                                 " in SLS {} is not formed as a"
 1415                                                 " single key dictionary"
 1416                                             ).format(req, body["__sls__"])
 1417                                             errors.append(err)
 1418                                             continue
 1419                                         req_key = next(iter(req))
 1420                                         req_val = req[req_key]
 1421                                         if "." in req_key:
 1422                                             errors.append(
 1423                                                 "Invalid requisite type '{}' "
 1424                                                 "in state '{}', in SLS "
 1425                                                 "'{}'. Requisite types must "
 1426                                                 "not contain dots, did you "
 1427                                                 "mean '{}'?".format(
 1428                                                     req_key,
 1429                                                     name,
 1430                                                     body["__sls__"],
 1431                                                     req_key[: req_key.find(".")],
 1432                                                 )
 1433                                             )
 1434                                         if not ishashable(req_val):
 1435                                             errors.append(
 1436                                                 (
 1437                                                     'Illegal requisite "{}", '
 1438                                                     "please check your syntax.\n"
 1439                                                 ).format(req_val)
 1440                                             )
 1441                                             continue
 1442 
 1443                                         # Check for global recursive requisites
 1444                                         reqs[name][req_val] = req_key
 1445                                         # I am going beyond 80 chars on
 1446                                         # purpose, this is just too much
 1447                                         # of a pain to deal with otherwise
 1448                                         if req_val in reqs:
 1449                                             if name in reqs[req_val]:
 1450                                                 if reqs[req_val][name] == state:
 1451                                                     if (
 1452                                                         reqs[req_val]["state"]
 1453                                                         == reqs[name][req_val]
 1454                                                     ):
 1455                                                         err = (
 1456                                                             "A recursive "
 1457                                                             "requisite was found, SLS "
 1458                                                             '"{}" ID "{}" ID "{}"'
 1459                                                         ).format(
 1460                                                             body["__sls__"],
 1461                                                             name,
 1462                                                             req_val,
 1463                                                         )
 1464                                                         errors.append(err)
 1465                                 # Make sure that there is only one key in the
 1466                                 # dict
 1467                                 if len(list(arg)) != 1:
 1468                                     errors.append(
 1469                                         "Multiple dictionaries defined in "
 1470                                         "argument of state '{}' in SLS '{}'".format(
 1471                                             name, body["__sls__"]
 1472                                         )
 1473                                     )
 1474                     if not fun:
 1475                         if state == "require" or state == "watch":
 1476                             continue
 1477                         errors.append(
 1478                             "No function declared in state '{}' in SLS '{}'".format(
 1479                                 state, body["__sls__"]
 1480                             )
 1481                         )
 1482                     elif fun > 1:
 1483                         errors.append(
 1484                             "Too many functions declared in state '{}' in "
 1485                             "SLS '{}'".format(state, body["__sls__"])
 1486                         )
 1487         return errors
 1488 
 1489     def verify_chunks(self, chunks):
 1490         """
 1491         Verify the chunks in a list of low data structures
 1492         """
 1493         err = []
 1494         for chunk in chunks:
 1495             err.extend(self.verify_data(chunk))
 1496         return err
 1497 
 1498     def order_chunks(self, chunks):
 1499         """
 1500         Sort the chunk list verifying that the chunks follow the order
 1501         specified in the order options.
 1502         """
 1503         cap = 1
 1504         for chunk in chunks:
 1505             if "order" in chunk:
 1506                 if not isinstance(chunk["order"], int):
 1507                     continue
 1508 
 1509                 chunk_order = chunk["order"]
 1510                 if chunk_order > cap - 1 and chunk_order > 0:
 1511                     cap = chunk_order + 100
 1512         for chunk in chunks:
 1513             if "order" not in chunk:
 1514                 chunk["order"] = cap
 1515                 continue
 1516 
 1517             if not isinstance(chunk["order"], (int, float)):
 1518                 if chunk["order"] == "last":
 1519                     chunk["order"] = cap + 1000000
 1520                 elif chunk["order"] == "first":
 1521                     chunk["order"] = 0
 1522                 else:
 1523                     chunk["order"] = cap
 1524             if "name_order" in chunk:
 1525                 chunk["order"] = chunk["order"] + chunk.pop("name_order") / 10000.0
 1526             if chunk["order"] < 0:
 1527                 chunk["order"] = cap + 1000000 + chunk["order"]
 1528         chunks.sort(
 1529             key=lambda chunk: (
 1530                 chunk["order"],
 1531                 "{0[state]}{0[name]}{0[fun]}".format(chunk),
 1532             )
 1533         )
 1534         return chunks
 1535 
 1536     def compile_high_data(self, high, orchestration_jid=None):
 1537         """
 1538         "Compile" the high data as it is retrieved from the CLI or YAML into
 1539         the individual state executor structures
 1540         """
 1541         chunks = []
 1542         for name, body in high.items():
 1543             if name.startswith("__"):
 1544                 continue
 1545             for state, run in body.items():
 1546                 funcs = set()
 1547                 names = []
 1548                 if state.startswith("__"):
 1549                     continue
 1550                 chunk = {"state": state, "name": name}
 1551                 if orchestration_jid is not None:
 1552                     chunk["__orchestration_jid__"] = orchestration_jid
 1553                 if "__sls__" in body:
 1554                     chunk["__sls__"] = body["__sls__"]
 1555                 if "__env__" in body:
 1556                     chunk["__env__"] = body["__env__"]
 1557                 chunk["__id__"] = name
 1558                 for arg in run:
 1559                     if isinstance(arg, str):
 1560                         funcs.add(arg)
 1561                         continue
 1562                     if isinstance(arg, dict):
 1563                         for key, val in arg.items():
 1564                             if key == "names":
 1565                                 for _name in val:
 1566                                     if _name not in names:
 1567                                         names.append(_name)
 1568                             elif key == "state":
 1569                                 # Don't pass down a state override
 1570                                 continue
 1571                             elif key == "name" and not isinstance(val, str):
 1572                                 # Invalid name, fall back to ID
 1573                                 chunk[key] = name
 1574                             else:
 1575                                 chunk[key] = val
 1576                 if names:
 1577                     name_order = 1
 1578                     for entry in names:
 1579                         live = copy.deepcopy(chunk)
 1580                         if isinstance(entry, dict):
 1581                             low_name = next(iter(entry.keys()))
 1582                             live["name"] = low_name
 1583                             list(map(live.update, entry[low_name]))
 1584                         else:
 1585                             live["name"] = entry
 1586                         live["name_order"] = name_order
 1587                         name_order += 1
 1588                         for fun in funcs:
 1589                             live["fun"] = fun
 1590                             chunks.append(live)
 1591                 else:
 1592                     live = copy.deepcopy(chunk)
 1593                     for fun in funcs:
 1594                         live["fun"] = fun
 1595                         chunks.append(live)
 1596         chunks = self.order_chunks(chunks)
 1597         return chunks
 1598 
 1599     def reconcile_extend(self, high):
 1600         """
 1601         Pull the extend data and add it to the respective high data
 1602         """
 1603         errors = []
 1604         if "__extend__" not in high:
 1605             return high, errors
 1606         ext = high.pop("__extend__")
 1607         for ext_chunk in ext:
 1608             for name, body in ext_chunk.items():
 1609                 if name not in high:
 1610                     state_type = next(x for x in body if not x.startswith("__"))
 1611                     # Check for a matching 'name' override in high data
 1612                     ids = find_name(name, state_type, high)
 1613                     if len(ids) != 1:
 1614                         errors.append(
 1615                             "Cannot extend ID '{0}' in '{1}:{2}'. It is not "
 1616                             "part of the high state.\n"
 1617                             "This is likely due to a missing include statement "
 1618                             "or an incorrectly typed ID.\nEnsure that a "
 1619                             "state with an ID of '{0}' is available\nin "
 1620                             "environment '{1}' and to SLS '{2}'".format(
 1621                                 name,
 1622                                 body.get("__env__", "base"),
 1623                                 body.get("__sls__", "base"),
 1624                             )
 1625                         )
 1626                         continue
 1627                     else:
 1628                         name = ids[0][0]
 1629 
 1630                 for state, run in body.items():
 1631                     if state.startswith("__"):
 1632                         continue
 1633                     if state not in high[name]:
 1634                         high[name][state] = run
 1635                         continue
 1636                     # high[name][state] is extended by run, both are lists
 1637                     for arg in run:
 1638                         update = False
 1639                         for hind in range(len(high[name][state])):
 1640                             if isinstance(arg, str) and isinstance(
 1641                                 high[name][state][hind], str
 1642                             ):
 1643                                 # replacing the function, replace the index
 1644                                 high[name][state].pop(hind)
 1645                                 high[name][state].insert(hind, arg)
 1646                                 update = True
 1647                                 continue
 1648                             if isinstance(arg, dict) and isinstance(
 1649                                 high[name][state][hind], dict
 1650                             ):
 1651                                 # It is an option, make sure the options match
 1652                                 argfirst = next(iter(arg))
 1653                                 if argfirst == next(iter(high[name][state][hind])):
 1654                                     # If argfirst is a requisite then we must merge
 1655                                     # our requisite with that of the target state
 1656                                     if argfirst in STATE_REQUISITE_KEYWORDS:
 1657                                         high[name][state][hind][argfirst].extend(
 1658                                             arg[argfirst]
 1659                                         )
 1660                                     # otherwise, its not a requisite and we are just extending (replacing)
 1661                                     else:
 1662                                         high[name][state][hind] = arg
 1663                                     update = True
 1664                                 if (
 1665                                     argfirst == "name"
 1666                                     and next(iter(high[name][state][hind])) == "names"
 1667                                 ):
 1668                                     # If names are overwritten by name use the name
 1669                                     high[name][state][hind] = arg
 1670                         if not update:
 1671                             high[name][state].append(arg)
 1672         return high, errors
 1673 
 1674     def apply_exclude(self, high):
 1675         """
 1676         Read in the __exclude__ list and remove all excluded objects from the
 1677         high data
 1678         """
 1679         if "__exclude__" not in high:
 1680             return high
 1681         ex_sls = set()
 1682         ex_id = set()
 1683         exclude = high.pop("__exclude__")
 1684         for exc in exclude:
 1685             if isinstance(exc, str):
 1686                 # The exclude statement is a string, assume it is an sls
 1687                 ex_sls.add(exc)
 1688             if isinstance(exc, dict):
 1689                 # Explicitly declared exclude
 1690                 if len(exc) != 1:
 1691                     continue
 1692                 key = next(iter(exc.keys()))
 1693                 if key == "sls":
 1694                     ex_sls.add(exc["sls"])
 1695                 elif key == "id":
 1696                     ex_id.add(exc["id"])
 1697         # Now the excludes have been simplified, use them
 1698         if ex_sls:
 1699             # There are sls excludes, find the associated ids
 1700             for name, body in high.items():
 1701                 if name.startswith("__"):
 1702                     continue
 1703                 sls = body.get("__sls__", "")
 1704                 if not sls:
 1705                     continue
 1706                 for ex_ in ex_sls:
 1707                     if fnmatch.fnmatch(sls, ex_):
 1708                         ex_id.add(name)
 1709         for id_ in ex_id:
 1710             if id_ in high:
 1711                 high.pop(id_)
 1712         return high
 1713 
 1714     def requisite_in(self, high):
 1715         """
 1716         Extend the data reference with requisite_in arguments
 1717         """
 1718         req_in = {
 1719             "require_in",
 1720             "watch_in",
 1721             "onfail_in",
 1722             "onchanges_in",
 1723             "use",
 1724             "use_in",
 1725             "prereq",
 1726             "prereq_in",
 1727         }
 1728         req_in_all = req_in.union(
 1729             {"require", "watch", "onfail", "onfail_stop", "onchanges"}
 1730         )
 1731         extend = {}
 1732         errors = []
 1733         disabled_reqs = self.opts.get("disabled_requisites", [])
 1734         if not isinstance(disabled_reqs, list):
 1735             disabled_reqs = [disabled_reqs]
 1736         for id_, body in high.items():
 1737             if not isinstance(body, dict):
 1738                 continue
 1739             for state, run in body.items():
 1740                 if state.startswith("__"):
 1741                     continue
 1742                 for arg in run:
 1743                     if isinstance(arg, dict):
 1744                         # It is not a function, verify that the arg is a
 1745                         # requisite in statement
 1746                         if len(arg) < 1:
 1747                             # Empty arg dict
 1748                             # How did we get this far?
 1749                             continue
 1750                         # Split out the components
 1751                         key = next(iter(arg))
 1752                         if key not in req_in:
 1753                             continue
 1754                         if key in disabled_reqs:
 1755                             log.warning(
 1756                                 "The %s requisite has been disabled, Ignoring.", key
 1757                             )
 1758                             continue
 1759                         rkey = key.split("_")[0]
 1760                         items = arg[key]
 1761                         if isinstance(items, dict):
 1762                             # Formatted as a single req_in
 1763                             for _state, name in items.items():
 1764 
 1765                                 # Not a use requisite_in
 1766                                 found = False
 1767                                 if name not in extend:
 1768                                     extend[name] = OrderedDict()
 1769                                 if "." in _state:
 1770                                     errors.append(
 1771                                         "Invalid requisite in {}: {} for "
 1772                                         "{}, in SLS '{}'. Requisites must "
 1773                                         "not contain dots, did you mean '{}'?".format(
 1774                                             rkey,
 1775                                             _state,
 1776                                             name,
 1777                                             body["__sls__"],
 1778                                             _state[: _state.find(".")],
 1779                                         )
 1780                                     )
 1781                                     _state = _state.split(".")[0]
 1782                                 if _state not in extend[name]:
 1783                                     extend[name][_state] = []
 1784                                 extend[name]["__env__"] = body["__env__"]
 1785                                 extend[name]["__sls__"] = body["__sls__"]
 1786                                 for ind in range(len(extend[name][_state])):
 1787                                     if next(iter(extend[name][_state][ind])) == rkey:
 1788                                         # Extending again
 1789                                         extend[name][_state][ind][rkey].append(
 1790                                             {state: id_}
 1791                                         )
 1792                                         found = True
 1793                                 if found:
 1794                                     continue
 1795                                 # The rkey is not present yet, create it
 1796                                 extend[name][_state].append({rkey: [{state: id_}]})
 1797 
 1798                         if isinstance(items, list):
 1799                             # Formed as a list of requisite additions
 1800                             hinges = []
 1801                             for ind in items:
 1802                                 if not isinstance(ind, dict):
 1803                                     # Malformed req_in
 1804                                     if ind in high:
 1805                                         _ind_high = [
 1806                                             x
 1807                                             for x in high[ind]
 1808                                             if not x.startswith("__")
 1809                                         ]
 1810                                         ind = {_ind_high[0]: ind}
 1811                                     else:
 1812                                         found = False
 1813                                         for _id in iter(high):
 1814                                             for state in [
 1815                                                 state
 1816                                                 for state in iter(high[_id])
 1817                                                 if not state.startswith("__")
 1818                                             ]:
 1819                                                 for j in iter(high[_id][state]):
 1820                                                     if (
 1821                                                         isinstance(j, dict)
 1822                                                         and "name" in j
 1823                                                     ):
 1824                                                         if j["name"] == ind:
 1825                                                             ind = {state: _id}
 1826                                                             found = True
 1827                                         if not found:
 1828                                             continue
 1829                                 if len(ind) < 1:
 1830                                     continue
 1831                                 pstate = next(iter(ind))
 1832                                 pname = ind[pstate]
 1833                                 if pstate == "sls":
 1834                                     # Expand hinges here
 1835                                     hinges = find_sls_ids(pname, high)
 1836                                 else:
 1837                                     hinges.append((pname, pstate))
 1838                                 if "." in pstate:
 1839                                     errors.append(
 1840                                         "Invalid requisite in {}: {} for "
 1841                                         "{}, in SLS '{}'. Requisites must "
 1842                                         "not contain dots, did you mean '{}'?".format(
 1843                                             rkey,
 1844                                             pstate,
 1845                                             pname,
 1846                                             body["__sls__"],
 1847                                             pstate[: pstate.find(".")],
 1848                                         )
 1849                                     )
 1850                                     pstate = pstate.split(".")[0]
 1851                                 for tup in hinges:
 1852                                     name, _state = tup
 1853                                     if key == "prereq_in":
 1854                                         # Add prerequired to origin
 1855                                         if id_ not in extend:
 1856                                             extend[id_] = OrderedDict()
 1857                                         if state not in extend[id_]:
 1858                                             extend[id_][state] = []
 1859                                         extend[id_][state].append(
 1860                                             {"prerequired": [{_state: name}]}
 1861                                         )
 1862                                     if key == "prereq":
 1863                                         # Add prerequired to prereqs
 1864                                         ext_ids = find_name(name, _state, high)
 1865                                         for ext_id, _req_state in ext_ids:
 1866                                             if ext_id not in extend:
 1867                                                 extend[ext_id] = OrderedDict()
 1868                                             if _req_state not in extend[ext_id]:
 1869                                                 extend[ext_id][_req_state] = []
 1870                                             extend[ext_id][_req_state].append(
 1871                                                 {"prerequired": [{state: id_}]}
 1872                                             )
 1873                                         continue
 1874                                     if key == "use_in":
 1875                                         # Add the running states args to the
 1876                                         # use_in states
 1877                                         ext_ids = find_name(name, _state, high)
 1878                                         for ext_id, _req_state in ext_ids:
 1879                                             if not ext_id:
 1880                                                 continue
 1881                                             ext_args = state_args(ext_id, _state, high)
 1882                                             if ext_id not in extend:
 1883                                                 extend[ext_id] = OrderedDict()
 1884                                             if _req_state not in extend[ext_id]:
 1885                                                 extend[ext_id][_req_state] = []
 1886                                             ignore_args = req_in_all.union(ext_args)
 1887                                             for arg in high[id_][state]:
 1888                                                 if not isinstance(arg, dict):
 1889                                                     continue
 1890                                                 if len(arg) != 1:
 1891                                                     continue
 1892                                                 if next(iter(arg)) in ignore_args:
 1893                                                     continue
 1894                                                 # Don't use name or names
 1895                                                 if next(iter(arg.keys())) == "name":
 1896                                                     continue
 1897                                                 if next(iter(arg.keys())) == "names":
 1898                                                     continue
 1899                                                 extend[ext_id][_req_state].append(arg)
 1900                                         continue
 1901                                     if key == "use":
 1902                                         # Add the use state's args to the
 1903                                         # running state
 1904                                         ext_ids = find_name(name, _state, high)
 1905                                         for ext_id, _req_state in ext_ids:
 1906                                             if not ext_id:
 1907                                                 continue
 1908                                             loc_args = state_args(id_, state, high)
 1909                                             if id_ not in extend:
 1910                                                 extend[id_] = OrderedDict()
 1911                                             if state not in extend[id_]:
 1912                                                 extend[id_][state] = []
 1913                                             ignore_args = req_in_all.union(loc_args)
 1914                                             for arg in high[ext_id][_req_state]:
 1915                                                 if not isinstance(arg, dict):
 1916                                                     continue
 1917                                                 if len(arg) != 1:
 1918                                                     continue
 1919                                                 if next(iter(arg)) in ignore_args:
 1920                                                     continue
 1921                                                 # Don't use name or names
 1922                                                 if next(iter(arg.keys())) == "name":
 1923                                                     continue
 1924                                                 if next(iter(arg.keys())) == "names":
 1925                                                     continue
 1926                                                 extend[id_][state].append(arg)
 1927                                         continue
 1928                                     found = False
 1929                                     if name not in extend:
 1930                                         extend[name] = OrderedDict()
 1931                                     if _state not in extend[name]:
 1932                                         extend[name][_state] = []
 1933                                     extend[name]["__env__"] = body["__env__"]
 1934                                     extend[name]["__sls__"] = body["__sls__"]
 1935                                     for ind in range(len(extend[name][_state])):
 1936                                         if (
 1937                                             next(iter(extend[name][_state][ind]))
 1938                                             == rkey
 1939                                         ):
 1940                                             # Extending again
 1941                                             extend[name][_state][ind][rkey].append(
 1942                                                 {state: id_}
 1943                                             )
 1944                                             found = True
 1945                                     if found:
 1946                                         continue
 1947                                     # The rkey is not present yet, create it
 1948                                     extend[name][_state].append({rkey: [{state: id_}]})
 1949         high["__extend__"] = []
 1950         for key, val in extend.items():
 1951             high["__extend__"].append({key: val})
 1952         req_in_high, req_in_errors = self.reconcile_extend(high)
 1953         errors.extend(req_in_errors)
 1954         return req_in_high, errors
 1955 
 1956     def _call_parallel_target(self, name, cdata, low):
 1957         """
 1958         The target function to call that will create the parallel thread/process
 1959         """
 1960         # we need to re-record start/end duration here because it is impossible to
 1961         # correctly calculate further down the chain
 1962         utc_start_time = datetime.datetime.utcnow()
 1963 
 1964         self.format_slots(cdata)
 1965         tag = _gen_tag(low)
 1966         try:
 1967             ret = self.states[cdata["full"]](*cdata["args"], **cdata["kwargs"])
 1968         except Exception as exc:  # pylint: disable=broad-except
 1969             log.debug(
 1970                 "An exception occurred in this state: %s",
 1971                 exc,
 1972                 exc_info_on_loglevel=logging.DEBUG,
 1973             )
 1974             trb = traceback.format_exc()
 1975             ret = {
 1976                 "result": False,
 1977                 "name": name,
 1978                 "changes": {},
 1979                 "comment": "An exception occurred in this state: {}".format(trb),
 1980             }
 1981 
 1982         utc_finish_time = datetime.datetime.utcnow()
 1983         delta = utc_finish_time - utc_start_time
 1984         # duration in milliseconds.microseconds
 1985         duration = (delta.seconds * 1000000 + delta.microseconds) / 1000.0
 1986         ret["duration"] = duration
 1987 
 1988         troot = os.path.join(self.opts["cachedir"], self.jid)
 1989         tfile = os.path.join(troot, salt.utils.hashutils.sha1_digest(tag))
 1990         if not os.path.isdir(troot):
 1991             try:
 1992                 os.makedirs(troot)
 1993             except OSError:
 1994                 # Looks like the directory was created between the check
 1995                 # and the attempt, we are safe to pass
 1996                 pass
 1997         with salt.utils.files.fopen(tfile, "wb+") as fp_:
 1998             fp_.write(msgpack_serialize(ret))
 1999 
 2000     def call_parallel(self, cdata, low):
 2001         """
 2002         Call the state defined in the given cdata in parallel
 2003         """
 2004         # There are a number of possibilities to not have the cdata
 2005         # populated with what we might have expected, so just be smart
 2006         # enough to not raise another KeyError as the name is easily
 2007         # guessable and fallback in all cases to present the real
 2008         # exception to the user
 2009         name = (cdata.get("args") or [None])[0] or cdata["kwargs"].get("name")
 2010         if not name:
 2011             name = low.get("name", low.get("__id__"))
 2012 
 2013         proc = salt.utils.process.Process(
 2014             target=self._call_parallel_target, args=(name, cdata, low)
 2015         )
 2016         proc.start()
 2017         ret = {
 2018             "name": name,
 2019             "result": None,
 2020             "changes": {},
 2021             "comment": "Started in a separate process",
 2022             "proc": proc,
 2023         }
 2024         return ret
 2025 
 2026     @salt.utils.decorators.state.OutputUnifier("content_check", "unify")
 2027     def call(self, low, chunks=None, running=None, retries=1):
 2028         """
 2029         Call a state directly with the low data structure, verify data
 2030         before processing.
 2031         """
 2032         utc_start_time = datetime.datetime.utcnow()
 2033         local_start_time = utc_start_time - (
 2034             datetime.datetime.utcnow() - datetime.datetime.now()
 2035         )
 2036         log.info(
 2037             "Running state [%s] at time %s",
 2038             low["name"].strip() if isinstance(low["name"], str) else low["name"],
 2039             local_start_time.time().isoformat(),
 2040         )
 2041         errors = self.verify_data(low)
 2042         if errors:
 2043             ret = {
 2044                 "result": False,
 2045                 "name": low["name"],
 2046                 "changes": {},
 2047                 "comment": "",
 2048             }
 2049             for err in errors:
 2050                 ret["comment"] += "{}\n".format(err)
 2051             ret["__run_num__"] = self.__run_num
 2052             self.__run_num += 1
 2053             format_log(ret)
 2054             self.check_refresh(low, ret)
 2055             return ret
 2056         else:
 2057             ret = {"result": False, "name": low["name"], "changes": {}}
 2058 
 2059         self.state_con["runas"] = low.get("runas", None)
 2060 
 2061         if low["state"] == "cmd" and "password" in low:
 2062             self.state_con["runas_password"] = low["password"]
 2063         else:
 2064             self.state_con["runas_password"] = low.get("runas_password", None)
 2065 
 2066         if not low.get("__prereq__"):
 2067             log.info(
 2068                 "Executing state %s.%s for [%s]",
 2069                 low["state"],
 2070                 low["fun"],
 2071                 low["name"].strip() if isinstance(low["name"], str) else low["name"],
 2072             )
 2073 
 2074         if "provider" in low:
 2075             self.load_modules(low)
 2076 
 2077         state_func_name = "{0[state]}.{0[fun]}".format(low)
 2078         cdata = salt.utils.args.format_call(
 2079             self.states[state_func_name],
 2080             low,
 2081             initial_ret={"full": state_func_name},
 2082             expected_extra_kws=STATE_INTERNAL_KEYWORDS,
 2083         )
 2084         inject_globals = {
 2085             # Pass a copy of the running dictionary, the low state chunks and
 2086             # the current state dictionaries.
 2087             # We pass deep copies here because we don't want any misbehaving
 2088             # state module to change these at runtime.
 2089             "__low__": immutabletypes.freeze(low),
 2090             "__running__": immutabletypes.freeze(running) if running else {},
 2091             "__instance_id__": self.instance_id,
 2092             "__lowstate__": immutabletypes.freeze(chunks) if chunks else {},
 2093         }
 2094 
 2095         if "__env__" in low:
 2096             inject_globals["__env__"] = str(low["__env__"])
 2097 
 2098         if self.inject_globals:
 2099             inject_globals.update(self.inject_globals)
 2100 
 2101         if low.get("__prereq__"):
 2102             test = sys.modules[self.states[cdata["full"]].__module__].__opts__["test"]
 2103             sys.modules[self.states[cdata["full"]].__module__].__opts__["test"] = True
 2104         try:
 2105             # Let's get a reference to the salt environment to use within this
 2106             # state call.
 2107             #
 2108             # If the state function accepts an 'env' keyword argument, it
 2109             # allows the state to be overridden(we look for that in cdata). If
 2110             # that's not found in cdata, we look for what we're being passed in
 2111             # the original data, namely, the special dunder __env__. If that's
 2112             # not found we default to 'base'
 2113             req_list = ("unless", "onlyif", "creates")
 2114             if (
 2115                 any(req in low for req in req_list)
 2116                 and "{0[state]}.mod_run_check".format(low) not in self.states
 2117             ):
 2118                 ret.update(self._run_check(low))
 2119 
 2120             if not self.opts.get("lock_saltenv", False):
 2121                 # NOTE: Overriding the saltenv when lock_saltenv is blocked in
 2122                 # salt/modules/state.py, before we ever get here, but this
 2123                 # additional check keeps use of the State class outside of the
 2124                 # salt/modules/state.py from getting around this setting.
 2125                 if "saltenv" in low:
 2126                     inject_globals["__env__"] = str(low["saltenv"])
 2127                 elif isinstance(cdata["kwargs"].get("env", None), str):
 2128                     # User is using a deprecated env setting which was parsed by
 2129                     # format_call.
 2130                     # We check for a string type since module functions which
 2131                     # allow setting the OS environ also make use of the "env"
 2132                     # keyword argument, which is not a string
 2133                     inject_globals["__env__"] = str(cdata["kwargs"]["env"])
 2134 
 2135             if "__env__" not in inject_globals:
 2136                 # Let's use the default environment
 2137                 inject_globals["__env__"] = "base"
 2138 
 2139             if "__orchestration_jid__" in low:
 2140                 inject_globals["__orchestration_jid__"] = low["__orchestration_jid__"]
 2141 
 2142             if "result" not in ret or ret["result"] is False:
 2143                 self.states.inject_globals = inject_globals
 2144                 if self.mocked:
 2145                     ret = mock_ret(cdata)
 2146                 else:
 2147                     # Execute the state function
 2148                     if not low.get("__prereq__") and low.get("parallel"):
 2149                         # run the state call in parallel, but only if not in a prereq
 2150                         ret = self.call_parallel(cdata, low)
 2151                     else:
 2152                         self.format_slots(cdata)
 2153                         ret = self.states[cdata["full"]](
 2154                             *cdata["args"], **cdata["kwargs"]
 2155                         )
 2156                 self.states.inject_globals = {}
 2157             if (
 2158                 "check_cmd" in low
 2159                 and "{0[state]}.mod_run_check_cmd".format(low) not in self.states
 2160             ):
 2161                 ret.update(self._run_check_cmd(low))
 2162         except Exception as exc:  # pylint: disable=broad-except
 2163             log.debug(
 2164                 "An exception occurred in this state: %s",
 2165                 exc,
 2166                 exc_info_on_loglevel=logging.DEBUG,
 2167             )
 2168             trb = traceback.format_exc()
 2169             # There are a number of possibilities to not have the cdata
 2170             # populated with what we might have expected, so just be smart
 2171             # enough to not raise another KeyError as the name is easily
 2172             # guessable and fallback in all cases to present the real
 2173             # exception to the user
 2174             name = (cdata.get("args") or [None])[0] or cdata["kwargs"].get("name")
 2175             if not name:
 2176                 name = low.get("name", low.get("__id__"))
 2177 
 2178             ret = {
 2179                 "result": False,
 2180                 "name": name,
 2181                 "changes": {},
 2182                 "comment": "An exception occurred in this state: {}".format(trb),
 2183             }
 2184         finally:
 2185             if low.get("__prereq__"):
 2186                 sys.modules[self.states[cdata["full"]].__module__].__opts__[
 2187                     "test"
 2188                 ] = test
 2189 
 2190             self.state_con.pop("runas", None)
 2191             self.state_con.pop("runas_password", None)
 2192 
 2193         if not isinstance(ret, dict):
 2194             return ret
 2195 
 2196         # If format_call got any warnings, let's show them to the user
 2197         if "warnings" in cdata:
 2198             ret.setdefault("warnings", []).extend(cdata["warnings"])
 2199 
 2200         if "provider" in low:
 2201             self.load_modules()
 2202 
 2203         if low.get("__prereq__"):
 2204             low["__prereq__"] = False
 2205             return ret
 2206 
 2207         ret["__sls__"] = low.get("__sls__")
 2208         ret["__run_num__"] = self.__run_num
 2209         self.__run_num += 1
 2210         format_log(ret)
 2211         self.check_refresh(low, ret)
 2212         utc_finish_time = datetime.datetime.utcnow()
 2213         timezone_delta = datetime.datetime.utcnow() - datetime.datetime.now()
 2214         local_finish_time = utc_finish_time - timezone_delta
 2215         local_start_time = utc_start_time - timezone_delta
 2216         ret["start_time"] = local_start_time.time().isoformat()
 2217         delta = utc_finish_time - utc_start_time
 2218         # duration in milliseconds.microseconds
 2219         duration = (delta.seconds * 1000000 + delta.microseconds) / 1000.0
 2220         ret["duration"] = duration
 2221         ret["__id__"] = low["__id__"]
 2222         log.info(
 2223             "Completed state [%s] at time %s (duration_in_ms=%s)",
 2224             low["name"].strip() if isinstance(low["name"], str) else low["name"],
 2225             local_finish_time.time().isoformat(),
 2226             duration,
 2227         )
 2228         if "retry" in low:
 2229             low["retry"] = self.verify_retry_data(low["retry"])
 2230             if not sys.modules[self.states[cdata["full"]].__module__].__opts__["test"]:
 2231                 if low["retry"]["until"] != ret["result"]:
 2232                     if low["retry"]["attempts"] > retries:
 2233                         interval = low["retry"]["interval"]
 2234                         if low["retry"]["splay"] != 0:
 2235                             interval = interval + random.randint(
 2236                                 0, low["retry"]["splay"]
 2237                             )
 2238                         log.info(
 2239                             "State result does not match retry until value, "
 2240                             "state will be re-run in %s seconds",
 2241                             interval,
 2242                         )
 2243                         self.functions["test.sleep"](interval)
 2244                         retry_ret = self.call(low, chunks, running, retries=retries + 1)
 2245                         orig_ret = ret
 2246                         ret = retry_ret
 2247                         ret["comment"] = "\n".join(
 2248                             [
 2249                                 (
 2250                                     'Attempt {}: Returned a result of "{}", '
 2251                                     'with the following comment: "{}"'.format(
 2252                                         retries, orig_ret["result"], orig_ret["comment"]
 2253                                     )
 2254                                 ),
 2255                                 "" if not ret["comment"] else ret["comment"],
 2256                             ]
 2257                         )
 2258                         ret["duration"] = (
 2259                             ret["duration"] + orig_ret["duration"] + (interval * 1000)
 2260                         )
 2261                         if retries == 1:
 2262                             ret["start_time"] = orig_ret["start_time"]
 2263             else:
 2264                 ret["comment"] = "  ".join(
 2265                     [
 2266                         "" if not ret["comment"] else str(ret["comment"]),
 2267                         (
 2268                             "The state would be retried every {1} seconds "
 2269                             "(with a splay of up to {3} seconds) "
 2270                             "a maximum of {0} times or until a result of {2} "
 2271                             "is returned"
 2272                         ).format(
 2273                             low["retry"]["attempts"],
 2274                             low["retry"]["interval"],
 2275                             low["retry"]["until"],
 2276                             low["retry"]["splay"],
 2277                         ),
 2278                     ]
 2279                 )
 2280         return ret
 2281 
 2282     def __eval_slot(self, slot):
 2283         log.debug("Evaluating slot: %s", slot)
 2284         fmt = slot.split(":", 2)
 2285         if len(fmt) != 3:
 2286             log.warning("Malformed slot: %s", slot)
 2287             return slot
 2288         if fmt[1] != "salt":
 2289             log.warning("Malformed slot: %s", slot)
 2290             log.warning(
 2291                 "Only execution modules are currently supported in slots. This means slot "
 2292                 'should start with "__slot__:salt:"'
 2293             )
 2294             return slot
 2295         fun, args, kwargs = salt.utils.args.parse_function(fmt[2])
 2296         if not fun or fun not in self.functions:
 2297             log.warning("Malformed slot: %s", slot)
 2298             log.warning(
 2299                 "Execution module should be specified in a function call format: "
 2300                 "test.arg('arg', kw='kwarg')"
 2301             )
 2302             return slot
 2303         log.debug("Calling slot: %s(%s, %s)", fun, args, kwargs)
 2304         slot_return = self.functions[fun](*args, **kwargs)
 2305 
 2306         # Given input  __slot__:salt:test.arg(somekey="value").not.exist ~ /appended
 2307         # slot_text should be __slot...).not.exist
 2308         # append_data should be ~ /appended
 2309         slot_text = fmt[2].split("~")[0]
 2310         append_data = fmt[2].split("~", 1)[1:]
 2311         log.debug("slot_text: %s", slot_text)
 2312         log.debug("append_data: %s", append_data)
 2313 
 2314         # Support parsing slot dict response
 2315         # return_get should result in a kwargs.nested.dict path by getting
 2316         # everything after first closing paren: )
 2317         return_get = None
 2318         try:
 2319             return_get = slot_text[slot_text.rindex(")") + 1 :]
 2320         except ValueError:
 2321             pass
 2322         if return_get:
 2323             # remove first period
 2324             return_get = return_get.split(".", 1)[1].strip()
 2325             log.debug("Searching slot result %s for %s", slot_return, return_get)
 2326             slot_return = salt.utils.data.traverse_dict_and_list(
 2327                 slot_return, return_get, default=None, delimiter="."
 2328             )
 2329 
 2330         if append_data:
 2331             if isinstance(slot_return, str):
 2332                 # Append text to slot string result
 2333                 append_data = " ".join(append_data).strip()
 2334                 log.debug("appending to slot result: %s", append_data)
 2335                 slot_return += append_data
 2336             else:
 2337                 log.error("Ignoring slot append, slot result is not a string")
 2338 
 2339         return slot_return
 2340 
 2341     def format_slots(self, cdata):
 2342         """
 2343         Read in the arguments from the low level slot syntax to make a last
 2344         minute runtime call to gather relevant data for the specific routine
 2345 
 2346         Will parse strings, first level of dictionary values, and strings and
 2347         first level dict values inside of lists
 2348         """
 2349         # __slot__:salt.cmd.run(foo, bar, baz=qux)
 2350         SLOT_TEXT = "__slot__:"
 2351         ctx = (("args", enumerate(cdata["args"])), ("kwargs", cdata["kwargs"].items()))
 2352         for atype, avalues in ctx:
 2353             for ind, arg in avalues:
 2354                 arg = salt.utils.data.decode(arg, keep=True)
 2355                 if isinstance(arg, dict):
 2356                     # Search dictionary values for __slot__:
 2357                     for key, value in arg.items():
 2358                         try:
 2359                             if value.startswith(SLOT_TEXT):
 2360                                 log.trace("Slot processsing dict value %s", value)
 2361                                 cdata[atype][ind][key] = self.__eval_slot(value)
 2362                         except AttributeError:
 2363                             # Not a string/slot
 2364                             continue
 2365                 elif isinstance(arg, list):
 2366                     for idx, listvalue in enumerate(arg):
 2367                         log.trace("Slot processing list value: %s", listvalue)
 2368                         if isinstance(listvalue, dict):
 2369                             # Search dict values in list for __slot__:
 2370                             for key, value in listvalue.items():
 2371                                 try:
 2372                                     if value.startswith(SLOT_TEXT):
 2373                                         log.trace(
 2374                                             "Slot processsing nested dict value %s",
 2375                                             value,
 2376                                         )
 2377                                         cdata[atype][ind][idx][key] = self.__eval_slot(
 2378                                             value
 2379                                         )
 2380                                 except AttributeError:
 2381                                     # Not a string/slot
 2382                                     continue
 2383                         if isinstance(listvalue, str):
 2384                             # Search strings in a list for __slot__:
 2385                             if listvalue.startswith(SLOT_TEXT):
 2386                                 log.trace(
 2387                                     "Slot processsing nested string %s", listvalue
 2388                                 )
 2389                                 cdata[atype][ind][idx] = self.__eval_slot(listvalue)
 2390                 elif isinstance(arg, str) and arg.startswith(SLOT_TEXT):
 2391                     # Search strings for __slot__:
 2392                     log.trace("Slot processsing %s", arg)
 2393                     cdata[atype][ind] = self.__eval_slot(arg)
 2394                 else:
 2395                     # Not a slot, skip it
 2396                     continue
 2397 
 2398     def verify_retry_data(self, retry_data):
 2399         """
 2400         verifies the specified retry data
 2401         """
 2402         retry_defaults = {
 2403             "until": True,
 2404             "attempts": 2,
 2405             "splay": 0,
 2406             "interval": 30,
 2407         }
 2408         expected_data = {
 2409             "until": bool,
 2410             "attempts": int,
 2411             "interval": int,
 2412             "splay": int,
 2413         }
 2414         validated_retry_data = {}
 2415         if isinstance(retry_data, dict):
 2416             for expected_key, value_type in expected_data.items():
 2417                 if expected_key in retry_data:
 2418                     if isinstance(retry_data[expected_key], value_type):
 2419                         validated_retry_data[expected_key] = retry_data[expected_key]
 2420                     else:
 2421                         log.warning(
 2422                             "An invalid value was passed for the retry %s, "
 2423                             "using default value '%s'",
 2424                             expected_key,
 2425                             retry_defaults[expected_key],
 2426                         )
 2427                         validated_retry_data[expected_key] = retry_defaults[
 2428                             expected_key
 2429                         ]
 2430                 else:
 2431                     validated_retry_data[expected_key] = retry_defaults[expected_key]
 2432         else:
 2433             log.warning(
 2434                 "State is set to retry, but a valid dict for retry "
 2435                 "configuration was not found.  Using retry defaults"
 2436             )
 2437             validated_retry_data = retry_defaults
 2438         return validated_retry_data
 2439 
 2440     def call_chunks(self, chunks):
 2441         """
 2442         Iterate over a list of chunks and call them, checking for requires.
 2443         """
 2444         # Check for any disabled states
 2445         disabled = {}
 2446         if "state_runs_disabled" in self.opts["grains"]:
 2447             for low in chunks[:]:
 2448                 state_ = "{}.{}".format(low["state"], low["fun"])
 2449                 for pat in self.opts["grains"]["state_runs_disabled"]:
 2450                     if fnmatch.fnmatch(state_, pat):
 2451                         comment = (
 2452                             'The state function "{0}" is currently disabled by "{1}", '
 2453                             "to re-enable, run state.enable {1}."
 2454                         ).format(state_, pat,)
 2455                         _tag = _gen_tag(low)
 2456                         disabled[_tag] = {
 2457                             "changes": {},
 2458                             "result": False,
 2459                             "comment": comment,
 2460                             "__run_num__": self.__run_num,
 2461                             "__sls__": low["__sls__"],
 2462                         }
 2463                         self.__run_num += 1
 2464                         chunks.remove(low)
 2465                         break
 2466         running = {}
 2467         for low in chunks:
 2468             if "__FAILHARD__" in running:
 2469                 running.pop("__FAILHARD__")
 2470                 return running
 2471             tag = _gen_tag(low)
 2472             if tag not in running:
 2473                 # Check if this low chunk is paused
 2474                 action = self.check_pause(low)
 2475                 if action == "kill":
 2476                     break
 2477                 running = self.call_chunk(low, running, chunks)
 2478                 if self.check_failhard(low, running):
 2479                     return running
 2480             self.active = set()
 2481         while True:
 2482             if self.reconcile_procs(running):
 2483                 break
 2484             time.sleep(0.01)
 2485         ret = dict(list(disabled.items()) + list(running.items()))
 2486         return ret
 2487 
 2488     def check_failhard(self, low, running):
 2489         """
 2490         Check if the low data chunk should send a failhard signal
 2491         """
 2492         tag = _gen_tag(low)
 2493         if self.opts.get("test", False):
 2494             return False
 2495         if low.get("failhard", self.opts["failhard"]) and tag in running:
 2496             if running[tag]["result"] is None:
 2497                 return False
 2498             return not running[tag]["result"]
 2499         return False
 2500 
 2501     def check_pause(self, low):
 2502         """
 2503         Check to see if this low chunk has been paused
 2504         """
 2505         if not self.jid:
 2506             # Can't pause on salt-ssh since we can't track continuous state
 2507             return
 2508         pause_path = os.path.join(self.opts["cachedir"], "state_pause", self.jid)
 2509         start = time.time()
 2510         if os.path.isfile(pause_path):
 2511             try:
 2512                 while True:
 2513                     tries = 0
 2514                     with salt.utils.files.fopen(pause_path, "rb") as fp_:
 2515                         try:
 2516                             pdat = msgpack_deserialize(fp_.read())
 2517                         except salt.utils.msgpack.exceptions.UnpackValueError:
 2518                             # Reading race condition
 2519                             if tries > 10:
 2520                                 # Break out if there are a ton of read errors
 2521                                 return
 2522                             tries += 1
 2523                             time.sleep(1)
 2524                             continue
 2525                         id_ = low["__id__"]
 2526                         key = ""
 2527                         if id_ in pdat:
 2528                             key = id_
 2529                         elif "__all__" in pdat:
 2530                             key = "__all__"
 2531                         if key:
 2532                             if "duration" in pdat[key]:
 2533                                 now = time.time()
 2534                                 if now - start > pdat[key]["duration"]:
 2535                                     return "run"
 2536                             if "kill" in pdat[key]:
 2537                                 return "kill"
 2538                         else:
 2539                             return "run"
 2540                         time.sleep(1)
 2541             except Exception as exc:  # pylint: disable=broad-except
 2542                 log.error(
 2543                     "Failed to read in pause data for file located at: %s", pause_path
 2544                 )
 2545                 return "run"
 2546         return "run"
 2547 
 2548     def reconcile_procs(self, running):
 2549         """
 2550         Check the running dict for processes and resolve them
 2551         """
 2552         retset = set()
 2553         for tag in running:
 2554             proc = running[tag].get("proc")
 2555             if proc:
 2556                 if not proc.is_alive():
 2557                     ret_cache = os.path.join(
 2558                         self.opts["cachedir"],
 2559                         self.jid,
 2560                         salt.utils.hashutils.sha1_digest(tag),
 2561                     )
 2562                     if not os.path.isfile(ret_cache):
 2563                         ret = {
 2564                             "result": False,
 2565                             "comment": "Parallel process failed to return",
 2566                             "name": running[tag]["name"],
 2567                             "changes": {},
 2568                         }
 2569                     try:
 2570                         with salt.utils.files.fopen(ret_cache, "rb") as fp_:
 2571                             ret = msgpack_deserialize(fp_.read())
 2572                     except OSError:
 2573                         ret = {
 2574                             "result": False,
 2575                             "comment": "Parallel cache failure",
 2576                             "name": running[tag]["name"],
 2577                             "changes": {},
 2578                         }
 2579                     running[tag].update(ret)
 2580                     running[tag].pop("proc")
 2581                 else:
 2582                     retset.add(False)
 2583         return False not in retset
 2584 
 2585     def check_requisite(self, low, running, chunks, pre=False):
 2586         """
 2587         Look into the running data to check the status of all requisite
 2588         states
 2589         """
 2590         disabled_reqs = self.opts.get("disabled_requisites", [])
 2591         if not isinstance(disabled_reqs, list):
 2592             disabled_reqs = [disabled_reqs]
 2593         present = False
 2594         # If mod_watch is not available make it a require
 2595         if "watch" in low:
 2596             if "{}.mod_watch".format(low["state"]) not in self.states:
 2597                 if "require" in low:
 2598                     low["require"].extend(low.pop("watch"))
 2599                 else:
 2600                     low["require"] = low.pop("watch")
 2601             else:
 2602                 present = True
 2603         if "watch_any" in low:
 2604             if "{}.mod_watch".format(low["state"]) not in self.states:
 2605                 if "require_any" in low:
 2606                     low["require_any"].extend(low.pop("watch_any"))
 2607                 else:
 2608                     low["require_any"] = low.pop("watch_any")
 2609             else:
 2610                 present = True
 2611         if "require" in low:
 2612             present = True
 2613         if "require_any" in low:
 2614             present = True
 2615         if "prerequired" in low:
 2616             present = True
 2617         if "prereq" in low:
 2618             present = True
 2619         if "onfail" in low:
 2620             present = True
 2621         if "onfail_any" in low:
 2622             present = True
 2623         if "onfail_all" in low:
 2624             present = True
 2625         if "onchanges" in low:
 2626             present = True
 2627         if "onchanges_any" in low:
 2628             present = True
 2629         if not present:
 2630             return "met", ()
 2631         self.reconcile_procs(running)
 2632         reqs = {
 2633             "require": [],
 2634             "require_any": [],
 2635             "watch": [],
 2636             "watch_any": [],
 2637             "prereq": [],
 2638             "onfail": [],
 2639             "onfail_any": [],
 2640             "onfail_all": [],
 2641             "onchanges": [],
 2642             "onchanges_any": [],
 2643         }
 2644         if pre:
 2645             reqs["prerequired"] = []
 2646         for r_state in reqs:
 2647             if r_state in low and low[r_state] is not None:
 2648                 if r_state in disabled_reqs:
 2649                     log.warning(
 2650                         "The %s requisite has been disabled, Ignoring.", r_state
 2651                     )
 2652                     continue
 2653                 for req in low[r_state]:
 2654                     if isinstance(req, str):
 2655                         req = {"id": req}
 2656                     req = trim_req(req)
 2657                     found = False
 2658                     for chunk in chunks:
 2659                         req_key = next(iter(req))
 2660                         req_val = req[req_key]
 2661                         if req_val is None:
 2662                             continue
 2663                         if req_key == "sls":
 2664                             # Allow requisite tracking of entire sls files
 2665                             if fnmatch.fnmatch(chunk["__sls__"], req_val):
 2666                                 found = True
 2667                                 reqs[r_state].append(chunk)
 2668                             continue
 2669                         try:
 2670                             if isinstance(req_val, str):
 2671                                 if fnmatch.fnmatch(
 2672                                     chunk["name"], req_val
 2673                                 ) or fnmatch.fnmatch(chunk["__id__"], req_val):
 2674                                     if req_key == "id" or chunk["state"] == req_key:
 2675                                         found = True
 2676                                         reqs[r_state].append(chunk)
 2677                             else:
 2678                                 raise KeyError
 2679                         except KeyError as exc:
 2680                             raise SaltRenderError(
 2681                                 "Could not locate requisite of [{}] present in state with name [{}]".format(
 2682                                     req_key, chunk["name"]
 2683                                 )
 2684                             )
 2685                         except TypeError:
 2686                             # On Python 2, the above req_val, being an OrderedDict, will raise a KeyError,
 2687                             # however on Python 3 it will raise a TypeError
 2688                             # This was found when running tests.unit.test_state.StateCompilerTestCase.test_render_error_on_invalid_requisite
 2689                             raise SaltRenderError(
 2690                                 "Could not locate requisite of [{}] present in state with name [{}]".format(
 2691                                     req_key, chunk["name"]
 2692                                 )
 2693                             )
 2694                     if not found:
 2695                         return "unmet", ()
 2696         fun_stats = set()
 2697         for r_state, chunks in reqs.items():
 2698             req_stats = set()
 2699             if r_state.startswith("prereq") and not r_state.startswith("prerequired"):
 2700                 run_dict = self.pre
 2701             else:
 2702                 run_dict = running
 2703 
 2704             while True:
 2705                 if self.reconcile_procs(run_dict):
 2706                     break
 2707                 time.sleep(0.01)
 2708 
 2709             for chunk in chunks:
 2710                 tag = _gen_tag(chunk)
 2711                 if tag not in run_dict:
 2712                     req_stats.add("unmet")
 2713                     continue
 2714                 if r_state.startswith("onfail"):
 2715                     if run_dict[tag]["result"] is True:
 2716                         req_stats.add("onfail")  # At least one state is OK
 2717                         continue
 2718                 else:
 2719                     if run_dict[tag]["result"] is False:
 2720                         req_stats.add("fail")
 2721                         continue
 2722                 if r_state.startswith("onchanges"):
 2723                     if not run_dict[tag]["changes"]:
 2724                         req_stats.add("onchanges")
 2725                     else:
 2726                         req_stats.add("onchangesmet")
 2727                     continue
 2728                 if r_state.startswith("watch") and run_dict[tag]["changes"]:
 2729                     req_stats.add("change")
 2730                     continue
 2731                 if r_state.startswith("prereq") and run_dict[tag]["result"] is None:
 2732                     if not r_state.startswith("prerequired"):
 2733                         req_stats.add("premet")
 2734                 if r_state.startswith("prereq") and not run_dict[tag]["result"] is None:
 2735                     if not r_state.startswith("prerequired"):
 2736                         req_stats.add("pre")
 2737                 else:
 2738                     if run_dict[tag].get("__state_ran__", True):
 2739                         req_stats.add("met")
 2740             if r_state.endswith("_any") or r_state == "onfail":
 2741                 if "met" in req_stats or "change" in req_stats:
 2742                     if "fail" in req_stats:
 2743                         req_stats.remove("fail")
 2744                 if "onchangesmet" in req_stats:
 2745                     if "onchanges" in req_stats:
 2746                         req_stats.remove("onchanges")
 2747                     if "fail" in req_stats:
 2748                         req_stats.remove("fail")
 2749                 if "onfail" in req_stats:
 2750                     # a met requisite in this case implies a success
 2751                     if "met" in req_stats:
 2752                         req_stats.remove("onfail")
 2753             if r_state.endswith("_all"):
 2754                 if "onfail" in req_stats:
 2755                     # a met requisite in this case implies a failure
 2756                     if "met" in req_stats:
 2757                         req_stats.remove("met")
 2758             fun_stats.update(req_stats)
 2759 
 2760         if "unmet" in fun_stats:
 2761             status = "unmet"
 2762         elif "fail" in fun_stats:
 2763             status = "fail"
 2764         elif "pre" in fun_stats:
 2765             if "premet" in fun_stats:
 2766                 status = "met"
 2767             else:
 2768                 status = "pre"
 2769         elif "onfail" in fun_stats and "onchangesmet" not in fun_stats:
 2770             status = "onfail"
 2771         elif "onchanges" in fun_stats and "onchangesmet" not in fun_stats:
 2772             status = "onchanges"
 2773         elif "change" in fun_stats:
 2774             status = "change"
 2775         else:
 2776             status = "met"
 2777 
 2778         return status, reqs
 2779 
 2780     def event(self, chunk_ret, length, fire_event=False):
 2781         """
 2782         Fire an event on the master bus
 2783 
 2784         If `fire_event` is set to True an event will be sent with the
 2785         chunk name in the tag and the chunk result in the event data.
 2786 
 2787         If `fire_event` is set to a string such as `mystate/is/finished`,
 2788         an event will be sent with the string added to the tag and the chunk
 2789         result in the event data.
 2790 
 2791         If the `state_events` is set to True in the config, then after the
 2792         chunk is evaluated an event will be set up to the master with the
 2793         results.
 2794         """
 2795         if not self.opts.get("local") and (
 2796             self.opts.get("state_events", True) or fire_event
 2797         ):
 2798             if not self.opts.get("master_uri"):
 2799                 ev_func = lambda ret, tag, preload=None: salt.utils.event.get_master_event(
 2800                     self.opts, self.opts["sock_dir"], listen=False
 2801                 ).fire_event(
 2802                     ret, tag
 2803                 )
 2804             else:
 2805                 ev_func = self.functions["event.fire_master"]
 2806 
 2807             ret = {"ret": chunk_ret}
 2808             if fire_event is True:
 2809                 tag = salt.utils.event.tagify(
 2810                     [self.jid, self.opts["id"], str(chunk_ret["name"])], "state_result",
 2811                 )
 2812             elif isinstance(fire_event, str):
 2813                 tag = salt.utils.event.tagify(
 2814                     [self.jid, self.opts["id"], str(fire_event)], "state_result",
 2815                 )
 2816             else:
 2817                 tag = salt.utils.event.tagify(
 2818                     [self.jid, "prog", self.opts["id"], str(chunk_ret["__run_num__"])],
 2819                     "job",
 2820                 )
 2821                 ret["len"] = length
 2822             preload = {"jid": self.jid}
 2823             ev_func(ret, tag, preload=preload)
 2824 
 2825     def call_chunk(self, low, running, chunks):
 2826         """
 2827         Check if a chunk has any requires, execute the requires and then
 2828         the chunk
 2829         """
 2830         low = self._mod_aggregate(low, running, chunks)
 2831         self._mod_init(low)
 2832         tag = _gen_tag(low)
 2833         if not low.get("prerequired"):
 2834             self.active.add(tag)
 2835         requisites = [
 2836             "require",
 2837             "require_any",
 2838             "watch",
 2839             "watch_any",
 2840             "prereq",
 2841             "onfail",
 2842             "onfail_any",
 2843             "onchanges",
 2844             "onchanges_any",
 2845         ]
 2846         if not low.get("__prereq__"):
 2847             requisites.append("prerequired")
 2848             status, reqs = self.check_requisite(low, running, chunks, pre=True)
 2849         else:
 2850             status, reqs = self.check_requisite(low, running, chunks)
 2851         if status == "unmet":
 2852             lost = {}
 2853             reqs = []
 2854             for requisite in requisites:
 2855                 lost[requisite] = []
 2856                 if requisite not in low:
 2857                     continue
 2858                 for req in low[requisite]:
 2859                     if isinstance(req, str):
 2860                         req = {"id": req}
 2861                     req = trim_req(req)
 2862                     found = False
 2863                     req_key = next(iter(req))
 2864                     req_val = req[req_key]
 2865                     for chunk in chunks:
 2866                         if req_val is None:
 2867                             continue
 2868                         if req_key == "sls":
 2869                             # Allow requisite tracking of entire sls files
 2870                             if fnmatch.fnmatch(chunk["__sls__"], req_val):
 2871                                 if requisite == "prereq":
 2872                                     chunk["__prereq__"] = True
 2873                                 reqs.append(chunk)
 2874                                 found = True
 2875                             continue
 2876                         if fnmatch.fnmatch(chunk["name"], req_val) or fnmatch.fnmatch(
 2877                             chunk["__id__"], req_val
 2878                         ):
 2879                             if req_key == "id" or chunk["state"] == req_key:
 2880                                 if requisite == "prereq":
 2881                                     chunk["__prereq__"] = True
 2882                                 elif requisite == "prerequired":
 2883                                     chunk["__prerequired__"] = True
 2884                                 reqs.append(chunk)
 2885                                 found = True
 2886                     if not found:
 2887                         lost[requisite].append(req)
 2888             if (
 2889                 lost["require"]
 2890                 or lost["watch"]
 2891                 or lost["prereq"]
 2892                 or lost["onfail"]
 2893                 or lost["onchanges"]
 2894                 or lost["require_any"]
 2895                 or lost["watch_any"]
 2896                 or lost["onfail_any"]
 2897                 or lost["onchanges_any"]
 2898                 or lost.get("prerequired")
 2899             ):
 2900                 comment = "The following requisites were not found:\n"
 2901                 for requisite, lreqs in lost.items():
 2902                     if not lreqs:
 2903                         continue
 2904                     comment += "{}{}:\n".format(" " * 19, requisite)
 2905                     for lreq in lreqs:
 2906                         req_key = next(iter(lreq))
 2907                         req_val = lreq[req_key]
 2908                         comment += "{}{}: {}\n".format(" " * 23, req_key, req_val)
 2909                 if low.get("__prereq__"):
 2910                     run_dict = self.pre
 2911                 else:
 2912                     run_dict = running
 2913                 start_time, duration = _calculate_fake_duration()
 2914                 run_dict[tag] = {
 2915                     "changes": {},
 2916                     "result": False,
 2917                     "duration": duration,
 2918                     "start_time": start_time,
 2919                     "comment": comment,
 2920                     "__run_num__": self.__run_num,
 2921                     "__sls__": low["__sls__"],
 2922                 }
 2923                 self.__run_num += 1
 2924                 self.event(run_dict[tag], len(chunks), fire_event=low.get("fire_event"))
 2925                 return running
 2926             for chunk in reqs:
 2927                 # Check to see if the chunk has been run, only run it if
 2928                 # it has not been run already
 2929                 ctag = _gen_tag(chunk)
 2930                 if ctag not in running:
 2931                     if ctag in self.active:
 2932                         if chunk.get("__prerequired__"):
 2933                             # Prereq recusive, run this chunk with prereq on
 2934                             if tag not in self.pre:
 2935                                 low["__prereq__"] = True
 2936                                 self.pre[ctag] = self.call(low, chunks, running)
 2937                                 return running
 2938                             else:
 2939                                 return running
 2940                         elif ctag not in running:
 2941                             log.error("Recursive requisite found")
 2942                             running[tag] = {
 2943                                 "changes": {},
 2944                                 "result": False,
 2945                                 "comment": "Recursive requisite found",
 2946                                 "__run_num__": self.__run_num,
 2947                                 "__sls__": low["__sls__"],
 2948                             }
 2949                         self.__run_num += 1
 2950                         self.event(
 2951                             running[tag], len(chunks), fire_event=low.get("fire_event")
 2952                         )
 2953                         return running
 2954                     running = self.call_chunk(chunk, running, chunks)
 2955                     if self.check_failhard(chunk, running):
 2956                         running["__FAILHARD__"] = True
 2957                         return running
 2958             if low.get("__prereq__"):
 2959                 status, reqs = self.check_requisite(low, running, chunks)
 2960                 self.pre[tag] = self.call(low, chunks, running)
 2961                 if not self.pre[tag]["changes"] and status == "change":
 2962                     self.pre[tag]["changes"] = {"watch": "watch"}
 2963                     self.pre[tag]["result"] = None
 2964             else:
 2965                 running = self.call_chunk(low, running, chunks)
 2966             if self.check_failhard(chunk, running):
 2967                 running["__FAILHARD__"] = True
 2968                 return running
 2969         elif status == "met":
 2970             if low.get("__prereq__"):
 2971                 self.pre[tag] = self.call(low, chunks, running)
 2972             else:
 2973                 running[tag] = self.call(low, chunks, running)
 2974         elif status == "fail":
 2975             # if the requisite that failed was due to a prereq on this low state
 2976             # show the normal error
 2977             if tag in self.pre:
 2978                 running[tag] = self.pre[tag]
 2979                 running[tag]["__run_num__"] = self.__run_num
 2980                 running[tag]["__sls__"] = low["__sls__"]
 2981             # otherwise the failure was due to a requisite down the chain
 2982             else:
 2983                 # determine what the requisite failures where, and return
 2984                 # a nice error message
 2985                 failed_requisites = set()
 2986                 # look at all requisite types for a failure
 2987                 for req_lows in reqs.values():
 2988                     for req_low in req_lows:
 2989                         req_tag = _gen_tag(req_low)
 2990                         req_ret = self.pre.get(req_tag, running.get(req_tag))
 2991                         # if there is no run output for the requisite it
 2992                         # can't be the failure
 2993                         if req_ret is None:
 2994                             continue
 2995                         # If the result was False (not None) it was a failure
 2996                         if req_ret["result"] is False:
 2997                             # use SLS.ID for the key-- so its easier to find
 2998                             key = "{sls}.{_id}".format(
 2999                                 sls=req_low["__sls__"], _id=req_low["__id__"]
 3000                             )
 3001                             failed_requisites.add(key)
 3002 
 3003                 _cmt = "One or more requisite failed: {}".format(
 3004                     ", ".join(str(i) for i in failed_requisites)
 3005                 )
 3006                 start_time, duration = _calculate_fake_duration()
 3007                 running[tag] = {
 3008                     "changes": {},
 3009                     "result": False,
 3010                     "duration": duration,
 3011                     "start_time": start_time,
 3012                     "comment": _cmt,
 3013                     "__run_num__": self.__run_num,
 3014                     "__sls__": low["__sls__"],
 3015                 }
 3016                 self.pre[tag] = running[tag]
 3017             self.__run_num += 1
 3018         elif status == "change" and not low.get("__prereq__"):
 3019             ret = self.call(low, chunks, running)
 3020             if not ret["changes"] and not ret.get("skip_watch", False):
 3021                 low = low.copy()
 3022                 low["sfun"] = low["fun"]
 3023                 low["fun"] = "mod_watch"
 3024                 low["__reqs__"] = reqs
 3025                 ret = self.call(low, chunks, running)
 3026             running[tag] = ret
 3027         elif status == "pre":
 3028             start_time, duration = _calculate_fake_duration()
 3029             pre_ret = {
 3030                 "changes": {},
 3031                 "result": True,
 3032                 "duration": duration,
 3033                 "start_time": start_time,
 3034                 "comment": "No changes detected",
 3035                 "__run_num__": self.__run_num,
 3036                 "__sls__": low["__sls__"],
 3037             }
 3038             running[tag] = pre_ret
 3039             self.pre[tag] = pre_ret
 3040             self.__run_num += 1
 3041         elif status == "onfail":
 3042             start_time, duration = _calculate_fake_duration()
 3043             running[tag] = {
 3044                 "changes": {},
 3045                 "result": True,
 3046                 "duration": duration,
 3047                 "start_time": start_time,
 3048                 "comment": "State was not run because onfail req did not change",
 3049                 "__state_ran__": False,
 3050                 "__run_num__": self.__run_num,
 3051                 "__sls__": low["__sls__"],
 3052             }
 3053             self.__run_num += 1
 3054         elif status == "onchanges":
 3055             start_time, duration = _calculate_fake_duration()
 3056             running[tag] = {
 3057                 "changes": {},
 3058                 "result": True,
 3059                 "duration": duration,
 3060                 "start_time": start_time,
 3061                 "comment": "State was not run because none of the onchanges reqs changed",
 3062                 "__state_ran__": False,
 3063                 "__run_num__": self.__run_num,
 3064                 "__sls__": low["__sls__"],
 3065             }
 3066             self.__run_num += 1
 3067         else:
 3068             if low.get("__prereq__"):
 3069                 self.pre[tag] = self.call(low, chunks, running)
 3070             else:
 3071                 running[tag] = self.call(low, chunks, running)
 3072         if tag in running:
 3073             self.event(running[tag], len(chunks), fire_event=low.get("fire_event"))
 3074 
 3075             for sub_state_data in running[tag].pop("sub_state_run", ()):
 3076                 start_time, duration = _calculate_fake_duration()
 3077                 self.__run_num += 1
 3078                 sub_tag = _gen_tag(sub_state_data["low"])
 3079                 running[sub_tag] = {
 3080                     "name": sub_state_data["low"]["name"],
 3081                     "changes": sub_state_data["changes"],
 3082                     "result": sub_state_data["result"],
 3083                     "duration": sub_state_data.get("duration", duration),
 3084                     "start_time": sub_state_data.get("start_time", start_time),
 3085                     "comment": sub_state_data.get("comment", ""),
 3086                     "__state_ran__": True,
 3087                     "__run_num__": self.__run_num,
 3088                     "__sls__": low["__sls__"],
 3089                 }
 3090 
 3091         return running
 3092 
 3093     def call_listen(self, chunks, running):
 3094         """
 3095         Find all of the listen routines and call the associated mod_watch runs
 3096         """
 3097         listeners = []
 3098         crefs = {}
 3099         for chunk in chunks:
 3100             crefs[(chunk["state"], chunk["__id__"], chunk["name"])] = chunk
 3101             if "listen" in chunk:
 3102                 listeners.append(
 3103                     {(chunk["state"], chunk["__id__"], chunk["name"]): chunk["listen"]}
 3104                 )
 3105             if "listen_in" in chunk:
 3106                 for l_in in chunk["listen_in"]:
 3107                     for key, val in l_in.items():
 3108                         listeners.append(
 3109                             {(key, val, "lookup"): [{chunk["state"]: chunk["__id__"]}]}
 3110                         )
 3111         mod_watchers = []
 3112         errors = {}
 3113         for l_dict in listeners:
 3114             for key, val in l_dict.items():
 3115                 for listen_to in val:
 3116                     if not isinstance(listen_to, dict):
 3117                         found = False
 3118                         for chunk in chunks:
 3119                             if (
 3120                                 chunk["__id__"] == listen_to
 3121                                 or chunk["name"] == listen_to
 3122                             ):
 3123                                 listen_to = {chunk["state"]: chunk["__id__"]}
 3124                                 found = True
 3125                         if not found:
 3126                             continue
 3127                     for lkey, lval in listen_to.items():
 3128                         if not any(lkey == cref[0] and lval in cref for cref in crefs):
 3129                             rerror = {
 3130                                 _l_tag(lkey, lval): {
 3131                                     "comment": "Referenced state {}: {} does not exist".format(
 3132                                         lkey, lval
 3133                                     ),
 3134                                     "name": "listen_{}:{}".format(lkey, lval),
 3135                                     "result": False,
 3136                                     "changes": {},
 3137                                 }
 3138                             }
 3139                             errors.update(rerror)
 3140                             continue
 3141                         to_tags = [
 3142                             _gen_tag(data)
 3143                             for cref, data in crefs.items()
 3144                             if lkey == cref[0] and lval in cref
 3145                         ]
 3146                         for to_tag in to_tags:
 3147                             if to_tag not in running:
 3148                                 continue
 3149                             if running[to_tag]["changes"]:
 3150                                 if not any(
 3151                                     key[0] == cref[0] and key[1] in cref
 3152                                     for cref in crefs
 3153                                 ):
 3154                                     rerror = {
 3155                                         _l_tag(key[0], key[1]): {
 3156                                             "comment": "Referenced state {}: {} does not exist".format(
 3157                                                 key[0], key[1]
 3158                                             ),
 3159                                             "name": "listen_{}:{}".format(
 3160                                                 key[0], key[1]
 3161                                             ),
 3162                                             "result": False,
 3163                                             "changes": {},
 3164                                         }
 3165                                     }
 3166                                     errors.update(rerror)
 3167                                     continue
 3168 
 3169                                 new_chunks = [
 3170                                     data
 3171                                     for cref, data in crefs.items()
 3172                                     if key[0] == cref[0] and key[1] in cref
 3173                                 ]
 3174                                 for chunk in new_chunks:
 3175                                     low = chunk.copy()
 3176                                     low["sfun"] = chunk["fun"]
 3177                                     low["fun"] = "mod_watch"
 3178                                     low["__id__"] = "listener_{}".format(low["__id__"])
 3179                                     for req in STATE_REQUISITE_KEYWORDS:
 3180                                         if req in low:
 3181                                             low.pop(req)
 3182                                     mod_watchers.append(low)
 3183         ret = self.call_chunks(mod_watchers)
 3184         running.update(ret)
 3185         for err in errors:
 3186             errors[err]["__run_num__"] = self.__run_num
 3187             self.__run_num += 1
 3188         running.update(errors)
 3189         return running
 3190 
 3191     def call_high(self, high, orchestration_jid=None):
 3192         """
 3193         Process a high data call and ensure the defined states.
 3194         """
 3195         errors = []
 3196         # If there is extension data reconcile it
 3197         high, ext_errors = self.reconcile_extend(high)
 3198         errors.extend(ext_errors)
 3199         errors.extend(self.verify_high(high))
 3200         if errors:
 3201             return errors
 3202         high, req_in_errors = self.requisite_in(high)
 3203         errors.extend(req_in_errors)
 3204         high = self.apply_exclude(high)
 3205         # Verify that the high data is structurally sound
 3206         if errors:
 3207             return errors
 3208         # Compile and verify the raw chunks
 3209         chunks = self.compile_high_data(high, orchestration_jid)
 3210 
 3211         # If there are extensions in the highstate, process them and update
 3212         # the low data chunks
 3213         if errors:
 3214             return errors
 3215         ret = self.call_chunks(chunks)
 3216         ret = self.call_listen(chunks, ret)
 3217 
 3218         def _cleanup_accumulator_data():
 3219             accum_data_path = os.path.join(
 3220                 get_accumulator_dir(self.opts["cachedir"]), self.instance_id
 3221             )
 3222             try:
 3223                 os.remove(accum_data_path)
 3224                 log.debug("Deleted accumulator data file %s", accum_data_path)
 3225             except OSError:
 3226                 log.debug("File %s does not exist, no need to cleanup", accum_data_path)
 3227 
 3228         _cleanup_accumulator_data()
 3229         if self.jid is not None:
 3230             pause_path = os.path.join(self.opts["cachedir"], "state_pause", self.jid)
 3231             if os.path.isfile(pause_path):
 3232                 try:
 3233                     os.remove(pause_path)
 3234                 except OSError:
 3235                     # File is not present, all is well
 3236                     pass
 3237 
 3238         return ret
 3239 
 3240     def render_template(self, high, template):
 3241         errors = []
 3242         if not high:
 3243             return high, errors
 3244 
 3245         if not isinstance(high, dict):
 3246             errors.append(
 3247                 "Template {} does not render to a dictionary".format(template)
 3248             )
 3249             return high, errors
 3250 
 3251         invalid_items = ("include", "exclude", "extends")
 3252         for item in invalid_items:
 3253             if item in high:
 3254                 errors.append(
 3255                     "The '{}' declaration found on '{}' is invalid when "
 3256                     "rendering single templates".format(item, template)
 3257                 )
 3258                 return high, errors
 3259 
 3260         for name in high:
 3261             if not isinstance(high[name], dict):
 3262                 if isinstance(high[name], str):
 3263                     # Is this is a short state, it needs to be padded
 3264                     if "." in high[name]:
 3265                         comps = high[name].split(".")
 3266                         high[name] = {
 3267                             # '__sls__': template,
 3268                             # '__env__': None,
 3269                             comps[0]: [comps[1]]
 3270                         }
 3271                         continue
 3272 
 3273                     errors.append(
 3274                         "ID {} in template {} is not a dictionary".format(
 3275                             name, template
 3276                         )
 3277                     )
 3278                     continue
 3279             skeys = set()
 3280             for key in sorted(high[name]):
 3281                 if key.startswith("_"):
 3282                     continue
 3283                 if high[name][key] is None:
 3284                     errors.append(
 3285                         "ID '{}' in template {} contains a short "
 3286                         "declaration ({}) with a trailing colon. When not "
 3287                         "passing any arguments to a state, the colon must be "
 3288                         "omitted.".format(name, template, key)
 3289                     )
 3290                     continue
 3291                 if not isinstance(high[name][key], list):
 3292                     continue
 3293                 if "." in key:
 3294                     comps = key.split(".")
 3295                     # Salt doesn't support state files such as:
 3296                     #
 3297                     # /etc/redis/redis.conf:
 3298                     #   file.managed:
 3299                     #     - user: redis
 3300                     #     - group: redis
 3301                     #     - mode: 644
 3302                     #   file.comment:
 3303                     #     - regex: ^requirepass
 3304                     if comps[0] in skeys:
 3305                         errors.append(
 3306                             "ID '{}' in template '{}' contains multiple "
 3307                             "state declarations of the same type".format(name, template)
 3308                         )
 3309                         continue
 3310                     high[name][comps[0]] = high[name].pop(key)
 3311                     high[name][comps[0]].append(comps[1])
 3312                     skeys.add(comps[0])
 3313                     continue
 3314                 skeys.add(key)
 3315 
 3316         return high, errors
 3317 
 3318     def call_template(self, template):
 3319         """
 3320         Enforce the states in a template
 3321         """
 3322         high = compile_template(
 3323             template,
 3324             self.rend,
 3325             self.opts["renderer"],
 3326             self.opts["renderer_blacklist"],
 3327             self.opts["renderer_whitelist"],
 3328         )
 3329         if not high:
 3330             return high
 3331         high, errors = self.render_template(high, template)
 3332         if errors:
 3333             return errors
 3334         return self.call_high(high)
 3335 
 3336     def call_template_str(self, template):
 3337         """
 3338         Enforce the states in a template, pass the template as a string
 3339         """
 3340         high = compile_template_str(
 3341             template,
 3342             self.rend,
 3343             self.opts["renderer"],
 3344             self.opts["renderer_blacklist"],
 3345             self.opts["renderer_whitelist"],
 3346         )
 3347         if not high:
 3348             return high
 3349         high, errors = self.render_template(high, "<template-str>")
 3350         if errors:
 3351             return errors
 3352         return self.call_high(high)
 3353 
 3354 
 3355 class LazyAvailStates:
 3356     """
 3357     The LazyAvailStates lazily loads the list of states of available
 3358     environments.
 3359 
 3360     This is particularly usefull when top_file_merging_strategy=same and there
 3361     are many environments.
 3362     """
 3363 
 3364     def __init__(self, hs):
 3365         self._hs = hs
 3366         self._avail = {"base": None}
 3367         self._filled = False
 3368 
 3369     def _fill(self):
 3370         if self._filled:
 3371             return
 3372         for saltenv in self._hs._get_envs():
 3373             if saltenv not in self._avail:
 3374                 self._avail[saltenv] = None
 3375         self._filled = True
 3376 
 3377     def __contains__(self, saltenv):
 3378         if saltenv == "base":
 3379             return True
 3380         self._fill()
 3381         return saltenv in self._avail
 3382 
 3383     def __getitem__(self, saltenv):
 3384         if saltenv != "base":
 3385             self._fill()
 3386         if self._avail[saltenv] is None:
 3387             self._avail[saltenv] = self._hs.client.list_states(saltenv)
 3388         return self._avail[saltenv]
 3389 
 3390     def items(self):
 3391         self._fill()
 3392         ret = []
 3393         for saltenv, states in self._avail:
 3394             ret.append((saltenv, self.__getitem__(saltenv)))
 3395         return ret
 3396 
 3397 
 3398 class BaseHighState:
 3399     """
 3400     The BaseHighState is an abstract base class that is the foundation of
 3401     running a highstate, extend it and add a self.state object of type State.
 3402 
 3403     When extending this class, please note that ``self.client`` and
 3404     ``self.matcher`` should be instantiated and handled.
 3405     """
 3406 
 3407     def __init__(self, opts):
 3408         self.opts = self.__gen_opts(opts)
 3409         self.iorder = 10000
 3410         self.avail = self.__gather_avail()
 3411         self.serial = salt.payload.Serial(self.opts)
 3412         self.building_highstate = OrderedDict()
 3413 
 3414     def __gather_avail(self):
 3415         """
 3416         Lazily gather the lists of available sls data from the master
 3417         """
 3418         return LazyAvailStates(self)
 3419 
 3420     def __gen_opts(self, opts):
 3421         """
 3422         The options used by the High State object are derived from options
 3423         on the minion and the master, or just the minion if the high state
 3424         call is entirely local.
 3425         """
 3426         # If the state is intended to be applied locally, then the local opts
 3427         # should have all of the needed data, otherwise overwrite the local
 3428         # data items with data from the master
 3429         if "local_state" in opts:
 3430             if opts["local_state"]:
 3431                 return opts
 3432         mopts = self.client.master_opts()
 3433         if not isinstance(mopts, dict):
 3434             # An error happened on the master
 3435             opts["renderer"] = "jinja|yaml"
 3436             opts["failhard"] = False
 3437             opts["state_top"] = salt.utils.url.create("top.sls")
 3438             opts["nodegroups"] = {}
 3439             opts["file_roots"] = {"base": [syspaths.BASE_FILE_ROOTS_DIR]}
 3440         else:
 3441             opts["renderer"] = mopts["renderer"]
 3442             opts["failhard"] = mopts.get("failhard", False)
 3443             if mopts["state_top"].startswith("salt://"):
 3444                 opts["state_top"] = mopts["state_top"]
 3445             elif mopts["state_top"].startswith("/"):
 3446                 opts["state_top"] = salt.utils.url.create(mopts["state_top"][1:])
 3447             else:
 3448                 opts["state_top"] = salt.utils.url.create(mopts["state_top"])
 3449             opts["state_top_saltenv"] = mopts.get("state_top_saltenv", None)
 3450             opts["nodegroups"] = mopts.get("nodegroups", {})
 3451             opts["state_auto_order"] = mopts.get(
 3452                 "state_auto_order", opts["state_auto_order"]
 3453             )
 3454             opts["file_roots"] = mopts["file_roots"]
 3455             opts["top_file_merging_strategy"] = mopts.get(
 3456                 "top_file_merging_strategy", opts.get("top_file_merging_strategy")
 3457             )
 3458             opts["env_order"] = mopts.get("env_order", opts.get("env_order", []))
 3459             opts["default_top"] = mopts.get("default_top", opts.get("default_top"))
 3460             opts["state_events"] = mopts.get("state_events")
 3461             opts["state_aggregate"] = mopts.get(
 3462                 "state_aggregate", opts.get("state_aggregate", False)
 3463             )
 3464             opts["jinja_env"] = mopts.get("jinja_env", {})
 3465             opts["jinja_sls_env"] = mopts.get("jinja_sls_env", {})
 3466             opts["jinja_lstrip_blocks"] = mopts.get("jinja_lstrip_blocks", False)
 3467             opts["jinja_trim_blocks"] = mopts.get("jinja_trim_blocks", False)
 3468         return opts
 3469 
 3470     def _get_envs(self):
 3471         """
 3472         Pull the file server environments out of the master options
 3473         """
 3474         envs = ["base"]
 3475         if "file_roots" in self.opts:
 3476             envs.extend([x for x in list(self.opts["file_roots"]) if x not in envs])
 3477         env_order = self.opts.get("env_order", [])
 3478         # Remove duplicates while preserving the order
 3479         members = set()
 3480         env_order = [
 3481             env for env in env_order if not (env in members or members.add(env))
 3482         ]
 3483         client_envs = self.client.envs()
 3484         if env_order and client_envs:
 3485             return [env for env in env_order if env in client_envs]
 3486 
 3487         elif env_order:
 3488             return env_order
 3489         else:
 3490             envs.extend([env for env in client_envs if env not in envs])
 3491             return envs
 3492 
 3493     def get_tops(self):
 3494         """
 3495         Gather the top files
 3496         """
 3497         tops = DefaultOrderedDict(list)
 3498         include = DefaultOrderedDict(list)
 3499         done = DefaultOrderedDict(list)
 3500         found = 0  # did we find any contents in the top files?
 3501         # Gather initial top files
 3502         merging_strategy = self.opts["top_file_merging_strategy"]
 3503         if merging_strategy == "same" and not self.opts["saltenv"]:
 3504             if not self.opts["default_top"]:
 3505                 raise SaltRenderError(
 3506                     "top_file_merging_strategy set to 'same', but no "
 3507                     "default_top configuration option was set"
 3508                 )
 3509 
 3510         if self.opts["saltenv"]:
 3511             contents = self.client.cache_file(
 3512                 self.opts["state_top"], self.opts["saltenv"]
 3513             )
 3514             if contents:
 3515                 found = 1
 3516                 tops[self.opts["saltenv"]] = [
 3517                     compile_template(
 3518                         contents,
 3519                         self.state.rend,
 3520                         self.state.opts["renderer"],
 3521                         self.state.opts["renderer_blacklist"],
 3522                         self.state.opts["renderer_whitelist"],
 3523                         saltenv=self.opts["saltenv"],
 3524                     )
 3525                 ]
 3526             else:
 3527                 tops[self.opts["saltenv"]] = [{}]
 3528 
 3529         else:
 3530             found = 0
 3531             state_top_saltenv = self.opts.get("state_top_saltenv", False)
 3532             if state_top_saltenv and not isinstance(state_top_saltenv, str):
 3533                 state_top_saltenv = str(state_top_saltenv)
 3534 
 3535             for saltenv in (
 3536                 [state_top_saltenv] if state_top_saltenv else self._get_envs()
 3537             ):
 3538                 contents = self.client.cache_file(self.opts["state_top"], saltenv)
 3539                 if contents:
 3540                     found = found + 1
 3541                     tops[saltenv].append(
 3542                         compile_template(
 3543                             contents,
 3544                             self.state.rend,
 3545                             self.state.opts["renderer"],
 3546                             self.state.opts["renderer_blacklist"],
 3547                             self.state.opts["renderer_whitelist"],
 3548                             saltenv=saltenv,
 3549                         )
 3550                     )
 3551                 else:
 3552                     tops[saltenv].append({})
 3553                     log.debug("No contents loaded for saltenv '%s'", saltenv)
 3554 
 3555             if (
 3556                 found > 1
 3557                 and merging_strategy == "merge"
 3558                 and not self.opts.get("env_order", None)
 3559             ):
 3560                 log.warning(
 3561                     "top_file_merging_strategy is set to '%s' and "
 3562                     "multiple top files were found. Merging order is not "
 3563                     "deterministic, it may be desirable to either set "
 3564                     "top_file_merging_strategy to 'same' or use the "
 3565                     "'env_order' configuration parameter to specify the "
 3566                     "merging order.",
 3567                     merging_strategy,
 3568                 )
 3569 
 3570         if found == 0:
 3571             log.debug(
 3572                 "No contents found in top file. If this is not expected, "
 3573                 "verify that the 'file_roots' specified in 'etc/master' "
 3574                 "are accessible. The 'file_roots' configuration is: %s",
 3575                 repr(self.state.opts["file_roots"]),
 3576             )
 3577 
 3578         # Search initial top files for includes
 3579         for saltenv, ctops in tops.items():
 3580             for ctop in ctops:
 3581                 if "include" not in ctop:
 3582                     continue
 3583                 for sls in ctop["include"]:
 3584                     include[saltenv].append(sls)
 3585                 ctop.pop("include")
 3586         # Go through the includes and pull out the extra tops and add them
 3587         while include:
 3588             pops = []
 3589             for saltenv, states in include.items():
 3590                 pops.append(saltenv)
 3591                 if not states:
 3592                     continue
 3593                 for sls_match in states:
 3594                     for sls in fnmatch.filter(self.avail[saltenv], sls_match):
 3595                         if sls in done[saltenv]:
 3596                             continue
 3597                         tops[saltenv].append(
 3598                             compile_template(
 3599                                 self.client.get_state(sls, saltenv).get("dest", False),
 3600                                 self.state.rend,
 3601                                 self.state.opts["renderer"],
 3602                                 self.state.opts["renderer_blacklist"],
 3603                                 self.state.opts["renderer_whitelist"],
 3604                                 saltenv,
 3605                             )
 3606                         )
 3607                         done[saltenv].append(sls)
 3608             for saltenv in pops:
 3609                 if saltenv in include:
 3610                     include.pop(saltenv)
 3611         return tops
 3612 
 3613     def merge_tops(self, tops):
 3614         """
 3615         Cleanly merge the top files
 3616         """
 3617         merging_strategy = self.opts["top_file_merging_strategy"]
 3618         try:
 3619             merge_attr = "_merge_tops_{}".format(merging_strategy)
 3620             merge_func = getattr(self, merge_attr)
 3621             if not hasattr(merge_func, "__call__"):
 3622                 msg = "'{}' is not callable".format(merge_attr)
 3623                 log.error(msg)
 3624                 raise TypeError(msg)
 3625         except (AttributeError, TypeError):
 3626             log.warning(
 3627                 "Invalid top_file_merging_strategy '%s', falling back to " "'merge'",
 3628                 merging_strategy,
 3629             )
 3630             merge_func = self._merge_tops_merge
 3631         return merge_func(tops)
 3632 
 3633     def _merge_tops_merge(self, tops):
 3634         """
 3635         The default merging strategy. The base env is authoritative, so it is
 3636         checked first, followed by the remaining environments. In top files
 3637         from environments other than "base", only the section matching the
 3638         environment from the top file will be considered, and it too will be
 3639         ignored if that environment was defined in the "base" top file.
 3640         """
 3641         top = DefaultOrderedDict(OrderedDict)
 3642 
 3643         # Check base env first as it is authoritative
 3644         base_tops = tops.pop("base", DefaultOrderedDict(OrderedDict))
 3645         for ctop in base_tops:
 3646             for saltenv, targets in ctop.items():
 3647                 if saltenv == "include":
 3648                     continue
 3649                 try:
 3650                     for tgt in targets:
 3651                         top[saltenv][tgt] = ctop[saltenv][tgt]
 3652                 except TypeError:
 3653                     raise SaltRenderError(
 3654                         "Unable to render top file. No targets found."
 3655                     )
 3656 
 3657         for cenv, ctops in tops.items():
 3658             for ctop in ctops:
 3659                 for saltenv, targets in ctop.items():
 3660                     if saltenv == "include":
 3661                         continue
 3662                     elif saltenv != cenv:
 3663                         log.debug(
 3664                             "Section for saltenv '%s' in the '%s' "
 3665                             "saltenv's top file will be ignored, as the "
 3666                             "top_file_merging_strategy is set to 'merge' "
 3667                             "and the saltenvs do not match",
 3668                             saltenv,
 3669                             cenv,
 3670                         )
 3671                         continue
 3672                     elif saltenv in top:
 3673                         log.debug(
 3674                             "Section for saltenv '%s' in the '%s' "
 3675                             "saltenv's top file will be ignored, as this "
 3676                             "saltenv was already defined in the 'base' top "
 3677                             "file",
 3678                             saltenv,
 3679                             cenv,
 3680                         )
 3681                         continue
 3682                     try:
 3683                         for tgt in targets:
 3684                             top[saltenv][tgt] = ctop[saltenv][tgt]
 3685                     except TypeError:
 3686                         raise SaltRenderError(
 3687                             "Unable to render top file. No targets found."
 3688                         )
 3689         return top
 3690 
 3691     def _merge_tops_same(self, tops):
 3692         """
 3693         For each saltenv, only consider the top file from that saltenv. All
 3694         sections matching a given saltenv, which appear in a different
 3695         saltenv's top file, will be ignored.
 3696         """
 3697         top = DefaultOrderedDict(OrderedDict)
 3698         for cenv, ctops in tops.items():
 3699             if all([x == {} for x in ctops]):
 3700                 # No top file found in this env, check the default_top
 3701                 default_top = self.opts["default_top"]
 3702                 fallback_tops = tops.get(default_top, [])
 3703                 if all([x == {} for x in fallback_tops]):
 3704                     # Nothing in the fallback top file
 3705                     log.error(
 3706                         "The '%s' saltenv has no top file, and the fallback "
 3707                         "saltenv specified by default_top (%s) also has no "
 3708                         "top file",
 3709                         cenv,
 3710                         default_top,
 3711                     )
 3712                     continue
 3713 
 3714                 for ctop in fallback_tops:
 3715                     for saltenv, targets in ctop.items():
 3716                         if saltenv != cenv:
 3717                             continue
 3718                         log.debug(
 3719                             "The '%s' saltenv has no top file, using the "
 3720                             "default_top saltenv (%s)",
 3721                             cenv,
 3722                             default_top,
 3723                         )
 3724                         for tgt in targets:
 3725                             top[saltenv][tgt] = ctop[saltenv][tgt]
 3726                         break
 3727                     else:
 3728                         log.error(
 3729                             "The '%s' saltenv has no top file, and no "
 3730                             "matches were found in the top file for the "
 3731                             "default_top saltenv (%s)",
 3732                             cenv,
 3733                             default_top,
 3734                         )
 3735 
 3736                 continue
 3737 
 3738             else:
 3739                 for ctop in ctops:
 3740                     for saltenv, targets in ctop.items():
 3741                         if saltenv == "include":
 3742                             continue
 3743                         elif saltenv != cenv:
 3744                             log.debug(
 3745                                 "Section for saltenv '%s' in the '%s' "
 3746                                 "saltenv's top file will be ignored, as the "
 3747                                 "top_file_merging_strategy is set to 'same' "
 3748                                 "and the saltenvs do not match",
 3749                                 saltenv,
 3750                                 cenv,
 3751                             )
 3752                             continue
 3753 
 3754                         try:
 3755                             for tgt in targets:
 3756                                 top[saltenv][tgt] = ctop[saltenv][tgt]
 3757                         except TypeError:
 3758                             raise SaltRenderError(
 3759                                 "Unable to render top file. No targets found."
 3760                             )
 3761         return top
 3762 
 3763     def _merge_tops_merge_all(self, tops):
 3764         """
 3765         Merge the top files into a single dictionary
 3766         """
 3767 
 3768         def _read_tgt(tgt):
 3769             match_type = None
 3770             states = []
 3771             for item in tgt:
 3772                 if isinstance(item, dict):
 3773                     match_type = item
 3774                 if isinstance(item, str):
 3775                     states.append(item)
 3776             return match_type, states
 3777 
 3778         top = DefaultOrderedDict(OrderedDict)
 3779         for ctops in tops.values():
 3780             for ctop in ctops:
 3781                 for saltenv, targets in ctop.items():
 3782                     if saltenv == "include":
 3783                         continue
 3784                     try:
 3785                         for tgt in targets:
 3786                             if tgt not in top[saltenv]:
 3787                                 top[saltenv][tgt] = ctop[saltenv][tgt]
 3788                                 continue
 3789                             m_type1, m_states1 = _read_tgt(top[saltenv][tgt])
 3790                             m_type2, m_states2 = _read_tgt(ctop[saltenv][tgt])
 3791                             merged = []
 3792                             match_type = m_type2 or m_type1
 3793                             if match_type is not None:
 3794                                 merged.append(match_type)
 3795                             merged.extend(m_states1)
 3796                             merged.extend([x for x in m_states2 if x not in merged])
 3797                             top[saltenv][tgt] = merged
 3798                     except TypeError:
 3799                         raise SaltRenderError(
 3800                             "Unable to render top file. No targets found."
 3801                         )
 3802         return top
 3803 
 3804     def verify_tops(self, tops):
 3805         """
 3806         Verify the contents of the top file data
 3807         """
 3808         errors = []
 3809         if not isinstance(tops, dict):
 3810             errors.append("Top data was not formed as a dict")
 3811             # No further checks will work, bail out
 3812             return errors
 3813         for saltenv, matches in tops.items():
 3814             if saltenv == "include":
 3815                 continue
 3816             if not isinstance(saltenv, str):
 3817                 errors.append(
 3818                     "Environment {} in top file is not formed as a "
 3819                     "string".format(saltenv)
 3820                 )
 3821             if saltenv == "":
 3822                 errors.append("Empty saltenv statement in top file")
 3823             if not isinstance(matches, dict):
 3824                 errors.append(
 3825                     "The top file matches for saltenv {} are not "
 3826                     "formatted as a dict".format(saltenv)
 3827                 )
 3828             for slsmods in matches.values():
 3829                 if not isinstance(slsmods, list):
 3830                     errors.append(
 3831                         "Malformed topfile (state declarations not " "formed as a list)"
 3832                     )
 3833                     continue
 3834                 for slsmod in slsmods:
 3835                     if isinstance(slsmod, dict):
 3836                         # This value is a match option
 3837                         for val in slsmod.values():
 3838                             if not val:
 3839                                 errors.append(
 3840                                     "Improperly formatted top file matcher "
 3841                                     "in saltenv {}: {} file".format(slsmod, val)
 3842                                 )
 3843                     elif isinstance(slsmod, str):
 3844                         # This is a sls module
 3845                         if not slsmod:
 3846                             errors.append(
 3847                                 "Environment {} contains an empty sls "
 3848                                 "index".format(saltenv)
 3849                             )
 3850 
 3851         return errors
 3852 
 3853     def get_top(self):
 3854         """
 3855         Returns the high data derived from the top file
 3856         """
 3857         try:
 3858             tops = self.get_tops()
 3859         except SaltRenderError as err:
 3860             log.error("Unable to render top file: %s", err.error)
 3861             return {}
 3862         return self.merge_tops(tops)
 3863 
 3864     def top_matches(self, top):
 3865         """
 3866         Search through the top high data for matches and return the states
 3867         that this minion needs to execute.
 3868 
 3869         Returns:
 3870         {'saltenv': ['state1', 'state2', ...]}
 3871         """
 3872         matches = DefaultOrderedDict(OrderedDict)
 3873         # pylint: disable=cell-var-from-loop
 3874         for saltenv, body in top.items():
 3875             if self.opts["saltenv"]:
 3876                 if saltenv != self.opts["saltenv"]:
 3877                     continue
 3878             for match, data in body.items():
 3879 
 3880                 def _filter_matches(_match, _data, _opts):
 3881                     if isinstance(_data, str):
 3882                         _data = [_data]
 3883                     if self.matchers["confirm_top.confirm_top"](_match, _data, _opts):
 3884                         if saltenv not in matches:
 3885                             matches[saltenv] = []
 3886                         for item in _data:
 3887                             if "subfilter" in item:
 3888                                 _tmpdata = item.pop("subfilter")
 3889                                 for match, data in _tmpdata.items():
 3890                                     _filter_matches(match, data, _opts)
 3891                             if isinstance(item, str):
 3892                                 matches[saltenv].append(item)
 3893                             elif isinstance(item, dict):
 3894                                 env_key, inc_sls = item.popitem()
 3895                                 if env_key not in self.avail:
 3896                                     continue
 3897                                 if env_key not in matches:
 3898                                     matches[env_key] = []
 3899                                 matches[env_key].append(inc_sls)
 3900 
 3901                 _filter_matches(match, data, self.opts["nodegroups"])
 3902         ext_matches = self._master_tops()
 3903         for saltenv in ext_matches:
 3904             top_file_matches = matches.get(saltenv, [])
 3905             if self.opts.get("master_tops_first"):
 3906                 first = ext_matches[saltenv]
 3907                 second = top_file_matches
 3908             else:
 3909                 first = top_file_matches
 3910                 second = ext_matches[saltenv]
 3911             matches[saltenv] = first + [x for x in second if x not in first]
 3912 
 3913         # pylint: enable=cell-var-from-loop
 3914         return matches
 3915 
 3916     def _master_tops(self):
 3917         """
 3918         Get results from the master_tops system. Override this function if the
 3919         execution of the master_tops needs customization.
 3920         """
 3921         return self.client.master_tops()
 3922 
 3923     def load_dynamic(self, matches):
 3924         """
 3925         If autoload_dynamic_modules is True then automatically load the
 3926         dynamic modules
 3927         """
 3928         if not self.opts["autoload_dynamic_modules"]:
 3929             return
 3930         syncd = self.state.functions["saltutil.sync_all"](list(matches), refresh=False)
 3931         if syncd["grains"]:
 3932             self.opts["grains"] = salt.loader.grains(self.opts)
 3933             self.state.opts["pillar"] = self.state._gather_pillar()
 3934         self.state.module_refresh()
 3935 
 3936     def render_state(self, sls, saltenv, mods, matches, local=False, context=None):
 3937         """
 3938         Render a state file and retrieve all of the include states
 3939         """
 3940         errors = []
 3941         if not local:
 3942             state_data = self.client.get_state(sls, saltenv)
 3943             fn_ = state_data.get("dest", False)
 3944         else:
 3945             fn_ = sls
 3946             if not os.path.isfile(fn_):
 3947                 errors.append(
 3948                     "Specified SLS {} on local filesystem cannot "
 3949                     "be found.".format(sls)
 3950                 )
 3951         state = None
 3952         if not fn_:
 3953             errors.append(
 3954                 "Specified SLS {} in saltenv {} is not "
 3955                 "available on the salt master or through a configured "
 3956                 "fileserver".format(sls, saltenv)
 3957             )
 3958         else:
 3959             try:
 3960                 state = compile_template(
 3961                     fn_,
 3962                     self.state.rend,
 3963                     self.state.opts["renderer"],
 3964                     self.state.opts["renderer_blacklist"],
 3965                     self.state.opts["renderer_whitelist"],
 3966                     saltenv,
 3967                     sls,
 3968                     rendered_sls=mods,
 3969                     context=context,
 3970                 )
 3971             except SaltRenderError as exc:
 3972                 msg = "Rendering SLS '{}:{}' failed: {}".format(saltenv, sls, exc)
 3973                 log.critical(msg)
 3974                 errors.append(msg)
 3975             except Exception as exc:  # pylint: disable=broad-except
 3976                 msg = "Rendering SLS {} failed, render error: {}".format(sls, exc)
 3977                 log.critical(
 3978                     msg,
 3979                     # Show the traceback if the debug logging level is enabled
 3980                     exc_info_on_loglevel=logging.DEBUG,
 3981                 )
 3982                 errors.append("{}\n{}".format(msg, traceback.format_exc()))
 3983             try:
 3984                 mods.add("{}:{}".format(saltenv, sls))
 3985             except AttributeError:
 3986                 pass
 3987 
 3988         if state:
 3989             if not isinstance(state, dict):
 3990                 errors.append("SLS {} does not render to a dictionary".format(sls))
 3991             else:
 3992                 include = []
 3993                 if "include" in state:
 3994                     if not isinstance(state["include"], list):
 3995                         err = (
 3996                             "Include Declaration in SLS {} is not formed "
 3997                             "as a list".format(sls)
 3998                         )
 3999                         errors.append(err)
 4000                     else:
 4001                         include = state.pop("include")
 4002 
 4003                 self._handle_extend(state, sls, saltenv, errors)
 4004                 self._handle_exclude(state, sls, saltenv, errors)
 4005                 self._handle_state_decls(state, sls, saltenv, errors)
 4006 
 4007                 for inc_sls in include:
 4008                     # inc_sls may take the form of:
 4009                     #   'sls.to.include' <- same as {<saltenv>: 'sls.to.include'}
 4010                     #   {<env_key>: 'sls.to.include'}
 4011                     #   {'_xenv': 'sls.to.resolve'}
 4012                     xenv_key = "_xenv"
 4013 
 4014                     if isinstance(inc_sls, dict):
 4015                         env_key, inc_sls = inc_sls.popitem()
 4016                     else:
 4017                         env_key = saltenv
 4018 
 4019                     if env_key not in self.avail:
 4020                         msg = (
 4021                             "Nonexistent saltenv '{}' found in include "
 4022                             "of '{}' within SLS '{}:{}'".format(
 4023                                 env_key, inc_sls, saltenv, sls
 4024                             )
 4025                         )
 4026                         log.error(msg)
 4027                         errors.append(msg)
 4028                         continue
 4029 
 4030                     if inc_sls.startswith("."):
 4031                         match = re.match(r"^(\.+)(.*)$", inc_sls)
 4032                         if match:
 4033                             levels, include = match.groups()
 4034                         else:
 4035                             msg = (
 4036                                 "Badly formatted include {} found in include "
 4037                                 "in SLS '{}:{}'".format(inc_sls, saltenv, sls)
 4038                             )
 4039                             log.error(msg)
 4040                             errors.append(msg)
 4041                             continue
 4042                         level_count = len(levels)
 4043                         p_comps = sls.split(".")
 4044                         if state_data.get("source", "").endswith("/init.sls"):
 4045                             p_comps.append("init")
 4046                         if level_count > len(p_comps):
 4047                             msg = (
 4048                                 "Attempted relative include of '{}' "
 4049                                 "within SLS '{}:{}' "
 4050                                 "goes beyond top level package ".format(
 4051                                     inc_sls, saltenv, sls
 4052                                 )
 4053                             )
 4054                             log.error(msg)
 4055                             errors.append(msg)
 4056                             continue
 4057                         inc_sls = ".".join(p_comps[:-level_count] + [include])
 4058 
 4059                     if env_key != xenv_key:
 4060                         if matches is None:
 4061                             matches = []
 4062                         # Resolve inc_sls in the specified environment
 4063                         if env_key in matches or fnmatch.filter(
 4064                             self.avail[env_key], inc_sls
 4065                         ):
 4066                             resolved_envs = [env_key]
 4067                         else:
 4068                             resolved_envs = []
 4069                     else:
 4070                         # Resolve inc_sls in the subset of environment matches
 4071                         resolved_envs = [
 4072                             aenv
 4073                             for aenv in matches
 4074                             if fnmatch.filter(self.avail[aenv], inc_sls)
 4075                         ]
 4076 
 4077                     # An include must be resolved to a single environment, or
 4078                     # the include must exist in the current environment
 4079                     if len(resolved_envs) == 1 or saltenv in resolved_envs:
 4080                         # Match inc_sls against the available states in the
 4081                         # resolved env, matching wildcards in the process. If
 4082                         # there were no matches, then leave inc_sls as the
 4083                         # target so that the next recursion of render_state
 4084                         # will recognize the error.
 4085                         sls_targets = fnmatch.filter(self.avail[saltenv], inc_sls) or [
 4086                             inc_sls
 4087                         ]
 4088 
 4089                         for sls_target in sls_targets:
 4090                             r_env = (
 4091                                 resolved_envs[0] if len(resolved_envs) == 1 else saltenv
 4092                             )
 4093                             mod_tgt = "{}:{}".format(r_env, sls_target)
 4094                             if mod_tgt not in mods:
 4095                                 nstate, err = self.render_state(
 4096                                     sls_target, r_env, mods, matches
 4097                                 )
 4098                                 if nstate:
 4099                                     self.merge_included_states(state, nstate, errors)
 4100                                     state.update(nstate)
 4101                                 if err:
 4102                                     errors.extend(err)
 4103                     else:
 4104                         msg = ""
 4105                         if not resolved_envs:
 4106                             msg = (
 4107                                 "Unknown include: Specified SLS {}: {} is not available on the salt "
 4108                                 "master in saltenv(s): {} "
 4109                             ).format(
 4110                                 env_key,
 4111                                 inc_sls,
 4112                                 ", ".join(matches) if env_key == xenv_key else env_key,
 4113                             )
 4114                         elif len(resolved_envs) > 1:
 4115                             msg = (
 4116                                 "Ambiguous include: Specified SLS {}: {} is available on the salt master "
 4117                                 "in multiple available saltenvs: {}"
 4118                             ).format(env_key, inc_sls, ", ".join(resolved_envs))
 4119                         log.critical(msg)
 4120                         errors.append(msg)
 4121                 try:
 4122                     self._handle_iorder(state)
 4123                 except TypeError:
 4124                     log.critical("Could not render SLS %s. Syntax error detected.", sls)
 4125         else:
 4126             state = {}
 4127         return state, errors
 4128 
 4129     def _handle_iorder(self, state):
 4130         """
 4131         Take a state and apply the iorder system
 4132         """
 4133         if self.opts["state_auto_order"]:
 4134             for name in state:
 4135                 for s_dec in state[name]:
 4136                     if not isinstance(s_dec, str):
 4137                         # PyDSL OrderedDict?
 4138                         continue
 4139 
 4140                     if not isinstance(state[name], dict):
 4141                         # Include's or excludes as lists?
 4142                         continue
 4143                     if not isinstance(state[name][s_dec], list):
 4144                         # Bad syntax, let the verify seq pick it up later on
 4145                         continue
 4146 
 4147                     found = False
 4148                     if s_dec.startswith("_"):
 4149                         continue
 4150 
 4151                     for arg in state[name][s_dec]:
 4152                         if isinstance(arg, dict):
 4153                             if len(arg) > 0:
 4154                                 if next(iter(arg.keys())) == "order":
 4155                                     found = True
 4156                     if not found:
 4157                         if not isinstance(state[name][s_dec], list):
 4158                             # quite certainly a syntax error, managed elsewhere
 4159                             continue
 4160                         state[name][s_dec].append({"order": self.iorder})
 4161                         self.iorder += 1
 4162         return state
 4163 
 4164     def _handle_state_decls(self, state, sls, saltenv, errors):
 4165         """
 4166         Add sls and saltenv components to the state
 4167         """
 4168         for name in state:
 4169             if not isinstance(state[name], dict):
 4170                 if name == "__extend__":
 4171                     continue
 4172                 if name == "__exclude__":
 4173                     continue
 4174 
 4175                 if isinstance(state[name], str):
 4176                     # Is this is a short state, it needs to be padded
 4177                     if "." in state[name]:
 4178                         comps = state[name].split(".")
 4179                         state[name] = {
 4180                             "__sls__": sls,
 4181                             "__env__": saltenv,
 4182                             comps[0]: [comps[1]],
 4183                         }
 4184                         continue
 4185                 errors.append("ID {} in SLS {} is not a dictionary".format(name, sls))
 4186                 continue
 4187             skeys = set()
 4188             for key in list(state[name]):
 4189                 if key.startswith("_"):
 4190                     continue
 4191                 if not isinstance(state[name][key], list):
 4192                     continue
 4193                 if "." in key:
 4194                     comps = key.split(".")
 4195                     # Salt doesn't support state files such as:
 4196                     #
 4197                     #     /etc/redis/redis.conf:
 4198                     #       file.managed:
 4199                     #         - source: salt://redis/redis.conf
 4200                     #         - user: redis
 4201                     #         - group: redis
 4202                     #         - mode: 644
 4203                     #       file.comment:
 4204                     #           - regex: ^requirepass
 4205                     if comps[0] in skeys:
 4206                         errors.append(
 4207                             "ID '{}' in SLS '{}' contains multiple state "
 4208                             "declarations of the same type".format(name, sls)
 4209                         )
 4210                         continue
 4211                     state[name][comps[0]] = state[name].pop(key)
 4212                     state[name][comps[0]].append(comps[1])
 4213                     skeys.add(comps[0])
 4214                     continue
 4215                 skeys.add(key)
 4216             if "__sls__" not in state[name]:
 4217                 state[name]["__sls__"] = sls
 4218             if "__env__" not in state[name]:
 4219                 state[name]["__env__"] = saltenv
 4220 
 4221     def _handle_extend(self, state, sls, saltenv, errors):
 4222         """
 4223         Take the extend dec out of state and apply to the highstate global
 4224         dec
 4225         """
 4226         if "extend" in state:
 4227             ext = state.pop("extend")
 4228             if not isinstance(ext, dict):
 4229                 errors.append(
 4230                     ("Extension value in SLS '{}' is not a " "dictionary").format(sls)
 4231                 )
 4232                 return
 4233             for name in ext:
 4234                 if not isinstance(ext[name], dict):
 4235                     errors.append(
 4236                         "Extension name '{}' in SLS '{}' is "
 4237                         "not a dictionary".format(name, sls)
 4238                     )
 4239                     continue
 4240                 if "__sls__" not in ext[name]:
 4241                     ext[name]["__sls__"] = sls
 4242                 if "__env__" not in ext[name]:
 4243                     ext[name]["__env__"] = saltenv
 4244                 for key in list(ext[name]):
 4245                     if key.startswith("_"):
 4246                         continue
 4247                     if not isinstance(ext[name][key], list):
 4248                         continue
 4249                     if "." in key:
 4250                         comps = key.split(".")
 4251                         ext[name][comps[0]] = ext[name].pop(key)
 4252                         ext[name][comps[0]].append(comps[1])
 4253             state.setdefault("__extend__", []).append(ext)
 4254 
 4255     def _handle_exclude(self, state, sls, saltenv, errors):
 4256         """
 4257         Take the exclude dec out of the state and apply it to the highstate
 4258         global dec
 4259         """
 4260         if "exclude" in state:
 4261             exc = state.pop("exclude")
 4262             if not isinstance(exc, list):
 4263                 err = "Exclude Declaration in SLS {} is not formed " "as a list".format(
 4264                     sls
 4265                 )
 4266                 errors.append(err)
 4267             state.setdefault("__exclude__", []).extend(exc)
 4268 
 4269     def render_highstate(self, matches, context=None):
 4270         """
 4271         Gather the state files and render them into a single unified salt
 4272         high data structure.
 4273         """
 4274         highstate = self.building_highstate
 4275         all_errors = []
 4276         mods = set()
 4277         statefiles = []
 4278         for saltenv, states in matches.items():
 4279             for sls_match in states:
 4280                 if saltenv in self.avail:
 4281                     statefiles = fnmatch.filter(self.avail[saltenv], sls_match)
 4282                 elif "__env__" in self.avail:
 4283                     statefiles = fnmatch.filter(self.avail["__env__"], sls_match)
 4284                 else:
 4285                     all_errors.append(
 4286                         "No matching salt environment for environment "
 4287                         "'{}' found".format(saltenv)
 4288                     )
 4289                 # if we did not found any sls in the fileserver listing, this
 4290                 # may be because the sls was generated or added later, we can
 4291                 # try to directly execute it, and if it fails, anyway it will
 4292                 # return the former error
 4293                 if not statefiles:
 4294                     statefiles = [sls_match]
 4295 
 4296                 for sls in statefiles:
 4297                     r_env = "{}:{}".format(saltenv, sls)
 4298                     if r_env in mods:
 4299                         continue
 4300                     state, errors = self.render_state(
 4301                         sls, saltenv, mods, matches, context=context
 4302                     )
 4303                     if state:
 4304                         self.merge_included_states(highstate, state, errors)
 4305                     for i, error in enumerate(errors[:]):
 4306                         if "is not available" in error:
 4307                             # match SLS foobar in environment
 4308                             this_sls = "SLS {} in saltenv".format(sls_match)
 4309                             if this_sls in error:
 4310                                 errors[i] = (
 4311                                     "No matching sls found for '{}' "
 4312                                     "in env '{}'".format(sls_match, saltenv)
 4313                                 )
 4314                     all_errors.extend(errors)
 4315 
 4316         self.clean_duplicate_extends(highstate)
 4317         return highstate, all_errors
 4318 
 4319     def clean_duplicate_extends(self, highstate):
 4320         if "__extend__" in highstate:
 4321             highext = []
 4322             for items in (ext.items() for ext in highstate["__extend__"]):
 4323                 for item in items:
 4324                     if item not in highext:
 4325                         highext.append(item)
 4326             highstate["__extend__"] = [{t[0]: t[1]} for t in highext]
 4327 
 4328     def merge_included_states(self, highstate, state, errors):
 4329         # The extend members can not be treated as globally unique:
 4330         if "__extend__" in state:
 4331             highstate.setdefault("__extend__", []).extend(state.pop("__extend__"))
 4332         if "__exclude__" in state:
 4333             highstate.setdefault("__exclude__", []).extend(state.pop("__exclude__"))
 4334         for id_ in state:
 4335             if id_ in highstate:
 4336                 if highstate[id_] != state[id_]:
 4337                     errors.append(
 4338                         (
 4339                             "Detected conflicting IDs, SLS"
 4340                             " IDs need to be globally unique.\n    The"
 4341                             " conflicting ID is '{}' and is found in SLS"
 4342                             " '{}:{}' and SLS '{}:{}'"
 4343                         ).format(
 4344                             id_,
 4345                             highstate[id_]["__env__"],
 4346                             highstate[id_]["__sls__"],
 4347                             state[id_]["__env__"],
 4348                             state[id_]["__sls__"],
 4349                         )
 4350                     )
 4351         try:
 4352             highstate.update(state)
 4353         except ValueError:
 4354             errors.append("Error when rendering state with contents: {}".format(state))
 4355 
 4356     def _check_pillar(self, force=False):
 4357         """
 4358         Check the pillar for errors, refuse to run the state if there are
 4359         errors in the pillar and return the pillar errors
 4360         """
 4361         if force:
 4362             return True
 4363         if "_errors" in self.state.opts["pillar"]:
 4364             return False
 4365         return True
 4366 
 4367     def matches_whitelist(self, matches, whitelist):
 4368         """
 4369         Reads over the matches and returns a matches dict with just the ones
 4370         that are in the whitelist
 4371         """
 4372         if not whitelist:
 4373             return matches
 4374         ret_matches = {}
 4375         if not isinstance(whitelist, list):
 4376             whitelist = whitelist.split(",")
 4377         for env in matches:
 4378             for sls in matches[env]:
 4379                 if sls in whitelist:
 4380                     ret_matches[env] = ret_matches[env] if env in ret_matches else []
 4381                     ret_matches[env].append(sls)
 4382         return ret_matches
 4383 
 4384     def call_highstate(
 4385         self,
 4386         exclude=None,
 4387         cache=None,
 4388         cache_name="highstate",
 4389         force=False,
 4390         whitelist=None,
 4391         orchestration_jid=None,
 4392     ):
 4393         """
 4394         Run the sequence to execute the salt highstate for this minion
 4395         """
 4396         # Check that top file exists
 4397         tag_name = "no_|-states_|-states_|-None"
 4398         ret = {
 4399             tag_name: {
 4400                 "result": False,
 4401                 "comment": "No states found for this minion",
 4402                 "name": "No States",
 4403                 "changes": {},
 4404                 "__run_num__": 0,
 4405             }
 4406         }
 4407         cfn = os.path.join(self.opts["cachedir"], "{}.cache.p".format(cache_name))
 4408 
 4409         if cache:
 4410             if os.path.isfile(cfn):
 4411                 with salt.utils.files.fopen(cfn, "rb") as fp_:
 4412                     high = self.serial.load(fp_)
 4413                     return self.state.call_high(high, orchestration_jid)
 4414         # File exists so continue
 4415         err = []
 4416         try:
 4417             top = self.get_top()
 4418         except SaltRenderError as err:
 4419             ret[tag_name]["comment"] = "Unable to render top file: "
 4420             ret[tag_name]["comment"] += str(err.error)
 4421             return ret
 4422         except Exception:  # pylint: disable=broad-except
 4423             trb = traceback.format_exc()
 4424             err.append(trb)
 4425             return err
 4426         err += self.verify_tops(top)
 4427         matches = self.top_matches(top)
 4428         if not matches:
 4429             msg = (
 4430                 "No Top file or master_tops data matches found. Please see "
 4431                 "master log for details."
 4432             )
 4433             ret[tag_name]["comment"] = msg
 4434             return ret
 4435         matches = self.matches_whitelist(matches, whitelist)
 4436         self.load_dynamic(matches)
 4437         if not self._check_pillar(force):
 4438             err += ["Pillar failed to render with the following messages:"]
 4439             err += self.state.opts["pillar"]["_errors"]
 4440         else:
 4441             high, errors = self.render_highstate(matches)
 4442             if exclude:
 4443                 if isinstance(exclude, str):
 4444                     exclude = exclude.split(",")
 4445                 if "__exclude__" in high:
 4446                     high["__exclude__"].extend(exclude)
 4447                 else:
 4448                     high["__exclude__"] = exclude
 4449             err += errors
 4450         if err:
 4451             return err
 4452         if not high:
 4453             return ret
 4454         with salt.utils.files.set_umask(0o077):
 4455             try:
 4456                 if salt.utils.platform.is_windows():
 4457                     # Make sure cache file isn't read-only
 4458                     self.state.functions["cmd.run"](
 4459                         ["attrib", "-R", cfn],
 4460                         python_shell=False,
 4461                         output_loglevel="quiet",
 4462                     )
 4463                 with salt.utils.files.fopen(cfn, "w+b") as fp_:
 4464                     try:
 4465                         self.serial.dump(high, fp_)
 4466                     except TypeError:
 4467                         # Can't serialize pydsl
 4468                         pass
 4469             except OSError:
 4470                 log.error('Unable to write to "state.highstate" cache file %s', cfn)
 4471 
 4472         return self.state.call_high(high, orchestration_jid)
 4473 
 4474     def compile_highstate(self):
 4475         """
 4476         Return just the highstate or the errors
 4477         """
 4478         err = []
 4479         top = self.get_top()
 4480         err += self.verify_tops(top)
 4481         matches = self.top_matches(top)
 4482         high, errors = self.render_highstate(matches)
 4483         err += errors
 4484 
 4485         if err:
 4486             return err
 4487 
 4488         return high
 4489 
 4490     def compile_low_chunks(self):
 4491         """
 4492         Compile the highstate but don't run it, return the low chunks to
 4493         see exactly what the highstate will execute
 4494         """
 4495         top = self.get_top()
 4496         matches = self.top_matches(top)
 4497         high, errors = self.render_highstate(matches)
 4498 
 4499         # If there is extension data reconcile it
 4500         high, ext_errors = self.state.reconcile_extend(high)
 4501         errors += ext_errors
 4502 
 4503         # Verify that the high data is structurally sound
 4504         errors += self.state.verify_high(high)
 4505         high, req_in_errors = self.state.requisite_in(high)
 4506         errors += req_in_errors
 4507         high = self.state.apply_exclude(high)
 4508 
 4509         if errors:
 4510             return errors
 4511 
 4512         # Compile and verify the raw chunks
 4513         chunks = self.state.compile_high_data(high)
 4514 
 4515         return chunks
 4516 
 4517     def compile_state_usage(self):
 4518         """
 4519         Return all used and unused states for the minion based on the top match data
 4520         """
 4521         err = []
 4522         top = self.get_top()
 4523         err += self.verify_tops(top)
 4524 
 4525         if err:
 4526             return err
 4527 
 4528         matches = self.top_matches(top)
 4529         state_usage = {}
 4530 
 4531         for saltenv, states in self.avail.items():
 4532             env_usage = {
 4533                 "used": [],
 4534                 "unused": [],
 4535                 "count_all": 0,
 4536                 "count_used": 0,
 4537                 "count_unused": 0,
 4538             }
 4539 
 4540             env_matches = matches.get(saltenv)
 4541 
 4542             for state in states:
 4543                 env_usage["count_all"] += 1
 4544                 if state in env_matches:
 4545                     env_usage["count_used"] += 1
 4546                     env_usage["used"].append(state)
 4547                 else:
 4548                     env_usage["count_unused"] += 1
 4549                     env_usage["unused"].append(state)
 4550 
 4551             state_usage[saltenv] = env_usage
 4552 
 4553         return state_usage
 4554 
 4555 
 4556 class HighState(BaseHighState):
 4557     """
 4558     Generate and execute the salt "High State". The High State is the
 4559     compound state derived from a group of template files stored on the
 4560     salt master or in the local cache.
 4561     """
 4562 
 4563     # a stack of active HighState objects during a state.highstate run
 4564     stack = []
 4565 
 4566     def __init__(
 4567         self,
 4568         opts,
 4569         pillar_override=None,
 4570         jid=None,
 4571         pillar_enc=None,
 4572         proxy=None,
 4573         context=None,
 4574         mocked=False,
 4575         loader="states",
 4576         initial_pillar=None,
 4577     ):
 4578         self.opts = opts
 4579         self.client = salt.fileclient.get_file_client(self.opts)
 4580         BaseHighState.__init__(self, opts)
 4581         self.state = State(
 4582             self.opts,
 4583             pillar_override,
 4584             jid,
 4585             pillar_enc,
 4586             proxy=proxy,
 4587             context=context,
 4588             mocked=mocked,
 4589             loader=loader,
 4590             initial_pillar=initial_pillar,
 4591         )
 4592         self.matchers = salt.loader.matchers(self.opts)
 4593         self.proxy = proxy
 4594 
 4595         # tracks all pydsl state declarations globally across sls files
 4596         self._pydsl_all_decls = {}
 4597 
 4598         # a stack of current rendering Sls objects, maintained and used by the pydsl renderer.
 4599         self._pydsl_render_stack = []
 4600 
 4601     def push_active(self):
 4602         self.stack.append(self)
 4603 
 4604     @classmethod
 4605     def clear_active(cls):
 4606         # Nuclear option
 4607         #
 4608         # Blow away the entire stack. Used primarily by the test runner but also
 4609         # useful in custom wrappers of the HighState class, to reset the stack
 4610         # to a fresh state.
 4611         cls.stack = []
 4612 
 4613     @classmethod
 4614     def pop_active(cls):
 4615         cls.stack.pop()
 4616 
 4617     @classmethod
 4618     def get_active(cls):
 4619         try:
 4620             return cls.stack[-1]
 4621         except IndexError:
 4622             return None
 4623 
 4624 
 4625 class MasterState(State):
 4626     """
 4627     Create a State object for master side compiling
 4628     """
 4629 
 4630     def __init__(self, opts, minion):
 4631         State.__init__(self, opts)
 4632 
 4633     def load_modules(self, data=None, proxy=None):
 4634         """
 4635         Load the modules into the state
 4636         """
 4637         log.info("Loading fresh modules for state activity")
 4638         # Load a modified client interface that looks like the interface used
 4639         # from the minion, but uses remote execution
 4640         #
 4641         self.functions = salt.client.FunctionWrapper(self.opts, self.opts["id"])
 4642         # Load the states, but they should not be used in this class apart
 4643         # from inspection
 4644         self.utils = salt.loader.utils(self.opts)
 4645         self.serializers = salt.loader.serializers(self.opts)
 4646         self.states = salt.loader.states(
 4647             self.opts, self.functions, self.utils, self.serializers
 4648         )
 4649         self.rend = salt.loader.render(
 4650             self.opts, self.functions, states=self.states, context=self.state_con
 4651         )
 4652 
 4653 
 4654 class MasterHighState(HighState):
 4655     """
 4656     Execute highstate compilation from the master
 4657     """
 4658 
 4659     def __init__(self, master_opts, minion_opts, grains, id_, saltenv=None):
 4660         # Force the fileclient to be local
 4661         opts = copy.deepcopy(minion_opts)
 4662         opts["file_client"] = "local"
 4663         opts["file_roots"] = master_opts["master_roots"]
 4664         opts["renderer"] = master_opts["renderer"]
 4665         opts["state_top"] = master_opts["state_top"]
 4666         opts["id"] = id_
 4667         opts["grains"] = grains
 4668         HighState.__init__(self, opts)
 4669 
 4670 
 4671 class RemoteHighState:
 4672     """
 4673     Manage gathering the data from the master
 4674     """
 4675 
 4676     # XXX: This class doesn't seem to be used anywhere
 4677     def __init__(self, opts, grains):
 4678         self.opts = opts
 4679         self.grains = grains
 4680         self.serial = salt.payload.Serial(self.opts)
 4681         # self.auth = salt.crypt.SAuth(opts)
 4682         self.channel = salt.transport.client.ReqChannel.factory(self.opts["master_uri"])
 4683         self._closing = False
 4684 
 4685     def compile_master(self):
 4686         """
 4687         Return the state data from the master
 4688         """
 4689         load = {"grains": self.grains, "opts": self.opts, "cmd": "_master_state"}
 4690         try:
 4691             return self.channel.send(load, tries=3, timeout=72000)
 4692         except SaltReqTimeoutError:
 4693             return {}
 4694 
 4695     def destroy(self):
 4696         if self._closing:
 4697             return
 4698 
 4699         self._closing = True
 4700         self.channel.close()
 4701 
 4702     # pylint: disable=W1701
 4703     def __del__(self):
 4704         self.destroy()
 4705 
 4706     # pylint: enable=W1701