"Fossies" - the Fresh Open Source Software Archive

Member "fail2ban-0.11.1/fail2ban/server/utils.py" (11 Jan 2020, 12405 Bytes) of package /linux/misc/fail2ban-0.11.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "utils.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 0.10.5_vs_0.11.1.

    1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
    2 # vi: set ft=python sts=4 ts=4 sw=4 noet :
    3 
    4 # This file is part of Fail2Ban.
    5 #
    6 # Fail2Ban is free software; you can redistribute it and/or modify
    7 # it under the terms of the GNU General Public License as published by
    8 # the Free Software Foundation; either version 2 of the License, or
    9 # (at your option) any later version.
   10 #
   11 # Fail2Ban is distributed in the hope that it will be useful,
   12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
   13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   14 # GNU General Public License for more details.
   15 #
   16 # You should have received a copy of the GNU General Public License
   17 # along with Fail2Ban; if not, write to the Free Software
   18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   19 
   20 __author__ = "Serg G. Brester (sebres) and Fail2Ban Contributors"
   21 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2012 Yaroslav Halchenko, 2012-2015 Serg G. Brester"
   22 __license__ = "GPL"
   23 
   24 import fcntl
   25 import logging
   26 import os
   27 import signal
   28 import subprocess
   29 import sys
   30 from     threading import Lock
   31 import time
   32 from ..helpers import getLogger, _merge_dicts, uni_decode
   33 
   34 try:
   35     from collections import OrderedDict
   36 except ImportError: # pragma: 3.x no cover
   37     OrderedDict = dict
   38 
   39 if sys.version_info >= (3, 3):
   40     import importlib.machinery
   41 else:
   42     import imp
   43 
   44 # Gets the instance of the logger.
   45 logSys = getLogger(__name__)
   46 
   47 # Some hints on common abnormal exit codes
   48 _RETCODE_HINTS = {
   49     127: '"Command not found".  Make sure that all commands in %(realCmd)r '
   50             'are in the PATH of fail2ban-server process '
   51             '(grep -a PATH= /proc/`pidof -x fail2ban-server`/environ). '
   52             'You may want to start '
   53             '"fail2ban-server -f" separately, initiate it with '
   54             '"fail2ban-client reload" in another shell session and observe if '
   55             'additional informative error messages appear in the terminals.'
   56     }
   57 
   58 # Dictionary to lookup signal name from number
   59 signame = dict((num, name)
   60     for name, num in signal.__dict__.iteritems() if name.startswith("SIG"))
   61 
   62 class Utils():
   63     """Utilities provide diverse static methods like executes OS shell commands, etc.
   64     """
   65 
   66     DEFAULT_SLEEP_TIME = 2
   67     DEFAULT_SLEEP_INTERVAL = 0.2
   68     DEFAULT_SHORT_INTERVAL = 0.001
   69     DEFAULT_SHORTEST_INTERVAL = DEFAULT_SHORT_INTERVAL / 100
   70 
   71 
   72     class Cache(object):
   73         """A simple cache with a TTL and limit on size
   74         """
   75 
   76         def __init__(self, *args, **kwargs):
   77             self.setOptions(*args, **kwargs)
   78             self._cache = OrderedDict()
   79             self.__lock = Lock()
   80 
   81         def setOptions(self, maxCount=1000, maxTime=60):
   82             self.maxCount = maxCount
   83             self.maxTime = maxTime
   84 
   85         def __len__(self):
   86             return len(self._cache)
   87 
   88         def get(self, k, defv=None):
   89             v = self._cache.get(k)
   90             if v: 
   91                 if v[1] > time.time():
   92                     return v[0]
   93                 self.unset(k)
   94             return defv
   95             
   96         def set(self, k, v):
   97             t = time.time()
   98             # avoid multiple modification of dict multi-threaded:
   99             cache = self._cache
  100             with self.__lock:
  101                 # clean cache if max count reached:
  102                 if len(cache) >= self.maxCount:
  103                     if OrderedDict is not dict:
  104                         # ordered (so remove some from ahead, FIFO)
  105                         while cache:
  106                             (ck, cv) = cache.popitem(last=False)
  107                             # if not yet expired (but has free slot for new entry):
  108                             if cv[1] > t and len(cache) < self.maxCount:
  109                                 break
  110                     else: # pragma: 3.x no cover (dict is in 2.6 only)
  111                         remlst = []
  112                         for (ck, cv) in cache.iteritems():
  113                             # if expired:
  114                             if cv[1] <= t:
  115                                 remlst.append(ck)
  116                         for ck in remlst:
  117                             self._cache.pop(ck, None)
  118                         # if still max count - remove any one:
  119                         while cache and len(cache) >= self.maxCount:
  120                             cache.popitem()
  121                 # set now:
  122                 cache[k] = (v, t + self.maxTime)
  123 
  124         def unset(self, k):
  125             with self.__lock:
  126                 self._cache.pop(k, None)
  127 
  128 
  129     @staticmethod
  130     def setFBlockMode(fhandle, value):
  131         flags = fcntl.fcntl(fhandle, fcntl.F_GETFL)
  132         if not value:
  133             flags |= os.O_NONBLOCK 
  134         else:
  135             flags &= ~os.O_NONBLOCK
  136         fcntl.fcntl(fhandle, fcntl.F_SETFL, flags)
  137         return flags
  138 
  139     @staticmethod
  140     def buildShellCmd(realCmd, varsDict):
  141         """Generates new shell command as array, contains map as variables to
  142         arguments statement (varsStat), the command (realCmd) used this variables and
  143         the list of the arguments, mapped from varsDict
  144 
  145         Example:
  146             buildShellCmd('echo "V2: $v2, V1: $v1"', {"v1": "val 1", "v2": "val 2", "vUnused": "unused var"})
  147         returns:
  148             ['v1=$0 v2=$1 vUnused=$2 \necho "V2: $v2, V1: $v1"', 'val 1', 'val 2', 'unused var']
  149         """
  150         # build map as array of vars and command line array:
  151         varsStat = ""
  152         if not isinstance(realCmd, list):
  153             realCmd = [realCmd]
  154         i = len(realCmd)-1
  155         for k, v in varsDict.iteritems():
  156             varsStat += "%s=$%s " % (k, i)
  157             realCmd.append(v)
  158             i += 1
  159         realCmd[0] = varsStat + "\n" + realCmd[0]
  160         return realCmd
  161 
  162     @staticmethod
  163     def executeCmd(realCmd, timeout=60, shell=True, output=False, tout_kill_tree=True, 
  164         success_codes=(0,), varsDict=None):
  165         """Executes a command.
  166 
  167         Parameters
  168         ----------
  169         realCmd : str
  170             The command to execute.
  171         timeout : int
  172             The time out in seconds for the command.
  173         shell : bool
  174             If shell is True (default), the specified command (may be a string) will be 
  175             executed through the shell.
  176         output : bool
  177             If output is True, the function returns tuple (success, stdoutdata, stderrdata, returncode).
  178             If False, just indication of success is returned
  179         varsDict: dict
  180             variables supplied to the command (or to the shell script)
  181 
  182         Returns
  183         -------
  184         bool or (bool, str, str, int)
  185             True if the command succeeded and with stdout, stderr, returncode if output was set to True
  186 
  187         Raises
  188         ------
  189         OSError
  190             If command fails to be executed.
  191         RuntimeError
  192             If command execution times out.
  193         """
  194         stdout = stderr = None
  195         retcode = None
  196         popen = env = None
  197         if varsDict:
  198             if shell:
  199                 # build map as array of vars and command line array:
  200                 realCmd = Utils.buildShellCmd(realCmd, varsDict)
  201             else: # pragma: no cover - currently unused
  202                 env = _merge_dicts(os.environ, varsDict)
  203         realCmdId = id(realCmd)
  204         logCmd = lambda level: logSys.log(level, "%x -- exec: %s", realCmdId, realCmd)
  205         try:
  206             popen = subprocess.Popen(
  207                 realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, env=env,
  208                 preexec_fn=os.setsid  # so that killpg does not kill our process
  209             )
  210             # wait with timeout for process has terminated:
  211             retcode = popen.poll()
  212             if retcode is None:
  213                 def _popen_wait_end():
  214                     retcode = popen.poll()
  215                     return (True, retcode) if retcode is not None else None
  216                 # popen.poll is fast operation so we can use the shortest sleep interval:
  217                 retcode = Utils.wait_for(_popen_wait_end, timeout, Utils.DEFAULT_SHORTEST_INTERVAL)
  218                 if retcode:
  219                     retcode = retcode[1]
  220             # if timeout:
  221             if retcode is None:
  222                 if logCmd: logCmd(logging.ERROR); logCmd = None
  223                 logSys.error("%x -- timed out after %s seconds." %
  224                     (realCmdId, timeout))
  225                 pgid = os.getpgid(popen.pid)
  226                 # if not tree - first try to terminate and then kill, otherwise - kill (-9) only:
  227                 os.killpg(pgid, signal.SIGTERM) # Terminate the process
  228                 time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
  229                 retcode = popen.poll()
  230                 #logSys.debug("%s -- terminated %s ", realCmd, retcode)
  231                 if retcode is None or tout_kill_tree: # Still going...
  232                     os.killpg(pgid, signal.SIGKILL) # Kill the process
  233                     time.sleep(Utils.DEFAULT_SLEEP_INTERVAL)
  234                     if retcode is None: # pragma: no cover - too sporadic
  235                         retcode = popen.poll()
  236                     #logSys.debug("%s -- killed %s ", realCmd, retcode)
  237                 if retcode is None and not Utils.pid_exists(pgid): # pragma: no cover
  238                     retcode = signal.SIGKILL
  239         except OSError as e:
  240             if logCmd: logCmd(logging.ERROR); logCmd = None
  241             stderr = "%s -- failed with %s" % (realCmd, e)
  242             logSys.error(stderr)
  243             if not popen:
  244                 return False if not output else (False, stdout, stderr, retcode)
  245 
  246         std_level = logging.DEBUG if retcode in success_codes else logging.ERROR
  247         if std_level >= logSys.getEffectiveLevel():
  248             if logCmd: logCmd(std_level-1 if std_level == logging.DEBUG else logging.ERROR); logCmd = None
  249         # if we need output (to return or to log it): 
  250         if output or std_level >= logSys.getEffectiveLevel():
  251 
  252             # if was timeouted (killed/terminated) - to prevent waiting, set std handles to non-blocking mode.
  253             if popen.stdout:
  254                 try:
  255                     if retcode is None or retcode < 0:
  256                         Utils.setFBlockMode(popen.stdout, False)
  257                     stdout = popen.stdout.read()
  258                 except IOError as e: # pragma: no cover
  259                     logSys.error(" ... -- failed to read stdout %s", e)
  260                 if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel():
  261                     for l in stdout.splitlines():
  262                         logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l))
  263                 popen.stdout.close()
  264             if popen.stderr:
  265                 try:
  266                     if retcode is None or retcode < 0:
  267                         Utils.setFBlockMode(popen.stderr, False)
  268                     stderr = popen.stderr.read()
  269                 except IOError as e: # pragma: no cover
  270                     logSys.error(" ... -- failed to read stderr %s", e)
  271                 if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel():
  272                     for l in stderr.splitlines():
  273                         logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l))
  274                 popen.stderr.close()
  275 
  276         success = False
  277         if retcode in success_codes:
  278             logSys.debug("%x -- returned successfully %i", realCmdId, retcode)
  279             success = True
  280         elif retcode is None:
  281             logSys.error("%x -- unable to kill PID %i", realCmdId, popen.pid)
  282         elif retcode < 0 or retcode > 128:
  283             # dash would return negative while bash 128 + n
  284             sigcode = -retcode if retcode < 0 else retcode - 128
  285             logSys.error("%x -- killed with %s (return code: %s)",
  286                 realCmdId, signame.get(sigcode, "signal %i" % sigcode), retcode)
  287         else:
  288             msg = _RETCODE_HINTS.get(retcode, None)
  289             logSys.error("%x -- returned %i", realCmdId, retcode)
  290             if msg:
  291                 logSys.info("HINT on %i: %s", retcode, msg % locals())
  292         if output:
  293             return success, stdout, stderr, retcode
  294         return success if len(success_codes) == 1 else (success, retcode)
  295     
  296     @staticmethod
  297     def wait_for(cond, timeout, interval=None):
  298         """Wait until condition expression `cond` is True, up to `timeout` sec
  299 
  300         Parameters
  301         ----------
  302         cond : callable
  303             The expression to check condition 
  304             (should return equivalent to bool True if wait successful).
  305         timeout : float or callable
  306             The time out for end of wait
  307             (in seconds or callable that returns True if timeout occurred).
  308         interval : float (optional)
  309             Polling start interval for wait cycle in seconds.
  310 
  311         Returns
  312         -------
  313         variable
  314             The return value of the last call of `cond`, 
  315             logical False (or None, 0, etc) if timeout occurred.
  316         """
  317         #logSys.log(5, "  wait for %r, tout: %r / %r", cond, timeout, interval)
  318         ini = 1  # to delay initializations until/when necessary
  319         while True:
  320             ret = cond()
  321             if ret:
  322                 return ret
  323             if ini:
  324                 ini = stm = 0
  325                 if not callable(timeout):
  326                     time0 = time.time() + timeout
  327                     timeout_expr = lambda: time.time() > time0
  328                 else:
  329                     timeout_expr = timeout
  330                 if not interval:
  331                     interval = Utils.DEFAULT_SLEEP_INTERVAL
  332             if timeout_expr():
  333                 break
  334             stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME)
  335             time.sleep(stm)
  336         return ret
  337 
  338     # Solution from http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid
  339     # under cc by-sa 3.0
  340     if os.name == 'posix':
  341         @staticmethod
  342         def pid_exists(pid):
  343             """Check whether pid exists in the current process table."""
  344             import errno
  345             if pid < 0:
  346                 return False
  347             try:
  348                 os.kill(pid, 0)
  349             except OSError as e:
  350                 return e.errno == errno.EPERM
  351             else:
  352                 return True
  353     else: # pragma: no cover (no windows currently supported)
  354         @staticmethod
  355         def pid_exists(pid):
  356             import ctypes
  357             kernel32 = ctypes.windll.kernel32
  358             SYNCHRONIZE = 0x100000
  359 
  360             process = kernel32.OpenProcess(SYNCHRONIZE, 0, pid)
  361             if process != 0:
  362                 kernel32.CloseHandle(process)
  363                 return True
  364             else:
  365                 return False
  366 
  367     @staticmethod
  368     def load_python_module(pythonModule):
  369         pythonModuleName = os.path.splitext(
  370             os.path.basename(pythonModule))[0]
  371         if sys.version_info >= (3, 3):
  372             mod = importlib.machinery.SourceFileLoader(
  373                 pythonModuleName, pythonModule).load_module()
  374         else:
  375             mod = imp.load_source(
  376                 pythonModuleName, pythonModule)
  377         return mod