"Fossies" - the Fresh Open Source Software Archive

Member "getmail-5.16/getmailcore/filters.py" (31 Oct 2021, 19843 Bytes) of package /linux/misc/getmail-5.16.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "filters.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 5.15_vs_5.16.

    1 #!/usr/bin/env python2
    2 '''Classes implementing message filters.
    3 
    4 Currently implemented:
    5 
    6 '''
    7 
    8 __all__ = [
    9     'FilterSkeleton',
   10     'Filter_external',
   11     'Filter_classifier',
   12     'Filter_TMDA',
   13 ]
   14 
   15 import os
   16 import tempfile
   17 import types
   18 
   19 from getmailcore.exceptions import *
   20 from getmailcore.compatibility import *
   21 from getmailcore.message import *
   22 from getmailcore.utilities import *
   23 from getmailcore.baseclasses import *
   24 
   25 #######################################
   26 class FilterSkeleton(ConfigurableBase):
   27     '''Base class for implementing message-filtering classes.
   28 
   29     Sub-classes should provide the following data attributes and methods:
   30 
   31       _confitems - a tuple of dictionaries representing the parameters the class
   32                    takes.  Each dictionary should contain the following key,
   33                    value pairs:
   34                      - name - parameter name
   35                      - type - a type function to compare the parameter value
   36                      against (i.e. str, int, bool)
   37                      - default - optional default value.  If not preseent, the
   38                      parameter is required.
   39 
   40       __str__(self) - return a simple string representing the class instance.
   41 
   42       showconf(self) - log a message representing the instance and configuration
   43                        from self._confstring().
   44 
   45       initialize(self) - process instantiation parameters from self.conf.
   46                          Raise getmailConfigurationError on errors.  Do any
   47                          other validation necessary, and set self.__initialized
   48                          when done.
   49 
   50       _filter_message(self, msg) - accept the message and deliver it, returning
   51                                    a tuple (exitcode, newmsg, err). exitcode
   52                                    should be 0 for success, 99 or 100 for
   53                                    success but drop the message, anything else
   54                                    for error. err should be an empty string on
   55                                    success, or an error message otherwise.
   56                                    newmsg is an email.Message() object
   57                                    representing the message in filtered form, or
   58                                    None on error or when dropping the message.
   59 
   60     See the Filter_external class for a good (though not simple) example.
   61     '''
   62     def __init__(self, **args):
   63         ConfigurableBase.__init__(self, **args)
   64         try:
   65             self.initialize()
   66         except KeyError, o:
   67             raise getmailConfigurationError(
   68                 'missing required configuration parameter %s' % o
   69             )
   70         self.log.trace('done\n')
   71 
   72     def filter_message(self, msg, retriever):
   73         self.log.trace()
   74         msg.received_from = retriever.received_from
   75         msg.received_with = retriever.received_with
   76         msg.received_by = retriever.received_by
   77         exitcode, newmsg, err = self._filter_message(msg)
   78         if exitcode in self.exitcodes_drop:
   79             # Drop message
   80             self.log.debug('filter %s returned %d; dropping message\n'
   81                            % (self, exitcode))
   82             return None
   83         elif (exitcode not in self.exitcodes_keep):
   84             raise getmailFilterError('filter %s returned %d (%s)\n'
   85                                      % (self, exitcode, err))
   86         elif err:
   87             if self.conf['ignore_stderr']:
   88                 self.log.info('filter %s: %s\n' % (self, err))
   89             else:
   90                 raise getmailFilterError(
   91                     'filter %s returned %d but wrote to stderr: %s\n'
   92                     % (self, exitcode, err)
   93                 )
   94 
   95         # Check the filter was sane
   96         if len(newmsg.headers()) < len(msg.headers()):
   97             if not self.conf.get('ignore_header_shrinkage', False):
   98                 # Warn user
   99                 self.log.warning(
  100                     'Warning: filter %s returned fewer headers (%d) than '
  101                         'supplied (%d)\n'
  102                     % (self, len(newmsg.headers()), len(msg.headers()))
  103                 )
  104 
  105         # Copy attributes from original message
  106         newmsg.copyattrs(msg)
  107 
  108         return newmsg
  109 
  110 #######################################
  111 class Filter_external(FilterSkeleton, ForkingBase):
  112     '''Arbitrary external filter destination.
  113 
  114     Parameters:
  115 
  116       path - path to the external filter binary.
  117 
  118       unixfrom - (boolean) whether to include a Unix From_ line at the beginning
  119                  of the message.  Defaults to False.
  120 
  121       arguments - a valid Python tuple of strings to be passed as arguments to
  122                   the command.  The following replacements are available if
  123                   supported by the retriever:
  124 
  125                     %(sender) - envelope return path
  126                     %(recipient) - recipient address
  127                     %(domain) - domain-part of recipient address
  128                     %(local) - local-part of recipient address
  129 
  130                   Warning: the text of these replacements is taken from the
  131                   message and is therefore under the control of a potential
  132                   attacker. DO NOT PASS THESE VALUES TO A SHELL -- they may
  133                   contain unsafe shell metacharacters or other hostile
  134                   constructions.
  135 
  136                   example:
  137 
  138                     path = /path/to/myfilter
  139                     arguments = ('--demime', '-f%(sender)', '--',
  140                         '%(recipient)')
  141 
  142       exitcodes_keep - if provided, a tuple of integers representing filter exit
  143                        codes that mean to pass the message to the next filter or
  144                        destination.  Default is (0, ).
  145 
  146       exitcodes_drop - if provided, a tuple of integers representing filter exit
  147                        codes that mean to drop the message.  Default is
  148                        (99, 100).
  149 
  150       user (string, optional) - if provided, the external command will be run as
  151                                 the specified user.  This requires that the main
  152                                 getmail process have permission to change the
  153                                 effective user ID.
  154 
  155       group (string, optional) -  if provided, the external command will be run
  156                                 with the specified group ID.  This requires that
  157                                 the main getmail process have permission to
  158                                 change the effective group ID.
  159 
  160       allow_root_commands (boolean, optional) - if set, external commands are
  161                                         allowed when running as root.  The
  162                                         default is not to allow such behaviour.
  163 
  164       ignore_stderr (boolean, optional) - if set, getmail will not consider the
  165             program writing to stderr to be an error.  The default is False.
  166     '''
  167     _confitems = (
  168         ConfFile(name='path'),
  169         ConfBool(name='unixfrom', required=False, default=False),
  170         ConfTupleOfStrings(name='arguments', required=False, default="()"),
  171         ConfTupleOfStrings(name='exitcodes_keep', required=False,
  172                            default="(0, )"),
  173         ConfTupleOfStrings(name='exitcodes_drop', required=False,
  174                            default="(99, 100)"),
  175         ConfString(name='user', required=False, default=None),
  176         ConfString(name='group', required=False, default=None),
  177         ConfBool(name='allow_root_commands', required=False, default=False),
  178         ConfBool(name='ignore_header_shrinkage', required=False, default=False),
  179         ConfBool(name='ignore_stderr', required=False, default=False),
  180         ConfInstance(name='configparser', required=False),
  181     )
  182 
  183     def initialize(self):
  184         self.log.trace()
  185         self.conf['command'] = os.path.basename(self.conf['path'])
  186         if not os.access(self.conf['path'], os.X_OK):
  187             raise getmailConfigurationError(
  188                 '%s not executable' % self.conf['path']
  189             )
  190         if type(self.conf['arguments']) != tuple:
  191             raise getmailConfigurationError(
  192                 'incorrect arguments format; see documentation (%s)'
  193                 % self.conf['arguments']
  194             )
  195         try:
  196             self.exitcodes_keep = [int(i) for i in self.conf['exitcodes_keep']
  197                                    if 0 <= int(i) <= 255]
  198             self.exitcodes_drop = [int(i) for i in self.conf['exitcodes_drop']
  199                                    if 0 <= int(i) <= 255]
  200             if not self.exitcodes_keep:
  201                 raise getmailConfigurationError('exitcodes_keep set empty')
  202             if frozenset(self.exitcodes_keep).intersection(
  203                 frozenset(self.exitcodes_drop)
  204             ):
  205                 raise getmailConfigurationError('exitcode sets intersect')
  206         except ValueError, o:
  207             raise getmailConfigurationError('invalid exit code specified (%s)'
  208                                             % o)
  209 
  210     def __str__(self):
  211         self.log.trace()
  212         return 'Filter_external %s (%s)' % (self.conf['command'],
  213                                             self._confstring())
  214 
  215     def showconf(self):
  216         self.log.trace()
  217         self.log.info('Filter_external(%s)\n' % self._confstring())
  218 
  219     def _filter_command(self, msg, msginfo, stdout, stderr):
  220         try:
  221             # Write out message with native EOL convention
  222             msgfile = tempfile.TemporaryFile()
  223             msgfile.write(msg.flatten(False, False,
  224                                       include_from=self.conf['unixfrom']))
  225             msgfile.flush()
  226             os.fsync(msgfile.fileno())
  227             # Rewind
  228             msgfile.seek(0)
  229             # Set stdin to read from this file
  230             os.dup2(msgfile.fileno(), 0)
  231             # Set stdout and stderr to write to files
  232             os.dup2(stdout.fileno(), 1)
  233             os.dup2(stderr.fileno(), 2)
  234             change_usergroup(None, self.conf['user'], self.conf['group'])
  235             args = [self.conf['path'], self.conf['path']]
  236             for arg in self.conf['arguments']:
  237                 arg = expand_user_vars(arg)
  238                 for (key, value) in msginfo.items():
  239                     arg = arg.replace('%%(%s)' % key, value)
  240                 args.append(arg)
  241             # Can't log this; if --trace is on, it will be written to the
  242             # message passed to the filter.
  243             #self.log.debug('about to execl() with args %s\n' % str(args))
  244             os.execl(*args)
  245         except StandardError, o:
  246             # Child process; any error must cause us to exit nonzero for parent
  247             # to detect it
  248             self.log.critical('exec of filter %s failed (%s)'
  249                               % (self.conf['command'], o))
  250             os._exit(127)
  251 
  252     def _filter_message(self, msg):
  253         self.log.trace()
  254         self._prepare_child()
  255         msginfo = {}
  256         msginfo['sender'] = msg.sender
  257         if msg.recipient != None:
  258             msginfo['recipient'] = msg.recipient
  259             msginfo['domain'] = msg.recipient.lower().split('@')[-1]
  260             msginfo['local'] = '@'.join(msg.recipient.split('@')[:-1])
  261         self.log.debug('msginfo "%s"\n' % msginfo)
  262 
  263         # At least some security...
  264         if (os.geteuid() == 0 and not self.conf['allow_root_commands']
  265                 and self.conf['user'] == None):
  266             raise getmailConfigurationError(
  267                 'refuse to invoke external commands as root by default'
  268             )
  269 
  270         stdout = tempfile.TemporaryFile()
  271         stderr = tempfile.TemporaryFile()
  272         childpid = os.fork()
  273 
  274         if not childpid:
  275             # Child
  276             self._filter_command(msg, msginfo, stdout, stderr)
  277         self.log.debug('spawned child %d\n' % childpid)
  278 
  279         # Parent
  280         exitcode = self._wait_for_child(childpid)
  281 
  282         stdout.seek(0)
  283         stderr.seek(0)
  284         err = stderr.read().strip()
  285 
  286         self.log.debug('command %s %d exited %d\n' % (self.conf['command'],
  287                                                       childpid, exitcode))
  288 
  289         newmsg = Message(fromfile=stdout)
  290 
  291         return (exitcode, newmsg, err)
  292 
  293 #######################################
  294 class Filter_classifier(Filter_external):
  295     '''Filter which runs the message through an external command, adding the
  296     command's output to the message header.  Takes the same parameters as
  297     Filter_external.  If the command prints nothing, no header fields are
  298     added.
  299     '''
  300     def __str__(self):
  301         self.log.trace()
  302         return 'Filter_classifier %s (%s)' % (self.conf['command'],
  303                                               self._confstring())
  304 
  305     def showconf(self):
  306         self.log.trace()
  307         self.log.info('Filter_classifier(%s)\n' % self._confstring())
  308 
  309     def _filter_message(self, msg):
  310         self.log.trace()
  311         self._prepare_child()
  312         msginfo = {}
  313         msginfo['sender'] = msg.sender
  314         if msg.recipient != None:
  315             msginfo['recipient'] = msg.recipient
  316             msginfo['domain'] = msg.recipient.lower().split('@')[-1]
  317             msginfo['local'] = '@'.join(msg.recipient.split('@')[:-1])
  318         self.log.debug('msginfo "%s"\n' % msginfo)
  319 
  320         # At least some security...
  321         if (os.geteuid() == 0 and not self.conf['allow_root_commands']
  322                 and self.conf['user'] == None):
  323             raise getmailConfigurationError(
  324                 'refuse to invoke external commands as root by default'
  325             )
  326 
  327         stdout = tempfile.TemporaryFile()
  328         stderr = tempfile.TemporaryFile()
  329         childpid = os.fork()
  330 
  331         if not childpid:
  332             # Child
  333             self._filter_command(msg, msginfo, stdout, stderr)
  334         self.log.debug('spawned child %d\n' % childpid)
  335 
  336         # Parent
  337         exitcode = self._wait_for_child(childpid)
  338 
  339         stdout.seek(0)
  340         stderr.seek(0)
  341         err = stderr.read().strip()
  342 
  343         self.log.debug('command %s %d exited %d\n' % (self.conf['command'],
  344                                                       childpid, exitcode))
  345 
  346         for line in [line.strip() for line in stdout.readlines()
  347                      if line.strip()]:
  348             # Output from filter can be in any random text encoding and may
  349             # not even be valid, which causes problems when trying to stick
  350             # that text into message headers.  Try to decode it to something
  351             # sane here first.
  352             line = decode_crappy_text(line)
  353             msg.add_header('X-getmail-filter-classifier', line)
  354 
  355         return (exitcode, msg, err)
  356 
  357 #######################################
  358 class Filter_TMDA(FilterSkeleton, ForkingBase):
  359     '''Filter which runs the message through TMDA's tmda-filter program
  360     to handle confirmations, etc.
  361 
  362     Parameters:
  363 
  364       path - path to the external tmda-filter binary.
  365 
  366       user (string, optional) - if provided, the external command will be run
  367                                 as the specified user.  This requires that the
  368                                 main getmail process have permission to change
  369                                 the effective user ID.
  370 
  371       group (string, optional) -  if provided, the external command will be run
  372                                 with the specified group ID.  This requires that
  373                                 the main getmail process have permission to
  374                                 change the effective group ID.
  375 
  376       allow_root_commands (boolean, optional) - if set, external commands are
  377                                 allowed when running as root.  The default is
  378                                 not to allow such behaviour.
  379 
  380       ignore_stderr (boolean, optional) - if set, getmail will not consider the
  381             program writing to stderr to be an error.  The default is False.
  382 
  383       conf-break - used to break envelope recipient to find EXT.  Defaults
  384                                 to "-".
  385     '''
  386     _confitems = (
  387         ConfFile(name='path', default='/usr/local/bin/tmda-filter'),
  388         ConfString(name='user', required=False, default=None),
  389         ConfString(name='group', required=False, default=None),
  390         ConfBool(name='allow_root_commands', required=False, default=False),
  391         ConfBool(name='ignore_stderr', required=False, default=False),
  392         ConfString(name='conf-break', required=False, default='-'),
  393         ConfInstance(name='configparser', required=False),
  394     )
  395 
  396     def initialize(self):
  397         self.log.trace()
  398         self.conf['command'] = os.path.basename(self.conf['path'])
  399         if not os.access(self.conf['path'], os.X_OK):
  400             raise getmailConfigurationError(
  401                 '%s not executable' % self.conf['path']
  402             )
  403         self.exitcodes_keep = (0, )
  404         self.exitcodes_drop = (99, )
  405 
  406     def __str__(self):
  407         self.log.trace()
  408         return 'Filter_TMDA %s' % self.conf['command']
  409 
  410     def showconf(self):
  411         self.log.trace()
  412         self.log.info('Filter_TMDA(%s)\n' % self._confstring())
  413 
  414     def _filter_command(self, msg, stdout, stderr):
  415         try:
  416             # Write out message with native EOL convention
  417             msgfile = tempfile.TemporaryFile()
  418             msgfile.write(msg.flatten(True, True, include_from=True))
  419             msgfile.flush()
  420             os.fsync(msgfile.fileno())
  421             # Rewind
  422             msgfile.seek(0)
  423             # Set stdin to read from this file
  424             os.dup2(msgfile.fileno(), 0)
  425             # Set stdout and stderr to write to files
  426             os.dup2(stdout.fileno(), 1)
  427             os.dup2(stderr.fileno(), 2)
  428             change_usergroup(None, self.conf['user'], self.conf['group'])
  429             args = [self.conf['path'], self.conf['path']]
  430             # Set environment for TMDA
  431             os.environ['SENDER'] = msg.sender
  432             os.environ['RECIPIENT'] = msg.recipient
  433             os.environ['EXT'] = self.conf['conf-break'].join(
  434                 '@'.join(msg.recipient.split('@')[:-1]).split(
  435                     self.conf['conf-break']
  436                 )[1:]
  437             )
  438             self.log.trace('SENDER="%(SENDER)s",RECIPIENT="%(RECIPIENT)s"'
  439                            ',EXT="%(EXT)s"' % os.environ)
  440             self.log.debug('about to execl() with args %s\n' % str(args))
  441             os.execl(*args)
  442         except StandardError, o:
  443             # Child process; any error must cause us to exit nonzero for parent
  444             # to detect it
  445             self.log.critical('exec of filter %s failed (%s)'
  446                               % (self.conf['command'], o))
  447             os._exit(127)
  448 
  449     def _filter_message(self, msg):
  450         self.log.trace()
  451         self._prepare_child()
  452         if msg.recipient == None or msg.sender == None:
  453             raise getmailConfigurationError(
  454                 'TMDA requires the message envelope and therefore a multidrop '
  455                 'retriever'
  456             )
  457 
  458         # At least some security...
  459         if (os.geteuid() == 0 and not self.conf['allow_root_commands']
  460                 and self.conf['user'] == None):
  461             raise getmailConfigurationError(
  462                 'refuse to invoke external commands as root by default'
  463             )
  464 
  465         stdout = tempfile.TemporaryFile()
  466         stderr = tempfile.TemporaryFile()
  467         childpid = os.fork()
  468 
  469         if not childpid:
  470             # Child
  471             self._filter_command(msg, stdout, stderr)
  472         self.log.debug('spawned child %d\n' % childpid)
  473 
  474         # Parent
  475         exitcode = self._wait_for_child(childpid)
  476 
  477         stderr.seek(0)
  478         err = stderr.read().strip()
  479 
  480         self.log.debug('command %s %d exited %d\n' % (self.conf['command'],
  481                                                       childpid, exitcode))
  482 
  483         return (exitcode, msg, err)