"Fossies" - the Fresh Open Source Software Archive

Member "getmail-5.16/getmailcore/_retrieverbases.py" (31 Oct 2021, 76421 Bytes) of package /linux/misc/getmail-5.16.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "_retrieverbases.py" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 5.15_vs_5.16.

    1 #!/usr/bin/env python2
    2 '''Base and mix-in classes implementing retrievers (message sources getmail can
    3 retrieve mail from).
    4 
    5 None of these classes can be instantiated directly.  In this module:
    6 
    7 Mix-in classes for SSL/non-SSL initialization:
    8 
    9   POP3initMixIn
   10   Py23POP3SSLinitMixIn
   11   Py24POP3SSLinitMixIn
   12   IMAPinitMixIn
   13   IMAPSSLinitMixIn
   14 
   15 Base classes:
   16 
   17   RetrieverSkeleton
   18   POP3RetrieverBase
   19   MultidropPOP3RetrieverBase
   20   IMAPRetrieverBase
   21   MultidropIMAPRetrieverBase
   22 '''
   23 
   24 __all__ = [
   25     'IMAPinitMixIn',
   26     'IMAPRetrieverBase',
   27     'IMAPSSLinitMixIn',
   28     'MultidropPOP3RetrieverBase',
   29     'MultidropIMAPRetrieverBase',
   30     'POP3_ssl_port',
   31     'POP3initMixIn',
   32     'POP3RetrieverBase',
   33     'POP3SSLinitMixIn',
   34     'RetrieverSkeleton',
   35 ]
   36 
   37 import sys
   38 import os
   39 import socket
   40 import time
   41 import email
   42 import poplib
   43 import imaplib
   44 import re
   45 import select
   46 
   47 try:
   48     # do we have a recent pykerberos?
   49     HAVE_KERBEROS_GSS = False
   50     import kerberos
   51     if 'authGSSClientWrap' in dir(kerberos):
   52         HAVE_KERBEROS_GSS = True
   53 except ImportError:
   54     pass
   55 
   56 # hashlib only present in python2.5, ssl in python2.6; used together
   57 # in SSL functionality below
   58 try:
   59     import ssl
   60 except ImportError:
   61     ssl = None
   62 try:
   63     import hashlib
   64 except ImportError:
   65     hashlib = None
   66 
   67 # If we have an ssl module:
   68 if ssl:
   69     has_sni = getattr(ssl, 'HAS_SNI', False)
   70     proto_best = getattr(ssl, 'PROTOCOL_TLS', None)
   71     if not proto_best:
   72         proto_best = getattr(ssl, 'PROTOCOL_SSLv23', None)
   73     has_ciphers = sys.hexversion >= 0x2070000
   74 
   75     # Monkey-patch SNI use into SSL.wrap_socket() if supported
   76     if has_sni:
   77         def _wrap_socket(sock, keyfile=None, certfile=None,
   78                          server_side=False, cert_reqs=ssl.CERT_NONE,
   79                          ssl_version=proto_best, ca_certs=None,
   80                          do_handshake_on_connect=True,
   81                          suppress_ragged_eofs=True,
   82                          ciphers=None, server_hostname=None):
   83             kwargs = dict(sock=sock, keyfile=keyfile, certfile=certfile,
   84                           server_side=server_side, cert_reqs=cert_reqs,
   85                           ssl_version=ssl_version, ca_certs=ca_certs,
   86                           do_handshake_on_connect=do_handshake_on_connect,
   87                           suppress_ragged_eofs=suppress_ragged_eofs,
   88                           ciphers=ciphers, server_hostname=server_hostname)
   89             if not has_ciphers:
   90                 kwargs.pop('ciphers', None)
   91             return ssl.SSLSocket(**kwargs)
   92     else:
   93         # no SNI support
   94         def _wrap_socket(sock, keyfile=None, certfile=None,
   95                          server_side=False, cert_reqs=ssl.CERT_NONE,
   96                          ssl_version=proto_best, ca_certs=None,
   97                          do_handshake_on_connect=True,
   98                          suppress_ragged_eofs=True,
   99                          ciphers=None, server_hostname=None):
  100             kwargs = dict(sock=sock, keyfile=keyfile, certfile=certfile,
  101                           server_side=server_side, cert_reqs=cert_reqs,
  102                           ssl_version=ssl_version, ca_certs=ca_certs,
  103                           do_handshake_on_connect=do_handshake_on_connect,
  104                           suppress_ragged_eofs=suppress_ragged_eofs,
  105                           ciphers=ciphers)
  106             if not has_ciphers:
  107                 kwargs.pop('ciphers', None)
  108             return ssl.SSLSocket(**kwargs)
  109     ssl.wrap_socket = _wrap_socket
  110 
  111     # Is it recent enough to have hostname matching (Python 3.2+)?
  112     try:
  113         ssl_match_hostname = ssl.match_hostname
  114     except AttributeError:
  115     # Running a Python with no hostname matching
  116         def _dnsname_match(dn, hostname, max_wildcards=1):
  117             """Matching according to RFC 6125, section 6.4.3
  118             http://tools.ietf.org/html/rfc6125#section-6.4.3
  119             """
  120             pats = []
  121             if not dn:
  122                 return False
  123         
  124             parts = dn.split(r'.')
  125             leftmost = parts[0]
  126             remainder = parts[1:]
  127         
  128             wildcards = leftmost.count('*')
  129             if wildcards > max_wildcards:
  130                 # Issue #17980: avoid denials of service by refusing more
  131                 # than one wildcard per fragment.  A survery of established
  132                 # policy among SSL implementations showed it to be a
  133                 # reasonable choice.
  134                 raise getmailOperationError(
  135                     "too many wildcards in certificate DNS name: " + repr(dn))
  136         
  137             # speed up common case w/o wildcards
  138             if not wildcards:
  139                 return dn.lower() == hostname.lower()
  140         
  141             # RFC 6125, section 6.4.3, subitem 1.
  142             # The client SHOULD NOT attempt to match a presented identifier
  143             # in which the wildcard character comprises a label other than
  144             # the left-most label.
  145             if leftmost == '*':
  146                 # When '*' is a fragment by itself, it matches a non-empty
  147                 # dotless fragment.
  148                 pats.append('[^.]+')
  149             elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
  150                 # RFC 6125, section 6.4.3, subitem 3.
  151                 # The client SHOULD NOT attempt to match a presented identifier
  152                 # where the wildcard character is embedded within an A-label or
  153                 # U-label of an internationalized domain name.
  154                 pats.append(re.escape(leftmost))
  155             else:
  156                 # Otherwise, '*' matches any dotless string, e.g. www*
  157                 pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
  158         
  159             # add the remaining fragments, ignore any wildcards
  160             for frag in remainder:
  161                 pats.append(re.escape(frag))
  162         
  163             pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
  164             return pat.match(hostname)
  165         
  166         
  167         def ssl_match_hostname(cert, hostname):
  168             """Verify that *cert* (in decoded format as returned by
  169             SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and
  170             RFC 6125 rules are followed, but IP addresses are not accepted
  171             for *hostname*.
  172         
  173             getmailOperationError is raised on failure. On success, the function
  174             returns nothing.
  175             """
  176             if not cert:
  177                 raise ValueError("empty or no certificate, ssl_match_hostname "
  178                                  "needs an SSL socket or SSL context with "
  179                                  "either CERT_OPTIONAL or CERT_REQUIRED")
  180             dnsnames = []
  181             san = cert.get('subjectAltName', ())
  182             for key, value in san:
  183                 if key == 'DNS':
  184                     if _dnsname_match(value, hostname):
  185                         return
  186                     dnsnames.append(value)
  187             if not dnsnames:
  188                 # The subject is only checked when there is no dNSName entry
  189                 # in subjectAltName
  190                 for sub in cert.get('subject', ()):
  191                     for key, value in sub:
  192                         # XXX according to RFC 2818, the most specific
  193                         # Common Name must be used.
  194                         if key == 'commonName':
  195                             if _dnsname_match(value, hostname):
  196                                 return
  197                             dnsnames.append(value)
  198             if len(dnsnames) > 1:
  199                 raise getmailOperationError("hostname %s "
  200                     "doesn't match either of %s"
  201                     % (hostname, ', '.join(map(repr, dnsnames))))
  202             elif len(dnsnames) == 1:
  203                 raise getmailOperationError("hostname %s "
  204                     "doesn't match %s"
  205                     % (hostname, dnsnames[0]))
  206             else:
  207                 raise getmailOperationError("no appropriate commonName or "
  208                     "subjectAltName fields were found")
  209         
  210 try:
  211     from email.header import decode_header
  212 except ImportError, o:
  213     # python < 2.5
  214     from email.Header import decode_header
  215 
  216 from getmailcore.compatibility import *
  217 from getmailcore.exceptions import *
  218 from getmailcore.constants import *
  219 from getmailcore.message import *
  220 from getmailcore.utilities import *
  221 from getmailcore._pop3ssl import POP3SSL, POP3_ssl_port
  222 from getmailcore.baseclasses import *
  223 import getmailcore.imap_utf7        # registers imap4-utf-7 codec
  224 
  225 
  226 NOT_ENVELOPE_RECIPIENT_HEADERS = (
  227     'to',
  228     'cc',
  229     'bcc',
  230     'received',
  231     'resent-to',
  232     'resent-cc',
  233     'resent-bcc'
  234 )
  235 
  236 # How long a vanished message is kept in the oldmail state file for IMAP
  237 # retrievers before we figure it's gone for good.  This is to allow users
  238 # to only occasionally retrieve mail from certain IMAP folders without
  239 # losing their oldmail state for that folder.  This is in seconds, so it's
  240 # 30 days.
  241 VANISHED_AGE = (60 * 60 * 24 * 30)
  242 
  243 # Regex used to remove problematic characters from oldmail filenames
  244 STRIP_CHAR_RE = r'[/\:;<>|]+'
  245 
  246 # Kerberos authentication state constants
  247 (GSS_STATE_STEP, GSS_STATE_WRAP) = (0, 1)
  248 
  249 # For matching imap LIST responses
  250 IMAP_LISTPARTS = re.compile(
  251     r'^\s*'
  252     r'\((?P<attributes>[^)]*)\)'
  253     r'\s+'
  254     r'"(?P<delimiter>[^"]+)"'
  255     r'\s+'
  256     # I *think* this should actually be a double-quoted string "like/this"
  257     # but in testing we saw an MSexChange response that violated that
  258     # expectation:
  259     #   (\HasNoChildren) "/" Calendar"
  260     # i.e. the leading quote on the mailbox name was missing.  The following
  261     # works for both by treating the leading/trailing double-quote as optional,
  262     # even when mismatched.
  263     r'("?)(?P<mailbox>.+?)("?)'
  264     r'\s*$'
  265 )
  266 
  267 
  268 # Constants used in socket module
  269 NO_OBJ = object()
  270 EAI_NONAME = getattr(socket, 'EAI_NONAME', NO_OBJ)
  271 EAI_NODATA = getattr(socket, 'EAI_NODATA', NO_OBJ)
  272 EAI_FAIL = getattr(socket, 'EAI_FAIL', NO_OBJ)
  273 
  274 
  275 # Constant for POPSSL
  276 POP3_SSL_PORT = 995
  277 
  278 
  279 # Python added poplib._MAXLINE somewhere along the way.  As far as I can
  280 # see, it serves no purpose except to introduce bugs into any software
  281 # using poplib.  Any computer running Python will have at least some megabytes
  282 # of userspace memory; arbitrarily causing message retrieval to break if any
  283 # "line" exceeds 2048 bytes is absolutely stupid.
  284 poplib._MAXLINE = 1 << 20   # 1MB; decrease this if you're running on a VIC-20
  285 
  286 
  287 #
  288 # Mix-in classes
  289 #
  290 
  291 #######################################
  292 class POP3initMixIn(object):
  293     '''Mix-In class to do POP3 non-SSL initialization.
  294     '''
  295     SSL = False
  296     def _connect(self):
  297         self.log.trace()
  298         try:
  299             self.conn = poplib.POP3(self.conf['server'], self.conf['port'])
  300             self.setup_received(self.conn.sock)
  301         except poplib.error_proto, o:
  302             raise getmailOperationError('POP error (%s)' % o)
  303         except socket.timeout:
  304             raise
  305             #raise getmailOperationError('timeout during connect')
  306         except socket.gaierror, o:
  307             raise getmailOperationError(
  308                 'error resolving name %s during connect (%s)'
  309                 % (self.conf['server'], o)
  310             )
  311 
  312         self.log.trace('POP3 connection %s established' % self.conn
  313                        + os.linesep)
  314 
  315 
  316 #######################################
  317 class POP3_SSL_EXTENDED(poplib.POP3_SSL):
  318     # Extended SSL support for POP3 (certificate checking, 
  319     # fingerprint matching, cipher selection, etc.)
  320 
  321     def __init__(self, host, port=POP3_SSL_PORT, keyfile=None,
  322                  certfile=None, ssl_version=None, ca_certs=None,
  323                  ssl_ciphers=None):
  324         self.host = host
  325         self.port = port
  326         self.keyfile = keyfile
  327         self.certfile = certfile
  328         self.ssl_version = ssl_version
  329         self.ca_certs = ca_certs
  330         self.ssl_ciphers = ssl_ciphers
  331 
  332         self.buffer = ''
  333         msg = "getaddrinfo returns an empty list"
  334         self.sock = None
  335         for res in socket.getaddrinfo(self.host, self.port, 0,
  336                                       socket.SOCK_STREAM):
  337             (af, socktype, proto, canonname, sa) = res
  338             try:
  339                 self.sock = socket.socket(af, socktype, proto)
  340                 self.sock.connect(sa)
  341             except socket.error, msg:
  342                 if self.sock:
  343                     self.sock.close()
  344                 self.sock = None
  345                 continue
  346             break
  347         if not self.sock:
  348             raise socket.error(msg)
  349         extra_args = { 'server_hostname': host }
  350         if self.ssl_version:
  351             extra_args['ssl_version'] = self.ssl_version
  352         if self.ca_certs:
  353             extra_args['cert_reqs'] = ssl.CERT_REQUIRED
  354             extra_args['ca_certs'] = self.ca_certs
  355         if self.ssl_ciphers:
  356             extra_args['ciphers'] = self.ssl_ciphers
  357 
  358         self.file = self.sock.makefile('rb')
  359         self.sslobj = ssl.wrap_socket(self.sock, self.keyfile,
  360                                       self.certfile, **extra_args)
  361         self._debugging = 0
  362         self.welcome = self._getresp()
  363 
  364 
  365 #######################################
  366 class Py24POP3SSLinitMixIn(object):
  367     '''Mix-In class to do POP3 over SSL initialization with Python 2.4's
  368     poplib.POP3_SSL class.
  369     '''
  370     SSL = True
  371     def _connect(self):
  372         self.log.trace()
  373         if not hasattr(socket, 'ssl'):
  374             raise getmailConfigurationError(
  375                 'SSL not supported by this installation of Python'
  376             )
  377         (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
  378         ca_certs = check_ca_certs(self.conf)
  379         ssl_version = check_ssl_version(self.conf)
  380         ssl_fingerprints = check_ssl_fingerprints(self.conf)
  381         ssl_ciphers = check_ssl_ciphers(self.conf)
  382         using_extended_certs_interface = False
  383         try:
  384             if ca_certs or ssl_version or ssl_ciphers:
  385                 using_extended_certs_interface = True
  386                 # Python 2.6 or higher required, use above class instead of
  387                 # vanilla stdlib one
  388                 msg = ''
  389                 if keyfile:
  390                     msg += 'with keyfile %s, certfile %s' % (keyfile, certfile)
  391                 if ssl_version:
  392                     if msg:
  393                         msg += ', '
  394                     msg += ('using protocol version %s'
  395                             % self.conf['ssl_version'].upper())
  396                 if ca_certs:
  397                     if msg:
  398                         msg += ', '
  399                     msg += 'with ca_certs %s' % ca_certs
  400 
  401                 self.log.trace(
  402                     'establishing POP3 SSL connection to %s:%d %s'
  403                     % (self.conf['server'], self.conf['port'], msg)
  404                     + os.linesep
  405                 )
  406                 self.conn = POP3_SSL_EXTENDED(
  407                     self.conf['server'], self.conf['port'], keyfile, certfile,
  408                     ssl_version, ca_certs, ssl_ciphers
  409                 )
  410             elif keyfile:
  411                 self.log.trace(
  412                     'establishing POP3 SSL connection to %s:%d with '
  413                     'keyfile %s, certfile %s'
  414                     % (self.conf['server'], self.conf['port'], keyfile,
  415                        certfile)
  416                     + os.linesep
  417                 )
  418                 self.conn = poplib.POP3_SSL(
  419                     self.conf['server'], self.conf['port'], keyfile, certfile
  420                 )
  421             else:
  422                 self.log.trace('establishing POP3 SSL connection to %s:%d'
  423                                % (self.conf['server'], self.conf['port'])
  424                                + os.linesep)
  425                 self.conn = poplib.POP3_SSL(self.conf['server'],
  426                                             self.conf['port'])
  427             self.setup_received(self.conn.sock)
  428             if ssl and hashlib:
  429                 sslobj = self.conn.sslobj
  430                 peercert = sslobj.getpeercert(True)
  431                 ssl_cipher = sslobj.cipher()
  432                 if ssl_cipher:
  433                     ssl_cipher = '%s:%s:%s' % ssl_cipher
  434                 if not peercert:
  435                     actual_hash = None
  436                 else:
  437                     actual_hash = hashlib.sha256(peercert).hexdigest().lower()
  438             else:
  439                 actual_hash = None
  440                 ssl_cipher = None
  441 
  442             # Ensure cert is for server we're connecting to
  443             if ssl and self.conf['ca_certs']:
  444                 ssl_match_hostname(
  445                     self.conn.sslobj.getpeercert(),
  446                     self.conf.get('ssl_cert_hostname', None) 
  447                         or self.conf['server']
  448                 )
  449 
  450             if ssl_fingerprints:
  451                 if not actual_hash:
  452                     raise getmailOperationError(
  453                         'socket ssl_fingerprints mismatch (no cert provided)'
  454                     )
  455 
  456                 any_matches = False
  457                 for expected_hash in ssl_fingerprints:
  458                     if expected_hash == actual_hash:
  459                         any_matches = True
  460                 if not any_matches:
  461                     raise getmailOperationError(
  462                         'socket ssl_fingerprints mismatch (got %s)'
  463                         % actual_hash
  464                     )
  465 
  466         except poplib.error_proto, o:
  467             raise getmailOperationError('POP error (%s)' % o)
  468         except socket.timeout:
  469             #raise getmailOperationError('timeout during connect')
  470             raise
  471         except socket.gaierror, o:
  472             raise getmailOperationError(
  473                 'error resolving name %s during connect (%s)'
  474                 % (self.conf['server'], o)
  475             )
  476 
  477         #self.conn.sock.setblocking(1)
  478 
  479         fingerprint_message = ('POP3 SSL connection %s established'
  480                                % self.conn)
  481         if actual_hash:
  482             fingerprint_message += ' with fingerprint %s' % actual_hash
  483         if ssl_cipher:
  484             fingerprint_message += ' using cipher %s' % ssl_cipher
  485         fingerprint_message += os.linesep
  486 
  487         if self.app_options.get('fingerprint', False):
  488             self.log.info(fingerprint_message)
  489         else:
  490             self.log.trace(fingerprint_message)
  491 
  492 
  493 #######################################
  494 class Py23POP3SSLinitMixIn(object):
  495     '''Mix-In class to do POP3 over SSL initialization with custom-implemented
  496     code to support SSL with Python 2.3's poplib.POP3 class.
  497     '''
  498     SSL = True
  499     def _connect(self):
  500         self.log.trace()
  501         if not hasattr(socket, 'ssl'):
  502             raise getmailConfigurationError(
  503                 'SSL not supported by this installation of Python'
  504             )
  505         (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
  506         ca_certs = check_ca_certs(self.conf)
  507         ssl_version = check_ssl_version(self.conf)
  508         ssl_fingerprints = check_ssl_fingerprints(self.conf)
  509         ssl_ciphers = check_ssl_ciphers(self.conf)
  510         if ca_certs or ssl_version or ssl_ciphers or ssl_fingerprints:
  511             raise getmailConfigurationError(
  512                 'SSL extended options are not supported by this'
  513                  ' installation of Python'
  514             )
  515         try:
  516             if keyfile:
  517                 self.log.trace(
  518                     'establishing POP3 SSL connection to %s:%d with keyfile '
  519                     '%s, certfile %s'
  520                     % (self.conf['server'], self.conf['port'],
  521                        keyfile, certfile)
  522                     + os.linesep
  523                 )
  524                 self.conn = POP3SSL(self.conf['server'], self.conf['port'],
  525                                     keyfile, certfile)
  526             else:
  527                 self.log.trace(
  528                     'establishing POP3 SSL connection to %s:%d'
  529                     % (self.conf['server'], self.conf['port'])
  530                     + os.linesep
  531                 )
  532                 self.conn = POP3SSL(self.conf['server'], self.conf['port'])
  533 
  534             self.setup_received(self.conn.rawsock)
  535         except poplib.error_proto, o:
  536             raise getmailOperationError('POP error (%s)' % o)
  537         except socket.timeout:
  538             #raise getmailOperationError('timeout during connect')
  539             raise
  540         except socket.gaierror, o:
  541             raise getmailOperationError('socket error during connect (%s)' % o)
  542         except socket.sslerror, o:
  543             raise getmailOperationError(
  544                 'socket sslerror during connect (%s)' % o
  545             )
  546 
  547         self.log.trace('POP3 SSL connection %s established' % self.conn
  548                        + os.linesep)
  549 
  550 
  551 #######################################
  552 class IMAPinitMixIn(object):
  553     '''Mix-In class to do IMAP non-SSL initialization.
  554     '''
  555     SSL = False
  556     def _connect(self):
  557         self.log.trace()
  558         try:
  559             self.conn = imaplib.IMAP4(self.conf['server'], self.conf['port'])
  560             self.setup_received(self.conn.sock)
  561         except imaplib.IMAP4.error, o:
  562             raise getmailOperationError('IMAP error (%s)' % o)
  563         except socket.timeout:
  564             #raise getmailOperationError('timeout during connect')
  565             raise
  566         except socket.gaierror, o:
  567             raise getmailOperationError('socket error during connect (%s)' % o)
  568 
  569         self.log.trace('IMAP connection %s established' % self.conn
  570                        + os.linesep)
  571 
  572 
  573 #######################################
  574 class IMAP4_SSL_EXTENDED(imaplib.IMAP4_SSL):
  575     # Similar to above, but with extended support for SSL certificate checking,
  576     # fingerprints, etc.
  577     def __init__(self, host='', port=imaplib.IMAP4_SSL_PORT, keyfile=None, 
  578                  certfile=None, ssl_version=None, ca_certs=None, 
  579                  ssl_ciphers=None):
  580        self.ssl_version = ssl_version
  581        self.ca_certs = ca_certs
  582        self.ssl_ciphers = ssl_ciphers
  583        imaplib.IMAP4_SSL.__init__(self, host, port, keyfile, certfile)
  584 
  585     def open(self, host='', port=imaplib.IMAP4_SSL_PORT):
  586        self.host = host
  587        self.port = port
  588        self.sock = socket.create_connection((host, port))
  589        extra_args = { 'server_hostname': host }
  590        if self.ssl_version:
  591            extra_args['ssl_version'] = self.ssl_version
  592        if self.ca_certs:
  593            extra_args['cert_reqs'] = ssl.CERT_REQUIRED
  594            extra_args['ca_certs'] = self.ca_certs
  595        if self.ssl_ciphers:
  596            extra_args['ciphers'] = self.ssl_ciphers
  597 
  598        self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, 
  599                                      **extra_args)
  600        self.file = self.sslobj.makefile('rb')
  601 
  602 
  603 #######################################
  604 class IMAPSSLinitMixIn(object):
  605     '''Mix-In class to do IMAP over SSL initialization.
  606     '''
  607     SSL = True
  608     def _connect(self):
  609         self.log.trace()
  610         if not hasattr(socket, 'ssl'):
  611             raise getmailConfigurationError(
  612                 'SSL not supported by this installation of Python'
  613             )
  614         (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
  615         ca_certs = check_ca_certs(self.conf)
  616         ssl_version = check_ssl_version(self.conf)
  617         ssl_fingerprints = check_ssl_fingerprints(self.conf)
  618         ssl_ciphers = check_ssl_ciphers(self.conf)
  619         using_extended_certs_interface = False
  620         try:
  621             if ca_certs or ssl_version or ssl_ciphers:
  622                 using_extended_certs_interface = True
  623                 # Python 2.6 or higher required, use above class instead of
  624                 # vanilla stdlib one
  625                 msg = ''
  626                 if keyfile:
  627                     msg += 'with keyfile %s, certfile %s' % (keyfile, certfile)
  628                 if ssl_version:
  629                     if msg:
  630                         msg += ', '
  631                     msg += ('using protocol version %s' 
  632                             % self.conf['ssl_version'].upper())
  633                 if ca_certs:
  634                     if msg:
  635                         msg += ', '
  636                     msg += 'with ca_certs %s' % ca_certs
  637 
  638                 self.log.trace(
  639                     'establishing IMAP SSL connection to %s:%d %s'
  640                     % (self.conf['server'], self.conf['port'], msg)
  641                     + os.linesep
  642                 )
  643                 self.conn = IMAP4_SSL_EXTENDED(
  644                     self.conf['server'], self.conf['port'], keyfile, certfile, 
  645                     ssl_version, ca_certs, ssl_ciphers
  646                 )
  647             elif keyfile:
  648                 self.log.trace(
  649                     'establishing IMAP SSL connection to %s:%d with keyfile '
  650                     '%s, certfile %s'
  651                     % (self.conf['server'], self.conf['port'],
  652                        keyfile, certfile)
  653                     + os.linesep
  654                 )
  655                 self.conn = imaplib.IMAP4_SSL(
  656                     self.conf['server'], self.conf['port'], keyfile, certfile
  657                 )
  658             else:
  659                 self.log.trace(
  660                     'establishing IMAP SSL connection to %s:%d'
  661                     % (self.conf['server'], self.conf['port']) + os.linesep
  662                 )
  663                 self.conn = imaplib.IMAP4_SSL(self.conf['server'],
  664                                               self.conf['port'])
  665             self.setup_received(self.conn.sock)
  666             if ssl and hashlib:
  667                 sslobj = self.conn.ssl()
  668                 peercert = sslobj.getpeercert(True)
  669                 ssl_cipher = sslobj.cipher()
  670                 if ssl_cipher:
  671                     ssl_cipher = '%s:%s:%s' % ssl_cipher
  672                 if not peercert:
  673                     actual_hash = None
  674                 else:
  675                     actual_hash = hashlib.sha256(peercert).hexdigest().lower()
  676             else:
  677                 actual_hash = None
  678                 ssl_cipher = None
  679 
  680             # Ensure cert is for server we're connecting to
  681             if ssl and self.conf['ca_certs']:
  682                 ssl_match_hostname(
  683                     self.conn.ssl().getpeercert(),
  684                     self.conf.get('ssl_cert_hostname', None) 
  685                         or self.conf['server']
  686                 )
  687 
  688             if ssl_fingerprints:
  689                 if not actual_hash:
  690                     raise getmailOperationError(
  691                         'socket ssl_fingerprints mismatch (no cert provided)'
  692                     )
  693 
  694                 any_matches = False
  695                 for expected_hash in ssl_fingerprints:
  696                     if expected_hash == actual_hash:
  697                         any_matches = True
  698                 if not any_matches:
  699                     raise getmailOperationError(
  700                         'socket ssl_fingerprints mismatch (got %s)' 
  701                         % actual_hash
  702                     )
  703 
  704         except imaplib.IMAP4.error, o:
  705             raise getmailOperationError('IMAP error (%s)' % o)
  706         except socket.timeout:
  707             #raise getmailOperationError('timeout during connect')
  708             raise
  709         except socket.gaierror, o:
  710             errcode = o[0]
  711             if errcode in (EAI_NONAME, EAI_NODATA):
  712                 # No such DNS name
  713                 raise getmailDnsLookupError(
  714                     'no address for %s (%s)' % (self.conf['server'], o)
  715                 )
  716             elif errcode == EAI_FAIL:
  717                 # DNS server failure
  718                 raise getmailDnsServerFailure(
  719                     'DNS server failure looking up address for %s (%s)' 
  720                     % (self.conf['server'], o)
  721                 )
  722             else:
  723                 raise getmailOperationError('socket error during connect (%s)' 
  724                                             % o)
  725         except socket.sslerror, o:
  726             raise getmailOperationError(
  727                 'socket sslerror during connect (%s)' % o
  728             )
  729 
  730         fingerprint_message = ('IMAP SSL connection %s established'
  731                                % self.conn)
  732         if actual_hash:
  733             fingerprint_message += ' with fingerprint %s' % actual_hash
  734         if ssl_cipher:
  735             fingerprint_message += ' using cipher %s' % ssl_cipher
  736         fingerprint_message += os.linesep
  737 
  738         if self.app_options['fingerprint']:
  739             self.log.info(fingerprint_message)
  740         else:
  741             self.log.trace(fingerprint_message)
  742 
  743 #
  744 # Base classes
  745 #
  746 
  747 #######################################
  748 class RetrieverSkeleton(ConfigurableBase):
  749     '''Base class for implementing message-retrieval classes.
  750 
  751     Sub-classes should provide the following data attributes and methods:
  752 
  753       _confitems - a tuple of dictionaries representing the parameters the class
  754                    takes.  Each dictionary should contain the following key,
  755                    value pairs:
  756                      - name - parameter name
  757                      - type - a type function to compare the parameter value
  758                      against (i.e. str, int, bool)
  759                      - default - optional default value.  If not present, the
  760                      parameter is required.
  761 
  762       __str__(self) - return a simple string representing the class instance.
  763 
  764       _getmsglist(self) - retieve a list of all available messages, and store
  765                           unique message identifiers in the dict
  766                           self.msgnum_by_msgid.
  767                           Message identifiers must be unique and persistent
  768                           across instantiations.  Also store message sizes (in
  769                           octets) in a dictionary self.msgsizes, using the
  770                           message identifiers as keys.
  771 
  772       _delmsgbyid(self, msgid) - delete a message from the message store based
  773                                  on its message identifier.
  774 
  775       _getmsgbyid(self, msgid) - retreive and return a message from the message
  776                                  store based on its message identifier.  The
  777                                  message is returned as a Message() class
  778                                  object. The message will have additional data
  779                                  attributes "sender" and "recipient".  sender
  780                                  should be present or "unknown".  recipient
  781                                  should be non-None if (and only if) the
  782                                  protocol/method of message retrieval preserves
  783                                  the original message envelope.
  784 
  785       _getheaderbyid(self, msgid) - similar to _getmsgbyid() above, but only the
  786                                  message header should be retrieved, if
  787                                  possible.  It should be returned in the same
  788                                  format.
  789 
  790       showconf(self) - should invoke self.log.info() to display the
  791                                 configuration of the class instance.
  792 
  793     Sub-classes may also wish to extend or over-ride the following base class
  794     methods:
  795 
  796       __init__(self, **args)
  797       __del__(self)
  798       initialize(self, options)
  799       checkconf(self)
  800     '''
  801     def __init__(self, **args):
  802         self.headercache = {}
  803         self.deleted = {}
  804         self.set_new_timestamp()
  805         self.__oldmail_written = False
  806         self.__initialized = False
  807         self.gotmsglist = False
  808         self._clear_state()
  809         self.conn = None
  810         self.supports_idle = False
  811         ConfigurableBase.__init__(self, **args)
  812 
  813     def set_new_timestamp(self):
  814         self.timestamp = int(time.time())
  815 
  816     def _clear_state(self):
  817         self.msgnum_by_msgid = {}
  818         self.msgid_by_msgnum = {}
  819         self.sorted_msgnum_msgid = ()
  820         self.msgsizes = {}
  821         self.oldmail = {}
  822         self.__delivered = {}
  823         self.deleted = {}
  824         self.mailbox_selected = False
  825         
  826     def setup_received(self, sock):
  827         serveraddr = sock.getpeername()
  828         if len(serveraddr) == 2:
  829             # IPv4
  830             self.remoteaddr = '%s:%s' % serveraddr
  831         elif len(serveraddr) == 4:
  832             # IPv6
  833             self.remoteaddr = '[%s]:%s' % serveraddr[:2]
  834         else:
  835             # Shouldn't happen
  836             self.log.warning('unexpected peer address format %s' % str(serveraddr))
  837             self.remoteaddr = str(serveraddr)
  838         self.received_from = '%s (%s)' % (self.conf['server'], 
  839                                           self.remoteaddr)
  840 
  841     def __str__(self):
  842         self.log.trace()
  843         return str(self.conf)
  844 
  845     def list_mailboxes(self):
  846         raise NotImplementedError('virtual')
  847 
  848     def select_mailbox(self, mailbox):
  849         raise NotImplementedError('virtual')
  850 
  851     def __len__(self):
  852         self.log.trace()
  853         return len(self.msgnum_by_msgid)
  854 
  855     def __getitem__(self, i):
  856         self.log.trace('i == %d' % i)
  857         if not self.__initialized:
  858             raise getmailOperationError('not initialized')
  859         return self.sorted_msgnum_msgid[i][1]
  860 
  861     def _oldmail_filename(self, mailbox):
  862         assert (mailbox is None 
  863                 or (isinstance(mailbox, (str, unicode)) and mailbox)), (
  864             'bad mailbox %s (%s)' % (mailbox, type(mailbox))
  865         )
  866         filename = self.oldmail_filename
  867         if mailbox is not None:
  868             if isinstance(mailbox, str):
  869                 mailbox = mailbox.decode('utf-8')
  870             mailbox = re.sub(STRIP_CHAR_RE, '.', mailbox)
  871             mailbox = mailbox.encode('utf-8')
  872             # Use oldmail file per IMAP folder
  873             filename += '-' + mailbox
  874         # else:
  875             # mailbox is None, is POP, just use filename
  876         return filename
  877 
  878     def oldmail_exists(self, mailbox):
  879         '''Test whether an oldmail file exists for a specified mailbox.'''
  880         return os.path.isfile(self._oldmail_filename(mailbox))
  881 
  882     def read_oldmailfile(self, mailbox):
  883         '''Read contents of an oldmail file.  For POP, mailbox must be 
  884         explicitly None.
  885         '''
  886         assert not self.oldmail, (
  887             'still have %d unflushed oldmail' % len(self.oldmail)
  888         )
  889         self.log.trace('mailbox=%s' % mailbox)
  890         
  891         filename = self._oldmail_filename(mailbox)
  892         logname = '%s:%s' % (self, mailbox or '')
  893         try:
  894             f = open(filename, 'rb')
  895         except IOError:
  896             self.log.moreinfo('no oldmail file for %s%s'
  897                               % (logname, os.linesep))
  898             return
  899             
  900         for line in f:
  901             line = line.strip()
  902             if not line or not '\0' in line:
  903                 # malformed
  904                 continue
  905             try:
  906                 (msgid, timestamp) = line.split('\0', 1)
  907                 if msgid.count('/') == 2:
  908                     # Was pre-4.22.0 file format, which includes the
  909                     # mailbox name in the msgid, in the format
  910                     # 'uidvalidity/mailbox/serveruid'.
  911                     # Strip it out.
  912                     fields = msgid.split('/')
  913                     msgid = '/'.join([fields[0], fields[2]])
  914                 self.oldmail[msgid] = int(timestamp)
  915             except ValueError:
  916                 # malformed
  917                 self.log.info(
  918                     'skipped malformed line "%r" for %s%s'
  919                     % (line, logname, os.linesep)
  920                 )
  921         self.log.moreinfo(
  922             'read %i uids for %s%s'
  923             % (len(self.oldmail), logname, os.linesep)
  924         )
  925         self.log.moreinfo('read %i uids in total for %s%s'
  926                           % (len(self.oldmail), logname, os.linesep))
  927 
  928     def write_oldmailfile(self, mailbox):
  929         '''Write oldmail info to oldmail file.'''
  930         self.log.trace('mailbox=%s' % mailbox)
  931         
  932         filename = self._oldmail_filename(mailbox)
  933         logname = '%s:%s' % (self, mailbox or '')
  934         
  935         oldmailfile = None
  936         wrote = 0
  937         msgids = frozenset(
  938             self.__delivered.keys()
  939         ).union(frozenset(self.oldmail.keys()))
  940         try:
  941             oldmailfile = updatefile(filename)
  942             for msgid in msgids:
  943                 self.log.debug('msgid %s ...' % msgid)
  944                 t = self.oldmail.get(msgid, self.timestamp)
  945                 self.log.debug(' timestamp %s' % t + os.linesep)
  946                 oldmailfile.write('%s\0%i%s' % (msgid, t, os.linesep))
  947                 wrote += 1
  948             oldmailfile.close()
  949             self.log.moreinfo('wrote %i uids for %s%s'
  950                               % (wrote, logname, os.linesep))
  951         except IOError, o:
  952             self.log.error('failed writing oldmail file for %s (%s)'
  953                            % (logname, o) + os.linesep)
  954             if oldmailfile:
  955                 oldmailfile.abort()
  956         self.__oldmail_written = True
  957 
  958     def initialize(self, options):
  959         # Options - dict of application-wide settings, including ones that 
  960         # aren't used in initializing the retriever.
  961         self.log.trace()
  962         self.checkconf()
  963         # socket.ssl() and socket timeouts are incompatible in Python 2.3
  964         if 'timeout' in self.conf:
  965             socket.setdefaulttimeout(self.conf['timeout'])
  966         else:
  967             # Explicitly set to None in case it was previously set
  968             socket.setdefaulttimeout(None)
  969 
  970         # Construct base filename for oldmail files.
  971         # strip problematic characters from oldmail filename.  Mostly for
  972         # non-Unix systems; only / is illegal in a Unix path component
  973         oldmail_filename = re.sub(
  974             STRIP_CHAR_RE, '-',
  975             'oldmail-%(server)s-%(port)i-%(username)s' % self.conf
  976         )
  977         self.oldmail_filename = os.path.join(self.conf['getmaildir'], 
  978                                              oldmail_filename)
  979 
  980         self.received_from = None
  981         self.app_options = options
  982         self.__initialized = True
  983 
  984     def quit(self):
  985         if self.mailbox_selected is not False:
  986             self.write_oldmailfile(self.mailbox_selected)
  987         self._clear_state()
  988 
  989     def abort(self):
  990         '''On error conditions where you do not want modified state to be saved,
  991         call this before .quit().
  992         '''
  993         self._clear_state()
  994 
  995     def delivered(self, msgid):
  996         self.__delivered[msgid] = None
  997 
  998     def getheader(self, msgid):
  999         if not self.__initialized:
 1000             raise getmailOperationError('not initialized')
 1001         if not msgid in self.headercache:
 1002             self.headercache[msgid] = self._getheaderbyid(msgid)
 1003         return self.headercache[msgid]
 1004 
 1005     def getmsg(self, msgid):
 1006         if not self.__initialized:
 1007             raise getmailOperationError('not initialized')
 1008         return self._getmsgbyid(msgid)
 1009 
 1010     def getmsgsize(self, msgid):
 1011         if not self.__initialized:
 1012             raise getmailOperationError('not initialized')
 1013         try:
 1014             return self.msgsizes[msgid]
 1015         except KeyError:
 1016             raise getmailOperationError('no such message ID %s' % msgid)
 1017 
 1018     def delmsg(self, msgid):
 1019         if not self.__initialized:
 1020             raise getmailOperationError('not initialized')
 1021         self._delmsgbyid(msgid)
 1022         self.deleted[msgid] = True
 1023 
 1024 
 1025 #######################################
 1026 class POP3RetrieverBase(RetrieverSkeleton):
 1027     '''Base class for single-user POP3 mailboxes.
 1028     '''
 1029     def __init__(self, **args):
 1030         RetrieverSkeleton.__init__(self, **args)
 1031         self.log.trace()
 1032 
 1033     def select_mailbox(self, mailbox):
 1034         assert mailbox is None, (
 1035             'POP does not support mailbox selection (%s)' % mailbox
 1036         )
 1037         if self.mailbox_selected is not False:
 1038             self.write_oldmailfile(self.mailbox_selected)
 1039 
 1040         self._clear_state()
 1041 
 1042         if self.oldmail_exists(mailbox):
 1043             self.read_oldmailfile(mailbox)
 1044         self.mailbox_selected = mailbox
 1045 
 1046         self._getmsglist()
 1047 
 1048     def _getmsgnumbyid(self, msgid):
 1049         self.log.trace()
 1050         if not msgid in self.msgnum_by_msgid:
 1051             raise getmailOperationError('no such message ID %s' % msgid)
 1052         return self.msgnum_by_msgid[msgid]
 1053 
 1054     def _getmsglist(self):
 1055         self.log.trace()
 1056         try:
 1057             (response, msglist, octets) = self.conn.uidl()
 1058             self.log.debug('UIDL response "%s", %d octets'
 1059                            % (response, octets) + os.linesep)
 1060             for (i, line) in enumerate(msglist):
 1061                 try:
 1062                     (msgnum, msgid) = line.split(None, 1)
 1063                     # Don't allow / in UIDs we store, as we look for that to 
 1064                     # detect old-style oldmail files.  Shouldn't occur in POP3
 1065                     # anyway.
 1066                     msgid = msgid.replace('/', '-')
 1067                 except ValueError:
 1068                     # Line didn't contain two tokens.  Server is broken.
 1069                     raise getmailOperationError(
 1070                         '%s failed to identify message index %d in UIDL output'
 1071                         ' -- see documentation or use '
 1072                         'BrokenUIDLPOP3Retriever instead'
 1073                         % (self, i)
 1074                     )
 1075                 msgnum = int(msgnum)
 1076                 if msgid in self.msgnum_by_msgid:
 1077                     # UIDL "unique" identifiers weren't unique.
 1078                     # Server is broken.
 1079                     if self.conf.get('delete_dup_msgids', False):
 1080                         self.log.debug('deleting message %s with duplicate '
 1081                                        'msgid %s' % (msgnum, msgid)
 1082                                        + os.linesep)
 1083                         self.conn.dele(msgnum)
 1084                     else:
 1085                         raise getmailOperationError(
 1086                             '%s does not uniquely identify messages '
 1087                             '(got %s twice) -- see documentation or use '
 1088                             'BrokenUIDLPOP3Retriever instead'
 1089                             % (self, msgid)
 1090                         )
 1091                 else:
 1092                     self.msgnum_by_msgid[msgid] = msgnum
 1093                     self.msgid_by_msgnum[msgnum] = msgid
 1094             self.log.debug('Message IDs: %s'
 1095                            % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
 1096             self.sorted_msgnum_msgid = sorted(self.msgid_by_msgnum.items())
 1097             (response, msglist, octets) = self.conn.list()
 1098             for line in msglist:
 1099                 msgnum = int(line.split()[0])
 1100                 msgsize = int(line.split()[1])
 1101                 msgid = self.msgid_by_msgnum.get(msgnum, None)
 1102                 # If no msgid found, it's a message that wasn't in the UIDL
 1103                 # response above.  Ignore it and we'll get it next time.
 1104                 if msgid is not None:
 1105                     self.msgsizes[msgid] = msgsize
 1106 
 1107             # Remove messages from state file that are no longer in mailbox,
 1108             # but only if the timestamp for them are old (30 days for now).
 1109             # This is because IMAP users can have one state file but multiple
 1110             # IMAP folders in different configuration rc files.
 1111             for msgid in self.oldmail.keys():
 1112                 timestamp = self.oldmail[msgid]
 1113                 age = self.timestamp - timestamp
 1114                 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
 1115                     self.log.debug('removing vanished old message id %s' % msgid
 1116                                    + os.linesep)
 1117                     del self.oldmail[msgid]
 1118 
 1119         except poplib.error_proto, o:
 1120             raise getmailOperationError(
 1121                 'POP error (%s) - if your server does not support the UIDL '
 1122                 'command, use BrokenUIDLPOP3Retriever instead'
 1123                  % o
 1124             )
 1125         self.gotmsglist = True
 1126 
 1127     def _delmsgbyid(self, msgid):
 1128         self.log.trace()
 1129         msgnum = self._getmsgnumbyid(msgid)
 1130         self.conn.dele(msgnum)
 1131 
 1132     def _getmsgbyid(self, msgid):
 1133         self.log.debug('msgid %s' % msgid + os.linesep)
 1134         msgnum = self._getmsgnumbyid(msgid)
 1135         self.log.debug('msgnum %i' % msgnum + os.linesep)
 1136         try:
 1137             response, lines, octets = self.conn.retr(msgnum)
 1138             self.log.debug('RETR response "%s", %d octets'
 1139                            % (response, octets) + os.linesep)
 1140             msg = Message(fromlines=lines+[''])
 1141             return msg
 1142         except poplib.error_proto, o:
 1143             raise getmailRetrievalError(
 1144                 'failed to retrieve msgid %s; server said %s' 
 1145                 % (msgid, o)
 1146             )
 1147 
 1148     def _getheaderbyid(self, msgid):
 1149         self.log.trace()
 1150         msgnum = self._getmsgnumbyid(msgid)
 1151         response, headerlist, octets = self.conn.top(msgnum, 0)
 1152         parser = email.Parser.HeaderParser()
 1153         return parser.parsestr(os.linesep.join(headerlist))
 1154 
 1155     def initialize(self, options):
 1156         self.log.trace()
 1157         # POP doesn't support different mailboxes
 1158         self.mailboxes = (None, )
 1159         # Handle password
 1160         if self.conf.get('password', None) is None:
 1161             if self.conf.get('password_command', None):
 1162                 # Retrieve from an arbitrary external command
 1163                 command = self.conf['password_command'][0]
 1164                 args = self.conf['password_command'][1:]
 1165                 (rc, stdout, stderr) = run_command(command, args)
 1166                 if stderr:
 1167                     self.log.warning(
 1168                         'External password program "%s" wrote to stderr: %s'
 1169                         % (command, stderr)
 1170                     )
 1171                 if rc:
 1172                     # program exited nonzero
 1173                     raise getmailOperationError(
 1174                         'External password program error (exited %d)' % rc
 1175                     )
 1176                 else:
 1177                     self.conf['password'] = stdout
 1178             else:
 1179                 self.conf['password'] = get_password(
 1180                     self, self.conf['username'], self.conf['server'],
 1181                     self.received_with, self.log
 1182                 )
 1183         RetrieverSkeleton.initialize(self, options)
 1184         try:
 1185             self._connect()
 1186             if self.conf['use_apop']:
 1187                 self.conn.apop(self.conf['username'], self.conf['password'])
 1188             else:
 1189                 self.conn.user(self.conf['username'])
 1190                 self.conn.pass_(self.conf['password'])
 1191             self._getmsglist()
 1192             self.log.debug('msgids: %s'
 1193                            % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
 1194             self.log.debug('msgsizes: %s' % self.msgsizes + os.linesep)
 1195             # Remove messages from state file that are no longer in mailbox
 1196             for msgid in self.oldmail.keys():
 1197                 if not self.msgsizes.has_key(msgid):
 1198                     self.log.debug('removing vanished message id %s' % msgid
 1199                                    + os.linesep)
 1200                     del self.oldmail[msgid]
 1201         except poplib.error_proto, o:
 1202             raise getmailOperationError('POP error (%s)' % o)
 1203 
 1204     def abort(self):
 1205         self.log.trace()
 1206         RetrieverSkeleton.abort(self)
 1207         if not self.conn:
 1208             return
 1209         try:
 1210             self.conn.rset()
 1211             self.conn.quit()
 1212         except (poplib.error_proto, socket.error), o:
 1213             pass
 1214         self.conn = None
 1215 
 1216     def quit(self):
 1217         RetrieverSkeleton.quit(self)
 1218         self.log.trace()
 1219         if not self.conn:
 1220             return
 1221         try:
 1222             self.conn.quit()
 1223         except (poplib.error_proto, socket.error), o:
 1224             raise getmailOperationError('POP error (%s)' % o)
 1225         except AttributeError:
 1226             pass
 1227         self.conn = None
 1228 
 1229 
 1230 #######################################
 1231 class MultidropPOP3RetrieverBase(POP3RetrieverBase):
 1232     '''Base retriever class for multi-drop POP3 mailboxes.
 1233 
 1234     Envelope is reconstructed from Return-Path: (sender) and a header specified
 1235     by the user (recipient).  This header is specified with the
 1236     "envelope_recipient" parameter, which takes the form <field-name>[:<field-
 1237     number>].  field-number defaults to 1 and is counted from top to bottom in
 1238     the message.  For instance, if the envelope recipient is present in the
 1239     second Delivered-To: header field of each message, envelope_recipient should
 1240     be specified as "delivered-to:2".
 1241     '''
 1242 
 1243     def initialize(self, options):
 1244         self.log.trace()
 1245         POP3RetrieverBase.initialize(self, options)
 1246         self.envrecipname = (
 1247             self.conf['envelope_recipient'].split(':')[0].lower()
 1248         )
 1249         if self.envrecipname in NOT_ENVELOPE_RECIPIENT_HEADERS:
 1250             raise getmailConfigurationError(
 1251                 'the %s header field does not record the envelope '
 1252                     'recipient address'
 1253                 % self.envrecipname
 1254             )
 1255         self.envrecipnum = 0
 1256         try:
 1257             self.envrecipnum = int(
 1258                 self.conf['envelope_recipient'].split(':', 1)[1]
 1259             ) - 1
 1260             if self.envrecipnum < 0:
 1261                 raise ValueError(self.conf['envelope_recipient'])
 1262         except IndexError:
 1263             pass
 1264         except ValueError, o:
 1265             raise getmailConfigurationError(
 1266                 'invalid envelope_recipient specification format (%s)' % o
 1267             )
 1268 
 1269     def _getmsgbyid(self, msgid):
 1270         self.log.trace()
 1271         msg = POP3RetrieverBase._getmsgbyid(self, msgid)
 1272         data = {}
 1273         for (name, val) in msg.headers():
 1274             name = name.lower()
 1275             val = val.strip()
 1276             if name in data:
 1277                 data[name].append(val)
 1278             else:
 1279                 data[name] = [val]
 1280 
 1281         try:
 1282             line = data[self.envrecipname][self.envrecipnum]
 1283         except (KeyError, IndexError), unused:
 1284             raise getmailConfigurationError(
 1285                 'envelope_recipient specified header missing (%s)'
 1286                 % self.conf['envelope_recipient']
 1287             )
 1288         msg.recipient = address_no_brackets(line.strip())
 1289         return msg
 1290 
 1291 
 1292 #######################################
 1293 class IMAPRetrieverBase(RetrieverSkeleton):
 1294     '''Base class for single-user IMAP mailboxes.
 1295     '''
 1296     def __init__(self, **args):
 1297         RetrieverSkeleton.__init__(self, **args)
 1298         self.log.trace()
 1299         self.gss_step = 0
 1300         self.gss_vc = None
 1301         self.gssapi = False
 1302 
 1303     def _clear_state(self):
 1304         RetrieverSkeleton._clear_state(self)
 1305         self.mailbox = None
 1306         self.uidvalidity = None
 1307         self.msgnum_by_msgid = {}
 1308         self.msgid_by_msgnum = {}
 1309         self.sorted_msgnum_msgid = ()
 1310         self._mboxuids = {}
 1311         self._mboxuidorder = []
 1312         self.msgsizes = {}
 1313         self.oldmail = {}
 1314         self.__delivered = {}
 1315 
 1316     def checkconf(self):
 1317         RetrieverSkeleton.checkconf(self)
 1318         if self.conf['use_kerberos'] and not HAVE_KERBEROS_GSS:
 1319             raise getmailConfigurationError(
 1320                 'cannot use kerberos authentication; Python kerberos support '
 1321                 'not installed or does not support GSS'
 1322             )
 1323 
 1324     def gssauth(self, response):
 1325         if not HAVE_KERBEROS_GSS:
 1326             # shouldn't get here
 1327             raise ValueError('kerberos GSS support not available')
 1328         data = ''.join(str(response).encode('base64').splitlines())
 1329         if self.gss_step == GSS_STATE_STEP:
 1330             if not self.gss_vc:
 1331                 (rc, self.gss_vc) = kerberos.authGSSClientInit(
 1332                     'imap@%s' % self.conf['server']
 1333                 )
 1334                 response = kerberos.authGSSClientResponse(self.gss_vc)
 1335             rc = kerberos.authGSSClientStep(self.gss_vc, data)
 1336             if rc != kerberos.AUTH_GSS_CONTINUE:
 1337                self.gss_step = GSS_STATE_WRAP
 1338         elif self.gss_step == GSS_STATE_WRAP:
 1339             rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
 1340             response = kerberos.authGSSClientResponse(self.gss_vc)
 1341             rc = kerberos.authGSSClientWrap(self.gss_vc, response,
 1342                                             self.conf['username'])
 1343         response = kerberos.authGSSClientResponse(self.gss_vc)
 1344         if not response:
 1345             response = ''
 1346         return response.decode('base64')
 1347  
 1348     def _getmboxuidbymsgid(self, msgid):
 1349         self.log.trace()
 1350         if not msgid in self.msgnum_by_msgid:
 1351             raise getmailOperationError('no such message ID %s' % msgid)
 1352         uid = self._mboxuids[msgid]
 1353         return uid
 1354 
 1355     def _parse_imapcmdresponse(self, cmd, *args):
 1356         self.log.trace()
 1357         try:
 1358             result, resplist = getattr(self.conn, cmd)(*args)
 1359         except imaplib.IMAP4.error, o:
 1360             if cmd == 'login':
 1361                 # Percolate up
 1362                 raise
 1363             else:
 1364                 raise getmailOperationError('IMAP error (%s)' % o)
 1365         if result != 'OK':
 1366             raise getmailOperationError(
 1367                 'IMAP error (command %s returned %s %s)'
 1368                 % ('%s %s' % (cmd, args), result, resplist)
 1369             )
 1370         if cmd.lower().startswith('login'):
 1371             self.log.debug('login command response %s' % resplist + os.linesep)
 1372         else:
 1373             self.log.debug(
 1374                 'command %s response %s'
 1375                 % ('%s %s' % (cmd, args), resplist)
 1376                 + os.linesep
 1377             )
 1378         return resplist
 1379 
 1380     def _parse_imapuidcmdresponse(self, cmd, *args):
 1381         self.log.trace()
 1382         try:
 1383             result, resplist = self.conn.uid(cmd, *args)
 1384         except imaplib.IMAP4.error, o:
 1385             if cmd == 'login':
 1386                 # Percolate up
 1387                 raise
 1388             else:
 1389                 raise getmailOperationError('IMAP error (%s)' % o)
 1390         if result != 'OK':
 1391             raise getmailOperationError(
 1392                 'IMAP error (command %s returned %s %s)'
 1393                 % ('%s %s' % (cmd, args), result, resplist)
 1394             )
 1395         self.log.debug('command uid %s response %s'
 1396                        % ('%s %s' % (cmd, args), resplist) + os.linesep)
 1397         return resplist
 1398 
 1399     def _parse_imapattrresponse(self, line):
 1400         self.log.trace('parsing attributes response line %s' % line
 1401                        + os.linesep)
 1402         r = {}
 1403         try:
 1404             parts = line[line.index('(') + 1:line.rindex(')')].split()
 1405             while parts:
 1406                 # Flags starts a parenthetical list of valueless flags
 1407                 if parts[0].lower() == 'flags' and parts[1].startswith('('):
 1408                     while parts and not parts[0].endswith(')'):
 1409                         del parts[0]
 1410                     if parts:
 1411                         # Last one, ends with ")"
 1412                         del parts[0]
 1413                     continue
 1414                 if len(parts) == 1:
 1415                     # Leftover part -- not name, value pair.
 1416                     raise ValueError
 1417                 name = parts.pop(0).lower()
 1418                 r[name] = parts.pop(0)
 1419         except (ValueError, IndexError, AttributeError), o:
 1420             raise getmailOperationError(
 1421                 'IMAP error (failed to parse attr response line "%s": %s)' 
 1422                 % (line, o)
 1423             )
 1424         self.log.trace('got %s' % r + os.linesep)
 1425         return r
 1426 
 1427     def list_mailboxes(self):
 1428         '''List (selectable) IMAP folders in account.'''
 1429         mailboxes = []
 1430         cmd = ('LIST', )
 1431         resplist = self._parse_imapcmdresponse(*cmd)
 1432         for item in resplist:
 1433             m = IMAP_LISTPARTS.match(item)
 1434             if not m:
 1435                 raise getmailOperationError(
 1436                     'no match for list response "%s"' % item
 1437                 )
 1438             g = m.groupdict()
 1439             attributes = g['attributes'].split()
 1440             if r'\Noselect' in attributes:
 1441                 # Can't select this mailbox, don't include it in output
 1442                 continue
 1443             try:
 1444                 mailbox = g['mailbox'].decode('imap4-utf-7')
 1445                 mailboxes.append(mailbox)
 1446                 #log.debug(u'%20s : delimiter %s, attributes: %s',
 1447                 #          mailbox, g['delimiter'], ', '.join(attributes))
 1448             except Exception, o:
 1449                 raise getmailOperationError('error decoding mailbox "%s"' 
 1450                                             % g['mailbox'])
 1451         return mailboxes
 1452 
 1453     def close_mailbox(self):
 1454         # Close current mailbox so deleted mail is expunged.  One getmail
 1455         # user had a buggy IMAP server that didn't do the automatic expunge,
 1456         # so we do it explicitly here if we've deleted any messages.
 1457         if self.deleted:
 1458             self.conn.expunge()
 1459         self.conn.close()
 1460         self.write_oldmailfile(self.mailbox_selected)
 1461         # And clear some state
 1462         self.mailbox_selected = False
 1463         self.mailbox = None
 1464         self.uidvalidity = None
 1465         self.msgnum_by_msgid = {}
 1466         self.msgid_by_msgnum = {}
 1467         self.sorted_msgnum_msgid = ()
 1468         self._mboxuids = {}
 1469         self._mboxuidorder = []
 1470         self.msgsizes = {}
 1471         self.oldmail = {}
 1472         self.__delivered = {}
 1473 
 1474     def select_mailbox(self, mailbox):
 1475         self.log.trace()
 1476         assert mailbox in self.mailboxes, (
 1477             'mailbox not in config (%s)' % mailbox
 1478         )
 1479         if self.mailbox_selected is not False:
 1480             self.close_mailbox()
 1481 
 1482         self._clear_state()
 1483 
 1484         if self.oldmail_exists(mailbox):
 1485             self.read_oldmailfile(mailbox)
 1486 
 1487         self.log.debug('selecting mailbox "%s"' % mailbox + os.linesep)
 1488         try:
 1489             if (self.app_options['delete'] or self.app_options['delete_after'] 
 1490                     or self.app_options['delete_bigger_than']):
 1491                 read_only = False
 1492             else:
 1493                 read_only = True
 1494             (status, count) = self.conn.select(mailbox.encode('imap4-utf-7'), 
 1495                                                read_only)
 1496             if status == 'NO':
 1497                 # Specified mailbox doesn't exist, no permissions, etc.
 1498                 raise getmailMailboxSelectError(mailbox)
 1499                 
 1500             self.mailbox_selected = mailbox
 1501             # use *last* EXISTS returned
 1502             count = int(count[-1])
 1503             uidvalidity = self.conn.response('UIDVALIDITY')[1][0]
 1504         except imaplib.IMAP4.error, o:
 1505             raise getmailOperationError('IMAP error (%s)' % o)
 1506         except (IndexError, ValueError), o:
 1507             raise getmailOperationError(
 1508                 'IMAP server failed to return correct SELECT response (%s)'
 1509                 % o
 1510             )
 1511         self.log.debug('select(%s) returned message count of %d'
 1512                        % (mailbox, count) + os.linesep)
 1513         self.mailbox = mailbox
 1514         self.uidvalidity = uidvalidity
 1515 
 1516         self._getmsglist(count)
 1517 
 1518         return count
 1519 
 1520     def _getmsglist(self, msgcount):
 1521         self.log.trace()
 1522         try:
 1523             if msgcount:
 1524                 # Get UIDs and sizes for all messages in mailbox
 1525                 response = self._parse_imapcmdresponse(
 1526                     'FETCH', '1:%d' % msgcount, '(UID RFC822.SIZE)'
 1527                 )
 1528                 for line in response:
 1529                     if not line:
 1530                         # One user had a server that returned a null response
 1531                         # somehow -- try to just skip.
 1532                         continue
 1533                     r = self._parse_imapattrresponse(line)
 1534                     # Don't allow / in UIDs we store, as we look for that to 
 1535                     # detect old-style oldmail files.  Can occur with IMAP, at 
 1536                     # least with some servers.
 1537                     uid = r['uid'].replace('/', '-')
 1538                     msgid = '%s/%s' % (self.uidvalidity, uid)
 1539                     self._mboxuids[msgid] = r['uid']
 1540                     self._mboxuidorder.append(msgid)
 1541                     self.msgnum_by_msgid[msgid] = None
 1542                     self.msgsizes[msgid] = int(r['rfc822.size'])
 1543 
 1544             # Remove messages from state file that are no longer in mailbox,
 1545             # but only if the timestamp for them are old (30 days for now).
 1546             # This is because IMAP users can have one state file but multiple
 1547             # IMAP folders in different configuration rc files.
 1548             for msgid in self.oldmail.keys():
 1549                 timestamp = self.oldmail[msgid]
 1550                 age = self.timestamp - timestamp
 1551                 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
 1552                     self.log.debug('removing vanished old message id %s' % msgid
 1553                                    + os.linesep)
 1554                     del self.oldmail[msgid]
 1555 
 1556         except imaplib.IMAP4.error, o:
 1557             raise getmailOperationError('IMAP error (%s)' % o)
 1558         self.gotmsglist = True
 1559 
 1560     def __getitem__(self, i):
 1561         return self._mboxuidorder[i]
 1562 
 1563     def _delmsgbyid(self, msgid):
 1564         self.log.trace()
 1565         try:
 1566             uid = self._getmboxuidbymsgid(msgid)
 1567             #self._selectmailbox(mailbox)
 1568             # Delete message
 1569             if self.conf['move_on_delete']:
 1570                 self.log.debug('copying message to folder "%s"'
 1571                                % self.conf['move_on_delete'] + os.linesep)
 1572                 response = self._parse_imapuidcmdresponse(
 1573                     'COPY', uid, self.conf['move_on_delete']
 1574                 )
 1575             self.log.debug('deleting message "%s"' % uid + os.linesep)
 1576             response = self._parse_imapuidcmdresponse(
 1577                 'STORE', uid, 'FLAGS', '(\Deleted \Seen)'
 1578             )
 1579         except imaplib.IMAP4.error, o:
 1580             raise getmailOperationError('IMAP error (%s)' % o)
 1581 
 1582     def _getmsgpartbyid(self, msgid, part):
 1583         self.log.trace()
 1584         try:
 1585             uid = self._getmboxuidbymsgid(msgid)
 1586             # Retrieve message
 1587             self.log.debug('retrieving body for message "%s"' % uid
 1588                            + os.linesep)
 1589             try:
 1590                 response = self._parse_imapuidcmdresponse('FETCH', uid, part)
 1591             except (imaplib.IMAP4.error, getmailOperationError), o:
 1592                 # server gave a negative/NO response, most likely.  Bad server,
 1593                 # no doughnut.
 1594                 raise getmailRetrievalError(
 1595                     'failed to retrieve msgid %s; server said %s' 
 1596                     % (msgid, o)
 1597                 )
 1598             # Response is really ugly:
 1599             #
 1600             # [
 1601             #   (
 1602             #       '1 (UID 1 RFC822 {704}',
 1603             #       'message text here with CRLF EOL'
 1604             #   ),
 1605             #   ')',
 1606             #   <maybe more>
 1607             # ]
 1608             
 1609             # MSExchange is broken -- if a message is badly formatted enough
 1610             # (virus, spam, trojan), it can completely fail to return the
 1611             # message when requested.
 1612             try:
 1613                 try:
 1614                     sbody = response[0][1]
 1615                 except Exception, o:
 1616                     sbody = None
 1617                 if not sbody:
 1618                     self.log.error('bad message from server!')
 1619                     sbody = str(response)
 1620                 msg = Message(fromstring=sbody)
 1621             except TypeError, o:
 1622                 # response[0] is None instead of a message tuple
 1623                 raise getmailRetrievalError('failed to retrieve msgid %s' 
 1624                                             % msgid)
 1625 
 1626             # record mailbox retrieved from in a header
 1627             if self.conf['record_mailbox']:
 1628                 msg.add_header('X-getmail-retrieved-from-mailbox', 
 1629                                self.mailbox_selected)
 1630 
 1631             # google extensions: apply labels, etc
 1632             if 'X-GM-EXT-1' in self.conn.capabilities:
 1633                 metadata = self._getgmailmetadata(uid, msg)
 1634                 for (header, value) in metadata.items():
 1635                     msg.add_header(header, value)
 1636 
 1637             return msg
 1638 
 1639         except imaplib.IMAP4.error, o:
 1640             raise getmailOperationError('IMAP error (%s)' % o)
 1641 
 1642     def _getgmailmetadata(self, uid, msg):
 1643         """
 1644         Add Gmail labels and other metadata which Google exposes through an
 1645         IMAP extension to headers in the message.
 1646         
 1647         See https://developers.google.com/google-apps/gmail/imap_extensions
 1648         """
 1649         try:
 1650             # ['976 (X-GM-THRID 1410134259107225671 X-GM-MSGID '
 1651             #   '1410134259107225671 X-GM-LABELS (labels space '
 1652             #   'separated) UID 167669)']
 1653             response = self._parse_imapuidcmdresponse('FETCH', uid,
 1654                 '(X-GM-LABELS X-GM-THRID X-GM-MSGID)')
 1655         except imaplib.IMAP4.error, o:
 1656             self.log.warning('Could not fetch google imap extensions: %s' % o)
 1657             return {}
 1658 
 1659         if not response or not response[0]:
 1660             return {}
 1661             
 1662         ext = re.search(
 1663             'X-GM-THRID (?P<THRID>\d+) X-GM-MSGID (?P<MSGID>\d+)'
 1664             ' X-GM-LABELS \((?P<LABELS>.*)\) UID',
 1665             response[0]
 1666         )
 1667         if not ext:
 1668             self.log.warning(
 1669                 'Could not parse google imap extensions. Server said: %s'
 1670                 % repr(response)
 1671             )
 1672             return {}
 1673 
 1674         results = ext.groupdict()
 1675         metadata = {}
 1676         for item in ('LABELS', 'THRID', 'MSGID'):
 1677             if item in results and results[item]:
 1678                 metadata['X-GMAIL-%s' % item] = results[item]
 1679 
 1680         return metadata
 1681 
 1682     def _getmsgbyid(self, msgid):
 1683         self.log.trace()
 1684         if self.conf.get('use_peek', True):
 1685             part = '(BODY.PEEK[])'
 1686         else:
 1687             part = '(RFC822)'
 1688         return self._getmsgpartbyid(msgid, part)
 1689 
 1690     def _getheaderbyid(self, msgid):
 1691         self.log.trace()
 1692         if self.conf.get('use_peek', True):
 1693             part = '(BODY.PEEK[header])'
 1694         else:
 1695             part = '(RFC822[header])'
 1696         return self._getmsgpartbyid(msgid, part)
 1697 
 1698     def initialize(self, options):
 1699         self.log.trace()
 1700         self.mailboxes = self.conf.get('mailboxes', ('INBOX', ))
 1701         # Handle password
 1702         if (self.conf.get('password', None) is None
 1703                 and not (HAVE_KERBEROS_GSS and self.conf['use_kerberos'])):
 1704             if self.conf['password_command']:
 1705                 # Retrieve from an arbitrary external command
 1706                 command = self.conf['password_command'][0]
 1707                 args = self.conf['password_command'][1:]
 1708                 (rc, stdout, stderr) = run_command(command, args)
 1709                 if stderr:
 1710                     self.log.warning(
 1711                         'External password program "%s" wrote to stderr: %s'
 1712                         % (command, stderr)
 1713                     )
 1714                 if rc:
 1715                     # program exited nonzero
 1716                     raise getmailOperationError(
 1717                         'External password program error (exited %d)' % rc
 1718                     )
 1719                 else:
 1720                     self.conf['password'] = stdout
 1721             else:
 1722                 self.conf['password'] = get_password(
 1723                     self, self.conf['username'], self.conf['server'], 
 1724                     self.received_with, self.log
 1725                 )
 1726             
 1727         RetrieverSkeleton.initialize(self, options)
 1728         try:
 1729             self.log.trace('trying self._connect()' + os.linesep)
 1730             self._connect()
 1731             try:
 1732                 self.log.trace('logging in' + os.linesep)
 1733                 if self.conf['use_kerberos'] and HAVE_KERBEROS_GSS:
 1734                     self.conn.authenticate('GSSAPI', self.gssauth)
 1735                 elif self.conf['use_cram_md5']:
 1736                     self._parse_imapcmdresponse(
 1737                         'login_cram_md5', self.conf['username'],
 1738                         self.conf['password']
 1739                     )
 1740                 elif self.conf['use_xoauth2']:
 1741                     # octal 1 / ctrl-A used as separator
 1742                     auth = 'user=%s\1auth=Bearer %s\1\1' % (self.conf['username'],
 1743                                                             self.conf['password'])
 1744                     self.conn.authenticate('XOAUTH2', lambda unused: auth)
 1745                 else:
 1746                     self._parse_imapcmdresponse('login', self.conf['username'],
 1747                                                 self.conf['password'])
 1748             except imaplib.IMAP4.abort, o:
 1749                 raise getmailLoginRefusedError(o)
 1750             except imaplib.IMAP4.error, o:
 1751                 if str(o).startswith('[UNAVAILABLE]'):
 1752                     raise getmailLoginRefusedError(o)
 1753                 else:
 1754                     raise getmailCredentialError(o)
 1755 
 1756             self.log.trace('logged in' + os.linesep)
 1757             """
 1758             self.log.trace('logged in, getting message list' + os.linesep)
 1759             self._getmsglist()
 1760             self.log.debug('msgids: %s'
 1761                            % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
 1762             self.log.debug('msgsizes: %s' % self.msgsizes + os.linesep)
 1763             # Remove messages from state file that are no longer in mailbox,
 1764             # but only if the timestamp for them are old (30 days for now).
 1765             # This is because IMAP users can have one state file but multiple
 1766             # IMAP folders in different configuration rc files.
 1767             for msgid in self.oldmail.keys():
 1768                 timestamp = self.oldmail[msgid]
 1769                 age = self.timestamp - timestamp
 1770                 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
 1771                     self.log.debug('removing vanished old message id %s' % msgid
 1772                                    + os.linesep)
 1773                     del self.oldmail[msgid]
 1774             """
 1775             # Some IMAP servers change the available capabilities after 
 1776             # authentication, i.e. they present a limited set before login.
 1777             # The Python stlib IMAP4 class doesn't take this into account
 1778             # and just checks the capabilities immediately after connecting.
 1779             # Force a re-check now that we've authenticated.
 1780             (typ, dat) = self.conn.capability()
 1781             if dat == [None]:
 1782                 # No response, don't update the stored capabilities
 1783                 self.log.warning('no post-login CAPABILITY response from server\n')
 1784             else:
 1785                 self.conn.capabilities = tuple(dat[-1].upper().split())
 1786 
 1787             if 'IDLE' in self.conn.capabilities:
 1788                 self.supports_idle = True
 1789                 imaplib.Commands['IDLE'] = ('AUTH', 'SELECTED')
 1790 
 1791             if self.mailboxes == ('ALL', ):
 1792                 # Special value meaning all mailboxes in account
 1793                 self.mailboxes = tuple(self.list_mailboxes())
 1794 
 1795         except imaplib.IMAP4.error, o:
 1796             raise getmailOperationError('IMAP error (%s)' % o)
 1797 
 1798     def abort(self):
 1799         self.log.trace()
 1800         RetrieverSkeleton.abort(self)
 1801         if not self.conn:
 1802             return
 1803         try:
 1804             self.quit()
 1805         except (imaplib.IMAP4.error, socket.error), o:
 1806             pass
 1807         self.conn = None
 1808 
 1809     def go_idle(self, folder, timeout=300):
 1810         """Initiates IMAP's IDLE mode if the server supports it
 1811 
 1812         Waits until state of current mailbox changes, and then returns. Returns
 1813         True if the connection still seems to be up, False otherwise.
 1814 
 1815         May throw getmailOperationError if the server refuses the IDLE setup
 1816         (e.g. if the server does not support IDLE)
 1817 
 1818         Default timeout is 5 minutes.
 1819         """
 1820 
 1821         if not self.supports_idle:
 1822             self.log.warning('IDLE not supported, so not idling\n')
 1823             raise getmailOperationError(
 1824                 'IMAP4 IDLE requested, but not supported by server'
 1825             )
 1826 
 1827 
 1828         if self.SSL:
 1829             sock = self.conn.ssl()
 1830         else:
 1831             sock = self.conn.socket()
 1832 
 1833         # Based on current imaplib IDLE patch: http://bugs.python.org/issue11245
 1834         self.conn.untagged_responses = {}
 1835         self.conn.select(folder)
 1836         tag = self.conn._command('IDLE')
 1837         data = self.conn._get_response() # read continuation response
 1838 
 1839         if data is not None:
 1840             raise getmailOperationError(
 1841                 'IMAP4 IDLE requested, but server refused IDLE request: %s' 
 1842                 % data
 1843             )
 1844 
 1845         self.log.debug('Entering IDLE mode (server says "%s")\n' 
 1846                        % self.conn.continuation_response)
 1847 
 1848         try:
 1849             aborted = None
 1850             (readable, unused, unused) = select.select([sock], [], [], timeout)
 1851         except KeyboardInterrupt, o:
 1852             # Delay raising this until we've stopped IDLE mode
 1853             aborted = o
 1854 
 1855         if aborted is not None:
 1856             self.log.debug('IDLE mode cancelled\n')
 1857         elif readable:
 1858             # The socket has data waiting; server has updated status
 1859             self.log.info('IDLE message received\n')
 1860         else:
 1861             self.log.debug('IDLE timeout (%ds)\n' % timeout)
 1862 
 1863         try:
 1864             self.conn.untagged_responses = {}
 1865             self.conn.send('DONE\r\n')
 1866             self.conn._command_complete('IDLE', tag)
 1867         except imaplib.IMAP4.error, o:
 1868             return False
 1869 
 1870         if aborted:
 1871             raise aborted
 1872 
 1873         return True
 1874 
 1875     def quit(self):
 1876         self.log.trace()
 1877         if not self.conn:
 1878             return
 1879         try:
 1880             if self.mailbox_selected is not False:
 1881                 self.close_mailbox()
 1882             self.conn.logout()
 1883         except imaplib.IMAP4.error, o:
 1884             #raise getmailOperationError('IMAP error (%s)' % o)
 1885             self.log.warning('IMAP error during logout (%s)' % o + os.linesep)
 1886         RetrieverSkeleton.quit(self)
 1887         self.conn = None
 1888 
 1889 
 1890 #######################################
 1891 class MultidropIMAPRetrieverBase(IMAPRetrieverBase):
 1892     '''Base retriever class for multi-drop IMAP mailboxes.
 1893 
 1894     Envelope is reconstructed from Return-Path: (sender) and a header specified
 1895     by the user (recipient).  This header is specified with the
 1896     "envelope_recipient" parameter, which takes the form <field-name>[:<field-
 1897     number>].  field-number defaults to 1 and is counted from top to bottom in
 1898     the message.  For instance, if the envelope recipient is present in the
 1899     second Delivered-To: header field of each message, envelope_recipient should
 1900     be specified as "delivered-to:2".
 1901     '''
 1902 
 1903     def initialize(self, options):
 1904         self.log.trace()
 1905         IMAPRetrieverBase.initialize(self, options)
 1906         self.envrecipname = (self.conf['envelope_recipient'].split(':')
 1907             [0].lower())
 1908         if self.envrecipname in NOT_ENVELOPE_RECIPIENT_HEADERS:
 1909             raise getmailConfigurationError(
 1910                 'the %s header field does not record the envelope recipient '
 1911                     'address'
 1912                 % self.envrecipname
 1913             )
 1914         self.envrecipnum = 0
 1915         try:
 1916             self.envrecipnum = int(
 1917                 self.conf['envelope_recipient'].split(':', 1)[1]
 1918             ) - 1
 1919             if self.envrecipnum < 0:
 1920                 raise ValueError(self.conf['envelope_recipient'])
 1921         except IndexError:
 1922             pass
 1923         except ValueError, o:
 1924             raise getmailConfigurationError(
 1925                 'invalid envelope_recipient specification format (%s)' % o
 1926             )
 1927 
 1928     def _getmsgbyid(self, msgid):
 1929         self.log.trace()
 1930         msg = IMAPRetrieverBase._getmsgbyid(self, msgid)
 1931         data = {}
 1932         for (name, encoded_value) in msg.headers():
 1933             name = name.lower()
 1934             for (val, encoding) in decode_header(encoded_value):
 1935                 val = val.strip()
 1936                 if name in data:
 1937                     data[name].append(val)
 1938                 else:
 1939                     data[name] = [val]
 1940 
 1941         try:
 1942             line = data[self.envrecipname][self.envrecipnum]
 1943         except (KeyError, IndexError), unused:
 1944             raise getmailConfigurationError(
 1945                 'envelope_recipient specified header missing (%s)'
 1946                 % self.conf['envelope_recipient']
 1947             )
 1948         msg.recipient = address_no_brackets(line.strip())
 1949         return msg
 1950 
 1951 
 1952 # Choose right POP-over-SSL mix-in based on Python version being used.
 1953 if sys.hexversion >= 0x02040000:
 1954     POP3SSLinitMixIn = Py24POP3SSLinitMixIn
 1955 else:
 1956     POP3SSLinitMixIn = Py23POP3SSLinitMixIn