"Fossies" - the Fresh Open Source Software Archive 
Member "getmail-5.16/getmailcore/_retrieverbases.py" (31 Oct 2021, 76421 Bytes) of package /linux/misc/getmail-5.16.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "_retrieverbases.py" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
5.15_vs_5.16.
1 #!/usr/bin/env python2
2 '''Base and mix-in classes implementing retrievers (message sources getmail can
3 retrieve mail from).
4
5 None of these classes can be instantiated directly. In this module:
6
7 Mix-in classes for SSL/non-SSL initialization:
8
9 POP3initMixIn
10 Py23POP3SSLinitMixIn
11 Py24POP3SSLinitMixIn
12 IMAPinitMixIn
13 IMAPSSLinitMixIn
14
15 Base classes:
16
17 RetrieverSkeleton
18 POP3RetrieverBase
19 MultidropPOP3RetrieverBase
20 IMAPRetrieverBase
21 MultidropIMAPRetrieverBase
22 '''
23
24 __all__ = [
25 'IMAPinitMixIn',
26 'IMAPRetrieverBase',
27 'IMAPSSLinitMixIn',
28 'MultidropPOP3RetrieverBase',
29 'MultidropIMAPRetrieverBase',
30 'POP3_ssl_port',
31 'POP3initMixIn',
32 'POP3RetrieverBase',
33 'POP3SSLinitMixIn',
34 'RetrieverSkeleton',
35 ]
36
37 import sys
38 import os
39 import socket
40 import time
41 import email
42 import poplib
43 import imaplib
44 import re
45 import select
46
47 try:
48 # do we have a recent pykerberos?
49 HAVE_KERBEROS_GSS = False
50 import kerberos
51 if 'authGSSClientWrap' in dir(kerberos):
52 HAVE_KERBEROS_GSS = True
53 except ImportError:
54 pass
55
56 # hashlib only present in python2.5, ssl in python2.6; used together
57 # in SSL functionality below
58 try:
59 import ssl
60 except ImportError:
61 ssl = None
62 try:
63 import hashlib
64 except ImportError:
65 hashlib = None
66
67 # If we have an ssl module:
68 if ssl:
69 has_sni = getattr(ssl, 'HAS_SNI', False)
70 proto_best = getattr(ssl, 'PROTOCOL_TLS', None)
71 if not proto_best:
72 proto_best = getattr(ssl, 'PROTOCOL_SSLv23', None)
73 has_ciphers = sys.hexversion >= 0x2070000
74
75 # Monkey-patch SNI use into SSL.wrap_socket() if supported
76 if has_sni:
77 def _wrap_socket(sock, keyfile=None, certfile=None,
78 server_side=False, cert_reqs=ssl.CERT_NONE,
79 ssl_version=proto_best, ca_certs=None,
80 do_handshake_on_connect=True,
81 suppress_ragged_eofs=True,
82 ciphers=None, server_hostname=None):
83 kwargs = dict(sock=sock, keyfile=keyfile, certfile=certfile,
84 server_side=server_side, cert_reqs=cert_reqs,
85 ssl_version=ssl_version, ca_certs=ca_certs,
86 do_handshake_on_connect=do_handshake_on_connect,
87 suppress_ragged_eofs=suppress_ragged_eofs,
88 ciphers=ciphers, server_hostname=server_hostname)
89 if not has_ciphers:
90 kwargs.pop('ciphers', None)
91 return ssl.SSLSocket(**kwargs)
92 else:
93 # no SNI support
94 def _wrap_socket(sock, keyfile=None, certfile=None,
95 server_side=False, cert_reqs=ssl.CERT_NONE,
96 ssl_version=proto_best, ca_certs=None,
97 do_handshake_on_connect=True,
98 suppress_ragged_eofs=True,
99 ciphers=None, server_hostname=None):
100 kwargs = dict(sock=sock, keyfile=keyfile, certfile=certfile,
101 server_side=server_side, cert_reqs=cert_reqs,
102 ssl_version=ssl_version, ca_certs=ca_certs,
103 do_handshake_on_connect=do_handshake_on_connect,
104 suppress_ragged_eofs=suppress_ragged_eofs,
105 ciphers=ciphers)
106 if not has_ciphers:
107 kwargs.pop('ciphers', None)
108 return ssl.SSLSocket(**kwargs)
109 ssl.wrap_socket = _wrap_socket
110
111 # Is it recent enough to have hostname matching (Python 3.2+)?
112 try:
113 ssl_match_hostname = ssl.match_hostname
114 except AttributeError:
115 # Running a Python with no hostname matching
116 def _dnsname_match(dn, hostname, max_wildcards=1):
117 """Matching according to RFC 6125, section 6.4.3
118 http://tools.ietf.org/html/rfc6125#section-6.4.3
119 """
120 pats = []
121 if not dn:
122 return False
123
124 parts = dn.split(r'.')
125 leftmost = parts[0]
126 remainder = parts[1:]
127
128 wildcards = leftmost.count('*')
129 if wildcards > max_wildcards:
130 # Issue #17980: avoid denials of service by refusing more
131 # than one wildcard per fragment. A survery of established
132 # policy among SSL implementations showed it to be a
133 # reasonable choice.
134 raise getmailOperationError(
135 "too many wildcards in certificate DNS name: " + repr(dn))
136
137 # speed up common case w/o wildcards
138 if not wildcards:
139 return dn.lower() == hostname.lower()
140
141 # RFC 6125, section 6.4.3, subitem 1.
142 # The client SHOULD NOT attempt to match a presented identifier
143 # in which the wildcard character comprises a label other than
144 # the left-most label.
145 if leftmost == '*':
146 # When '*' is a fragment by itself, it matches a non-empty
147 # dotless fragment.
148 pats.append('[^.]+')
149 elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
150 # RFC 6125, section 6.4.3, subitem 3.
151 # The client SHOULD NOT attempt to match a presented identifier
152 # where the wildcard character is embedded within an A-label or
153 # U-label of an internationalized domain name.
154 pats.append(re.escape(leftmost))
155 else:
156 # Otherwise, '*' matches any dotless string, e.g. www*
157 pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
158
159 # add the remaining fragments, ignore any wildcards
160 for frag in remainder:
161 pats.append(re.escape(frag))
162
163 pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
164 return pat.match(hostname)
165
166
167 def ssl_match_hostname(cert, hostname):
168 """Verify that *cert* (in decoded format as returned by
169 SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and
170 RFC 6125 rules are followed, but IP addresses are not accepted
171 for *hostname*.
172
173 getmailOperationError is raised on failure. On success, the function
174 returns nothing.
175 """
176 if not cert:
177 raise ValueError("empty or no certificate, ssl_match_hostname "
178 "needs an SSL socket or SSL context with "
179 "either CERT_OPTIONAL or CERT_REQUIRED")
180 dnsnames = []
181 san = cert.get('subjectAltName', ())
182 for key, value in san:
183 if key == 'DNS':
184 if _dnsname_match(value, hostname):
185 return
186 dnsnames.append(value)
187 if not dnsnames:
188 # The subject is only checked when there is no dNSName entry
189 # in subjectAltName
190 for sub in cert.get('subject', ()):
191 for key, value in sub:
192 # XXX according to RFC 2818, the most specific
193 # Common Name must be used.
194 if key == 'commonName':
195 if _dnsname_match(value, hostname):
196 return
197 dnsnames.append(value)
198 if len(dnsnames) > 1:
199 raise getmailOperationError("hostname %s "
200 "doesn't match either of %s"
201 % (hostname, ', '.join(map(repr, dnsnames))))
202 elif len(dnsnames) == 1:
203 raise getmailOperationError("hostname %s "
204 "doesn't match %s"
205 % (hostname, dnsnames[0]))
206 else:
207 raise getmailOperationError("no appropriate commonName or "
208 "subjectAltName fields were found")
209
210 try:
211 from email.header import decode_header
212 except ImportError, o:
213 # python < 2.5
214 from email.Header import decode_header
215
216 from getmailcore.compatibility import *
217 from getmailcore.exceptions import *
218 from getmailcore.constants import *
219 from getmailcore.message import *
220 from getmailcore.utilities import *
221 from getmailcore._pop3ssl import POP3SSL, POP3_ssl_port
222 from getmailcore.baseclasses import *
223 import getmailcore.imap_utf7 # registers imap4-utf-7 codec
224
225
226 NOT_ENVELOPE_RECIPIENT_HEADERS = (
227 'to',
228 'cc',
229 'bcc',
230 'received',
231 'resent-to',
232 'resent-cc',
233 'resent-bcc'
234 )
235
236 # How long a vanished message is kept in the oldmail state file for IMAP
237 # retrievers before we figure it's gone for good. This is to allow users
238 # to only occasionally retrieve mail from certain IMAP folders without
239 # losing their oldmail state for that folder. This is in seconds, so it's
240 # 30 days.
241 VANISHED_AGE = (60 * 60 * 24 * 30)
242
243 # Regex used to remove problematic characters from oldmail filenames
244 STRIP_CHAR_RE = r'[/\:;<>|]+'
245
246 # Kerberos authentication state constants
247 (GSS_STATE_STEP, GSS_STATE_WRAP) = (0, 1)
248
249 # For matching imap LIST responses
250 IMAP_LISTPARTS = re.compile(
251 r'^\s*'
252 r'\((?P<attributes>[^)]*)\)'
253 r'\s+'
254 r'"(?P<delimiter>[^"]+)"'
255 r'\s+'
256 # I *think* this should actually be a double-quoted string "like/this"
257 # but in testing we saw an MSexChange response that violated that
258 # expectation:
259 # (\HasNoChildren) "/" Calendar"
260 # i.e. the leading quote on the mailbox name was missing. The following
261 # works for both by treating the leading/trailing double-quote as optional,
262 # even when mismatched.
263 r'("?)(?P<mailbox>.+?)("?)'
264 r'\s*$'
265 )
266
267
268 # Constants used in socket module
269 NO_OBJ = object()
270 EAI_NONAME = getattr(socket, 'EAI_NONAME', NO_OBJ)
271 EAI_NODATA = getattr(socket, 'EAI_NODATA', NO_OBJ)
272 EAI_FAIL = getattr(socket, 'EAI_FAIL', NO_OBJ)
273
274
275 # Constant for POPSSL
276 POP3_SSL_PORT = 995
277
278
279 # Python added poplib._MAXLINE somewhere along the way. As far as I can
280 # see, it serves no purpose except to introduce bugs into any software
281 # using poplib. Any computer running Python will have at least some megabytes
282 # of userspace memory; arbitrarily causing message retrieval to break if any
283 # "line" exceeds 2048 bytes is absolutely stupid.
284 poplib._MAXLINE = 1 << 20 # 1MB; decrease this if you're running on a VIC-20
285
286
287 #
288 # Mix-in classes
289 #
290
291 #######################################
292 class POP3initMixIn(object):
293 '''Mix-In class to do POP3 non-SSL initialization.
294 '''
295 SSL = False
296 def _connect(self):
297 self.log.trace()
298 try:
299 self.conn = poplib.POP3(self.conf['server'], self.conf['port'])
300 self.setup_received(self.conn.sock)
301 except poplib.error_proto, o:
302 raise getmailOperationError('POP error (%s)' % o)
303 except socket.timeout:
304 raise
305 #raise getmailOperationError('timeout during connect')
306 except socket.gaierror, o:
307 raise getmailOperationError(
308 'error resolving name %s during connect (%s)'
309 % (self.conf['server'], o)
310 )
311
312 self.log.trace('POP3 connection %s established' % self.conn
313 + os.linesep)
314
315
316 #######################################
317 class POP3_SSL_EXTENDED(poplib.POP3_SSL):
318 # Extended SSL support for POP3 (certificate checking,
319 # fingerprint matching, cipher selection, etc.)
320
321 def __init__(self, host, port=POP3_SSL_PORT, keyfile=None,
322 certfile=None, ssl_version=None, ca_certs=None,
323 ssl_ciphers=None):
324 self.host = host
325 self.port = port
326 self.keyfile = keyfile
327 self.certfile = certfile
328 self.ssl_version = ssl_version
329 self.ca_certs = ca_certs
330 self.ssl_ciphers = ssl_ciphers
331
332 self.buffer = ''
333 msg = "getaddrinfo returns an empty list"
334 self.sock = None
335 for res in socket.getaddrinfo(self.host, self.port, 0,
336 socket.SOCK_STREAM):
337 (af, socktype, proto, canonname, sa) = res
338 try:
339 self.sock = socket.socket(af, socktype, proto)
340 self.sock.connect(sa)
341 except socket.error, msg:
342 if self.sock:
343 self.sock.close()
344 self.sock = None
345 continue
346 break
347 if not self.sock:
348 raise socket.error(msg)
349 extra_args = { 'server_hostname': host }
350 if self.ssl_version:
351 extra_args['ssl_version'] = self.ssl_version
352 if self.ca_certs:
353 extra_args['cert_reqs'] = ssl.CERT_REQUIRED
354 extra_args['ca_certs'] = self.ca_certs
355 if self.ssl_ciphers:
356 extra_args['ciphers'] = self.ssl_ciphers
357
358 self.file = self.sock.makefile('rb')
359 self.sslobj = ssl.wrap_socket(self.sock, self.keyfile,
360 self.certfile, **extra_args)
361 self._debugging = 0
362 self.welcome = self._getresp()
363
364
365 #######################################
366 class Py24POP3SSLinitMixIn(object):
367 '''Mix-In class to do POP3 over SSL initialization with Python 2.4's
368 poplib.POP3_SSL class.
369 '''
370 SSL = True
371 def _connect(self):
372 self.log.trace()
373 if not hasattr(socket, 'ssl'):
374 raise getmailConfigurationError(
375 'SSL not supported by this installation of Python'
376 )
377 (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
378 ca_certs = check_ca_certs(self.conf)
379 ssl_version = check_ssl_version(self.conf)
380 ssl_fingerprints = check_ssl_fingerprints(self.conf)
381 ssl_ciphers = check_ssl_ciphers(self.conf)
382 using_extended_certs_interface = False
383 try:
384 if ca_certs or ssl_version or ssl_ciphers:
385 using_extended_certs_interface = True
386 # Python 2.6 or higher required, use above class instead of
387 # vanilla stdlib one
388 msg = ''
389 if keyfile:
390 msg += 'with keyfile %s, certfile %s' % (keyfile, certfile)
391 if ssl_version:
392 if msg:
393 msg += ', '
394 msg += ('using protocol version %s'
395 % self.conf['ssl_version'].upper())
396 if ca_certs:
397 if msg:
398 msg += ', '
399 msg += 'with ca_certs %s' % ca_certs
400
401 self.log.trace(
402 'establishing POP3 SSL connection to %s:%d %s'
403 % (self.conf['server'], self.conf['port'], msg)
404 + os.linesep
405 )
406 self.conn = POP3_SSL_EXTENDED(
407 self.conf['server'], self.conf['port'], keyfile, certfile,
408 ssl_version, ca_certs, ssl_ciphers
409 )
410 elif keyfile:
411 self.log.trace(
412 'establishing POP3 SSL connection to %s:%d with '
413 'keyfile %s, certfile %s'
414 % (self.conf['server'], self.conf['port'], keyfile,
415 certfile)
416 + os.linesep
417 )
418 self.conn = poplib.POP3_SSL(
419 self.conf['server'], self.conf['port'], keyfile, certfile
420 )
421 else:
422 self.log.trace('establishing POP3 SSL connection to %s:%d'
423 % (self.conf['server'], self.conf['port'])
424 + os.linesep)
425 self.conn = poplib.POP3_SSL(self.conf['server'],
426 self.conf['port'])
427 self.setup_received(self.conn.sock)
428 if ssl and hashlib:
429 sslobj = self.conn.sslobj
430 peercert = sslobj.getpeercert(True)
431 ssl_cipher = sslobj.cipher()
432 if ssl_cipher:
433 ssl_cipher = '%s:%s:%s' % ssl_cipher
434 if not peercert:
435 actual_hash = None
436 else:
437 actual_hash = hashlib.sha256(peercert).hexdigest().lower()
438 else:
439 actual_hash = None
440 ssl_cipher = None
441
442 # Ensure cert is for server we're connecting to
443 if ssl and self.conf['ca_certs']:
444 ssl_match_hostname(
445 self.conn.sslobj.getpeercert(),
446 self.conf.get('ssl_cert_hostname', None)
447 or self.conf['server']
448 )
449
450 if ssl_fingerprints:
451 if not actual_hash:
452 raise getmailOperationError(
453 'socket ssl_fingerprints mismatch (no cert provided)'
454 )
455
456 any_matches = False
457 for expected_hash in ssl_fingerprints:
458 if expected_hash == actual_hash:
459 any_matches = True
460 if not any_matches:
461 raise getmailOperationError(
462 'socket ssl_fingerprints mismatch (got %s)'
463 % actual_hash
464 )
465
466 except poplib.error_proto, o:
467 raise getmailOperationError('POP error (%s)' % o)
468 except socket.timeout:
469 #raise getmailOperationError('timeout during connect')
470 raise
471 except socket.gaierror, o:
472 raise getmailOperationError(
473 'error resolving name %s during connect (%s)'
474 % (self.conf['server'], o)
475 )
476
477 #self.conn.sock.setblocking(1)
478
479 fingerprint_message = ('POP3 SSL connection %s established'
480 % self.conn)
481 if actual_hash:
482 fingerprint_message += ' with fingerprint %s' % actual_hash
483 if ssl_cipher:
484 fingerprint_message += ' using cipher %s' % ssl_cipher
485 fingerprint_message += os.linesep
486
487 if self.app_options.get('fingerprint', False):
488 self.log.info(fingerprint_message)
489 else:
490 self.log.trace(fingerprint_message)
491
492
493 #######################################
494 class Py23POP3SSLinitMixIn(object):
495 '''Mix-In class to do POP3 over SSL initialization with custom-implemented
496 code to support SSL with Python 2.3's poplib.POP3 class.
497 '''
498 SSL = True
499 def _connect(self):
500 self.log.trace()
501 if not hasattr(socket, 'ssl'):
502 raise getmailConfigurationError(
503 'SSL not supported by this installation of Python'
504 )
505 (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
506 ca_certs = check_ca_certs(self.conf)
507 ssl_version = check_ssl_version(self.conf)
508 ssl_fingerprints = check_ssl_fingerprints(self.conf)
509 ssl_ciphers = check_ssl_ciphers(self.conf)
510 if ca_certs or ssl_version or ssl_ciphers or ssl_fingerprints:
511 raise getmailConfigurationError(
512 'SSL extended options are not supported by this'
513 ' installation of Python'
514 )
515 try:
516 if keyfile:
517 self.log.trace(
518 'establishing POP3 SSL connection to %s:%d with keyfile '
519 '%s, certfile %s'
520 % (self.conf['server'], self.conf['port'],
521 keyfile, certfile)
522 + os.linesep
523 )
524 self.conn = POP3SSL(self.conf['server'], self.conf['port'],
525 keyfile, certfile)
526 else:
527 self.log.trace(
528 'establishing POP3 SSL connection to %s:%d'
529 % (self.conf['server'], self.conf['port'])
530 + os.linesep
531 )
532 self.conn = POP3SSL(self.conf['server'], self.conf['port'])
533
534 self.setup_received(self.conn.rawsock)
535 except poplib.error_proto, o:
536 raise getmailOperationError('POP error (%s)' % o)
537 except socket.timeout:
538 #raise getmailOperationError('timeout during connect')
539 raise
540 except socket.gaierror, o:
541 raise getmailOperationError('socket error during connect (%s)' % o)
542 except socket.sslerror, o:
543 raise getmailOperationError(
544 'socket sslerror during connect (%s)' % o
545 )
546
547 self.log.trace('POP3 SSL connection %s established' % self.conn
548 + os.linesep)
549
550
551 #######################################
552 class IMAPinitMixIn(object):
553 '''Mix-In class to do IMAP non-SSL initialization.
554 '''
555 SSL = False
556 def _connect(self):
557 self.log.trace()
558 try:
559 self.conn = imaplib.IMAP4(self.conf['server'], self.conf['port'])
560 self.setup_received(self.conn.sock)
561 except imaplib.IMAP4.error, o:
562 raise getmailOperationError('IMAP error (%s)' % o)
563 except socket.timeout:
564 #raise getmailOperationError('timeout during connect')
565 raise
566 except socket.gaierror, o:
567 raise getmailOperationError('socket error during connect (%s)' % o)
568
569 self.log.trace('IMAP connection %s established' % self.conn
570 + os.linesep)
571
572
573 #######################################
574 class IMAP4_SSL_EXTENDED(imaplib.IMAP4_SSL):
575 # Similar to above, but with extended support for SSL certificate checking,
576 # fingerprints, etc.
577 def __init__(self, host='', port=imaplib.IMAP4_SSL_PORT, keyfile=None,
578 certfile=None, ssl_version=None, ca_certs=None,
579 ssl_ciphers=None):
580 self.ssl_version = ssl_version
581 self.ca_certs = ca_certs
582 self.ssl_ciphers = ssl_ciphers
583 imaplib.IMAP4_SSL.__init__(self, host, port, keyfile, certfile)
584
585 def open(self, host='', port=imaplib.IMAP4_SSL_PORT):
586 self.host = host
587 self.port = port
588 self.sock = socket.create_connection((host, port))
589 extra_args = { 'server_hostname': host }
590 if self.ssl_version:
591 extra_args['ssl_version'] = self.ssl_version
592 if self.ca_certs:
593 extra_args['cert_reqs'] = ssl.CERT_REQUIRED
594 extra_args['ca_certs'] = self.ca_certs
595 if self.ssl_ciphers:
596 extra_args['ciphers'] = self.ssl_ciphers
597
598 self.sslobj = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
599 **extra_args)
600 self.file = self.sslobj.makefile('rb')
601
602
603 #######################################
604 class IMAPSSLinitMixIn(object):
605 '''Mix-In class to do IMAP over SSL initialization.
606 '''
607 SSL = True
608 def _connect(self):
609 self.log.trace()
610 if not hasattr(socket, 'ssl'):
611 raise getmailConfigurationError(
612 'SSL not supported by this installation of Python'
613 )
614 (keyfile, certfile) = check_ssl_key_and_cert(self.conf)
615 ca_certs = check_ca_certs(self.conf)
616 ssl_version = check_ssl_version(self.conf)
617 ssl_fingerprints = check_ssl_fingerprints(self.conf)
618 ssl_ciphers = check_ssl_ciphers(self.conf)
619 using_extended_certs_interface = False
620 try:
621 if ca_certs or ssl_version or ssl_ciphers:
622 using_extended_certs_interface = True
623 # Python 2.6 or higher required, use above class instead of
624 # vanilla stdlib one
625 msg = ''
626 if keyfile:
627 msg += 'with keyfile %s, certfile %s' % (keyfile, certfile)
628 if ssl_version:
629 if msg:
630 msg += ', '
631 msg += ('using protocol version %s'
632 % self.conf['ssl_version'].upper())
633 if ca_certs:
634 if msg:
635 msg += ', '
636 msg += 'with ca_certs %s' % ca_certs
637
638 self.log.trace(
639 'establishing IMAP SSL connection to %s:%d %s'
640 % (self.conf['server'], self.conf['port'], msg)
641 + os.linesep
642 )
643 self.conn = IMAP4_SSL_EXTENDED(
644 self.conf['server'], self.conf['port'], keyfile, certfile,
645 ssl_version, ca_certs, ssl_ciphers
646 )
647 elif keyfile:
648 self.log.trace(
649 'establishing IMAP SSL connection to %s:%d with keyfile '
650 '%s, certfile %s'
651 % (self.conf['server'], self.conf['port'],
652 keyfile, certfile)
653 + os.linesep
654 )
655 self.conn = imaplib.IMAP4_SSL(
656 self.conf['server'], self.conf['port'], keyfile, certfile
657 )
658 else:
659 self.log.trace(
660 'establishing IMAP SSL connection to %s:%d'
661 % (self.conf['server'], self.conf['port']) + os.linesep
662 )
663 self.conn = imaplib.IMAP4_SSL(self.conf['server'],
664 self.conf['port'])
665 self.setup_received(self.conn.sock)
666 if ssl and hashlib:
667 sslobj = self.conn.ssl()
668 peercert = sslobj.getpeercert(True)
669 ssl_cipher = sslobj.cipher()
670 if ssl_cipher:
671 ssl_cipher = '%s:%s:%s' % ssl_cipher
672 if not peercert:
673 actual_hash = None
674 else:
675 actual_hash = hashlib.sha256(peercert).hexdigest().lower()
676 else:
677 actual_hash = None
678 ssl_cipher = None
679
680 # Ensure cert is for server we're connecting to
681 if ssl and self.conf['ca_certs']:
682 ssl_match_hostname(
683 self.conn.ssl().getpeercert(),
684 self.conf.get('ssl_cert_hostname', None)
685 or self.conf['server']
686 )
687
688 if ssl_fingerprints:
689 if not actual_hash:
690 raise getmailOperationError(
691 'socket ssl_fingerprints mismatch (no cert provided)'
692 )
693
694 any_matches = False
695 for expected_hash in ssl_fingerprints:
696 if expected_hash == actual_hash:
697 any_matches = True
698 if not any_matches:
699 raise getmailOperationError(
700 'socket ssl_fingerprints mismatch (got %s)'
701 % actual_hash
702 )
703
704 except imaplib.IMAP4.error, o:
705 raise getmailOperationError('IMAP error (%s)' % o)
706 except socket.timeout:
707 #raise getmailOperationError('timeout during connect')
708 raise
709 except socket.gaierror, o:
710 errcode = o[0]
711 if errcode in (EAI_NONAME, EAI_NODATA):
712 # No such DNS name
713 raise getmailDnsLookupError(
714 'no address for %s (%s)' % (self.conf['server'], o)
715 )
716 elif errcode == EAI_FAIL:
717 # DNS server failure
718 raise getmailDnsServerFailure(
719 'DNS server failure looking up address for %s (%s)'
720 % (self.conf['server'], o)
721 )
722 else:
723 raise getmailOperationError('socket error during connect (%s)'
724 % o)
725 except socket.sslerror, o:
726 raise getmailOperationError(
727 'socket sslerror during connect (%s)' % o
728 )
729
730 fingerprint_message = ('IMAP SSL connection %s established'
731 % self.conn)
732 if actual_hash:
733 fingerprint_message += ' with fingerprint %s' % actual_hash
734 if ssl_cipher:
735 fingerprint_message += ' using cipher %s' % ssl_cipher
736 fingerprint_message += os.linesep
737
738 if self.app_options['fingerprint']:
739 self.log.info(fingerprint_message)
740 else:
741 self.log.trace(fingerprint_message)
742
743 #
744 # Base classes
745 #
746
747 #######################################
748 class RetrieverSkeleton(ConfigurableBase):
749 '''Base class for implementing message-retrieval classes.
750
751 Sub-classes should provide the following data attributes and methods:
752
753 _confitems - a tuple of dictionaries representing the parameters the class
754 takes. Each dictionary should contain the following key,
755 value pairs:
756 - name - parameter name
757 - type - a type function to compare the parameter value
758 against (i.e. str, int, bool)
759 - default - optional default value. If not present, the
760 parameter is required.
761
762 __str__(self) - return a simple string representing the class instance.
763
764 _getmsglist(self) - retieve a list of all available messages, and store
765 unique message identifiers in the dict
766 self.msgnum_by_msgid.
767 Message identifiers must be unique and persistent
768 across instantiations. Also store message sizes (in
769 octets) in a dictionary self.msgsizes, using the
770 message identifiers as keys.
771
772 _delmsgbyid(self, msgid) - delete a message from the message store based
773 on its message identifier.
774
775 _getmsgbyid(self, msgid) - retreive and return a message from the message
776 store based on its message identifier. The
777 message is returned as a Message() class
778 object. The message will have additional data
779 attributes "sender" and "recipient". sender
780 should be present or "unknown". recipient
781 should be non-None if (and only if) the
782 protocol/method of message retrieval preserves
783 the original message envelope.
784
785 _getheaderbyid(self, msgid) - similar to _getmsgbyid() above, but only the
786 message header should be retrieved, if
787 possible. It should be returned in the same
788 format.
789
790 showconf(self) - should invoke self.log.info() to display the
791 configuration of the class instance.
792
793 Sub-classes may also wish to extend or over-ride the following base class
794 methods:
795
796 __init__(self, **args)
797 __del__(self)
798 initialize(self, options)
799 checkconf(self)
800 '''
801 def __init__(self, **args):
802 self.headercache = {}
803 self.deleted = {}
804 self.set_new_timestamp()
805 self.__oldmail_written = False
806 self.__initialized = False
807 self.gotmsglist = False
808 self._clear_state()
809 self.conn = None
810 self.supports_idle = False
811 ConfigurableBase.__init__(self, **args)
812
813 def set_new_timestamp(self):
814 self.timestamp = int(time.time())
815
816 def _clear_state(self):
817 self.msgnum_by_msgid = {}
818 self.msgid_by_msgnum = {}
819 self.sorted_msgnum_msgid = ()
820 self.msgsizes = {}
821 self.oldmail = {}
822 self.__delivered = {}
823 self.deleted = {}
824 self.mailbox_selected = False
825
826 def setup_received(self, sock):
827 serveraddr = sock.getpeername()
828 if len(serveraddr) == 2:
829 # IPv4
830 self.remoteaddr = '%s:%s' % serveraddr
831 elif len(serveraddr) == 4:
832 # IPv6
833 self.remoteaddr = '[%s]:%s' % serveraddr[:2]
834 else:
835 # Shouldn't happen
836 self.log.warning('unexpected peer address format %s' % str(serveraddr))
837 self.remoteaddr = str(serveraddr)
838 self.received_from = '%s (%s)' % (self.conf['server'],
839 self.remoteaddr)
840
841 def __str__(self):
842 self.log.trace()
843 return str(self.conf)
844
845 def list_mailboxes(self):
846 raise NotImplementedError('virtual')
847
848 def select_mailbox(self, mailbox):
849 raise NotImplementedError('virtual')
850
851 def __len__(self):
852 self.log.trace()
853 return len(self.msgnum_by_msgid)
854
855 def __getitem__(self, i):
856 self.log.trace('i == %d' % i)
857 if not self.__initialized:
858 raise getmailOperationError('not initialized')
859 return self.sorted_msgnum_msgid[i][1]
860
861 def _oldmail_filename(self, mailbox):
862 assert (mailbox is None
863 or (isinstance(mailbox, (str, unicode)) and mailbox)), (
864 'bad mailbox %s (%s)' % (mailbox, type(mailbox))
865 )
866 filename = self.oldmail_filename
867 if mailbox is not None:
868 if isinstance(mailbox, str):
869 mailbox = mailbox.decode('utf-8')
870 mailbox = re.sub(STRIP_CHAR_RE, '.', mailbox)
871 mailbox = mailbox.encode('utf-8')
872 # Use oldmail file per IMAP folder
873 filename += '-' + mailbox
874 # else:
875 # mailbox is None, is POP, just use filename
876 return filename
877
878 def oldmail_exists(self, mailbox):
879 '''Test whether an oldmail file exists for a specified mailbox.'''
880 return os.path.isfile(self._oldmail_filename(mailbox))
881
882 def read_oldmailfile(self, mailbox):
883 '''Read contents of an oldmail file. For POP, mailbox must be
884 explicitly None.
885 '''
886 assert not self.oldmail, (
887 'still have %d unflushed oldmail' % len(self.oldmail)
888 )
889 self.log.trace('mailbox=%s' % mailbox)
890
891 filename = self._oldmail_filename(mailbox)
892 logname = '%s:%s' % (self, mailbox or '')
893 try:
894 f = open(filename, 'rb')
895 except IOError:
896 self.log.moreinfo('no oldmail file for %s%s'
897 % (logname, os.linesep))
898 return
899
900 for line in f:
901 line = line.strip()
902 if not line or not '\0' in line:
903 # malformed
904 continue
905 try:
906 (msgid, timestamp) = line.split('\0', 1)
907 if msgid.count('/') == 2:
908 # Was pre-4.22.0 file format, which includes the
909 # mailbox name in the msgid, in the format
910 # 'uidvalidity/mailbox/serveruid'.
911 # Strip it out.
912 fields = msgid.split('/')
913 msgid = '/'.join([fields[0], fields[2]])
914 self.oldmail[msgid] = int(timestamp)
915 except ValueError:
916 # malformed
917 self.log.info(
918 'skipped malformed line "%r" for %s%s'
919 % (line, logname, os.linesep)
920 )
921 self.log.moreinfo(
922 'read %i uids for %s%s'
923 % (len(self.oldmail), logname, os.linesep)
924 )
925 self.log.moreinfo('read %i uids in total for %s%s'
926 % (len(self.oldmail), logname, os.linesep))
927
928 def write_oldmailfile(self, mailbox):
929 '''Write oldmail info to oldmail file.'''
930 self.log.trace('mailbox=%s' % mailbox)
931
932 filename = self._oldmail_filename(mailbox)
933 logname = '%s:%s' % (self, mailbox or '')
934
935 oldmailfile = None
936 wrote = 0
937 msgids = frozenset(
938 self.__delivered.keys()
939 ).union(frozenset(self.oldmail.keys()))
940 try:
941 oldmailfile = updatefile(filename)
942 for msgid in msgids:
943 self.log.debug('msgid %s ...' % msgid)
944 t = self.oldmail.get(msgid, self.timestamp)
945 self.log.debug(' timestamp %s' % t + os.linesep)
946 oldmailfile.write('%s\0%i%s' % (msgid, t, os.linesep))
947 wrote += 1
948 oldmailfile.close()
949 self.log.moreinfo('wrote %i uids for %s%s'
950 % (wrote, logname, os.linesep))
951 except IOError, o:
952 self.log.error('failed writing oldmail file for %s (%s)'
953 % (logname, o) + os.linesep)
954 if oldmailfile:
955 oldmailfile.abort()
956 self.__oldmail_written = True
957
958 def initialize(self, options):
959 # Options - dict of application-wide settings, including ones that
960 # aren't used in initializing the retriever.
961 self.log.trace()
962 self.checkconf()
963 # socket.ssl() and socket timeouts are incompatible in Python 2.3
964 if 'timeout' in self.conf:
965 socket.setdefaulttimeout(self.conf['timeout'])
966 else:
967 # Explicitly set to None in case it was previously set
968 socket.setdefaulttimeout(None)
969
970 # Construct base filename for oldmail files.
971 # strip problematic characters from oldmail filename. Mostly for
972 # non-Unix systems; only / is illegal in a Unix path component
973 oldmail_filename = re.sub(
974 STRIP_CHAR_RE, '-',
975 'oldmail-%(server)s-%(port)i-%(username)s' % self.conf
976 )
977 self.oldmail_filename = os.path.join(self.conf['getmaildir'],
978 oldmail_filename)
979
980 self.received_from = None
981 self.app_options = options
982 self.__initialized = True
983
984 def quit(self):
985 if self.mailbox_selected is not False:
986 self.write_oldmailfile(self.mailbox_selected)
987 self._clear_state()
988
989 def abort(self):
990 '''On error conditions where you do not want modified state to be saved,
991 call this before .quit().
992 '''
993 self._clear_state()
994
995 def delivered(self, msgid):
996 self.__delivered[msgid] = None
997
998 def getheader(self, msgid):
999 if not self.__initialized:
1000 raise getmailOperationError('not initialized')
1001 if not msgid in self.headercache:
1002 self.headercache[msgid] = self._getheaderbyid(msgid)
1003 return self.headercache[msgid]
1004
1005 def getmsg(self, msgid):
1006 if not self.__initialized:
1007 raise getmailOperationError('not initialized')
1008 return self._getmsgbyid(msgid)
1009
1010 def getmsgsize(self, msgid):
1011 if not self.__initialized:
1012 raise getmailOperationError('not initialized')
1013 try:
1014 return self.msgsizes[msgid]
1015 except KeyError:
1016 raise getmailOperationError('no such message ID %s' % msgid)
1017
1018 def delmsg(self, msgid):
1019 if not self.__initialized:
1020 raise getmailOperationError('not initialized')
1021 self._delmsgbyid(msgid)
1022 self.deleted[msgid] = True
1023
1024
1025 #######################################
1026 class POP3RetrieverBase(RetrieverSkeleton):
1027 '''Base class for single-user POP3 mailboxes.
1028 '''
1029 def __init__(self, **args):
1030 RetrieverSkeleton.__init__(self, **args)
1031 self.log.trace()
1032
1033 def select_mailbox(self, mailbox):
1034 assert mailbox is None, (
1035 'POP does not support mailbox selection (%s)' % mailbox
1036 )
1037 if self.mailbox_selected is not False:
1038 self.write_oldmailfile(self.mailbox_selected)
1039
1040 self._clear_state()
1041
1042 if self.oldmail_exists(mailbox):
1043 self.read_oldmailfile(mailbox)
1044 self.mailbox_selected = mailbox
1045
1046 self._getmsglist()
1047
1048 def _getmsgnumbyid(self, msgid):
1049 self.log.trace()
1050 if not msgid in self.msgnum_by_msgid:
1051 raise getmailOperationError('no such message ID %s' % msgid)
1052 return self.msgnum_by_msgid[msgid]
1053
1054 def _getmsglist(self):
1055 self.log.trace()
1056 try:
1057 (response, msglist, octets) = self.conn.uidl()
1058 self.log.debug('UIDL response "%s", %d octets'
1059 % (response, octets) + os.linesep)
1060 for (i, line) in enumerate(msglist):
1061 try:
1062 (msgnum, msgid) = line.split(None, 1)
1063 # Don't allow / in UIDs we store, as we look for that to
1064 # detect old-style oldmail files. Shouldn't occur in POP3
1065 # anyway.
1066 msgid = msgid.replace('/', '-')
1067 except ValueError:
1068 # Line didn't contain two tokens. Server is broken.
1069 raise getmailOperationError(
1070 '%s failed to identify message index %d in UIDL output'
1071 ' -- see documentation or use '
1072 'BrokenUIDLPOP3Retriever instead'
1073 % (self, i)
1074 )
1075 msgnum = int(msgnum)
1076 if msgid in self.msgnum_by_msgid:
1077 # UIDL "unique" identifiers weren't unique.
1078 # Server is broken.
1079 if self.conf.get('delete_dup_msgids', False):
1080 self.log.debug('deleting message %s with duplicate '
1081 'msgid %s' % (msgnum, msgid)
1082 + os.linesep)
1083 self.conn.dele(msgnum)
1084 else:
1085 raise getmailOperationError(
1086 '%s does not uniquely identify messages '
1087 '(got %s twice) -- see documentation or use '
1088 'BrokenUIDLPOP3Retriever instead'
1089 % (self, msgid)
1090 )
1091 else:
1092 self.msgnum_by_msgid[msgid] = msgnum
1093 self.msgid_by_msgnum[msgnum] = msgid
1094 self.log.debug('Message IDs: %s'
1095 % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
1096 self.sorted_msgnum_msgid = sorted(self.msgid_by_msgnum.items())
1097 (response, msglist, octets) = self.conn.list()
1098 for line in msglist:
1099 msgnum = int(line.split()[0])
1100 msgsize = int(line.split()[1])
1101 msgid = self.msgid_by_msgnum.get(msgnum, None)
1102 # If no msgid found, it's a message that wasn't in the UIDL
1103 # response above. Ignore it and we'll get it next time.
1104 if msgid is not None:
1105 self.msgsizes[msgid] = msgsize
1106
1107 # Remove messages from state file that are no longer in mailbox,
1108 # but only if the timestamp for them are old (30 days for now).
1109 # This is because IMAP users can have one state file but multiple
1110 # IMAP folders in different configuration rc files.
1111 for msgid in self.oldmail.keys():
1112 timestamp = self.oldmail[msgid]
1113 age = self.timestamp - timestamp
1114 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
1115 self.log.debug('removing vanished old message id %s' % msgid
1116 + os.linesep)
1117 del self.oldmail[msgid]
1118
1119 except poplib.error_proto, o:
1120 raise getmailOperationError(
1121 'POP error (%s) - if your server does not support the UIDL '
1122 'command, use BrokenUIDLPOP3Retriever instead'
1123 % o
1124 )
1125 self.gotmsglist = True
1126
1127 def _delmsgbyid(self, msgid):
1128 self.log.trace()
1129 msgnum = self._getmsgnumbyid(msgid)
1130 self.conn.dele(msgnum)
1131
1132 def _getmsgbyid(self, msgid):
1133 self.log.debug('msgid %s' % msgid + os.linesep)
1134 msgnum = self._getmsgnumbyid(msgid)
1135 self.log.debug('msgnum %i' % msgnum + os.linesep)
1136 try:
1137 response, lines, octets = self.conn.retr(msgnum)
1138 self.log.debug('RETR response "%s", %d octets'
1139 % (response, octets) + os.linesep)
1140 msg = Message(fromlines=lines+[''])
1141 return msg
1142 except poplib.error_proto, o:
1143 raise getmailRetrievalError(
1144 'failed to retrieve msgid %s; server said %s'
1145 % (msgid, o)
1146 )
1147
1148 def _getheaderbyid(self, msgid):
1149 self.log.trace()
1150 msgnum = self._getmsgnumbyid(msgid)
1151 response, headerlist, octets = self.conn.top(msgnum, 0)
1152 parser = email.Parser.HeaderParser()
1153 return parser.parsestr(os.linesep.join(headerlist))
1154
1155 def initialize(self, options):
1156 self.log.trace()
1157 # POP doesn't support different mailboxes
1158 self.mailboxes = (None, )
1159 # Handle password
1160 if self.conf.get('password', None) is None:
1161 if self.conf.get('password_command', None):
1162 # Retrieve from an arbitrary external command
1163 command = self.conf['password_command'][0]
1164 args = self.conf['password_command'][1:]
1165 (rc, stdout, stderr) = run_command(command, args)
1166 if stderr:
1167 self.log.warning(
1168 'External password program "%s" wrote to stderr: %s'
1169 % (command, stderr)
1170 )
1171 if rc:
1172 # program exited nonzero
1173 raise getmailOperationError(
1174 'External password program error (exited %d)' % rc
1175 )
1176 else:
1177 self.conf['password'] = stdout
1178 else:
1179 self.conf['password'] = get_password(
1180 self, self.conf['username'], self.conf['server'],
1181 self.received_with, self.log
1182 )
1183 RetrieverSkeleton.initialize(self, options)
1184 try:
1185 self._connect()
1186 if self.conf['use_apop']:
1187 self.conn.apop(self.conf['username'], self.conf['password'])
1188 else:
1189 self.conn.user(self.conf['username'])
1190 self.conn.pass_(self.conf['password'])
1191 self._getmsglist()
1192 self.log.debug('msgids: %s'
1193 % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
1194 self.log.debug('msgsizes: %s' % self.msgsizes + os.linesep)
1195 # Remove messages from state file that are no longer in mailbox
1196 for msgid in self.oldmail.keys():
1197 if not self.msgsizes.has_key(msgid):
1198 self.log.debug('removing vanished message id %s' % msgid
1199 + os.linesep)
1200 del self.oldmail[msgid]
1201 except poplib.error_proto, o:
1202 raise getmailOperationError('POP error (%s)' % o)
1203
1204 def abort(self):
1205 self.log.trace()
1206 RetrieverSkeleton.abort(self)
1207 if not self.conn:
1208 return
1209 try:
1210 self.conn.rset()
1211 self.conn.quit()
1212 except (poplib.error_proto, socket.error), o:
1213 pass
1214 self.conn = None
1215
1216 def quit(self):
1217 RetrieverSkeleton.quit(self)
1218 self.log.trace()
1219 if not self.conn:
1220 return
1221 try:
1222 self.conn.quit()
1223 except (poplib.error_proto, socket.error), o:
1224 raise getmailOperationError('POP error (%s)' % o)
1225 except AttributeError:
1226 pass
1227 self.conn = None
1228
1229
1230 #######################################
1231 class MultidropPOP3RetrieverBase(POP3RetrieverBase):
1232 '''Base retriever class for multi-drop POP3 mailboxes.
1233
1234 Envelope is reconstructed from Return-Path: (sender) and a header specified
1235 by the user (recipient). This header is specified with the
1236 "envelope_recipient" parameter, which takes the form <field-name>[:<field-
1237 number>]. field-number defaults to 1 and is counted from top to bottom in
1238 the message. For instance, if the envelope recipient is present in the
1239 second Delivered-To: header field of each message, envelope_recipient should
1240 be specified as "delivered-to:2".
1241 '''
1242
1243 def initialize(self, options):
1244 self.log.trace()
1245 POP3RetrieverBase.initialize(self, options)
1246 self.envrecipname = (
1247 self.conf['envelope_recipient'].split(':')[0].lower()
1248 )
1249 if self.envrecipname in NOT_ENVELOPE_RECIPIENT_HEADERS:
1250 raise getmailConfigurationError(
1251 'the %s header field does not record the envelope '
1252 'recipient address'
1253 % self.envrecipname
1254 )
1255 self.envrecipnum = 0
1256 try:
1257 self.envrecipnum = int(
1258 self.conf['envelope_recipient'].split(':', 1)[1]
1259 ) - 1
1260 if self.envrecipnum < 0:
1261 raise ValueError(self.conf['envelope_recipient'])
1262 except IndexError:
1263 pass
1264 except ValueError, o:
1265 raise getmailConfigurationError(
1266 'invalid envelope_recipient specification format (%s)' % o
1267 )
1268
1269 def _getmsgbyid(self, msgid):
1270 self.log.trace()
1271 msg = POP3RetrieverBase._getmsgbyid(self, msgid)
1272 data = {}
1273 for (name, val) in msg.headers():
1274 name = name.lower()
1275 val = val.strip()
1276 if name in data:
1277 data[name].append(val)
1278 else:
1279 data[name] = [val]
1280
1281 try:
1282 line = data[self.envrecipname][self.envrecipnum]
1283 except (KeyError, IndexError), unused:
1284 raise getmailConfigurationError(
1285 'envelope_recipient specified header missing (%s)'
1286 % self.conf['envelope_recipient']
1287 )
1288 msg.recipient = address_no_brackets(line.strip())
1289 return msg
1290
1291
1292 #######################################
1293 class IMAPRetrieverBase(RetrieverSkeleton):
1294 '''Base class for single-user IMAP mailboxes.
1295 '''
1296 def __init__(self, **args):
1297 RetrieverSkeleton.__init__(self, **args)
1298 self.log.trace()
1299 self.gss_step = 0
1300 self.gss_vc = None
1301 self.gssapi = False
1302
1303 def _clear_state(self):
1304 RetrieverSkeleton._clear_state(self)
1305 self.mailbox = None
1306 self.uidvalidity = None
1307 self.msgnum_by_msgid = {}
1308 self.msgid_by_msgnum = {}
1309 self.sorted_msgnum_msgid = ()
1310 self._mboxuids = {}
1311 self._mboxuidorder = []
1312 self.msgsizes = {}
1313 self.oldmail = {}
1314 self.__delivered = {}
1315
1316 def checkconf(self):
1317 RetrieverSkeleton.checkconf(self)
1318 if self.conf['use_kerberos'] and not HAVE_KERBEROS_GSS:
1319 raise getmailConfigurationError(
1320 'cannot use kerberos authentication; Python kerberos support '
1321 'not installed or does not support GSS'
1322 )
1323
1324 def gssauth(self, response):
1325 if not HAVE_KERBEROS_GSS:
1326 # shouldn't get here
1327 raise ValueError('kerberos GSS support not available')
1328 data = ''.join(str(response).encode('base64').splitlines())
1329 if self.gss_step == GSS_STATE_STEP:
1330 if not self.gss_vc:
1331 (rc, self.gss_vc) = kerberos.authGSSClientInit(
1332 'imap@%s' % self.conf['server']
1333 )
1334 response = kerberos.authGSSClientResponse(self.gss_vc)
1335 rc = kerberos.authGSSClientStep(self.gss_vc, data)
1336 if rc != kerberos.AUTH_GSS_CONTINUE:
1337 self.gss_step = GSS_STATE_WRAP
1338 elif self.gss_step == GSS_STATE_WRAP:
1339 rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
1340 response = kerberos.authGSSClientResponse(self.gss_vc)
1341 rc = kerberos.authGSSClientWrap(self.gss_vc, response,
1342 self.conf['username'])
1343 response = kerberos.authGSSClientResponse(self.gss_vc)
1344 if not response:
1345 response = ''
1346 return response.decode('base64')
1347
1348 def _getmboxuidbymsgid(self, msgid):
1349 self.log.trace()
1350 if not msgid in self.msgnum_by_msgid:
1351 raise getmailOperationError('no such message ID %s' % msgid)
1352 uid = self._mboxuids[msgid]
1353 return uid
1354
1355 def _parse_imapcmdresponse(self, cmd, *args):
1356 self.log.trace()
1357 try:
1358 result, resplist = getattr(self.conn, cmd)(*args)
1359 except imaplib.IMAP4.error, o:
1360 if cmd == 'login':
1361 # Percolate up
1362 raise
1363 else:
1364 raise getmailOperationError('IMAP error (%s)' % o)
1365 if result != 'OK':
1366 raise getmailOperationError(
1367 'IMAP error (command %s returned %s %s)'
1368 % ('%s %s' % (cmd, args), result, resplist)
1369 )
1370 if cmd.lower().startswith('login'):
1371 self.log.debug('login command response %s' % resplist + os.linesep)
1372 else:
1373 self.log.debug(
1374 'command %s response %s'
1375 % ('%s %s' % (cmd, args), resplist)
1376 + os.linesep
1377 )
1378 return resplist
1379
1380 def _parse_imapuidcmdresponse(self, cmd, *args):
1381 self.log.trace()
1382 try:
1383 result, resplist = self.conn.uid(cmd, *args)
1384 except imaplib.IMAP4.error, o:
1385 if cmd == 'login':
1386 # Percolate up
1387 raise
1388 else:
1389 raise getmailOperationError('IMAP error (%s)' % o)
1390 if result != 'OK':
1391 raise getmailOperationError(
1392 'IMAP error (command %s returned %s %s)'
1393 % ('%s %s' % (cmd, args), result, resplist)
1394 )
1395 self.log.debug('command uid %s response %s'
1396 % ('%s %s' % (cmd, args), resplist) + os.linesep)
1397 return resplist
1398
1399 def _parse_imapattrresponse(self, line):
1400 self.log.trace('parsing attributes response line %s' % line
1401 + os.linesep)
1402 r = {}
1403 try:
1404 parts = line[line.index('(') + 1:line.rindex(')')].split()
1405 while parts:
1406 # Flags starts a parenthetical list of valueless flags
1407 if parts[0].lower() == 'flags' and parts[1].startswith('('):
1408 while parts and not parts[0].endswith(')'):
1409 del parts[0]
1410 if parts:
1411 # Last one, ends with ")"
1412 del parts[0]
1413 continue
1414 if len(parts) == 1:
1415 # Leftover part -- not name, value pair.
1416 raise ValueError
1417 name = parts.pop(0).lower()
1418 r[name] = parts.pop(0)
1419 except (ValueError, IndexError, AttributeError), o:
1420 raise getmailOperationError(
1421 'IMAP error (failed to parse attr response line "%s": %s)'
1422 % (line, o)
1423 )
1424 self.log.trace('got %s' % r + os.linesep)
1425 return r
1426
1427 def list_mailboxes(self):
1428 '''List (selectable) IMAP folders in account.'''
1429 mailboxes = []
1430 cmd = ('LIST', )
1431 resplist = self._parse_imapcmdresponse(*cmd)
1432 for item in resplist:
1433 m = IMAP_LISTPARTS.match(item)
1434 if not m:
1435 raise getmailOperationError(
1436 'no match for list response "%s"' % item
1437 )
1438 g = m.groupdict()
1439 attributes = g['attributes'].split()
1440 if r'\Noselect' in attributes:
1441 # Can't select this mailbox, don't include it in output
1442 continue
1443 try:
1444 mailbox = g['mailbox'].decode('imap4-utf-7')
1445 mailboxes.append(mailbox)
1446 #log.debug(u'%20s : delimiter %s, attributes: %s',
1447 # mailbox, g['delimiter'], ', '.join(attributes))
1448 except Exception, o:
1449 raise getmailOperationError('error decoding mailbox "%s"'
1450 % g['mailbox'])
1451 return mailboxes
1452
1453 def close_mailbox(self):
1454 # Close current mailbox so deleted mail is expunged. One getmail
1455 # user had a buggy IMAP server that didn't do the automatic expunge,
1456 # so we do it explicitly here if we've deleted any messages.
1457 if self.deleted:
1458 self.conn.expunge()
1459 self.conn.close()
1460 self.write_oldmailfile(self.mailbox_selected)
1461 # And clear some state
1462 self.mailbox_selected = False
1463 self.mailbox = None
1464 self.uidvalidity = None
1465 self.msgnum_by_msgid = {}
1466 self.msgid_by_msgnum = {}
1467 self.sorted_msgnum_msgid = ()
1468 self._mboxuids = {}
1469 self._mboxuidorder = []
1470 self.msgsizes = {}
1471 self.oldmail = {}
1472 self.__delivered = {}
1473
1474 def select_mailbox(self, mailbox):
1475 self.log.trace()
1476 assert mailbox in self.mailboxes, (
1477 'mailbox not in config (%s)' % mailbox
1478 )
1479 if self.mailbox_selected is not False:
1480 self.close_mailbox()
1481
1482 self._clear_state()
1483
1484 if self.oldmail_exists(mailbox):
1485 self.read_oldmailfile(mailbox)
1486
1487 self.log.debug('selecting mailbox "%s"' % mailbox + os.linesep)
1488 try:
1489 if (self.app_options['delete'] or self.app_options['delete_after']
1490 or self.app_options['delete_bigger_than']):
1491 read_only = False
1492 else:
1493 read_only = True
1494 (status, count) = self.conn.select(mailbox.encode('imap4-utf-7'),
1495 read_only)
1496 if status == 'NO':
1497 # Specified mailbox doesn't exist, no permissions, etc.
1498 raise getmailMailboxSelectError(mailbox)
1499
1500 self.mailbox_selected = mailbox
1501 # use *last* EXISTS returned
1502 count = int(count[-1])
1503 uidvalidity = self.conn.response('UIDVALIDITY')[1][0]
1504 except imaplib.IMAP4.error, o:
1505 raise getmailOperationError('IMAP error (%s)' % o)
1506 except (IndexError, ValueError), o:
1507 raise getmailOperationError(
1508 'IMAP server failed to return correct SELECT response (%s)'
1509 % o
1510 )
1511 self.log.debug('select(%s) returned message count of %d'
1512 % (mailbox, count) + os.linesep)
1513 self.mailbox = mailbox
1514 self.uidvalidity = uidvalidity
1515
1516 self._getmsglist(count)
1517
1518 return count
1519
1520 def _getmsglist(self, msgcount):
1521 self.log.trace()
1522 try:
1523 if msgcount:
1524 # Get UIDs and sizes for all messages in mailbox
1525 response = self._parse_imapcmdresponse(
1526 'FETCH', '1:%d' % msgcount, '(UID RFC822.SIZE)'
1527 )
1528 for line in response:
1529 if not line:
1530 # One user had a server that returned a null response
1531 # somehow -- try to just skip.
1532 continue
1533 r = self._parse_imapattrresponse(line)
1534 # Don't allow / in UIDs we store, as we look for that to
1535 # detect old-style oldmail files. Can occur with IMAP, at
1536 # least with some servers.
1537 uid = r['uid'].replace('/', '-')
1538 msgid = '%s/%s' % (self.uidvalidity, uid)
1539 self._mboxuids[msgid] = r['uid']
1540 self._mboxuidorder.append(msgid)
1541 self.msgnum_by_msgid[msgid] = None
1542 self.msgsizes[msgid] = int(r['rfc822.size'])
1543
1544 # Remove messages from state file that are no longer in mailbox,
1545 # but only if the timestamp for them are old (30 days for now).
1546 # This is because IMAP users can have one state file but multiple
1547 # IMAP folders in different configuration rc files.
1548 for msgid in self.oldmail.keys():
1549 timestamp = self.oldmail[msgid]
1550 age = self.timestamp - timestamp
1551 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
1552 self.log.debug('removing vanished old message id %s' % msgid
1553 + os.linesep)
1554 del self.oldmail[msgid]
1555
1556 except imaplib.IMAP4.error, o:
1557 raise getmailOperationError('IMAP error (%s)' % o)
1558 self.gotmsglist = True
1559
1560 def __getitem__(self, i):
1561 return self._mboxuidorder[i]
1562
1563 def _delmsgbyid(self, msgid):
1564 self.log.trace()
1565 try:
1566 uid = self._getmboxuidbymsgid(msgid)
1567 #self._selectmailbox(mailbox)
1568 # Delete message
1569 if self.conf['move_on_delete']:
1570 self.log.debug('copying message to folder "%s"'
1571 % self.conf['move_on_delete'] + os.linesep)
1572 response = self._parse_imapuidcmdresponse(
1573 'COPY', uid, self.conf['move_on_delete']
1574 )
1575 self.log.debug('deleting message "%s"' % uid + os.linesep)
1576 response = self._parse_imapuidcmdresponse(
1577 'STORE', uid, 'FLAGS', '(\Deleted \Seen)'
1578 )
1579 except imaplib.IMAP4.error, o:
1580 raise getmailOperationError('IMAP error (%s)' % o)
1581
1582 def _getmsgpartbyid(self, msgid, part):
1583 self.log.trace()
1584 try:
1585 uid = self._getmboxuidbymsgid(msgid)
1586 # Retrieve message
1587 self.log.debug('retrieving body for message "%s"' % uid
1588 + os.linesep)
1589 try:
1590 response = self._parse_imapuidcmdresponse('FETCH', uid, part)
1591 except (imaplib.IMAP4.error, getmailOperationError), o:
1592 # server gave a negative/NO response, most likely. Bad server,
1593 # no doughnut.
1594 raise getmailRetrievalError(
1595 'failed to retrieve msgid %s; server said %s'
1596 % (msgid, o)
1597 )
1598 # Response is really ugly:
1599 #
1600 # [
1601 # (
1602 # '1 (UID 1 RFC822 {704}',
1603 # 'message text here with CRLF EOL'
1604 # ),
1605 # ')',
1606 # <maybe more>
1607 # ]
1608
1609 # MSExchange is broken -- if a message is badly formatted enough
1610 # (virus, spam, trojan), it can completely fail to return the
1611 # message when requested.
1612 try:
1613 try:
1614 sbody = response[0][1]
1615 except Exception, o:
1616 sbody = None
1617 if not sbody:
1618 self.log.error('bad message from server!')
1619 sbody = str(response)
1620 msg = Message(fromstring=sbody)
1621 except TypeError, o:
1622 # response[0] is None instead of a message tuple
1623 raise getmailRetrievalError('failed to retrieve msgid %s'
1624 % msgid)
1625
1626 # record mailbox retrieved from in a header
1627 if self.conf['record_mailbox']:
1628 msg.add_header('X-getmail-retrieved-from-mailbox',
1629 self.mailbox_selected)
1630
1631 # google extensions: apply labels, etc
1632 if 'X-GM-EXT-1' in self.conn.capabilities:
1633 metadata = self._getgmailmetadata(uid, msg)
1634 for (header, value) in metadata.items():
1635 msg.add_header(header, value)
1636
1637 return msg
1638
1639 except imaplib.IMAP4.error, o:
1640 raise getmailOperationError('IMAP error (%s)' % o)
1641
1642 def _getgmailmetadata(self, uid, msg):
1643 """
1644 Add Gmail labels and other metadata which Google exposes through an
1645 IMAP extension to headers in the message.
1646
1647 See https://developers.google.com/google-apps/gmail/imap_extensions
1648 """
1649 try:
1650 # ['976 (X-GM-THRID 1410134259107225671 X-GM-MSGID '
1651 # '1410134259107225671 X-GM-LABELS (labels space '
1652 # 'separated) UID 167669)']
1653 response = self._parse_imapuidcmdresponse('FETCH', uid,
1654 '(X-GM-LABELS X-GM-THRID X-GM-MSGID)')
1655 except imaplib.IMAP4.error, o:
1656 self.log.warning('Could not fetch google imap extensions: %s' % o)
1657 return {}
1658
1659 if not response or not response[0]:
1660 return {}
1661
1662 ext = re.search(
1663 'X-GM-THRID (?P<THRID>\d+) X-GM-MSGID (?P<MSGID>\d+)'
1664 ' X-GM-LABELS \((?P<LABELS>.*)\) UID',
1665 response[0]
1666 )
1667 if not ext:
1668 self.log.warning(
1669 'Could not parse google imap extensions. Server said: %s'
1670 % repr(response)
1671 )
1672 return {}
1673
1674 results = ext.groupdict()
1675 metadata = {}
1676 for item in ('LABELS', 'THRID', 'MSGID'):
1677 if item in results and results[item]:
1678 metadata['X-GMAIL-%s' % item] = results[item]
1679
1680 return metadata
1681
1682 def _getmsgbyid(self, msgid):
1683 self.log.trace()
1684 if self.conf.get('use_peek', True):
1685 part = '(BODY.PEEK[])'
1686 else:
1687 part = '(RFC822)'
1688 return self._getmsgpartbyid(msgid, part)
1689
1690 def _getheaderbyid(self, msgid):
1691 self.log.trace()
1692 if self.conf.get('use_peek', True):
1693 part = '(BODY.PEEK[header])'
1694 else:
1695 part = '(RFC822[header])'
1696 return self._getmsgpartbyid(msgid, part)
1697
1698 def initialize(self, options):
1699 self.log.trace()
1700 self.mailboxes = self.conf.get('mailboxes', ('INBOX', ))
1701 # Handle password
1702 if (self.conf.get('password', None) is None
1703 and not (HAVE_KERBEROS_GSS and self.conf['use_kerberos'])):
1704 if self.conf['password_command']:
1705 # Retrieve from an arbitrary external command
1706 command = self.conf['password_command'][0]
1707 args = self.conf['password_command'][1:]
1708 (rc, stdout, stderr) = run_command(command, args)
1709 if stderr:
1710 self.log.warning(
1711 'External password program "%s" wrote to stderr: %s'
1712 % (command, stderr)
1713 )
1714 if rc:
1715 # program exited nonzero
1716 raise getmailOperationError(
1717 'External password program error (exited %d)' % rc
1718 )
1719 else:
1720 self.conf['password'] = stdout
1721 else:
1722 self.conf['password'] = get_password(
1723 self, self.conf['username'], self.conf['server'],
1724 self.received_with, self.log
1725 )
1726
1727 RetrieverSkeleton.initialize(self, options)
1728 try:
1729 self.log.trace('trying self._connect()' + os.linesep)
1730 self._connect()
1731 try:
1732 self.log.trace('logging in' + os.linesep)
1733 if self.conf['use_kerberos'] and HAVE_KERBEROS_GSS:
1734 self.conn.authenticate('GSSAPI', self.gssauth)
1735 elif self.conf['use_cram_md5']:
1736 self._parse_imapcmdresponse(
1737 'login_cram_md5', self.conf['username'],
1738 self.conf['password']
1739 )
1740 elif self.conf['use_xoauth2']:
1741 # octal 1 / ctrl-A used as separator
1742 auth = 'user=%s\1auth=Bearer %s\1\1' % (self.conf['username'],
1743 self.conf['password'])
1744 self.conn.authenticate('XOAUTH2', lambda unused: auth)
1745 else:
1746 self._parse_imapcmdresponse('login', self.conf['username'],
1747 self.conf['password'])
1748 except imaplib.IMAP4.abort, o:
1749 raise getmailLoginRefusedError(o)
1750 except imaplib.IMAP4.error, o:
1751 if str(o).startswith('[UNAVAILABLE]'):
1752 raise getmailLoginRefusedError(o)
1753 else:
1754 raise getmailCredentialError(o)
1755
1756 self.log.trace('logged in' + os.linesep)
1757 """
1758 self.log.trace('logged in, getting message list' + os.linesep)
1759 self._getmsglist()
1760 self.log.debug('msgids: %s'
1761 % sorted(self.msgnum_by_msgid.keys()) + os.linesep)
1762 self.log.debug('msgsizes: %s' % self.msgsizes + os.linesep)
1763 # Remove messages from state file that are no longer in mailbox,
1764 # but only if the timestamp for them are old (30 days for now).
1765 # This is because IMAP users can have one state file but multiple
1766 # IMAP folders in different configuration rc files.
1767 for msgid in self.oldmail.keys():
1768 timestamp = self.oldmail[msgid]
1769 age = self.timestamp - timestamp
1770 if not self.msgsizes.has_key(msgid) and age > VANISHED_AGE:
1771 self.log.debug('removing vanished old message id %s' % msgid
1772 + os.linesep)
1773 del self.oldmail[msgid]
1774 """
1775 # Some IMAP servers change the available capabilities after
1776 # authentication, i.e. they present a limited set before login.
1777 # The Python stlib IMAP4 class doesn't take this into account
1778 # and just checks the capabilities immediately after connecting.
1779 # Force a re-check now that we've authenticated.
1780 (typ, dat) = self.conn.capability()
1781 if dat == [None]:
1782 # No response, don't update the stored capabilities
1783 self.log.warning('no post-login CAPABILITY response from server\n')
1784 else:
1785 self.conn.capabilities = tuple(dat[-1].upper().split())
1786
1787 if 'IDLE' in self.conn.capabilities:
1788 self.supports_idle = True
1789 imaplib.Commands['IDLE'] = ('AUTH', 'SELECTED')
1790
1791 if self.mailboxes == ('ALL', ):
1792 # Special value meaning all mailboxes in account
1793 self.mailboxes = tuple(self.list_mailboxes())
1794
1795 except imaplib.IMAP4.error, o:
1796 raise getmailOperationError('IMAP error (%s)' % o)
1797
1798 def abort(self):
1799 self.log.trace()
1800 RetrieverSkeleton.abort(self)
1801 if not self.conn:
1802 return
1803 try:
1804 self.quit()
1805 except (imaplib.IMAP4.error, socket.error), o:
1806 pass
1807 self.conn = None
1808
1809 def go_idle(self, folder, timeout=300):
1810 """Initiates IMAP's IDLE mode if the server supports it
1811
1812 Waits until state of current mailbox changes, and then returns. Returns
1813 True if the connection still seems to be up, False otherwise.
1814
1815 May throw getmailOperationError if the server refuses the IDLE setup
1816 (e.g. if the server does not support IDLE)
1817
1818 Default timeout is 5 minutes.
1819 """
1820
1821 if not self.supports_idle:
1822 self.log.warning('IDLE not supported, so not idling\n')
1823 raise getmailOperationError(
1824 'IMAP4 IDLE requested, but not supported by server'
1825 )
1826
1827
1828 if self.SSL:
1829 sock = self.conn.ssl()
1830 else:
1831 sock = self.conn.socket()
1832
1833 # Based on current imaplib IDLE patch: http://bugs.python.org/issue11245
1834 self.conn.untagged_responses = {}
1835 self.conn.select(folder)
1836 tag = self.conn._command('IDLE')
1837 data = self.conn._get_response() # read continuation response
1838
1839 if data is not None:
1840 raise getmailOperationError(
1841 'IMAP4 IDLE requested, but server refused IDLE request: %s'
1842 % data
1843 )
1844
1845 self.log.debug('Entering IDLE mode (server says "%s")\n'
1846 % self.conn.continuation_response)
1847
1848 try:
1849 aborted = None
1850 (readable, unused, unused) = select.select([sock], [], [], timeout)
1851 except KeyboardInterrupt, o:
1852 # Delay raising this until we've stopped IDLE mode
1853 aborted = o
1854
1855 if aborted is not None:
1856 self.log.debug('IDLE mode cancelled\n')
1857 elif readable:
1858 # The socket has data waiting; server has updated status
1859 self.log.info('IDLE message received\n')
1860 else:
1861 self.log.debug('IDLE timeout (%ds)\n' % timeout)
1862
1863 try:
1864 self.conn.untagged_responses = {}
1865 self.conn.send('DONE\r\n')
1866 self.conn._command_complete('IDLE', tag)
1867 except imaplib.IMAP4.error, o:
1868 return False
1869
1870 if aborted:
1871 raise aborted
1872
1873 return True
1874
1875 def quit(self):
1876 self.log.trace()
1877 if not self.conn:
1878 return
1879 try:
1880 if self.mailbox_selected is not False:
1881 self.close_mailbox()
1882 self.conn.logout()
1883 except imaplib.IMAP4.error, o:
1884 #raise getmailOperationError('IMAP error (%s)' % o)
1885 self.log.warning('IMAP error during logout (%s)' % o + os.linesep)
1886 RetrieverSkeleton.quit(self)
1887 self.conn = None
1888
1889
1890 #######################################
1891 class MultidropIMAPRetrieverBase(IMAPRetrieverBase):
1892 '''Base retriever class for multi-drop IMAP mailboxes.
1893
1894 Envelope is reconstructed from Return-Path: (sender) and a header specified
1895 by the user (recipient). This header is specified with the
1896 "envelope_recipient" parameter, which takes the form <field-name>[:<field-
1897 number>]. field-number defaults to 1 and is counted from top to bottom in
1898 the message. For instance, if the envelope recipient is present in the
1899 second Delivered-To: header field of each message, envelope_recipient should
1900 be specified as "delivered-to:2".
1901 '''
1902
1903 def initialize(self, options):
1904 self.log.trace()
1905 IMAPRetrieverBase.initialize(self, options)
1906 self.envrecipname = (self.conf['envelope_recipient'].split(':')
1907 [0].lower())
1908 if self.envrecipname in NOT_ENVELOPE_RECIPIENT_HEADERS:
1909 raise getmailConfigurationError(
1910 'the %s header field does not record the envelope recipient '
1911 'address'
1912 % self.envrecipname
1913 )
1914 self.envrecipnum = 0
1915 try:
1916 self.envrecipnum = int(
1917 self.conf['envelope_recipient'].split(':', 1)[1]
1918 ) - 1
1919 if self.envrecipnum < 0:
1920 raise ValueError(self.conf['envelope_recipient'])
1921 except IndexError:
1922 pass
1923 except ValueError, o:
1924 raise getmailConfigurationError(
1925 'invalid envelope_recipient specification format (%s)' % o
1926 )
1927
1928 def _getmsgbyid(self, msgid):
1929 self.log.trace()
1930 msg = IMAPRetrieverBase._getmsgbyid(self, msgid)
1931 data = {}
1932 for (name, encoded_value) in msg.headers():
1933 name = name.lower()
1934 for (val, encoding) in decode_header(encoded_value):
1935 val = val.strip()
1936 if name in data:
1937 data[name].append(val)
1938 else:
1939 data[name] = [val]
1940
1941 try:
1942 line = data[self.envrecipname][self.envrecipnum]
1943 except (KeyError, IndexError), unused:
1944 raise getmailConfigurationError(
1945 'envelope_recipient specified header missing (%s)'
1946 % self.conf['envelope_recipient']
1947 )
1948 msg.recipient = address_no_brackets(line.strip())
1949 return msg
1950
1951
1952 # Choose right POP-over-SSL mix-in based on Python version being used.
1953 if sys.hexversion >= 0x02040000:
1954 POP3SSLinitMixIn = Py24POP3SSLinitMixIn
1955 else:
1956 POP3SSLinitMixIn = Py23POP3SSLinitMixIn