"Fossies" - the Fresh Open Source Software Archive 
Member "getmail-5.16/getmailcore/utilities.py" (31 Oct 2021, 24460 Bytes) of package /linux/misc/getmail-5.16.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style:
standard) with prefixed line numbers.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "utilities.py" see the
Fossies "Dox" file reference documentation and the latest
Fossies "Diffs" side-by-side code changes report:
5.15_vs_5.16.
1 #!/usr/bin/env python2
2 '''Utility classes and functions for getmail.
3 '''
4
5 __all__ = [
6 'address_no_brackets',
7 'change_usergroup',
8 'change_uidgid',
9 'decode_crappy_text',
10 'format_header',
11 'check_ssl_key_and_cert',
12 'check_ca_certs',
13 'check_ssl_version',
14 'check_ssl_fingerprints',
15 'check_ssl_ciphers',
16 'deliver_maildir',
17 'eval_bool',
18 'expand_user_vars',
19 'is_maildir',
20 'localhostname',
21 'lock_file',
22 'logfile',
23 'mbox_from_escape',
24 'safe_open',
25 'unlock_file',
26 'gid_of_uid',
27 'uid_of_user',
28 'updatefile',
29 'get_password',
30 'run_command',
31 ]
32
33
34 import os
35 import os.path
36 import socket
37 import signal
38 import stat
39 import time
40 import glob
41 import re
42 import fcntl
43 import pwd
44 import grp
45 import getpass
46 import commands
47 import sys
48 import tempfile
49 import errno
50 try:
51 import subprocess
52 except ImportError, o:
53 subprocess = None
54
55 # hashlib only present in python2.5, ssl in python2.6; used together
56 # in SSL functionality below
57 try:
58 import ssl
59 except ImportError:
60 ssl = None
61 try:
62 import hashlib
63 except ImportError:
64 hashlib = None
65
66 # Optional gnome-keyring integration
67 try:
68 import gnomekeyring
69 # And test to see if it's actually available
70 if not gnomekeyring.is_available():
71 gnomekeyring = None
72 except ImportError:
73 gnomekeyring = None
74
75 from getmailcore.exceptions import *
76
77 logtimeformat = '%Y-%m-%d %H:%M:%S'
78 _bool_values = {
79 'true' : True,
80 'yes' : True,
81 'on' : True,
82 '1' : True,
83 'false' : False,
84 'no' : False,
85 'off' : False,
86 '0' : False
87 }
88 osx_keychain_binary = '/usr/bin/security'
89
90
91 #######################################
92 def lock_file(file, locktype):
93 '''Do file locking.'''
94 assert locktype in ('lockf', 'flock'), 'unknown lock type %s' % locktype
95 if locktype == 'lockf':
96 fcntl.lockf(file, fcntl.LOCK_EX)
97 elif locktype == 'flock':
98 fcntl.flock(file, fcntl.LOCK_EX)
99
100 #######################################
101 def unlock_file(file, locktype):
102 '''Do file unlocking.'''
103 assert locktype in ('lockf', 'flock'), 'unknown lock type %s' % locktype
104 if locktype == 'lockf':
105 fcntl.lockf(file, fcntl.LOCK_UN)
106 elif locktype == 'flock':
107 fcntl.flock(file, fcntl.LOCK_UN)
108
109 #######################################
110 def safe_open(path, mode, permissions=0600):
111 '''Open a file path safely.
112 '''
113 if os.name != 'posix':
114 return open(path, mode)
115 try:
116 fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_EXCL, permissions)
117 file = os.fdopen(fd, mode)
118 except OSError, o:
119 raise getmailDeliveryError('failure opening %s (%s)' % (path, o))
120 return file
121
122 #######################################
123 class updatefile(object):
124 '''A class for atomically updating files.
125
126 A new, temporary file is created when this class is instantiated. When the
127 object's close() method is called, the file is synced to disk and atomically
128 renamed to replace the original file. close() is automatically called when
129 the object is deleted.
130 '''
131 def __init__(self, filename):
132 self.closed = False
133 self.filename = filename
134 self.tmpname = filename + '.tmp.%d' % os.getpid()
135 # If the target is a symlink, the rename-on-close semantics of this
136 # class would break the symlink, replacing it with the new file.
137 # Instead, follow the symlink here, and replace the target file on
138 # close.
139 while os.path.islink(filename):
140 filename = os.path.join(os.path.dirname(filename),
141 os.readlink(filename))
142 try:
143 f = safe_open(self.tmpname, 'wb')
144 except IOError, (code, msg):
145 raise IOError('%s, opening output file "%s"' % (msg, self.tmpname))
146 self.file = f
147 self.write = f.write
148 self.flush = f.flush
149
150 def __del__(self):
151 self.close()
152
153 def abort(self):
154 try:
155 if hasattr(self, 'file'):
156 self.file.close()
157 except IOError:
158 pass
159 self.closed = True
160
161 def close(self):
162 if self.closed or not hasattr(self, 'file'):
163 return
164 self.file.flush()
165 os.fsync(self.file.fileno())
166 self.file.close()
167 os.rename(self.tmpname, self.filename)
168 self.closed = True
169
170 #######################################
171 class logfile(object):
172 '''A class for locking and appending timestamped data lines to a log file.
173 '''
174 def __init__(self, filename):
175 self.closed = False
176 self.filename = filename
177 try:
178 self.file = open(expand_user_vars(self.filename), 'ab')
179 except IOError, (code, msg):
180 raise IOError('%s, opening file "%s"' % (msg, self.filename))
181
182 def __del__(self):
183 self.close()
184
185 def __str__(self):
186 return 'logfile(filename="%s")' % self.filename
187
188 def close(self):
189 if self.closed:
190 return
191 self.file.flush()
192 self.file.close()
193 self.closed = True
194
195 def write(self, s):
196 try:
197 lock_file(self.file, 'flock')
198 # Seek to end
199 self.file.seek(0, 2)
200 self.file.write(time.strftime(logtimeformat, time.localtime())
201 + ' ' + s.rstrip() + os.linesep)
202 self.file.flush()
203 finally:
204 unlock_file(self.file, 'flock')
205
206 #######################################
207 def format_params(d, maskitems=('password', ), skipitems=()):
208 '''Take a dictionary of parameters and return a string summary.
209 '''
210 s = ''
211 keys = d.keys()
212 keys.sort()
213 for key in keys:
214 if key in skipitems:
215 continue
216 if s:
217 s += ','
218 if key in maskitems:
219 s += '%s=*' % key
220 else:
221 s += '%s="%s"' % (key, d[key])
222 return s
223
224 ###################################
225 def alarm_handler(*unused):
226 '''Handle an alarm during maildir delivery.
227
228 Should never happen.
229 '''
230 raise getmailDeliveryError('Delivery timeout')
231
232 #######################################
233 def is_maildir(d):
234 '''Verify a path is a maildir.
235 '''
236 dir_parent = os.path.dirname(d.endswith('/') and d[:-1] or d)
237 if not os.access(dir_parent, os.X_OK):
238 raise getmailConfigurationError(
239 'cannot read contents of parent directory of %s '
240 '- check permissions and ownership' % d
241 )
242 if not os.path.isdir(d):
243 return False
244 if not os.access(d, os.X_OK):
245 raise getmailConfigurationError(
246 'cannot read contents of directory %s '
247 '- check permissions and ownership' % d
248 )
249 for sub in ('tmp', 'cur', 'new'):
250 subdir = os.path.join(d, sub)
251 if not os.path.isdir(subdir):
252 return False
253 if not os.access(subdir, os.W_OK):
254 raise getmailConfigurationError(
255 'cannot write to maildir %s '
256 '- check permissions and ownership' % d
257 )
258 return True
259
260 #######################################
261 def deliver_maildir(maildirpath, data, hostname, dcount=None, filemode=0600):
262 '''Reliably deliver a mail message into a Maildir. Uses Dan Bernstein's
263 documented rules for maildir delivery, and the updated naming convention
264 for new files (modern delivery identifiers). See
265 http://cr.yp.to/proto/maildir.html and
266 http://qmail.org/man/man5/maildir.html for details.
267 '''
268 if not is_maildir(maildirpath):
269 raise getmailDeliveryError('not a Maildir (%s)' % maildirpath)
270
271 # Set a 24-hour alarm for this delivery
272 signal.signal(signal.SIGALRM, alarm_handler)
273 signal.alarm(24 * 60 * 60)
274
275 info = {
276 'deliverycount' : dcount,
277 'hostname' : hostname.split('.')[0].replace('/', '\\057').replace(
278 ':', '\\072'),
279 'pid' : os.getpid(),
280 }
281 dir_tmp = os.path.join(maildirpath, 'tmp')
282 dir_new = os.path.join(maildirpath, 'new')
283
284 for unused in range(3):
285 t = time.time()
286 info['secs'] = int(t)
287 info['usecs'] = int((t - int(t)) * 1000000)
288 info['unique'] = 'M%(usecs)dP%(pid)s' % info
289 if info['deliverycount'] is not None:
290 info['unique'] += 'Q%(deliverycount)s' % info
291 try:
292 info['unique'] += 'R%s' % ''.join(
293 ['%02x' % ord(char)
294 for char in open('/dev/urandom', 'rb').read(8)]
295 )
296 except StandardError:
297 pass
298
299 filename = '%(secs)s.%(unique)s.%(hostname)s' % info
300 fname_tmp = os.path.join(dir_tmp, filename)
301 fname_new = os.path.join(dir_new, filename)
302
303 # File must not already exist
304 if os.path.exists(fname_tmp):
305 # djb says sleep two seconds and try again
306 time.sleep(2)
307 continue
308
309 # Be generous and check cur/file[:...] just in case some other, dumber
310 # MDA is in use. We wouldn't want them to clobber us and have the user
311 # blame us for their bugs.
312 curpat = os.path.join(maildirpath, 'cur', filename) + ':*'
313 collision = glob.glob(curpat)
314 if collision:
315 # There is a message in maildir/cur/ which could be clobbered by
316 # a dumb MUA, and which shouldn't be there. Abort.
317 raise getmailDeliveryError('collision with %s' % collision)
318
319 # Found an unused filename
320 break
321 else:
322 signal.alarm(0)
323 raise getmailDeliveryError('failed to allocate file in maildir')
324
325 # Get user & group of maildir
326 s_maildir = os.stat(maildirpath)
327
328 # Open file to write
329 try:
330 f = safe_open(fname_tmp, 'wb', filemode)
331 f.write(data)
332 f.flush()
333 os.fsync(f.fileno())
334 f.close()
335
336 except IOError, o:
337 signal.alarm(0)
338 raise getmailDeliveryError('failure writing file %s (%s)'
339 % (fname_tmp, o))
340
341 # Move message file from Maildir/tmp to Maildir/new
342 try:
343 os.link(fname_tmp, fname_new)
344 os.unlink(fname_tmp)
345
346 except OSError:
347 signal.alarm(0)
348 try:
349 os.unlink(fname_tmp)
350 except KeyboardInterrupt:
351 raise
352 except StandardError:
353 pass
354 raise getmailDeliveryError('failure renaming "%s" to "%s"'
355 % (fname_tmp, fname_new))
356
357 # Delivery done
358
359 # Cancel alarm
360 signal.alarm(0)
361 signal.signal(signal.SIGALRM, signal.SIG_DFL)
362
363 return filename
364
365 #######################################
366 def mbox_from_escape(s):
367 '''Escape spaces, tabs, and newlines in the envelope sender address.'''
368 return ''.join([(c in (' ', '\t', '\n')) and '-' or c for c in s]) or '<>'
369
370 #######################################
371 def address_no_brackets(addr):
372 '''Strip surrounding <> on an email address, if present.'''
373 if addr.startswith('<') and addr.endswith('>'):
374 return addr[1:-1]
375 else:
376 return addr
377
378 #######################################
379 def eval_bool(s):
380 '''Handle boolean values intelligently.
381 '''
382 try:
383 return _bool_values[str(s).lower()]
384 except KeyError:
385 raise getmailConfigurationError(
386 'boolean parameter requires value to be one of true or false, '
387 'not "%s"' % s
388 )
389
390 #######################################
391 def gid_of_uid(uid):
392 try:
393 return pwd.getpwuid(uid).pw_gid
394 except KeyError, o:
395 raise getmailConfigurationError('no such specified uid (%s)' % o)
396
397 #######################################
398 def uid_of_user(user):
399 try:
400 return pwd.getpwnam(user).pw_uid
401 except KeyError, o:
402 raise getmailConfigurationError('no such specified user (%s)' % o)
403
404 #######################################
405 def change_usergroup(logger=None, user=None, _group=None):
406 '''
407 Change the current effective GID and UID to those specified by user and
408 _group.
409 '''
410 uid = None
411 gid = None
412 if _group:
413 if logger:
414 logger.debug('Getting GID for specified group %s\n' % _group)
415 try:
416 gid = grp.getgrnam(_group).gr_gid
417 except KeyError, o:
418 raise getmailConfigurationError('no such specified group (%s)' % o)
419 if user:
420 if logger:
421 logger.debug('Getting UID for specified user %s\n' % user)
422 uid = uid_of_user(user)
423
424 change_uidgid(logger, uid, gid)
425
426 #######################################
427 def change_uidgid(logger=None, uid=None, gid=None):
428 '''
429 Change the current effective GID and UID to those specified by uid
430 and gid.
431 '''
432 try:
433 if gid:
434 if os.getegid() != gid:
435 if logger:
436 logger.debug('Setting egid to %d\n' % gid)
437 os.setregid(gid, gid)
438 if uid:
439 if os.geteuid() != uid:
440 if logger:
441 logger.debug('Setting euid to %d\n' % uid)
442 os.setreuid(uid, uid)
443 except OSError, o:
444 raise getmailDeliveryError('change UID/GID to %s/%s failed (%s)'
445 % (uid, gid, o))
446
447 #######################################
448 def decode_crappy_text(s):
449 '''Take a line of text in arbitrary and possibly broken bytestring encoding
450 and return an ASCII or unicode version of it.
451 '''
452 # first, assume it was written in the encoding of the user's terminal
453 lang = os.environ.get('LANG')
454 if lang:
455 try:
456 (lang, encoding) = lang.split('.')
457 return s.decode(encoding)
458 except (UnicodeError, ValueError), o:
459 pass
460 # that failed; try well-formed in various common encodings next
461 for encoding in ('ascii', 'utf-8', 'latin-1', 'utf-16'):
462 try:
463 return s.decode(encoding)
464 except UnicodeError, o:
465 continue
466 # all failed - force it
467 return s.decode('utf-8', 'replace')
468
469
470 #######################################
471 def format_header(name, line):
472 '''Take a long line and return rfc822-style multiline header.
473 '''
474 header = ''
475 line = (name.strip() + ': '
476 + ' '.join([part.strip() for part in line.splitlines()]))
477 # Split into lines of maximum 78 characters long plus newline, if
478 # possible. A long line may result if no space characters are present.
479 while line and len(line) > 78:
480 i = line.rfind(' ', 0, 78)
481 if i == -1:
482 # No space in first 78 characters, try a long line
483 i = line.rfind(' ')
484 if i == -1:
485 # No space at all
486 break
487 if header:
488 header += os.linesep + ' '
489 header += line[:i]
490 line = line[i:].lstrip()
491 if header:
492 header += os.linesep + ' '
493 if line:
494 header += line.strip() + os.linesep
495 return header
496
497 #######################################
498 def expand_user_vars(s):
499 '''Return a string expanded for both leading "~/" or "~username/" and
500 environment variables in the form "$varname" or "${varname}".
501 '''
502 return os.path.expanduser(os.path.expandvars(s))
503
504 #######################################
505 def localhostname():
506 '''Return a name for localhost which is (hopefully) the "correct" FQDN.
507 '''
508 n = socket.gethostname()
509 if '.' in n:
510 return n
511 return socket.getfqdn()
512
513 #######################################
514 def check_ssl_key_and_cert(conf):
515 keyfile = conf['keyfile']
516 if keyfile is not None:
517 keyfile = expand_user_vars(keyfile)
518 certfile = conf['certfile']
519 if certfile is not None:
520 certfile = expand_user_vars(certfile)
521 if keyfile and not os.path.isfile(keyfile):
522 raise getmailConfigurationError(
523 'optional keyfile must be path to a valid file'
524 )
525 if certfile and not os.path.isfile(certfile):
526 raise getmailConfigurationError(
527 'optional certfile must be path to a valid file'
528 )
529 if (keyfile is None) ^ (certfile is None):
530 raise getmailConfigurationError(
531 'optional certfile and keyfile must be supplied together'
532 )
533 return (keyfile, certfile)
534
535 #######################################
536 def check_ca_certs(conf):
537 ca_certs = conf['ca_certs']
538 if ca_certs is not None:
539 ca_certs = expand_user_vars(ca_certs)
540 if ssl is None:
541 raise getmailConfigurationError(
542 'specifying ca_certs not supported by this installation of '
543 'Python; requires Python 2.6'
544 )
545 if ca_certs and not os.path.isfile(ca_certs):
546 raise getmailConfigurationError(
547 'optional ca_certs must be path to a valid file'
548 )
549 return ca_certs
550
551 #######################################
552 def check_ssl_version(conf):
553 ssl_version = conf['ssl_version']
554 if ssl_version is None:
555 return None
556 if ssl is None:
557 raise getmailConfigurationError(
558 'specifying ssl_version not supported by this installation of '
559 'Python; requires Python 2.6'
560 )
561 def get_or_fail(version, symbol):
562 if symbol is not None:
563 v = getattr(ssl, symbol, None)
564 if v is not None:
565 return v
566 raise getmailConfigurationError(
567 'unknown or unsupported ssl_version "%s"' % version
568 )
569
570 ssl_version = ssl_version.lower()
571 if ssl_version == 'sslv23':
572 return get_or_fail(ssl_version, 'PROTOCOL_SSLv23')
573 elif ssl_version == 'sslv3':
574 return get_or_fail(ssl_version, 'PROTOCOL_SSLv3')
575 elif ssl_version == 'tlsv1':
576 return get_or_fail(ssl_version, 'PROTOCOL_TLSv1')
577 elif ssl_version == 'tlsv1_1' and 'PROTOCOL_TLSv1_1' in dir(ssl):
578 return get_or_fail(ssl_version, 'PROTOCOL_TLSv1_1')
579 elif ssl_version == 'tlsv1_2' and 'PROTOCOL_TLSv1_2' in dir(ssl):
580 return get_or_fail(ssl_version, 'PROTOCOL_TLSv1_2')
581 return get_or_fail(ssl_version, None)
582
583 #######################################
584 def check_ssl_fingerprints(conf):
585 ssl_fingerprints = conf['ssl_fingerprints']
586 if ssl_fingerprints is ():
587 return ()
588 if ssl is None or hashlib is None:
589 raise getmailConfigurationError(
590 'specifying ssl_fingerprints not supported by this installation of '
591 'Python; requires Python 2.6'
592 )
593
594 normalized_fprs = []
595 for fpr in ssl_fingerprints:
596 fpr = fpr.lower().replace(':','')
597 if len(fpr) != 64:
598 raise getmailConfigurationError(
599 'ssl_fingerprints must each be the SHA256 certificate hash in hex (with or without colons)'
600 )
601 normalized_fprs.append(fpr)
602 return normalized_fprs
603
604 #######################################
605 def check_ssl_ciphers(conf):
606 ssl_ciphers = conf['ssl_ciphers']
607 if ssl_ciphers:
608 if sys.version_info < (2, 7, 0):
609 raise getmailConfigurationError(
610 'specifying ssl_ciphers not supported by this installation of '
611 'Python; requires Python 2.7'
612 )
613 if re.search(r'[^a-zA-z0-9, :!\-+@]', ssl_ciphers):
614 raise getmailConfigurationError(
615 'invalid character in ssl_ciphers'
616 )
617 return ssl_ciphers
618
619 #######################################
620 keychain_password = None
621 if os.name == 'posix':
622 if os.path.isfile(osx_keychain_binary):
623 def keychain_password(user, server, protocol, logger):
624 """Mac OSX: return a keychain password, if it exists. Otherwise, return
625
626 None.
627 """
628 # OSX protocol is not an arbitrary string; it's a code limited to
629 # 4 case-sensitive chars, and only specific values.
630 protocol = protocol.lower()
631 if 'imap' in protocol:
632 protocol = 'imap'
633 elif 'pop' in protocol:
634 protocol = 'pop3'
635 else:
636 # This will break.
637 protocol = '????'
638
639 # wish we could pass along a comment to this thing for the user prompt
640 cmd = "%s find-internet-password -g -a '%s' -s '%s' -r '%s'" % (
641 osx_keychain_binary, user, server, protocol
642 )
643 (status, output) = commands.getstatusoutput(cmd)
644 if status != os.EX_OK or not output:
645 logger.error('keychain command %s failed: %s %s'
646 % (cmd, status, output))
647 return None
648 password = None
649 for line in output.split('\n'):
650 #match = re.match(r'password: "([^"]+)"', line)
651 #if match:
652 # password = match.group(1)
653 if 'password:' in line:
654 pw = line.split(':', 1)[1].strip()
655 if pw.startswith('"') and pw.endswith('"'):
656 pw = pw[1:-1]
657 password = pw
658 if password is None:
659 logger.debug('No keychain password found for %s %s %s'
660 % (user, server, protocol))
661 return password
662 elif gnomekeyring:
663 def keychain_password(user, server, protocol, logger):
664 """Gnome: return a keyring password, if it exists. Otherwise, return
665 None.
666 """
667 #logger.trace('trying Gnome keyring for user="%s", server="%s", protocol="%s"\n'
668 # % (user, server, protocol))
669 try:
670 # http://developer.gnome.org/gnome-keyring/3.5/gnome-keyring
671 # -Network-Passwords.html#gnome-keyring-find-network-password-sync
672 secret = gnomekeyring.find_network_password_sync(
673 # user, domain=None, server, object=None, protocol,
674 # authtype=None, port=0
675 user, None, server, None, protocol, None, 0
676 )
677
678 #logger.trace('got keyring result %s' % str(secret))
679 except gnomekeyring.NoMatchError:
680 logger.debug('gnome-keyring does not know password for %s %s %s'
681 % (user, server, protocol))
682 return None
683
684 # secret looks like this:
685 # [{'protocol': 'imap', 'keyring': 'Default', 'server': 'gmail.com',
686 # 'user': 'hiciu', 'item_id': 1L, 'password': 'kielbasa'}]
687 if secret and 'password' in secret[0]:
688 return secret[0]['password']
689
690 return None
691 #else:
692 # Posix but no OSX keychain or Gnome keyring.
693 # Fallthrough
694 if keychain_password is None:
695 def keychain_password(user, server, protocol, logger):
696 """Neither Mac OSX keychain or Gnome keyring available: always return
697 None.
698 """
699 return None
700
701
702 #######################################
703 def get_password(label, user, server, protocol, logger):
704 # try keychain/keyrings first, where available
705 password = keychain_password(user, server, protocol, logger)
706 if password:
707 logger.debug('using password from keychain/keyring')
708 else:
709 # no password found (or not on OSX), prompt in the usual way
710 password = getpass.getpass('Enter password for %s: ' % label)
711 return password
712
713
714 #######################################
715 def run_command(command, args):
716 # Simple subprocess wrapper for running a command and fetching its exit
717 # status and output/stderr.
718 if args is None:
719 args = []
720 if type(args) == tuple:
721 args = list(args)
722
723 # Programmer sanity checks
724 assert type(command) in (str, unicode), (
725 'command is %s (%s)' % (command, type(command))
726 )
727 assert type(args) == list, (
728 'args is %s (%s)' % (args, type(args))
729 )
730 for arg in args:
731 assert type(arg) in (str, unicode), 'arg is %s (%s)' % (arg, type(arg))
732
733 stdout = tempfile.TemporaryFile()
734 stderr = tempfile.TemporaryFile()
735
736 cmd = [command] + args
737
738 try:
739 p = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
740 except OSError, o:
741 if o.errno == errno.ENOENT:
742 # no such file, command not found
743 raise getmailConfigurationError('Program "%s" not found' % command)
744 #else:
745 raise
746
747 rc = p.wait()
748 stdout.seek(0)
749 stderr.seek(0)
750 return (rc, stdout.read().strip(), stderr.read().strip())