"Fossies" - the Fresh Open Source Software Archive

Member "etmtk-3.2.31/etmTk/data.py" (31 Mar 2017, 276691 Bytes) of package /linux/privat/etmtk-3.2.31.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "data.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 3.2.27_vs_3.2.30.

A hint: This file contains one or more very long lines, so maybe it is better readable using the pure text view mode that shows the contents as wrapped lines within the browser window.


    1 #!/usr/bin/env python3
    2 # -*- coding: utf-8 -*-
    3 from __future__ import (absolute_import, division, print_function,
    4                         unicode_literals)
    5 import os
    6 import os.path
    7 # import pwd
    8 from copy import deepcopy
    9 from textwrap import wrap
   10 import platform
   11 import json
   12 
   13 import logging
   14 import logging.config
   15 logger = logging.getLogger()
   16 
   17 this_dir, this_filename = os.path.split(__file__)
   18 LANGUAGES = os.path.normpath(os.path.join(this_dir, "locale"))
   19 
   20 BGCOLOR = HLCOLOR = FGCOLOR = CALENDAR_COLORS = None
   21 
   22 def _(x):
   23     return(x)
   24 
   25 def setup_logging(level, etmdir=None):
   26     """
   27     Setup logging configuration. Override root:level in
   28     logging.yaml with default_level.
   29     """
   30     if etmdir:
   31         etmdir = os.path.normpath(etmdir)
   32     else:
   33         etmdir = os.path.normpath(os.path.join(os.path.expanduser("~/.etm")))
   34     log_levels = {
   35         '1': logging.DEBUG,
   36         '2': logging.INFO,
   37         '3': logging.WARN,
   38         '4': logging.ERROR,
   39         '5': logging.CRITICAL
   40     }
   41 
   42     if level in log_levels:
   43         loglevel = log_levels[level]
   44     else:
   45         loglevel = log_levels['3']
   46 
   47     if os.path.isdir(etmdir):
   48         logfile = os.path.normpath(os.path.abspath(os.path.join(etmdir, "etmtk_log.txt")))
   49         if not os.path.isfile(logfile):
   50             open(logfile, 'a').close()
   51 
   52         config = {'disable_existing_loggers': False,
   53                   'formatters': {'simple': {
   54                       'format': '--- %(asctime)s - %(levelname)s - %(module)s.%(funcName)s\n    %(message)s'}},
   55                   'handlers': {'console': {'class': 'logging.StreamHandler',
   56                                            'formatter': 'simple',
   57                                            'level': loglevel,
   58                                            'stream': 'ext://sys.stdout'},
   59                                'file': {'backupCount': 5,
   60                                         'class': 'logging.handlers.RotatingFileHandler',
   61                                         'encoding': 'utf8',
   62                                         'filename': logfile,
   63                                         'formatter': 'simple',
   64                                         'level': 'WARN',
   65                                         'maxBytes': 1048576}},
   66                   'loggers': {'etmtk': {'handlers': ['console'],
   67                                         'level': 'DEBUG',
   68                                         'propagate': False}},
   69                   'root': {'handlers': ['console', 'file'], 'level': 'DEBUG'},
   70                   'version': 1}
   71         logging.config.dictConfig(config)
   72         logger.info('logging at level: {0}\n    writing exceptions to: {1}'.format(loglevel, logfile))
   73     else:  # no etmdir - first use
   74         config = {'disable_existing_loggers': False,
   75                   'formatters': {'simple': {
   76                       'format': '--- %(asctime)s - %(levelname)s - %(module)s.%(funcName)s\n    %(message)s'}},
   77                   'handlers': {'console': {'class': 'logging.StreamHandler',
   78                                            'formatter': 'simple',
   79                                            'level': loglevel,
   80                                            'stream': 'ext://sys.stdout'}},
   81                   'loggers': {'etmtk': {'handlers': ['console'],
   82                                         'level': 'DEBUG',
   83                                         'propagate': False}},
   84                   'root': {'handlers': ['console'], 'level': 'DEBUG'},
   85                   'version': 1}
   86         logging.config.dictConfig(config)
   87         logger.info('logging at level: {0}'.format(loglevel))
   88 
   89 import subprocess
   90 
   91 # setup gettext in get_options once locale is known
   92 import gettext
   93 
   94 if platform.python_version() >= '3':
   95     python_version = 3
   96     python_version2 = False
   97     from io import StringIO
   98     unicode = str
   99     u = lambda x: x
  100     raw_input = input
  101     from urllib.parse import quote
  102 else:
  103     python_version = 2
  104     python_version2 = True
  105     from cStringIO import StringIO
  106     from urllib2 import quote
  107 
  108 def s2or3(s):
  109     if python_version == 2:
  110         if type(s) is unicode:
  111             return s
  112         elif type(s) is str:
  113             try:
  114                 return unicode(s, term_encoding)
  115             except ValueError:
  116                 logger.error('s2or3 exception: {0}'.format(s))
  117         else:
  118             return s
  119     else:
  120         return s
  121 
  122 from random import random, uniform
  123 from math import log
  124 
  125 
  126 class Node(object):
  127     __slots__ = 'value', 'next', 'width'
  128 
  129     def __init__(self, value, next, width):
  130         self.value, self.next, self.width = value, next, width
  131 
  132 
  133 class End(object):
  134     """
  135     Sentinel object that always compares greater than another object.
  136     Replaced __cmp__ to work with python3.x
  137     """
  138 
  139     def __eq__(self, other):
  140         return 0
  141 
  142     def __ne__(self, other):
  143         return 1
  144 
  145     def __gt__(self, other):
  146         return 1
  147 
  148     def __ge__(self, other):
  149         return 1
  150 
  151     def __le__(self, other):
  152         return 0
  153 
  154     def __lt__(self, other):
  155         return 0
  156 
  157 # Singleton terminator node
  158 NIL = Node(End(), [], [])
  159 
  160 
  161 class IndexableSkiplist:
  162     """Sorted collection supporting O(lg n) insertion, removal, and lookup by rank."""
  163 
  164     def __init__(self, expected_size=100, type=""):
  165         self.size = 0
  166         self.type = type
  167         self.maxlevels = int(1 + log(expected_size, 2))
  168         self.head = Node('HEAD', [NIL] * self.maxlevels, [1] * self.maxlevels)
  169 
  170     def __len__(self):
  171         return self.size
  172 
  173     def __getitem__(self, i):
  174         node = self.head
  175         i += 1
  176         for level in reversed(range(self.maxlevels)):
  177             while node.width[level] <= i:
  178                 i -= node.width[level]
  179                 node = node.next[level]
  180         return node.value
  181 
  182     def insert(self, value):
  183         # find first node on each level where node.next[levels].value > value
  184         chain = [None] * self.maxlevels
  185         steps_at_level = [0] * self.maxlevels
  186         node = self.head
  187         for level in reversed(range(self.maxlevels)):
  188             try:
  189                 while node.next[level].value <= value:
  190                     steps_at_level[level] += node.width[level]
  191                     node = node.next[level]
  192                 chain[level] = node
  193             except:
  194                 logger.exception('Error comparing {0}:\n    {1}\n    with the value to be inserted\n    {2}'.format(self.type, node.next[level].value, value))
  195                 return
  196 
  197         # insert a link to the newnode at each level
  198         d = min(self.maxlevels, 1 - int(log(random(), 2.0)))
  199         newnode = Node(value, [None] * d, [None] * d)
  200         steps = 0
  201         for level in range(d):
  202             prevnode = chain[level]
  203             newnode.next[level] = prevnode.next[level]
  204             prevnode.next[level] = newnode
  205             newnode.width[level] = prevnode.width[level] - steps
  206             prevnode.width[level] = steps + 1
  207             steps += steps_at_level[level]
  208         for level in range(d, self.maxlevels):
  209             chain[level].width[level] += 1
  210         self.size += 1
  211 
  212     def remove(self, value):
  213         # find first node on each level where node.next[levels].value >= value
  214         chain = [None] * self.maxlevels
  215         node = self.head
  216         for level in reversed(range(self.maxlevels)):
  217             try:
  218                 while node.next[level].value < value:
  219                     node = node.next[level]
  220                 chain[level] = node
  221             except:
  222                 logger.exception('Error comparing {0}:\n    {1}\n    with the value to be removed\n    {2}'.format(self.type, node.next[level].value, value))
  223         if value != chain[0].next[0].value:
  224             raise KeyError('Not Found')
  225 
  226         # remove one link at each level
  227         d = len(chain[0].next[0].next)
  228         for level in range(d):
  229             prevnode = chain[level]
  230             prevnode.width[level] += prevnode.next[level].width[level] - 1
  231             prevnode.next[level] = prevnode.next[level].next[level]
  232         for level in range(d, self.maxlevels):
  233             chain[level].width[level] -= 1
  234         self.size -= 1
  235 
  236     def __iter__(self):
  237         'Iterate over values in sorted order'
  238         node = self.head.next[0]
  239         while node is not NIL:
  240             yield node.value
  241             node = node.next[0]
  242 
  243 # initial instances
  244 
  245 itemsSL = IndexableSkiplist(5000, "items")
  246 alertsSL = IndexableSkiplist(100, "alerts")
  247 datetimesSL = IndexableSkiplist(1000, "datetimes")
  248 datesSL = IndexableSkiplist(1000, "dates")
  249 busytimesSL = {}
  250 occasionsSL = {}
  251 items = []
  252 alerts = []
  253 datetimes = []
  254 busytimes = {}
  255 occasions = {}
  256 file2uuids = {}
  257 uuid2hash = {}
  258 file2data = {}
  259 
  260 name2list = {
  261     "items": items,
  262     "alerts": alerts,
  263     "datetimes": datetimes
  264 }
  265 name2SL = {
  266     "items": itemsSL,
  267     "alerts": alertsSL,
  268     "datetimes": datetimesSL
  269 }
  270 
  271 
  272 def clear_all_data():
  273     global itemsSL, alertsSL, datetimesSL, datesSL, busytimesSL, occasionsSL, items, alerts, datetimes, busytimes, occasions, file2uuids, uuid2hash, file2data, name2list, name2SL
  274     itemsSL = IndexableSkiplist(5000, "items")
  275     alertsSL = IndexableSkiplist(100, "alerts")
  276     datetimesSL = IndexableSkiplist(1000, "datetimes")
  277     datesSL = IndexableSkiplist(1000, "dates")
  278     busytimesSL = {}
  279     occasionsSL = {}
  280     items = []
  281     alerts = []
  282     datetimes = []
  283     busytimes = {}
  284     occasions = {}
  285     file2uuids = {}
  286     uuid2hash = {}
  287     file2data = {}
  288 
  289     name2list = {
  290         "items": items,
  291         "alerts": alerts,
  292         "datetimes": datetimes
  293     }
  294     name2SL = {
  295         "items": itemsSL,
  296         "alerts": alertsSL,
  297         "datetimes": datetimesSL
  298     }
  299 
  300 
  301 dayfirst = False
  302 yearfirst = True
  303 
  304 IGNORE = """\
  305 syntax: glob
  306 .*
  307 """
  308 
  309 from datetime import datetime, timedelta, time
  310 from dateutil.tz import (tzlocal, tzutc)
  311 from dateutil.easter import easter
  312 
  313 
  314 def get_current_time():
  315     return datetime.now(tzlocal())
  316 
  317 # this task will be created for first time users
  318 SAMPLE = """\
  319 # Sample entries - edit or delete at your pleasure
  320 = @t sample, tasks
  321 ? lose weight and exercise more
  322 - milk and eggs @c errands
  323 - reservation for Saturday dinner @c phone
  324 - hair cut @s -1 @r w &i 2 &o r @c errands
  325 - put out trash @s 1 @r w &w MO @o s
  326 
  327 = @t sample, occasions
  328 ^ etm's !2009! birthday @s 2010-02-27 @r y @d initial release 2009-02-27
  329 ^ payday @s 1/1 @r m &w MO, TU, WE, TH, FR &m -1, -2, -3 &s -1 @d the last weekday of each month
  330 
  331 = @t sample, events
  332 * sales meeting @s +7 9a @e 1h @a 5 @a 2d: e; who@when.com, what@where.org @u jsmith
  333 * stationary bike @s 1 5:30p @e 30 @r d @a 0
  334 * Tête-à-têtes @s 1 3p @e 90 @r w &w fri @l conference room @t meetings
  335 * Book club @s -1/1 7pm @e 2h @z US/Eastern @r w &w TH
  336 * Tennis @s -1/1 9am @e 1h30m @z US/Eastern @r w &w SA
  337 * Dinner @s -1/1 7:30pm @e 2h30m @z US/Eastern @a 1h, 40m: m @u dag @r w &w SA
  338 * Appt with Dr Burns @s 2014-05-15 10am @e 1h @r m &i 9 &w 1TU &t 2
  339 """
  340 
  341 HOLIDAYS = """\
  342 ^ Martin Luther King Day @s 2010-01-18 @r y &w 3MO &M 1
  343 ^ Valentine's Day @s 2010-02-14 @r y &M 2 &m 14
  344 ^ President's Day @s 2010-02-15 @c holiday @r y &w 3MO &M 2
  345 ^ Daylight saving time begins @s 2010-03-14 @r y &w 2SU &M 3
  346 ^ St Patrick's Day @s 2010-03-17 @r y &M 3 &m 17
  347 ^ Easter Sunday @s 2010-01-01 @r y &E 0
  348 ^ Mother's Day @s 2010-05-09 @r y &w 2SU &M 5
  349 ^ Memorial Day @s 2010-05-31 @r y &w -1MO &M 5
  350 ^ Father's Day @s 2010-06-20 @r y &w 3SU &M 6
  351 ^ The !1776! Independence Day @s 2010-07-04 @r y &M 7 &m 4
  352 ^ Labor Day @s 2010-09-06 @r y &w 1MO &M 9
  353 ^ Daylight saving time ends @s 2010-11-01 @r y &w 1SU &M 11
  354 ^ Thanksgiving @s 2010-11-26 @r y &w 4TH &M 11
  355 ^ Christmas @s 2010-12-25 @r y &M 12 &m 25
  356 ^ Presidential election day @s 2004-11-01 12am @r y &i 4 &m 2, 3, 4, 5, 6, 7, 8 &M 11 &w TU
  357 """
  358 
  359 JOIN = "- join the etm discussion group @s +14 @b 10 @c computer @g http://groups.google.com/group/eventandtaskmanager/topics"
  360 
  361 COMPETIONS = """\
  362 # put completion phrases here, one per line. E.g.:
  363 @z US/Eastern
  364 @z US/Central
  365 @z US/Mountain
  366 @z US/Pacific
  367 
  368 @c errands
  369 @c phone
  370 @c home
  371 @c office
  372 
  373 # empty lines and lines that begin with '#' are ignored.
  374 """
  375 
  376 USERS = """\
  377 jsmith:
  378     - Smith, John
  379     - jsmth@whatever.com
  380     - wife Rebecca
  381     - children Tom, Dick and Harry
  382 """
  383 
  384 REPORTS = """\
  385 # put report specifications here, one per line. E.g.:
  386 
  387 # scheduled items this week:
  388 c ddd, MMM dd yyyy -b mon - 7d -e +7
  389 
  390 # this and next week:
  391 c ddd, MMM dd yyyy -b mon - 7d -e +14
  392 
  393 # this month:
  394 c ddd, MMM dd yyyy -b 1 -e +1/1
  395 
  396 # this and next month:
  397 c ddd, MMM dd yyyy -b 1 -e +2/1
  398 
  399 # last month's actions:
  400 a MMM yyyy; u; k[0]; k[1:] -b -1/1 -e 1
  401 
  402 # this month's actions:
  403 a MMM yyyy; u; k[0]; k[1:] -b 1 -e +1/1
  404 
  405 # this week's actions:
  406 a w; u; k[0]; k[1:] -b sun - 6d -e sun
  407 
  408 # all items by folder:
  409 c f
  410 
  411 # all items by keyword:
  412 c k
  413 
  414 # all items by tag:
  415 c t
  416 
  417 # all items by user:
  418 c u
  419 
  420 # empty lines and lines that begin with '#' are ignored.
  421 """
  422 
  423 # command line usage
  424 USAGE = """\
  425 Usage:
  426 
  427     etm [logging level] [path] [?] [acmsv]
  428 
  429 With no arguments, etm will set logging level 3 (warn), use settings from
  430 the configuration file ~/.etm/etmtk.cfg, and open the GUI.
  431 
  432 If the first argument is an integer not less than 1 (debug) and not greater
  433 than 5 (critical), then set that logging level and remove the argument.
  434 
  435 If the first (remaining) argument is the path to a directory that contains
  436 a file named etmtk.cfg, then use that configuration file and remove the
  437 argument.
  438 
  439 If the first (remaining) argument is one of the commands listed below, then
  440 execute the remaining arguments without opening the GUI.
  441 
  442     a ARG   display the agenda view using ARG, if given, as a filter.
  443     c ARGS  display a custom view using the remaining arguments as the
  444             specification. (Enclose ARGS in single quotes to prevent shell
  445             expansion.)
  446     d ARG   display the day view using ARG, if given, as a filter.
  447     k ARG   display the keywords view using ARG, if given, as a filter.
  448     m INT   display a report using the remaining argument, which must be a
  449             positive integer, to display a report using the corresponding
  450             entry from the file given by report_specifications in etmtk.cfg.
  451             Use ? m to display the numbered list of entries from this file.
  452     n ARG   display the notes view using ARG, if given, as a filter.
  453     N ARGS  Create a new item using the remaining arguments as the item
  454             specification. (Enclose ARGS in single quotes to prevent shell
  455             expansion.)
  456     p ARG   display the path view using ARG, if given, as a filter.
  457     t ARG   display the tags view using ARG, if given, as a filter.
  458     v       display information about etm and the operating system.
  459     ? ARG   display (this) command line help information if ARGS = '' or,
  460             if ARGS = X where X is one of the above commands, then display
  461             details about command X. 'X ?' is equivalent to '? X'.\
  462 """
  463 
  464 import re
  465 import sys
  466 import locale
  467 
  468 # term_encoding = locale.getdefaultlocale()[1]
  469 term_locale = locale.getdefaultlocale()[0]
  470 
  471 qt2dt = [
  472     ('a', '%p'),
  473     ('dddd', '%A'),
  474     ('ddd', '%a'),
  475     ('dd', '%d'),
  476     ('MMMM', '%B'),
  477     ('MMM', '%b'),
  478     ('MM', '%m'),
  479     ('yyyy', '%Y'),
  480     ('yy', '%y'),
  481     ('hh', '%H'),
  482     ('h', '%I'),
  483     ('mm', '%M'),
  484     ('w', 'WEEK')
  485 ]
  486 
  487 
  488 def commandShortcut(s):
  489     """
  490     Produce label, command pairs from s based on Command for OSX
  491     and Control otherwise.
  492     """
  493     if s.upper() == s and s.lower() != s:
  494         shift = "Shift-"
  495     else:
  496         shift = ""
  497     if mac:
  498         # return "{0}Cmd-{1}".format(shift, s), "<{0}Command-{1}>".format(shift, s)
  499         return "{0}Ctrl-{1}".format(shift, s.upper()), "<{0}Control-{1}>".format(shift, s)
  500     else:
  501         return "{0}Ctrl-{1}".format(shift, s.upper()), "<{0}Control-{1}>".format(shift, s)
  502 
  503 
  504 def optionShortcut(s):
  505     """
  506     Produce label, command pairs from s based on Command for OSX
  507     and Control otherwise.
  508     """
  509     if s.upper() == s and s.lower() != s:
  510         shift = "Shift-"
  511     else:
  512         shift = ""
  513     if mac:
  514         return "{0}Alt-{1}".format(shift, s.upper()), "<{0}Option-{1}>".format(shift, s)
  515     else:
  516         return "{0}Alt-{1}".format(shift, s.upper()), "<{0}Alt-{1}>".format(shift, s)
  517 
  518 
  519 def d_to_str(d, s):
  520     for key, val in qt2dt:
  521         s = s.replace(key, val)
  522     ret = s2or3(d.strftime(s))
  523     if 'WEEK' in ret:
  524         theweek = get_week(d)
  525         ret = ret.replace('WEEK', theweek)
  526     return ret
  527 
  528 
  529 def dt_to_str(dt, s):
  530     for key, val in qt2dt:
  531         s = s.replace(key, val)
  532     ret = s2or3(dt.strftime(s))
  533     if 'WEEK' in ret:
  534         theweek = get_week(dt)
  535         ret = ret.replace('WEEK', theweek)
  536     return ret
  537 
  538 
  539 def get_week(dt):
  540     yn, wn, dn = dt.isocalendar()
  541     if dn > 1:
  542         days = dn - 1
  543     else:
  544         days = 0
  545     weekbeg = dt - days * ONEDAY
  546     weekend = dt + (6 - days) * ONEDAY
  547     ybeg = weekbeg.year
  548     yend = weekend.year
  549     mbeg = weekbeg.month
  550     mend = weekend.month
  551     if mbeg == mend:
  552         header = "{0} - {1}".format(
  553             fmt_dt(weekbeg, '%b %d'), fmt_dt(weekend, '%d'))
  554     elif ybeg == yend:
  555         header = "{0} - {1}".format(
  556             fmt_dt(weekbeg, '%b %d'), fmt_dt(weekend, '%b %d'))
  557     else:
  558         header = "{0} - {1}".format(
  559             fmt_dt(weekbeg, '%b %d, %Y'), fmt_dt(weekend, '%b %d, %Y'))
  560     header = leadingzero.sub('', header)
  561     theweek = "{0} {1}: {2}".format(_("Week"), "{0:02d}".format(wn), header)
  562     return theweek
  563 
  564 
  565 from etmTk.v import version
  566 from etmTk.version import version as fullversion
  567 
  568 last_version = version
  569 
  570 from re import split as rsplit
  571 
  572 sys_platform = platform.system()
  573 if sys_platform in ('Windows', 'Microsoft'):
  574     windoz = True
  575     from time import clock as timer
  576 else:
  577     windoz = False
  578     from time import time as timer
  579 
  580 if sys.platform == 'darwin':
  581     mac = True
  582     CMD = "Command"
  583     default_style = 'aqua'
  584 else:
  585     mac = False
  586     CMD = "Control"
  587     default_style = 'default'
  588 
  589 # used in hack to prevent dialog from hanging under os x
  590 if mac:
  591     AFTER = 200
  592 else:
  593     AFTER = 1
  594 
  595 
  596 class TimeIt(object):
  597     def __init__(self, loglevel=1, label=""):
  598         self.loglevel = loglevel
  599         self.label = label
  600         msg = "{0} timer started".format(self.label)
  601         if self.loglevel == 1:
  602             logger.debug(msg)
  603         elif self.loglevel == 2:
  604             logger.info(msg)
  605         self.start = timer()
  606 
  607     def stop(self, *args):
  608         self.end = timer()
  609         self.secs = self.end - self.start
  610         self.msecs = self.secs * 1000  # millisecs
  611         msg = "{0} timer stopped; elapsed time: {1} milliseconds".format(self.label, self.msecs)
  612         if self.loglevel == 1:
  613             logger.debug(msg)
  614         elif self.loglevel == 2:
  615             logger.info(msg)
  616 
  617 has_icalendar = False
  618 try:
  619     from icalendar import Calendar, Event, Todo, Journal
  620     from icalendar.caselessdict import CaselessDict
  621     from icalendar.prop import vDate, vDatetime
  622     has_icalendar = True
  623     import pytz
  624 except ImportError:
  625     if has_icalendar:
  626         logger.info('Could not import pytz')
  627     else:
  628         logger.info('Could not import icalendar and/or pytz')
  629     has_icalendar = False
  630 
  631 from time import sleep
  632 import dateutil.rrule as dtR
  633 from dateutil.parser import parse as dparse
  634 from dateutil import __version__ as dateutil_version
  635 # noinspection PyPep8Naming
  636 from dateutil.tz import gettz as getTz
  637 
  638 
  639 def memoize(fn):
  640     memo = {}
  641 
  642     def memoizer(*param_tuple, **kwds_dict):
  643         if kwds_dict:
  644             memoizer.namedargs += 1
  645             return fn(*param_tuple, **kwds_dict)
  646         try:
  647             memoizer.cacheable += 1
  648             try:
  649                 return memo[param_tuple]
  650             except KeyError:
  651                 memoizer.misses += 1
  652                 memo[param_tuple] = result = fn(*param_tuple)
  653                 return result
  654         except TypeError:
  655             memoizer.cacheable -= 1
  656             memoizer.noncacheable += 1
  657             return fn(*param_tuple)
  658 
  659     memoizer.namedargs = memoizer.cacheable = memoizer.noncacheable = 0
  660     memoizer.misses = 0
  661     return memoizer
  662 
  663 
  664 @memoize
  665 def gettz(z=None):
  666     return getTz(z)
  667 
  668 
  669 import calendar
  670 
  671 import yaml
  672 from itertools import groupby
  673 # from dateutil.rrule import *
  674 from dateutil.rrule import (DAILY, rrule)
  675 
  676 import bisect
  677 import uuid
  678 import codecs
  679 import shutil
  680 import fnmatch
  681 
  682 
  683 def term_print(s):
  684     if python_version2:
  685         try:
  686             print(unicode(s).encode(term_encoding))
  687         except Exception:
  688             logger.exception("error printing: '{0}', {1}".format(s, type(s)))
  689     else:
  690         print(s)
  691 
  692 
  693 parse = None
  694 
  695 
  696 def setup_parse(day_first, year_first):
  697     global parse
  698 
  699     # noinspection PyRedeclaration
  700     def parse(s):
  701         """
  702         Return a datetime object
  703         """
  704         try:
  705             res = dparse(str(s), dayfirst=day_first, yearfirst=year_first)
  706         except:
  707             return 'Could not parse: {0}'.format(s)
  708 
  709         return res
  710 
  711 
  712 try:
  713     from os.path import relpath
  714 except ImportError:  # python < 2.6
  715     from os.path import curdir, abspath, sep, commonprefix, pardir, join
  716 
  717     def relpath(path, start=curdir):
  718         """Return a relative version of a path"""
  719         if not path:
  720             raise ValueError("no path specified")
  721         start_list = abspath(start).split(sep)
  722         path_list = abspath(path).split(sep)
  723         # Work out how much of the filepath is shared by start and path.
  724         i = len(commonprefix([start_list, path_list]))
  725         rel_list = [pardir] * (len(start_list) - i) + path_list[i:]
  726         if not rel_list:
  727             return curdir
  728         return join(*rel_list)
  729 
  730 cwd = os.getcwd()
  731 
  732 
  733 def pathSearch(filename):
  734     search_path = os.getenv('PATH').split(os.pathsep)
  735     for path in search_path:
  736         candidate = os.path.normpath(os.path.join(path, filename))
  737         # logger.debug('checking for: {0}'.format(candidate))
  738         if os.path.isfile(candidate):
  739             # return os.path.abspath(candidate)
  740             return candidate
  741     return ''
  742 
  743 
  744 def getMercurial():
  745     if windoz:
  746         hg = pathSearch('hg.exe')
  747     else:
  748         hg = pathSearch('hg')
  749     if hg:
  750         logger.debug('found hg: {0}'.format(hg))
  751         base_command = "hg -R {work}"
  752         history_command = 'hg log --style compact --template "{desc}\\n" -R {work} -p {numchanges} {file}'
  753         commit_command = 'hg commit -q -A -R {work} -m "{mesg}"'
  754         init = 'hg init {work}'
  755         init_command = "%s && %s" % (init, commit_command)
  756         logger.debug('hg base_command: {0}; history_command: {1}; commit_command: {2}; init_command: {3}'.format(base_command, history_command, commit_command, init_command))
  757     else:
  758         logger.debug('could not find hg in path')
  759         base_command = history_command = commit_command = init_command = ''
  760     return base_command, history_command, commit_command, init_command
  761 
  762 
  763 def getGit():
  764     if windoz:
  765         git = pathSearch('git.exe')
  766     else:
  767         git = pathSearch('git')
  768     if git:
  769         logger.debug('found git: {0}'.format(git))
  770         base_command = "git --git-dir {repo} --work-tree {work}"
  771         history_command = "git --git-dir {repo} --work-tree {work} log --pretty=format:'- %ai %an: %s' -U0 {numchanges} {file}"
  772         init = 'git init {work}'
  773         add = 'git --git-dir {repo} --work-tree {work} add */\*.txt > /dev/null'
  774         commit = 'git --git-dir {repo} --work-tree {work} commit -a -m "{mesg}" > /dev/null'
  775         commit_command = '%s && %s' % (add, commit)
  776         init_command = '%s && %s && %s' % (init, add, commit)
  777         logger.debug('git base_command: {0}; history_command: {1}; commit_command: {2}; init_command: {3}'.format(base_command, history_command, commit_command, init_command))
  778     else:
  779         logger.debug('could not find git in path')
  780         base_command = history_command = commit_command = init_command = ''
  781     return base_command, history_command, commit_command, init_command
  782 
  783 
  784 zonelist = [
  785     'Africa/Cairo',
  786     'Africa/Casablanca',
  787     'Africa/Johannesburg',
  788     'Africa/Mogadishu',
  789     'Africa/Nairobi',
  790     'America/Belize',
  791     'America/Buenos_Aires',
  792     'America/Edmonton',
  793     'America/Mexico_City',
  794     'America/Monterrey',
  795     'America/Montreal',
  796     'America/Toronto',
  797     'America/Vancouver',
  798     'America/Winnipeg',
  799     'Asia/Baghdad',
  800     'Asia/Bahrain',
  801     'Asia/Calcutta',
  802     'Asia/Damascus',
  803     'Asia/Dubai',
  804     'Asia/Hong_Kong',
  805     'Asia/Istanbul',
  806     'Asia/Jakarta',
  807     'Asia/Jerusalem',
  808     'Asia/Katmandu',
  809     'Asia/Kuwait',
  810     'Asia/Macao',
  811     'Asia/Pyongyang',
  812     'Asia/Saigon',
  813     'Asia/Seoul',
  814     'Asia/Shanghai',
  815     'Asia/Singapore',
  816     'Asia/Tehran',
  817     'Asia/Tokyo',
  818     'Asia/Vladivostok',
  819     'Atlantic/Azores',
  820     'Atlantic/Bermuda',
  821     'Atlantic/Reykjavik',
  822     'Australia/Sydney',
  823     'Europe/Amsterdam',
  824     'Europe/Berlin',
  825     'Europe/Lisbon',
  826     'Europe/London',
  827     'Europe/Madrid',
  828     'Europe/Minsk',
  829     'Europe/Monaco',
  830     'Europe/Moscow',
  831     'Europe/Oslo',
  832     'Europe/Paris',
  833     'Europe/Prague',
  834     'Europe/Rome',
  835     'Europe/Stockholm',
  836     'Europe/Vienna',
  837     'Pacific/Auckland',
  838     'Pacific/Fiji',
  839     'Pacific/Samoa',
  840     'Pacific/Tahiti',
  841     'Turkey',
  842     'US/Alaska',
  843     'US/Aleutian',
  844     'US/Arizona',
  845     'US/Central',
  846     'US/East-Indiana',
  847     'US/Eastern',
  848     'US/Hawaii',
  849     'US/Indiana-Starke',
  850     'US/Michigan',
  851     'US/Mountain',
  852     'US/Pacific']
  853 
  854 
  855 def get_localtz(zones=zonelist):
  856     """
  857 
  858     :param zones: list of timezone strings
  859     :return: timezone string
  860     """
  861     linfo = gettz()
  862     now = get_current_time()
  863     # get the abbreviation for the local timezone, e.g, EDT
  864     possible = []
  865     # try the zone list first unless windows system
  866     if not windoz:
  867         for i in range(len(zones)):
  868             z = zones[i]
  869             zinfo = gettz(z)
  870             if zinfo and zinfo == linfo:
  871                 possible.append(i)
  872                 break
  873     if not possible:
  874         for i in range(len(zones)):
  875             z = zones[i]
  876             zinfo = gettz(z)
  877             if zinfo and zinfo.utcoffset(now) == linfo.utcoffset(now):
  878                 possible.append(i)
  879     if not possible:
  880         # the local zone needs to be added to timezones
  881         return ['']
  882     return [zonelist[i] for i in possible]
  883 
  884 
  885 def calyear(advance=0, options=None):
  886     """
  887     """
  888     if not options:
  889         options = {}
  890     lcl = options['lcl']
  891     if 'sundayfirst' in options and options['sundayfirst']:
  892         week_begin = 6
  893     else:
  894         week_begin = 0
  895         # hack to set locale on darwin, windoz and linux
  896     try:
  897         if mac:
  898             # locale test
  899             c = calendar.LocaleTextCalendar(week_begin, lcl[0])
  900         elif windoz:
  901             locale.setlocale(locale.LC_ALL, '')
  902             lcl = locale.getlocale()
  903             c = calendar.LocaleTextCalendar(week_begin, lcl)
  904         else:
  905             lcl = locale.getdefaultlocale()
  906             c = calendar.LocaleTextCalendar(week_begin, lcl)
  907     except:
  908         logger.exception('Could not set locale: {0}'.format(lcl))
  909         c = calendar.LocaleTextCalendar(week_begin)
  910     cal = []
  911     y = int(today.strftime("%Y"))
  912     m = 1
  913     # d = 1
  914     y += advance
  915     for i in range(12):
  916         cal.append(c.formatmonth(y, m).split('\n'))
  917         m += 1
  918         if m > 12:
  919             y += 1
  920             m = 1
  921     ret = []
  922     for r in range(0, 12, 3):
  923         l = max(len(cal[r]), len(cal[r + 1]), len(cal[r + 2]))
  924         for i in range(3):
  925             if len(cal[r + i]) < l:
  926                 for j in range(len(cal[r + i]), l + 1):
  927                     cal[r + i].append('')
  928         for j in range(l):
  929             if python_version2:
  930                 ret.append(s2or3(u'  %-20s    %-20s    %-20s' % (cal[r][j], cal[r + 1][j], cal[r + 2][j])))
  931             else:
  932                 ret.append((u'  %-20s    %-20s    %-20s' % (cal[r][j], cal[r + 1][j], cal[r + 2][j])))
  933     return ret
  934 
  935 
  936 def date_calculator(s, options=None):
  937     """
  938         x [+-] y
  939         where x is a datetime and y is either a datetime or a timeperiod
  940     :param s:
  941     """
  942     estr = estr_regex.search(s)
  943     if estr:
  944         y = estr.group(1)
  945         e = easter(int(y))
  946         E = e.strftime("%Y-%m-%d")
  947         s = estr_regex.sub(E, s)
  948 
  949     m = date_calc_regex.match(s)
  950     if not m:
  951         return 'Could not parse "%s"' % s
  952     x, pm, y = [z.strip() for z in m.groups()]
  953     xzs = None
  954     nx = timezone_regex.match(x)
  955     if nx:
  956         x, xzs = nx.groups()
  957     yz = tzlocal()
  958     yzs = None
  959     ny = timezone_regex.match(y)
  960     if ny:
  961         y, yzs = ny.groups()
  962         yz = gettz(yzs)
  963     windoz_epoch = _("Warning: any timezone information in dates prior to 1970 is ignored under Windows.")
  964     warn = ""
  965     try:
  966         dt_x = parse_str(x, timezone=xzs)
  967         pmy = "%s%s" % (pm, y)
  968         if period_string_regex.match(pmy):
  969             dt = (dt_x + parse_period(pmy, minutes=False))
  970             if windoz and (dt_x.year < 1970 or dt.year < 1970):
  971                 warn = "\n\n{0}".format(windoz_epoch)
  972             else:
  973                 dt.astimezone(yz)
  974 
  975             res = dt.strftime("%Y-%m-%d %H:%M%z")
  976             prompt = "{0}:\n\n{1}{2}".format(s.strip(), res.strip(), warn)
  977             return prompt
  978         else:
  979             dt_y = parse_str(y, timezone=yzs)
  980             if windoz and (dt_x.year < 1970 or dt_y.year < 1970):
  981                 warn = "\n\n{0}".format(windoz_epoch)
  982                 dt_x = dt_x.replace(tzinfo=None)
  983                 dt_y = dt_y.replace(tzinfo=None)
  984             if pm == '-':
  985                 res = fmt_period(dt_x - dt_y)
  986                 prompt = "{0}:\n\n{1}{2}".format(s.strip(), res.strip(), warn)
  987                 return prompt
  988             else:
  989                 return 'error: datetimes cannot be added'
  990     except ValueError:
  991         return 'error parsing "%s"' % s
  992 
  993 
  994 def mail_report(message, smtp_from=None, smtp_server=None,
  995                 smtp_id=None, smtp_pw=None, smtp_to=None):
  996     """
  997     """
  998     import smtplib
  999     from email.MIMEMultipart import MIMEMultipart
 1000     from email.MIMEText import MIMEText
 1001     from email.Utils import COMMASPACE, formatdate
 1002     # from email import Encoders
 1003 
 1004     assert type(smtp_to) == list
 1005 
 1006     msg = MIMEMultipart()
 1007     msg['From'] = smtp_from
 1008     msg['To'] = COMMASPACE.join(smtp_to)
 1009     msg['Date'] = formatdate(localtime=True)
 1010     msg['Subject'] = "etm agenda"
 1011 
 1012     msg.attach(MIMEText(message, 'html'))
 1013 
 1014     smtp = smtplib.SMTP_SSL(smtp_server)
 1015     smtp.login(smtp_id, smtp_pw)
 1016     smtp.sendmail(smtp_from, smtp_to, msg.as_string())
 1017     smtp.close()
 1018 
 1019 
 1020 def send_mail(smtp_to, subject, message, files=None, smtp_from=None, smtp_server=None,
 1021               smtp_id=None, smtp_pw=None):
 1022     """
 1023     """
 1024     if not files:
 1025         files = []
 1026     import smtplib
 1027     from email.mime.multipart import MIMEMultipart
 1028     from email.mime.base import MIMEBase
 1029     from email.mime.text import MIMEText
 1030     from email.utils import COMMASPACE, formatdate
 1031     from email import encoders as Encoders
 1032     assert type(smtp_to) == list
 1033     assert type(files) == list
 1034     msg = MIMEMultipart()
 1035     msg['From'] = smtp_from
 1036     msg['To'] = COMMASPACE.join(smtp_to)
 1037     msg['Date'] = formatdate(localtime=True)
 1038     msg['Subject'] = subject
 1039     msg.attach(MIMEText(message))
 1040     for f in files:
 1041         part = MIMEBase('application', "octet-stream")
 1042         part.set_payload(open(f, "rb").read())
 1043         Encoders.encode_base64(part)
 1044         part.add_header(
 1045             'Content-Disposition',
 1046             'attachment; filename="%s"' % os.path.basename(f))
 1047         msg.attach(part)
 1048     smtp = smtplib.SMTP_SSL(smtp_server)
 1049     smtp.login(smtp_id, smtp_pw)
 1050     smtp.sendmail(smtp_from, smtp_to, msg.as_string())
 1051     smtp.close()
 1052 
 1053 
 1054 def send_text(sms_phone, subject, message, sms_from, sms_server, sms_pw):
 1055     sms_phone = "%s" % sms_phone
 1056     import smtplib
 1057     from email.mime.text import MIMEText
 1058 
 1059     sms = smtplib.SMTP(sms_server)
 1060     sms.starttls()
 1061     sms.login(sms_from, sms_pw)
 1062     for num in sms_phone.split(','):
 1063         msg = MIMEText(message)
 1064         msg["From"] = sms_from
 1065         msg["Subject"] = subject
 1066         msg['To'] = num
 1067         sms.sendmail(sms_from, sms_phone, msg.as_string())
 1068     sms.quit()
 1069 
 1070 
 1071 item_regex = re.compile(r'^([\$\^\*~!%\?#=\+\-])\s')
 1072 email_regex = re.compile('([\w\-\.]+@(\w[\w\-]+\.)+[\w\-]+)')
 1073 sign_regex = re.compile(r'(^\s*([+-])?)')
 1074 week_regex = re.compile(r'[+-]?(\d+)w', flags=re.I)
 1075 estr_regex = re.compile(r'easter\((\d{4,4})\)', flags=re.I)
 1076 day_regex = re.compile(r'[+-]?(\d+)d', flags=re.I)
 1077 hour_regex = re.compile(r'[+-]?(\d+)h', flags=re.I)
 1078 minute_regex = re.compile(r'[+-]?(\d+)m', flags=re.I)
 1079 date_calc_regex = re.compile(r'^\s*(.+)\s+([+-])\s+(.+)\s*$')
 1080 period_string_regex = re.compile(r'^\s*([+-]?(\d+[wWdDhHmM])+\s*$)')
 1081 timezone_regex = re.compile(r'^(.+)\s+([A-Za-z]+/[A-Za-z]+)$')
 1082 int_regex = re.compile(r'^\s*([+-]?\d+)\s*$')
 1083 leadingzero = re.compile(r'(?<!(:|\d|-))0+(?=\d)')
 1084 trailingzeros = re.compile(r'(:00)')
 1085 at_regex = re.compile(r'\s+@', re.MULTILINE)
 1086 minus_regex = re.compile(r'\s+\-(?=[a-zA-Z])')
 1087 amp_regex = re.compile(r'\s+&')
 1088 comma_regex = re.compile(r',\s*')
 1089 range_regex = re.compile(r'range\((\d+)(\s*,\s*(\d+))?\)')
 1090 id_regex = re.compile(r'^\s*@i')
 1091 anniversary_regex = re.compile(r'!(\d{4})!')
 1092 group_regex = re.compile(r'^\s*(.*)\s+(\d+)/(\d+):\s*(.*)')
 1093 groupdate_regex = re.compile(r'\by{2}\b|\by{4}\b|\b[dM]{1,4}\b|\bw\b')
 1094 options_regex = re.compile(r'^\s*(!?[fk](\[[:\d]+\])?)|(!?[clostu])\s*$')
 1095 # completion_regex = re.compile(r'(?:^.*?)((?:\@[a-zA-Z] ?)?\b\S*)$')
 1096 completion_regex = re.compile(r'((?:[@&][a-zA-Z]? ?)?(?:\b[a-zA-Z0-9_/:]+)?)$')
 1097 
 1098 # what about other languages?
 1099 # lun mar mer jeu ven sam dim
 1100 # we'll use this to reduce abbrevs to 2 letters for weekdays in rrule
 1101 threeday_regex = re.compile(r'(MON|TUE|WED|THU|FRI|SAT|SUN)',
 1102                             re.IGNORECASE)
 1103 
 1104 ONEMINUTE = timedelta(minutes=1)
 1105 ONEHOUR = timedelta(hours=1)
 1106 ONEDAY = timedelta(days=1)
 1107 ONEWEEK = timedelta(weeks=1)
 1108 
 1109 rel_date_regex = re.compile(r'(?<![0-9])([-+][0-9]+)')
 1110 rel_month_regex = re.compile(r'(?<![0-9])([-+][0-9]+)/([0-9]+)')
 1111 blank_lines_regex = re.compile(r'^(\s*$){2,}', re.MULTILINE)
 1112 
 1113 fmt = "%a %Y-%m-%d %H:%M %Z"
 1114 
 1115 rfmt = "%Y-%m-%d %H:%M%z"
 1116 efmt = "%H:%M %a %b %d"
 1117 
 1118 sfmt = "%Y%m%dT%H%M"
 1119 
 1120 # finish and due dates
 1121 zfmt = "%Y%m%dT%H%M"
 1122 
 1123 sortdatefmt = "%Y%m%d"
 1124 reprdatefmt = "%a %b %d, %Y"
 1125 shortdatefmt = "%a %b %d %Y"
 1126 shortyearlessfmt = "%a %b %d"
 1127 weekdayfmt = "%a %d"
 1128 sorttimefmt = "%H%M"
 1129 etmdatefmt = "%Y-%m-%d"
 1130 etmtimefmt = "%H:%M"
 1131 rrulefmt = "%a %b %d %Y %H:%M %Z %z"
 1132 
 1133 today = datetime.now(tzlocal()).replace(
 1134     hour=0, minute=0, second=0, microsecond=0)
 1135 yesterday = today - ONEDAY
 1136 tomorrow = today + ONEDAY
 1137 
 1138 day_begin = time(0, 0)
 1139 day_end = time(23, 59)
 1140 day_end_minutes = 23 * 60 + 59
 1141 actions = ["s", "d", "e", "p", "v"]
 1142 
 1143 
 1144 def setConfig(options):
 1145     dfile_encoding = options['encoding']['file']
 1146     cal_regex = None
 1147     if 'calendars' in options:
 1148         cal_pattern = r'^%s' % '|'.join(
 1149             [x[2] for x in options['calendars'] if x[1]])
 1150         cal_regex = re.compile(cal_pattern)
 1151 
 1152     options['user_data'] = {}
 1153     options['completions'] = []
 1154     options['reports'] = []
 1155     completions = set([])
 1156     reports = set([])
 1157     completion_files = []
 1158     report_files = []
 1159     user_files = []
 1160 
 1161     # get info from files in datadir
 1162     prefix, filelist = getFiles(options['datadir'], include=r'*.cfg')
 1163     for rp in ['completions.cfg', 'users.cfg', 'reports.cfg']:
 1164         fp = os.path.join(options['etmdir'], rp)
 1165         if os.path.isfile(fp):
 1166             filelist.append((fp, rp))
 1167     logger.info('prefix: {0}; files: {1}'.format(prefix, filelist))
 1168     for fp, rp in filelist:
 1169         if os.path.split(rp)[0] and cal_regex and not cal_regex.match(rp):
 1170             continue
 1171         np = relpath(fp, options['etmdir'])
 1172         drive, parts = os_path_splitall(fp)
 1173         n, e = os.path.splitext(parts[-1])
 1174         # skip etmtk and any other .cfg files other than the following
 1175         if n == "completions":
 1176             completion_files.append((np, fp, False))
 1177             with codecs.open(fp, 'r', dfile_encoding) as fo:
 1178                 for x in fo.readlines():
 1179                     x = x.rstrip()
 1180                     if x and x[0] != "#":
 1181                         completions.add(x)
 1182 
 1183         elif n == "reports":
 1184             report_files.append((np, fp, False))
 1185             with codecs.open(fp, 'r', dfile_encoding) as fo:
 1186                 for x in fo.readlines():
 1187                     x = x.rstrip()
 1188                     if x and x[0] != "#":
 1189                         reports.add(x)
 1190 
 1191         elif n == "users":
 1192             user_files.append((np, fp, False))
 1193             fo = codecs.open(fp, 'r', dfile_encoding)
 1194             tmp = yaml.load(fo)
 1195             fo.close()
 1196             try:
 1197                 # if a key already exists, use the tmp value
 1198                 options['user_data'].update(tmp)
 1199                 for x in tmp.keys():
 1200                     completions.add("@u {0}".format(x))
 1201                     completions.add("&u {0}".format(x))
 1202             except:
 1203                 logger.exception("Error loading {0}".format(fp))
 1204 
 1205     # get info from cfg_files
 1206     if 'cfg_files' in options and options['cfg_files']:
 1207         if 'completions' in options['cfg_files'] and options['cfg_files']['completions']:
 1208             for fp in options['cfg_files']['completions']:
 1209                 completion_files.append((relpath(fp, options['etmdir']), fp, False))
 1210                 with codecs.open(fp, 'r', dfile_encoding) as fo:
 1211                     for x in fo.readlines():
 1212                         x = x.rstrip()
 1213                         if x and x[0] != "#":
 1214                             completions.add(x)
 1215         if 'reports' in options['cfg_files'] and options['cfg_files']['reports']:
 1216             for fp in options['cfg_files']['reports']:
 1217                 report_files.append((relpath(fp, options['etmdir']), fp, False))
 1218                 with codecs.open(fp, 'r', dfile_encoding) as fo:
 1219                     for x in fo.readlines():
 1220                         x = x.rstrip()
 1221                         if x and x[0] != "#":
 1222                             reports.add(x)
 1223         if 'users' in options['cfg_files'] and options['cfg_files']['users']:
 1224             for fp in options['cfg_files']['users']:
 1225                 user_files.append((relpath(fp, options['etmdir']), fp, False))
 1226                 fo = codecs.open(fp, 'r', dfile_encoding)
 1227                 tmp = yaml.load(fo)
 1228                 fo.close()
 1229                 # if a key already exists, use this value
 1230                 options['user_data'].update(tmp)
 1231                 for x in tmp.keys():
 1232                     completions.add("@u {0}".format(x))
 1233                     completions.add("&u {0}".format(x))
 1234 
 1235     if completions:
 1236         completions = list(completions)
 1237         completions.sort()
 1238         options['completions'] = completions
 1239         options['keywords'] = [x[3:] for x in completions if x.startswith('@k')]
 1240     else:
 1241         logger.info('no completions')
 1242     if reports:
 1243         reports = list(reports)
 1244         reports.sort()
 1245         options['reports'] = reports
 1246     else:
 1247         logger.info('no reports')
 1248 
 1249     options['completion_files'] = completion_files
 1250     options['report_files'] = report_files
 1251     options['user_files'] = user_files
 1252 
 1253 
 1254 # noinspection PyGlobalUndefined
 1255 term_encoding = None
 1256 file_encoding = None
 1257 gui_encoding = None
 1258 local_timezone = None
 1259 
 1260 NONE = YESTERDAY = TODAY = TOMORROW = ""
 1261 trans = lang = None
 1262 
 1263 def get_options(d=''):
 1264     """
 1265     """
 1266     logger.debug('starting get_options with directory: "{0}"'.format(d))
 1267     global parse, lang, trans, s2or3, term_encoding, file_encoding, gui_encoding, local_timezone, NONE, YESTERDAY, TODAY, TOMORROW, BGCOLOR, FGCOLOR, tstr2SCI, CALENDAR_COLORS
 1268 
 1269     from locale import getpreferredencoding
 1270     from sys import stdout
 1271     try:
 1272         dterm_encoding = stdout.term_encoding
 1273     except AttributeError:
 1274         dterm_encoding = None
 1275     if not dterm_encoding:
 1276         dterm_encoding = getpreferredencoding()
 1277 
 1278     term_encoding = dterm_encoding = dfile_encoding = codecs.lookup(dterm_encoding).name
 1279 
 1280     use_locale = ()
 1281     etmdir = ''
 1282     NEWCFG = "etmtk.cfg"
 1283     OLDCFG = "etm.cfg"
 1284     using_oldcfg = False
 1285     if d and os.path.isdir(d):
 1286         etmdir = os.path.abspath(d)
 1287     else:
 1288         homedir = os.path.expanduser("~")
 1289         etmdir = os.path.normpath(os.path.join(homedir, ".etm"))
 1290     newconfig = os.path.normpath(os.path.join(etmdir, NEWCFG))
 1291     oldconfig = os.path.normpath(os.path.join(etmdir, OLDCFG))
 1292     default_datadir = os.path.normpath(os.path.join(etmdir, 'data'))
 1293     logger.debug('checking first for: {0}; then: {1}'.format(newconfig, oldconfig))
 1294 
 1295     colors_cfg = os.path.normpath(os.path.join(etmdir, 'colors.cfg'))
 1296     # the default colors
 1297     FGCOLOR = BASE_COLORS['foreground']
 1298     HLCOLOR = BASE_COLORS['highlight']
 1299     BGCOLOR = BASE_COLORS['background']
 1300     item_colors = ITEM_COLORS
 1301     if os.path.isfile(colors_cfg):
 1302         logger.info('using colors file: {0}'.format(colors_cfg))
 1303         fo = codecs.open(colors_cfg, 'r', dfile_encoding)
 1304         use_colors = yaml.load(fo)
 1305         fo.close()
 1306 
 1307         if use_colors:
 1308             FGCOLOR = use_colors['base']['foreground']
 1309             HLCOLOR = use_colors['base']['highlight']
 1310             BGCOLOR = use_colors['base']['background']
 1311             CALENDAR_COLORS = use_colors['calendar']
 1312             item_colors = use_colors['item']
 1313     elif os.path.isdir(etmdir):
 1314         fo = codecs.open(colors_cfg, 'w', dfile_encoding)
 1315         fo.writelines(colors_light)
 1316         fo.close()
 1317 
 1318     for key in tstr2SCI:
 1319         # update the item colors
 1320         tstr2SCI[key][1] = item_colors[key]
 1321 
 1322     locale_cfg = os.path.normpath(os.path.join(etmdir, 'locale.cfg'))
 1323     if os.path.isfile(locale_cfg):
 1324         logger.info('using locale file: {0}'.format(locale_cfg))
 1325         fo = codecs.open(locale_cfg, 'r', dfile_encoding)
 1326         use_locale = yaml.load(fo)
 1327         fo.close()
 1328         if use_locale:
 1329             dgui_encoding = use_locale[0][1]
 1330         else:
 1331             use_locale = ()
 1332             tmp = locale.getdefaultlocale()
 1333             dgui_encoding = tmp[1]
 1334     else:
 1335         use_locale = ()
 1336         tmp = locale.getdefaultlocale()
 1337         dgui_encoding = tmp[1]
 1338 
 1339     if use_locale:
 1340         locale.setlocale(locale.LC_ALL, map(str, use_locale[0]))
 1341         lcl = locale.getlocale()
 1342         lang = use_locale[0][0]
 1343     else:
 1344         lcl = locale.getdefaultlocale()
 1345 
 1346 
 1347     NONE = '~ {0} ~'.format(_("none"))
 1348     YESTERDAY = _('Yesterday')
 1349     TODAY = _('Today')
 1350     TOMORROW = _('Tomorrow')
 1351 
 1352     try:
 1353         dgui_encoding = codecs.lookup(dgui_encoding).name
 1354     except (TypeError, LookupError):
 1355         dgui_encoding = codecs.lookup(locale.getpreferredencoding()).name
 1356 
 1357     time_zone = get_localtz()[0]
 1358 
 1359     default_freetimes = {'opening': 8 * 60, 'closing': 17 * 60, 'minimum': 30, 'buffer': 15}
 1360 
 1361     git_command, git_history, git_commit, git_init = getGit()
 1362     hg_command, hg_history, hg_commit, hg_init = getMercurial()
 1363 
 1364     default_vcs = ''
 1365 
 1366     default_options = {
 1367         'action_markups': {'default': 1.0, },
 1368         'action_minutes': 6,
 1369         'action_interval': 1,
 1370         'action_keys': 'k',
 1371         'action_timer': {'running': '', 'paused': ''},
 1372         'action_rates': {'default': 100.0, },
 1373         'action_template': '!hours!h $!value!) !label! (!count!)',
 1374 
 1375         'agenda_colors': 2,
 1376         'agenda_days': 2,
 1377         'agenda_indent': 3,
 1378         'agenda_width1': 32,
 1379         'agenda_width2': 18,
 1380         'agenda_omit': ['ac', 'fn', 'ns'],
 1381 
 1382         'alert_default': ['m'],
 1383         'alert_displaycmd': '',
 1384         'alert_soundcmd': '',
 1385         'alert_template': '!time_span!\n!l!\n\n!d!',
 1386         'alert_voicecmd': '',
 1387         'alert_wakecmd': '',
 1388 
 1389         'ampm': True,
 1390         'completions_width': 36,
 1391 
 1392         'calendars': [],
 1393 
 1394         'cfg_files': {'completions': [], 'reports': [], 'users': []},
 1395 
 1396         'countdown_command': '',
 1397         'countdown_minutes': 10,
 1398 
 1399         'current_textfile': '',
 1400         'current_htmlfile': '',
 1401         'current_icsfolder': '',
 1402         'current_indent': 3,
 1403         'current_opts': '',
 1404         'current_width1': 48,
 1405         'current_width2': 18,
 1406 
 1407         'datadir': default_datadir,
 1408         'dayfirst': dayfirst,
 1409 
 1410         'details_rows': 4,
 1411 
 1412         'display_idletime': True,
 1413 
 1414         'early_hour': 6,
 1415 
 1416         'edit_cmd': '',
 1417         'email_template': "!time_span!\n!l!\n\n!d!",
 1418         'etmdir': etmdir,
 1419         'exportdir': etmdir,
 1420         'encoding': {'file': dfile_encoding, 'gui': dgui_encoding,
 1421                      'term': dterm_encoding},
 1422         'filechange_alert': '',
 1423         'fontsize_fixed': 0,
 1424         'fontsize_tree': 0,
 1425         'freetimes': default_freetimes,
 1426         'icscal_file': os.path.normpath(os.path.join(etmdir, 'etmcal.ics')),
 1427         'icsitem_file': os.path.normpath(os.path.join(etmdir, 'etmitem.ics')),
 1428         'icssync_folder': '',
 1429         'ics_subscriptions': [],
 1430 
 1431         'local_timezone': time_zone,
 1432 
 1433         'message_last': 0,
 1434         'message_next': 0,
 1435 
 1436         # 'monthly': os.path.join('personal', 'monthly'),
 1437         'monthly': os.path.join('personal', 'monthly'),
 1438         'outline_depth': 0,
 1439         'prefix': "\n  ",
 1440         'prefix_uses': 'dfjlmrtz+-',
 1441         'report_begin': '1',
 1442         'report_end': '+1/1',
 1443         'report_colors': 2,
 1444         'report_indent': 3,
 1445         'report_width1': 43,
 1446         'report_width2': 17,
 1447 
 1448         'show_finished': 1,
 1449 
 1450         'snooze_command': '',
 1451         'snooze_minutes': 7,
 1452 
 1453         'smtp_from': '',
 1454         'smtp_id': '',
 1455         'smtp_pw': '',
 1456         'smtp_server': '',
 1457 
 1458         'sms_from': '',
 1459         'sms_message': '!summary!',
 1460         'sms_phone': '',
 1461         'sms_pw': '',
 1462         'sms_server': '',
 1463         'sms_subject': '!time_span!',
 1464 
 1465         'style': default_style,
 1466 
 1467         'sundayfirst': False,
 1468         'update_minutes': 15,
 1469         'vcs_system': default_vcs,
 1470         'vcs_settings': {'command': '', 'commit': '', 'dir': '', 'file': '', 'history': '', 'init': '', 'limit': ''},
 1471         'weeks_after': 52,
 1472         'yearfirst': yearfirst}
 1473 
 1474     if not os.path.isdir(etmdir):
 1475         # first etm use, no etmdir
 1476         os.makedirs(etmdir)
 1477     logfile = os.path.normpath(os.path.abspath(os.path.join(etmdir, "etmtk_log.txt")))
 1478     if not os.path.isfile(logfile):
 1479         fo = codecs.open(logfile, 'w', dfile_encoding)
 1480         fo.write("")
 1481         fo.close()
 1482 
 1483     if os.path.isfile(newconfig):
 1484         try:
 1485             logger.info('user options: {0}'.format(newconfig))
 1486             fo = codecs.open(newconfig, 'r', dfile_encoding)
 1487             user_options = yaml.load(fo)
 1488             fo.close()
 1489         except yaml.parser.ParserError:
 1490             logger.exception(
 1491                 'Exception loading {0}. Using default options.'.format(newconfig))
 1492             user_options = {}
 1493     elif os.path.isfile(oldconfig):
 1494         try:
 1495             using_oldcfg = True
 1496             logger.info('user options: {0}'.format(oldconfig))
 1497             fo = codecs.open(oldconfig, 'r', dfile_encoding)
 1498             user_options = yaml.load(fo)
 1499             fo.close()
 1500         except yaml.parser.ParserError:
 1501             logger.exception(
 1502                 'Exception loading {0}. Using default options.'.format(oldconfig))
 1503             user_options = {}
 1504     else:
 1505         logger.info('using default options')
 1506         user_options = {'datadir': default_datadir}
 1507         fo = codecs.open(newconfig, 'w', dfile_encoding)
 1508         yaml.safe_dump(user_options, fo)
 1509         fo.close()
 1510 
 1511     options = deepcopy(default_options)
 1512     changed = False
 1513     if user_options:
 1514         if 'agenda_omit' in user_options:
 1515             tmp = [x for x in user_options['agenda_omit'] if x in ['ac', 'by', 'fn', 'ns', 'oc']]
 1516             if tmp != user_options['agenda_omit']:
 1517                 user_options['agenda_omit'] = tmp
 1518                 changed = True
 1519         if ('actions_timercmd' in user_options and
 1520                 user_options['actions_timercmd']):
 1521             user_options['action_timer']['running'] = \
 1522                 user_options['actions_timercmd']
 1523             del user_options['actions_timercmd']
 1524             changed = True
 1525         options.update(user_options)
 1526     else:
 1527         user_options = {}
 1528     # logger.debug("user_options: {0}".format(user_options))
 1529 
 1530     for key in default_options:
 1531         if key in ['action_keys', 'show_finished', 'fontsize_busy', 'fontsize_fixed', 'fontsize_tree', 'outline_depth', 'prefix', 'prefix_uses', 'icssyc_folder', 'ics_subscriptions', 'agenda_days', 'message_next', 'message_last', 'agenda_omit']:
 1532             if key not in user_options:
 1533                 # we want to allow 0 as an entry
 1534                 options[key] = default_options[key]
 1535                 changed = True
 1536         elif key in ['ampm', 'dayfirst', 'yearfirst', 'retain_ids', 'display_idletime']:
 1537             if key not in user_options:
 1538                 # we want to allow False as an entry
 1539                 options[key] = default_options[key]
 1540                 changed = True
 1541 
 1542         elif default_options[key] and (key not in user_options or not user_options[key]):
 1543             options[key] = default_options[key]
 1544             changed = True
 1545 
 1546     if type(options['update_minutes']) is not int or options['update_minutes'] <= 0 or options['update_minutes'] > 59:
 1547         options['update_minutes'] = default_options['update_minutes']
 1548 
 1549     remove_keys = []
 1550     for key in options:
 1551         if key not in default_options:
 1552             remove_keys.append(key)
 1553             changed = True
 1554     for key in remove_keys:
 1555         del options[key]
 1556 
 1557     action_keys = [x for x in options['action_keys']]
 1558     if action_keys:
 1559         for at_key in action_keys:
 1560             if at_key not in key2type or "~" not in key2type[at_key]:
 1561                 action_keys.remove(at_key)
 1562                 changed = True
 1563         if changed:
 1564             options['action_keys'] = "".join(action_keys)
 1565 
 1566     # check freetimes
 1567     for key in default_freetimes:
 1568         if key not in options['freetimes']:
 1569             options['freetimes'][key] = default_freetimes[key]
 1570             logger.warn('A value was not provided for freetimes[{0}] - using the default value.'.format(key))
 1571             changed = True
 1572         else:
 1573             if type(options['freetimes'][key]) is not int:
 1574                 changed = True
 1575                 try:
 1576                     options['freetimes'][key] = int(eval(options['freetimes'][key]))
 1577                 except:
 1578                     logger.warn('The value provided for freetimes[{0}], "{1}", could not be converted to an integer - using the default value instead.'.format(key, options['freetimes'][key]))
 1579                     options['freetimes'][key] = default_freetimes[key]
 1580 
 1581     free_keys = [x for x in options['freetimes'].keys()]
 1582     for key in free_keys:
 1583         if key not in default_freetimes:
 1584             del options['freetimes'][key]
 1585             logger.warn('A value was provided for freetimes[{0}], but this is an invalid option and has been deleted.'.format(key))
 1586             changed = True
 1587 
 1588     if not os.path.isdir(options['datadir']):
 1589         """
 1590         <datadir>
 1591             personal/
 1592                 monthly/
 1593             sample/
 1594                 completions.cfg
 1595                 reports.cfg
 1596                 sample.txt
 1597                 users.cfg
 1598             shared/
 1599                 holidays.txt
 1600 
 1601         etmtk.cfg
 1602             calendars:
 1603             - - personal
 1604               - true
 1605               - personal
 1606             - - sample
 1607               - true
 1608               - sample
 1609             - - shared
 1610               - true
 1611               - shared
 1612         """
 1613         changed = True
 1614         term_print('creating datadir: {0}'.format(options['datadir']))
 1615         # first use of this datadir - first use of new etm?
 1616         os.makedirs(options['datadir'])
 1617         # create one task for new users to join the etm discussion group
 1618         currfile = ensureMonthly(options)
 1619         with open(currfile, 'w') as fo:
 1620             fo.write(JOIN)
 1621         sample = os.path.normpath(os.path.join(options['datadir'], 'sample'))
 1622         os.makedirs(sample)
 1623         with codecs.open(os.path.join(sample, 'sample.txt'), 'w', dfile_encoding) as fo:
 1624             fo.write(SAMPLE)
 1625         holidays = os.path.normpath(os.path.join(options['datadir'], 'shared'))
 1626         os.makedirs(holidays)
 1627         with codecs.open(os.path.join(holidays, 'holidays.txt'), 'w', dfile_encoding) as fo:
 1628             fo.write(HOLIDAYS)
 1629         with codecs.open(os.path.join(options['datadir'], 'sample', 'completions.cfg'), 'w', dfile_encoding) as fo:
 1630             fo.write(COMPETIONS)
 1631         with codecs.open(os.path.join(options['datadir'], 'sample', 'reports.cfg'), 'w', dfile_encoding) as fo:
 1632             fo.write(REPORTS)
 1633         with codecs.open(os.path.join(options['datadir'], 'sample', 'users.cfg'), 'w', dfile_encoding) as fo:
 1634             fo.write(USERS)
 1635         if not options['calendars']:
 1636             options['calendars'] = [['personal', True, 'personal'], ['sample', True, 'sample'], ['shared', True, 'shared']]
 1637     logger.info('using datadir: {0}'.format(options['datadir']))
 1638     logger.debug('changed: {0}; user: {1}; options: {2}'.format(changed, (user_options != default_options), (options != default_options)))
 1639     if changed or using_oldcfg:
 1640         # save options to newconfig even if user options came from oldconfig
 1641         logger.debug('Writing etmtk.cfg changes to {0}'.format(newconfig))
 1642         fo = codecs.open(newconfig, 'w', options['encoding']['file'])
 1643         yaml.safe_dump(options, fo, default_flow_style=False)
 1644         fo.close()
 1645 
 1646     # add derived options
 1647     if options['vcs_system'] == 'git':
 1648         if git_command:
 1649             options['vcs'] = {'command': git_command, 'history': git_history, 'commit': git_commit, 'init': git_init, 'dir': '.git', 'limit': '-n', 'file': ""}
 1650             repo = os.path.normpath(os.path.join(options['datadir'], options['vcs']['dir']))
 1651             work = options['datadir']
 1652             # logger.debug('{0} options: {1}'.format(options['vcs_system'], options['vcs']))
 1653         else:
 1654             logger.warn('could not setup "git" vcs')
 1655             options['vcs'] = {}
 1656             options['vcs_system'] = ''
 1657     elif options['vcs_system'] == 'mercurial':
 1658         if hg_command:
 1659             options['vcs'] = {'command': hg_command, 'history': hg_history, 'commit': hg_commit, 'init': hg_init, 'dir': '.hg', 'limit': '-l', 'file': ''}
 1660             repo = os.path.normpath(os.path.join(options['datadir'], options['vcs']['dir']))
 1661             work = options['datadir']
 1662             # logger.debug('{0} options: {1}'.format(options['vcs_system'], options['vcs']))
 1663         else:
 1664             logger.warn('could not setup "mercurial" vcs')
 1665             options['vcs'] = {}
 1666             options['vcs_system'] = ''
 1667     else:
 1668         options['vcs_system'] = ''
 1669         options['vcs'] = {}
 1670 
 1671     # overrule the defaults if any custom settings are given
 1672     if options['vcs_system']:
 1673         if options['vcs_settings']:
 1674             # update any settings with custom modifications
 1675             for key in options['vcs_settings']:
 1676                 if options['vcs_settings'][key]:
 1677                     options['vcs'][key] = options['vcs_settings'][key]
 1678         # add the derived options
 1679         options['vcs']['repo'] = repo
 1680         options['vcs']['work'] = work
 1681 
 1682     if options['vcs']:
 1683         vcs_lst = []
 1684         keys = [x for x in options['vcs'].keys()]
 1685         keys.sort()
 1686         for key in keys:
 1687             vcs_lst.append("{0}: {1}".format(key, options['vcs'][key]))
 1688         vcs_str = "\n      ".join(vcs_lst)
 1689     else:
 1690         vcs_str = ""
 1691     logger.info('using vcs {0}; options:\n      {1}'.format(options['vcs_system'], vcs_str))
 1692 
 1693     (options['daybegin_fmt'], options['dayend_fmt'], options['reprtimefmt'], options['longreprtimefmt'], options['reprdatetimefmt'], options['etmdatetimefmt'],
 1694      options['rfmt'], options['efmt']) = get_fmts(options)
 1695     options['config'] = newconfig
 1696     options['scratchpad'] = os.path.normpath(os.path.join(options['etmdir'], _("scratchpad")))
 1697     options['colors'] = os.path.normpath(os.path.join(options['etmdir'], "colors.cfg"))
 1698 
 1699     if options['action_minutes'] not in [1, 6, 12, 15, 30, 60]:
 1700         term_print(
 1701             "Invalid action_minutes setting: %s. Reset to 1." %
 1702             options['action_minutes'])
 1703         options['action_minutes'] = 1
 1704 
 1705     setConfig(options)
 1706 
 1707     z = gettz(options['local_timezone'])
 1708     if z is None:
 1709         term_print(
 1710             "Error: bad entry for local_timezone in etmtk.cfg: '%s'" %
 1711             options['local_timezone'])
 1712         options['local_timezone'] = ''
 1713 
 1714     if 'vcs_system' in options and options['vcs_system']:
 1715         logger.debug('vcs_system: {0}'.format(options['vcs_system']))
 1716         f = ''
 1717         if options['vcs_system'] == 'mercurial':
 1718             f = os.path.normpath(os.path.join(options['datadir'], '.hgignore'))
 1719         elif options['vcs_system'] == 'git':
 1720             f = os.path.normpath(os.path.join(options['datadir'], '.gitignore'))
 1721         if f and not os.path.isfile(f):
 1722             fo = open(f, 'w')
 1723             fo.write(IGNORE)
 1724             fo.close()
 1725             logger.info('created: {0}'.format(f))
 1726         logger.debug('checking for {0}'.format(options['vcs']['repo']))
 1727         if not os.path.isdir(options['vcs']['repo']):
 1728             init = options['vcs']['init']
 1729             # work = (options['vcs']['work'])
 1730             command = init.format(work=options['vcs']['work'], repo=options['vcs']['repo'], mesg="initial commit")
 1731             logger.debug('initializing vcs: {0}'.format(command))
 1732             # run_cmd(command)
 1733             subprocess.call(command, shell=True)
 1734 
 1735     if options['current_icsfolder']:
 1736         if not os.path.isdir(options['current_icsfolder']):
 1737             os.makedirs(options['current_icsfolder'])
 1738 
 1739     options['lcl'] = lcl
 1740     logger.info('using lcl: {0}'.format(lcl))
 1741 
 1742     options['hide_finished'] = False
 1743     # define parse using dayfirst and yearfirst
 1744     setup_parse(options['dayfirst'], options['yearfirst'])
 1745     term_encoding = options['encoding']['term']
 1746     file_encoding = options['encoding']['file']
 1747     gui_encoding = options['encoding']['gui']
 1748     local_timezone = options['local_timezone']
 1749     options['background_color'] = BGCOLOR
 1750     options['highlight_color'] = HLCOLOR
 1751     options['foreground_color'] = FGCOLOR
 1752     options['calendar_colors'] = CALENDAR_COLORS
 1753 
 1754     # set 'bef' here and update on newday using a naive datetime
 1755     now = datetime.now()
 1756     year, wn, dn = now.isocalendar()
 1757     weeks_after = options['weeks_after']
 1758     if dn > 1:
 1759         days = dn - 1
 1760     else:
 1761         days = 0
 1762     week_beg = now - days * ONEDAY
 1763     bef = (week_beg + (7 * (weeks_after + 1)) * ONEDAY)
 1764     options['bef'] = bef
 1765 
 1766     logger.debug("ending get_options")
 1767     return user_options, options, use_locale
 1768 
 1769 
 1770 def get_fmts(options):
 1771     global rfmt, efmt
 1772     df = "%x"
 1773     ef = "%a %b %d"
 1774     if 'ampm' in options and options['ampm']:
 1775         reprtimefmt = "%I:%M%p"
 1776         longreprtimefmt = "%I:%M:%S%p"
 1777         daybegin_fmt = "12am"
 1778         dayend_fmt = "11:59pm"
 1779         rfmt = "{0} %I:%M%p %z".format(df)
 1780         efmt = "%I:%M%p {0}".format(ef)
 1781 
 1782     else:
 1783         reprtimefmt = "%H:%M"
 1784         longreprtimefmt = "%H:%M:%S"
 1785         daybegin_fmt = "0:00"
 1786         dayend_fmt = "23:59"
 1787         rfmt = "{0} %H:%M%z".format(df)
 1788         efmt = "%H:%M {0}".format(ef)
 1789 
 1790     reprdatetimefmt = "%s %s %%Z" % (reprdatefmt, reprtimefmt)
 1791     etmdatetimefmt = "%s %s" % (etmdatefmt, reprtimefmt)
 1792     return (daybegin_fmt, dayend_fmt, reprtimefmt, longreprtimefmt, reprdatetimefmt, etmdatetimefmt, rfmt, efmt)
 1793 
 1794 
 1795 def checkForNewerVersion():
 1796     global python_version2
 1797     import socket
 1798 
 1799     timeout = 10
 1800     socket.setdefaulttimeout(timeout)
 1801     if platform.python_version() >= '3':
 1802         python_version2 = False
 1803         from urllib.request import urlopen
 1804         from urllib.error import URLError
 1805         # from urllib.parse import urlencode
 1806     else:
 1807         python_version2 = True
 1808         from urllib2 import urlopen, URLError
 1809 
 1810     url = "http://people.duke.edu/~dgraham/etmtk/version.txt"
 1811     try:
 1812         response = urlopen(url)
 1813     except URLError as e:
 1814         if hasattr(e, 'reason'):
 1815             msg = """\
 1816 The latest version could not be determined.
 1817 Reason: %s.""" % e.reason
 1818         elif hasattr(e, 'code'):
 1819             msg = """\
 1820 The server couldn\'t fulfill the request.
 1821 Error code: %s.""" % e.code
 1822         return 0, msg
 1823     else:
 1824         # everything is fine
 1825         if python_version2:
 1826             res = response.read()
 1827             vstr = rsplit('\s+', res)[0]
 1828         else:
 1829             res = response.read().decode(term_encoding)
 1830             vstr = rsplit('\s+', res)[0]
 1831 
 1832         if version < vstr:
 1833             return (1, """\
 1834 A newer version of etm, %s, is available at \
 1835 people.duke.edu/~dgraham/etmtk.""" % vstr)
 1836         else:
 1837             return 1, 'You are using the latest version.'
 1838 
 1839 
 1840 type_keys = [x for x in '=^*-+%~$?!#']
 1841 
 1842 type2Str = {
 1843     '$': "ib",
 1844     '^': "oc",
 1845     '*': "ev",
 1846     '~': "ac",
 1847     '!': "nu",  # undated only appear in folders
 1848     '-': "un",  # for next view
 1849     '+': "un",  # for next view
 1850     '%': "du",
 1851     '?': "so",
 1852     '#': "dl"}
 1853 
 1854 id2Type = {
 1855     #   TStr  TNum Forground Color   Icon         view
 1856     "ac": '~',
 1857     "av": '-',
 1858     "by": '>',
 1859     "cs": '+',  # job
 1860     "cu": '+',  # job with unfinished prereqs
 1861     "dl": '#',
 1862     "ds": '%',
 1863     "du": '%',
 1864     "ev": '*',
 1865     "fn": u"\u2713",
 1866     "ib": '$',
 1867     "ns": '!',
 1868     "nu": '!',
 1869     "oc": '^',
 1870     "pc": '+',  # job pastdue
 1871     "pu": '+',  # job pastdue with unfinished prereqs
 1872     "pd": '%',
 1873     "pt": '-',
 1874     "rm": '*',
 1875     "so": '?',
 1876     "un": '-',
 1877 }
 1878 
 1879 # the named colors are listed in colors.py.
 1880 
 1881 # the contents of colors_light.cfg:
 1882 colors_light = """\
 1883 base:
 1884   foreground: 'black'           # default font color
 1885   highlight: '#B2B2AF'          # default highlight color
 1886   background: '#FEFEFC'         # default background color
 1887 
 1888 item:                           # font colors for items in tree views
 1889   ac: 'darkorchid'              # action
 1890   av: 'RoyalBlue3'              # scheduled, available task
 1891   by: 'DarkGoldenRod3'          # begin by
 1892   cs: 'RoyalBlue3'              # scheduled job
 1893   cu: 'gray65'                  # scheduled job with unfinished prereqs
 1894   dl: 'gray70'                  # hidden (folder view)
 1895   ds: 'darkslategray'           # scheduled, delegated task
 1896   du: 'darkslategray'           # unscheduled, delegated task
 1897   ev: 'springgreen4'            # event
 1898   fn: 'gray70'                  # finished task
 1899   ib: 'coral2'                  # inbox
 1900   ns: 'saddlebrown'             # note
 1901   nu: 'saddlebrown'             # unscheduled noted
 1902   oc: 'peachpuff4'              # occasion
 1903   pc: 'firebrick1'              # pastdue job
 1904   pu: 'firebrick1'              # pastdue job with unfinished prereqs
 1905   pd: 'firebrick1'              # pastdue, delegated task
 1906   pt: 'firebrick1'              # pastdue task
 1907   rm: 'seagreen'                # reminder
 1908   so: 'SteelBlue3'              # someday
 1909   un: 'RoyalBlue3'              # unscheduled task (next)
 1910 
 1911 calendar:
 1912   date: 'RoyalBlue3'            # week/month calendar dates
 1913   grid: 'gray85'                # week/month calendar grid lines
 1914   busybar: 'RoyalBlue3'         # week/month busy bars
 1915   current: '#DCEAFC'            # current date calendar background
 1916   active: '#FCFCD9'             # active/selected date background
 1917   occasion: 'gray92'            # occasion background
 1918   conflict: '#FF3300'           # conflict flag
 1919   year_past: 'springgreen4'     # calendar, past years font color
 1920   year_current: 'black'         # calendar, current year font color
 1921   year_future: 'RoyalBlue3'     # calendar, future years font color
 1922 """
 1923 # The contents of colors_light should duplicate the default
 1924 # colors below.
 1925 
 1926 # default colors
 1927 BASE_COLORS = {
 1928     'foreground': "black",
 1929     'highlight': "#B2B2AF",
 1930     'background': "#FEFEFC"
 1931 }
 1932 
 1933 ITEM_COLORS = {
 1934     "ac": "darkorchid",
 1935     "av": "RoyalBlue3",
 1936     "by": "DarkGoldenRod3",
 1937     "cs": "RoyalBlue3",
 1938     "cu": "gray65",
 1939     "dl": "gray70",
 1940     "ds": "darkslategray",
 1941     "du": "darkslategray",
 1942     "ev": "springgreen4",
 1943     "fn": "gray70",
 1944     "ib": "coral2",
 1945     "ns": "saddlebrown",
 1946     "nu": "saddlebrown",
 1947     "oc": "peachpuff4",
 1948     "pc": "firebrick1",
 1949     "pu": "firebrick1",
 1950     "pd": "firebrick1",
 1951     "pt": "firebrick1",
 1952     "rm": "seagreen",
 1953     "so": "SteelBlue3",
 1954     "un": "RoyalBlue3",
 1955 }
 1956 
 1957 CALENDAR_COLORS = {
 1958     "date": "RoyalBlue3",
 1959     "grid": "gray85",
 1960     "busybar": "RoyalBlue3",
 1961     "current": "#DCEAFC",
 1962     "active": "#FCFCD9",
 1963     "occasion": "gray92",
 1964     "conflict": "#FF3300",
 1965     "year_past": "springgreen4",
 1966     "year_current": 'black',
 1967     "year_future": 'RoyalBlue3',
 1968 }
 1969 
 1970 # type string to Sort Color Icon. The color will be added in
 1971 # get_options either from colors.cfg or from the above defaults
 1972 tstr2SCI = {
 1973     #   TStr  TNum Forground Color   Icon         view
 1974     "ac": [23, "", "action", "day"],
 1975     "av": [16, "", "task", "day"],
 1976     "by": [19, "", "beginby", "now"],
 1977     "cs": [18, "", "child", "day"],
 1978     "cu": [22, "", "child", "day"],
 1979     "dl": [28, "", "delete", "folder"],
 1980     "ds": [17, "", "delegated", "day"],
 1981     "du": [21, "", "delegated", "day"],
 1982     "ev": [12, "", "event", "day"],
 1983     "fn": [27, "", "finished", "day"],
 1984     "ib": [10, "", "inbox", "now"],
 1985     "ns": [24, "", "note", "day"],
 1986     "nu": [25, "", "note", "day"],
 1987     "oc": [11, "", "occasion", "day"],
 1988     "pc": [15, "", "child", "now"],
 1989     "pu": [15, "", "child", "now"],
 1990     "pd": [14, "", "delegated", "now"],
 1991     "pt": [13, "", "task", "now"],
 1992     "rm": [12, "", "reminder", "day"],
 1993     "so": [26, "", "someday", "now"],
 1994     "un": [20, "", "task", "next"],
 1995 }
 1996 
 1997 def fmt_period(td, parent=None, short=False):
 1998     if type(td) is not timedelta:
 1999         return td
 2000     if td < ONEMINUTE * 0:
 2001         return '0m'
 2002     if td == ONEMINUTE * 0:
 2003         return '0m'
 2004     until = []
 2005     td_days = td.days
 2006     td_hours = td.seconds // (60 * 60)
 2007     td_minutes = (td.seconds % (60 * 60)) // 60
 2008 
 2009     if short:
 2010         if td_days > 1:
 2011             if td_minutes > 30:
 2012                 td_hours += 1
 2013             td_minutes = 0
 2014         if td_days > 7:
 2015             if td_hours > 12:
 2016                 td_days += 1
 2017             td_hours = 0
 2018 
 2019     if td_days:
 2020         until.append("%dd" % td_days)
 2021     if td_hours:
 2022         until.append("%dh" % td_hours)
 2023     if td_minutes:
 2024         until.append("%dm" % td_minutes)
 2025     if not until:
 2026         until = "0m"
 2027     return "".join(until)
 2028 
 2029 
 2030 def fmt_time(dt, omitMidnight=False, seconds=False, options=None):
 2031     # fmt time, omit leading zeros and, if ampm, convert to lowercase
 2032     # and omit trailing m's
 2033     if not options:
 2034         options = {}
 2035     if omitMidnight and dt.hour == 0 and dt.minute == 0:
 2036         return u''
 2037     # logger.debug('dt before fmt: {0}'.format(dt))
 2038     if seconds:
 2039         dt_fmt = dt.strftime(options['longreprtimefmt'])
 2040     else:
 2041         dt_fmt = dt.strftime(options['reprtimefmt'])
 2042     # logger.debug('dt dt_fmt: {0}'.format(dt_fmt))
 2043     if dt_fmt[0] == "0":
 2044         dt_fmt = dt_fmt[1:]
 2045     # The 3rd test is for Poland where am, pm = ''
 2046     if 'ampm' in options and options['ampm'] and not dt_fmt[-1].isdigit():
 2047         # dt_fmt = dt_fmt.lower()[:-1]
 2048         dt_fmt = dt_fmt.lower()
 2049         dt_fmt = leadingzero.sub('', dt_fmt)
 2050         dt_fmt = trailingzeros.sub('', dt_fmt)
 2051     return s2or3(dt_fmt)
 2052 
 2053 
 2054 def fmt_date(dt, short=False):
 2055     if type(dt) in [str, unicode]:
 2056         return unicode(dt)
 2057     if short:
 2058         tdy = datetime.today()
 2059         if type(dt) == datetime:
 2060             dt = dt.date()
 2061         if dt == tdy.date():
 2062             if python_version2:
 2063                 dt_fmt = u"{0} ({1})".format(unicode(dt.strftime(shortyearlessfmt), gui_encoding, 'ignore'), TODAY)
 2064             else:
 2065                 dt_fmt = "{0} ({1})".format(dt.strftime(shortyearlessfmt), TODAY)
 2066         elif dt == tdy.date() - ONEDAY:
 2067             if python_version2:
 2068                 dt_fmt = u"{0} ({1})".format(unicode(dt.strftime(shortyearlessfmt), gui_encoding, 'ignore'), YESTERDAY)
 2069             else:
 2070                 dt_fmt = "{0} ({1})".format(dt.strftime(shortyearlessfmt), YESTERDAY)
 2071         elif dt == tdy.date() + ONEDAY:
 2072             if python_version2:
 2073                 dt_fmt = u"{0} ({1})".format(unicode(dt.strftime(shortyearlessfmt), gui_encoding, 'ignore'), TOMORROW)
 2074             else:
 2075                 dt_fmt = "{0} ({1})".format(dt.strftime(shortyearlessfmt), TOMORROW)
 2076         elif dt.year == tdy.year:
 2077             dt_fmt = dt.strftime(shortyearlessfmt)
 2078         else:
 2079             dt_fmt = dt.strftime(shortdatefmt)
 2080     else:
 2081         if python_version2:
 2082             dt_fmt = unicode(dt.strftime(reprdatefmt), term_encoding)
 2083         else:
 2084             dt_fmt = dt.strftime(reprdatefmt)
 2085     dt_fmt = leadingzero.sub('', s2or3(dt_fmt))
 2086     return dt_fmt
 2087 
 2088 
 2089 def fmt_shortdatetime(dt, options=None):
 2090     if not options:
 2091         options = {}
 2092     if type(dt) in [str, unicode]:
 2093         return unicode(dt)
 2094     tdy = datetime.today()
 2095     if dt.date() == tdy.date():
 2096         dt_fmt = "%s %s" % (fmt_time(dt, options=options), TODAY)
 2097     elif dt.date() == tdy.date() - ONEDAY:
 2098         dt_fmt = "%s %s" % (fmt_time(dt, options=options), YESTERDAY)
 2099     elif dt.date() == tdy.date() + ONEDAY:
 2100         dt_fmt = "%s %s" % (fmt_time(dt, options=options), TOMORROW)
 2101     elif dt.year == tdy.year:
 2102         try:
 2103             x1 = unicode(fmt_time(dt, options=options))
 2104             x2 = unicode(dt.strftime(shortyearlessfmt))
 2105             dt_fmt = "%s %s" % (x1, x2)
 2106         except:
 2107             dt_fmt = dt.strftime("%X %x")
 2108     else:
 2109         try:
 2110             dt_fmt = dt.strftime(shortdatefmt)
 2111             dt_fmt = leadingzero.sub('', dt_fmt)
 2112         except:
 2113             dt_fmt = dt.strftime("%X %x")
 2114     return s2or3(dt_fmt)
 2115 
 2116 
 2117 def fmt_datetime(dt, options=None):
 2118     if not options:
 2119         options = {}
 2120     t_fmt = fmt_time(dt, options=options)
 2121     dt_fmt = "%s %s" % (dt.strftime(etmdatefmt), t_fmt)
 2122     return s2or3(dt_fmt)
 2123 
 2124 
 2125 def fmt_weekday(dt):
 2126     return fmt_dt(dt, weekdayfmt)
 2127 
 2128 
 2129 def fmt_dt(dt, f):
 2130     dt_fmt = dt.strftime(f)
 2131     return s2or3(dt_fmt)
 2132 
 2133 
 2134 rrule_hsh = {
 2135     'f': 'FREQUENCY',  # unicode
 2136     'i': 'INTERVAL',  # positive integer
 2137     't': 'COUNT',  # total count positive integer
 2138     's': 'BYSETPOS',  # integer
 2139     'u': 'UNTIL',  # unicode
 2140     'M': 'BYMONTH',  # integer 1...12
 2141     'm': 'BYMONTHDAY',  # positive integer
 2142     'W': 'BYWEEKNO',  # positive integer
 2143     'w': 'BYWEEKDAY',  # integer 0 (SU) ... 6 (SA)
 2144     'h': 'BYHOUR',  # positive integer
 2145     'n': 'BYMINUTE',  # positive integer
 2146     'E': 'BYEASTER',  # non-negative integer number of days after easter
 2147 }
 2148 
 2149 # for icalendar export we need BYDAY instead of BYWEEKDAY
 2150 ical_hsh = deepcopy(rrule_hsh)
 2151 ical_hsh['w'] = 'BYDAY'
 2152 ical_hsh['f'] = 'FREQ'
 2153 
 2154 ical_rrule_hsh = {
 2155     'FREQ': 'r',  # unicode
 2156     'INTERVAL': 'i',  # positive integer
 2157     'COUNT': 't',  # total count positive integer
 2158     'BYSETPOS': 's',  # integer
 2159     'UNTIL': 'u',  # unicode
 2160     'BYMONTH': 'M',  # integer 1...12
 2161     'BYMONTHDAY': 'm',  # positive integer
 2162     'BYWEEKNO': 'W',  # positive integer
 2163     'BYDAY': 'w',  # integer 0 (SU) ... 6 (SA)
 2164     # 'BYWEEKDAY': 'w',  # integer 0 (SU) ... 6 (SA)
 2165     'BYHOUR': 'h',  # positive integer
 2166     'BYMINUTE': 'n',  # positive integer
 2167     'BYEASTER': 'E',  # non negative integer number of days after easter
 2168 }
 2169 
 2170 # don't add f and u - they require special processing in get_rrulestr
 2171 rrule_keys = ['i', 'm', 'M', 'w', 'W', 'h', 'n', 't', 's', 'E']
 2172 ical_rrule_keys = ['f', 'i', 'm', 'M', 'w', 'W', 'h', 'n', 't', 's', 'u']
 2173 
 2174 # ^ Presidential election day @s 2004-11-01 12am
 2175 #   @r y &i 4 &m 2, 3, 4, 5, 6, 7, 8 &M 11 &w TU
 2176 
 2177 # don't add l (list) - handeled separately
 2178 freq_hsh = {
 2179     'y': 'YEARLY',
 2180     'm': 'MONTHLY',
 2181     'w': 'WEEKLY',
 2182     'd': 'DAILY',
 2183     'h': 'HOURLY',
 2184     'n': 'MINUTELY',
 2185     'E': 'EASTERLY',
 2186 }
 2187 
 2188 ical_freq_hsh = {
 2189     'YEARLY': 'y',
 2190     'MONTHLY': 'm',
 2191     'WEEKLY': 'w',
 2192     'DAILY': 'd',
 2193     'HOURLY': 'h',
 2194     'MINUTELY': 'n',
 2195     # 'EASTERLY': 'e'
 2196 }
 2197 
 2198 amp_hsh = {
 2199     'r': 'f',    # the starting value for an @r entry is frequency
 2200     'a': 't'     # the starting value for an @a entry is *triggers*
 2201 }
 2202 
 2203 at_keys = [
 2204     's',  # start datetime
 2205     'e',  # extent time spent
 2206     'x',  # expense money spent
 2207     'a',  # alert
 2208     'b',  # begin
 2209     'c',  # context
 2210     'k',  # keyword
 2211     't',  # tags
 2212     'l',  # location
 2213     'n',  # noshow, tasks only. list of views in a, d, k, t.
 2214     'u',  # user
 2215     'f',  # finish date
 2216     'h',  # history (task group)
 2217     'i',  # invitees
 2218     'g',  # goto
 2219     'j',  # job
 2220     'p',  # priority
 2221     'q',  # queue
 2222     'r',  # repetition rule
 2223     '+',  # include
 2224     '-',  # exclude
 2225     'o',  # overdue
 2226     'd',  # description
 2227     'm',  # memo
 2228     'z',  # time zone
 2229     'I',  # id',
 2230     'v',  # action rate key
 2231     'w',  # expense markup key
 2232 ]
 2233 
 2234 all_keys = at_keys + ['entry', 'fileinfo', 'itemtype', 'rrule', '_summary', '_group_summary', '_a', '_j', '_p', '_r', 'prereqs']
 2235 
 2236 all_types = [u'=', u'^', u'*', u'-', u'+', u'%', u'~', u'$',  u'?', u'!',  u'#']
 2237 # job_types = [u'-', u'+', u'%', u'$', u'?', u'#']
 2238 job_types = [u'-', u'+', u'%']
 2239 any_types = [u'=', u'$', u'?', u'#']
 2240 
 2241 # @key to item types - used to check for valid key usage
 2242 key2type = {
 2243     u'+': all_types,
 2244     u'-': all_types,
 2245     u'a': all_types,
 2246     u'b': all_types,
 2247     u'c': all_types,
 2248     u'd': all_types,
 2249     u'e': all_types,
 2250     u'f': job_types + any_types,
 2251     u'g': all_types + any_types,
 2252     u'h': [u'+'] + any_types,
 2253     u'i': [u'*', u'^'] + any_types,
 2254     u'I': all_types,
 2255     u'j': [u'+'] + any_types,
 2256     u'k': all_types,
 2257     u'l': all_types,
 2258     u'm': all_types,
 2259     u'o': job_types + any_types,
 2260     u'n': [u'-', u'%', u'*'] + any_types,
 2261     # u'n': [u'-', u'%'] + any_types,
 2262     u'p': job_types + any_types,
 2263     u'q': all_types,
 2264     u'r': all_types,
 2265     u's': all_types,
 2266     u't': all_types,
 2267     u'u': all_types,
 2268     u'v': [u'~'] + any_types,
 2269     u'w': [u'~'] + any_types,
 2270     u'x': [u'~'] + any_types,
 2271     u'z': all_types,
 2272 }
 2273 
 2274 label_keys = [
 2275     # 'f',  # finish date
 2276     '_a',  # alert
 2277     'b',  # begin
 2278     'c',  # context
 2279     'd',  # description
 2280     'g',  # goto
 2281     'i',  # invitees
 2282     'k',  # keyword
 2283     'l',  # location
 2284     'm',  # memo
 2285     'p',  # priority
 2286     '_r',  # repetition rule
 2287     't',  # tags
 2288     'u',  # user
 2289 ]
 2290 
 2291 amp_keys = {
 2292     'r': [
 2293         u'f',   # r frequency
 2294         u'i',   # r interval
 2295         u'm',   # r monthday
 2296         u'M',   # r month
 2297         u'w',   # r weekday
 2298         u'W',   # r week
 2299         u'h',   # r hour
 2300         u'n',   # r minute
 2301         u'E',   # r easter
 2302         u't',   # r total (dateutil COUNT) (c is context in j)
 2303         u'u',   # r until
 2304         u's'],  # r set position
 2305     'j': [
 2306         u'j',   # j job summary
 2307         u'b',   # j beginby
 2308         u'c',   # j context
 2309         u'd',   # j description
 2310         u'e',   # e extent
 2311         u'f',   # j finish
 2312         u'h',   # h history (task group jobs)
 2313         u'p',   # j priority
 2314         u'u',   # user
 2315         u'q'],  # j queue position
 2316 }
 2317 
 2318 
 2319 @memoize
 2320 def makeTree(tree_rows, view=None, calendars=None, sort=True, fltr=None, hide_finished=False):
 2321     """
 2322     e.g. row:
 2323     [('now', (1, 13), datetime.datetime(2015, 8, 20, 0, 0), 10, 'Call Saul', 'personal/dag/monthly/2015/08.txt'),
 2324       'Now',
 2325       'Available',
 2326       ('e2d85baae43140d5966f63ccabe455dcetm', 'pt', 'Call Saul', '-38d', datetime.datetime(2015, 8, 20, 0, 0))]
 2327     """
 2328     tree = {}
 2329     lofl = []
 2330     root = '_'
 2331     empty = True
 2332     cal_regex = None
 2333     if calendars:
 2334         cal_pattern = r'^%s' % '|'.join([x[2] for x in calendars if x[1]])
 2335         cal_regex = re.compile(cal_pattern)
 2336     if fltr is not None:
 2337         mtch = True
 2338         if fltr[0] == '!':
 2339             mtch = False
 2340             fltr = fltr[1:]
 2341         filter_regex = re.compile(r'{0}'.format(fltr), re.IGNORECASE)
 2342         logger.debug('filter: {0} ({1})'.format(fltr, mtch))
 2343     else:
 2344         filter_regex = None
 2345     root_key = tuple(["", root])
 2346     tree.setdefault(root_key, [])
 2347     for pc in tree_rows:
 2348         if hide_finished and pc[-1][1] == 'fn':
 2349             continue
 2350         if cal_regex and not cal_regex.match(pc[0][-1]):
 2351             continue
 2352         if view and pc[0][0] != view:
 2353             continue
 2354         if filter_regex is not None:
 2355             s = "{0} {1}".format(pc[-1][2], " ".join(pc[1:-1]))
 2356             # logger.debug('looking in "{0}"'.format(s))
 2357             m = filter_regex.search(s)
 2358             if not ((mtch and m) or (not mtch and not m)):
 2359                 continue
 2360         if sort:
 2361             pc.pop(0)
 2362         empty = False
 2363         key = tuple([root, pc[0]])
 2364         if key not in tree[root_key]:
 2365             tree[root_key].append(key)
 2366         # logger.debug('key: {0}'.format(key))
 2367         lofl.append(pc)
 2368         for i in range(len(pc) - 1):
 2369             if pc[:i]:
 2370                 parent_key = tuple([":".join(pc[:i]), pc[i]])
 2371             else:
 2372                 parent_key = tuple([root, pc[i]])
 2373             child_key = tuple([":".join(pc[:i + 1]), pc[i + 1]])
 2374             # logger.debug('parent: {0}; child: {1}'.format(parent_key, child_key))
 2375             if pc[:i + 1] not in lofl:
 2376                 lofl.append(pc[:i + 1])
 2377             tree.setdefault(parent_key, [])
 2378             if child_key not in tree[parent_key]:
 2379                 tree[parent_key].append(child_key)
 2380     if empty:
 2381         return {}
 2382     return tree
 2383 
 2384 
 2385 def truncate(s, l):
 2386     if l > 0 and len(s) > l:
 2387         if re.search(' ~ ', s):
 2388             s = s.split(' ~ ')[0]
 2389         s = "%s.." % s[:l - 2]
 2390     return s
 2391 
 2392 
 2393 def tree2Html(tree, indent=2, width1=54, width2=20, colors=2):
 2394     global html_lst
 2395     html_lst = []
 2396     if colors:
 2397         e_c = "</font>"
 2398     else:
 2399         e_c = ""
 2400     tab = " " * indent
 2401 
 2402     def t2H(tree_hsh, node=('', '_'), level=0):
 2403         if type(node) == tuple:
 2404             if type(node[1]) == tuple:
 2405                 t = id2Type[node[1][1]]
 2406                 col2 = "{0:^{width}}".format(
 2407                     truncate(node[1][3], width2), width=width2)
 2408                 if colors == 2:
 2409                     s_c = '<font color="%s">' % tstr2SCI[node[1][1]][1]
 2410                 elif colors == 1:
 2411                     if node[1][1][0] == 'p':
 2412                         # past due
 2413                         s_c = '<font color="%s">' % tstr2SCI[node[1][1]][1]
 2414                     else:
 2415                         s_c = '<font color="black">'
 2416                 else:
 2417                     s_c = ''
 2418                 if width1 > 0:
 2419                     rmlft = width1 - indent * level
 2420                 else:
 2421                     rmlft = 0
 2422                 s = "%s%s%s %-*s %s%s" % (tab * level, s_c, unicode(t),
 2423                                           rmlft, unicode(truncate(node[1][2], rmlft)),
 2424                                           col2, e_c)
 2425                 html_lst.append(s)
 2426             else:
 2427                 html_lst.append("%s%s" % (tab * level, node[1]))
 2428         else:
 2429             html_lst.append("%s%s" % (tab * level, node))
 2430         if node not in tree_hsh:
 2431             return ()
 2432         level += 1
 2433         nodes = tree_hsh[node]
 2434         for n in nodes:
 2435             t2H(tree_hsh, n, level)
 2436     t2H(tree)
 2437     return [x[indent:] for x in html_lst]
 2438 
 2439 
 2440 def tree2Rst(tree, indent=2, width1=54, width2=14, colors=0,
 2441              number=False, count=0, count2id=None):
 2442     global text_lst
 2443     args = [count, count2id]
 2444     text_lst = []
 2445     if colors:
 2446         e_c = ""
 2447     else:
 2448         e_c = ""
 2449     tab = "   " * indent
 2450 
 2451     def t2H(tree_hsh, node=('', '_'), level=0):
 2452         if args[1] is None:
 2453             args[1] = {}
 2454         if type(node) == tuple:
 2455             if type(node[1]) == tuple:
 2456                 args[0] += 1
 2457                 # join the uuid and the datetime of the instance
 2458                 args[1][args[0]] = "{0}::{1}".format(node[-1][0], node[-1][-1])
 2459                 t = id2Type[node[1][1]]
 2460                 s_c = ''
 2461                 col2 = "{0:^{width}}".format(
 2462                     truncate(node[1][3], width2), width=width2)
 2463                 if number:
 2464                     rmlft = width1 - indent * level - 2 - len(str(args[0]))
 2465                     s = "%s\%s%s [%s] %-*s %s%s" % (
 2466                         tab * (level - 1), s_c, unicode(t),
 2467                         args[0], rmlft,
 2468                         unicode(truncate(node[1][2], rmlft)),
 2469                         col2, e_c)
 2470                 else:
 2471                     rmlft = width1 - indent * level
 2472                     s = "%s\%s%s %-*s %s%s" % (tab * (level - 1), s_c, unicode(t),
 2473                                                rmlft,
 2474                                                unicode(truncate(node[1][2], rmlft)),
 2475                                                col2, e_c)
 2476                 text_lst.append(s)
 2477             else:
 2478                 if node[1].strip() != '_':
 2479                     text_lst.append("%s[b]%s[/b]" % (tab * (level - 1), node[1]))
 2480         else:
 2481             text_lst.append("%s%s" % (tab * (level - 1), node))
 2482         if node not in tree_hsh:
 2483             return ()
 2484         level += 1
 2485         nodes = tree_hsh[node]
 2486         for n in nodes:
 2487             t2H(tree_hsh, n, level)
 2488 
 2489     t2H(tree)
 2490     return [x for x in text_lst], args[0], args[1]
 2491 
 2492 
 2493 def tree2Text(tree, indent=4, width1=43, width2=20, colors=0,
 2494               number=False, count=0, count2id=None, depth=0):
 2495     global text_lst
 2496     logger.debug("data.tree2Text: width1={0}, width2={1}, colors={2}".format(width1, width2, colors))
 2497     args = [count, count2id]
 2498     text_lst = []
 2499     if colors:
 2500         e_c = ""
 2501     else:
 2502         e_c = ""
 2503     tab = " " * indent
 2504 
 2505     def t2H(tree_hsh, node=('', '_'), level=0):
 2506         if depth and level > depth:
 2507             return
 2508         if args[1] is None:
 2509             args[1] = {}
 2510         if type(node) == tuple:
 2511             if type(node[1]) == tuple:
 2512                 args[0] += 1
 2513                 # join the uuid and the datetime of the instance
 2514                 args[1][args[0]] = "{0}::{1}".format(node[-1][0], node[-1][-1])
 2515                 t = id2Type[node[1][1]]
 2516                 s_c = ''
 2517                 # logger.debug("node13: {0}; width2: {1}".format(node[1][3],  width2))
 2518                 if node[1][3]:
 2519                     col2 = "{0:^{width}}".format(
 2520                         truncate(node[1][3], width2), width=width2)
 2521                 else:
 2522                     col2 = ""
 2523                 if number:
 2524                     if width1 > 0:
 2525                         rmlft = width1 - indent * level - 2 - len(str(args[0]))
 2526                     else:
 2527                         rmlft = 0
 2528                     s = u"{0:s}{1:s}{2:s} [{3:s}] {4:<*s} {5:s}{6:s}".format(
 2529                         tab * level,
 2530                         s_c,
 2531                         unicode(t),
 2532                         args[0],
 2533                         rmlft,
 2534                         unicode(truncate(node[1][2], rmlft)),
 2535                         col2, e_c)
 2536                 else:
 2537                     if width1 > 0:
 2538                         rmlft = width1 - indent * level
 2539                     else:
 2540                         rmlft = 0
 2541                     s = "%s%s%s %-*s %s%s" % (tab * level, s_c, unicode(t), rmlft, unicode(truncate(node[1][2], rmlft)), col2, e_c)
 2542                 text_lst.append(s)
 2543             else:
 2544                 aug = "%s%s" % (tab * level, node[1])
 2545                 text_lst.append(aug.split('!!')[0])
 2546         else:
 2547             text_lst.append("%s%s" % (tab * level, node))
 2548         if node not in tree_hsh:
 2549             return ()
 2550         level += 1
 2551         nodes = tree_hsh[node]
 2552         for n in nodes:
 2553             t2H(tree_hsh, n, level)
 2554 
 2555     t2H(tree)
 2556     return [x[indent:] for x in text_lst], args[0], args[1]
 2557 
 2558 
 2559 lst = None
 2560 rows = None
 2561 row = None
 2562 
 2563 
 2564 def tallyByGroup(list_of_tuples, max_level=0, indnt=3, options=None, export=False):
 2565     """
 2566 list_of_tuples should already be sorted and the last component
 2567 in each tuple should be a tuple (minutes, value, expense, charge)
 2568 to be tallied.
 2569 
 2570      ('Scotland', 'Glasgow', 'North', 'summary sgn', (306, 10, 20.00, 30.00)),
 2571      ('Scotland', 'Glasgow', 'South', 'summary sgs', (960, 10, 45.00, 60.00)),
 2572      ('Wales', 'Cardiff', 'summary wc', (396, 10, 22.50, 30.00)),
 2573      ('Wales', 'Bangor', 'summary wb', (126, 10, 37.00, 37.00)),
 2574 
 2575 Recursively process groups and accumulate the totals.
 2576     """
 2577     if not options:
 2578         options = {}
 2579     if not max_level:
 2580         max_level = len(list_of_tuples[0]) - 1
 2581     level = -1
 2582     global lst
 2583     global head
 2584     global auglst
 2585     head = []
 2586     auglst = []
 2587     lst = []
 2588     if 'action_template' in options:
 2589         action_template = options['action_template']
 2590     else:
 2591         action_template = "!hours! $!value!) !label! (!count!)"
 2592 
 2593     action_template = "!indent!%s" % action_template
 2594 
 2595     if 'action_minutes' in options and options['action_minutes'] in [6, 12, 15, 30, 60]:
 2596         # floating point hours
 2597         m = options['action_minutes']
 2598 
 2599     tab = " " * indnt
 2600 
 2601     global rows, row
 2602     rows = []
 2603     row = ['' for i in range(max_level + 1)]
 2604 
 2605     def doLeaf(tup, lvl):
 2606         global row, rows, head, auglst
 2607         if len(tup) < 2:
 2608             rows.append(deepcopy(row))
 2609             return ()
 2610         k = tup[0]
 2611         g = tup[1:]
 2612         t = tup[-1]
 2613         lvl += 1
 2614         row[lvl] = k
 2615         row[-1] = t
 2616         hsh = {}
 2617         if max_level and lvl > max_level - 1:
 2618             rows.append(deepcopy(row))
 2619             return ()
 2620         indent = " " * indnt
 2621         hsh['indent'] = indent * lvl
 2622         hsh['count'] = 1
 2623         hsh['minutes'] = t[0]
 2624         hsh['value'] = "%.2f" % t[1]  # only 2 digits after the decimal point
 2625         hsh['expense'] = t[2]
 2626         hsh['charge'] = t[3]
 2627         hsh['total'] = t[1] + t[3]
 2628         if options['action_minutes'] in [6, 12, 15, 30, 60]:
 2629             # floating point hours
 2630             hsh['hours'] = "{0:n}".format(
 2631                 ((t[0] // m + (t[0] % m > 0)) * m) / 60.0)
 2632         else:
 2633             # hours and minutes
 2634             hsh['hours'] = "%d:%02d" % (t[0] // 60, t[0] % 60)
 2635         hsh['label'] = k
 2636         lst.append(expand_template(action_template, hsh, complain=True))
 2637         head.append(lst[-1].lstrip())
 2638         auglst.append(head)
 2639 
 2640         if len(g) >= 1:
 2641             doLeaf(g, lvl)
 2642 
 2643     def doGroups(tuple_list, lvl):
 2644         global row, rows, head, auglst
 2645         hsh = {}
 2646         lvl += 1
 2647         if max_level and lvl > max_level - 1:
 2648             rows.append(deepcopy(row))
 2649             return
 2650         hsh['indent'] = tab * lvl
 2651         for k, g, t in group_sort(tuple_list):
 2652             head = head[:lvl]
 2653             row[lvl] = k[-1]
 2654             row[-1] = t
 2655             hsh['count'] = len(g)
 2656             hsh['minutes'] = t[0]  # only 2 digits after the decimal point
 2657             hsh['value'] = "%.2f" % t[1]
 2658             hsh['expense'] = t[2]
 2659             hsh['charge'] = t[3]
 2660             hsh['total'] = t[1] + t[3]
 2661             if options['action_minutes'] in [6, 12, 15, 30, 60]:
 2662                 # hours and tenths
 2663                 hsh['hours'] = "{0:n}".format(
 2664                     ((t[0] // m + (t[0] % m > 0)) * m) / 60.0)
 2665             else:
 2666                 # hours and minutes
 2667                 hsh['hours'] = "%d:%02d" % (t[0] // 60, t[0] % 60)
 2668 
 2669             hsh['label'] = k[-1]
 2670             lst.append(expand_template(action_template, hsh, complain=True))
 2671             head.append(lst[-1].lstrip())
 2672             if len(head) == max_level:
 2673                 auglst.append(head)
 2674             if len(g) > 1:
 2675                 doGroups(g, lvl)
 2676             else:
 2677                 doLeaf(g[0], lvl)
 2678 
 2679     doGroups(list_of_tuples, level)
 2680 
 2681     for i in range(len(auglst)):
 2682         if type(auglst[i][-1]) in [str, unicode]:
 2683             res = auglst[i][-1].split('!!')
 2684             if len(res) > 1:
 2685                 summary = res[0]
 2686                 uid = res[1]
 2687                 auglst[i][-1] = tuple((uid, 'ac', summary, ''))
 2688     res = makeTree(auglst, sort=False)
 2689 
 2690     if export:
 2691         for i in range(len(rows)):
 2692             # remove the uuid from the summary
 2693             summary = rows[i][-2].split('!!')[0]
 2694             rows[i][-2] = summary
 2695         return rows
 2696     else:
 2697         return res
 2698 
 2699 
 2700 def group_sort(row_lst):
 2701     # last element of each list component is a (minutes, value,
 2702     # expense, charge) tuple.
 2703     # next to last element is a summary string.
 2704     key = lambda cols: [cols[0]]
 2705     for k, group in groupby(row_lst, key):
 2706         t = []
 2707         g = []
 2708         for x in group:
 2709             t.append(x[-1])
 2710             g.append(x[1:])
 2711         s = tupleSum(t)
 2712         yield k, g, s
 2713 
 2714 
 2715 def uniqueId():
 2716     # return unicode(thistime.strftime("%Y%m%dT%H%M%S@etmtk"))
 2717     return unicode("{0}etm".format(uuid.uuid4().hex))
 2718 
 2719 
 2720 def nowAsUTC():
 2721     return datetime.now(tzlocal()).astimezone(tzutc()).replace(tzinfo=None)
 2722 
 2723 
 2724 def datetime2minutes(dt):
 2725     if type(dt) != datetime:
 2726         return ()
 2727     t = dt.time()
 2728     return t.hour * 60 + t.minute
 2729 
 2730 
 2731 def parse_str(dt, timezone=None, fmt=None):
 2732     """
 2733     E.g., ('2/5/12', 'US/Pacific', rfmt) => "20120205T0000-0800"
 2734     Return datetime object if fmt is None
 2735     Return
 2736     """
 2737     if type(dt) in [str, unicode]:
 2738         if dt == 'now':
 2739             if timezone is None:
 2740                 dt = datetime.now()
 2741             else:
 2742                 dt = datetime.now().replace(
 2743                     tzinfo=tzlocal()).astimezone(
 2744                     gettz(timezone)).replace(tzinfo=None)
 2745         elif period_string_regex.match(dt):
 2746             dt = datetime.now() + parse_period(dt, minutes=False)
 2747         else:
 2748             now = datetime.now()
 2749 
 2750             estr = estr_regex.search(dt)
 2751             rel_mnth = rel_month_regex.search(dt)
 2752             rel_date = rel_date_regex.search(dt)
 2753 
 2754             if estr:
 2755                 y = estr.group(1)
 2756                 e = easter(int(y))
 2757                 E = e.strftime("%Y-%m-%d")
 2758                 dt = estr_regex.sub(E, dt)
 2759 
 2760             if rel_mnth:
 2761                 new_y = now.year
 2762                 now_m = now.month
 2763                 mnth, day = map(int, rel_mnth.groups())
 2764                 new_m = now_m + mnth
 2765                 new_d = day
 2766                 if new_m <= 0:
 2767                     new_y -= 1
 2768                     new_m += 12
 2769                 elif new_m > 12:
 2770                     new_y += 1
 2771                     new_m -= 12
 2772                 new_date = "%s-%02d-%02d" % (new_y, new_m, new_d)
 2773                 dt = rel_month_regex.sub(new_date, dt)
 2774             elif rel_date:
 2775                 days = int(rel_date.group(0))
 2776                 new_date = (now + days * ONEDAY).strftime("%Y-%m-%d")
 2777                 dt = rel_date_regex.sub(new_date, dt)
 2778 
 2779             dt = parse(dt)
 2780             if type(dt) is not datetime:
 2781                 # we have a problem, return the error message
 2782                 return dt
 2783     else:
 2784         # dt is a datetime
 2785         if dt.utcoffset() is None:
 2786             dt = dt.replace(tzinfo=tzlocal())
 2787 
 2788     if timezone is None:
 2789         dtz = dt.replace(tzinfo=tzlocal())
 2790     else:
 2791         dtz = dt.replace(tzinfo=gettz(timezone))
 2792 
 2793     if windoz and dtz.year < 1970:
 2794         y = dtz.year
 2795         m = dtz.month
 2796         d = dtz.day
 2797         H = dtz.hour
 2798         M = dtz.minute
 2799         dtz = datetime(y, m, d, H, M, 0, 0)
 2800         epoch = datetime(1970, 1, 1, 0, 0, 0, 0)
 2801 
 2802         # dtz.replace(tzinfo=None)
 2803         td = epoch - dtz
 2804         seconds = td.days * 24 * 60 * 60 + td.seconds
 2805         dtz = epoch - timedelta(seconds=seconds)
 2806 
 2807     if fmt is None:
 2808         return dtz
 2809     else:
 2810         return dtz.strftime(fmt)
 2811 
 2812 
 2813 def parse_date_period(s):
 2814     """
 2815     fuzzy_date [ (+|-) period string]
 2816     e.g. mon + 7d: the 2nd Monday on or after today
 2817     used in reports to handle begin and end options
 2818     """
 2819     parts = [x.strip() for x in rsplit(' [+-] ', s)]
 2820     try:
 2821         dt = parse_str(parts[0])
 2822     except Exception:
 2823         return 'error: could not parse date "{0}"'.format(parts[0])
 2824     if len(parts) > 1:
 2825         try:
 2826             pr = parse_period(parts[1])
 2827         except Exception:
 2828             return 'error: could not parse period "{0}"'.format(parts[1])
 2829         if ' + ' in s:
 2830             return dt + pr
 2831         else:
 2832             return dt - pr
 2833     else:
 2834         return dt
 2835 
 2836 
 2837 def parse_period(s, minutes=True):
 2838     """\
 2839     Take a case-insensitive period string and return a corresponding timedelta.
 2840     Examples:
 2841         parse_period('-2W3D4H5M')= -timedelta(weeks=2,days=3,hours=4,minutes=5)
 2842         parse_period('1h30m') = timedelta(hours=1, minutes=30)
 2843         parse_period('-10') = timedelta(minutes= 10)
 2844     where:
 2845         W or w: weeks
 2846         D or d: days
 2847         H or h: hours
 2848         M or m: minutes
 2849     If an integer is passed or a string that can be converted to an
 2850     integer, then return a timedelta corresponding to this number of
 2851     minutes if 'minutes = True', and this number of days otherwise.
 2852     Minutes will be True for alerts and False for beginbys.
 2853     """
 2854     td = timedelta(seconds=0)
 2855     if minutes:
 2856         unitperiod = ONEMINUTE
 2857     else:
 2858         unitperiod = ONEDAY
 2859     try:
 2860         m = int(s)
 2861         return m * unitperiod
 2862     except Exception:
 2863         m = int_regex.match(s)
 2864         if m:
 2865             return td + int(m.group(1)) * unitperiod, ""
 2866             # if we get here we should have a period string
 2867     m = period_string_regex.match(s)
 2868     if not m:
 2869         logger.error("Invalid period string: '{0}'".format(s))
 2870         return "Invalid period string: '{0}'".format(s)
 2871     m = week_regex.search(s)
 2872     if m:
 2873         td += int(m.group(1)) * ONEWEEK
 2874     m = day_regex.search(s)
 2875     if m:
 2876         td += int(m.group(1)) * ONEDAY
 2877     m = hour_regex.search(s)
 2878     if m:
 2879         td += int(m.group(1)) * ONEHOUR
 2880     m = minute_regex.search(s)
 2881     if m:
 2882         td += int(m.group(1)) * ONEMINUTE
 2883     if type(td) is not timedelta:
 2884         return "Could not parse {0}".format(s)
 2885     m = sign_regex.match(s)
 2886     if m and m.group(1) == '-':
 2887         return -1 * td
 2888     else:
 2889         return td
 2890 
 2891 
 2892 def year2string(startyear, endyear):
 2893     """compute difference and append suffix"""
 2894     diff = int(endyear) - int(startyear)
 2895     suffix = 'th'
 2896     if diff < 4 or diff > 20:
 2897         if diff % 10 == 1:
 2898             suffix = 'st'
 2899         elif diff % 10 == 2:
 2900             suffix = 'nd'
 2901         elif diff % 10 == 3:
 2902             suffix = 'rd'
 2903     return "%d%s" % (diff, suffix)
 2904 
 2905 
 2906 def lst2str(l):
 2907     if type(l) != list:
 2908         return l
 2909     tmp = []
 2910     for item in l:
 2911         if type(item) in [datetime]:
 2912             tmp.append(parse_str(item, fmt=zfmt))
 2913         elif type(item) in [timedelta]:
 2914             tmp.append(timedelta2Str(item))
 2915         elif isinstance(item, unicode):
 2916             tmp.append(item)
 2917         else:  # type(i) in [str,]:
 2918             tmp.append(str(item))
 2919     return ", ".join(tmp)
 2920 
 2921 
 2922 def hsh2str(hsh, options=None, include_uid=False):
 2923     """
 2924 For editing one or more, but not all, instances of an item. Needed:
 2925 1. Add @+ datetime to orig and make copy sans all repeating info and
 2926    with @s datetime.
 2927 2. Add &r datetime - ONEMINUTE to each _r in orig and make copy with
 2928    @s datetime
 2929 3. Add &f datetime to selected job.
 2930     """
 2931     if not options:
 2932         options = {}
 2933     msg = []
 2934     if '_summary' not in hsh:
 2935         hsh['_summary'] = ''
 2936     if '_group_summary' in hsh:
 2937         sl = ["%s %s" % (hsh['itemtype'], hsh['_group_summary'])]
 2938         if 'I' in hsh:
 2939             # fix the item index
 2940             hsh['I'] = hsh['I'].split(':')[0]
 2941     else:
 2942         sl = ["%s %s" % (hsh['itemtype'], hsh['_summary'])]
 2943     if 'I' not in hsh or not hsh['I']:
 2944         hsh['I'] = uniqueId()
 2945     bad_keys = [x for x in hsh.keys() if x not in all_keys]
 2946     if bad_keys:
 2947         omitted = []
 2948         for key in bad_keys:
 2949             omitted.append('@{0} {1}'.format(key, hsh[key]))
 2950         msg.append("unrecognized entries: {0}".format(", ".join(omitted)))
 2951     for key in at_keys:
 2952         amp_key = None
 2953         if hsh['itemtype'] == "=":
 2954             prefix = ""
 2955         elif 'prefix_uses' in options and key in options['prefix_uses']:
 2956             prefix = options['prefix']
 2957         else:
 2958             prefix = ""
 2959         if key == 'a' and '_a' in hsh:
 2960             alerts = []
 2961             for alert in hsh["_a"]:
 2962                 triggers, acts, arguments = alert
 2963                 _ = "@a %s" % ", ".join([fmt_period(x) for x in triggers])
 2964                 if acts:
 2965                     _ += ": %s" % ", ".join(acts)
 2966                     if arguments:
 2967                         arg_strings = []
 2968                         for arg in arguments:
 2969                             arg_strings.append(", ".join(arg))
 2970                         _ += "; %s" % "; ".join(arg_strings)
 2971                 alerts.append(_)
 2972             sl.extend(alerts)
 2973         elif key in ['r', 'j']:
 2974             at_key = key
 2975             keys = amp_keys[key]
 2976             key = "_%s" % key
 2977         elif key in ['+', '-']:
 2978             keys = []
 2979         elif key in ['t', 'l', 'd']:
 2980             keys = []
 2981         else:
 2982             keys = []
 2983 
 2984         if key in hsh and hsh[key] is not None:
 2985             # since r and j can repeat, value will be a list
 2986             value = hsh[key]
 2987             if keys:
 2988                 # @r or @j --- value will be a list of hashes or
 2989                 # possibly, in the  case of @a, a list of lists. f
 2990                 # will be the first key for @r and t will be the
 2991                 # first for @a
 2992                 omitted = []
 2993                 for v in value:
 2994                     for k in v.keys():
 2995                         if k not in keys:
 2996                             omitted.append('&{0} {1}'.format(k, v[k]))
 2997                 if omitted:
 2998                     msg.append("unrecognized entries: {0}".format(", ".join(omitted)))
 2999 
 3000                 tmp = []
 3001                 for h in value:
 3002                     if unicode(keys[0]) not in h:
 3003                         logger.warning("{0} not in {1}".format(keys[0], h))
 3004                         continue
 3005                     tmp.append('%s@%s %s' % (prefix, at_key,
 3006                                              lst2str(h[unicode(keys[0])])))
 3007                     for amp_key in keys[1:]:
 3008                         if amp_key in h:
 3009                             if at_key == 'j' and amp_key == 'f':
 3010                                 pairs = []
 3011                                 for pair in h['f']:
 3012                                     pairs.append(";".join([
 3013                                         x.strftime(zfmt) for x in pair if x]))
 3014                                 v = (', '.join(pairs))
 3015                             elif at_key == 'j' and amp_key == 'h':
 3016                                 pairs = []
 3017                                 for pair in h['h']:
 3018                                     pairs.append(";".join([
 3019                                         x.strftime(zfmt) for x in pair if x]))
 3020                                 v = (', '.join(pairs))
 3021                             elif amp_key == 'e':
 3022                                 try:
 3023                                     v = fmt_period(h['e'])
 3024                                 except Exception:
 3025                                     v = h['e']
 3026                                     logger.error(
 3027                                         "error: could not parse h['e']: '{0}'".format(
 3028                                             h['e']))
 3029                             else:
 3030                                 v = lst2str(h[amp_key])
 3031                             tmp.append('&%s %s' % (amp_key, v))
 3032                 if tmp:
 3033                     sl.append(" ".join(tmp))
 3034             elif key == 's':
 3035                 try:
 3036                     sl.append("%s@%s %s" % (prefix, key, fmt_datetime(value, options=options)))
 3037                 except:
 3038                     msg.append("problem with @{0}: {1}".format(key, value))
 3039             elif key == 'q':
 3040                 # Added this for abused women to record place in a queue - value should be the datetime the person entered the queue. Entering "now" would record the current datetime.
 3041                 if type(value) is datetime:
 3042                     sl.append("%s@%s %s" % (
 3043                         prefix, key,
 3044                         value.strftime(zfmt),
 3045                     ))
 3046                 else:
 3047                     sl.append("%s@%s %s" % (prefix, key, value))
 3048             elif key == 'e':
 3049                 try:
 3050                     sl.append("%s@%s %s" % (prefix, key, fmt_period(value)))
 3051                 except:
 3052                     msg.append("problem with @{0}: {1}".format(key, value))
 3053             elif key == 'f':
 3054                 tmp = []
 3055                 for pair in hsh['f']:
 3056                     tmp.append(";".join([x.strftime(zfmt) for x in pair if x]))
 3057                 sl.append("%s@f %s" % (prefix, ", {0}".format(prefix).join(tmp)))
 3058             elif key == 'I':
 3059                 if include_uid and hsh['itemtype'] != "=":
 3060                     sl.append("prefix@i {0}".format(prefix, value))
 3061             elif key == 'h':
 3062                 tmp = []
 3063                 for pair in hsh['h']:
 3064                     tmp.append(";".join([x.strftime(zfmt) for x in pair if x]))
 3065                 sl.append("%s@h %s" % (prefix, ", {0}".format(prefix).join(tmp)))
 3066             else:
 3067                 sl.append("%s@%s %s" % (prefix, key, lst2str(value)))
 3068     return " ".join(sl), msg
 3069 
 3070 
 3071 def process_all_datafiles(options):
 3072     prefix, filelist = getFiles(options['datadir'])
 3073     return process_data_file_list(filelist, options=options)
 3074 
 3075 
 3076 def process_data_file_list(filelist, options=None):
 3077     if not options:
 3078         options = {}
 3079     messages = []
 3080     file2lastmodified = {}
 3081     bad_datafiles = {}
 3082     file2uuids = {}
 3083     uuid2hashes = {}
 3084     uuid2labels = {}
 3085     for f, r in filelist:
 3086         file2lastmodified[(f, r)] = os.path.getmtime(f)
 3087         msg, hashes, u2l = process_one_file(f, r, options)
 3088         uuid2labels.update(u2l)
 3089         if msg:
 3090             messages.append("errors loading %s:" % r)
 3091             messages.extend(msg)
 3092         try:
 3093             for hsh in hashes:
 3094                 if hsh['itemtype'] == '=':
 3095                     continue
 3096                 uid = hsh['I']
 3097                 uuid2hashes[uid] = hsh
 3098                 file2uuids.setdefault(r, []).append(uid)
 3099         except Exception:
 3100             fio = StringIO()
 3101             msg = fio.getvalue()
 3102             bad_datafiles[r] = msg
 3103             logger.error('Error processing: {0}\n{1}'.format(r, msg))
 3104     return uuid2hashes, uuid2labels, file2uuids, file2lastmodified, bad_datafiles, messages
 3105 
 3106 
 3107 def process_one_file(full_filename, rel_filename, options=None):
 3108     if not options:
 3109         options = {}
 3110     file_items = getFileItems(full_filename, rel_filename)
 3111     return items2Hashes(file_items, options)
 3112 
 3113 
 3114 def getFiles(root, include=r'*.txt', exclude=r'.*', other=[]):
 3115     """
 3116     Return the common prefix and a list of full paths from root
 3117     :param root: directory
 3118     :return: common prefix of files and a list of full file paths
 3119     """
 3120     # includes = r'*.txt'
 3121     # excludes = r'.*'
 3122     paths = [root]
 3123     filelist = []
 3124     other.sort()
 3125     for path in other:
 3126         paths.append(path)
 3127     common_prefix = os.path.commonprefix(paths)
 3128     for path in other:
 3129         rel_path = relpath(path, common_prefix)
 3130         filelist.append((path, rel_path))
 3131     for path, dirs, files in os.walk(root):
 3132         # exclude dirs
 3133         dirs[:] = [os.path.join(path, d) for d in dirs
 3134                    if not fnmatch.fnmatch(d, exclude)]
 3135 
 3136         # exclude/include files
 3137         files = [os.path.join(path, f) for f in files
 3138                  if not fnmatch.fnmatch(f, exclude)]
 3139         files = [os.path.normpath(f) for f in files if fnmatch.fnmatch(f, include)]
 3140 
 3141         for fname in files:
 3142             rel_path = relpath(fname, common_prefix)
 3143             filelist.append((fname, rel_path))
 3144     return common_prefix, filelist
 3145 
 3146 
 3147 def getAllFiles(root, include=r'*', exclude=r'.*', other=[]):
 3148     """
 3149     Return the common prefix and a list of full paths from root
 3150     :param root: directory
 3151     :return: common prefix of files and a list of full file paths
 3152     """
 3153     paths = [root]
 3154     filelist = []
 3155     for path in other:
 3156         paths.append(path)
 3157     other.sort()
 3158     common_prefix = os.path.commonprefix(paths)
 3159     for path in other:
 3160         rel_path = relpath(path, common_prefix)
 3161         filelist.append((path, rel_path))
 3162     for path, dirs, files in os.walk(root):
 3163         # exclude dirs
 3164         dirs[:] = [os.path.join(path, d) for d in dirs
 3165                    if not fnmatch.fnmatch(d, exclude)]
 3166         # exclude/include files
 3167         files = [os.path.join(path, f) for f in files
 3168                  if not fnmatch.fnmatch(f, exclude)]
 3169         files = [os.path.normpath(f) for f in files if fnmatch.fnmatch(f, include)]
 3170         for fname in files:
 3171             rel_path = relpath(fname, common_prefix)
 3172             filelist.append((fname, rel_path))
 3173         if not (dirs or files):
 3174             # empty
 3175             rel_path = relpath(path, common_prefix)
 3176             filelist.append((path, rel_path))
 3177     return common_prefix, filelist
 3178 
 3179 
 3180 def getFileTuples(root, include=r'*.txt', exclude=r'.*', all=False, other=[]):
 3181     """
 3182     Used in view to get config files
 3183     """
 3184     if all:
 3185         common_prefix, filelist = getAllFiles(root, include, exclude, other=other)
 3186     else:
 3187         common_prefix, filelist = getFiles(root, include, exclude, other=other)
 3188     lst = []
 3189     prior = []
 3190     for fp, rp in filelist:
 3191         drive, tup = os_path_splitall(rp)
 3192         for i in range(0, len(tup)):
 3193             if len(prior) > i and tup[i] == prior[i]:
 3194                 continue
 3195             prior = tup[:i]
 3196             disable = (i < len(tup) - 1) or os.path.isdir(fp)
 3197             lst.append(("{0}{1}".format(" " * 6 * i, tup[i]), rp, disable))
 3198     return common_prefix, lst
 3199 
 3200 
 3201 def os_path_splitall(path, debug=False):
 3202     parts = []
 3203     drive, path = os.path.splitdrive(path)
 3204     while True:
 3205         newpath, tail = os.path.split(path)
 3206         if newpath == path:
 3207             assert not tail
 3208             if path:
 3209                 parts.append(path)
 3210             break
 3211         parts.append(tail)
 3212         path = newpath
 3213     parts.reverse()
 3214     return drive, parts
 3215 
 3216 
 3217 def getFileItems(full_name, rel_name, append_newline=True):
 3218     """
 3219         Group the lines in file f into logical items and return them.
 3220     :param full_name: including datadir
 3221     :param rel_name: from datadir
 3222     :param append_newline: bool, default True
 3223     """
 3224     fo = codecs.open(full_name, 'r', file_encoding)
 3225     lines = fo.readlines()
 3226     fo.close()
 3227     # make sure we have a trailing new-line. Yes, we really need this.
 3228     if append_newline:
 3229         lines.append('\n')
 3230     linenum = 0
 3231     linenums = []
 3232     logical_line = []
 3233     for line in lines:
 3234         linenums.append(linenum)
 3235         linenum += 1
 3236         # preserve new lines and leading whitespace within logical lines
 3237         stripped = line.rstrip()
 3238         m = item_regex.match(stripped)
 3239         if m is not None or stripped == '=':
 3240             if logical_line:
 3241                 yield (''.join(logical_line), rel_name, linenums)
 3242             logical_line = []
 3243             linenums = []
 3244             logical_line.append("%s\n" % line.rstrip())
 3245         elif stripped:
 3246             # a line which does not continue, end of logical line
 3247             logical_line.append("%s\n" % line.rstrip())
 3248         elif logical_line:
 3249             # preserve interior empty lines
 3250             logical_line.append("\n")
 3251     if logical_line:
 3252         # end of sequence implies end of last logical line
 3253         yield (''.join(logical_line), rel_name, linenums)
 3254 
 3255 
 3256 def items2Hashes(list_of_items, options=None):
 3257     """
 3258         Return a list of messages and a list of hashes corresponding to items in
 3259         list_of_items.
 3260     """
 3261     if not options:
 3262         options = {}
 3263     messages = []
 3264     hashes = []
 3265     uuid2labels = {}
 3266     defaults = {}
 3267     # in_task_group = False
 3268     for item, rel_name, linenums in list_of_items:
 3269         hsh, msg = str2hsh(item, options=options)
 3270         logger.debug("items2Hashes:\n  item='{0}'  hsh={1}\n  msg={2}".format(item, hsh, msg))
 3271 
 3272         if item.strip() == "=":
 3273             # reset defaults
 3274             defaults = {}
 3275 
 3276         tmp_hsh = {}
 3277         tmp_hsh.update(defaults)
 3278         tmp_hsh.update(hsh)
 3279         hsh = tmp_hsh
 3280         try:
 3281             hsh['fileinfo'] = (rel_name, linenums[0], linenums[-1])
 3282         except:
 3283             raise ValueError("exception in fileinfo:",
 3284                              rel_name, linenums, "\n", hsh)
 3285         if msg:
 3286             lines = []
 3287             item = item.strip()
 3288             if len(item) > 56:
 3289                 lines.extend(wrap(item, 56))
 3290             else:
 3291                 lines.append(item)
 3292             for line in lines:
 3293                 messages.append("{0}".format(line))
 3294             for m in msg:
 3295                 messages.append('{0}'.format(m))
 3296             msg.append('    {0}'.format(hsh['fileinfo']))
 3297             # put the bad item in the inbox for repairs
 3298             hsh['_summary'] = "{0} {1}".format(hsh['itemtype'], hsh['_summary'])
 3299             hsh['itemtype'] = "$"
 3300             hsh['I'] = uniqueId()
 3301             hsh['errors'] = "\n    ".join(msg)
 3302             logger.warn("{0}".format(hsh['errors']))
 3303             # no more processing
 3304             # ('hsh:', hsh)
 3305             hashes.append(hsh)
 3306             continue
 3307 
 3308         itemtype = hsh['itemtype']
 3309         if itemtype == '$':
 3310             # inbasket item
 3311             hashes.append(hsh)
 3312         elif itemtype == '#':
 3313             # deleted item
 3314             # yield this so that hidden entries are in file2uuids
 3315             hashes.append(hsh)
 3316         elif itemtype == '=':
 3317             # set group defaults
 3318             # hashes.append(this so that default entries are in file2uuids
 3319             logger.debug("items2Hashes defaults: {0}".format(hsh))
 3320             defaults = hsh
 3321             hashes.append(hsh)
 3322         elif itemtype == '+':
 3323             # needed for task group:
 3324             #   the original hsh with the summary adjusted to show
 3325             #       the number of tasks and type changed to '-' and the
 3326             #       date updated to refect the due (keep) due date
 3327             #   a non-repeating hash with type '+' for each job
 3328             #       with current due date for unfinished jobs and
 3329             #       otherwise the finished date. These will appear
 3330             #       in days but not folders
 3331             #   '+' items will be not be added to folders
 3332             # Finishing a group task should be handled separately
 3333             # when the last job is finished and 'f' is updated.
 3334             # Here we assume that one or more jobs are unfinished.
 3335             queue_hsh = {}
 3336             tmp_hsh = {}
 3337             for at_key in defaults:
 3338                 if at_key in key2type and itemtype in key2type[at_key]:
 3339                     tmp_hsh[at_key] = defaults[at_key]
 3340 
 3341             # tmp_hsh.update(defaults)
 3342             tmp_hsh.update(hsh)
 3343             group_defaults = tmp_hsh
 3344             group_task = deepcopy(group_defaults)
 3345             done, due, following = getDoneAndTwo(group_task)
 3346             if 'f' in group_defaults and due:
 3347                 del group_defaults['f']
 3348                 group_defaults['s'] = due
 3349             if 'rrule' in group_defaults:
 3350                 del group_defaults['rrule']
 3351             prereqs = []
 3352             last_level = 1
 3353             uid = hsh['I']
 3354             summary = hsh['_summary']
 3355             if 'j' not in hsh:
 3356                 continue
 3357             job_num = 0
 3358             jobs = [x for x in hsh['j']]
 3359             completed = []
 3360             num_jobs = len(jobs)
 3361             del group_defaults['j']
 3362             if following:
 3363                 del group_task['j']
 3364                 # group_task['s'] = following
 3365                 group_task['s'] = following
 3366                 group_task['_summary'] = "%s [%s jobs]" % (
 3367                     summary, len(jobs))
 3368                 hashes.append(group_task)
 3369             for job in jobs:
 3370                 tmp_hsh = {}
 3371                 tmp_hsh.update(group_defaults)
 3372                 tmp_hsh.update(job)
 3373                 job = tmp_hsh
 3374                 job['itemtype'] = '+'
 3375                 job_num += 1
 3376                 current_id = "%s:%02d" % (uid, job_num)
 3377                 if 'f' in job:
 3378                     # this will be a done:due pair with the due
 3379                     # of the current group task
 3380                         completed.append(current_id)
 3381                 job["_summary"] = "%s %d/%d: %s" % (
 3382                     summary, job_num, num_jobs, job['j'])
 3383                 del job['j']
 3384                 if 'q' not in job:
 3385                     logger.warn('error: q missing from job')
 3386                     continue
 3387                 try:
 3388                     current_level = int(job['q'])
 3389                 except:
 3390                     logger.warn('error: bad value for q', job['q'])
 3391                     continue
 3392                 job['I'] = current_id
 3393 
 3394                 queue_hsh.setdefault(current_level, set([])).add(current_id)
 3395 
 3396                 if current_level < last_level:
 3397                     prereqs = []
 3398                     for k in queue_hsh:
 3399                         if k > current_level:
 3400                             queue_hsh[k] = set([])
 3401                 for k in queue_hsh:
 3402                     if k < current_level:
 3403                         prereqs.extend(list(queue_hsh[k]))
 3404                 job['prereqs'] = [x for x in prereqs if x not in completed]
 3405 
 3406                 last_level = current_level
 3407                 try:
 3408                     job['fileinfo'] = (rel_name, linenums[0], linenums[-1])
 3409                 except:
 3410                     logger.exception("fileinfo: {0}.{1}".format(rel_name, linenums))
 3411                 logger.debug('appending job: {0}'.format(job))
 3412                 hashes.append(job)
 3413         else:
 3414             tmp_hsh = {}
 3415             for at_key in defaults:
 3416                 if at_key in key2type and itemtype in key2type[at_key]:
 3417                     tmp_hsh[at_key] = defaults[at_key]
 3418 
 3419             # tmp_hsh.update(defaults)
 3420             tmp_hsh.update(hsh)
 3421             hsh = tmp_hsh
 3422             try:
 3423                 hsh['fileinfo'] = (rel_name, linenums[0], linenums[-1])
 3424             except:
 3425                 raise ValueError("exception in fileinfo:",
 3426                                  rel_name, linenums, "\n", hsh)
 3427             hashes.append(hsh)
 3428         if itemtype not in ['=', '$']:
 3429             tmp = [' ']
 3430             for key in label_keys:
 3431                 if key in hsh and hsh[key]:
 3432                     # dump the '_'
 3433                     key = key[-1]
 3434                     tmp.append(key)
 3435                 # else:
 3436                 #     tmp.append(' ')
 3437             uuid2labels[hsh['I']] = "".join(tmp)
 3438     return messages, hashes, uuid2labels
 3439 
 3440 
 3441 def get_reps(bef, hsh):
 3442     if hsh['itemtype'] in ['+', '-', '%']:
 3443         done, due, following = getDoneAndTwo(hsh)
 3444         if hsh['itemtype'] == '+':
 3445             if done and following:
 3446                 start = following
 3447             elif due:
 3448                 start = due
 3449         elif due:
 3450             start = due
 3451         else:
 3452             start = done
 3453     else:
 3454         start = hsh['s'].replace(tzinfo=None)
 3455     tmp = []
 3456     if not start:
 3457         return False, []
 3458     for hsh_r in hsh['_r']:
 3459         tests = [
 3460             u'f' in hsh_r and hsh_r['f'] == 'l',
 3461             u't' in hsh_r,
 3462             u'u' in hsh_r
 3463         ]
 3464         for test in tests:
 3465             passed = False
 3466             if test:
 3467                 passed = True
 3468                 break
 3469         if not passed:
 3470             break
 3471 
 3472     if passed:
 3473         # finite, get instances after start
 3474         try:
 3475             tmp.extend([x for x in hsh['rrule'] if x >= start])
 3476         except:
 3477             logger.exception('done: {0}; due: {1}; following: {2}; start: {3}; rrule: {4}'.format(done, due, following, start, hsh['rrule']))
 3478     else:
 3479         tmp.extend(list(hsh['rrule'].between(start, bef, inc=True)))
 3480         tmp.append(hsh['rrule'].after(bef, inc=False))
 3481 
 3482     if windoz:
 3483         ret = []
 3484         epoch = datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=None)
 3485         for i in tmp:
 3486             if not i:
 3487                 continue
 3488             # i.replace(tzinfo=gettz(hsh['z']))
 3489             if i.year < 1970:
 3490                 # i.replace(tzinfo=gettz(hsh['z']))
 3491                 td = epoch - i
 3492                 i = epoch - td
 3493             else:
 3494                 i.replace(tzinfo=gettz(hsh['z'])).astimezone(tzlocal()).replace(tzinfo=None)
 3495             ret.append(i)
 3496         return passed, ret
 3497 
 3498     return passed, [j.replace(tzinfo=gettz(hsh['z'])).astimezone(tzlocal()).replace(tzinfo=None) for j in tmp if j]
 3499 
 3500 
 3501 def get_rrulestr(hsh, key_hsh=rrule_hsh):
 3502     """
 3503         Parse the rrule relevant information in hsh and return a
 3504         corresponding RRULE string.
 3505 
 3506         First pass - replace hsh['r'] with an equivalent rrulestr.
 3507     """
 3508     if 'r' not in hsh:
 3509         return ()
 3510     try:
 3511         lofh = hsh['r']
 3512     except:
 3513         raise ValueError("Could not load rrule:", hsh['r'])
 3514     ret = []
 3515     l = []
 3516     if type(lofh) == dict:
 3517         lofh = [lofh]
 3518     for h in lofh:
 3519         if 'f' in h and h['f'] == 'l':
 3520             # list only
 3521             l = []
 3522         else:
 3523             try:
 3524                 l = ["RRULE:FREQ=%s" % freq_hsh[h['f']]]
 3525             except:
 3526                 logger.exception("bad rrule: {0}, {1}, {2}\n{3}".format(rrule, "\nh:", h, hsh))
 3527 
 3528         for k in rrule_keys:
 3529             if k in h and h[k]:
 3530                 v = h[k]
 3531                 if type(v) == list:
 3532                     v = ",".join(map(str, v))
 3533                 if k == 'w':
 3534                     # make weekdays upper case
 3535                     v = v.upper()
 3536                     m = threeday_regex.search(v)
 3537                     while m:
 3538                         v = threeday_regex.sub("%s" % m.group(1)[:2],
 3539                                                v, count=1)
 3540                         m = threeday_regex.search(v)
 3541                 l.append("%s=%s" % (rrule_hsh[k], v))
 3542         if 'u' in h:
 3543             dt = parse_str(h['u'], hsh['z']).replace(tzinfo=None)
 3544             l.append("UNTIL=%s" % dt.strftime(sfmt))
 3545         ret.append(";".join(l))
 3546     return "\n".join(ret)
 3547 
 3548 
 3549 def get_rrule(hsh):
 3550     """
 3551         Used to process the rulestr entry. Dates and times in *rstr*
 3552         will be datetimes with offsets. Parameters *aft* and *bef* are
 3553         UTC datetimes. Datetimes from *rule* will be returned as local
 3554         times.
 3555 
 3556         Second pass - use the hsh['r'] rrulestr entry
 3557     """
 3558     rlst = []
 3559     warn = []
 3560     if 'z' not in hsh:
 3561         hsh['z'] = local_timezone
 3562     if 'o' in hsh and hsh['o'] == 'r' and 'f' in hsh:
 3563         dtstart = hsh['f'][-1][0].replace(tzinfo=gettz(hsh['z']))
 3564     elif 's' in hsh:
 3565         dtstart = parse_str(hsh['s'], hsh['z']).replace(tzinfo=None)
 3566     else:
 3567         dtstart = datetime.now()
 3568     if 'r' in hsh:
 3569         if hsh['r']:
 3570             rlst.append(hsh['r'])
 3571         if dtstart:
 3572             rlst.insert(0, "DTSTART:%s" % dtstart.strftime(sfmt))
 3573     if '+' in hsh:
 3574         parts = hsh['+']
 3575         if type(parts) != list:
 3576             parts = [parts]
 3577         if parts:
 3578             start = parse_str(dtstart, fmt=sfmt)
 3579             for part in map(str, parts):
 3580                 # rlst.append("RDATE:%s" % parse(part).strftime(sfmt))
 3581                 plus = parse_str(part, fmt=sfmt)
 3582                 if plus >= start:
 3583                     rlst.append("RDATE:%s" % plus)
 3584     if '-' in hsh:
 3585         tmprule = dtR.rrulestr("\n".join(rlst))
 3586         parts = hsh['-']
 3587         if type(parts) != list:
 3588             parts = [parts]
 3589         if parts:
 3590             for part in map(str, parts):
 3591                 thisdatetime = parse_str(part, hsh['z']).replace(tzinfo=None)
 3592                 beforedatetime = tmprule.before(thisdatetime, inc=True)
 3593                 if beforedatetime != thisdatetime:
 3594                     warn.append(_(
 3595                         "{0} is listed in @- but doesn't match any datetimes generated by @r.").format(
 3596                         thisdatetime.strftime(rfmt)))
 3597                 rlst.append("EXDATE:%s" % parse_str(part, fmt=sfmt))
 3598     rulestr = "\n".join(rlst)
 3599     try:
 3600         rule = dtR.rrulestr(rulestr)
 3601     except ValueError as e:
 3602         rule = None
 3603         warn.append("{0}".format(e))
 3604         # raise ValueError("could not create rule from", rulestr)
 3605     return rulestr, rule, warn
 3606 
 3607 
 3608 def checkhsh(hsh):
 3609     messages = []
 3610     if hsh['itemtype'] in ['*', '~', '^'] and 's' not in hsh:
 3611         messages.append(
 3612             "An entry for @s is required for events, actions and occasions.")
 3613     elif hsh['itemtype'] in ['~'] and 'e' not in hsh and 'x' not in hsh:
 3614         messages.append("An entry for either @e or @x is required for actions.")
 3615     if ('a' in hsh or 'r' in hsh) and 's' not in hsh:
 3616         messages.append(
 3617             "An entry for @s is required for items with either @a or @r entries.")
 3618     if ('+' in hsh or '-' in hsh) and 'r' not in hsh:
 3619         messages.extend(
 3620             ["An entry for @r is required for items with",
 3621              "either @+ or @- entries."])
 3622     if ('n' in hsh and hsh['n']):
 3623         n_views = ['d', 't', 'k']
 3624         bad = []
 3625         for v in hsh['n']:
 3626             if v not in n_views:
 3627                 bad.append(v)
 3628         if bad:
 3629             messages.extend(
 3630                 ["Not allowed in @n: {0}. Only values from".format(', '.join(bad)),
 3631                 "{0} are allowed.".format(", ".join(n_views)),
 3632                  ])
 3633     return messages
 3634 
 3635 
 3636 def str2opts(s, options=None, cli=True):
 3637     if not options:
 3638         options = {}
 3639     filters = {}
 3640     if 'calendars' in options:
 3641         cal_pattern = r'^%s' % '|'.join(
 3642             [x[2] for x in options['calendars'] if x[1]])
 3643         filters['cal_regex'] = re.compile(cal_pattern)
 3644     s = s2or3(s)
 3645     op_str = s.split('#')[0]
 3646     parts = minus_regex.split(op_str)
 3647     head = parts.pop(0)
 3648     report = head[0]
 3649     groupbystr = head[1:].strip()
 3650     if not report or report not in ['c', 'a'] or not groupbystr:
 3651         return {}
 3652     grpby = {'report': report}
 3653     filters['dates'] = False
 3654     dated = {'grpby': False}
 3655     filters['report'] = unicode(report)
 3656     filters['omit'] = [True, []]
 3657     filters['neg_fields'] = []
 3658     filters['pos_fields'] = []
 3659     groupbylst = [unicode(x.strip()) for x in groupbystr.split(';')]
 3660     grpby['lst'] = groupbylst
 3661     for part in groupbylst:
 3662         if groupdate_regex.search(part):
 3663             dated['grpby'] = True
 3664             filters['dates'] = True
 3665         elif part not in ['c', 'u', 'l'] and part[0] not in ['k', 'f', 't']:
 3666             term_print(
 3667                 str(_('Ignoring invalid grpby part: "{0}"'.format(part))))
 3668             groupbylst.remove(part)
 3669     if not groupbylst:
 3670         return '', '', ''
 3671         # we'll split cols on :: after applying fmts to the string
 3672     grpby['cols'] = "::".join(["{%d}" % i for i in range(len(groupbylst))])
 3673     grpby['fmts'] = []
 3674     grpby['tuples'] = []
 3675     filters['grpby'] = ['_summary']
 3676     filters['missing'] = False
 3677     # include = {'y', 'm', 'w', 'd'}
 3678     include = {'y', 'm', 'd'}
 3679     for group in groupbylst:
 3680         d_lst = []
 3681         if groupdate_regex.search(group):
 3682             if 'w' in group:
 3683                 # groupby week or some other date spec,  not both
 3684                 group = "w"
 3685                 d_lst.append('w')
 3686                 include.discard('w')
 3687                 if 'y' in group:
 3688                     include.discard('y')
 3689                 if 'M' in group:
 3690                     include.discard('m')
 3691                 if 'd' in group:
 3692                     include.discard('d')
 3693             else:
 3694                 if 'y' in group:
 3695                     d_lst.append('yyyy')
 3696                     include.discard('y')
 3697                 if 'M' in group:
 3698                     d_lst.append('MM')
 3699                     include.discard('m')
 3700                 if 'd' in group:
 3701                     d_lst.append('dd')
 3702                     include.discard('d')
 3703             grpby['tuples'].append(" ".join(d_lst))
 3704             grpby['fmts'].append(
 3705                 "d_to_str(tup[-3], '%s')" % group)
 3706 
 3707         elif '[' in group:
 3708             if group[0] == 'f':
 3709                 if ':' in group:
 3710                     grpby['fmts'].append(
 3711                         "'/'.join(rsplit('/', hsh['fileinfo'][0])%s)" %
 3712                         (group[1:]))
 3713                     grpby['tuples'].append(
 3714                         "'/'.join(rsplit('/', hsh['fileinfo'][0])%s)" %
 3715                         (group[1:]))
 3716                 else:
 3717                     grpby['fmts'].append(
 3718                         "rsplit('/', hsh['fileinfo'][0])%s" % (group[1:]))
 3719                     grpby['tuples'].append(
 3720                         "rsplit('/', hsh['fileinfo'][0])%s" % (group[1:]))
 3721             elif group[0] == 'k':
 3722                 if ':' in group:
 3723                     grpby['fmts'].append(
 3724                         "':'.join(rsplit(':', hsh['%s'])%s)" %
 3725                         (group[0], group[1:]))
 3726                     grpby['tuples'].append(
 3727                         "':'.join(rsplit(':', hsh['%s'])%s)" %
 3728                         (group[0], group[1:]))
 3729                 else:
 3730                     grpby['fmts'].append(
 3731                         "rsplit(':', hsh['%s'])%s" % (group[0], group[1:]))
 3732                     grpby['tuples'].append(
 3733                         "rsplit(':', hsh['%s'])%s" % (group[0], group[1:]))
 3734             filters['grpby'].append(group[0])
 3735         else:
 3736             if 'f' in group:
 3737                 grpby['fmts'].append("hsh['fileinfo'][0]")
 3738                 grpby['tuples'].append("hsh['fileinfo'][0]")
 3739             else:
 3740                 grpby['fmts'].append("hsh['%s']" % group.strip())
 3741                 grpby['tuples'].append("hsh['%s']" % group.strip())
 3742             filters['grpby'].append(group[0])
 3743         if include:
 3744             if include == {'y', 'm', 'd'}:
 3745                 grpby['include'] = "yyyy-MM-dd"
 3746             elif include == {'m', 'd'}:
 3747                 grpby['include'] = "MMM d"
 3748             elif include == {'y', 'd'}:
 3749                 grpby['include'] = "yyyy-MM-dd"
 3750             elif include == set(['y', 'w']):
 3751                 groupby['include'] = "w"
 3752             elif include == {'d'}:
 3753                 grpby['include'] = "MMM dd"
 3754             elif include == set(['w']):
 3755                 grpby['include'] = "w"
 3756             else:
 3757                 grpby['include'] = ""
 3758         else:
 3759             grpby['include'] = ""
 3760         logger.debug('grpby final: {0}'.format(grpby))
 3761 
 3762     for part in parts:
 3763         key = unicode(part[0])
 3764         if key in ['b', 'e']:
 3765             dt = parse_date_period(part[1:])
 3766             dated[key] = dt.replace(tzinfo=None)
 3767 
 3768         elif key == 'm':
 3769             value = unicode(part[1:].strip())
 3770             if value == '1':
 3771                 filters['missing'] = True
 3772 
 3773         elif key == 'f':
 3774             value = unicode(part[1:].strip())
 3775             if value[0] == '!':
 3776                 filters['folder'] = (False, re.compile(r'%s' % value[1:],
 3777                                                        re.IGNORECASE))
 3778             else:
 3779                 filters['folder'] = (True, re.compile(r'%s' % value,
 3780                                                       re.IGNORECASE))
 3781         elif key == 's':
 3782             value = unicode(part[1:].strip())
 3783             if value[0] == '!':
 3784                 filters['search'] = (False, re.compile(r'%s' % value[1:],
 3785                                                        re.IGNORECASE))
 3786             else:
 3787                 filters['search'] = (True, re.compile(r'%s' % value,
 3788                                                       re.IGNORECASE))
 3789         elif key == 'S':
 3790             value = unicode(part[1:].strip())
 3791             if value[0] == '!':
 3792                 filters['search-all'] = (False, re.compile(r'%s' % value[1:], re.IGNORECASE | re.DOTALL))
 3793             else:
 3794                 filters['search-all'] = (True, re.compile(r'%s' % value, re.IGNORECASE | re.DOTALL))
 3795         elif key == 'd':
 3796             if cli:
 3797                 if grpby['report'] == 'a':
 3798                     d = int(part[1:])
 3799                     if d:
 3800                         d += 1
 3801                     grpby['depth'] = d
 3802             else:
 3803                 pass
 3804 
 3805         elif key == 't':
 3806             value = [x.strip() for x in part[1:].split(',')]
 3807             for t in value:
 3808                 if t[0] == '!':
 3809                     filters['neg_fields'].append((
 3810                         't', re.compile(r'%s' % t[1:], re.IGNORECASE)))
 3811                 else:
 3812                     filters['pos_fields'].append((
 3813                         't', re.compile(r'%s' % t, re.IGNORECASE)))
 3814         elif key == 'o':
 3815             value = unicode(part[1:].strip())
 3816             if value[0] == '!':
 3817                 filters['omit'][0] = False
 3818                 filters['omit'][1] = [x for x in value[1:]]
 3819             else:
 3820                 filters['omit'][0] = True
 3821                 filters['omit'][1] = [x for x in value]
 3822         elif key == 'h':
 3823             grpby['colors'] = int(part[1:])
 3824         elif key == 'w':
 3825             grpby['width1'] = int(part[1:])
 3826         elif key == 'W':
 3827             grpby['width2'] = int(part[1:])
 3828         else:
 3829             value = unicode(part[1:].strip())
 3830             if value[0] == '!':
 3831                 filters['neg_fields'].append((
 3832                     key, re.compile(r'%s' % value[1:], re.IGNORECASE)))
 3833             else:
 3834                 filters['pos_fields'].append((
 3835                     key, re.compile(r'%s' % value, re.IGNORECASE)))
 3836     if 'b' not in dated:
 3837         dated['b'] = parse_str(options['report_begin']).replace(tzinfo=None)
 3838     if 'e' not in dated:
 3839         dated['e'] = parse_str(options['report_end']).replace(tzinfo=None)
 3840     if 'colors' not in grpby or grpby['colors'] not in [0, 1, 2]:
 3841         grpby['colors'] = options['report_colors']
 3842     if 'width1' not in grpby:
 3843         grpby['width1'] = options['report_width1']
 3844     if 'width2' not in grpby:
 3845         grpby['width2'] = options['report_width2']
 3846     grpby['lst'].append(u'summary')
 3847     logger.debug('grpby: {0}; dated: {1}; filters: {2}'.format(grpby, dated, filters))
 3848     return grpby, dated, filters
 3849 
 3850 
 3851 def applyFilters(file2uuids, uuid2hash, filters):
 3852     """
 3853         Apply all filters except begin and end and return a list of
 3854         the uid's of the passing hashes.
 3855 
 3856         TODO: memoize?
 3857     """
 3858 
 3859     typeHsh = {
 3860         'a': '~',
 3861         'd': '%',
 3862         'e': '*',
 3863         'g': '+',
 3864         'o': '^',
 3865         'n': '!',
 3866         't': '-',
 3867         's': '?',
 3868     }
 3869     uuids = []
 3870 
 3871     omit = None
 3872     if 'omit' in filters:
 3873         omit, omit_types = filters['omit']
 3874         omit_chars = [typeHsh[x] for x in omit_types]
 3875 
 3876     for f in file2uuids:
 3877         if 'cal_regex' in filters and not filters['cal_regex'].match(f):
 3878             continue
 3879         if 'folder' in filters:
 3880             tf, folder_regex = filters['folder']
 3881             if tf and not folder_regex.search(f):
 3882                 continue
 3883             if not tf and folder_regex.search(f):
 3884                 continue
 3885         for uid in file2uuids[f]:
 3886             hsh = uuid2hash[uid]
 3887             skip = False
 3888             type_char = hsh['itemtype']
 3889             if type_char in ['=', '#', '$']:
 3890                 # omit defaults, hidden, inbox and someday
 3891                 continue
 3892             if filters['dates'] and 's' not in hsh:
 3893                 # groupby includes a date specification and this item is undated
 3894                 continue
 3895             if filters['report'] == 'a' and type_char != '~':
 3896                 continue
 3897             if filters['report'] == 'c' and omit is not None:
 3898                 if omit and type_char in omit_chars:
 3899                     # we're omitting this type
 3900                     continue
 3901                 if not omit and type_char not in omit_chars:
 3902                     # we're not showing this type
 3903                     continue
 3904             if 'search' in filters:
 3905                 tf, rx = filters['search']
 3906                 l = []
 3907                 for g in filters['grpby']:
 3908                     # search over the leaf summary and the branch
 3909                     for t in ['_summary', u'c', u'k', u'f', u'u']:
 3910                         if t not in g:
 3911                             continue
 3912                         if t == 'f':
 3913                             v = hsh['fileinfo'][0]
 3914                         elif t in hsh:
 3915                             v = hsh[t]
 3916                         else:
 3917                             continue
 3918                             # add v to l
 3919                         l.append(v)
 3920                 s = ' '.join(l)
 3921                 res = rx.search(s)
 3922                 if tf and not res:
 3923                     skip = True
 3924                 if not tf and res:
 3925                     skip = True
 3926             if 'search-all' in filters:
 3927                 tf, rx = filters['search-all']
 3928                 # search over the entire entry and the file path
 3929                 l = [hsh['entry'], hsh['fileinfo'][0]]
 3930                 s = ' '.join(l)
 3931                 res = rx.search(s)
 3932                 if tf and not res:
 3933                     skip = True
 3934                 if not tf and res:
 3935                     skip = True
 3936             for t in ['c', 'k', 'u', 'l']:
 3937                 if t in filters['grpby']:
 3938                     if filters['missing']:
 3939                         if t not in hsh:
 3940                             hsh[t] = NONE
 3941                     else:
 3942                         if t in hsh and hsh[t] == NONE:
 3943                             # we added this on an earlier report
 3944                             del hsh[t]
 3945                         if t not in hsh:
 3946                             skip = True
 3947                             break
 3948             if skip:
 3949                 # try the next uid
 3950                 continue
 3951             for flt, rgx in filters['pos_fields']:
 3952                 if flt == 't':
 3953                     if 't' not in hsh or not rgx.search(" ".join(hsh['t'])):
 3954                         skip = True
 3955                         break
 3956                 elif flt not in hsh or not rgx.search(hsh[flt]):
 3957                     skip = True
 3958                     break
 3959             if skip:
 3960                 # try the next uid
 3961                 continue
 3962             for flt, rgx in filters['neg_fields']:
 3963                 if flt == 't':
 3964                     if 't' in hsh and rgx.search(" ".join(hsh['t'])):
 3965                         skip = True
 3966                         break
 3967                 elif flt in hsh and rgx.search(hsh[flt]):
 3968                     skip = True
 3969                     break
 3970             if skip:
 3971                 # try the next uid
 3972                 continue
 3973                 # passed all tests
 3974             uuids.append(uid)
 3975     return uuids
 3976 
 3977 
 3978 def reportDT(dt, include, options=None):
 3979     # include will be something like "MMM d yyyy"
 3980     if not options:
 3981         options = {}
 3982     res = ''
 3983     if dt.hour == 0 and dt.minute == 0:
 3984         if not include:
 3985             return ''
 3986         return d_to_str(dt, "yyyy-MM-dd")
 3987     else:
 3988         if options['ampm']:
 3989             if include:
 3990                 res = dt_to_str(dt, "%s h:mma" % include)
 3991             else:
 3992                 res = dt_to_str(dt, "h:mma")
 3993         else:
 3994             if include:
 3995                 res = dt_to_str(dt, "%s hh:mm" % include)
 3996             else:
 3997                 res = dt_to_str(dt, "hh:mm")
 3998         return leadingzero.sub('', res.lower())
 3999 
 4000 
 4001 # noinspection PyChainedComparisons
 4002 def makeReportTuples(uuids, uuid2hash, grpby, dated, options=None):
 4003     """
 4004         Using filtered uuids, and dates: grpby, b and e, return a sorted
 4005         list of tuples
 4006             (sort1, sort2, ... typenum, dt or '', uid)
 4007         using dt takes care of time when needed or date and time when
 4008         grpby has no date specification
 4009     """
 4010     if not options:
 4011         options = {}
 4012     today_datetime = datetime.now().replace(
 4013         hour=0, minute=0, second=0, microsecond=0)
 4014     today_date = datetime.now().date()
 4015     tups = []
 4016     for uid in uuids:
 4017         try:
 4018             hsh = {}
 4019             for k, v in uuid2hash[uid].items():
 4020                 hsh[k] = v
 4021                 # we'll make anniversary subs to a copy later
 4022             hsh['summary'] = hsh['_summary']
 4023             tchr = hsh['itemtype']
 4024             tstr = type2Str[tchr]
 4025             if 't' not in hsh:
 4026                 hsh['t'] = []
 4027             if dated['grpby']:
 4028                 dates = []
 4029                 if 'f' in hsh and hsh['f']:
 4030                     next = getDoneAndTwo(hsh)[1]
 4031                     if next:
 4032                         start = next
 4033                 else:
 4034                     start = parse_str(hsh['s'], hsh['z']).astimezone(tzlocal()).replace(tzinfo=None)
 4035                 if 'rrule' in hsh:
 4036                     if dated['b'] > start:
 4037                         start = dated['b']
 4038                     for date in hsh['rrule'].between(dated['b'], dated['e'], inc=True):
 4039                         # to local time
 4040                         date = date.replace(tzinfo=gettz(hsh['z'])).astimezone(tzlocal()).replace(tzinfo=None)
 4041                         if date < dated['e']:
 4042                             bisect.insort(dates, date)
 4043                 elif 's' in hsh and hsh['s'] and 'f' not in hsh:
 4044                     if dated['b'] <= hsh['s'] < dated['e']:
 4045                         bisect.insort(dates, start)
 4046                         # datesSL.insert(start)
 4047                 if 'f' in hsh and hsh['f']:
 4048                     dt = parse_str(
 4049                         hsh['f'][-1][0], hsh['z']).astimezone(
 4050                         tzlocal()).replace(tzinfo=None)
 4051                     if dated['b'] <= dt <= dated['e']:
 4052                         bisect.insort(dates, dt)
 4053                 for dt in dates:
 4054                     if not (dated['b'] <= dt <= dated['e']):
 4055                         continue
 4056                     item = []
 4057                     # ('dt', type(dt), dt)
 4058                     for g in grpby['tuples']:
 4059                         if groupdate_regex.search(g):
 4060                             item.append(d_to_str(dt, g))
 4061                         elif g in ['c', 'u']:
 4062                             item.append(hsh[g])
 4063                         else:  # should be f or k
 4064                             item.append(eval(g))
 4065                     item.extend([
 4066                         tstr2SCI[tstr][0],
 4067                         tstr,
 4068                         dt,
 4069                         reportDT(dt, grpby['include'], options),
 4070                         uid])
 4071                     bisect.insort(tups, tuple(item))
 4072 
 4073             else:  # no date spec in grpby
 4074                 item = []
 4075                 dt = ''
 4076                 if hsh['itemtype'] in [u'+', u'-', u'%']:
 4077                     # task type
 4078                     done, due, following = getDoneAndTwo(hsh)
 4079                     if due:
 4080                         # add a due entry
 4081                         if due.date() < today_date:
 4082                             if tchr == '+':
 4083                                 tstr = 'pc'
 4084                             elif tchr == '-':
 4085                                 tstr = 'pt'
 4086                             elif tchr == '%':
 4087                                 tstr = 'pd'
 4088                         dt = due
 4089                     elif done:
 4090                         dt = done
 4091                 else:
 4092                     # not a task type
 4093                     if 's' in hsh:
 4094                         if 'rrule' in hsh:
 4095                             if tchr in ['^', '*', '~']:
 4096                                 dt = (hsh['rrule'].after(today_datetime, inc=True) or hsh['rrule'].before(today_datetime, inc=True))
 4097                                 if dt is None:
 4098                                     logger.warning('No valid datetimes for {0}, {1}'.format(hsh['_summary'], hsh['fileinfo']))
 4099                                     continue
 4100                             else:
 4101                                 dt = hsh['rrule'].after(hsh['s'], inc=True)
 4102                         else:
 4103                             dt = parse_str(hsh['s'], hsh['z']).replace(tzinfo=None)
 4104                     else:
 4105                         # undated
 4106                         dt = ''
 4107                 for g in grpby['tuples']:
 4108                     if groupdate_regex.search(g):
 4109                         item.append(dt_to_str(dt, g))
 4110                     else:
 4111                         try:
 4112                             res = eval(g)
 4113                             item.append(res)
 4114                         except:
 4115                             pass
 4116                 if type(dt) == datetime:
 4117                     if dated['b'] <= dt <= dated['e']:
 4118                         dtstr = reportDT(dt, grpby['include'], options)
 4119                         dt = dt.strftime(etmdatefmt)
 4120                     else:
 4121                         dt = None
 4122                 else:
 4123                     dtstr = dt
 4124                 if dt is not None:
 4125                     item.extend([
 4126                         tstr2SCI[tstr][0],
 4127                         tstr,
 4128                         dt,
 4129                         dtstr,
 4130                         uid])
 4131                     bisect.insort(tups, tuple(item))
 4132         except:
 4133             logger.exception('Error processing: {0}, {1}'.format(hsh['_summary'], hsh['fileinfo']))
 4134     return tups
 4135 
 4136 
 4137 def getAgenda(allrows, colors=2, days=4, indent=2, width1=54,
 4138               width2=14, calendars=None, omit=[], mode='html', fltr=None):
 4139     if not calendars:
 4140         calendars = []
 4141     items = deepcopy(allrows)
 4142     day = []
 4143     inbasket = []
 4144     now = []
 4145     next = []
 4146     someday = []
 4147     if colors and mode == 'html':
 4148         bb = "<b>"
 4149         eb = "</b>"
 4150     else:
 4151         bb = ""
 4152         eb = ""
 4153     # show day items starting with beg and ending with lst
 4154     beg = datetime.today()
 4155     tom = beg + ONEDAY
 4156     lst = beg + (days - 1) * ONEDAY
 4157     beg_fmt = beg.strftime(u"%Y%m%d")
 4158     tom_fmt = tom.strftime(u"%Y%m%d")
 4159     lst_fmt = lst.strftime(u"%Y%m%d")
 4160     if not items:
 4161         return {}
 4162     for item in items:
 4163         if item[0][0] == 'day':
 4164             if item[0][1] >= beg_fmt and item[0][1] <= lst_fmt:
 4165                 # if item[2][1] in ['fn', 'ac', 'ns']:
 4166                 if omit and item[2][1] in omit:
 4167                     # skip omitted items
 4168                     continue
 4169                 if item[0][1] == beg_fmt:
 4170                     item[1] = u"{0}".format(fmt_date(beg, short=True))
 4171                 elif item[0][1] == tom_fmt:
 4172                     item[1] = u"{0}".format(fmt_date(tom, short=True))
 4173                 day.append(item)
 4174         elif item[0][0] == 'inbasket':
 4175             item.insert(1, u"{0}{1}{2}".format(bb, _("In Basket"), eb))
 4176             inbasket.append(item)
 4177         elif item[0][0] == 'now':
 4178             item.insert(1, u"{0}{1}{2}".format(bb, _("Now"), eb))
 4179             now.append(item)
 4180         elif item[0][0] == 'next':
 4181             item.insert(1, u"{0}{1}{2}".format(bb, _("Next"), eb))
 4182             next.append(item)
 4183         elif item[0][0] == 'someday':
 4184             item.insert(1, u"{0}{1}{2}".format(bb, _("Someday"), eb))
 4185             someday.append(item)
 4186     tree = {}
 4187     nv = 0
 4188     for l in [day, inbasket, now, next, someday]:
 4189         if l:
 4190             nv += 1
 4191             update = makeTree(l, calendars=calendars, fltr=fltr)
 4192             for key in update.keys():
 4193                 tree.setdefault(key, []).extend(update[key])
 4194     logger.debug("called makeTree for {0} views".format(nv))
 4195     return tree
 4196 
 4197 
 4198 # @memoize
 4199 def getReportData(s, file2uuids, uuid2hash, options=None, export=False,
 4200                   colors=None, cli=True):
 4201     """
 4202         getViewData returns items with the format:
 4203             [(view, (sort)), node1, node2, ...,
 4204                 (uuid, typestr, summary, col_2, dt_sort_str) ]
 4205         pop item[0] after sort leaving
 4206             [node1, node2, ... (xxx) ]
 4207 
 4208         for actions (tallyByGroup) we need
 4209             (node1, node2, ... (minutes, value, expense, charge))
 4210     """
 4211     if not options:
 4212         options = {}
 4213     try:
 4214         grpby, dated, filters = str2opts(s, options, cli)
 4215     except:
 4216         e = "{0}: {1}".format(_("Could not process"), s)
 4217         logger.exception(e)
 4218         return e
 4219     if not grpby:
 4220         return ["{0}: grpby".format(_('invalid setting'))]
 4221     uuids = applyFilters(file2uuids, uuid2hash, filters)
 4222     tups = makeReportTuples(uuids, uuid2hash, grpby, dated, options)
 4223     items = []
 4224     cols = grpby['cols']
 4225     fmts = grpby['fmts']
 4226     for tup in tups:
 4227         uuid = tup[-1]
 4228         hsh = uuid2hash[tup[-1]]
 4229 
 4230         # for eval we need to be sure that t is in hsh
 4231         if 't' not in hsh:
 4232             hsh['t'] = []
 4233 
 4234         try:
 4235             # for eval: {} is the global namespace
 4236             # and {'tup' ... dt_to_str} is the local namespace
 4237             eval_fmts = [
 4238                 eval(x, {},
 4239                      {'tup': tup, 'hsh': hsh, 'rsplit': rsplit,
 4240                       'd_to_str': d_to_str, 'dt_to_str': dt_to_str})
 4241                 for x in fmts]
 4242         except Exception:
 4243             logger.exception('fmts: {0}'.format(fmts))
 4244             continue
 4245         if filters['dates']:
 4246             dt = reportDT(tup[-3], grpby['include'], options)
 4247             if dt == '00:00':
 4248                 dt = ''
 4249                 dtl = None
 4250             else:
 4251                 dtl = tup[-3]
 4252         else:
 4253             # the datetime (sort string) will be in tup[-3],
 4254             # the display string in tup[-2]
 4255             dt = tup[-2]
 4256             dtl = tup[-3]
 4257         if dtl:
 4258             etmdt = parse_str(dtl, hsh['z'], fmt=rfmt)
 4259             if etmdt is None:
 4260                 etmdt = ""
 4261         else:
 4262             etmdt = ''
 4263 
 4264         try:
 4265             item = (cols.format(*eval_fmts)).split('::')
 4266         except:
 4267             logger.exception("eval_fmts: {0}".format(*eval_fmts))
 4268 
 4269         if grpby['report'] == 'c':
 4270             if fmts.count(u"hsh['t']"):
 4271                 position = fmts.index(u"hsh['t']")
 4272                 for tag in hsh['t']:
 4273                     rowcpy = deepcopy(item)
 4274                     rowcpy[position] = tag
 4275                     rowcpy.append(
 4276                         (tup[-1], tup[-4],
 4277                          setSummary(hsh, parse(dtl)), dt, etmdt))
 4278                     items.append(rowcpy)
 4279             else:
 4280                 item.append((tup[-1], tup[-4],
 4281                              setSummary(hsh, parse(dtl)), dt, etmdt))
 4282                 items.append(item)
 4283         else:  # action report
 4284             summary = format(setSummary(hsh, parse(dt)))
 4285             item.append("{0}!!{1}!!".format(summary, uuid))
 4286             temp = []
 4287             temp.extend(timeValue(hsh, options))
 4288             temp.extend(expenseCharge(hsh, options))
 4289             item.append(temp)
 4290             items.append(item)
 4291     if grpby['report'] == 'c' and not export:
 4292         tree = makeTree(items, sort=False)
 4293         return tree
 4294     else:
 4295         if grpby['report'] == 'a' and 'depth' in grpby and grpby['depth']:
 4296             depth = min(grpby['depth']-1, len(grpby['lst']))
 4297         else:
 4298             depth = len(grpby['lst'])
 4299         logger.debug('using depth: {0}'.format(depth))
 4300         if export:
 4301             data = []
 4302             head = [x for x in grpby['lst'][:depth]]
 4303             logger.debug('head: {0}\nlst: {1}\ndepth: {2}'.format(head, grpby['lst'], depth))
 4304             if grpby['report'] == 'c':
 4305                 for row in items:
 4306                     tup = ['"{0}"'.format(x) for x in row.pop(-1)[2:6]]
 4307                     row.extend(tup)
 4308                     data.append(row)
 4309             else:
 4310                 head.extend(['minutes', 'value', 'expense', 'charge'])
 4311                 data.append(head)
 4312                 lst = tallyByGroup(
 4313                     items, max_level=depth, options=options, export=True)
 4314                 for row in lst:
 4315                     tup = [x for x in list(row.pop(-1))]
 4316                     row.extend(tup)
 4317                     data.append(row)
 4318             return data
 4319         else:
 4320             res = tallyByGroup(items, max_level=depth, options=options)
 4321             return res
 4322 
 4323 
 4324 def str2hsh(s, uid=None, options=None):
 4325     if not options:
 4326         options = {}
 4327     msg = []
 4328     try:
 4329         hsh = {}
 4330         alerts = []
 4331         at_parts = at_regex.split(s)
 4332         # logger.debug('at_parts: {0}'.format(at_parts))
 4333         head = at_parts.pop(0).strip()
 4334         if head and head[0] in type_keys:
 4335             itemtype = unicode(head[0])
 4336             summary = head[1:].strip()
 4337         else:
 4338             # in basket
 4339             itemtype = u'$'
 4340             summary = head
 4341         hsh['itemtype'] = itemtype
 4342         hsh['_summary'] = summary
 4343         if uid:
 4344             hsh['I'] = uid
 4345         if itemtype == u'+':
 4346             hsh['_group_summary'] = summary
 4347         hsh['entry'] = s
 4348         for at_part in at_parts:
 4349             at_key = unicode(at_part[0])
 4350             at_val = at_part[1:].strip()
 4351             if itemtype not in key2type[at_key]:
 4352                 msg.append("An entry for @{0} is not allowed in items of type '{1}'.".format(at_key, itemtype))
 4353                 continue
 4354             if at_key == 'a':
 4355                 actns = options['alert_default']
 4356                 arguments = []
 4357                 # alert_parts = at_val.split(':', maxsplit=1)
 4358                 alert_parts = re.split(':', at_val, maxsplit=1)
 4359                 t_lst = alert_parts.pop(0).split(',')
 4360                 periods = []
 4361                 for x in t_lst:
 4362                     p = parse_period(x)
 4363                     if type(p) is timedelta:
 4364                         periods.append(p)
 4365                     else:
 4366                         msg.append(p)
 4367                 periods = tuple(periods)
 4368                 triggers = [x for x in periods]
 4369                 if alert_parts:
 4370                     action_parts = [
 4371                         x.strip() for x in alert_parts[0].split(';')]
 4372                     actns = [
 4373                         unicode(x.strip()) for x in
 4374                         action_parts.pop(0).split(',')]
 4375                     if action_parts:
 4376                         arguments = []
 4377                         for action_part in action_parts:
 4378                             tmp = action_part.split(',')
 4379                             arguments.append(tmp)
 4380                 alerts.append([triggers, actns, arguments])
 4381             elif at_key in ['+', '-', 'i', 'n']:
 4382                 parts = comma_regex.split(at_val)
 4383                 tmp = []
 4384                 for part in parts:
 4385                     tmp.append(part)
 4386                 hsh[at_key] = tmp
 4387             elif at_key in ['r', 'j']:
 4388                 amp_parts = amp_regex.split(at_val)
 4389                 part_hsh = {}
 4390                 this_key = unicode(amp_hsh.get(at_key, at_key))
 4391                 amp_0 = amp_parts.pop(0)
 4392                 part_hsh[this_key] = amp_0
 4393                 for amp_part in amp_parts:
 4394                     amp_key = unicode(amp_part[0])
 4395                     amp_val = amp_part[1:].strip()
 4396                     if amp_key in ['q', 'i', 't']:
 4397                         try:
 4398                             part_hsh[amp_key] = int(amp_val)
 4399                         except ValueError:
 4400                             msg.append('"&{0} {1}" is invalid - a positive integer is required.'.format(amp_key, amp_val))
 4401                             logger.exception('Bad entry "{0}" given for "&{1}" in "{2}". An integer is required.'.format(amp_val, amp_key, hsh['entry']))
 4402                         else:
 4403                             if part_hsh[amp_key] < 1:
 4404                                 msg.append('"&{0} {1}" is invalid - a positive integer is required.'.format(amp_key, amp_val))
 4405 
 4406                     elif amp_key == 'e':
 4407                         p = parse_period(amp_val)
 4408                         if type(p) is timedelta:
 4409                             part_hsh['e'] = p
 4410                         else:
 4411                             msg.append(p)
 4412                     else:
 4413                         m = range_regex.search(amp_val)
 4414                         if m:
 4415                             if m.group(3):
 4416                                 part_hsh[amp_key] = [
 4417                                     x for x in range(
 4418                                         int(m.group(1)),
 4419                                         int(m.group(3)))]
 4420                             else:
 4421                                 part_hsh[amp_key] = range(int(m.group(1)))
 4422                         # value will be a scalar or list
 4423                         elif comma_regex.search(amp_val):
 4424                             part_hsh[amp_key] = comma_regex.split(amp_val)
 4425                         else:
 4426                             part_hsh[amp_key] = amp_val
 4427                 try:
 4428                     hsh.setdefault("%s" % at_key, []).append(part_hsh)
 4429                 except:
 4430                     msg.append("error appending '%s' to hsh[%s]" %
 4431                                (part_hsh, at_key))
 4432             else:
 4433                 # value will be a scalar or list
 4434                 if at_key in ['a', 't']:
 4435                     if comma_regex.search(at_val):
 4436                         hsh[at_key] = [
 4437                             x for x in comma_regex.split(at_val) if x]
 4438                     else:
 4439                         hsh[at_key] = [at_val]
 4440                 elif at_key == 's':
 4441                     # we'll parse this after we get the timezone
 4442                     hsh['s'] = at_val
 4443                 elif at_key == 'k':
 4444                     hsh['k'] = ":".join([x.strip() for x in at_val.split(':')])
 4445                 elif at_key == 'e':
 4446                     p = parse_period(at_val)
 4447                     if type(p) is timedelta:
 4448                         hsh['e'] = p
 4449                     else:
 4450                         msg.append(p)
 4451 
 4452                 elif at_key == 'p':
 4453                     hsh['p'] = int(at_val)
 4454                     if hsh['p'] <= 0 or hsh['p'] >= 10:
 4455                         hsh['p'] = 10
 4456                 else:
 4457                     hsh[at_key] = at_val
 4458         if alerts:
 4459             hsh['_a'] = alerts
 4460         if 'z' not in hsh:
 4461             if 's' in hsh or 'f' in hsh or 'q' in hsh:
 4462                 hsh['z'] = options['local_timezone']
 4463         if 'z' in hsh:
 4464             z = gettz(hsh['z'])
 4465             if z is None:
 4466                 msg.append("error: bad timezone: '%s'" % hsh['z'])
 4467                 hsh['z'] = ''
 4468         if 's' in hsh:
 4469             dt = parse_str(hsh['s'], hsh['z'])
 4470             if type(dt) is datetime:
 4471                 hsh['s'] = dt.replace(tzinfo=None)
 4472             else:
 4473                 msg.append(dt)
 4474         if 'q' in hsh:
 4475             try:
 4476                 hsh['q'] = parse_str(hsh['q'], hsh['z']).replace(tzinfo=None)
 4477             except:
 4478                 err = "error: could not parse '@q {0}'".format(hsh['q'])
 4479                 msg.append(err)
 4480         if '+' in hsh:
 4481             tmp = []
 4482             for part in hsh['+']:
 4483                 r = parse_str(part, hsh['z']).replace(tzinfo=None)
 4484                 tmp.append(r)
 4485             hsh['+'] = tmp
 4486         if '-' in hsh:
 4487             tmp = []
 4488             for part in hsh['-']:
 4489                 r = parse_str(part, hsh['z']).replace(tzinfo=None)
 4490                 tmp.append(r)
 4491             hsh['-'] = tmp
 4492         if 'b' in hsh:
 4493             try:
 4494                 hsh['b'] = int(hsh['b'])
 4495             except:
 4496                 msg.append(
 4497                     '"@b {0}" is invalid - a positive integer is required'.format(hsh['b']))
 4498             else:
 4499                 if hsh['b'] < 1:
 4500                     msg.append(
 4501                         '"@b {0}" is invalid - a positive integer is required'.format(hsh['b']))
 4502 
 4503         if 'f' in hsh:
 4504             # this will be a list of done:due pairs
 4505             # 20120201T1325;20120202T1400, ...
 4506             # logger.debug('hsh["f"]: {0}'.format(hsh['f']))
 4507             pairs = [x.strip() for x in hsh['f'].split(',') if x.strip()]
 4508             # logger.debug('pairs: {0}'.format(pairs))
 4509             hsh['f'] = []
 4510             for pair in pairs:
 4511                 pair = pair.split(';')
 4512                 done = parse_str(
 4513                         pair[0], hsh['z']).replace(tzinfo=None)
 4514                 if len(pair) > 1:
 4515                     due = parse_str(pair[1], hsh['z']).replace(tzinfo=None)
 4516                 else:
 4517                     due = done
 4518                     # logger.debug("appending {0} to {1}".format(done, hsh['entry']))
 4519                 hsh['f'].append((done, due))
 4520         if 'h' in hsh:
 4521             # this will be a list of done:due pairs
 4522             # 20120201T1325;20120202T1400, ...
 4523             # logger.debug('hsh["f"]: {0}'.format(hsh['f']))
 4524             pairs = [x.strip() for x in hsh['h'].split(',') if x.strip()]
 4525             # logger.debug('pairs: {0}'.format(pairs))
 4526             hsh['h'] = []
 4527             for pair in pairs:
 4528                 pair = pair.split(';')
 4529                 done = parse_str(
 4530                         pair[0], hsh['z']).replace(tzinfo=None)
 4531                 if len(pair) > 1:
 4532                     due = parse_str(
 4533                             pair[1], hsh['z']).replace(tzinfo=None)
 4534                 else:
 4535                     due = done
 4536                     # logger.debug("appending {0} to {1}".format(done, hsh['entry']))
 4537                 hsh['h'].append((done, due))
 4538         if 'j' in hsh:
 4539             for i in range(len(hsh['j'])):
 4540                 job = hsh['j'][i]
 4541                 if 'q' not in job:
 4542                     msg.append("@j: %s" % job['j'])
 4543                     msg.append("an &q entry is required for jobs")
 4544                 if 'f' in job:
 4545                     if 'z' not in hsh:
 4546                         hsh['z'] = options['local_timezone']
 4547 
 4548                     pair = job['f'].split(';')
 4549                     done = parse_str(
 4550                             pair[0], hsh['z']).replace(tzinfo=None)
 4551                     if len(pair) > 1:
 4552                         due = parse_str(
 4553                                 pair[1], hsh['z']).replace(tzinfo=None)
 4554                     else:
 4555                         due = ''
 4556 
 4557                     job['f'] = [(done, due)]
 4558 
 4559                 if 'h' in job:
 4560                     # this will be a list of done:due pairs
 4561                     # 20120201T1325;20120202T1400, ...
 4562                     logger.debug("job['h']: {0}, {1}".format(job['h'], type(job['h'])))
 4563                     if type(job['h']) is str:
 4564                         pairs = job['h'].split(',')
 4565                     else:
 4566                         pairs = job['h']
 4567                     logger.debug('starting pairs: {0}, {1}'.format(pairs, type(pairs)))
 4568                     job['h'] = []
 4569                     # if type(pairs) in [unicode, str]:
 4570                     if type(pairs) not in [list]:
 4571                         pairs = [pairs]
 4572                     for pair in pairs:
 4573                         logger.debug('splitting pair: {0}'.format(pair))
 4574                         pair = pair.split(';')
 4575                         logger.debug('processing done, due: {0}'.format(pair))
 4576                         done = parse_str(
 4577                                 pair[0], hsh['z']).replace(tzinfo=None)
 4578                         if len(pair) > 1:
 4579                             logger.debug('parsing due: {0}, {1}'.format(pair[1], type(pair[1])))
 4580                             due = parse_str(
 4581                                     pair[1], hsh['z']).replace(tzinfo=None)
 4582                         else:
 4583                             due = done
 4584                         logger.debug("appending ({0}, {1}) to {2} ".format(done, due, job['j']))
 4585                         job['h'].append((done, due))
 4586                         logger.debug("job['h']: {0}".format(job['h']))
 4587                 # put the modified job back in the hash
 4588                     hsh['j'][i] = job
 4589         for k, v in hsh.items():
 4590             if type(v) in [datet