"Fossies" - the Fresh Open Source Software Archive

Member "roundup-2.0.0/roundup/hyperdb.py" (29 Jun 2020, 77814 Bytes) of package /linux/www/roundup-2.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "hyperdb.py": 1.6.1_vs_2.0.0.

    1 #
    2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
    3 # This module is free software, and you may redistribute it and/or modify
    4 # under the same terms as Python, so long as this copyright message and
    5 # disclaimer are retained in their original form.
    6 #
    7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
    8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
    9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
   10 # POSSIBILITY OF SUCH DAMAGE.
   11 #
   12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
   13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   14 # FOR A PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
   15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
   16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
   17 #
   18 
   19 """Hyperdatabase implementation, especially field types.
   20 """
   21 __docformat__ = 'restructuredtext'
   22 
   23 # standard python modules
   24 import os, re, shutil, sys, weakref
   25 import traceback
   26 import logging
   27 
   28 # roundup modules
   29 from . import date, password
   30 from .support import ensureParentsExist, PrioList
   31 from roundup.i18n import _
   32 from roundup.cgi.exceptions import DetectorError
   33 from roundup.anypy.cmp_ import NoneAndDictComparable
   34 from roundup.anypy.strings import eval_import
   35 
   36 logger = logging.getLogger('roundup.hyperdb')
   37 
   38 
   39 #
   40 # Types
   41 #
   42 class _Type(object):
   43     """A roundup property type."""
   44     def __init__(self, required=False, default_value=None, quiet=False):
   45         self.required = required
   46         self.__default_value = default_value
   47         self.quiet = quiet
   48         # We do not allow updates if self.computed is True
   49         # For now only Multilinks (using the rev_multilink) can be computed
   50         self.computed = False
   51 
   52     def __repr__(self):
   53         ' more useful for dumps '
   54         return '<%s.%s>' % (self.__class__.__module__, self.__class__.__name__)
   55 
   56     def get_default_value(self):
   57         """The default value when creating a new instance of this property."""
   58         return self.__default_value
   59 
   60     def register (self, cls, propname):
   61         """Register myself to the class of which we are a property
   62            the given propname is the name we have in our class.
   63         """
   64         assert not getattr(self, 'cls', None)
   65         self.name = propname
   66         self.cls  = cls
   67 
   68     def sort_repr(self, cls, val, name):
   69         """Representation used for sorting. This should be a python
   70         built-in type, otherwise sorting will take ages. Note that
   71         individual backends may chose to use something different for
   72         sorting as long as the outcome is the same.
   73         """
   74         return val
   75 
   76 
   77 class String(_Type):
   78     """An object designating a String property."""
   79     def __init__(self, indexme='no', required=False, default_value="",
   80                  quiet=False):
   81         super(String, self).__init__(required, default_value, quiet)
   82         self.indexme = indexme == 'yes'
   83 
   84     def from_raw(self, value, propname='', **kw):
   85         """fix the CRLF/CR -> LF stuff"""
   86         if propname == 'content':
   87             # Why oh why wasn't the FileClass content property a File
   88             # type from the beginning?
   89             return value
   90         return fixNewlines(value)
   91 
   92     def sort_repr(self, cls, val, name):
   93         if not val:
   94             return val
   95         if name == 'id':
   96             return int(val)
   97         return val.lower()
   98 
   99 
  100 class Password(_Type):
  101     """An object designating a Password property."""
  102     def __init__(self, scheme=None, required=False, default_value=None,
  103                  quiet=False):
  104         super(Password, self).__init__(required, default_value, quiet)
  105         self.scheme = scheme
  106 
  107     def from_raw(self, value, **kw):
  108         if not value:
  109             return None
  110         try:
  111             return password.Password(encrypted=value, scheme=self.scheme,
  112                                      strict=True)
  113         except password.PasswordValueError as message:
  114             raise HyperdbValueError(_('property %s: %s') %
  115                                     (kw['propname'], message))
  116 
  117     def sort_repr(self, cls, val, name):
  118         if not val:
  119             return val
  120         return str(val)
  121 
  122 
  123 class Date(_Type):
  124     """An object designating a Date property."""
  125     def __init__(self, offset=None, required=False, default_value=None,
  126                  quiet=False):
  127         super(Date, self).__init__(required=required,
  128                                    default_value=default_value,
  129                                    quiet=quiet)
  130         self._offset = offset
  131 
  132     def offset(self, db):
  133         if self._offset is not None:
  134             return self._offset
  135         return db.getUserTimezone()
  136 
  137     def from_raw(self, value, db, **kw):
  138         try:
  139             value = date.Date(value, self.offset(db))
  140         except ValueError as message:
  141             raise HyperdbValueError(_('property %s: %r is an invalid '
  142                                       'date (%s)') % (kw['propname'],
  143                                                       value, message))
  144         return value
  145 
  146     def range_from_raw(self, value, db):
  147         """return Range value from given raw value with offset correction"""
  148         return date.Range(value, date.Date, offset=self.offset(db))
  149 
  150     def sort_repr(self, cls, val, name):
  151         if not val:
  152             return val
  153         return str(val)
  154 
  155 
  156 class Interval(_Type):
  157     """An object designating an Interval property."""
  158     def from_raw(self, value, **kw):
  159         try:
  160             value = date.Interval(value)
  161         except ValueError as message:
  162             raise HyperdbValueError(_('property %s: %r is an invalid '
  163                                       'date interval (%s)') %
  164                                     (kw['propname'], value, message))
  165         return value
  166 
  167     def sort_repr(self, cls, val, name):
  168         if not val:
  169             return val
  170         return val.as_seconds()
  171 
  172 
  173 class _Pointer(_Type):
  174     """An object designating a Pointer property that links or multilinks
  175     to a node in a specified class."""
  176     def __init__(self, classname, do_journal='yes', try_id_parsing='yes',
  177                  required=False, default_value=None,
  178                  msg_header_property=None, quiet=False, rev_multilink=None):
  179         """ Default is to journal link and unlink events.
  180             When try_id_parsing is false, we don't allow IDs in input
  181             fields (the key of the Link or Multilink property must be
  182             given instead). This is useful when the name of a property
  183             can be numeric. It will only work if the linked item has a
  184             key property and is a questionable feature for multilinks.
  185             The msg_header_property is used in the mail gateway when
  186             sending out messages: By default roundup creates headers of
  187             the form: 'X-Roundup-issue-prop: value' for all properties
  188             prop of issue that have a 'name' property. This definition
  189             allows to override the 'name' property. A common use-case is
  190             adding a mail-header with the assigned_to property to allow
  191             user mail-filtering of issue-emails for which they're
  192             responsible. In that case setting
  193             'msg_header_property="username"' for the assigned_to
  194             property will generated message headers of the form:
  195             'X-Roundup-issue-assigned_to: joe_user'.
  196             The rev_multilink is used to inject a reverse multilink into
  197             the Class linked by a Link or Multilink property. Note that
  198             the result is always a Multilink. The name given with
  199             rev_multilink is the name in the class where it is injected.
  200         """
  201         super(_Pointer, self).__init__(required, default_value, quiet)
  202         self.classname = classname
  203         self.do_journal = do_journal == 'yes'
  204         self.try_id_parsing = try_id_parsing == 'yes'
  205         self.msg_header_property = msg_header_property
  206         self.rev_multilink = rev_multilink
  207 
  208     def __repr__(self):
  209         """more useful for dumps. But beware: This is also used in schema
  210         storage in SQL backends!
  211         """
  212         return '<%s.%s to "%s">' % (self.__class__.__module__,
  213                                     self.__class__.__name__, self.classname)
  214 
  215 
  216 class Link(_Pointer):
  217     """An object designating a Link property that links to a
  218        node in a specified class."""
  219     def from_raw(self, value, db, propname, **kw):
  220         if (self.try_id_parsing and value == '-1') or not value:
  221             value = None
  222         else:
  223             if self.try_id_parsing:
  224                 value = convertLinkValue(db, propname, self, value)
  225             else:
  226                 value = convertLinkValue(db, propname, self, value, None)
  227         return value
  228 
  229     def sort_repr(self, cls, val, name):
  230         if not val:
  231             return val
  232         op = cls.labelprop()
  233         if op == 'id':
  234             return int(cls.get(val, op))
  235         return cls.get(val, op)
  236 
  237 
  238 class Multilink(_Pointer):
  239     """An object designating a Multilink property that links
  240        to nodes in a specified class.
  241 
  242        "classname" indicates the class to link to
  243 
  244        "do_journal" indicates whether the linked-to nodes should have
  245                     'link' and 'unlink' events placed in their journal
  246        "rev_property" is used when injecting reverse multilinks. By
  247                     default (for a normal multilink) the table name is
  248                     <name_of_linking_class>_<name_of_link_property>
  249                     e.g. for the messages multilink in issue in the
  250                     classic schema it would be "issue_messages". The
  251                     multilink table in that case has two columns, the
  252                     nodeid contains the ID of the linking class while
  253                     the linkid contains the ID of the linked-to class.
  254                     When injecting backlinks, for a backlink resulting
  255                     from a Link or Multilink the table_name,
  256                     linkid_name, and nodeid_name must be explicitly
  257                     specified. So when specifying a rev_multilink
  258                     property for the messages attribute in the example
  259                     above, we would get 'issue_messages' for the
  260                     table_name, 'nodeid' for the linkid_name and
  261                     'linkid' for the nodeid_name (note the reversal).
  262                     For a rev_multilink resulting, e.g. from the
  263                     standard 'status' Link in the Class 'issue' in the
  264                     classic template we would set table_name to '_issue'
  265                     (table names in the database get a leading
  266                     underscore), the nodeid_name to 'status' and the
  267                     linkid_name to 'id'. With these settings we can use
  268                     the standard query engine (with minor modifications
  269                     for the computed names) to resolve reverse
  270                     multilinks.
  271     """
  272 
  273     def __init__(self, classname, do_journal='yes', required=False,
  274                  quiet=False, try_id_parsing='yes', rev_multilink=None,
  275                  rev_property=None):
  276 
  277         super(Multilink, self).__init__(classname,
  278                                         do_journal,
  279                                         required=required,
  280                                         default_value=[], quiet=quiet,
  281                                         try_id_parsing=try_id_parsing,
  282                                         rev_multilink=rev_multilink)
  283         self.rev_property  = rev_property
  284         self.rev_classname = None
  285         self.rev_propname  = None
  286         self.table_name    = None # computed in 'register' below
  287         self.linkid_name   = 'linkid'
  288         self.nodeid_name   = 'nodeid'
  289         if self.rev_property:
  290             # Do not allow updates if this is a reverse multilink
  291             self.computed = True
  292             self.rev_classname = rev_property.cls.classname
  293             self.rev_propname  = rev_property.name
  294             if isinstance(self.rev_property, Link):
  295                 self.table_name  = '_' + self.rev_classname
  296                 self.linkid_name = 'id'
  297                 self.nodeid_name = '_' + self.rev_propname
  298             else:
  299                 self.table_name  = self.rev_classname + '_' + self.rev_propname
  300                 self.linkid_name = 'nodeid'
  301                 self.nodeid_name = 'linkid'
  302 
  303     def from_raw(self, value, db, klass, propname, itemid, **kw):
  304         if not value:
  305             return []
  306 
  307         # get the current item value if it's not a new item
  308         if itemid and not itemid.startswith('-'):
  309             curvalue = klass.get(itemid, propname)
  310         else:
  311             curvalue = []
  312 
  313         # if the value is a comma-separated string then split it now
  314         if isinstance(value, type('')):
  315             value = value.split(',')
  316 
  317         # handle each add/remove in turn
  318         # keep an extra list for all items that are
  319         # definitely in the new list (in case of e.g.
  320         # <propname>=A,+B, which should replace the old
  321         # list with A,B)
  322         do_set = 1
  323         newvalue = []
  324         for item in value:
  325             item = item.strip()
  326 
  327             # skip blanks
  328             if not item: continue
  329 
  330             # handle +/-
  331             remove = 0
  332             if item.startswith('-'):
  333                 remove = 1
  334                 item = item[1:].strip()
  335                 do_set = 0
  336             elif item.startswith('+'):
  337                 item = item[1:].strip()
  338                 do_set = 0
  339 
  340             # look up the value
  341             if self.try_id_parsing:
  342                 itemid = convertLinkValue(db, propname, self, item)
  343             else:
  344                 itemid = convertLinkValue(db, propname, self, item, None)
  345 
  346             # perform the add/remove
  347             if remove:
  348                 try:
  349                     curvalue.remove(itemid)
  350                 except ValueError:
  351                     # This can occur if the edit adding the element
  352                     # produced an error, so the form has it in the
  353                     # "no selection" choice but it's not set in the
  354                     # database.
  355                     pass
  356             else:
  357                 newvalue.append(itemid)
  358                 if itemid not in curvalue:
  359                     curvalue.append(itemid)
  360 
  361         # that's it, set the new Multilink property value,
  362         # or overwrite it completely
  363         if do_set:
  364             value = newvalue
  365         else:
  366             value = curvalue
  367 
  368         # TODO: one day, we'll switch to numeric ids and this will be
  369         # unnecessary :(
  370         value = [int(x) for x in value]
  371         value.sort()
  372         value = [str(x) for x in value]
  373         return value
  374 
  375     def register(self, cls, propname):
  376         super(Multilink, self).register(cls, propname)
  377         if self.table_name is None:
  378             self.table_name = self.cls.classname + '_' + self.name
  379 
  380     def sort_repr(self, cls, val, name):
  381         if not val:
  382             return val
  383         op = cls.labelprop()
  384         if op == 'id':
  385             return [int(cls.get(v, op)) for v in val]
  386         return [cls.get(v, op) for v in val]
  387 
  388 
  389 class Boolean(_Type):
  390     """An object designating a boolean property"""
  391     def from_raw(self, value, **kw):
  392         value = value.strip()
  393         # checked is a common HTML checkbox value
  394         value = value.lower() in ('checked', 'yes', 'true', 'on', '1')
  395         return value
  396 
  397 
  398 class Number(_Type):
  399     """An object designating a numeric property"""
  400     def __init__(self, use_double=False, **kw):
  401         """ The value use_double tells the database backend to use a
  402             floating-point format with more precision than the default.
  403             Usually implemented by type 'double precision' in the sql
  404             backend. The default is to use single-precision float (aka
  405             'real') in the db. Note that sqlite already uses 8-byte for
  406             floating point numbers.
  407         """
  408         self.use_double = use_double
  409         super(Number, self).__init__(**kw)
  410 
  411     def from_raw(self, value, **kw):
  412         value = value.strip()
  413         try:
  414             value = float(value)
  415         except ValueError:
  416             raise HyperdbValueError(_('property %s: %r is not a number') %
  417                                     (kw['propname'], value))
  418         return value
  419 
  420 
  421 class Integer(_Type):
  422     """An object designating an integer property"""
  423     def from_raw(self, value, **kw):
  424         value = value.strip()
  425         try:
  426             value = int(value)
  427         except ValueError:
  428             raise HyperdbValueError(_('property %s: %r is not an integer') %
  429                                     (kw['propname'], value))
  430         return value
  431 
  432 
  433 #
  434 # Support for splitting designators
  435 #
  436 class DesignatorError(ValueError):
  437     pass
  438 
  439 
  440 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
  441     """ Take a foo123 and return ('foo', 123)
  442     """
  443     m = dre.match(designator)
  444     if m is None:
  445         raise DesignatorError(_('"%s" not a node designator') % designator)
  446     return m.group(1), m.group(2)
  447 
  448 
  449 class Exact_Match(object):
  450     """ Used to encapsulate exact match semantics search values
  451     """
  452     def __init__(self, value):
  453         self.value = value
  454 
  455 
  456 class Proptree(object):
  457     """ Simple tree data structure for property lookup. Each node in
  458     the tree is a roundup Class Property that has to be navigated to
  459     find given property. The need_for attribute is used to mark nodes
  460     that are used for sorting, searching or retrieval: The attribute
  461     is a dictionary containing one or several of the values 'sort',
  462     'search', 'retrieve'.
  463 
  464     The Proptree is also used for transitively searching attributes for
  465     backends that do not support transitive search (e.g. anydbm). The
  466     _val attribute with set_val is used for this.
  467     """
  468 
  469     def __init__(self, db, cls, name, props, parent=None, retr=False):
  470         self.db = db
  471         self.name = name
  472         self.props = props
  473         self.parent = parent
  474         self._val = None
  475         self.has_values = False
  476         self.cls = cls
  477         self.classname = None
  478         self.uniqname = None
  479         self.children = []
  480         self.sortattr = []
  481         self.propdict = {}
  482         self.need_for = {'search': True}
  483         self.sort_direction = None
  484         self.sort_ids = None
  485         self.sort_ids_needed = False
  486         self.sort_result = None
  487         self.attr_sort_done = False
  488         self.tree_sort_done = False
  489         self.propclass = None
  490         self.orderby = []
  491         self.sql_idx = None  # index of retrieved column in sql result
  492         self.need_retired = False
  493         self.need_child_retired = False
  494         if parent:
  495             self.root = parent.root
  496             self.depth = parent.depth + 1
  497         else:
  498             self.root = self
  499             self.seqno = 1
  500             self.depth = 0
  501             self.need_for['sort'] = True
  502         self.id = self.root.seqno
  503         self.root.seqno += 1
  504         if self.cls:
  505             self.classname = self.cls.classname
  506             self.uniqname = '%s%s' % (self.cls.classname, self.id)
  507         if not self.parent:
  508             self.uniqname = self.cls.classname
  509         if retr:
  510             self.append_retr_props()
  511 
  512     def append(self, name, need_for='search', retr=False):
  513         """Append a property to self.children. Will create a new
  514         propclass for the child.
  515         """
  516         if name in self.propdict:
  517             pt = self.propdict[name]
  518             pt.need_for[need_for] = True
  519             if retr and isinstance(pt.propclass, Link):
  520                 pt.append_retr_props()
  521             return pt
  522         propclass = self.props[name]
  523         cls = None
  524         props = None
  525         if isinstance(propclass, (Link, Multilink)):
  526             cls = self.db.getclass(propclass.classname)
  527             props = cls.getprops()
  528         child = self.__class__(self.db, cls, name, props, parent=self)
  529         child.need_for = {need_for: True}
  530         child.propclass = propclass
  531         if isinstance(propclass, Multilink) and self.props[name].computed:
  532             if isinstance(self.props[name].rev_property, Link):
  533                 child.need_retired = True
  534             else:
  535                 child.need_child_retired = True
  536         self.children.append(child)
  537         self.propdict[name] = child
  538         if retr and isinstance(child.propclass, Link):
  539             child.append_retr_props()
  540         return child
  541 
  542     def append_retr_props(self):
  543         """Append properties for retrieval."""
  544         for name, prop in self.cls.getprops(protected=1).items():
  545             if isinstance(prop, Multilink):
  546                 continue
  547             self.append(name, need_for='retrieve')
  548 
  549     def compute_sort_done(self, mlseen=False):
  550         """ Recursively check if attribute is needed for sorting
  551         ('sort' in self.need_for) or all children have tree_sort_done set and
  552         sort_ids_needed unset: set self.tree_sort_done if one of the conditions
  553         holds. Also remove sort_ids_needed recursively once having seen a
  554         Multilink that is used for sorting.
  555         """
  556         if isinstance(self.propclass, Multilink) and 'sort' in self.need_for:
  557             mlseen = True
  558         if mlseen:
  559             self.sort_ids_needed = False
  560         self.tree_sort_done = True
  561         for p in self.children:
  562             p.compute_sort_done(mlseen)
  563             if not p.tree_sort_done:
  564                 self.tree_sort_done = False
  565         if 'sort' not in self.need_for:
  566             self.tree_sort_done = True
  567         if mlseen:
  568             self.tree_sort_done = False
  569 
  570     def ancestors(self):
  571         p = self
  572         while p.parent:
  573             yield p
  574             p = p.parent
  575 
  576     def search(self, search_matches=None, sort=True, retired=False):
  577         """ Recursively search for the given properties in a proptree.
  578         Once all properties are non-transitive, the search generates a
  579         simple _filter call which does the real work
  580         """
  581         filterspec = {}
  582         exact_match_spec = {}
  583         for p in self.children:
  584             if 'search' in p.need_for:
  585                 if p.children:
  586                     p.search(sort=False)
  587                 if getattr(p.propclass,'rev_property',None):
  588                     pn = p.propclass.rev_property.name
  589                     cl = p.propclass.rev_property.cls
  590                     if not isinstance(p.val, type([])):
  591                         p.val = [p.val]
  592                     if p.val == ['-1'] :
  593                         s1 = set(self.cls.getnodeids(retired=False))
  594                         s2 = set()
  595                         for id in cl.getnodeids(retired=False):
  596                             node = cl.getnode(id)
  597                             if node[pn]:
  598                                 if isinstance(node [pn], type([])):
  599                                     s2.update(node [pn])
  600                                 else:
  601                                     s2.add(node [pn])
  602                         items = s1.difference(s2)
  603                     elif isinstance(p.propclass.rev_property, Link):
  604                         items = set(cl.get(x, pn) for x in p.val
  605                             if not cl.is_retired(x))
  606                     else:
  607                         items = set().union(*(cl.get(x, pn) for x in p.val
  608                             if not cl.is_retired(x)))
  609                     filterspec[p.name] = list(sorted(items))
  610                 elif isinstance(p.val, type([])):
  611                     exact = []
  612                     subst = []
  613                     for v in p.val:
  614                         if isinstance(v, Exact_Match):
  615                             exact.append(v.value)
  616                         else:
  617                             subst.append(v)
  618                     if exact:
  619                         exact_match_spec[p.name] = exact
  620                     if subst:
  621                         filterspec[p.name] = subst
  622                 else:
  623                     assert not isinstance(p.val, Exact_Match)
  624                     filterspec[p.name] = p.val
  625         self.val = self.cls._filter(search_matches, filterspec, sort and self,
  626                                     retired=retired,
  627                                     exact_match_spec=exact_match_spec)
  628         return self.val
  629 
  630     def sort(self, ids=None):
  631         """ Sort ids by the order information stored in self. With
  632         optimisations: Some order attributes may be precomputed (by the
  633         backend) and some properties may already be sorted.
  634         """
  635         if ids is None:
  636             ids = self.val
  637         if self.sortattr and [s for s in self.sortattr
  638                               if not s.attr_sort_done]:
  639             return self._searchsort(ids, True, True)
  640         return ids
  641 
  642     def sortable_children(self, intermediate=False):
  643         """ All children needed for sorting. If intermediate is True,
  644         intermediate nodes (not being a sort attribute) are returned,
  645         too.
  646         """
  647         return [p for p in self.children
  648                 if 'sort' in p.need_for and (intermediate or p.sort_direction)]
  649 
  650     def __iter__(self):
  651         """ Yield nodes in depth-first order -- visited nodes first """
  652         for p in self.children:
  653             yield p
  654             for c in p:
  655                 yield c
  656 
  657     def _get(self, ids):
  658         """Lookup given ids -- possibly a list of list. We recurse until
  659         we have a list of ids.
  660         """
  661         if not ids:
  662             return ids
  663         if isinstance(ids[0], list):
  664             cids = [self._get(i) for i in ids]
  665         else:
  666             cids = [i and self.parent.cls.get(i, self.name) for i in ids]
  667             if self.sortattr:
  668                 cids = [self._searchsort(i, False, True) for i in cids]
  669         return cids
  670 
  671     def _searchsort(self, ids=None, update=True, dosort=True):
  672         """ Recursively compute the sort attributes. Note that ids
  673         may be a deeply nested list of lists of ids if several
  674         multilinks are encountered on the way from the root to an
  675         individual attribute. We make sure that everything is properly
  676         sorted on the way up. Note that the individual backend may
  677         already have precomputed self.result or self.sort_ids. In this
  678         case we do nothing for existing sa.result and recurse further if
  679         self.sort_ids is available.
  680 
  681         Yech, Multilinks: This gets especially complicated if somebody
  682         sorts by different attributes of the same multilink (or
  683         transitively across several multilinks). My use-case is sorting
  684         by issue.messages.author and (reverse) by issue.messages.date.
  685         In this case we sort the messages by author and date and use
  686         this sorted list twice for sorting issues. This means that
  687         issues are sorted by author and then by the time of the messages
  688         *of this author*. Probably what the user intends in that case,
  689         so we do *not* use two sorted lists of messages, one sorted by
  690         author and one sorted by date for sorting issues.
  691         """
  692         for pt in self.sortable_children(intermediate=True):
  693             # ids can be an empty list
  694             if pt.tree_sort_done or not ids:
  695                 continue
  696             if pt.sort_ids:  # cached or computed by backend
  697                 cids = pt.sort_ids
  698             else:
  699                 cids = pt._get(ids)
  700             if pt.sort_direction and not pt.sort_result:
  701                 sortrep = pt.propclass.sort_repr
  702                 pt.sort_result = pt._sort_repr(sortrep, cids)
  703             pt.sort_ids = cids
  704             if pt.children:
  705                 pt._searchsort(cids, update, False)
  706         if self.sortattr and dosort:
  707             ids = self._sort(ids)
  708         if not update:
  709             for pt in self.sortable_children(intermediate=True):
  710                 pt.sort_ids = None
  711             for pt in self.sortattr:
  712                 pt.sort_result = None
  713         return ids
  714 
  715     def _set_val(self, val):
  716         """ Check if self._val is already defined. If yes, we compute the
  717             intersection of the old and the new value(s)
  718             Note: If self is a Leaf node we need to compute a
  719             union: Normally we intersect (logical and) different
  720             subqueries into a Link or Multilink property. But for
  721             leaves we might have a part of a query in a filterspec and
  722             in an exact_match_spec. These have to be all there, the
  723             generated search will ensure a logical and of all tests for
  724             equality/substring search.
  725         """
  726         if self.has_values:
  727             v = self._val
  728             if not isinstance(self._val, type([])):
  729                 v = [self._val]
  730             vals = set(v)
  731             if not isinstance(val, type([])):
  732                 val = [val]
  733             # if cls is None we're a leaf
  734             if self.cls:
  735                 vals.intersection_update(val)
  736             else:
  737                 vals.update(val)
  738             self._val = [v for v in vals]
  739         else:
  740             self._val = val
  741         self.has_values = True
  742 
  743     val = property(lambda self: self._val, _set_val)
  744 
  745     def _sort(self, val):
  746         """Finally sort by the given sortattr.sort_result. Note that we
  747         do not sort by attrs having attr_sort_done set. The caller is
  748         responsible for setting attr_sort_done only for trailing
  749         attributes (otherwise the sort order is wrong). Since pythons
  750         sort is stable, we can sort already sorted lists without
  751         destroying the sort-order for items that compare equal with the
  752         current sort.
  753 
  754         Sorting-Strategy: We sort repeatedly by different sort-keys from
  755         right to left. Since pythons sort is stable, we can safely do
  756         that. An optimisation is a "run-length encoding" of the
  757         sort-directions: If several sort attributes sort in the same
  758         direction we can combine them into a single sort. Note that
  759         repeated sorting is probably more efficient than using
  760         compare-methods in python due to the overhead added by compare
  761         methods.
  762         """
  763         if not val:
  764             return val
  765         sortattr = []
  766         directions = []
  767         dir_idx = []
  768         idx = 0
  769         curdir = None
  770         for sa in self.sortattr:
  771             if sa.attr_sort_done:
  772                 break
  773             if sortattr:
  774                 assert len(sortattr[0]) == len(sa.sort_result)
  775             sortattr.append(sa.sort_result)
  776             if curdir != sa.sort_direction:
  777                 dir_idx.append(idx)
  778                 directions.append(sa.sort_direction)
  779                 curdir = sa.sort_direction
  780             idx += 1
  781         sortattr.append(val)
  782         sortattr = zip(*sortattr)
  783         for dir, i in reversed(list(zip(directions, dir_idx))):
  784             rev = dir == '-'
  785             sortattr = sorted(sortattr,
  786                               key=lambda x: NoneAndDictComparable(x[i:idx]),
  787                               reverse=rev)
  788             idx = i
  789         return [x[-1] for x in sortattr]
  790 
  791     def _sort_repr(self, sortrep, ids):
  792         """Call sortrep for given ids -- possibly a list of list. We
  793         recurse until we have a list of ids.
  794         """
  795         if not ids:
  796             return ids
  797         if isinstance(ids[0], list):
  798             res = [self._sort_repr(sortrep, i) for i in ids]
  799         else:
  800             res = [sortrep(self.cls, i, self.name) for i in ids]
  801         return res
  802 
  803     def __repr__(self):
  804         r = ["proptree:" + self.name]
  805         for n in self:
  806             r.append("proptree:" + "    " * n.depth + n.name)
  807         return '\n'.join(r)
  808     __str__ = __repr__
  809 
  810 
  811 #
  812 # the base Database class
  813 #
  814 class DatabaseError(ValueError):
  815     """Error to be raised when there is some problem in the database code
  816     """
  817     pass
  818 
  819 
  820 class Database(object):
  821     """A database for storing records containing flexible data types.
  822 
  823 This class defines a hyperdatabase storage layer, which the Classes use to
  824 store their data.
  825 
  826 
  827 Transactions
  828 ------------
  829 The Database should support transactions through the commit() and
  830 rollback() methods. All other Database methods should be transaction-aware,
  831 using data from the current transaction before looking up the database.
  832 
  833 An implementation must provide an override for the get() method so that the
  834 in-database value is returned in preference to the in-transaction value.
  835 This is necessary to determine if any values have changed during a
  836 transaction.
  837 
  838 
  839 Implementation
  840 --------------
  841 
  842 All methods except __repr__ must be implemented by a concrete backend Database.
  843 
  844 """
  845 
  846     # flag to set on retired entries
  847     RETIRED_FLAG = '__hyperdb_retired'
  848 
  849     BACKEND_MISSING_STRING = None
  850     BACKEND_MISSING_NUMBER = None
  851     BACKEND_MISSING_BOOLEAN = None
  852 
  853     def __init__(self, config, journaltag=None):
  854         """Open a hyperdatabase given a specifier to some storage.
  855 
  856         The 'storagelocator' is obtained from config.DATABASE.
  857         The meaning of 'storagelocator' depends on the particular
  858         implementation of the hyperdatabase.  It could be a file name,
  859         a directory path, a socket descriptor for a connection to a
  860         database over the network, etc.
  861 
  862         The 'journaltag' is a token that will be attached to the journal
  863         entries for any edits done on the database.  If 'journaltag' is
  864         None, the database is opened in read-only mode: the Class.create(),
  865         Class.set(), and Class.retire() methods are disabled.
  866         """
  867         raise NotImplementedError
  868 
  869     def post_init(self):
  870         """Called once the schema initialisation has finished.
  871            If 'refresh' is true, we want to rebuild the backend
  872            structures. Note that post_init can be called multiple times,
  873            at least during regression testing.
  874         """
  875         done = getattr(self, 'post_init_done', None)
  876         for cn in self.getclasses():
  877             cl = self.getclass(cn)
  878             for p in cl.properties:
  879                 prop = cl.properties[p]
  880                 if not isinstance (prop, (Link, Multilink)):
  881                     continue
  882                 if prop.rev_multilink:
  883                     linkcls = self.getclass(prop.classname)
  884                     if prop.rev_multilink in linkcls.properties:
  885                         if not done:
  886                             raise ValueError(
  887                                 "%s already a property of class %s"%
  888                                 (prop.rev_multilink, linkcls.classname))
  889                     else:
  890                         linkcls.properties[prop.rev_multilink] = Multilink(
  891                             cl.classname, rev_property=prop)
  892         self.post_init_done = True
  893 
  894     def refresh_database(self):
  895         """Called to indicate that the backend should rebuild all tables
  896            and structures. Not called in normal usage."""
  897         raise NotImplementedError
  898 
  899     def __getattr__(self, classname):
  900         """A convenient way of calling self.getclass(classname)."""
  901         raise NotImplementedError
  902 
  903     def addclass(self, cl):
  904         """Add a Class to the hyperdatabase.
  905         """
  906         raise NotImplementedError
  907 
  908     def getclasses(self):
  909         """Return a list of the names of all existing classes."""
  910         raise NotImplementedError
  911 
  912     def getclass(self, classname):
  913         """Get the Class object representing a particular class.
  914 
  915         If 'classname' is not a valid class name, a KeyError is raised.
  916         """
  917         raise NotImplementedError
  918 
  919     def clear(self):
  920         """Delete all database contents.
  921         """
  922         raise NotImplementedError
  923 
  924     def getclassdb(self, classname, mode='r'):
  925         """Obtain a connection to the class db that will be used for
  926            multiple actions.
  927         """
  928         raise NotImplementedError
  929 
  930     def addnode(self, classname, nodeid, node):
  931         """Add the specified node to its class's db.
  932         """
  933         raise NotImplementedError
  934 
  935     def serialise(self, classname, node):
  936         """Copy the node contents, converting non-marshallable data into
  937            marshallable data.
  938         """
  939         return node
  940 
  941     def setnode(self, classname, nodeid, node):
  942         """Change the specified node.
  943         """
  944         raise NotImplementedError
  945 
  946     def unserialise(self, classname, node):
  947         """Decode the marshalled node data
  948         """
  949         return node
  950 
  951     def getnode(self, classname, nodeid):
  952         """Get a node from the database.
  953 
  954         'cache' exists for backwards compatibility, and is not used.
  955         """
  956         raise NotImplementedError
  957 
  958     def hasnode(self, classname, nodeid):
  959         """Determine if the database has a given node.
  960         """
  961         raise NotImplementedError
  962 
  963     def countnodes(self, classname):
  964         """Count the number of nodes that exist for a particular Class.
  965         """
  966         raise NotImplementedError
  967 
  968     def storefile(self, classname, nodeid, property, content):
  969         """Store the content of the file in the database.
  970 
  971            The property may be None, in which case the filename does not
  972            indicate which property is being saved.
  973         """
  974         raise NotImplementedError
  975 
  976     def getfile(self, classname, nodeid, property):
  977         """Get the content of the file in the database.
  978         """
  979         raise NotImplementedError
  980 
  981     def addjournal(self, classname, nodeid, action, params):
  982         """ Journal the Action
  983         'action' may be:
  984 
  985             'set' -- 'params' is a dictionary of property values
  986             'create' -- 'params' is an empty dictionary as of
  987                       Wed Nov 06 11:38:43 2002 +0000
  988             'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
  989             'retired' or 'restored'-- 'params' is None
  990         """
  991         raise NotImplementedError
  992 
  993     def getjournal(self, classname, nodeid):
  994         """ get the journal for id
  995         """
  996         raise NotImplementedError
  997 
  998     def pack(self, pack_before):
  999         """ pack the database
 1000         """
 1001         raise NotImplementedError
 1002 
 1003     def commit(self):
 1004         """ Commit the current transactions.
 1005 
 1006         Save all data changed since the database was opened or since the
 1007         last commit() or rollback().
 1008         """
 1009         raise NotImplementedError
 1010 
 1011     def rollback(self):
 1012         """ Reverse all actions from the current transaction.
 1013 
 1014         Undo all the changes made since the database was opened or the last
 1015         commit() or rollback() was performed.
 1016         """
 1017         raise NotImplementedError
 1018 
 1019     def close(self):
 1020         """Close the database.
 1021 
 1022         This method must be called at the end of processing.
 1023 
 1024         """
 1025 
 1026 
 1027 def iter_roles(roles):
 1028     ''' handle the text processing of turning the roles list
 1029         into something python can use more easily
 1030     '''
 1031     if not roles or not roles.strip():
 1032         return
 1033     for role in [x.lower().strip() for x in roles.split(',')]:
 1034         yield role
 1035 
 1036 
 1037 #
 1038 # The base Class class
 1039 #
 1040 class Class:
 1041     """ The handle to a particular class of nodes in a hyperdatabase.
 1042 
 1043         All methods except __repr__ and getnode must be implemented by a
 1044         concrete backend Class.
 1045     """
 1046 
 1047     def __init__(self, db, classname, **properties):
 1048         """Create a new class with a given name and property specification.
 1049 
 1050         'classname' must not collide with the name of an existing class,
 1051         or a ValueError is raised.  The keyword arguments in 'properties'
 1052         must map names to property objects, or a TypeError is raised.
 1053         """
 1054         for name in 'creation activity creator actor'.split():
 1055             if name in properties:
 1056                 raise ValueError('"creation", "activity", "creator" and '
 1057                                  '"actor" are reserved')
 1058 
 1059         self.classname = classname
 1060         self.properties = properties
 1061         # Make the class and property name known to the property
 1062         for p in properties:
 1063             properties[p].register(self, p)
 1064         self.db = weakref.proxy(db)       # use a weak ref to avoid circularity
 1065         self.key = ''
 1066 
 1067         # should we journal changes (default yes)
 1068         self.do_journal = 1
 1069 
 1070         # do the db-related init stuff
 1071         db.addclass(self)
 1072 
 1073         actions = "create set retire restore".split()
 1074         self.auditors = dict([(a, PrioList()) for a in actions])
 1075         self.reactors = dict([(a, PrioList()) for a in actions])
 1076 
 1077     def __repr__(self):
 1078         """Slightly more useful representation
 1079            Note that an error message can be raised at a point
 1080            where self.classname isn't known yet if the error
 1081            occurs during schema parsing.
 1082         """
 1083         cn = getattr (self, 'classname', 'Unknown')
 1084         return '<hyperdb.Class "%s">' % cn
 1085 
 1086     # Editing nodes:
 1087 
 1088     def create(self, **propvalues):
 1089         """Create a new node of this class and return its id.
 1090 
 1091         The keyword arguments in 'propvalues' map property names to values.
 1092 
 1093         The values of arguments must be acceptable for the types of their
 1094         corresponding properties or a TypeError is raised.
 1095 
 1096         If this class has a key property, it must be present and its value
 1097         must not collide with other key strings or a ValueError is raised.
 1098 
 1099         Any other properties on this class that are missing from the
 1100         'propvalues' dictionary are set to None.
 1101 
 1102         If an id in a link or multilink property does not refer to a valid
 1103         node, an IndexError is raised.
 1104         """
 1105         raise NotImplementedError
 1106 
 1107     _marker = []
 1108 
 1109     def get(self, nodeid, propname, default=_marker, cache=1):
 1110         """Get the value of a property on an existing node of this class.
 1111 
 1112         'nodeid' must be the id of an existing node of this class or an
 1113         IndexError is raised.  'propname' must be the name of a property
 1114         of this class or a KeyError is raised.
 1115 
 1116         'cache' exists for backwards compatibility, and is not used.
 1117         """
 1118         raise NotImplementedError
 1119 
 1120     # not in spec
 1121     def getnode(self, nodeid):
 1122         """ Return a convenience wrapper for the node.
 1123 
 1124         'nodeid' must be the id of an existing node of this class or an
 1125         IndexError is raised.
 1126 
 1127         'cache' exists for backwards compatibility, and is not used.
 1128         """
 1129         return Node(self, nodeid)
 1130 
 1131     def getnodeids(self, retired=None):
 1132         """Retrieve all the ids of the nodes for a particular Class.
 1133         """
 1134         raise NotImplementedError
 1135 
 1136     def set(self, nodeid, **propvalues):
 1137         """Modify a property on an existing node of this class.
 1138 
 1139         'nodeid' must be the id of an existing node of this class or an
 1140         IndexError is raised.
 1141 
 1142         Each key in 'propvalues' must be the name of a property of this
 1143         class or a KeyError is raised.
 1144 
 1145         All values in 'propvalues' must be acceptable types for their
 1146         corresponding properties or a TypeError is raised.
 1147 
 1148         If the value of the key property is set, it must not collide with
 1149         other key strings or a ValueError is raised.
 1150 
 1151         If the value of a Link or Multilink property contains an invalid
 1152         node id, a ValueError is raised.
 1153         """
 1154         raise NotImplementedError
 1155 
 1156     def retire(self, nodeid):
 1157         """Retire a node.
 1158 
 1159         The properties on the node remain available from the get() method,
 1160         and the node's id is never reused.
 1161 
 1162         Retired nodes are not returned by the find(), list(), or lookup()
 1163         methods, and other nodes may reuse the values of their key properties.
 1164         """
 1165         raise NotImplementedError
 1166 
 1167     def restore(self, nodeid):
 1168         """Restpre a retired node.
 1169 
 1170         Make node available for all operations like it was before retirement.
 1171         """
 1172         raise NotImplementedError
 1173 
 1174     def is_retired(self, nodeid):
 1175         """Return true if the node is rerired
 1176         """
 1177         raise NotImplementedError
 1178 
 1179     def destroy(self, nodeid):
 1180         """Destroy a node.
 1181 
 1182         WARNING: this method should never be used except in extremely rare
 1183                  situations where there could never be links to the node being
 1184                  deleted
 1185 
 1186         WARNING: use retire() instead
 1187 
 1188         WARNING: the properties of this node will not be available ever again
 1189 
 1190         WARNING: really, use retire() instead
 1191 
 1192         Well, I think that's enough warnings. This method exists mostly to
 1193         support the session storage of the cgi interface.
 1194 
 1195         The node is completely removed from the hyperdb, including all journal
 1196         entries. It will no longer be available, and will generally break code
 1197         if there are any references to the node.
 1198         """
 1199 
 1200     def history(self, nodeid, enforceperm=True, skipquiet=True):
 1201         """Retrieve the journal of edits on a particular node.
 1202 
 1203         'nodeid' must be the id of an existing node of this class or an
 1204         IndexError is raised.
 1205 
 1206         The returned list contains tuples of the form
 1207 
 1208             (date, tag, action, params)
 1209 
 1210         'date' is a Timestamp object specifying the time of the change and
 1211         'tag' is the journaltag specified when the database was opened.
 1212 
 1213         If the property to be displayed is a quiet property, it will
 1214         not be shown. This can be disabled by setting skipquiet=False.
 1215 
 1216         If the user requesting the history does not have View access
 1217         to the property, the journal entry will not be shown. This can
 1218         be disabled by setting enforceperm=False.
 1219 
 1220         Note that there is a check for obsolete properties and classes
 1221         resulting from history changes. These are also only checked if
 1222         enforceperm is True.
 1223         """
 1224         if not self.do_journal:
 1225             raise ValueError('Journalling is disabled for this class')
 1226 
 1227         perm = self.db.security.hasPermission
 1228         journal = []
 1229 
 1230         uid = self.db.getuid()  # id of the person requesting the history
 1231 
 1232         # Roles of the user and the configured obsolete_history_roles
 1233         hr = set(iter_roles(self.db.config.OBSOLETE_HISTORY_ROLES))
 1234         ur = set(self.db.user.get_roles(uid))
 1235         allow_obsolete = bool(hr & ur)
 1236 
 1237         for j in self.db.getjournal(self.classname, nodeid):
 1238             # hide/remove journal entry if:
 1239             #   property is quiet
 1240             #   property is not (viewable or editable)
 1241             #   property is obsolete and not allow_obsolete
 1242             id, evt_date, user, action, args = j
 1243             if logger.isEnabledFor(logging.DEBUG):
 1244                 j_repr = "%s" % (j,)
 1245             else:
 1246                 j_repr = ''
 1247             if args and isinstance(args, type({})):
 1248                 for key in list(args.keys()):
 1249                     if key not in self.properties:
 1250                         if enforceperm and not allow_obsolete:
 1251                             del args[key]
 1252                         continue
 1253                     if skipquiet and self.properties[key].quiet:
 1254                         logger.debug("skipping quiet property"
 1255                                      " %s::%s in %s",
 1256                                      self.classname, key, j_repr)
 1257                         del args[key]
 1258                         continue
 1259                     if enforceperm and not (perm("View",
 1260                                                  uid,
 1261                                                  self.classname,
 1262                                                  property=key) or
 1263                                             perm("Edit",
 1264                                                  uid,
 1265                                                  self.classname,
 1266                                                  property=key)):
 1267                         logger.debug("skipping unaccessible property "
 1268                                      "%s::%s seen by user%s in %s",
 1269                                      self.classname, key, uid, j_repr)
 1270                         del args[key]
 1271                         continue
 1272                 if not args:
 1273                     logger.debug("Omitting journal entry for  %s%s"
 1274                                  " all props removed in: %s",
 1275                                  self.classname, nodeid, j_repr)
 1276                     continue
 1277                 journal.append(j)
 1278             elif action in ['link', 'unlink'] and isinstance(args, type(())):
 1279                 # definitions:
 1280                 # myself - object whose history is being filtered
 1281                 # linkee - object/class whose property is changing to
 1282                 #          include/remove myself
 1283                 # link property - property of the linkee class that is changing
 1284                 #
 1285                 # Remove the history item if
 1286                 #   linkee.link property (key) is quiet
 1287                 #   linkee class.link property is not (viewable or editable)
 1288                 #       to user
 1289                 #   [ should linkee object.link property is not
 1290                 #      (viewable or editable) to user be included?? ]
 1291                 #   linkee object (linkcl, linkid) is not
 1292                 #       (viewable or editable) to user
 1293                 if len(args) == 3:
 1294                     # e.g. for issue3 blockedby adds link to issue5 with:
 1295                     # j = id, evt_date, user, action, args
 1296                     # 3|20170528045201.484|5|link|('issue', '5', 'blockedby')
 1297                     linkcl, linkid, key = args
 1298                     cls = None
 1299                     try:
 1300                         cls = self.db.getclass(linkcl)
 1301                     except KeyError:
 1302                         pass
 1303                     # obsolete property or class
 1304                     if not cls or key not in cls.properties:
 1305                         if not enforceperm or allow_obsolete:
 1306                             journal.append(j)
 1307                         continue
 1308                     # obsolete linked-to item
 1309                     try:
 1310                         cls.get(linkid, key)  # does linkid exist
 1311                     except IndexError:
 1312                         if not enforceperm or allow_obsolete:
 1313                             journal.append(j)
 1314                         continue
 1315                     # is the updated property quiet?
 1316                     if skipquiet and cls.properties[key].quiet:
 1317                         logger.debug("skipping quiet property: "
 1318                                      "%s %sed %s%s",
 1319                                      j_repr, action, self.classname, nodeid)
 1320                         continue
 1321                     # can user view the property in linkee class
 1322                     if enforceperm and not (perm("View",
 1323                                                  uid,
 1324                                                  linkcl,
 1325                                                  property=key) or
 1326                                             perm("Edit",
 1327                                                  uid,
 1328                                                  linkcl,
 1329                                                  property=key)):
 1330                         logger.debug("skipping unaccessible property: "
 1331                                      "%s with uid %s %sed %s%s",
 1332                                      j_repr, uid, action,
 1333                                      self.classname, nodeid)
 1334                         continue
 1335                     # check access to linkee object
 1336                     if enforceperm and not (perm("View",
 1337                                                  uid,
 1338                                                  cls.classname,
 1339                                                  itemid=linkid) or
 1340                                             perm("Edit",
 1341                                                  uid,
 1342                                                  cls.classname,
 1343                                                  itemid=linkid)):
 1344                         logger.debug("skipping unaccessible object: "
 1345                                      "%s uid %s %sed %s%s",
 1346                                      j_repr, uid, action,
 1347                                      self.classname, nodeid)
 1348                         continue
 1349                     journal.append(j)
 1350                 else:
 1351                     logger.error("Invalid %s journal entry for %s%s: %s",
 1352                                  action, self.classname, nodeid, j)
 1353             elif action in ['create', 'retired', 'restored']:
 1354                 journal.append(j)
 1355             else:
 1356                 logger.warning("Possibly malformed journal for %s%s %s",
 1357                                self.classname, nodeid, j)
 1358         return journal
 1359 
 1360     # Locating nodes:
 1361     def hasnode(self, nodeid):
 1362         """Determine if the given nodeid actually exists
 1363         """
 1364         raise NotImplementedError
 1365 
 1366     def setkey(self, propname):
 1367         """Select a String property of this class to be the key property.
 1368 
 1369         'propname' must be the name of a String property of this class or
 1370         None, or a TypeError is raised.  The values of the key property on
 1371         all existing nodes must be unique or a ValueError is raised.
 1372         """
 1373         raise NotImplementedError
 1374 
 1375     def setlabelprop(self, labelprop):
 1376         """Set the label property. Used for override of labelprop
 1377            resolution order.
 1378         """
 1379         if labelprop not in self.getprops():
 1380             raise ValueError(_("Not a property name: %s") % labelprop)
 1381         self._labelprop = labelprop
 1382 
 1383     def setorderprop(self, orderprop):
 1384         """Set the order property. Used for override of orderprop
 1385            resolution order
 1386         """
 1387         if orderprop not in self.getprops():
 1388             raise ValueError(_("Not a property name: %s") % orderprop)
 1389         self._orderprop = orderprop
 1390 
 1391     def getkey(self):
 1392         """Return the name of the key property for this class or None."""
 1393         raise NotImplementedError
 1394 
 1395     def labelprop(self, default_to_id=0):
 1396         """Return the property name for a label for the given node.
 1397 
 1398         This method attempts to generate a consistent label for the node.
 1399         It tries the following in order:
 1400 
 1401         0. self._labelprop if set
 1402         1. key property
 1403         2. "name" property
 1404         3. "title" property
 1405         4. first property from the sorted property name list
 1406         """
 1407         if hasattr(self, '_labelprop'):
 1408             return self._labelprop
 1409         k = self.getkey()
 1410         if k:
 1411             return k
 1412         props = self.getprops()
 1413         if 'name' in props:
 1414             return 'name'
 1415         elif 'title' in props:
 1416             return 'title'
 1417         if default_to_id:
 1418             return 'id'
 1419         props = sorted(props.keys())
 1420         return props[0]
 1421 
 1422     def orderprop(self):
 1423         """Return the property name to use for sorting for the given node.
 1424 
 1425         This method computes the property for sorting.
 1426         It tries the following in order:
 1427 
 1428         0. self._orderprop if set
 1429         1. "order" property
 1430         2. self.labelprop()
 1431         """
 1432 
 1433         if hasattr(self, '_orderprop'):
 1434             return self._orderprop
 1435         props = self.getprops()
 1436         if 'order' in props:
 1437             return 'order'
 1438         return self.labelprop()
 1439 
 1440     def lookup(self, keyvalue):
 1441         """Locate a particular node by its key property and return its id.
 1442 
 1443         If this class has no key property, a TypeError is raised.  If the
 1444         'keyvalue' matches one of the values for the key property among
 1445         the nodes in this class, the matching node's id is returned;
 1446         otherwise a KeyError is raised.
 1447         """
 1448         raise NotImplementedError
 1449 
 1450     def find(self, **propspec):
 1451         """Get the ids of nodes in this class which link to the given nodes.
 1452 
 1453         'propspec' consists of keyword args propname={nodeid:1,}
 1454         'propname' must be the name of a property in this class, or a
 1455         KeyError is raised.  That property must be a Link or Multilink
 1456         property, or a TypeError is raised.
 1457 
 1458         Any node in this class whose 'propname' property links to any of the
 1459         nodeids will be returned. Used by the full text indexing, which knows
 1460         that "foo" occurs in msg1, msg3 and file7, so we have hits on these
 1461         issues:
 1462 
 1463             db.issue.find(messages={'1':1,'3':1}, files={'7':1})
 1464         """
 1465         raise NotImplementedError
 1466 
 1467     def _filter(self, search_matches, filterspec, sort=(None, None),
 1468                 group=(None, None), retired=False, exact_match_spec={}):
 1469         """For some backends this implements the non-transitive
 1470         search, for more information see the filter method.
 1471         """
 1472         raise NotImplementedError
 1473 
 1474     def _proptree(self, filterspec, exact_match_spec={}, sortattr=[],
 1475                   retr=False):
 1476         """Build a tree of all transitive properties in the given
 1477         exact_match_spec/filterspec.
 1478         If we retrieve (retr is True) linked items we don't follow
 1479         across multilinks. We also don't follow if the searched value
 1480         can contain NULL values.
 1481         """
 1482         proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
 1483         for exact, spec in enumerate((filterspec, exact_match_spec)):
 1484             for key, v in spec.items():
 1485                 keys = key.split('.')
 1486                 p = proptree
 1487                 mlseen = False
 1488                 for k in keys:
 1489                     if isinstance(p.propclass, Multilink):
 1490                         mlseen = True
 1491                     isnull = v == '-1' or v is None
 1492                     islist = isinstance(v, type([]))
 1493                     nullin = islist and ('-1' in v or None in v)
 1494                     r = retr and not mlseen and not isnull and not nullin
 1495                     p = p.append(k, retr=r)
 1496                 if exact:
 1497                     if isinstance(v, type([])):
 1498                         vv = []
 1499                         for x in v:
 1500                             vv.append(Exact_Match(x))
 1501                         p.val = vv
 1502                     else:
 1503                         p.val = [Exact_Match(v)]
 1504                 else:
 1505                     p.val = v
 1506         multilinks = {}
 1507         for s in sortattr:
 1508             keys = s[1].split('.')
 1509             p = proptree
 1510             mlseen = False
 1511             for k in keys:
 1512                 if isinstance(p.propclass, Multilink):
 1513                     mlseen = True
 1514                 r = retr and not mlseen
 1515                 p = p.append(k, need_for='sort', retr=r)
 1516                 if isinstance(p.propclass, Multilink):
 1517                     multilinks[p] = True
 1518             if p.cls:
 1519                 p = p.append(p.cls.orderprop(), need_for='sort')
 1520             if p.sort_direction:  # if orderprop is also specified explicitly
 1521                 continue
 1522             p.sort_direction = s[0]
 1523             proptree.sortattr.append(p)
 1524         for p in multilinks.keys():
 1525             sattr = {}
 1526             for c in p:
 1527                 if c.sort_direction:
 1528                     sattr[c] = True
 1529             for sa in proptree.sortattr:
 1530                 if sa in sattr:
 1531                     p.sortattr.append(sa)
 1532         return proptree
 1533 
 1534     def get_transitive_prop(self, propname_path, default=None):
 1535         """Expand a transitive property (individual property names
 1536         separated by '.' into a new property at the end of the path. If
 1537         one of the names does not refer to a valid property, we return
 1538         None.
 1539         Example propname_path (for class issue): "messages.author"
 1540         """
 1541         props = self.db.getclass(self.classname).getprops()
 1542         for k in propname_path.split('.'):
 1543             try:
 1544                 prop = props[k]
 1545             except (KeyError, TypeError):
 1546                 return default
 1547             cl = getattr(prop, 'classname', None)
 1548             props = None
 1549             if cl:
 1550                 props = self.db.getclass(cl).getprops()
 1551         return prop
 1552 
 1553     def _sortattr(self, sort=[], group=[]):
 1554         """Build a single list of sort attributes in the correct order
 1555         with sanity checks (no duplicate properties) included. Always
 1556         sort last by id -- if id is not already in sortattr.
 1557         """
 1558         seen = {}
 1559         sortattr = []
 1560         for srt in group, sort:
 1561             if not isinstance(srt, list):
 1562                 srt = [srt]
 1563             for s in srt:
 1564                 if s[1] and s[1] not in seen:
 1565                     sortattr.append((s[0] or '+', s[1]))
 1566                     seen[s[1]] = True
 1567         if 'id' not in seen:
 1568             sortattr.append(('+', 'id'))
 1569         return sortattr
 1570 
 1571     def filter(self, search_matches, filterspec, sort=[], group=[],
 1572                retired=False, exact_match_spec={}, limit=None, offset=None):
 1573         """Return a list of the ids of the active nodes in this class that
 1574         match the 'filter' spec, sorted by the group spec and then the
 1575         sort spec.
 1576 
 1577         "search_matches" is a container type which by default is None
 1578         and optionally contains IDs of items to match. If non-empty only
 1579         IDs of the initial set are returned.
 1580 
 1581         "filterspec" is {propname: value(s)}
 1582         "exact_match_spec" is the same format as "filterspec" but
 1583         specifies exact match for the given propnames. This only makes a
 1584         difference for String properties, these specify case insensitive
 1585         substring search when in "filterspec" and exact match when in
 1586         exact_match_spec.
 1587 
 1588         "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
 1589         or None and prop is a prop name or None. Note that for
 1590         backward-compatibility reasons a single (dir, prop) tuple is
 1591         also allowed.
 1592 
 1593         The parameter retired when set to False, returns only live
 1594         (un-retired) results. When setting it to True, only retired
 1595         items are returned. If None, both retired and unretired items
 1596         are returned. The default is False, i.e. only live items are
 1597         returned by default.
 1598 
 1599         The "limit" and "offset" parameters define a limit on the number
 1600         of results returned and an offset before returning any results,
 1601         respectively. These can be used when displaying a number of
 1602         items in a pagination application or similar. A common use-case
 1603         is returning the first item of a sorted search by specifying
 1604         limit=1 (i.e. the maximum or minimum depending on sort order).
 1605 
 1606         The filter must match all properties specificed. If the property
 1607         value to match is a list:
 1608 
 1609         1. String properties must match all elements in the list, and
 1610         2. Other properties must match any of the elements in the list.
 1611 
 1612         This also means that for strings in exact_match_spec it doesn't
 1613         make sense to specify multiple values because those cannot all
 1614         be matched exactly.
 1615 
 1616         The propname in filterspec and prop in a sort/group spec may be
 1617         transitive, i.e., it may contain properties of the form
 1618         link.link.link.name, e.g. you can search for all issues where a
 1619         message was added by a certain user in the last week with a
 1620         filterspec of
 1621         {'messages.author' : '42', 'messages.creation' : '.-1w;'}
 1622 
 1623         Implementation note:
 1624         This implements a non-optimized version of Transitive search
 1625         using _filter implemented in a backend class. A more efficient
 1626         version can be implemented in the individual backends -- e.g.,
 1627         an SQL backend will want to create a single SQL statement and
 1628         override the filter method instead of implementing _filter.
 1629         """
 1630         sortattr = self._sortattr(sort=sort, group=group)
 1631         proptree = self._proptree(filterspec, exact_match_spec, sortattr)
 1632         proptree.search(search_matches, retired=retired)
 1633         if offset is not None or limit is not None:
 1634             items = proptree.sort()
 1635             if limit and offset:
 1636                 return items[offset:offset+limit]
 1637             elif offset is not None:
 1638                 return items[offset:]
 1639             else:
 1640                 return items[:limit]
 1641         return proptree.sort()
 1642 
 1643     # non-optimized filter_iter, a backend may chose to implement a
 1644     # better version that provides a real iterator that pre-fills the
 1645     # cache for each id returned. Note that the filter_iter doesn't
 1646     # promise to correctly sort by multilink (which isn't sane to do
 1647     # anyway).
 1648     filter_iter = filter
 1649 
 1650     def count(self):
 1651         """Get the number of nodes in this class.
 1652 
 1653         If the returned integer is 'numnodes', the ids of all the nodes
 1654         in this class run from 1 to numnodes, and numnodes+1 will be the
 1655         id of the next node to be created in this class.
 1656         """
 1657         raise NotImplementedError
 1658 
 1659     # Manipulating properties:
 1660     def getprops(self, protected=1):
 1661         """Return a dictionary mapping property names to property objects.
 1662            If the "protected" flag is true, we include protected properties -
 1663            those which may not be modified.
 1664         """
 1665         raise NotImplementedError
 1666 
 1667     def get_required_props(self, propnames=[]):
 1668         """Return a dict of property names mapping to property objects.
 1669         All properties that have the "required" flag set will be
 1670         returned in addition to all properties in the propnames
 1671         parameter.
 1672         """
 1673         props = self.getprops(protected=False)
 1674         pdict = dict([(p, props[p]) for p in propnames])
 1675         pdict.update([(k, v) for k, v in props.items() if v.required])
 1676         return pdict
 1677 
 1678     def addprop(self, **properties):
 1679         """Add properties to this class.
 1680 
 1681         The keyword arguments in 'properties' must map names to property
 1682         objects, or a TypeError is raised.  None of the keys in 'properties'
 1683         may collide with the names of existing properties, or a ValueError
 1684         is raised before any properties have been added.
 1685         """
 1686         raise NotImplementedError
 1687 
 1688     def index(self, nodeid):
 1689         """Add (or refresh) the node to search indexes"""
 1690         raise NotImplementedError
 1691 
 1692     #
 1693     # Detector interface
 1694     #
 1695     def audit(self, event, detector, priority=100):
 1696         """Register an auditor detector"""
 1697         self.auditors[event].append((priority, detector.__name__, detector))
 1698 
 1699     def fireAuditors(self, event, nodeid, newvalues):
 1700         """Fire all registered auditors"""
 1701         for _prio, _name, audit in self.auditors[event]:
 1702             try:
 1703                 audit(self.db, self, nodeid, newvalues)
 1704             except (EnvironmentError, ArithmeticError) as e:
 1705                 tb = traceback.format_exc()
 1706                 html = ("<h1>Traceback</h1>" + str(tb).replace('\n', '<br>').
 1707                         replace(' ', '&nbsp;'))
 1708                 txt = 'Caught exception %s: %s\n%s' % (str(type(e)), e, tb)
 1709                 exc_info = sys.exc_info()
 1710                 subject = "Error: %s" % exc_info[1]
 1711                 raise DetectorError(subject, html, txt)
 1712 
 1713     def react(self, event, detector, priority=100):
 1714         """Register a reactor detector"""
 1715         self.reactors[event].append((priority, detector.__name__, detector))
 1716 
 1717     def fireReactors(self, event, nodeid, oldvalues):
 1718         """Fire all registered reactors"""
 1719         for _prio, _name, react in self.reactors[event]:
 1720             try:
 1721                 react(self.db, self, nodeid, oldvalues)
 1722             except (EnvironmentError, ArithmeticError) as e:
 1723                 tb = traceback.format_exc()
 1724                 html = ("<h1>Traceback</h1>" + str(tb).replace('\n', '<br>').
 1725                         replace(' ', '&nbsp;'))
 1726                 txt = 'Caught exception %s: %s\n%s' % (str(type(e)), e, tb)
 1727                 exc_info = sys.exc_info()
 1728                 subject = "Error: %s" % exc_info[1]
 1729                 raise DetectorError(subject, html, txt)
 1730 
 1731     #
 1732     # import / export support
 1733     #
 1734     def export_propnames(self):
 1735         """List the property names for export from this Class"""
 1736         propnames = sorted(self.getprops().keys())
 1737         return propnames
 1738 
 1739     def import_journals(self, entries):
 1740         """Import a class's journal.
 1741 
 1742         Uses setjournal() to set the journal for each item.
 1743         Strategy for import: Sort first by id, then import journals for
 1744         each id, this way the memory footprint is a lot smaller than the
 1745         initial implementation which stored everything in a big hash by
 1746         id and then proceeded to import journals for each id."""
 1747         properties = self.getprops()
 1748         a = []
 1749         for l in entries:
 1750             # first element in sorted list is the (numeric) id
 1751             # in python2.4 and up we would use sorted with a key...
 1752             a.append((int(l[0].strip("'")), l))
 1753         a.sort()
 1754 
 1755         last = 0
 1756         r = []
 1757         for n, l in a:
 1758             nodeid, jdate, user, action, params = map(eval_import, l)
 1759             assert (str(n) == nodeid)
 1760             if n != last:
 1761                 if r:
 1762                     self.db.setjournal(self.classname, str(last), r)
 1763                 last = n
 1764                 r = []
 1765 
 1766             if action == 'set':
 1767                 for propname, value in params.items():
 1768                     prop = properties[propname]
 1769                     if value is None:
 1770                         pass
 1771                     elif isinstance(prop, Date):
 1772                         value = date.Date(value)
 1773                     elif isinstance(prop, Interval):
 1774                         value = date.Interval(value)
 1775                     elif isinstance(prop, Password):
 1776                         value = password.JournalPassword(encrypted=value)
 1777                     params[propname] = value
 1778             elif action == 'create' and params:
 1779                 # old tracker with data stored in the create!
 1780                 params = {}
 1781             r.append((nodeid, date.Date(jdate), user, action, params))
 1782         if r:
 1783             self.db.setjournal(self.classname, nodeid, r)
 1784 
 1785     #
 1786     # convenience methods
 1787     #
 1788     def get_roles(self, nodeid):
 1789         """Return iterator for all roles for this nodeid.
 1790 
 1791            Yields string-processed roles.
 1792            This method can be overridden to provide a hook where we can
 1793            insert other permission models (e.g. get roles from database)
 1794            In standard schemas only a user has a roles property but
 1795            this may be different in customized schemas.
 1796            Note that this is the *central place* where role
 1797            processing happens!
 1798         """
 1799         node = self.db.getnode(self.classname, nodeid)
 1800         return iter_roles(node['roles'])
 1801 
 1802     def has_role(self, nodeid, *roles):
 1803         '''See if this node has any roles that appear in roles.
 1804 
 1805            For convenience reasons we take a list.
 1806            In standard schemas only a user has a roles property but
 1807            this may be different in customized schemas.
 1808         '''
 1809         roles = dict.fromkeys([r.strip().lower() for r in roles])
 1810         for role in self.get_roles(nodeid):
 1811             if role in roles:
 1812                 return True
 1813         return False
 1814 
 1815 
 1816 class HyperdbValueError(ValueError):
 1817     """ Error converting a raw value into a Hyperdb value """
 1818     pass
 1819 
 1820 
 1821 def convertLinkValue(db, propname, prop, value, idre=re.compile(r'^\d+$')):
 1822     """ Convert the link value (may be id or key value) to an id value. """
 1823     linkcl = db.classes[prop.classname]
 1824     if not idre or not idre.match(value):
 1825         if linkcl.getkey():
 1826             try:
 1827                 value = linkcl.lookup(value)
 1828             except KeyError:
 1829                 raise HyperdbValueError(_('property %s: %r is not a %s.') % (
 1830                     propname, value, prop.classname))
 1831         else:
 1832             raise HyperdbValueError(_('you may only enter ID values '
 1833                                       'for property %s') % propname)
 1834     return value
 1835 
 1836 
 1837 def fixNewlines(text):
 1838     """ Homogenise line endings.
 1839 
 1840         Different web clients send different line ending values, but
 1841         other systems (eg. email) don't necessarily handle those line
 1842         endings. Our solution is to convert all line endings to LF.
 1843     """
 1844     if text is not None:
 1845         text = text.replace('\r\n', '\n')
 1846         return text.replace('\r', '\n')
 1847     return text
 1848 
 1849 
 1850 def rawToHyperdb(db, klass, itemid, propname, value, **kw):
 1851     """ Convert the raw (user-input) value to a hyperdb-storable value. The
 1852         value is for the "propname" property on itemid (may be None for a
 1853         new item) of "klass" in "db".
 1854 
 1855         The value is usually a string, but in the case of multilink inputs
 1856         it may be either a list of strings or a string with comma-separated
 1857         values.
 1858     """
 1859     properties = klass.getprops()
 1860 
 1861     # ensure it's a valid property name
 1862     propname = propname.strip()
 1863     try:
 1864         proptype = properties[propname]
 1865     except KeyError:
 1866         raise HyperdbValueError(_('%r is not a property of %s') % (
 1867             propname, klass.classname))
 1868 
 1869     # if we got a string, strip it now
 1870     if isinstance(value, type('')):
 1871         value = value.strip()
 1872 
 1873     # convert the input value to a real property value
 1874     value = proptype.from_raw(value, db=db, klass=klass,
 1875                               propname=propname, itemid=itemid, **kw)
 1876 
 1877     return value
 1878 
 1879 
 1880 class FileClass:
 1881     """ A class that requires the "content" property and stores it on
 1882         disk.
 1883     """
 1884     default_mime_type = 'text/plain'
 1885 
 1886     def __init__(self, db, classname, **properties):
 1887         """The newly-created class automatically includes the "content"
 1888         property.
 1889         """
 1890         if 'content' not in properties:
 1891             properties['content'] = String(indexme='yes')
 1892 
 1893     def export_propnames(self):
 1894         """ Don't export the "content" property
 1895         """
 1896         propnames = list(self.getprops().keys())
 1897         propnames.remove('content')
 1898         propnames.sort()
 1899         return propnames
 1900 
 1901     def exportFilename(self, dirname, nodeid):
 1902         """ Returns destination filename for a exported file
 1903 
 1904             Called by export function in roundup admin to generate
 1905             the <class>-files subdirectory
 1906         """
 1907         subdir_filename = self.db.subdirFilename(self.classname, nodeid)
 1908         return os.path.join(dirname, self.classname+'-files', subdir_filename)
 1909 
 1910     def export_files(self, dirname, nodeid):
 1911         """ Export the "content" property as a file, not csv column
 1912         """
 1913         source = self.db.filename(self.classname, nodeid)
 1914 
 1915         dest = self.exportFilename(dirname, nodeid)
 1916         ensureParentsExist(dest)
 1917         shutil.copyfile(source, dest)
 1918 
 1919     def import_files(self, dirname, nodeid):
 1920         """ Import the "content" property as a file
 1921         """
 1922         source = self.exportFilename(dirname, nodeid)
 1923 
 1924         dest = self.db.filename(self.classname, nodeid, create=1)
 1925         ensureParentsExist(dest)
 1926         shutil.copyfile(source, dest)
 1927 
 1928         mime_type = None
 1929         props = self.getprops()
 1930         if 'type' in props:
 1931             mime_type = self.get(nodeid, 'type')
 1932         if not mime_type:
 1933             mime_type = self.default_mime_type
 1934         if props['content'].indexme:
 1935             index_content = self.get(nodeid, 'binary_content')
 1936             if bytes != str and isinstance(index_content, bytes):
 1937                 index_content = index_content.decode('utf-8', errors='ignore')
 1938             # indexer will only index text mime type. It will skip
 1939             # other types. So if mime type of file is correct, we
 1940             # call add_text on content.
 1941             self.db.indexer.add_text((self.classname, nodeid, 'content'),
 1942                                      index_content, mime_type)
 1943 
 1944 
 1945 class Node:
 1946     """ A convenience wrapper for the given node
 1947     """
 1948     def __init__(self, cl, nodeid, cache=1):
 1949         self.__dict__['cl'] = cl
 1950         self.__dict__['nodeid'] = nodeid
 1951 
 1952     def keys(self, protected=1):
 1953         return list(self.cl.getprops(protected=protected).keys())
 1954 
 1955     def values(self, protected=1):
 1956         l = []
 1957         for name in self.cl.getprops(protected=protected).keys():
 1958             l.append(self.cl.get(self.nodeid, name))
 1959         return l
 1960 
 1961     def items(self, protected=1):
 1962         l = []
 1963         for name in self.cl.getprops(protected=protected).keys():
 1964             l.append((name, self.cl.get(self.nodeid, name)))
 1965         return l
 1966 
 1967     def has_key(self, name):
 1968         return name in self.cl.getprops()
 1969 
 1970     def get(self, name, default=None):
 1971         if name in self:
 1972             return self[name]
 1973         else:
 1974             return default
 1975 
 1976     def __getattr__(self, name):
 1977         if name in self.__dict__:
 1978             return self.__dict__[name]
 1979         try:
 1980             return self.cl.get(self.nodeid, name)
 1981         except KeyError as value:
 1982             # we trap this but re-raise it as AttributeError - all other
 1983             # exceptions should pass through untrapped
 1984             raise AttributeError(str(value))
 1985 
 1986     def __getitem__(self, name):
 1987         return self.cl.get(self.nodeid, name)
 1988 
 1989     def __setattr__(self, name, value):
 1990         try:
 1991             return self.cl.set(self.nodeid, **{name: value})
 1992         except KeyError as value:
 1993             # we trap this but re-raise it as AttributeError - all other
 1994             # exceptions should pass through untrapped
 1995             raise AttributeError(str(value))
 1996 
 1997     def __setitem__(self, name, value):
 1998         self.cl.set(self.nodeid, **{name: value})
 1999 
 2000     def history(self, enforceperm=True, skipquiet=True):
 2001         return self.cl.history(self.nodeid,
 2002                                enforceperm=enforceperm,
 2003                                skipquiet=skipquiet)
 2004 
 2005     def retire(self):
 2006         return self.cl.retire(self.nodeid)
 2007 
 2008 
 2009 def Choice(name, db, *options):
 2010     """Quick helper to create a simple class with choices
 2011     """
 2012     cl = Class(db, name, name=String(), order=String())
 2013     for i in range(len(options)):
 2014         cl.create(name=options[i], order=i)
 2015     return Link(name)
 2016 
 2017 # vim: set filetype=python sts=4 sw=4 et si :