"Fossies" - the Fresh Open Source Software Archive

Member "roundup-2.0.0/test/db_test_base.py" (19 May 2020, 159658 Bytes) of package /linux/www/roundup-2.0.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "db_test_base.py": 1.6.1_vs_2.0.0.

    1 #
    2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
    3 # This module is free software, and you may redistribute it and/or modify
    4 # under the same terms as Python, so long as this copyright message and
    5 # disclaimer are retained in their original form.
    6 #
    7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
    8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
    9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
   10 # POSSIBILITY OF SUCH DAMAGE.
   11 #
   12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
   13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   14 # FOR A PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
   15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
   16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
   17 
   18 from __future__ import print_function
   19 import unittest, os, shutil, errno, imp, sys, time, pprint, os.path
   20 
   21 try:
   22     from base64 import encodebytes as base64_encode  # python3 only
   23 except ImportError:
   24     # python2 and deplricated in 3
   25     from base64 import encodestring as base64_encode
   26 
   27 import logging, cgi
   28 from . import gpgmelib
   29 from email import message_from_string
   30 
   31 import pytest
   32 from roundup.hyperdb import String, Password, Link, Multilink, Date, \
   33     Interval, DatabaseError, Boolean, Number, Node, Integer
   34 from roundup.mailer import Mailer
   35 from roundup import date, password, init, instance, configuration, \
   36     roundupdb, i18n, hyperdb
   37 from roundup.cgi.templating import HTMLItem
   38 from roundup.cgi.templating import HTMLProperty, _HTMLItem, anti_csrf_nonce
   39 from roundup.cgi import client, actions
   40 from roundup.cgi.engine_zopetal import RoundupPageTemplate
   41 from roundup.cgi.templating import HTMLItem
   42 from roundup.exceptions import UsageError, Reject
   43 
   44 from roundup.anypy.strings import b2s, s2b, u2s
   45 from roundup.anypy.cmp_ import NoneAndDictComparable
   46 from roundup.anypy.email_ import message_from_bytes
   47 
   48 from .mocknull import MockNull
   49 
   50 config = configuration.CoreConfig()
   51 config.DATABASE = "db"
   52 config.RDBMS_NAME = "rounduptest"
   53 config.RDBMS_HOST = "localhost"
   54 config.RDBMS_USER = "rounduptest"
   55 config.RDBMS_PASSWORD = "rounduptest"
   56 config.RDBMS_TEMPLATE = "template0"
   57 # these TRACKER_WEB and MAIL_DOMAIN values are used in mailgw tests
   58 config.MAIL_DOMAIN = "your.tracker.email.domain.example"
   59 config.TRACKER_WEB = "http://tracker.example/cgi-bin/roundup.cgi/bugs/"
   60 # uncomment the following to have excessive debug output from test cases
   61 # FIXME: tracker logging level should be increased by -v arguments
   62 #   to 'run_tests.py' script
   63 #config.LOGGING_FILENAME = "/tmp/logfile"
   64 #config.LOGGING_LEVEL = "DEBUG"
   65 config.init_logging()
   66 
   67 def setupTracker(dirname, backend="anydbm", optimize=False):
   68     """Install and initialize new tracker in dirname; return tracker instance.
   69 
   70     If the directory exists, it is wiped out before the operation.
   71 
   72     """
   73     global config
   74     try:
   75         shutil.rmtree(dirname)
   76     except OSError as error:
   77         if error.errno not in (errno.ENOENT, errno.ESRCH): raise
   78     # create the instance
   79     init.install(dirname, os.path.join(os.path.dirname(__file__),
   80                                        '..',
   81                                        'share',
   82                                        'roundup',
   83                                        'templates',
   84                                        'classic'))
   85     config.RDBMS_BACKEND = backend
   86     config.save(os.path.join(dirname, 'config.ini'))
   87     tracker = instance.open(dirname, optimize=optimize)
   88     if tracker.exists():
   89         tracker.nuke()
   90     tracker.init(password.Password('sekrit'))
   91     return tracker
   92 
   93 def setupSchema(db, create, module):
   94     mls = module.Class(db, "mls", name=String())
   95     mls.setkey("name")
   96     status = module.Class(db, "status", name=String(), mls=Multilink("mls"))
   97     status.setkey("name")
   98     priority = module.Class(db, "priority", name=String(), order=String())
   99     priority.setkey("name")
  100     user = module.Class(db, "user", username=String(),
  101         password=Password(quiet=True), assignable=Boolean(quiet=True),
  102         age=Number(quiet=True), roles=String(), address=String(),
  103         rating=Integer(quiet=True), supervisor=Link('user'),
  104         realname=String(quiet=True), longnumber=Number(use_double=True))
  105     user.setkey("username")
  106     file = module.FileClass(db, "file", name=String(), type=String(),
  107         comment=String(indexme="yes"), fooz=Password())
  108     file_nidx = module.FileClass(db, "file_nidx", content=String(indexme='no'))
  109 
  110     # initialize quiet mode a second way without using Multilink("user", quiet=True)
  111     mynosy = Multilink("user", rev_multilink='nosy_issues')
  112     mynosy.quiet = True
  113     issue = module.IssueClass(db, "issue", title=String(indexme="yes"),
  114         status=Link("status"), nosy=mynosy, deadline=Date(quiet=True),
  115         foo=Interval(quiet=True, default_value=date.Interval('-1w')),
  116         files=Multilink("file"), assignedto=Link('user', quiet=True,
  117         rev_multilink='issues'), priority=Link('priority'),
  118         spam=Multilink('msg'), feedback=Link('msg'))
  119     stuff = module.Class(db, "stuff", stuff=String())
  120     session = module.Class(db, 'session', title=String())
  121     msg = module.FileClass(db, "msg", date=Date(),
  122         author=Link("user", do_journal='no'), files=Multilink('file'),
  123         inreplyto=String(), messageid=String(),
  124         recipients=Multilink("user", do_journal='no'))
  125     session.disableJournalling()
  126     db.post_init()
  127     if create:
  128         user.create(username="admin", roles='Admin',
  129             password=password.Password('sekrit'))
  130         user.create(username="fred", roles='User',
  131             password=password.Password('sekrit'), address='fred@example.com')
  132         u1 = mls.create(name="unread_1")
  133         u2 = mls.create(name="unread_2")
  134         status.create(name="unread",mls=[u1, u2])
  135         status.create(name="in-progress")
  136         status.create(name="testing")
  137         status.create(name="resolved")
  138         priority.create(name="feature", order="2")
  139         priority.create(name="wish", order="3")
  140         priority.create(name="bug", order="1")
  141     db.commit()
  142 
  143     # nosy tests require this
  144     db.security.addPermissionToRole('User', 'View', 'msg')
  145 
  146     # quiet journal tests require this
  147     # QuietJournal - reference used later in tests
  148     v1 = db.security.addPermission(name='View', klass='user',
  149                 properties=['username', 'supervisor', 'assignable'],
  150                 description="Prevent users from seeing roles")
  151 
  152     db.security.addPermissionToRole("User", v1)
  153 
  154 class MyTestCase(object):
  155     def tearDown(self):
  156         if hasattr(self, 'db'):
  157             self.db.close()
  158         if os.path.exists(config.DATABASE):
  159             shutil.rmtree(config.DATABASE)
  160 
  161     def open_database(self, user='admin'):
  162         self.db = self.module.Database(config, user)
  163 
  164 
  165 if 'LOGGING_LEVEL' in os.environ:
  166     logger = logging.getLogger('roundup.hyperdb')
  167     logger.setLevel(os.environ['LOGGING_LEVEL'])
  168 
  169 class commonDBTest(MyTestCase):
  170     def setUp(self):
  171         # remove previous test, ignore errors
  172         if os.path.exists(config.DATABASE):
  173             shutil.rmtree(config.DATABASE)
  174         os.makedirs(config.DATABASE + '/files')
  175         self.open_database()
  176         setupSchema(self.db, 1, self.module)
  177 
  178     def iterSetup(self, classname='issue'):
  179         cls = getattr(self.db, classname)
  180         def filt_iter(*args, **kw):
  181             """ for checking equivalence of filter and filter_iter """
  182             return list(cls.filter_iter(*args, **kw))
  183         return self.assertEqual, cls.filter, filt_iter
  184 
  185     def filteringSetupTransitiveSearch(self, classname='issue'):
  186         u_m = {}
  187         k = 30
  188         for user in (
  189                 {'username': 'ceo', 'age': 129},
  190                 {'username': 'grouplead1', 'age': 29, 'supervisor': '3'},
  191                 {'username': 'grouplead2', 'age': 29, 'supervisor': '3'},
  192                 {'username': 'worker1', 'age': 25, 'supervisor' : '4'},
  193                 {'username': 'worker2', 'age': 24, 'supervisor' : '4'},
  194                 {'username': 'worker3', 'age': 23, 'supervisor' : '5'},
  195                 {'username': 'worker4', 'age': 22, 'supervisor' : '5'},
  196                 {'username': 'worker5', 'age': 21, 'supervisor' : '5'}):
  197             u = self.db.user.create(**user)
  198             u_m [u] = self.db.msg.create(author = u, content = ' '
  199                 , date = date.Date ('2006-01-%s' % k))
  200             k -= 1
  201         i = date.Interval('-1d')
  202         for issue in (
  203                 {'title': 'ts1', 'status': '2', 'assignedto': '6',
  204                     'priority': '3', 'messages' : [u_m ['6']], 'nosy' : ['4']},
  205                 {'title': 'ts2', 'status': '1', 'assignedto': '6',
  206                     'priority': '3', 'messages' : [u_m ['6']], 'nosy' : ['5']},
  207                 {'title': 'ts4', 'status': '2', 'assignedto': '7',
  208                     'priority': '3', 'messages' : [u_m ['7']]},
  209                 {'title': 'ts5', 'status': '1', 'assignedto': '8',
  210                     'priority': '3', 'messages' : [u_m ['8']]},
  211                 {'title': 'ts6', 'status': '2', 'assignedto': '9',
  212                     'priority': '3', 'messages' : [u_m ['9']]},
  213                 {'title': 'ts7', 'status': '1', 'assignedto': '10',
  214                     'priority': '3', 'messages' : [u_m ['10']]},
  215                 {'title': 'ts8', 'status': '2', 'assignedto': '10',
  216                     'priority': '3', 'messages' : [u_m ['10']], 'foo' : i},
  217                 {'title': 'ts9', 'status': '1', 'assignedto': '10',
  218                     'priority': '3', 'messages' : [u_m ['10'], u_m ['9']]}):
  219             self.db.issue.create(**issue)
  220         return self.iterSetup(classname)
  221 
  222 
  223 class DBTest(commonDBTest):
  224 
  225     def testRefresh(self):
  226         self.db.refresh_database()
  227 
  228     #
  229     # automatic properties (well, the two easy ones anyway)
  230     #
  231     def testCreatorProperty(self):
  232         i = self.db.issue
  233         id1 = i.create(title='spam')
  234         self.db.journaltag = 'fred'
  235         id2 = i.create(title='spam')
  236         self.assertNotEqual(id1, id2)
  237         self.assertNotEqual(i.get(id1, 'creator'), i.get(id2, 'creator'))
  238 
  239     def testActorProperty(self):
  240         i = self.db.issue
  241         id1 = i.create(title='spam')
  242         self.db.journaltag = 'fred'
  243         i.set(id1, title='asfasd')
  244         self.assertNotEqual(i.get(id1, 'creator'), i.get(id1, 'actor'))
  245 
  246     # ID number controls
  247     def testIDGeneration(self):
  248         id1 = self.db.issue.create(title="spam", status='1')
  249         id2 = self.db.issue.create(title="eggs", status='2')
  250         self.assertNotEqual(id1, id2)
  251     def testIDSetting(self):
  252         # XXX numeric ids
  253         self.db.setid('issue', 10)
  254         id2 = self.db.issue.create(title="eggs", status='2')
  255         self.assertEqual('11', id2)
  256 
  257     #
  258     # basic operations
  259     #
  260     def testEmptySet(self):
  261         id1 = self.db.issue.create(title="spam", status='1')
  262         self.db.issue.set(id1)
  263 
  264     # String
  265     def testStringChange(self):
  266         for commit in (0,1):
  267             # test set & retrieve
  268             nid = self.db.issue.create(title="spam", status='1')
  269             self.assertEqual(self.db.issue.get(nid, 'title'), 'spam')
  270 
  271             # change and make sure we retrieve the correct value
  272             self.db.issue.set(nid, title='eggs')
  273             if commit: self.db.commit()
  274             self.assertEqual(self.db.issue.get(nid, 'title'), 'eggs')
  275 
  276     def testStringUnset(self):
  277         for commit in (0,1):
  278             nid = self.db.issue.create(title="spam", status='1')
  279             if commit: self.db.commit()
  280             self.assertEqual(self.db.issue.get(nid, 'title'), 'spam')
  281             # make sure we can unset
  282             self.db.issue.set(nid, title=None)
  283             if commit: self.db.commit()
  284             self.assertEqual(self.db.issue.get(nid, "title"), None)
  285 
  286     # FileClass "content" property (no unset test)
  287     def testFileClassContentChange(self):
  288         for commit in (0,1):
  289             # test set & retrieve
  290             nid = self.db.file.create(content="spam")
  291             self.assertEqual(self.db.file.get(nid, 'content'), 'spam')
  292 
  293             # change and make sure we retrieve the correct value
  294             self.db.file.set(nid, content='eggs')
  295             if commit: self.db.commit()
  296             self.assertEqual(self.db.file.get(nid, 'content'), 'eggs')
  297 
  298     def testStringUnicode(self):
  299         # test set & retrieve
  300         ustr = u2s(u'\xe4\xf6\xfc\u20ac')
  301         nid = self.db.issue.create(title=ustr, status='1')
  302         self.assertEqual(self.db.issue.get(nid, 'title'), ustr)
  303 
  304         # change and make sure we retrieve the correct value
  305         ustr2 = u2s(u'change \u20ac change')
  306         self.db.issue.set(nid, title=ustr2)
  307         self.db.commit()
  308         self.assertEqual(self.db.issue.get(nid, 'title'), ustr2)
  309 
  310         # test set & retrieve (this time for file contents)
  311         nid = self.db.file.create(content=ustr)
  312         self.assertEqual(self.db.file.get(nid, 'content'), ustr)
  313         self.assertEqual(self.db.file.get(nid, 'binary_content'), s2b(ustr))
  314 
  315     def testStringBinary(self):
  316         ''' Create file with binary content that is not able
  317             to be interpreted as unicode. Try to cause file module
  318             trigger and handle UnicodeDecodeError
  319             and get valid output
  320         '''
  321         # test set & retrieve
  322         bstr = b'\x00\xF0\x34\x33' # random binary data
  323 
  324         # test set & retrieve (this time for file contents)
  325         nid = self.db.file.create(content=bstr)
  326         print(nid)
  327         print(repr(self.db.file.get(nid, 'content')))
  328         print(repr(self.db.file.get(nid, 'binary_content')))
  329         p3val='file1 is not text, retrieve using binary_content property. mdsum: 0e1d1b47e4bd1beab3afc9b79f596c1d'
  330 
  331         if sys.version_info[0] > 2:
  332             # python 3
  333             self.assertEqual(self.db.file.get(nid, 'content'), p3val)
  334             self.assertEqual(self.db.file.get(nid, 'binary_content'),
  335                              bstr)
  336         else:
  337             # python 2
  338             self.assertEqual(self.db.file.get(nid, 'content'), bstr)
  339             self.assertEqual(self.db.file.get(nid, 'binary_content'), bstr)
  340 
  341     # Link
  342     def testLinkChange(self):
  343         self.assertRaises(IndexError, self.db.issue.create, title="spam",
  344             status='100')
  345         for commit in (0,1):
  346             nid = self.db.issue.create(title="spam", status='1')
  347             if commit: self.db.commit()
  348             self.assertEqual(self.db.issue.get(nid, "status"), '1')
  349             self.db.issue.set(nid, status='2')
  350             if commit: self.db.commit()
  351             self.assertEqual(self.db.issue.get(nid, "status"), '2')
  352 
  353     def testLinkUnset(self):
  354         for commit in (0,1):
  355             nid = self.db.issue.create(title="spam", status='1')
  356             if commit: self.db.commit()
  357             self.db.issue.set(nid, status=None)
  358             if commit: self.db.commit()
  359             self.assertEqual(self.db.issue.get(nid, "status"), None)
  360 
  361     # Multilink
  362     def testMultilinkChange(self):
  363         for commit in (0,1):
  364             self.assertRaises(IndexError, self.db.issue.create, title="spam",
  365                 nosy=['foo%s'%commit])
  366             u1 = self.db.user.create(username='foo%s'%commit)
  367             u2 = self.db.user.create(username='bar%s'%commit)
  368             nid = self.db.issue.create(title="spam", nosy=[u1])
  369             if commit: self.db.commit()
  370             self.assertEqual(self.db.issue.get(nid, "nosy"), [u1])
  371             self.db.issue.set(nid, nosy=[])
  372             if commit: self.db.commit()
  373             self.assertEqual(self.db.issue.get(nid, "nosy"), [])
  374             self.db.issue.set(nid, nosy=[u1,u2])
  375             if commit: self.db.commit()
  376             l = [u1,u2]; l.sort()
  377             m = self.db.issue.get(nid, "nosy"); m.sort()
  378             self.assertEqual(l, m)
  379 
  380             # verify that when we pass None to an Multilink it sets
  381             # it to an empty list
  382             self.db.issue.set(nid, nosy=None)
  383             if commit: self.db.commit()
  384             self.assertEqual(self.db.issue.get(nid, "nosy"), [])
  385 
  386     def testMakeSeveralMultilinkedNodes(self):
  387         for commit in (0,1):
  388             u1 = self.db.user.create(username='foo%s'%commit)
  389             u2 = self.db.user.create(username='bar%s'%commit)
  390             u3 = self.db.user.create(username='baz%s'%commit)
  391             nid = self.db.issue.create(title="spam", nosy=[u1])
  392             if commit: self.db.commit()
  393             self.assertEqual(self.db.issue.get(nid, "nosy"), [u1])
  394             self.db.issue.set(nid, deadline=date.Date('.'))
  395             self.db.issue.set(nid, nosy=[u1,u2], title='ta%s'%commit)
  396             if commit: self.db.commit()
  397             self.assertEqual(self.db.issue.get(nid, "nosy"), [u1,u2])
  398             self.db.issue.set(nid, deadline=date.Date('.'))
  399             self.db.issue.set(nid, nosy=[u1,u2,u3], title='tb%s'%commit)
  400             if commit: self.db.commit()
  401             self.assertEqual(self.db.issue.get(nid, "nosy"), [u1,u2,u3])
  402 
  403     def testMultilinkChangeIterable(self):
  404         for commit in (0,1):
  405             # invalid nosy value assertion
  406             self.assertRaises(IndexError, self.db.issue.create, title='spam',
  407                 nosy=['foo%s'%commit])
  408             # invalid type for nosy create
  409             self.assertRaises(TypeError, self.db.issue.create, title='spam',
  410                 nosy=1)
  411             u1 = self.db.user.create(username='foo%s'%commit)
  412             u2 = self.db.user.create(username='bar%s'%commit)
  413             # try a couple of the built-in iterable types to make
  414             # sure that we accept them and handle them properly
  415             # try a set as input for the multilink
  416             nid = self.db.issue.create(title="spam", nosy=set(u1))
  417             if commit: self.db.commit()
  418             self.assertEqual(self.db.issue.get(nid, "nosy"), [u1])
  419             self.assertRaises(TypeError, self.db.issue.set, nid,
  420                 nosy='invalid type')
  421             # test with a tuple
  422             self.db.issue.set(nid, nosy=tuple())
  423             if commit: self.db.commit()
  424             self.assertEqual(self.db.issue.get(nid, "nosy"), [])
  425             # make sure we accept a frozen set
  426             self.db.issue.set(nid, nosy=set([u1,u2]))
  427             if commit: self.db.commit()
  428             l = [u1,u2]; l.sort()
  429             m = self.db.issue.get(nid, "nosy"); m.sort()
  430             self.assertEqual(l, m)
  431 
  432 
  433 # XXX one day, maybe...
  434 #    def testMultilinkOrdering(self):
  435 #        for i in range(10):
  436 #            self.db.user.create(username='foo%s'%i)
  437 #        i = self.db.issue.create(title="spam", nosy=['5','3','12','4'])
  438 #        self.db.commit()
  439 #        l = self.db.issue.get(i, "nosy")
  440 #        # all backends should return the Multilink numeric-id-sorted
  441 #        self.assertEqual(l, ['3', '4', '5', '12'])
  442 
  443     # Date
  444     def testDateChange(self):
  445         self.assertRaises(TypeError, self.db.issue.create,
  446             title='spam', deadline=1)
  447         for commit in (0,1):
  448             nid = self.db.issue.create(title="spam", status='1')
  449             self.assertRaises(TypeError, self.db.issue.set, nid, deadline=1)
  450             a = self.db.issue.get(nid, "deadline")
  451             if commit: self.db.commit()
  452             self.db.issue.set(nid, deadline=date.Date())
  453             b = self.db.issue.get(nid, "deadline")
  454             if commit: self.db.commit()
  455             self.assertNotEqual(a, b)
  456             self.assertNotEqual(b, date.Date('1970-1-1.00:00:00'))
  457             # The 1970 date will fail for metakit -- it is used
  458             # internally for storing NULL. The others would, too
  459             # because metakit tries to convert date.timestamp to an int
  460             # for storing and fails with an overflow.
  461             for d in [date.Date (x) for x in ('2038', '1970', '0033', '9999')]:
  462                 self.db.issue.set(nid, deadline=d)
  463                 if commit: self.db.commit()
  464                 c = self.db.issue.get(nid, "deadline")
  465                 self.assertEqual(c, d)
  466 
  467     def testDateLeapYear(self):
  468         nid = self.db.issue.create(title='spam', status='1',
  469             deadline=date.Date('2008-02-29'))
  470         self.assertEqual(str(self.db.issue.get(nid, 'deadline')),
  471             '2008-02-29.00:00:00')
  472         self.assertEqual(self.db.issue.filter(None,
  473             {'deadline': '2008-02-29'}), [nid])
  474         self.assertEqual(list(self.db.issue.filter_iter(None,
  475             {'deadline': '2008-02-29'})), [nid])
  476         self.db.issue.set(nid, deadline=date.Date('2008-03-01'))
  477         self.assertEqual(str(self.db.issue.get(nid, 'deadline')),
  478             '2008-03-01.00:00:00')
  479         self.assertEqual(self.db.issue.filter(None,
  480             {'deadline': '2008-02-29'}), [])
  481         self.assertEqual(list(self.db.issue.filter_iter(None,
  482             {'deadline': '2008-02-29'})), [])
  483 
  484     def testDateUnset(self):
  485         for commit in (0,1):
  486             nid = self.db.issue.create(title="spam", status='1')
  487             self.db.issue.set(nid, deadline=date.Date())
  488             if commit: self.db.commit()
  489             self.assertNotEqual(self.db.issue.get(nid, "deadline"), None)
  490             self.db.issue.set(nid, deadline=None)
  491             if commit: self.db.commit()
  492             self.assertEqual(self.db.issue.get(nid, "deadline"), None)
  493 
  494     def testDateSort(self):
  495         d1 = date.Date('.')
  496         ae, filter, filter_iter = self.filteringSetup()
  497         nid = self.db.issue.create(title="nodeadline", status='1')
  498         self.db.commit()
  499         for filt in filter, filter_iter:
  500             ae(filt(None, {}, ('+','deadline')), ['5', '2', '1', '3', '4'])
  501             ae(filt(None, {}, ('+','id'), ('+', 'deadline')),
  502                 ['5', '2', '1', '3', '4'])
  503             ae(filt(None, {}, ('-','id'), ('-', 'deadline')),
  504                 ['4', '3', '1', '2', '5'])
  505 
  506     def testDateSortMultilink(self):
  507         d1 = date.Date('.')
  508         ae, filter, filter_iter = self.filteringSetup()
  509         nid = self.db.issue.create(title="nodeadline", status='1')
  510         self.db.commit()
  511         ae(sorted(self.db.issue.get('1','nosy')), [])
  512         ae(sorted(self.db.issue.get('2','nosy')), [])
  513         ae(sorted(self.db.issue.get('3','nosy')), ['1','2'])
  514         ae(sorted(self.db.issue.get('4','nosy')), ['1','2','3'])
  515         ae(sorted(self.db.issue.get('5','nosy')), [])
  516         ae(self.db.user.get('1','username'), 'admin')
  517         ae(self.db.user.get('2','username'), 'fred')
  518         ae(self.db.user.get('3','username'), 'bleep')
  519         # filter_iter currently doesn't work for Multilink sort
  520         # so testing only filter
  521         ae(filter(None, {}, ('+', 'id'), ('+','nosy')),
  522             ['1', '2', '5', '4', '3'])
  523         ae(filter(None, {}, ('+','deadline'), ('+', 'nosy')),
  524             ['5', '2', '1', '4', '3'])
  525         ae(filter(None, {}, ('+','nosy'), ('+', 'deadline')),
  526             ['5', '2', '1', '3', '4'])
  527 
  528     # Interval
  529     def testIntervalChange(self):
  530         self.assertRaises(TypeError, self.db.issue.create,
  531             title='spam', foo=1)
  532         for commit in (0,1):
  533             nid = self.db.issue.create(title="spam", status='1')
  534             self.assertRaises(TypeError, self.db.issue.set, nid, foo=1)
  535             if commit: self.db.commit()
  536             a = self.db.issue.get(nid, "foo")
  537             i = date.Interval('-1d')
  538             self.db.issue.set(nid, foo=i)
  539             if commit: self.db.commit()
  540             self.assertNotEqual(self.db.issue.get(nid, "foo"), a)
  541             self.assertEqual(i, self.db.issue.get(nid, "foo"))
  542             j = date.Interval('1y')
  543             self.db.issue.set(nid, foo=j)
  544             if commit: self.db.commit()
  545             self.assertNotEqual(self.db.issue.get(nid, "foo"), i)
  546             self.assertEqual(j, self.db.issue.get(nid, "foo"))
  547 
  548     def testIntervalUnset(self):
  549         for commit in (0,1):
  550             nid = self.db.issue.create(title="spam", status='1')
  551             self.db.issue.set(nid, foo=date.Interval('-1d'))
  552             if commit: self.db.commit()
  553             self.assertNotEqual(self.db.issue.get(nid, "foo"), None)
  554             self.db.issue.set(nid, foo=None)
  555             if commit: self.db.commit()
  556             self.assertEqual(self.db.issue.get(nid, "foo"), None)
  557 
  558     # Boolean
  559     def testBooleanSet(self):
  560         nid = self.db.user.create(username='one', assignable=1)
  561         self.assertEqual(self.db.user.get(nid, "assignable"), 1)
  562         nid = self.db.user.create(username='two', assignable=0)
  563         self.assertEqual(self.db.user.get(nid, "assignable"), 0)
  564 
  565     def testBooleanChange(self):
  566         userid = self.db.user.create(username='foo', assignable=1)
  567         self.assertEqual(1, self.db.user.get(userid, 'assignable'))
  568         self.db.user.set(userid, assignable=0)
  569         self.assertEqual(self.db.user.get(userid, 'assignable'), 0)
  570         self.db.user.set(userid, assignable=1)
  571         self.assertEqual(self.db.user.get(userid, 'assignable'), 1)
  572 
  573     def testBooleanUnset(self):
  574         nid = self.db.user.create(username='foo', assignable=1)
  575         self.db.user.set(nid, assignable=None)
  576         self.assertEqual(self.db.user.get(nid, "assignable"), None)
  577 
  578     # Number
  579     def testNumberChange(self):
  580         nid = self.db.user.create(username='foo', age=1)
  581         self.assertEqual(1, self.db.user.get(nid, 'age'))
  582         self.db.user.set(nid, age=3)
  583         self.assertNotEqual(self.db.user.get(nid, 'age'), 1)
  584         self.db.user.set(nid, age=1.0)
  585         self.assertEqual(self.db.user.get(nid, 'age'), 1)
  586         self.db.user.set(nid, age=0)
  587         self.assertEqual(self.db.user.get(nid, 'age'), 0)
  588 
  589         nid = self.db.user.create(username='bar', age=0)
  590         self.assertEqual(self.db.user.get(nid, 'age'), 0)
  591 
  592     def testNumberUnset(self):
  593         nid = self.db.user.create(username='foo', age=1)
  594         self.db.user.set(nid, age=None)
  595         self.assertEqual(self.db.user.get(nid, "age"), None)
  596 
  597     # Long number
  598     def testDoubleChange(self):
  599         lnl = 100.12345678
  600         ln  = 100.123456789
  601         lng = 100.12345679
  602         nid = self.db.user.create(username='foo', longnumber=ln)
  603         self.assertEqual(self.db.user.get(nid, 'longnumber') < lng, True)
  604         self.assertEqual(self.db.user.get(nid, 'longnumber') > lnl, True)
  605         lnl = 1.0012345678e55
  606         ln  = 1.00123456789e55
  607         lng = 1.0012345679e55
  608         self.db.user.set(nid, longnumber=ln)
  609         self.assertEqual(self.db.user.get(nid, 'longnumber') < lng, True)
  610         self.assertEqual(self.db.user.get(nid, 'longnumber') > lnl, True)
  611         self.db.user.set(nid, longnumber=-1)
  612         self.assertEqual(self.db.user.get(nid, 'longnumber'), -1)
  613         self.db.user.set(nid, longnumber=0)
  614         self.assertEqual(self.db.user.get(nid, 'longnumber'), 0)
  615 
  616         nid = self.db.user.create(username='bar', longnumber=0)
  617         self.assertEqual(self.db.user.get(nid, 'longnumber'), 0)
  618 
  619     def testDoubleUnset(self):
  620         nid = self.db.user.create(username='foo', longnumber=1.2345)
  621         self.db.user.set(nid, longnumber=None)
  622         self.assertEqual(self.db.user.get(nid, "longnumber"), None)
  623 
  624 
  625     # Integer
  626     def testIntegerChange(self):
  627         nid = self.db.user.create(username='foo', rating=100)
  628         self.assertEqual(100, self.db.user.get(nid, 'rating'))
  629         self.db.user.set(nid, rating=300)
  630         self.assertNotEqual(self.db.user.get(nid, 'rating'), 100)
  631         self.db.user.set(nid, rating=-1)
  632         self.assertEqual(self.db.user.get(nid, 'rating'), -1)
  633         self.db.user.set(nid, rating=0)
  634         self.assertEqual(self.db.user.get(nid, 'rating'), 0)
  635 
  636         nid = self.db.user.create(username='bar', rating=0)
  637         self.assertEqual(self.db.user.get(nid, 'rating'), 0)
  638 
  639     def testIntegerUnset(self):
  640         nid = self.db.user.create(username='foo', rating=1)
  641         self.db.user.set(nid, rating=None)
  642         self.assertEqual(self.db.user.get(nid, "rating"), None)
  643 
  644     # Password
  645     def testPasswordChange(self):
  646         x = password.Password('x')
  647         userid = self.db.user.create(username='foo', password=x)
  648         self.assertEqual(x, self.db.user.get(userid, 'password'))
  649         self.assertEqual(self.db.user.get(userid, 'password'), 'x')
  650         y = password.Password('y')
  651         self.db.user.set(userid, password=y)
  652         self.assertEqual(self.db.user.get(userid, 'password'), 'y')
  653         self.assertRaises(TypeError, self.db.user.create, userid,
  654             username='bar', password='x')
  655         self.assertRaises(TypeError, self.db.user.set, userid, password='x')
  656 
  657     def testPasswordUnset(self):
  658         x = password.Password('x')
  659         nid = self.db.user.create(username='foo', password=x)
  660         self.db.user.set(nid, assignable=None)
  661         self.assertEqual(self.db.user.get(nid, "assignable"), None)
  662 
  663     # key value
  664     def testKeyValue(self):
  665         self.assertRaises(ValueError, self.db.user.create)
  666 
  667         newid = self.db.user.create(username="spam")
  668         self.assertEqual(self.db.user.lookup('spam'), newid)
  669         self.db.commit()
  670         self.assertEqual(self.db.user.lookup('spam'), newid)
  671         self.db.user.retire(newid)
  672         self.assertRaises(KeyError, self.db.user.lookup, 'spam')
  673 
  674         # use the key again now that the old is retired
  675         newid2 = self.db.user.create(username="spam")
  676         self.assertNotEqual(newid, newid2)
  677         # try to restore old node. this shouldn't succeed!
  678         self.assertRaises(KeyError, self.db.user.restore, newid)
  679 
  680         self.assertRaises(TypeError, self.db.issue.lookup, 'fubar')
  681 
  682     # label property
  683     def testLabelProp(self):
  684         # key prop
  685         self.assertEqual(self.db.status.labelprop(), 'name')
  686         self.assertEqual(self.db.user.labelprop(), 'username')
  687         # title
  688         self.assertEqual(self.db.issue.labelprop(), 'title')
  689         # name
  690         self.assertEqual(self.db.file.labelprop(), 'name')
  691         # id
  692         self.assertEqual(self.db.stuff.labelprop(default_to_id=1), 'id')
  693 
  694     # retirement
  695     def testRetire(self):
  696         self.db.issue.create(title="spam", status='1')
  697         b = self.db.status.get('1', 'name')
  698         a = self.db.status.list()
  699         nodeids = self.db.status.getnodeids()
  700         self.db.status.retire('1')
  701         others = nodeids[:]
  702         others.remove('1')
  703 
  704         self.assertEqual(set(self.db.status.getnodeids()),
  705             set(nodeids))
  706         self.assertEqual(set(self.db.status.getnodeids(retired=True)),
  707             set(['1']))
  708         self.assertEqual(set(self.db.status.getnodeids(retired=False)),
  709             set(others))
  710 
  711         self.assertTrue(self.db.status.is_retired('1'))
  712 
  713         # make sure the list is different
  714         self.assertNotEqual(a, self.db.status.list())
  715 
  716         # can still access the node if necessary
  717         self.assertEqual(self.db.status.get('1', 'name'), b)
  718         self.assertRaises(IndexError, self.db.status.set, '1', name='hello')
  719         self.db.commit()
  720         self.assertTrue(self.db.status.is_retired('1'))
  721         self.assertEqual(self.db.status.get('1', 'name'), b)
  722         self.assertNotEqual(a, self.db.status.list())
  723 
  724         # try to restore retired node
  725         self.db.status.restore('1')
  726 
  727         self.assertTrue(not self.db.status.is_retired('1'))
  728 
  729     def testCacheCreateSet(self):
  730         self.db.issue.create(title="spam", status='1')
  731         a = self.db.issue.get('1', 'title')
  732         self.assertEqual(a, 'spam')
  733         self.db.issue.set('1', title='ham')
  734         b = self.db.issue.get('1', 'title')
  735         self.assertEqual(b, 'ham')
  736 
  737     def testSerialisation(self):
  738         nid = self.db.issue.create(title="spam", status='1',
  739             deadline=date.Date(), foo=date.Interval('-1d'))
  740         self.db.commit()
  741         assert isinstance(self.db.issue.get(nid, 'deadline'), date.Date)
  742         assert isinstance(self.db.issue.get(nid, 'foo'), date.Interval)
  743         uid = self.db.user.create(username="fozzy",
  744             password=password.Password('t. bear'))
  745         self.db.commit()
  746         assert isinstance(self.db.user.get(uid, 'password'), password.Password)
  747 
  748     def testTransactions(self):
  749         # remember the number of items we started
  750         num_issues = len(self.db.issue.list())
  751         num_files = self.db.numfiles()
  752         self.db.issue.create(title="don't commit me!", status='1')
  753         self.assertNotEqual(num_issues, len(self.db.issue.list()))
  754         self.db.rollback()
  755         self.assertEqual(num_issues, len(self.db.issue.list()))
  756         self.db.issue.create(title="please commit me!", status='1')
  757         self.assertNotEqual(num_issues, len(self.db.issue.list()))
  758         self.db.commit()
  759         self.assertNotEqual(num_issues, len(self.db.issue.list()))
  760         self.db.rollback()
  761         self.assertNotEqual(num_issues, len(self.db.issue.list()))
  762         self.db.file.create(name="test", type="text/plain", content="hi")
  763         self.db.rollback()
  764         self.assertEqual(num_files, self.db.numfiles())
  765         for i in range(10):
  766             self.db.file.create(name="test", type="text/plain",
  767                     content="hi %d"%(i))
  768             self.db.commit()
  769         num_files2 = self.db.numfiles()
  770         self.assertNotEqual(num_files, num_files2)
  771         self.db.file.create(name="test", type="text/plain", content="hi")
  772         self.db.rollback()
  773         self.assertNotEqual(num_files, self.db.numfiles())
  774         self.assertEqual(num_files2, self.db.numfiles())
  775 
  776         # rollback / cache interaction
  777         name1 = self.db.user.get('1', 'username')
  778         self.db.user.set('1', username = name1+name1)
  779         # get the prop so the info's forced into the cache (if there is one)
  780         self.db.user.get('1', 'username')
  781         self.db.rollback()
  782         name2 = self.db.user.get('1', 'username')
  783         self.assertEqual(name1, name2)
  784 
  785     def testDestroyBlob(self):
  786         # destroy an uncommitted blob
  787         f1 = self.db.file.create(content='hello', type="text/plain")
  788         self.db.commit()
  789         fn = self.db.filename('file', f1)
  790         self.db.file.destroy(f1)
  791         self.db.commit()
  792         self.assertEqual(os.path.exists(fn), False)
  793 
  794     def testDestroyNoJournalling(self):
  795         self.innerTestDestroy(klass=self.db.session)
  796 
  797     def testDestroyJournalling(self):
  798         self.innerTestDestroy(klass=self.db.issue)
  799 
  800     def innerTestDestroy(self, klass):
  801         newid = klass.create(title='Mr Friendly')
  802         n = len(klass.list())
  803         self.assertEqual(klass.get(newid, 'title'), 'Mr Friendly')
  804         count = klass.count()
  805         klass.destroy(newid)
  806         self.assertNotEqual(count, klass.count())
  807         self.assertRaises(IndexError, klass.get, newid, 'title')
  808         self.assertNotEqual(len(klass.list()), n)
  809         if klass.do_journal:
  810             self.assertRaises(IndexError, klass.history, newid)
  811 
  812         # now with a commit
  813         newid = klass.create(title='Mr Friendly')
  814         n = len(klass.list())
  815         self.assertEqual(klass.get(newid, 'title'), 'Mr Friendly')
  816         self.db.commit()
  817         count = klass.count()
  818         klass.destroy(newid)
  819         self.assertNotEqual(count, klass.count())
  820         self.assertRaises(IndexError, klass.get, newid, 'title')
  821         self.db.commit()
  822         self.assertRaises(IndexError, klass.get, newid, 'title')
  823         self.assertNotEqual(len(klass.list()), n)
  824         if klass.do_journal:
  825             self.assertRaises(IndexError, klass.history, newid)
  826 
  827         # now with a rollback
  828         newid = klass.create(title='Mr Friendly')
  829         n = len(klass.list())
  830         self.assertEqual(klass.get(newid, 'title'), 'Mr Friendly')
  831         self.db.commit()
  832         count = klass.count()
  833         klass.destroy(newid)
  834         self.assertNotEqual(len(klass.list()), n)
  835         self.assertRaises(IndexError, klass.get, newid, 'title')
  836         self.db.rollback()
  837         self.assertEqual(count, klass.count())
  838         self.assertEqual(klass.get(newid, 'title'), 'Mr Friendly')
  839         self.assertEqual(len(klass.list()), n)
  840         if klass.do_journal:
  841             self.assertNotEqual(klass.history(newid), [])
  842 
  843     def testExceptions(self):
  844         # this tests the exceptions that should be raised
  845         ar = self.assertRaises
  846 
  847         ar(KeyError, self.db.getclass, 'fubar')
  848 
  849         #
  850         # class create
  851         #
  852         # string property
  853         ar(TypeError, self.db.status.create, name=1)
  854         # id, creation, creator and activity properties are reserved
  855         ar(KeyError, self.db.status.create, id=1)
  856         ar(KeyError, self.db.status.create, creation=1)
  857         ar(KeyError, self.db.status.create, creator=1)
  858         ar(KeyError, self.db.status.create, activity=1)
  859         ar(KeyError, self.db.status.create, actor=1)
  860         # invalid property name
  861         ar(KeyError, self.db.status.create, foo='foo')
  862         # key name clash
  863         ar(ValueError, self.db.status.create, name='unread')
  864         # invalid link index
  865         ar(IndexError, self.db.issue.create, title='foo', status='bar')
  866         # invalid link value
  867         ar(ValueError, self.db.issue.create, title='foo', status=1)
  868         # invalid multilink type
  869         ar(TypeError, self.db.issue.create, title='foo', status='1',
  870             nosy='hello')
  871         # invalid multilink index type
  872         ar(ValueError, self.db.issue.create, title='foo', status='1',
  873             nosy=[1])
  874         # invalid multilink index
  875         ar(IndexError, self.db.issue.create, title='foo', status='1',
  876             nosy=['10'])
  877 
  878         #
  879         # key property
  880         #
  881         # key must be a String
  882         ar(TypeError, self.db.file.setkey, 'fooz')
  883         # key must exist
  884         ar(KeyError, self.db.file.setkey, 'fubar')
  885 
  886         #
  887         # class get
  888         #
  889         # invalid node id
  890         ar(IndexError, self.db.issue.get, '99', 'title')
  891         # invalid property name
  892         ar(KeyError, self.db.status.get, '2', 'foo')
  893 
  894         #
  895         # class set
  896         #
  897         # invalid node id
  898         ar(IndexError, self.db.issue.set, '99', title='foo')
  899         # invalid property name
  900         ar(KeyError, self.db.status.set, '1', foo='foo')
  901         # string property
  902         ar(TypeError, self.db.status.set, '1', name=1)
  903         # key name clash
  904         ar(ValueError, self.db.status.set, '2', name='unread')
  905         # set up a valid issue for me to work on
  906         id = self.db.issue.create(title="spam", status='1')
  907         # invalid link index
  908         ar(IndexError, self.db.issue.set, id, title='foo', status='bar')
  909         # invalid link value
  910         ar(ValueError, self.db.issue.set, id, title='foo', status=1)
  911         # invalid multilink type
  912         ar(TypeError, self.db.issue.set, id, title='foo', status='1',
  913             nosy='hello')
  914         # invalid multilink index type
  915         ar(ValueError, self.db.issue.set, id, title='foo', status='1',
  916             nosy=[1])
  917         # invalid multilink index
  918         ar(IndexError, self.db.issue.set, id, title='foo', status='1',
  919             nosy=['10'])
  920         # NOTE: the following increment the username to avoid problems
  921         # within metakit's backend (it creates the node, and then sets the
  922         # info, so the create (and by a fluke the username set) go through
  923         # before the age/assignable/etc. set, which raises the exception)
  924         # invalid number value
  925         ar(TypeError, self.db.user.create, username='foo', age='a')
  926         # invalid boolean value
  927         ar(TypeError, self.db.user.create, username='foo2', assignable='true')
  928         nid = self.db.user.create(username='foo3')
  929         # invalid number value
  930         ar(TypeError, self.db.user.set, nid, age='a')
  931         # invalid boolean value
  932         ar(TypeError, self.db.user.set, nid, assignable='true')
  933 
  934     def testAuditors(self):
  935         class test:
  936             called = False
  937             def call(self, *args): self.called = True
  938         create = test()
  939 
  940         self.db.user.audit('create', create.call)
  941         self.db.user.create(username="mary")
  942         self.assertEqual(create.called, True)
  943 
  944         set = test()
  945         self.db.user.audit('set', set.call)
  946         self.db.user.set('1', username="joe")
  947         self.assertEqual(set.called, True)
  948 
  949         retire = test()
  950         self.db.user.audit('retire', retire.call)
  951         self.db.user.retire('1')
  952         self.assertEqual(retire.called, True)
  953 
  954     def testAuditorTwo(self):
  955         class test:
  956             n = 0
  957             def a(self, *args): self.call_a = self.n; self.n += 1
  958             def b(self, *args): self.call_b = self.n; self.n += 1
  959             def c(self, *args): self.call_c = self.n; self.n += 1
  960         test = test()
  961         self.db.user.audit('create', test.b, 1)
  962         self.db.user.audit('create', test.a, 1)
  963         self.db.user.audit('create', test.c, 2)
  964         self.db.user.create(username="mary")
  965         self.assertEqual(test.call_a, 0)
  966         self.assertEqual(test.call_b, 1)
  967         self.assertEqual(test.call_c, 2)
  968 
  969     def testDefault_Value(self):
  970         new_issue=self.db.issue.create(title="title", deadline=date.Date('2016-6-30.22:39'))
  971 
  972         # John Rouillard claims this should return the default value of 1 week for foo,
  973         # but the hyperdb doesn't assign the default value for missing properties in the
  974         # db on creation.
  975         result=self.db.issue.get(new_issue, 'foo')
  976         # When the defaultis automatically set by the hyperdb, change this to
  977         # match the Interval test below.
  978         self.assertEqual(result, None)
  979 
  980         # but verify that the default value is retreivable
  981         result=self.db.issue.properties['foo'].get_default_value()
  982         self.assertEqual(result, date.Interval('-7d'))
  983 
  984     def testQuietProperty(self):
  985         # make sure that the quiet properties: "assignable" and "age" are not
  986         # returned as part of the proplist
  987         new_user=self.db.user.create(username="pete", age=10, assignable=False)
  988         new_issue=self.db.issue.create(title="title", deadline=date.Date('2016-6-30.22:39'))
  989         # change all quiet params. Verify they aren't returned in object.
  990         # between this and the issue class every type represented in hyperdb
  991         # should be initalized with a quiet parameter.
  992         result=self.db.user.set(new_user, username="new", age=20, supervisor='3', assignable=True,
  993                                 password=password.Password("3456"), rating=4, realname="newname")
  994         self.assertEqual(result, {'supervisor': '3', 'username': "new"})
  995         result=self.db.user.get(new_user, 'age')
  996         self.assertEqual(result, 20)
  997 
  998         # change all quiet params. Verify they aren't returned in object.
  999         result=self.db.issue.set(new_issue, title="title2", deadline=date.Date('2016-7-13.22:39'),
 1000                                  assignedto="2", nosy=["3", "2"])
 1001         self.assertEqual(result, {'title': 'title2'})
 1002 
 1003         # also test that we can make a property noisy
 1004         self.db.user.properties['age'].quiet=False
 1005         result=self.db.user.set(new_user, username="old", age=30, supervisor='2', assignable=False)
 1006         self.assertEqual(result, {'age': 30, 'supervisor': '2', 'username': "old"})
 1007         self.db.user.properties['age'].quiet=True
 1008 
 1009     def testQuietChangenote(self):
 1010         # create user 3 for later use
 1011         self.db.user.create(username="pete", age=10, assignable=False)
 1012 
 1013         new_issue=self.db.issue.create(title="title", deadline=date.Date('2016-6-30.22:39'))
 1014 
 1015         # change all quiet params. Verify they aren't returned in CreateNote.
 1016         result=self.db.issue.set(new_issue, title="title2", deadline=date.Date('2016-6-30.22:39'),
 1017                                  assignedto="2", nosy=["3", "2"])
 1018         result=self.db.issue.generateCreateNote(new_issue)
 1019         self.assertEqual(result, '\n----------\ntitle: title2')
 1020 
 1021         # also test that we can make a property noisy
 1022         self.db.issue.properties['nosy'].quiet=False
 1023         self.db.issue.properties['deadline'].quiet=False
 1024         result=self.db.issue.set(new_issue, title="title2", deadline=date.Date('2016-7-13.22:39'),
 1025                                  assignedto="2", nosy=["1", "2"])
 1026         result=self.db.issue.generateCreateNote(new_issue)
 1027         self.assertEqual(result, '\n----------\ndeadline: 2016-07-13.22:39:00\nnosy: admin, fred\ntitle: title2')
 1028         self.db.issue.properties['nosy'].quiet=True
 1029         self.db.issue.properties['deadline'].quiet=True
 1030 
 1031     def testViewPremJournal(self):
 1032         pass
 1033 
 1034     def testQuietJournal(self):
 1035         ## This is an example of how to enable logging module
 1036         ## and report the results. It uses testfixtures
 1037         ## that can be installed via pip.
 1038         ## Uncomment below 2 lines:
 1039         #import logging
 1040         #from testfixtures import LogCapture
 1041         ## then run every call to roundup functions with:
 1042         #with LogCapture('roundup.hyperdb', level=logging.DEBUG) as l:
 1043         #    result=self.db.user.history('2')
 1044         #print l
 1045         ## change 'roundup.hyperdb' to the logging name you want to capture.
 1046         ## print l just prints the output. Run using:
 1047         ## ./run_tests.py --capture=no -k testQuietJournal test/test_anydbm.py
 1048 
 1049         # FIXME There should be a test via
 1050         # template.py::_HTMLItem::history() and verify the output.
 1051         # not sure how to get there from here. -- rouilj
 1052 
 1053         # The Class::history() method now does filtering of quiet
 1054         # props. Make sure that the quiet properties: "assignable"
 1055         # and "age" are not returned as part of the journal
 1056         new_user=self.db.user.create(username="pete", age=10, assignable=False)
 1057         new_issue=self.db.issue.create(title="title", deadline=date.Date('2016-6-30.22:39'))
 1058 
 1059         # change all quiet params. Verify they aren't returned in journal.
 1060         # between this and the issue class every type represented in hyperdb
 1061         # should be initalized with a quiet parameter.
 1062         result=self.db.user.set(new_user, username="new", age=20,
 1063                                 supervisor='1', assignable=True,
 1064                                 password=password.Password("3456"),
 1065                                 rating=4, realname="newname")
 1066         result=self.db.user.history(new_user, skipquiet=False)
 1067         '''
 1068         [('3', <Date 2017-04-14.02:12:20.922>, '1', 'create', {}),
 1069          ('3', <Date 2017-04-14.02:12:20.922>, '1', 'set',
 1070            {'username': 'pete', 'assignable': False,
 1071             'supervisor': None, 'realname': None, 'rating': None,
 1072             'age': 10, 'password': None})]
 1073         '''
 1074         expected = {'username': 'pete', 'assignable': False,
 1075             'supervisor': None, 'realname': None, 'rating': None,
 1076             'age': 10, 'password': None}
 1077 
 1078         result.sort()
 1079         (id, tx_date, user, action, args) = result[-1]
 1080         # check piecewise ignoring date of transaction
 1081         self.assertEqual('3', id)
 1082         self.assertEqual('1', user)
 1083         self.assertEqual('set', action)
 1084         self.assertEqual(expected, args)
 1085 
 1086         # change all quiet params on issue.
 1087         result=self.db.issue.set(new_issue, title="title2",
 1088                                  deadline=date.Date('2016-07-30.22:39'),
 1089                                  assignedto="2", nosy=["3", "2"])
 1090         result=self.db.issue.generateCreateNote(new_issue)
 1091         self.assertEqual(result, '\n----------\ntitle: title2')
 1092 
 1093         # check history including quiet properties
 1094         result=self.db.issue.history(new_issue, skipquiet=False)
 1095         print(result)
 1096         ''' output should be like:
 1097              [ ... ('1', <Date 2017-04-14.01:41:08.466>, '1', 'set',
 1098                  {'assignedto': None, 'nosy': (('+', ['3', '2']),),
 1099                      'deadline': <Date 2016-06-30.22:39:00.000>,
 1100                      'title': 'title'})
 1101         '''
 1102         expected = {'assignedto': None,
 1103                     'nosy': (('+', ['3', '2']),),
 1104                     'deadline': date.Date('2016-06-30.22:39'),
 1105                     'title': 'title'}
 1106 
 1107         result.sort()
 1108         print("history include quiet props", result[-1])
 1109         (id, tx_date, user, action, args) = result[-1]
 1110         # check piecewise ignoring date of transaction
 1111         self.assertEqual('1', id)
 1112         self.assertEqual('1', user)
 1113         self.assertEqual('set', action)
 1114         self.assertEqual(expected, args)
 1115 
 1116         # check history removing quiet properties
 1117         result=self.db.issue.history(new_issue)
 1118         ''' output should be like:
 1119              [ ... ('1', <Date 2017-04-14.01:41:08.466>, '1', 'set',
 1120                  {'title': 'title'})
 1121         '''
 1122         expected = {'title': 'title'}
 1123 
 1124         result.sort()
 1125         print("history remove quiet props", result[-1])
 1126         (id, tx_date, user, action, args) = result[-1]
 1127         # check piecewise
 1128         self.assertEqual('1', id)
 1129         self.assertEqual('1', user)
 1130         self.assertEqual('set', action)
 1131         self.assertEqual(expected, args)
 1132 
 1133         # also test that we can make a property noisy
 1134         self.db.issue.properties['nosy'].quiet=False
 1135         self.db.issue.properties['deadline'].quiet=False
 1136 
 1137         # FIXME: mysql use should be fixed or
 1138         # a different way of checking this should be done.
 1139         # this sleep is a hack.
 1140         # mysql transation timestamps are in whole
 1141         # seconds. To get the history to sort in proper
 1142         # order by using timestamps we have to sleep 2 seconds
 1143         # here tomake sure the timestamp between this transaction
 1144         # and the last transaction is at least 1 second apart.
 1145         import time; time.sleep(2)
 1146         result=self.db.issue.set(new_issue, title="title2",
 1147                                  deadline=date.Date('2016-7-13.22:39'),
 1148                                  assignedto="2", nosy=["1", "2"])
 1149         result=self.db.issue.generateCreateNote(new_issue)
 1150         self.assertEqual(result, '\n----------\ndeadline: 2016-07-13.22:39:00\nnosy: admin, fred\ntitle: title2')
 1151 
 1152 
 1153         # check history removing the current quiet properties
 1154         result=self.db.issue.history(new_issue)
 1155         expected = {'nosy': (('+', ['1']), ('-', ['3'])),
 1156                     'deadline': date.Date("2016-07-30.22:39:00.000")}
 1157 
 1158         result.sort()
 1159         print("result unquiet", result)
 1160         (id, tx_date, user, action, args) = result[-1]
 1161         # check piecewise
 1162         self.assertEqual('1', id)
 1163         self.assertEqual('1', user)
 1164         self.assertEqual('set', action)
 1165         self.assertEqual(expected, args)
 1166 
 1167         result=self.db.user.history('2')
 1168         result.sort()
 1169 
 1170         # result should look like:
 1171         #  [('2', <Date 2017-08-29.01:42:40.227>, '1', 'create', {}),
 1172         #   ('2', <Date 2017-08-29.01:42:44.283>, '1', 'link',
 1173         #      ('issue', '1', 'nosy')) ]
 1174 
 1175         expected2 = ('issue', '1', 'nosy')
 1176 
 1177         (id, tx_date, user, action, args) = result[-1]
 1178 
 1179         self.assertEqual(len(result),2)
 1180 
 1181         self.assertEqual('2', id)
 1182         self.assertEqual('1', user)
 1183         self.assertEqual('link', action)
 1184         self.assertEqual(expected2, args)
 1185 
 1186         # reset quiet props
 1187         self.db.issue.properties['nosy'].quiet=True
 1188         self.db.issue.properties['deadline'].quiet=True
 1189 
 1190         # Change the role for the new_user.
 1191         # If journal is retrieved by admin this adds the role
 1192         # change as the last element. If retreived by non-admin
 1193         # it should not be returned because the user has no
 1194         # View permissons on role.
 1195         # FIXME delay by two seconds due to mysql missing
 1196         # fractional seconds. See sleep above for details
 1197         time.sleep(2)
 1198         result=self.db.user.set(new_user, roles="foo, bar")
 1199 
 1200         # Verify last journal entry as admin is a role change
 1201         # from None
 1202         result=self.db.user.history(new_user, skipquiet=False)
 1203         result.sort()
 1204         ''' result should end like:
 1205           [ ...
 1206           ('3', <Date 2017-04-15.02:06:11.482>, '1', 'set',
 1207                 {'username': 'pete', 'assignable': False,
 1208                  'supervisor': None, 'realname': None,
 1209                   'rating': None, 'age': 10, 'password': None}),
 1210           ('3', <Date 2017-04-15.02:06:11.482>, '1', 'link',
 1211                 ('issue', '1', 'nosy')),
 1212           ('3', <Date 2017-04-15.02:06:11.482>, '1', 'unlink',
 1213                 ('issue', '1', 'nosy')),
 1214           ('3', <Date 2017-04-15.02:06:11.482>, '1', 'set',
 1215              {'roles': None})]
 1216         '''
 1217         (id, tx_date, user, action, args) = result[-1]
 1218         expected = {'roles': None }
 1219 
 1220         self.assertEqual('3', id)
 1221         self.assertEqual('1', user)
 1222         self.assertEqual('set', action)
 1223         self.assertEqual(expected, args)
 1224 
 1225         # set an existing user's role to User so it can
 1226         # view some props of the user class (search backwards
 1227         # for QuietJournal to see the properties, they should be:
 1228         # 'username', 'supervisor', 'assignable' i.e. age is not
 1229         # one of them.
 1230         id = self.db.user.lookup("fred")
 1231         # FIXME mysql timestamp issue see sleeps above
 1232         time.sleep(2)
 1233         result=self.db.user.set(id, roles="User")
 1234         # make the user fred current.
 1235         self.db.setCurrentUser('fred')
 1236         self.assertEqual(self.db.getuid(), id)
 1237 
 1238         # check history as the user fred
 1239         #   include quiet properties
 1240         #   but require View perms
 1241         result=self.db.user.history(new_user, skipquiet=False)
 1242         result.sort()
 1243         ''' result should look like
 1244         [('3', <Date 2017-04-15.01:43:26.911>, '1', 'create', {}),
 1245         ('3', <Date 2017-04-15.01:43:26.911>, '1', 'set',
 1246             {'username': 'pete', 'assignable': False,
 1247               'supervisor': None, 'age': 10})]
 1248         '''
 1249         # analyze last item
 1250         (id, tx_date, user, action, args) = result[-1]
 1251         expected= {'username': 'pete', 'assignable': False,
 1252                    'supervisor': None}
 1253 
 1254         self.assertEqual('3', id)
 1255         self.assertEqual('1', user)
 1256         self.assertEqual('set', action)
 1257         self.assertEqual(expected, args)
 1258 
 1259         # reset the user to admin
 1260         self.db.setCurrentUser('admin')
 1261         self.assertEqual(self.db.getuid(), '1') # admin is always 1
 1262 
 1263     def testJournals(self):
 1264         muid = self.db.user.create(username="mary")
 1265         self.db.user.create(username="pete")
 1266         self.db.issue.create(title="spam", status='1')
 1267         self.db.commit()
 1268 
 1269         # journal entry for issue create
 1270         journal = self.db.getjournal('issue', '1')
 1271         self.assertEqual(1, len(journal))
 1272         (nodeid, date_stamp, journaltag, action, params) = journal[0]
 1273         self.assertEqual(nodeid, '1')
 1274         self.assertEqual(journaltag, self.db.user.lookup('admin'))
 1275         self.assertEqual(action, 'create')
 1276         keys = sorted(params.keys())
 1277         self.assertEqual(keys, [])
 1278 
 1279         # journal entry for link
 1280         journal = self.db.getjournal('user', '1')
 1281         self.assertEqual(1, len(journal))
 1282         self.db.issue.set('1', assignedto='1')
 1283         self.db.commit()
 1284         journal = self.db.getjournal('user', '1')
 1285         self.assertEqual(2, len(journal))
 1286         (nodeid, date_stamp, journaltag, action, params) = journal[1]
 1287         self.assertEqual('1', nodeid)
 1288         self.assertEqual('1', journaltag)
 1289         self.assertEqual('link', action)
 1290         self.assertEqual(('issue', '1', 'assignedto'), params)
 1291 
 1292         # wait a bit to keep proper order of journal entries
 1293         time.sleep(0.01)
 1294         # journal entry for unlink
 1295         self.db.setCurrentUser('mary')
 1296         self.db.issue.set('1', assignedto='2')
 1297         self.db.commit()
 1298         journal = self.db.getjournal('user', '1')
 1299         self.assertEqual(3, len(journal))
 1300         (nodeid, date_stamp, journaltag, action, params) = journal[2]
 1301         self.assertEqual('1', nodeid)
 1302         self.assertEqual(muid, journaltag)
 1303         self.assertEqual('unlink', action)
 1304         self.assertEqual(('issue', '1', 'assignedto'), params)
 1305 
 1306         # test disabling journalling
 1307         # ... get the last entry
 1308         jlen = len(self.db.getjournal('user', '1'))
 1309         self.db.issue.disableJournalling()
 1310         self.db.issue.set('1', title='hello world')
 1311         self.db.commit()
 1312         # see if the change was journalled when it shouldn't have been
 1313         self.assertEqual(jlen,  len(self.db.getjournal('user', '1')))
 1314         jlen = len(self.db.getjournal('issue', '1'))
 1315         self.db.issue.enableJournalling()
 1316         self.db.issue.set('1', title='hello world 2')
 1317         self.db.commit()
 1318         # see if the change was journalled
 1319         self.assertNotEqual(jlen,  len(self.db.getjournal('issue', '1')))
 1320 
 1321     def testJournalNonexistingProperty(self):
 1322         # Test for non-existing properties, link/unlink events to
 1323         # non-existing classes and link/unlink events to non-existing
 1324         # properties in a class: These all may be the result of a schema
 1325         # change and should not lead to a traceback.
 1326         self.db.user.create(username="mary", roles="User")
 1327         id = self.db.issue.create(title="spam", status='1')
 1328         # FIXME delay by two seconds due to mysql missing
 1329         # fractional seconds. This keeps the journal order correct.
 1330         time.sleep(2)
 1331         self.db.issue.set(id, title='green eggs')
 1332         time.sleep(2)
 1333         self.db.commit()
 1334         journal = self.db.getjournal('issue', id)
 1335         now     = date.Date('.')
 1336         sec     = date.Interval('0:00:01')
 1337         sec2    = date.Interval('0:00:02')
 1338         jp0 = dict(title = 'spam')
 1339         # Non-existing property changed
 1340         jp1 = dict(nonexisting = None)
 1341         journal.append ((id, now, '1', 'set', jp1))
 1342         # Link from user-class to non-existing property
 1343         jp2 = ('user', '1', 'xyzzy')
 1344         journal.append ((id, now+sec, '1', 'link', jp2))
 1345         # Link from non-existing class
 1346         jp3 = ('frobozz', '1', 'xyzzy')
 1347         journal.append ((id, now+sec2, '1', 'link', jp3))
 1348         self.db.setjournal('issue', id, journal)
 1349         self.db.commit()
 1350         result=self.db.issue.history(id)
 1351         result.sort()
 1352         # anydbm drops unknown properties during serialisation
 1353         if self.db.dbtype == 'anydbm':
 1354             self.assertEqual(len(result), 4)
 1355             self.assertEqual(result [1][4], jp0)
 1356             self.assertEqual(result [2][4], jp2)
 1357             self.assertEqual(result [3][4], jp3)
 1358         else:
 1359             self.assertEqual(len(result), 5)
 1360             self.assertEqual(result [1][4], jp0)
 1361             print(result) # following test fails sometimes under sqlite
 1362                           # in travis. Looks like an ordering issue
 1363                           # in python 3.5. Print result to debug.
 1364             self.assertEqual(result [2][4], jp1)
 1365             self.assertEqual(result [3][4], jp2)
 1366             self.assertEqual(result [4][4], jp3)
 1367         self.db.close()
 1368         # Verify that normal user doesn't see obsolete props/classes
 1369         # Backend memorydb cannot re-open db for different user
 1370         if self.db.dbtype != 'memorydb':
 1371             self.open_database('mary')
 1372             setupSchema(self.db, 0, self.module)
 1373             # allow mary to see issue fields like title
 1374             self.db.security.addPermissionToRole('User', 'View', 'issue')
 1375             result=self.db.issue.history(id)
 1376             self.assertEqual(len(result), 2)
 1377             self.assertEqual(result [1][4], jp0)
 1378 
 1379     def testJournalPreCommit(self):
 1380         id = self.db.user.create(username="mary")
 1381         self.assertEqual(len(self.db.getjournal('user', id)), 1)
 1382         self.db.commit()
 1383 
 1384     def testPack(self):
 1385         id = self.db.issue.create(title="spam", status='1')
 1386         self.db.commit()
 1387         time.sleep(1)
 1388         self.db.issue.set(id, status='2')
 1389         self.db.commit()
 1390 
 1391         # sleep for at least a second, then get a date to pack at
 1392         time.sleep(1)
 1393         pack_before = date.Date('.')
 1394 
 1395         # wait another second and add one more entry
 1396         time.sleep(1)
 1397         self.db.issue.set(id, status='3')
 1398         self.db.commit()
 1399         jlen = len(self.db.getjournal('issue', id))
 1400 
 1401         # pack
 1402         self.db.pack(pack_before)
 1403 
 1404         # we should have the create and last set entries now
 1405         self.assertEqual(jlen-1, len(self.db.getjournal('issue', id)))
 1406 
 1407     def testIndexerSearching(self):
 1408         f1 = self.db.file.create(content='hello', type="text/plain")
 1409         # content='world' has the wrong content-type and won't be indexed
 1410         f2 = self.db.file.create(content='world', type="text/frozz",
 1411             comment='blah blah')
 1412         i1 = self.db.issue.create(files=[f1, f2], title="flebble plop")
 1413         i2 = self.db.issue.create(title="flebble the frooz")
 1414         self.db.commit()
 1415         self.assertEqual(self.db.indexer.search([], self.db.issue), {})
 1416         self.assertEqual(self.db.indexer.search(['hello'], self.db.issue),
 1417             {i1: {'files': [f1]}})
 1418         # content='world' has the wrong content-type and shouldn't be indexed
 1419         self.assertEqual(self.db.indexer.search(['world'], self.db.issue), {})
 1420         self.assertEqual(self.db.indexer.search(['frooz'], self.db.issue),
 1421             {i2: {}})
 1422         self.assertEqual(self.db.indexer.search(['flebble'], self.db.issue),
 1423             {i1: {}, i2: {}})
 1424 
 1425         # test AND'ing of search terms
 1426         self.assertEqual(self.db.indexer.search(['frooz', 'flebble'],
 1427             self.db.issue), {i2: {}})
 1428 
 1429         # unindexed stopword
 1430         self.assertEqual(self.db.indexer.search(['the'], self.db.issue), {})
 1431 
 1432     def testIndexerSearchingLink(self):
 1433         m1 = self.db.msg.create(content="one two")
 1434         i1 = self.db.issue.create(messages=[m1])
 1435         m2 = self.db.msg.create(content="two three")
 1436         i2 = self.db.issue.create(feedback=m2)
 1437         self.db.commit()
 1438         self.assertEqual(self.db.indexer.search(['two'], self.db.issue),
 1439             {i1: {'messages': [m1]}, i2: {'feedback': [m2]}})
 1440 
 1441     def testIndexerSearchMulti(self):
 1442         m1 = self.db.msg.create(content="one two")
 1443         m2 = self.db.msg.create(content="two three")
 1444         i1 = self.db.issue.create(messages=[m1])
 1445         i2 = self.db.issue.create(spam=[m2])
 1446         self.db.commit()
 1447         self.assertEqual(self.db.indexer.search([], self.db.issue), {})
 1448         self.assertEqual(self.db.indexer.search(['one'], self.db.issue),
 1449             {i1: {'messages': [m1]}})
 1450         self.assertEqual(self.db.indexer.search(['two'], self.db.issue),
 1451             {i1: {'messages': [m1]}, i2: {'spam': [m2]}})
 1452         self.assertEqual(self.db.indexer.search(['three'], self.db.issue),
 1453             {i2: {'spam': [m2]}})
 1454 
 1455     def testReindexingChange(self):
 1456         search = self.db.indexer.search
 1457         issue = self.db.issue
 1458         i1 = issue.create(title="flebble plop")
 1459         i2 = issue.create(title="flebble frooz")
 1460         self.db.commit()
 1461         self.assertEqual(search(['plop'], issue), {i1: {}})
 1462         self.assertEqual(search(['flebble'], issue), {i1: {}, i2: {}})
 1463 
 1464         # change i1's title
 1465         issue.set(i1, title="plop")
 1466         self.db.commit()
 1467         self.assertEqual(search(['plop'], issue), {i1: {}})
 1468         self.assertEqual(search(['flebble'], issue), {i2: {}})
 1469 
 1470     def testReindexingClear(self):
 1471         search = self.db.indexer.search
 1472         issue = self.db.issue
 1473         i1 = issue.create(title="flebble plop")
 1474         i2 = issue.create(title="flebble frooz")
 1475         self.db.commit()
 1476         self.assertEqual(search(['plop'], issue), {i1: {}})
 1477         self.assertEqual(search(['flebble'], issue), {i1: {}, i2: {}})
 1478 
 1479         # unset i1's title
 1480         issue.set(i1, title="")
 1481         self.db.commit()
 1482         self.assertEqual(search(['plop'], issue), {})
 1483         self.assertEqual(search(['flebble'], issue), {i2: {}})
 1484 
 1485     def testFileClassReindexing(self):
 1486         f1 = self.db.file.create(content='hello')
 1487         f2 = self.db.file.create(content='hello, world')
 1488         i1 = self.db.issue.create(files=[f1, f2])
 1489         self.db.commit()
 1490         d = self.db.indexer.search(['hello'], self.db.issue)
 1491         self.assertTrue(i1 in d)
 1492         d[i1]['files'].sort()
 1493         self.assertEqual(d, {i1: {'files': [f1, f2]}})
 1494         self.assertEqual(self.db.indexer.search(['world'], self.db.issue),
 1495             {i1: {'files': [f2]}})
 1496         self.db.file.set(f1, content="world")
 1497         self.db.commit()
 1498         d = self.db.indexer.search(['world'], self.db.issue)
 1499         d[i1]['files'].sort()
 1500         self.assertEqual(d, {i1: {'files': [f1, f2]}})
 1501         self.assertEqual(self.db.indexer.search(['hello'], self.db.issue),
 1502             {i1: {'files': [f2]}})
 1503 
 1504     def testFileClassIndexingNoNoNo(self):
 1505         f1 = self.db.file.create(content='hello')
 1506         self.db.commit()
 1507         self.assertEqual(self.db.indexer.search(['hello'], self.db.file),
 1508             {'1': {}})
 1509 
 1510         f1 = self.db.file_nidx.create(content='hello')
 1511         self.db.commit()
 1512         self.assertEqual(self.db.indexer.search(['hello'], self.db.file_nidx),
 1513             {})
 1514 
 1515     def testForcedReindexing(self):
 1516         self.db.issue.create(title="flebble frooz")
 1517         self.db.commit()
 1518         self.assertEqual(self.db.indexer.search(['flebble'], self.db.issue),
 1519             {'1': {}})
 1520         self.db.indexer.quiet = 1
 1521         self.db.indexer.force_reindex()
 1522         self.db.post_init()
 1523         self.db.indexer.quiet = 9
 1524         self.assertEqual(self.db.indexer.search(['flebble'], self.db.issue),
 1525             {'1': {}})
 1526 
 1527     def testIndexingPropertiesOnImport(self):
 1528         # import an issue
 1529         title = 'Bzzt'
 1530         nodeid = self.db.issue.import_list(['title', 'messages', 'files',
 1531             'spam', 'nosy', 'superseder'], [repr(title), '[]', '[]',
 1532             '[]', '[]', '[]'])
 1533         self.db.commit()
 1534 
 1535         # Content of title attribute is indexed
 1536         self.assertEqual(self.db.indexer.search([title], self.db.issue),
 1537             {str(nodeid):{}})
 1538 
 1539 
 1540     #
 1541     # searching tests follow
 1542     #
 1543     def testFindIncorrectProperty(self):
 1544         self.assertRaises(TypeError, self.db.issue.find, title='fubar')
 1545 
 1546     def _find_test_setup(self):
 1547         self.db.file.create(content='')
 1548         self.db.file.create(content='')
 1549         self.db.user.create(username='')
 1550         one = self.db.issue.create(status="1", nosy=['1'])
 1551         two = self.db.issue.create(status="2", nosy=['2'], files=['1'],
 1552             assignedto='2')
 1553         three = self.db.issue.create(status="1", nosy=['1','2'])
 1554         four = self.db.issue.create(status="3", assignedto='1',
 1555             files=['1','2'])
 1556         return one, two, three, four
 1557 
 1558     def testFindLink(self):
 1559         one, two, three, four = self._find_test_setup()
 1560         got = self.db.issue.find(status='1')
 1561         got.sort()
 1562         self.assertEqual(got, [one, three])
 1563         got = self.db.issue.find(status={'1':1})
 1564         got.sort()
 1565         self.assertEqual(got, [one, three])
 1566 
 1567     def testFindRevLinkMultilink(self):
 1568         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 1569         ni = 'nosy_issues'
 1570         self.db.issue.set('6', nosy=['3', '4', '5'])
 1571         self.db.issue.set('7', nosy=['5'])
 1572         # After this setup we have the following values for nosy:
 1573         # issue  assignedto  nosy
 1574         # 1:      6          4
 1575         # 2:      6          5
 1576         # 3:      7
 1577         # 4:      8
 1578         # 5:      9
 1579         # 6:      10         3, 4, 5
 1580         # 7:      10         5
 1581         # 8:      10
 1582         # assignedto links back from 'issues'
 1583         # nosy links back from 'nosy_issues'
 1584         self.assertEqual(self.db.user.find(issues={'1':1}), ['6'])
 1585         self.assertEqual(self.db.user.find(issues={'8':1}), ['10'])
 1586         self.assertEqual(self.db.user.find(issues={'2':1, '5':1}), ['6', '9'])
 1587         self.assertEqual(self.db.user.find(nosy_issues={'8':1}), [])
 1588         self.assertEqual(self.db.user.find(nosy_issues={'6':1}),
 1589             ['3', '4', '5'])
 1590         self.assertEqual(self.db.user.find(nosy_issues={'3':1, '5':1}), [])
 1591         self.assertEqual(self.db.user.find(nosy_issues={'2':1, '6':1, '7':1}),
 1592             ['3', '4', '5'])
 1593 
 1594     def testFindLinkFail(self):
 1595         self._find_test_setup()
 1596         self.assertEqual(self.db.issue.find(status='4'), [])
 1597         self.assertEqual(self.db.issue.find(status={'4':1}), [])
 1598 
 1599     def testFindLinkUnset(self):
 1600         one, two, three, four = self._find_test_setup()
 1601         got = self.db.issue.find(assignedto=None)
 1602         got.sort()
 1603         self.assertEqual(got, [one, three])
 1604         got = self.db.issue.find(assignedto={None:1})
 1605         got.sort()
 1606         self.assertEqual(got, [one, three])
 1607 
 1608     def testFindMultipleLink(self):
 1609         one, two, three, four = self._find_test_setup()
 1610         l = self.db.issue.find(status={'1':1, '3':1})
 1611         l.sort()
 1612         self.assertEqual(l, [one, three, four])
 1613         l = self.db.issue.find(status=('1', '3'))
 1614         l.sort()
 1615         self.assertEqual(l, [one, three, four])
 1616         l = self.db.issue.find(status=['1', '3'])
 1617         l.sort()
 1618         self.assertEqual(l, [one, three, four])
 1619         l = self.db.issue.find(assignedto={None:1, '1':1})
 1620         l.sort()
 1621         self.assertEqual(l, [one, three, four])
 1622 
 1623     def testFindMultilink(self):
 1624         one, two, three, four = self._find_test_setup()
 1625         got = self.db.issue.find(nosy='2')
 1626         got.sort()
 1627         self.assertEqual(got, [two, three])
 1628         got = self.db.issue.find(nosy={'2':1})
 1629         got.sort()
 1630         self.assertEqual(got, [two, three])
 1631         got = self.db.issue.find(nosy={'2':1}, files={})
 1632         got.sort()
 1633         self.assertEqual(got, [two, three])
 1634 
 1635     def testFindMultiMultilink(self):
 1636         one, two, three, four = self._find_test_setup()
 1637         got = self.db.issue.find(nosy='2', files='1')
 1638         got.sort()
 1639         self.assertEqual(got, [two, three, four])
 1640         got = self.db.issue.find(nosy={'2':1}, files={'1':1})
 1641         got.sort()
 1642         self.assertEqual(got, [two, three, four])
 1643 
 1644     def testFindMultilinkFail(self):
 1645         self._find_test_setup()
 1646         self.assertEqual(self.db.issue.find(nosy='3'), [])
 1647         self.assertEqual(self.db.issue.find(nosy={'3':1}), [])
 1648 
 1649     def testFindMultilinkUnset(self):
 1650         self._find_test_setup()
 1651         self.assertEqual(self.db.issue.find(nosy={}), [])
 1652 
 1653     def testFindLinkAndMultilink(self):
 1654         one, two, three, four = self._find_test_setup()
 1655         got = self.db.issue.find(status='1', nosy='2')
 1656         got.sort()
 1657         self.assertEqual(got, [one, two, three])
 1658         got = self.db.issue.find(status={'1':1}, nosy={'2':1})
 1659         got.sort()
 1660         self.assertEqual(got, [one, two, three])
 1661 
 1662     def testFindRetired(self):
 1663         one, two, three, four = self._find_test_setup()
 1664         self.assertEqual(len(self.db.issue.find(status='1')), 2)
 1665         self.db.issue.retire(one)
 1666         self.assertEqual(len(self.db.issue.find(status='1')), 1)
 1667 
 1668     def testStringFind(self):
 1669         self.assertRaises(TypeError, self.db.issue.stringFind, status='1')
 1670 
 1671         ids = []
 1672         ids.append(self.db.issue.create(title="spam"))
 1673         self.db.issue.create(title="not spam")
 1674         ids.append(self.db.issue.create(title="spam"))
 1675         ids.sort()
 1676         got = self.db.issue.stringFind(title='spam')
 1677         got.sort()
 1678         self.assertEqual(got, ids)
 1679         self.assertEqual(self.db.issue.stringFind(title='fubar'), [])
 1680 
 1681         # test retiring a node
 1682         self.db.issue.retire(ids[0])
 1683         self.assertEqual(len(self.db.issue.stringFind(title='spam')), 1)
 1684 
 1685     def filteringSetup(self, classname='issue'):
 1686         for user in (
 1687                 {'username': 'bleep', 'age': 1, 'assignable': True},
 1688                 {'username': 'blop', 'age': 1.5, 'assignable': True},
 1689                 {'username': 'blorp', 'age': 2, 'assignable': False}):
 1690             self.db.user.create(**user)
 1691         file_content = ''.join([chr(i) for i in range(255)])
 1692         f = self.db.file.create(content=file_content)
 1693         for issue in (
 1694                 {'title': 'issue one', 'status': '2', 'assignedto': '1',
 1695                     'foo': date.Interval('1:10'), 'priority': '3',
 1696                     'deadline': date.Date('2003-02-16.22:50')},
 1697                 {'title': 'issue two', 'status': '1', 'assignedto': '2',
 1698                     'foo': date.Interval('1d'), 'priority': '3',
 1699                     'deadline': date.Date('2003-01-01.00:00')},
 1700                 {'title': 'issue three', 'status': '1', 'priority': '2',
 1701                     'nosy': ['1','2'], 'deadline': date.Date('2003-02-18')},
 1702                 {'title': 'non four', 'status': '3',
 1703                     'foo': date.Interval('0:10'), 'priority': '2',
 1704                     'nosy': ['1','2','3'], 'deadline': date.Date('2004-03-08'),
 1705                     'files': [f]}):
 1706             self.db.issue.create(**issue)
 1707         self.db.commit()
 1708         return self.iterSetup(classname)
 1709 
 1710     def testFilteringID(self):
 1711         ae, filter, filter_iter = self.filteringSetup()
 1712         for filt in filter, filter_iter:
 1713             ae(filt(None, {'id': '1'}, ('+','id'), (None,None)), ['1'])
 1714             ae(filt(None, {'id': '2'}, ('+','id'), (None,None)), ['2'])
 1715             ae(filt(None, {'id': '100'}, ('+','id'), (None,None)), [])
 1716 
 1717     def testFilteringBoolean(self):
 1718         ae, filter, filter_iter = self.filteringSetup('user')
 1719         a = 'assignable'
 1720         for filt in filter, filter_iter:
 1721             ae(filt(None, {a: '1'}, ('+','id'), (None,None)), ['3','4'])
 1722             ae(filt(None, {a: '0'}, ('+','id'), (None,None)), ['5'])
 1723             ae(filt(None, {a: ['1']}, ('+','id'), (None,None)), ['3','4'])
 1724             ae(filt(None, {a: ['0']}, ('+','id'), (None,None)), ['5'])
 1725             ae(filt(None, {a: ['0','1']}, ('+','id'), (None,None)),
 1726                 ['3','4','5'])
 1727             ae(filt(None, {a: 'True'}, ('+','id'), (None,None)), ['3','4'])
 1728             ae(filt(None, {a: 'False'}, ('+','id'), (None,None)), ['5'])
 1729             ae(filt(None, {a: ['True']}, ('+','id'), (None,None)), ['3','4'])
 1730             ae(filt(None, {a: ['False']}, ('+','id'), (None,None)), ['5'])
 1731             ae(filt(None, {a: ['False','True']}, ('+','id'), (None,None)),
 1732                 ['3','4','5'])
 1733             ae(filt(None, {a: True}, ('+','id'), (None,None)), ['3','4'])
 1734             ae(filt(None, {a: False}, ('+','id'), (None,None)), ['5'])
 1735             ae(filt(None, {a: 1}, ('+','id'), (None,None)), ['3','4'])
 1736             ae(filt(None, {a: 0}, ('+','id'), (None,None)), ['5'])
 1737             ae(filt(None, {a: [1]}, ('+','id'), (None,None)), ['3','4'])
 1738             ae(filt(None, {a: [0]}, ('+','id'), (None,None)), ['5'])
 1739             ae(filt(None, {a: [0,1]}, ('+','id'), (None,None)), ['3','4','5'])
 1740             ae(filt(None, {a: [True]}, ('+','id'), (None,None)), ['3','4'])
 1741             ae(filt(None, {a: [False]}, ('+','id'), (None,None)), ['5'])
 1742             ae(filt(None, {a: [False,True]}, ('+','id'), (None,None)),
 1743                 ['3','4','5'])
 1744 
 1745     def testFilteringNumber(self):
 1746         ae, filter, filter_iter = self.filteringSetup('user')
 1747         for filt in filter, filter_iter:
 1748             ae(filt(None, {'age': '1'}, ('+','id'), (None,None)), ['3'])
 1749             ae(filt(None, {'age': '1.5'}, ('+','id'), (None,None)), ['4'])
 1750             ae(filt(None, {'age': '2'}, ('+','id'), (None,None)), ['5'])
 1751             ae(filt(None, {'age': ['1','2']}, ('+','id'), (None,None)),
 1752                 ['3','5'])
 1753             ae(filt(None, {'age': 2}, ('+','id'), (None,None)), ['5'])
 1754             ae(filt(None, {'age': [1,2]}, ('+','id'), (None,None)), ['3','5'])
 1755 
 1756     def testFilteringString(self):
 1757         ae, filter, filter_iter = self.filteringSetup()
 1758         for filt in filter, filter_iter:
 1759             ae(filt(None, {'title': ['one']}, ('+','id'), (None,None)), ['1'])
 1760             ae(filt(None, {'title': ['issue one']}, ('+','id'), (None,None)),
 1761                 ['1'])
 1762             ae(filt(None, {'title': ['issue', 'one']}, ('+','id'), (None,None)),
 1763                 ['1'])
 1764             ae(filt(None, {'title': ['issue']}, ('+','id'), (None,None)),
 1765                 ['1','2','3'])
 1766             ae(filt(None, {'title': ['one', 'two']}, ('+','id'), (None,None)),
 1767                 [])
 1768 
 1769     def testFilteringStringCase(self):
 1770         """
 1771         Similar to testFilteringString except the search parameters
 1772         have different capitalization.
 1773         """
 1774         ae, filter, filter_iter = self.filteringSetup()
 1775         for filt in filter, filter_iter:
 1776             ae(filt(None, {'title': ['One']}, ('+','id'), (None,None)), ['1'])
 1777             ae(filt(None, {'title': ['Issue One']}, ('+','id'), (None,None)),
 1778                 ['1'])
 1779             ae(filt(None, {'title': ['ISSUE', 'ONE']}, ('+','id'), (None,None)),
 1780                 ['1'])
 1781             ae(filt(None, {'title': ['iSSUE']}, ('+','id'), (None,None)),
 1782                 ['1','2','3'])
 1783             ae(filt(None, {'title': ['One', 'Two']}, ('+','id'), (None,None)),
 1784                 [])
 1785 
 1786     def testFilteringStringExactMatch(self):
 1787         ae, filter, filter_iter = self.filteringSetup()
 1788         # Change title of issue2 to 'issue' so we can test substring
 1789         # search vs exact search
 1790         self.db.issue.set('2', title='issue')
 1791         #self.db.commit()
 1792         for filt in filter, filter_iter:
 1793             ae(filt(None, {}, exact_match_spec =
 1794                {'title': ['one']}), [])
 1795             ae(filt(None, {}, exact_match_spec =
 1796                {'title': ['issue one']}), ['1'])
 1797             ae(filt(None, {}, exact_match_spec =
 1798                {'title': ['issue', 'one']}), [])
 1799             ae(filt(None, {}, exact_match_spec =
 1800                {'title': ['issue']}), ['2'])
 1801             ae(filt(None, {}, exact_match_spec =
 1802                {'title': ['one', 'two']}), [])
 1803             ae(filt(None, {}, exact_match_spec =
 1804                {'title': ['One']}), [])
 1805             ae(filt(None, {}, exact_match_spec =
 1806                {'title': ['Issue One']}), [])
 1807             ae(filt(None, {}, exact_match_spec =
 1808                {'title': ['ISSUE', 'ONE']}), [])
 1809             ae(filt(None, {}, exact_match_spec =
 1810                {'title': ['iSSUE']}), [])
 1811             ae(filt(None, {}, exact_match_spec =
 1812                {'title': ['One', 'Two']}), [])
 1813             ae(filt(None, {}, exact_match_spec =
 1814                {'title': ['non four']}), ['4'])
 1815             # Both, filterspec and exact_match_spec on same prop
 1816             ae(filt(None, {'title': 'iSSUE'}, exact_match_spec =
 1817                {'title': ['issue']}), ['2'])
 1818 
 1819     def testFilteringSpecialChars(self):
 1820         """ Special characters in SQL search are '%' and '_', some used
 1821             to lead to a traceback.
 1822         """
 1823         ae, filter, filter_iter = self.filteringSetup()
 1824         self.db.issue.set('1', title="With % symbol")
 1825         self.db.issue.set('2', title="With _ symbol")
 1826         self.db.issue.set('3', title="With \\ symbol")
 1827         self.db.issue.set('4', title="With ' symbol")
 1828         d = dict (status = '1')
 1829         for filt in filter, filter_iter:
 1830             ae(filt(None, dict(title='%'), ('+','id'), (None,None)), ['1'])
 1831             ae(filt(None, dict(title='_'), ('+','id'), (None,None)), ['2'])
 1832             ae(filt(None, dict(title='\\'), ('+','id'), (None,None)), ['3'])
 1833             ae(filt(None, dict(title="'"), ('+','id'), (None,None)), ['4'])
 1834 
 1835     def testFilteringLink(self):
 1836         ae, filter, filter_iter = self.filteringSetup()
 1837         a = 'assignedto'
 1838         grp = (None, None)
 1839         for filt in filter, filter_iter:
 1840             ae(filt(None, {'status': '1'}, ('+','id'), grp), ['2','3'])
 1841             ae(filt(None, {a: '-1'}, ('+','id'), grp), ['3','4'])
 1842             ae(filt(None, {a: None}, ('+','id'), grp), ['3','4'])
 1843             ae(filt(None, {a: [None]}, ('+','id'), grp), ['3','4'])
 1844             ae(filt(None, {a: ['-1', None]}, ('+','id'), grp), ['3','4'])
 1845             ae(filt(None, {a: ['1', None]}, ('+','id'), grp), ['1', '3','4'])
 1846 
 1847     def testFilteringRevLink(self):
 1848         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 1849         # We have
 1850         # issue assignedto
 1851         # 1:    6
 1852         # 2:    6
 1853         # 3:    7
 1854         # 4:    8
 1855         # 5:    9
 1856         # 6:    10
 1857         # 7:    10
 1858         # 8:    10
 1859         for filt in filter, filter_iter:
 1860             ae(filt(None, {'issues': ['3', '4']}), ['7', '8'])
 1861             ae(filt(None, {'issues': ['1', '4', '8']}), ['6', '8', '10'])
 1862             ae(filt(None, {'issues.title': ['ts2']}), ['6'])
 1863             ae(filt(None, {'issues': ['-1']}), ['1', '2', '3', '4', '5'])
 1864             ae(filt(None, {'issues': '-1'}), ['1', '2', '3', '4', '5'])
 1865         def ls(x):
 1866             return list(sorted(x))
 1867         self.assertEqual(ls(self.db.user.get('6', 'issues')), ['1', '2'])
 1868         self.assertEqual(ls(self.db.user.get('7', 'issues')), ['3'])
 1869         self.assertEqual(ls(self.db.user.get('10', 'issues')), ['6', '7', '8'])
 1870         n = self.db.user.getnode('6')
 1871         self.assertEqual(ls(n.issues), ['1', '2'])
 1872         # Now retire some linked-to issues and retry
 1873         self.db.issue.retire('6')
 1874         self.db.issue.retire('2')
 1875         self.db.issue.retire('3')
 1876         self.db.commit()
 1877         for filt in filter, filter_iter:
 1878             ae(filt(None, {'issues': ['3', '4']}), ['8'])
 1879             ae(filt(None, {'issues': ['1', '4', '8']}), ['6', '8', '10'])
 1880             ae(filt(None, {'issues.title': ['ts2']}), [])
 1881             ae(filt(None, {'issues': ['-1']}), ['1', '2', '3', '4', '5', '7'])
 1882             ae(filt(None, {'issues': '-1'}), ['1', '2', '3', '4', '5', '7'])
 1883         self.assertEqual(ls(self.db.user.get('6', 'issues')), ['1'])
 1884         self.assertEqual(ls(self.db.user.get('7', 'issues')), [])
 1885         self.assertEqual(ls(self.db.user.get('10', 'issues')), ['7', '8'])
 1886 
 1887     def testFilteringLinkSortSearchMultilink(self):
 1888         ae, filter, filter_iter = self.filteringSetup()
 1889         a = 'assignedto'
 1890         grp = (None, None)
 1891         for filt in filter, filter_iter:
 1892             ae(filt(None, {'status.mls': '1'}, ('+','status')), ['2','3'])
 1893             ae(filt(None, {'status.mls': '2'}, ('+','status')), ['2','3'])
 1894 
 1895     def testFilteringMultilinkAndGroup(self):
 1896         """testFilteringMultilinkAndGroup:
 1897         See roundup Bug 1541128: apparently grouping by something and
 1898         searching a Multilink failed with MySQL 5.0
 1899         """
 1900         ae, filter, filter_iter = self.filteringSetup()
 1901         for f in filter, filter_iter:
 1902             ae(f(None, {'files': '1'}, ('-','activity'), ('+','status')), ['4'])
 1903 
 1904     def testFilteringRetired(self):
 1905         ae, filter, filter_iter = self.filteringSetup()
 1906         self.db.issue.retire('2')
 1907         for f in filter, filter_iter:
 1908             ae(f(None, {'status': '1'}, ('+','id'), (None,None)), ['3'])
 1909 
 1910     def testFilteringMultilink(self):
 1911         ae, filter, filter_iter = self.filteringSetup()
 1912         for filt in filter, filter_iter:
 1913             ae(filt(None, {'nosy': '3'}, ('+','id'), (None,None)), ['4'])
 1914             ae(filt(None, {'nosy': '-1'}, ('+','id'), (None,None)), ['1', '2'])
 1915             ae(filt(None, {'nosy': ['1','2']}, ('+', 'status'),
 1916                 ('-', 'deadline')), ['4', '3'])
 1917 
 1918     def testFilteringRevMultilink(self):
 1919         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 1920         ni = 'nosy_issues'
 1921         self.db.issue.set('6', nosy=['3', '4', '5'])
 1922         self.db.issue.set('7', nosy=['5'])
 1923         # After this setup we have the following values for nosy:
 1924         # issue   nosy
 1925         # 1:      4
 1926         # 2:      5
 1927         # 3:
 1928         # 4:
 1929         # 5:
 1930         # 6:      3, 4, 5
 1931         # 7:      5
 1932         # 8:
 1933         for filt in filter, filter_iter:
 1934             ae(filt(None, {ni: ['1', '2']}), ['4', '5'])
 1935             ae(filt(None, {ni: ['6','7']}), ['3', '4', '5'])
 1936             ae(filt(None, {'nosy_issues.title': ['ts2']}), ['5'])
 1937             ae(filt(None, {ni: ['-1']}), ['1', '2', '6', '7', '8', '9', '10'])
 1938             ae(filt(None, {ni: '-1'}), ['1', '2', '6', '7', '8', '9', '10'])
 1939         def ls(x):
 1940             return list(sorted(x))
 1941         self.assertEqual(ls(self.db.user.get('4', ni)), ['1', '6'])
 1942         self.assertEqual(ls(self.db.user.get('5', ni)), ['2', '6', '7'])
 1943         n = self.db.user.getnode('4')
 1944         self.assertEqual(ls(n.nosy_issues), ['1', '6'])
 1945         # Now retire some linked-to issues and retry
 1946         self.db.issue.retire('2')
 1947         self.db.issue.retire('6')
 1948         self.db.commit()
 1949         for filt in filter, filter_iter:
 1950             ae(filt(None, {ni: ['1', '2']}), ['4'])
 1951             ae(filt(None, {ni: ['6','7']}), ['5'])
 1952             ae(filt(None, {'nosy_issues.title': ['ts2']}), [])
 1953             ae(filt(None, {ni: ['-1']}),
 1954                 ['1', '2', '3', '6', '7', '8', '9', '10'])
 1955             ae(filt(None, {ni: '-1'}),
 1956                 ['1', '2', '3', '6', '7', '8', '9', '10'])
 1957         self.assertEqual(ls(self.db.user.get('4', ni)), ['1'])
 1958         self.assertEqual(ls(self.db.user.get('5', ni)), ['7'])
 1959 
 1960     def testFilteringMany(self):
 1961         ae, filter, filter_iter = self.filteringSetup()
 1962         for f in filter, filter_iter:
 1963             ae(f(None, {'nosy': '2', 'status': '1'}, ('+','id'), (None,None)),
 1964                 ['3'])
 1965 
 1966     def testFilteringRangeBasic(self):
 1967         ae, filter, filter_iter = self.filteringSetup()
 1968         d = 'deadline'
 1969         for f in filter, filter_iter:
 1970             ae(f(None, {d: 'from 2003-02-10 to 2003-02-23'}), ['1','3'])
 1971             ae(f(None, {d: '2003-02-10; 2003-02-23'}), ['1','3'])
 1972             ae(f(None, {d: '; 2003-02-16'}), ['2'])
 1973 
 1974     def testFilteringRangeTwoSyntaxes(self):
 1975         ae, filter, filter_iter = self.filteringSetup()
 1976         for filt in filter, filter_iter:
 1977             ae(filt(None, {'deadline': 'from 2003-02-16'}), ['1', '3', '4'])
 1978             ae(filt(None, {'deadline': '2003-02-16;'}), ['1', '3', '4'])
 1979 
 1980     def testFilteringRangeYearMonthDay(self):
 1981         ae, filter, filter_iter = self.filteringSetup()
 1982         for filt in filter, filter_iter:
 1983             ae(filt(None, {'deadline': '2002'}), [])
 1984             ae(filt(None, {'deadline': '2003'}), ['1', '2', '3'])
 1985             ae(filt(None, {'deadline': '2004'}), ['4'])
 1986             ae(filt(None, {'deadline': '2003-02-16'}), ['1'])
 1987             ae(filt(None, {'deadline': '2003-02-17'}), [])
 1988 
 1989     def testFilteringRangeMonths(self):
 1990         ae, filter, filter_iter = self.filteringSetup()
 1991         for month in range(1, 13):
 1992             for n in range(1, month+1):
 1993                 i = self.db.issue.create(title='%d.%d'%(month, n),
 1994                     deadline=date.Date('2001-%02d-%02d.00:00'%(month, n)))
 1995         self.db.commit()
 1996 
 1997         for month in range(1, 13):
 1998             for filt in filter, filter_iter:
 1999                 r = filt(None, dict(deadline='2001-%02d'%month))
 2000                 assert len(r) == month, 'month %d != length %d'%(month, len(r))
 2001 
 2002     def testFilteringDateRangeMulti(self):
 2003         ae, filter, filter_iter = self.filteringSetup()
 2004         self.db.issue.create(title='no deadline')
 2005         self.db.commit()
 2006         for filt in filter, filter_iter:
 2007             r = filt (None, dict(deadline='-'))
 2008             self.assertEqual(r, ['5'])
 2009             r = filt (None, dict(deadline=';2003-02-01,2004;'))
 2010             self.assertEqual(r, ['2', '4'])
 2011             r = filt (None, dict(deadline='-,;2003-02-01,2004;'))
 2012             self.assertEqual(r, ['2', '4', '5'])
 2013 
 2014     def testFilteringRangeInterval(self):
 2015         ae, filter, filter_iter = self.filteringSetup()
 2016         for filt in filter, filter_iter:
 2017             ae(filt(None, {'foo': 'from 0:50 to 2:00'}), ['1'])
 2018             ae(filt(None, {'foo': 'from 0:50 to 1d 2:00'}), ['1', '2'])
 2019             ae(filt(None, {'foo': 'from 5:50'}), ['2'])
 2020             ae(filt(None, {'foo': 'to 0:05'}), [])
 2021 
 2022     def testFilteringRangeGeekInterval(self):
 2023         ae, filter, filter_iter = self.filteringSetup()
 2024         # Note: When querying, create date one minute later than the
 2025         # timespan later queried to avoid race conditions where the
 2026         # creation of the deadline is more than a second ago when
 2027         # queried -- in that case we wouldn't get the expected result.
 2028         # By extending the interval by a minute we would need a very
 2029         # slow machine for this test to fail :-)
 2030         for issue in (
 2031                 { 'deadline': date.Date('. -2d') + date.Interval ('00:01')},
 2032                 { 'deadline': date.Date('. -1d') + date.Interval ('00:01')},
 2033                 { 'deadline': date.Date('. -8d') + date.Interval ('00:01')},
 2034                 ):
 2035             self.db.issue.create(**issue)
 2036         for filt in filter, filter_iter:
 2037             ae(filt(None, {'deadline': '-2d;'}), ['5', '6'])
 2038             ae(filt(None, {'deadline': '-1d;'}), ['6'])
 2039             ae(filt(None, {'deadline': '-1w;'}), ['5', '6'])
 2040             ae(filt(None, {'deadline': '. -2d;'}), ['5', '6'])
 2041             ae(filt(None, {'deadline': '. -1d;'}), ['6'])
 2042             ae(filt(None, {'deadline': '. -1w;'}), ['5', '6'])
 2043 
 2044     def testFilteringIntervalSort(self):
 2045         # 1: '1:10'
 2046         # 2: '1d'
 2047         # 3: None
 2048         # 4: '0:10'
 2049         ae, filter, filter_iter = self.filteringSetup()
 2050         for filt in filter, filter_iter:
 2051             # ascending should sort None, 1:10, 1d
 2052             ae(filt(None, {}, ('+','foo'), (None,None)), ['3', '4', '1', '2'])
 2053             # descending should sort 1d, 1:10, None
 2054             ae(filt(None, {}, ('-','foo'), (None,None)), ['2', '1', '4', '3'])
 2055 
 2056     def testFilteringStringSort(self):
 2057         # 1: 'issue one'
 2058         # 2: 'issue two'
 2059         # 3: 'issue three'
 2060         # 4: 'non four'
 2061         ae, filter, filter_iter = self.filteringSetup()
 2062         for filt in filter, filter_iter:
 2063             ae(filt(None, {}, ('+','title')), ['1', '3', '2', '4'])
 2064             ae(filt(None, {}, ('-','title')), ['4', '2', '3', '1'])
 2065         # Test string case: For now allow both, w/wo case matching.
 2066         # 1: 'issue one'
 2067         # 2: 'issue two'
 2068         # 3: 'Issue three'
 2069         # 4: 'non four'
 2070         self.db.issue.set('3', title='Issue three')
 2071         for filt in filter, filter_iter:
 2072             ae(filt(None, {}, ('+','title')), ['1', '3', '2', '4'])
 2073             ae(filt(None, {}, ('-','title')), ['4', '2', '3', '1'])
 2074         # Obscure bug in anydbm backend trying to convert to number
 2075         # 1: '1st issue'
 2076         # 2: '2'
 2077         # 3: 'Issue three'
 2078         # 4: 'non four'
 2079         self.db.issue.set('1', title='1st issue')
 2080         self.db.issue.set('2', title='2')
 2081         for filt in filter, filter_iter:
 2082             ae(filt(None, {}, ('+','title')), ['1', '2', '3', '4'])
 2083             ae(filt(None, {}, ('-','title')), ['4', '3', '2', '1'])
 2084 
 2085     def testFilteringMultilinkSort(self):
 2086         # 1: []                 Reverse:  1: []
 2087         # 2: []                           2: []
 2088         # 3: ['admin','fred']             3: ['fred','admin']
 2089         # 4: ['admin','bleep','fred']     4: ['fred','bleep','admin']
 2090         # Note the sort order for the multilink doen't change when
 2091         # reversing the sort direction due to the re-sorting of the
 2092         # multilink!
 2093         # Note that we don't test filter_iter here, Multilink sort-order
 2094         # isn't defined for that.
 2095         ae, filt, dummy = self.filteringSetup()
 2096         ae(filt(None, {}, ('+','nosy'), (None,None)), ['1', '2', '4', '3'])
 2097         ae(filt(None, {}, ('-','nosy'), (None,None)), ['4', '3', '1', '2'])
 2098 
 2099     def testFilteringMultilinkSortGroup(self):
 2100         # 1: status: 2 "in-progress" nosy: []
 2101         # 2: status: 1 "unread"      nosy: []
 2102         # 3: status: 1 "unread"      nosy: ['admin','fred']
 2103         # 4: status: 3 "testing"     nosy: ['admin','bleep','fred']
 2104         # Note that we don't test filter_iter here, Multilink sort-order
 2105         # isn't defined for that.
 2106         ae, filt, dummy = self.filteringSetup()
 2107         ae(filt(None, {}, ('+','nosy'), ('+','status')), ['1', '4', '2', '3'])
 2108         ae(filt(None, {}, ('-','nosy'), ('+','status')), ['1', '4', '3', '2'])
 2109         ae(filt(None, {}, ('+','nosy'), ('-','status')), ['2', '3', '4', '1'])
 2110         ae(filt(None, {}, ('-','nosy'), ('-','status')), ['3', '2', '4', '1'])
 2111         ae(filt(None, {}, ('+','status'), ('+','nosy')), ['1', '2', '4', '3'])
 2112         ae(filt(None, {}, ('-','status'), ('+','nosy')), ['2', '1', '4', '3'])
 2113         ae(filt(None, {}, ('+','status'), ('-','nosy')), ['4', '3', '1', '2'])
 2114         ae(filt(None, {}, ('-','status'), ('-','nosy')), ['4', '3', '2', '1'])
 2115 
 2116     def testFilteringLinkSortGroup(self):
 2117         # 1: status: 2 -> 'i', priority: 3 -> 1
 2118         # 2: status: 1 -> 'u', priority: 3 -> 1
 2119         # 3: status: 1 -> 'u', priority: 2 -> 3
 2120         # 4: status: 3 -> 't', priority: 2 -> 3
 2121         ae, filter, filter_iter = self.filteringSetup()
 2122         for filt in filter, filter_iter:
 2123             ae(filt(None, {}, ('+','status'), ('+','priority')),
 2124                 ['1', '2', '4', '3'])
 2125             ae(filt(None, {'priority':'2'}, ('+','status'), ('+','priority')),
 2126                 ['4', '3'])
 2127             ae(filt(None, {'priority.order':'3'}, ('+','status'),
 2128                 ('+','priority')), ['4', '3'])
 2129             ae(filt(None, {'priority':['2','3']}, ('+','priority'),
 2130                 ('+','status')), ['1', '4', '2', '3'])
 2131             ae(filt(None, {}, ('+','priority'), ('+','status')),
 2132                 ['1', '4', '2', '3'])
 2133 
 2134     def testFilteringDateSort(self):
 2135         # '1': '2003-02-16.22:50'
 2136         # '2': '2003-01-01.00:00'
 2137         # '3': '2003-02-18'
 2138         # '4': '2004-03-08'
 2139         ae, filter, filter_iter = self.filteringSetup()
 2140         for f in filter, filter_iter:
 2141             # ascending
 2142             ae(f(None, {}, ('+','deadline'), (None,None)), ['2', '1', '3', '4'])
 2143             # descending
 2144             ae(f(None, {}, ('-','deadline'), (None,None)), ['4', '3', '1', '2'])
 2145 
 2146     def testFilteringDateSortPriorityGroup(self):
 2147         # '1': '2003-02-16.22:50'  1 => 2
 2148         # '2': '2003-01-01.00:00'  3 => 1
 2149         # '3': '2003-02-18'        2 => 3
 2150         # '4': '2004-03-08'        1 => 2
 2151         ae, filter, filter_iter = self.filteringSetup()
 2152 
 2153         for filt in filter, filter_iter:
 2154             # ascending
 2155             ae(filt(None, {}, ('+','deadline'), ('+','priority')),
 2156                 ['2', '1', '3', '4'])
 2157             ae(filt(None, {}, ('-','deadline'), ('+','priority')),
 2158                 ['1', '2', '4', '3'])
 2159             # descending
 2160             ae(filt(None, {}, ('+','deadline'), ('-','priority')),
 2161                 ['3', '4', '2', '1'])
 2162             ae(filt(None, {}, ('-','deadline'), ('-','priority')),
 2163                 ['4', '3', '1', '2'])
 2164 
 2165     def testFilteringTransitiveLinkUser(self):
 2166         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 2167         for f in filter, filter_iter:
 2168             ae(f(None, {'supervisor.username': 'ceo'}, ('+','username')),
 2169                 ['4', '5'])
 2170             ae(f(None, {'supervisor.supervisor.username': 'ceo'},
 2171                 ('+','username')), ['6', '7', '8', '9', '10'])
 2172             ae(f(None, {'supervisor.supervisor': '3'}, ('+','username')),
 2173                 ['6', '7', '8', '9', '10'])
 2174             ae(f(None, {'supervisor.supervisor.id': '3'}, ('+','username')),
 2175                 ['6', '7', '8', '9', '10'])
 2176             ae(f(None, {'supervisor.username': 'grouplead1'}, ('+','username')),
 2177                 ['6', '7'])
 2178             ae(f(None, {'supervisor.username': 'grouplead2'}, ('+','username')),
 2179                 ['8', '9', '10'])
 2180             ae(f(None, {'supervisor.username': 'grouplead2',
 2181                 'supervisor.supervisor.username': 'ceo'}, ('+','username')),
 2182                 ['8', '9', '10'])
 2183             ae(f(None, {'supervisor.supervisor': '3', 'supervisor': '4'},
 2184                 ('+','username')), ['6', '7'])
 2185 
 2186     def testFilteringTransitiveLinkUserLimit(self):
 2187         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 2188         for f in filter, filter_iter:
 2189             ae(f(None, {'supervisor.username': 'ceo'}, ('+','username'),
 2190                  limit=1), ['4'])
 2191             ae(f(None, {'supervisor.supervisor.username': 'ceo'},
 2192                 ('+','username'), limit=4), ['6', '7', '8', '9'])
 2193             ae(f(None, {'supervisor.supervisor': '3'}, ('+','username'),
 2194                 limit=2, offset=2), ['8', '9'])
 2195             ae(f(None, {'supervisor.supervisor.id': '3'}, ('+','username'),
 2196                 limit=3, offset=1), ['7', '8', '9'])
 2197             ae(f(None, {'supervisor.username': 'grouplead2'}, ('+','username'),
 2198                 limit=2, offset=2), ['10'])
 2199             ae(f(None, {'supervisor.username': 'grouplead2',
 2200                 'supervisor.supervisor.username': 'ceo'}, ('+','username'),
 2201                 limit=4, offset=3), [])
 2202             ae(f(None, {'supervisor.supervisor': '3', 'supervisor': '4'},
 2203                 ('+','username'), limit=1, offset=5), [])
 2204 
 2205     def testFilteringTransitiveLinkSort(self):
 2206         ae, filter, filter_iter = self.filteringSetupTransitiveSearch()
 2207         ae, ufilter, ufilter_iter = self.iterSetup('user')
 2208         # Need to make ceo his own (and first two users') supervisor,
 2209         # otherwise we will depend on sorting order of NULL values.
 2210         # Leave that to a separate test.
 2211         self.db.user.set('1', supervisor = '3')
 2212         self.db.user.set('2', supervisor = '3')
 2213         self.db.user.set('3', supervisor = '3')
 2214         for ufilt in ufilter, ufilter_iter:
 2215             ae(ufilt(None, {'supervisor':'3'}, []), ['1', '2', '3', '4', '5'])
 2216             ae(ufilt(None, {}, [('+','supervisor.supervisor.supervisor'),
 2217                 ('+','supervisor.supervisor'), ('+','supervisor'),
 2218                 ('+','username')]),
 2219                 ['1', '3', '2', '4', '5', '6', '7', '8', '9', '10'])
 2220             ae(ufilt(None, {}, [('+','supervisor.supervisor.supervisor'),
 2221                 ('-','supervisor.supervisor'), ('-','supervisor'),
 2222                 ('+','username')]),
 2223                 ['8', '9', '10', '6', '7', '1', '3', '2', '4', '5'])
 2224         for f in filter, filter_iter:
 2225             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2226                 ('+','assignedto.supervisor.supervisor'),
 2227                 ('+','assignedto.supervisor'), ('+','assignedto')]),
 2228                 ['1', '2', '3', '4', '5', '6', '7', '8'])
 2229             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2230                 ('+','assignedto.supervisor.supervisor'),
 2231                 ('-','assignedto.supervisor'), ('+','assignedto')]),
 2232                 ['4', '5', '6', '7', '8', '1', '2', '3'])
 2233             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2234                 ('+','assignedto.supervisor.supervisor'),
 2235                 ('+','assignedto.supervisor'), ('+','assignedto'),
 2236                 ('-','status')]),
 2237                 ['2', '1', '3', '4', '5', '6', '8', '7'])
 2238             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2239                 ('+','assignedto.supervisor.supervisor'),
 2240                 ('+','assignedto.supervisor'), ('+','assignedto'),
 2241                 ('+','status')]),
 2242                 ['1', '2', '3', '4', '5', '7', '6', '8'])
 2243             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2244                 ('+','assignedto.supervisor.supervisor'),
 2245                 ('-','assignedto.supervisor'), ('+','assignedto'),
 2246                 ('+','status')]), ['4', '5', '7', '6', '8', '1', '2', '3'])
 2247             ae(f(None, {'assignedto':['6','7','8','9','10']},
 2248                 [('+','assignedto.supervisor.supervisor.supervisor'),
 2249                 ('+','assignedto.supervisor.supervisor'),
 2250                 ('-','assignedto.supervisor'), ('+','assignedto'),
 2251                 ('+','status')]), ['4', '5', '7', '6', '8', '1', '2', '3'])
 2252             ae(f(None, {'assignedto':['6','7','8','9']},
 2253                 [('+','assignedto.supervisor.supervisor.supervisor'),
 2254                 ('+','assignedto.supervisor.supervisor'),
 2255                 ('-','assignedto.supervisor'), ('+','assignedto'),
 2256                 ('+','status')]), ['4', '5', '1', '2', '3'])
 2257 
 2258     def testFilteringTransitiveLinkSortNull(self):
 2259         """Check sorting of NULL values"""
 2260         ae, filter, filter_iter = self.filteringSetupTransitiveSearch()
 2261         ae, ufilter, ufilter_iter = self.iterSetup('user')
 2262         for ufilt in ufilter, ufilter_iter:
 2263             ae(ufilt(None, {}, [('+','supervisor.supervisor.supervisor'),
 2264                 ('+','supervisor.supervisor'), ('+','supervisor'),
 2265                 ('+','username')]),
 2266                 ['1', '3', '2', '4', '5', '6', '7', '8', '9', '10'])
 2267             ae(ufilt(None, {}, [('+','supervisor.supervisor.supervisor'),
 2268                 ('-','supervisor.supervisor'), ('-','supervisor'),
 2269                 ('+','username')]),
 2270                 ['8', '9', '10', '6', '7', '4', '5', '1', '3', '2'])
 2271         for f in filter, filter_iter:
 2272             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2273                 ('+','assignedto.supervisor.supervisor'),
 2274                 ('+','assignedto.supervisor'), ('+','assignedto')]),
 2275                 ['1', '2', '3', '4', '5', '6', '7', '8'])
 2276             ae(f(None, {}, [('+','assignedto.supervisor.supervisor.supervisor'),
 2277                 ('+','assignedto.supervisor.supervisor'),
 2278                 ('-','assignedto.supervisor'), ('+','assignedto')]),
 2279                 ['4', '5', '6', '7', '8', '1', '2', '3'])
 2280 
 2281     def testFilteringTransitiveLinkIssue(self):
 2282         ae, filter, filter_iter = self.filteringSetupTransitiveSearch()
 2283         for filt in filter, filter_iter:
 2284             ae(filt(None, {'assignedto.supervisor.username': 'grouplead1'},
 2285                 ('+','id')), ['1', '2', '3'])
 2286             ae(filt(None, {'assignedto.supervisor.username': 'grouplead2'},
 2287                 ('+','id')), ['4', '5', '6', '7', '8'])
 2288             ae(filt(None, {'assignedto.supervisor.username': 'grouplead2',
 2289                            'status': '1'}, ('+','id')), ['4', '6', '8'])
 2290             ae(filt(None, {'assignedto.supervisor.username': 'grouplead2',
 2291                            'status': '2'}, ('+','id')), ['5', '7'])
 2292             ae(filt(None, {'assignedto.supervisor.username': ['grouplead2'],
 2293                            'status': '2'}, ('+','id')), ['5', '7'])
 2294             ae(filt(None, {'assignedto.supervisor': ['4', '5'], 'status': '2'},
 2295                 ('+','id')), ['1', '3', '5', '7'])
 2296 
 2297     def testFilteringTransitiveMultilink(self):
 2298         ae, filter, filter_iter = self.filteringSetupTransitiveSearch()
 2299         for filt in filter, filter_iter:
 2300             ae(filt(None, {'messages.author.username': 'grouplead1'},
 2301                 ('+','id')), [])
 2302             ae(filt(None, {'messages.author': '6'},
 2303                 ('+','id')), ['1', '2'])
 2304             ae(filt(None, {'messages.author.id': '6'},
 2305                 ('+','id')), ['1', '2'])
 2306             ae(filt(None, {'messages.author.username': 'worker1'},
 2307                 ('+','id')), ['1', '2'])
 2308             ae(filt(None, {'messages.author': '10'},
 2309                 ('+','id')), ['6', '7', '8'])
 2310             ae(filt(None, {'messages.author': '9'},
 2311                 ('+','id')), ['5', '8'])
 2312             ae(filt(None, {'messages.author': ['9', '10']},
 2313                 ('+','id')), ['5', '6', '7', '8'])
 2314             ae(filt(None, {'messages.author': ['8', '9']},
 2315                 ('+','id')), ['4', '5', '8'])
 2316             ae(filt(None, {'messages.author': ['8', '9'], 'status' : '1'},
 2317                 ('+','id')), ['4', '8'])
 2318             ae(filt(None, {'messages.author': ['8', '9'], 'status' : '2'},
 2319                 ('+','id')), ['5'])
 2320             ae(filt(None, {'messages.author': ['8', '9', '10'],
 2321                 'messages.date': '2006-01-22.21:00;2006-01-23'}, ('+','id')),
 2322                 ['6', '7', '8'])
 2323             ae(filt(None, {'nosy.supervisor.username': 'ceo'},
 2324                 ('+','id')), ['1', '2'])
 2325             ae(filt(None, {'messages.author': ['6', '9']},
 2326                 ('+','id')), ['1', '2', '5', '8'])
 2327             ae(filt(None, {'messages': ['5', '7']},
 2328                 ('+','id')), ['3', '5', '8'])
 2329             ae(filt(None, {'messages.author': ['6', '9'],
 2330                 'messages': ['5', '7']}, ('+','id')), ['5', '8'])
 2331 
 2332     def testFilteringTransitiveMultilinkSort(self):
 2333         # Note that we don't test filter_iter here, Multilink sort-order
 2334         # isn't defined for that.
 2335         ae, filt, dummy = self.filteringSetupTransitiveSearch()
 2336         ae(filt(None, {}, [('+','messages.author')]),
 2337             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2338         ae(filt(None, {}, [('-','messages.author')]),
 2339             ['8', '6', '7', '5', '4', '3', '1', '2'])
 2340         ae(filt(None, {}, [('+','messages.date')]),
 2341             ['6', '7', '8', '5', '4', '3', '1', '2'])
 2342         ae(filt(None, {}, [('-','messages.date')]),
 2343             ['1', '2', '3', '4', '8', '5', '6', '7'])
 2344         ae(filt(None, {}, [('+','messages.author'),('+','messages.date')]),
 2345             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2346         ae(filt(None, {}, [('-','messages.author'),('+','messages.date')]),
 2347             ['8', '6', '7', '5', '4', '3', '1', '2'])
 2348         ae(filt(None, {}, [('+','messages.author'),('-','messages.date')]),
 2349             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2350         ae(filt(None, {}, [('-','messages.author'),('-','messages.date')]),
 2351             ['8', '6', '7', '5', '4', '3', '1', '2'])
 2352         ae(filt(None, {}, [('+','messages.author'),('+','assignedto')]),
 2353             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2354         ae(filt(None, {}, [('+','messages.author'),
 2355             ('-','assignedto.supervisor'),('-','assignedto')]),
 2356             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2357         ae(filt(None, {},
 2358             [('+','messages.author.supervisor.supervisor.supervisor'),
 2359             ('+','messages.author.supervisor.supervisor'),
 2360             ('+','messages.author.supervisor'), ('+','messages.author')]),
 2361             ['1', '2', '3', '4', '5', '6', '7', '8'])
 2362         self.db.user.setorderprop('age')
 2363         self.db.msg.setorderprop('date')
 2364         ae(filt(None, {}, [('+','messages'), ('+','messages.author')]),
 2365             ['6', '7', '8', '5', '4', '3', '1', '2'])
 2366         ae(filt(None, {}, [('+','messages.author'), ('+','messages')]),
 2367             ['6', '7', '8', '5', '4', '3', '1', '2'])
 2368         self.db.msg.setorderprop('author')
 2369         # Orderprop is a Link/Multilink:
 2370         # messages are sorted by orderprop().labelprop(), i.e. by
 2371         # author.username, *not* by author.orderprop() (author.age)!
 2372         ae(filt(None, {}, [('+','messages')]),
 2373             ['1', '2', '3', '4', '5', '8', '6', '7'])
 2374         ae(filt(None, {}, [('+','messages.author'), ('+','messages')]),
 2375             ['6', '7', '8', '5', '4', '3', '1', '2'])
 2376         # The following will sort by
 2377         # author.supervisor.username and then by
 2378         # author.username
 2379         # I've resited the tempation to implement recursive orderprop
 2380         # here: There could even be loops if several classes specify a
 2381         # Link or Multilink as the orderprop...
 2382         # msg: 4: worker1 (id  5) : grouplead1 (id 4) ceo (id 3)
 2383         # msg: 5: worker2 (id  7) : grouplead1 (id 4) ceo (id 3)
 2384         # msg: 6: worker3 (id  8) : grouplead2 (id 5) ceo (id 3)
 2385         # msg: 7: worker4 (id  9) : grouplead2 (id 5) ceo (id 3)
 2386         # msg: 8: worker5 (id 10) : grouplead2 (id 5) ceo (id 3)
 2387         # issue 1: messages 4   sortkey:[[grouplead1], [worker1], 1]
 2388         # issue 2: messages 4   sortkey:[[grouplead1], [worker1], 2]
 2389         # issue 3: messages 5   sortkey:[[grouplead1], [worker2], 3]
 2390         # issue 4: messages 6   sortkey:[[grouplead2], [worker3], 4]
 2391         # issue 5: messages 7   sortkey:[[grouplead2], [worker4], 5]
 2392         # issue 6: messages 8   sortkey:[[grouplead2], [worker5], 6]
 2393         # issue 7: messages 8   sortkey:[[grouplead2], [worker5], 7]
 2394         # issue 8: messages 7,8 sortkey:[[grouplead2, grouplead2], ...]
 2395         self.db.user.setorderprop('supervisor')
 2396         ae(filt(None, {}, [('+','messages.author'), ('-','messages')]),
 2397             ['3', '1', '2', '6', '7', '5', '4', '8'])
 2398 
 2399     def testFilteringSortId(self):
 2400         ae, filter, filter_iter = self.filteringSetupTransitiveSearch('user')
 2401         for filt in filter, filter_iter:
 2402             ae(filt(None, {}, ('+','id')),
 2403                 ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'])
 2404 
 2405     def testFilteringRetiredString(self):
 2406         ae, filter, filter_iter = self.filteringSetup()
 2407         self.db.issue.retire('1')
 2408         self.db.commit()
 2409         r = { None: (['1'], ['1'], ['1'], ['1', '2', '3'], [])
 2410             , True: (['1'], ['1'], ['1'], ['1'], [])
 2411             , False: ([], [], [], ['2', '3'], [])
 2412             }
 2413         for filt in filter, filter_iter:
 2414             for retire in True, False, None:
 2415                 ae(filt(None, {'title': ['one']}, ('+','id'),
 2416                    retired=retire), r[retire][0])
 2417                 ae(filt(None, {'title': ['issue one']}, ('+','id'),
 2418                    retired=retire), r[retire][1])
 2419                 ae(filt(None, {'title': ['issue', 'one']}, ('+','id'),
 2420                    retired=retire), r[retire][2])
 2421                 ae(filt(None, {'title': ['issue']}, ('+','id'),
 2422                    retired=retire), r[retire][3])
 2423                 ae(filt(None, {'title': ['one', 'two']}, ('+','id'),
 2424                    retired=retire), r[retire][4])
 2425 
 2426 # XXX add sorting tests for other types
 2427 
 2428     # nuke and re-create db for restore
 2429     def nukeAndCreate(self):
 2430         # shut down this db and nuke it
 2431         self.db.close()
 2432         self.nuke_database()
 2433 
 2434         # open a new, empty database
 2435         os.makedirs(config.DATABASE + '/files')
 2436         self.db = self.module.Database(config, 'admin')
 2437         setupSchema(self.db, 0, self.module)
 2438 
 2439     def testImportExport(self):
 2440         # use the filtering setup to create a bunch of items
 2441         ae, dummy1, dummy2 = self.filteringSetup()
 2442         # Get some stuff into the journal for testing import/export of
 2443         # journal data:
 2444         self.db.user.set('4', password = password.Password('xyzzy'))
 2445         self.db.user.set('4', age = 3)
 2446         self.db.user.set('4', assignable = True)
 2447         self.db.issue.set('1', title = 'i1', status = '3')
 2448         self.db.issue.set('1', deadline = date.Date('2007'))
 2449         self.db.issue.set('1', foo = date.Interval('1:20'))
 2450         p = self.db.priority.create(name = 'some_prio_without_order')
 2451         self.db.commit()
 2452         self.db.user.set('4', password = password.Password('123xyzzy'))
 2453         self.db.user.set('4', assignable = False)
 2454         self.db.priority.set(p, order = '4711')
 2455         self.db.commit()
 2456 
 2457         self.db.user.retire('3')
 2458         self.db.issue.retire('2')
 2459 
 2460         # grab snapshot of the current database
 2461         orig = {}
 2462         origj = {}
 2463         for cn,klass in self.db.classes.items():
 2464             cl = orig[cn] = {}
 2465             jn = origj[cn] = {}
 2466             for id in klass.list():
 2467                 it = cl[id] = {}
 2468                 jn[id] = self.db.getjournal(cn, id)
 2469                 for name in klass.getprops().keys():
 2470                     it[name] = klass.get(id, name)
 2471 
 2472         os.mkdir('_test_export')
 2473         try:
 2474             # grab the export
 2475             export = {}
 2476             journals = {}
 2477             for cn,klass in self.db.classes.items():
 2478                 names = klass.export_propnames()
 2479                 cl = export[cn] = [names+['is retired']]
 2480                 for id in klass.getnodeids():
 2481                     cl.append(klass.export_list(names, id))
 2482                     if hasattr(klass, 'export_files'):
 2483                         klass.export_files('_test_export', id)
 2484                 journals[cn] = klass.export_journals()
 2485 
 2486             self.nukeAndCreate()
 2487 
 2488             # import
 2489             for cn, items in export.items():
 2490                 klass = self.db.classes[cn]
 2491                 names = items[0]
 2492                 maxid = 1
 2493                 for itemprops in items[1:]:
 2494                     id = int(klass.import_list(names, itemprops))
 2495                     if hasattr(klass, 'import_files'):
 2496                         klass.import_files('_test_export', str(id))
 2497                     maxid = max(maxid, id)
 2498                 self.db.setid(cn, str(maxid+1))
 2499                 klass.import_journals(journals[cn])
 2500             # This is needed, otherwise journals won't be there for anydbm
 2501             self.db.commit()
 2502         finally:
 2503             shutil.rmtree('_test_export')
 2504 
 2505         # compare with snapshot of the database
 2506         for cn, items in orig.items():
 2507             klass = self.db.classes[cn]
 2508             propdefs = klass.getprops(1)
 2509             # ensure retired items are retired :)
 2510             l = sorted(items.keys())
 2511             m = klass.list(); m.sort()
 2512             ae(l, m, '%s id list wrong %r vs. %r'%(cn, l, m))
 2513             for id, props in items.items():
 2514                 for name, value in props.items():
 2515                     l = klass.get(id, name)
 2516                     if isinstance(value, type([])):
 2517                         value.sort()
 2518                         l.sort()
 2519                     try:
 2520                         ae(l, value)
 2521                     except AssertionError:
 2522                         if not isinstance(propdefs[name], Date):
 2523                             raise
 2524                         # don't get hung up on rounding errors
 2525                         assert not l.__cmp__(value, int_seconds=1)
 2526         for jc, items in origj.items():
 2527             for id, oj in items.items():
 2528                 rj = self.db.getjournal(jc, id)
 2529                 # Both mysql and postgresql have some minor issues with
 2530                 # rounded seconds on export/import, so we compare only
 2531                 # the integer part.
 2532                 for j in oj:
 2533                     j[1].second = float(int(j[1].second))
 2534                 for j in rj:
 2535                     j[1].second = float(int(j[1].second))
 2536                 oj.sort(key = NoneAndDictComparable)
 2537                 rj.sort(key = NoneAndDictComparable)
 2538                 ae(oj, rj)
 2539 
 2540         # make sure the retired items are actually imported
 2541         ae(self.db.user.get('4', 'username'), 'blop')
 2542         ae(self.db.issue.get('2', 'title'), 'issue two')
 2543 
 2544         # make sure id counters are set correctly
 2545         maxid = max([int(id) for id in self.db.user.list()])
 2546         newid = int(self.db.user.create(username='testing'))
 2547         assert newid > maxid
 2548 
 2549     # test import/export via admin interface
 2550     def testAdminImportExport(self):
 2551         import roundup.admin
 2552         import csv
 2553         # use the filtering setup to create a bunch of items
 2554         ae, dummy1, dummy2 = self.filteringSetup()
 2555         # create large field
 2556         self.db.priority.create(name = 'X' * 500)
 2557         self.db.config.CSV_FIELD_SIZE = 400
 2558         self.db.commit()
 2559         output = []
 2560         # ugly hack to get stderr output and disable stdout output
 2561         # during regression test. Depends on roundup.admin not using
 2562         # anything but stdout/stderr from sys (which is currently the
 2563         # case)
 2564         def stderrwrite(s):
 2565             output.append(s)
 2566         roundup.admin.sys = MockNull ()
 2567         try:
 2568             roundup.admin.sys.stderr.write = stderrwrite
 2569             tool = roundup.admin.AdminTool()
 2570             home = '.'
 2571             tool.tracker_home = home
 2572             tool.db = self.db
 2573             tool.verbose = False
 2574             tool.do_export (['_test_export'])
 2575             self.assertEqual(len(output), 2)
 2576             self.assertEqual(output [1], '\n')
 2577             self.assertTrue(output [0].startswith
 2578                 ('Warning: config csv_field_size should be at least'))
 2579             self.assertTrue(int(output[0].split()[-1]) > 500)
 2580 
 2581             if hasattr(roundup.admin.csv, 'field_size_limit'):
 2582                 self.nukeAndCreate()
 2583                 self.db.config.CSV_FIELD_SIZE = 400
 2584                 tool = roundup.admin.AdminTool()
 2585                 tool.tracker_home = home
 2586                 tool.db = self.db
 2587                 tool.verbose = False
 2588                 self.assertRaises(csv.Error, tool.do_import, ['_test_export'])
 2589 
 2590             self.nukeAndCreate()
 2591             self.db.config.CSV_FIELD_SIZE = 3200
 2592             tool = roundup.admin.AdminTool()
 2593             tool.tracker_home = home
 2594             tool.db = self.db
 2595             tool.verbose = False
 2596             tool.do_import(['_test_export'])
 2597         finally:
 2598             roundup.admin.sys = sys
 2599             shutil.rmtree('_test_export')
 2600 
 2601     # test props from args parsing
 2602     def testAdminOtherCommands(self):
 2603         import roundup.admin
 2604 
 2605         # use the filtering setup to create a bunch of items
 2606         ae, dummy1, dummy2 = self.filteringSetup()
 2607         # create large field
 2608         self.db.priority.create(name = 'X' * 500)
 2609         self.db.config.CSV_FIELD_SIZE = 400
 2610         self.db.commit()
 2611 
 2612         eoutput = [] # stderr output
 2613         soutput = [] # stdout output
 2614 
 2615         def stderrwrite(s):
 2616             eoutput.append(s)
 2617         def stdoutwrite(s):
 2618             soutput.append(s)
 2619         roundup.admin.sys = MockNull ()
 2620         try:
 2621             roundup.admin.sys.stderr.write = stderrwrite
 2622             roundup.admin.sys.stdout.write = stdoutwrite
 2623 
 2624             tool = roundup.admin.AdminTool()
 2625             home = '.'
 2626             tool.tracker_home = home
 2627             tool.db = self.db
 2628             tool.verbose = False
 2629             tool.separator = "\n"
 2630             tool.print_designator = True
 2631 
 2632             # test props_from_args
 2633             self.assertRaises(UsageError, tool.props_from_args, "fullname") # invalid propname
 2634 
 2635             self.assertEqual(tool.props_from_args("="), {'': None}) # not sure this desired, I'd expect UsageError
 2636 
 2637             props = tool.props_from_args(["fullname=robert", "friends=+rouilj,+other", "key="])
 2638             self.assertEqual(props, {'fullname': 'robert', 'friends': '+rouilj,+other', 'key': None})
 2639 
 2640             # test get_class()
 2641             self.assertRaises(UsageError, tool.get_class, "bar") # invalid class
 2642 
 2643             # This writes to stdout, need to figure out how to redirect to a variable.
 2644             # classhandle = tool.get_class("user") # valid class
 2645             # FIXME there should be some test here
 2646 
 2647             issue_class_spec = tool.do_specification(["issue"])
 2648             self.assertEqual(sorted (soutput),
 2649                              ['assignedto: <roundup.hyperdb.Link to "user">\n',
 2650                               'deadline: <roundup.hyperdb.Date>\n',
 2651                               'feedback: <roundup.hyperdb.Link to "msg">\n',
 2652                               'files: <roundup.hyperdb.Multilink to "file">\n',
 2653                               'foo: <roundup.hyperdb.Interval>\n',
 2654                               'messages: <roundup.hyperdb.Multilink to "msg">\n',
 2655                               'nosy: <roundup.hyperdb.Multilink to "user">\n',
 2656                               'priority: <roundup.hyperdb.Link to "priority">\n',
 2657                               'spam: <roundup.hyperdb.Multilink to "msg">\n',
 2658                               'status: <roundup.hyperdb.Link to "status">\n',
 2659                               'superseder: <roundup.hyperdb.Multilink to "issue">\n',
 2660                               'title: <roundup.hyperdb.String>\n'])
 2661 
 2662             #userclassprop=tool.do_list(["mls"])
 2663             #tool.print_designator = False
 2664             #userclassprop=tool.do_get(["realname","user1"])
 2665 
 2666             # test do_create
 2667             soutput[:] = [] # empty for next round of output
 2668             userclass=tool.do_create(["issue", "title='title1 title'", "nosy=1,3"]) # should be issue 5
 2669             userclass=tool.do_create(["issue", "title='title2 title'", "nosy=2,3"]) # should be issue 6
 2670             self.assertEqual(soutput, ['5\n', '6\n'])
 2671             # verify nosy setting
 2672             props=self.db.issue.get('5', "nosy")
 2673             self.assertEqual(props, ['1','3'])
 2674 
 2675             # test do_set using newly created issues
 2676             # remove user 3 from issues
 2677             # verifies issue2550572
 2678             userclass=tool.do_set(["issue5,issue6", "nosy=-3"])
 2679             # verify proper result
 2680             props=self.db.issue.get('5', "nosy")
 2681             self.assertEqual(props, ['1'])
 2682             props=self.db.issue.get('6', "nosy")
 2683             self.assertEqual(props, ['2'])
 2684 
 2685             # basic usage test. TODO add full output verification
 2686             soutput[:] = [] # empty for next round of output
 2687             tool.usage(message="Hello World")
 2688             self.assertTrue(soutput[0].startswith('Problem: Hello World'), None)
 2689 
 2690             # check security output
 2691             soutput[:] = [] # empty for next round of output
 2692             tool.do_security("Admin")
 2693             expected =  [ 'New Web users get the Role "User"\n',
 2694                           'New Email users get the Role "User"\n',
 2695                           'Role "admin":\n',
 2696                           ' User may create everything (Create)\n',
 2697                           ' User may edit everything (Edit)\n',
 2698                           ' User may restore everything (Restore)\n',
 2699                           ' User may retire everything (Retire)\n',
 2700                           ' User may view everything (View)\n',
 2701                           ' User may access the web interface (Web Access)\n',
 2702                           ' User may access the rest interface (Rest Access)\n',
 2703                           ' User may access the xmlrpc interface (Xmlrpc Access)\n',
 2704                           ' User may manipulate user Roles through the web (Web Roles)\n',
 2705                           ' User may use the email interface (Email Access)\n',
 2706                           'Role "anonymous":\n', 'Role "user":\n',
 2707                           ' User is allowed to access msg (View for "msg" only)\n',
 2708                           ' Prevent users from seeing roles (View for "user": [\'username\', \'supervisor\', \'assignable\'] only)\n']
 2709 
 2710             self.assertEqual(soutput, expected)
 2711 
 2712 
 2713             self.nukeAndCreate()
 2714             tool = roundup.admin.AdminTool()
 2715             tool.tracker_home = home
 2716             tool.db = self.db
 2717             tool.verbose = False
 2718         finally:
 2719             roundup.admin.sys = sys
 2720 
 2721 
 2722     # test duplicate relative tracker home initialisation (issue2550757)
 2723     def testAdminDuplicateInitialisation(self):
 2724         import roundup.admin
 2725         output = []
 2726         def stderrwrite(s):
 2727             output.append(s)
 2728         roundup.admin.sys = MockNull ()
 2729         t = '_test_initialise'
 2730         try:
 2731             roundup.admin.sys.stderr.write = stderrwrite
 2732             tool = roundup.admin.AdminTool()
 2733             tool.force = True
 2734             args = (None, 'classic', 'anydbm',
 2735                     'MAIL_DOMAIN=%s' % config.MAIL_DOMAIN)
 2736             tool.do_install(t, args=args)
 2737             args = (None, 'mypasswd')
 2738             tool.do_initialise(t, args=args)
 2739             tool.do_initialise(t, args=args)
 2740             try:  # python >=2.7
 2741                 self.assertNotIn(t, os.listdir(t))
 2742             except AttributeError:
 2743                 self.assertFalse('db' in os.listdir(t))
 2744         finally:
 2745             roundup.admin.sys = sys
 2746             if os.path.exists(t):
 2747                 shutil.rmtree(t)
 2748 
 2749     def testAddProperty(self):
 2750         self.db.issue.create(title="spam", status='1')
 2751         self.db.commit()
 2752 
 2753         self.db.issue.addprop(fixer=Link("user"))
 2754         # force any post-init stuff to happen
 2755         self.db.post_init()
 2756         props = self.db.issue.getprops()
 2757         keys = sorted(props.keys())
 2758         self.assertEqual(keys, ['activity', 'actor', 'assignedto', 'creation',
 2759             'creator', 'deadline', 'feedback', 'files', 'fixer', 'foo', 'id', 'messages',
 2760             'nosy', 'priority', 'spam', 'status', 'superseder', 'title'])
 2761         self.assertEqual(self.db.issue.get('1', "fixer"), None)
 2762 
 2763     def testRemoveProperty(self):
 2764         self.db.issue.create(title="spam", status='1')
 2765         self.db.commit()
 2766 
 2767         del self.db.issue.properties['title']
 2768         self.db.post_init()
 2769         props = self.db.issue.getprops()
 2770         keys = sorted(props.keys())
 2771         self.assertEqual(keys, ['activity', 'actor', 'assignedto', 'creation',
 2772             'creator', 'deadline', 'feedback', 'files', 'foo', 'id', 'messages',
 2773             'nosy', 'priority', 'spam', 'status', 'superseder'])
 2774         self.assertEqual(self.db.issue.list(), ['1'])
 2775 
 2776     def testAddRemoveProperty(self):
 2777         self.db.issue.create(title="spam", status='1')
 2778         self.db.commit()
 2779 
 2780         self.db.issue.addprop(fixer=Link("user"))
 2781         del self.db.issue.properties['title']
 2782         self.db.post_init()
 2783         props = self.db.issue.getprops()
 2784         keys = sorted(props.keys())
 2785         self.assertEqual(keys, ['activity', 'actor', 'assignedto', 'creation',
 2786             'creator', 'deadline', 'feedback', 'files', 'fixer', 'foo', 'id',
 2787             'messages', 'nosy', 'priority', 'spam', 'status', 'superseder'])
 2788         self.assertEqual(self.db.issue.list(), ['1'])
 2789 
 2790     def testNosyMail(self) :
 2791         """Creates one issue with two attachments, one smaller and one larger
 2792            than the set max_attachment_size.
 2793         """
 2794         old_translate_ = roundupdb._
 2795         roundupdb._ = i18n.get_translation(language='C').gettext
 2796         db = self.db
 2797         db.config.NOSY_MAX_ATTACHMENT_SIZE = 4096
 2798         res = dict(mail_to = None, mail_msg = None)
 2799         def dummy_snd(s, to, msg, res=res) :
 2800             res["mail_to"], res["mail_msg"] = to, msg
 2801         backup, Mailer.smtp_send = Mailer.smtp_send, dummy_snd
 2802         try :
 2803             f1 = db.file.create(name="test1.txt", content="x" * 20, type="application/octet-stream")
 2804             f2 = db.file.create(name="test2.txt", content="y" * 5000, type="application/octet-stream")
 2805             m  = db.msg.create(content="one two", author="admin",
 2806                 files = [f1, f2])
 2807             i  = db.issue.create(title='spam', files = [f1, f2],
 2808                 messages = [m], nosy = [db.user.lookup("fred")])
 2809 
 2810             db.issue.nosymessage(i, m, {})
 2811             mail_msg = str(res["mail_msg"])
 2812             self.assertEqual(res["mail_to"], ["fred@example.com"])
 2813             self.assertTrue("From: admin" in mail_msg)
 2814             self.assertTrue("Subject: [issue1] spam" in mail_msg)
 2815             self.assertTrue("New submission from admin" in mail_msg)
 2816             self.assertTrue("one two" in mail_msg)
 2817             self.assertTrue("File 'test1.txt' not attached" not in mail_msg)
 2818             self.assertTrue(b2s(base64_encode(s2b("xxx"))).rstrip() in mail_msg)
 2819             self.assertTrue("File 'test2.txt' not attached" in mail_msg)
 2820             self.assertTrue(b2s(base64_encode(s2b("yyy"))).rstrip() not in mail_msg)
 2821         finally :
 2822             roundupdb._ = old_translate_
 2823             Mailer.smtp_send = backup
 2824 
 2825     def testNosyMailTextAndBinary(self) :
 2826         """Creates one issue with two attachments, one as text and one as binary.
 2827         """
 2828         old_translate_ = roundupdb._
 2829         roundupdb._ = i18n.get_translation(language='C').gettext
 2830         db = self.db
 2831         res = dict(mail_to = None, mail_msg = None)
 2832         def dummy_snd(s, to, msg, res=res) :
 2833             res["mail_to"], res["mail_msg"] = to, msg
 2834         backup, Mailer.smtp_send = Mailer.smtp_send, dummy_snd
 2835         try :
 2836             f1 = db.file.create(name="test1.txt", content="Hello world", type="text/plain")
 2837             f2 = db.file.create(name="test2.bin", content=b"\x01\x02\x03\xfe\xff", type="application/octet-stream")
 2838             m  = db.msg.create(content="one two", author="admin",
 2839                 files = [f1, f2])
 2840             i  = db.issue.create(title='spam', files = [f1, f2],
 2841                 messages = [m], nosy = [db.user.lookup("fred")])
 2842 
 2843             db.issue.nosymessage(i, m, {})
 2844             mail_msg = str(res["mail_msg"])
 2845             self.assertEqual(res["mail_to"], ["fred@example.com"])
 2846             self.assertTrue("From: admin" in mail_msg)
 2847             self.assertTrue("Subject: [issue1] spam" in mail_msg)
 2848             self.assertTrue("New submission from admin" in mail_msg)
 2849             self.assertTrue("one two" in mail_msg)
 2850             self.assertTrue("Hello world" in mail_msg)
 2851             self.assertTrue(b2s(base64_encode(b"\x01\x02\x03\xfe\xff")).rstrip() in mail_msg)
 2852         finally :
 2853             roundupdb._ = old_translate_
 2854             Mailer.smtp_send = backup
 2855 
 2856     @pytest.mark.skipif(gpgmelib.gpg is None, reason='Skipping PGPNosy test')
 2857     def testPGPNosyMail(self) :
 2858         """Creates one issue with two attachments, one smaller and one larger
 2859            than the set max_attachment_size. Recipients are one with and
 2860            one without encryption enabled via a gpg group.
 2861         """
 2862         old_translate_ = roundupdb._
 2863         roundupdb._ = i18n.get_translation(language='C').gettext
 2864         db = self.db
 2865         db.config.NOSY_MAX_ATTACHMENT_SIZE = 4096
 2866         db.config['PGP_HOMEDIR'] = gpgmelib.pgphome
 2867         db.config['PGP_ROLES'] = 'pgp'
 2868         db.config['PGP_ENABLE'] = True
 2869         db.config['PGP_ENCRYPT'] = True
 2870         gpgmelib.setUpPGP()
 2871         res = []
 2872         def dummy_snd(s, to, msg, res=res) :
 2873             res.append (dict (mail_to = to, mail_msg = msg))
 2874         backup, Mailer.smtp_send = Mailer.smtp_send, dummy_snd
 2875         try :
 2876             john = db.user.create(username="john", roles='User,pgp',
 2877                 address='john@test.test', realname='John Doe')
 2878             f1 = db.file.create(name="test1.txt", content="x" * 20, type="application/octet-stream")
 2879             f2 = db.file.create(name="test2.txt", content="y" * 5000, type="application/octet-stream")
 2880             m  = db.msg.create(content="one two", author="admin",
 2881                 files = [f1, f2])
 2882             i  = db.issue.create(title='spam', files = [f1, f2],
 2883                 messages = [m], nosy = [db.user.lookup("fred"), john])
 2884 
 2885             db.issue.nosymessage(i, m, {})
 2886             res.sort(key=lambda x: x['mail_to'])
 2887             self.assertEqual(res[0]["mail_to"], ["fred@example.com"])
 2888             self.assertEqual(res[1]["mail_to"], ["john@test.test"])
 2889             mail_msg = str(res[0]["mail_msg"])
 2890             self.assertTrue("From: admin" in mail_msg)
 2891             self.assertTrue("Subject: [issue1] spam" in mail_msg)
 2892             self.assertTrue("New submission from admin" in mail_msg)
 2893             self.assertTrue("one two" in mail_msg)
 2894             self.assertTrue("File 'test1.txt' not attached" not in mail_msg)
 2895             self.assertTrue(b2s(base64_encode(s2b("xxx"))).rstrip() in mail_msg)
 2896             self.assertTrue("File 'test2.txt' not attached" in mail_msg)
 2897             self.assertTrue(b2s(base64_encode(s2b("yyy"))).rstrip() not in mail_msg)
 2898             mail_msg = str(res[1]["mail_msg"])
 2899             parts = message_from_string(mail_msg).get_payload()
 2900             self.assertEqual(len(parts),2)
 2901             self.assertEqual(parts[0].get_payload().strip(), 'Version: 1')
 2902             crypt = gpgmelib.gpg.core.Data(parts[1].get_payload())
 2903             plain = gpgmelib.gpg.core.Data()
 2904             ctx = gpgmelib.gpg.core.Context()
 2905             res = ctx.op_decrypt(crypt, plain)
 2906             self.assertEqual(res, None)
 2907             plain.seek(0,0)
 2908             self.assertTrue("From: admin" in mail_msg)
 2909             self.assertTrue("Subject: [issue1] spam" in mail_msg)
 2910             mail_msg = str(message_from_bytes(plain.read()))
 2911             self.assertTrue("New submission from admin" in mail_msg)
 2912             self.assertTrue("one two" in mail_msg)
 2913             self.assertTrue("File 'test1.txt' not attached" not in mail_msg)
 2914             self.assertTrue(b2s(base64_encode(s2b("xxx"))).rstrip() in mail_msg)
 2915             self.assertTrue("File 'test2.txt' not attached" in mail_msg)
 2916             self.assertTrue(b2s(base64_encode(s2b("yyy"))).rstrip() not in mail_msg)
 2917         finally :
 2918             roundupdb._ = old_translate_
 2919             Mailer.smtp_send = backup
 2920             gpgmelib.tearDownPGP()
 2921 
 2922 class ROTest(MyTestCase):
 2923     def setUp(self):
 2924         # remove previous test, ignore errors
 2925         if os.path.exists(config.DATABASE):
 2926             shutil.rmtree(config.DATABASE)
 2927         os.makedirs(config.DATABASE + '/files')
 2928         self.db = self.module.Database(config, 'admin')
 2929         setupSchema(self.db, 1, self.module)
 2930         self.db.close()
 2931 
 2932         self.db = self.module.Database(config)
 2933         setupSchema(self.db, 0, self.module)
 2934 
 2935     def testExceptions(self):
 2936         # this tests the exceptions that should be raised
 2937         ar = self.assertRaises
 2938 
 2939         # this tests the exceptions that should be raised
 2940         ar(DatabaseError, self.db.status.create, name="foo")
 2941         ar(DatabaseError, self.db.status.set, '1', name="foo")
 2942         ar(DatabaseError, self.db.status.retire, '1')
 2943 
 2944 
 2945 class SchemaTest(MyTestCase):
 2946     def setUp(self):
 2947         # remove previous test, ignore errors
 2948         if os.path.exists(config.DATABASE):
 2949             shutil.rmtree(config.DATABASE)
 2950         os.makedirs(config.DATABASE + '/files')
 2951 
 2952     def test_reservedProperties(self):
 2953         self.open_database()
 2954         self.assertRaises(ValueError, self.module.Class, self.db, "a",
 2955             creation=String())
 2956         self.assertRaises(ValueError, self.module.Class, self.db, "a",
 2957             activity=String())
 2958         self.assertRaises(ValueError, self.module.Class, self.db, "a",
 2959             creator=String())
 2960         self.assertRaises(ValueError, self.module.Class, self.db, "a",
 2961             actor=String())
 2962 
 2963     def init_a(self):
 2964         self.open_database()
 2965         a = self.module.Class(self.db, "a", name=String())
 2966         a.setkey("name")
 2967         self.db.post_init()
 2968 
 2969     def test_fileClassProps(self):
 2970         self.open_database()
 2971         a = self.module.FileClass(self.db, 'a')
 2972         l = sorted(a.getprops().keys())
 2973         self.assertTrue(l, ['activity', 'actor', 'content', 'created',
 2974             'creation', 'type'])
 2975 
 2976     def init_ab(self):
 2977         self.open_database()
 2978         a = self.module.Class(self.db, "a", name=String())
 2979         a.setkey("name")
 2980         b = self.module.Class(self.db, "b", name=String(),
 2981             fooz=Multilink('a'))
 2982         b.setkey("name")
 2983         self.db.post_init()
 2984 
 2985     def test_addNewClass(self):
 2986         self.init_a()
 2987 
 2988         self.assertRaises(ValueError, self.module.Class, self.db, "a",
 2989             name=String())
 2990 
 2991         aid = self.db.a.create(name='apple')
 2992         self.db.commit(); self.db.close()
 2993 
 2994         # add a new class to the schema and check creation of new items
 2995         # (and existence of old ones)
 2996         self.init_ab()
 2997         bid = self.db.b.create(name='bear', fooz=[aid])
 2998         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 2999         self.db.commit()
 3000         self.db.close()
 3001 
 3002         # now check we can recall the added class' items
 3003         self.init_ab()
 3004         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3005         self.assertEqual(self.db.a.lookup('apple'), aid)
 3006         self.assertEqual(self.db.b.get(bid, 'name'), 'bear')
 3007         self.assertEqual(self.db.b.get(bid, 'fooz'), [aid])
 3008         self.assertEqual(self.db.b.lookup('bear'), bid)
 3009 
 3010         # confirm journal's ok
 3011         self.db.getjournal('a', aid)
 3012         self.db.getjournal('b', bid)
 3013 
 3014     def init_amod(self):
 3015         self.open_database()
 3016         a = self.module.Class(self.db, "a", name=String(), newstr=String(),
 3017             newint=Interval(), newnum=Number(), newbool=Boolean(),
 3018             newdate=Date())
 3019         a.setkey("name")
 3020         b = self.module.Class(self.db, "b", name=String())
 3021         b.setkey("name")
 3022         self.db.post_init()
 3023 
 3024     def test_modifyClass(self):
 3025         self.init_ab()
 3026 
 3027         # add item to user and issue class
 3028         aid = self.db.a.create(name='apple')
 3029         bid = self.db.b.create(name='bear')
 3030         self.db.commit(); self.db.close()
 3031 
 3032         # modify "a" schema
 3033         self.init_amod()
 3034         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3035         self.assertEqual(self.db.a.get(aid, 'newstr'), None)
 3036         self.assertEqual(self.db.a.get(aid, 'newint'), None)
 3037         # hack - metakit can't return None for missing values, and we're not
 3038         # really checking for that behavior here anyway
 3039         self.assertTrue(not self.db.a.get(aid, 'newnum'))
 3040         self.assertTrue(not self.db.a.get(aid, 'newbool'))
 3041         self.assertEqual(self.db.a.get(aid, 'newdate'), None)
 3042         self.assertEqual(self.db.b.get(aid, 'name'), 'bear')
 3043         aid2 = self.db.a.create(name='aardvark', newstr='booz')
 3044         self.db.commit(); self.db.close()
 3045 
 3046         # test
 3047         self.init_amod()
 3048         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3049         self.assertEqual(self.db.a.get(aid, 'newstr'), None)
 3050         self.assertEqual(self.db.b.get(aid, 'name'), 'bear')
 3051         self.assertEqual(self.db.a.get(aid2, 'name'), 'aardvark')
 3052         self.assertEqual(self.db.a.get(aid2, 'newstr'), 'booz')
 3053 
 3054         # confirm journal's ok
 3055         self.db.getjournal('a', aid)
 3056         self.db.getjournal('a', aid2)
 3057 
 3058     def init_amodkey(self):
 3059         self.open_database()
 3060         a = self.module.Class(self.db, "a", name=String(), newstr=String())
 3061         a.setkey("newstr")
 3062         b = self.module.Class(self.db, "b", name=String())
 3063         b.setkey("name")
 3064         self.db.post_init()
 3065 
 3066     def test_changeClassKey(self):
 3067         self.init_amod()
 3068         aid = self.db.a.create(name='apple')
 3069         self.assertEqual(self.db.a.lookup('apple'), aid)
 3070         self.db.commit(); self.db.close()
 3071 
 3072         # change the key to newstr on a
 3073         self.init_amodkey()
 3074         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3075         self.assertEqual(self.db.a.get(aid, 'newstr'), None)
 3076         self.assertRaises(KeyError, self.db.a.lookup, 'apple')
 3077         aid2 = self.db.a.create(name='aardvark', newstr='booz')
 3078         self.db.commit(); self.db.close()
 3079 
 3080         # check
 3081         self.init_amodkey()
 3082         self.assertEqual(self.db.a.lookup('booz'), aid2)
 3083 
 3084         # confirm journal's ok
 3085         self.db.getjournal('a', aid)
 3086 
 3087     def test_removeClassKey(self):
 3088         self.init_amod()
 3089         aid = self.db.a.create(name='apple')
 3090         self.assertEqual(self.db.a.lookup('apple'), aid)
 3091         self.db.commit(); self.db.close()
 3092 
 3093         self.db = self.module.Database(config, 'admin')
 3094         a = self.module.Class(self.db, "a", name=String(), newstr=String())
 3095         self.db.post_init()
 3096 
 3097         aid2 = self.db.a.create(name='apple', newstr='booz')
 3098         self.db.commit()
 3099 
 3100 
 3101     def init_amodml(self):
 3102         self.open_database()
 3103         a = self.module.Class(self.db, "a", name=String(),
 3104             newml=Multilink('a'))
 3105         a.setkey('name')
 3106         self.db.post_init()
 3107 
 3108     def test_makeNewMultilink(self):
 3109         self.init_a()
 3110         aid = self.db.a.create(name='apple')
 3111         self.assertEqual(self.db.a.lookup('apple'), aid)
 3112         self.db.commit(); self.db.close()
 3113 
 3114         # add a multilink prop
 3115         self.init_amodml()
 3116         bid = self.db.a.create(name='bear', newml=[aid])
 3117         self.assertEqual(self.db.a.find(newml=aid), [bid])
 3118         self.assertEqual(self.db.a.lookup('apple'), aid)
 3119         self.db.commit(); self.db.close()
 3120 
 3121         # check
 3122         self.init_amodml()
 3123         self.assertEqual(self.db.a.find(newml=aid), [bid])
 3124         self.assertEqual(self.db.a.lookup('apple'), aid)
 3125         self.assertEqual(self.db.a.lookup('bear'), bid)
 3126 
 3127         # confirm journal's ok
 3128         self.db.getjournal('a', aid)
 3129         self.db.getjournal('a', bid)
 3130 
 3131     def test_removeMultilink(self):
 3132         # add a multilink prop
 3133         self.init_amodml()
 3134         aid = self.db.a.create(name='apple')
 3135         bid = self.db.a.create(name='bear', newml=[aid])
 3136         self.assertEqual(self.db.a.find(newml=aid), [bid])
 3137         self.assertEqual(self.db.a.lookup('apple'), aid)
 3138         self.assertEqual(self.db.a.lookup('bear'), bid)
 3139         self.db.commit(); self.db.close()
 3140 
 3141         # remove the multilink
 3142         self.init_a()
 3143         self.assertEqual(self.db.a.lookup('apple'), aid)
 3144         self.assertEqual(self.db.a.lookup('bear'), bid)
 3145 
 3146         # confirm journal's ok
 3147         self.db.getjournal('a', aid)
 3148         self.db.getjournal('a', bid)
 3149 
 3150     def test_removeClass(self):
 3151         self.init_ab()
 3152         aid = self.db.a.create(name='apple')
 3153         bid = self.db.b.create(name='bear')
 3154         self.db.commit(); self.db.close()
 3155 
 3156         # drop the b class
 3157         self.init_a()
 3158         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3159         self.assertEqual(self.db.a.lookup('apple'), aid)
 3160         self.db.commit(); self.db.close()
 3161 
 3162         # now check we can recall the added class' items
 3163         self.init_a()
 3164         self.assertEqual(self.db.a.get(aid, 'name'), 'apple')
 3165         self.assertEqual(self.db.a.lookup('apple'), aid)
 3166 
 3167         # confirm journal's ok
 3168         self.db.getjournal('a', aid)
 3169 
 3170 class RDBMSTest:
 3171     """ tests specific to RDBMS backends """
 3172     def test_indexTest(self):
 3173         self.assertEqual(self.db.sql_index_exists('_issue', '_issue_id_idx'), 1)
 3174         self.assertEqual(self.db.sql_index_exists('_issue', '_issue_x_idx'), 0)
 3175 
 3176 class FilterCacheTest(commonDBTest):
 3177     def testFilteringTransitiveLinkCache(self):
 3178         ae, filter, filter_iter = self.filteringSetupTransitiveSearch()
 3179         ae, ufilter, ufilter_iter = self.iterSetup('user')
 3180         # Need to make ceo his own (and first two users') supervisor
 3181         self.db.user.set('1', supervisor = '3')
 3182         self.db.user.set('2', supervisor = '3')
 3183         self.db.user.set('3', supervisor = '3')
 3184         # test bool value
 3185         self.db.user.set('4', assignable = True)
 3186         self.db.user.set('3', assignable = False)
 3187         filt = self.db.issue.filter_iter
 3188         ufilt = self.db.user.filter_iter
 3189         user_result = \
 3190             {  '1' : {'username': 'admin', 'assignable': None,
 3191                       'supervisor': '3', 'realname': None, 'roles': 'Admin',
 3192                       'creator': '1', 'age': None, 'actor': '1',
 3193                       'address': None}
 3194             ,  '2' : {'username': 'fred', 'assignable': None,
 3195                       'supervisor': '3', 'realname': None, 'roles': 'User',
 3196                       'creator': '1', 'age': None, 'actor': '1',
 3197                       'address': 'fred@example.com'}
 3198             ,  '3' : {'username': 'ceo', 'assignable': False,
 3199                       'supervisor': '3', 'realname': None, 'roles': None,
 3200                       'creator': '1', 'age': 129.0, 'actor': '1',
 3201                       'address': None}
 3202             ,  '4' : {'username': 'grouplead1', 'assignable': True,
 3203                       'supervisor': '3', 'realname': None, 'roles': None,
 3204                       'creator': '1', 'age': 29.0, 'actor': '1',
 3205                       'address': None}
 3206             ,  '5' : {'username': 'grouplead2', 'assignable': None,
 3207                       'supervisor': '3', 'realname': None, 'roles': None,
 3208                       'creator': '1', 'age': 29.0, 'actor': '1',
 3209                       'address': None}
 3210             ,  '6' : {'username': 'worker1', 'assignable': None,
 3211                       'supervisor': '4', 'realname': None, 'roles': None,
 3212                       'creator': '1', 'age': 25.0, 'actor': '1',
 3213                       'address': None}
 3214             ,  '7' : {'username': 'worker2', 'assignable': None,
 3215                       'supervisor': '4', 'realname': None, 'roles': None,
 3216                       'creator': '1', 'age': 24.0, 'actor': '1',
 3217                       'address': None}
 3218             ,  '8' : {'username': 'worker3', 'assignable': None,
 3219                       'supervisor': '5', 'realname': None, 'roles': None,
 3220                       'creator': '1', 'age': 23.0, 'actor': '1',
 3221                       'address': None}
 3222             ,  '9' : {'username': 'worker4', 'assignable': None,
 3223                       'supervisor': '5', 'realname': None, 'roles': None,
 3224                       'creator': '1', 'age': 22.0, 'actor': '1',
 3225                       'address': None}
 3226             , '10' : {'username': 'worker5', 'assignable': None,
 3227                       'supervisor': '5', 'realname': None, 'roles': None,
 3228                       'creator': '1', 'age': 21.0, 'actor': '1',
 3229                       'address': None}
 3230             }
 3231         foo = date.Interval('-1d')
 3232         issue_result = \
 3233             { '1' : {'title': 'ts1', 'status': '2', 'assignedto': '6',
 3234                      'priority': '3', 'messages' : ['4'], 'nosy' : ['4']}
 3235             , '2' : {'title': 'ts2', 'status': '1', 'assignedto': '6',
 3236                      'priority': '3', 'messages' : ['4'], 'nosy' : ['5']}
 3237             , '3' : {'title': 'ts4', 'status': '2', 'assignedto': '7',
 3238                      'priority': '3', 'messages' : ['5']}
 3239             , '4' : {'title': 'ts5', 'status': '1', 'assignedto': '8',
 3240                      'priority': '3', 'messages' : ['6']}
 3241             , '5' : {'title': 'ts6', 'status': '2', 'assignedto': '9',
 3242                      'priority': '3', 'messages' : ['7']}
 3243             , '6' : {'title': 'ts7', 'status': '1', 'assignedto': '10',
 3244                      'priority': '3', 'messages' : ['8'], 'foo' : None}
 3245             , '7' : {'title': 'ts8', 'status': '2', 'assignedto': '10',
 3246                      'priority': '3', 'messages' : ['8'], 'foo' : foo}
 3247             , '8' : {'title': 'ts9', 'status': '1', 'assignedto': '10',
 3248                      'priority': '3', 'messages' : ['7', '8']}
 3249             }
 3250         result = []
 3251         self.db.clearCache()
 3252         for id in ufilt(None, {}, [('+','supervisor.supervisor.supervisor'),
 3253             ('-','supervisor.supervisor'), ('-','supervisor'),
 3254             ('+','username')]):
 3255             result.append(id)
 3256             nodeid = id
 3257             for x in range(4):
 3258                 assert(('user', nodeid) in self.db.cache)
 3259                 n = self.db.user.getnode(nodeid)
 3260                 for k, v in user_result[nodeid].items():
 3261                     ae((k, n[k]), (k, v))
 3262                 for k in 'creation', 'activity':
 3263                     assert(n[k])
 3264                 nodeid = n.supervisor
 3265             self.db.clearCache()
 3266         ae (result, ['8', '9', '10', '6', '7', '1', '3', '2', '4', '5'])
 3267 
 3268         result = []
 3269         self.db.clearCache()
 3270         for id in filt(None, {},
 3271             [('+','assignedto.supervisor.supervisor.supervisor'),
 3272             ('+','assignedto.supervisor.supervisor'),
 3273             ('-','assignedto.supervisor'), ('+','assignedto')]):
 3274             result.append(id)
 3275             assert(('issue', id) in self.db.cache)
 3276             n = self.db.issue.getnode(id)
 3277             for k, v in issue_result[id].items():
 3278                 ae((k, n[k]), (k, v))
 3279             for k in 'creation', 'activity':
 3280                 assert(n[k])
 3281             nodeid = n.assignedto
 3282             for x in range(4):
 3283                 assert(('user', nodeid) in self.db.cache)
 3284                 n = self.db.user.getnode(nodeid)
 3285                 for k, v in user_result[nodeid].items():
 3286                     ae((k, n[k]), (k, v))
 3287                 for k in 'creation', 'activity':
 3288                     assert(n[k])
 3289                 nodeid = n.supervisor
 3290             self.db.clearCache()
 3291         ae (result, ['4', '5', '6', '7', '8', '1', '2', '3'])
 3292 
 3293 
 3294 class ClassicInitBase(object):
 3295     count = 0
 3296     db = None
 3297 
 3298     def setUp(self):
 3299         ClassicInitBase.count = ClassicInitBase.count + 1
 3300         self.dirname = '_test_init_%s'%self.count
 3301         try:
 3302             shutil.rmtree(self.dirname)
 3303         except OSError as error:
 3304             if error.errno not in (errno.ENOENT, errno.ESRCH): raise
 3305 
 3306     def tearDown(self):
 3307         if self.db is not None:
 3308             self.db.close()
 3309         try:
 3310             shutil.rmtree(self.dirname)
 3311         except OSError as error:
 3312             if error.errno not in (errno.ENOENT, errno.ESRCH): raise
 3313 
 3314 class ClassicInitTest(ClassicInitBase):
 3315     def testCreation(self):
 3316         ae = self.assertEqual
 3317 
 3318         # set up and open a tracker
 3319         tracker = setupTracker(self.dirname, self.backend)
 3320         # open the database
 3321         db = self.db = tracker.open('test')
 3322 
 3323         # check the basics of the schema and initial data set
 3324         l = db.priority.list()
 3325         l.sort()
 3326         ae(l, ['1', '2', '3', '4', '5'])
 3327         l = db.status.list()
 3328         l.sort()
 3329         ae(l, ['1', '2', '3', '4', '5', '6', '7', '8'])
 3330         l = db.keyword.list()
 3331         ae(l, [])
 3332         l = db.user.list()
 3333         l.sort()
 3334         ae(l, ['1', '2'])
 3335         l = db.msg.list()
 3336         ae(l, [])
 3337         l = db.file.list()
 3338         ae(l, [])
 3339         l = db.issue.list()
 3340         ae(l, [])
 3341 
 3342 
 3343 class ConcurrentDBTest(ClassicInitBase):
 3344     def testConcurrency(self):
 3345         # The idea here is a read-modify-update cycle in the presence of
 3346         # a cache that has to be properly handled. The same applies if
 3347         # we extend a String or otherwise modify something that depends
 3348         # on the previous value.
 3349 
 3350         # set up and open a tracker
 3351         tracker = setupTracker(self.dirname, self.backend)
 3352         # open the database
 3353         self.db = tracker.open('admin')
 3354 
 3355         prio = '1'
 3356         self.assertEqual(self.db.priority.get(prio, 'order'), 1.0)
 3357         def inc(db):
 3358             db.priority.set(prio, order=db.priority.get(prio, 'order') + 1)
 3359 
 3360         inc(self.db)
 3361 
 3362         db2 = tracker.open("admin")
 3363         self.assertEqual(db2.priority.get(prio, 'order'), 1.0)
 3364         db2.commit()
 3365         self.db.commit()
 3366         self.assertEqual(self.db.priority.get(prio, 'order'), 2.0)
 3367 
 3368         inc(db2)
 3369         db2.commit()
 3370         db2.clearCache()
 3371         self.assertEqual(db2.priority.get(prio, 'order'), 3.0)
 3372         db2.close()
 3373 
 3374 class HTMLItemTest(ClassicInitBase):
 3375     class Request :
 3376         """ Fake html request """
 3377         rfile = None
 3378         def start_response (self, a, b) :
 3379             pass
 3380         # end def start_response
 3381     # end class Request
 3382 
 3383     def setUp(self):
 3384         super(HTMLItemTest, self).setUp()
 3385         self.tracker = tracker = setupTracker(self.dirname, self.backend)
 3386         db = self.db = tracker.open('admin')
 3387         req = self.Request()
 3388         env = dict (PATH_INFO='', REQUEST_METHOD='GET', QUERY_STRING='')
 3389         self.client = self.tracker.Client(self.tracker, req, env, None)
 3390         self.client.db = db
 3391         self.client.language = None
 3392         self.client.userid = db.getuid()
 3393         self.client.classname = 'issue'
 3394         user = {'username': 'worker5', 'realname': 'Worker', 'roles': 'User'}
 3395         u = self.db.user.create(**user)
 3396         u_m = self.db.msg.create(author = u, content = 'bla'
 3397             , date = date.Date ('2006-01-01'))
 3398         issue = {'title': 'ts1', 'status': '2', 'assignedto': '3',
 3399                 'priority': '3', 'messages' : [u_m], 'nosy' : ['3']}
 3400         self.db.issue.create(**issue)
 3401         issue = {'title': 'ts2', 'status': '2',
 3402                 'messages' : [u_m], 'nosy' : ['3']}
 3403         self.db.issue.create(**issue)
 3404 
 3405     def testHTMLItemAttributes(self):
 3406         issue = HTMLItem(self.client, 'issue', '1')
 3407         ae = self.assertEqual
 3408         ae(issue.title.plain(),'ts1')
 3409         ae(issue ['title'].plain(),'ts1')
 3410         ae(issue.status.plain(),'deferred')
 3411         ae(issue ['status'].plain(),'deferred')
 3412         ae(issue.assignedto.plain(),'worker5')
 3413         ae(issue ['assignedto'].plain(),'worker5')
 3414         ae(issue.priority.plain(),'bug')
 3415         ae(issue ['priority'].plain(),'bug')
 3416         ae(issue.messages.plain(),'1')
 3417         ae(issue ['messages'].plain(),'1')
 3418         ae(issue.nosy.plain(),'worker5')
 3419         ae(issue ['nosy'].plain(),'worker5')
 3420         ae(len(issue.messages),1)
 3421         ae(len(issue ['messages']),1)
 3422         ae(len(issue.nosy),1)
 3423         ae(len(issue ['nosy']),1)
 3424 
 3425     def testHTMLItemDereference(self):
 3426         issue = HTMLItem(self.client, 'issue', '1')
 3427         ae = self.assertEqual
 3428         ae(str(issue.priority.name),'bug')
 3429         ae(str(issue.priority['name']),'bug')
 3430         ae(str(issue ['priority']['name']),'bug')
 3431         ae(str(issue ['priority'].name),'bug')
 3432