"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/test/unit/test_changes_p4poller.py" (23 May 2019, 18472 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "test_changes_p4poller.py": 2.1.0_vs_2.2.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 import datetime
   17 
   18 import dateutil.tz
   19 
   20 from twisted.internet import defer
   21 from twisted.internet import error
   22 from twisted.internet import reactor
   23 from twisted.python import failure
   24 from twisted.trial import unittest
   25 
   26 from buildbot.changes.p4poller import P4PollerError
   27 from buildbot.changes.p4poller import P4Source
   28 from buildbot.changes.p4poller import get_simple_split
   29 from buildbot.test.util import changesource
   30 from buildbot.test.util import config
   31 from buildbot.test.util import gpo
   32 from buildbot.test.util.misc import TestReactorMixin
   33 from buildbot.util import datetime2epoch
   34 
   35 first_p4changes = \
   36     b"""Change 1 on 2006/04/13 by slamb@testclient 'first rev'
   37 """
   38 
   39 second_p4changes = \
   40     b"""Change 3 on 2006/04/13 by bob@testclient 'short desc truncated'
   41 Change 2 on 2006/04/13 by slamb@testclient 'bar'
   42 """
   43 
   44 third_p4changes = \
   45     b"""Change 5 on 2006/04/13 by mpatel@testclient 'first rev'
   46 """
   47 
   48 fourth_p4changes = \
   49     b"""Change 6 on 2006/04/14 by mpatel@testclient 'bar \xd0\x91'
   50 """
   51 
   52 p4_describe_2 = \
   53     b"""Change 2 by slamb@testclient on 2006/04/13 21:46:23
   54 
   55 \tcreation
   56 
   57 Affected files ...
   58 
   59 ... //depot/myproject/trunk/whatbranch#1 add
   60 ... //depot/otherproject/trunk/something#1 add
   61 """
   62 
   63 p4_describe_3 = \
   64     """Change 3 by bob@testclient on 2006/04/13 21:51:39
   65 
   66 \tshort desc truncated because this is a long description.
   67 \tASDF-GUI-P3-\u2018Upgrade Icon\u2019 disappears sometimes.
   68 
   69 Affected files ...
   70 
   71 ... //depot/myproject/branch_b/branch_b_file#1 add
   72 ... //depot/myproject/branch_b/whatbranch#1 branch
   73 ... //depot/myproject/branch_c/whatbranch#1 branch
   74 """
   75 
   76 p4_describe_4 = \
   77     b"""Change 4 by mpatel@testclient on 2006/04/13 21:55:39
   78 
   79 \tThis is a multiline comment with tabs and spaces
   80 \t
   81 \tA list:
   82 \t  Item 1
   83 \t\tItem 2
   84 
   85 Affected files ...
   86 
   87 ... //depot/myproject/branch_b/branch_b_file#1 add
   88 ... //depot/myproject/branch_b#75 edit
   89 ... //depot/myproject/branch_c/branch_c_file#1 add
   90 """
   91 
   92 p4change = {
   93     3: p4_describe_3,
   94     2: p4_describe_2,
   95     5: p4_describe_4,
   96 }
   97 
   98 
   99 class FakeTransport:
  100 
  101     def __init__(self):
  102         self.msg = None
  103 
  104     def write(self, msg):
  105         self.msg = msg
  106 
  107     def closeStdin(self):
  108         pass
  109 
  110 
  111 class TestP4Poller(changesource.ChangeSourceMixin,
  112                    gpo.GetProcessOutputMixin,
  113                    config.ConfigErrorsMixin,
  114                    TestReactorMixin,
  115                    unittest.TestCase):
  116 
  117     def setUp(self):
  118         self.setUpTestReactor()
  119         self.setUpGetProcessOutput()
  120         return self.setUpChangeSource()
  121 
  122     def tearDown(self):
  123         return self.tearDownChangeSource()
  124 
  125     def add_p4_describe_result(self, number, result):
  126         self.expectCommands(
  127             gpo.Expect('p4', 'describe', '-s', str(number)).stdout(result))
  128 
  129     def makeTime(self, timestring):
  130         datefmt = '%Y/%m/%d %H:%M:%S'
  131         when = datetime.datetime.strptime(timestring, datefmt)
  132         return when
  133 
  134     # tests
  135 
  136     def test_describe(self):
  137         self.attachChangeSource(
  138             P4Source(p4port=None, p4user=None,
  139                      p4base='//depot/myproject/',
  140                      split_file=lambda x: x.split('/', 1)))
  141         self.assertSubstring("p4source", self.changesource.describe())
  142 
  143     def test_name(self):
  144         # no name:
  145         cs1 = P4Source(p4port=None, p4user=None,
  146                        p4base='//depot/myproject/',
  147                        split_file=lambda x: x.split('/', 1))
  148         self.assertEqual("P4Source:None://depot/myproject/", cs1.name)
  149 
  150         # explicit name:
  151         cs2 = P4Source(p4port=None, p4user=None, name='MyName',
  152                        p4base='//depot/myproject/',
  153                        split_file=lambda x: x.split('/', 1))
  154         self.assertEqual("MyName", cs2.name)
  155 
  156     @defer.inlineCallbacks
  157     def do_test_poll_successful(self, **kwargs):
  158         encoding = kwargs.get('encoding', 'utf8')
  159         self.attachChangeSource(
  160             P4Source(p4port=None, p4user=None,
  161                      p4base='//depot/myproject/',
  162                      split_file=lambda x: x.split('/', 1),
  163                      **kwargs))
  164         self.expectCommands(
  165             gpo.Expect(
  166                 'p4', 'changes', '-m', '1', '//depot/myproject/...').stdout(first_p4changes),
  167             gpo.Expect(
  168                 'p4', 'changes', '//depot/myproject/...@2,#head').stdout(second_p4changes),
  169         )
  170         encoded_p4change = p4change.copy()
  171         encoded_p4change[3] = encoded_p4change[3].encode(encoding)
  172         self.add_p4_describe_result(2, encoded_p4change[2])
  173         self.add_p4_describe_result(3, encoded_p4change[3])
  174 
  175         # The first time, it just learns the change to start at.
  176         self.assertTrue(self.changesource.last_change is None)
  177         yield self.changesource.poll()
  178 
  179         self.assertEqual(self.master.data.updates.changesAdded, [])
  180         self.assertEqual(self.changesource.last_change, 1)
  181 
  182         # Subsequent times, it returns Change objects for new changes.
  183         yield self.changesource.poll()
  184 
  185         # when_timestamp is converted from a local time spec, so just
  186         # replicate that here
  187         when1 = self.makeTime("2006/04/13 21:46:23")
  188         when2 = self.makeTime("2006/04/13 21:51:39")
  189 
  190         # these two can happen in either order, since they're from the same
  191         # perforce change.
  192         changesAdded = self.master.data.updates.changesAdded
  193         if changesAdded[1]['branch'] == 'branch_c':
  194             changesAdded[1:] = reversed(changesAdded[1:])
  195         self.assertEqual(self.master.data.updates.changesAdded, [{
  196             'author': 'slamb',
  197             'branch': 'trunk',
  198             'category': None,
  199             'codebase': None,
  200             'comments': 'creation',
  201             'files': ['whatbranch'],
  202             'project': '',
  203             'properties': {},
  204             'repository': '',
  205             'revision': '2',
  206             'revlink': '',
  207             'src': None,
  208             'when_timestamp': datetime2epoch(when1),
  209         }, {
  210             'author': 'bob',
  211             'branch': 'branch_b',
  212             'category': None,
  213             'codebase': None,
  214             'comments':
  215                 'short desc truncated because this is a long description.\n'
  216                 'ASDF-GUI-P3-\u2018Upgrade Icon\u2019 disappears sometimes.',
  217             'files': ['branch_b_file', 'whatbranch'],
  218             'project': '',
  219             'properties': {},
  220             'repository': '',
  221             'revision': '3',
  222             'revlink': '',
  223             'src': None,
  224             'when_timestamp': datetime2epoch(when2),
  225         }, {
  226             'author': 'bob',
  227             'branch': 'branch_c',
  228             'category': None,
  229             'codebase': None,
  230             'comments':
  231                 'short desc truncated because this is a long description.\n'
  232                 'ASDF-GUI-P3-\u2018Upgrade Icon\u2019 disappears sometimes.',
  233             'files': ['whatbranch'],
  234             'project': '',
  235             'properties': {},
  236             'repository': '',
  237             'revision': '3',
  238             'revlink': '',
  239             'src': None,
  240             'when_timestamp': datetime2epoch(when2),
  241         }])
  242         self.assertAllCommandsRan()
  243 
  244     def test_poll_successful_default_encoding(self):
  245         return self.do_test_poll_successful()
  246 
  247     def test_poll_successful_macroman_encoding(self):
  248         return self.do_test_poll_successful(encoding='macroman')
  249 
  250     def test_poll_failed_changes(self):
  251         self.attachChangeSource(
  252             P4Source(p4port=None, p4user=None,
  253                      p4base='//depot/myproject/',
  254                      split_file=lambda x: x.split('/', 1)))
  255         self.expectCommands(
  256             gpo.Expect('p4', 'changes', '-m', '1', '//depot/myproject/...').stdout(b'Perforce client error:\n...'))
  257 
  258         # call _poll, so we can catch the failure
  259         d = self.changesource._poll()
  260         return self.assertFailure(d, P4PollerError)
  261 
  262     @defer.inlineCallbacks
  263     def test_poll_failed_describe(self):
  264         self.attachChangeSource(
  265             P4Source(p4port=None, p4user=None,
  266                      p4base='//depot/myproject/',
  267                      split_file=lambda x: x.split('/', 1)))
  268         self.expectCommands(
  269             gpo.Expect(
  270                 'p4', 'changes', '//depot/myproject/...@3,#head').stdout(second_p4changes),
  271         )
  272         self.add_p4_describe_result(2, p4change[2])
  273         self.add_p4_describe_result(3, b'Perforce client error:\n...')
  274 
  275         # tell poll() that it's already been called once
  276         self.changesource.last_change = 2
  277 
  278         # call _poll, so we can catch the failure
  279         with self.assertRaises(P4PollerError):
  280             yield self.changesource._poll()
  281 
  282         # check that 2 was processed OK
  283         self.assertEqual(self.changesource.last_change, 2)
  284         self.assertAllCommandsRan()
  285 
  286     def test_poll_unicode_error(self):
  287         self.attachChangeSource(
  288             P4Source(p4port=None, p4user=None,
  289                      p4base='//depot/myproject/',
  290                      split_file=lambda x: x.split('/', 1)))
  291         self.expectCommands(
  292             gpo.Expect(
  293                 'p4', 'changes', '//depot/myproject/...@3,#head').stdout(second_p4changes),
  294         )
  295         # Add a character which cannot be decoded with utf-8
  296         undecodableText = p4change[2] + b"\x81"
  297         self.add_p4_describe_result(2, undecodableText)
  298 
  299         # tell poll() that it's already been called once
  300         self.changesource.last_change = 2
  301 
  302         # call _poll, so we can catch the failure
  303         d = self.changesource._poll()
  304         return self.assertFailure(d, UnicodeError)
  305 
  306     def test_poll_unicode_error2(self):
  307         self.attachChangeSource(
  308             P4Source(p4port=None, p4user=None,
  309                      p4base='//depot/myproject/',
  310                      split_file=lambda x: x.split('/', 1),
  311                      encoding='ascii'))
  312         # Trying to decode a certain character with ascii codec should fail.
  313         self.expectCommands(
  314             gpo.Expect(
  315                 'p4', 'changes', '-m', '1', '//depot/myproject/...').stdout(fourth_p4changes),
  316         )
  317 
  318         d = self.changesource._poll()
  319         return d
  320 
  321     @defer.inlineCallbacks
  322     def test_acquire_ticket_auth(self):
  323         self.attachChangeSource(
  324             P4Source(p4port=None, p4user=None, p4passwd='pass',
  325                      p4base='//depot/myproject/',
  326                      split_file=lambda x: x.split('/', 1),
  327                      use_tickets=True))
  328         self.expectCommands(
  329             gpo.Expect('p4', '-P', 'TICKET_ID_GOES_HERE',
  330                        'changes', '-m', '1', '//depot/myproject/...').stdout(first_p4changes)
  331         )
  332 
  333         transport = FakeTransport()
  334 
  335         # p4poller uses only those arguments at the moment
  336         def spawnProcess(pp, cmd, argv, env):
  337             self.assertEqual([cmd, argv],
  338                              ['p4', [b'p4', b'login', b'-p']])
  339             pp.makeConnection(transport)
  340             self.assertEqual('pass\n', transport.msg)
  341             pp.outReceived('Enter password:\nSuccess:  Password verified.\nTICKET_ID_GOES_HERE\n')
  342             so = error.ProcessDone(None)
  343             pp.processEnded(failure.Failure(so))
  344         self.patch(reactor, 'spawnProcess', spawnProcess)
  345 
  346         yield self.changesource.poll()
  347 
  348         self.assertEqual(
  349             self.changesource._ticket_passwd, 'TICKET_ID_GOES_HERE')
  350 
  351     @defer.inlineCallbacks
  352     def test_acquire_ticket_auth2(self):
  353         self.attachChangeSource(
  354             P4Source(p4port=None, p4user=None, p4passwd='pass',
  355                      p4base='//depot/myproject/',
  356                      split_file=lambda x: x.split('/', 1),
  357                      use_tickets=True))
  358         self.expectCommands(
  359             gpo.Expect('p4', '-P', 'TICKET_ID_GOES_HERE',
  360                        'changes', '-m', '1', '//depot/myproject/...').stdout(first_p4changes)
  361         )
  362 
  363         transport = FakeTransport()
  364 
  365         # p4poller uses only those arguments at the moment
  366         def spawnProcess(pp, cmd, argv, env):
  367             self.assertEqual([cmd, argv],
  368                              ['p4', [b'p4', b'login', b'-p']])
  369             pp.makeConnection(transport)
  370             self.assertEqual('pass\n', transport.msg)
  371             pp.outReceived('Enter password:\nTICKET_ID_GOES_HERE\n')
  372             so = error.ProcessDone(None)
  373             pp.processEnded(failure.Failure(so))
  374         self.patch(reactor, 'spawnProcess', spawnProcess)
  375 
  376         yield self.changesource.poll()
  377 
  378         self.assertEqual(
  379             self.changesource._ticket_passwd, 'TICKET_ID_GOES_HERE')
  380 
  381     @defer.inlineCallbacks
  382     def test_acquire_ticket_auth2_fail(self):
  383         self.attachChangeSource(
  384             P4Source(p4port=None, p4user=None, p4passwd='pass',
  385                      p4base='//depot/myproject/',
  386                      split_file=lambda x: x.split('/', 1),
  387                      use_tickets=True))
  388         self.expectCommands(
  389             gpo.Expect('p4', '-P', None,
  390                        'changes', '-m', '1', '//depot/myproject/...').stdout(first_p4changes)
  391         )
  392 
  393         transport = FakeTransport()
  394 
  395         # p4poller uses only those arguments at the moment
  396         def spawnProcess(pp, cmd, argv, env):
  397             self.assertEqual([cmd, argv],
  398                              ['p4', [b'p4', b'login', b'-p']])
  399             pp.makeConnection(transport)
  400             self.assertEqual('pass\n', transport.msg)
  401             pp.outReceived('Enter password:\n')
  402             pp.errReceived("Password invalid.\n'auth-check' validation failed: Incorrect password!\n")
  403             so = error.ProcessDone(status=1)
  404             pp.processEnded(failure.Failure(so))
  405         self.patch(reactor, 'spawnProcess', spawnProcess)
  406 
  407         yield self.changesource.poll()
  408 
  409         self.assertEqual(
  410             self.changesource._ticket_passwd, None)
  411 
  412     @defer.inlineCallbacks
  413     def test_poll_split_file(self):
  414         """Make sure split file works on branch only changes"""
  415         self.attachChangeSource(
  416             P4Source(p4port=None, p4user=None,
  417                      p4base='//depot/myproject/',
  418                      split_file=get_simple_split))
  419         self.expectCommands(
  420             gpo.Expect(
  421                 'p4', 'changes', '//depot/myproject/...@51,#head').stdout(third_p4changes),
  422         )
  423         self.add_p4_describe_result(5, p4change[5])
  424 
  425         self.changesource.last_change = 50
  426         yield self.changesource.poll()
  427 
  428         # when_timestamp is converted from a local time spec, so just
  429         # replicate that here
  430         when = self.makeTime("2006/04/13 21:55:39")
  431 
  432         def changeKey(change):
  433             """ Let's sort the array of changes by branch,
  434                 because in P4Source._poll(), changeAdded()
  435                 is called by iterating over a dictionary of
  436                 branches"""
  437             return change['branch']
  438 
  439         self.assertEqual(sorted(self.master.data.updates.changesAdded, key=changeKey),
  440             sorted([{
  441             'author': 'mpatel',
  442             'branch': 'branch_c',
  443             'category': None,
  444             'codebase': None,
  445             'comments': 'This is a multiline comment with tabs and spaces\n\nA list:\n  Item 1\n\tItem 2',
  446             'files': ['branch_c_file'],
  447             'project': '',
  448             'properties': {},
  449             'repository': '',
  450             'revision': '5',
  451             'revlink': '',
  452             'src': None,
  453             'when_timestamp': datetime2epoch(when),
  454         }, {
  455             'author': 'mpatel',
  456             'branch': 'branch_b',
  457             'category': None,
  458             'codebase': None,
  459             'comments': 'This is a multiline comment with tabs and spaces\n\nA list:\n  Item 1\n\tItem 2',
  460             'files': ['branch_b_file'],
  461             'project': '',
  462             'properties': {},
  463             'repository': '',
  464             'revision': '5',
  465             'revlink': '',
  466             'src': None,
  467             'when_timestamp': datetime2epoch(when),
  468         }], key=changeKey))
  469         self.assertEqual(self.changesource.last_change, 5)
  470         self.assertAllCommandsRan()
  471 
  472     @defer.inlineCallbacks
  473     def test_server_tz(self):
  474         """Verify that the server_tz parameter is handled correctly"""
  475         self.attachChangeSource(
  476             P4Source(p4port=None, p4user=None,
  477                      p4base='//depot/myproject/',
  478                      split_file=get_simple_split,
  479                      server_tz="Europe/Berlin"))
  480         self.expectCommands(
  481             gpo.Expect(
  482                 'p4', 'changes', '//depot/myproject/...@51,#head').stdout(third_p4changes),
  483         )
  484         self.add_p4_describe_result(5, p4change[5])
  485 
  486         self.changesource.last_change = 50
  487         yield self.changesource.poll()
  488 
  489         # when_timestamp is converted from 21:55:39 Berlin time to UTC
  490         when_berlin = self.makeTime("2006/04/13 21:55:39")
  491         when_berlin = when_berlin.replace(
  492             tzinfo=dateutil.tz.gettz('Europe/Berlin'))
  493         when = datetime2epoch(when_berlin)
  494 
  495         self.assertEqual([ch['when_timestamp']
  496                           for ch in self.master.data.updates.changesAdded],
  497                          [when, when])
  498         self.assertAllCommandsRan()
  499 
  500     def test_resolveWho_callable(self):
  501         with self.assertRaisesConfigError(
  502                 "You need to provide a valid callable for resolvewho"):
  503             P4Source(resolvewho=None)
  504 
  505 
  506 class TestSplit(unittest.TestCase):
  507 
  508     def test_get_simple_split(self):
  509         self.assertEqual(get_simple_split('foo/bar'), ('foo', 'bar'))
  510         self.assertEqual(get_simple_split('foo-bar'), (None, None))
  511         self.assertEqual(get_simple_split('/bar'), ('', 'bar'))
  512         self.assertEqual(get_simple_split('foo/'), ('foo', ''))