"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/changes/p4poller.py" (23 May 2019, 13141 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "p4poller.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.1.0_vs_2.2.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Portions Copyright Buildbot Team Members
   15 # Portions Copyright 2011 National Instruments
   16 
   17 
   18 # Many thanks to Dave Peticolas for contributing this module
   19 
   20 import datetime
   21 import os
   22 import re
   23 
   24 import dateutil.tz
   25 
   26 from twisted.internet import defer
   27 from twisted.internet import protocol
   28 from twisted.internet import reactor
   29 from twisted.internet import utils
   30 from twisted.python import log
   31 
   32 from buildbot import config
   33 from buildbot import util
   34 from buildbot.changes import base
   35 from buildbot.util import bytes2unicode
   36 
   37 debug_logging = False
   38 
   39 
   40 class P4PollerError(Exception):
   41 
   42     """Something went wrong with the poll. This is used as a distinctive
   43     exception type so that unit tests can detect and ignore it."""
   44 
   45 
   46 class TicketLoginProtocol(protocol.ProcessProtocol):
   47 
   48     """ Twisted process protocol to run `p4 login` and enter our password
   49         in the stdin."""
   50 
   51     def __init__(self, stdin, p4base):
   52         self.deferred = defer.Deferred()
   53         self.stdin = stdin
   54         self.stdout = ''
   55         self.stderr = ''
   56         self.p4base = p4base
   57 
   58     def connectionMade(self):
   59         if self.stdin:
   60             if debug_logging:
   61                 log.msg("P4Poller: entering password for %s: %s" %
   62                         (self.p4base, self.stdin))
   63             self.transport.write(self.stdin)
   64         self.transport.closeStdin()
   65 
   66     def processEnded(self, reason):
   67         if debug_logging:
   68             log.msg("P4Poller: login process finished for %s: %s" %
   69                     (self.p4base, reason.value.exitCode))
   70         self.deferred.callback(reason.value.exitCode)
   71 
   72     def outReceived(self, data):
   73         if debug_logging:
   74             log.msg("P4Poller: login stdout for %s: %s" % (self.p4base, data))
   75         self.stdout += data
   76 
   77     def errReceived(self, data):
   78         if debug_logging:
   79             log.msg("P4Poller: login stderr for %s: %s" % (self.p4base, data))
   80         self.stderr += data
   81 
   82 
   83 def get_simple_split(branchfile):
   84     """Splits the branchfile argument and assuming branch is
   85        the first path component in branchfile, will return
   86        branch and file else None."""
   87 
   88     index = branchfile.find('/')
   89     if index == -1:
   90         return None, None
   91     branch, file = branchfile.split('/', 1)
   92     return branch, file
   93 
   94 
   95 class P4Source(base.PollingChangeSource, util.ComparableMixin):
   96 
   97     """This source will poll a perforce repository for changes and submit
   98     them to the change master."""
   99 
  100     compare_attrs = ("p4port", "p4user", "p4passwd", "p4base",
  101                      "p4bin", "pollInterval", "pollAtLaunch",
  102                      "server_tz")
  103 
  104     env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER",
  105                 "P4CHARSET", "PATH"]
  106 
  107     changes_line_re = re.compile(
  108         r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.*'$")
  109     describe_header_re = re.compile(
  110         r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$")
  111     file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ [/\w]+$")
  112     datefmt = '%Y/%m/%d %H:%M:%S'
  113 
  114     parent = None  # filled in when we're added
  115     last_change = None
  116     loop = None
  117 
  118     def __init__(self, p4port=None, p4user=None, p4passwd=None,
  119                  p4base='//', p4bin='p4',
  120                  split_file=lambda branchfile: (None, branchfile),
  121                  pollInterval=60 * 10, histmax=None, pollinterval=-2,
  122                  encoding='utf8', project=None, name=None,
  123                  use_tickets=False, ticket_login_interval=60 * 60 * 24,
  124                  server_tz=None, pollAtLaunch=False, revlink=lambda branch, revision: (''), resolvewho=lambda who: (who)):
  125 
  126         # for backward compatibility; the parameter used to be spelled with 'i'
  127         if pollinterval != -2:
  128             pollInterval = pollinterval
  129 
  130         if name is None:
  131             name = "P4Source:%s:%s" % (p4port, p4base)
  132 
  133         super().__init__(name=name,
  134                          pollInterval=pollInterval,
  135                          pollAtLaunch=pollAtLaunch)
  136 
  137         if project is None:
  138             project = ''
  139 
  140         if use_tickets and not p4passwd:
  141             config.error(
  142                 "You need to provide a P4 password to use ticket authentication")
  143 
  144         if not callable(revlink):
  145             config.error(
  146                 "You need to provide a valid callable for revlink")
  147 
  148         if not callable(resolvewho):
  149             config.error(
  150                 "You need to provide a valid callable for resolvewho")
  151 
  152         self.p4port = p4port
  153         self.p4user = p4user
  154         self.p4passwd = p4passwd
  155         self.p4base = p4base
  156         self.p4bin = p4bin
  157         self.split_file = split_file
  158         self.encoding = encoding
  159         self.project = util.bytes2unicode(project)
  160         self.use_tickets = use_tickets
  161         self.ticket_login_interval = ticket_login_interval
  162         self.revlink_callable = revlink
  163         self.resolvewho_callable = resolvewho
  164         self.server_tz = dateutil.tz.gettz(server_tz) if server_tz else None
  165         if server_tz is not None and self.server_tz is None:
  166             raise P4PollerError("Failed to get timezone from server_tz string '{}'".format(server_tz))
  167 
  168         self._ticket_passwd = None
  169         self._ticket_login_counter = 0
  170 
  171     def describe(self):
  172         return "p4source %s %s" % (self.p4port, self.p4base)
  173 
  174     def poll(self):
  175         d = self._poll()
  176         d.addErrback(log.err, 'P4 poll failed on %s, %s' %
  177                      (self.p4port, self.p4base))
  178         return d
  179 
  180     def _get_process_output(self, args):
  181         env = {e: os.environ.get(e)
  182                for e in self.env_vars if os.environ.get(e)}
  183         d = utils.getProcessOutput(self.p4bin, args, env)
  184         return d
  185 
  186     def _acquireTicket(self, protocol):
  187         command = [self.p4bin, ]
  188         if self.p4port:
  189             command.extend(['-p', self.p4port])
  190         if self.p4user:
  191             command.extend(['-u', self.p4user])
  192         command.extend(['login', '-p'])
  193         command = [c.encode('utf-8') for c in command]
  194 
  195         reactor.spawnProcess(protocol, self.p4bin, command, env=os.environ)
  196 
  197     def _parseTicketPassword(self, text):
  198         lines = text.splitlines()
  199         if len(lines) < 2:
  200             return None
  201         return lines[-1].strip()
  202 
  203     def _getPasswd(self):
  204         if self.use_tickets:
  205             return self._ticket_passwd
  206         return self.p4passwd
  207 
  208     @defer.inlineCallbacks
  209     def _poll(self):
  210         if self.use_tickets:
  211             self._ticket_login_counter -= 1
  212             if self._ticket_login_counter <= 0:
  213                 # Re-acquire the ticket and reset the counter.
  214                 log.msg("P4Poller: (re)acquiring P4 ticket for %s..." %
  215                         self.p4base)
  216                 protocol = TicketLoginProtocol(
  217                     self.p4passwd + "\n", self.p4base)
  218                 self._acquireTicket(protocol)
  219                 yield protocol.deferred
  220 
  221                 self._ticket_passwd = self._parseTicketPassword(
  222                     protocol.stdout)
  223                 self._ticket_login_counter = max(
  224                     self.ticket_login_interval / self.pollInterval, 1)
  225                 if debug_logging:
  226                     log.msg("P4Poller: got ticket password: %s" %
  227                             self._ticket_passwd)
  228                     log.msg(
  229                         "P4Poller: next ticket acquisition in %d polls" % self._ticket_login_counter)
  230 
  231         args = []
  232         if self.p4port:
  233             args.extend(['-p', self.p4port])
  234         if self.p4user:
  235             args.extend(['-u', self.p4user])
  236         if self.p4passwd:
  237             args.extend(['-P', self._getPasswd()])
  238         args.extend(['changes'])
  239         if self.last_change is not None:
  240             args.extend(
  241                 ['%s...@%d,#head' % (self.p4base, self.last_change + 1)])
  242         else:
  243             args.extend(['-m', '1', '%s...' % (self.p4base,)])
  244 
  245         result = yield self._get_process_output(args)
  246         # decode the result from its designated encoding
  247         try:
  248             result = bytes2unicode(result, self.encoding)
  249         except UnicodeError as ex:
  250             log.msg("{}: cannot fully decode {} in {}".format(
  251                     ex, repr(result), self.encoding))
  252             result = bytes2unicode(result, encoding=self.encoding, errors="replace")
  253 
  254         last_change = self.last_change
  255         changelists = []
  256         for line in result.split('\n'):
  257             line = line.strip()
  258             if not line:
  259                 continue
  260             m = self.changes_line_re.match(line)
  261             if not m:
  262                 raise P4PollerError(
  263                     "Unexpected 'p4 changes' output: %r" % result)
  264             num = int(m.group('num'))
  265             if last_change is None:
  266                 # first time through, the poller just gets a "baseline" for where to
  267                 # start on the next poll
  268                 log.msg('P4Poller: starting at change %d' % num)
  269                 self.last_change = num
  270                 return
  271             changelists.append(num)
  272         changelists.reverse()  # oldest first
  273 
  274         # Retrieve each sequentially.
  275         for num in changelists:
  276             args = []
  277             if self.p4port:
  278                 args.extend(['-p', self.p4port])
  279             if self.p4user:
  280                 args.extend(['-u', self.p4user])
  281             if self.p4passwd:
  282                 args.extend(['-P', self._getPasswd()])
  283             args.extend(['describe', '-s', str(num)])
  284             result = yield self._get_process_output(args)
  285 
  286             # decode the result from its designated encoding
  287             try:
  288                 result = bytes2unicode(result, self.encoding)
  289             except UnicodeError as ex:
  290                 log.msg(
  291                     "P4Poller: couldn't decode changelist description: %s" % ex.encoding)
  292                 log.msg("P4Poller: in object: %s" % ex.object)
  293                 log.err("P4Poller: poll failed on %s, %s" %
  294                         (self.p4port, self.p4base))
  295                 raise
  296 
  297             lines = result.split('\n')
  298             # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date
  299             # field. The rstrip() is intended to remove that.
  300             lines[0] = lines[0].rstrip()
  301             m = self.describe_header_re.match(lines[0])
  302             if not m:
  303                 raise P4PollerError(
  304                     "Unexpected 'p4 describe -s' result: %r" % result)
  305             who = self.resolvewho_callable(m.group('who'))
  306             when = datetime.datetime.strptime(m.group('when'), self.datefmt)
  307             if self.server_tz:
  308                 # Convert from the server's timezone to the local timezone.
  309                 when = when.replace(tzinfo=self.server_tz)
  310             when = util.datetime2epoch(when)
  311 
  312             comment_lines = []
  313             lines.pop(0)  # describe header
  314             lines.pop(0)  # blank line
  315             while not lines[0].startswith('Affected files'):
  316                 if lines[0].startswith('\t'):  # comment is indented by one tab
  317                     comment_lines.append(lines.pop(0)[1:])
  318                 else:
  319                     lines.pop(0)  # discard non comment line
  320             comments = '\n'.join(comment_lines)
  321 
  322             lines.pop(0)  # affected files
  323             branch_files = {}  # dict for branch mapped to file(s)
  324             while lines:
  325                 line = lines.pop(0).strip()
  326                 if not line:
  327                     continue
  328                 m = self.file_re.match(line)
  329                 if not m:
  330                     raise P4PollerError("Invalid file line: %r" % line)
  331                 path = m.group('path')
  332                 if path.startswith(self.p4base):
  333                     branch, file = self.split_file(path[len(self.p4base):])
  334                     if (branch is None and file is None):
  335                         continue
  336                     if branch in branch_files:
  337                         branch_files[branch].append(file)
  338                     else:
  339                         branch_files[branch] = [file]
  340 
  341             for branch in branch_files:
  342                 yield self.master.data.updates.addChange(
  343                     author=who,
  344                     files=branch_files[branch],
  345                     comments=comments,
  346                     revision=str(num),
  347                     when_timestamp=when,
  348                     branch=branch,
  349                     project=self.project,
  350                     revlink=self.revlink_callable(branch, str(num)))
  351 
  352             self.last_change = num