"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/steps/source/p4.py" (23 May 2019, 15682 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "p4.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.1.0_vs_2.2.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 # Portions Copyright 2013 Bad Dog Consulting
   16 
   17 import re
   18 
   19 from twisted.internet import defer
   20 from twisted.python import log
   21 
   22 from buildbot import config
   23 from buildbot import interfaces
   24 from buildbot.interfaces import WorkerTooOldError
   25 from buildbot.process import buildstep
   26 from buildbot.process.properties import Interpolate
   27 from buildbot.steps.source import Source
   28 
   29 # Notes:
   30 #  see
   31 # http://perforce.com/perforce/doc.current/manuals/cmdref/o.gopts.html#1040647
   32 #   for getting p4 command to output marshalled python dictionaries as output
   33 #   for commands.
   34 #   Perhaps switch to using 'p4 -G' :  From URL above:
   35 #   -G Causes all output (and batch input for form commands with -i) to be
   36 #   formatted as marshalled Python dictionary objects. This is most often used
   37 #   when scripting.
   38 
   39 debug_logging = False
   40 
   41 
   42 class P4(Source):
   43 
   44     """Perform Perforce checkout/update operations."""
   45 
   46     name = 'p4'
   47 
   48     renderables = ['mode', 'p4base', 'p4client', 'p4viewspec', 'p4branch']
   49     possible_modes = ('incremental', 'full')
   50 
   51     def __init__(self, mode='incremental',
   52                  method=None, p4base=None, p4branch=None,
   53                  p4port=None, p4user=None,
   54                  p4passwd=None, p4extra_views=(), p4line_end='local',
   55                  p4viewspec=None, p4viewspec_suffix='...',
   56                  p4client=Interpolate(
   57                      'buildbot_%(prop:workername)s_%(prop:buildername)s'),
   58                  p4client_spec_options='allwrite rmdir',
   59                  p4extra_args=None,
   60                  p4bin='p4',
   61                  use_tickets=False,
   62                  **kwargs):
   63         self.method = method
   64         self.mode = mode
   65         self.p4branch = p4branch
   66         self.p4bin = p4bin
   67         self.p4base = p4base
   68         self.p4port = p4port
   69         self.p4user = p4user
   70         self.p4passwd = p4passwd
   71         self.p4extra_views = p4extra_views
   72         self.p4viewspec = p4viewspec
   73         self.p4viewspec_suffix = p4viewspec_suffix
   74         self.p4line_end = p4line_end
   75         self.p4client = p4client
   76         self.p4client_spec_options = p4client_spec_options
   77         self.p4extra_args = p4extra_args
   78         self.use_tickets = use_tickets
   79 
   80         super().__init__(**kwargs)
   81 
   82         if self.mode not in self.possible_modes and not interfaces.IRenderable.providedBy(self.mode):
   83             config.error("mode %s is not an IRenderable, or one of %s" % (
   84                 self.mode, self.possible_modes))
   85 
   86         if not p4viewspec and p4base is None:
   87             config.error("You must provide p4base or p4viewspec")
   88 
   89         if p4viewspec and (p4base or p4branch or p4extra_views):
   90             config.error(
   91                 "Either provide p4viewspec or p4base and p4branch (and optionally p4extra_views")
   92 
   93         if p4viewspec and isinstance(p4viewspec, str):
   94             config.error(
   95                 "p4viewspec must not be a string, and should be a sequence of 2 element sequences")
   96 
   97         if not interfaces.IRenderable.providedBy(p4base) and p4base and p4base.endswith('/'):
   98             config.error(
   99                 'p4base should not end with a trailing / [p4base = %s]' % p4base)
  100 
  101         if not interfaces.IRenderable.providedBy(p4branch) and p4branch and p4branch.endswith('/'):
  102             config.error(
  103                 'p4branch should not end with a trailing / [p4branch = %s]' % p4branch)
  104 
  105         if (p4branch or p4extra_views) and not p4base:
  106             config.error(
  107                 'If you specify either p4branch or p4extra_views you must also specify p4base')
  108 
  109         if self.p4client_spec_options is None:
  110             self.p4client_spec_options = ''
  111 
  112     def startVC(self, branch, revision, patch):
  113         if debug_logging:
  114             log.msg('in startVC')
  115 
  116         self.revision = revision
  117         self.method = self._getMethod()
  118         self.stdio_log = self.addLogForRemoteCommands("stdio")
  119 
  120         d = self.checkP4()
  121 
  122         @d.addCallback
  123         def checkInstall(p4Installed):
  124             if not p4Installed:
  125                 raise WorkerTooOldError("p4 is not installed on worker")
  126             return 0
  127 
  128         # Try to obfuscate the password when used as an argument to commands.
  129         if self.p4passwd is not None:
  130             if not self.workerVersionIsOlderThan('shell', '2.16'):
  131                 self.p4passwd_arg = ('obfuscated', self.p4passwd, 'XXXXXX')
  132             else:
  133                 self.p4passwd_arg = self.p4passwd
  134                 log.msg("Worker does not understand obfuscation; "
  135                         "p4 password will be logged")
  136 
  137         if self.use_tickets and self.p4passwd:
  138             d.addCallback(self._acquireTicket)
  139 
  140         d.addCallback(self._getAttrGroupMember('mode', self.mode))
  141 
  142         d.addCallback(self.parseGotRevision)
  143         d.addCallback(self.finish)
  144         d.addErrback(self.failed)
  145         return d
  146 
  147     @defer.inlineCallbacks
  148     def mode_full(self, _):
  149         if debug_logging:
  150             log.msg("P4:full()..")
  151 
  152         # First we need to create the client
  153         yield self._createClientSpec()
  154 
  155         # Then p4 sync #none
  156         yield self._dovccmd(['sync', '#none'])
  157 
  158         # Then remove directory.
  159         yield self.runRmdir(self.workdir)
  160 
  161         # Then we need to sync the client
  162         if self.revision:
  163             if debug_logging:
  164                 log.msg("P4: full() sync command based on :base:%s changeset:%d",
  165                         self._getP4BaseForLog(), int(self.revision))
  166             yield self._dovccmd(['sync', '%s...@%d' % (
  167                 self._getP4BaseForCommand(), int(self.revision))], collectStdout=True)
  168         else:
  169             if debug_logging:
  170                 log.msg(
  171                     "P4: full() sync command based on :base:%s no revision", self._getP4BaseForLog())
  172             yield self._dovccmd(['sync'], collectStdout=True)
  173 
  174         if debug_logging:
  175             log.msg("P4: full() sync done.")
  176 
  177     @defer.inlineCallbacks
  178     def mode_incremental(self, _):
  179         if debug_logging:
  180             log.msg("P4:incremental()")
  181 
  182         # First we need to create the client
  183         yield self._createClientSpec()
  184 
  185         # and plan to do a checkout
  186         command = ['sync', ]
  187 
  188         if self.revision:
  189             command.extend(
  190                 ['%s...@%d' % (self._getP4BaseForCommand(), int(self.revision))])
  191 
  192         if debug_logging:
  193             log.msg(
  194                 "P4:incremental() command:%s revision:%s", command, self.revision)
  195         yield self._dovccmd(command)
  196 
  197     def finish(self, res):
  198         d = defer.succeed(res)
  199 
  200         @d.addCallback
  201         def _gotResults(results):
  202             self.setStatus(self.cmd, results)
  203             return results
  204         d.addCallback(self.finished)
  205         return d
  206 
  207     def _getP4BaseForLog(self):
  208         return self.p4base or '<custom viewspec>'
  209 
  210     def _getP4BaseForCommand(self):
  211         return self.p4base or ''
  212 
  213     def _buildVCCommand(self, doCommand):
  214         assert doCommand, "No command specified"
  215 
  216         command = [self.p4bin, ]
  217 
  218         if self.p4port:
  219             command.extend(['-p', self.p4port])
  220         if self.p4user:
  221             command.extend(['-u', self.p4user])
  222         if not self.use_tickets and self.p4passwd:
  223             command.extend(['-P', self.p4passwd_arg])
  224         if self.p4client:
  225             command.extend(['-c', self.p4client])
  226 
  227         # Only add the extra arguments for the `sync` command.
  228         if doCommand[0] == 'sync' and self.p4extra_args:
  229             command.extend(self.p4extra_args)
  230 
  231         command.extend(doCommand)
  232         return command
  233 
  234     def _dovccmd(self, command, collectStdout=False, initialStdin=None):
  235         command = self._buildVCCommand(command)
  236 
  237         if debug_logging:
  238             log.msg("P4:_dovccmd():workdir->%s" % self.workdir)
  239 
  240         cmd = buildstep.RemoteShellCommand(self.workdir, command,
  241                                            env=self.env,
  242                                            logEnviron=self.logEnviron,
  243                                            timeout=self.timeout,
  244                                            collectStdout=collectStdout,
  245                                            initialStdin=initialStdin,)
  246         cmd.useLog(self.stdio_log, False)
  247         if debug_logging:
  248             log.msg("Starting p4 command : p4 %s" % (" ".join(command),))
  249 
  250         d = self.runCommand(cmd)
  251 
  252         @d.addCallback
  253         def evaluateCommand(_):
  254             if cmd.rc != 0:
  255                 if debug_logging:
  256                     log.msg(
  257                         "P4:_dovccmd():Source step failed while running command %s" % cmd)
  258                 raise buildstep.BuildStepFailed()
  259             if collectStdout:
  260                 return cmd.stdout
  261             return cmd.rc
  262         return d
  263 
  264     def _getMethod(self):
  265         if self.method is not None and self.mode != 'incremental':
  266             return self.method
  267         elif self.mode == 'incremental':
  268             return None
  269         elif self.method is None and self.mode == 'full':
  270             return 'fresh'
  271 
  272     def _sourcedirIsUpdatable(self):
  273         # In general you should always be able to write to the directory
  274         # You just specified as the root of your client
  275         # So just return.
  276         # If we find a case where this is no longer true, then this
  277         # needs to be implemented
  278         return defer.succeed(True)
  279 
  280     @defer.inlineCallbacks
  281     def _createClientSpec(self):
  282         builddir = self.getProperty('builddir')
  283 
  284         if debug_logging:
  285             log.msg("P4:_createClientSpec() builddir:%s" % builddir)
  286             log.msg("P4:_createClientSpec() SELF.workdir:%s" % self.workdir)
  287 
  288         prop_dict = self.getProperties().asDict()
  289         prop_dict['p4client'] = self.p4client
  290 
  291         client_spec = ''
  292         client_spec += "Client: %s\n\n" % self.p4client
  293         client_spec += "Owner: %s\n\n" % self.p4user
  294         client_spec += "Description:\n\tCreated by %s\n\n" % self.p4user
  295         client_spec += "Root:\t%s\n\n" % self.build.path_module.normpath(
  296             self.build.path_module.join(builddir, self.workdir)
  297         )
  298         client_spec += "Options:\t%s\n\n" % self.p4client_spec_options
  299         if self.p4line_end:
  300             client_spec += "LineEnd:\t%s\n\n" % self.p4line_end
  301         else:
  302             client_spec += "LineEnd:\tlocal\n\n"
  303 
  304         # Setup a view
  305         client_spec += "View:\n"
  306 
  307         def has_whitespace(*args):
  308             return any([re.search(r'\s', i) for i in args if i is not None])
  309 
  310         if self.p4viewspec:
  311             # uses only p4viewspec array of tuples to build view
  312             # If the user specifies a viewspec via an array of tuples then
  313             # Ignore any specified p4base,p4branch, and/or p4extra_views
  314             suffix = self.p4viewspec_suffix or ''
  315             for k, v in self.p4viewspec:
  316                 if debug_logging:
  317                     log.msg('P4:_createClientSpec():key:%s value:%s' % (k, v))
  318 
  319                 qa = '"' if has_whitespace(k, suffix) else ''
  320                 qb = '"' if has_whitespace(self.p4client, v, suffix) else ''
  321                 client_spec += '\t%s%s%s%s %s//%s/%s%s%s\n' % (qa, k, suffix, qa,
  322                                                                qb, self.p4client, v, suffix, qb)
  323         else:
  324             # Uses p4base, p4branch, p4extra_views
  325 
  326             qa = '"' if has_whitespace(self.p4base, self.p4branch) else ''
  327 
  328             client_spec += "\t%s%s" % (qa, self.p4base)
  329 
  330             if self.p4branch:
  331                 client_spec += "/%s" % (self.p4branch)
  332 
  333             client_spec += "/...%s " % qa
  334 
  335             qb = '"' if has_whitespace(self.p4client) else ''
  336             client_spec += "%s//%s/...%s\n" % (qb, self.p4client, qb)
  337 
  338             if self.p4extra_views:
  339                 for k, v in self.p4extra_views:
  340                     qa = '"' if has_whitespace(k) else ''
  341                     qb = '"' if has_whitespace(k, self.p4client, v) else ''
  342 
  343                     client_spec += "\t%s%s/...%s %s//%s/%s/...%s\n" % (qa, k, qa,
  344                                                                        qb, self.p4client, v, qb)
  345 
  346         if debug_logging:
  347             log.msg(client_spec)
  348 
  349         stdout = yield self._dovccmd(['client', '-i'], collectStdout=True, initialStdin=client_spec)
  350         mo = re.search(r'Client (\S+) (.+)$', stdout, re.M)
  351         return mo and (mo.group(2) == 'saved.' or mo.group(2) == 'not changed.')
  352 
  353     @defer.inlineCallbacks
  354     def _acquireTicket(self, _):
  355         if debug_logging:
  356             log.msg("P4:acquireTicket()")
  357 
  358         # TODO: check first if the ticket is still valid?
  359         initialStdin = self.p4passwd + "\n"
  360         yield self._dovccmd(['login'], initialStdin=initialStdin)
  361 
  362     def parseGotRevision(self, _):
  363         command = self._buildVCCommand(['changes', '-m1', '#have'])
  364 
  365         cmd = buildstep.RemoteShellCommand(self.workdir, command,
  366                                            env=self.env,
  367                                            timeout=self.timeout,
  368                                            logEnviron=self.logEnviron,
  369                                            collectStdout=True)
  370         cmd.useLog(self.stdio_log, False)
  371         d = self.runCommand(cmd)
  372 
  373         @d.addCallback
  374         def _setrev(_):
  375             stdout = cmd.stdout.strip()
  376             # Example output from p4 changes -m1 #have
  377             # Change 212798 on 2012/04/13 by user@user-unix-bldng2 'change to
  378             # pickup build'
  379             revision = stdout.split()[1]
  380             try:
  381                 int(revision)
  382             except ValueError:
  383                 msg = ("p4.parseGotRevision unable to parse output "
  384                        "of 'p4 changes -m1 \"#have\"': '%s'" % stdout)
  385                 log.msg(msg)
  386                 raise buildstep.BuildStepFailed()
  387 
  388             if debug_logging:
  389                 log.msg("Got p4 revision %s" % (revision,))
  390             self.updateSourceProperty('got_revision', revision)
  391             return 0
  392         return d
  393 
  394     def purge(self, ignore_ignores):
  395         """Delete everything that shown up on status."""
  396         command = ['sync', '#none']
  397         if ignore_ignores:
  398             command.append('--no-ignore')
  399         d = self._dovccmd(command, collectStdout=True)
  400 
  401         # add deferred to rm tree
  402 
  403         # then add defer to sync to revision
  404         return d
  405 
  406     def checkP4(self):
  407         cmd = buildstep.RemoteShellCommand(self.workdir, ['p4', '-V'],
  408                                            env=self.env,
  409                                            logEnviron=self.logEnviron)
  410         cmd.useLog(self.stdio_log, False)
  411         d = self.runCommand(cmd)
  412 
  413         @d.addCallback
  414         def evaluate(_):
  415             if cmd.rc != 0:
  416                 return False
  417             return True
  418         return d
  419 
  420     def computeSourceRevision(self, changes):
  421         if not changes or None in [c.revision for c in changes]:
  422             return None
  423         lastChange = max([int(c.revision) for c in changes])
  424         return lastChange