"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/clients/tryclient.py" (23 May 2019, 32196 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "tryclient.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.0.1_vs_2.1.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 import json
   17 import os
   18 import random
   19 import re
   20 import shlex
   21 import string
   22 import sys
   23 import time
   24 
   25 from twisted.cred import credentials
   26 from twisted.internet import defer
   27 from twisted.internet import protocol
   28 from twisted.internet import reactor
   29 from twisted.internet import task
   30 from twisted.internet import utils
   31 from twisted.python import log
   32 from twisted.python import runtime
   33 from twisted.python.procutils import which
   34 from twisted.spread import pb
   35 
   36 from buildbot.status import builder
   37 from buildbot.util import now
   38 from buildbot.util import unicode2bytes
   39 from buildbot.util.eventual import fireEventually
   40 
   41 
   42 class SourceStamp:
   43 
   44     def __init__(self, branch, revision, patch, repository=''):
   45         self.branch = branch
   46         self.revision = revision
   47         self.patch = patch
   48         self.repository = repository
   49 
   50 
   51 def output(*msg):
   52     print(' '.join([str(m)for m in msg]))
   53 
   54 
   55 class SourceStampExtractor:
   56 
   57     def __init__(self, treetop, branch, repository):
   58         self.treetop = treetop
   59         self.repository = repository
   60         self.branch = branch
   61         exes = which(self.vcexe)
   62         if not exes:
   63             output("Could not find executable '{}'.".format(self.vcexe))
   64             sys.exit(1)
   65         self.exe = exes[0]
   66 
   67     def dovc(self, cmd):
   68         """This accepts the arguments of a command, without the actual
   69         command itself."""
   70         env = os.environ.copy()
   71         env['LC_ALL'] = "C"
   72         d = utils.getProcessOutputAndValue(self.exe, cmd, env=env,
   73                                            path=self.treetop)
   74         d.addCallback(self._didvc, cmd)
   75         return d
   76 
   77     def _didvc(self, res, cmd):
   78         (stdout, stderr, code) = res
   79         # 'bzr diff' sets rc=1 if there were any differences.
   80         # cvs does something similar, so don't bother requiring rc=0.
   81         return stdout
   82 
   83     def get(self):
   84         """Return a Deferred that fires with a SourceStamp instance."""
   85         d = self.getBaseRevision()
   86         d.addCallback(self.getPatch)
   87         d.addCallback(self.done)
   88         return d
   89 
   90     def readPatch(self, diff, patchlevel):
   91         if not diff:
   92             diff = None
   93         self.patch = (patchlevel, diff)
   94 
   95     def done(self, res):
   96         if not self.repository:
   97             self.repository = self.treetop
   98         # TODO: figure out the branch and project too
   99         ss = SourceStamp(self.branch, self.baserev, self.patch,
  100                          repository=self.repository)
  101         return ss
  102 
  103 
  104 class CVSExtractor(SourceStampExtractor):
  105     patchlevel = 0
  106     vcexe = "cvs"
  107 
  108     def getBaseRevision(self):
  109         # this depends upon our local clock and the repository's clock being
  110         # reasonably synchronized with each other. We express everything in
  111         # UTC because the '%z' format specifier for strftime doesn't always
  112         # work.
  113         self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000",
  114                                      time.gmtime(now()))
  115         return defer.succeed(None)
  116 
  117     def getPatch(self, res):
  118         # the -q tells CVS to not announce each directory as it works
  119         if self.branch is not None:
  120             # 'cvs diff' won't take both -r and -D at the same time (it
  121             # ignores the -r). As best I can tell, there is no way to make
  122             # cvs give you a diff relative to a timestamp on the non-trunk
  123             # branch. A bare 'cvs diff' will tell you about the changes
  124             # relative to your checked-out versions, but I know of no way to
  125             # find out what those checked-out versions are.
  126             output("Sorry, CVS 'try' builds don't work with branches")
  127             sys.exit(1)
  128         args = ['-q', 'diff', '-u', '-D', self.baserev]
  129         d = self.dovc(args)
  130         d.addCallback(self.readPatch, self.patchlevel)
  131         return d
  132 
  133 
  134 class SVNExtractor(SourceStampExtractor):
  135     patchlevel = 0
  136     vcexe = "svn"
  137 
  138     def getBaseRevision(self):
  139         d = self.dovc(["status", "-u"])
  140         d.addCallback(self.parseStatus)
  141         return d
  142 
  143     def parseStatus(self, res):
  144         # svn shows the base revision for each file that has been modified or
  145         # which needs an update. You can update each file to a different
  146         # version, so each file is displayed with its individual base
  147         # revision. It also shows the repository-wide latest revision number
  148         # on the last line ("Status against revision: \d+").
  149 
  150         # for our purposes, we use the latest revision number as the "base"
  151         # revision, and get a diff against that. This means we will get
  152         # reverse-diffs for local files that need updating, but the resulting
  153         # tree will still be correct. The only weirdness is that the baserev
  154         # that we emit may be different than the version of the tree that we
  155         # first checked out.
  156 
  157         # to do this differently would probably involve scanning the revision
  158         # numbers to find the max (or perhaps the min) revision, and then
  159         # using that as a base.
  160 
  161         for line in res.split(b"\n"):
  162             m = re.search(br'^Status against revision:\s+(\d+)', line)
  163             if m:
  164                 self.baserev = m.group(1)
  165                 return
  166         output(
  167             b"Could not find 'Status against revision' in SVN output: " + res)
  168         sys.exit(1)
  169 
  170     def getPatch(self, res):
  171         d = self.dovc(["diff", "-r{}".format(self.baserev)])
  172         d.addCallback(self.readPatch, self.patchlevel)
  173         return d
  174 
  175 
  176 class BzrExtractor(SourceStampExtractor):
  177     patchlevel = 0
  178     vcexe = "bzr"
  179 
  180     def getBaseRevision(self):
  181         d = self.dovc(["revision-info", "-rsubmit:"])
  182         d.addCallback(self.get_revision_number)
  183         return d
  184 
  185     def get_revision_number(self, out):
  186         revno, revid = out.split()
  187         self.baserev = 'revid:' + revid
  188         return
  189 
  190     def getPatch(self, res):
  191         d = self.dovc(["diff", "-r{}..".format(self.baserev)])
  192         d.addCallback(self.readPatch, self.patchlevel)
  193         return d
  194 
  195 
  196 class MercurialExtractor(SourceStampExtractor):
  197     patchlevel = 1
  198     vcexe = "hg"
  199 
  200     def _didvc(self, res, cmd):
  201         (stdout, stderr, code) = res
  202 
  203         if code:
  204             cs = ' '.join(['hg'] + cmd)
  205             if stderr:
  206                 stderr = '\n' + stderr.rstrip()
  207             raise RuntimeError("{} returned {} {}".format(cs, code, stderr))
  208 
  209         return stdout
  210 
  211     @defer.inlineCallbacks
  212     def getBaseRevision(self):
  213         upstream = ""
  214         if self.repository:
  215             upstream = "r'%s'" % self.repository
  216         output = ''
  217         try:
  218             output = yield self.dovc(["log", "--template", "{node}\\n", "-r",
  219                                       "max(::. - outgoing(%s))" % upstream])
  220         except RuntimeError:
  221             # outgoing() will abort if no default-push/default path is
  222             # configured
  223             if upstream:
  224                 raise
  225             # fall back to current working directory parent
  226             output = yield self.dovc(["log", "--template", "{node}\\n", "-r", "p1()"])
  227         m = re.search(br'^(\w+)', output)
  228         if not m:
  229             raise RuntimeError(
  230                 "Revision {!r} is not in the right format".format(output))
  231         self.baserev = m.group(0)
  232 
  233     def getPatch(self, res):
  234         d = self.dovc(["diff", "-r", self.baserev])
  235         d.addCallback(self.readPatch, self.patchlevel)
  236         return d
  237 
  238 
  239 class PerforceExtractor(SourceStampExtractor):
  240     patchlevel = 0
  241     vcexe = "p4"
  242 
  243     def getBaseRevision(self):
  244         d = self.dovc(["changes", "-m1", "..."])
  245         d.addCallback(self.parseStatus)
  246         return d
  247 
  248     def parseStatus(self, res):
  249         #
  250         # extract the base change number
  251         #
  252         m = re.search(br'Change (\d+)', res)
  253         if m:
  254             self.baserev = m.group(1)
  255             return
  256 
  257         output(b"Could not find change number in output: " + res)
  258         sys.exit(1)
  259 
  260     def readPatch(self, res, patchlevel):
  261         #
  262         # extract the actual patch from "res"
  263         #
  264         if not self.branch:
  265             output("you must specify a branch")
  266             sys.exit(1)
  267         mpatch = ""
  268         found = False
  269         for line in res.split("\n"):
  270             m = re.search('==== //depot/' + self.branch
  271                           + r'/([\w/\.\d\-_]+)#(\d+) -', line)
  272             if m:
  273                 mpatch += "--- %s#%s\n" % (m.group(1), m.group(2))
  274                 mpatch += "+++ %s\n" % (m.group(1))
  275                 found = True
  276             else:
  277                 mpatch += line
  278                 mpatch += "\n"
  279         if not found:
  280             output(b"could not parse patch file")
  281             sys.exit(1)
  282         self.patch = (patchlevel, mpatch)
  283 
  284     def getPatch(self, res):
  285         d = self.dovc(["diff"])
  286         d.addCallback(self.readPatch, self.patchlevel)
  287         return d
  288 
  289 
  290 class DarcsExtractor(SourceStampExtractor):
  291     patchlevel = 1
  292     vcexe = "darcs"
  293 
  294     def getBaseRevision(self):
  295         d = self.dovc(["changes", "--context"])
  296         d.addCallback(self.parseStatus)
  297         return d
  298 
  299     def parseStatus(self, res):
  300         self.baserev = res              # the whole context file
  301 
  302     def getPatch(self, res):
  303         d = self.dovc(["diff", "-u"])
  304         d.addCallback(self.readPatch, self.patchlevel)
  305         return d
  306 
  307 
  308 class GitExtractor(SourceStampExtractor):
  309     patchlevel = 1
  310     vcexe = "git"
  311     config = None
  312 
  313     def getBaseRevision(self):
  314         # If a branch is specified, parse out the rev it points to
  315         # and extract the local name.
  316         if self.branch:
  317             d = self.dovc(["rev-parse", self.branch])
  318             d.addCallback(self.override_baserev)
  319             d.addCallback(self.extractLocalBranch)
  320             return d
  321         d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"])
  322         d.addCallback(self.parseStatus)
  323         return d
  324 
  325     # remove remote-prefix from self.branch (assumes format <prefix>/<branch>)
  326     # this uses "git remote" to retrieve all configured remote names
  327     def extractLocalBranch(self, res):
  328         if '/' in self.branch:
  329             d = self.dovc(["remote"])
  330             d.addCallback(self.fixBranch)
  331             return d
  332 
  333     # strip remote prefix from self.branch
  334     def fixBranch(self, remotes):
  335         for l in remotes.split("\n"):
  336             r = l.strip()
  337             if r and self.branch.startswith(r):
  338                 self.branch = self.branch[len(r) + 1:]
  339                 break
  340 
  341     def readConfig(self):
  342         if self.config:
  343             return defer.succeed(self.config)
  344         d = self.dovc(["config", "-l"])
  345         d.addCallback(self.parseConfig)
  346         return d
  347 
  348     def parseConfig(self, res):
  349         self.config = {}
  350         for l in res.split(b"\n"):
  351             if l.strip():
  352                 parts = l.strip().split(b"=", 2)
  353                 if len(parts) < 2:
  354                     parts.append('true')
  355                 self.config[parts[0]] = parts[1]
  356         return self.config
  357 
  358     def parseTrackingBranch(self, res):
  359         # If we're tracking a remote, consider that the base.
  360         remote = self.config.get(b"branch." + self.branch + b".remote")
  361         ref = self.config.get(b"branch." + self.branch + b".merge")
  362         if remote and ref:
  363             remote_branch = ref.split(b"/", 2)[-1]
  364             baserev = remote + b"/" + remote_branch
  365         else:
  366             baserev = b"master"
  367 
  368         d = self.dovc(["rev-parse", baserev])
  369         d.addCallback(self.override_baserev)
  370         return d
  371 
  372     def override_baserev(self, res):
  373         self.baserev = res.strip()
  374 
  375     def parseStatus(self, res):
  376         # The current branch is marked by '*' at the start of the
  377         # line, followed by the branch name and the SHA1.
  378         #
  379         # Branch names may contain pretty much anything but whitespace.
  380         m = re.search(br'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE)
  381         if m:
  382             self.baserev = m.group(2)
  383             self.branch = m.group(1)
  384             d = self.readConfig()
  385             d.addCallback(self.parseTrackingBranch)
  386             return d
  387         output(b"Could not find current GIT branch: " + res)
  388         sys.exit(1)
  389 
  390     def getPatch(self, res):
  391         d = self.dovc(["diff", "--src-prefix=a/", "--dst-prefix=b/",
  392                        "--no-textconv", "--no-ext-diff", self.baserev])
  393         d.addCallback(self.readPatch, self.patchlevel)
  394         return d
  395 
  396 
  397 class MonotoneExtractor(SourceStampExtractor):
  398     patchlevel = 0
  399     vcexe = "mtn"
  400 
  401     def getBaseRevision(self):
  402         d = self.dovc(["automate", "get_base_revision_id"])
  403         d.addCallback(self.parseStatus)
  404         return d
  405 
  406     def parseStatus(self, output):
  407         hash = output.strip()
  408         if len(hash) != 40:
  409             self.baserev = None
  410         self.baserev = hash
  411 
  412     def getPatch(self, res):
  413         d = self.dovc(["diff"])
  414         d.addCallback(self.readPatch, self.patchlevel)
  415         return d
  416 
  417 
  418 def getSourceStamp(vctype, treetop, branch=None, repository=None):
  419     if vctype == "cvs":
  420         cls = CVSExtractor
  421     elif vctype == "svn":
  422         cls = SVNExtractor
  423     elif vctype == "bzr":
  424         cls = BzrExtractor
  425     elif vctype == "hg":
  426         cls = MercurialExtractor
  427     elif vctype == "p4":
  428         cls = PerforceExtractor
  429     elif vctype == "darcs":
  430         cls = DarcsExtractor
  431     elif vctype == "git":
  432         cls = GitExtractor
  433     elif vctype == "mtn":
  434         cls = MonotoneExtractor
  435     elif vctype == "none":
  436         return defer.succeed(SourceStamp("", "", (1, ""), ""))
  437     else:
  438         output("unknown vctype '{}'".format(vctype))
  439         sys.exit(1)
  440     return cls(treetop, branch, repository).get()
  441 
  442 
  443 def ns(s):
  444     return "{}:{},".format(len(s), s)
  445 
  446 
  447 def createJobfile(jobid, branch, baserev, patch_level, patch_body, repository,
  448                   project, who, comment, builderNames, properties):
  449     # Determine job file version from provided arguments
  450     if properties:
  451         version = 5
  452     elif comment:
  453         version = 4
  454     elif who:
  455         version = 3
  456     else:
  457         version = 2
  458     job = ""
  459     job += ns(str(version))
  460     if version < 5:
  461         job += ns(jobid)
  462         job += ns(branch)
  463         job += ns(str(baserev))
  464         job += ns("{}".format(patch_level))
  465         job += ns(patch_body or "")
  466         job += ns(repository)
  467         job += ns(project)
  468         if (version >= 3):
  469             job += ns(who)
  470         if (version >= 4):
  471             job += ns(comment)
  472         for bn in builderNames:
  473             job += ns(bn)
  474     else:
  475         job += ns(
  476             json.dumps({
  477                 'jobid': jobid, 'branch': branch, 'baserev': str(baserev),
  478                 'patch_level': patch_level, 'patch_body': patch_body,
  479                 'repository': repository, 'project': project, 'who': who,
  480                 'comment': comment, 'builderNames': builderNames,
  481                 'properties': properties,
  482             }))
  483     return job
  484 
  485 
  486 def getTopdir(topfile, start=None):
  487     """walk upwards from the current directory until we find this topfile"""
  488     if not start:
  489         start = os.getcwd()
  490     here = start
  491     toomany = 20
  492     while toomany > 0:
  493         if os.path.exists(os.path.join(here, topfile)):
  494             return here
  495         next = os.path.dirname(here)
  496         if next == here:
  497             break                       # we've hit the root
  498         here = next
  499         toomany -= 1
  500     output("Unable to find topfile '{}' anywhere "
  501            "from {} upwards".format(topfile, start))
  502     sys.exit(1)
  503 
  504 
  505 class RemoteTryPP(protocol.ProcessProtocol):
  506 
  507     def __init__(self, job):
  508         self.job = job
  509         self.d = defer.Deferred()
  510 
  511     def connectionMade(self):
  512         self.transport.write(self.job)
  513         self.transport.closeStdin()
  514 
  515     def outReceived(self, data):
  516         sys.stdout.write(data)
  517 
  518     def errReceived(self, data):
  519         sys.stderr.write(data)
  520 
  521     def processEnded(self, status_object):
  522         sig = status_object.value.signal
  523         rc = status_object.value.exitCode
  524         if sig is not None or rc != 0:
  525             self.d.errback(RuntimeError("remote 'buildbot tryserver' failed"
  526                                         ": sig={}, rc={}".format(sig, rc)))
  527             return
  528         self.d.callback((sig, rc))
  529 
  530 
  531 class Try(pb.Referenceable):
  532     buildsetStatus = None
  533     quiet = False
  534     printloop = False
  535 
  536     def __init__(self, config):
  537         self.config = config
  538         self.connect = self.getopt('connect')
  539         if self.connect not in ['ssh', 'pb']:
  540             output("you must specify a connect style: ssh or pb")
  541             sys.exit(1)
  542         self.builderNames = self.getopt('builders')
  543         self.project = self.getopt('project', '')
  544         self.who = self.getopt('who')
  545         self.comment = self.getopt('comment')
  546 
  547     def getopt(self, config_name, default=None):
  548         value = self.config.get(config_name)
  549         if value is None or value == []:
  550             value = default
  551         return value
  552 
  553     def createJob(self):
  554         # returns a Deferred which fires when the job parameters have been
  555         # created
  556 
  557         # generate a random (unique) string. It would make sense to add a
  558         # hostname and process ID here, but a) I suspect that would cause
  559         # windows portability problems, and b) really this is good enough
  560         self.bsid = "{}-{}".format(time.time(), random.randint(0, 1000000))
  561 
  562         # common options
  563         branch = self.getopt("branch")
  564 
  565         difffile = self.config.get("diff")
  566         if difffile:
  567             baserev = self.config.get("baserev")
  568             if difffile == "-":
  569                 diff = sys.stdin.read()
  570             else:
  571                 with open(difffile, "r") as f:
  572                     diff = f.read()
  573             if not diff:
  574                 diff = None
  575             patch = (self.config['patchlevel'], diff)
  576             ss = SourceStamp(
  577                 branch, baserev, patch, repository=self.getopt("repository"))
  578             d = defer.succeed(ss)
  579         else:
  580             vc = self.getopt("vc")
  581             if vc in ("cvs", "svn"):
  582                 # we need to find the tree-top
  583                 topdir = self.getopt("topdir")
  584                 if topdir:
  585                     treedir = os.path.expanduser(topdir)
  586                 else:
  587                     topfile = self.getopt("topfile")
  588                     if topfile:
  589                         treedir = getTopdir(topfile)
  590                     else:
  591                         output("Must specify topdir or topfile.")
  592                         sys.exit(1)
  593             else:
  594                 treedir = os.getcwd()
  595             d = getSourceStamp(vc, treedir, branch, self.getopt("repository"))
  596         d.addCallback(self._createJob_1)
  597         return d
  598 
  599     def _createJob_1(self, ss):
  600         self.sourcestamp = ss
  601         patchlevel, diff = ss.patch
  602         if diff is None:
  603             raise RuntimeError("There is no patch to try, diff is empty.")
  604 
  605         if self.connect == "ssh":
  606             revspec = ss.revision
  607             if revspec is None:
  608                 revspec = ""
  609             self.jobfile = createJobfile(
  610                 self.bsid, ss.branch or "", revspec, patchlevel, diff,
  611                 ss.repository, self.project, self.who, self.comment,
  612                 self.builderNames, self.config.get('properties', {}))
  613 
  614     def fakeDeliverJob(self):
  615         # Display the job to be delivered, but don't perform delivery.
  616         ss = self.sourcestamp
  617         output("Job:\n\tRepository: {}\n\tProject: {}\n\tBranch: {}\n\t"
  618                "Revision: {}\n\tBuilders: {}\n{}".format(
  619                ss.repository, self.project, ss.branch,
  620                ss.revision,
  621                self.builderNames,
  622                ss.patch[1]))
  623         d = defer.Deferred()
  624         d.callback(True)
  625         return d
  626 
  627     def deliverJob(self):
  628         # returns a Deferred that fires when the job has been delivered
  629         if self.connect == "ssh":
  630             tryhost = self.getopt("host")
  631             tryport = self.getopt("port")
  632             tryuser = self.getopt("username")
  633             trydir = self.getopt("jobdir")
  634             buildbotbin = self.getopt("buildbotbin")
  635             ssh_command = self.getopt("ssh")
  636             if not ssh_command:
  637                 ssh_commands = which("ssh")
  638                 if not ssh_commands:
  639                     raise RuntimeError("couldn't find ssh executable, make sure "
  640                                        "it is available in the PATH")
  641 
  642                 argv = [ssh_commands[0]]
  643             else:
  644                 # Split the string on whitespace to allow passing options in
  645                 # ssh command too, but preserving whitespace inside quotes to
  646                 # allow using paths with spaces in them which is common under
  647                 # Windows. And because Windows uses backslashes in paths, we
  648                 # can't just use shlex.split there as it would interpret them
  649                 # specially, so do it by hand.
  650                 if runtime.platformType == 'win32':
  651                     # Note that regex here matches the arguments, not the
  652                     # separators, as it's simpler to do it like this. And then we
  653                     # just need to get all of them together using the slice and
  654                     # also remove the quotes from those that were quoted.
  655                     argv = [string.strip(a, '"') for a in
  656                             re.split(r'''([^" ]+|"[^"]+")''', ssh_command)[1::2]]
  657                 else:
  658                     # Do use standard tokenization logic under POSIX.
  659                     argv = shlex.split(ssh_command)
  660 
  661             if tryuser:
  662                 argv += ["-l", tryuser]
  663 
  664             if tryport:
  665                 argv += ["-p", tryport]
  666 
  667             argv += [tryhost, buildbotbin, "tryserver", "--jobdir", trydir]
  668             pp = RemoteTryPP(self.jobfile)
  669             reactor.spawnProcess(pp, argv[0], argv, os.environ)
  670             d = pp.d
  671             return d
  672         if self.connect == "pb":
  673             user = self.getopt("username")
  674             passwd = self.getopt("passwd")
  675             master = self.getopt("master")
  676             tryhost, tryport = master.split(":")
  677             tryport = int(tryport)
  678             f = pb.PBClientFactory()
  679             d = f.login(credentials.UsernamePassword(unicode2bytes(user), unicode2bytes(passwd)))
  680             reactor.connectTCP(tryhost, tryport, f)
  681             d.addCallback(self._deliverJob_pb)
  682             return d
  683         raise RuntimeError("unknown connecttype '{}', "
  684                            "should be 'ssh' or 'pb'".format(self.connect))
  685 
  686     def _deliverJob_pb(self, remote):
  687         ss = self.sourcestamp
  688         output("Delivering job; comment=", self.comment)
  689 
  690         d = remote.callRemote("try",
  691                               ss.branch,
  692                               ss.revision,
  693                               ss.patch,
  694                               ss.repository,
  695                               self.project,
  696                               self.builderNames,
  697                               self.who,
  698                               self.comment,
  699                               self.config.get('properties', {}))
  700         d.addCallback(self._deliverJob_pb2)
  701         return d
  702 
  703     def _deliverJob_pb2(self, status):
  704         self.buildsetStatus = status
  705         return status
  706 
  707     def getStatus(self):
  708         # returns a Deferred that fires when the builds have finished, and
  709         # may emit status messages while we wait
  710         wait = bool(self.getopt("wait"))
  711         if not wait:
  712             output("not waiting for builds to finish")
  713         elif self.connect == "ssh":
  714             output("waiting for builds with ssh is not supported")
  715         else:
  716             self.running = defer.Deferred()
  717             assert self.buildsetStatus
  718             self._getStatus_1()
  719             return self.running
  720 
  721     def _getStatus_1(self, res=None):
  722         if res:
  723             self.buildsetStatus = res
  724         # gather the set of BuildRequests
  725         d = self.buildsetStatus.callRemote("getBuildRequests")
  726         d.addCallback(self._getStatus_2)
  727 
  728     def _getStatus_2(self, brs):
  729         self.builderNames = []
  730         self.buildRequests = {}
  731 
  732         # self.builds holds the current BuildStatus object for each one
  733         self.builds = {}
  734 
  735         # self.outstanding holds the list of builderNames which haven't
  736         # finished yet
  737         self.outstanding = []
  738 
  739         # self.results holds the list of build results. It holds a tuple of
  740         # (result, text)
  741         self.results = {}
  742 
  743         # self.currentStep holds the name of the Step that each build is
  744         # currently running
  745         self.currentStep = {}
  746 
  747         # self.ETA holds the expected finishing time (absolute time since
  748         # epoch)
  749         self.ETA = {}
  750 
  751         for n, br in brs:
  752             self.builderNames.append(n)
  753             self.buildRequests[n] = br
  754             self.builds[n] = None
  755             self.outstanding.append(n)
  756             self.results[n] = [None, None]
  757             self.currentStep[n] = None
  758             self.ETA[n] = None
  759             # get new Builds for this buildrequest. We follow each one until
  760             # it finishes or is interrupted.
  761             br.callRemote("subscribe", self)
  762 
  763         # now that those queries are in transit, we can start the
  764         # display-status-every-30-seconds loop
  765         if not self.getopt("quiet"):
  766             self.printloop = task.LoopingCall(self.printStatus)
  767             self.printloop.start(3, now=False)
  768 
  769     # these methods are invoked by the status objects we've subscribed to
  770 
  771     def remote_newbuild(self, bs, builderName):
  772         if self.builds[builderName]:
  773             self.builds[builderName].callRemote("unsubscribe", self)
  774         self.builds[builderName] = bs
  775         bs.callRemote("subscribe", self, 20)
  776         d = bs.callRemote("waitUntilFinished")
  777         d.addCallback(self._build_finished, builderName)
  778 
  779     def remote_stepStarted(self, buildername, build, stepname, step):
  780         self.currentStep[buildername] = stepname
  781 
  782     def remote_stepFinished(self, buildername, build, stepname, step, results):
  783         pass
  784 
  785     def remote_buildETAUpdate(self, buildername, build, eta):
  786         self.ETA[buildername] = now() + eta
  787 
  788     def _build_finished(self, bs, builderName):
  789         # we need to collect status from the newly-finished build. We don't
  790         # remove the build from self.outstanding until we've collected
  791         # everything we want.
  792         self.builds[builderName] = None
  793         self.ETA[builderName] = None
  794         self.currentStep[builderName] = "finished"
  795         d = bs.callRemote("getResults")
  796         d.addCallback(self._build_finished_2, bs, builderName)
  797         return d
  798 
  799     def _build_finished_2(self, results, bs, builderName):
  800         self.results[builderName][0] = results
  801         d = bs.callRemote("getText")
  802         d.addCallback(self._build_finished_3, builderName)
  803         return d
  804 
  805     def _build_finished_3(self, text, builderName):
  806         self.results[builderName][1] = text
  807 
  808         self.outstanding.remove(builderName)
  809         if not self.outstanding:
  810             # all done
  811             return self.statusDone()
  812 
  813     def printStatus(self):
  814         try:
  815             names = sorted(self.buildRequests.keys())
  816             for n in names:
  817                 if n not in self.outstanding:
  818                     # the build is finished, and we have results
  819                     code, text = self.results[n]
  820                     t = builder.Results[code]
  821                     if text:
  822                         t += " ({})".format(" ".join(text))
  823                 elif self.builds[n]:
  824                     t = self.currentStep[n] or "building"
  825                     if self.ETA[n]:
  826                         t += " [ETA {}s]".format(self.ETA[n] - now())
  827                 else:
  828                     t = "no build"
  829                 self.announce("{}: {}".format(n, t))
  830             self.announce("")
  831         except Exception:
  832             log.err(None, "printing status")
  833 
  834     def statusDone(self):
  835         if self.printloop:
  836             self.printloop.stop()
  837             self.printloop = None
  838         output("All Builds Complete")
  839         # TODO: include a URL for all failing builds
  840         names = sorted(self.buildRequests.keys())
  841         happy = True
  842         for n in names:
  843             code, text = self.results[n]
  844             t = "{}: {}".format(n, builder.Results[code])
  845             if text:
  846                 t += " ({})".format(" ".join(text))
  847             output(t)
  848             if code != builder.SUCCESS:
  849                 happy = False
  850 
  851         if happy:
  852             self.exitcode = 0
  853         else:
  854             self.exitcode = 1
  855         self.running.callback(self.exitcode)
  856 
  857     def getAvailableBuilderNames(self):
  858         # This logs into the master using the PB protocol to
  859         # get the names of the configured builders that can
  860         # be used for the --builder argument
  861         if self.connect == "pb":
  862             user = self.getopt("username")
  863             passwd = self.getopt("passwd")
  864             master = self.getopt("master")
  865             tryhost, tryport = master.split(":")
  866             tryport = int(tryport)
  867             f = pb.PBClientFactory()
  868             d = f.login(credentials.UsernamePassword(unicode2bytes(user), unicode2bytes(passwd)))
  869             reactor.connectTCP(tryhost, tryport, f)
  870             d.addCallback(self._getBuilderNames)
  871             return d
  872         if self.connect == "ssh":
  873             output("Cannot get available builders over ssh.")
  874             sys.exit(1)
  875         raise RuntimeError(
  876             "unknown connecttype '{}', should be 'pb'".format(self.connect))
  877 
  878     def _getBuilderNames(self, remote):
  879         d = remote.callRemote("getAvailableBuilderNames")
  880         d.addCallback(self._getBuilderNames2)
  881         d.addCallback(lambda _: remote.broker.transport.loseConnection())
  882         return d
  883 
  884     def _getBuilderNames2(self, buildernames):
  885         output("The following builders are available for the try scheduler: ")
  886         for buildername in buildernames:
  887             output(buildername)
  888 
  889     def announce(self, message):
  890         if not self.quiet:
  891             output(message)
  892 
  893     def run(self, _inTests=False):
  894         # we can't do spawnProcess until we're inside reactor.run(), so get
  895         # funky
  896         output("using '{}' connect method".format(self.connect))
  897         self.exitcode = 0
  898         d = fireEventually(None)
  899         if bool(self.config.get("get-builder-names")):
  900             d.addCallback(lambda res: self.getAvailableBuilderNames())
  901         else:
  902             d.addCallback(lambda res: self.createJob())
  903             d.addCallback(lambda res: self.announce("job created"))
  904             deliver = self.deliverJob
  905             if bool(self.config.get("dryrun")):
  906                 deliver = self.fakeDeliverJob
  907             d.addCallback(lambda res: deliver())
  908             d.addCallback(lambda res: self.announce("job has been delivered"))
  909             d.addCallback(lambda res: self.getStatus())
  910         d.addErrback(self.trapSystemExit)
  911         d.addErrback(log.err)
  912         d.addCallback(self.cleanup)
  913         if _inTests:
  914             return d
  915         d.addCallback(lambda res: reactor.stop())
  916 
  917         reactor.run()
  918         sys.exit(self.exitcode)
  919 
  920     def trapSystemExit(self, why):
  921         why.trap(SystemExit)
  922         self.exitcode = why.value.code
  923 
  924     def cleanup(self, res=None):
  925         if self.buildsetStatus:
  926             self.buildsetStatus.broker.transport.loseConnection()