"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.5.1/buildbot/changes/gitpoller.py" (24 Nov 2019, 17000 Bytes) of package /linux/misc/buildbot-2.5.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "gitpoller.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.3.1_vs_2.4.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 import os
   17 import re
   18 import stat
   19 from urllib.parse import quote as urlquote
   20 
   21 from twisted.internet import defer
   22 from twisted.internet import utils
   23 from twisted.python import log
   24 
   25 from buildbot import config
   26 from buildbot.changes import base
   27 from buildbot.util import bytes2unicode
   28 from buildbot.util import private_tempdir
   29 from buildbot.util.git import GitMixin
   30 from buildbot.util.git import getSshKnownHostsContents
   31 from buildbot.util.misc import writeLocalFile
   32 from buildbot.util.state import StateMixin
   33 
   34 
   35 class GitError(Exception):
   36 
   37     """Raised when git exits with code 128."""
   38 
   39 
   40 class GitPoller(base.PollingChangeSource, StateMixin, GitMixin):
   41 
   42     """This source will poll a remote git repo for changes and submit
   43     them to the change master."""
   44 
   45     compare_attrs = ("repourl", "branches", "workdir",
   46                      "pollInterval", "gitbin", "usetimestamps",
   47                      "category", "project", "pollAtLaunch",
   48                      "buildPushesWithNoCommits", "sshPrivateKey", "sshHostKey",
   49                      "sshKnownHosts")
   50 
   51     secrets = ("sshPrivateKey", "sshHostKey", "sshKnownHosts")
   52 
   53     def __init__(self, repourl, branches=None, branch=None,
   54                  workdir=None, pollInterval=10 * 60,
   55                  gitbin='git', usetimestamps=True,
   56                  category=None, project=None,
   57                  pollinterval=-2, fetch_refspec=None,
   58                  encoding='utf-8', name=None, pollAtLaunch=False,
   59                  buildPushesWithNoCommits=False, only_tags=False,
   60                  sshPrivateKey=None, sshHostKey=None, sshKnownHosts=None):
   61 
   62         # for backward compatibility; the parameter used to be spelled with 'i'
   63         if pollinterval != -2:
   64             pollInterval = pollinterval
   65 
   66         if name is None:
   67             name = repourl
   68 
   69         super().__init__(name=name,
   70                          pollInterval=pollInterval,
   71                          pollAtLaunch=pollAtLaunch,
   72                          sshPrivateKey=sshPrivateKey,
   73                          sshHostKey=sshHostKey,
   74                          sshKnownHosts=sshKnownHosts)
   75 
   76         if project is None:
   77             project = ''
   78 
   79         if only_tags and (branch or branches):
   80             config.error("GitPoller: can't specify only_tags and branch/branches")
   81         if branch and branches:
   82             config.error("GitPoller: can't specify both branch and branches")
   83         elif branch:
   84             branches = [branch]
   85         elif not branches:
   86             if only_tags:
   87                 branches = lambda ref: ref.startswith('refs/tags/')  # noqa: E731
   88             else:
   89                 branches = ['master']
   90 
   91         self.repourl = repourl
   92         self.branches = branches
   93         self.encoding = encoding
   94         self.buildPushesWithNoCommits = buildPushesWithNoCommits
   95         self.gitbin = gitbin
   96         self.workdir = workdir
   97         self.usetimestamps = usetimestamps
   98         self.category = category if callable(
   99             category) else bytes2unicode(category, encoding=self.encoding)
  100         self.project = bytes2unicode(project, encoding=self.encoding)
  101         self.changeCount = 0
  102         self.lastRev = {}
  103         self.sshPrivateKey = sshPrivateKey
  104         self.sshHostKey = sshHostKey
  105         self.sshKnownHosts = sshKnownHosts
  106         self.setupGit(logname='GitPoller')
  107 
  108         if fetch_refspec is not None:
  109             config.error("GitPoller: fetch_refspec is no longer supported. "
  110                          "Instead, only the given branches are downloaded.")
  111 
  112         if self.workdir is None:
  113             self.workdir = 'gitpoller-work'
  114 
  115     @defer.inlineCallbacks
  116     def _checkGitFeatures(self):
  117         stdout = yield self._dovccmd('--version', [])
  118 
  119         self.parseGitFeatures(stdout)
  120         if not self.gitInstalled:
  121             raise EnvironmentError('Git is not installed')
  122 
  123         if (self.sshPrivateKey is not None and
  124                 not self.supportsSshPrivateKeyAsEnvOption):
  125             raise EnvironmentError('SSH private keys require Git 2.3.0 or newer')
  126 
  127     @defer.inlineCallbacks
  128     def activate(self):
  129         # make our workdir absolute, relative to the master's basedir
  130         if not os.path.isabs(self.workdir):
  131             self.workdir = os.path.join(self.master.basedir, self.workdir)
  132             log.msg("gitpoller: using workdir '{}'".format(self.workdir))
  133 
  134         try:
  135             self.lastRev = yield self.getState('lastRev', {})
  136 
  137             super().activate()
  138         except Exception as e:
  139             log.err(e, 'while initializing GitPoller repository')
  140 
  141     def describe(self):
  142         str = ('GitPoller watching the remote git repository ' +
  143                bytes2unicode(self.repourl, self.encoding))
  144 
  145         if self.branches:
  146             if self.branches is True:
  147                 str += ', branches: ALL'
  148             elif not callable(self.branches):
  149                 str += ', branches: ' + ', '.join(self.branches)
  150 
  151         if not self.master:
  152             str += " [STOPPED - check log]"
  153 
  154         return str
  155 
  156     def _getBranches(self):
  157         d = self._dovccmd('ls-remote', ['--refs', self.repourl])
  158 
  159         @d.addCallback
  160         def parseRemote(rows):
  161             branches = []
  162             for row in rows.splitlines():
  163                 if '\t' not in row:
  164                     # Not a useful line
  165                     continue
  166                 sha, ref = row.split("\t")
  167                 branches.append(ref)
  168             return branches
  169         return d
  170 
  171     def _headsFilter(self, branch):
  172         """Filter out remote references that don't begin with 'refs/heads'."""
  173         return branch.startswith("refs/heads/")
  174 
  175     def _removeHeads(self, branch):
  176         """Remove 'refs/heads/' prefix from remote references."""
  177         if branch.startswith("refs/heads/"):
  178             branch = branch[11:]
  179         return branch
  180 
  181     def _trackerBranch(self, branch):
  182         # manually quote tilde for Python 3.7
  183         url = urlquote(self.repourl, '').replace('~', '%7E')
  184         return "refs/buildbot/{}/{}".format(url, self._removeHeads(branch))
  185 
  186     @defer.inlineCallbacks
  187     def poll(self):
  188         yield self._checkGitFeatures()
  189 
  190         try:
  191             yield self._dovccmd('init', ['--bare', self.workdir])
  192         except GitError as e:
  193             log.msg(e.args[0])
  194             return
  195 
  196         branches = self.branches if self.branches else []
  197         remote_refs = yield self._getBranches()
  198         if branches is True or callable(branches):
  199             if callable(self.branches):
  200                 branches = [b for b in remote_refs if self.branches(b)]
  201             else:
  202                 branches = [b for b in remote_refs if self._headsFilter(b)]
  203         elif branches and remote_refs:
  204             remote_branches = [self._removeHeads(b) for b in remote_refs]
  205             branches = sorted(list(set(branches) & set(remote_branches)))
  206 
  207         refspecs = [
  208             '+{}:{}'.format(self._removeHeads(branch), self._trackerBranch(branch))
  209             for branch in branches
  210         ]
  211 
  212         try:
  213             yield self._dovccmd('fetch', [self.repourl] + refspecs,
  214                                 path=self.workdir)
  215         except GitError as e:
  216             log.msg(e.args[0])
  217             return
  218 
  219         revs = {}
  220         log.msg('gitpoller: processing changes from "{}"'.format(self.repourl))
  221         for branch in branches:
  222             try:
  223                 rev = yield self._dovccmd(
  224                     'rev-parse', [self._trackerBranch(branch)], path=self.workdir)
  225                 revs[branch] = bytes2unicode(rev, self.encoding)
  226                 yield self._process_changes(revs[branch], branch)
  227             except Exception:
  228                 log.err(_why="trying to poll branch {} of {}".format(
  229                         branch, self.repourl))
  230 
  231         self.lastRev.update(revs)
  232         yield self.setState('lastRev', self.lastRev)
  233 
  234     def _get_commit_comments(self, rev):
  235         args = ['--no-walk', r'--format=%s%n%b', rev, '--']
  236         d = self._dovccmd('log', args, path=self.workdir)
  237         return d
  238 
  239     def _get_commit_timestamp(self, rev):
  240         # unix timestamp
  241         args = ['--no-walk', r'--format=%ct', rev, '--']
  242         d = self._dovccmd('log', args, path=self.workdir)
  243 
  244         @d.addCallback
  245         def process(git_output):
  246             if self.usetimestamps:
  247                 try:
  248                     stamp = int(git_output)
  249                 except Exception as e:
  250                     log.msg(
  251                         'gitpoller: caught exception converting output \'{}\' to timestamp'.format(git_output))
  252                     raise e
  253                 return stamp
  254             return None
  255         return d
  256 
  257     def _get_commit_files(self, rev):
  258         args = ['--name-only', '--no-walk', r'--format=%n', rev, '--']
  259         d = self._dovccmd('log', args, path=self.workdir)
  260 
  261         def decode_file(file):
  262             # git use octal char sequences in quotes when non ASCII
  263             match = re.match('^"(.*)"$', file)
  264             if match:
  265                 file = bytes2unicode(match.groups()[0], encoding=self.encoding,
  266                                      errors='unicode_escape')
  267             return bytes2unicode(file, encoding=self.encoding)
  268 
  269         @d.addCallback
  270         def process(git_output):
  271             fileList = [decode_file(file)
  272                         for file in
  273                         [s for s in git_output.splitlines() if len(s)]]
  274             return fileList
  275         return d
  276 
  277     def _get_commit_author(self, rev):
  278         args = ['--no-walk', r'--format=%aN <%aE>', rev, '--']
  279         d = self._dovccmd('log', args, path=self.workdir)
  280 
  281         @d.addCallback
  282         def process(git_output):
  283             if not git_output:
  284                 raise EnvironmentError('could not get commit author for rev')
  285             return git_output
  286         return d
  287 
  288     @defer.inlineCallbacks
  289     def _get_commit_committer(self, rev):
  290         args = ['--no-walk', r'--format=%cN <%cE>', rev, '--']
  291         res = yield self._dovccmd('log', args, path=self.workdir)
  292         if not res:
  293             raise EnvironmentError('could not get commit committer for rev')
  294         return res
  295 
  296     @defer.inlineCallbacks
  297     def _process_changes(self, newRev, branch):
  298         """
  299         Read changes since last change.
  300 
  301         - Read list of commit hashes.
  302         - Extract details from each commit.
  303         - Add changes to database.
  304         """
  305 
  306         # initial run, don't parse all history
  307         if not self.lastRev:
  308             return
  309 
  310         # get the change list
  311         revListArgs = (['--format=%H', '{}'.format(newRev)] +
  312                        ['^' + rev
  313                         for rev in sorted(self.lastRev.values())] +
  314                        ['--'])
  315         self.changeCount = 0
  316         results = yield self._dovccmd('log', revListArgs, path=self.workdir)
  317 
  318         # process oldest change first
  319         revList = results.split()
  320         revList.reverse()
  321 
  322         if self.buildPushesWithNoCommits and not revList:
  323             existingRev = self.lastRev.get(branch)
  324             if existingRev != newRev:
  325                 revList = [newRev]
  326                 if existingRev is None:
  327                     # This branch was completely unknown, rebuild
  328                     log.msg('gitpoller: rebuilding {} for new branch "{}"'.format(
  329                         newRev, branch))
  330                 else:
  331                     # This branch is known, but it now points to a different
  332                     # commit than last time we saw it, rebuild.
  333                     log.msg('gitpoller: rebuilding {} for updated branch "{}"'.format(
  334                         newRev, branch))
  335 
  336         self.changeCount = len(revList)
  337         self.lastRev[branch] = newRev
  338 
  339         if self.changeCount:
  340             log.msg('gitpoller: processing {} changes: {} from "{}" branch "{}"'.format(
  341                     self.changeCount, revList, self.repourl, branch))
  342 
  343         for rev in revList:
  344             dl = defer.DeferredList([
  345                 self._get_commit_timestamp(rev),
  346                 self._get_commit_author(rev),
  347                 self._get_commit_committer(rev),
  348                 self._get_commit_files(rev),
  349                 self._get_commit_comments(rev),
  350             ], consumeErrors=True)
  351 
  352             results = yield dl
  353 
  354             # check for failures
  355             failures = [r[1] for r in results if not r[0]]
  356             if failures:
  357                 for failure in failures:
  358                     log.err(
  359                         failure, "while processing changes for {} {}".format(newRev, branch))
  360                 # just fail on the first error; they're probably all related!
  361                 failures[0].raiseException()
  362 
  363             timestamp, author, committer, files, comments = [r[1] for r in results]
  364 
  365             yield self.master.data.updates.addChange(
  366                 author=author,
  367                 committer=committer,
  368                 revision=bytes2unicode(rev, encoding=self.encoding),
  369                 files=files, comments=comments, when_timestamp=timestamp,
  370                 branch=bytes2unicode(self._removeHeads(branch)),
  371                 project=self.project,
  372                 repository=bytes2unicode(self.repourl, encoding=self.encoding),
  373                 category=self.category, src='git')
  374 
  375     def _isSshPrivateKeyNeededForCommand(self, command):
  376         commandsThatNeedKey = [
  377             'fetch',
  378             'ls-remote',
  379         ]
  380         if self.sshPrivateKey is not None and command in commandsThatNeedKey:
  381             return True
  382         return False
  383 
  384     def _downloadSshPrivateKey(self, keyPath):
  385         # We change the permissions of the key file to be user-readable only so
  386         # that ssh does not complain. This is not used for security because the
  387         # parent directory will have proper permissions.
  388         writeLocalFile(keyPath, self.sshPrivateKey, mode=stat.S_IRUSR)
  389 
  390     def _downloadSshKnownHosts(self, path):
  391         if self.sshKnownHosts is not None:
  392             contents = self.sshKnownHosts
  393         else:
  394             contents = getSshKnownHostsContents(self.sshHostKey)
  395         writeLocalFile(path, contents)
  396 
  397     def _getSshPrivateKeyPath(self, ssh_data_path):
  398         return os.path.join(ssh_data_path, 'ssh-key')
  399 
  400     def _getSshKnownHostsPath(self, ssh_data_path):
  401         return os.path.join(ssh_data_path, 'ssh-known-hosts')
  402 
  403     @defer.inlineCallbacks
  404     def _dovccmd(self, command, args, path=None):
  405         if self._isSshPrivateKeyNeededForCommand(command):
  406             with private_tempdir.PrivateTemporaryDirectory(
  407                     dir=self.workdir, prefix='.buildbot-ssh') as tmp_path:
  408                 stdout = yield self._dovccmdImpl(command, args, path, tmp_path)
  409         else:
  410             stdout = yield self._dovccmdImpl(command, args, path, None)
  411         return stdout
  412 
  413     @defer.inlineCallbacks
  414     def _dovccmdImpl(self, command, args, path, ssh_workdir):
  415         full_args = []
  416         full_env = os.environ.copy()
  417 
  418         if self._isSshPrivateKeyNeededForCommand(command):
  419             key_path = self._getSshPrivateKeyPath(ssh_workdir)
  420             self._downloadSshPrivateKey(key_path)
  421 
  422             known_hosts_path = None
  423             if self.sshHostKey is not None or self.sshKnownHosts is not None:
  424                 known_hosts_path = self._getSshKnownHostsPath(ssh_workdir)
  425                 self._downloadSshKnownHosts(known_hosts_path)
  426 
  427             self.adjustCommandParamsForSshPrivateKey(full_args, full_env,
  428                                                      key_path, None,
  429                                                      known_hosts_path)
  430 
  431         full_args += [command] + args
  432 
  433         res = yield utils.getProcessOutputAndValue(self.gitbin,
  434             full_args, path=path, env=full_env)
  435         (stdout, stderr, code) = res
  436         stdout = bytes2unicode(stdout, self.encoding)
  437         stderr = bytes2unicode(stderr, self.encoding)
  438         if code != 0:
  439             if code == 128:
  440                 raise GitError('command {} in {} on repourl {} failed with exit code {}: {}'.format(
  441                                full_args, path, self.repourl, code, stderr))
  442             raise EnvironmentError('command {} in {} on repourl {} failed with exit code {}: {}'.format(
  443                                    full_args, path, self.repourl, code, stderr))
  444         return stdout.strip()