"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.5.1/buildbot/changes/svnpoller.py" (24 Nov 2019, 17685 Bytes) of package /linux/misc/buildbot-2.5.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "svnpoller.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.3.1_vs_2.4.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 # Based on the work of Dave Peticolas for the P4poll
   16 # Changed to svn (using xml.dom.minidom) by Niklaus Giger
   17 # Hacked beyond recognition by Brian Warner
   18 
   19 import os
   20 import xml.dom.minidom
   21 from urllib.parse import quote_plus as urlquote_plus
   22 
   23 from twisted.internet import defer
   24 from twisted.internet import utils
   25 from twisted.python import log
   26 
   27 from buildbot import util
   28 from buildbot.changes import base
   29 from buildbot.util import bytes2unicode
   30 
   31 # these split_file_* functions are available for use as values to the
   32 # split_file= argument.
   33 
   34 
   35 def split_file_alwaystrunk(path):
   36     return dict(path=path)
   37 
   38 
   39 def split_file_branches(path):
   40     # turn "trunk/subdir/file.c" into (None, "subdir/file.c")
   41     # and "trunk/subdir/" into (None, "subdir/")
   42     # and "trunk/" into (None, "")
   43     # and "branches/1.5.x/subdir/file.c" into ("branches/1.5.x", "subdir/file.c")
   44     # and "branches/1.5.x/subdir/" into ("branches/1.5.x", "subdir/")
   45     # and "branches/1.5.x/" into ("branches/1.5.x", "")
   46     pieces = path.split('/')
   47     if len(pieces) > 1 and pieces[0] == 'trunk':
   48         return (None, '/'.join(pieces[1:]))
   49     elif len(pieces) > 2 and pieces[0] == 'branches':
   50         return ('/'.join(pieces[0:2]), '/'.join(pieces[2:]))
   51     return None
   52 
   53 
   54 def split_file_projects_branches(path):
   55     # turn projectname/trunk/subdir/file.c into dict(project=projectname,
   56     # branch=trunk, path=subdir/file.c)
   57     if "/" not in path:
   58         return None
   59     project, path = path.split("/", 1)
   60     f = split_file_branches(path)
   61     if f:
   62         info = dict(project=project, path=f[1])
   63         if f[0]:
   64             info['branch'] = f[0]
   65         return info
   66     return f
   67 
   68 
   69 class SVNPoller(base.PollingChangeSource, util.ComparableMixin):
   70 
   71     """
   72     Poll a Subversion repository for changes and submit them to the change
   73     master.
   74     """
   75 
   76     compare_attrs = ("repourl", "split_file",
   77                      "svnuser", "svnpasswd", "project",
   78                      "pollInterval", "histmax",
   79                      "svnbin", "category", "cachepath", "pollAtLaunch")
   80     secrets = ("svnuser", "svnpasswd")
   81     parent = None  # filled in when we're added
   82     last_change = None
   83     loop = None
   84 
   85     def __init__(self, repourl, split_file=None,
   86                  svnuser=None, svnpasswd=None,
   87                  pollInterval=10 * 60, histmax=100,
   88                  svnbin='svn', revlinktmpl='', category=None,
   89                  project='', cachepath=None, pollinterval=-2,
   90                  extra_args=None, name=None, pollAtLaunch=False):
   91 
   92         # for backward compatibility; the parameter used to be spelled with 'i'
   93         if pollinterval != -2:
   94             pollInterval = pollinterval
   95 
   96         if name is None:
   97             name = repourl
   98 
   99         super().__init__(name=name,
  100                          pollInterval=pollInterval,
  101                          pollAtLaunch=pollAtLaunch,
  102                          svnuser=svnuser, svnpasswd=svnpasswd)
  103 
  104         if repourl.endswith("/"):
  105             repourl = repourl[:-1]  # strip the trailing slash
  106         self.repourl = repourl
  107         self.extra_args = extra_args
  108         self.split_file = split_file or split_file_alwaystrunk
  109         self.svnuser = svnuser
  110         self.svnpasswd = svnpasswd
  111 
  112         self.revlinktmpl = revlinktmpl
  113 
  114         # include environment variables required for ssh-agent auth
  115         self.environ = os.environ.copy()
  116 
  117         self.svnbin = svnbin
  118         self.histmax = histmax
  119         self._prefix = None
  120         self.category = category if callable(
  121             category) else util.bytes2unicode(category)
  122         self.project = util.bytes2unicode(project)
  123 
  124         self.cachepath = cachepath
  125         if self.cachepath and os.path.exists(self.cachepath):
  126             try:
  127                 with open(self.cachepath, "r") as f:
  128                     self.last_change = int(f.read().strip())
  129                     log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % (
  130                         self.repourl, self.last_change))
  131                 # try writing it, too
  132                 with open(self.cachepath, "w") as f:
  133                     f.write(str(self.last_change))
  134             except Exception:
  135                 self.cachepath = None
  136                 log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " +
  137                          "skipping and not using") % self.repourl)
  138                 log.err()
  139 
  140     def describe(self):
  141         return "SVNPoller: watching %s" % self.repourl
  142 
  143     def poll(self):
  144         # Our return value is only used for unit testing.
  145 
  146         # we need to figure out the repository root, so we can figure out
  147         # repository-relative pathnames later. Each REPOURL is in the form
  148         # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something
  149         # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a
  150         # physical repository at /svn/Twisted on that host), (PROJECT) is
  151         # something like Projects/Twisted (i.e. within the repository's
  152         # internal namespace, everything under Projects/Twisted/ has
  153         # something to do with Twisted, but these directory names do not
  154         # actually appear on the repository host), (BRANCH) is something like
  155         # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative
  156         # filename like "twisted/internet/defer.py".
  157 
  158         # our self.repourl attribute contains (ROOT)/(PROJECT) combined
  159         # together in a way that we can't separate without svn's help. If the
  160         # user is not using the split_file= argument, then self.repourl might
  161         # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will
  162         # get back from 'svn log' will be of the form
  163         # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove
  164         # that (PROJECT) prefix from them. To do this without requiring the
  165         # user to tell us how repourl is split into ROOT and PROJECT, we do an
  166         # 'svn info --xml' command at startup. This command will include a
  167         # <root> element that tells us ROOT. We then strip this prefix from
  168         # self.repourl to determine PROJECT, and then later we strip the
  169         # PROJECT prefix from the filenames reported by 'svn log --xml' to
  170         # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to
  171         # turn into separate BRANCH and FILEPATH values.
  172 
  173         # whew.
  174 
  175         if self.project:
  176             log.msg("SVNPoller: polling " + self.project)
  177         else:
  178             log.msg("SVNPoller: polling")
  179 
  180         d = defer.succeed(None)
  181         if not self._prefix:
  182             d.addCallback(lambda _: self.get_prefix())
  183 
  184             @d.addCallback
  185             def set_prefix(prefix):
  186                 self._prefix = prefix
  187 
  188         d.addCallback(self.get_logs)
  189         d.addCallback(self.parse_logs)
  190         d.addCallback(self.get_new_logentries)
  191         d.addCallback(self.create_changes)
  192         d.addCallback(self.submit_changes)
  193         d.addCallback(self.finished_ok)
  194         # eat errors
  195         d.addErrback(log.err, 'SVNPoller: Error in  while polling')
  196         return d
  197 
  198     def getProcessOutput(self, args):
  199         # this exists so we can override it during the unit tests
  200         d = utils.getProcessOutput(self.svnbin, args, self.environ)
  201         return d
  202 
  203     def get_prefix(self):
  204         args = ["info", "--xml", "--non-interactive", self.repourl]
  205         if self.svnuser:
  206             args.append("--username=%s" % self.svnuser)
  207         if self.svnpasswd is not None:
  208             args.append("--password=%s" % self.svnpasswd)
  209         if self.extra_args:
  210             args.extend(self.extra_args)
  211         d = self.getProcessOutput(args)
  212 
  213         @d.addCallback
  214         def determine_prefix(output):
  215             try:
  216                 doc = xml.dom.minidom.parseString(output)
  217             except xml.parsers.expat.ExpatError:
  218                 log.msg("SVNPoller: SVNPoller.get_prefix: ExpatError in '%s'"
  219                         % output)
  220                 raise
  221             rootnodes = doc.getElementsByTagName("root")
  222             if not rootnodes:
  223                 # this happens if the URL we gave was already the root. In this
  224                 # case, our prefix is empty.
  225                 self._prefix = ""
  226                 return self._prefix
  227             rootnode = rootnodes[0]
  228             root = "".join([c.data for c in rootnode.childNodes])
  229             # root will be a unicode string
  230             if not self.repourl.startswith(root):
  231                 log.msg(format="Got root %(root)r from `svn info`, but it is "
  232                                "not a prefix of the configured repourl",
  233                         repourl=self.repourl, root=root)
  234                 raise RuntimeError("Configured repourl doesn't match svn root")
  235             prefix = self.repourl[len(root):]
  236             if prefix.startswith("/"):
  237                 prefix = prefix[1:]
  238             log.msg("SVNPoller: repourl=%s, root=%s, so prefix=%s" %
  239                     (self.repourl, root, prefix))
  240             return prefix
  241         return d
  242 
  243     def get_logs(self, _):
  244         args = []
  245         args.extend(["log", "--xml", "--verbose", "--non-interactive"])
  246         if self.svnuser:
  247             args.extend(["--username=%s" % self.svnuser])
  248         if self.svnpasswd is not None:
  249             args.extend(["--password=%s" % self.svnpasswd])
  250         if self.extra_args:
  251             args.extend(self.extra_args)
  252         args.extend(["--limit=%d" % (self.histmax), self.repourl])
  253         d = self.getProcessOutput(args)
  254         return d
  255 
  256     def parse_logs(self, output):
  257         # parse the XML output, return a list of <logentry> nodes
  258         try:
  259             doc = xml.dom.minidom.parseString(output)
  260         except xml.parsers.expat.ExpatError:
  261             log.msg(
  262                 "SVNPoller: SVNPoller.parse_logs: ExpatError in '%s'" % output)
  263             raise
  264         logentries = doc.getElementsByTagName("logentry")
  265         return logentries
  266 
  267     def get_new_logentries(self, logentries):
  268         last_change = old_last_change = self.last_change
  269 
  270         # given a list of logentries, calculate new_last_change, and
  271         # new_logentries, where new_logentries contains only the ones after
  272         # last_change
  273 
  274         new_last_change = None
  275         new_logentries = []
  276         if logentries:
  277             new_last_change = int(logentries[0].getAttribute("revision"))
  278 
  279             if last_change is None:
  280                 # if this is the first time we've been run, ignore any changes
  281                 # that occurred before now. This prevents a build at every
  282                 # startup.
  283                 log.msg('SVNPoller: starting at change %s' % new_last_change)
  284             elif last_change == new_last_change:
  285                 # an unmodified repository will hit this case
  286                 log.msg('SVNPoller: no changes')
  287             else:
  288                 for el in logentries:
  289                     if last_change == int(el.getAttribute("revision")):
  290                         break
  291                     new_logentries.append(el)
  292                 new_logentries.reverse()  # return oldest first
  293 
  294         self.last_change = new_last_change
  295         log.msg('SVNPoller: _process_changes %s .. %s' %
  296                 (old_last_change, new_last_change))
  297         return new_logentries
  298 
  299     def _get_text(self, element, tag_name):
  300         try:
  301             child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
  302             text = "".join([t.data for t in child_nodes])
  303         except IndexError:
  304             text = "unknown"
  305         return text
  306 
  307     def _transform_path(self, path):
  308         if not path.startswith(self._prefix):
  309             log.msg(format="SVNPoller: ignoring path '%(path)s' which doesn't"
  310                     "start with prefix '%(prefix)s'",
  311                     path=path, prefix=self._prefix)
  312             return
  313         relative_path = path[len(self._prefix):]
  314         if relative_path.startswith("/"):
  315             relative_path = relative_path[1:]
  316         where = self.split_file(relative_path)
  317         # 'where' is either None, (branch, final_path) or a dict
  318         if not where:
  319             return
  320         if isinstance(where, tuple):
  321             where = dict(branch=where[0], path=where[1])
  322         return where
  323 
  324     def create_changes(self, new_logentries):
  325         changes = []
  326 
  327         for el in new_logentries:
  328             revision = str(el.getAttribute("revision"))
  329 
  330             revlink = ''
  331 
  332             if self.revlinktmpl and revision:
  333                 revlink = self.revlinktmpl % urlquote_plus(revision)
  334                 revlink = str(revlink)
  335 
  336             log.msg("Adding change revision %s" % (revision,))
  337             author = self._get_text(el, "author")
  338             comments = self._get_text(el, "msg")
  339             # there is a "date" field, but it provides localtime in the
  340             # repository's timezone, whereas we care about buildmaster's
  341             # localtime (since this will get used to position the boxes on
  342             # the Waterfall display, etc). So ignore the date field, and
  343             # addChange will fill in with the current time
  344             branches = {}
  345             try:
  346                 pathlist = el.getElementsByTagName("paths")[0]
  347             except IndexError:  # weird, we got an empty revision
  348                 log.msg("ignoring commit with no paths")
  349                 continue
  350 
  351             for p in pathlist.getElementsByTagName("path"):
  352                 kind = p.getAttribute("kind")
  353                 action = p.getAttribute("action")
  354                 path = "".join([t.data for t in p.childNodes])
  355                 if path.startswith("/"):
  356                     path = path[1:]
  357                 if kind == "dir" and not path.endswith("/"):
  358                     path += "/"
  359                 where = self._transform_path(path)
  360 
  361                 # if 'where' is None, the file was outside any project that
  362                 # we care about and we should ignore it
  363                 if where:
  364                     branch = where.get("branch", None)
  365                     filename = where["path"]
  366                     if branch not in branches:
  367                         branches[branch] = {
  368                             'files': [], 'number_of_directories': 0}
  369                     if filename == "":
  370                         # root directory of branch
  371                         branches[branch]['files'].append(filename)
  372                         branches[branch]['number_of_directories'] += 1
  373                     elif filename.endswith("/"):
  374                         # subdirectory of branch
  375                         branches[branch]['files'].append(filename[:-1])
  376                         branches[branch]['number_of_directories'] += 1
  377                     else:
  378                         branches[branch]['files'].append(filename)
  379 
  380                     if "action" not in branches[branch]:
  381                         branches[branch]['action'] = action
  382 
  383                     for key in ("repository", "project", "codebase"):
  384                         if key in where:
  385                             branches[branch][key] = where[key]
  386 
  387             for branch in branches:
  388                 action = branches[branch]['action']
  389                 files = branches[branch]['files']
  390 
  391                 number_of_directories_changed = branches[
  392                     branch]['number_of_directories']
  393                 number_of_files_changed = len(files)
  394 
  395                 if (action == 'D' and number_of_directories_changed == 1 and
  396                         number_of_files_changed == 1 and files[0] == ''):
  397                     log.msg("Ignoring deletion of branch '%s'" % branch)
  398                 else:
  399                     chdict = dict(
  400                         author=author,
  401                         committer=None,
  402                         # weakly assume filenames are utf-8
  403                         files=[bytes2unicode(f, 'utf-8', 'replace')
  404                                for f in files],
  405                         comments=comments,
  406                         revision=revision,
  407                         branch=util.bytes2unicode(branch),
  408                         revlink=revlink,
  409                         category=self.category,
  410                         repository=util.bytes2unicode(
  411                             branches[branch].get('repository', self.repourl)),
  412                         project=util.bytes2unicode(
  413                             branches[branch].get('project', self.project)),
  414                         codebase=util.bytes2unicode(
  415                             branches[branch].get('codebase', None)))
  416                     changes.append(chdict)
  417 
  418         return changes
  419 
  420     @defer.inlineCallbacks
  421     def submit_changes(self, changes):
  422         for chdict in changes:
  423             yield self.master.data.updates.addChange(src='svn', **chdict)
  424 
  425     def finished_ok(self, res):
  426         if self.cachepath:
  427             with open(self.cachepath, "w") as f:
  428                 f.write(str(self.last_change))
  429 
  430         log.msg("SVNPoller: finished polling %s" % res)
  431         return res