"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.5.1/buildbot/steps/worker.py" (24 Nov 2019, 11796 Bytes) of package /linux/misc/buildbot-2.5.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "worker.py" see the Fossies "Dox" file reference documentation.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 
   17 import os
   18 import stat
   19 
   20 from buildbot.process import buildstep
   21 from buildbot.process import remotecommand
   22 from buildbot.process import remotetransfer
   23 from buildbot.process.results import FAILURE
   24 from buildbot.process.results import SUCCESS
   25 from buildbot.util import bytes2unicode
   26 
   27 
   28 class WorkerBuildStep(buildstep.BuildStep):
   29     pass
   30 
   31 
   32 class SetPropertiesFromEnv(WorkerBuildStep):
   33 
   34     """
   35     Sets properties from environment variables on the worker.
   36 
   37     Note this is transferred when the worker first connects
   38     """
   39     name = 'SetPropertiesFromEnv'
   40     description = ['Setting']
   41     descriptionDone = ['Set']
   42 
   43     def __init__(self, variables, source="WorkerEnvironment", **kwargs):
   44         super().__init__(**kwargs)
   45         self.variables = variables
   46         self.source = source
   47 
   48     def start(self):
   49         # on Windows, environment variables are case-insensitive, but we have
   50         # a case-sensitive dictionary in worker_environ.  Fortunately, that
   51         # dictionary is also folded to uppercase, so we can simply fold the
   52         # variable names to uppercase to duplicate the case-insensitivity.
   53         fold_to_uppercase = (self.worker.worker_system == 'win32')
   54 
   55         properties = self.build.getProperties()
   56         environ = self.worker.worker_environ
   57         variables = self.variables
   58         log = []
   59         if isinstance(variables, str):
   60             variables = [self.variables]
   61         for variable in variables:
   62             key = variable
   63             if fold_to_uppercase:
   64                 key = variable.upper()
   65             value = environ.get(key, None)
   66             if value:
   67                 # note that the property is not uppercased
   68                 properties.setProperty(variable, value, self.source,
   69                                        runtime=True)
   70                 log.append("%s = %r" % (variable, value))
   71         self.addCompleteLog("properties", "\n".join(log))
   72         self.finished(SUCCESS)
   73 
   74 
   75 class FileExists(WorkerBuildStep):
   76 
   77     """
   78     Check for the existence of a file on the worker.
   79     """
   80     name = 'FileExists'
   81     renderables = ['file']
   82     haltOnFailure = True
   83     flunkOnFailure = True
   84 
   85     def __init__(self, file, **kwargs):
   86         super().__init__(**kwargs)
   87         self.file = file
   88 
   89     def start(self):
   90         self.checkWorkerHasCommand('stat')
   91         cmd = remotecommand.RemoteCommand('stat', {'file': self.file})
   92         d = self.runCommand(cmd)
   93         d.addCallback(lambda res: self.commandComplete(cmd))
   94         d.addErrback(self.failed)
   95 
   96     def commandComplete(self, cmd):
   97         if cmd.didFail():
   98             self.descriptionDone = ["File not found."]
   99             self.finished(FAILURE)
  100             return
  101         s = cmd.updates["stat"][-1]
  102         if stat.S_ISREG(s[stat.ST_MODE]):
  103             self.descriptionDone = ["File found."]
  104             self.finished(SUCCESS)
  105         else:
  106             self.descriptionDone = ["Not a file."]
  107             self.finished(FAILURE)
  108 
  109 
  110 class CopyDirectory(WorkerBuildStep):
  111 
  112     """
  113     Copy a directory tree on the worker.
  114     """
  115     name = 'CopyDirectory'
  116     description = ['Copying']
  117     descriptionDone = ['Copied']
  118 
  119     renderables = ['src', 'dest']
  120 
  121     haltOnFailure = True
  122     flunkOnFailure = True
  123 
  124     def __init__(self, src, dest, timeout=None, maxTime=None, **kwargs):
  125         super().__init__(**kwargs)
  126         self.src = src
  127         self.dest = dest
  128         self.timeout = timeout
  129         self.maxTime = maxTime
  130 
  131     def start(self):
  132         self.checkWorkerHasCommand('cpdir')
  133 
  134         args = {'fromdir': self.src, 'todir': self.dest}
  135         if self.timeout:
  136             args['timeout'] = self.timeout
  137         if self.maxTime:
  138             args['maxTime'] = self.maxTime
  139 
  140         cmd = remotecommand.RemoteCommand('cpdir', args)
  141         d = self.runCommand(cmd)
  142         d.addCallback(lambda res: self.commandComplete(cmd))
  143         d.addErrback(self.failed)
  144 
  145     def commandComplete(self, cmd):
  146         if cmd.didFail():
  147             self.step_status.setText(["Copying", self.src, "to", self.dest, "failed."])
  148             self.finished(FAILURE)
  149             return
  150         self.step_status.setText(self.describe(done=True))
  151         self.finished(SUCCESS)
  152 
  153     # TODO: BuildStep subclasses don't have a describe()....
  154     def getResultSummary(self):
  155         src = bytes2unicode(self.src, errors='replace')
  156         dest = bytes2unicode(self.dest, errors='replace')
  157         copy = "{} to {}".format(src, dest)
  158         if self.results == SUCCESS:
  159             rv = 'Copied ' + copy
  160         else:
  161             rv = 'Copying ' + copy + ' failed.'
  162         return {'step': rv}
  163 
  164 
  165 class RemoveDirectory(WorkerBuildStep):
  166 
  167     """
  168     Remove a directory tree on the worker.
  169     """
  170     name = 'RemoveDirectory'
  171     description = ['Deleting']
  172     descriptionDone = ['Deleted']
  173 
  174     renderables = ['dir']
  175 
  176     haltOnFailure = True
  177     flunkOnFailure = True
  178 
  179     def __init__(self, dir, **kwargs):
  180         super().__init__(**kwargs)
  181         self.dir = dir
  182 
  183     def start(self):
  184         self.checkWorkerHasCommand('rmdir')
  185         cmd = remotecommand.RemoteCommand('rmdir', {'dir': self.dir})
  186         d = self.runCommand(cmd)
  187         d.addCallback(lambda res: self.commandComplete(cmd))
  188         d.addErrback(self.failed)
  189 
  190     def commandComplete(self, cmd):
  191         if cmd.didFail():
  192             self.step_status.setText(["Delete failed."])
  193             self.finished(FAILURE)
  194             return
  195         self.finished(SUCCESS)
  196 
  197 
  198 class MakeDirectory(WorkerBuildStep):
  199 
  200     """
  201     Create a directory on the worker.
  202     """
  203     name = 'MakeDirectory'
  204     description = ['Creating']
  205     descriptionDone = ['Created']
  206 
  207     renderables = ['dir']
  208 
  209     haltOnFailure = True
  210     flunkOnFailure = True
  211 
  212     def __init__(self, dir, **kwargs):
  213         super().__init__(**kwargs)
  214         self.dir = dir
  215 
  216     def start(self):
  217         self.checkWorkerHasCommand('mkdir')
  218         cmd = remotecommand.RemoteCommand('mkdir', {'dir': self.dir})
  219         d = self.runCommand(cmd)
  220         d.addCallback(lambda res: self.commandComplete(cmd))
  221         d.addErrback(self.failed)
  222 
  223     def commandComplete(self, cmd):
  224         if cmd.didFail():
  225             self.step_status.setText(["Create failed."])
  226             self.finished(FAILURE)
  227             return
  228         self.finished(SUCCESS)
  229 
  230 
  231 class CompositeStepMixin():
  232 
  233     def workerPathToMasterPath(self, path):
  234         return os.path.join(*self.worker.path_module.split(path))
  235 
  236     def addLogForRemoteCommands(self, logname):
  237         """This method must be called by user classes
  238         composite steps could create several logs, this mixin functions will write
  239         to the last one.
  240         """
  241         self.rc_log = self.addLog(logname)
  242         return self.rc_log
  243 
  244     def runRemoteCommand(self, cmd, args, abandonOnFailure=True,
  245                          evaluateCommand=lambda cmd: cmd.didFail()):
  246         """generic RemoteCommand boilerplate"""
  247         cmd = remotecommand.RemoteCommand(cmd, args)
  248         if hasattr(self, "rc_log"):
  249             cmd.useLog(self.rc_log, False)
  250         d = self.runCommand(cmd)
  251 
  252         def commandComplete(cmd):
  253             if abandonOnFailure and cmd.didFail():
  254                 raise buildstep.BuildStepFailed()
  255             return evaluateCommand(cmd)
  256 
  257         d.addCallback(lambda res: commandComplete(cmd))
  258         return d
  259 
  260     def runRmdir(self, dir, timeout=None, **kwargs):
  261         """ remove a directory from the worker """
  262         cmd_args = {'dir': dir, 'logEnviron': self.logEnviron}
  263         if timeout:
  264             cmd_args['timeout'] = timeout
  265         return self.runRemoteCommand('rmdir', cmd_args, **kwargs)
  266 
  267     def runRmFile(self, path, timeout=None, **kwargs):
  268         """ remove a file from the worker """
  269         cmd_args = {'path': path, 'logEnviron': self.logEnviron}
  270         if timeout:
  271             cmd_args['timeout'] = timeout
  272         if self.workerVersionIsOlderThan('rmfile', '3.1'):
  273             cmd_args['dir'] = os.path.abspath(path)
  274             return self.runRemoteCommand('rmdir', cmd_args, **kwargs)
  275         return self.runRemoteCommand('rmfile', cmd_args, **kwargs)
  276 
  277     def pathExists(self, path):
  278         """ test whether path exists"""
  279         def commandComplete(cmd):
  280             return not cmd.didFail()
  281 
  282         return self.runRemoteCommand('stat', {'file': path,
  283                                               'logEnviron': self.logEnviron, },
  284                                      abandonOnFailure=False,
  285                                      evaluateCommand=commandComplete)
  286 
  287     def runMkdir(self, _dir, **kwargs):
  288         """ create a directory and its parents"""
  289         return self.runRemoteCommand('mkdir', {'dir': _dir,
  290                                                'logEnviron': self.logEnviron, },
  291                                      **kwargs)
  292 
  293     def runGlob(self, path, **kwargs):
  294         """ find files matching a shell-style pattern"""
  295         def commandComplete(cmd):
  296             return cmd.updates['files'][-1]
  297 
  298         return self.runRemoteCommand('glob', {'path': path,
  299                                               'logEnviron': self.logEnviron, },
  300                                      evaluateCommand=commandComplete, **kwargs)
  301 
  302     def getFileContentFromWorker(self, filename, abandonOnFailure=False):
  303         self.checkWorkerHasCommand("uploadFile")
  304         fileWriter = remotetransfer.StringFileWriter()
  305         # default arguments
  306         args = {
  307             'workdir': self.workdir,
  308             'writer': fileWriter,
  309             'maxsize': None,
  310             'blocksize': 32 * 1024,
  311         }
  312 
  313         if self.workerVersionIsOlderThan('uploadFile', '3.0'):
  314             args['slavesrc'] = filename
  315         else:
  316             args['workersrc'] = filename
  317 
  318         def commandComplete(cmd):
  319             if cmd.didFail():
  320                 return None
  321             return fileWriter.buffer
  322 
  323         return self.runRemoteCommand('uploadFile', args,
  324                                      abandonOnFailure=abandonOnFailure,
  325                                      evaluateCommand=commandComplete)
  326 
  327     def downloadFileContentToWorker(self, workerdest, strfile,
  328                                     abandonOnFailure=False, mode=None,
  329                                     workdir=None):
  330         if workdir is None:
  331             workdir = self.workdir
  332 
  333         self.checkWorkerHasCommand("downloadFile")
  334         fileReader = remotetransfer.StringFileReader(strfile)
  335         # default arguments
  336         args = {
  337             'workdir': workdir,
  338             'maxsize': None,
  339             'mode': mode,
  340             'reader': fileReader,
  341             'blocksize': 32 * 1024,
  342         }
  343 
  344         if self.workerVersionIsOlderThan('downloadFile', '3.0'):
  345             args['slavedest'] = workerdest
  346         else:
  347             args['workerdest'] = workerdest
  348 
  349         def commandComplete(cmd):
  350             if cmd.didFail():
  351                 return None
  352             return fileReader
  353         return self.runRemoteCommand('downloadFile', args,
  354                                      abandonOnFailure=abandonOnFailure,
  355                                      evaluateCommand=commandComplete)