"Fossies" - the Fresh Open Source Software Archive

Member "buildbot-2.3.1/buildbot/master.py" (23 May 2019, 17190 Bytes) of package /linux/misc/buildbot-2.3.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers. Alternatively you can here view or download the uninterpreted source code file. For more information about "master.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.2.0_vs_2.3.0.

    1 # This file is part of Buildbot.  Buildbot is free software: you can
    2 # redistribute it and/or modify it under the terms of the GNU General Public
    3 # License as published by the Free Software Foundation, version 2.
    4 #
    5 # This program is distributed in the hope that it will be useful, but WITHOUT
    6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    8 # details.
    9 #
   10 # You should have received a copy of the GNU General Public License along with
   11 # this program; if not, write to the Free Software Foundation, Inc., 51
   12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
   13 #
   14 # Copyright Buildbot Team Members
   15 
   16 
   17 import os
   18 import signal
   19 import socket
   20 
   21 from twisted.application import internet
   22 from twisted.internet import defer
   23 from twisted.internet import task
   24 from twisted.internet import threads
   25 from twisted.python import failure
   26 from twisted.python import log
   27 
   28 import buildbot
   29 import buildbot.pbmanager
   30 from buildbot import config
   31 from buildbot import monkeypatches
   32 from buildbot.buildbot_net_usage_data import sendBuildbotNetUsageData
   33 from buildbot.changes.manager import ChangeManager
   34 from buildbot.data import connector as dataconnector
   35 from buildbot.db import connector as dbconnector
   36 from buildbot.db import exceptions
   37 from buildbot.machine.manager import MachineManager
   38 from buildbot.mq import connector as mqconnector
   39 from buildbot.process import cache
   40 from buildbot.process import debug
   41 from buildbot.process import metrics
   42 from buildbot.process.botmaster import BotMaster
   43 from buildbot.process.users.manager import UserManagerManager
   44 from buildbot.schedulers.manager import SchedulerManager
   45 from buildbot.secrets.manager import SecretManager
   46 from buildbot.status.master import Status
   47 from buildbot.util import check_functional_environment
   48 from buildbot.util import service
   49 from buildbot.util.eventual import eventually
   50 from buildbot.wamp import connector as wampconnector
   51 from buildbot.worker import manager as workermanager
   52 from buildbot.www import service as wwwservice
   53 
   54 
   55 class LogRotation:
   56 
   57     def __init__(self):
   58         self.rotateLength = 1 * 1000 * 1000
   59         self.maxRotatedFiles = 10
   60 
   61 
   62 class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService):
   63 
   64     # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
   65     # unclaimed; this should be at least 2 to avoid false positives
   66     UNCLAIMED_BUILD_FACTOR = 6
   67 
   68     def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
   69         super().__init__()
   70 
   71         if reactor is None:
   72             from twisted.internet import reactor
   73         self.reactor = reactor
   74 
   75         self.setName("buildmaster")
   76 
   77         self.umask = umask
   78 
   79         self.basedir = basedir
   80         if basedir is not None:  # None is used in tests
   81             assert os.path.isdir(self.basedir)
   82 
   83         if config_loader is not None and configFileName is not None:
   84             raise config.ConfigErrors([
   85                 "Can't specify both `config_loader` and `configFilename`.",
   86             ])
   87         if config_loader is None:
   88             if configFileName is None:
   89                 configFileName = 'master.cfg'
   90             config_loader = config.FileLoader(self.basedir, configFileName)
   91         self.config_loader = config_loader
   92         self.configFileName = configFileName
   93 
   94         # flag so we don't try to do fancy things before the master is ready
   95         self._master_initialized = False
   96         self.initLock = defer.DeferredLock()
   97 
   98         # set up child services
   99         self.create_child_services()
  100 
  101         # db configured values
  102         self.configured_db_url = None
  103 
  104         # configuration / reconfiguration handling
  105         self.config = config.MasterConfig()
  106         self.reconfig_active = False
  107         self.reconfig_requested = False
  108         self.reconfig_notifier = None
  109 
  110         # this stores parameters used in the tac file, and is accessed by the
  111         # WebStatus to duplicate those values.
  112         self.log_rotation = LogRotation()
  113 
  114         # local cache for this master's object ID
  115         self._object_id = None
  116 
  117         # Check environment is sensible
  118         check_functional_environment(self.config)
  119 
  120         # figure out local hostname
  121         try:
  122             self.hostname = os.uname()[1]  # only on unix
  123         except AttributeError:
  124             self.hostname = socket.getfqdn()
  125 
  126         # public attributes
  127         self.name = ("%s:%s" % (self.hostname,
  128                                 os.path.abspath(self.basedir or '.')))
  129         if isinstance(self.name, bytes):
  130             self.name = self.name.decode('ascii', 'replace')
  131         self.masterid = None
  132 
  133     def create_child_services(self):
  134         # note that these are order-dependent.  If you get the order wrong,
  135         # you'll know it, as the master will fail to start.
  136 
  137         self.metrics = metrics.MetricLogObserver()
  138         self.metrics.setServiceParent(self)
  139 
  140         self.caches = cache.CacheManager()
  141         self.caches.setServiceParent(self)
  142 
  143         self.pbmanager = buildbot.pbmanager.PBManager()
  144         self.pbmanager.setServiceParent(self)
  145 
  146         self.workers = workermanager.WorkerManager(self)
  147         self.workers.setServiceParent(self)
  148 
  149         self.change_svc = ChangeManager()
  150         self.change_svc.setServiceParent(self)
  151 
  152         self.botmaster = BotMaster()
  153         self.botmaster.setServiceParent(self)
  154 
  155         self.machine_manager = MachineManager()
  156         self.machine_manager.setServiceParent(self)
  157 
  158         self.scheduler_manager = SchedulerManager()
  159         self.scheduler_manager.setServiceParent(self)
  160 
  161         self.user_manager = UserManagerManager(self)
  162         self.user_manager.setServiceParent(self)
  163 
  164         self.db = dbconnector.DBConnector(self.basedir)
  165         self.db.setServiceParent(self)
  166 
  167         self.wamp = wampconnector.WampConnector()
  168         self.wamp.setServiceParent(self)
  169 
  170         self.mq = mqconnector.MQConnector()
  171         self.mq.setServiceParent(self)
  172 
  173         self.data = dataconnector.DataConnector()
  174         self.data.setServiceParent(self)
  175 
  176         self.www = wwwservice.WWWService()
  177         self.www.setServiceParent(self)
  178 
  179         self.debug = debug.DebugServices()
  180         self.debug.setServiceParent(self)
  181 
  182         self.status = Status()
  183         self.status.setServiceParent(self)
  184 
  185         self.secrets_manager = SecretManager()
  186         self.secrets_manager.setServiceParent(self)
  187         self.secrets_manager.reconfig_priority = 2000
  188 
  189         self.service_manager = service.BuildbotServiceManager()
  190         self.service_manager.setServiceParent(self)
  191         self.service_manager.reconfig_priority = 1000
  192 
  193         self.masterHouskeepingTimer = 0
  194 
  195         @defer.inlineCallbacks
  196         def heartbeat():
  197             if self.masterid is not None:
  198                 yield self.data.updates.masterActive(name=self.name,
  199                                                      masterid=self.masterid)
  200             yield self.data.updates.expireMasters()
  201         self.masterHeartbeatService = internet.TimerService(60, heartbeat)
  202         self.masterHeartbeatService.clock = self.reactor
  203         # we do setServiceParent only when the master is configured
  204         # master should advertise itself only at that time
  205 
  206     # setup and reconfig handling
  207 
  208     _already_started = False
  209 
  210     @defer.inlineCallbacks
  211     def startService(self):
  212         assert not self._already_started, "can only start the master once"
  213         self._already_started = True
  214 
  215         log.msg("Starting BuildMaster -- buildbot.version: %s" %
  216                 buildbot.version)
  217 
  218         # Set umask
  219         if self.umask is not None:
  220             os.umask(self.umask)
  221 
  222         # first, apply all monkeypatches
  223         monkeypatches.patch_all()
  224 
  225         # we want to wait until the reactor is running, so we can call
  226         # reactor.stop() for fatal errors
  227         d = defer.Deferred()
  228         self.reactor.callWhenRunning(d.callback, None)
  229         yield d
  230 
  231         startup_succeed = False
  232         try:
  233             yield self.initLock.acquire()
  234             # load the configuration file, treating errors as fatal
  235             try:
  236                 # run the master.cfg in thread, so that it can use blocking
  237                 # code
  238                 self.config = yield threads.deferToThreadPool(
  239                     self.reactor, self.reactor.getThreadPool(),
  240                     self.config_loader.loadConfig)
  241 
  242             except config.ConfigErrors as e:
  243                 log.msg("Configuration Errors:")
  244                 for msg in e.errors:
  245                     log.msg("  " + msg)
  246                 log.msg("Halting master.")
  247                 self.reactor.stop()
  248                 return
  249             except Exception:
  250                 log.err(failure.Failure(), 'while starting BuildMaster')
  251                 self.reactor.stop()
  252                 return
  253 
  254             # set up services that need access to the config before everything
  255             # else gets told to reconfig
  256             try:
  257                 yield self.db.setup()
  258             except exceptions.DatabaseNotReadyError:
  259                 # (message was already logged)
  260                 self.reactor.stop()
  261                 return
  262 
  263             self.mq.setup()
  264 
  265             if hasattr(signal, "SIGHUP"):
  266                 def sighup(*args):
  267                     eventually(self.reconfig)
  268                 signal.signal(signal.SIGHUP, sighup)
  269 
  270             if hasattr(signal, "SIGUSR1"):
  271                 def sigusr1(*args):
  272                     eventually(self.botmaster.cleanShutdown)
  273                 signal.signal(signal.SIGUSR1, sigusr1)
  274 
  275             # get the masterid so other services can use it in
  276             # startup/reconfig.  This goes directly to the DB since the data
  277             # API isn't initialized yet, and anyway, this method is aware of
  278             # the DB API since it just called its setup function
  279             self.masterid = yield self.db.masters.findMasterId(
  280                 name=self.name)
  281 
  282             # mark this master as stopped, in case it crashed before
  283             yield self.data.updates.masterStopped(name=self.name,
  284                                                   masterid=self.masterid)
  285 
  286             # call the parent method
  287             yield super().startService()
  288 
  289             # We make sure the housekeeping is done before configuring in order to cleanup
  290             # any remaining claimed schedulers or change sources from zombie
  291             # masters
  292             yield self.data.updates.expireMasters(forceHouseKeeping=True)
  293 
  294             # give all services a chance to load the new configuration, rather
  295             # than the base configuration
  296             yield self.reconfigServiceWithBuildbotConfig(self.config)
  297 
  298             # Mark the master as active now that mq is running
  299             yield self.data.updates.masterActive(name=self.name,
  300                                                  masterid=self.masterid)
  301 
  302             # Start the heartbeat timer
  303             yield self.masterHeartbeatService.setServiceParent(self)
  304 
  305             # send the statistics to buildbot.net, without waiting
  306             self.sendBuildbotNetUsageData()
  307             startup_succeed = True
  308         except Exception:
  309             f = failure.Failure()
  310             log.err(f, 'while starting BuildMaster')
  311             self.reactor.stop()
  312 
  313         finally:
  314             if startup_succeed:
  315                 log.msg("BuildMaster is running")
  316             else:
  317                 log.msg("BuildMaster startup failed")
  318 
  319             yield self.initLock.release()
  320             self._master_initialized = True
  321 
  322     def sendBuildbotNetUsageData(self):
  323         if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
  324             raise RuntimeError(
  325                 "Should not enable buildbotNetUsageData in trial tests!")
  326         sendBuildbotNetUsageData(self)
  327 
  328     @defer.inlineCallbacks
  329     def stopService(self):
  330         try:
  331             yield self.initLock.acquire()
  332             if self.masterid is not None:
  333                 yield self.data.updates.masterStopped(
  334                     name=self.name, masterid=self.masterid)
  335             if self.running:
  336                 yield self.botmaster.cleanShutdown(
  337                     quickMode=True, stopReactor=False)
  338                 yield super().stopService()
  339 
  340             log.msg("BuildMaster is stopped")
  341             self._master_initialized = False
  342         finally:
  343             yield self.initLock.release()
  344 
  345     def reconfig(self):
  346         # this method wraps doConfig, ensuring it is only ever called once at
  347         # a time, and alerting the user if the reconfig takes too long
  348         if self.reconfig_active:
  349             log.msg("reconfig already active; will reconfig again after")
  350             self.reconfig_requested = True
  351             return
  352 
  353         self.reconfig_active = self.reactor.seconds()
  354         metrics.MetricCountEvent.log("loaded_config", 1)
  355 
  356         # notify every 10 seconds that the reconfig is still going on, although
  357         # reconfigs should not take that long!
  358         self.reconfig_notifier = task.LoopingCall(lambda:
  359                                                   log.msg("reconfig is ongoing for %d s" %
  360                                                           (self.reactor.seconds() - self.reconfig_active)))
  361         self.reconfig_notifier.start(10, now=False)
  362 
  363         timer = metrics.Timer("BuildMaster.reconfig")
  364         timer.start()
  365 
  366         d = self.doReconfig()
  367 
  368         @d.addBoth
  369         def cleanup(res):
  370             timer.stop()
  371             self.reconfig_notifier.stop()
  372             self.reconfig_notifier = None
  373             self.reconfig_active = False
  374             if self.reconfig_requested:
  375                 self.reconfig_requested = False
  376                 self.reconfig()
  377             return res
  378 
  379         d.addErrback(log.err, 'while reconfiguring')
  380 
  381         return d  # for tests
  382 
  383     @defer.inlineCallbacks
  384     def doReconfig(self):
  385         log.msg("beginning configuration update")
  386         changes_made = False
  387         failed = False
  388         try:
  389             yield self.initLock.acquire()
  390             # Run the master.cfg in thread, so that it can use blocking code
  391             new_config = yield threads.deferToThreadPool(
  392                 self.reactor, self.reactor.getThreadPool(),
  393                 self.config_loader.loadConfig)
  394             changes_made = True
  395             self.config = new_config
  396 
  397             yield self.reconfigServiceWithBuildbotConfig(new_config)
  398 
  399         except config.ConfigErrors as e:
  400             for msg in e.errors:
  401                 log.msg(msg)
  402             failed = True
  403 
  404         except Exception:
  405             log.err(failure.Failure(), 'during reconfig:')
  406             failed = True
  407 
  408         finally:
  409             yield self.initLock.release()
  410 
  411         if failed:
  412             if changes_made:
  413                 log.msg("WARNING: reconfig partially applied; master "
  414                         "may malfunction")
  415             else:
  416                 log.msg("reconfig aborted without making any changes")
  417         else:
  418             log.msg("configuration update complete")
  419 
  420     def reconfigServiceWithBuildbotConfig(self, new_config):
  421         if self.configured_db_url is None:
  422             self.configured_db_url = new_config.db['db_url']
  423         elif (self.configured_db_url != new_config.db['db_url']):
  424             config.error(
  425                 "Cannot change c['db']['db_url'] after the master has started",
  426             )
  427 
  428         if self.config.mq['type'] != new_config.mq['type']:
  429             raise config.ConfigErrors([
  430                 "Cannot change c['mq']['type'] after the master has started",
  431             ])
  432 
  433         return super().reconfigServiceWithBuildbotConfig(new_config)
  434 
  435     # informational methods
  436     def allSchedulers(self):
  437         return list(self.scheduler_manager)
  438 
  439     def getStatus(self):
  440         """
  441         @rtype: L{buildbot.status.builder.Status}
  442         """
  443         return self.status
  444 
  445     # state maintenance (private)
  446     def getObjectId(self):
  447         """
  448         Return the object id for this master, for associating state with the
  449         master.
  450 
  451         @returns: ID, via Deferred
  452         """
  453         # try to get the cached value
  454         if self._object_id is not None:
  455             return defer.succeed(self._object_id)
  456 
  457         # failing that, get it from the DB; multiple calls to this function
  458         # at the same time will not hurt
  459 
  460         d = self.db.state.getObjectId(self.name,
  461                                       "buildbot.master.BuildMaster")
  462 
  463         @d.addCallback
  464         def keep(id):
  465             self._object_id = id
  466             return id
  467         return d
  468 
  469     def _getState(self, name, default=None):
  470         "private wrapper around C{self.db.state.getState}"
  471         d = self.getObjectId()
  472 
  473         @d.addCallback
  474         def get(objectid):
  475             return self.db.state.getState(objectid, name, default)
  476         return d
  477 
  478     def _setState(self, name, value):
  479         "private wrapper around C{self.db.state.setState}"
  480         d = self.getObjectId()
  481 
  482         @d.addCallback
  483         def set(objectid):
  484             return self.db.state.setState(objectid, name, value)
  485         return d