jhbuild r2304 - in trunk: . buildbot jhbuild/buildbot jhbuild/commands
- From: fpeters svn gnome org
- To: svn-commits-list gnome org
- Subject: jhbuild r2304 - in trunk: . buildbot jhbuild/buildbot jhbuild/commands
- Date: Sun, 24 Aug 2008 11:44:26 +0000 (UTC)
Author: fpeters
Date: Sun Aug 24 11:44:26 2008
New Revision: 2304
URL: http://svn.gnome.org/viewvc/jhbuild?rev=2304&view=rev
Log:
* buildbot/master.cfg, jhbuild/buildbot/__init__.py,
jhbuild/commands/bot.py: moved build master "unconfiguration" stuff
out of master.cfg.
Modified:
trunk/ChangeLog
trunk/buildbot/master.cfg
trunk/jhbuild/buildbot/__init__.py
trunk/jhbuild/commands/bot.py
Modified: trunk/buildbot/master.cfg
==============================================================================
--- trunk/buildbot/master.cfg (original)
+++ trunk/buildbot/master.cfg Sun Aug 24 11:44:26 2008
@@ -10,112 +10,16 @@
# buildmaster. They are documented in docs/config.xhtml .
import buildbot
-import sys
-import os
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
-####### BUILDSLAVES
-
-# the 'slaves' list is read from the 'slaves.csv' file in the current directory
-# it is a CSV file structured like this:
-# slavename,password
-from buildbot.buildslave import BuildSlave
-import csv
-
-if not os.path.exists('slaves.csv'):
- c['slaves'] = []
-else:
- c['slaves'] = [BuildSlave(x[0], x[1])
- for x in csv.reader(file('slaves.csv'))
- if x and not x[0].startswith('#')]
-
-if len(c['slaves']) == 0:
- print >> sys.stderr, 'You must fill slaves.csv with slaves'
-
-# to limit to two concurrent builds on a slave, use
-# c['slaves'] = [BuildSlave("bot1name", "bot1passwd", max_builds=2)]
-
-
# 'slavePortnum' defines the TCP port to listen on. This must match the value
# configured into the buildslaves (with their --master option)
c['slavePortnum'] = 9070
-####### PROJECTS
-
-# the 'projects' list defines which jhbuild modules that to build
-from jhbuild.buildbot import jhbuild_list
-c['projects'] = jhbuild_list()
-
-####### CHANGESOURCES
-
-# the 'change_source' setting tells the buildmaster how it should find out
-# about source code changes. Any class which implements IChangeSource can be
-# put here: there are several in buildbot/changes/*.py to choose from.
-
-from jhbuild.buildbot.changes import GnomeMaildirSource
-if jhbuild_config.jhbuildbot_svn_commits_box:
- # trigger builds from mails to svn-commit-list
- # (note Maildir must be correct, or everything will fail)
- c['change_source'] = GnomeMaildirSource(
- jhbuild_config.jhbuildbot_svn_commits_box, prefix=None)
-else:
- # support injection (use 'buildbot sendchange')
- from buildbot.changes.pb import PBChangeSource
- c['change_source'] = PBChangeSource()
-
-####### SCHEDULERS
-
-## configure the Schedulers
-from jhbuild.buildbot.scheduler import SerialScheduler, OnCommitScheduler
-c['schedulers'] = []
-for slave in c['slaves']:
- s = None
- for project in c['projects']:
- buildername = str('%s-%s' % (project, slave.slavename))
- s = SerialScheduler(buildername, project, upstream=s,
- builderNames=[buildername])
- c['schedulers'].append(s)
- if jhbuild_config.jhbuildbot_svn_commits_box:
- s2 = OnCommitScheduler('oc-' + buildername, project, builderNames=[buildername])
- c['schedulers'].append(s2)
-
-####### BUILDERS
-
-# the 'builders' list defines the Builders. Each one is configured with a
-# dictionary, using the following keys:
-# name (required): the name used to describe this bilder
-# slavename (required): which slave to use, must appear in c['bots']
-# builddir (required): which subdirectory to run the builder in
-# factory (required): a BuildFactory to define how the build is run
-# periodicBuildTime (optional): if set, force a build every N seconds
-
-# buildbot/process/factory.py provides several BuildFactory classes you can
-# start with, which implement build processes for common targets (GNU
-# autoconf projects, CPAN perl modules, etc). The factory.BuildFactory is the
-# base class, and is configured with a series of BuildSteps. When the build
-# is run, the appropriate buildslave is told to execute each Step in turn.
-
-# the first BuildStep is typically responsible for obtaining a copy of the
-# sources. There are source-obtaining Steps in buildbot/steps/source.py for
-# CVS, SVN, and others.
-
-from jhbuild.buildbot.factory import JHBuildFactory
-c['builders'] = []
-for project in c['projects']:
- for slave in c['slaves']:
- f = JHBuildFactory(project)
- c['builders'].append({
- 'name' : "%s-%s" % (project, slave.slavename),
- 'slavename' : slave.slavename,
- 'builddir' : 'builddir/%s.%s' % (project, slave.slavename),
- 'factory' : f,
- 'category' : project
- })
-
####### STATUS TARGETS
# 'status' is a list of Status Targets. The results of each build will be
@@ -124,15 +28,6 @@
c['status'] = []
-from jhbuild.buildbot.status.web import JHBuildWebStatus
-c['status'].append(
- JHBuildWebStatus(
- jhbuild_config.moduleset,
- c['projects'],
- [x.slavename for x in c['slaves']],
- http_port=8080, allowForce=True)
-)
-
# from buildbot.status import mail
# c['status'].append(mail.MailNotifier(fromaddr="buildbot localhost",
# extraRecipients=["builds example com"],
Modified: trunk/jhbuild/buildbot/__init__.py
==============================================================================
--- trunk/jhbuild/buildbot/__init__.py (original)
+++ trunk/jhbuild/buildbot/__init__.py Sun Aug 24 11:44:26 2008
@@ -1,28 +0,0 @@
-# jhbuild - a build script for GNOME 1.x and 2.x
-# Copyright (C) 2008 apinheiro igalia com, John Carr, Frederic Peters
-#
-# __init__.py: utility functions for master.cfg
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-import jhbuild.moduleset
-
-def jhbuild_list():
- module_set = jhbuild.moduleset.load(jhbuild_config)
- module_list = module_set.get_module_list(
- jhbuild_config.modules,
- jhbuild_config.skip,
- include_optional_modules=True)
- return [x.name for x in module_list if not x.name.startswith('meta-')]
Modified: trunk/jhbuild/commands/bot.py
==============================================================================
--- trunk/jhbuild/commands/bot.py (original)
+++ trunk/jhbuild/commands/bot.py Sun Aug 24 11:44:26 2008
@@ -2,7 +2,7 @@
# Copyright (C) 2001-2006 James Henstridge
# Copyright (C) 2008 Frederic Peters
#
-# bot.py: buildbot slave commands
+# bot.py: buildbot control commands
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,6 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+#
+# Some methods are derived from Buildbot own methods (when it was not possible
+# to override just some parts of them). Buildbot is also licensed under the
+# GNU General Public License.
import os
import sys
@@ -24,6 +29,7 @@
from optparse import make_option
import socket
import __builtin__
+import csv
import jhbuild.moduleset
import jhbuild.frontends
@@ -177,9 +183,382 @@
def createOrGetApplication(self):
return self.application
- from twisted.application import service
+ from twisted.application import service, strports
from buildbot.master import BuildMaster
application = service.Application('buildmaster')
+ from buildbot.buildslave import BuildSlave
+
+ from twisted.python import log
+ from twisted.internet import defer
+ from buildbot import interfaces
+ from buildbot.process.properties import Properties
+
+
+ class JhBuildMaster(BuildMaster):
+ jhbuild_config = config
+ def loadConfig(self, f):
+ # modified from parent method to get slaves, projects, change
+ # sources, schedulers, builders and web status ouf of
+ # master.cfg [it would have been cleaner if jhbuild didn't
+ # have to copy all that code.]
+ localDict = {'basedir': os.path.expanduser(self.basedir)}
+ try:
+ exec f in localDict
+ except:
+ log.msg("error while parsing config file")
+ raise
+
+ try:
+ config = localDict['BuildmasterConfig']
+ except KeyError:
+ log.err("missing config dictionary")
+ log.err("config file must define BuildmasterConfig")
+ raise
+
+ known_keys = ("bots", "slaves",
+ "sources", "change_source",
+ "schedulers", "builders",
+ "slavePortnum", "debugPassword", "manhole",
+ "status", "projectName", "projectURL", "buildbotURL",
+ "properties"
+ )
+ for k in config.keys():
+ if k not in known_keys:
+ log.msg("unknown key '%s' defined in config dictionary" % k)
+
+ # the 'slaves' list is read from the 'slaves.csv' file in the
+ # current directory it is a CSV file structured like this:
+ # slavename,password
+ # it is also possible to define build slave options, for
+ # example:
+ # slavename,password,max_builds=2
+ # (recognized build slave options are max_build and
+ # missing_timeout)
+ config['slaves'] = []
+ if os.path.exists('slaves.csv'):
+ for x in csv.reader(file('slaves.csv')):
+ if not x or x[0].startswith('#'):
+ continue
+ kw = {}
+ for option in x[2:]:
+ if not '=' in option:
+ continue
+ k, v = option.split('=', 1)
+ if k in ('max_builds', 'missing_timeout'):
+ v = int(v)
+ else:
+ # unrecognized option
+ continue
+ kw[k] = v
+ config['slaves'].append(BuildSlave(x[0], x[1], **kw))
+
+ if len(config['slaves']) == 0:
+ log.msg('you must fill slaves.csv with slaves')
+
+ module_set = jhbuild.moduleset.load(self.jhbuild_config)
+ module_list = module_set.get_module_list(
+ self.jhbuild_config.modules,
+ self.jhbuild_config.skip,
+ include_optional_modules=True)
+ config['projects'] = [x.name for x in module_list \
+ if not x.name.startswith('meta-')]
+
+ if self.jhbuild_config.jhbuildbot_svn_commits_box:
+ # trigger builds from mails to svn-commit-list
+ # (note Maildir must be correct, or everything will fail)
+ from jhbuild.buildbot.changes import GnomeMaildirSource
+ config['change_source'] = GnomeMaildirSource(
+ self.jhbuild_config.jhbuildbot_svn_commits_box,
+ prefix=None)
+ else:
+ # support injection (use 'buildbot sendchange')
+ from buildbot.changes.pb import PBChangeSource
+ c['change_source'] = PBChangeSource()
+
+ # Schedulers
+ from jhbuild.buildbot.scheduler import SerialScheduler, OnCommitScheduler
+ config['schedulers'] = []
+ for slave in config['slaves']:
+ s = None
+ for project in config['projects']:
+ buildername = str('%s-%s' % (project, slave.slavename))
+ s = SerialScheduler(buildername, project, upstream=s,
+ builderNames=[buildername])
+ config['schedulers'].append(s)
+ if self.jhbuild_config.jhbuildbot_svn_commits_box:
+ # schedulers that will launch job when receiving
+ # change notifications
+ s2 = OnCommitScheduler('oc-' + buildername,
+ project, builderNames=[buildername])
+ config['schedulers'].append(s2)
+
+ # Builders
+ from jhbuild.buildbot.factory import JHBuildFactory
+ config['builders'] = []
+ for project in config['projects']:
+ for slave in config['slaves']:
+ f = JHBuildFactory(project)
+ config['builders'].append({
+ 'name' : "%s-%s" % (project, slave.slavename),
+ 'slavename' : slave.slavename,
+ 'builddir' : 'builddir/%s.%s' % (project, slave.slavename),
+ 'factory' : f,
+ 'category' : project
+ })
+
+ # Status targets
+ if not config.has_key('status'):
+ # let it be possible to define additional status in
+ # master.cfg
+ config['status'] = []
+
+ from jhbuild.buildbot.status.web import JHBuildWebStatus
+ config['status'].append(
+ JHBuildWebStatus(
+ self.jhbuild_config.moduleset,
+ config['projects'],
+ [x.slavename for x in config['slaves']],
+ http_port=8080, allowForce=True)
+ )
+
+ # remaining of the method is a straight copy from buildbot
+ # ...
+ try:
+ # required
+ schedulers = config['schedulers']
+ builders = config['builders']
+ for k in builders:
+ if k['name'].startswith("_"):
+ errmsg = ("builder names must not start with an "
+ "underscore: " + k['name'])
+ log.err(errmsg)
+ raise ValueError(errmsg)
+
+ slavePortnum = config['slavePortnum']
+ #slaves = config['slaves']
+ #change_source = config['change_source']
+
+ # optional
+ debugPassword = config.get('debugPassword')
+ manhole = config.get('manhole')
+ status = config.get('status', [])
+ projectName = config.get('projectName')
+ projectURL = config.get('projectURL')
+ buildbotURL = config.get('buildbotURL')
+ properties = config.get('properties', {})
+
+ except KeyError, e:
+ log.msg("config dictionary is missing a required parameter")
+ log.msg("leaving old configuration in place")
+ raise
+
+ #if "bots" in config:
+ # raise KeyError("c['bots'] is no longer accepted")
+
+ slaves = config.get('slaves', [])
+ if "bots" in config:
+ m = ("c['bots'] is deprecated as of 0.7.6 and will be "
+ "removed by 0.8.0 . Please use c['slaves'] instead.")
+ log.msg(m)
+ warnings.warn(m, DeprecationWarning)
+ for name, passwd in config['bots']:
+ slaves.append(BuildSlave(name, passwd))
+
+ if "bots" not in config and "slaves" not in config:
+ log.msg("config dictionary must have either 'bots' or 'slaves'")
+ log.msg("leaving old configuration in place")
+ raise KeyError("must have either 'bots' or 'slaves'")
+
+ #if "sources" in config:
+ # raise KeyError("c['sources'] is no longer accepted")
+
+ change_source = config.get('change_source', [])
+ if isinstance(change_source, (list, tuple)):
+ change_sources = change_source
+ else:
+ change_sources = [change_source]
+ if "sources" in config:
+ m = ("c['sources'] is deprecated as of 0.7.6 and will be "
+ "removed by 0.8.0 . Please use c['change_source'] instead.")
+ log.msg(m)
+ warnings.warn(m, DeprecationWarning)
+ for s in config['sources']:
+ change_sources.append(s)
+
+ # do some validation first
+ for s in slaves:
+ assert isinstance(s, BuildSlave)
+ if s.slavename in ("debug", "change", "status"):
+ raise KeyError, "reserved name '%s' used for a bot" % s.slavename
+ if config.has_key('interlocks'):
+ raise KeyError("c['interlocks'] is no longer accepted")
+
+ assert isinstance(change_sources, (list, tuple))
+ for s in change_sources:
+ assert interfaces.IChangeSource(s, None)
+ # this assertion catches c['schedulers'] = Scheduler(), since
+ # Schedulers are service.MultiServices and thus iterable.
+ errmsg = "c['schedulers'] must be a list of Scheduler instances"
+ assert isinstance(schedulers, (list, tuple)), errmsg
+ for s in schedulers:
+ assert interfaces.IScheduler(s, None), errmsg
+ assert isinstance(status, (list, tuple))
+ for s in status:
+ assert interfaces.IStatusReceiver(s, None)
+
+ slavenames = [s.slavename for s in slaves]
+ buildernames = []
+ dirnames = []
+ for b in builders:
+ if type(b) is tuple:
+ raise ValueError("builder %s must be defined with a dict, "
+ "not a tuple" % b[0])
+ if b.has_key('slavename') and b['slavename'] not in slavenames:
+ raise ValueError("builder %s uses undefined slave %s" \
+ % (b['name'], b['slavename']))
+ for n in b.get('slavenames', []):
+ if n not in slavenames:
+ raise ValueError("builder %s uses undefined slave %s" \
+ % (b['name'], n))
+ if b['name'] in buildernames:
+ raise ValueError("duplicate builder name %s"
+ % b['name'])
+ buildernames.append(b['name'])
+ if b['builddir'] in dirnames:
+ raise ValueError("builder %s reuses builddir %s"
+ % (b['name'], b['builddir']))
+ dirnames.append(b['builddir'])
+
+ unscheduled_buildernames = buildernames[:]
+ schedulernames = []
+ for s in schedulers:
+ for b in s.listBuilderNames():
+ assert b in buildernames, \
+ "%s uses unknown builder %s" % (s, b)
+ if b in unscheduled_buildernames:
+ unscheduled_buildernames.remove(b)
+
+ if s.name in schedulernames:
+ # TODO: schedulers share a namespace with other Service
+ # children of the BuildMaster node, like status plugins, the
+ # Manhole, the ChangeMaster, and the BotMaster (although most
+ # of these don't have names)
+ msg = ("Schedulers must have unique names, but "
+ "'%s' was a duplicate" % (s.name,))
+ raise ValueError(msg)
+ schedulernames.append(s.name)
+
+ if unscheduled_buildernames:
+ log.msg("Warning: some Builders have no Schedulers to drive them:"
+ " %s" % (unscheduled_buildernames,))
+
+ # assert that all locks used by the Builds and their Steps are
+ # uniquely named.
+ locks = {}
+ for b in builders:
+ for l in b.get('locks', []):
+ if locks.has_key(l.name):
+ if locks[l.name] is not l:
+ raise ValueError("Two different locks (%s and %s) "
+ "share the name %s"
+ % (l, locks[l.name], l.name))
+ else:
+ locks[l.name] = l
+ # TODO: this will break with any BuildFactory that doesn't use a
+ # .steps list, but I think the verification step is more
+ # important.
+ for s in b['factory'].steps:
+ for l in s[1].get('locks', []):
+ if locks.has_key(l.name):
+ if locks[l.name] is not l:
+ raise ValueError("Two different locks (%s and %s)"
+ " share the name %s"
+ % (l, locks[l.name], l.name))
+ else:
+ locks[l.name] = l
+
+ if not isinstance(properties, dict):
+ raise ValueError("c['properties'] must be a dictionary")
+
+ # slavePortnum supposed to be a strports specification
+ if type(slavePortnum) is int:
+ slavePortnum = "tcp:%d" % slavePortnum
+
+ # now we're committed to implementing the new configuration, so do
+ # it atomically
+ # TODO: actually, this is spread across a couple of Deferreds, so it
+ # really isn't atomic.
+
+ d = defer.succeed(None)
+
+ self.projectName = projectName
+ self.projectURL = projectURL
+ self.buildbotURL = buildbotURL
+
+ self.properties = Properties()
+ self.properties.update(properties, self.configFileName)
+
+ # self.slaves: Disconnect any that were attached and removed from the
+ # list. Update self.checker with the new list of passwords, including
+ # debug/change/status.
+ d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
+
+ # self.debugPassword
+ if debugPassword:
+ self.checker.addUser("debug", debugPassword)
+ self.debugPassword = debugPassword
+
+ # self.manhole
+ if manhole != self.manhole:
+ # changing
+ if self.manhole:
+ # disownServiceParent may return a Deferred
+ d.addCallback(lambda res: self.manhole.disownServiceParent())
+ def _remove(res):
+ self.manhole = None
+ return res
+ d.addCallback(_remove)
+ if manhole:
+ def _add(res):
+ self.manhole = manhole
+ manhole.setServiceParent(self)
+ d.addCallback(_add)
+
+ # add/remove self.botmaster.builders to match builders. The
+ # botmaster will handle startup/shutdown issues.
+ d.addCallback(lambda res: self.loadConfig_Builders(builders))
+
+ d.addCallback(lambda res: self.loadConfig_status(status))
+
+ # Schedulers are added after Builders in case they start right away
+ d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
+ # and Sources go after Schedulers for the same reason
+ d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
+
+ # self.slavePort
+ if self.slavePortnum != slavePortnum:
+ if self.slavePort:
+ def closeSlavePort(res):
+ d1 = self.slavePort.disownServiceParent()
+ self.slavePort = None
+ return d1
+ d.addCallback(closeSlavePort)
+ if slavePortnum is not None:
+ def openSlavePort(res):
+ self.slavePort = strports.service(slavePortnum,
+ self.slaveFactory)
+ self.slavePort.setServiceParent(self)
+ d.addCallback(openSlavePort)
+ log.msg("BuildMaster listening on port %s" % slavePortnum)
+ self.slavePortnum = slavePortnum
+
+ log.msg("configuration update started")
+ def _done(res):
+ self.readConfig = True
+ log.msg("configuration update complete")
+ d.addCallback(_done)
+ d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
+ return d
basedir = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
@@ -189,7 +568,7 @@
os.makedirs(os.path.join(basedir, 'builddir'))
master_cfg_path = os.path.join(basedir, 'master.cfg')
- BuildMaster(basedir, master_cfg_path).setServiceParent(application)
+ JhBuildMaster(basedir, master_cfg_path).setServiceParent(application)
JhBuildbotApplicationRunner.application = application
JhBuildbotApplicationRunner(options).run()
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
