extras-buildsys-temp/automation README, NONE, 1.1 archwelder.py, NONE, 1.1 aw-server.py, NONE, 1.1 buildqueue.py, NONE, 1.1
Seth Vidal (skvidal)
fedora-extras-commits at redhat.com
Thu May 5 08:07:43 UTC 2005
Author: skvidal
Update of /cvs/fedora/extras-buildsys-temp/automation
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv27622
Added Files:
README archwelder.py aw-server.py buildqueue.py
Log Message:
add the automation bits to the automation dir
--- NEW FILE README ---
These are the scripts used to automate the fedora extras build system.
They use the cvs infrastructure to obtain and build packages. For
more information on these scripts contact skvidal at fedoraproject.org or
see the fedora buildsys-list.
skvidal
5/5/2005
--- NEW FILE archwelder.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
# TODO: xml-rpc communication using 2-way ssl-cert-verified xmlrpc connection
# TODO: config from file not cli
import SimpleXMLRPCServer
import xmlrpclib
import socket
import os
import popen2
import sha
import time
import base64
DEBUG = False
def debugprint(stuff=''):
if DEBUG:
print stuff
def log(stuff=''):
print stuff
class ArchWelderMach:
"""puts things together for an arch - baseclass for handling builds for
other arches"""
def die(self, sig=15):
os.kill(self.pobj.pid, sig)
return True
def _unlock(self):
cmd = '%s /usr/bin/mach -r %s unlock' % (self.arch_command, self.buildroot)
self.pobj = popen2.Popen4(cmd=cmd)
self._status = 'unlocking'
while self.pobj.poll() == -1:
debugprint('still running unlock')
time.sleep(10)
def _clean(self):
cmd = '%s /usr/bin/mach -r %s clean' % (self.arch_command, self.buildroot)
self.pobj = popen2.Popen4(cmd=cmd)
self._status = 'cleaning'
while self.pobj.poll() == -1:
debugprint('still running clean')
time.sleep(10)
def _prep(self):
cmd = '%s /usr/bin/mach -r %s setup prep' % (self.arch_command, self.buildroot)
self.pobj = popen2.Popen4(cmd=cmd)
self._status = 'prep'
while self.pobj.poll() == -1:
debugprint('still running prep')
time.sleep(10)
def build(self):
# check for existence of srpm before going on
self._unlock()
self._clean()
self._prep()
os.chdir(self.mydir + '/' + self.buildarch)
cmd = '%s /usr/bin/mach -r %s -c rebuild %s' % (self.arch_command, self.buildroot, self.srpm)
self.pobj = popen2.Popen4(cmd=cmd)
self._status = 'building'
return self.status()
def status(self):
return self.pobj.poll()
def logs(self):
self._output.extend(self.pobj.fromchild.readlines())
return self._output
class i386Arch(ArchWelderMach):
def __init__(self, srpm, target, mydir, buildarch='i386'):
self._status = None
self.target = target
self.mydir = mydir
self.buildroot = 'fedora-%s-i386-core' % (target)
self.srpm = srpm
self.buildarch = buildarch
self.arch_command = '/usr/bin/setarch i686'
self._output = []
class x86_64Arch(ArchWelderMach):
def __init__(self, srpm, target, mydir, buildarch='x86_64'):
self._status = None
self.target = target
self.mydir = mydir
self.buildroot = 'fedora-%s-x86_64-core' % (target)
self.srpm = srpm
self.buildarch = buildarch
self.arch_command = ''
self._output = []
class PPCArch(ArchWelderMach):
def __init__(self, srpm, target, mydir, buildarch='ppc'):
self._status = None
self.target = target
self.mydir = mydir
self.buildroot = 'fedora-%s-ppc-core' % (target)
self.srpm = srpm
self.buildarch = buildarch
self.arch_command = ''
self._output = []
class PPC64Arch(ArchWelderMach):
def __init__(self, srpm, target, mydir, buildarch='ppc64'):
self._status = None
self.target = target
self.mydir = mydir
self.buildroot = 'fedora-%s-ppc64-core' % (target)
self.srpm = srpm
self.buildarch = buildarch
self.arch_command = ''
self._output = []
class XMLArchWelder:
""" this class handles all xmlrpc connections transparently to the
calling functions"""
def __init__(self, srpm, target, mydir, buildarch):
self._status = None
self.target = target
self.mydir = mydir
self.buildroot = 'fedora-%s-ppc-core' % (target)
self.srpm = srpm
self.buildarch = buildarch
self.arch_command = ''
self._output = []
self.uniqid = None
self.ourservers = {'ppc':'http://ppcextras.linux.duke.edu:8888'}
self.rpc = xmlrpclib.Server(self.ourservers[self.buildarch])
def build(self):
self.uniqid = self.rpc.start(self.srpm, self.target, self.mydir, self.buildarch)
def status(self):
return self.rpc.status(self.uniqid)
def die(self):
return self.rpc.die(self.uniqid)
def logs(self):
result = []
enclogs = self.rpc.logs(self.uniqid)
for line in enclogs:
txt = base64.decodestring(line)
result.append(txt)
return result
def myArchWelder(srpm, target, mydir, buildarch, localarches):
"""hand it an arch it hands you back the archwelder instance you need"""
welder_dict = {'i386':i386Arch,
'i486':i386Arch,
'i586':i386Arch,
'i686':i386Arch,
'athlon':i386Arch,
'x86_64':x86_64Arch,
'amd64':x86_64Arch,
'ia32e':x86_64Arch,
'ppc':PPCArch,
'ppc32':PPCArch,
'ppc64':PPC64Arch,
}
if not welder_dict.has_key(buildarch):
# raise an exception here bitching about no place to build for that arch
pass
if buildarch == 'noarch':
if len(localarches) > 0:
welder = welder_dict[localarches[0]] # whatever our first localarch is, fine
else:
welder = XMLArchWelder
else:
if buildarch in localarches:
welder = welder_dict[buildarch]
else:
welder = XMLArchWelder
awp = welder(srpm, target, mydir, buildarch)
return awp
class XMLRPCArchWelderServer:
def __init__(self, localarches):
self.ids = {} # unique id => awclass instance
self.localarches = localarches
def start(self, srpm, target, stagedir, buildarch):
check = '%s %s %s' % (srpm, target, buildarch)
sum = sha.new()
sum.update(check)
uniqid = sum.hexdigest()
awp = myArchWelder(srpm, target, stagedir, buildarch, self.localarches)
self.ids[uniqid] = awp
awp.build()
log("Build request for %s on %s arch %s" % (srpm, target, buildarch))
return uniqid
def status(self, uniqid=None):
awp = self.ids[uniqid]
return awp.status()
def die(self, uniqid=None):
awp = self.ids[uniqid]
return awp.die()
def logs(self, uniqid=None):
awp = self.ids[uniqid]
result = []
for line in awp.logs():
enc = base64.encodestring(line)
result.append(enc)
return result
def listjobs(self):
return self.ids.keys()
def start_xml_server(hostname, localarches=['ppc']):
server = SimpleXMLRPCServer.SimpleXMLRPCServer((hostname, 8888))
server.register_instance(XMLRPCArchWelderServer(localarches))
server.serve_forever()
--- NEW FILE aw-server.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
import archwelder
import sys
import string
if len(sys.argv) < 2:
print "Error: need hostname and list of arches buildable on this system\n\n"
sys.exit(1)
hostname = sys.argv[1]
arches = sys.argv[2:]
print 'Binding on %s with arches: %s' % (hostname, string.join(arches))
archwelder.start_xml_server(hostname, arches)
--- NEW FILE buildqueue.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
# TODO: update the 'tobuild' file with the status
# TODO: write the status files out per-dir
# TODO: Logfile for queuer/monitor to track status w/o debugprint
# TODO: file output configuration from config file
# TODO: After a successful build we should look through the active and failed
# dirs and look for empty 'name' dirs and remove them
import os
import os.path
import sys
import commands
import time
import popen2
import rpmUtils
import exceptions
import shutil
import tempfile
import smtplib
from email.MIMEText import MIMEText
import string
import archwelder
DEBUG = False
def debugprint(stuff=''):
if DEBUG:
print stuff
def log(stuff=''):
print stuff
tmpdir = '/var/tmp'
cvscmd = '/usr/bin/cvs'
makecmd = '/usr/bin/make'
poll = 5 # time in minutes to repoll the tobuild file
os.environ['CVSROOT'] = ':ext:buildsys at cvs.fedora.redhat.com:/cvs/extras'
os.environ['CVS_RSH'] = '/usr/bin/ssh'
class PrepError(exceptions.Exception):
def __init__(self, errno=0, args=None):
exceptions.Exception.__init__(self)
self.args = args
self.errno = errno
def __str__(self):
return self.args
class BuildError(exceptions.Exception):
def __init__(self, errno=0, args=None):
exceptions.Exception.__init__(self)
self.args = args
self.errno = errno
def __str__(self):
return self.args
class MonitorError(exceptions.Exception):
def __init__(self, errno=0, args=None):
exceptions.Exception.__init__(self)
self.args = args
self.errno = errno
def __str__(self):
return self.args
class Monitor:
# acts as a list when instance is referenced that way - returns
# tuple of build request information
def __init__(self, tobuild):
self.tobuild = tobuild
self.needbuilt = {}
self.buildorder = []
self.tmpdir = None
self.cvs_check()
def __iter__(self):
mylist = []
for item in self.buildorder:
if self.needbuilt[item] == 1:
mylist.append(item)
return mylist.__iter__()
def cvs_check(self):
if self.tmpdir is None:
self.tmpdir = tempfile.mkdtemp(prefix='common', dir=tmpdir)
os.chdir(self.tmpdir)
cmd = '%s co %s' % (cvscmd, self.tobuild)
debugprint("Running %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
raise MonitorError(1, "Error updating output is: %s" % o)
self.local_tobuild = os.path.join(self.tmpdir, self.tobuild)
self.parse_tobuild()
def parse_tobuild(self):
debugprint("parsing tobuild")
tb = open(self.local_tobuild, 'r')
for line in tb.readlines():
line.replace("\n", "")
if len(line) > 0 and line.strip().startswith('#'):
continue
if line.strip() == '':
continue
buildstuff = line.strip().split('\t')
if len(buildstuff) == 5: # we have a status
continue
buildstuff = tuple(buildstuff)
if not self.needbuilt.has_key(buildstuff):
self.needbuilt[buildstuff] = 1
self.buildorder.append(buildstuff)
tb.close()
def reset(self):
self.cvs_check()
def buildprocessed(self, tup):
if self.needbuilt.has_key(tup):
self.needbuilt[tup] = 0
self.buildorder.remove(tup)
def newitems(self):
for pkginfo in self.needbuilt.keys():
if self.needbuilt[pkginfo] == 1:
return True
return False
class Queuer:
def __init__(self, username, repo, tag, targ_name):
self.email = '%s at fedora.redhat.com' % username
self.email_from = 'buildsys at fedoraproject.org'
self.stages_root = '/rpmbuild/extras/stages'
targetdict = { 'fc-3': '3', 'devel':'development',
'3': '3', 'development': 'development',
'fc3': '3', 'rawhide': 'development'}
targetarches = {'3': ['i386', 'x86_64'],
'development': ['ppc','i386', 'x86_64']}
self.repo = repo
self.tag = tag
if targetdict.has_key(targ_name.lower()):
self.target = targetdict[targ_name.lower()]
else:
msg = 'Target %s unknown for %s' % (targ_name, self.tag)
subj = 'Prep Error: %s on %s' % (self.tag, targ_name)
self.email_result(resultstring=msg, subject=subj)
raise PrepError(4, 'target %s does not exist' % targ_name)
self.arches = targetarches[self.target]
self.buildarches = []
self.archprocess = []
self.build_status = {}
self.anyfail = False
self.timeout = 3600
self.timeout_reached = False
self.localarches = ['i386', 'x86_64']
self.logs = {}
def prep(self):
self.checkout()
self.srpmpath = self.make_srpm()
ts = rpmUtils.transaction.initReadOnlyTransaction()
hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpmpath)
self.name = hdr['name']
self.ver = hdr['version']
self.release = hdr['release']
self.buildarches = self.arch_handling(hdr)
if len(self.buildarches) == 0:
raise PrepError(5, 'no buildable arches available for %s - %s.' % (self.name, self.tag))
self.curstage = 'active'
self.make_stage_dir()
self._createrepo(stage='needsign')
for arch in self.buildarches:
thisdir = os.path.join(self.stage_dir, arch)
if not os.path.exists(thisdir):
os.makedirs(thisdir)
srpm = os.path.basename(self.srpmpath)
srpm_in_dir = os.path.join(self.stage_dir, srpm)
if os.path.exists(srpm_in_dir):
os.unlink(srpm_in_dir)
shutil.move(self.srpmpath, self.stage_dir)
self.srpmpath = srpm_in_dir
del hdr
del ts
def arch_handling(self, hdr):
archs = []
buildable_arches = self.arches
ba = hdr['buildarchs']
exclusive = hdr['exclusivearch']
exclude = hdr['excludearch']
if ba == ['noarch']:
return ba
tmparchs = self.arches
if ba:
tmparchs = ba
else:
if exclusive:
tmparchs = exclusive
if exclude:
for arch in exclude:
if arch in tmparchs:
tmparchs.remove(arch)
# we probably need to check the archs, and make sure they are what
# we can build for
for thisarch in tmparchs:
if thisarch in buildable_arches:
archs.append(thisarch)
return archs
def make_stage_dir(self):
self.pkgsubdir = '%s/%s-%s' % (self.name, self.ver, self.release)
self.target_and_pkg_subdir = '%s/%s' % (self.target, self.pkgsubdir)
self.stage_dir = os.path.join(self.stages_root, self.curstage, self.target_and_pkg_subdir)
if not os.path.exists(self.stage_dir):
os.makedirs(self.stage_dir)
return self.stage_dir
def checkout(self):
self.tmpdir = tempfile.mkdtemp(prefix=self.tag, dir=tmpdir)
os.chdir(self.tmpdir)
cmd = '%s co -r %s %s' % (cvscmd, self.tag, self.repo)
debugprint("Running %s" % (cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
subj = 'Prep Error: %s on %s' % (self.tag, self.target)
msg = "could not check out %s from %s - output was:\n %s" % (self.tag, self.target, o)
self.email_result(resultstring=msg, subject=subj)
raise PrepError(1, msg)
def make_srpm(self):
repodir = os.path.join(self.tmpdir, self.repo)
os.chdir(repodir)
cmd = '%s srpm' % (makecmd)
debugprint("Running %s in %s" % (cmd, repodir))
s, o = commands.getstatusoutput(cmd)
if s != 0:
subj = 'Prep Error: %s on %s' % (self.tag, self.target)
msg = "could not make srpm for %s - output was:\n %s" % (self.tag, o)
self.email_result(resultstring=msg, subject=subj)
raise PrepError(2, msg)
srpmpath = None
for line in o.split("\n"):
if line.startswith("Wrote:"):
line.replace("\n", "")
(garbage, path) = line.split(':')
srpmpath = path.strip()
return srpmpath
if not srpmpath:
subj = 'Prep Error: %s on %s' % (self.tag, self.target)
msg = "could not find srpm for %s - output was:\n %s" % (self.tag, o)
self.email_result(resultstring=msg, subject=subj)
raise PrepError(3, msg)
def run(self):
for arch in self.buildarches:
awp = archwelder.myArchWelder(self.srpmpath, self.target, self.stage_dir, arch, self.localarches)
self.archprocess.append(awp)
# spawn off the builds
starttime = time.time()
for awp in self.archprocess:
debugprint('Spawning build of %s for %s' % (self.name, awp.buildarch))
awp.build()
# check their status
debugprint('Monitoring builds of %s' % self.name)
while len(self.archprocess) > 0 and not self.anyfail:
curtime = time.time()
if curtime > starttime + self.timeout:
self.timeout_reached = True
self.anyfail = True
for awp in self.archprocess:
status = awp.status()
debugprint('%s: %s' % (awp.buildarch, status))
if status == -1:
continue
else:
self.archprocess.remove(awp)
self.build_status[awp] = status
# if we'd rather have all builds finish
# even if an arch fails we should remove this check
if status != 0:
self.anyfail = True
time.sleep(10) # pause for 10s before retrying the loop
if self.anyfail:
for awp in self.archprocess:
awp.die()
status = -2 # killed it even though it was still running
if self.timeout_reached: status = -3 # killed b/c of a timeout
self.build_status[awp] = status
for (awp, status) in self.build_status.iteritems(): # go through stuff in build status
if status not in [0, -2, -3]:
self.logs[awp.buildarch] = awp.logs()
elif status == -2:
self.logs[awp.buildarch] = ['Build process terminated due to failure on another arch\n']
elif status == -3:
self.logs[awp.buildarch] = ['Build process terminated by timeout\n']
if self.anyfail:
return self.failed()
return self.succeeded()
def failed(self):
self.curstage = 'failed'
old_stage = self.stage_dir
dest = self.make_stage_dir()
if os.path.exists(dest):
shutil.rmtree(dest, ignore_errors=True)
shutil.move(old_stage, dest)
for arch in self.logs.keys():
buildroot = 'fedora-%s-%s-core' % (self.target, arch)
stage_arch = os.path.join(self.stage_dir, arch)
build_log = '%s/mach/%s/%s-%s-%s/rpm.log' % (tmpdir, buildroot,
self.name, self.ver, self.release)
if os.path.exists(build_log):
bl = open(build_log, 'r')
else:
bl = None
if not os.path.exists(stage_arch):
os.makedirs(stage_arch)
fn = '%s/%s-%s-%s.failure.log' % (stage_arch, self.name, self.ver, self.release)
logfile = open(fn, 'w')
for line in self.logs[arch]:
logfile.write(line)
if bl:
for line in bl.readlines():
logfile.write(line)
bl.close()
logfile.close()
# markup status file
resultstring = """
Build of %s on %s failed to complete on one or more archs. Please see logs at:
http://extras64.linux.duke.edu/failed/%s/%s""" % (self.name, self.target, self.target, self.name)
self.email_result(resultstring)
return False
def succeeded(self):
self.curstage = 'needsign'
old_stage = self.stage_dir
dest = self.make_stage_dir()
if os.path.exists(dest):
shutil.rmtree(dest, ignore_errors=True)
shutil.move(old_stage, dest)
# markup status file
resultstring = """
Build of %s on %s succeeded.
""" % (self.name, self.target)
self.email_result(resultstring)
self._createrepo()
return True
def email_result(self, resultstring, subject=None):
"""send 'resultstring' to self.email from self.email_from"""
msg = MIMEText(resultstring)
if not subject:
subject = 'Build Result: %s on %s' % (self.name, self.target)
msg['Subject'] = subject
msg['From'] = self.email_from
msg['To'] = self.email
s = smtplib.SMTP()
s.connect()
s.sendmail(self.email_from, [self.email], msg.as_string())
s.close()
def _createrepo(self, stage=None):
# createrepo on the needsign tree for new changes
if not stage:
stage = self.curstage
repodir = os.path.join(self.stages_root, stage, self.target)
if not os.path.exists(repodir):
os.makedirs(repodir)
s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
if s != 0:
raise PrepError(5, 'Error generating repodata for %s: %s' % repodir)
def main(localarches):
try:
tobuild = 'common/tobuild'
mon = Monitor(tobuild)
except MonitorError, e:
print "Error monitoring tobuild was: %s" % e
sys.exit(1)
while True:
while mon.newitems():
for buildtup in mon:
(username, repo, tag, target) = buildtup
result = None
try:
log('Setting up queue for: %s target distro is %s' % (tag, target))
queue = Queuer(username, repo, tag, target)
queue.localarches = localarches
queue.prep()
log('Build of %s on %s for arches: %s' % (queue.name, queue.target, string.join(queue.buildarches)))
except PrepError, e:
print "Error setting up build for %s on %s: %s" % (tag, target, e)
else:
result = queue.run()
mon.buildprocessed(buildtup)
if result:
log("Success for %s on %s" % (tag, target))
else:
log("Failed to build %s on %s" % (tag, target))
debugprint('sleeping for %s minutes' % poll)
try:
time.sleep(poll*60)
except KeyboardInterrupt:
pass
try:
mon.reset()
except MonitorError, e:
print "Error monitoring tobuild was: %s" % e
sys.exit(1)
if __name__ == '__main__':
me = os.getcwd()
localarches = []
if len(sys.argv) > 1:
localarches = sys.argv[1:]
main(localarches)
os.chdir(me)
More information about the fedora-extras-commits
mailing list