extras-buildsys/builder Builder.py, 1.10, 1.11 BuilderMock.py, 1.5, 1.6 main.py, 1.2, 1.3
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Sun May 14 05:43:08 UTC 2006
Author: dcbw
Update of /cvs/fedora/extras-buildsys/builder
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv2964/builder
Modified Files:
Builder.py BuilderMock.py main.py
Log Message:
2006-05-14 Dan Williams <dcbw at redhat.com>
* Rework archjob handling. They are now processed from the owning
PackageJob object, and only one is spawned for the lifetime of the
PackageJob for each architecture (previously one was spawned for each
individual build job on the builder). Archjob UIDs are generated on
the server now rather than the builder.
* Correctly handle builders going away before they've had a chance to
notify the server that they have started an archjob. Previously, these
jobs would just be lost. This is the real reason for the rework of the
archjob handling above.
* On the builder, don't use die() to end jobs that have failed; do it
properly and upload files to the server if we weren't killed
* Deal with active builders whose hostnames can't be resolved when
the server starts
* Kill any existing jobs on builders when the server starts up
* Consolidate and simplify logging functions in PackageJob
* More pylint-induced cleanups
Index: Builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/builder/Builder.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -r1.10 -r1.11
--- Builder.py 12 May 2006 04:10:46 -0000 1.10
+++ Builder.py 14 May 2006 05:43:06 -0000 1.11
@@ -46,7 +46,7 @@
return ''
return socket.gethostname()
-def determine_max_jobs(cfg):
+def determine_max_jobs():
""" Simple max job calculator based on number of CPUs """
import commands
@@ -65,7 +65,7 @@
""" Abstract builder base object """
def __init__(self, cfg):
self._cfg = cfg
- self._max_slots = determine_max_jobs(cfg)
+ self._max_slots = determine_max_jobs()
self._seq_gen = Commands.SequenceGenerator()
self._building_jobs_lock = threading.Lock()
@@ -162,7 +162,7 @@
return target_cfg
- def _new_job_for_arch(self, target_cfg, buildarch, srpm_url):
+ def _new_job_for_arch(self, target_cfg, buildarch, srpm_url, uniqid):
"""Creates a new mock build job given a particular build architecture."""
if buildarch != 'noarch' and not BuilderMock.BuilderClassDict.has_key(buildarch):
@@ -177,30 +177,31 @@
builder_class = BuilderMock.BuilderClassDict[buildarch]
# We'll throw a TypeError here if there's no available builder_class for this arch
- return builder_class(self, target_cfg, buildarch, srpm_url)
+ return builder_class(self, target_cfg, buildarch, srpm_url, uniqid)
- def _start_new_job(self, target_dict, srpm_url):
+ def _start_new_job(self, cmd):
+ target_dict = cmd.target_dict()
+ srpm_url = cmd.srpm_url()
+ uniqid = cmd.archjob_id()
target_str = Config.make_target_string(target_dict['distro'], target_dict['target'], target_dict['arch'], target_dict['repo'])
- uniqid = -1
msg = "Success"
(free_slots, max_slots) = self.slots()
if free_slots <= 0:
msg = "Error: Tried to build '%s' on target %s when already building" \
" maximum (%d) jobs" % (srpm_url, target_str, max_slots)
self._log(msg)
- return (uniqid, msg)
+ return (-1, msg)
target_cfg = self._get_target_cfg(target_dict)
if not target_cfg:
msg = "Error: Tried to build '%s' on target %s which isn't supported" % (srpm_url, target_str)
self._log(msg)
- return (uniqid, msg)
+ return (-1, msg)
archjob = None
try:
- archjob = self._new_job_for_arch(target_cfg, target_dict['arch'], srpm_url)
- uniqid = archjob.uniqid()
+ archjob = self._new_job_for_arch(target_cfg, target_dict['arch'], srpm_url, uniqid)
self._all_jobs[uniqid] = archjob
self._building_jobs_lock.acquire()
self._building_jobs.append(archjob)
@@ -259,9 +260,9 @@
self._building_jobs = []
self._cfg = cfg
- def _log(self, string):
+ def _log(self, msg):
if self._cfg.get_bool("General", "debug"):
- print string
+ print msg
def die(self, uniqid):
try:
@@ -526,7 +527,7 @@
"""Process a single command from the server."""
if isinstance(cmd, Commands.PlgCommandNewJobReq):
- (uniqid, msg) = self._start_new_job(cmd.target_dict(), cmd.srpm_url())
+ (uniqid, msg) = self._start_new_job(cmd)
ack = Commands.PlgCommandNewJobAck(uniqid, msg, cmd.seq(), self._seq_gen.next())
self._queued_cmds.append(ack)
elif isinstance(cmd, Commands.PlgCommandUnlockRepo):
Index: BuilderMock.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/builder/BuilderMock.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- BuilderMock.py 11 May 2006 15:43:39 -0000 1.5
+++ BuilderMock.py 14 May 2006 05:43:06 -0000 1.6
@@ -26,8 +26,8 @@
import urllib
import errno
import exceptions
-import sha
import time
+import Builder
from plague import ExecUtils
from plague import FileDownloader
@@ -47,27 +47,22 @@
method = "https://"
else:
method = "http://"
- hostname = get_hostname(cfg, False)
+ hostname = Builder.get_hostname(cfg, False)
full_url = "%s%s:%s/%s" % (method, hostname, port, file_part)
return urllib.quote(full_url)
-def _generate_uniqid(target_str, srpm_url):
- sha_hash = sha.new()
- sha_hash.update('%d %s %s' % (time.time(), target_str, srpm_url))
- return sha_hash.hexdigest()
-
class BuilderMock(threading.Thread):
"""puts things together for an arch - baseclass for handling builds for
other arches"""
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self._controller = controller
self._buildarch = buildarch
self._starttime = time.time()
self._endtime = 0
self._mockstarttime = 0
- self._uniqid = _generate_uniqid(str(target_cfg), srpm_url)
+ self._uniqid = uniqid
self._status = 'init'
self._die = False
self._repo_locked = True
@@ -131,14 +126,14 @@
try:
# Kill all members of the child's process group
os.kill(child_pgroup, 9)
- except OSError, e:
- self._log("ERROR: Couldn't kill child process group %d: %s\n" % (child_pgroup, e))
+ except OSError, exc:
+ self._log("ERROR: Couldn't kill child process group %d: %s\n" % (child_pgroup, exc))
else:
# Ensure child process is reaped
self._log("Waiting for mock process %d to exit...\n" % self._childpid)
try:
(pid, status) = os.waitpid(self._childpid, 0)
- except OSError, e:
+ except OSError:
pass
self._log("Mock process %d exited.\n" % self._childpid)
self._childpid = 0
@@ -261,8 +256,8 @@
try:
f.seek(0, 0)
mockstat = f.read(4)
- except OSError, e:
- if e.errno == errno.EAGAIN:
+ except OSError, exc:
+ if exc.errno == errno.EAGAIN:
try:
time.sleep(0.25)
except KeyboardInterrupt:
@@ -299,8 +294,9 @@
# If job was cancelled, just return
if self.is_done_status():
return
- self._status = 'failed'
+ self._done_status = 'failed'
self._log("ERROR: Failed to retrieve %s.\n" % url)
+ self._post_cleanup()
elif dl_status == FileTransfer.FT_RESULT_CANCELED:
# Ignore cancelation
pass
@@ -313,10 +309,11 @@
try:
self._downloader = self._controller.download_srpm(self._srpm_url,
target_dir, self.dl_callback, self._srpm_url)
- except FileDownloader.FileNameException, e:
- self._status = 'failed'
- self._log("ERROR: Failed to begin SRPM download. Error: '%s' URL: %s\n" % (e, self._srpm_url))
-
+ except FileDownloader.FileNameException, exc:
+ self._done_status = 'failed'
+ self._log("ERROR: Failed to begin SRPM download. Error: '%s' URL: %s\n" % (exc, self._srpm_url))
+ self._post_cleanup()
+
def _status_downloading(self):
pass
@@ -334,8 +331,9 @@
# Kill a job in 'downloaded' state after 30 minutes because
# it's likely orphaned
if time.time() > (self._repo_wait_start + (60 * 30)):
+ self._done_status = 'failed'
self._log("Job waited too long for repo to unlock. Killing it...\n")
- self.die()
+ self._post_cleanup()
def _watch_mock(self, good_exit, bad_exit):
(aux_pid, status) = os.waitpid(self._childpid, os.WNOHANG)
@@ -364,8 +362,9 @@
# something is wrong if mock takes more than 15s to write the status file
if time.time() > self._mockstarttime + 15:
self._mockstarttime = 0
+ self._done_status = 'failed'
self._log("ERROR: Timed out waiting for the mock status file! %s\n" % mockstatusfile)
- self.die()
+ self._post_cleanup()
else:
if not self._mock_config and self._mock_is_prepping():
self._mock_config = self._read_mock_config()
@@ -400,15 +399,23 @@
try:
self._log("Waiting for child process %d to exit.\n" % self._childpid)
(pid, status) = os.waitpid(self._childpid, 0)
- except OSError, e:
+ except OSError:
self._childpid = 0
pass
- self._copy_mock_output_to_log()
- self._files = self._find_files()
- self._uploader = self._controller.upload_files(self._uniqid, self._files,
- self.ul_callback, None)
- self._status = "uploading"
+ if self._done_status is not 'killed':
+ self._copy_mock_output_to_log()
+
+ self._post_cleanup()
+
+ def _post_cleanup(self):
+ if self._done_status is not 'killed':
+ self._files = self._find_files()
+ self._uploader = self._controller.upload_files(self._uniqid, self._files,
+ self.ul_callback, None)
+ self._status = "uploading"
+ else:
+ self._status = self._done_status
def ul_callback(self, status, cb_data, msg):
if status == FileTransfer.FT_RESULT_SUCCESS:
@@ -450,9 +457,10 @@
try:
srpm_filename = FileDownloader.get_base_filename_from_url(self._srpm_url, ['.src.rpm'])
self._srpm_path = os.path.join(self._work_dir, self._uniqid, "source", srpm_filename)
- except FileDownloader.FileNameException, e:
- self._log("ERROR: SRPM file name was invalid. Message: '%s'\n" % e)
- self._status = 'failed'
+ except FileDownloader.FileNameException, exc:
+ self._done_status = 'failed'
+ self._log("ERROR: SRPM file name was invalid. Message: '%s'\n" % exc)
+ self._post_cleanup()
# Main build job work loop
while not self.is_done_status():
@@ -463,8 +471,9 @@
try:
func = getattr(self, "_status_%s" % self._status)
except AttributeError:
+ self._done_status = 'failed'
self._log("ERROR: internal builder inconsistency, didn't recognize status '%s'.\n" % self._status)
- self._status = 'failed'
+ self._post_cleanup()
else:
func()
time.sleep(3)
@@ -524,34 +533,34 @@
class InvalidTargetError(exceptions.Exception): pass
class i386Arch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = '/usr/bin/setarch i686'
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
class x86_64Arch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = ''
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
class PPCArch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = '/usr/bin/setarch ppc32'
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
class PPC64Arch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = ''
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
class SparcArch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = '/usr/bin/sparc32'
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
class Sparc64Arch(BuilderMock):
- def __init__(self, controller, target_cfg, buildarch, srpm_url):
+ def __init__(self, controller, target_cfg, buildarch, srpm_url, uniqid):
self.arch_command = '/usr/bin/sparc64'
- BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url)
+ BuilderMock.__init__(self, controller, target_cfg, buildarch, srpm_url, uniqid)
BuilderClassDict = {
Index: main.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/builder/main.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- main.py 10 May 2006 14:28:12 -0000 1.2
+++ main.py 14 May 2006 05:43:06 -0000 1.3
@@ -202,7 +202,10 @@
log("Shutting down...\n")
builder.stop()
builder.cleanup()
- time.sleep(2)
+ try:
+ time.sleep(2)
+ except KeyboardInterrupt:
+ pass
log(" done.\n");
sys.stdout.flush()
More information about the fedora-extras-commits
mailing list