extras-buildsys/server ArchJob.py, 1.13, 1.14 BuildMaster.py, 1.33, 1.34 Builder.py, 1.13, 1.14 BuilderManager.py, 1.13, 1.14 Config.py, 1.3, 1.4 PackageJob.py, 1.24, 1.25 UserInterface.py, 1.50, 1.51
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Mon Aug 29 17:48:21 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv4707/server
Modified Files:
ArchJob.py BuildMaster.py Builder.py BuilderManager.py
Config.py PackageJob.py UserInterface.py
Log Message:
2005-08-29 Dan Williams <dcbw at redhat.com>
* Make more stuff actually work
Index: ArchJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/ArchJob.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- ArchJob.py 25 Aug 2005 18:15:14 -0000 1.13
+++ ArchJob.py 29 Aug 2005 17:48:19 -0000 1.14
@@ -73,11 +73,14 @@
return True
return False
+ def arch(self):
+ return self._target_dict['arch']
+
def _to_dict(self):
attrdict = {}
attrdict['jobid'] = self.jobid
attrdict['parent_uid'] = self.par_job.uid
- attrdict['arch'] = self.arch
+ attrdict['arch'] = self._target_dict['arch']
addr = self.builder.address()
# for some reason, splithost doesn't like the protocol
# method, you have to give it a string starting with "//"
@@ -93,8 +96,8 @@
attrdict['endtime'] = self.endtime
return attrdict
- def set_status(self, status):
- if status != 'idle':
+ def set_builder_job_status(self, status):
+ if status != 'idle':
oldstatus = self.builder_status
self.builder_status = status
if oldstatus != self.builder_status:
@@ -119,7 +122,7 @@
except socket.error, e:
if not CommonErrors.canIgnoreSocketError(e):
print "%s (%s/%s): [ %s ] Unknown error when signalling repo unlocked: '%s'" % (self.par_job.uid,
- self.par_job.package, self.arch, self.bci.address(), e)
+ self.par_job.package, self._target_dict['arch'], self.bci.address(), e)
except xmlrpclib.ProtocolError, e:
pass
@@ -130,7 +133,7 @@
except socket.error, e:
if not CommonErrors.canIgnoreSocketError(e):
print "%s (%s/%s): [ %s ] Unknown error when signalling repo unlocked: '%s'" % (self.par_job.uid,
- self.par_job.package, self.arch, self.bci.address(), e)
+ self.par_job.package, self._target_dict['arch'], self.bci.address(), e)
except xmlrpclib.ProtocolError, e:
pass
return files
@@ -176,7 +179,7 @@
dl_status = self.downloads[url]
if dl_status == 0:
# spawn the download
- target_dir = os.path.join(self.par_job.get_stage_dir(), self.arch)
+ target_dir = os.path.join(self.par_job.get_stage_dir(), self._target_dict['arch'])
if not os.path.exists(target_dir):
os.makedirs(target_dir)
if self._use_ssl:
@@ -220,7 +223,7 @@
file_string = file_string + ", "
print "%s (%s/%s): Build result files - [ %s ]" % (self.par_job.uid,
- self.par_job.package, self.arch, file_string)
+ self.par_job.package, self._target_dict['arch'], file_string)
def dl_callback(self, status, cb_data):
url = cb_data
@@ -243,7 +246,7 @@
except FileDownloader.FileNameError, e:
# Just ignore the file then
print "%s (%s/%s): Illegal file name. Error: '%s', URL: %s" % (self.par_job.uid,
- self.par_job.package, self.arch, e, url)
+ self.par_job.package, self._target_dict['arch'], e, url)
pass
return files
Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.33
retrieving revision 1.34
diff -u -r1.33 -r1.34
--- BuildMaster.py 25 Aug 2005 18:15:14 -0000 1.33
+++ BuildMaster.py 29 Aug 2005 17:48:19 -0000 1.34
@@ -202,8 +202,8 @@
try:
repo = self._repos[target_str]
except KeyError:
- print "%s (%s): Target '%s' not found." % (uid,
- row['package'], target_str)
+ print "%s (%s): Target '%s' not found." % (uid, row['package'],
+ target_str)
else:
job = PackageJob.PackageJob(uid, row['username'], row['package'],
row['source'], repo, self)
@@ -348,8 +348,7 @@
try:
self.curs.execute('UPDATE archjobs SET status="%s", builder_status="%s", endtime=%d ' \
'WHERE jobid="%s" AND parent_uid=%d' % (attrdict['status'],
- attrdict['builder_status'], attrdict['endtime'], uid,
- attrdict['parent_uid']))
+ attrdict['builder_status'], attrdict['endtime'], uid, attrdict['parent_uid']))
except sqlite.OperationalError, e:
print "DB Error: could not access jobs database. Reason: '%s'" % e
@@ -378,7 +377,7 @@
self.curs.execute('INSERT INTO jobs (uid, username, package,' \
' source, target_distro, target_target, target_repo, buildreq,' \
' starttime, endtime, status, result)' \
- ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, 0, "%s", "")' \
+ ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", "%s", "%s", %d, 0, "%s", "")' \
% (item['email'], item['package'], item['source'], item['target_distro'], \
item['target_target'], item['target_repo'], item['buildreq'], \
item['time'], 'initialize'))
@@ -390,7 +389,7 @@
' target_target="%s" AND target_repo = "%s" AND' \
' buildreq="%s" AND starttime=%d AND status="initialize"' \
% (item['email'], item['package'], item['source'], \
- item['target_disro'], item['target_target'], item['target_repo'], \
+ item['target_distro'], item['target_target'], item['target_repo'], \
item['buildreq'], item['time']))
self.dbcx.commit()
Index: Builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Builder.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- Builder.py 25 Aug 2005 18:15:14 -0000 1.13
+++ Builder.py 29 Aug 2005 17:48:19 -0000 1.14
@@ -82,8 +82,10 @@
def _building_jobs(self):
jobs = {}
try:
- jobs = self._server.building_jobs()
+ (jobs, free_slots) = self._server.building_jobs()
self._unavail_count = 0
+ self._alive = True
+ self._free_slots = free_slots
except (socket.error, socket.timeout, OpenSSL.SSL.SysCallError, xmlrpclib.ProtocolError):
self._unavail_count = self._unavail_count + 1
return jobs
@@ -97,14 +99,6 @@
alive = False
return (alive, target_list)
- def _free_slots(self):
- try:
- free_slots = self._server.free_slots()
- except (socket.error, socket.timeout, OpenSSL.SSL.SysCallError, xmlrpclib.ProtocolError):
- self._unavail_count = self._unavail_count + 1
- free_slots = 0
- return free_slots
-
def arches(self, target_dict):
for td in self._target_list:
if td['distro'] == target_dict['distro'] and td['target'] == target_dict['target'] and td['repo'] == target_dict['repo']:
@@ -115,10 +109,11 @@
return arches
return None
- def can_build_arch_on_target(self, arch, target):
- if self._target_arches.has_key(target) and len(self._target_arches[target]) > 0:
- if arch in self._target_arches[target]:
- return True
+ def can_build_for_target(self, target_dict):
+ for td in self._target_list:
+ if td['distro'] == target_dict['distro'] and td['target'] == target_dict['target'] and td['repo'] == target_dict['repo']:
+ if target_dict['arch'] in td['supported_arches']:
+ return True
return False
def address(self):
@@ -134,9 +129,7 @@
def start_job(self, par_job, target_dict, srpm_url):
if not self.available():
raise RuntimeError
- if not self._target_arches.has_key(target) or len(self._target_arches[target]) == 0:
- raise RuntimeError
- if not arch in self._target_arches[target]:
+ if not self.can_build_for_target(target_dict):
raise RuntimeError
self._server_lock.acquire()
@@ -158,7 +151,6 @@
def _update_building_jobs(self):
jobs = self._building_jobs()
- self._free_slots = self.free_slots()
# Update the current job's status
if self._unavail_count == 0:
@@ -167,7 +159,7 @@
try:
job = self._jobs[jobid]
status = jobs[jobid]
- job.set_status(status)
+ job.set_builder_job_status(status)
if status == 'prepping':
self._prepping_jobs = True
except KeyError:
@@ -282,12 +274,6 @@
arches.append(arch)
builder_dict['arches'] = arches
- if self._alive:
- if self._cur_jobid:
- builder_dict['status'] = 'building'
- else:
- builder_dict['status'] = 'idle'
- else:
- builder_dict['status'] = 'unavailable'
+ builder_dict['alive'] = self._alive
return builder_dict
Index: BuilderManager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuilderManager.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- BuilderManager.py 25 Aug 2005 18:15:14 -0000 1.13
+++ BuilderManager.py 29 Aug 2005 17:48:19 -0000 1.14
@@ -48,7 +48,10 @@
for arch in builder_dict['arches']:
string = string + arch + " "
string = string + " " * (80 - len(string))
- string = string + builder_dict['status']
+ status = "unavailable"
+ if builder_dict['alive']:
+ status = "alive"
+ string = string + status
del builder_dict
print string
print ""
@@ -138,11 +141,10 @@
self._queue.remove(req)
continue
# Find a free builder for this request
- target_dict = req['target_dict']
for builder in self._builders:
- if builder.available() and builder.can_build_arch_on_target(target_dict):
+ if builder.available() and builder.can_build_for_target(req['target_dict']):
try:
- job = builder.start_job(parent, target_dict, req['srpm_url'])
+ job = builder.start_job(parent, req['target_dict'], req['srpm_url'])
except RuntimeError:
pass
else:
@@ -166,7 +168,6 @@
req = {}
req['parent'] = par_job
req['target_dict'] = target_dict
- req['arch'] = arch
req['srpm_url'] = srpm_url
self._queue_lock.acquire()
Index: Config.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Config.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- Config.py 29 Aug 2005 14:31:43 -0000 1.3
+++ Config.py 29 Aug 2005 17:48:19 -0000 1.4
@@ -47,7 +47,13 @@
cfg = self._target_aliases[user_alias.lower()]
except KeyError:
raise InvalidUserAliasException("Invalid user alias.")
+ return cfg
+ def resolve_target_string(self, target_str):
+ try:
+ cfg = self._target_strings[target_str]
+ except KeyError:
+ raise InvalidTargetException("Invalid target string.")
return cfg
def _add_target(self, target_cfg):
@@ -64,7 +70,7 @@
# We're OK, add it to our list
self._targets.append(target_cfg)
- self._target_names[target_cfg.target_string()] = target_cfg
+ self._target_strings[target_cfg.target_string()] = target_cfg
for alias in aliases:
self._target_aliases[alias.lower()] = target_cfg
Index: PackageJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/PackageJob.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -r1.24 -r1.25
--- PackageJob.py 25 Aug 2005 18:15:14 -0000 1.24
+++ PackageJob.py 29 Aug 2005 17:48:19 -0000 1.25
@@ -96,13 +96,13 @@
return True
return False
-def make_job_log_url(base_url, target, uid, name, ver, release):
- if target and uid and name and ver and release:
- if config_opts['log_url'].endswith('/'):
+def make_job_log_url(base_url, target_str, uid, name, ver, release):
+ if target_str and uid and name and ver and release:
+ if base_url.endswith('/'):
slash=''
else:
slash='/'
- return "%s%s%s/%s-%s-%s-%s/" % (config_opts['log_url'], slash, target, uid, name, ver, release)
+ return "%s%s%s/%s-%s-%s-%s/" % (base_url, slash, target_str, uid, name, ver, release)
return None
@@ -435,9 +435,11 @@
def add_arch_job(self, job):
""" Called by the BuilderManager when it's started a new arch job for us """
self._archjobs_lock.acquire()
- if self.archjobs[job.arch] != None:
- log("%s (%s/%s): Already have archjob for this arch (%s). New job UID is %s." % (self.uid, self.package, job.arch, self.archjobs[job.arch].jobid, job.jobid))
- self.archjobs[job.arch] = job
+ jobarch = job.arch()
+ if self.archjobs[jobarch] != None:
+ log("%s (%s/%s): Already have archjob for this arch (%s). New job UID is %s." % (self.uid, \
+ self.package, job.arch, self.archjobs[job.arch].jobid, job.jobid))
+ self.archjobs[jobarch] = job
# If this is the first archjob, that means we are now building.
# So we start up the second PackageJobController thread.
@@ -446,14 +448,15 @@
t.start()
self._archjobs_lock.release()
- log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, job.arch, job.jobid))
+ log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, jobarch, job.jobid))
def remove_arch_job(self, job):
""" Removes an arch job when its builder is no longer responding """
self._archjobs_lock.acquire()
- print "%s (%s/%s): Builder disappeared. Requeuing arch..." % (self.uid, self.package, job.arch)
+ jobarch = job.arch()
+ print "%s (%s/%s): Builder disappeared. Requeuing arch..." % (self.uid, self.package, jobarch)
self.archjobs[job.arch] = None
- self._request_one_arch_job(job.arch, True)
+ self._request_one_arch_job(jobarch, True)
self._archjobs_lock.release()
def is_done(self):
@@ -550,9 +553,10 @@
# a particular arch fails.
if not job.failure_noticed():
job.set_failure_noticed()
- msg = "Job failed on arch %s\n" % (job.arch)
+ jobarch = job.arch()
+ msg = "Job failed on arch %s\n" % (jobarch)
self._archjobs_lock.release()
- raise BuildError(msg, job.arch)
+ raise BuildError(msg, jobarch)
self._archjobs_lock.release()
@@ -592,7 +596,7 @@
for f in job.get_files():
if not f.endswith(".rpm"):
continue
- src_file = os.path.join(self.result_dir, job.arch, f)
+ src_file = os.path.join(self.result_dir, job.arch(), f)
if src_file.endswith(".src.rpm"):
# Keep an SRPM. We prefer built SRPMs from builders over
# the original SRPM.
@@ -617,12 +621,13 @@
for f in job.get_files():
if not f.endswith(".rpm"):
continue
- src_file = os.path.join(self.result_dir, job.arch, f)
+ jobarch = job.arch()
+ src_file = os.path.join(self.result_dir, jobarch, f)
verrel = "%s-%s" % (self.ver, self.release)
if f.endswith(".src.rpm"):
dst_path = os.path.join(repo_dir, self._target_str, self.name, verrel, "SRPM")
else:
- dst_path = os.path.join(repo_dir, self._target_str, self.name, verrel, job.arch)
+ dst_path = os.path.join(repo_dir, self._target_str, self.name, verrel, jobarch)
self.repofiles[src_file] = os.path.join(dst_path, f)
self._event.clear()
Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.50
retrieving revision 1.51
diff -u -r1.50 -r1.51
--- UserInterface.py 29 Aug 2005 14:31:43 -0000 1.50
+++ UserInterface.py 29 Aug 2005 17:48:19 -0000 1.51
@@ -201,7 +201,7 @@
def list_jobs(self, args_dict):
""" Query job information and return it to the user """
- sql = 'SELECT uid, username, package, cvs_tag, target_distro, target_target, ' \
+ sql = 'SELECT uid, username, package, source, target_distro, target_target, ' \
'target_repo, starttime, endtime, status, result FROM jobs WHERE '
sql_args = []
@@ -316,8 +316,10 @@
jobrec['uid'] = row['uid']
jobrec['username'] = row['username']
jobrec['package'] = row['package']
- jobrec['source'] = row['cvs_tag']
- jobrec['target'] = row['target']
+ jobrec['source'] = row['source']
+ jobrec['target_distro'] = row['target_distro']
+ jobrec['target_target'] = row['target_target']
+ jobrec['target_repo'] = row['target_repo']
jobrec['starttime'] = row['starttime']
jobrec['endtime'] = row['endtime']
jobrec['status'] = row['status']
@@ -366,8 +368,9 @@
if not uid:
return (-1, "Error: Invalid job UID.", {})
- sql = 'SELECT uid, username, package, cvs_tag, target, starttime, endtime, status, '\
- 'epoch, version, release, archlist, result_msg FROM jobs WHERE uid=%d' % uid
+ sql = 'SELECT uid, username, package, source, target_distro, target_target, ' \
+ 'target_repo, starttime, endtime, status, epoch, version, release, ' \
+ 'archlist, result_msg FROM jobs WHERE uid=%d' % uid
# Run the query for the job
try:
@@ -382,8 +385,10 @@
jobrec['uid'] = job['uid']
jobrec['username'] = job['username']
jobrec['package'] = job['package']
- jobrec['source'] = job['cvs_tag']
- jobrec['target'] = job['target']
+ jobrec['source'] = job['source']
+ jobrec['target_distro'] = job['target_distro']
+ jobrec['target_target'] = job['target_target']
+ jobrec['target_repo'] = job['target_repo']
jobrec['starttime'] = job['starttime']
jobrec['endtime'] = job['endtime']
jobrec['status'] = job['status']
@@ -487,18 +492,11 @@
Allow/Deny operations based on user privileges
"""
- def enqueue(self, package, cvs_tag, target, buildreq=None):
+ def enqueue(self, package, source, target_alias, buildreq=None):
user = AuthedXMLRPCServer.get_authinfo()
if not user or not user.own_jobs:
return (-1, "Insufficient privileges.", -1)
- return UserInterface.enqueue(self, user.email, package, cvs_tag, target, buildreq)
-
-
- def enqueue_srpm(self, package, srpm_file, target, buildreq=None):
- user = AuthedXMLRPCServer.get_authinfo()
- if not user or not user.own_jobs:
- return (-1, "Insufficient privileges.", -1)
- return UserInterface.enqueue_srpm(self, user.email, package, srpm_file, target, buildreq)
+ return UserInterface.enqueue(self, user.email, package, source, target_alias, buildreq)
def requeue(self, uid):
More information about the fedora-extras-commits
mailing list