[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/server BuildJob.py, 1.1, 1.2 CONFIG.py, 1.5, 1.6 build-server, 1.1, 1.2 buildmaster.py, 1.3, 1.4 buildserver.py, 1.3, 1.4 client_manager.py, 1.5, 1.6



Author: dcbw

Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv28967/server

Modified Files:
	BuildJob.py CONFIG.py build-server buildmaster.py 
	buildserver.py client_manager.py 
Log Message:
2005-06-12  Dan Williams <dcbw redhat com>

    * Require a "hostname" argument to the build server.  But we get to remove
      it as a config option from CONFIG.py then, which is good :)

    * Implement Additional Package Architectures support.  This is essentially
      an look-aside file listing packages and other arches these packages should
      build on, but that a whole distribution isn't built on.  Examples include
      OpenSSL on SPARC, where sparcv8 and sparcv9 optimized builds are produced,
      but not an entire sparcv9 distro.  See the "addl_pkg_arches" directory
      for an example file, and the CONFIG.py file for more information.

    * Simplify build client tracking on the server.  There is now only 1 instance
      of a BuildClient (formerly BuildClientInstance) class per build client.
      Previously there was one instance for each arch the client supported.
      Since at this time we don't allow more than one build job per client,
      its un-necessary to track anything per-client-per-arch.

    * Return the build job UID when queuing a job

    * Start jobs on clients immediately when creating them, instead of putting
      the server-side tracking object in 'initialize' state first and then
      starting the job on the client in the next process() iteration.

    * Do some locking when the BuildClient class gets the job the client
      is currently running.  Because BuildJob instances run in their own threads,
      the BuildJob could be starting a new job on the BuildClient while the
      BuildClient instance is getting status.  This could result in a race
      condition and potentially two jobs thinking they have started concurrently.

    * Add a "Getting Started" section to the README file




Index: BuildJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildJob.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- BuildJob.py	9 Jun 2005 19:31:28 -0000	1.1
+++ BuildJob.py	12 Jun 2005 08:23:43 -0000	1.2
@@ -72,9 +72,10 @@
 class BuildJob(threading.Thread):
     """ Controller object for building 1 SRPM on multiple arches """
 
-    def __init__(self, uid, username, package, cvs_tag, target, client_manager):
+    def __init__(self, uid, username, package, cvs_tag, target, client_manager, hostname):
         self.curstage = 'initialize'
         self.bcm = client_manager
+        self.hostname = hostname
         self.uid = uid
         self.username = username
         self.starttime = time.time()
@@ -101,21 +102,80 @@
         return self.uid
         
     def arch_handling(self, hdr):
-        archs = []
+        # Associate sub-architectures with their "master" architecture.
+        # This is only used to determine which arches to build on by default,
+        # so that if we have an Additional Package Arches file that specifies
+        # 'sparcv9' for a package that we don't try to build sparcv9 for that
+        # package unless 'sparc' is listed in our 'targets' config option.
+        sub_arches = {
+                        'athlon'    : 'i386',
+                        'i686'      : 'i386',
+                        'i586'      : 'i386',
+                        'i486'      : 'i386',
+                        'amd64'     : 'x86_64',
+                        'ia32e'     : 'x86_64',
+                        'ppc32'     : 'ppc',
+                        'sparcv8'   : 'sparc',
+                        'sparcv9'   : 'sparc'
+                     }
+
+        # Grab additional build arches out of the Additional Package
+        # Arches file
+        apa_file_name = self.target + "addl-arches"
+        apa_file = os.path.join(CONFIG.get('addl_package_arches_dir'), apa_file_name)
+        addl_arches = []
+        try:
+            f = open(apa_file, "r")
+        except IOError, e:
+            pass
+        else:
+            for line in f.readlines():
+                line = line.strip()
+                tmp_split = line.split(':')
+                if len(tmp_split) == 2:
+                    package = tmp_split[0]
+                    if package == self.name:
+                        tmp_arches = tmp_split[1]
+                        tmp_arches = tmp_arches.strip()
+                        addl_arches = tmp_arches.split(' ')
+                        break
+
         targets = CONFIG.get('targets')
         buildable_arches = targets[self.target]
-        
+
+        target_opt_arches = CONFIG.get('target_optional_arches')
+        opt_arches = target_opt_arches[self.target]
+
+        # Remove arches we don't support from addl_arches
+        for arch in addl_arches:
+            if sub_arches.has_key(arch):
+                master_addl_arch = sub_arches[arch]
+                if master_addl_arch not in buildable_arches:
+                    if master_addl_arch not in opt_arches:
+                        addl_arches.remove(arch)
+
         ba = hdr['buildarchs']
-        exclusive = hdr['exclusivearch']
+        exclusive = hdr['exclusivearch'] 
         exclude = hdr['excludearch']
         
         if ba == ['noarch']:
             return ba
 
-        # default to building all arches
+        # default to building all base arches the 'target'
+        # supports, and any additional arches from the
+        # Additional Package Arches file whose "master" arch
+        # is enabled for this target
         tmparchs = []
+        allowed_arches = []
         for arch in buildable_arches:
             tmparchs.append(arch)
+            allowed_arches.append(arch)
+        for arch in addl_arches:
+            tmparchs.append(arch)
+            allowed_arches.append(arch)
+        # Optional arches don't get built by default but are "allowed"
+        for arch in opt_arches:
+            allowed_arches.append(arch)
 
         if ba:
             tmparchs = ba
@@ -127,16 +187,17 @@
             for arch in exclude:
                 if arch in tmparchs:
                     tmparchs.remove(arch)
-        
+
         # we probably need to check the archs, and make sure they are what 
         # we can build for
+        archs = []
         for thisarch in tmparchs:
-            if thisarch in buildable_arches:
+            if thisarch in allowed_arches:
                 archs.append(thisarch)
 
         return archs
 
-        
+
     def _make_stage_dir(self, rootdir):
         # The dir will look like this:
         # <rootdir>/devel/95-foo-1.1.0-23
@@ -209,7 +270,7 @@
             self.email_result(resultstring=msg, subject=subj)
             self.curstage = 'finished'
             self.failed = True
-            #shutil.rmtree(self.checkout_tmpdir, True)
+            shutil.rmtree(self.checkout_tmpdir, True)
             return
         
         srpmpath = None
@@ -296,30 +357,26 @@
                 else:
                     self._succeeded()
 
-
-    def job_server_gone(self, job):
-        """ Remove a job from our building queue if its server went away """
-
-        print "%s (%s/%s): Build client disappeared.  Requeuing arch..." % (self.uid, self.package, job.arch)
-        del self.sub_jobs[job.arch]
-
     def _start_unspawned_builds(self):
         for arch in self.buildarches:
             if not self.sub_jobs.has_key(arch):
                 # Construct SPRM URL
                 srpm_http_base = self.srpm_http_path[len(http_dir):]
-                srpm_url = "https://"; + CONFIG.get('hostname') + ":8886" + srpm_http_base
-                job = self.bcm.new_job_on_arch(self, self.target, arch, srpm_url)
+                srpm_url = "https://"; + self.hostname + ":8886" + srpm_http_base
+                job = self.bcm.start_arch_job(self, self.target, arch, srpm_url)
                 if job:
-                    if job.start() == True:
-                        self.bcm.track_job(job)
-                        self.sub_jobs[arch] = job
-                        log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
-                    else:
-                        del job
+                    self.sub_jobs[arch] = job
+                    log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
 
     def _monitor(self):
         self.curstage = 'building'
+
+        # Deal with jobs whose build client is no longer responding
+        for job in self.sub_jobs.values():
+            if job.builder_gone:
+                print "%s (%s/%s): Build client disappeared.  Requeuing arch..." % (self.uid, self.package, job.arch)
+                del self.sub_jobs[job.arch]
+
         self._start_unspawned_builds()
 
         have_jobs = False


Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- CONFIG.py	9 Jun 2005 19:31:28 -0000	1.5
+++ CONFIG.py	12 Jun 2005 08:23:43 -0000	1.6
@@ -1,7 +1,6 @@
 # Configuration file for buildmaster.py
 
 config_opts = {}
-config_opts['hostname'] = "127.0.0.1"
 config_opts['email_to_domain'] = "redhat.com"
 config_opts['email_from'] = "buildsys fedoraproject org"
 config_opts['pkg_cvs_root'] = ":pserver:anonymous cvs fedora redhat com:/cvs/dist"
@@ -11,11 +10,13 @@
 config_opts['tmpdir'] = "/tmp"
 config_opts['log_url'] = "http://foo.foo.org/logs/";
 
+SERVER_BASE_DIR = "/work/fedora-cvs/extras-buildsys/server"
+
 # SSL Cert and key bits
 # MUST be full path to cert
-config_opts['server_cert'] = "/work/fedora-cvs/extras-buildsys/server/certs/server_cert.pem"
-config_opts['server_key'] = "/work/fedora-cvs/extras-buildsys/server/certs/server_key.pem"
-config_opts['ca_cert'] = "/work/fedora-cvs/extras-buildsys/server/certs/ca_cert.pem"
+config_opts['server_cert'] = SERVER_BASE_DIR + "/certs/server_cert.pem"
+config_opts['server_key'] = SERVER_BASE_DIR + "/certs/server_key.pem"
+config_opts['ca_cert'] = SERVER_BASE_DIR + "/certs/ca_cert.pem"
 
 # server_work_dir
 #   - Where logs and finished RPMs are stored
@@ -31,15 +32,60 @@
 # do this unless you really really want to.
 config_opts['use_srpm_not_cvs'] = False
 
-config_opts['targets'] = {  'FC-3' : ['i386', 'x86_64'],
-                            'devel' : ['i386']
+# Targets
+#
+# Architectures for each target should be the _base_ architectures (ie, x86_64,
+# not ia32e or amd64).  The build system will build all packages on these arches
+# be default, unless the package ExcludeArch-es any.
+#
+# Further architectures on a per-package basis are configured in each target's
+# package file.  See the config option 'addl_package_arches_dir'.
+#
+config_opts['targets'] = {  'FC-3' :	['i386', 'x86_64'],
+                            'devel' :	['i386']
                          }
 
+# Target Optional Arches
+#
+# These are arches that submitters _may_ build packages on, but packages won't
+# be built on these arches by default.
+#
+config_opts['target_optional_arches'] = {   'FC-3' :	[],
+                                            'devel' :	[]
+                                        }
+
+
+# Additional Package Arches files
+#
+# Additional package arches files describe additional sub-architectures
+# that packages may build on in a particular target.  For example, in
+# OpenSSL packages on SPARC get optimized for sparcv9 and sparcv8,
+# but not all packages are built for those architectures.  Additional
+# architectures are added to the base architectures listed in the
+# 'targets' config option above, on a per-package basis.
+#
+# One Additional Package Arches file exists per target named in the 'targets'
+# config_opts item above.  Each Additional Package Arches file follows this
+# format: one package per line, package name terminated by a ':' character.
+# Pound/Hash characters denote comments.
+#
+# Example:
+# 
+# openssl: sparcv8 sparcv9
+#
+config_opts['addl_package_arches_dir'] = SERVER_BASE_DIR + "/addl_pkg_arches"
+
+
+# Builder Clients
 config_opts['builders'] = [ 'https://127.0.0.1:8888' ]
 
+
+
+####################################################################################
+
 def get(key):
     if config_opts.has_key(key):
         return config_opts[key]
     else:
-        print "Bad request for key '%s'" % (key)
+        print "Bad request for config option '%s'" % (key)
         exit (1)


Index: build-server
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/build-server,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- build-server	8 Jun 2005 15:55:57 -0000	1.1
+++ build-server	12 Jun 2005 08:23:43 -0000	1.2
@@ -1,4 +1,4 @@
 #!/bin/bash
 
 export PYTHONPATH="$PYTHONPATH:../common"
-python buildserver.py
+python buildserver.py $@


Index: buildmaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildmaster.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- buildmaster.py	9 Jun 2005 19:31:28 -0000	1.3
+++ buildmaster.py	12 Jun 2005 08:23:43 -0000	1.4
@@ -41,8 +41,9 @@
 
 
 class BuildMaster(threading.Thread):
-    def __init__(self, client_manager):
+    def __init__(self, hostname, client_manager):
         self.bcm = client_manager
+        self.hostname = hostname
         self.building_jobs = []
         self.should_stop = False
         self.dbcx = sqlite.connect("buildmaster_db", encoding="utf-8",
@@ -84,7 +85,7 @@
                 print "%s (%s): Starting tag '%s' on target '%s'" % (item['uid'], \
                         item['package'], item['cvs_tag'], item['target'])
                 job = BuildJob.BuildJob(item['uid'], item['username'], item['package'],
-                        item['cvs_tag'], item['target'], self.bcm)
+                        item['cvs_tag'], item['target'], self.bcm, self.hostname)
                 self.building_jobs.append(job)
                 job.start()
 


Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- buildserver.py	9 Jun 2005 19:31:28 -0000	1.3
+++ buildserver.py	12 Jun 2005 08:23:43 -0000	1.4
@@ -19,6 +19,7 @@
 import time
 import CONFIG
 import socket
+import sys
 import os
 import SimpleXMLRPCServer
 import xmlrpclib
@@ -58,6 +59,23 @@
     def __del__(self):
         self.dbcx.close()
 
+    def _insert_job(self, username, package, cvs_tag, target, buildreq, time):
+        # Insert request into the database
+        self.curs.execute('INSERT INTO jobs (uid, username, package,' \
+                ' cvs_tag, target, buildreq, time_submitted, status)' \
+                ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, "%s")' \
+                % (username, package, cvs_tag, target, buildreq, time, 'waiting'))
+        self.dbcx.commit()
+
+        # Find the UID
+        self.curs.execute('SELECT uid FROM jobs WHERE username="%s" AND' \
+                ' package="%s" AND cvs_tag="%s" AND target="%s" AND' \
+                ' buildreq="%s" AND time_submitted=%d AND status="waiting"' \
+                % (username, package, cvs_tag, target, buildreq, time))
+        self.dbcx.commit()
+        item = self.curs.fetchone()
+        return item['uid']
+
     def enqueue(self, username, package, cvs_tag, target, buildreq=None):
         """ Accept a job to build and stuff it into the job database """
 
@@ -69,6 +87,7 @@
         print "Request to enqueue '%s' tag '%s' for target '%s' (user '%s')" \
                 % (package, cvs_tag, target, username)
         targets = CONFIG.get('targets')
+        jobid = -1
         if not targets.has_key(target):
             print "Error setting up build for %s on %s: target does not exist."\
                     % (cvs_tag, target)
@@ -76,14 +95,8 @@
                     "%s: target does not exist." % (cvs_tag, target))
             return (-1, "This build server does not support the target %s." % target)
         else:
-            # Insert request into the database
-            self.curs.execute('INSERT INTO jobs (uid, username, package,' \
-                    ' cvs_tag, target, buildreq, time_submitted, status)' \
-                    ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, "%s")' \
-                    % (username, package, cvs_tag, target, buildreq,        \
-                    time.time(), 'waiting'))
-            self.dbcx.commit()
-        return (0, "Success: package has been queued.")
+            jobid = self._insert_job(username, package, cvs_tag, target, buildreq, time.time())
+        return (0, "Success: package has been queued.", jobid)
 
     def enqueue_srpm(self, username, package, srpm_file, target, buildreq=None):
         """ Accept a job to build from SRPM file and stuff it into the job database """
@@ -102,6 +115,7 @@
         print "Request to enqueue '%s' file '%s' for target '%s' (user '%s')" \
                 % (package, srpm_file, target, username)
         targets = CONFIG.get('targets')
+        jobid = -1
         if not targets.has_key(target):
             print "Error setting up build for %s on %s: target does not exist."\
                     % (srpm_file, target)
@@ -109,14 +123,8 @@
                     "%s: target does not exist." % (srpm_file, target))
             return (-1, "This build server does not support the target %s." % target)
         else:
-            # Insert request into the database
-            self.curs.execute('INSERT INTO jobs (uid, username, package,' \
-                    ' cvs_tag, target, buildreq, time_submitted, status)' \
-                    ' VALUES (NULL, "%s", "%s", "%s", "%s", "%s", %d, "%s")' \
-                    % (username, package, srpm_file, target, buildreq,        \
-                    time.time(), 'waiting'))
-            self.dbcx.commit()
-        return (0, "Success: package has been queued.")
+            jobid = self._insert_job(username, package, cvs_tag, target, buildreq, time.time())
+        return (0, "Success: package has been queued.", jobid)
 
     def list_waiting_jobs(self):
         self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
@@ -140,11 +148,11 @@
             job_list.append(tempX)
         return job_list
 
-    def look_for_clients(self):
+    def update_clients(self):
         reload(CONFIG)
         print "-----------------------------------------------------"
         print " Looking for Build Clients..."
-        self.bcm.update_client_instances()
+        self.bcm.update_clients()
         print "-----------------------------------------------------\n"
         return 0
 
@@ -161,15 +169,22 @@
 
 
 if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print "Usage:\n"
+        print "   %s <hostname>\n" % sys.argv[0]
+        sys.exit(1)
+
+    hostname = sys.argv[1]
+
     bcm = BuildClientManager()
 
     # Create the BuildMaster thread
-    bm = BuildMaster(bcm)
+    bm = BuildMaster(hostname, bcm)
     bm.start()
 
     # Create the BuildMaster XMLRPC server
     xmlrpc_bm = XMLRPCBuildMaster(bcm)
-    bm_server = MyXMLRPCServer((CONFIG.get('hostname'), 8887))
+    bm_server = MyXMLRPCServer((hostname, 8887))
     bm_server.register_instance(xmlrpc_bm)
 
     # SSL certificate and key filenames
@@ -179,11 +194,10 @@
 
     # SRPM fileserver
     http_dir = os.path.join(CONFIG.get('server_work_dir'), "srpm_http_dir")
-    srpm_server = SimpleHTTPSServer.SimpleHTTPSServer(server_cert, server_key, ca_cert,
-                    (CONFIG.get('hostname'), 8886), http_dir)
+    srpm_server = SimpleHTTPSServer.SimpleHTTPSServer(server_cert, server_key, ca_cert, (hostname, 8886), http_dir)
     srpm_server.start()
 
-    print "BuildMaster accepting requests on %s:8887.\n" % CONFIG.get('hostname')
+    print "BuildMaster accepting requests on %s:8887.\n" % hostname
     try:
         bm_server.serve_forever()
     except Exception:


Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- client_manager.py	9 Jun 2005 19:31:28 -0000	1.5
+++ client_manager.py	12 Jun 2005 08:23:43 -0000	1.6
@@ -23,6 +23,7 @@
 import socket
 import FileDownloader
 import os
+import threading
 import urllib
 import SSLXMLRPCServerProxy
 from M2Crypto import SSL
@@ -54,9 +55,9 @@
         self.srpm_url = srpm_url
         self._server = server
         self.starttime = None
+        self.builder_gone = False
         self.downloads = {}
 
-    def start(self):
         failed = False
         try:
             self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
@@ -66,13 +67,15 @@
             failed = True
 
         if failed or self.jobid == 0:
-            self.status = 'failed'
-            return False
+            self._valid = False
         else:
+            self._valid = True
             self.starttime = time.time()
             self.status = 'running'
             self.process()
-            return True
+
+    def valid(self):
+        return self._valid
 
     def process(self):
         if self.status == 'done':
@@ -171,41 +174,63 @@
         if self.downloads.has_key(url):
             self.downloads[url] = status
 
-    def server_gone(self):
-        if self.status == 'done':
-            return
-        self.parent_job.job_server_gone(self)
+    def builder_gone(self):
+        if self.status != 'done':
+            self.builer_gone = True
 
     def die(self):
         self._server.die(self.jobid)
         self.status = 'done'
 
 
-class BuildClientInstance:
-    """ Tracks an single arch on an BuildClient """
+class BuildClient:
+    """ Tracks all jobs on a build client """
 
-    def __init__(self, bcm, address, arch):
-        self._bcm = bcm
+    def __init__(self, manager, address):
+        self._cur_job_lock = threading.Lock()
+        self._manager = manager
         self._jobs = []
-        self._arch = arch
         self._address = address
         self._server = SSLXMLRPCServerProxy.SSLXMLRPCServerProxy(server_cert, server_key, ca_cert, self._address)
         self._unavail_count = 0
-        self._cur_job = self._get_cur_job()
+        self._arches = []
+        try:
+            self._arches = self._server.supported_arches()
+        except socket.error, e:
+            del self._server
+            self._valid = False
+            return
+        self._valid = True
+        self._arches.append('noarch')
+        self._update_cur_job()
 
-    def arch(self):
-        return self._arch
+    def arches(self):
+        return self._arches
 
     def address(self):
         return self._address
+
+    def valid(self):
+        return self._valid
     
-    def new_job(self, parent_job, target, srpm_url):
-        return BuildClientJob(self, parent_job, self._server, target, self._arch, srpm_url)
+    def start_arch_job(self, parent_job, target, arch, srpm_url):
+        # Can be called from other threads (notably from BuildJob threads)
+        job = None
+        if arch in self._arches and self.available():
+            job = BuildClientJob(self, parent_job, self._server, target, arch, srpm_url)
+            if job.valid():
+                self._jobs.append(job)
+                self._update_cur_job()
+            else:
+                del job
+                job = None
+        return job
 
-    def track_job(self, job):
-        self._jobs.append(job)
+    def _update_cur_job(self):
+        # Need to do some locking here since BuildJobs (which are their own thread)
+        # can call through to our start_arch_job() method at any time
+        self._cur_job_lock.acquire()
 
-    def _get_cur_job(self):
         cur_job = None
         try:
             cur_job = self._server.get_cur_job()
@@ -214,7 +239,7 @@
             if e[0] == 111 or e[0] == 104:
                 self._unavail_count = self._unavail_count + 1
             else:
-                print "bci(%s): got error '%s' from build client while trying to get " \
+                print "BuildClient(%s): got error '%s' from build client while trying to get " \
                         "current job number" % (self._address, e)
         except SSL.SSLError, e:
             if e == "unexpected eof":
@@ -223,100 +248,93 @@
             self._unavail_count = 0
             if cur_job == 0:
                 cur_job = None
-        return cur_job
 
-    def process(self):
-        self._cur_job = self._get_cur_job()
+        self._cur_job = cur_job
+        self._cur_job_lock.release()
 
-        # Update status of all jobs
-        for j in self._jobs:
-            j.process()
+    def process(self):
+        self._update_cur_job()
 
         # If we haven't been able to contact the BuildClient for a bit, kill build
         # jobs on this BuildClient
         if self._unavail_count > 2:
             for job in self._jobs:
-                job.server_gone()
-                del job
-            # Return 1 to indicate we should be killed
-            return 1
+                job.builder_gone()
+                self._jobs.remove(job)
+            # Return -1 to indicate we should be killed
+            return -1
+
+        # Update status of all jobs
+        for j in self._jobs:
+            j.process()
 
         return 0
     
     def available(self):
-        if self._cur_job:
+        if self._unavail_count > 2:
+            return False
+
+        self._cur_job_lock.acquire()
+        cur_job = self._cur_job
+        self._cur_job_lock.release()
+
+        if cur_job:
             return False
         else:
             return True
 
+
 class BuildClientManager:
     def __init__(self):
         # List of addresses of possible builders
         self.possible_clients = CONFIG.get('builders')
-
-        # Dict:  arches => available builders
-        # Like so:  [ 'i386':['10.0.0.1', '10.0.0.2'],
-        #             'x86_64':['10.0.0.3']
-        #           ]
-        self.running_clients = {}
+        self.running_clients = []
 
         print "-----------------------------------------------------"
         print " Looking for BuildClients..."
-        self.update_client_instances()
+        self.update_clients()
         print "-----------------------------------------------------\n"
 
-    def update_client_instances(self):
-        # Figure out which clients are alive, and what they support
-        # We create a separate client instance for each arch on each builder,
-        # even though both instances talk to the same XMLRPC server on the builder
+    def update_clients(self):
         for address in self.possible_clients:
             # If the address is already in our running_clients list, skip it
             skip = False
-            for bci_list in self.running_clients.values():
-                for bci in bci_list:
-                    if address == bci.address():
-                        skip = True
+            for client in self.running_clients:
+                if address == client.address():
+                    skip = True
             if skip == True:
                 continue
 
-            arches = None
-            server_cert = CONFIG.get('server_cert')
-            server_key = CONFIG.get('server_key')
-            ca_cert = CONFIG.get('ca_cert')
-            server = SSLXMLRPCServerProxy.SSLXMLRPCServerProxy(server_cert, server_key, ca_cert, address)
-            try:
-                arches = server.supported_arches()
-            except socket.error, e:
-                pass
-            if arches:
-                arches.append('noarch')
-                print "   New Client: '%s' [%s]" % (address, string.join(arches))
-                for a in arches:
-                    if not self.running_clients.has_key(a):
-                        self.running_clients[a] = []
-                    bci = BuildClientInstance(self, address, a)
-                    self.running_clients[a].append(bci)
-            del server
+            # Try to connect to client and add it to our client
+            # list if we can
+            client = BuildClient(self, address)
+            if client.valid():
+                print "   New Client: '%s' [%s]" % (address, string.join(client.arches()))
+                self.running_clients.append(client)
+            else:
+                del client
 
     def process(self):
-        """ Allow each BuildClientInstance to update its status and do some processing """
-        for bci_list in self.running_clients.values():
-            for bci in bci_list:
-                if bci.process() == 1:
-                    # Remove the BuildClientInstance from our lists
-                    print "Removing BuildClient '%s'/%s because it timed out." % (bci.address(), bci.arch())
-                    bci_list.remove(bci)
+        """ Allow each BuildClient to update its status and do some processing """
+        for client in self.running_clients:
+            if client.process() == -1:
+                print "Removing BuildClient '%s' because it timed out." % client.address()
+                self.running_clients.remove(client)
 
     def track_job(self, job):
         if job:
             job.bci.track_job(job)
 
-    def new_job_on_arch(self, parent_job, target, arch, srpm_url):
+    def start_arch_job(self, parent_job, target, arch, srpm_url):
         """ Create a job on a free builder for this arch """
 
-        if self.running_clients.has_key(arch):
-            for client in self.running_clients[arch]:
-                if client.available():
-                    return client.new_job(parent_job, target, srpm_url)
-        return None
+        job = None
+        for client in self.running_clients:
+            if not client.available():
+                continue
+            client_arches = client.arches()
+            if arch in client_arches:
+                job = client.start_arch_job(parent_job, target, arch, srpm_url)
+
+        return job
 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]