extras-buildsys/server BuildJob.py, 1.6, 1.7 BuildMaster.py, 1.1, 1.2 client_manager.py, 1.12, 1.13

Daniel Williams (dcbw) fedora-extras-commits at redhat.com
Sun Jun 19 02:47:56 UTC 2005


Author: dcbw

Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv19730/server

Modified Files:
	BuildJob.py BuildMaster.py client_manager.py 
Log Message:
2005-06-17  Dan Williams <dcbw at redhat.com>

    * common/CommonErrors.py
      server/client_manager.py
      common/SSLCommon.py
      common/FileDownloader.py

    * server/BuildJob.py
        - better check for unspawned jobs
        - Add a '/' to SRPM URLs that clients download to protect against CONFIG.py
            errors
        - Mark the repo as invalid after we copy RPMs to it

    * server/BuildMaster.py
        - Only run createrepo when the repository has actually changed

    * server/client_manager.py
       - Fix bug that caused jobs to simultaneously get started on all clients
           that supported an architecture




Index: BuildJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildJob.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- BuildJob.py	17 Jun 2005 16:27:08 -0000	1.6
+++ BuildJob.py	19 Jun 2005 02:47:53 -0000	1.7
@@ -360,10 +360,10 @@
 
     def _start_unspawned_builds(self):
         for arch in self.buildarches:
-            if not self.sub_jobs.has_key(arch):
+            if not self.sub_jobs.has_key(arch) or not self.sub_jobs[arch]:
                 # Construct SPRM URL
                 srpm_http_base = self.srpm_http_path[len(http_dir):]
-                srpm_url = "https://" + self.hostname + ":8886" + srpm_http_base
+                srpm_url = "https://" + self.hostname + ":8886/" + srpm_http_base
                 job = self.bm.bcm.start_arch_job(self, self.target, arch, srpm_url)
                 if job:
                     self.sub_jobs[arch] = job
@@ -431,6 +431,8 @@
         self.curstage = 'needsign'
 
         # Copy completed RPMs to repo dir
+        # FIXME: possible concurrency issue, what if createrepo
+        # is being run when we are copying RPMs to the repo dir?
         for job in self.sub_jobs.values():
             file_list = job.get_files()
             for f in file_list:
@@ -442,6 +444,7 @@
                 if not os.path.exists(dst_path):
                     os.makedirs(dst_path)
                 shutil.copy(src_file, dst_path)
+        self.bm.invalidate_repo()
 
         resultstring = "%s (%s): Build on target %s succeeded." % (self.uid, self.name, self.target)
         self.email_result(resultstring)


Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- BuildMaster.py	15 Jun 2005 04:10:07 -0000	1.1
+++ BuildMaster.py	19 Jun 2005 02:47:53 -0000	1.2
@@ -51,12 +51,16 @@
         self.dbcx = sqlite.connect("jobdb", encoding="utf-8", timeout=2)
         self.curs = self.dbcx.cursor()
         self.createrepo_lock = threading.Lock()
+        self._repo_invalid = True
         ensure_build_db_tables(self.dbcx)
         threading.Thread.__init__(self)
 
     def __del__(self):
         self.dbcx.close()
 
+    def invalidate_repo(self):
+        self._repo_invalid = True
+
     def stop(self):
         self.should_stop = True
 
@@ -70,14 +74,21 @@
         return self.bcm
 
     def createrepo(self, repodir):
+        if not self._repo_invalid:
+            return
+
         """ We need to lock calls to createrepo so they don't get run at the same time """
+        # FIXME: possibly concurrency issue here, what if clients are
+        # trying to pull repodata while we are recreating it?
         if not os.path.exists(repodir):
             os.makedirs(repodir)
         self.createrepo_lock.acquire()
         s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
         self.createrepo_lock.release()
         if s != 0:
-            print "createrepo failed with exit status %d" % s
+            print "createrepo failed with exit status %d!" % s
+
+        self._repo_invalid = False
 
     def run(self):
         while self.should_stop == False:


Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -r1.12 -r1.13
--- client_manager.py	17 Jun 2005 16:27:08 -0000	1.12
+++ client_manager.py	19 Jun 2005 02:47:53 -0000	1.13
@@ -27,6 +27,7 @@
 import urllib
 import SSLXMLRPCServerProxy
 from M2Crypto import SSL
+import CommonErrors
 
 
 # SSL certificate and key filenames
@@ -241,7 +242,7 @@
             cur_job = self._server.get_cur_job()
         except socket.error, e:
             # Check for "Connection refused" or "Connection reset by peer"
-            if e[0] == 111 or e[0] == 104 or e[0] == 61:
+            if CommonErrors.canIgnoreSocketError(e):
                 self._unavail_count = self._unavail_count + 1
             else:
                 print "BuildClient(%s): got error '%s' from build client while trying to get " \
@@ -359,13 +360,16 @@
     def start_arch_job(self, parent_job, target, arch, srpm_url):
         """ Create a job on a free builder for this arch """
 
-        job = None
         for client in self.running_clients:
             if not client.available():
                 continue
             client_arches = client.arches()
             if arch in client_arches:
                 job = client.start_arch_job(parent_job, target, arch, srpm_url)
+                if not job.valid():
+                    del job
+                else:
+                    return job
 
-        return job
+        return None
 




More information about the fedora-extras-commits mailing list