[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/client buildclient.py,1.16,1.17



Author: dcbw

Update of /cvs/fedora/extras-buildsys/client
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv11449/client

Modified Files:
	buildclient.py 
Log Message:
2005-06-25  Dan Williams <dcbw redhat com>

    * client/buildclient.py
        - Break out some functions from process() to make code easier to follow
        - Use unique extensions on buildroots so more than one build client
            can run at the same time on the same machine
        - Execute mock after the build to clean up the buildroot before
            deleting the buildroot directory

        Requires latest mock from CVS




Index: buildclient.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/client/buildclient.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -r1.16 -r1.17
--- buildclient.py	24 Jun 2005 21:07:30 -0000	1.16
+++ buildclient.py	25 Jun 2005 19:19:04 -0000	1.17
@@ -71,6 +71,7 @@
         self._target = target
         self._srpm_url = srpm_url
         self._log_fd = None
+        self._mock_config = None
 
         self._result_dir = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "result")
         if not os.path.exists(self._result_dir):
@@ -150,9 +151,9 @@
             os.makedirs(self._result_dir)
         if not os.path.exists(self._result_dir):
             os.makedirs(self._result_dir)
-        cmd = '%s %s -r %s --resultdir=%s --statedir=%s %s' % (self.arch_command,
+        cmd = '%s %s -r %s --resultdir=%s --statedir=%s --uniqueext=%s %s' % (self.arch_command,
                             CONFIG.get('builder_cmd'), self.buildroot, 
-                            self._result_dir, self._state_dir, self._srpm_path)
+                            self._result_dir, self._state_dir, self._uniqid, self._srpm_path)
         self.log("   %s\n" % cmd)
         self._pobj = popen2.Popen4(cmd=cmd, bufsize=1024)
         fcntl.fcntl(self._pobj.fromchild.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
@@ -164,6 +165,14 @@
         mockstatusfile = os.path.join(self._state_dir, 'status')
         while not os.path.exists(mockstatusfile):
             time.sleep(0.5)
+            # if mock exited with an error report that error and not
+            # the missing status file.
+            exit_status = self._pobj.poll()
+            if exit_status > 0:
+                self._status = 'failed'
+                break
+
+            # Kill mock after 5s if it didn't dump the status file
             if time.time() - start_time > 5:
                 self.log("Timed out waiting for the mock status file!  %s\n" % mockstatusfile)
                 try:
@@ -176,9 +185,27 @@
                 self._status = 'failed'
                 break
 
+    def _cleanup(self):
+        self.log("Cleaning up the buildroot...\n")
+        cmd = '%s %s clean --uniqueext=%s -r %s' % (self.arch_command,
+                            CONFIG.get('builder_cmd'), self._uniqid,
+                            self.buildroot)
+        self.log("   %s\n" % cmd)
+        self._pobj = popen2.Popen4(cmd=cmd)
+        self._status = 'cleanup'
+
     def _mock_is_prepping(self):
         mock_status = self._get_mock_status()
         if mock_status:
+            if mock_status == 'prep':
+                return True
+            elif mock_status == 'setu':
+                return True
+        return False
+
+    def _mock_using_repo(self):
+        mock_status = self._get_mock_status()
+        if mock_status:
             if mock_status == 'init':
                 return True
             elif mock_status == 'clea':
@@ -198,7 +225,6 @@
     def _get_mock_status(self):
         mockstatusfile = os.path.join(self._state_dir, 'status')
         if not os.path.exists(mockstatusfile):
-            self.log("mock status file doesn't exist!  %s" % mockstatusfile)
             return None
 
         f = open(mockstatusfile, "r")
@@ -220,64 +246,96 @@
         string = string.lower()
         return string
 
-    def process(self):
-        if not self.is_done_status():
-            if self._status == 'downloading':
-                pass
-            elif self._status == 'downloaded':
-                # We can't start doing anything with yum until the build
-                # server tells us the repo is unlocked.
-                if not self._repo_locked:
-                    self._build()
-                else:
-                    # Only show this message once
-                    if not self._repo_locked_msg:
-                        self.log("Waiting for repository to unlock before starting the build...\n")
-                        self._repo_locked_msg = True
-            elif self._status == 'prepping':
-                if not self._mock_is_prepping():
-                    self._status = 'building'
-            elif self._status == 'building':
-                exit_status = self._pobj.poll()
-                if exit_status == 0:
-                    # mock completed successfully
-                    if self._status != 'building':
-                        self.log("Bad job end status %s encountered!" % self._status)
-                    self._status = 'done'
-                elif exit_status > 0:
-                    # mock exited with an error
-                    self._status = 'failed'
-
-            if self._pobj:
-                # Grab any mock output and write it to a log
-                string = ' '
-                while len(string) > 0:
-                    try:
-                        string = os.read(self._pobj.fromchild.fileno(), 1024)
-                    except OSError, e:
-                        if e.errno == errno.EAGAIN:     # Resource temporarily unavailable
-                            break
-                        else:
-                            self.log("Error reading mock output: %s\n" % e)
+    def _read_mock_config(self):
+        mockconfigfile = os.path.join(self._result_dir, 'mockconfig.log')
+        if not os.path.exists(mockconfigfile):
+            return None
+
+        f = open(mockconfigfile, "r")
+        contents = {}
+        for line in f:
+            (item, loc) = line.split('=')
+            item = item.strip()
+            loc = loc.strip()
+            contents[item] = loc
+        f.close()
+        return contents
+
+    def _grab_mock_output(self):
+        # We don't care about output from the 'cleanup' stage
+        if self._pobj and self._status != 'cleanup':
+            # Grab any mock output and write it to a log
+            string = ' '
+            while len(string) > 0:
+                try:
+                    string = os.read(self._pobj.fromchild.fileno(), 1024)
+                except OSError, e:
+                    if e.errno == errno.EAGAIN:     # Resource temporarily unavailable
+                        break
                     else:
-                        self._log_fd.write(string)
-                        self._log_fd.flush()
-                        os.fsync(self._log_fd.fileno())
-
-            if self.is_done_status():
-                self._files = self._find_files()
-                self.log("\n\n-----------------------\n\n")
-                if self._status == 'done':
-                    self.log("Job completed successfully.\n")
-                elif self._status == 'failed':
-                    exit_status = self._pobj.poll()
-                    self.log("Job failed due to mock errors!  mock exit status: %d\n" % exit_status)
-                elif self._status == 'killed':
-                    self.log("Job failed because it was killed.\n")
-        else:
-            if self._log_fd:
-                self._log_fd.close()
-                self._log_fd = None
+                        self.log("Error reading mock output: %s\n" % e)
+                else:
+                    self._log_fd.write(string)
+                    self._log_fd.flush()
+                    os.fsync(self._log_fd.fileno())
+
+    def _mock_done(self):
+        self._files = self._find_files()
+        self.log("\n\n-----------------------\n\n")
+        if self._status == 'done':
+            self.log("Job completed successfully.\n")
+        elif self._status == 'failed':
+            exit_status = self._pobj.poll()
+            self.log("Job failed due to mock errors!  mock exit status: %d\n" % exit_status)
+        elif self._status == 'killed':
+            self.log("Job failed because it was killed.\n")
+
+        if self._log_fd:
+            self._log_fd.close()
+            self._log_fd = None
+
+    def process(self):
+        if self.is_done_status():
+            return
+
+        if self._status == 'downloading':
+            pass
+        elif self._status == 'downloaded':
+            # We can't start doing anything with yum until the build
+            # server tells us the repo is unlocked.
+            if not self._repo_locked:
+                self._build()
+            else:
+                # Only show this message once
+                if not self._repo_locked_msg:
+                    self.log("Waiting for repository to unlock before starting the build...\n")
+                    self._repo_locked_msg = True
+        elif self._status == 'prepping':
+            if not self._mock_config and self._mock_is_prepping():
+                self._mock_config = self._read_mock_config()
+            if not self._mock_using_repo():
+                self._status = 'building'
+        elif self._status == 'building':
+            exit_status = self._pobj.poll()
+            if exit_status == 0:
+                # mock completed successfully
+                if self._status != 'building':
+                    self.log("Bad job end status %s encountered!" % self._status)
+                self._cleanup()
+            elif exit_status > 0:
+                # mock exited with an error
+                self._status = 'failed'
+        elif self._status == 'cleanup':
+            exit_status = self._pobj.poll()
+            if exit_status >= 0:
+                # We ignore mock errors when cleaning the buildroot
+                self._status = 'done'
+                if self._mock_config.has_key('rootdir'):
+                    shutil.rmtree(self._mock_config['rootdir'], ignore_errors=True)
+
+        self._grab_mock_output()
+        if self.is_done_status():
+            self._mock_done()
 
     def _find_files(self):
         # Grab the list of files in our job's result dir and URL encode them
@@ -397,12 +455,12 @@
         self.localarches = localarches
         self.cur_job = 0
 
-    def _process(self):
+    def process(self):
         # Give jobs some time to update their status and do their thing
         job = 0
         for (uniqid, bcp) in self.ids.iteritems():
-            bcp.process()
             if not bcp.is_done_status():
+                bcp.process()
                 job = uniqid
         self.cur_job = job  # Update current job
 
@@ -514,7 +572,7 @@
         cur_time = time.time()
         if cur_time >= last_time + 3:
             # do some work every 3s or so
-            bcs._process()
+            bcs.process()
             last_time = time.time()
 
     os._exit(0)


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]