[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/server BuildJob.py, NONE, 1.1 CONFIG.py, 1.4, 1.5 buildmaster.py, 1.2, 1.3 buildserver.py, 1.2, 1.3 client_manager.py, 1.4, 1.5 buildjob.py, 1.6, NONE



Author: dcbw

Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv8175

Modified Files:
	CONFIG.py buildmaster.py buildserver.py client_manager.py 
Added Files:
	BuildJob.py 
Removed Files:
	buildjob.py 
Log Message:
2005-06-09  Dan Williams <dcbw redhat com>

    * common/FileDownloader.py
        - Don't traceback on refused or dropped connections

    * server/buildjob.py -> server/BuildJob.py
        - BuildJob objects are now threads
        - Clean up console output a lot
        - Delete arch-specific jobs that don't actually start

    * server/buildmaster.py
        - Clean up console output

    * server/buildserver.py
        - Clean up 'enqueue' command return messages

    * server/client_manager.py
        - Try to tighten up exception handling around socket operations
        - Don't print so much stuff to the console
        - Prettier printing of downloaded file lists from client




--- NEW FILE BuildJob.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal


import os
import os.path
import sys
import commands
import threading
import time
import popen2
import rpmUtils
import exceptions
import shutil
import tempfile
import smtplib
from email.MIMEText import MIMEText
import string
import SimpleXMLRPCServer
import xmlrpclib
import CONFIG
import socket
from client_manager import BuildClientManager
from client_manager import BuildClientJob

os.environ['CVSROOT'] = CONFIG.get('pkg_cvs_root')
if len(CONFIG.get('pkg_cvs_rsh')) > 0:
    os.environ['CVS_RSH'] = CONFIG.get('pkg_cvs_rsh')

DEBUG = False
def debugprint(stuff=''):
    if DEBUG:
        print stuff

def log(stuff=''):
    print stuff

class PrepError(exceptions.Exception):
    def __init__(self, errno=0, args=None):
        exceptions.Exception.__init__(self)
        self.args = args
        self.errno = errno
    def __str__(self):
        return self.args
        
class BuildError(exceptions.Exception):
    def __init__(self, errno=0, args=None):
        exceptions.Exception.__init__(self)
        self.args = args
        self.errno = errno
    def __str__(self):
        return self.args

        

http_dir = os.path.join(CONFIG.get('server_work_dir'), "srpm_http_dir")

class BuildJob(threading.Thread):
    """ Controller object for building 1 SRPM on multiple arches """

    def __init__(self, uid, username, package, cvs_tag, target, client_manager):
        self.curstage = 'initialize'
        self.bcm = client_manager
        self.uid = uid
        self.username = username
        self.starttime = time.time()
        self.endtime = None
        self.package = package
        self.target = target
        self.buildarches = []
        self.sub_jobs = {}
        self.failed = False
        self.no_cvs = CONFIG.get('use_srpm_not_cvs')
        self.cvs_tag = cvs_tag
        self.stage_dir = None
        self.srpm_path = None
        self.srpm_http_path = None
        # Deal with straight SRPM builds
        if self.no_cvs and self.curstage is 'initialize':
            self.curstage = 'make_srpm'
        threading.Thread.__init__(self)

    def get_cur_stage(self):
        return self.curstage
        
    def get_uid(self):
        return self.uid
        
    def arch_handling(self, hdr):
        archs = []
        targets = CONFIG.get('targets')
        buildable_arches = targets[self.target]
        
        ba = hdr['buildarchs']
        exclusive = hdr['exclusivearch']
        exclude = hdr['excludearch']
        
        if ba == ['noarch']:
            return ba

        # default to building all arches
        tmparchs = []
        for arch in buildable_arches:
            tmparchs.append(arch)

        if ba:
            tmparchs = ba
        else:
            if exclusive:
                tmparchs = exclusive
                
        if exclude:
            for arch in exclude:
                if arch in tmparchs:
                    tmparchs.remove(arch)
        
        # we probably need to check the archs, and make sure they are what 
        # we can build for
        for thisarch in tmparchs:
            if thisarch in buildable_arches:
                archs.append(thisarch)

        return archs

        
    def _make_stage_dir(self, rootdir):
        # The dir will look like this:
        # <rootdir>/devel/95-foo-1.1.0-23
        pkgsubdir = '%d-%s-%s-%s' % (self.uid, self.name, self.ver, self.release)
        stage_dir = os.path.join(rootdir, self.target, pkgsubdir)
        if os.path.exists(stage_dir):
            shutil.rmtree(stage_dir, ignore_errors=True)
        os.makedirs(stage_dir)
        return stage_dir

        
    def _checkout(self):
        self.curstage = 'checkout'
        dir_prefix = self.cvs_tag + "-"
        self.checkout_tmpdir = tempfile.mkdtemp(prefix=dir_prefix, dir=CONFIG.get('tmpdir'))
        os.chdir(self.checkout_tmpdir)

        # Checkout the module
        cmd = '%s co -r %s %s' % (CONFIG.get('cvs_cmd'), self.cvs_tag, self.package)
        debugprint("%d: Running %s" % (self.uid, cmd))
        s, o = commands.getstatusoutput(cmd)
        if s != 0:
            subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
            msg = "could not check out %s from %s - output was:\n %s" % (self.cvs_tag, self.target, o)
            self.email_result(resultstring=msg, subject=subj)
            self.curstage = 'finished'
            self.failed = True
            shutil.rmtree(self.checkout_tmpdir, True)
            return

        # Just in case the 'common' directory didn't come along for the ride,
        # get it from CVS
        pkg_path = os.path.join(self.checkout_tmpdir, self.package)
        if not os.path.exists(os.path.join(pkg_path, "common")):
            os.chdir(pkg_path)
            cmd = '%s co common' % CONFIG.get('cvs_cmd')
            debugprint("%d: Running %s" % (self.uid, cmd))
            s, o = commands.getstatusoutput(cmd)
            os.chdir(self.checkout_tmpdir)
            if s != 0:
                subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
                msg = "could not check out common directory - output was:\n %s" % (self.cvs_tag, self.target, o)
                self.email_result(resultstring=msg, subject=subj)
                self.curstage = 'finished'
                self.failed = True
                shutil.rmtree(self.checkout_tmpdir, True)
                return

    def _make_srpm(self):
        self.curstage = 'make_srpm'
        self.srpm_path = None
        srpm_dir = os.path.join(self.checkout_tmpdir, self.package, self.target)
        if not os.path.exists(srpm_dir):
            subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
            msg = "could not find path %s for %s." % (srpm_dir, self.cvs_tag)
            self.email_result(resultstring=msg, subject=subj)
            self.curstage = 'finished'
            self.failed = True
            shutil.rmtree(self.checkout_tmpdir, True)
            return

        os.chdir(srpm_dir)

        cmd = '%s srpm' % CONFIG.get('make_cmd')
        debugprint("%d: Running %s in %s" % (self.uid, cmd, srpm_dir))
        s, o = commands.getstatusoutput(cmd)
        if s != 0:
            subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
            msg = "could not make srpm for %s - output was:\n %s" % (self.cvs_tag, o)
            self.email_result(resultstring=msg, subject=subj)
            self.curstage = 'finished'
            self.failed = True
            #shutil.rmtree(self.checkout_tmpdir, True)
            return
        
        srpmpath = None
        for line in o.split("\n"):
            if line.startswith("Wrote:"):
                line.replace("\n", "")
                (garbage, path) = line.split(':')
                srpmpath = path.strip()
                break
        if not srpmpath:
            subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
            msg = "could not find srpm for %s - output was:\n %s" % (self.cvs_tag, o)
            self.email_result(resultstring=msg, subject=subj)
            self.curstage = 'finished'
            self.failed = True
            shutil.rmtree(self.checkout_tmpdir, True)
            return
        self.srpm_path = srpmpath

    def _prep(self):
        self.curstage = 'prep'

        # In SRPM-only mode, cvs_tag is path to the SRPM to build
        if self.no_cvs:
            self.srpm_path = self.cvs_tag

        ts = rpmUtils.transaction.initReadOnlyTransaction()
        hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpm_path)
        self.name = hdr['name']
        self.ver = hdr['version']
        self.release = hdr['release']
        self.buildarches = self.arch_handling(hdr)
        del hdr
        del ts

        if len(self.buildarches) == 0:
            self.failed = True
            self.curstage = 'finished'
            return

        # Make sure build clients see latest packages
        self._createrepo()

        self.stage_dir = self._make_stage_dir(CONFIG.get('server_work_dir'))
        for arch in self.buildarches:
            thisdir = os.path.join(self.stage_dir, arch)
            if not os.path.exists(thisdir):
                os.makedirs(thisdir)

        # Copy the SRPM to the final package product dir
        srpm = os.path.basename(self.srpm_path)
        srpm_in_dir = os.path.join(self.stage_dir, srpm)
        if os.path.exists(srpm_in_dir):
            os.unlink(srpm_in_dir)
        shutil.copy(self.srpm_path, self.stage_dir)

        # Must also copy SRPM to where the build client can get it over HTTP
        http_pkg_path = self._make_stage_dir(http_dir)
        self.srpm_http_path = os.path.join(http_pkg_path, srpm)
        shutil.copy(self.srpm_path, self.srpm_http_path)
        self.srpm_path = srpm_in_dir

        # Remove CVS checkout and make_srpm dirs
        if not self.no_cvs:
            shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)

    def run(self):
        while self.curstage != 'needsign' and self.curstage != 'failed':
            # Advance to next stage based on current stage
            oldstage = self.curstage
            if oldstage == 'initialize':
                self._checkout()
            elif oldstage == 'checkout':
                self._make_srpm()
            elif oldstage == 'make_srpm':
                self._prep()
            elif oldstage == 'prep' or oldstage == 'building':
                self._monitor()
            elif oldstage == 'finished':
                self._cleanup()
            elif oldstage == 'cleanup':
                if self.failed:
                    self._failed()
                else:
                    self._succeeded()


    def job_server_gone(self, job):
        """ Remove a job from our building queue if its server went away """

        print "%s (%s/%s): Build client disappeared.  Requeuing arch..." % (self.uid, self.package, job.arch)
        del self.sub_jobs[job.arch]

    def _start_unspawned_builds(self):
        for arch in self.buildarches:
            if not self.sub_jobs.has_key(arch):
                # Construct SPRM URL
                srpm_http_base = self.srpm_http_path[len(http_dir):]
                srpm_url = "https://"; + CONFIG.get('hostname') + ":8886" + srpm_http_base
                job = self.bcm.new_job_on_arch(self, self.target, arch, srpm_url)
                if job:
                    if job.start() == True:
                        self.bcm.track_job(job)
                        self.sub_jobs[arch] = job
                        log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
                    else:
                        del job

    def _monitor(self):
        self.curstage = 'building'
        self._start_unspawned_builds()

        have_jobs = False
        jobs_running = False
        for job in self.sub_jobs.values():
            have_jobs = True
            job_status = job.get_status()
            if job_status is 'downloading' or job_status is 'running':
                jobs_running = True
                continue
            elif job_status is 'done':
                builder_result = job.get_builder_result()
                if builder_result == 'failed' or builder_result == 'killed': 
                    self.failed = True

        if self.failed or (have_jobs == True and jobs_running == False):
            self.curstage = 'finished'

        time.sleep(5)

    def _cleanup(self):
        self.curstage = 'cleanup'
        if self.failed:
            # Kill remaining jobs on other arches
            for job in self.sub_jobs.values():
                job_status = job.get_status()
                if job_status is 'initialize' or job_status is 'running':
                    job.die()

    def get_stage_dir(self):
        return self.stage_dir

    def _failed(self):
        old_stage = self.curstage
        self.curstage = 'failed'

        resultstring = """
    %s (%s): %s on %s failed to complete on one or more archs.
""" % (self.uid, self.package, self.cvs_tag, self.target)

        self.email_result(resultstring)
        
    def _succeeded(self):
        self.curstage = 'needsign'

        # Copy completed RPMs to repo dir here
        for job in self.sub_jobs.values():
            file_list = job.get_files()
            for f in file_list:
                if not f.endswith(".rpm"):
                    continue
                src_file = os.path.join(self.stage_dir, job.arch, f)
                verrel = "%s-%s" % (self.ver, self.release)
                dst_path = os.path.join(CONFIG.get('repo_dir'), self.target, self.name, verrel, job.arch)
                if not os.path.exists(dst_path):
                    os.makedirs(dst_path)
                shutil.copy(src_file, dst_path)

        resultstring = "%s (%s): Build on target %s succeeded." % (self.uid, self.name, self.target)
        self.email_result(resultstring)

        # Udpate the repo with new packages
        self._createrepo()

    def email_result(self, resultstring, subject=None):
        """send 'resultstring' to self.email from self.email_from"""
        
        print "%s (%s): Result: '%s'" % (self.uid, self.package, resultstring)
        return
        msg = MIMEText(resultstring)
        if not subject:
            subject = 'Build Result: %s on %s' % (self.name, self.target)
        msg['Subject'] = subject
        msg['From'] = CONFIG.get('email_from')
        email_to = '%s %s' % (self.username, CONFIG.get('email_to_domain'))
        msg['To'] = email_to
        s = smtplib.SMTP()
        s.connect()
        s.sendmail(CONFIG.get('email_from'), [email_to], msg.as_string())
        s.close()

    def _createrepo(self):
        # createrepo on the needsign tree for new changes
        repodir = os.path.join(CONFIG.get('repo_dir'), self.target)
        debugprint("%d: updating repodir %s" % (self.uid, repodir))
        if not os.path.exists(repodir):
            os.makedirs(repodir)
        s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
        if s != 0:
            print "Createrepo failed!!!"
            #self.curstage = 'failed'
            #raise PrepError(5, 'Error generating repodata for %s: %s' % (repodir, o))




Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- CONFIG.py	9 Jun 2005 01:57:41 -0000	1.4
+++ CONFIG.py	9 Jun 2005 19:31:28 -0000	1.5
@@ -35,8 +35,7 @@
                             'devel' : ['i386']
                          }
 
-#config_opts['builders'] = [ 'http://172.16.83.112:8888', 'http://172.16.83.22:8888' ]
-config_opts['builders'] = [ 'http://127.0.0.1:8888' ]
+config_opts['builders'] = [ 'https://127.0.0.1:8888' ]
 
 def get(key):
     if config_opts.has_key(key):


Index: buildmaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildmaster.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- buildmaster.py	7 Jun 2005 12:10:23 -0000	1.2
+++ buildmaster.py	9 Jun 2005 19:31:28 -0000	1.3
@@ -18,8 +18,7 @@
 
 import time
 import CONFIG
-from buildjob import BuildJob
-from buildjob import PrepError
+import BuildJob
 import sqlite
 import threading
 
@@ -71,10 +70,9 @@
 
             # Allow each job some processing time
             for job in self.building_jobs:
-                job.process()
                 self.set_job_status(job)
                 if job.get_cur_stage() == 'failed' or job.get_cur_stage() == 'needsign':
-                    print "%d: Job finished." % job.get_uid()
+                    print "%s (%s): Job finished." % (job.get_uid(), job.package)
                     self.building_jobs.remove(job)
 
             # Grab one waiting job from database and start it
@@ -83,11 +81,12 @@
             self.dbcx.commit()
             item = self.curs.fetchone()
             if item:
-                print "%d: Adding (%s/'%s'/%s) to build queue" % (item['uid'], \
+                print "%s (%s): Starting tag '%s' on target '%s'" % (item['uid'], \
                         item['package'], item['cvs_tag'], item['target'])
-                job = BuildJob(item['uid'], item['username'], item['package'],
+                job = BuildJob.BuildJob(item['uid'], item['username'], item['package'],
                         item['cvs_tag'], item['target'], self.bcm)
                 self.building_jobs.append(job)
+                job.start()
 
             time.sleep(5)
             if self.should_stop == True:


Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- buildserver.py	9 Jun 2005 01:57:41 -0000	1.2
+++ buildserver.py	9 Jun 2005 19:31:28 -0000	1.3
@@ -22,7 +22,6 @@
 import os
 import SimpleXMLRPCServer
 import xmlrpclib
-from buildjob import BuildJob
 import sqlite
 import smtplib
 from email.MIMEText import MIMEText
@@ -62,12 +61,10 @@
     def enqueue(self, username, package, cvs_tag, target, buildreq=None):
         """ Accept a job to build and stuff it into the job database """
 
-        # FIXME  we should be passing back a tuple of (error/success, reason)
-
         if CONFIG.get('use_srpm_not_cvs') == True:
             email_result(username, cvs_tag, "Error setting up build for %s on "\
                     "%s: this server builds SRPMs, not CVS checkouts." % (cvs_tag, target))
-            return -1
+            return (-1, "This build server is set up for building SRPMS only.  Use the 'enqueue_srpm' command instead.")
 
         print "Request to enqueue '%s' tag '%s' for target '%s' (user '%s')" \
                 % (package, cvs_tag, target, username)
@@ -77,7 +74,7 @@
                     % (cvs_tag, target)
             email_result(username, cvs_tag, "Error setting up build for %s on "\
                     "%s: target does not exist." % (cvs_tag, target))
-            return -1
+            return (-1, "This build server does not support the target %s." % target)
         else:
             # Insert request into the database
             self.curs.execute('INSERT INTO jobs (uid, username, package,' \
@@ -86,23 +83,21 @@
                     % (username, package, cvs_tag, target, buildreq,        \
                     time.time(), 'waiting'))
             self.dbcx.commit()
-        return 0
+        return (0, "Success: package has been queued.")
 
     def enqueue_srpm(self, username, package, srpm_file, target, buildreq=None):
         """ Accept a job to build from SRPM file and stuff it into the job database """
 
-        # FIXME  we should be passing back a tuple of (error/success, reason)
-
         if CONFIG.get('use_srpm_not_cvs') == False:
             email_result(username, srpm_file, "Error setting up build for %s on "\
                     "%s: this server builds CVS checkouts, not SRPMS." % (srpm_file, target))
-            return -1
+            return (-1, "This build server is set up for building from CVS.  Use the 'enqueue' command instead.")
 
         # We limit the database field to 255 chars
         if len(srpm_file) > 255:
             email_result(username, srpm_file, "Error setting up build for %s on "\
                     "%s: try using a shorter path to the SRPM (< 255 chars)." % (srpm_file, target))
-            return -1
+            return (-1, "Pathname to SRPM is limited to 255 characters.")
 
         print "Request to enqueue '%s' file '%s' for target '%s' (user '%s')" \
                 % (package, srpm_file, target, username)
@@ -112,7 +107,7 @@
                     % (srpm_file, target)
             email_result(username, srpm_file, "Error setting up build for %s on "\
                     "%s: target does not exist." % (srpm_file, target))
-            return -1
+            return (-1, "This build server does not support the target %s." % target)
         else:
             # Insert request into the database
             self.curs.execute('INSERT INTO jobs (uid, username, package,' \
@@ -121,7 +116,7 @@
                     % (username, package, srpm_file, target, buildreq,        \
                     time.time(), 'waiting'))
             self.dbcx.commit()
-        return 0
+        return (0, "Success: package has been queued.")
 
     def list_waiting_jobs(self):
         self.curs.execute('SELECT uid, username, package, cvs_tag, target' \


Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- client_manager.py	9 Jun 2005 03:18:03 -0000	1.4
+++ client_manager.py	9 Jun 2005 19:31:28 -0000	1.5
@@ -60,11 +60,12 @@
         failed = False
         try:
             self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
-        except Exception, e:
+        except socket.error, e:
+            if e[0] != 104 and e[0] != 111:
+                print "Error starting job on host %s\n\t---error: %s" % (self.bci.address(), e)
             failed = True
 
         if failed or self.jobid == 0:
-            print "Error starting job on host %s\n\t---error: %s" % (self.bci.address(), e)
             self.status = 'failed'
             return False
         else:
@@ -80,20 +81,18 @@
             result = ''
             try:
                 result = self._server.status(self.jobid)
-            except Exception, e:
-                print "BuildClientJob(%s): got error '%s' from build client while attempting " \
-                        "to get its status." % (self.bci.address(), e)
+            except socket.error, e:
+                if e[0] != 104 and e[0] != 111:
+                    print "%s (%s/%s): [ %s ] Unknown error when getting status: '%s'" % (self.parent_job.uid,
+                                self.parent_job.package, self.arch, self.bci.address(), e)
             self.client_result = result
 
             # if the builder is done, grab list of files to download
             if result == 'done' or result == 'killed' or result == 'failed':
                 self.status = 'downloading'
-                print "%s (%s): Files to download from build client:" % (self.jobid, self.arch)
                 for f in self._server.files(self.jobid):
                     uf = urllib.unquote(f)
-                    print "     %s" % uf
                     self.downloads[uf] = 0
-                print "%s (%s): Done" % (self.jobid, self.arch)
         elif self.status == 'downloading':
             # Start grabbing the next undownloaded file, but only
             # if we aren't already pulling one down
@@ -119,6 +118,7 @@
                     break
                 elif dl_status == 1:
                     # in progress
+                    undownloaded = True
                     break
                 elif dl_status == 2:
                     # error
@@ -126,17 +126,32 @@
                 elif dl_status == 3:
                     # this one is done
                     continue
+
+            # All done downloading?
             if not undownloaded:
+                # Print out downloaded file list
+                file_string = ""
+                ndownloads = len(self.downloads.keys())
+                for url in self.downloads.keys():
+                    filename = os.path.basename(url)
+                    dl_status = self.downloads[url]
+                    string = "'" + filename + "'"
+                    if dl_status == 2:
+                        string = string + " (failed)"
+                    file_string = file_string + string
+                    if url != self.downloads.keys()[ndownloads - 1]:
+                        file_string = file_string + ", "
+
+                print "%s (%s/%s): Build result files - [ %s ]" % (self.parent_job.uid,
+                            self.parent_job.package, self.arch, file_string)
                 self.status = 'done'
 
     def dl_callback(self, status, cb_data):
         url = cb_data
         if status == 'done':
             self.downloads[url] = 3
-            print("%s/%s: Finished downloading %s" % (self.jobid, self.arch, url))
         elif status == 'failed':
             self.downloads[url] = 2
-            print("%s/%s: Failed to download %s" % (self.jobid, self.arch, url))
 
     def get_status(self):
         return self.status


--- buildjob.py DELETED ---


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]