[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/server BuildMaster.py, NONE, 1.1 BuildJob.py, 1.2, 1.3 CONFIG.py, 1.8, 1.9 UserInterface.py, 1.1, 1.2 buildserver.py, 1.6, 1.7 client_manager.py, 1.8, 1.9 buildmaster.py, 1.6, NONE



Author: dcbw

Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv23432/server

Modified Files:
	BuildJob.py CONFIG.py UserInterface.py buildserver.py 
	client_manager.py 
Added Files:
	BuildMaster.py 
Removed Files:
	buildmaster.py 
Log Message:
2005-06-14  Dan Williams <dcbw redhat com>

    * client/buildclient.py
        - Write mock output to a log on failure

    * server/buildmaster.py -> server/BuildMaster.py

    * server/BuildJob.py
      server/UserInterface.py
      server/buildserver.py
        - Fix for buildmaster.py -> BuildMaster.py

    * server/BuildJob.py
      server/BuildMaster.py
        - BuildMaster object now has a 'createrepo' method, which BuildJobs
            call when they need to update the repo.  BuildMaster.createrepo() is
            locked so that we never run two createrepos at the same time

    * server/CONFIG.py
        - Clarify some options

    * server/UserInterface.py
        - Remove list_waiting_jobs and list_building_jobs, we now have a more
            flexible "list_jobs" interface that will be extended to allow more
            search parameters
        - In update_clients(), return new clients that we've found

    * server/client_manager.py
        - Print out clients we find when we start up
        - In update_clients(), return new clients that we've found

    * utils/package-builder.py
        - Implement most of remaining functionality:
            - hook up enqueue and enqueue_srpm commands
            - Add a list_own_jobs command
            - grab user's email address from certificate or ~/.package-builder
        - Allow non-SSL connections to build server




--- NEW FILE BuildMaster.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal


import time
import CONFIG
import BuildJob
import sqlite
import threading
import os
import commands


def ensure_build_db_tables(dbcx):
    """ Central routine to create the database table structure """

    curs = dbcx.cursor()
    try:
        curs.execute('SELECT * FROM jobs')
        dbcx.commit()
    except Exception, e:
        # If DB wasn't created, try to create it
        curs.execute('CREATE TABLE jobs (uid INTEGER PRIMARY KEY, ' \
                'username VARCHAR(20), package VARCHAR(50), ' \
                'cvs_tag VARCHAR(255), target VARCHAR(20), ' \
                'buildreq VARCHAR(75), time_submitted BIGINT, ' \
                'status VARCHAR(15))')
        dbcx.commit()


class BuildMaster(threading.Thread):
    def __init__(self, hostname, client_manager):
        self.bcm = client_manager
        self.hostname = hostname
        self.building_jobs = []
        self.should_stop = False
        self.dbcx = sqlite.connect("jobdb", encoding="utf-8", timeout=2)
        self.curs = self.dbcx.cursor()
        self.createrepo_lock = threading.Lock()
        ensure_build_db_tables(self.dbcx)
        threading.Thread.__init__(self)

    def __del__(self):
        self.dbcx.close()

    def stop(self):
        self.should_stop = True

    def set_job_status(self, job):
        status = job.get_cur_stage()
        job_uid = job.get_uid()
        self.curs.execute('UPDATE jobs SET status="%s" WHERE uid=%d' \
                % (status, job_uid))

    def getClientManager(self):
        return self.bcm

    def createrepo(self, repodir):
        """ We need to lock calls to createrepo so they don't get run at the same time """
        if not os.path.exists(repodir):
            os.makedirs(repodir)
        self.createrepo_lock.acquire()
        s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
        self.createrepo_lock.release()
        if s != 0:
            print "createrepo failed with exit status %d" % s

    def run(self):
        while self.should_stop == False:
            # Update all build clients and known jobs
            self.bcm.process()

            # Allow each job some processing time
            for job in self.building_jobs:
                self.set_job_status(job)
                if job.get_cur_stage() == 'failed' or job.get_cur_stage() == 'needsign':
                    print "%s (%s): Job finished." % (job.get_uid(), job.package)
                    self.building_jobs.remove(job)

            # Grab one waiting job from database and start it
            self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
                    ' FROM jobs WHERE status="waiting"')
            self.dbcx.commit()
            item = self.curs.fetchone()
            if item:
                print "%s (%s): Starting tag '%s' on target '%s'" % (item['uid'], \
                        item['package'], item['cvs_tag'], item['target'])
                job = BuildJob.BuildJob(item['uid'], item['username'], item['package'],
                        item['cvs_tag'], item['target'], self, self.hostname)
                self.building_jobs.append(job)
                job.start()

            time.sleep(5)



Index: BuildJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildJob.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- BuildJob.py	12 Jun 2005 08:23:43 -0000	1.2
+++ BuildJob.py	15 Jun 2005 04:10:07 -0000	1.3
@@ -72,9 +72,9 @@
 class BuildJob(threading.Thread):
     """ Controller object for building 1 SRPM on multiple arches """
 
-    def __init__(self, uid, username, package, cvs_tag, target, client_manager, hostname):
+    def __init__(self, uid, username, package, cvs_tag, target, buildmaster, hostname):
         self.curstage = 'initialize'
-        self.bcm = client_manager
+        self.bm = buildmaster
         self.hostname = hostname
         self.uid = uid
         self.username = username
@@ -363,7 +363,7 @@
                 # Construct SPRM URL
                 srpm_http_base = self.srpm_http_path[len(http_dir):]
                 srpm_url = "https://"; + self.hostname + ":8886" + srpm_http_base
-                job = self.bcm.start_arch_job(self, self.target, arch, srpm_url)
+                job = self.bm.bcm.start_arch_job(self, self.target, arch, srpm_url)
                 if job:
                     self.sub_jobs[arch] = job
                     log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
@@ -461,13 +461,7 @@
     def _createrepo(self):
         # createrepo on the needsign tree for new changes
         repodir = os.path.join(CONFIG.get('repo_dir'), self.target)
-        debugprint("%d: updating repodir %s" % (self.uid, repodir))
-        if not os.path.exists(repodir):
-            os.makedirs(repodir)
-        s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
-        if s != 0:
-            print "Createrepo failed!!!"
-            #self.curstage = 'failed'
-            #raise PrepError(5, 'Error generating repodata for %s: %s' % (repodir, o))
+        self.bm.createrepo(repodir)
+        debugprint("%d: updated repodir %s" % (self.uid, repodir))
 
 


Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- CONFIG.py	14 Jun 2005 13:37:42 -0000	1.8
+++ CONFIG.py	15 Jun 2005 04:10:07 -0000	1.9
@@ -1,4 +1,4 @@
-# Configuration file for buildmaster.py
+# Configuration file for build system server
 
 config_opts = {}
 config_opts['email_to_domain'] = "redhat.com"
@@ -26,9 +26,14 @@
 # MUST be full path to cert
 config_opts['server_cert'] = SERVER_BASE_DIR + "/certs/server_cert.pem"
 config_opts['server_key'] = SERVER_BASE_DIR + "/certs/server_key.pem"
+
+# CA cert that signs build client certificates
 config_opts['ca_cert'] = SERVER_BASE_DIR + "/certs/ca_cert.pem"
+
+# CA cert that signed build job submitters' certificates
 config_opts['ui_ca_cert'] = SERVER_BASE_DIR + "/certs/fedora-upload-ca.pem"
 
+
 # server_work_dir
 #   - Where logs and finished RPMs are stored
 config_opts['server_work_dir'] = "/rpmbuild"


Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- UserInterface.py	14 Jun 2005 13:37:42 -0000	1.1
+++ UserInterface.py	15 Jun 2005 04:10:08 -0000	1.2
@@ -22,7 +22,7 @@
 import time
 import sys
 import os
-from buildmaster import ensure_build_db_tables
+import BuildMaster
 
 
 def email_result(email, cvs_tag, resultstring, subject=None):
@@ -51,7 +51,7 @@
         self.bcm = client_manager
         self.dbcx = sqlite.connect("jobdb", encoding="utf-8", timeout=2)
         self.curs = self.dbcx.cursor()
-        ensure_build_db_tables(self.dbcx)
+        BuildMaster.ensure_build_db_tables(self.dbcx)
 
 
     def __del__(self):
@@ -128,21 +128,22 @@
         return (0, "Success: package has been queued.", jobid)
 
 
-    def list_waiting_jobs(self):
-        self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
-                ' FROM jobs WHERE status="waiting"')
-        self.dbcx.commit()
-        data = self.curs.fetchall()
-        job_list = []
-        for row in data:
-            tempX = [ item for item in row]
-            job_list.append(tempX)
-        return job_list
-
-
-    def list_building_jobs(self):
-        self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
-                ' FROM jobs WHERE status="building"')
+    def list_jobs(self, args_dict):
+        sql = 'SELECT uid, username, package, cvs_tag, target, status FROM jobs'
+        search = ''
+        first = True
+
+        if args_dict.has_key('email') and args_dict['email']:
+            if first:
+                search = search +  " WHERE "
+            else:
+                search = search + "AND "
+            search = search + 'username LIKE "%%%s%%" ' % args_dict['email']
+            first = False
+
+        if len(search):
+            sql = sql + search
+        self.curs.execute(sql)
         self.dbcx.commit()
         data = self.curs.fetchall()
         job_list = []
@@ -154,11 +155,8 @@
 
     def update_clients(self):
         reload(CONFIG)
-        print "-----------------------------------------------------"
-        print " Looking for Build Clients..."
-        self.bcm.update_clients()
-        print "-----------------------------------------------------\n"
-        return (0, "Success.")
+        client_list = self.bcm.update_clients()
+        return (0, "Success.", client_list)
 
 
 
@@ -179,12 +177,8 @@
         return UserInterface.enqueue_srpm(self, user.email, package, cvs_tag, target, buildreq)
 
 
-    def list_waiting_jobs(self, user):
-        return UserInterface.list_waiting_jobs(self)
-
-
-    def list_building_jobs(self, user):
-        return UserInterface.list_building_jobs(self)
+    def list_jobs(self, user, args_dict):
+        return UserInterface.list_jobs(self, args_dict)
 
 
     def update_clients(self, user):
@@ -198,22 +192,3 @@
     Allow all operations, NULL authentication
     """
 
-    def enqueue(self, email, package, cvs_tag, target, buildreq=None):
-        return UserInterface.enqueue(self, email, package, cvs_tag, target, buildreq)
-
-
-    def enqueue_srpm(self, email, package, srpm_file, target, buildreq=None):
-        return UserInterface.enqueue_srpm(self, email, package, cvs_tag, target, buildreq)
-
-
-    def list_waiting_jobs(self):
-        return UserInterface.list_waiting_jobs(self)
-
-
-    def list_building_jobs(self):
-        return UserInterface.list_building_jobs(self)
-
-
-    def update_clients(self):
-        return UserInterface.update_clients(self)
-


Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- buildserver.py	14 Jun 2005 13:37:42 -0000	1.6
+++ buildserver.py	15 Jun 2005 04:10:08 -0000	1.7
@@ -19,7 +19,7 @@
 import CONFIG
 import sys
 import os
-from buildmaster import BuildMaster
+import BuildMaster
 from client_manager import BuildClientManager
 import SimpleHTTPSServer
 import SimpleSSLXMLRPCServer
@@ -75,7 +75,7 @@
     bcm = BuildClientManager()
 
     # Create the BuildMaster thread
-    bm = BuildMaster(hostname, bcm)
+    bm = BuildMaster.BuildMaster(hostname, bcm)
     bm.start()
 
     # SSL certificate and key filenames
@@ -106,7 +106,7 @@
     srpm_server = SimpleHTTPSServer.SimpleHTTPSServer(srpm_server_certs, (hostname, 8886), http_dir)
     srpm_server.start()
 
-    print "BuildMaster accepting requests on %s:8887.\n" % hostname
+    print "Build Server accepting requests on %s:8887.\n" % hostname
     try:
         bm_server.serve_forever()
     except KeyboardInterrupt:


Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- client_manager.py	15 Jun 2005 01:41:12 -0000	1.8
+++ client_manager.py	15 Jun 2005 04:10:08 -0000	1.9
@@ -285,19 +285,37 @@
         else:
             return True
 
+    def to_dict(self):
+        client_dict = {}
+        client_dict['address'] = self._address
+        client_dict['arches'] = self._arches
+        if self._cur_job:
+            client_dict['status'] = 'building'
+        else:
+            client_dict['status'] = 'idle'
+        return client_dict
 
 class BuildClientManager:
     def __init__(self):
         # List of addresses of possible builders
         self.possible_clients = CONFIG.get('builders')
         self.running_clients = []
+        client_list = self.update_clients()
+
+        # Print out client list when starting up
+        print "\nBuild Clients:"
+        print "-" * 60
+        for client in client_list:
+            string = "  " + client['address']
+            string = string + " " * (40 - len(client['address']))
+            for arch in client['arches']:
+                string = string + arch + " "
+            print string
+        print ""
 
-        print "-----------------------------------------------------"
-        print " Looking for BuildClients..."
-        self.update_clients()
-        print "-----------------------------------------------------\n"
 
     def update_clients(self):
+        client_list = []
         for address in self.possible_clients:
             # If the address is already in our running_clients list, skip it
             skip = False
@@ -311,11 +329,13 @@
             # list if we can
             client = BuildClient(self, address)
             if client.valid():
-                print "   New Client: '%s' [%s]" % (address, string.join(client.arches()))
+                client_list.append(client.to_dict())
                 self.running_clients.append(client)
             else:
                 del client
 
+        return client_list
+
     def process(self):
         """ Allow each BuildClient to update its status and do some processing """
         for client in self.running_clients:


--- buildmaster.py DELETED ---


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]