[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/server fileserver.py, NONE, 1.1 CONFIG.py, 1.1.1.1, 1.2 aw_manager.py, 1.1.1.1, 1.2 bm_server.py, 1.1.1.1, 1.2 buildjob.py, 1.1.1.1, 1.2



Author: dcbw

Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv10030/server

Modified Files:
	CONFIG.py aw_manager.py bm_server.py buildjob.py 
Added Files:
	fileserver.py 
Log Message:
First (incomplete) cut of HTTP transfer stuff.



--- NEW FILE fileserver.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#

import SimpleHTTPServer
import SocketServer
import threading
import os
import urllib
import posixpath
import CONFIG

BaseRequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
BaseHttpServer = SocketServer.TCPServer

class BMHttpRequestHandler(BaseRequestHandler):

    def __init__(self, request, client_address, server):
        self._server = server
        BaseRequestHandler.__init__(self, request, client_address, server)

    def list_directory(self, path):
        self.send_error(404, "No permission to list directory")

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        This code is lifted from SimpleHTTPRequestHandler so that we can
        make sure the request is always based in our srpm_http_dir.
        """
        path = posixpath.normpath(urllib.unquote(path))
        words = path.split('/')
        words = filter(None, words)
        path = CONFIG.get('srpm_http_dir')
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)
        return path

    def do_GET(self):
        BaseRequestHandler.do_GET(self)
#        try:
#            BaseRequestHandler.do_GET(self)
#        except Exception, e:
#            # We get an exception if the client drops the transfer
#            pass

class BMThreadingHttpServer(SocketServer.ThreadingTCPServer):

    def __init__(self, server_address, RequestHandlerClass):
        self.protocol_version = "HTTP/1.0"    # Don't want keepalive
        self.allow_reuse_address = 1
        os.chdir(CONFIG.get('srpm_http_dir'))
        BaseHttpServer.__init__(self, server_address, RequestHandlerClass)


class BMHttpServerThread(threading.Thread):

    def __init__(self, address_tuple):
        self._server = BMThreadingHttpServer(address_tuple, BMHttpRequestHandler)
        threading.Thread.__init__(self)

    def run(self):
        self._server.serve_forever()


Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- CONFIG.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ CONFIG.py	4 Jun 2005 23:14:51 -0000	1.2
@@ -13,6 +13,8 @@
 config_opts['redhat_internal_cvs'] = 1
 config_opts['log_url'] = "http://foo.foo.org/logs/";
 
+config_opts['srpm_http_dir'] = "/tmp/srpm_stage_dir"
+
 # This option disables pulling from CVS.  Allowing jobs to be submitted
 # as unknown SRPMs from random people may be a security risk, so don't
 # do this unless you really really want to.


Index: aw_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/aw_manager.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- aw_manager.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ aw_manager.py	4 Jun 2005 23:14:51 -0000	1.2
@@ -32,21 +32,20 @@
 class ArchWelderJob:
     """ Tracks a single build instance for a single arch on an ArchWelder """
 
-    def __init__(self, parent_job, awi, server, srpm, target, mydir, arch):
+    def __init__(self, awi, parent_job, server, target, arch, srpm_url):
         self.parent_job = parent_job
         self.awi = awi
         self.jobid = None
         self.status = None
-        self.srpm = srpm
         self.target = target
-        self.mydir = mydir
         self.arch = arch
+        self.srpm_url = srpm_url
         self._server = server
         self.starttime = None
 
     def start(self):
         try:
-            self.jobid = self._server.start(self.srpm, self.target, self.mydir, self.arch)
+            self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
         except Exception, e:
             print "Error starting job on host %s\n\t---error: %s" % (self.awi.address(), e)
             return False
@@ -103,8 +102,8 @@
     def address(self):
         return self._address
     
-    def new_job(self, parent_job, srpm, target, mydir):
-        return ArchWelderJob(parent_job, self, self._server, srpm, target, mydir, self._arch)
+    def new_job(self, parent_job, target, srpm_url):
+        return ArchWelderJob(self, parent_job, self._server, target, self._arch, srpm_url)
 
     def track_job(self, job):
         self._jobs.append(job)
@@ -184,7 +183,7 @@
             server = xmlrpclib.Server(address)
             try:
                 arches = server.supported_arches()
-            except Exception, e:
+            except socket.error, e:
                 pass
             if arches:
                 arches.append('noarch')
@@ -194,8 +193,6 @@
                         self.running_aw[a] = []
                     awi = ArchWelderInstance(self, address, a)
                     self.running_aw[a].append(awi)
-            else:
-                self.possible_aw.remove(address)
             del server
 
     def process(self):
@@ -211,12 +208,12 @@
         if job:
             job.awi.track_job(job)
 
-    def new_job_on_arch(self, parent_job, arch, srpm, target, mydir):
+    def new_job_on_arch(self, parent_job, target, arch, srpm_url):
         """ Create a job on a free builder for this arch """
 
         if self.running_aw.has_key(arch):
             for aw in self.running_aw[arch]:
                 if aw.available():
-                    return aw.new_job(parent_job, srpm, target, mydir)
+                    return aw.new_job(parent_job, target, srpm_url)
         return None
 


Index: bm_server.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/bm_server.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- bm_server.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ bm_server.py	4 Jun 2005 23:14:51 -0000	1.2
@@ -19,6 +19,7 @@
 import time
 import CONFIG
 import socket
+import os
 import SimpleXMLRPCServer
 import xmlrpclib
 from buildjob import BuildJob
@@ -30,6 +31,7 @@
 from buildmaster import BuildMaster
 from buildmaster import ensure_build_db_tables
 from aw_manager import ArchWelderManager
+from fileserver import BMHttpServerThread
 
 def email_result(username, cvs_tag, resultstring, subject=None):
     """send 'resultstring' to username"""
@@ -176,6 +178,11 @@
     xmlrpc_bm = XMLRPCBuildMaster(awm)
     bm_server = MyXMLRPCServer((CONFIG.get('hostname'), 8887))
     bm_server.register_instance(xmlrpc_bm)
+
+    # SRPM fileserver
+    fileserver = BMHttpServerThread((CONFIG.get('hostname'), 8886))
+    fileserver.start()
+
     print "BuildMaster accepting requests on %s:8887.\n" % CONFIG.get('hostname')
     try:
         bm_server.serve_forever()
@@ -184,3 +191,7 @@
         print "Shutting down..."
         bm.stop()
 
+    print "Done."
+    os._exit(0)
+
+


Index: buildjob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildjob.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- buildjob.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ buildjob.py	4 Jun 2005 23:14:51 -0000	1.2
@@ -69,6 +69,7 @@
     """ Controller object for building 1 SRPM on multiple arches """
 
     def __init__(self, uid, username, package, cvs_tag, target, arch_welder_manager):
+        self.curstage = 'initialize'
         self.awm = arch_welder_manager
         self.uid = uid
         self.username = username
@@ -80,17 +81,14 @@
         self.buildarches = []
         self.sub_jobs = {}
         self.failed = False
-
-        # If we are building and SRPM, cvs_tag will be the SRPM file
-        if CONFIG.get('use_srpm_not_cvs') == True:
-            self.srpmpath = cvs_tag
-            # SRPM builds short-circut to the prep stage (which is right after make_srpm)
+        self.no_cvs = CONFIG.get('use_srpm_not_cvs')
+        self.cvs_tag = cvs_tag
+        self.stage_dir = None
+        self.srpm_path = None
+        self.srpm_http_path = None
+        # Deal with straight SRPM builds
+        if self.no_cvs and self.curstage is 'initialize':
             self.curstage = 'make_srpm'
-            self.cvs_tag = None
-        else:
-            self.srpmpath = None
-            self.curstage = 'initialize'
-            self.cvs_tag = cvs_tag
 
     def get_cur_stage(self):
         return self.curstage
@@ -135,19 +133,21 @@
         return archs
 
         
-    def _make_stage_dir(self):
-        self.pkgsubdir = '%s-%d/%s-%s' % (self.name, self.starttime, self.ver, self.release)
-        self.target_and_pkg_subdir = '%s/%s' % (self.target, self.pkgsubdir)
-        self.stage_dir = os.path.join(self.stages_root, self.curstage, self.target_and_pkg_subdir)
-        if not os.path.exists(self.stage_dir):
-            os.makedirs(self.stage_dir)
-        return self.stage_dir
+    def _make_stage_dir(self, rootdir):
+        # The dir will look like this:
+        # /builder/devel/finished/foo-3463467347734/1.1.0-23
+        pkgsubdir = '%s-%d/%s-%s' % (self.name, self.starttime, self.ver, self.release)
+        stage_dir = os.path.join(rootdir, self.target, self.curstage, pkgsubdir)
+        if not os.path.exists(stage_dir):
+            os.makedirs(stage_dir)
+        return stage_dir
 
         
     def _checkout(self):
         self.curstage = 'checkout'
-        self.tmpdir = tempfile.mkdtemp(prefix=self.cvs_tag, dir=CONFIG.get('tmpdir'))
-        os.chdir(self.tmpdir)
+        dir_prefix = self.cvs_tag + "-"
+        self.checkout_tmpdir = tempfile.mkdtemp(prefix=dir_prefix, dir=CONFIG.get('tmpdir'))
+        os.chdir(self.checkout_tmpdir)
         cmd = '%s co -r %s %s' % (CONFIG.get('cvs_cmd'), self.cvs_tag, self.package)
         debugprint("%d: Running %s" % (self.uid, cmd))
         s, o = commands.getstatusoutput(cmd)
@@ -157,10 +157,11 @@
             self.email_result(resultstring=msg, subject=subj)
             self.curstage = 'finished'
             self.failed = True
+            shutil.rmtree(self.checkout_tmpdir, True)
             return
 
         if CONFIG.get('redhat_internal_cvs') == 1:
-            os.chdir(os.path.join(self.tmpdir, self.package))
+            os.chdir(os.path.join(self.checkout_tmpdir, self.package))
             cmd = '%s co common' % CONFIG.get('cvs_cmd')
             debugprint("%d: Running %s" % (self.uid, cmd))
             s, o = commands.getstatusoutput(cmd)
@@ -170,18 +171,20 @@
                 self.email_result(resultstring=msg, subject=subj)
                 self.curstage = 'finished'
                 self.failed = True
+                shutil.rmtree(self.checkout_tmpdir, True)
                 return
 
     def _make_srpm(self):
         self.curstage = 'make_srpm'
-        self.srpmpath = None
-        packagedir = os.path.join(self.tmpdir, self.package)
+        self.srpm_path = None
+        packagedir = os.path.join(self.checkout_tmpdir, self.package)
         if not os.path.exists(packagedir):
             subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
             msg = "could not find path %s for %s." % (packagedir, self.cvs_tag)
             self.email_result(resultstring=msg, subject=subj)
             self.curstage = 'finished'
             self.failed = True
+            shutil.rmtree(self.checkout_tmpdir, True)
             return
 
         if CONFIG.get('redhat_internal_cvs') == 1:
@@ -199,6 +202,7 @@
             self.email_result(resultstring=msg, subject=subj)
             self.curstage = 'finished'
             self.failed = True
+            shutil.rmtree(self.checkout_tmpdir, True)
             return
         
         srpmpath = None
@@ -214,53 +218,62 @@
             self.email_result(resultstring=msg, subject=subj)
             self.curstage = 'finished'
             self.failed = True
+            shutil.rmtree(self.checkout_tmpdir, True)
             return
-        self.srpmpath = srpmpath
+        self.srpm_path = srpmpath
 
     def _prep(self):
         self.curstage = 'prep'
         ts = rpmUtils.transaction.initReadOnlyTransaction()
-        hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpmpath)
+        hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpm_path)
         self.name = hdr['name']
         self.ver = hdr['version']
         self.release = hdr['release']
         self.buildarches = self.arch_handling(hdr)
+        del hdr
+        del ts
+
         if len(self.buildarches) == 0:
             self.failed = True
             self.curstage = 'finished'
-            del hdr
-            del ts
-            return;
-        self._make_stage_dir()
+            return
+        self.stage_dir = self._make_stage_dir(self.stages_root)
         self._createrepo(stage='needsign')
-        
+
         for arch in self.buildarches:
             thisdir = os.path.join(self.stage_dir, arch)
             if not os.path.exists(thisdir):
                 os.makedirs(thisdir)
         
-        srpm = os.path.basename(self.srpmpath)
+        srpm = os.path.basename(self.srpm_path)
         srpm_in_dir = os.path.join(self.stage_dir, srpm)
         if os.path.exists(srpm_in_dir):
             os.unlink(srpm_in_dir)
-        shutil.move(self.srpmpath, self.stage_dir)
-        self.srpmpath = srpm_in_dir
-        del hdr
-        del ts
-        
+        shutil.copy(self.srpm_path, self.stage_dir)
+
+        # Must also copy it to where the build client can get it over HTTP
+        http_pkg_path = self._make_stage_dir(CONFIG.get('srpm_http_dir'))
+        self.srpm_http_path = os.path.join(http_pkg_path, srpm)
+        shutil.copy(self.srpm_path, self.srpm_http_path)
+        self.srpm_path = srpm_in_dir
+
+        # Remove CVS checkout and make_srpm dirs
+        shutil.rmtree(self.checkout_tmpdir, True)
+
     def process(self):
         # Advance to next stage based on current stage
-        if self.curstage == 'initialize':
+        oldstage = self.curstage
+        if oldstage == 'initialize':
             self._checkout()
-        elif self.curstage == 'checkout':
+        elif oldstage == 'checkout':
             self._make_srpm()
-        elif self.curstage == 'make_srpm':
+        elif oldstage == 'make_srpm':
             self._prep()
-        elif self.curstage == 'prep' or self.curstage == 'building':
+        elif oldstage == 'prep' or oldstage == 'building':
             self._monitor()
-        elif self.curstage == 'finished':
+        elif oldstage == 'finished':
             self._cleanup()
-        elif self.curstage == 'cleanup':
+        elif oldstage == 'cleanup':
             if self.failed:
                 self._failed()
             else:
@@ -276,7 +289,11 @@
     def _start_unspawned_builds(self):
         for arch in self.buildarches:
             if not self.sub_jobs.has_key(arch):
-                job = self.awm.new_job_on_arch(self, arch, self.srpmpath, self.target, self.stage_dir)
+                # Construct SPRM URL
+                srpm_http_base = self.srpm_http_path[len(CONFIG.get('srpm_http_dir')):]
+                srpm_url = "http://"; + CONFIG.get('hostname') + ":8886" + srpm_http_base
+                print "SRPM URL: " + srpm_url
+                job = self.awm.new_job_on_arch(self, self.target, arch, srpm_url)
                 if job:
                     if job.start() == True:
                         self.awm.track_job(job)
@@ -314,51 +331,57 @@
                     job.die()
             
     def _failed(self):
+        old_stage = self.curstage
         self.curstage = 'failed'
-        old_stage = self.stage_dir
-        dest = self._make_stage_dir()
-        if os.path.exists(dest):
-            shutil.rmtree(dest, ignore_errors=True)
-        shutil.move(old_stage, dest)
-        for job in self.sub_jobs.values():
-            buildroot = 'fedora-%s-%s-core' % (self.target, job.arch)
-            stage_arch = os.path.join(self.stage_dir, job.arch)
-            build_log = '%s/mach/%s/%s-%s-%s/rpm.log' % (CONFIG.get('tmpdir'), buildroot,
-                                            self.name, self.ver, self.release)
-            if os.path.exists(build_log):
-                bl = open(build_log, 'r')
-            else:
-                bl = None
-            if not os.path.exists(stage_arch):
-                os.makedirs(stage_arch)
-            fn = '%s/%s-%s-%s.failure.log' % (stage_arch, self.name, self.ver, self.release)
-            print "%d: Logfile (%s) is: %s" % (self.uid, job.arch, fn)
-            logfile = open(fn, 'w')
+        if old_stage is not 'initialize' and old_stage is not 'checkout' and old_stage is not 'make_srpm':
+            old_stage_dir = self.stage_dir
+            dest = self._make_stage_dir(self.stages_root)
+            if os.path.exists(dest):
+                shutil.rmtree(dest, ignore_errors=True)
+            shutil.move(old_stage_dir, dest)
+            for job in self.sub_jobs.values():
+                buildroot = 'fedora-%s-%s-core' % (self.target, job.arch)
+                stage_arch = os.path.join(self.stage_dir, job.arch)
+                build_log = '%s/mach/%s/%s-%s-%s/rpm.log' % (CONFIG.get('tmpdir'), buildroot,
+                                                self.name, self.ver, self.release)
+                if os.path.exists(build_log):
+                    bl = open(build_log, 'r')
+                else:
+                    bl = None
+                if not os.path.exists(stage_arch):
+                    os.makedirs(stage_arch)
+                fn = '%s/%s-%s-%s.failure.log' % (stage_arch, self.name, self.ver, self.release)
+                print "%d: Logfile (%s) is: %s" % (self.uid, job.arch, fn)
+                logfile = open(fn, 'w')
 
-            if job.status == 'killed':
-                lines = ['Build process terminated due to failure on another arch\n']
-            else:
-                lines = job.logs()
+                if job.status == 'killed':
+                    lines = ['Build process terminated due to failure on another arch\n']
+                else:
+                    lines = job.logs()
 
-            for line in lines:
-                logfile.write(line)
-            if bl:
-                for line in bl.readlines():
+                for line in lines:
                     logfile.write(line)
-                bl.close()
-            logfile.close()
-            
-        # markup status file
-        resultstring = """
-%s: Build of %s on %s failed to complete on one or more archs. Please see logs at:
-%s/%s/%s""" % (self.uid, self.name, self.target, CONFIG.get('log_url'), self.target, self.name)
+                if bl:
+                    for line in bl.readlines():
+                        logfile.write(line)
+                    bl.close()
+                logfile.close()
+                
+            # markup status file
+            resultstring = """
+    %s: Build of %s on %s failed to complete on one or more archs. Please see logs at:
+    %s/%s/%s""" % (self.uid, self.name, self.target, CONFIG.get('log_url'), self.target, self.name)
+        else:
+            resultstring = """
+    %s: Build of %s (%s) on %s failed to complete on one or more archs.
+""" % (self.uid, self.package, self.cvs_tag, self.target)
         self.email_result(resultstring)
         return False
         
     def _succeeded(self):
         self.curstage = 'needsign'
         old_stage = self.stage_dir
-        dest = self._make_stage_dir()
+        dest = self._make_stage_dir(self.stages_root)
         if os.path.exists(dest):
             shutil.rmtree(dest, ignore_errors=True)
         shutil.move(old_stage, dest)


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]