[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/client srpm_download.py, NONE, 1.1 archwelder.py, 1.1.1.1, 1.2 archwelder_config.py, 1.1.1.1, 1.2 fileserver.py, 1.1.1.1, 1.2



Author: dcbw

Update of /cvs/fedora/extras-buildsys/client
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv10030/client

Modified Files:
	archwelder.py archwelder_config.py fileserver.py 
Added Files:
	srpm_download.py 
Log Message:
First (incomplete) cut of HTTP transfer stuff.



--- NEW FILE srpm_download.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#

import threading
import urlgrabber
import os
import shutil

def log(stuff=''):
    print stuff


class SRPMDownloadThread(threading.Thread):

    def __init__(self, awm):
        self._awm = awm
        threading.Thread.__init__(self)

    def run(self):
        if self._awm.status() is not 'downloading':
            return
        srpm_url = self._awm.srpm_url()
        srpm_path = self._awm.srpm_path()
        srpm_dirname = os.path.dirname(srpm_path)
        success = False
        if srpm_url and srpm_path:
            os.makedirs(srpm_dirname)
            result = urlgrabber.urlgrab(srpm_url, srpm_path)

#            try:
#                # Get the SRPM from the build server
#                result = urlgrabber.urlgrab(srpm_url, srpm_path)
#            except Exception, e:
#                pass
#            else:
            if result:
                success = True

        if success:
            self._awm.set_status('downloaded')
            log("%s: Finished downloading %s" % (self._awm.uniqid(), srpm_url))
        else:
            # Don't set failed status if the job was cancelled while downloading
            # since status is set in the cancellation call
            if not self._awm.is_done_status():
                self._awm.set_status('failed')
                log("%s: Failed to download %s" % (self._awm.uniqid(), srpm_url))
            shutil.rmtree(srpm_dirname, ignore_errors=True)


Index: archwelder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/client/archwelder.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- archwelder.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ archwelder.py	4 Jun 2005 23:14:50 -0000	1.2
@@ -29,8 +29,10 @@
 import sys
 import string
 import time
+import urllib
 from archwelder_config import CONFIG
 from fileserver import HttpServerThread
+from srpm_download import SRPMDownloadThread
 
 DEBUG = False
 def debugprint(stuff=''):
@@ -40,104 +42,196 @@
 def log(stuff=''):
     print stuff
 
-def is_done_status(status):
-    if status == 'done' or status == 'killed' or status == 'failed':
-        return True
-    return False
+
+def get_base_filename_from_url(url):
+    """ Safely unquotes a URL and gets the base file name from it.
+        We're not using urlparse here because it doesn't un-escape stuff """
+
+    unquoted = url
+    last_unquoted = None
+    count = 5
+    
+    # Keep unquoting the string until the last two unquote operations
+    # produce the same string
+    while (unquoted != last_unquoted) and (count > 0):
+        last_unquoted = unquoted
+        unquoted = urllib.unquote_plus(unquoted)
+        count = count - 1
+
+    # If after 5 iterations of unquoting, the strings still aren't the same,
+    # something is wrong.
+    if (count == 0) and (unquoted != last_unquoted):
+        return None
+
+    # Try to grab the filename off the end of the URL
+    index = url.rfind('/')
+    if index is -1:
+        return None
+    filename = url[index+1:]
+
+    if not filename.endswith('.src.rpm'):
+        return None
+
+    # FIXME: what other validation can we do here?
+    for c in filename:
+        # For now, legal characters are '_-.' plus alphanumeric
+        if (c == '_') or (c == '-') or (c == '.') or c.isalnum():
+            pass
+        else:
+            return None
+
+    return filename
+
 
 class ArchWelderMach:
     """puts things together for an arch - baseclass for handling builds for 
        other arches"""
-    def __init__(self, uniqid, srpm, target, mydir):
+    def __init__(self, uniqid, target, srpm_url):
+        self._uniqid = uniqid
         self._status = 'init'
-        self.target = target
-        self.mydir = mydir
         self._output = []
-        self.srpm = srpm
-        self.uniqid = uniqid
+        self._pobj = None
+        self._target = target
+        self._srpm_url = srpm_url
+        srpm_filename = get_base_filename_from_url(srpm_url)
+        if not srpm_filename:
+            self._status = 'failed'
+            self._srpm_path = None
+        else:
+            self._srpm_path = os.path.join(CONFIG('srpm_stage_dir'), self._uniqid, srpm_filename)
 
     def die(self, sig=15):
-        try:
-            os.kill(self.pobj.pid, sig)
-        except OSError, e:
-            print "Couldn't kill process %d: %s" % (self.pobj.pid, e)
+        if self._pobj and self._pobj.pid:  # Can't kill the package download from build server
+            try:
+                os.kill(self._pobj.pid, sig)
+            except OSError, e:
+                print "Couldn't kill process %d: %s" % (self._pobj.pid, e)
         self._status = 'killed'
         return True
 
+    def _get_srpm(self):
+        self._status = 'downloading'
+        dl_thread = SRPMDownloadThread(self)
+        dl_thread.start()
+
+    def _unlock(self):
+        print "%s: starting step 'unlocking'" % self._uniqid
+        cmd = '%s %s -r %s unlock' % (self.arch_command,
+                            CONFIG('mach_cmd'), self.buildroot)
+        self._pobj = popen2.Popen4(cmd=cmd)
+        self._status = 'unlocking'
+
+    def _clean(self):
+        print "%s: starting step 'cleaning'" % self._uniqid
+        cmd = '%s %s -r %s clean' % (self.arch_command,
+                            CONFIG('mach_cmd'), self.buildroot)
+        self._pobj = popen2.Popen4(cmd=cmd)
+        self._status = 'cleaning'
+        
+    
+    def _prep(self):
+        print "%s: starting step 'prepping'" % self._uniqid
+        cmd = '%s /%s -r %s setup prep' % (self.arch_command,
+                            CONFIG('mach_cmd'), self.buildroot)
+        self._pobj = popen2.Popen4(cmd=cmd)
+        self._status = 'prepping'
         
     def _build(self):
-        print "%s: starting step 'building'" % self.uniqid
-        resultdir = self.mydir + '/' + self.buildarch
-        cmd = '%s %s -r %s --resultdir=%s %s' % (self.arch_command,
-                            CONFIG('builder_cmd'), self.buildroot, 
-                            resultdir, self.srpm)
-        self.pobj = popen2.Popen4(cmd=cmd)
+        print "%s: starting step 'building'" % self._uniqid
+#??        os.chdir(self._mydir + '/' + self.buildarch)
+        cmd = '%s %s -r %s -c rebuild %s' % (self.arch_command,
+                            CONFIG('mach_cmd'), self.buildroot, self._srpm_path)
+        self._pobj = popen2.Popen4(cmd=cmd)
         self._status = 'building'
 
     def start(self):
         # check for existence of srpm before going on
-        pass
+        self._get_srpm()
 
     def process(self):
-        if not is_done_status(self._status):
-            # If we're done with a step, advance to next one
-            exit_status = self.pobj.poll()
-            if exit_status == 0:
-                self._build()
-                if self._status == 'building':
-                    # TODO: Should have 'moving' stage
-                    # where files are copied to the directory
-                    # that fileserver.py looks in.
-                    print "%s: Job done." % self.uniqid
-                    self._status = 'done'
-                else:
-                    print "Bad status %s encountered!" % self._status
-            elif exit_status > 0:
-                print "%s: job failed!\n" % self.uniqid
-                self._status = 'failed'
-            else:
-                # mock process still running
+        if not self.is_done_status():
+            if self._status == 'downloading':
                 pass
+            elif self._status == 'downloaded':
+                self._unlock()
+            else:
+                # If we're done with a step, advance to next one
+                exit_status = self._pobj.poll()
+                if exit_status == 0:
+                    if self._status == 'unlocking':
+                        self._clean()
+                    elif self._status == 'cleaning':
+                        self._prep()
+                    elif self._status == 'prepping':
+                        self._build()
+                    elif self._status == 'building':
+                        # TODO: Should have 'moving' stage
+                        # where files are copied to the directory
+                        # that fileserver.py looks in.
+                        print "%s: Job done." % self.uniqid
+                        self._status = 'done'
+                    else:
+                        print "Bad status %s encountered!" % self._status
+                elif exit_status > 0:
+                    print "%s: job failed!\n" % self._uniqid
+                    self._status = 'failed'
+                else:
+                    # mach process still running
+                    pass
 
     def status(self):
         return self._status
 
+    def set_status(self, status):
+        self._status = status
+
     def logs(self):
-        self._output.extend(self.pobj.fromchild.readlines())
+        if self._pobj:
+            self._output.extend(self._pobj.fromchild.readlines())
         return self._output
-        
+
+    def srpm_url(self):
+        return self._srpm_url
+
+    def srpm_path(self):
+        return self._srpm_path
+
+    def is_done_status(self):
+        if (self._status is 'done') or (self._status is 'killed') or (self._status is 'failed'):
+            return True
+        return False
+
+    def uniqid(self):
+        return self._uniqid
+
 
 class i386Arch(ArchWelderMach):
-    def __init__(self, uniqid, srpm, target, mydir, buildarch='i386'):
-        self.buildroot = '%s-%s-i386-%s' % (CONFIG('distro_name'), target,
-                            CONFIG('repo_name'))
+    def __init__(self, uniqid, target, buildarch, srpm_url):
+        self.buildroot = '%s-%s-i386-%s' % (CONFIG('distro_name'), target, CONFIG('repo_name'))
         self.buildarch = buildarch
         self.arch_command = '/usr/bin/setarch i686'
-        ArchWelderMach.__init__(self, uniqid, srpm, target, mydir)
+        ArchWelderMach.__init__(self, uniqid, target, srpm_url)
 
 class x86_64Arch(ArchWelderMach):
-    def __init__(self, uniqid, srpm, target, mydir, buildarch='x86_64'):
-        self.buildroot = '%s-%s-x86_64-%s' % (CONFIG('distro_name'), target,
-                            CONFIG('repo_name'))
+    def __init__(self, uniqid, target, buildarch, srpm_url):
+        self.buildroot = '%s-%s-x86_64-%s' % (CONFIG('distro_name'), target, CONFIG('repo_name'))
         self.buildarch = buildarch
         self.arch_command = ''
-        ArchWelderMach.__init__(self, uniqid, srpm, target, mydir)
+        ArchWelderMach.__init__(self, uniqid, target, srpm_url)
 
 class PPCArch(ArchWelderMach):
-    def __init__(self, uniqid, srpm, target, mydir, buildarch='ppc'):
-        self.buildroot = '%s-%s-ppc-%s' % (CONFIG('distro_name'), target,
-                            CONFIG('repo_name'))
+    def __init__(self, uniqid, target, buildarch, srpm_url):
+        self.buildroot = '%s-%s-ppc-%s' % (CONFIG('distro_name'), target, CONFIG('repo_name'))
         self.buildarch = buildarch
         self.arch_command = ''
-        ArchWelderMach.__init__(self, uniqid, srpm, target, mydir)
+        ArchWelderMach.__init__(self, uniqid, target, srpm_url)
 
 class PPC64Arch(ArchWelderMach):
-    def __init__(self, uniqid, srpm, target, mydir, buildarch='ppc64'):
-        self.buildroot = '%s-%s-ppc64-%s' % (CONFIG('distro_name'), target,
-                            CONFIG('repo_name'))
+    def __init__(self, uniqid, target, buildarch, srpm_url):
+        self.buildroot = '%s-%s-ppc64-%s' % (CONFIG('distro_name'), target, CONFIG('repo_name'))
         self.buildarch = buildarch
         self.arch_command = ''
-        ArchWelderMach.__init__(self, uniqid, srpm, target, mydir)
+        ArchWelderMach.__init__(self, uniqid, target, srpm_url)
 
 
 # Keep this global scope, used in __main__
@@ -154,7 +248,7 @@
                 'ppc64':PPC64Arch,
                }
 
-def getArchWelder(uniqid, srpm, target, mydir, buildarch, localarches):
+def getArchWelder(uniqid, target, buildarch, srpm_url, localarches):
     """hand it an arch it hands you back the archwelder instance you need"""
     
     if not welder_dict.has_key(buildarch):
@@ -168,7 +262,7 @@
         if buildarch in localarches:
             welder = welder_dict[buildarch]
     
-    awp = welder(uniqid, srpm, target, mydir, buildarch)
+    awp = welder(uniqid, target, buildarch, srpm_url)
     return awp
 
 class XMLRPCArchWelderServer:
@@ -182,46 +276,53 @@
         job = 0
         for (uniqid, awp) in self.ids.iteritems():
             awp.process()
-            if not is_done_status(awp.status()):
+            if not awp.is_done_status():
                 job = uniqid
         self.cur_job = job  # Update current job
 
-    def start(self, srpm, target, stagedir, buildarch):
+    def start(self, target, buildarch, srpm_url):
         if self.cur_job != 0:
             print "Tried to build '%s' when already buiding something" % srpm
             return 0
 
         cur_time = time.time()
-        check = '%d %s %s %s' % (cur_time, srpm, target, buildarch)
+        check = '%d %s %s %s' % (cur_time, target, buildarch, srpm_url)
         sum = sha.new()
         sum.update(check)
         uniqid = sum.hexdigest()
         if target == 'devel':
             target = 'development'
-        awp = getArchWelder(uniqid, srpm, target, stagedir, buildarch, self.localarches)
+        awp = getArchWelder(uniqid, target, buildarch, srpm_url, self.localarches)
         if awp != None:
             self.ids[uniqid] = awp
             awp.start()
-            log("%s: started %s on %s arch %s at time %d" % (uniqid, srpm,
+            log("%s: started %s on %s arch %s at time %d" % (uniqid, srpm_url,
                         target, buildarch, cur_time))
         else:
             log("%s: Failed request for %s on %s UNSUPPORTED arch %s at time %d" %
-                        (uniqid, srpm, target, buildarch, cur_time))
+                        (uniqid, srpm_url, target, buildarch, cur_time))
             uniqid = 0
         self.cur_job = uniqid
         return uniqid
 
     def status(self, uniqid=None):
-        awp = self.ids[uniqid]
+        if not uniqid:
+            uniqid = self.cur_job
+        if not uniqid:
+            return 'idle'
+        try:
+            awp = self.ids[uniqid]
+        except KeyError, e:
+            awp = None
         if not awp:
-            return ''
+            return 'idle'
         return awp.status()
     
-    def die(self, uniqid=None):
+    def die(self, uniqid):
         awp = self.ids[uniqid]
         return awp.die()
     
-    def logs(self, uniqid=None):
+    def logs(self, uniqid):
         awp = self.ids[uniqid]
         result = []
         for line in awp.logs():
@@ -243,12 +344,9 @@
     """ XMLRPC server subclass that turns on SO_REUSEADDR """
 
     def __init__(self, address):
+        self.allow_reuse_address = 1
         SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr=address, logRequests=False)
 
-    def server_bind(self):
-        self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
-        self.socket.bind(self.server_address)
-
 if __name__ == '__main__':
     if len(sys.argv) < 3:
         print "Usage:\n"


Index: archwelder_config.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/client/archwelder_config.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- archwelder_config.py	1 Jun 2005 04:15:01 -0000	1.1.1.1
+++ archwelder_config.py	4 Jun 2005 23:14:50 -0000	1.2
@@ -1,10 +1,18 @@
 # Configuration file for archwelder.py
 
 config_opts = {}
-config_opts['builder_cmd'] = "/usr/bin/mock"
+config_opts['mach_cmd'] = "/usr/bin/mach"
 config_opts['distro_name'] = "fedora"
 config_opts['repo_name'] = "core"
-config_opts['built_rpm_download_basedir'] = "/tmp"
+
+# Where to allow the build server to download built
+# packages from.
+# WARNING: this is world readable!
+config_opts['built_rpm_download_basedir'] = "/tmp/completed"
+
+# Where to put SRPMs downloaded from the build server
+# that we need to build.
+config_opts['srpm_stage_dir'] = "/tmp/to-build"
 
 
 def CONFIG (key):


Index: fileserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/client/fileserver.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- fileserver.py	1 Jun 2005 04:15:00 -0000	1.1.1.1
+++ fileserver.py	4 Jun 2005 23:14:50 -0000	1.2
@@ -12,8 +12,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-# copyright 2005 Duke University
-# written by Seth Vidal
+#
+# Copyright 2005 Dan Williams and Red Hat, Inc.
+#
 
 import SimpleHTTPServer
 import SocketServer
@@ -30,8 +31,29 @@
         self._server = server
         BaseRequestHandler.__init__(self, request, client_address, server)
 
-    def list_directory(self):
-        self.send_error (404, "File not found")
+    def list_directory(self, path):
+        self.send_error(404, "No permission to list directory")
+
+    def translate_path(self, path):
+        """Translate a /-separated PATH to the local filename syntax.
+
+        Components that mean special things to the local file system
+        (e.g. drive or directory names) are ignored.  (XXX They should
+        probably be diagnosed.)
+
+        This code is lifted from SimpleHTTPRequestHandler so that we can
+        make sure the request is always based in our srpm_http_dir.
+        """
+        path = posixpath.normpath(urllib.unquote(path))
+        words = path.split('/')
+        words = filter(None, words)
+        path = CONFIG('built_rpm_download_basedir')
+        for word in words:
+            drive, word = os.path.splitdrive(word)
+            head, word = os.path.split(word)
+            if word in (os.curdir, os.pardir): continue
+            path = os.path.join(path, word)
+        return path
 
     def do_GET(self):
         self._server.set_busy (True)
@@ -42,12 +64,13 @@
             pass
         self._server.set_busy (False)
 
-class AWHttpServer(BaseHttpServer):
+class AWThreadingHttpServer(SocketServer.ThreadingTCPServer):
 
     def __init__(self, server_address, RequestHandlerClass, xmlaw):
+        self.protocol_version = "HTTP/1.0"    # Don't want keepalive
+        self.allow_reuse_address = 1
         os.chdir(CONFIG('built_rpm_download_basedir'))
         BaseHttpServer.__init__(self, server_address, RequestHandlerClass)
-        self.allow_reuse_address = True
 
         # Local variables
         self._xmlaw = xmlaw
@@ -66,9 +89,8 @@
 class HttpServerThread(threading.Thread):
 
     def __init__(self, address_tuple, xmlaw):
-        self._stop = False
         self._xmlaw = xmlaw
-        self._server = AWHttpServer(address_tuple, AWHttpRequestHandler, xmlaw)
+        self._server = AWThreadingHttpServer(address_tuple, AWHttpRequestHandler, xmlaw)
         threading.Thread.__init__(self)
 
     def run(self):


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]