[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

extras-buildsys/client CONFIG.py, NONE, 1.1 FileDownloader.py, NONE, 1.1 FileServer.py, NONE, 1.1 buildclient.py, NONE, 1.1 archwelder.py, 1.3, NONE archwelder_config.py, 1.3, NONE fileserver.py, 1.2, NONE srpm_download.py, 1.1, NONE



Author: dcbw

Update of /cvs/fedora/extras-buildsys/client
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv28321/client

Added Files:
	CONFIG.py FileDownloader.py FileServer.py buildclient.py 
Removed Files:
	archwelder.py archwelder_config.py fileserver.py 
	srpm_download.py 
Log Message:
2005-06-07  Dan Williams <dcbw redhat com>

    * Rework much of the build system to support HTTP transfer of SRPMs to
      build clients, and of logs & RPMs back to the build server.  Simple
      SRPM building is broken right now but will be fixed soon.




--- NEW FILE CONFIG.py ---
# Configuration file for archwelder.py

config_opts = {}
config_opts['builder_cmd'] = "/usr/bin/mock"
config_opts['distro_name'] = "fedora"
config_opts['repo_name'] = "core"

# Where to keep SRPMs to build and the finished products
# and logs.
# WARNING: this directory is world-readable via HTTP!
config_opts['client_work_dir'] = "/tmp/build_client_work"

def get(key):
    if config_opts.has_key(key):
        return config_opts[key]
    else:
        print "Bad request for key '%s'" % (key)
        exit (1)


--- NEW FILE FileDownloader.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#

import threading
import urlgrabber
import os
import shutil

def log(stuff=''):
    print stuff


class FileDownloader(threading.Thread):

    def __init__(self, build_client):
        self._build_client = build_client
        threading.Thread.__init__(self)

    def run(self):
        if self._build_client.status() is not 'downloading':
            return
        srpm_url = self._build_client.srpm_url()
        srpm_path = self._build_client.srpm_path()
        srpm_dirname = os.path.dirname(srpm_path)
        success = False
        if srpm_url and srpm_path:
            os.makedirs(srpm_dirname)
            result = urlgrabber.urlgrab(srpm_url, srpm_path)

#            try:
#                # Get the SRPM from the build server
#                result = urlgrabber.urlgrab(srpm_url, srpm_path)
#            except Exception, e:
#                pass
#            else:
            if result:
                success = True

        if success:
            self._build_client.set_status('downloaded')
            log("%s: Finished downloading %s" % (self._build_client.uniqid(), srpm_url))
        else:
            # Don't set failed status if the job was cancelled while downloading
            # since status is set in the cancellation call
            if not self._build_client.is_done_status():
                self._build_client.set_status('failed')
                log("%s: Failed to download %s" % (self._build_client.uniqid(), srpm_url))
            shutil.rmtree(srpm_dirname, ignore_errors=True)


--- NEW FILE FileServer.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#

import SimpleHTTPServer
import SocketServer
import threading
import os
import urllib
import posixpath
import CONFIG

BaseRequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
BaseHttpServer = SocketServer.TCPServer

class HttpRequestHandler(BaseRequestHandler):

    def __init__(self, request, client_address, server):
        self._server = server
        BaseRequestHandler.__init__(self, request, client_address, server)

    def list_directory(self, path):
        self.send_error(404, "No permission to list directory")

    def log_request(self, code='-', size='-'):
        # Don't log requests
        pass

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        This code is lifted from SimpleHTTPRequestHandler so that we can
        make sure the request is always based in our download directory,
        not the current directory.
        """
        path = posixpath.normpath(urllib.unquote(path))
        words = path.split('/')
        words = filter(None, words)
        path = CONFIG.get('client_work_dir')
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)
        return path

    def do_GET(self):
        self._server.set_busy (True)
        BaseRequestHandler.do_GET(self)
#        try:
#            BaseRequestHandler.do_GET(self)
#        except Exception, e:
#            # We get an exception if the client drops the transfer
#            pass
        self._server.set_busy (False)

class ThreadingHttpServer(SocketServer.ThreadingTCPServer):

    def __init__(self, server_address, RequestHandlerClass, xmlbc):
        self.protocol_version = "HTTP/1.0"    # Don't want keepalive
        self.allow_reuse_address = 1
        BaseHttpServer.__init__(self, server_address, RequestHandlerClass)

        # Local variables
        self._xmlbc = xmlbc
        self._busy = False

    def get_xmlbc(self):
        return self._xmlbc

    def get_busy(self):
        return self._busy

    def set_busy(self, busy):
        self._busy = busy


class FileServer(threading.Thread):

    def __init__(self, address_tuple, xmlbc):
        self._xmlbc = xmlbc
        self._server = ThreadingHttpServer(address_tuple, HttpRequestHandler, xmlbc)
        threading.Thread.__init__(self)

    def run(self):
        self._server.serve_forever()



--- NEW FILE buildclient.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal

# TODO: xml-rpc communication using 2-way ssl-cert-verified xmlrpc connection


import SimpleXMLRPCServer
import xmlrpclib
import socket
import os
import popen2
import sha
import time
import sys
import string
import time
import urllib
import CONFIG
import FileServer
import FileDownloader

DEBUG = False
def debugprint(stuff=''):
    if DEBUG:
        print stuff

def log(stuff=''):
    print stuff

g_our_hostname = None

def get_url_for_file(file_path):
    """ Return a URL pointing to a particular file in our work dir """

    # Ensure the file we're turning into a URL lives in our client work dir
    if not file_path.startswith(CONFIG.get("client_work_dir")):
        return None
    file_part = file_path[len(CONFIG.get("client_work_dir")) + 1:]
    full_url = "http://"; + g_our_hostname + ":8889/" + file_part
    return urllib.quote(full_url)


def get_base_filename_from_url(url):
    """ Safely unquotes a URL and gets the base file name from it.
        We're not using urlparse here because it doesn't un-escape stuff """

    unquoted = url
    last_unquoted = None
    count = 5
    
    # Keep unquoting the string until the last two unquote operations
    # produce the same string
    while (unquoted != last_unquoted) and (count > 0):
        last_unquoted = unquoted
        unquoted = urllib.unquote_plus(unquoted)
        count = count - 1

    # If after 5 iterations of unquoting, the strings still aren't the same,
    # something is wrong.
    if (count == 0) and (unquoted != last_unquoted):
        return None

    # Try to grab the filename off the end of the URL
    index = url.rfind('/')
    if index is -1:
        return None
    filename = url[index+1:]

    if not filename.endswith('.src.rpm'):
        return None

    # FIXME: what other validation can we do here?
    for c in filename:
        # For now, legal characters are '_-.' plus alphanumeric
        if (c == '_') or (c == '-') or (c == '.') or c.isalnum():
            pass
        else:
            return None

    return filename


class BuildClientMach:
    """puts things together for an arch - baseclass for handling builds for 
       other arches"""
    def __init__(self, uniqid, target, srpm_url):
        self._uniqid = uniqid
        self._status = 'init'
        self._files = []
        self._pobj = None
        self._target = target
        self._srpm_url = srpm_url

        srpm_filename = get_base_filename_from_url(srpm_url)
        if not srpm_filename:
            self._status = 'failed'
            self._srpm_path = None
        else:
            self._srpm_path = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "source", srpm_filename)

        self._result_dir = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "result")
        if not os.path.exists(self._result_dir):
            os.makedirs(self._result_dir)

    def die(self, sig=15):
        if self._pobj and self._pobj.pid:  # Can't kill the package download from build server
            try:
                os.kill(self._pobj.pid, sig)
            except OSError, e:
                print "Couldn't kill process %d: %s" % (self._pobj.pid, e)
        self._status = 'killed'
        return True

    def _download_srpm(self):
        self._status = 'downloading'
        dl_thread = FileDownloader.FileDownloader(self)
        dl_thread.start()

    def _build(self):
        print "%s: starting step 'building'" % self._uniqid
        if not os.path.exists(self._result_dir):
            os.makedirs(self._result_dir)
        cmd = '%s %s -r %s --resultdir=%s %s' % (self.arch_command,
                            CONFIG.get('builder_cmd'), self.buildroot, 
                            self._result_dir, self._srpm_path)
        self._pobj = popen2.Popen4(cmd=cmd)
        self._status = 'building'

    def start(self):
        # check for existence of srpm before going on
        self._download_srpm()

    def process(self):
        if not self.is_done_status():
            if self._status == 'downloading':
                pass
            elif self._status == 'downloaded':
                self._build()
            else:
                # If we're done with a step, advance to next one
                exit_status = self._pobj.poll()
                if exit_status == 0:
                    if self._status == 'building':
                        print "%s: Job done." % self._uniqid
                        self._status = 'done'
                        self._files = self._find_files()
                    else:
                        print "Bad status %s encountered!" % self._status
                elif exit_status > 0:
                    print "%s: job failed! mock exit status was %d\n" % (self._uniqid, exit_status)
                    self._status = 'failed'
                    self._files = self._find_files()
                else:
                    # builder process still running
                    pass

    def _find_files(self):
        # Grab the list of files in our job's result dir and URL encode them
        files_in_dir = os.listdir(self._result_dir)
        file_list = []
        for f in files_in_dir:
            file_url = get_url_for_file(os.path.join(self._result_dir, f))
            if file_url:
                file_list.append(file_url)
            else:
                print "Couldn't get file URL for %s" % f
        return file_list

    def status(self):
        return self._status

    def set_status(self, status):
        self._status = status

    def files(self):
        # ?? What to do with mock output...  another log file?
        #if self._pobj:
        #    self._output.extend(self._pobj.fromchild.readlines())
        return self._files

    def srpm_url(self):
        return self._srpm_url

    def srpm_path(self):
        return self._srpm_path

    def is_done_status(self):
        if (self._status is 'done') or (self._status is 'killed') or (self._status is 'failed'):
            return True
        return False

    def uniqid(self):
        return self._uniqid


class i386Arch(BuildClientMach):
    def __init__(self, uniqid, target, buildarch, srpm_url):
        self.buildroot = '%s-%s-i386-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
        self.buildarch = buildarch
        self.arch_command = '/usr/bin/setarch i686'
        BuildClientMach.__init__(self, uniqid, target, srpm_url)

class x86_64Arch(BuildClientMach):
    def __init__(self, uniqid, target, buildarch, srpm_url):
        self.buildroot = '%s-%s-x86_64-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
        self.buildarch = buildarch
        self.arch_command = ''
        BuildClientMach.__init__(self, uniqid, target, srpm_url)

class PPCArch(BuildClientMach):
    def __init__(self, uniqid, target, buildarch, srpm_url):
        self.buildroot = '%s-%s-ppc-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
        self.buildarch = buildarch
        self.arch_command = ''
        BuildClientMach.__init__(self, uniqid, target, srpm_url)

class PPC64Arch(BuildClientMach):
    def __init__(self, uniqid, target, buildarch, srpm_url):
        self.buildroot = '%s-%s-ppc64-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
        self.buildarch = buildarch
        self.arch_command = ''
        BuildClientMach.__init__(self, uniqid, target, srpm_url)


# Keep this global scope, used in __main__
builder_dict = { 'i386':i386Arch,
                'i486':i386Arch,
                'i586':i386Arch,
                'i686':i386Arch,
                'athlon':i386Arch,
                'x86_64':x86_64Arch,
                'amd64':x86_64Arch,
                'ia32e':x86_64Arch,
                'ppc':PPCArch,
                'ppc32':PPCArch,
                'ppc64':PPC64Arch,
               }

def getBuildClient(uniqid, target, buildarch, srpm_url, localarches):
    """hand it an arch it hands you back the build client instance you need"""
    
    if not builder_dict.has_key(buildarch):
        # raise an exception here bitching about no place to build for that arch
        pass

    if buildarch == 'noarch':
        if len(localarches) > 0:
            builder = builder_dict[localarches[0]]
    else:
        if buildarch in localarches:
            builder = builder_dict[buildarch]
    
    bcp = builder(uniqid, target, buildarch, srpm_url)
    return bcp

class XMLRPCBuildClientServer:
    def __init__(self, localarches):
        self.ids = {} # unique id => awclass instance
        self.localarches = localarches
        self.cur_job = 0

    def _process(self):
        # Give jobs some time to update their status and do their thing
        job = 0
        for (uniqid, bcp) in self.ids.iteritems():
            bcp.process()
            if not bcp.is_done_status():
                job = uniqid
        self.cur_job = job  # Update current job

    def start(self, target, buildarch, srpm_url):
        if self.cur_job != 0:
            print "Tried to build '%s' when already buiding something" % srpm_url
            return 0

        cur_time = time.time()
        check = '%d %s %s %s' % (cur_time, target, buildarch, srpm_url)
        sum = sha.new()
        sum.update(check)
        uniqid = sum.hexdigest()
        if target == 'devel':
            target = 'development'
        bcp = getBuildClient(uniqid, target, buildarch, srpm_url, self.localarches)
        if bcp != None:
            self.ids[uniqid] = bcp
            bcp.start()
            log("%s: started %s on %s arch %s at time %d" % (uniqid, srpm_url,
                        target, buildarch, cur_time))
        else:
            log("%s: Failed request for %s on %s UNSUPPORTED arch %s at time %d" %
                        (uniqid, srpm_url, target, buildarch, cur_time))
            uniqid = 0
        self.cur_job = uniqid
        return uniqid

    def status(self, uniqid=None):
        if not uniqid:
            uniqid = self.cur_job
        if not uniqid:
            return 'idle'
        try:
            bcp = self.ids[uniqid]
        except KeyError, e:
            bcp = None
        if not bcp:
            return 'idle'
        return bcp.status()
    
    def die(self, uniqid):
        bcp = self.ids[uniqid]
        return bcp.die()
    
    def files(self, uniqid):
        bcp = self.ids[uniqid]
        return bcp.files()
    
    def listjobs(self):
        return self.ids.keys()

    def get_cur_job(self):
        """ Are we currently building something? """
        return self.cur_job

    def supported_arches(self):
        return self.localarches

class MyXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
    """ XMLRPC server subclass that turns on SO_REUSEADDR """

    def __init__(self, address):
        self.allow_reuse_address = 1
        SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr=address, logRequests=False)


if __name__ == '__main__':
    if len(sys.argv) < 3:
        print "Usage:\n"
        print "   %s <hostname> <archlist>\n" % sys.argv[0]
        print "   <hostname> - hostname or IP address of this machine"
        print "   <archlist> - space-separated list of arches this machine can build"

        # pretty-print the available archlist
        archlist = ""
        avail_arches = builder_dict.keys()
        avail_arches.sort()
        for a in avail_arches:
            archlist = archlist + a
            if a != avail_arches[len(avail_arches)-1]:
                archlist = archlist + ", "
        print "                Available arches: [ %s ]\n" % archlist
        sys.exit(1)

    g_our_hostname = sys.argv[1]
    localarches = sys.argv[2:]

    print "Binding to address '%s' with arches: [%s]" % (g_our_hostname, string.join(localarches))

    xmlserver = MyXMLRPCServer((g_our_hostname, 8888))
    bcs = XMLRPCBuildClientServer(localarches)
    xmlserver.register_instance(bcs)

    # Start up the HTTP server thread which the build server
    # pulls completed RPMs from
    http_server = FileServer.FileServer((g_our_hostname, 8889), xmlserver)
    http_server.start()

    last_time = time.time()
    while True:
        try:
            xmlserver.handle_request()
        except KeyboardInterrupt, e:
            print "Shutting down..."
            break

        cur_time = time.time()
        if cur_time >= last_time + 5:
            # do some work every 5s or so
            bcs._process()
            last_time = time.time()

    os._exit(0)


--- archwelder.py DELETED ---


--- archwelder_config.py DELETED ---


--- fileserver.py DELETED ---


--- srpm_download.py DELETED ---


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]