extras-buildsys/client CONFIG.py, NONE, 1.1 FileDownloader.py, NONE, 1.1 FileServer.py, NONE, 1.1 buildclient.py, NONE, 1.1 archwelder.py, 1.3, NONE archwelder_config.py, 1.3, NONE fileserver.py, 1.2, NONE srpm_download.py, 1.1, NONE
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue Jun 7 12:10:25 UTC 2005
- Previous message (by thread): extras-buildsys ChangeLog,1.1.1.1,1.2
- Next message (by thread): extras-buildsys/server FileDownloader.py, NONE, 1.1 FileServer.py, NONE, 1.1 buildserver.py, NONE, 1.1 client_manager.py, NONE, 1.1 CONFIG.py, 1.2, 1.3 buildjob.py, 1.2, 1.3 buildmaster.py, 1.1.1.1, 1.2 aw_manager.py, 1.2, NONE bm_server.py, 1.2, NONE fileserver.py, 1.1, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: dcbw
Update of /cvs/fedora/extras-buildsys/client
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv28321/client
Added Files:
CONFIG.py FileDownloader.py FileServer.py buildclient.py
Removed Files:
archwelder.py archwelder_config.py fileserver.py
srpm_download.py
Log Message:
2005-06-07 Dan Williams <dcbw at redhat.com>
* Rework much of the build system to support HTTP transfer of SRPMs to
build clients, and of logs & RPMs back to the build server. Simple
SRPM building is broken right now but will be fixed soon.
--- NEW FILE CONFIG.py ---
# Configuration file for archwelder.py
config_opts = {}
config_opts['builder_cmd'] = "/usr/bin/mock"
config_opts['distro_name'] = "fedora"
config_opts['repo_name'] = "core"
# Where to keep SRPMs to build and the finished products
# and logs.
# WARNING: this directory is world-readable via HTTP!
config_opts['client_work_dir'] = "/tmp/build_client_work"
def get(key):
if config_opts.has_key(key):
return config_opts[key]
else:
print "Bad request for key '%s'" % (key)
exit (1)
--- NEW FILE FileDownloader.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#
import threading
import urlgrabber
import os
import shutil
def log(stuff=''):
print stuff
class FileDownloader(threading.Thread):
def __init__(self, build_client):
self._build_client = build_client
threading.Thread.__init__(self)
def run(self):
if self._build_client.status() is not 'downloading':
return
srpm_url = self._build_client.srpm_url()
srpm_path = self._build_client.srpm_path()
srpm_dirname = os.path.dirname(srpm_path)
success = False
if srpm_url and srpm_path:
os.makedirs(srpm_dirname)
result = urlgrabber.urlgrab(srpm_url, srpm_path)
# try:
# # Get the SRPM from the build server
# result = urlgrabber.urlgrab(srpm_url, srpm_path)
# except Exception, e:
# pass
# else:
if result:
success = True
if success:
self._build_client.set_status('downloaded')
log("%s: Finished downloading %s" % (self._build_client.uniqid(), srpm_url))
else:
# Don't set failed status if the job was cancelled while downloading
# since status is set in the cancellation call
if not self._build_client.is_done_status():
self._build_client.set_status('failed')
log("%s: Failed to download %s" % (self._build_client.uniqid(), srpm_url))
shutil.rmtree(srpm_dirname, ignore_errors=True)
--- NEW FILE FileServer.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#
import SimpleHTTPServer
import SocketServer
import threading
import os
import urllib
import posixpath
import CONFIG
BaseRequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
BaseHttpServer = SocketServer.TCPServer
class HttpRequestHandler(BaseRequestHandler):
def __init__(self, request, client_address, server):
self._server = server
BaseRequestHandler.__init__(self, request, client_address, server)
def list_directory(self, path):
self.send_error(404, "No permission to list directory")
def log_request(self, code='-', size='-'):
# Don't log requests
pass
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
This code is lifted from SimpleHTTPRequestHandler so that we can
make sure the request is always based in our download directory,
not the current directory.
"""
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = CONFIG.get('client_work_dir')
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def do_GET(self):
self._server.set_busy (True)
BaseRequestHandler.do_GET(self)
# try:
# BaseRequestHandler.do_GET(self)
# except Exception, e:
# # We get an exception if the client drops the transfer
# pass
self._server.set_busy (False)
class ThreadingHttpServer(SocketServer.ThreadingTCPServer):
def __init__(self, server_address, RequestHandlerClass, xmlbc):
self.protocol_version = "HTTP/1.0" # Don't want keepalive
self.allow_reuse_address = 1
BaseHttpServer.__init__(self, server_address, RequestHandlerClass)
# Local variables
self._xmlbc = xmlbc
self._busy = False
def get_xmlbc(self):
return self._xmlbc
def get_busy(self):
return self._busy
def set_busy(self, busy):
self._busy = busy
class FileServer(threading.Thread):
def __init__(self, address_tuple, xmlbc):
self._xmlbc = xmlbc
self._server = ThreadingHttpServer(address_tuple, HttpRequestHandler, xmlbc)
threading.Thread.__init__(self)
def run(self):
self._server.serve_forever()
--- NEW FILE buildclient.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
# TODO: xml-rpc communication using 2-way ssl-cert-verified xmlrpc connection
import SimpleXMLRPCServer
import xmlrpclib
import socket
import os
import popen2
import sha
import time
import sys
import string
import time
import urllib
import CONFIG
import FileServer
import FileDownloader
DEBUG = False
def debugprint(stuff=''):
if DEBUG:
print stuff
def log(stuff=''):
print stuff
g_our_hostname = None
def get_url_for_file(file_path):
""" Return a URL pointing to a particular file in our work dir """
# Ensure the file we're turning into a URL lives in our client work dir
if not file_path.startswith(CONFIG.get("client_work_dir")):
return None
file_part = file_path[len(CONFIG.get("client_work_dir")) + 1:]
full_url = "http://" + g_our_hostname + ":8889/" + file_part
return urllib.quote(full_url)
def get_base_filename_from_url(url):
""" Safely unquotes a URL and gets the base file name from it.
We're not using urlparse here because it doesn't un-escape stuff """
unquoted = url
last_unquoted = None
count = 5
# Keep unquoting the string until the last two unquote operations
# produce the same string
while (unquoted != last_unquoted) and (count > 0):
last_unquoted = unquoted
unquoted = urllib.unquote_plus(unquoted)
count = count - 1
# If after 5 iterations of unquoting, the strings still aren't the same,
# something is wrong.
if (count == 0) and (unquoted != last_unquoted):
return None
# Try to grab the filename off the end of the URL
index = url.rfind('/')
if index is -1:
return None
filename = url[index+1:]
if not filename.endswith('.src.rpm'):
return None
# FIXME: what other validation can we do here?
for c in filename:
# For now, legal characters are '_-.' plus alphanumeric
if (c == '_') or (c == '-') or (c == '.') or c.isalnum():
pass
else:
return None
return filename
class BuildClientMach:
"""puts things together for an arch - baseclass for handling builds for
other arches"""
def __init__(self, uniqid, target, srpm_url):
self._uniqid = uniqid
self._status = 'init'
self._files = []
self._pobj = None
self._target = target
self._srpm_url = srpm_url
srpm_filename = get_base_filename_from_url(srpm_url)
if not srpm_filename:
self._status = 'failed'
self._srpm_path = None
else:
self._srpm_path = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "source", srpm_filename)
self._result_dir = os.path.join(CONFIG.get('client_work_dir'), self._uniqid, "result")
if not os.path.exists(self._result_dir):
os.makedirs(self._result_dir)
def die(self, sig=15):
if self._pobj and self._pobj.pid: # Can't kill the package download from build server
try:
os.kill(self._pobj.pid, sig)
except OSError, e:
print "Couldn't kill process %d: %s" % (self._pobj.pid, e)
self._status = 'killed'
return True
def _download_srpm(self):
self._status = 'downloading'
dl_thread = FileDownloader.FileDownloader(self)
dl_thread.start()
def _build(self):
print "%s: starting step 'building'" % self._uniqid
if not os.path.exists(self._result_dir):
os.makedirs(self._result_dir)
cmd = '%s %s -r %s --resultdir=%s %s' % (self.arch_command,
CONFIG.get('builder_cmd'), self.buildroot,
self._result_dir, self._srpm_path)
self._pobj = popen2.Popen4(cmd=cmd)
self._status = 'building'
def start(self):
# check for existence of srpm before going on
self._download_srpm()
def process(self):
if not self.is_done_status():
if self._status == 'downloading':
pass
elif self._status == 'downloaded':
self._build()
else:
# If we're done with a step, advance to next one
exit_status = self._pobj.poll()
if exit_status == 0:
if self._status == 'building':
print "%s: Job done." % self._uniqid
self._status = 'done'
self._files = self._find_files()
else:
print "Bad status %s encountered!" % self._status
elif exit_status > 0:
print "%s: job failed! mock exit status was %d\n" % (self._uniqid, exit_status)
self._status = 'failed'
self._files = self._find_files()
else:
# builder process still running
pass
def _find_files(self):
# Grab the list of files in our job's result dir and URL encode them
files_in_dir = os.listdir(self._result_dir)
file_list = []
for f in files_in_dir:
file_url = get_url_for_file(os.path.join(self._result_dir, f))
if file_url:
file_list.append(file_url)
else:
print "Couldn't get file URL for %s" % f
return file_list
def status(self):
return self._status
def set_status(self, status):
self._status = status
def files(self):
# ?? What to do with mock output... another log file?
#if self._pobj:
# self._output.extend(self._pobj.fromchild.readlines())
return self._files
def srpm_url(self):
return self._srpm_url
def srpm_path(self):
return self._srpm_path
def is_done_status(self):
if (self._status is 'done') or (self._status is 'killed') or (self._status is 'failed'):
return True
return False
def uniqid(self):
return self._uniqid
class i386Arch(BuildClientMach):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-i386-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
self.buildarch = buildarch
self.arch_command = '/usr/bin/setarch i686'
BuildClientMach.__init__(self, uniqid, target, srpm_url)
class x86_64Arch(BuildClientMach):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-x86_64-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
self.buildarch = buildarch
self.arch_command = ''
BuildClientMach.__init__(self, uniqid, target, srpm_url)
class PPCArch(BuildClientMach):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-ppc-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
self.buildarch = buildarch
self.arch_command = ''
BuildClientMach.__init__(self, uniqid, target, srpm_url)
class PPC64Arch(BuildClientMach):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-ppc64-%s' % (CONFIG.get('distro_name'), target, CONFIG.get('repo_name'))
self.buildarch = buildarch
self.arch_command = ''
BuildClientMach.__init__(self, uniqid, target, srpm_url)
# Keep this global scope, used in __main__
builder_dict = { 'i386':i386Arch,
'i486':i386Arch,
'i586':i386Arch,
'i686':i386Arch,
'athlon':i386Arch,
'x86_64':x86_64Arch,
'amd64':x86_64Arch,
'ia32e':x86_64Arch,
'ppc':PPCArch,
'ppc32':PPCArch,
'ppc64':PPC64Arch,
}
def getBuildClient(uniqid, target, buildarch, srpm_url, localarches):
"""hand it an arch it hands you back the build client instance you need"""
if not builder_dict.has_key(buildarch):
# raise an exception here bitching about no place to build for that arch
pass
if buildarch == 'noarch':
if len(localarches) > 0:
builder = builder_dict[localarches[0]]
else:
if buildarch in localarches:
builder = builder_dict[buildarch]
bcp = builder(uniqid, target, buildarch, srpm_url)
return bcp
class XMLRPCBuildClientServer:
def __init__(self, localarches):
self.ids = {} # unique id => awclass instance
self.localarches = localarches
self.cur_job = 0
def _process(self):
# Give jobs some time to update their status and do their thing
job = 0
for (uniqid, bcp) in self.ids.iteritems():
bcp.process()
if not bcp.is_done_status():
job = uniqid
self.cur_job = job # Update current job
def start(self, target, buildarch, srpm_url):
if self.cur_job != 0:
print "Tried to build '%s' when already buiding something" % srpm_url
return 0
cur_time = time.time()
check = '%d %s %s %s' % (cur_time, target, buildarch, srpm_url)
sum = sha.new()
sum.update(check)
uniqid = sum.hexdigest()
if target == 'devel':
target = 'development'
bcp = getBuildClient(uniqid, target, buildarch, srpm_url, self.localarches)
if bcp != None:
self.ids[uniqid] = bcp
bcp.start()
log("%s: started %s on %s arch %s at time %d" % (uniqid, srpm_url,
target, buildarch, cur_time))
else:
log("%s: Failed request for %s on %s UNSUPPORTED arch %s at time %d" %
(uniqid, srpm_url, target, buildarch, cur_time))
uniqid = 0
self.cur_job = uniqid
return uniqid
def status(self, uniqid=None):
if not uniqid:
uniqid = self.cur_job
if not uniqid:
return 'idle'
try:
bcp = self.ids[uniqid]
except KeyError, e:
bcp = None
if not bcp:
return 'idle'
return bcp.status()
def die(self, uniqid):
bcp = self.ids[uniqid]
return bcp.die()
def files(self, uniqid):
bcp = self.ids[uniqid]
return bcp.files()
def listjobs(self):
return self.ids.keys()
def get_cur_job(self):
""" Are we currently building something? """
return self.cur_job
def supported_arches(self):
return self.localarches
class MyXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
""" XMLRPC server subclass that turns on SO_REUSEADDR """
def __init__(self, address):
self.allow_reuse_address = 1
SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, addr=address, logRequests=False)
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage:\n"
print " %s <hostname> <archlist>\n" % sys.argv[0]
print " <hostname> - hostname or IP address of this machine"
print " <archlist> - space-separated list of arches this machine can build"
# pretty-print the available archlist
archlist = ""
avail_arches = builder_dict.keys()
avail_arches.sort()
for a in avail_arches:
archlist = archlist + a
if a != avail_arches[len(avail_arches)-1]:
archlist = archlist + ", "
print " Available arches: [ %s ]\n" % archlist
sys.exit(1)
g_our_hostname = sys.argv[1]
localarches = sys.argv[2:]
print "Binding to address '%s' with arches: [%s]" % (g_our_hostname, string.join(localarches))
xmlserver = MyXMLRPCServer((g_our_hostname, 8888))
bcs = XMLRPCBuildClientServer(localarches)
xmlserver.register_instance(bcs)
# Start up the HTTP server thread which the build server
# pulls completed RPMs from
http_server = FileServer.FileServer((g_our_hostname, 8889), xmlserver)
http_server.start()
last_time = time.time()
while True:
try:
xmlserver.handle_request()
except KeyboardInterrupt, e:
print "Shutting down..."
break
cur_time = time.time()
if cur_time >= last_time + 5:
# do some work every 5s or so
bcs._process()
last_time = time.time()
os._exit(0)
--- archwelder.py DELETED ---
--- archwelder_config.py DELETED ---
--- fileserver.py DELETED ---
--- srpm_download.py DELETED ---
- Previous message (by thread): extras-buildsys ChangeLog,1.1.1.1,1.2
- Next message (by thread): extras-buildsys/server FileDownloader.py, NONE, 1.1 FileServer.py, NONE, 1.1 buildserver.py, NONE, 1.1 client_manager.py, NONE, 1.1 CONFIG.py, 1.2, 1.3 buildjob.py, 1.2, 1.3 buildmaster.py, 1.1.1.1, 1.2 aw_manager.py, 1.2, NONE bm_server.py, 1.2, NONE fileserver.py, 1.1, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list