extras-buildsys/common AuthedXMLRPCServer.py, NONE, 1.1 HTTPServer.py, NONE, 1.1 XMLRPCServerProxy.py, NONE, 1.1 FileDownloader.py, 1.9, 1.10 Makefile, 1.3, 1.4 SSLCommon.py, 1.7, 1.8 lighttpdManager.py, 1.1, 1.2 SSLXMLRPCServerProxy.py, 1.3, NONE SimpleHTTPSServer.py, 1.6, NONE SimpleSSLXMLRPCServer.py, 1.7, NONE
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue Jul 5 21:08:00 UTC 2005
- Previous message (by thread): extras-buildsys/client client.py,1.5,1.6
- Next message (by thread): extras-buildsys/server ArchJob.py, NONE, 1.1 Builder.py, NONE, 1.1 BuilderManager.py, NONE, 1.1 PackageJob.py, NONE, 1.1 main.py, NONE, 1.1 BuildMaster.py, 1.9, 1.10 CONFIG.py, 1.14, 1.15 Makefile, 1.4, 1.5 Repo.py, 1.4, 1.5 UserInterface.py, 1.14, 1.15 BuildJob.py, 1.15, NONE buildserver.py, 1.12, NONE client_manager.py, 1.27, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: dcbw
Update of /cvs/fedora/extras-buildsys/common
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv3768/common
Modified Files:
FileDownloader.py Makefile SSLCommon.py lighttpdManager.py
Added Files:
AuthedXMLRPCServer.py HTTPServer.py XMLRPCServerProxy.py
Removed Files:
SSLXMLRPCServerProxy.py SimpleHTTPSServer.py
SimpleSSLXMLRPCServer.py
Log Message:
2005-07-05 Dan Williams <dcbw at redhat.com>
* Rework a bunch of stuff so the build server doesn't use quite as much CPU,
also split out stuff from server/client_manager.py
* Generalize the common/ classes to provide both SSL and non-SSL facilities,
renaming a lot of those files in the process
* Fix non-SSL builder/server and client/server communication
Note: At this time, SSL may be broken.
--- NEW FILE AuthedXMLRPCServer.py ---
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Konstantin Ryabitsev <icon at phy.duke.edu>
# Modified by Dan Williams <dcbw at redhat.com>
import os, sys
import socket
import time
import SocketServer
from M2Crypto import SSL
import xmlrpclib
import SimpleXMLRPCServer
import SSLCommon
class AuthedSimpleXMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
def do_POST(self):
"""
See documentation on SimpleXMLRPCRequestHandler. Modifications
include the ability to pass through local request-specific data,
and removal of code to deal with historical _dispatch methods.
"""
if 0:
# ONLY FOR DEBUGGING, so we can see exceptions on the console
# get arguments
data = self.rfile.read(int(self.headers["content-length"]))
authinfo = self.server.get_authinfo(self.request, self.client_address)
response = self.server._marshaled_dispatch(data, authinfo)
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
else:
try:
# get arguments
data = self.rfile.read(int(self.headers["content-length"]))
authinfo = self.server.get_authinfo(self.request, self.client_address)
response = self.server._marshaled_dispatch(data, authinfo)
except: # This should only happen if the module is buggy
# internal error, report as HTTP server error
self.send_response(500)
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
# shut down the connection
self.wfile.flush()
self.connection.shutdown(1)
class BaseAuthedXMLRPCServer:
def __init__(self, address, authinfo_callback=None):
self.allow_reuse_address = 1
self.logRequests = 0
self.authinfo_callback = authinfo_callback
if sys.version_info[:3] > (2, 2, 3):
SimpleXMLRPCServer.SimpleXMLRPCDispatcher.__init__(self)
def get_authinfo(self, request, client_address):
if self.authinfo_callback:
return self.authinfo_callback(request, client_address)
return None
def verify_request(self, request, client_address):
"""
Allow ourselves a chance to verify the client
"""
if self.authinfo_callback:
authinfo = self.get_authinfo(request, client_address)
if authinfo:
return True
return False
# Allow all requests if there's no authinfo_callback
return True
def _marshaled_dispatch(self, data, authinfo):
"""
Allow the ability to pass request-dependent objects into
the handler instance.
"""
params, method = xmlrpclib.loads(data)
# generate response
try:
response = self._dispatch(authinfo, method, params)
# wrap response in a singleton tuple
response = (response,)
response = xmlrpclib.dumps(response, methodresponse=1)
except xmlrpclib.Fault, fault:
response = xmlrpclib.dumps(fault)
except:
# report exception back to server
response = xmlrpclib.dumps(
xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
)
return response
def _dispatch(self, authinfo, method, params):
"""
Allow the ability to pass request-dependent objects into
the handler instance.
"""
func = None
try:
# check to see if a matching function has been registered
func = self.funcs[method]
except KeyError:
if self.instance is not None:
# call instance method directly
try:
func = self.lcl_resolve_dotted_attribute(method)
except AttributeError:
pass
if func is not None:
# Only pass authinfo into Instance if we have are set
# up to get authinfo
if self.authinfo_callback is not None:
return func(authinfo, *params)
else:
return func(*params)
else:
raise Exception('method "%s" is not supported' % method)
# Sigh. Different versions of M2Crypto have this in different places
def lcl_resolve_dotted_attribute(self, method):
pyver = sys.version_info[:3]
if pyver <= (2, 3, 4):
return SimpleXMLRPCServer.resolve_dotted_attribute(
self.instance, method)
elif pyver > (2, 3, 4):
return SimpleXMLRPCServer.resolve_dotted_attribute(
self.instance, method, self.allow_dotted_names)
def serve_forever(self):
while True:
try:
self.handle_request()
except KeyboardInterrupt, e:
print "Shutting down..."
break
except socket.error, e:
if e[0] == 11: #Resource temporarily unavailable
try:
time.sleep(0.1)
except KeyboardInterrupt, e:
print "Shutting down..."
break
class AuthedSSLXMLRPCServer(BaseAuthedXMLRPCServer, SSLCommon.QuietSSLServer, SimpleXMLRPCServer.SimpleXMLRPCServer):
""" Extension to allow more fine-tuned SSL handling """
def __init__(self, address, authinfo_callback=None, certs=None):
BaseAuthedXMLRPCServer.__init__(self, address, authinfo_callback)
ctx = SSLCommon.getSSLContext(certs)
SSLCommon.QuietSSLServer.__init__(self, address, AuthedSimpleXMLRPCRequestHandler, ctx)
class AuthedXMLRPCServer(BaseAuthedXMLRPCServer, SocketServer.ThreadingTCPServer, SimpleXMLRPCServer.SimpleXMLRPCServer):
def __init__(self, address, authinfo_callback=None):
BaseAuthedXMLRPCServer.__init__(self, address, authinfo_callback)
SocketServer.ThreadingTCPServer.__init__(self, address, AuthedSimpleXMLRPCRequestHandler)
--- NEW FILE HTTPServer.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#
import SimpleHTTPServer
import SocketServer
import threading
import urllib
import posixpath
import os, sys
from SimpleHTTPServer import SimpleHTTPRequestHandler
from M2Crypto import Rand, SSL
from M2Crypto.SSL.SSLServer import ThreadingSSLServer
import SSLCommon
import socket
import time
from M2Crypto import threading as m2thread
class HttpRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
def __init__(self, request, client_address, server):
self._server = server
SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, request, client_address, server)
def list_directory(self, path):
self.send_error(404, "No permission to list directory")
def log_request(self, code='-', size='-'):
# Don't log requests
pass
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
This code is lifted from SimpleHTTPRequestHandler so that we can
make sure the request is always based in our download directory,
not the current directory.
"""
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = self._server.http_dir
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def do_GET(self):
SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
# try:
# SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
# except Exception, e:
# # We get an exception if the client drops the transfer
# pass
class ThreadingHTTPSServer(ThreadingSSLServer):
""" SSL-enabled variant """
def __init__(self, server_addr, http_dir, certs):
self.allow_reuse_address = 1
self.http_dir = http_dir
self.ctx = SSLCommon.getSSLContext(certs)
ThreadingSSLServer.__init__(self, server_addr, HttpRequestHandler, self.ctx)
self.server_name = server_addr[0]
self.server_port = server_addr[1]
# About the last thing we want is a client blocking the server
# because it hung or tracebacked
timeout = SSL.timeout(10)
self.socket.set_socket_read_timeout(timeout)
self.socket.set_socket_write_timeout(timeout)
def finish(self):
if self.request:
self.request.set_shutdown(SSL.SSL_RECEIVED_SHUTDOWN | SSL.SSL_SENT_SHUTDOWN)
self.request.close()
def serve_forever(self):
while True:
try:
self.handle_request()
except KeyboardInterrupt, e:
print "Shutting down..."
break
except socket.error, e:
if e[0] == 11: # Resource temporarily unavailable
try:
time.sleep(0.1)
except KeyboardInterrupt, e:
print "Shutting down..."
break
class ThreadingHTTPServer(SocketServer.ThreadingTCPServer):
""" Non-SSL variant """
def __init__(self, certs, server_addr, http_dir):
self.allow_reuse_address = 1
self.http_dir = http_dir
ThreadingTCPServer.__init__(self, server_addr, HttpRequestHandler)
def serve_forever(self):
while True:
try:
self.handle_request()
except KeyboardInterrupt, e:
print "Shutting down..."
break
except socket.error, e:
if e[0] == 11: # Resource temporarily unavailable
try:
time.sleep(0.1)
except KeyboardInterrupt, e:
print "Shutting down..."
break
class HTTPServer(threading.Thread):
def __init__(self, addr, http_dir, certs):
if certs and len(certs) > 0:
self._server = ThreadingHTTPSServer(addr, http_dir, certs)
else:
self._server = ThreadingHTTPServer(addr, http_dir)
self._stop = False
threading.Thread.__init__(self)
def run(self):
while not self._stop:
self._server.handle_request()
def stop(self):
self._stop = True
###########################################################
# Testing stuff
###########################################################
if __name__ == '__main__':
if len(sys.argv) < 4:
print "Usage: python SimpleHTTPSServer.py key_and_cert ca_cert peer_ca_cert"
sys.exit(1)
certs = {}
certs['key_and_cert'] = sys.argv[1]
certs['ca_cert'] = sys.argv[2]
certs['peer_ca_cert'] = sys.argv[3]
print "Starting..."
srcdir = os.path.join("/tmp", "server_dir")
if not os.path.exists(srcdir):
os.makedirs(srcdir)
print "Creating random test data..."
f = open(os.path.join(srcdir, "testfile.dat"), "w")
x = 1
while x < 10000:
f.write(os.urandom(50))
x = x + 1
f.close()
m2thread.init()
print "Starting the server."
server = SimpleHTTPSServer(certs, ('localhost', 8886), srcdir)
server.start()
while True:
try:
time.sleep(1)
except KeyboardInterrupt:
print "Quitting..."
m2thread.cleanup()
os._exit(0)
--- NEW FILE XMLRPCServerProxy.py ---
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Modified by Dan Williams <dcbw at redhat.com>
import os, sys
from M2Crypto import SSL
from M2Crypto.m2xmlrpclib import SSL_Transport, ServerProxy
import SSLCommon
class XMLRPCServerProxy(ServerProxy):
def __init__(self, uri, certs):
if certs and len(certs) > 0:
self.ctx = SSLCommon.getSSLContext(certs)
ServerProxy.__init__(self, uri, SSL_Transport(ssl_context=self.ctx))
else:
ServerProxy.__init__(self, uri)
Index: FileDownloader.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/FileDownloader.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -r1.9 -r1.10
--- FileDownloader.py 1 Jul 2005 11:51:33 -0000 1.9
+++ FileDownloader.py 5 Jul 2005 21:07:57 -0000 1.10
@@ -83,7 +83,10 @@
self._filename = get_base_filename_from_url(self._url, legal_exts)
if not self._filename:
print "Couldn't get base filename from url!! target_dir=%s, url=%s" % (target_dir, url)
- self._opener = HTTPSURLopener.HTTPSURLopener(certs)
+ if certs and len(certs) > 0:
+ self._opener = HTTPSURLopener.HTTPSURLopener(certs)
+ else:
+ self._opener = urllib.URLopener()
threading.Thread.__init__(self)
def run(self):
Index: Makefile
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/Makefile,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- Makefile 1 Jul 2005 11:51:33 -0000 1.3
+++ Makefile 5 Jul 2005 21:07:57 -0000 1.4
@@ -11,10 +11,10 @@
CommonErrors.py \
FileDownloader.py \
HTTPSURLopener.py \
- SimpleHTTPSServer.py \
- SimpleSSLXMLRPCServer.py \
+ HTTPServer.py \
+ AuthedXMLRPCServer.py \
SSLCommon.py \
- SSLXMLRPCServerProxy.py \
+ XMLRPCServerProxy.py \
lighttpdManager.py \
__init__.py
Index: SSLCommon.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/SSLCommon.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- SSLCommon.py 1 Jul 2005 11:51:33 -0000 1.7
+++ SSLCommon.py 5 Jul 2005 21:07:57 -0000 1.8
@@ -51,7 +51,11 @@
class QuietSSLServer(SSL.ThreadingSSLServer):
def __init__(self, server_address, RequestHandlerClass, ssl_context):
- SSL.SSLServer.__init__(self, server_address, RequestHandlerClass, ssl_context)
+ try:
+ SSL.SSLServer.__init__(self, server_address, RequestHandlerClass, ssl_context)
+ except socket.error, e:
+ if e[0] == 49: # Cannot assign requested address
+ print "Error: requested address '%s' couldn't be used." % server_address
# About the last thing we want is a client blocking the server
# because it hung or tracebacked
Index: lighttpdManager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/lighttpdManager.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- lighttpdManager.py 1 Jul 2005 11:51:33 -0000 1.1
+++ lighttpdManager.py 5 Jul 2005 21:07:58 -0000 1.2
@@ -25,7 +25,7 @@
class lighttpdManager(threading.Thread):
- def __init__(self, conffile, hostname, port, basedir, key_and_cert):
+ def __init__(self, conffile, hostname, port, basedir, exit_if_dead, key_and_cert):
self._stop = False
self._pobj = None
self._conffile = conffile
@@ -33,6 +33,7 @@
self._hostname = hostname
self._basedir = basedir
self._key_and_cert = key_and_cert
+ self._exit_if_dead = exit_if_dead
threading.Thread.__init__(self)
if not self._setup():
os._exit(1)
@@ -41,7 +42,7 @@
def _setup(self):
if not os.path.exists(self._basedir):
os.makedirs(self._basedir)
- if not os.access(self._key_and_cert, os.R_OK):
+ if self._key_and_cert and not os.access(self._key_and_cert, os.R_OK):
print "%s does not exist or is not readable." % self._key_and_cert
return False
if not os.path.exists(os.path.dirname(self._conffile)):
@@ -59,9 +60,13 @@
server.username = "lighttpd"
server.groupname = "lighttpd"
server.dir-listing = "disable"
+''' % (self._basedir, self._port, self._hostname)
+
+ if self._key_and_cert:
+ cnf = cnf + '''
ssl.engine = "enable"
ssl.pemfile = "%s"
-''' % (self._basedir, self._port, self._hostname, self._key_and_cert)
+''' % (self._key_and_cert)
f = open(self._conffile, "w")
f.write(cnf)
@@ -79,21 +84,27 @@
st = self._pobj.poll()
# Restart it if it fails
if st >= 0:
- print "HTTP server failed, trying to restart..."
- print "----------------------------------------"
- for line in self._pobj.fromchild.readlines():
- print line
- print "----------------------------------------\n"
- del self._pobj
- if not self._setup():
+ if self._exit_if_dead:
+ print "HTTP server failed! exiting..."
os._exit(1)
- self._start_server()
+ else:
+ print "HTTP server failed, trying to restart..."
+ print "----------------------------------------"
+ for line in self._pobj.fromchild.readlines():
+ print line
+ print "----------------------------------------\n"
+ del self._pobj
+ if not self._setup():
+ os._exit(1)
+ self._start_server()
time.sleep(5)
if self._pobj:
- os.kill(self._pobj.pid, 15)
- os.remove(self._conffile)
-
+ try:
+ os.kill(self._pobj.pid, 15)
+ os.remove(self._conffile)
+ except:
+ pass
def stop(self):
self._stop = True
--- SSLXMLRPCServerProxy.py DELETED ---
--- SimpleHTTPSServer.py DELETED ---
--- SimpleSSLXMLRPCServer.py DELETED ---
- Previous message (by thread): extras-buildsys/client client.py,1.5,1.6
- Next message (by thread): extras-buildsys/server ArchJob.py, NONE, 1.1 Builder.py, NONE, 1.1 BuilderManager.py, NONE, 1.1 PackageJob.py, NONE, 1.1 main.py, NONE, 1.1 BuildMaster.py, 1.9, 1.10 CONFIG.py, 1.14, 1.15 Makefile, 1.4, 1.5 Repo.py, 1.4, 1.5 UserInterface.py, 1.14, 1.15 BuildJob.py, 1.15, NONE buildserver.py, 1.12, NONE client_manager.py, 1.27, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list