[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Cluster-devel] conga/luci cluster/index_html site/luci/Extens ...



CVSROOT:	/cvs/cluster
Module name:	conga
Branch: 	RHEL5
Changes by:	rmccabe sourceware org	2007-06-27 06:19:22

Modified files:
	luci/cluster   : index_html 
	luci/site/luci/Extensions: HelperFunctions.py LuciDB.py 
	                           RicciQueries.py cluster_adapters.py 
	                           ricci_communicator.py 
	luci/storage   : index_html 

Log message:
	Add support for issuing ricci batch jobs on all cluster nodes simultaneously

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/cluster/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.20.2.10&r2=1.20.2.11
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/HelperFunctions.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.4.2.3&r2=1.4.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/LuciDB.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/RicciQueries.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.4.1&r2=1.1.4.2
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/cluster_adapters.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.120.2.31&r2=1.120.2.32
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/site/luci/Extensions/ricci_communicator.py.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.9.2.11&r2=1.9.2.12
http://sourceware.org/cgi-bin/cvsweb.cgi/conga/luci/storage/index_html.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.7.2.2&r2=1.7.2.3

--- conga/luci/cluster/index_html	2007/06/18 20:14:55	1.20.2.10
+++ conga/luci/cluster/index_html	2007/06/27 06:19:21	1.20.2.11
@@ -44,7 +44,7 @@
 			<tal:block tal:condition="firsttime">
 				<tal:block tal:define="global busywaiting python:True" />
 				<meta http-equiv="refresh"
-					tal:attributes="content isBusy/refreshurl | string:." />
+					tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
 			</tal:block>
 
 			<tal:block tal:define="global busy isBusy/busy | nothing" />
@@ -52,7 +52,7 @@
 			<tal:block tal:condition="busy">
 				<tal:block tal:define="global busywaiting python:True" />
 				<meta http-equiv="refresh"
-					tal:attributes="content isBusy/refreshurl | string:." />
+					tal:attributes="content isBusy/refreshurl | python:'3%surl=/luci/cluster' % chr(0x3b)" />
 			</tal:block>
 		</tal:block>
     </metal:headslot>
--- conga/luci/site/luci/Extensions/HelperFunctions.py	2007/06/18 18:39:32	1.4.2.3
+++ conga/luci/site/luci/Extensions/HelperFunctions.py	2007/06/27 06:19:22	1.4.2.4
@@ -18,11 +18,13 @@
 	return '%s; %s' % (str1, str2)
 
 class Worker(threading.Thread):
-	def __init__(self, mutex, hosts, riccis):
+	def __init__(self, mutex, hosts, riccis, func=None, *args):
 		threading.Thread.__init__(self)
 		self.mutex = mutex
 		self.hosts = hosts
 		self.riccis = riccis
+		self.query_func = func
+		self.query_args = args
 
 	def run(self):
 		while True:
@@ -32,21 +34,45 @@
 				return
 			host = self.hosts.pop()
 			self.mutex.release()
-			r = None
+
+			r = { 'ricci': None }
 
 			try:
-				r = RicciCommunicator(host)
+				rc = RicciCommunicator(host)
+				r['ricci'] = rc
+
+				if self.query_func is not None:
+					if self.query_args:
+						args = list(self.query_args)
+					else:
+						args = list()
+					args.insert(0, rc)
+					r['batch_result'] = self.query_func(*args)
 			except Exception, e:
-				#print host, 'failed', str(e)
-				pass
-			except:
-				#print host, 'failed'
 				pass
 
 			self.mutex.acquire()
 			self.riccis[host] = r
 			self.mutex.release()
 
+def send_batch_to_hosts(system_list, max_threads, func, *args):
+	mutex = threading.RLock()
+	threads = list()
+	hosts = list()
+	num_hosts = 0
+	ret = {}
+
+	for host in system_list:
+		hosts.append(host)
+		num_hosts += 1
+		if num_hosts <= max_threads:
+			threads.append(Worker(mutex, hosts, ret, func, *args))
+	for thread in threads:
+		thread.start()
+	for thread in threads:
+		thread.join()
+	return ret
+
 def get_system_info(self, system_list):
 	mutex = threading.RLock()
 	hive  = [] # workers
@@ -72,7 +98,7 @@
 		trusted = ''
 		authed = False
 		trusted = False
-		ricci = ss[hostname]
+		ricci = ss[hostname]['ricci']
 
 		if ricci is not None:
 			OS = ricci.os()
--- conga/luci/site/luci/Extensions/LuciDB.py	2007/06/18 18:39:32	1.1.4.1
+++ conga/luci/site/luci/Extensions/LuciDB.py	2007/06/27 06:19:22	1.1.4.2
@@ -6,7 +6,6 @@
 # Free Software Foundation.
 
 from AccessControl import getSecurityManager
-import RicciQueries as rq
 from ricci_communicator import RicciCommunicator
 from LuciZopePerm import isAdmin
 from LuciSyslog import get_logger
@@ -158,7 +157,7 @@
 			return True
 
 		batch_id = item[1].getProperty(BATCH_ID)
-		batch_ret = rq.checkBatch(rc, batch_id)
+		batch_ret = rc.batch_status(batch_id)
 		finished = batch_ret[0]
 		if finished is True or finished == -1:
 			if finished == -1:
--- conga/luci/site/luci/Extensions/RicciQueries.py	2007/06/18 18:39:32	1.1.4.1
+++ conga/luci/site/luci/Extensions/RicciQueries.py	2007/06/27 06:19:22	1.1.4.2
@@ -6,52 +6,12 @@
 # Free Software Foundation.
 
 from xml.dom import minidom
-from ricci_communicator import RicciCommunicator, extract_module_status
+from ricci_communicator import RicciCommunicator
 from LuciSyslog import get_logger
 from conga_constants import LUCI_DEBUG_MODE
 
 luci_log = get_logger()
 
-def checkBatch(rc, batch_id):
-	err_msg = 'An unknown Ricci error occurred on %s' % rc.hostname()
-
-	try:
-		batch = rc.batch_report(batch_id)
-		if batch is None:
-			if LUCI_DEBUG_MODE is True:
-				luci_log.debug_verbose('checkBatch0: batch id %s not found' \
-					% batch_id)
-			return (True, 'batch id %s was not found' % batch_id)
-	except Exception, e:
-		if LUCI_DEBUG_MODE is True:
-			luci_log.debug_verbose('checkBatch1: %s: %r %s' \
-				% (rc.hostname(), e, str(e)))
-		return (-1, err_msg)
-
-	try:
-		code, new_err_msg = extract_module_status(batch, 1)
-		if new_err_msg:
-			err_msg = 'A Ricci error occurred on %s: %s' \
-				% (rc.hostname(), str(new_err_msg))
-	except Exception, e:
-		if LUCI_DEBUG_MODE is True:
-			luci_log.debug_verbose('checkBatch2: %s: %r %s' \
-				% (rc.hostname(), e, str(e)))
-		return (-1, err_msg)
-
-	# In progress.
-	if code == -101 or code == -102:
-		return (False, 'in progress')
-
-	# Done successfully.
-	if code == 0:
-		return (True, 'completed sucessfully')
-
-	# Error
-	if LUCI_DEBUG_MODE is True:
-		luci_log.debug_verbose('checkBatch3: %d: %s' % (code, rc.hostname()))
-	return (-1, err_msg)
-
 def addClusterNodeBatch(cluster_name,
 						install_base,
 						install_services,
@@ -310,9 +270,9 @@
 
 	for i in batch:
 		try:
-			batch_number = i.getAttribute('batch_id')
-			result = i.getAttribute('status')
-			return (str(batch_number), str(result))
+			batch_number = str(i.getAttribute('batch_id'))
+			result = str(i.getAttribute('status'))
+			return (batch_number, result)
 		except Exception, e:
 			if LUCI_DEBUG_MODE is True:
 				luci_log.debug_verbose('BAR1: %s' % e)
--- conga/luci/site/luci/Extensions/cluster_adapters.py	2007/06/19 15:54:10	1.120.2.31
+++ conga/luci/site/luci/Extensions/cluster_adapters.py	2007/06/27 06:19:22	1.120.2.32
@@ -640,7 +640,7 @@
 	# abort the whole process.
 	try:
 		while True:
-			batch_ret = rq.checkBatch(cluster_ricci, batch_number)
+			batch_ret = cluster_ricci.batch_status(batch_number)
 			code = batch_ret[0]
 			if code is True:
 				break
@@ -3106,7 +3106,7 @@
 
 			if rc is not None:
 				batch_num = item[1].getProperty(BATCH_ID)
-				batch_res = rq.checkBatch(rc, batch_num)
+				batch_res = rc.batch_status(batch_num)
 				finished = batch_res[0]
 				err_msg = batch_res[1]
 
--- conga/luci/site/luci/Extensions/ricci_communicator.py	2007/06/18 18:39:33	1.9.2.11
+++ conga/luci/site/luci/Extensions/ricci_communicator.py	2007/06/27 06:19:22	1.9.2.12
@@ -53,7 +53,10 @@
 					% (self.__hostname, hello.toxml()))
 		except:
 			if LUCI_DEBUG_MODE is True:
-				luci_log.debug_verbose('RC:init0: error receiving header from %s: %r %s' % (self.__hostname, e, str(e)))
+				luci_log.debug_verbose('RC:init0: error receiving header from %s:%d: %r %s' % (self.__hostname, self.__port, e, str(e)))
+			else:
+				luci_log.info('Error reading from %s:%d: %s' \
+					% (self.__hostname, self.__port, str(e)))
 
 		self.__authed = hello.firstChild.getAttribute('authenticated') == 'true'
 		self.__cluname = hello.firstChild.getAttribute('clustername')
@@ -231,8 +234,8 @@
 		if doc.firstChild.getAttribute('success') != '0':
 			if LUCI_DEBUG_MODE is True:
 				luci_log.debug_verbose('RC:PB4: batch command failed')
-			raise RicciError, 'The last ricci command to host %s failed' \
-					% self.__hostname
+			raise RicciError, 'The last ricci command to host %s:%d failed' \
+					% (self.__hostname, self.__port)
 
 		batch_node = None
 		for node in doc.firstChild.childNodes:
@@ -243,8 +246,8 @@
 			if LUCI_DEBUG_MODE is True:
 				luci_log.debug_verbose('RC:PB5: no <batch/> from %s' \
 					% self.__hostname)
-			raise RicciError, 'missing <batch/> in ricci response from "%s"' \
-					% self.__hostname
+			raise RicciError, 'missing <batch/> in ricci response from %s:%d' \
+					% (self.__hostname, self.__port)
 
 		return batch_node
 
@@ -264,7 +267,7 @@
 			ricci_xml = self.process_batch(batch_xml, async)
 			if LUCI_DEBUG_NET is True:
 				try:
-					luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command.' % (ricci_xml.toxml(), self.__hostname))
+					luci_log.debug_net_priv('RC:BRun2: received XML "%s" from host %s in response to batch command' % (ricci_xml.toxml(), self.__hostname))
 				except:
 					pass
 		except:
@@ -276,6 +279,48 @@
 		doc.appendChild(ricci_xml)
 		return doc
 
+	def batch_status(self, batch_id):
+		err_msg = 'An unknown ricci error occurred on %s:%d' \
+					% (self.__hostname, self.__port)
+
+		try:
+			batch = self.batch_report(batch_id)
+			if batch is None:
+				if LUCI_DEBUG_MODE is True:
+					luci_log.debug_verbose('RCCB0: batch id %s not found' \
+						% batch_id)
+				return (True, 'batch id %s was not found' % batch_id)
+		except Exception, e:
+			if LUCI_DEBUG_MODE is True:
+				luci_log.debug_verbose('RCCB1: %s:%d: %r %s' \
+					% (self.__hostname, self.__port, e, str(e)))
+			return (-1, err_msg)
+
+		try:
+			code, new_err_msg = extract_module_status(batch, 1)
+			if new_err_msg:
+				err_msg = 'A ricci error occurred on %s:%d: %s' \
+							% (self.__hostname, self.__port, str(new_err_msg))
+		except Exception, e:
+			if LUCI_DEBUG_MODE is True:
+				luci_log.debug_verbose('RCCB2: %s:%d %r %s: %s' \
+					% (self.__hostname, self.__port, e, str(e), err_msg))
+			return (-1, err_msg)
+
+		# In progress.
+		if code == -101 or code == -102:
+			return (False, 'in progress')
+
+		# Done successfully.
+		if code == 0:
+			return (True, 'completed sucessfully')
+
+		# Error
+		if LUCI_DEBUG_MODE is True:
+			luci_log.debug_verbose('RCCB3: %s:%d code %d: %s' \
+				% (self.__hostname, self.__port, code, err_msg))
+		return (-1, err_msg)
+
 	def batch_report(self, batch_id):
 		if LUCI_DEBUG_NET is True:
 			luci_log.debug_net('RC:BRep0: [auth=%d] asking for batchid# %s for host %s' % (self.__authed, batch_id, self.__hostname))
@@ -374,6 +419,9 @@
 		return doc
 
 def get_ricci_communicator(self, hostname, allowed_systems):
+	if not hostname:
+		return None
+
 	if not self.access_to_host_allowed(hostname, allowed_systems):
 		if LUCI_DEBUG_MODE is True:
 			luci_log.debug_verbose('GRC0: access to host %s is not allowed' % hostname)
--- conga/luci/storage/index_html	2007/06/26 17:00:53	1.7.2.2
+++ conga/luci/storage/index_html	2007/06/27 06:19:22	1.7.2.3
@@ -222,7 +222,7 @@
                 <h1>Future Site of Forms</h1>
               </metal:main-form-content>
 				<div style="padding-top: 10px;"
-					tal:condition="report_cached">
+					tal:condition="python: report_cached and not (request.has_key('pagetype') and request['pagetype']=='commit_changes')">
 					<form>
 						<input type="hidden" name="reprobe_url"
 							tal:attributes="value


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]