[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Cluster-devel] cluster/group/daemon app.c cman.c cpg.c gd_int ...



CVSROOT:	/cvs/cluster
Module name:	cluster
Changes by:	teigland sourceware org	2006-12-01 20:26:01

Modified files:
	group/daemon   : app.c cman.c cpg.c gd_internal.h 

Log message:
	Be more intelligent about handling recovery sets so we can deal with
	cases where a node fails, rejoins, then fails again before recovery has
	completed for the first failure.  Also handles case where the groupd
	process exits without the node going down.  If that happens, we want
	to kill the node (via cman) if the node was in any groups and ignore
	it otherwise.

Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&r1=1.52&r2=1.53
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cman.c.diff?cvsroot=cluster&r1=1.27&r2=1.28
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&r1=1.36&r2=1.37
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&r1=1.44&r2=1.45

--- cluster/group/daemon/app.c	2006/10/16 15:52:05	1.52
+++ cluster/group/daemon/app.c	2006/12/01 20:26:01	1.53
@@ -209,20 +209,61 @@
    level must complete recovery (on all nodes) before recovery can begin for
    the next level. */
 
-void add_recovery_set(int nodeid)
+/* FIXME: do we need to worry about the case where we get an
+   add_recovery_set_cman() that finds an old rs, the old rs completes
+   and goes away, and then we get the add_recovery_set_cpg() matching
+   the _cman() variant that we ignored? */
+
+void add_recovery_set_cman(int nodeid)
+{
+	struct recovery_set *rs;
+
+	log_debug("add_recovery_set_cman nodeid %d", nodeid);
+
+	rs = get_recovery_set(nodeid);
+	if (rs->cman_update) {
+		log_debug("old recovery for %d still in progress", nodeid);
+		return;
+	}
+	rs->cman_update = 1;
+
+	if (!rs->cpg_update && !in_groupd_cpg(nodeid)) {
+		log_debug("free recovery set %d not running groupd", nodeid);
+		list_del(&rs->list);
+		free(rs);
+		return;
+	}
+
+	if (list_empty(&rs->entries) && rs->cpg_update) {
+		log_debug("free unused recovery set %d cman", nodeid);
+		list_del(&rs->list);
+		free(rs);
+	}
+}
+
+/* procdown of 1 means the groupd daemon process exited, but the node didn't
+   fail.  when only the process fails, we won't get a cman callback which is
+   only for nodedown.  if the node wasn't in any groups we don't add a recovery
+   set and don't care about the exited groupd; if the node with the failed
+   groupd _was_ in any groups, we add a rs and process_groupd_confchg() will do
+   cman_kill_node() to make the node really fail (and we'll get an
+   add_recovery_set_cman()). */
+
+struct recovery_set *add_recovery_set_cpg(int nodeid, int procdown)
 {
 	struct recovery_set *rs;
 	struct recovery_entry *re;
 	group_t *g;
 	node_t *node;
 
-	log_debug("add_recovery_set for nodeid %d", nodeid);
+	log_debug("add_recovery_set_cpg nodeid %d procdown %d",
+		  nodeid, procdown);
 
 	rs = get_recovery_set(nodeid);
-
-	ASSERT(list_empty(&rs->entries));
-	ASSERT(!rs->cpg_update);
-
+	if (rs->cpg_update) {
+		log_debug("old recovery for %d still in progress", nodeid);
+		return rs;
+	}
 	rs->cpg_update = 1;
 
 	list_for_each_entry(g, &gd_groups, list) {
@@ -238,11 +279,16 @@
 		}
 	}
 
-	if (list_empty(&rs->entries) && rs->cman_update) {
-		log_debug("free unused recovery set %d cpg", nodeid);
-		list_del(&rs->list);
-		free(rs);
+	if (list_empty(&rs->entries)) {
+		if (rs->cman_update || procdown) {
+			log_debug("free unused recovery set %d cpg", nodeid);
+			list_del(&rs->list);
+			free(rs);
+			return NULL;
+		}
 	}
+
+	return rs;
 }
 
 void _del_recovery_set(group_t *g, int nodeid, int purge)
--- cluster/group/daemon/cman.c	2006/06/30 21:00:33	1.27
+++ cluster/group/daemon/cman.c	2006/12/01 20:26:01	1.28
@@ -53,7 +53,6 @@
 
 static void statechange(void)
 {
-	struct recovery_set *rs;
 	int i, rv;
 
 	old_quorate = cman_quorate;
@@ -95,24 +94,7 @@
 
 			log_debug("cman: node %d removed",
 				  old_nodes[i].cn_nodeid);
-
-			rs = get_recovery_set(old_nodes[i].cn_nodeid);
-			rs->cman_update = 1;
-
-			if (!rs->cpg_update && !in_groupd_cpg(rs->nodeid)) {
-				log_debug("free recovery set %d not in cpg",
-					  rs->nodeid);
-				list_del(&rs->list);
-				free(rs);
-				continue;
-			}
-
-			if (rs->cpg_update && list_empty(&rs->entries)) {
-				log_debug("free unused recovery set %d cman",
-					  rs->nodeid);
-				list_del(&rs->list);
-				free(rs);
-			}
+			add_recovery_set_cman(old_nodes[i].cn_nodeid);
 		}
 	}
 
--- cluster/group/daemon/cpg.c	2006/10/13 15:57:23	1.36
+++ cluster/group/daemon/cpg.c	2006/12/01 20:26:01	1.37
@@ -151,6 +151,7 @@
 
 void process_groupd_confchg(void)
 {
+	struct recovery_set *rs;
 	int i, found = 0;
 
 	log_debug("groupd confchg total %d left %d joined %d",
@@ -172,14 +173,26 @@
 		log_print("we are not in groupd confchg: %u %u",
 			  our_nodeid, (uint32_t) getpid());
 
-	/* FIXME: we probably want to do a cman_kill_node() on a node
-	   where groupd exits but cman is still running. */
-
 	for (i = 0; i < saved_left_count; i++) {
-		if (saved_left[i].reason != CPG_REASON_LEAVE) {
-			add_recovery_set(saved_left[i].nodeid);
-			groupd_down(saved_left[i].nodeid);
+		if (saved_left[i].reason == CPG_REASON_LEAVE)
+			continue;
+
+		if (saved_left[i].reason == CPG_REASON_NODEDOWN) {
+			/* a nice clean failure */
+			add_recovery_set_cpg(saved_left[i].nodeid, 0);
+		} else if (saved_left[i].reason == CPG_REASON_PROCDOWN) {
+			/* groupd failed, but the node is still up; if
+			   the node was in any groups (non-NULL rs is
+			   returned), then kill the node so it'll be a
+			   real nodedown */
+			rs = add_recovery_set_cpg(saved_left[i].nodeid, 1);
+			if (rs) {
+				log_print("kill node %d - groupd PROCDOWN",
+					  saved_left[i].nodeid);
+				kill_cman(saved_left[i].nodeid);
+			}
 		}
+		groupd_down(saved_left[i].nodeid);
 	}
 }
 
--- cluster/group/daemon/gd_internal.h	2006/10/06 16:55:19	1.44
+++ cluster/group/daemon/gd_internal.h	2006/12/01 20:26:01	1.45
@@ -236,7 +236,8 @@
 
 
 /* app.c */
-void add_recovery_set(int nodeid);
+void add_recovery_set_cman(int nodeid);
+struct recovery_set *add_recovery_set_cpg(int nodeid, int procdown);
 int queue_app_recover(group_t *g, int nodeid);
 int queue_app_join(group_t *g, int nodeid);
 int queue_app_leave(group_t *g, int nodeid);


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]