[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[Cluster-devel] [PATCH] qdiskd: backport dual socket connection to cman



From: "Fabio M. Di Nitto" <fdinitto redhat com>

Patch 76741bb2a94ae94e493c609d50f570d02e2f3029 had a not so obvious
dependency on 08ae3ce147b2771c5ee6e1d364a5e48c88384427.

Backport portion of 08ae3ce147b2771c5ee6e1d364a5e48c88384427 to handle
dual cman socket (admin and user) and use the correct socket (user)
for send/receive data.

Move cman_alive check and heartbeat (for dispatch) to ch_user.

Resolves: rhbz#782900

Signed-off-by: Fabio M. Di Nitto <fdinitto redhat com>
---
 cman/qdisk/disk.h      |    5 ++-
 cman/qdisk/disk_util.c |    7 +++--
 cman/qdisk/iostate.c   |    8 +++---
 cman/qdisk/main.c      |   69 ++++++++++++++++++++++-------------------------
 4 files changed, 43 insertions(+), 46 deletions(-)

diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h
index d491de1..83167ea 100644
--- a/cman/qdisk/disk.h
+++ b/cman/qdisk/disk.h
@@ -270,7 +270,8 @@ typedef struct {
 	int qc_master;		/* Master?! */
 	int qc_status_sock;
 	run_flag_t qc_flags;
-	cman_handle_t qc_ch;
+	cman_handle_t qc_ch_admin;
+	cman_handle_t qc_ch_user;
 	char *qc_device;
 	char *qc_label;
 	char *qc_status_file;
@@ -299,7 +300,7 @@ typedef struct {
 int qd_write_status(qd_ctx *ctx, int nid, disk_node_state_t state,
 		    disk_msg_t *msg, memb_mask_t mask, memb_mask_t master);
 int qd_read_print_status(target_info_t *disk, int nid);
-int qd_init(qd_ctx *ctx, cman_handle_t ch, int me);
+int qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me);
 void qd_destroy(qd_ctx *ctx);
 
 /* proc.c */
diff --git a/cman/qdisk/disk_util.c b/cman/qdisk/disk_util.c
index f5539c0..25f4013 100644
--- a/cman/qdisk/disk_util.c
+++ b/cman/qdisk/disk_util.c
@@ -312,16 +312,17 @@ generate_token(void)
   Initialize a quorum disk context, given a CMAN handle and a nodeid.
  */
 int
-qd_init(qd_ctx *ctx, cman_handle_t ch, int me)
+qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me)
 {
-	if (!ctx || !ch || !me) {
+	if (!ctx || !ch_admin || !ch_user || !me) {
 		errno = EINVAL;
 		return -1;
 	}	
 
 	memset(ctx, 0, sizeof(*ctx));
 	ctx->qc_incarnation = generate_token();
-	ctx->qc_ch = ch;
+	ctx->qc_ch_admin = ch_admin;
+	ctx->qc_ch_user = ch_user;
 	ctx->qc_my_id = me;
 	ctx->qc_status_sock = -1;
 
diff --git a/cman/qdisk/iostate.c b/cman/qdisk/iostate.c
index eb74ad2..ba7ad12 100644
--- a/cman/qdisk/iostate.c
+++ b/cman/qdisk/iostate.c
@@ -69,7 +69,7 @@ io_nanny_thread(void *arg)
 	iostate_t last_main_state = 0, current_main_state = 0;
 	int last_main_incarnation = 0, current_main_incarnation = 0;
 	int logged_incarnation = 0;
-	cman_handle_t ch = (cman_handle_t)arg;
+	cman_handle_t ch_user = (cman_handle_t)arg;
 	int32_t whine_state;
 
 	/* Start with wherever we're at now */
@@ -105,7 +105,7 @@ io_nanny_thread(void *arg)
 		/* Whine on CMAN api */
 		whine_state = (int32_t)current_main_state;
 		swab32(whine_state);
-		cman_send_data(ch, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0);
+		cman_send_data(ch_user, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0);
 
 		/* Don't log things twice */
 		if (logged_incarnation == current_main_incarnation)
@@ -125,7 +125,7 @@ io_nanny_thread(void *arg)
 
 
 int
-io_nanny_start(cman_handle_t ch, int timeout)
+io_nanny_start(cman_handle_t ch_user, int timeout)
 {
 	int ret;
 
@@ -135,7 +135,7 @@ io_nanny_start(cman_handle_t ch, int timeout)
 	qdisk_timeout = timeout;
 	thread_active = 1;
 
-	ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch);
+	ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch_user);
 	pthread_mutex_unlock(&state_mutex);
 
 	return ret;
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 90d00ab..72a3c07 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -287,7 +287,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask)
 				if (ctx->qc_flags & RF_ALLOW_KILL) {
 					clulog(LOG_DEBUG, "Telling CMAN to "
 						"kill the node\n");
-					cman_kill_node(ctx->qc_ch,
+					cman_kill_node(ctx->qc_ch_admin,
 						ni[x].ni_status.ps_nodeid);
 				}
 			}
@@ -325,7 +325,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask)
 			if (ctx->qc_flags & RF_ALLOW_KILL) {
 				clulog(LOG_DEBUG, "Telling CMAN to "
 					"kill the node\n");
-				cman_kill_node(ctx->qc_ch,
+				cman_kill_node(ctx->qc_ch_admin,
 					ni[x].ni_status.ps_nodeid);
 			}
 			continue;
@@ -580,7 +580,7 @@ check_cman(qd_ctx *ctx, memb_mask_t mask, memb_mask_t master_mask)
 	cman_node_t nodes[MAX_NODES_DISK];
 	int retnodes, x;
 
-	if (cman_get_nodes(ctx->qc_ch, MAX_NODES_DISK,
+	if (cman_get_nodes(ctx->qc_ch_admin, MAX_NODES_DISK,
 			   &retnodes, nodes) <0 )
 		return;
 
@@ -894,7 +894,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 	_running = 1;
 	while (_running) {
 		/* perform a forceful cman dispatch */
-		if (cman_alive(ctx->qc_ch) < 0) {
+		if (cman_alive(ctx->qc_ch_user) < 0) {
 			clulog(LOG_ERR, "cman: %s\n", strerror(errno));
 		}
 
@@ -931,11 +931,11 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 				msg.m_msg = M_NONE;
 				++msg.m_seq;
 				bid_pending = 0;
-				if (cman_alive(ctx->qc_ch) < 0) {
+				if (cman_alive(ctx->qc_ch_user) < 0) {
 					clulog(LOG_ERR, "cman: %s\n",
 					       strerror(errno));
 				} else {
-					cman_poll_quorum_device(ctx->qc_ch, 0);
+					cman_poll_quorum_device(ctx->qc_ch_admin, 0);
 				}
 				if (ctx->qc_flags & RF_REBOOT)
 					reboot(RB_AUTOBOOT);
@@ -1039,7 +1039,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 			/* We are the master.  Poll the quorum device.
 			   We can't be the master unless we score high
 			   enough on our heuristics. */
-			if (cman_alive(ctx->qc_ch) < 0) {
+			if (cman_alive(ctx->qc_ch_user) < 0) {
 				clulog(LOG_ERR, "cman_dispatch: %s\n",
 				       strerror(errno));
 				clulog(LOG_ERR,
@@ -1048,7 +1048,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 			}
 			check_cman(ctx, mask, master_mask);
 			if (!errors)
-				cman_poll_quorum_device(ctx->qc_ch, 1);
+				cman_poll_quorum_device(ctx->qc_ch_admin, 1);
 
 		} else if (ctx->qc_status == S_RUN && ctx->qc_master &&
 			   ctx->qc_master != ctx->qc_my_id) {
@@ -1060,7 +1060,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 			      ni[ctx->qc_master-1].ni_status.ps_master_mask,
 				       ctx->qc_my_id-1,
 				       sizeof(memb_mask_t))) {
-				if (cman_alive(ctx->qc_ch) < 0) {
+				if (cman_alive(ctx->qc_ch_user) < 0) {
 					clulog(LOG_ERR, "cman_dispatch: %s\n",
 						strerror(errno));
 					clulog(LOG_ERR,
@@ -1069,7 +1069,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
 				}
 				if (!errors && 
 				    (!(ctx->qc_flags & RF_MASTER_WINS)))
-					cman_poll_quorum_device(ctx->qc_ch, 1);
+					cman_poll_quorum_device(ctx->qc_ch_admin, 1);
 			}
 		}
 		
@@ -1506,12 +1506,7 @@ check_stop_cman(qd_ctx *ctx)
 	
 	clulog(LOG_WARNING, "Telling CMAN to leave the cluster; qdisk is not"
 		" available\n");
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
-	if (cman_shutdown(ctx->qc_ch, 0) < 0) {
-#else
-	int x = 0;
-	if (ioctl(cman_get_fd(ctx->qc_ch), SIOCCLUSTER_LEAVE_CLUSTER, &x) < 0) {
-#endif
+	if (cman_shutdown(ctx->qc_ch_admin, 0) < 0) {
 		clulog(LOG_CRIT, "Could not leave the cluster - rebooting\n");
 		sleep(5);
 		if (ctx->qc_flags & RF_DEBUG)
@@ -1552,7 +1547,8 @@ main(int argc, char **argv)
 	cman_node_t me;
 	int cfh, rv, forked = 0, nfd = -1, ret = -1;
 	qd_ctx ctx;
-	cman_handle_t ch = NULL;
+	cman_handle_t ch_admin = NULL;
+	cman_handle_t ch_user = NULL;
 	node_info_t ni[MAX_NODES_DISK];
 	struct h_data h[10];
 	char debug = 0, foreground = 0;
@@ -1597,12 +1593,8 @@ main(int argc, char **argv)
 	qp.ni = &ni[0];
 	qp.ni_len = MAX_NODES_DISK;
 
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
-	ch = cman_admin_init(&qp);
-#else
-	ch = cman_init(&qp);
-#endif
-	if (!ch) {
+	ch_admin = cman_admin_init(&qp);
+	if (!ch_admin) {
 		if (!foreground && !forked) {
 			if (daemon_init(argv[0]) < 0)
 				goto out;
@@ -1614,22 +1606,24 @@ main(int argc, char **argv)
 		
 		do {
 			sleep(5);
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
-			ch = cman_admin_init(&qp);
-#else
-			ch = cman_init(&qp);
-#endif
-		} while (!ch);
+			ch_admin = cman_admin_init(&qp);
+		} while (!ch_admin);
+	}
+
+	ch_user = cman_init(&qp);
+	if (!ch_user) {
+		clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n", strerror(errno));
+		goto out;
 	}
 
-	if (cman_start_recv_data(ch, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) {
+	if (cman_start_recv_data(ch_user, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) {
 		clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n",
 				 strerror(errno));
 		goto out;
 	}
 
 	memset(&me, 0, sizeof(me));
-	while (cman_get_node(ch, CMAN_NODEID_US, &me) < 0) {
+	while (cman_get_node(ch_admin, CMAN_NODEID_US, &me) < 0) {
 		if (!foreground && !forked) {
 			if (daemon_init(argv[0]) < 0)
 				goto out;
@@ -1639,7 +1633,7 @@ main(int argc, char **argv)
 		sleep(5);
 	}
 
-	qd_init(&ctx, ch, me.cn_nodeid);
+	qd_init(&ctx, ch_admin, ch_user, me.cn_nodeid);
 
 	signal(SIGINT, int_handler);
 	signal(SIGTERM, int_handler);
@@ -1715,13 +1709,13 @@ main(int argc, char **argv)
 	if (!_running)
 		goto out;
 
-	if ((rv = cman_register_quorum_device(ctx.qc_ch,
+	if ((rv = cman_register_quorum_device(ctx.qc_ch_admin,
 				    (ctx.qc_flags&RF_CMAN_LABEL)? 
 				        ctx.qc_cman_label:
                                         ctx.qc_device,
 				    ctx.qc_votes)) < 0) {
 		if (errno == EBUSY) {
-			if ((rv = cman_update_quorum_device(ctx.qc_ch,
+			if ((rv = cman_update_quorum_device(ctx.qc_ch_admin,
 				    (ctx.qc_flags&RF_CMAN_LABEL)?
 					ctx.qc_cman_label:
 					ctx.qc_device,
@@ -1740,17 +1734,18 @@ main(int argc, char **argv)
 		}
 	}
 
-	io_nanny_start(ch, ctx.qc_tko * ctx.qc_interval);
+	io_nanny_start(ch_user, ctx.qc_tko * ctx.qc_interval);
 
 	if (quorum_loop(&ctx, ni, MAX_NODES_DISK) == 0)
-		cman_unregister_quorum_device(ctx.qc_ch);
+		cman_unregister_quorum_device(ctx.qc_ch_admin);
 
 	io_nanny_stop();
 
 	quorum_logout(&ctx);
 	/* free cman handle to avoid leak in cman */
 out:
-	cman_finish(ctx.qc_ch);
+	cman_finish(ctx.qc_ch_admin);
+	cman_finish(ctx.qc_ch_user);
 	qd_destroy(&ctx);
 
 	return ret;
-- 
1.7.7.6


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]