[Cluster-devel] Cluster Project branch, RHEL5, updated. cmirror_1_1_15-73-ga21d1c0
jbrassow at sourceware.org
jbrassow at sourceware.org
Thu May 15 18:02:53 UTC 2008
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "Cluster Project".
http://sources.redhat.com/git/gitweb.cgi?p=cluster.git;a=commitdiff;h=a21d1c03600e011cf1ee132100ac823bf3478877
The branch, RHEL5 has been updated
via a21d1c03600e011cf1ee132100ac823bf3478877 (commit)
from 607cd9680d773fff362144249328c3b6d75a4443 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit a21d1c03600e011cf1ee132100ac823bf3478877
Author: Jonathan Brassow <jbrassow at redhat.com>
Date: Thu May 15 13:02:19 2008 -0500
clogd + dm-log-clustered: Improve communication error handling (K <-> U)
Connector (netlink) is an unreliable protocol. From time to time, I get
a connector header in userspace that has all the right fields (seq #, size
of data, etc), but the data is all zeros. When this happens, I use the seq
number in the header to request the kernel to resend the request. This
should avoid those nasty 'Invalid request' messages... which are then
followed by 'Request timed out' (because the invalid request was ignored).
-----------------------------------------------------------------------
Summary of changes:
cmirror-kernel/src/dm-clog-tfr.c | 124 +++++++++++++++++++++---------------
cmirror/src/local.c | 130 +++++++++++++++++++++++++-------------
2 files changed, 157 insertions(+), 97 deletions(-)
diff --git a/cmirror-kernel/src/dm-clog-tfr.c b/cmirror-kernel/src/dm-clog-tfr.c
index 61f6f21..7abd554 100644
--- a/cmirror-kernel/src/dm-clog-tfr.c
+++ b/cmirror-kernel/src/dm-clog-tfr.c
@@ -62,7 +62,7 @@ static int dm_clog_sendto_server(struct clog_tfr *tfr)
msg->id.idx = cn_clog_id.idx;
msg->id.val = cn_clog_id.val;
msg->ack = 0;
- msg->seq = 0;
+ msg->seq = tfr->seq;
msg->len = sizeof(struct clog_tfr) + tfr->data_size;
r = cn_netlink_send(msg, 0, gfp_any());
@@ -73,47 +73,67 @@ static int dm_clog_sendto_server(struct clog_tfr *tfr)
return r;
}
-static void cn_clog_callback(void *data)
+/*
+ * fill_pkg
+ * @msg
+ * @tfr
+ *
+ * Parameters can be either msg or tfr, but not both. This
+ * function fills in the reply for a waiting request. If just
+ * msg is given, then the reply is simply an ACK from userspace
+ * that the request was received.
+ *
+ * Returns: 0 on success, -ENOENT on failure
+ */
+static int fill_pkg(struct cn_msg *msg, struct clog_tfr *tfr)
{
- int found = 0;
- struct cn_msg *msg = (struct cn_msg *)data;
- struct clog_tfr *tfr = (struct clog_tfr *)(msg + 1);
+ uint32_t rtn_seq = (msg) ? msg->seq : (tfr) ? tfr->seq : 0;
struct receiving_pkg *pkg;
- spin_lock(&receiving_list_lock);
- if (msg->len < sizeof(*tfr)) {
- DMERR("Incomplete message received from cluster log server");
- spin_unlock(&receiving_list_lock);
- return;
- }
-
list_for_each_entry(pkg, &receiving_list, list) {
- /*
- DMINFO("Msg from userspace received [%s].", RQ_TYPE(tfr->request_type));
- DMINFO(" Seq # received: %u Seq # wanted: %u",
- pkg->seq, tfr->seq);
- */
- if (tfr->seq == pkg->seq) {
- if (tfr->data_size > *(pkg->data_size)) {
- DMERR("Insufficient space to receive package [%s]",
- RQ_TYPE(tfr->request_type));
- *(pkg->data_size) = 0;
- pkg->error = -ENOSPC;
- } else {
- pkg->error = tfr->error;
- memcpy(pkg->data, tfr->data, tfr->data_size);
- *(pkg->data_size) = tfr->data_size;
- }
- found = 1;
- complete(&pkg->complete);
- break;
+ if (rtn_seq != pkg->seq)
+ continue;
+
+ if (msg) {
+ pkg->error = -msg->ack;
+ *(pkg->data_size) = 0;
+ } else if (tfr->data_size > *(pkg->data_size)) {
+ DMERR("Insufficient space to receive package [%s]",
+ RQ_TYPE(tfr->request_type));
+ *(pkg->data_size) = 0;
+ pkg->error = -ENOSPC;
+ } else {
+ pkg->error = tfr->error;
+ memcpy(pkg->data, tfr->data, tfr->data_size);
+ *(pkg->data_size) = tfr->data_size;
}
+ complete(&pkg->complete);
+ return 0;
}
+ return -ENOENT;
+}
+
+/*
+ * cn_clog_callback
+ * @data
+ *
+ * This is the connector callback that delivers data
+ * that was sent from userspace.
+ */
+static void cn_clog_callback(void *data)
+{
+ struct cn_msg *msg = (struct cn_msg *)data;
+ struct clog_tfr *tfr = (struct clog_tfr *)(msg + 1);
+
+ spin_lock(&receiving_list_lock);
+ if (msg->len == 0)
+ fill_pkg(msg, NULL);
+ else if (msg->len < sizeof(*tfr))
+ DMERR("Incomplete message received: [%u]", msg->seq);
+ else
+ fill_pkg(NULL, tfr);
spin_unlock(&receiving_list_lock);
- if (!found)
- DMERR("Stray request returned: %s, %u",
- RQ_TYPE(tfr->request_type), tfr->seq);
}
/*
@@ -183,37 +203,35 @@ resend:
if (r) {
DMERR("Unable to send cluster log request [%s] to server: %d",
RQ_TYPE(request_type), r);
+ spin_lock(&receiving_list_lock);
+ list_del_init(&(pkg.list));
+ spin_unlock(&receiving_list_lock);
+
goto out;
}
r = wait_for_completion_timeout(&(pkg.complete), 15 * HZ);
+ spin_lock(&receiving_list_lock);
+ list_del_init(&(pkg.list));
+ spin_unlock(&receiving_list_lock);
if (!r) {
DMWARN("[%s] Request timed out: [%s/%u] - retrying",
SHORT_UUID(uuid), RQ_TYPE(request_type), pkg.seq);
- spin_lock(&receiving_list_lock);
- list_del_init(&(pkg.list));
- spin_unlock(&receiving_list_lock);
-
goto resend;
- } else {
- /* FIXME: Pull time checking code - it's just for profiling */
- pkg.start_time = (jiffies - pkg.start_time);
- do_div(pkg.start_time, HZ);
- if (pkg.start_time > 0)
- DMWARN("Excessive delay in request processing, %llu sec: [%s/%u]",
- pkg.start_time, RQ_TYPE(request_type),
- pkg.seq);
}
+
+ /* FIXME: Pull time checking code - it's just for profiling */
+ pkg.start_time = (jiffies - pkg.start_time);
+ do_div(pkg.start_time, HZ);
+ if (pkg.start_time > 0)
+ DMWARN("Excessive delay in request processing, %llu sec: [%s/%u]",
+ pkg.start_time, RQ_TYPE(request_type), pkg.seq);
+
r = pkg.error;
- if (r)
- DMERR("[%s] Server error while processing request [%s]: %d",
- SHORT_UUID(uuid), RQ_TYPE(request_type), r);
+ if (r == -EAGAIN)
+ goto resend;
out:
- spin_lock(&receiving_list_lock);
- list_del_init(&(pkg.list));
- spin_unlock(&receiving_list_lock);
-
if (tfr != (struct clog_tfr *)prealloced_clog_tfr)
kfree(tfr);
diff --git a/cmirror/src/local.c b/cmirror/src/local.c
index 0a9c581..4d66e17 100644
--- a/cmirror/src/local.c
+++ b/cmirror/src/local.c
@@ -17,58 +17,42 @@
static int cn_fd; /* Connector (netlink) socket fd */
-static int kernel_recv_helper(void *data, int in_size)
+/* FIXME: merge this function with kernel_send_helper */
+static int kernel_ack(uint32_t seq, int error)
{
- int len;
- struct cn_msg *msg;
- unsigned char buf[2048];
-
- /* FIXME: get rid of buf and use passed in 'data' */
- memset(buf, 0, sizeof(buf));
+ int r;
+ unsigned char buf[sizeof(struct nlmsghdr) + sizeof(struct cn_msg)];
+ struct nlmsghdr *nlh = (struct nlmsghdr *)buf;
+ struct cn_msg *msg = NLMSG_DATA(nlh);
- len = recv(cn_fd, buf, sizeof(buf), 0);
- if (len < 0) {
- LOG_ERROR("Failed to recv message from kernel");
- return -errno;
+ if (error < 0) {
+ LOG_ERROR("Programmer error: error codes must be positive");
+ return -EINVAL;
}
- switch (((struct nlmsghdr *)buf)->nlmsg_type) {
- case NLMSG_ERROR:
- LOG_ERROR("Unable to recv message from kernel: NLMSG_ERROR");
- return -EBADE;
- case NLMSG_DONE:
- msg = (struct cn_msg *) NLMSG_DATA((struct nlmsghdr *)buf);
- len -= sizeof(struct nlmsghdr);
-
- if (len < sizeof(struct cn_msg)) {
- LOG_ERROR("Incomplete request from kernel received");
- return -EBADE;
- }
-
- if (msg->len > in_size) {
- LOG_ERROR("Not enough space to receive kernel request (%d/%d)",
- msg->len, in_size);
- return -EBADE;
- }
-
- if (!msg->len)
- LOG_ERROR("Zero length message received");
+ memset(buf, 0, sizeof(buf));
- len -= sizeof(struct cn_msg);
+ nlh->nlmsg_seq = 0;
+ nlh->nlmsg_pid = getpid();
+ nlh->nlmsg_type = NLMSG_DONE;
+ nlh->nlmsg_len = NLMSG_LENGTH(sizeof(struct cn_msg));
+ nlh->nlmsg_flags = 0;
- if (len < msg->len)
- LOG_ERROR("len = %d, msg->len = %d", len, msg->len);
+ msg->len = 0;
+ msg->id.idx = 0x4;
+ msg->id.val = 0x1;
+ msg->seq = seq;
+ msg->ack = error;
- memcpy(data, msg->data, msg->len);
- break;
- default:
- LOG_ERROR("Unknown nlmsg_type");
+ r = send(cn_fd, nlh, NLMSG_LENGTH(sizeof(struct cn_msg)), 0);
+ /* FIXME: do better error processing */
+ if (r <= 0)
return -EBADE;
- }
return 0;
}
+
/*
* kernel_recv
* @tfr: the newly allocated request from kernel
@@ -82,6 +66,9 @@ static int kernel_recv_helper(void *data, int in_size)
static int kernel_recv(struct clog_tfr **tfr)
{
int r = 0;
+ int len;
+ struct cn_msg *msg;
+ char buf[2048];
/*
* A failure to allocate space means the request is lost
@@ -97,14 +84,69 @@ static int kernel_recv(struct clog_tfr **tfr)
}
memset(*tfr, 0, DM_CLOG_TFR_SIZE);
+ memset(buf, 0, sizeof(buf));
+
+ len = recv(cn_fd, buf, sizeof(buf), 0);
+ if (len < 0) {
+ LOG_ERROR("Failed to recv message from kernel");
+ r = -errno;
+ goto fail;
+ }
+
+ switch (((struct nlmsghdr *)buf)->nlmsg_type) {
+ case NLMSG_ERROR:
+ LOG_ERROR("Unable to recv message from kernel: NLMSG_ERROR");
+ r = -EBADE;
+ goto fail;
+ case NLMSG_DONE:
+ msg = (struct cn_msg *)NLMSG_DATA((struct nlmsghdr *)buf);
+ len -= sizeof(struct nlmsghdr);
+
+ if (len < sizeof(struct cn_msg)) {
+ LOG_ERROR("Incomplete request from kernel received");
+ r = -EBADE;
+ goto fail;
+ }
- r = kernel_recv_helper(*tfr, DM_CLOG_TFR_SIZE);
+ if (msg->len > DM_CLOG_TFR_SIZE) {
+ LOG_ERROR("Not enough space to receive kernel request (%d/%d)",
+ msg->len, DM_CLOG_TFR_SIZE);
+ r = -EBADE;
+ goto fail;
+ }
+
+ if (!msg->len)
+ LOG_ERROR("Zero length message received");
+
+ len -= sizeof(struct cn_msg);
+
+ if (len < msg->len)
+ LOG_ERROR("len = %d, msg->len = %d", len, msg->len);
+
+ memcpy(*tfr, msg->data, msg->len);
+
+ if (!(*tfr)->request_type) {
+ LOG_ERROR("Bad transmission, requesting resend [%u]", msg->seq);
+ if (kernel_ack(msg->seq, EAGAIN)) {
+ LOG_ERROR("Failed to NACK kernel transmission [%u]",
+ msg->seq);
+ r = -EBADE;
+ } else
+ r = -EAGAIN;
+ }
+ break;
+ default:
+ LOG_ERROR("Unknown nlmsg_type");
+ r = -EBADE;
+ }
+
+fail:
if (r) {
queue_add(*tfr, free_queue);
*tfr = NULL;
}
- return (r == -EAGAIN) ? 0 : r;
+ return r;
}
static int kernel_send_helper(void *data, int out_size)
hooks/post-receive
--
Cluster Project
More information about the Cluster-devel
mailing list