[Cluster-devel] cluster/cmirror-kernel/src dm-cmirror-client.c ...
jbrassow at sourceware.org
jbrassow at sourceware.org
Thu Jul 27 23:11:56 UTC 2006
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL4
Changes by: jbrassow at sourceware.org 2006-07-27 23:11:55
Modified files:
cmirror-kernel/src: dm-cmirror-client.c dm-cmirror-common.h
dm-cmirror-server.c
Log message:
- further tightening for 199826
We now refuse to read/write the disk log if we are suspended. We also
add BUG_ON(<suspended>) to operations that do I/O to the log device.
The reason for the BUG_ON() is that it is better to drop the machine
than to have it hang the cluster while it attempts to read/write from
a suspended device. That being said, it should now be impossible to
get to those functions which would perform I/O operations during
suspension.
I have still seen cases where the mirror will stall. However, I think
this is due to LVM (clvmd), because it happens when a mirror is created
while the log device is suspended - which must not happen. I've only
seen this when doing simultaneous create/convert/remove from all nodes
in the cluster.
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-client.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.22&r2=1.1.2.23
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-common.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.9&r2=1.1.2.10
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror-kernel/src/dm-cmirror-server.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.1.2.15&r2=1.1.2.16
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-client.c 2006/07/22 22:19:34 1.1.2.22
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-client.c 2006/07/27 23:11:55 1.1.2.23
@@ -302,7 +302,7 @@
lc->server_id = lr.u.lr_coordinator;
} else {
/* ATTENTION -- what do we do with this ? */
- DMWARN("Failed to receive election results from server");
+ DMWARN("Failed to receive election results from server: %d", len);
error = len;
}
@@ -363,21 +363,7 @@
iov.iov_len = sizeof(struct log_request);
iov.iov_base = lr;
-/*
- DMERR("To :: 0x%x, %s",
- saddr_in.sin_addr.s_addr,
- (lr->lr_type == LRT_IS_CLEAN)? "LRT_IS_CLEAN":
- (lr->lr_type == LRT_IN_SYNC)? "LRT_IN_SYNC":
- (lr->lr_type == LRT_MARK_REGION)? "LRT_MARK_REGION":
- (lr->lr_type == LRT_GET_RESYNC_WORK)? "LRT_GET_RESYNC_WORK":
- (lr->lr_type == LRT_GET_SYNC_COUNT)? "LRT_GET_SYNC_COUNT":
- (lr->lr_type == LRT_CLEAR_REGION)? "LRT_CLEAR_REGION":
- (lr->lr_type == LRT_COMPLETE_RESYNC_WORK)? "LRT_COMPLETE_RESYNC_WORK":
- (lr->lr_type == LRT_MASTER_LEAVING)? "LRT_MASTER_LEAVING":
- (lr->lr_type == LRT_ELECTION)? "LRT_ELECTION":
- (lr->lr_type == LRT_SELECTION)? "LRT_SELECTION": "UNKNOWN"
- );
-*/
+
if(lr->lr_type == LRT_MARK_REGION){
mark_req2ser++;
}
@@ -453,25 +439,28 @@
request_retry_count,
request_count,
dm_div_up(request_retry_count*100, request_count));
+ DMDEBUG("Last request:");
+ DMDEBUG(" - my_id :: %u", my_id);
+ DMDEBUG(" - server :: %u", lc->server_id);
+ DMDEBUG(" - log uuid:: %s (%s)",
+ lc->uuid + (strlen(lc->uuid) - 8),
+ atomic_read(&lc->suspended) ? "suspended" : "active");
+ DMDEBUG(" - request :: %s",
+ (type == LRT_IS_CLEAN)? "LRT_IS_CLEAN":
+ (type == LRT_IN_SYNC)? "LRT_IN_SYNC":
+ (type == LRT_MARK_REGION)? "LRT_MARK_REGION":
+ (type == LRT_GET_RESYNC_WORK)? "LRT_GET_RESYNC_WORK":
+ (type == LRT_GET_SYNC_COUNT)? "LRT_GET_SYNC_COUNT":
+ (type == LRT_CLEAR_REGION)? "LRT_CLEAR_REGION":
+ (type == LRT_COMPLETE_RESYNC_WORK)? "LRT_COMPLETE_RESYNC_WORK":
+ (type == LRT_MASTER_LEAVING)? "LRT_MASTER_LEAVING":
+ (type == LRT_ELECTION)? "LRT_ELECTION":
+ (type == LRT_SELECTION)? "LRT_SELECTION": "UNKNOWN");
+ DMDEBUG(" - error :: %d", error);
}
}
if(lr) kfree(lr);
-#if 0
- DMINFO("My (%u) request (%s) to server (%u) failed :: %d",
- my_id,
- (type == LRT_IS_CLEAN)? "LRT_IS_CLEAN":
- (type == LRT_IN_SYNC)? "LRT_IN_SYNC":
- (type == LRT_MARK_REGION)? "LRT_MARK_REGION":
- (type == LRT_GET_RESYNC_WORK)? "LRT_GET_RESYNC_WORK":
- (type == LRT_GET_SYNC_COUNT)? "LRT_GET_SYNC_COUNT":
- (type == LRT_CLEAR_REGION)? "LRT_CLEAR_REGION":
- (type == LRT_COMPLETE_RESYNC_WORK)? "LRT_COMPLETE_RESYNC_WORK":
- (type == LRT_MASTER_LEAVING)? "LRT_MASTER_LEAVING":
- (type == LRT_ELECTION)? "LRT_ELECTION":
- (type == LRT_SELECTION)? "LRT_SELECTION": "UNKNOWN",
- lc->server_id, error);
-#endif
return error;
}
@@ -739,7 +728,7 @@
DMINFO("Leaving while clear region requests remain.");
list_del_init(&lc->log_list);
- if(lc->server_id == my_id)
+ if ((lc->server_id == my_id) && !atomic_read(&lc->suspended))
consult_server(lc, 0, LRT_MASTER_LEAVING, NULL);
sock_release(lc->client_sock);
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-common.h 2006/07/22 22:19:34 1.1.2.9
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-common.h 2006/07/27 23:11:55 1.1.2.10
@@ -22,6 +22,7 @@
#define DMWARN(f, x...) printk(KERN_WARNING DM_NAME ": " f "\n" , ## x)
#define DMERR(f, x...) printk(KERN_ERR DM_NAME ": " f "\n" , ## x)
#define DMINFO(f, x...) printk(KERN_INFO DM_NAME ": " f "\n" , ## x)
+#define DMDEBUG(f, x...) printk(KERN_DEBUG DM_NAME ": " f "\n" , ## x)
#define DMEMIT(x...) sz += ((sz >= maxlen) ? \
0 : scnprintf(result + sz, maxlen - sz, x))
--- cluster/cmirror-kernel/src/Attic/dm-cmirror-server.c 2006/07/22 22:50:38 1.1.2.15
+++ cluster/cmirror-kernel/src/Attic/dm-cmirror-server.c 2006/07/27 23:11:55 1.1.2.16
@@ -107,6 +107,7 @@
if (!log->log_dev)
return 0;
+ BUG_ON(atomic_read(&log->suspended));
r = dm_io_sync_vm(1, &log->header_location, READ,
log->disk_header, &ebits);
if (unlikely(r))
@@ -137,6 +138,7 @@
if (!log->log_dev)
return 0;
+ BUG_ON(atomic_read(&log->suspended));
header_to_disk(&log->header, log->disk_header);
return dm_io_sync_vm(1, &log->header_location, WRITE,
log->disk_header, &ebits);
@@ -180,6 +182,7 @@
if (!log->log_dev)
return 0;
+ BUG_ON(atomic_read(&log->suspended));
r = dm_io_sync_vm(1, &log->bits_location, READ,
log->clean_bits, &ebits);
@@ -196,6 +199,7 @@
if (!log->log_dev)
return 0;
+ BUG_ON(atomic_read(&log->suspended));
return dm_io_sync_vm(1, &log->bits_location, WRITE,
log->clean_bits, &ebits);
}
@@ -295,7 +299,8 @@
struct region_user *tmp_ru, *ru;
unsigned char live_nodes[16]; /* Attention -- max of 128 nodes... */
- DMINFO("Disk Resume::");
+ DMDEBUG("Disk Resume:: %s (%s)", lc->uuid + (strlen(lc->uuid) - 8),
+ atomic_read(&lc->suspended) ? "suspended" : "active");
debug_disk_write = 1;
memset(live_nodes, 0, sizeof(live_nodes));
@@ -355,20 +360,20 @@
}
}
- DMINFO(" Live nodes :: %d", global_count);
- DMINFO(" In-Use Regions :: %d", good_count+bad_count);
- DMINFO(" Good IUR's :: %d", good_count);
- DMINFO(" Bad IUR's :: %d", bad_count);
+ DMDEBUG(" Live nodes :: %d", global_count);
+ DMDEBUG(" In-Use Regions :: %d", good_count+bad_count);
+ DMDEBUG(" Good IUR's :: %d", good_count);
+ DMDEBUG(" Bad IUR's :: %d", bad_count);
lc->sync_count = count_bits32(lc->sync_bits, lc->bitset_uint32_count);
lc->sync_search = 0;
- DMINFO(" Sync count :: %Lu", lc->sync_count);
- DMINFO(" Disk Region count :: %Lu", lc->header.nr_regions);
- DMINFO(" Region count :: %Lu", lc->region_count);
+ DMDEBUG(" Sync count :: %Lu", lc->sync_count);
+ DMDEBUG(" Disk Region count :: %Lu", lc->header.nr_regions);
+ DMDEBUG(" Region count :: %Lu", lc->region_count);
if(lc->header.nr_regions != lc->region_count){
- DMINFO(" NOTE: Mapping has changed.");
+ DMDEBUG(" NOTE: Mapping has changed.");
}
/* Take this out for now.
if(list_empty(&lc->region_users) && (lc->sync_count != lc->header.nr_regions)){
@@ -398,13 +403,13 @@
}
*/
- DMINFO("Marked regions::");
+ DMDEBUG("Marked regions::");
i = print_zero_bits((unsigned char *)lc->clean_bits, 0, lc->region_count);
- DMINFO(" Total = %d", i);
+ DMDEBUG(" Total = %d", i);
- DMINFO("Out-of-sync regions::");
+ DMDEBUG("Out-of-sync regions::");
i = print_zero_bits((unsigned char *)lc->sync_bits, 0, lc->region_count);
- DMINFO(" Total = %d", i);
+ DMDEBUG(" Total = %d", i);
/* set the correct number of regions in the header */
lc->header.nr_regions = lc->region_count;
@@ -529,7 +534,7 @@
ru = find_ru(lc, who, lr->u.lr_region);
if(!ru){
- DMINFO("Request to remove unrecorded region user (%u/%Lu)",
+ DMDEBUG("Request to remove unrecorded region user (%u/%Lu)",
who, lr->u.lr_region);
return -EINVAL;
} else {
@@ -596,7 +601,7 @@
(info < 1000 && !(info%100)) ||
(info < 200 && !(info%25)) ||
(info < 6)){
- DMINFO(SECTOR_FORMAT " out-of-sync regions remaining for %s.",
+ DMDEBUG(SECTOR_FORMAT " out-of-sync regions remaining for %s.",
lc->region_count - lc->sync_count,
lc->uuid + (strlen(lc->uuid) - 8));
}
@@ -843,9 +848,12 @@
return -1;
}
if(lc && (old != lc->server_id) && (my_id == lc->server_id)){
- DMINFO("I'm the cluster mirror log server for %s",
+ DMDEBUG("I'm the cluster mirror log server for %s",
lc->uuid + (strlen(lc->uuid) - 8));
- disk_resume(lc);
+ if (!atomic_read(&lc->suspended))
+ disk_resume(lc);
+ else
+ DMDEBUG("Not reading disk log because I'm suspended.");
}
goto reply;
}
@@ -860,6 +868,30 @@
goto reply;
}
+ if (atomic_read(&lc->suspended)) {
+ nodeid = ipaddr_to_nodeid((struct sockaddr *)msg.msg_name);
+ /*
+ DMDEBUG("Getting request while server (%u) is suspended:", my_id);
+ DMDEBUG(" - Requester :: %u", nodeid);
+ DMDEBUG(" - log uuid :: %s", lc->uuid + (strlen(lc->uuid) - 8));
+ DMDEBUG(" - req type :: %s",
+ (lr.lr_type == LRT_IS_CLEAN)? "LRT_IS_C LEAN":
+ (lr.lr_type == LRT_IN_SYNC)? "LRT_IN_SYNC":
+ (lr.lr_type == LRT_MARK_REGION)? "LRT_MARK_REGION":
+ (lr.lr_type == LRT_GET_RESYNC_WORK)? "LRT_GET_RESYNC_WORK":
+ (lr.lr_type == LRT_GET_SYNC_COUNT)? "LRT_GET_SYNC_COUNT":
+ (lr.lr_type == LRT_CLEAR_REGION)? "LRT_CLEAR_REGION":
+ (lr.lr_type == LRT_COMPLETE_RESYNC_WORK)? "LRT_COMPLETE_RESYNC_WORK":
+ (lr.lr_type == LRT_MASTER_LEAVING)? "LRT_MASTER_LEAVING":
+ (lr.lr_type == LRT_ELECTION)? "LRT_ELECTION":
+ (lr.lr_type == LRT_SELECTION)? "LRT_SELECTION": "UNKNOWN");
+ */
+ if (my_id != nodeid) {
+ lr.u.lr_int_rtn = -ENXIO;
+ goto reply;
+ }
+ }
+
switch(lr.lr_type){
case LRT_IS_CLEAN:
error = server_is_clean(lc, &lr);
@@ -1004,7 +1036,10 @@
list_for_each_entry(lc, &log_list_head, log_list){
if(lc->server_id == my_id){
- disk_resume(lc);
+ if (!atomic_read(&lc->suspended))
+ disk_resume(lc);
+ else
+ DMDEBUG("Not reading disk log because I'm suspended.");
}
}
break;
More information about the Cluster-devel
mailing list