[Linux-cluster] GFS file system corruption?
AJ Lewis
alewis at redhat.com
Mon Aug 29 16:48:54 UTC 2005
On Sun, Aug 28, 2005 at 02:31:13PM -0600, Matthew B. Brookover wrote:
> I have 6 computers running Redhat Enterrpise 3 release 5, running kernel
> 2.4.21-32.0.1.ELsmp.
>
> >From the source code I compiled GFS 6.0.2.20-2. The SAN is an ISCSI
> based storage system from LeftHand Networks. Using ext3, the postmark
> disk test works fine, on a GFS file system, we get a number of errors.
> The output from both postmark runs is below.
Looks like you're running into bugs
https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=160835
and
https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=160525
which have been fixed in the latest code. Unfortunately there is not a GFS
6.0 srpm released yet that contains this fix, but I've attached a patch that
clears them up, as well as a couple other minor issues.
I'm not sure about the postmark errors you're seeing - hopefully someone else
on the list can shed some light on them. But try the fsck with these patches
and see if there are any actual problems with the fs shown.
Regards,
--
AJ Lewis Voice: 612-638-0500
Red Hat E-Mail: alewis at redhat.com
One Main Street SE, Suite 209
Minneapolis, MN 55414
Current GPG fingerprint = D9F8 EDCE 4242 855F A03D 9B63 F50C 54A8 578C 8715
Grab the key at: http://people.redhat.com/alewis/gpg.html or one of the
many keyservers out there...
-------------- next part --------------
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/fs_recovery.c gfs_fsck-6.0-cvs/fs_recovery.c
--- gfs_fsck-6.0.2.20/fs_recovery.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/fs_recovery.c 2005-06-21 13:39:10.000000000 -0500
@@ -37,6 +37,8 @@ static int reconstruct_single_journal(st
srandom(time(NULL));
sequence = jdesc->ji_nsegment / (RAND_MAX + 1.0) * random();
+ log_info("Clearing journal %d\n", jnum);
+
for (seg = 0; seg < jdesc->ji_nsegment; seg++){
memset(buf, 0, sdp->sb.sb_bsize);
memset(&lh, 0, sizeof(struct gfs_log_header));
@@ -75,9 +77,13 @@ static int reconstruct_single_journal(st
int reconstruct_journals(struct fsck_sb *sdp){
int i;
- for(i=0; i < sdp->journals; i++)
+ log_warn("Clearing journals (this may take a while)\n");
+ for(i=0; i < sdp->journals; i++) {
+ if((i % 10) == 0)
+ log_at_notice(".");
if(reconstruct_single_journal(sdp, i))
return -1;
-
+ }
+ log_notice("Cleared journals\n");
return 0;
}
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/fsck.h gfs_fsck-6.0-cvs/fsck.h
--- gfs_fsck-6.0.2.20/fsck.h 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/fsck.h 2005-06-20 09:55:52.000000000 -0500
@@ -46,6 +46,7 @@ struct options {
int initialize(struct fsck_sb *sbp);
void destroy(struct fsck_sb *sbp);
+int block_mounters(struct fsck_sb *sbp, int block_em);
int pass1(struct fsck_sb *sbp);
int pass1b(struct fsck_sb *sbp);
int pass1c(struct fsck_sb *sbp);
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/initialize.c gfs_fsck-6.0-cvs/initialize.c
--- gfs_fsck-6.0.2.20/initialize.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/initialize.c 2005-06-20 09:55:52.000000000 -0500
@@ -52,6 +52,37 @@ int init_journals(struct fsck_sb *sbp)
return 0;
}
+/**
+ * block_mounters
+ *
+ * Change the lock protocol so nobody can mount the fs
+ *
+ */
+int block_mounters(struct fsck_sb *sbp, int block_em)
+{
+ if(block_em) {
+ /* verify it starts with lock_ */
+ if(!strncmp(sbp->sb.sb_lockproto, "lock_", 5)) {
+ /* Change lock_ to fsck_ */
+ memcpy(sbp->sb.sb_lockproto, "fsck_", 5);
+ }
+ /* FIXME: Need to do other verification in the else
+ * case */
+ } else {
+ /* verify it starts with fsck_ */
+ /* verify it starts with lock_ */
+ if(!strncmp(sbp->sb.sb_lockproto, "fsck_", 5)) {
+ /* Change fsck_ to lock_ */
+ memcpy(sbp->sb.sb_lockproto, "lock_", 5);
+ }
+ }
+
+ if(write_sb(sbp)) {
+ stack;
+ return -1;
+ }
+ return 0;
+}
/*
@@ -330,8 +361,18 @@ int init_sbp(struct fsck_sb *sbp)
}
}
if (fill_super_block(sbp)) {
+ stack;
return -1;
}
+
+ /* Change lock protocol to be fsck_* instead of lock_* */
+ if(!sbp->opts->no) {
+ if(block_mounters(sbp, 1)) {
+ log_err("Unable to block other mounters\n");
+ return -1;
+ }
+ }
+
/* verify various things */
if(init_journals(sbp)) {
@@ -344,9 +385,14 @@ int init_sbp(struct fsck_sb *sbp)
void destroy_sbp(struct fsck_sb *sbp)
{
- empty_super_block(sbp);
- if(!sbp->opts->no)
+ if(!sbp->opts->no) {
+ if(block_mounters(sbp, 0)) {
+ log_warn("Unable to unblock other mounters - manual intevention required\n");
+ log_warn("Use 'gfs_tool sb <device> proto' to fix\n");
+ }
fsync(sbp->diskfd);
+ }
+ empty_super_block(sbp);
close(sbp->diskfd);
}
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/log.c gfs_fsck-6.0-cvs/log.c
--- gfs_fsck-6.0.2.20/log.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/log.c 2005-06-21 13:39:10.000000000 -0500
@@ -60,7 +60,8 @@ void print_msg(int priority, char *file,
return;
}
-void print_fsck_log(int priority, char *file, int line, const char *format, ...)
+
+void print_fsck_log(int iif, int priority, char *file, int line, const char *format, ...)
{
va_list args;
@@ -70,10 +71,9 @@ void print_fsck_log(int priority, char *
transform = _(format);
- if(_state.print_level >= priority) {
+ if((_state.print_level == priority) ||
+ (!iif && (_state.print_level >= priority)))
print_msg(priority, file, line, transform, args);
- }
-
va_end(args);
}
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/log.h gfs_fsck-6.0-cvs/log.h
--- gfs_fsck-6.0.2.20/log.h 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/log.h 2005-06-21 13:39:10.000000000 -0500
@@ -23,46 +23,76 @@
-#define print_log(priority, format...) \
+#define print_log(iif, priority, format...) \
do { \
- print_fsck_log(priority, __FILE__, __LINE__, ## format); \
+ print_fsck_log(iif, priority, __FILE__, __LINE__, ## format); \
} while(0)
#define log_debug(format...) \
do { \
- print_log(MSG_DEBUG, format); \
+ print_log(0, MSG_DEBUG, format); \
} while(0)
#define log_info(format...) \
do { \
- print_log(MSG_INFO, format); \
+ print_log(0, MSG_INFO, format); \
} while(0)
#define log_notice(format...) \
do { \
- print_log(MSG_NOTICE, format); \
+ print_log(0, MSG_NOTICE, format); \
} while(0)
#define log_warn(format...) \
do { \
- print_log(MSG_WARN, format); \
+ print_log(0, MSG_WARN, format); \
} while(0)
#define log_err(format...) \
do { \
- print_log(MSG_ERROR, format); \
+ print_log(0, MSG_ERROR, format); \
} while(0)
#define log_crit(format...) \
do { \
- print_log(MSG_CRITICAL, format); \
+ print_log(0, MSG_CRITICAL, format); \
} while(0)
#define stack log_debug("<backtrace> - %s()\n", __func__)
+#define log_at_debug(format...) \
+do { \
+ print_log(1, MSG_DEBUG, format); \
+} while(0)
+
+#define log_at_info(format...) \
+do { \
+ print_log(1, MSG_INFO, format); \
+} while(0)
+
+#define log_at_notice(format...) \
+do { \
+ print_log(1, MSG_NOTICE, format); \
+} while(0)
+
+#define log_at_warn(format...) \
+do { \
+ print_log(1, MSG_WARN, format); \
+} while(0)
+
+#define log_at_err(format...) \
+do { \
+ print_log(1, MSG_ERROR, format); \
+} while(0)
+
+#define log_at_crit(format...) \
+do { \
+ print_log(1, MSG_CRITICAL, format); \
+} while(0)
+
void increase_verbosity(void);
void decrease_verbosity(void);
-void print_fsck_log(int priority, char *file, int line, const char *format, ...);
+void print_fsck_log(int iif, int priority, char *file, int line, const char *format, ...);
int query(struct fsck_sb *sbp, const char *format, ...);
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/metawalk.c gfs_fsck-6.0-cvs/metawalk.c
--- gfs_fsck-6.0.2.20/metawalk.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/metawalk.c 2005-07-11 13:22:39.000000000 -0500
@@ -464,6 +464,8 @@ static int build_metalist(struct fsck_in
for (ptr = (uint64 *)(bh->b_data + head_size);
(char *)ptr < (bh->b_data + bh->b_size);
ptr++) {
+ nbh = NULL;
+
if (!*ptr)
continue;
@@ -481,9 +483,11 @@ static int build_metalist(struct fsck_in
continue;
}
if(!nbh) {
- /* FIXME: error checking */
- get_and_read_buf(ip->i_sbd, block,
- &nbh, 0);
+ if(get_and_read_buf(ip->i_sbd, block,
+ &nbh, 0)) {
+ stack;
+ goto fail;
+ }
}
osi_list_add(&nbh->b_list, cur_list);
}
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/pass1b.c gfs_fsck-6.0-cvs/pass1b.c
--- gfs_fsck-6.0.2.20/pass1b.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/pass1b.c 2005-07-11 13:22:39.000000000 -0500
@@ -77,7 +77,17 @@ static int check_data(struct fsck_inode
static int check_eattr_indir(struct fsck_inode *ip, uint64_t block,
uint64_t parent, osi_buf_t **bh, void *private)
{
+ struct fsck_sb *sbp = ip->i_sbd;
+ osi_buf_t *indir_bh = NULL;
+
inc_if_found(block, 0, private);
+ if(get_and_read_buf(sbp, block, &indir_bh, 0)){
+ log_warn("Unable to read EA leaf block #%"PRIu64".\n",
+ block);
+ return 1;
+ }
+
+ *bh = indir_bh;
return 0;
}
@@ -85,8 +95,17 @@ static int check_eattr_indir(struct fsck
static int check_eattr_leaf(struct fsck_inode *ip, uint64_t block,
uint64_t parent, osi_buf_t **bh, void *private)
{
+ struct fsck_sb *sbp = ip->i_sbd;
+ osi_buf_t *leaf_bh = NULL;
+
inc_if_found(block, 0, private);
+ if(get_and_read_buf(sbp, block, &leaf_bh, 0)){
+ log_warn("Unable to read EA leaf block #%"PRIu64".\n",
+ block);
+ return 1;
+ }
+ *bh = leaf_bh;
return 0;
}
@@ -326,11 +345,14 @@ int find_block_ref(struct fsck_sb *sbp,
stack;
return -1;
}
+ log_info("Checking inode %"PRIu64"'s metatree for references to block %"PRIu64"\n",
+ inode, b->block_no);
if(check_metatree(ip, &find_refs)) {
stack;
free_inode(&ip);
return -1;
}
+ log_info("Done checking metatree\n");
if (myfi.found) {
if(!(id = malloc(sizeof(*id)))) {
@@ -463,7 +485,9 @@ int pass1b(struct fsck_sb *sbp)
/* Rescan the fs looking for pointers to blocks that are in
* the duplicate block map */
log_info("Scanning filesystem for inodes containing duplicate blocks...\n");
- for(i = 0; i < sbp->last_fs_block; i++) {
+ log_debug("Filesystem has %"PRIu64" blocks total\n", sbp->last_fs_block);
+ for(i = 0; i < sbp->last_fs_block; i += 1) {
+ log_debug("Scanning block %"PRIu64" for inodes\n", i);
if(block_check(sbp->bl, i, &q)) {
stack;
return -1;
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/pass1c.c gfs_fsck-6.0-cvs/pass1c.c
--- gfs_fsck-6.0.2.20/pass1c.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/pass1c.c 2005-06-15 10:51:08.000000000 -0500
@@ -46,6 +46,7 @@ int check_eattr_indir(struct fsck_inode
int *update = (int *) private;
struct fsck_sb *sbp = ip->i_sbd;
struct block_query q;
+ osi_buf_t *indir_bh;
if(check_range(sbp, block)) {
log_err("Extended attributes indirect block out of range...removing\n");
@@ -63,7 +64,15 @@ int check_eattr_indir(struct fsck_inode
*update = 1;
return 1;
}
+ else if(get_and_read_buf(sbp, block, &indir_bh, 0)){
+ log_warn("Unable to read EA leaf block #%"PRIu64".\n",
+ block);
+ ip->i_di.di_eattr = 0;
+ *update = 1;
+ return 1;
+ }
+ *bh = indir_bh;
return 0;
}
int check_eattr_leaf(struct fsck_inode *ip, uint64_t block,
@@ -72,6 +81,7 @@ int check_eattr_leaf(struct fsck_inode *
int *update = (int *) private;
struct fsck_sb *sbp = ip->i_sbd;
struct block_query q;
+ osi_buf_t *leaf_bh;
if(check_range(sbp, block)) {
log_err("Extended attributes block out of range...removing\n");
@@ -89,6 +99,15 @@ int check_eattr_leaf(struct fsck_inode *
*update = 1;
return 1;
}
+ else if(get_and_read_buf(sbp, block, &leaf_bh, 0)){
+ log_warn("Unable to read EA leaf block #%"PRIu64".\n",
+ block);
+ ip->i_di.di_eattr = 0;
+ *update = 1;
+ return 1;
+ }
+
+ *bh = leaf_bh;
return 0;
@@ -200,7 +219,7 @@ int pass1c(struct fsck_sb *sbp)
while (!find_next_block_type(sbp->bl, eattr_block, &block_no)) {
log_info("EA in inode %"PRIu64"\n", block_no);
- if(get_and_read_buf(sbp, eattr_block, &bh, 0)) {
+ if(get_and_read_buf(sbp, block_no, &bh, 0)) {
stack;
return -1;
}
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/pass5.c gfs_fsck-6.0-cvs/pass5.c
--- gfs_fsck-6.0.2.20/pass5.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/pass5.c 2005-06-20 16:22:23.000000000 -0500
@@ -196,21 +196,38 @@ int check_block_status(struct fsck_sb *s
block_status = convert_mark(q.block_type, count);
if(rg_status != block_status) {
- log_err("ondisk and fsck bitmaps differ at block %"
- PRIu64"\n", block);
log_debug("Ondisk is %u - FSCK thinks it is %u (%u)\n",
rg_status, block_status, q.block_type);
- if(query(sbp, "Fix bitmap for block %"PRIu64"? (y/n) ",
- block)) {
- if(fs_set_bitmap(sbp, block, block_status)) {
- log_err("Failed.\n");
+ if((rg_status == GFS_BLKST_FREEMETA) &&
+ (block_status == GFS_BLKST_FREE)) {
+ log_info("Converting free metadata block at %"
+ PRIu64" to a free data block\n", block);
+ if(!sbp->opts->no) {
+ if(fs_set_bitmap(sbp, block, block_status)) {
+ log_warn("Failed to convert free metadata block to free data block at %PRIu64.\n", block);
+ }
+ else {
+ log_info("Succeeded.\n");
+ }
}
- else {
- log_err("Succeeded.\n");
+ }
+ else {
+
+ log_err("ondisk and fsck bitmaps differ at"
+ " block %"PRIu64"\n", block);
+
+ if(query(sbp, "Fix bitmap for block %"
+ PRIu64"? (y/n) ", block)) {
+ if(fs_set_bitmap(sbp, block, block_status)) {
+ log_err("Failed.\n");
+ }
+ else {
+ log_err("Succeeded.\n");
+ }
+ } else {
+ log_err("Bitmap at block %"PRIu64
+ " left inconsistent\n", block);
}
- } else {
- log_err("Bitmap at block %"PRIu64
- " left inconsistent\n", block);
}
}
(*rg_block)++;
@@ -224,13 +241,19 @@ int check_block_status(struct fsck_sb *s
return 0;
}
+#define FREE_COUNT 1
+#define USED_INODE_COUNT 2
+#define FREE_INODE_COUNT 4
+#define USED_META_COUNT 8
+#define FREE_META_COUNT 16
+#define CONVERT_FREEMETA_TO_FREE (FREE_COUNT | FREE_META_COUNT)
int update_rgrp(struct fsck_rgrp *rgp, uint32_t *count)
{
- int update = 0;
uint32_t i;
fs_bitmap_t *bits;
uint64_t rg_block = 0;
+ uint8_t bmap = 0;
for(i = 0; i < rgp->rd_ri.ri_length; i++) {
bits = &rgp->rd_bits[i];
@@ -243,42 +266,63 @@ int update_rgrp(struct fsck_rgrp *rgp, u
}
/* Compare the rgrps counters with what we found */
- /* actually adjust counters and write out to disk */
if(rgp->rd_rg.rg_free != count[0]) {
- log_err("free count inconsistent: is %u should be %u\n",
- rgp->rd_rg.rg_free, count[0] );
- rgp->rd_rg.rg_free = count[0];
- update = 1;
+ bmap |= FREE_COUNT;
}
if(rgp->rd_rg.rg_useddi != count[1]) {
- log_err("used inode count inconsistent: is %u should be %u\n",
- rgp->rd_rg.rg_useddi, count[1]);
- rgp->rd_rg.rg_useddi = count[1];
- update = 1;
+ bmap |= USED_INODE_COUNT;
}
if(rgp->rd_rg.rg_freedi != count[2]) {
- log_err("free inode count inconsistent: is %u should be %u\n",
- rgp->rd_rg.rg_freedi, count[2]);
- rgp->rd_rg.rg_freedi = count[2];
- update = 1;
+ bmap |= FREE_INODE_COUNT;
}
if(rgp->rd_rg.rg_usedmeta != count[3]) {
- log_err("used meta count inconsistent: is %u should be %u\n",
- rgp->rd_rg.rg_usedmeta, count[3]);
- rgp->rd_rg.rg_usedmeta = count[3];
- update = 1;
+ bmap |= USED_META_COUNT;
}
if(rgp->rd_rg.rg_freemeta != count[4]) {
- log_err("free meta count inconsistent: is %u should be %u\n",
- rgp->rd_rg.rg_freemeta, count[4]);
- rgp->rd_rg.rg_freemeta = count[4];
- update = 1;
+ bmap |= FREE_META_COUNT;
}
- if(update) {
+ if(bmap && !(bmap & ~CONVERT_FREEMETA_TO_FREE)) {
+ log_notice("Converting %d unused metadata blocks to free data blocks...\n",
+ rgp->rd_rg.rg_freemeta - count[4]);
+ rgp->rd_rg.rg_free = count[0];
+ rgp->rd_rg.rg_freemeta = count[4];
+ gfs_rgrp_out(&rgp->rd_rg, BH_DATA(rgp->rd_bh[0]));
+ if(!rgp->rd_sbd->opts->no) {
+ write_buf(rgp->rd_sbd, rgp->rd_bh[0], 0);
+ }
+ } else if(bmap) {
+ /* actually adjust counters and write out to disk */
+ if(bmap & FREE_COUNT) {
+ log_err("free count inconsistent: is %u should be %u\n",
+ rgp->rd_rg.rg_free, count[0] );
+ rgp->rd_rg.rg_free = count[0];
+ }
+ if(bmap & USED_INODE_COUNT) {
+ log_err("used inode count inconsistent: is %u should be %u\n",
+ rgp->rd_rg.rg_useddi, count[1]);
+ rgp->rd_rg.rg_useddi = count[1];
+ }
+ if(bmap & FREE_INODE_COUNT) {
+ log_err("free inode count inconsistent: is %u should be %u\n",
+ rgp->rd_rg.rg_freedi, count[2]);
+ rgp->rd_rg.rg_freedi = count[2];
+ }
+ if(bmap & USED_META_COUNT) {
+ log_err("used meta count inconsistent: is %u should be %u\n",
+ rgp->rd_rg.rg_usedmeta, count[3]);
+ rgp->rd_rg.rg_usedmeta = count[3];
+ }
+ if(bmap & FREE_META_COUNT) {
+ log_err("free meta count inconsistent: is %u should be %u\n",
+ rgp->rd_rg.rg_freemeta, count[4]);
+ rgp->rd_rg.rg_freemeta = count[4];
+ }
+
if(query(rgp->rd_sbd,
"Update resource group counts? (y/n) ")) {
- /* write out the rgrp */
+ log_warn("Resource group counts updated\n");
+ /* write out the rgrp */
gfs_rgrp_out(&rgp->rd_rg, BH_DATA(rgp->rd_bh[0]));
write_buf(rgp->rd_sbd, rgp->rd_bh[0], 0);
} else {
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/super.c gfs_fsck-6.0-cvs/super.c
--- gfs_fsck-6.0.2.20/super.c 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/super.c 2005-06-20 10:02:58.000000000 -0500
@@ -303,3 +303,30 @@ int ri_update(struct fsck_sb *sdp)
return -1;
}
+
+int write_sb(struct fsck_sb *sbp)
+{
+ int error = 0;
+ osi_buf_t *bh;
+
+ error = get_and_read_buf(sbp, GFS_SB_ADDR >> sbp->fsb2bb_shift,
+ &bh, 0);
+ if (error){
+ log_crit("Unable to read superblock\n");
+ goto out;
+ }
+
+ gfs_sb_out(&sbp->sb, BH_DATA(bh));
+
+ /* FIXME: Should this set the BW_WAIT flag? */
+ if((error = write_buf(sbp, bh, 0))) {
+ stack;
+ goto out;
+ }
+
+ relse_buf(sbp, bh);
+out:
+ return error;
+
+}
+
diff -P -Nurp --exclude CVS gfs_fsck-6.0.2.20/super.h gfs_fsck-6.0-cvs/super.h
--- gfs_fsck-6.0.2.20/super.h 2005-08-29 11:27:21.000000000 -0500
+++ gfs_fsck-6.0-cvs/super.h 2005-06-20 09:50:35.000000000 -0500
@@ -19,5 +19,6 @@
int read_sb(struct fsck_sb *sdp);
int ji_update(struct fsck_sb *sdp);
int ri_update(struct fsck_sb *sdp);
+int write_sb(struct fsck_sb *sdp);
#endif /* _SUPER_H */
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 189 bytes
Desc: not available
URL: <http://listman.redhat.com/archives/linux-cluster/attachments/20050829/eaa435fa/attachment.sig>
More information about the Linux-cluster
mailing list