[Crash-utility] Question on how to check NFS Exported Filesystems in crash utility while analyzing vmcore file.
Dave Anderson
anderson at redhat.com
Thu Aug 16 14:25:58 UTC 2012
----- Original Message -----
>
>
> Hi All,
>
> I hope someone might have answer to this, I am looking for a way to
> dump the exported Filesystems from node in crash utility while
> analyzing vmcore file. Any pointers will be useful.
>
> Regards,
> Tipu.
>
I'm by no means at all versed in the intracacies of NFS exports,
but here's how I found the instance of an exported filesystem
on a RHEL5 machine that exports its "/usr/dumps" directory.
The nfsd module has a global "svc_export_cache" data structure
that is declared in "fs/nfsd/export.c":
struct cache_detail svc_export_cache = {
.owner = THIS_MODULE,
.hash_size = EXPORT_HASHMAX,
.hash_table = export_table,
.name = "nfsd.export",
.cache_put = svc_export_put,
.cache_request = svc_export_request,
.cache_parse = svc_export_parse,
.cache_show = svc_export_show,
.match = svc_export_match,
.init = svc_export_init,
.update = export_update,
.alloc = svc_export_alloc,
};
Load the nfsd module's debuginfo data first:
crash> mod -s nfsd
MODULE NAME SIZE OBJECT FILE
ffffffff88f2ae00 nfsd 285065 /lib/modules/2.6.18-128.el5/kernel/fs/nfsd/nfsd.ko
crash>
As I understand the kernel sources, each of the exported NFS
filesystems are referenced by the by a "cache_head" structure
that is contained in its "hash_table" array, which has
"hash_size" entries:
crash> cache_detail
struct cache_detail {
struct module *owner;
int hash_size; <====
struct cache_head **hash_table; <====
rwlock_t hash_lock;
atomic_t inuse;
char *name;
void (*cache_put)(struct kref *);
void (*cache_request)(struct cache_detail *, struct cache_head *, char **, int *);
int (*cache_parse)(struct cache_detail *, char *, int);
int (*cache_show)(struct seq_file *, struct cache_detail *, struct cache_head *);
struct cache_head *(*alloc)(void);
int (*match)(struct cache_head *, struct cache_head *);
void (*init)(struct cache_head *, struct cache_head *);
void (*update)(struct cache_head *, struct cache_head *);
time_t flush_time;
struct list_head others;
time_t nextcheck;
int entries;
struct list_head queue;
struct proc_dir_entry *proc_ent;
struct proc_dir_entry *flush_ent;
struct proc_dir_entry *channel_ent;
struct proc_dir_entry *content_ent;
atomic_t readers;
time_t last_close;
time_t last_warn;
void (*warn_no_listener)(struct cache_detail *);
}
SIZE: 224
crash>
So if you look at the structure contents, there is an array
of 256 cache_head structures at 0xffffffff88499b00:
crash> svc_export_cache
svc_export_cache = $4 = {
owner = 0xffffffff88490f80,
hash_size = 256, <====
hash_table = 0xffffffff88499b00, <====
hash_lock = {
raw_lock = {
lock = 16777216
}
},
inuse = {
counter = 0
},
name = 0xffffffff88481405 "nfsd.export",
cache_put = 0xffffffff88469be3 <svc_export_put>,
cache_request = 0xffffffff8846a581 <svc_export_request>,
cache_parse = 0xffffffff8846ab86 <svc_export_parse>,
cache_show = 0xffffffff88469306 <svc_export_show>,
alloc = 0xffffffff8846924a <svc_export_alloc>,
match = 0xffffffff884690a4 <svc_export_match>,
init = 0xffffffff8846a762 <svc_export_init>,
update = 0xffffffff884690ca <export_update>,
flush_time = 1333389558,
others = {
next = 0xffffffff88428a50,
prev = 0xffffffff8848f490
},
nextcheck = 1345127263,
entries = 1,
queue = {
next = 0xffff810028f73cc0,
prev = 0xffff810028f73cc0
},
proc_ent = 0xffff81003a7d4180,
flush_ent = 0xffff8100389678c0,
channel_ent = 0xffff8100389679c0,
content_ent = 0xffff810038646b80,
readers = {
counter = 1
},
last_close = 1333389553,
last_warn = -1,
warn_no_listener = 0
}
crash>
And by dumping the 256 entries in the hash table, I see
just the single entry down at address ffffffff8849a040:
crash> rd 0xffffffff88499b00 256
ffffffff88499b00: 0000000000000000 0000000000000000 ................
ffffffff88499b10: 0000000000000000 0000000000000000 ................
ffffffff88499b20: 0000000000000000 0000000000000000 ................
ffffffff88499b30: 0000000000000000 0000000000000000 ................
ffffffff88499b40: 0000000000000000 0000000000000000 ................
ffffffff88499b50: 0000000000000000 0000000000000000 ................
ffffffff88499b60: 0000000000000000 0000000000000000 ................
ffffffff88499b70: 0000000000000000 0000000000000000 ................
ffffffff88499b80: 0000000000000000 0000000000000000 ................
ffffffff88499b90: 0000000000000000 0000000000000000 ................
ffffffff88499ba0: 0000000000000000 0000000000000000 ................
ffffffff88499bb0: 0000000000000000 0000000000000000 ................
ffffffff88499bc0: 0000000000000000 0000000000000000 ................
ffffffff88499bd0: 0000000000000000 0000000000000000 ................
ffffffff88499be0: 0000000000000000 0000000000000000 ................
ffffffff88499bf0: 0000000000000000 0000000000000000 ................
ffffffff88499c00: 0000000000000000 0000000000000000 ................
ffffffff88499c10: 0000000000000000 0000000000000000 ................
ffffffff88499c20: 0000000000000000 0000000000000000 ................
ffffffff88499c30: 0000000000000000 0000000000000000 ................
ffffffff88499c40: 0000000000000000 0000000000000000 ................
ffffffff88499c50: 0000000000000000 0000000000000000 ................
ffffffff88499c60: 0000000000000000 0000000000000000 ................
ffffffff88499c70: 0000000000000000 0000000000000000 ................
ffffffff88499c80: 0000000000000000 0000000000000000 ................
ffffffff88499c90: 0000000000000000 0000000000000000 ................
ffffffff88499ca0: 0000000000000000 0000000000000000 ................
ffffffff88499cb0: 0000000000000000 0000000000000000 ................
ffffffff88499cc0: 0000000000000000 0000000000000000 ................
ffffffff88499cd0: 0000000000000000 0000000000000000 ................
ffffffff88499ce0: 0000000000000000 0000000000000000 ................
ffffffff88499cf0: 0000000000000000 0000000000000000 ................
ffffffff88499d00: 0000000000000000 0000000000000000 ................
ffffffff88499d10: 0000000000000000 0000000000000000 ................
ffffffff88499d20: 0000000000000000 0000000000000000 ................
ffffffff88499d30: 0000000000000000 0000000000000000 ................
ffffffff88499d40: 0000000000000000 0000000000000000 ................
ffffffff88499d50: 0000000000000000 0000000000000000 ................
ffffffff88499d60: 0000000000000000 0000000000000000 ................
ffffffff88499d70: 0000000000000000 0000000000000000 ................
ffffffff88499d80: 0000000000000000 0000000000000000 ................
ffffffff88499d90: 0000000000000000 0000000000000000 ................
ffffffff88499da0: 0000000000000000 0000000000000000 ................
ffffffff88499db0: 0000000000000000 0000000000000000 ................
ffffffff88499dc0: 0000000000000000 0000000000000000 ................
ffffffff88499dd0: 0000000000000000 0000000000000000 ................
ffffffff88499de0: 0000000000000000 0000000000000000 ................
ffffffff88499df0: 0000000000000000 0000000000000000 ................
ffffffff88499e00: 0000000000000000 0000000000000000 ................
ffffffff88499e10: 0000000000000000 0000000000000000 ................
ffffffff88499e20: 0000000000000000 0000000000000000 ................
ffffffff88499e30: 0000000000000000 0000000000000000 ................
ffffffff88499e40: 0000000000000000 0000000000000000 ................
ffffffff88499e50: 0000000000000000 0000000000000000 ................
ffffffff88499e60: 0000000000000000 0000000000000000 ................
ffffffff88499e70: 0000000000000000 0000000000000000 ................
ffffffff88499e80: 0000000000000000 0000000000000000 ................
ffffffff88499e90: 0000000000000000 0000000000000000 ................
ffffffff88499ea0: 0000000000000000 0000000000000000 ................
ffffffff88499eb0: 0000000000000000 0000000000000000 ................
ffffffff88499ec0: 0000000000000000 0000000000000000 ................
ffffffff88499ed0: 0000000000000000 0000000000000000 ................
ffffffff88499ee0: 0000000000000000 0000000000000000 ................
ffffffff88499ef0: 0000000000000000 0000000000000000 ................
ffffffff88499f00: 0000000000000000 0000000000000000 ................
ffffffff88499f10: 0000000000000000 0000000000000000 ................
ffffffff88499f20: 0000000000000000 0000000000000000 ................
ffffffff88499f30: 0000000000000000 0000000000000000 ................
ffffffff88499f40: 0000000000000000 0000000000000000 ................
ffffffff88499f50: 0000000000000000 0000000000000000 ................
ffffffff88499f60: 0000000000000000 0000000000000000 ................
ffffffff88499f70: 0000000000000000 0000000000000000 ................
ffffffff88499f80: 0000000000000000 0000000000000000 ................
ffffffff88499f90: 0000000000000000 0000000000000000 ................
ffffffff88499fa0: 0000000000000000 0000000000000000 ................
ffffffff88499fb0: 0000000000000000 0000000000000000 ................
ffffffff88499fc0: 0000000000000000 0000000000000000 ................
ffffffff88499fd0: 0000000000000000 0000000000000000 ................
ffffffff88499fe0: 0000000000000000 0000000000000000 ................
ffffffff88499ff0: 0000000000000000 0000000000000000 ................
ffffffff8849a000: 0000000000000000 0000000000000000 ................
ffffffff8849a010: 0000000000000000 0000000000000000 ................
ffffffff8849a020: 0000000000000000 0000000000000000 ................
ffffffff8849a030: 0000000000000000 0000000000000000 ................
ffffffff8849a040: ffff810036155280 0000000000000000 .R.6............
ffffffff8849a050: 0000000000000000 0000000000000000 ................
ffffffff8849a060: 0000000000000000 0000000000000000 ................
ffffffff8849a070: 0000000000000000 0000000000000000 ................
ffffffff8849a080: 0000000000000000 0000000000000000 ................
ffffffff8849a090: 0000000000000000 0000000000000000 ................
ffffffff8849a0a0: 0000000000000000 0000000000000000 ................
ffffffff8849a0b0: 0000000000000000 0000000000000000 ................
ffffffff8849a0c0: 0000000000000000 0000000000000000 ................
ffffffff8849a0d0: 0000000000000000 0000000000000000 ................
ffffffff8849a0e0: 0000000000000000 0000000000000000 ................
ffffffff8849a0f0: 0000000000000000 0000000000000000 ................
ffffffff8849a100: 0000000000000000 0000000000000000 ................
ffffffff8849a110: 0000000000000000 0000000000000000 ................
ffffffff8849a120: 0000000000000000 0000000000000000 ................
ffffffff8849a130: 0000000000000000 0000000000000000 ................
ffffffff8849a140: 0000000000000000 0000000000000000 ................
ffffffff8849a150: 0000000000000000 0000000000000000 ................
ffffffff8849a160: 0000000000000000 0000000000000000 ................
ffffffff8849a170: 0000000000000000 0000000000000000 ................
ffffffff8849a180: 0000000000000000 0000000000000000 ................
ffffffff8849a190: 0000000000000000 0000000000000000 ................
ffffffff8849a1a0: 0000000000000000 0000000000000000 ................
ffffffff8849a1b0: 0000000000000000 0000000000000000 ................
ffffffff8849a1c0: 0000000000000000 0000000000000000 ................
ffffffff8849a1d0: 0000000000000000 0000000000000000 ................
ffffffff8849a1e0: 0000000000000000 0000000000000000 ................
ffffffff8849a1f0: 0000000000000000 0000000000000000 ................
ffffffff8849a200: 0000000000000000 0000000000000000 ................
ffffffff8849a210: 0000000000000000 0000000000000000 ................
ffffffff8849a220: 0000000000000000 0000000000000000 ................
ffffffff8849a230: 0000000000000000 0000000000000000 ................
ffffffff8849a240: 0000000000000000 0000000000000000 ................
ffffffff8849a250: 0000000000000000 0000000000000000 ................
ffffffff8849a260: 0000000000000000 0000000000000000 ................
ffffffff8849a270: 0000000000000000 0000000000000000 ................
ffffffff8849a280: 0000000000000000 0000000000000000 ................
ffffffff8849a290: 0000000000000000 0000000000000000 ................
ffffffff8849a2a0: 0000000000000000 0000000000000000 ................
ffffffff8849a2b0: 0000000000000000 0000000000000000 ................
ffffffff8849a2c0: 0000000000000000 0000000000000000 ................
ffffffff8849a2d0: 0000000000000000 0000000000000000 ................
ffffffff8849a2e0: 0000000000000000 0000000000000000 ................
ffffffff8849a2f0: 0000000000000000 0000000000000000 ................
crash>
That cache_head looks like this:
crash> cache_head ffff810036155280
struct cache_head {
next = 0x0,
expiry_time = 1345127262,
last_refresh = 1345125462,
ref = {
refcount = {
counter = 1
}
},
flags = 1
}
crash>
But the cache_head is embedded in an svc_export structure:
crash> svc_export
struct svc_export {
struct cache_head h;
struct auth_domain *ex_client;
int ex_flags;
struct vfsmount *ex_mnt;
struct dentry *ex_dentry;
char *ex_path;
uid_t ex_anon_uid;
gid_t ex_anon_gid;
int ex_fsid;
struct nfsd4_fs_locations ex_fslocs;
int ex_nflavors;
struct exp_flavor_info ex_flavors[8];
}
SIZE: 192
crash>
And so looking at that entry as an svc_export, voila, we see
the "/usr/dumps" export:
crash> svc_export ffff810036155280
struct svc_export {
h = {
next = 0x0,
expiry_time = 1345127262,
last_refresh = 1345125462,
ref = {
refcount = {
counter = 1
}
},
flags = 1
},
ex_client = 0xffff810038abeb80,
ex_flags = 47,
ex_mnt = 0xffff81003fe13480,
ex_dentry = 0xffff81001dee0660,
ex_path = 0xffff81002a09c3e0 "/usr/dumps", <====
ex_anon_uid = 65534,
ex_anon_gid = 65534,
ex_fsid = 0,
ex_fslocs = {
locations_count = 0,
locations = 0x0,
migrated = 0
},
ex_nflavors = 0,
ex_flavors = {{
pseudoflavor = 4294934784,
flags = 835776256
}, {
pseudoflavor = 4294934784,
flags = 613611416
}, {
pseudoflavor = 4294934784,
flags = 687289760
}, {
pseudoflavor = 4294934784,
flags = 4
}, {
pseudoflavor = 0,
flags = 4
}, {
pseudoflavor = 0,
flags = 2148721614
}, {
pseudoflavor = 4294967295,
flags = 0
}, {
pseudoflavor = 0,
flags = 1352
}}
}
crash>
Now, that all being done, that search could have been signicantly simplified
given that the hash_table array is initialized to point to the static data
strucure "export_table":
struct cache_detail svc_export_cache = {
.owner = THIS_MODULE,
.hash_size = EXPORT_HASHMAX,
.hash_table = export_table, <=====
...
So if you just dump its contents directly:
crash> p export_table
export_table = $5 =
{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x
0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0
x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0
, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xffff810036155280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x
0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0
x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}
crash>
And then cast each non-zero entry in the array as an svc_export as done above.
At least I *think* that's how it works...
Dave
More information about the Crash-utility
mailing list