Fwd: md_thread and kjournald race w/ raid1 on 2-way x86

Srinivas Murthy codevana at gmail.com
Wed Jun 4 03:29:51 UTC 2008 

[0]kdb> btp 1684
Stack traceback for pid 1684
0xc39db580     1684        2  1    1   R  0xc39db760  kjournald
esp        eip        Function (args)
kdb_bb: address 0xffffffff not recognised
Using old style backtrace, unreliable with no arguments
esp        eip        Function (args)
0xc6549eb8 0xc02c1149 journal_commit_transaction+0x879
0xc6549f28 0xc0227945 lock_timer_base+0x25
0xc6549f40 0xc0227b6a try_to_del_timer_sync+0x4a
0xc6549f60 0xc02c3845 kjournald+0xb5
0xc6549f88 0xc0233040 autoremove_wake_function
0xc6549f94 0xc062f8e1 __sched_text_start+0x1f1
0xc6549fa8 0xc0233040 autoremove_wake_function


Based on this code below :

0xc02c10e3 journal_commit_transaction+0x813:   jmp    0xc02c10e3
journal_commit_transaction+0x813
0xc02c10e5 journal_commit_transaction+0x815:   movl   $0xc0651de8,(%esp)
0xc02c10ec journal_commit_transaction+0x81c:   mov    $0xc0651e44,%ecx
0xc02c10f1 journal_commit_transaction+0x821:   mov    $0xc0651dcd,%edx
0xc02c10f6 journal_commit_transaction+0x826:   mov    %ecx,0x8(%esp)
0xc02c10fa journal_commit_transaction+0x82a:   mov    $0xc0651f8a,%esi
0xc02c10ff journal_commit_transaction+0x82f:   mov    $0x2bd,%ebx
0xc02c1104 journal_commit_transaction+0x834:   mov    %esi,0x10(%esp)
0xc02c1108 journal_commit_transaction+0x838:   mov    %ebx,0xc(%esp)
0xc02c110c journal_commit_transaction+0x83c:   mov    %edx,0x4(%esp)
0xc02c1110 journal_commit_transaction+0x840:   call   0xc021efa0 printk
[0]kdb>
0xc02c1115 journal_commit_transaction+0x845:   ud2a
0xc02c1117 journal_commit_transaction+0x847:   jmp    0xc02c1117
journal_commit_transaction+0x847
0xc02c1119 journal_commit_transaction+0x849:   movl   $0xc0651de8,(%esp)
0xc02c1120 journal_commit_transaction+0x850:   mov    $0xc0651fa0,%eax
0xc02c1125 journal_commit_transaction+0x855:   mov    $0xc0651dcd,%edi
0xc02c112a journal_commit_transaction+0x85a:   mov    %eax,0x10(%esp)
0xc02c112e journal_commit_transaction+0x85e:   mov    $0x2b5,%eax
0xc02c1133 journal_commit_transaction+0x863:   mov    %eax,0xc(%esp)
0xc02c1137 journal_commit_transaction+0x867:   mov    $0xc0651e44,%eax
0xc02c113c journal_commit_transaction+0x86c:   mov    %edi,0x4(%esp)
0xc02c1140 journal_commit_transaction+0x870:   mov    %eax,0x8(%esp)
0xc02c1144 journal_commit_transaction+0x874:   call   0xc021efa0 printk
0xc02c1149 journal_commit_transaction+0x879:   ud2a
0xc02c114b journal_commit_transaction+0x87b:   jmp    0xc02c114b
journal_commit_transaction+0x87b
0xc02c114d journal_commit_transaction+0x87d:   mov    0x34(%ebx),%eax
0xc02c1150 journal_commit_transaction+0x880:   test   %eax,%eax
[0]kdb>
0xc02c1152 journal_commit_transaction+0x882:   jne    0xc02c11a2
journal_commit_transaction+0x8d2
0xc02c1154 journal_commit_transaction+0x884:   mov    0x38(%ebx),%edx
0xc02c1157 journal_commit_transaction+0x887:   test   %edx,%edx
0xc02c1159 journal_commit_transaction+0x889:   je     0xc02c11fd
journal_commit_transaction+0x92d
0xc02c115f journal_commit_transaction+0x88f:   mov    0x24(%edx),%edi
0xc02c1162 journal_commit_transaction+0x892:   mov    (%edi),%esi
0xc02c1164 journal_commit_transaction+0x894:   mov    (%esi),%eax
0xc02c1166 journal_commit_transaction+0x896:   test   $0x4,%al
0xc02c1168 journal_commit_transaction+0x898:   jne    0xc02c11e0
journal_commit_transaction+0x910
0xc02c116a journal_commit_transaction+0x89a:   call   0xc06302f0 cond_resched
0xc02c116f journal_commit_transaction+0x89f:   test   %eax,%eax
0xc02c1171 journal_commit_transaction+0x8a1:   jne    0xc02c1154
journal_commit_transaction+0x884
0xc02c1173 journal_commit_transaction+0x8a3:   mov    (%esi),%eax
0xc02c1175 journal_commit_transaction+0x8a5:   test   $0x1,%al
0xc02c1177 journal_commit_transaction+0x8a7:   mov    $0xfffffffb,%eax
0xc02c117c journal_commit_transaction+0x8ac:   cmovne 0xffffff98(%ebp),%eax
[0]kdb> rd
eax = 0x00000096 ebx = 0xf76bcf00 ecx = 0xffffffff edx = 0xf7588ac0
esi = 0xf6c66f88 edi = 0xc0651dcd esp = 0xc6549ec4 eip = 0xc02c1149
ebp = 0xc6549f5c xss = 0xc0580068 xcs = 0x00000060 eflags = 0x00010296
xds = 0xc065007b xes = 0xc654007b origeax = 0xffffffff &regs = 0xc6549e8c

and,

(gdb) p &(((struct buffer_head *)0)->b_count)
$1 = (atomic_t *) 0x34

I think bh is, 0xf76bcf00

but,

[0]kdb> md 0xf76bcf00
0xf76bcf00 f7f63400 00701310 00000004 000001ca   .4....p.........
0xf76bcf10 00000000 00000000 00000000 00000000   ................
0xf76bcf20 00000000 c6320b98 00000000 00000000   ......2.........
0xf76bcf30 00000000 f7386498 f7386b28 00000001   .....d8.(k8.....
0xf76bcf40 00000000 00000000 00000000 00000000   ................
0xf76bcf50 00000000 ffffefab 00000008 00000000   ................
0xf76bcf60 f76bc4e0 00100100 00200200 f76bcf70   ..k....... .p.k.
0xf76bcf70 00000001 00000000 f88eef70 f76bcf7c   ........p...|.k.
[0]kdb>
0xf76bcf80 f76bcf7c f76251e0 0000000d 0011ffff   |.k..Qb.........
0xf76bcf90 00000000 00000001 00000000 00000000   ................
0xf76bcfa0 00000000 f7ae9840 f88f140c deadc0de   .... at ...........
0xf76bcfb0 00000019 00000000 00000000 00000004   ................
0xf76bcfc0 00000000 00000000 00000000 00000000   ................
0xf76bcfd0-0xf76bcfef zero suppressed
0xf76bcff0 00000000 00000000 00000000 00000000   ................
[0]kdb>
0xf76bd000 00000000 00000000 00000000 00000000   ................
0xf76bd010-0xf76bd06f zero suppressed
0xf76bd070 00000000 00000000 00000000 00000000   ................

Not sure I'm reading bh correctly.


On Tue, Jun 3, 2008 at 8:06 PM, Eric Sandeen <sandeen at redhat.com> wrote:
> Srinivas Murthy wrote:
>
>> <6>EXT3-fs: mounted filesystem with ordered data mode.
>> <0>Assertion failure in journal_commit_transaction() at
>> fs/jbd/commit.c:693: "((&bh->b_count)->counter) == 0"
>> <0>------------[ cut here ]------------
>> <2>kernel BUG at fs/jbd/commit.c:693!
>> <0>invalid opcode: 0000 [#1]
>> <0>PREEMPT SMP
>> <0>CPU:    1
>> <0>EIP:    0060:[<c02c1149>]    Tainted: P        VLI
>> <0>EFLAGS: 00010296   (2.6.23.waas #4)
>> <0>EIP is at journal_commit_transaction+0x879/0xe00
>> <0>eax: 00000096   ebx: f76bcf00   ecx: ffffffff   edx: f7588ac0
>> <0>esi: f6c66f88   edi: c0651dcd   ebp: c6549f5c   esp: c6549ec4
>> <0>ds: 007b   es: 007b   fs: 00d8  gs: 0000  ss: 0068
>> <0>Process kjournald (pid: 1684, ti=c6548000 task=c39db580 task.ti=c6548000)
>> <0>Stack: c0651de8 c0651dcd c0651e44 000002b5 c0651fa0 00000000
>> 00000000 f7f63414
>> <0>       f7f634dc 00000000 00000fcc f7435034 00000000 00000000
>> c6402000 00000000
>> <0>       f7f63400 f7386fc0 000005d7 f77fb580 c39db580 f70bdd74
>> 00000202 c70f8000
>> <0>Call Trace:
>> <0> [<c0203a1a>] show_trace_log_lvl+0x1a/0x30
>> <0> [<c0203aea>] show_stack_log_lvl+0x9a/0xc0
>> <0> [<c0203d46>] show_registers+0x1d6/0x340
>> <0> [<c020403d>] die+0x10d/0x220
>> <0> [<c02041e1>] do_trap+0x91/0xd0
>> <0> [<c0204419>] do_invalid_op+0x89/0xa0
>> <0> [<c06317e2>] error_code+0x72/0x78
>> <0> [<c02c3845>] kjournald+0xb5/0x1f0
>> <0> [<c0232a5c>] kthread+0x5c/0xa0
>> <0> [<c020388b>] kernel_thread_helper+0x7/0x1c
>> <0> =======================
>> <0>Code: 65 c0 b8 a0 1f 65 c0 bf cd 1d 65 c0 89 44 24 10 b8 b5 02 00
>> 00 89 44 24 0c b8 44 1e 65 c0 89 7c 24 04 89 44 24 08 e8 57 de f5 ff
>> <0f> 0b eb fe 8b 43 34 85 c0 75 4e 8b 53 38 85 d2 0f 84 9e 00 00
>> <0>EIP: [<c02c1149>] journal_commit_transaction+0x879/0xe00 SS:ESP 0068:c6549ec4
>> <6>SysRq : Changing Loglevel
>> <4>Loglevel set to 7
>>
>> [0]kdb> btc
>> btc: cpu status: Currently on cpu 0
>
> Also, I'd backtrace pid 1684 (kjournald) and dump the bh, see what it
> looks like...
>
> kdb> btp 1684
> kdb> bh <whatever the address of the buffer head is>
>
> if i remember correctly...
>
> -Eric
>
>

More information about the Ext3-users mailing list