Re: [issue1504] hammer crash on cleanups

[Eugene <dfuser@xxxxxxxxxxxxx>]

Hello again.

Simon 'corecode' Schubert wrote:
> > > Also, please configure a dumpdev and capture a crash dump.
Before updating I've decided to run %hammer reblock on a slave pfs with
%hammer reblock /mirrorvar
without remounting it before executing a command and got next (reported 
with kgdb):

Unread portion of the kernel message buffer:
panic: assertion: cursor->trans->sync_lock_refs > 0 in hammer_recover_cursor
Trace beginning at frame 0xd743e714
panic(d743e738,d743e7b8,d743e89c,d743e814,d743e744) at panic+0x8c
panic(c059aed8,c060299c,c05826fc,d743e7b8,d743e9ac) at panic+0x8c
hammer_recover_cursor(d743e7b8,0,0,5,d743e778) at hammer_recover_cursor+0x2c
hammer_ioc_mirror_write(d743ea84,d2c52550,c2a72998,c2b13fa8,1) at 
hammer_ioc_mirror_write+0x928
hammer_ioctl(d2c52550,c0c46808,c2a72998,1,c29a7768) at hammer_ioctl+0x8f2
hammer_vop_ioctl(d743eae0,c0664560,c2ae8b50,d60e9b00,46) at 
hammer_vop_ioctl+0x2f
vop_ioctl(c2ae8b50,c2b13fa8,c0c46808,c2a72998,1) at vop_ioctl+0x38
vn_ioctl(d0291148,c0c46808,c2a72998,c29a7768,d0291148) at vn_ioctl+0xdd
mapped_ioctl(3,c0c46808,bfbff794,0,d743ed34) at mapped_ioctl+0x3e1
sys_ioctl(d743ecf0,6,24de3699,0,d2aecfd8) at sys_ioctl+0x16
syscall2(d743ed40) at syscall2+0x1ef
Xint0x80_syscall() at Xint0x80_syscall+0x36
(kgdb) backtrace
#0  dumpsys () at ./machine/thread.h:83
#1  0xc0332666 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:375
#2  0xc0332787 in panic (fmt=0xc059e8fc "from debugger")
   at /usr/src/sys/kern/kern_shutdown.c:801
#3  0xc0182745 in db_panic (addr=-1068281776, have_addr=0, count=-1,
   modif=0xd743e5c4 "") at /usr/src/sys/ddb/db_command.c:447
#4  0xc0182db0 in db_command_loop () at /usr/src/sys/ddb/db_command.c:343
#5  0xc0185380 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_trap.c:71
#6  0xc05351bc in kdb_trap (type=3, code=0, regs=0xd743e6c0)
   at /usr/src/sys/platform/pc32/i386/db_interface.c:152
#7  0xc0543693 in trap (frame=0xd743e6c0)
   at /usr/src/sys/platform/pc32/i386/trap.c:797
#8  0xc0535ef7 in calltrap ()
   at /usr/src/sys/platform/pc32/i386/exception.s:785
#9  0xc0535050 in Debugger (msg=0xc05b6499 "panic") at ./cpu/cpufunc.h:73
#10 0xc033277e in panic (fmt=0xc059aed8 "assertion: %s in %s")
   at /usr/src/sys/kern/kern_shutdown.c:799
#11 0xc04a1dc4 in hammer_recover_cursor (cursor=0xd743e7b8)
   at /usr/src/sys/vfs/hammer/hammer_cursor.c:582
#12 0xc04ab39d in hammer_ioc_mirror_write (trans=0xd743ea84, ip=0xd2c52550,
   mirror=0xc2a72998) at /usr/src/sys/vfs/hammer/hammer_mirror.c:457
#13 0xc04aa686 in hammer_ioctl (ip=0xd2c52550, com=3234097160,
   data=0xc2a72998 "", fflag=1, cred=0xc29a7768)
   at /usr/src/sys/vfs/hammer/hammer_ioctl.c:134
#14 0xc04ba16d in hammer_vop_ioctl (ap=0xd743eae0)
   at /usr/src/sys/vfs/hammer/hammer_vnops.c:2191
#15 0xc0387c4e in vop_ioctl (ops=0xc2ae8b50, vp=0xc2b13fa8,
   command=3234097160, data=0xc2a72998 "", fflag=1, cred=0xc29a7768)
   at /usr/src/sys/kern/vfs_vopops.c:372
#16 0xc0387018 in vn_ioctl (fp=0xd0291148, com=3234097160, 
data=0xc2a72998 "",
   ucred=0xc29a7768) at /usr/src/sys/kern/vfs_vnops.c:1120
#17 0xc035295b in mapped_ioctl (fd=3, com=3234097160,
   uspc_data=0xbfbff794 <Address 0xbfbff794 out of bounds>, map=0x0)
   at /usr/src/sys/sys/file2.h:87
#18 0xc03529e3 in sys_ioctl (uap=0xd743ecf0)
   at /usr/src/sys/kern/sys_generic.c:525
#19 0xc0543033 in syscall2 (frame=0xd743ed40)
   at /usr/src/sys/platform/pc32/i386/trap.c:1339
#20 0xc0535fa6 in Xint0x80_syscall ()
   at /usr/src/sys/platform/pc32/i386/exception.s:876
#21 0x080552d7 in ?? ()
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
(kgdb) list *0xc04a1dc4
0xc04a1dc4 is in hammer_recover_cursor 
(/usr/src/sys/vfs/hammer/hammer_cursor.c:587).
582             KKASSERT(cursor->trans->sync_lock_refs > 0);
583
584             /*
585              * Wait for the deadlock to clear
586              */
587             if (cursor->deadlk_node) {
588                     hammer_lock_ex_ident(&cursor->deadlk_node->lock, 
"hmrdlk");
589                     hammer_unlock(&cursor->deadlk_node->lock);
590                     hammer_rel_node(cursor->deadlk_node);
591                     cursor->deadlk_node = NULL;
(kgdb) list *0xc04ab39d
0xc04ab39d is in hammer_ioc_mirror_write 
(/usr/src/sys/vfs/hammer/hammer_mirror.c:458).
453                      * for the next loop.
454                      */
455                     if (error == EDEADLK) {
456                             while (error == EDEADLK) {
457                                     hammer_recover_cursor(&cursor);
458                                     error = 
hammer_cursor_upgrade(&cursor);
459                             }
460                     } else {
461                             if (error == EALREADY)
462                                     error = 0;
(kgdb) list *0xc04aa686
0xc04aa686 is in hammer_ioctl (/usr/src/sys/vfs/hammer/hammer_ioctl.c:134).
129                                         (struct hammer_ioc_mirror_rw 
*)data);
130                     }
131                     break;
132             case HAMMERIOC_MIRROR_WRITE:
133                     if (error == 0) {
134                             error = hammer_ioc_mirror_write(&trans, ip,
135                                         (struct hammer_ioc_mirror_rw 
*)data);
136                     }
137                     break;
138             case HAMMERIOC_GET_VERSION:
(kgdb) list *0xc04ba16d
0xc04ba16d is in hammer_vop_ioctl 
(/usr/src/sys/vfs/hammer/hammer_vnops.c:2193).
2188            struct hammer_inode *ip = ap->a_vp->v_data;
2189
2190            ++hammer_stats_file_iopsr;
2191            return(hammer_ioctl(ip, ap->a_command, ap->a_data,
2192                                ap->a_fflag, ap->a_cred));
2193    }
2194
2195    static
2196    int
2197    hammer_vop_mountctl(struct vop_mountctl_args *ap)
(kgdb) list *0xc0387c4e
0xc0387c4e is in vop_ioctl (/usr/src/sys/kern/vfs_vopops.c:374).
369             ap.a_fflag = fflag;
370             ap.a_cred = cred;
371
372             DO_OPS(ops, error, &ap, vop_ioctl);
373             return(error);
374     }
375
376     int
377     vop_poll(struct vop_ops *ops, struct vnode *vp, int events, 
struct ucred *cred)
378     {
(kgdb) list *0xc0387018
0xc0387018 is in vn_ioctl (/usr/src/sys/kern/vfs_vnops.c:1120).
1115                            }
1116                            *(int *)data = dev_dflags(vp->v_rdev) & 
D_TYPEMASK;
1117                            error = 0;
1118                            break;
1119                    }
1120                    error = VOP_IOCTL(vp, com, data, fp->f_flag, ucred);
1121                    if (error == 0 && com == TIOCSCTTY) {
1122                            struct proc *p = curthread->td_proc;
1123                            struct session *sess;
1124
(kgdb)
> Best update to latest master, there were many fixes since August.
Now I'm planning to start updating the system and watch if crashes will 
go on. Please tell, what can I dig from crash dumps more, or how to send 
those dumps to You?