[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [libvirt] libvirtd (from git) no longer responds to 'kill'



This seems to be some sort of deadlock, easily reproduced by running
the libguestfs test suite, or even just 'libguestfs-test-tool'.

Here is a stack trace:

(gdb) t a a bt

Thread 11 (Thread 0x7fe6505d7700 (LWP 20021)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943c80, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque entry=0x1934930)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe6505d7700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 10 (Thread 0x7fe64fdd6700 (LWP 20022)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943c80, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque entry=0x1934b50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64fdd6700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 9 (Thread 0x7fe64f5d5700 (LWP 20023)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943c80, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque entry=0x1934c50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64f5d5700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 8 (Thread 0x7fe64edd4700 (LWP 20024)):
#0  __lll_lock_wait ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
---Type <return> to continue, or q <return> to quit---
#1  0x000000328ca09ca6 in _L_lock_836 () from /lib64/libpthread.so.0
#2  0x000000328ca09ba8 in __GI___pthread_mutex_lock (
    mutex=mutex entry=0x7fe64006eb30) at pthread_mutex_lock.c:64
#3  0x00007fe650ad142d in virMutexLock (m=m entry=0x7fe64006eb30)
    at util/virthreadpthread.c:85
#4  0x00007fe650ac38de in virObjectLock (anyobj=anyobj entry=0x7fe64006eb20)
    at util/virobject.c:322
#5  0x00007fe650ce65b1 in virSecurityManagerGetModel (
    mgr=mgr entry=0x7fe64006eb20) at security/security_manager.c:236
#6  0x00007fe650ce994c in virSecuritySELinuxSecurityVerify (
    mgr=0x7fe64006eb20, def=<optimized out>)
    at security/security_selinux.c:1806
#7  0x00007fe650ce7251 in virSecurityManagerVerify (mgr=0x7fe64006eb20, 
    def=def entry=0x7fe63400ac20) at security/security_manager.c:573
#8  0x00007fe650ce3cd4 in virSecurityStackVerify (mgr=<optimized out>, 
    def=0x7fe63400ac20) at security/security_stack.c:125
#9  0x00007fe650ce7251 in virSecurityManagerVerify (mgr=0x7fe64001cc50, 
    def=def entry=0x7fe63400ac20) at security/security_manager.c:573
#10 0x00007fe64893e63d in qemuDomainCreate (conn=0x7fe638000bd0, 
    xml=0x7fe6340009a0 "<?xml version=\"1.0\"?>\n<domain type=\"kvm\" xmlns:qemu=\"http://libvirt.org/schemas/domain/qemu/1.0\";>\n  <name>guestfs-492qa31a2ntfmk0j</name>\n  <memory unit=\"MiB\">500</memory>\n  <currentMemory unit=\"MiB\">"..., flags=<optimized out>) at qemu/qemu_driver.c:1538
#11 0x00007fe650b4fa39 in virDomainCreateXML (conn=0x7fe638000bd0, 
    xmlDesc=0x7fe6340009a0 "<?xml version=\"1.0\"?>\n<domain type=\"kvm\" xmlns:qemu=\"http://libvirt.org/schemas/domain/qemu/1.0\";>\n  <name>guestfs-492qa31a2ntfmk0j</name>\n  <memory unit=\"MiB\">500</memory>\n  <currentMemory unit=\"MiB\">"..., flags=2) at libvirt.c:1988
#12 0x000000000042c915 in remoteDispatchDomainCreateXML (
    server=<optimized out>, msg=<optimized out>, ret=0x7fe6340008c0, 
    args=0x7fe634000970, rerr=0x7fe64edd3c50, client=0x196eb60)
    at remote_dispatch.h:1172
#13 remoteDispatchDomainCreateXMLHelper (server=<optimized out>, 
    client=0x196eb60, msg=<optimized out>, rerr=0x7fe64edd3c50, 
    args=0x7fe634000970, ret=0x7fe6340008c0) at remote_dispatch.h:1152
#14 0x00007fe650bbe602 in virNetServerProgramDispatchCall (msg=0x196cd40, 
    client=0x196eb60, server=0x1943b10, prog=0x1968460)
    at rpc/virnetserverprogram.c:432
#15 virNetServerProgramDispatch (prog=0x1968460, 
    server=server entry=0x1943b10, client=0x196eb60, msg=0x196cd40)
    at rpc/virnetserverprogram.c:305
#16 0x00007fe650bb8838 in virNetServerProcessMsg (msg=<optimized out>, 
    prog=<optimized out>, client=<optimized out>, srv=0x1943b10)
    at rpc/virnetserver.c:162
#17 virNetServerHandleJob (jobOpaque=<optimized out>, opaque=0x1943b10)
    at rpc/virnetserver.c:183
---Type <return> to continue, or q <return> to quit---
#18 0x00007fe650ad1b3e in virThreadPoolWorker (opaque=opaque entry=0x1934b50)
    at util/virthreadpool.c:144
#19 0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#20 0x000000328ca07d15 in start_thread (arg=0x7fe64edd4700)
    at pthread_create.c:308
#21 0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 7 (Thread 0x7fe64e5d3700 (LWP 20025)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943c80, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bdb in virThreadPoolWorker (opaque=opaque entry=0x1934c50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64e5d3700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 6 (Thread 0x7fe64ddd2700 (LWP 20026)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943d18, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque entry=0x1934930)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64ddd2700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 5 (Thread 0x7fe64d5d1700 (LWP 20027)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943d18, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque entry=0x1934b50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
---Type <return> to continue, or q <return> to quit---
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64d5d1700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 4 (Thread 0x7fe64cdd0700 (LWP 20028)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943d18, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque entry=0x1934930)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64cdd0700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 3 (Thread 0x7fe64c5cf700 (LWP 20029)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943d18, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque entry=0x1934c50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64c5cf700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114

Thread 2 (Thread 0x7fe64bdce700 (LWP 20030)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:165
#1  0x00007fe650ad155a in virCondWait (c=c entry=0x1943d18, 
    m=m entry=0x1943c58) at util/virthreadpthread.c:117
#2  0x00007fe650ad1bfb in virThreadPoolWorker (opaque=opaque entry=0x1934b50)
    at util/virthreadpool.c:103
#3  0x00007fe650ad11f6 in virThreadHelper (data=<optimized out>)
    at util/virthreadpthread.c:161
#4  0x000000328ca07d15 in start_thread (arg=0x7fe64bdce700)
    at pthread_create.c:308
#5  0x000000328c6f246d in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:114
---Type <return> to continue, or q <return> to quit---

Thread 1 (Thread 0x7fe6509f9840 (LWP 20007)):
#0  0x000000328c6e998d in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007fe650aa96e5 in poll (__timeout=-1, __nfds=9, __fds=<optimized out>)
    at /usr/include/bits/poll2.h:46
#2  virEventPollRunOnce () at util/vireventpoll.c:630
#3  0x00007fe650aa805b in virEventRunDefaultImpl () at util/virevent.c:273
#4  0x00007fe650bb9e55 in virNetServerRun (srv=srv entry=0x1943b10)
    at rpc/virnetserver.c:1108
#5  0x000000000040c749 in main (argc=<optimized out>, argv=<optimized out>)
    at libvirtd.c:1481

Rich.

-- 
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Fedora Windows cross-compiler. Compile Windows programs, test, and
build Windows installers. Over 100 libraries supported.
http://fedoraproject.org/wiki/MinGW


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]