[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] Fwd: There seems a deadlock in libvirt




Hi, all,

We were running OpenStack with Ubuntu and libvirt 0.9.10. We found that libvirt monitor command not working well. 
There were a lot of error in libvirtd.log like this
2013-02-07 06:07:39.000+0000: 18112: error : qemuDomainObjBeginJobInternal:773 : Timed out during operation: cannot acquire state change lock

We dig into libvirtd by strace and find one of the thread only have the following command
futex(0x7f69ac0ec0ec, FUTEX_WAIT_PRIVATE, 2717, NULL

It seems this thread waiting for reply but nothing came back thus other threads would wait for it. We also saw there is a function called virCondWaitUntil(). Is it safe for us to modify the code from virCondWait() to virCondWaitUntil() to prevent such deadlock scenario? Thanks.

Following is the gdb -p 'libvirt.pid' and 'thread id' and 'bt full'
#0  0x00007f69c8c1dd84 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
No symbol table info available.
#1  0x00007f69c9ee884a in virCondWait (c=<optimized out>, m=<optimized out>) at util/threads-pthread.c:117
        ret = <optimized out>
#2  0x000000000049c749 in qemuMonitorSend (mon=0x7f69ac0ec0c0, msg=<optimized out>) at qemu/qemu_monitor.c:826
        ret = -1
        __func__ = "qemuMonitorSend"
        __FUNCTION__ = "qemuMonitorSend"
#3  0x00000000004ac8ed in qemuMonitorJSONCommandWithFd (mon=0x7f69ac0ec0c0, cmd=0x7f6998028280, scm_fd=-1, reply=0x7f69c57829f8)
    at qemu/qemu_monitor_json.c:230
        ret = -1
        msg = {txFD = -1, txBuffer = 0x7f69980e9b00 "{\"execute\":\"query-balloon\",\"id\":\"libvirt-1359\"}\r\n", txOffset = 49, txLength = 49,
          rxBuffer = 0x0, rxLength = 0, rxObject = 0x0, finished = false, passwordHandler = 0, passwordOpaque = 0x0}
        cmdstr = 0x7f69980ef2f0 "{\"execute\":\"query-balloon\",\"id\":\"libvirt-1359\"}"
        id = 0x7f69980b0a20 "libvirt-1359"
        exe = <optimized out>
        __FUNCTION__ = "qemuMonitorJSONCommandWithFd"
        __func__ = "qemuMonitorJSONCommandWithFd"
#4  0x00000000004ae794 in qemuMonitorJSONGetBalloonInfo (mon=0x7f69ac0ec0c0, currmem=0x7f69c5782a48) at qemu/qemu_monitor_json.c:1190
        ret = <optimized out>
        cmd = 0x7f6998028280
        reply = 0x0
        __FUNCTION__ = "qemuMonitorJSONGetBalloonInfo"
#5  0x0000000000457451 in qemudDomainGetInfo (dom=<optimized out>, info=0x7f69c5782b50) at qemu/qemu_driver.c:2181
        priv = 0x7f69a0093b00
        driver = 0x7f69b80ca8e0
        vm = 0x7f69a0093370 
        ret = -1
        err = <optimized out>
        balloon = <optimized out>
        __FUNCTION__ = "qemudDomainGetInfo"
#6  0x00007f69c9f63eda in virDomainGetInfo (domain=0x7f69980e3650, info=0x7f69c5782b50) at libvirt.c:4230
        ret = <optimized out>
        conn = <optimized out>
        __func__ = "virDomainGetInfo"
        __FUNCTION__ = "virDomainGetInfo"
#7  0x0000000000439bca in remoteDispatchDomainGetInfo (ret=0x7f6998000c20, args=<optimized out>, rerr=0x7f69c5782c50, client=0x157e730,
    server=<optimized out>, msg=<optimized out>) at remote_dispatch.h:1640
        rv = -1
        tmp = {state = 1 '\001', maxMem = 2097152, memory = 0, nrVirtCpu = 0, cpuTime = 5981880000000}
        dom = 0x7f69980e3650
        priv = <optimized out>
#8  remoteDispatchDomainGetInfoHelper (server=<optimized out>, client=0x157e730, msg=<optimized out>, rerr=0x7f69c5782c50, args=<optimized out>, 
    ret=0x7f6998000c20) at remote_dispatch.h:1616
        __func__ = "remoteDispatchDomainGetInfoHelper"
#9  0x00007f69c9fbb915 in virNetServerProgramDispatchCall (msg=0x1689cc0, client=0x157e730, server=0x1577c90, prog=0x15825d0)
    at rpc/virnetserverprogram.c:416
        ret = 0x7f6998000c20 ""
        rv = -1
        i = <optimized out>
        arg = 0x7f6998027950 "\360e\n\230i\177"
        dispatcher = 0x73de40
        rerr = {code = 0, domain = 0, message = 0x0, level = 0, dom = 0x0, str1 = 0x0, str2 = 0x0, str3 = 0x0, int1 = 0, int2 = 0, net = 0x0}
#10 virNetServerProgramDispatch (prog=0x15825d0, server=0x1577c90, client=0x157e730, msg=0x1689cc0) at rpc/virnetserverprogram.c:289
        ret = -1
        rerr = {code = 0, domain = 0, message = 0x0, level = 0, dom = 0x0, str1 = 0x0, str2 = 0x0, str3 = 0x0, int1 = 0, int2 = 0, net = 0x0}
        __func__ = "virNetServerProgramDispatch"
        __FUNCTION__ = "virNetServerProgramDispatch"
#11 0x00007f69c9fb6461 in virNetServerHandleJob (jobOpaque=<optimized out>, opaque=0x1577c90) at rpc/virnetserver.c:164
        srv = 0x1577c90
        job = 0x155dfa0
        __func__ = "virNetServerHandleJob"
#12 0x00007f69c9ee8e3e in virThreadPoolWorker (opaque=<optimized out>) at util/threadpool.c:144
        data = "">
        pool = 0x1577d80
        cond = 0x1577de0
        priority = false
        job = 0x162dd20
#13 0x00007f69c9ee84e6 in virThreadHelper (data="" out>) at util/threads-pthread.c:161
        args = 0x0
        local = {func = 0x7f69c9ee8d00 <virThreadPoolWorker>, opaque = 0x1559f90}
#14 0x00007f69c8c19e9a in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
No symbol table info available.
#15 0x00007f69c89474bd in clone () from /lib/x86_64-linux-gnu/libc.so.6
No symbol table info available.
#16 0x0000000000000000 in ?? ()
No symbol table info available.

Regards,
Chun-Hung


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]