[libvirt] libvirtd segfault
Scott Sullivan
ssullivan at liquidweb.com
Fri Dec 28 13:24:29 UTC 2012
In case its useful, here's the 'bt' output from this session as well:
(gdb) bt
#0 qemuDomainObjBeginJobInternal (driver=0x7fffe401d740,
driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY,
asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
#1 0x00007fffeac2b223 in qemuDomainDestroyFlags (dom=<value optimized
out>, flags=<value optimized out>) at qemu/qemu_driver.c:2052
#2 0x00000039f10f97df in virDomainDestroy (domain=0x7fff741b8540) at
libvirt.c:2201
#3 0x0000000000428e22 in remoteDispatchDomainDestroy (server=<value
optimized out>, client=<value optimized out>, msg=<value optimized out>,
rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized
out>) at remote_dispatch.h:1277
#4 remoteDispatchDomainDestroyHelper (server=<value optimized out>,
client=<value optimized out>, msg=<value optimized out>,
rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized
out>) at remote_dispatch.h:1255
#5 0x00000039f1146152 in virNetServerProgramDispatchCall
(prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at
rpc/virnetserverprogram.c:431
#6 virNetServerProgramDispatch (prog=0x6884a0, server=0x67fe60,
client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:304
#7 0x00000039f1143fee in virNetServerProcessMsg (srv=<value optimized
out>, client=0x68eb50, prog=<value optimized out>, msg=0x6930b0) at
rpc/virnetserver.c:171
#8 0x00000039f1144a8b in virNetServerHandleJob (jobOpaque=<value
optimized out>, opaque=<value optimized out>) at rpc/virnetserver.c:192
#9 0x00000039f10643ec in virThreadPoolWorker (opaque=<value optimized
out>) at util/threadpool.c:144
#10 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
at util/threads-pthread.c:161
#11 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
#12 0x0000003009ae570d in clone () from /lib64/libc.so.6
(gdb)
On 12/27/2012 01:15 PM, Scott Sullivan wrote:
> ### Libvirt version: ###
>
> libvirt tagged v1.0.0 from git , with
> f0e72b2f5c675f927d04545dc5095f9e5998f171 applied
>
> ### Problem: ###
>
> Libvirtd segfaults
>
> ### Steps to reproduce: ###
>
> This is difficult to reproduce; but appears to happen only when doing
> 'virsh destroys', and when they are ran in concurrent intervals, over
> extended periods of time.
>
> Here are the steps I have taken to reproduce this problem
> (debian_6.0_amd64_kvm.img.gz is just a zipped up squeeze stock ISO
> install):
>
> First, copy this script to ease the steps (assumed to be at
> /root/libvirt_crash.pl later):
>
>
> #!/usr/bin/env perl
>
> use strict;
> use warnings;
>
> use Getopt::Long qw(:config no_ignore_case);
> use String::Random;
> use UUID::Random;
>
> my ($lvms,$sleep,$restarts_per_lvm);
> GetOptions(
> 'lvms=i' => \$lvms,
> 'restarts_per_lvm=i' => \$restarts_per_lvm,
> 'sleep=i' => \$sleep,
> );
>
>
> die "USAGE:\n$0 [--lvms int | --restarts_per_lvm int | --sleep int
> (optional) ]\n" unless ( $lvms && $restarts_per_lvm );
>
> sub _create_lvm {
> my $opts = shift;
> print "\tlvcreate -n $opts->{lvm} -L 5G /dev/LVM\n";
> system("lvcreate -n $opts->{lvm} -L 5G /dev/LVM");
> }
>
> sub _kpartx_lvm {
> my $opts = shift;
> die "_kpartx_lvm needs modifier!\n" unless ( $opts->{modifier} );
> print "\tkpartx -p p -$opts->{modifier} -v
> /dev/LVM/$opts->{lvm}\n";
> system("kpartx -p p -$opts->{modifier} -v
> /dev/LVM/$opts->{lvm}");
> }
>
> sub _remove_lvm {
> my $opts = shift;
> print "\tlvremove -f /dev/LVM/$opts->{lvm}\n";
> system("lvremove -f /dev/LVM/$opts->{lvm}");
> unlink("/xen/configs/$opts->{lvm}.cfg");
> }
>
> sub _display_lvm {
> my $opts = shift;
> print "\tparted -s /dev/LVM/$opts->{lvm} unit gb p\n";
> system("parted -s /dev/LVM/$opts->{lvm} unit gb p");
> }
>
> sub _dd_template {
> my $opts = shift;
> print "\tdd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz
> bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M\n";
> system("dd if=/root/ssullivan/debian_6.0_amd64_kvm.img.gz
> bs=1M | gunzip -c | dd of=/dev/LVM/$opts->{lvm} bs=1M");
> }
>
> sub _e2fsck_lvm {
> my $opts = shift;
> print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1\n";
> system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p1");
> print "\te2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3\n";
> system("e2fsck -y -f /dev/mapper/LVM-$opts->{lvm}p3")
> }
>
> sub _place_virtconf {
> my $opts = shift;
>
> my $uuid = UUID::Random::generate;
> my $virt_conf = <<END;
> <domain type="kvm">
> <name>$opts->{lvm}</name>
> <uuid>$uuid</uuid>
> <memory>450560</memory>
> <currentMemory>450560</currentMemory>
> <vcpu>2</vcpu>
> <cpu>
> <topology sockets="2" cores="4" threads="1"/>
> </cpu>
> <os>
> <type arch="x86_64" machine="pc-1.1">hvm</type>
> <boot dev="hd"/>
> </os>
> <features>
> <acpi/>
> <apic/>
> <pae/>
> </features>
> <clock offset="localtime"/>
> <on_poweroff>destroy</on_poweroff>
> <on_reboot>restart</on_reboot>
> <on_crash>restart</on_crash>
> <devices>
> <emulator>/opt/libexec/qemu-kvm-wrapper</emulator>
> <disk type="block" device="disk">
> <driver name="qemu" cache="none"/>
> <source dev="/dev/LVM/$opts->{lvm}"/>
> <target dev="vda" bus="virtio"/>
> </disk>
>
>
> <serial type="pty">
> <target port="1"/>
> </serial>
> <console type="pty">
> <target port="1"/>
> </console>
>
> <input type="tablet" bus="usb"/>
> <input type="mouse" bus="ps2"/>
> <graphics type="vnc" port="-1" autoport="yes" keymap="en-us"/>
> </devices>
> </domain>
> END
>
> open(my $fh, '>', "/xen/configs/$opts->{lvm}.cfg") || die
> "Error opening [/xen/configs/$opts->{lvm}.cfg] [$!]";
> print $fh "$virt_conf\n";
> close($fh) || die "Error closing
> [/xen/configs/$opts->{lvm}.cfg] [$!]";
> }
>
> sub _shutdown_lvm {
> my $opts = shift;
> print "\tvirsh destroy $opts->{lvm}\n";
> system("virsh destroy $opts->{lvm}");
> _virsh_list();
> }
>
> sub _virsh_list {
> print "\tvirsh list\n";
> system('virsh list');
> }
>
> sub _start_lvm {
> my $opts = shift;
> print "\tvirsh create /xen/configs/$opts->{lvm}.cfg\n";
> system("virsh create /xen/configs/$opts->{lvm}.cfg");
> _virsh_list();
> }
>
> my $rnd = new String::Random;
> my $count = 0;
> while ( $count < $lvms ) {
>
> my $id = $rnd->randpattern("cccccccc");
> print "####\nLVM PASS: $count\nLVM: $id\n####\n";
>
> print "Creating LVM...\n";
> _create_lvm({ lvm => $id });
>
> print "DD'ing template...\n";
> _dd_template({ lvm => $id });
>
> print "kpartxA LVM...\n";
> _kpartx_lvm({ lvm => $id, modifier => 'a' });
>
> print "e2fsck LVM..\n";
> _e2fsck_lvm({ lvm => $id });
>
> print "kpartxD LVM...\n";
> _kpartx_lvm({ lvm => $id, modifier => 'd' });
>
> print "Placing libvirt conf...\n";
> _place_virtconf({ lvm => $id });
>
> print "Partition setup:\n";
> _display_lvm({ lvm => $id });
>
> print "Performing $restarts_per_lvm restarts on $id...\n";
> my $restart_cnt = 0;
> while ( $restart_cnt < $restarts_per_lvm ) {
>
> print "####\nRESTART PASS: $restart_cnt\nLVM:
> $id\n####\n";
>
> print "Shutting down LVM...\n";
> _shutdown_lvm({ lvm => $id });
>
> sleep $sleep if ( $sleep );
>
> print "Starting LVM...\n";
> _start_lvm({ lvm => $id });
>
> $restart_cnt++;
> }
>
> print "Removing LVM...\n";
> _shutdown_lvm({ lvm => $id });
> _remove_lvm({ lvm => $id });
>
> $count++;
> }
>
> I then started 21 of these scripts, with the below commands
> (preferably in a screen to reduce spamming):
>
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000
> --sleep 10 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000
> --sleep 10 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 1000 --restarts_per_lvm 2000 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
> perl /root/libvirt_crash.pl --lvms 6000 --restarts_per_lvm 30 &
>
> I already had libvirtd started under GDB at this point. I waited about
> 3-6 hours, and I found this in my GDB session:
>
> 2012-12-27 17:23:35.922+0000: 7445: error : qemuMonitorIO:614 :
> internal error End of file from monitor
> 2012-12-27 17:23:36.558+0000: 7445: error : qemuMonitorIO:614 :
> internal error End of file from monitor
>
> Program received signal SIGSEGV, Segmentation fault.
> [Switching to Thread 0x7fff8e1fc700 (LWP 31142)]
> qemuDomainObjBeginJobInternal (driver=0x7fffe401d740,
> driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY,
> asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
> 768 priv->jobs_queued++;
> Missing separate debuginfos, use: debuginfo-install
> audit-libs-2.1.3-3.el6.x86_64 augeas-libs-0.9.0-1.el6.x86_64
> avahi-libs-0.6.25-11.el6.x86_64 cyrus-sasl-gssapi-2.1.23-13.el6.x86_64
> cyrus-sasl-lib-2.1.23-13.el6.x86_64
> cyrus-sasl-md5-2.1.23-13.el6.x86_64
> cyrus-sasl-plain-2.1.23-13.el6.x86_64 db4-4.7.25-16.el6.x86_64
> dbus-libs-1.2.24-5.el6_1.x86_64
> device-mapper-libs-1.02.66-6.el6.x86_64 glibc-2.12-1.47.el6.x86_64
> gnutls-2.8.5-4.el6.x86_64 keyutils-libs-1.4-3.el6.x86_64
> krb5-libs-1.9-22.el6_2.1.x86_64 libblkid-2.17.2-12.4.el6.x86_64
> libcap-ng-0.6.4-3.el6_0.1.x86_64 libcom_err-1.41.12-11.el6.x86_64
> libcurl-7.19.7-26.el6_1.2.x86_64 libgcrypt-1.4.5-9.el6.x86_64
> libgpg-error-1.7-4.el6.x86_64 libidn-1.18-2.el6.x86_64
> libnl-1.1-14.el6.x86_64 libpcap-1.0.0-6.20091201git117cb5.el6.x86_64
> libpciaccess-0.12.1-1.el6.x86_64 libselinux-2.0.94-5.2.el6.x86_64
> libsepol-2.0.41-4.el6.x86_64 libtasn1-2.3-3.el6.x86_64
> libudev-147-2.40.el6.x86_64 libuuid-2.17.2-12.4.el6.x86_64
> libxml2-2.7.6-4.el6.x86_64 libxslt-1.1.26-2.el6.x86_64
> netcf-libs-0.1.9-2.el6.x86_64 nspr-4.8.8-3.el6.x86_64
> nss-3.12.10-17.el6_2.x86_64 nss-softokn-freebl-3.12.9-11.el6.x86_64
> nss-util-3.12.10-2.el6.x86_64 numactl-2.0.3-9.el6.x86_64
> openldap-2.4.23-20.el6.x86_64 openssl-1.0.0-20.el6.x86_64
> yajl-1.0.7-3.el6.x86_64 zlib-1.2.3-27.el6.x86_64
> (gdb)
>
> Here is the output of 'thread apply all bt' from this session:
>
> Thread 30 (Thread 0x7fff8e1fc700 (LWP 31142)):
> #0 qemuDomainObjBeginJobInternal (driver=0x7fffe401d740,
> driver_locked=true, obj=0x7fff80001b00, job=QEMU_JOB_DESTROY,
> asyncJob=QEMU_ASYNC_JOB_NONE) at qemu/qemu_domain.c:768
> #1 0x00007fffeac2b223 in qemuDomainDestroyFlags (dom=<value optimized
> out>, flags=<value optimized out>) at qemu/qemu_driver.c:2052
> #2 0x00000039f10f97df in virDomainDestroy (domain=0x7fff741b8540) at
> libvirt.c:2201
> #3 0x0000000000428e22 in remoteDispatchDomainDestroy (server=<value
> optimized out>, client=<value optimized out>, msg=<value optimized
> out>, rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value
> optimized out>) at remote_dispatch.h:1277
> #4 remoteDispatchDomainDestroyHelper (server=<value optimized out>,
> client=<value optimized out>, msg=<value optimized out>,
> rerr=0x7fff8e1fbbe0, args=<value optimized out>, ret=<value optimized
> out>) at remote_dispatch.h:1255
> #5 0x00000039f1146152 in virNetServerProgramDispatchCall
> (prog=0x6884a0, server=0x67fe60, client=0x68eb50, msg=0x6930b0) at
> rpc/virnetserverprogram.c:431
> #6 virNetServerProgramDispatch (prog=0x6884a0, server=0x67fe60,
> client=0x68eb50, msg=0x6930b0) at rpc/virnetserverprogram.c:304
> #7 0x00000039f1143fee in virNetServerProcessMsg (srv=<value optimized
> out>, client=0x68eb50, prog=<value optimized out>, msg=0x6930b0) at
> rpc/virnetserver.c:171
> #8 0x00000039f1144a8b in virNetServerHandleJob (jobOpaque=<value
> optimized out>, opaque=<value optimized out>) at rpc/virnetserver.c:192
> #9 0x00000039f10643ec in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:144
> #10 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #11 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #12 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 29 (Thread 0x7fff8ebfd700 (LWP 20241)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 28 (Thread 0x7fff8f5fe700 (LWP 20235)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 27 (Thread 0x7fff8ffff700 (LWP 29349)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 26 (Thread 0x7fffb8dfa700 (LWP 29348)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 25 (Thread 0x7fffb97fb700 (LWP 28577)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 24 (Thread 0x7fffba1fc700 (LWP 28576)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 23 (Thread 0x7fffbabfd700 (LWP 28180)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 22 (Thread 0x7fffbb5fe700 (LWP 28178)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 21 (Thread 0x7fffe0dfa700 (LWP 12076)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 20 (Thread 0x7fffe17fb700 (LWP 5934)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 19 (Thread 0x7fffe21fc700 (LWP 5904)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 18 (Thread 0x7fffe2bfd700 (LWP 5860)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 17 (Thread 0x7fffbbfff700 (LWP 855)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 16 (Thread 0x7fffe97c4700 (LWP 835)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 11 (Thread 0x7fffec73f700 (LWP 7457)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 10 (Thread 0x7fffed140700 (LWP 7456)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 9 (Thread 0x7fffedb41700 (LWP 7455)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 8 (Thread 0x7fffee542700 (LWP 7454)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 7 (Thread 0x7fffeef43700 (LWP 7453)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 6 (Thread 0x7fffef944700 (LWP 7452)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 5 (Thread 0x7ffff0345700 (LWP 7451)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 4 (Thread 0x7ffff0d46700 (LWP 7450)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 3 (Thread 0x7ffff1747700 (LWP 7449)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 2 (Thread 0x7ffff2148700 (LWP 7448)):
> #0 0x000000300a20b3dc in pthread_cond_wait@@GLIBC_2.3.2 () from
> /lib64/libpthread.so.0
> #1 0x00000039f1063eb6 in virCondWait (c=<value optimized out>,
> m=<value optimized out>) at util/threads-pthread.c:117
> #2 0x00000039f1064483 in virThreadPoolWorker (opaque=<value optimized
> out>) at util/threadpool.c:103
> #3 0x00000039f1063cd9 in virThreadHelper (data=<value optimized out>)
> at util/threads-pthread.c:161
> #4 0x000000300a2077f1 in start_thread () from /lib64/libpthread.so.0
> #5 0x0000003009ae570d in clone () from /lib64/libc.so.6
>
> Thread 1 (Thread 0x7ffff7fda800 (LWP 7445)):
> #0 0x0000003009a46f60 in vfprintf () from /lib64/libc.so.6
> #1 0x0000003009afc970 in __vsnprintf_chk () from /lib64/libc.so.6
> #2 0x0000003009afc8aa in __snprintf_chk () from /lib64/libc.so.6
> #3 0x00000039f108144a in snprintf (when=<value optimized out>,
> buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at
> /usr/include/bits/stdio2.h:65
> #4 virTimeStringThenRaw (when=<value optimized out>,
> buf=0x7fffffffd470 "2012-12-27 17:23:36.561+0000") at util/virtime.c:219
> #5 0x00000039f108167a in virTimeStringNowRaw (buf=0x7fffffffd470
> "2012-12-27 17:23:36.561+0000") at util/virtime.c:195
> #6 0x00000039f105834b in virLogVMessage (source=VIR_LOG_FROM_FILE,
> priority=<value optimized out>, filename=0x39f11ad70b
> "util/event_poll.c", linenr=378, funcname=0x39f11adcc0
> "virEventPollMakePollFDs", fmt=<value optimized out>,
> vargs=0x7fffffffd4f0) at util/logging.c:822
> #7 0x00000039f10586ec in virLogMessage (source=<value optimized out>,
> priority=<value optimized out>, filename=<value optimized out>,
> linenr=<value optimized out>, funcname=<value optimized out>,
> fmt=<value optimized out>) at util/logging.c:753
> #8 0x00000039f1052241 in virEventPollMakePollFDs () at
> util/event_poll.c:374
> #9 virEventPollRunOnce () at util/event_poll.c:605
> #10 0x00000039f1051666 in virEventRunDefaultImpl () at util/event.c:247
> #11 0x00000039f11437dd in virNetServerRun (srv=0x67fe60) at
> rpc/virnetserver.c:1004
> #12 0x000000000040c5a3 in main (argc=<value optimized out>,
> argv=<value optimized out>) at libvirtd.c:1354
> (gdb)
>
> Does anyone have any insights as to the cause of this, or any
> potential solutions? If there's anything I can add that would help let
> me know.
More information about the libvir-list
mailing list