[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

[libvirt] Help writing bug report: 99% system time when running >1 VM



Hi,

I'm experiencing guest freezes and top is showing 99% system time
whenever I try to run more than one guest on my VM host.

Please help me to make a decent bug report!

libvirt: 1.2.1
qemu: 1.7.0

All individual VMs run perfectly on their own, only when two or more
are started at the same time I see these problems.  I've not idea how
to find where the processes are getting "stuck".  I've had this
working in the past, with the only differences I'm aware of being
version increments and that I previously used to use openvswitch, but
switched to simple linux bridge after rebuilding the setup.

If I strace for a few seconds on one of these processes which are
sucking up CPU, all I see is, e.g.:

[root kvmhost1 ~]# strace -c -p 12392
Process 12392 attached
^CProcess 12392 detached
% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
  0.00    0.000000           0         1           rt_sigpending
  0.00    0.000000           0         1         1 rt_sigtimedwait
------ ----------- ----------- --------- --------- ----------------
100.00    0.000000                     2         1 total


Any advice on what to do next appreciated!

Kind Regards,
Kevin Shanahan.

-- example top output --

top - 12:03:42 up 22:15,  3 users,  load average: 3.27, 2.21, 1.64
Threads: 112 total,   4 running, 108 sleeping,   0 stopped,   0 zombie
%Cpu(s):  2.3 us, 97.7 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem:   8169596 total,  3119260 used,  5050336 free,        0 buffers
KiB Swap:        0 total,        0 used,        0 free.   435704 cached Mem

  PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND       
12392 root      20   0 2886636 138272   6380 R 59.2  1.7   3:19.38 qemu-system-x+
11053 root      20   0 3473808 2.031g   6408 R 57.5 26.1  28:47.17 qemu-system-x+
11054 root      20   0 3473808 2.031g   6408 R 36.9 26.1  19:29.52 qemu-system-x+
12393 root      20   0 2886636 138272   6380 S 36.3  1.7   2:14.98 qemu-system-x+
11051 root      20   0 3473808 2.031g   6408 S  7.7 26.1   8:11.11 qemu-system-x+
11057 root      20   0 3473808 2.031g   6408 S  0.0 26.1   0:00.00 qemu-system-x+
12342 root      20   0 1080460 173156   6312 S  0.0  2.1   0:02.44 qemu-system-x+
12344 root      20   0 1080460 173156   6312 S  0.0  2.1   0:29.59 qemu-system-x+
12346 root      20   0 1080460 173156   6312 S  0.0  2.1   0:00.00 qemu-system-x+
12390 root      20   0 2886636 138272   6380 S  0.0  1.7   0:01.71 qemu-system-x+
12396 root      20   0 2886636 138272   6380 S  0.0  1.7   0:00.00 qemu-system-x+

-- perf top output --
(seems pretty normal, I think?)

Samples: 3M of event 'cycles', Event count (approx.): 62626938657                 
 20.90%  [kernel]                 [k] read_hpet                                  ◆
  4.47%  [kernel]                 [k] __schedule                                 ▒
  3.75%  [kernel]                 [k] update_cfs_shares                          ▒
  3.50%  [kernel]                 [k] kvm_arch_vcpu_ioctl_run                    ▒
  3.03%  [kernel]                 [k] native_read_tsc                            ▒
  2.42%  [kernel]                 [k] update_curr                                ▒
  2.22%  [kernel]                 [k] native_write_msr_safe                      ▒

-- virsh dumpxml win7 --

<domain type='kvm' id='4'>
  <name>win7</name>
  <uuid>fb683081-01c0-ee11-3e6f-580c62216b59</uuid>
  <memory unit='KiB'>2097152</memory>
  <currentMemory unit='KiB'>2097152</currentMemory>
  <vcpu placement='static'>2</vcpu>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.7'>hvm</type>
    <boot dev='cdrom'/>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <clock offset='localtime'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='volume' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source pool='vmspace' volume='win7.img'/>
      <target dev='vda' bus='virtio'/>
      <alias name='virtio-disk0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='volume' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <target dev='hdc' bus='ide'/>
      <readonly/>
      <alias name='ide0-1-0'/>
      <address type='drive' controller='0' bus='1' target='0' unit='0'/>
    </disk>
    <controller type='ide' index='0'>
      <alias name='ide0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <controller type='usb' index='0'>
      <alias name='usb0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'>
      <alias name='pci.0'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:cd:7a:f1'/>
      <source bridge='br0'/>
      <target dev='vnet1'/>
      <model type='virtio'/>
      <alias name='net0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <source path='/dev/pts/2'/>
      <target port='0'/>
      <alias name='serial0'/>
    </serial>
    <console type='pty' tty='/dev/pts/2'>
      <source path='/dev/pts/2'/>
      <target type='serial' port='0'/>
      <alias name='serial0'/>
    </console>
    <input type='tablet' bus='usb'>
      <alias name='input0'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='5901' autoport='yes' listen='127.0.0.1'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <alias name='video0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <alias name='balloon0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
  <seclabel type='none'/>
</domain>

-- virsh dumpxml arch1 --

<domain type='kvm' id='6'>
  <name>arch1</name>
  <uuid>fe10ccad-c4f0-b1ac-5b5b-fab023bb1176</uuid>
  <memory unit='KiB'>1572864</memory>
  <currentMemory unit='KiB'>1572864</currentMemory>
  <vcpu placement='static'>2</vcpu>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='pc-i440fx-1.7'>hvm</type>
    <boot dev='cdrom'/>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='volume' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source pool='vmspace' volume='arch1.img'/>
      <target dev='vda' bus='virtio'/>
      <alias name='virtio-disk0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='block' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <target dev='hdc' bus='ide'/>
      <readonly/>
      <alias name='ide0-1-0'/>
      <address type='drive' controller='0' bus='1' target='0' unit='0'/>
    </disk>
    <controller type='ide' index='0'>
      <alias name='ide0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <controller type='usb' index='0'>
      <alias name='usb0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'>
      <alias name='pci.0'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:a8:d8:4e'/>
      <source bridge='br0'/>
      <target dev='vnet2'/>
      <model type='virtio'/>
      <alias name='net0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <source path='/dev/pts/5'/>
      <target port='0'/>
      <alias name='serial0'/>
    </serial>
    <console type='pty' tty='/dev/pts/5'>
      <source path='/dev/pts/5'/>
      <target type='serial' port='0'/>
      <alias name='serial0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='5902' autoport='yes' listen='127.0.0.1'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <alias name='video0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <alias name='balloon0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </memballoon>
  </devices>
  <seclabel type='none'/>
</domain>

-- virsh pool-dumpxml vmspace --

<pool type='netfs'>
  <name>vmspace</name>
  <uuid>29cd171a-5727-4881-b858-e6e2e118d56d</uuid>
  <capacity unit='bytes'>527373959168</capacity>
  <allocation unit='bytes'>114542247936</allocation>
  <available unit='bytes'>412831711232</available>
  <source>
    <host name='10.48.50.20'/>
    <dir path='/vmspace/images'/>
    <format type='nfs'/>
  </source>
  <target>
    <path>/srv/kvm/images</path>
    <permissions>
      <mode>0755</mode>
      <owner>-1</owner>
      <group>-1</group>
    </permissions>
  </target>
</pool>


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]