[libvirt-users] libvirt possibly ignoring cache=none ?

Brano Zarnovican zarnovican at gmail.com
Wed Aug 7 12:24:37 UTC 2013


Hi,

I have an instance with 8G ram assigned. All block devices have cache
disabled (cache=none) on host. However, cgroup is reporting 4G of
cache associated to the instance (on host)

# cgget -r memory.stat libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.stat: cache 4318011392
rss 8676360192
...

When I drop all system caches on host..

# echo 3 > /proc/sys/vm/drop_caches
#

..cache associated to the instance drops too.

# cgget -r memory.stat libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.stat: cache 122880
rss 8674291712
...

Can somebody explain what is cached, if there is cache=none everywhere ?

Thanks,

Brano Zarnovican

PS: versions:
Scientific Linux release 6.4 (Carbon)
kernel-2.6.32-358.11.1.el6.x86_64
qemu-kvm-0.12.1.2-2.355.el6_4.5.x86_64
libvirt-0.10.2-18.el6_4.5.x86_64
-------------- next part --------------
[root at dev-cmp08 ~]# ps auxw | head -1
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
[root at dev-cmp08 ~]# ps auxw | grep 12007
qemu     12007 72.8  1.5 13751712 8462604 ?    Sl   Jul08 31441:33 /usr/libexec/qemu-kvm -name i-000009fa -S -M rhel6.3.0 -cpu SandyBridge,+pdpe1gb,+osxsave,+dca,+pcid,+pdcm,+xtpr,+tm2,+est,+smx,+vmx,+ds_cpl,+monitor,+dtes64,+pbe,+tm,+ht,+ss,+acpi,+ds,+vme -enable-kvm -m 8196 -smp 8,sockets=8,cores=1,threads=1 -uuid 23e7ad1f-e79d-41d0-bbf5-a2f95c26c1da -nodefconfig -nodefaults -chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/i-000009fa.monitor,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -drive file=/var/lib/nova/instances/i-000009fa/disk,if=none,id=drive-virtio-disk0,format=qcow2,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -drive file=/var/lib/nova/instances/i-000009fa/disk.local,if=none,id=drive-virtio-disk1,format=qcow2,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk1,id=virtio-disk1 -drive file=/dev/disk/by-path/ip-172.30.128.3:3260-iscsi-iqn.1992-08.com.netapp:node.netapp02-lun-14,if=none,id=drive-virtio-disk5,format=raw,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x7,drive=drive-virtio-disk5,id=virtio-disk5 -netdev tap,fd=21,id=hostnet0,vhost=on,vhostfd=24 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=fa:16:3e:4f:02:ea,bus=pci.0,addr=0x3 -chardev file,id=charserial0,path=/var/lib/nova/instances/i-000009fa/console.log -device isa-serial,chardev=charserial0,id=serial0 -chardev pty,id=charserial1 -device isa-serial,chardev=charserial1,id=serial1 -device usb-tablet,id=input0 -vnc 172.30.129.221:2 -k en-us -vga cirrus -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x6
root     12012  0.5  0.0      0     0 ?        S    Jul08 225:22 [vhost-12007]
root     49616  0.0  0.0 103244   852 pts/23   S+   12:31   0:00 grep 12007
[root at dev-cmp08 ~]# 
[root at dev-cmp08 ~]# 
[root at dev-cmp08 ~]# 


[root at dev-cmp08 ~]# cgget -r memory.limit_in_bytes -r memory.usage_in_bytes libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.limit_in_bytes: 13215727616
memory.usage_in_bytes: 12998287360


[root at dev-cmp08 ~]# virsh dumpxml i-000009fa
<domain type='kvm' id='3'>
  <name>i-000009fa</name>
  <uuid>23e7ad1f-e79d-41d0-bbf5-a2f95c26c1da</uuid>
  <memory unit='KiB'>8392704</memory>
  <currentMemory unit='KiB'>8392704</currentMemory>
  <vcpu placement='static'>8</vcpu>
  <os>
    <type arch='x86_64' machine='rhel6.3.0'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <cpu mode='custom' match='exact'>
    <model fallback='allow'>SandyBridge</model>
    <vendor>Intel</vendor>
    <feature policy='require' name='pdpe1gb'/>
    <feature policy='require' name='osxsave'/>
    <feature policy='require' name='dca'/>
    <feature policy='require' name='pcid'/>
    <feature policy='require' name='pdcm'/>
    <feature policy='require' name='xtpr'/>
    <feature policy='require' name='tm2'/>
    <feature policy='require' name='est'/>
    <feature policy='require' name='smx'/>
    <feature policy='require' name='vmx'/>
    <feature policy='require' name='ds_cpl'/>
    <feature policy='require' name='monitor'/>
    <feature policy='require' name='dtes64'/>
    <feature policy='require' name='pbe'/>
    <feature policy='require' name='tm'/>
    <feature policy='require' name='ht'/>
    <feature policy='require' name='ss'/>
    <feature policy='require' name='acpi'/>
    <feature policy='require' name='ds'/>
    <feature policy='require' name='vme'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <devices>
    <emulator>/usr/libexec/qemu-kvm</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/var/lib/nova/instances/i-000009fa/disk'/>
      <target dev='vda' bus='virtio'/>
      <alias name='virtio-disk0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </disk>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2' cache='none'/>
      <source file='/var/lib/nova/instances/i-000009fa/disk.local'/>
      <target dev='vdb' bus='virtio'/>
      <alias name='virtio-disk1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </disk>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none'/>
      <source dev='/dev/disk/by-path/ip-172.30.128.3:3260-iscsi-iqn.1992-08.com.netapp:node.netapp02-lun-14'/>
      <target dev='sdf' bus='virtio'/>
      <alias name='virtio-disk5'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
    </disk>
    <controller type='usb' index='0'>
      <alias name='usb0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <interface type='bridge'>
      <mac address='fa:16:3e:4f:02:ea'/>
      <source bridge='br_v101'/>
      <target dev='vnet2'/>
      <model type='virtio'/>
      <filterref filter='nova-instance-i-000009fa-fa163e4f02ea'>
        <parameter name='DHCPSERVER' value='172.30.40.175'/>
        <parameter name='IP' value='172.30.40.181'/>
      </filterref>
      <alias name='net0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='file'>
      <source path='/var/lib/nova/instances/i-000009fa/console.log'/>
      <target port='0'/>
      <alias name='serial0'/>
    </serial>
    <serial type='pty'>
      <source path='/dev/pts/2'/>
      <target port='1'/>
      <alias name='serial1'/>
    </serial>
    <console type='file'>
      <source path='/var/lib/nova/instances/i-000009fa/console.log'/>
      <target type='serial' port='0'/>
      <alias name='serial0'/>
    </console>
    <input type='tablet' bus='usb'>
      <alias name='input0'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <graphics type='vnc' port='5902' autoport='yes' listen='172.30.129.221' keymap='en-us'>
      <listen type='address' address='172.30.129.221'/>
    </graphics>
    <video>
      <model type='cirrus' vram='9216' heads='1'/>
      <alias name='video0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <alias name='balloon0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </memballoon>
  </devices>
  <seclabel type='dynamic' model='selinux' relabel='yes'>
    <label>system_u:system_r:svirt_t:s0:c248,c684</label>
    <imagelabel>system_u:object_r:svirt_image_t:s0:c248,c684</imagelabel>
  </seclabel>
</domain>

[root at dev-cmp08 ~]# cgget -r memory.stat libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.stat: cache 4318011392
    rss 8676360192
    mapped_file 20480
    pgpgin 16765353
    pgpgout 17492340
    swap 921600
    inactive_anon 1866690560
    active_anon 6809690112
    inactive_file 2909306880
    active_file 1408684032
    unevictable 0
    hierarchical_memory_limit 13215727616
    hierarchical_memsw_limit 9223372036854775807
    total_cache 4318011392
    total_rss 8676360192
    total_mapped_file 20480
    total_pgpgin 16765353
    total_pgpgout 17492340
    total_swap 921600
    total_inactive_anon 1866690560
    total_active_anon 6809690112
    total_inactive_file 2909306880
    total_active_file 1408684032
    total_unevictable 0

[root at dev-cmp08 ~]# free -m
             total       used       free     shared    buffers     cached
Mem:        516809     361926     154882          0        465       4860
-/+ buffers/cache:     356600     160208
Swap:       102399       2438      99961
[root at dev-cmp08 ~]# 
[root at dev-cmp08 ~]# 
[root at dev-cmp08 ~]# 




;
; cache dropped
;





[root at dev-cmp08 ~]# echo 3 > /proc/sys/vm/drop_caches


[root at dev-cmp08 ~]# cgget -r memory.limit_in_bytes -r memory.usage_in_bytes libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.limit_in_bytes: 13215727616
memory.usage_in_bytes: 8675569664



[root at dev-cmp08 ~]# cgget -r memory.stat libvirt/qemu/i-000009fa
libvirt/qemu/i-000009fa:
memory.stat: cache 122880
    rss 8674291712
    mapped_file 20480
    pgpgin 16765443
    pgpgout 18547107
    swap 921600
    inactive_anon 1865596928
    active_anon 6808715264
    inactive_file 24576
    active_file 77824
    unevictable 0
    hierarchical_memory_limit 13215727616
    hierarchical_memsw_limit 9223372036854775807
    total_cache 122880
    total_rss 8674291712
    total_mapped_file 20480
    total_pgpgin 16765443
    total_pgpgout 18547107
    total_swap 921600
    total_inactive_anon 1865596928
    total_active_anon 6808715264
    total_inactive_file 24576
    total_active_file 77824
    total_unevictable 0



More information about the libvirt-users mailing list