[Linux-cluster] pvmove locking problem CLVM on RHEL 6
emmanuel segura
emi2fast at gmail.com
Fri Jun 22 07:37:29 UTC 2012
Hello Jay
The error is so clear, you have lvm configured in exclusive mode, that
means you can't access to your vg from more then one node at a time
2012/6/21 Jay Tingle <yogsothoth at sinistar.org>
> Hi All, I am having a problem using pvmove during some testing with Red
> Hat
> Cluster using CLVM on RHEL 6.2. I have 3 nodes which are ESXi 5u1 VMs
> with the
> 'multi-writer' flag set for the shared vmdk devices. I keep getting locking
> errors during the pvmove. Everything else seems to be working great as
> far as
> CLVM goes. Searching through the list archives and consulting the manuals
> it
> looks like all you need is to have cmirrord running. The RHEL 6 manual
> mentions cmirror-kmod which doesn't seem to exist anymore. Is there still
> a
> kernel module on RHEL 6? I am standard clvm with ext4 in an active/passive
> cluster. Anyone know what I am doing wrong? Below is my lvm config and my
> cluster config. Thanks in advance.
>
> [root at rhc6esx1 ~]# rpm -qa|grep -i lvm
> lvm2-libs-2.02.87-6.el6.x86_64
> lvm2-2.02.87-6.el6.x86_64
> lvm2-cluster-2.02.87-6.el6.**x86_64
> [root at rhc6esx1 ~]# rpm -q cman
> cman-3.0.12.1-23.el6.x86_64
> [root at rhc6esx1 ~]# rpm -q cmirror
> cmirror-2.02.87-6.el6.x86_64
>
> [root at rhc6esx1 ~]# ps -ef|grep cmirror
> root 21253 20692 0 13:37 pts/1 00:00:00 grep cmirror
> root 31858 1 0 13:18 ? 00:00:00 cmirrord
> [root at rhc6esx1 ~]# pvs|grep cfq888dbvg
> /dev/sdf1 cfq888dbvg lvm2 a-- 20.00g 0
> /dev/sdi1 cfq888dbvg lvm2 a-- 20.00g 0
> /dev/sdj1 cfq888dbvg lvm2 a-- 20.00g 0
> /dev/sdk1 cfq888dbvg lvm2 a-- 80.00g 80.00g
>
> [root at rhc6esx1 ~]# pvmove -v /dev/sdi1 /dev/sdk1
> Finding volume group "cfq888dbvg"
> Executing: /sbin/modprobe dm-log-userspace
> Archiving volume group "cfq888dbvg" metadata (seqno 7).
> Creating logical volume pvmove0
> Moving 5119 extents of logical volume cfq888dbvg/cfq888_db
> Error locking on node rhc6esx1-priv: Device or resource busy
> Error locking on node rhc6esx3-priv: Volume is busy on another node
> Error locking on node rhc6esx2-priv: Volume is busy on another node
> Failed to activate cfq888_db
>
> [root at rhc6esx1 ~]# clustat
> Cluster Status for rhc6 @ Thu Jun 21 13:35:49 2012
> Member Status: Quorate
>
> Member Name ID Status
> ------ ---- ---- ------
> rhc6esx1-priv 1 Online,
> Local, rgmanager
> rhc6esx2-priv 2 Online,
> rgmanager
> rhc6esx3-priv 3 Online,
> rgmanager
> /dev/block/8:33 0 Online,
> Quorum Disk
>
> Service Name Owner (Last)
> State
> ------- ---- ----- ------
> -----
> service:cfq888_grp rhc6esx1-priv
> started
>
> [root at rhc6esx1 ~]# lvm dumpconfig
> devices {
> dir="/dev"
> scan="/dev"
> obtain_device_list_from_udev=1
> preferred_names=["^/dev/mpath/**", "^/dev/mapper/mpath",
> "^/dev/[hs]d"]
> filter="a/.*/"
> cache_dir="/etc/lvm/cache"
> cache_file_prefix=""
> write_cache_state=1
> sysfs_scan=1
> md_component_detection=1
> md_chunk_alignment=1
> data_alignment_detection=1
> data_alignment=0
> data_alignment_offset_**detection=1
> ignore_suspended_devices=0
> disable_after_error_count=0
> require_restorefile_with_uuid=**1
> pv_min_size=2048
> issue_discards=0
> }
> dmeventd {
> mirror_library="libdevmapper-**event-lvm2mirror.so"
> snapshot_library="**libdevmapper-event-**lvm2snapshot.so"
> }
> activation {
> checks=0
> udev_sync=1
> udev_rules=1
> verify_udev_operations=0
> missing_stripe_filler="error"
> reserved_stack=256
> reserved_memory=8192
> process_priority=-18
> mirror_region_size=512
> readahead="auto"
> mirror_log_fault_policy="**allocate"
> mirror_image_fault_policy="**remove"
> snapshot_autoextend_threshold=**100
> snapshot_autoextend_percent=20
> use_mlockall=0
> monitoring=1
> polling_interval=15
> }
> global {
> umask=63
> test=0
> units="h"
> si_unit_consistency=1
> activation=1
> proc="/proc"
> locking_type=3
> wait_for_locks=1
> fallback_to_clustered_locking=**1
> fallback_to_local_locking=1
> locking_dir="/var/lock/lvm"
> prioritise_write_locks=1
> abort_on_internal_errors=0
> detect_internal_vg_cache_**corruption=0
> metadata_read_only=0
> mirror_segtype_default="**mirror"
> }
> shell {
> history_size=100
> }
> backup {
> backup=1
> backup_dir="/etc/lvm/backup"
> archive=1
> archive_dir="/etc/lvm/archive"
> retain_min=10
> retain_days=30
> }
> log {
> verbose=0
> syslog=1
> overwrite=0
> level=0
> indent=1
> command_names=0
> prefix=" "
> }
>
>
> [root at rhc6esx1 ~]# ccs -h localhost --getconf
> <cluster config_version="273" name="rhc6">
> <fence_daemon clean_start="0" post_fail_delay="20" post_join_delay="60"/>
> <clusternodes>
> <clusternode name="rhc6esx1-priv" nodeid="1">
> <fence>
> <method name="1">
> <device name="fence_vmware" uuid="422a2b6a-4093-2694-65e0-**
> a01332ef54bd"/>
> </method>
> </fence>
> </clusternode>
> <clusternode name="rhc6esx2-priv" nodeid="2">
> <fence>
> <method name="1">
> <device name="fence_vmware" uuid="422a9c5d-f9e2-8150-340b-**
> c84b834ba068"/>
> </method>
> </fence>
> </clusternode>
> <clusternode name="rhc6esx3-priv" nodeid="3">
> <fence>
> <method name="1">
> <device name="fence_vmware" uuid="422af24c-909f-187d-4e64-**
> 2a28cbe5d09d"/>
> </method>
> </fence>
> </clusternode>
> </clusternodes>
> <cman expected_votes="5"/>
> <fencedevices>
> <fencedevice agent="fence_vmware_soap" ipaddr="192.168.1.111"
> login="mrfence" name="fence_vmware" passwd="FenceM3" ssl="yes"
> verbose="yes"/>
> </fencedevices>
> <totem token="30000"/>
> <quorumd interval="1" label="rhc6esx-quorum" stop_cman="1" tko="10"
> votes="2"/>
> <logging logfile_priority="info" syslog_facility="daemon"
> syslog_priority="warning" to_logfile="yes" to_syslog="yes">
> <logging_daemon logfile="/var/log/cluster/**qdiskd.log" name="qdiskd"/>
> <logging_daemon logfile="/var/log/cluster/**fenced.log" name="fenced"/>
> <logging_daemon logfile="/var/log/cluster/dlm_**controld.log"
> name="dlm_controld"/>
> <logging_daemon logfile="/var/log/cluster/gfs_**controld.log"
> name="gfs_controld"/>
> <logging_daemon debug="on" logfile="/var/log/cluster/**rgmanager.log"
> name="rgmanager"/>
> <logging_daemon name="corosync" to_logfile="no"/>
> </logging>
> <rm log_level="7">
> <failoverdomains>
> <failoverdomain name="rhc6esx3_home" nofailback="1" ordered="1"
> restricted="1">
> <failoverdomainnode name="rhc6esx3-priv" priority="1"/>
> <failoverdomainnode name="rhc6esx2-priv" priority="2"/>
> <failoverdomainnode name="rhc6esx1-priv" priority="3"/>
> </failoverdomain>
> <failoverdomain name="rhc6esx2_home" nofailback="1" ordered="1"
> restricted="1">
> <failoverdomainnode name="rhc6esx2-priv" priority="1"/>
> <failoverdomainnode name="rhc6esx1-priv" priority="2"/>
> <failoverdomainnode name="rhc6esx3-priv" priority="3"/>
> </failoverdomain>
> <failoverdomain name="rhc6esx1_home" nofailback="1" ordered="1"
> restricted="1">
> <failoverdomainnode name="rhc6esx1-priv" priority="1"/>
> <failoverdomainnode name="rhc6esx2-priv" priority="2"/>
> <failoverdomainnode name="rhc6esx3-priv" priority="3"/>
> </failoverdomain>
> </failoverdomains>
> <resources>
> <lvm name="cfq888vg_lvm" self_fence="1" vg_name="cfq888vg"/>
> <lvm name="cfq888bkpvg_lvm" self_fence="1" vg_name="cfq888bkpvg"/>
> <lvm name="cfq888dbvg_lvm" self_fence="1" vg_name="cfq888dbvg"/>
> <lvm name="cfq888revg_lvm" vg_name="cfq888revg"/>
> <lvm name="cfq888flashvg_lvm" self_fence="1" vg_name="cfq888flashvg"/>
> <ip address="192.168.1.31" monitor_link="1"/>
> <fs device="/dev/cfq888vg/cfq888" force_fsck="0" force_unmount="1"
> fstype="ext4" mountpoint="/cfq888" name="cfq888_mnt" self_fence="0"/>
> <fs device="/dev/cfq888vg/cfq888_**ar" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_ar"
> name="cfq888_ar_mnt" self_fence="0"/>
> <fs device="/dev/cfq888vg/cfq888_**sw" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_sw"
> name="cfq888_sw_mnt" self_fence="0"/>
> <fs device="/dev/cfq888bkpvg/**cfq888_dmp" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_**dmp"
> name="cfq888_dmp_mnt" self_fence="0"/>
> <fs device="/dev/cfq888bkpvg/**cfq888_bk" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_bk"
> name="cfq888_bk_mnt" self_fence="0"/>
> <fs device="/dev/cfq888dbvg/**cfq888_db" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_db"
> name="cfq888_db_mnt" self_fence="0"/>
> <fs device="/dev/cfq888flashvg/**cfq888_flash" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_bk/**cfq888_flash"
> name="cfq888_flash_mnt" self_fence="0"/>
> <fs device="/dev/cfq888revg/**cfq888_rd" force_fsck="0"
> force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_rd"
> name="cfq888_rd_mnt" self_fence="0"/>
> <oracledb home="/u01/app/oracle/product/**11.2.0/dbhome_1"
> listener_name="cfq888_lsnr" name="cfq888" type="base" user="oracle"/>
> </resources>
> <service autostart="1" domain="rhc6esx1_home" exclusive="0"
> name="cfq888_grp" recovery="restart">
> <lvm ref="cfq888vg_lvm"/>
> <lvm ref="cfq888bkpvg_lvm"/>
> <lvm ref="cfq888dbvg_lvm"/>
> <lvm ref="cfq888revg_lvm"/>
> <lvm ref="cfq888flashvg_lvm"/>
> <fs ref="cfq888_mnt">
> <fs ref="cfq888_ar_mnt"/>
> <fs ref="cfq888_sw_mnt"/>
> <fs ref="cfq888_dmp_mnt"/>
> <fs ref="cfq888_bk_mnt">
> <fs ref="cfq888_flash_mnt"/>
> </fs>
> <fs ref="cfq888_db_mnt"/>
> <fs ref="cfq888_rd_mnt"/>
> </fs>
> <ip ref="192.168.1.31"/>
> <oracledb ref="cfq888"/>
> </service>
> </rm>
> </cluster>
>
>
> thanks,
> --Jason
>
>
>
> --
> Linux-cluster mailing list
> Linux-cluster at redhat.com
> https://www.redhat.com/**mailman/listinfo/linux-cluster<https://www.redhat.com/mailman/listinfo/linux-cluster>
>
--
esta es mi vida e me la vivo hasta que dios quiera
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://listman.redhat.com/archives/linux-cluster/attachments/20120622/33943b37/attachment.htm>
More information about the Linux-cluster
mailing list