[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [Linux-cluster] pvmove locking problem CLVM on RHEL 6



Hello Jay

The error is so clear, you have lvm configured in exclusive mode, that means you can't access to your vg from more then one node at a time
2012/6/21 Jay Tingle <yogsothoth sinistar org>
Hi All,    I am having a problem using pvmove during some testing with Red Hat
Cluster using CLVM on RHEL 6.2.  I have 3 nodes which are ESXi 5u1 VMs with the
'multi-writer' flag set for the shared vmdk devices. I keep getting locking
errors during the pvmove.  Everything else seems to be working great as far as
CLVM goes.  Searching through the list archives and consulting the manuals it
looks like all you need is to have cmirrord running.  The RHEL 6 manual
mentions cmirror-kmod which doesn't seem to exist anymore.  Is there still a
kernel module on RHEL 6?  I am standard clvm with ext4 in an active/passive
cluster.  Anyone know what I am doing wrong?  Below is my lvm config and my
cluster config.  Thanks in advance.

[root rhc6esx1 ~]# rpm -qa|grep -i lvm
lvm2-libs-2.02.87-6.el6.x86_64
lvm2-2.02.87-6.el6.x86_64
lvm2-cluster-2.02.87-6.el6.x86_64
[root rhc6esx1 ~]# rpm -q cman
cman-3.0.12.1-23.el6.x86_64
[root rhc6esx1 ~]# rpm -q cmirror
cmirror-2.02.87-6.el6.x86_64

[root rhc6esx1 ~]# ps -ef|grep cmirror
root     21253 20692  0 13:37 pts/1    00:00:00 grep cmirror
root     31858     1  0 13:18 ?        00:00:00 cmirrord
      [root rhc6esx1 ~]# pvs|grep cfq888dbvg
 /dev/sdf1  cfq888dbvg        lvm2 a--  20.00g     0
 /dev/sdi1  cfq888dbvg        lvm2 a--  20.00g     0
 /dev/sdj1  cfq888dbvg        lvm2 a--  20.00g     0
 /dev/sdk1  cfq888dbvg        lvm2 a--  80.00g 80.00g

[root rhc6esx1 ~]# pvmove -v /dev/sdi1 /dev/sdk1
  Finding volume group "cfq888dbvg"
  Executing: /sbin/modprobe dm-log-userspace
  Archiving volume group "cfq888dbvg" metadata (seqno 7).
  Creating logical volume pvmove0
  Moving 5119 extents of logical volume cfq888dbvg/cfq888_db
 Error locking on node rhc6esx1-priv: Device or resource busy
 Error locking on node rhc6esx3-priv: Volume is busy on another node
 Error locking on node rhc6esx2-priv: Volume is busy on another node
 Failed to activate cfq888_db

[root rhc6esx1 ~]# clustat
Cluster Status for rhc6 @ Thu Jun 21 13:35:49 2012
Member Status: Quorate

Member Name                                              ID   Status
------ ----                                              ---- ------
rhc6esx1-priv                                                1 Online, Local, rgmanager
rhc6esx2-priv                                                2 Online, rgmanager
rhc6esx3-priv                                                3 Online, rgmanager
/dev/block/8:33                                              0 Online, Quorum Disk

Service Name                                    Owner (Last)                                    State
------- ----                                    ----- ------                                    -----
service:cfq888_grp                              rhc6esx1-priv                                   started

[root rhc6esx1 ~]# lvm dumpconfig
 devices {
      dir="/dev"
      scan="/dev"
      obtain_device_list_from_udev=1
      preferred_names=["^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d"]
      filter="a/.*/"
      cache_dir="/etc/lvm/cache"
      cache_file_prefix=""
      write_cache_state=1
      sysfs_scan=1
      md_component_detection=1
      md_chunk_alignment=1
      data_alignment_detection=1
      data_alignment=0
      data_alignment_offset_detection=1
      ignore_suspended_devices=0
      disable_after_error_count=0
      require_restorefile_with_uuid=1
      pv_min_size=2048
      issue_discards=0
 }
 dmeventd {
      mirror_library="libdevmapper-event-lvm2mirror.so"
      snapshot_library="libdevmapper-event-lvm2snapshot.so"
 }
 activation {
      checks=0
      udev_sync=1
      udev_rules=1
      verify_udev_operations=0
      missing_stripe_filler="error"
      reserved_stack=256
      reserved_memory=8192
      process_priority=-18
      mirror_region_size=512
      readahead="auto"
      mirror_log_fault_policy="allocate"
      mirror_image_fault_policy="remove"
      snapshot_autoextend_threshold=100
      snapshot_autoextend_percent=20
      use_mlockall=0
      monitoring=1
      polling_interval=15
 }
 global {
      umask=63
      test=0
      units="h"
      si_unit_consistency=1
      activation=1
      proc="/proc"
      locking_type=3
      wait_for_locks=1
      fallback_to_clustered_locking=1
      fallback_to_local_locking=1
      locking_dir="/var/lock/lvm"
      prioritise_write_locks=1
      abort_on_internal_errors=0
      detect_internal_vg_cache_corruption=0
      metadata_read_only=0
      mirror_segtype_default="mirror"
 }
 shell {
      history_size=100
 }
 backup {
      backup=1
      backup_dir="/etc/lvm/backup"
      archive=1
      archive_dir="/etc/lvm/archive"
      retain_min=10
      retain_days=30
 }
 log {
      verbose=0
      syslog=1
      overwrite=0
      level=0
      indent=1
      command_names=0
      prefix="  "
 }


[root rhc6esx1 ~]# ccs -h localhost --getconf
<cluster config_version="273" name="rhc6">
 <fence_daemon clean_start="0" post_fail_delay="20" post_join_delay="60"/>
 <clusternodes>
  <clusternode name="rhc6esx1-priv" nodeid="1">
    <fence>
      <method name="1">
        <device name="fence_vmware" uuid="422a2b6a-4093-2694-65e0-a01332ef54bd"/>
      </method>
    </fence>
  </clusternode>
  <clusternode name="rhc6esx2-priv" nodeid="2">
    <fence>
      <method name="1">
        <device name="fence_vmware" uuid="422a9c5d-f9e2-8150-340b-c84b834ba068"/>
      </method>
    </fence>
  </clusternode>
  <clusternode name="rhc6esx3-priv" nodeid="3">
    <fence>
      <method name="1">
        <device name="fence_vmware" uuid="422af24c-909f-187d-4e64-2a28cbe5d09d"/>
      </method>
    </fence>
  </clusternode>
 </clusternodes>
 <cman expected_votes="5"/>
 <fencedevices>
  <fencedevice agent="fence_vmware_soap" ipaddr="192.168.1.111" login="mrfence" name="fence_vmware" passwd="FenceM3" ssl="yes" verbose="yes"/>
 </fencedevices>
 <totem token="30000"/>
 <quorumd interval="1" label="rhc6esx-quorum" stop_cman="1" tko="10" votes="2"/>
 <logging logfile_priority="info" syslog_facility="daemon" syslog_priority="warning" to_logfile="yes" to_syslog="yes">
  <logging_daemon logfile="/var/log/cluster/qdiskd.log" name="qdiskd"/>
  <logging_daemon logfile="/var/log/cluster/fenced.log" name="fenced"/>
  <logging_daemon logfile="/var/log/cluster/dlm_controld.log" name="dlm_controld"/>
  <logging_daemon logfile="/var/log/cluster/gfs_controld.log" name="gfs_controld"/>
  <logging_daemon debug="on" logfile="/var/log/cluster/rgmanager.log" name="rgmanager"/>
  <logging_daemon name="corosync" to_logfile="no"/>
 </logging>
 <rm log_level="7">
  <failoverdomains>
    <failoverdomain name="rhc6esx3_home" nofailback="1" ordered="1" restricted="1">
      <failoverdomainnode name="rhc6esx3-priv" priority="1"/>
      <failoverdomainnode name="rhc6esx2-priv" priority="2"/>
      <failoverdomainnode name="rhc6esx1-priv" priority="3"/>
    </failoverdomain>
    <failoverdomain name="rhc6esx2_home" nofailback="1" ordered="1" restricted="1">
      <failoverdomainnode name="rhc6esx2-priv" priority="1"/>
      <failoverdomainnode name="rhc6esx1-priv" priority="2"/>
      <failoverdomainnode name="rhc6esx3-priv" priority="3"/>
    </failoverdomain>
    <failoverdomain name="rhc6esx1_home" nofailback="1" ordered="1" restricted="1">
      <failoverdomainnode name="rhc6esx1-priv" priority="1"/>
      <failoverdomainnode name="rhc6esx2-priv" priority="2"/>
      <failoverdomainnode name="rhc6esx3-priv" priority="3"/>
    </failoverdomain>
  </failoverdomains>
  <resources>
    <lvm name="cfq888vg_lvm" self_fence="1" vg_name="cfq888vg"/>
    <lvm name="cfq888bkpvg_lvm" self_fence="1" vg_name="cfq888bkpvg"/>
    <lvm name="cfq888dbvg_lvm" self_fence="1" vg_name="cfq888dbvg"/>
    <lvm name="cfq888revg_lvm" vg_name="cfq888revg"/>
    <lvm name="cfq888flashvg_lvm" self_fence="1" vg_name="cfq888flashvg"/>
    <ip address="192.168.1.31" monitor_link="1"/>
    <fs device="/dev/cfq888vg/cfq888" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888" name="cfq888_mnt" self_fence="0"/>
    <fs device="/dev/cfq888vg/cfq888_ar" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_ar" name="cfq888_ar_mnt" self_fence="0"/>
    <fs device="/dev/cfq888vg/cfq888_sw" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_sw" name="cfq888_sw_mnt" self_fence="0"/>
    <fs device="/dev/cfq888bkpvg/cfq888_dmp" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_dmp" name="cfq888_dmp_mnt" self_fence="0"/>
    <fs device="/dev/cfq888bkpvg/cfq888_bk" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_bk" name="cfq888_bk_mnt" self_fence="0"/>
    <fs device="/dev/cfq888dbvg/cfq888_db" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_db" name="cfq888_db_mnt" self_fence="0"/>
    <fs device="/dev/cfq888flashvg/cfq888_flash" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_bk/cfq888_flash" name="cfq888_flash_mnt" self_fence="0"/>
    <fs device="/dev/cfq888revg/cfq888_rd" force_fsck="0" force_unmount="1" fstype="ext4" mountpoint="/cfq888/cfq888_rd" name="cfq888_rd_mnt" self_fence="0"/>
    <oracledb home="/u01/app/oracle/product/11.2.0/dbhome_1" listener_name="cfq888_lsnr" name="cfq888" type="base" user="oracle"/>
  </resources>
  <service autostart="1" domain="rhc6esx1_home" exclusive="0" name="cfq888_grp" recovery="restart">
    <lvm ref="cfq888vg_lvm"/>
    <lvm ref="cfq888bkpvg_lvm"/>
    <lvm ref="cfq888dbvg_lvm"/>
    <lvm ref="cfq888revg_lvm"/>
    <lvm ref="cfq888flashvg_lvm"/>
    <fs ref="cfq888_mnt">
      <fs ref="cfq888_ar_mnt"/>
      <fs ref="cfq888_sw_mnt"/>
      <fs ref="cfq888_dmp_mnt"/>
      <fs ref="cfq888_bk_mnt">
        <fs ref="cfq888_flash_mnt"/>
      </fs>
      <fs ref="cfq888_db_mnt"/>
      <fs ref="cfq888_rd_mnt"/>
    </fs>
    <ip ref="192.168.1.31"/>
    <oracledb ref="cfq888"/>
  </service>
 </rm>
</cluster>


thanks,
--Jason



--
Linux-cluster mailing list
Linux-cluster redhat com
https://www.redhat.com/mailman/listinfo/linux-cluster



--
esta es mi vida e me la vivo hasta que dios quiera

[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]