[Linux-cluster] Fwd: CLVM exclusive mode
Rafael Micó Miranda
rmicmirregs at gmail.com
Thu Jul 30 14:15:04 UTC 2009
Hi Brem
El jue, 30-07-2009 a las 09:15 +0200, brem belguebli escribió:
> Hi,
>
> does it look like we're hiting some "undesired feature" ;-)
>
> Concerning the 0 nodeid, I think I read that on some Redhat documents
> or bugzilla report, I could find it out.
>
> Brem
>
>
>
>
> --
> Linux-cluster mailing list
> Linux-cluster at redhat.com
> https://www.redhat.com/mailman/listinfo/linux-cluster
I made some test on my lab environment too, i attach the results in the
TXT file.
My conclusions:
1.- lovgols with exclusive flag must be used over clustered volume
groups (obvious and already known)
2.- logvols activated with exclusive flag must be handled EXCLUSIVELY
with the exclusive flag
---> as part of my lvm-cluster.sh resource script, the exclusive flag is
part of the resource definition in cluster.conf so this is correctly
handled
3.- you can activate an already active exclusive logvol on any node if
you dont take into accout, during the activation, the exclusive flag
4.- in use (opened) logvols are protected from deactivation from
secondary nodes, even from main node
5.- after a node failure (hang-up, fencing...) logvol is not open
anymore, so it can be exclusively activated on a new node
All this was tested manually, but this is the expected behaviour on
lvm-cluster.sh resource script.
Link to lvm-cluster.sh resource script:
https://www.redhat.com/archives/cluster-devel/2009-June/msg00020.html
Cheers,
Rafael
--
Rafael Micó Miranda
-------------- next part --------------
# Nodes are ccopru08 and ccopru17
# ccopru17 will act as the main node, where i try to get exclusive access to logvols
# ccopru08 will act as the secondary node, trying to get resources and trying to produce "bad things"
# volgrp = volume group
# logvol = logical volume over a volume group
# general system information
[root at ccopru08 dlm]# cat /etc/issue
Red Hat Enterprise Linux Server release 5.3 (Tikanga)
Kernel \r on an \m
[root at ccopru08 ~]# rpm -qa | grep -e "\(cman\|openais\|rgmanager\|lvm\)"
cman-2.0.98-1.el5
rgmanager-2.0.46-1.el5
lvm2-2.02.40-6.el5
openais-0.80.3-22.el5
lvm2-cluster-2.02.40-7.el5
system-config-lvm-1.1.5-1.0.el5
[root at ccopru08 etc]# greplimpio.sh /etc/multipath.conf
blacklist {
devnode "sda"
}
blacklist_exceptions {
devnode "sda[a-z]"
}
defaults {
user_friendly_names yes
}
multipaths {
multipath {
wwid 360014380024d2e260000900000770000
alias quorum
}
multipath {
wwid 360014380024d2e2600009000007d0000
alias clvm01
}
multipath {
wwid 360014380024d2e260000900000830000
alias clvm02
}
multipath {
wwid 360014380024d2e260000900000890000
alias clvm03
}
multipath {
wwid 360014380024d2e2600009000008f0000
alias vol31
}
multipath {
wwid 360014380024d2e260000900000950000
alias vol33
}
multipath {
wwid 360014380024d2e2600009000009b0000
alias vol34
}
multipath {
wwid 360014380024d2e260000900000a10000
alias vol55
}
}
devices {
device {
vendor "(COMPAQ|HP)"
product "HSV1[01]1|HSV2[01]0|HSV300|HSV4[05]0"
getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
prio_callout "/sbin/mpath_prio_alua /dev/%n"
hardware_handler "0"
path_selector "round-robin 0"
path_grouping_policy group_by_prio
failback immediate
rr_weight uniform
no_path_retry 12
rr_min_io 100
path_checker tur
}
}
[root at ccopru08 etc]# greplimpio.sh /etc/lvm/lvm.conf
devices {
dir = "/dev"
scan = [ "/dev" ]
preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ]
filter = [ "r|/dev/sd|", "a|/dev/mpath/.*|", "r|.*|" ]
cache_dir = "/etc/lvm/cache"
cache_file_prefix = ""
write_cache_state = 1
sysfs_scan = 1
md_component_detection = 1
md_chunk_alignment = 1
ignore_suspended_devices = 0
}
log {
verbose = 0
syslog = 1
file = "/var/log/lvm2.log"
overwrite = 0
level = 0
indent = 1
command_names = 0
prefix = " "
}
backup {
backup = 1
backup_dir = "/etc/lvm/backup"
archive = 1
archive_dir = "/etc/lvm/archive"
retain_min = 10
retain_days = 30
}
shell {
history_size = 100
}
global {
library_dir = "/usr/lib64"
umask = 077
test = 0
units = "h"
activation = 1
proc = "/proc"
locking_type = 3
fallback_to_clustered_locking = 1
fallback_to_local_locking = 1
locking_dir = "/var/lock/lvm"
}
activation {
missing_stripe_filler = "error"
reserved_stack = 256
reserved_memory = 8192
process_priority = -18
mirror_region_size = 512
readahead = "auto"
mirror_log_fault_policy = "allocate"
mirror_device_fault_policy = "remove"
}
dmeventd {
mirror_library = "libdevmapper-event-lvm2mirror.so"
snapshot_library = "libdevmapper-event-lvm2snapshot.so"
}
[root at ccopru08 etc]# cat /etc/cluster/cluster.conf
<?xml version="1.0" ?>
<cluster alias="cluster01" config_version="1" name="cluster01">
<fence_daemon post_fail_delay="0" post_join_delay="3"/>
<clusternodes>
<clusternode name="ccopru08" nodeid="1" votes="1">
<fence/>
</clusternode>
<clusternode name="ccopru17" nodeid="2" votes="1">
<fence/>
</clusternode>
</clusternodes>
<cman expected_votes="1" two_node="1">
</cman>
<fencedevices>
<fencedevice agent="fence_drac" hostname="ccopru08-ilo" login="***" name="ccopru08-fence" passwd="***"/>
<fencedevice agent="fence_drac" hostname="ccopru17-ilo" login="***" name="ccopru17-fence" passwd="***"/>
</fencedevices>
<rm>
<failoverdomains/>
<resources/>
</rm>
</cluster>
# end of general system information
# start of tests
# creation of pvs and vgs
[root at ccopru08 etc]# pvcreate /dev/mpath/clvm01
Physical volume "/dev/mpath/clvm01" successfully created
[root at ccopru17 dlm]# pvs
PV VG Fmt Attr PSize PFree
/dev/mpath/clvm01 lvm2 -- 269,00G 269,00G
[root at ccopru17 dlm]# vgcreate -c n volgrp01 /dev/mpath/clvm01
Non-clustered volume group "volgrp01" successfully created
[root at ccopru08 dlm]# vgs
VG #PV #LV #SN Attr VSize VFree
volgrp01 1 0 0 wz--n- 269,00G 269,00G
[root at ccopru17 dlm]# vgchange -a n volgrp01
0 logical volume(s) in volume group "volgrp01" now active
# change to clustered vg
[root at ccopru17 dlm]# vgchange -c y volgrp01
Volume group "volgrp01" successfully changed
[root at ccopru17 dlm]# vgs
VG #PV #LV #SN Attr VSize VFree
volgrp01 1 0 0 wz--nc 269,00G 269,00G
# creation of logvol01
[root at ccopru17 dlm]# lvcreate -n logvol01 -L 5G volgrp01
Logical volume "logvol01" created
# logvol is reachable on all nodes
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru08 dlm]# lvdisplay -v /dev/volgrp01/logvol01
Using logical volume(s) on command line
--- Logical volume ---
LV Name /dev/volgrp01/logvol01
VG Name volgrp01
LV UUID oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
LV Write Access read/write
LV Status available
# open 0
LV Size 5,00 GB
Current LE 1280
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:8
[root at ccopru17 dlm]# lvdisplay -v /dev/volgrp01/logvol01
Using logical volume(s) on command line
--- Logical volume ---
LV Name /dev/volgrp01/logvol01
VG Name volgrp01
LV UUID oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
LV Write Access read/write
LV Status available
# open 0
LV Size 5,00 GB
Current LE 1280
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 256
Block device 253:8
Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
26a0001 2 1a70001 12991 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
2790001 0 0 7844 0 1 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81083a61a600 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
026a0001 CR Remote: 2 01a70001
02790001 CR
Conversion Queue
Waiting Queue
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
1a70001 1 26a0001 12991 0 1 1 2 1 -1 0 1 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81011cfc0600 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 1
Granted Queue
01a70001 CR Master: 026a0001
Conversion Queue
Waiting Queue
# deactivation of logvol01
[root at ccopru17 dlm]# lvchange -an volgrp01/logvol01
Node ccopru08
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
Node ccopru17
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
# activation with exclusive flag of logvol01 as part of full volgrp01
[root at ccopru17 dlm]# lvchange -aey volgrp01
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
Node ccopru08
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3bd0001 0 0 12991 0 1 1 2 5 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81083db55e00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
03bd0001 EX
Conversion Queue
Waiting Queue
# activation of logvol01 on the other node, without using the exclusive flag
[root at ccopru08 dlm]# lvchange -ay volgrp01/logvol01
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
1790001 2 3720001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81076e0ca800 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 2
Granted Queue
01790001 CR Master: 03720001
Conversion Queue
Waiting Queue
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3720001 1 1790001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
3bd0001 0 0 12991 0 5 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81083db55e00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
03720001 CR Remote: 1 01790001
03bd0001 CR
Conversion Queue
Waiting Queue
# deactivation, again, of logvol01
[root at ccopru17 dlm]# lvchange -an volgrp01/logvol01
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
Node ccopru08
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
Node ccopru17
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
# creation of a second logvol, logvol02
[root at ccopru17 dlm]# lvcreate -n logvol02 -L 5G volgrp01
Logical volume "logvol02" created
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
31e0001 2 3680001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Contents of /debug/dlm/clvmd
Resource ffff8107c196de00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Local Copy, Master is node 2
Granted Queue
031e0001 CR Master: 03680001
Conversion Queue
Waiting Queue
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3680001 1 31e0001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
3940001 0 0 12991 0 1 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Contents of /debug/dlm/clvmd
Resource ffff81083f9b1000 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Master Copy
Granted Queue
03680001 CR Remote: 1 031e0001
03940001 CR
Conversion Queue
Waiting Queue
# deactivation of logvol02
[root at ccopru17 dlm]# lvchange -an volgrp01/logvol02
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
Node ccopru08
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
Node ccopru17
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
# activation of logvol01 with exclusive flag, no activation of logvol02
[root at ccopru17 dlm]# lvchange -aey volgrp01/logvol01
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
Node ccopru08
Contents of /debug/dlm/clvmd_locks
Contents of /debug/dlm/clvmd
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
e80001 0 0 12991 0 1 1 2 5 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff810825026400 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
00e80001 EX
Conversion Queue
Waiting Queue
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# try to activate logvol01 on other node, with exclusive flag
[root at ccopru08 dlm]# lvchange -aey volgrp01/logvol01
Error locking on node ccopru08: Volume is busy on another node
# so this is OK
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvchange -aly volgrp01/logvol01
Error locking on node ccopru08: Volume is busy on another node
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# second try, activation without using the exclusive flag
[root at ccopru08 dlm]# lvchange -ay volgrp01/logvol01
# success, undesired behaviour
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
2b00001 2 23b0001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff81083b9f4200 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 2
Granted Queue
02b00001 CR Master: 023b0001
Conversion Queue
Waiting Queue
Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
23b0001 1 2b00001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
e80001 0 0 12991 0 5 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Contents of /debug/dlm/clvmd
Resource ffff810825026400 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
023b0001 CR Remote: 1 02b00001
00e80001 CR
Conversion Queue
Waiting Queue
# deactivation of logvol01 on both nodes
[root at ccopru17 dlm]# lvchange -an volgrp01/logvol01
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# activation of logvol01 with exclusive flag
[root at ccopru17 dlm]# lvchange -aey volgrp01/logvol01
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru17 dlm]# mkfs.ext3 /dev/volgrp01/logvol01
[root at ccopru17 dlm]# mkdir /media/aux
[root at ccopru17 dlm]# mount /dev/volgrp01/logvol01 /media/aux/
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-ao 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# deactivation of logvol01, in use (opened), from the active node
[root at ccopru17 dlm]# lvchange -aen volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
# good behaviour
# try to deactivate from the other node
[root at ccopru08 dlm]# lvchange -aen volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
[root at ccopru08 dlm]# lvchange -an volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
# good behaviour
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# try to activate in the other node with exclusive flag
[root at ccopru08 dlm]# lvchange -aey volgrp01/logvol01
Error locking on node ccopru08: Volume is busy on another node
#good behaviour
# try to activate in the other node without exclusive flag
[root at ccopru08 dlm]# lvchange -ay volgrp01/logvol01
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-a- 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
# try to deactivate from the other node
[root at ccopru08 dlm]# lvchange -an volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
[root at ccopru08 dlm]# lvchange -aen volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
[root at ccopru08 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru17 dlm]# lvs -v
Finding all logical volumes
LV VG #Seg Attr LSize Maj Min KMaj KMin Origin Snap% Move Copy% Log Convert LV UUID
logvol01 volgrp01 1 -wi-ao 5,00G -1 -1 253 8 oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
logvol02 volgrp01 1 -wi--- 5,00G -1 -1 -1 -1 fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
[root at ccopru08 dlm]# lvchange -aen volgrp01/logvol01
Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
# volume is yet in the main node
Conclusions:
1.- lovgols with exclusive flag must be used over clustered volume groups (obvious and already known)
2.- logvols activated with exclusive flag must be handled EXCLUSIVELY with the exclusive flag
---> as part of my lvm-cluster.sh resource script, the exclusive flag is part of the resource definition in cluster.conf so this is correctly handled
3.- you can activate an already active exclusive logvol on any node if you dont take into accout, during the activation, the exclusive flag
4.- in use (opened) logvols are protected from deactivation from secondary nodes, even from main node
5.- after a node faiule (hang-up, fencing...) logvol is not open anymore, so it can be exclusively activated on a new node
All this was tested manually, but this is the expected behaviour on lvm-cluster.sh resurce script.
Link to lvm-cluster.sh resoruce script:
https://www.redhat.com/archives/cluster-devel/2009-June/msg00020.html
Cheers,
Rafael
More information about the Linux-cluster
mailing list