[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [Linux-cluster] Fwd: CLVM exclusive mode



Hi Brem

El jue, 30-07-2009 a las 09:15 +0200, brem belguebli escribió:
> Hi,
>  
> does it look like we're hiting some "undesired feature" ;-)
>  
> Concerning the 0 nodeid, I think I read that on some Redhat documents
> or bugzilla report, I could find it out.
>  
> Brem 
> 
>  
>         
> 
> --
> Linux-cluster mailing list
> Linux-cluster redhat com
> https://www.redhat.com/mailman/listinfo/linux-cluster

I made some test on my lab environment too, i attach the results in the
TXT file.

My conclusions:

1.- lovgols with exclusive flag must be used over clustered volume
groups (obvious and already known)
2.- logvols activated with exclusive flag must be handled EXCLUSIVELY
with the exclusive flag

---> as part of my lvm-cluster.sh resource script, the exclusive flag is
part of the resource definition in cluster.conf so this is correctly
handled
	
3.- you can activate an already active exclusive logvol on any node if
you dont take into accout, during the activation, the exclusive flag
4.- in use (opened) logvols are protected from deactivation from
secondary nodes, even from main node
5.- after a node failure (hang-up, fencing...) logvol is not open
anymore, so it can be exclusively activated on a new node

All this was tested manually, but this is the expected behaviour on
lvm-cluster.sh resource script.

Link to lvm-cluster.sh resource script:

https://www.redhat.com/archives/cluster-devel/2009-June/msg00020.html

Cheers,

Rafael

-- 
Rafael Micó Miranda
# Nodes are ccopru08 and ccopru17
# ccopru17 will act as the main node, where i try to get exclusive access to logvols
# ccopru08 will act as the secondary node, trying to get resources and trying to produce "bad things"

# volgrp = volume group
# logvol = logical volume over a volume group

# general system information

[root ccopru08 dlm]# cat /etc/issue
Red Hat Enterprise Linux Server release 5.3 (Tikanga)
Kernel \r on an \m



[root ccopru08 ~]# rpm -qa | grep -e "\(cman\|openais\|rgmanager\|lvm\)"
cman-2.0.98-1.el5
rgmanager-2.0.46-1.el5
lvm2-2.02.40-6.el5
openais-0.80.3-22.el5
lvm2-cluster-2.02.40-7.el5
system-config-lvm-1.1.5-1.0.el5


[root ccopru08 etc]# greplimpio.sh /etc/multipath.conf 
blacklist {
        devnode         "sda"
}
blacklist_exceptions {
        devnode "sda[a-z]"
}
defaults {
        user_friendly_names yes
}
multipaths {
        multipath {
                wwid                    360014380024d2e260000900000770000
                alias                   quorum
        }
        multipath {
                wwid                    360014380024d2e2600009000007d0000
                alias                   clvm01
        }
        multipath {
                wwid                    360014380024d2e260000900000830000
                alias                   clvm02
        }
        multipath {
                wwid                    360014380024d2e260000900000890000
                alias                   clvm03
        }
        multipath {
                wwid                    360014380024d2e2600009000008f0000
                alias                   vol31
        }
        multipath {
                wwid                    360014380024d2e260000900000950000
                alias                   vol33
        }
        multipath {
                wwid                    360014380024d2e2600009000009b0000
                alias                   vol34
        }
        multipath {
                wwid                    360014380024d2e260000900000a10000
                alias                   vol55
        }
}
devices {
        device {
                vendor "(COMPAQ|HP)"
                product "HSV1[01]1|HSV2[01]0|HSV300|HSV4[05]0"
                getuid_callout "/sbin/scsi_id -g -u -s /block/%n"
                prio_callout "/sbin/mpath_prio_alua /dev/%n"
                hardware_handler "0"
                path_selector "round-robin 0"
                path_grouping_policy group_by_prio
                failback immediate
                rr_weight uniform
                no_path_retry 12
                rr_min_io 100
                path_checker tur
        }
}

[root ccopru08 etc]# greplimpio.sh /etc/lvm/lvm.conf
devices {
    dir = "/dev"
    scan = [ "/dev" ]
    preferred_names = [ "^/dev/mpath/", "^/dev/mapper/mpath", "^/dev/[hs]d" ]
    filter = [ "r|/dev/sd|", "a|/dev/mpath/.*|", "r|.*|" ]
    cache_dir = "/etc/lvm/cache"
    cache_file_prefix = ""
    write_cache_state = 1
    sysfs_scan = 1
    md_component_detection = 1
    md_chunk_alignment = 1
    ignore_suspended_devices = 0
}
log {
    verbose = 0
    syslog = 1
    file = "/var/log/lvm2.log"
    overwrite = 0
    level = 0
    indent = 1
    command_names = 0
    prefix = "  "
}
backup {
    backup = 1
    backup_dir = "/etc/lvm/backup"
    archive = 1
    archive_dir = "/etc/lvm/archive"
    retain_min = 10
    retain_days = 30
}
shell {
    history_size = 100
}
global {
    library_dir = "/usr/lib64"
    umask = 077
    test = 0
    units = "h"
    activation = 1
    proc = "/proc"
    locking_type = 3
    fallback_to_clustered_locking = 1
    fallback_to_local_locking = 1
    locking_dir = "/var/lock/lvm"
}
activation {
    missing_stripe_filler = "error"
    reserved_stack = 256
    reserved_memory = 8192
    process_priority = -18
    mirror_region_size = 512
    readahead = "auto"
    mirror_log_fault_policy = "allocate"
    mirror_device_fault_policy = "remove"
}
dmeventd {
    mirror_library = "libdevmapper-event-lvm2mirror.so"
    snapshot_library = "libdevmapper-event-lvm2snapshot.so"
}

[root ccopru08 etc]# cat /etc/cluster/cluster.conf
<?xml version="1.0" ?>
<cluster alias="cluster01" config_version="1" name="cluster01">
        <fence_daemon post_fail_delay="0" post_join_delay="3"/>
        <clusternodes>
                <clusternode name="ccopru08" nodeid="1" votes="1">
                        <fence/>
                </clusternode>
                <clusternode name="ccopru17" nodeid="2" votes="1">
                        <fence/>
                </clusternode>
        </clusternodes>
        <cman expected_votes="1" two_node="1">
        </cman>
        <fencedevices>
                <fencedevice agent="fence_drac" hostname="ccopru08-ilo" login="***" name="ccopru08-fence" passwd="***"/>
                <fencedevice agent="fence_drac" hostname="ccopru17-ilo" login="***" name="ccopru17-fence" passwd="***"/>
        </fencedevices>
        <rm>
                <failoverdomains/>
                <resources/>
        </rm>
</cluster>

# end of general system information

# start of tests

# creation of pvs and vgs

[root ccopru08 etc]# pvcreate /dev/mpath/clvm01 
  Physical volume "/dev/mpath/clvm01" successfully created
 
[root ccopru17 dlm]# pvs
  PV                VG   Fmt  Attr PSize   PFree  
  /dev/mpath/clvm01      lvm2 --   269,00G 269,00G

[root ccopru17 dlm]# vgcreate -c n volgrp01 /dev/mpath/clvm01 
  Non-clustered volume group "volgrp01" successfully created

[root ccopru08 dlm]# vgs
  VG       #PV #LV #SN Attr   VSize   VFree  
  volgrp01   1   0   0 wz--n- 269,00G 269,00G

[root ccopru17 dlm]# vgchange -a n volgrp01
  0 logical volume(s) in volume group "volgrp01" now active
  
# change to clustered vg

[root ccopru17 dlm]# vgchange -c y volgrp01 
  Volume group "volgrp01" successfully changed
  
[root ccopru17 dlm]# vgs
  VG       #PV #LV #SN Attr   VSize   VFree  
  volgrp01   1   0   0 wz--nc 269,00G 269,00G

# creation of logvol01

[root ccopru17 dlm]# lvcreate -n logvol01 -L 5G volgrp01
  Logical volume "logvol01" created
  
# logvol is reachable on all nodes

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2


[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2



[root ccopru08 dlm]# lvdisplay -v /dev/volgrp01/logvol01
    Using logical volume(s) on command line
  --- Logical volume ---
  LV Name                /dev/volgrp01/logvol01
  VG Name                volgrp01
  LV UUID                oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  LV Write Access        read/write
  LV Status              available
  # open                 0
  LV Size                5,00 GB
  Current LE             1280
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     256
  Block device           253:8

[root ccopru17 dlm]# lvdisplay -v /dev/volgrp01/logvol01
    Using logical volume(s) on command line
  --- Logical volume ---
  LV Name                /dev/volgrp01/logvol01
  VG Name                volgrp01
  LV UUID                oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  LV Write Access        read/write
  LV Status              available
  # open                 0
  LV Size                5,00 GB
  Current LE             1280
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     256
  Block device           253:8


Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
26a0001 2 1a70001 12991 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
2790001 0 0 7844 0 1 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81083a61a600 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
026a0001 CR Remote:   2 01a70001
02790001 CR
Conversion Queue
Waiting Queue



Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
1a70001 1 26a0001 12991 0 1 1 2 1 -1 0 1 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81011cfc0600 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 1
Granted Queue
01a70001 CR Master:     026a0001
Conversion Queue
Waiting Queue

# deactivation of logvol01

[root ccopru17 dlm]# lvchange -an volgrp01/logvol01

Node ccopru08
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

Node ccopru17
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

# activation with exclusive flag of logvol01 as part of full volgrp01

[root ccopru17 dlm]# lvchange -aey volgrp01
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

Node ccopru08
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3bd0001 0 0 12991 0 1 1 2 5 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81083db55e00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
03bd0001 EX
Conversion Queue
Waiting Queue

# activation of logvol01 on the other node, without using the exclusive flag

[root ccopru08 dlm]# lvchange -ay volgrp01/logvol01
[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
1790001 2 3720001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81076e0ca800 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 2
Granted Queue
01790001 CR Master:     03720001
Conversion Queue
Waiting Queue


Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3720001 1 1790001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
3bd0001 0 0 12991 0 5 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81083db55e00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
03720001 CR Remote:   1 01790001
03bd0001 CR
Conversion Queue
Waiting Queue

# deactivation, again, of logvol01

[root ccopru17 dlm]# lvchange -an volgrp01/logvol01
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2

Node ccopru08
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

Node ccopru17
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

# creation of a second logvol, logvol02

[root ccopru17 dlm]# lvcreate -n logvol02 -L 5G volgrp01
  Logical volume "logvol02" created
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
31e0001 2 3680001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"


Contents of /debug/dlm/clvmd

Resource ffff8107c196de00 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Local Copy, Master is node 2
Granted Queue
031e0001 CR Master:     03680001
Conversion Queue
Waiting Queue

Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
3680001 1 31e0001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
3940001 0 0 12991 0 1 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"


Contents of /debug/dlm/clvmd

Resource ffff81083f9b1000 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88tfIeS8eMkJIehccljM0vSW56gdaLYujYV"
Master Copy
Granted Queue
03680001 CR Remote:   1 031e0001
03940001 CR
Conversion Queue
Waiting Queue

# deactivation of logvol02

[root ccopru17 dlm]# lvchange -an volgrp01/logvol02
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

Node ccopru08
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

Node ccopru17
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd

# activation of logvol01 with exclusive flag, no activation of logvol02

[root ccopru17 dlm]# lvchange -aey volgrp01/logvol01
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV


Node ccopru08
Contents of /debug/dlm/clvmd_locks


Contents of /debug/dlm/clvmd


Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
e80001 0 0 12991 0 1 1 2 5 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff810825026400 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
00e80001 EX
Conversion Queue
Waiting Queue


[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

# try to activate logvol01 on other node, with exclusive flag

[root ccopru08 dlm]# lvchange -aey volgrp01/logvol01 
  Error locking on node ccopru08: Volume is busy on another node

# so this is OK

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvchange -aly volgrp01/logvol01 
  Error locking on node ccopru08: Volume is busy on another node

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

# second try, activation without using the exclusive flag

[root ccopru08 dlm]# lvchange -ay volgrp01/logvol01

# success, undesired behaviour

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

Node ccopru08
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
2b00001 2 23b0001 7844 0 1 1 2 1 -1 0 2 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff81083b9f4200 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Local Copy, Master is node 2
Granted Queue
02b00001 CR Master:     023b0001
Conversion Queue
Waiting Queue

Node ccopru17
Contents of /debug/dlm/clvmd_locks
id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name
23b0001 1 2b00001 7844 0 1 10001 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
e80001 0 0 12991 0 5 1 2 1 -1 0 0 64 "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"


Contents of /debug/dlm/clvmd

Resource ffff810825026400 Name (len=64) "q3J161vB2DUWbzULDf2LIA5iotmPU88toYLF252lZuSSFzIw9N1fJQ1dHJTATNp2"
Master Copy
Granted Queue
023b0001 CR Remote:   1 02b00001
00e80001 CR
Conversion Queue
Waiting Queue

# deactivation of logvol01 on both nodes

[root ccopru17 dlm]# lvchange -an volgrp01/logvol01

[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV
  
# activation of logvol01 with exclusive flag
 
[root ccopru17 dlm]# lvchange -aey volgrp01/logvol01

[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV


[root ccopru17 dlm]# mkfs.ext3 /dev/volgrp01/logvol01

[root ccopru17 dlm]# mkdir /media/aux
[root ccopru17 dlm]# mount /dev/volgrp01/logvol01 /media/aux/
[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-ao 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

# deactivation of logvol01, in use (opened), from the active node

[root ccopru17 dlm]# lvchange -aen volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating
# good behaviour

# try to deactivate from the other node

[root ccopru08 dlm]# lvchange -aen volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating

[root ccopru08 dlm]# lvchange -an volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating

# good behaviour

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

# try to activate in the other node with exclusive flag

[root ccopru08 dlm]# lvchange -aey volgrp01/logvol01
  Error locking on node ccopru08: Volume is busy on another node
#good behaviour

# try to activate in the other node without exclusive flag

[root ccopru08 dlm]# lvchange -ay volgrp01/logvol01

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-a- 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

# try to deactivate from the other node

[root ccopru08 dlm]# lvchange -an volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating

[root ccopru08 dlm]# lvchange -aen volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating

[root ccopru08 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru17 dlm]# lvs -v 
    Finding all logical volumes
  LV       VG       #Seg Attr   LSize Maj Min KMaj KMin Origin Snap%  Move Copy%  Log Convert LV UUID                               
  logvol01 volgrp01    1 -wi-ao 5,00G  -1  -1 253  8                                          oYLF25-2lZu-SSFz-Iw9N-1fJQ-1dHJ-TATNp2
  logvol02 volgrp01    1 -wi--- 5,00G  -1  -1 -1   -1                                         fIeS8e-MkJI-ehcc-ljM0-vSW5-6gda-LYujYV

[root ccopru08 dlm]# lvchange -aen volgrp01/logvol01
  Error locking on node ccopru17: LV volgrp01/logvol01 in use: not deactivating


# volume is yet in the main node



Conclusions:

1.- lovgols with exclusive flag must be used over clustered volume groups (obvious and already known)
2.- logvols activated with exclusive flag must be handled EXCLUSIVELY with the exclusive flag

---> as part of my lvm-cluster.sh resource script, the exclusive flag is part of the resource definition in cluster.conf so this is correctly handled
	
3.- you can activate an already active exclusive logvol on any node if you dont take into accout, during the activation, the exclusive flag
4.- in use (opened) logvols are protected from deactivation from secondary nodes, even from main node
5.- after a node faiule (hang-up, fencing...) logvol is not open anymore, so it can be exclusively activated on a new node

All this was tested manually, but this is the expected behaviour on lvm-cluster.sh resurce script.

Link to lvm-cluster.sh resoruce script:

https://www.redhat.com/archives/cluster-devel/2009-June/msg00020.html

Cheers,

Rafael

[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]