[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [Linux-cluster] Monitoring shared disk



Hi, again

I have three different configuration (im testing :) RH CS )

Note:
Hardwer is virtual (VMware server 1.0.4 on windows xp host)
Linux is CentOS 4.6

First:
====> LVM partition with GFS <====
in wmx file:
#SHARED Disk 1
scsi1.present = "TRUE"
scsi1.virtualDev = "lsilogic"
scsi1.sharedBus = "virtual"
scsi1:0.present = "TRUE"
scsi1:0.fileName = "D:\VmWare\clusterdisks\SharedDisk1.vmdk"
scsi1:0.mode = "independent-persistent"
scsi1:0.redo = ""
scsi1:0.deviceType = "disk"

#SHARED Disk 2
scsi2.present = "TRUE"
scsi2.virtualDev = "lsilogic"
scsi2.sharedBus = "virtual"
scsi2:0.present = "TRUE"
scsi2:0.fileName = "D:\VmWare\clusterdisks\SharedDisk2.vmdk"
scsi2:0.mode = "independent-persistent"
scsi2:0.redo = ""
scsi2:0.deviceType = "disk"

#SHARED BUS settings
disk.locking = "FALSE"
diskLib.dataCacheMaxReadAheadSize = "0"
diskLib.dataCacheMinReadAheadSize = "0"
diskLib.dataCacheMaxSize = "0"
diskLib.dataCachePageSize = "4096"
diskLib.maxUnsyncedWrites = "0"

[root ankebut]# cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.

10.81.4.98      ankebut.domain.com       ankebut
172.16.40.10    node1.localdomain       node1
10.81.4.104     yildiz.domain.com        yildiz
172.16.40.11    node2.localdomain       node2
127.0.0.1       localhost.localdomain   localhost

Installing this packages from
http://mirror.centos.org/centos/4/csgfs/CentOS-csgfs.repo:
rgmanager-1.9.72-1
system-config-cluster-1.0.51-2.0
ccs-1.0.11-1
magma-1.0.8-1
magma-plugins-1.0.12-0
cman-1.0.17-0
cman-kernel-2.6.9-53.5
gulm-1.0.10-0
dlm-1.0.7-1
dlm-kernel-2.6.9-52.2
fence-1.32.50-2
iddev-2.0.0-4
perl-Net-Telnet-3.03-3
GFS-6.1.15-1
GFS-kernel-2.6.9-75.9
GFS-kernheaders-2.6.9-75.9
gnbd-1.0.9-1
gnbd-kernel-2.6.9-10.29
gnbd-kernheaders-2.6.9-10.29
lvm2-cluster-2.02.27-2.el4.centos.1

[root ankebut ~]# vgdisplay -v /dev/VG_SHARE
    Using volume group(s) on command line
    Finding volume group "VG_SHARE"
  --- Volume group ---
  VG Name               VG_SHARE
  System ID
  Format                lvm2
  Metadata Areas        2
  Metadata Sequence No  3
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               0
  Max PV                0
  Cur PV                2
  Act PV                2
  VG Size               15.99 GB
  PE Size               4.00 MB
  Total PE              4094
  Alloc PE / Size       1002 / 3.91 GB
  Free  PE / Size       3092 / 12.08 GB
  VG UUID               C6D0HQ-AeIW-4k8a-Tzl5-ayGx-Tv14-DwkTe7

  --- Logical volume ---
  LV Name                /dev/VG_SHARE/LV_SHARE
  VG Name                VG_SHARE
  LV UUID                7zA1Wf-bn5J-XmD4-azfK-CM3F-y90F-XviVNn
  LV Write Access        read/write
  LV Status              available
  # open                 0
  LV Size                3.91 GB
  Current LE             1002
  Segments               1
  Allocation             inherit
  Read ahead sectors     0
  Block device           253:2

  --- Physical volumes ---
  PV Name               /dev/sda1
  PV UUID               qn9dKj-CIHQ-hyxr-JBdi-rsna-cKkL-SY2OM0
  PV Status             allocatable
  Total PE / Free PE    2047 / 1045

  PV Name               /dev/sdb1
  PV UUID               Kr4yil-DAks-i7jB-HU7M-CLjW-BK2e-9oO91W
  PV Status             allocatable
  Total PE / Free PE    2047 / 2047

[root ankebut ~]# gfs_mkfs -j 8 -t CentOS_RHCS:sharedd -p lock_dlm
/dev/VG_SHARE/LV_SHARE

[root ankebut]# cat /etc/cluster/cluster.conf
<?xml version="1.0"?>
<cluster alias="CentOS_RHCS" config_version="1" name="CentOS_RHCS">
  <fence_daemon post_fail_delay="0" post_join_delay="3"/>
  <clusternodes>
    <clusternode name="node1.localdomain" votes="1">
      <com_info>
        <syslog name="node1.localdomain"/>
          <sharedd name="/dev/VG_SHARE/LV_SHARE"/>
      </com_info>
      <fence>
        <method name="1">
          <device name="ManualFence" nodename="node1.localdomain"/>
        </method>
      </fence>
    </clusternode>
    <clusternode name="node2.localdomain" votes="1">
      <com_info>
        <syslog name="node2.localdomain"/>
          <sharedd name="/dev/VG_SHARE/LV_SHARE"/>
      </com_info>
      <fence>
        <method name="1">
          <device name="ManualFence" nodename="node2.localdomain"/>
        </method>
      </fence>
    </clusternode>
  </clusternodes>
  <cman expected_votes="1" two_node="1"/>
  <fencedevices>
    <fencedevice agent="fence_manual" name="ManualFence"/>
  </fencedevices>
  <rm>
    <failoverdomains/>
    <resources/>
  </rm>
</cluster>

[root ankebut]# mount -t gfs /dev/VG_SHARE/LV_SHARE /share/

[root ankebut]# cat /etc/fstab
/dev/VG_SHARE/LV_SHARE  /share                  gfs     _netdev 0 0

TEST SHARING
[root ankebut]# echo 'Date from ankebut is:' `date` >> /share/date
[root ankebut]# cat /share/date
Date from ankebut is: Fri Feb 1 12:40:17 CET 2008

[root yildiz ~]# echo 'Date from yildiz is:' `date` >> /share/date
[root yildiz ~]# cat /share/date
Date from ankebut is: Fri Feb 1 12:40:17 CET 2008
Date from yildiz is: Fri Feb 1 13:40:39 CET 2008


====> LVM partition with GFS <====
in vmw file
#SHARED Disk 1
scsi1.present = "TRUE"
scsi1.virtualDev = "lsilogic"
scsi1.sharedBus = "virtual"
scsi1:0.present = "TRUE"
scsi1:0.fileName = "D:\VmWare\clusterdisks\QuorumDisk.vmdk"
scsi1:0.mode = "independent-persistent"
scsi1:0.redo = ""
scsi1:0.deviceType = "disk"

#SHARED BUS settings
disk.locking = "FALSE"
diskLib.dataCacheMaxReadAheadSize = "0"
diskLib.dataCacheMinReadAheadSize = "0"
diskLib.dataCacheMaxSize = "0"
diskLib.dataCachePageSize = "4096"
diskLib.maxUnsyncedWrites = "0"

[root ankebut]# cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.

10.81.4.98      ankebut.domain.com      ankebut
172.16.40.10    node1.localdomain       node1
10.81.4.104     yildiz.domain.com        yildiz
172.16.40.11    node2.localdomain       node2
127.0.0.1       localhost.localdomain   localhost

Installing this packages from
http://mirror.centos.org/centos/4/csgfs/CentOS-csgfs.repo:
rgmanager-1.9.72-1
system-config-cluster-1.0.51-2.0
ccs-1.0.11-1
magma-1.0.8-1
magma-plugins-1.0.12-0
cman-1.0.17-0
cman-kernel-2.6.9-53.5
gulm-1.0.10-0
dlm-1.0.7-1
dlm-kernel-2.6.9-52.2
fence-1.32.50-2
iddev-2.0.0-4
perl-Net-Telnet-3.03-3

[root ankebut]# parted /dev/sda
(parted) p
Disk geometry for /dev/sda: 0.000-8192.000 megabytes
Disk label type: msdos
Minor    Start       End     Type      Filesystem  Flags
1          0.031      7.844  primary
2          7.844   8189.384  primary

[root ankebut]# mkqdisk -c /dev/sda1 -l QuorumLabel

[root ankebut]# mkqdisk -L
mkqdisk v0.5.1
/dev/sda1:
        Magic:   eb7a62c2
        Label:   QuorumLabel
        Created: Mon Feb  4 09:58:58 2008
        Host:    ankebut.domain.com

[root ankebut]# cat /etc/cluster/cluster.conf
<?xml version="1.0"?>
<cluster alias="CentOS_RHCS" config_version="1" name="CentOS_RHCS">
  <fence_daemon post_fail_delay="0" post_join_delay="3"/>
  <clusternodes>
    <clusternode name="node1.localdomain" votes="1">
      <fence>
        <method name="1">
          <device name="ManualFence" nodename="node1.localdomain"/>
        </method>
      </fence>
    </clusternode>
    <clusternode name="node2.localdomain" votes="1">
      <fence>
        <method name="1">
          <device name="ManualFence" nodename="node2.localdomain"/>
        </method>
      </fence>
    </clusternode>
  </clusternodes>
  <quorumd interval="1" tko="5" votes="3" log_level="7"
label="QuorumLabel" status_file="/tmp/quorum-status">
    <heuristic interval="2" program="/bin/ping node1.localdomain -c1
-t1" score="1"/>
    <heuristic interval="2" program="/bin/ping node2.localdomain -c1
-t1" score="1"/>
  </quorumd>
  <cman expected_votes="1" two_node="1"/>
  <fencedevices>
    <fencedevice agent="fence_manual" name="ManualFence"/>
  </fencedevices>
  <rm>
    <failoverdomains/>
    <resources/>
  </rm>
</cluster>

TEST QUORUM DISK
[root ankebut]# cman_tool nodes
Node  Votes Exp Sts  Name
   0    3    0   M   /dev/sda1
   1    1    1   M   node2.localdomain
   2    1    1   M   node1.localdomain

[root ankebut]# cman_tool status
Protocol version: 5.0.1
Config version: 1
Cluster name: CentOS_RHCS
Cluster ID: 42649
Cluster Member: Yes
Membership state: Cluster-Member
Nodes: 2
Expected_votes: 1
Total_votes: 5
Quorum: 1
Active subsystems: 2
Node name: node1.localdomain
Node ID: 2

[root ankebut]# cat /tmp/quorum-status
Time Stamp: Mon Feb  4 10:29:08 2008
Node ID: 2
Score: 1/1 (Minimum required = 1)
Current state: Master
Initializing Set: { }
Visible Set: { 1 2 }
Master Node ID: 2
Quorate Set: { 1 2 }


====> RAW partitions for Oracle <====
in vmx file:
#SHARED Disk 1
scsi1.present = "TRUE"
scsi1.virtualDev = "lsilogic"
scsi1.sharedBus = "virtual"
scsi1:0.present = "TRUE"
scsi1:0.fileName = "D:\VmWare\clusterdisks\SharedDisk.vmdk"
scsi1:0.mode = "independent-persistent"
scsi1:0.redo = ""
scsi1:0.deviceType = "disk"

#SHARED BUS settings
disk.locking = "FALSE"
diskLib.dataCacheMaxReadAheadSize = "0"
diskLib.dataCacheMinReadAheadSize = "0"
diskLib.dataCacheMaxSize = "0"
diskLib.dataCachePageSize = "4096"
diskLib.maxUnsyncedWrites = "0"

[root ankebut]# cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.

10.81.4.98      ankebut.domain.com       ankebut
172.16.40.10    node1.localdomain       node1
10.81.4.104     yildiz.domain.com        yildiz
172.16.40.11    node2.localdomain       node2
127.0.0.1       localhost.localdomain   localhost

Installing this packages from
http://mirror.centos.org/centos/4/csgfs/CentOS-csgfs.repo:
rgmanager-1.9.72-1
system-config-cluster-1.0.51-2.0
ccs-1.0.11-1
magma-1.0.8-1
magma-plugins-1.0.12-0
cman-1.0.17-0
cman-kernel-2.6.9-53.5
gulm-1.0.10-0
dlm-1.0.7-1
dlm-kernel-2.6.9-52.2
fence-1.32.50-2
iddev-2.0.0-4
perl-Net-Telnet-3.03-3

[root ankebut ~]# fdisk /dev/sda
Command (m for help): p

Disk /dev/sda: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/sda1               1         522     4192933+  83  Linux
/dev/sda2             523        1044     4192965   83  Linux

[root ankebut]# cat /etc/sysconfig/rawdevices
# This file and interface are deprecated.
# Applications needing raw device access should open regular
# block devices with O_DIRECT.
# raw device bindings
# format:  <rawdev> <major> <minor>
#          <rawdev> <blockdev>
# example: /dev/raw/raw1 /dev/sda1
#          /dev/raw/raw2 8 5
/dev/raw/raw1 /dev/sda1
/dev/raw/raw2 /dev/sda2

[root ankebut]# service rawdevices restart
Assigning devices:
           /dev/raw/raw1  -->   /dev/sda1
/dev/raw/raw1:  bound to major 8, minor 1
           /dev/raw/raw2  -->   /dev/sda2
/dev/raw/raw2:  bound to major 8, minor 2
Done

[root ankebut]# raw -qa
/dev/raw/raw1:  bound to major 8, minor 1
/dev/raw/raw2:  bound to major 8, minor 2


[root ankebut /]# mkfs.ext3 -j -b 4096 /dev/sda1
[root ankebut /]# mkfs.ext3 -j -b 4096 /dev/sda2

[root ankebut]# cat /etc/cluster/cluster.conf
:(, I dont know how to create cluster.conf for raw configuration in my case

TEST RAW DISK
I dont know how to test RAW configuration


Please, check my configurations and please tell me where i make
mistake, if i make :)
Thanks.

With Regards
Asim

On Feb 7, 2008 4:40 PM, Asim Husanovic <asim husanovic gmail com> wrote:
> Thanks.
> Asim
>
>
> On Feb 7, 2008 4:28 PM, Brian Kroth <bpkroth wisc edu> wrote:
> > Here are some quick examples.  There are almost certainly other ways to do
> > it.
> >
> > Asim Husanovic <asim husanovic gmail com>:
> > > Hi
> > >
> > > How to identify shared disk/volumes?
> >
> > scsi_id
> >
> > > How to collect the cluster FS information?
> >
> > gfs_tool
> >
> > > How to display shared disks on node/s? Which node/s?
> >
> > gfs_tool list
> > cman_tool services
> >
> >
> > You can wrap all of these inside snmp scripts/oids or use nagios passive
> > checks if you want to monitor them remotely/automatically.
> >
> > Brian
> >
>


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]