[Cluster-devel] cluster/gfs-kernel/src/gfs daemon.c gfs_ondisk ...
wcheng at sourceware.org
wcheng at sourceware.org
Tue Mar 13 21:21:22 UTC 2007
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL4
Changes by: wcheng at sourceware.org 2007-03-13 21:21:21
Modified files:
gfs-kernel/src/gfs: daemon.c gfs_ondisk.h incore.h ioctl.c
ops_fstype.c ops_super.c rgrp.c rgrp.h
super.c super.h
Log message:
Bugzilla 220622: backport GFS2 statfs() implementation to GFS1.
There are few compromises made while porting GFS2 approach over, mostly
to avoid on-disk structure changes. GFS2 allocates (number-of-nodes + 1)
physical files into disk during mkfs time but GFS1 only has one extra
space (the unused license file) for this purpose. We deviate from GFS2
by writing the local per-node changes into a memory buffer.
Doing a "df" on a quiet filesystem now results:
dhcp145 (1 cpu HP): old df took 0.875 seconds, new df 0.008 second
dhcp146 (4 cpus DELL): old df took 0.808 seconds, new df 0.006 second.
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/daemon.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.5&r2=1.5.2.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/gfs_ondisk.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.7.2.1&r2=1.7.2.2
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/incore.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.17.2.8&r2=1.17.2.9
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/ioctl.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.7.2.6&r2=1.7.2.7
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/ops_fstype.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.13.2.3&r2=1.13.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/ops_super.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.12.2.5&r2=1.12.2.6
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/rgrp.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.14.2.3&r2=1.14.2.4
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/rgrp.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.3&r2=1.3.2.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/super.c.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.12.2.7&r2=1.12.2.8
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/gfs-kernel/src/gfs/super.h.diff?cvsroot=cluster&only_with_tag=RHEL4&r1=1.2&r2=1.2.2.1
--- cluster/gfs-kernel/src/gfs/daemon.c 2005/01/04 10:07:11 1.5
+++ cluster/gfs-kernel/src/gfs/daemon.c 2007/03/13 21:21:21 1.5.2.1
@@ -207,6 +207,19 @@
complete(&sdp->sd_thread_completion);
for (;;) {
+ /* Update statfs file */
+ if (gfs_tune_get(sdp, gt_statfs_fast) &&
+ time_after_eq(jiffies,
+ sdp->sd_statfs_sync_time +
+ gfs_tune_get(sdp, gt_statfs_fast) * HZ)) {
+ error = gfs_statfs_sync(sdp);
+ if (error &&
+ error != -EROFS &&
+ !test_bit(SDF_SHUTDOWN, &sdp->sd_flags))
+ printk("GFS: fsid=%s: statfs: error = %d\n",
+ sdp->sd_fsname, error);
+ sdp->sd_statfs_sync_time = jiffies;
+ }
/* Update quota file */
if (time_after_eq(jiffies,
sdp->sd_quota_sync_time +
--- cluster/gfs-kernel/src/gfs/gfs_ondisk.h 2006/11/06 21:01:36 1.7.2.1
+++ cluster/gfs-kernel/src/gfs/gfs_ondisk.h 2007/03/13 21:21:21 1.7.2.2
@@ -616,6 +616,31 @@
uint32_t ea_pad;
};
+/*
+ * Statfs change
+ * Describes an change to the pool of free and allocated
+ * blocks.
+ */
+
+struct gfs_statfs_change {
+ __be64 sc_total;
+ __be64 sc_free;
+ __be64 sc_dinodes;
+};
+
+/*
+struct gfs_statfs_change_host {
+ __u64 sc_total;
+ __u64 sc_free;
+ __u64 sc_dinodes;
+};
+*/
+struct gfs_statfs_change_host {
+ int64_t sc_total;
+ int64_t sc_free;
+ int64_t sc_dinodes;
+};
+
/* Endian functions */
#define GFS_ENDIAN_BIG
--- cluster/gfs-kernel/src/gfs/incore.h 2007/01/22 07:43:52 1.17.2.8
+++ cluster/gfs-kernel/src/gfs/incore.h 2007/03/13 21:21:21 1.17.2.9
@@ -893,6 +893,7 @@
unsigned int gt_greedy_quantum;
unsigned int gt_greedy_max;
unsigned int gt_rgrp_try_threshold;
+ unsigned int gt_statfs_fast;
};
/*
@@ -963,6 +964,13 @@
struct gfs_tune sd_tune; /* Filesystem tuning structure */
+ /* statfs */
+ struct inode *sd_statfs_inode;
+ spinlock_t sd_statfs_spin;
+ struct gfs_statfs_change_host sd_statfs_master;
+ struct gfs_statfs_change_host sd_statfs_local;
+ unsigned long sd_statfs_sync_time;
+
/* Resource group stuff */
struct gfs_inode *sd_riinode; /* Resource Index (rindex) inode */
--- cluster/gfs-kernel/src/gfs/ioctl.c 2007/01/22 07:43:52 1.7.2.6
+++ cluster/gfs-kernel/src/gfs/ioctl.c 2007/03/13 21:21:21 1.7.2.7
@@ -483,6 +483,7 @@
gfs_printf("greedy_quantum %u\n", gt->gt_greedy_quantum);
gfs_printf("greedy_max %u\n", gt->gt_greedy_max);
gfs_printf("rgrp_try_threshold %u\n", gt->gt_rgrp_try_threshold);
+ gfs_printf("statfs_fast %u\n", gt->gt_statfs_fast);
error = 0;
@@ -513,6 +514,7 @@
struct gfs_tune *gt = &sdp->sd_tune;
char param[ARG_SIZE], value[ARG_SIZE];
unsigned int x;
+ int error;
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
@@ -752,6 +754,15 @@
return -EINVAL;
tune_set(gt_rgrp_try_threshold, x);
+ } else if (strcmp(param, "statfs_fast") == 0) {
+ if (sscanf(value, "%u", &x) != 1)
+ return -EINVAL;
+ error = gfs_statfs_init(sdp, x);
+ if (error)
+ return error;
+ else
+ tune_set(gt_statfs_fast, x);
+
} else
return -EINVAL;
@@ -1496,5 +1507,3 @@
return error;
}
-
-
--- cluster/gfs-kernel/src/gfs/ops_fstype.c 2006/09/15 21:42:05 1.13.2.3
+++ cluster/gfs-kernel/src/gfs/ops_fstype.c 2007/03/13 21:21:21 1.13.2.4
@@ -100,6 +100,8 @@
spin_lock_init(&sdp->sd_rg_recent_lock);
spin_lock_init(&sdp->sd_rg_forward_lock);
+ spin_lock_init(&sdp->sd_statfs_spin);
+
for (x = 0; x < GFS_GL_HASH_SIZE; x++) {
sdp->sd_gl_hash[x].hb_lock = RW_LOCK_UNLOCKED;
INIT_LIST_HEAD(&sdp->sd_gl_hash[x].hb_list);
@@ -472,9 +474,16 @@
/* Read in the license inode */
+ /* Piggy back fast df on this inode location
+ * sb->sb_quota_di.no_formal_ino = jindex_dinode + 2;
+ * sb->sb_quota_di.no_addr = jindex_dinode + 2;
+ * sb->sb_license_di.no_formal_ino = jindex_dinode + 3;
+ * sb->sb_license_di.no_addr = jindex_dinode + 3;
+ */
+
error = gfs_get_linode(sdp);
if (error) {
- printk("GFS: fsid=%s: can't get license file inode: %d\n",
+ printk("GFS: fsid=%s: can't get license/statfs file inode: %d\n",
sdp->sd_fsname, error);
goto fail_qi_free;
}
--- cluster/gfs-kernel/src/gfs/ops_super.c 2006/09/15 21:42:05 1.12.2.5
+++ cluster/gfs-kernel/src/gfs/ops_super.c 2007/03/13 21:21:21 1.12.2.6
@@ -283,6 +283,9 @@
atomic_inc(&sdp->sd_ops_super);
+ if (gfs_tune_get(sdp, gt_statfs_fast))
+ return(gfs_statfs_fast(sdp, (void *)buf));
+
error = gfs_stat_gfs(sdp, &sg, TRUE);
if (error)
return error;
--- cluster/gfs-kernel/src/gfs/rgrp.c 2005/10/24 15:53:03 1.14.2.3
+++ cluster/gfs-kernel/src/gfs/rgrp.c 2007/03/13 21:21:21 1.14.2.4
@@ -18,6 +18,7 @@
#include <asm/semaphore.h>
#include <linux/completion.h>
#include <linux/buffer_head.h>
+#include <linux/types.h>
#include "gfs.h"
#include "bits.h"
@@ -1657,6 +1658,9 @@
al->al_alloced_data++;
gfs_trans_add_quota(sdp, +1, ip->i_di.di_uid, ip->i_di.di_gid);
+
+ /* total=0, free=-1, dinodes=0 */
+ gfs_statfs_modify(sdp, 0, -1, 0);
}
/**
@@ -1711,6 +1715,9 @@
gfs_trans_add_quota(sdp, +1, ip->i_di.di_uid, ip->i_di.di_gid);
+ /* total=0, free=-1, dinode=0 */
+ gfs_statfs_modify(sdp, 0, -1, 0);
+
return 0;
}
@@ -1726,6 +1733,7 @@
int
gfs_dialloc(struct gfs_inode *dip, uint64_t *block)
{
+ struct gfs_sbd *sdp = dip->i_sbd;
struct gfs_alloc *al = dip->i_alloc;
struct gfs_rgrpd *rgd = al->al_rgd;
uint32_t goal, blk;
@@ -1765,6 +1773,9 @@
al->al_alloced_di++;
al->al_alloced_meta++;
+ /* total=0, free=-1, dinodes=1 */
+ gfs_statfs_modify(sdp, 0, -1, +1);
+
return error;
}
@@ -1797,6 +1808,8 @@
gfs_trans_add_quota(sdp, -(int64_t)blen,
ip->i_di.di_uid,
ip->i_di.di_gid);
+ /* total=0, free=+blen, dinodes=0 */
+ gfs_statfs_modify(sdp, 0, blen, 0);
}
/**
@@ -1831,6 +1844,9 @@
gfs_trans_add_bh(rgd->rd_gl, rgd->rd_bh[0]);
gfs_rgrp_out(&rgd->rd_rg, rgd->rd_bh[0]->b_data);
+ /* total=0, free=blen, dinode=0 */
+ gfs_statfs_modify(sdp, 0, blen, 0);
+
gfs_trans_add_quota(sdp, -(int64_t)blen,
ip->i_di.di_uid,
ip->i_di.di_gid);
@@ -1865,6 +1881,9 @@
gfs_trans_add_bh(rgd->rd_gl, rgd->rd_bh[0]);
gfs_rgrp_out(&rgd->rd_rg, rgd->rd_bh[0]->b_data);
+
+ /* total=0, free=1, dinodes=-1 */
+ gfs_statfs_modify(rgd->rd_sbd, 0, +1, -1);
}
/**
--- cluster/gfs-kernel/src/gfs/rgrp.h 2005/01/04 10:07:12 1.3
+++ cluster/gfs-kernel/src/gfs/rgrp.h 2007/03/13 21:21:21 1.3.2.1
@@ -57,6 +57,10 @@
void gfs_difree_uninit(struct gfs_rgrpd *rgd, uint64_t addr);
void gfs_difree(struct gfs_rgrpd *rgd, struct gfs_inode *ip);
+extern void gfs_statfs_modify(struct gfs_sbd *sdp,
+ int64_t total,
+ int64_t free,
+ int64_t dinodes);
/*
* gfs_rgrp_list
*
--- cluster/gfs-kernel/src/gfs/super.c 2007/02/06 22:08:36 1.12.2.7
+++ cluster/gfs-kernel/src/gfs/super.c 2007/03/13 21:21:21 1.12.2.8
@@ -17,6 +17,8 @@
#include <linux/spinlock.h>
#include <asm/semaphore.h>
#include <linux/completion.h>
+#include <linux/statfs.h>
+#include <linux/types.h>
#include <linux/buffer_head.h>
#include <linux/vmalloc.h>
@@ -33,6 +35,7 @@
#include "rgrp.h"
#include "super.h"
#include "unlinked.h"
+#include "trans.h"
/**
* gfs_tune_init - Fill a gfs_tune structure with default values
@@ -85,6 +88,7 @@
gt->gt_greedy_quantum = HZ / 40;
gt->gt_greedy_max = HZ / 4;
gt->gt_rgrp_try_threshold = 100;
+ gt->gt_statfs_fast = 0;
}
/**
@@ -682,6 +686,7 @@
if (error)
return error;
+ /* iopen obtained in via gfs_glock_get(..gfs_iopen_glops) */
error = gfs_inode_get(i_gh.gh_gl, &sdp->sd_sb.sb_license_di,
CREATE, &sdp->sd_linode);
@@ -770,6 +775,7 @@
!test_bit(SDF_SHUTDOWN, &sdp->sd_flags))
return error;
+ gfs_statfs_sync(sdp);
gfs_log_flush(sdp);
gfs_quota_sync(sdp);
gfs_quota_scan(sdp);
@@ -1083,3 +1089,199 @@
up(&sdp->sd_freeze_lock);
}
+
+/*
+ * 03/02/07 wcheng at redhat.com
+ * Fast statfs implementation - mostly based on GFS2 implementation.
+ */
+
+void gfs_statfs_change_in(struct gfs_statfs_change_host *sc, const void *buf)
+{
+ const struct gfs_statfs_change *str = buf;
+
+ sc->sc_total = be64_to_cpu(str->sc_total);
+ sc->sc_free = be64_to_cpu(str->sc_free);
+ sc->sc_dinodes = be64_to_cpu(str->sc_dinodes);
+}
+
+void gfs_statfs_change_out(const struct gfs_statfs_change_host *sc, void *buf)
+{
+ struct gfs_statfs_change *str = buf;
+
+ str->sc_total = cpu_to_be64(sc->sc_total);
+ str->sc_free = cpu_to_be64(sc->sc_free);
+ str->sc_dinodes = cpu_to_be64(sc->sc_dinodes);
+}
+
+int gfs_statfs_init(struct gfs_sbd *sdp, int flag)
+{
+ int error;
+
+ /* if flag == 0, do we want to turn this off ? */
+ if (!flag)
+ return 0;
+
+ error = gfs_statfs_start(sdp);
+ if (error)
+ printk("GFS: fsid=%s: can't initialize statfs subsystem: %d\n",
+ sdp->sd_fsname, error);
+
+ return error;
+}
+int gfs_statfs_start(struct gfs_sbd *sdp)
+{
+ struct gfs_stat_gfs sg;
+ struct gfs_inode *m_ip;
+ struct gfs_statfs_change_host *m_sc = &sdp->sd_statfs_master;
+ struct gfs_statfs_change_host *l_sc = &sdp->sd_statfs_local;
+ struct buffer_head *m_bh;
+ struct gfs_holder gh;
+ int error;
+
+ printk("GFS: fsid=%s: fast statfs start time = %lu\n",
+ sdp->sd_fsname, get_seconds());
+
+ /* created via gfs_get_linode() in fill_super(). */
+ /* gfs_inode_glops */
+ m_ip = sdp->sd_linode;
+
+ /* get real statistics */
+ error = gfs_stat_gfs(sdp, &sg, TRUE);
+ if (error)
+ return error;
+
+ /* make sure the page is refreshed via glock flushing */
+ error = gfs_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE,
+ &gh);
+ if (error)
+ goto gfs_statfs_start_out;
+
+ error = gfs_get_inode_buffer(m_ip, &m_bh);
+ if (error)
+ goto gfs_statfs_start_unlock;
+
+ error = gfs_trans_begin(sdp, 1, 0);
+ if (error)
+ goto gfs_statfs_start_bh;
+
+ spin_lock(&sdp->sd_statfs_spin);
+ m_sc->sc_total = sg.sg_total_blocks;
+ m_sc->sc_free = sg.sg_free + sg.sg_free_dinode + sg.sg_free_meta;
+ m_sc->sc_dinodes = sg.sg_used_dinode;
+ memset(l_sc, 0, sizeof(struct gfs_statfs_change_host));
+ spin_unlock(&sdp->sd_statfs_spin);
+
+ gfs_trans_add_bh(m_ip->i_gl, m_bh);
+ gfs_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs_dinode));
+
+ gfs_trans_end(sdp);
+
+gfs_statfs_start_bh:
+ brelse(m_bh);
+
+gfs_statfs_start_unlock:
+ gfs_glock_dq_uninit(&gh);
+
+gfs_statfs_start_out:
+ return 0;
+}
+
+void gfs_statfs_modify(struct gfs_sbd *sdp,
+ int64_t total,
+ int64_t free,
+ int64_t dinodes)
+{
+ struct gfs_statfs_change_host *l_sc = &sdp->sd_statfs_local;
+
+ spin_lock(&sdp->sd_statfs_spin);
+ l_sc->sc_total += total;
+ l_sc->sc_free += free;
+ l_sc->sc_dinodes += dinodes;
+ spin_unlock(&sdp->sd_statfs_spin);
+}
+
+int gfs_statfs_sync(struct gfs_sbd *sdp)
+{
+ struct gfs_inode *m_ip = sdp->sd_linode;
+ struct gfs_statfs_change_host *m_sc = &sdp->sd_statfs_master;
+ struct gfs_statfs_change_host *l_sc = &sdp->sd_statfs_local;
+ struct gfs_holder gh;
+ struct buffer_head *m_bh;
+ int error;
+
+ error = gfs_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE,
+ &gh);
+ if (error)
+ return error;
+
+ error = gfs_get_inode_buffer(m_ip, &m_bh);
+ if (error)
+ goto gfs_statfs_sync_out;
+
+ /* if no change, simply return */
+ spin_lock(&sdp->sd_statfs_spin);
+ gfs_statfs_change_in(m_sc, m_bh->b_data +
+ sizeof(struct gfs_dinode));
+ if (!l_sc->sc_total && !l_sc->sc_free && !l_sc->sc_dinodes) {
+ spin_unlock(&sdp->sd_statfs_spin);
+ goto out_bh;
+ }
+ spin_unlock(&sdp->sd_statfs_spin);
+
+ error = gfs_trans_begin(sdp, 1, 0);
+ if (error)
+ goto out_bh;
+
+ spin_lock(&sdp->sd_statfs_spin);
+ m_sc->sc_total += l_sc->sc_total;
+ m_sc->sc_free += l_sc->sc_free;
+ m_sc->sc_dinodes += l_sc->sc_dinodes;
+ memset(l_sc, 0, sizeof(struct gfs_statfs_change_host));
+ spin_unlock(&sdp->sd_statfs_spin);
+
+ gfs_trans_add_bh(m_ip->i_gl, m_bh);
+ gfs_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs_dinode));
+
+ gfs_trans_end(sdp);
+
+out_bh:
+ brelse(m_bh);
+
+gfs_statfs_sync_out:
+ gfs_glock_dq_uninit(&gh);
+ return error;
+}
+
+int gfs_statfs_fast(struct gfs_sbd *sdp, void *b)
+{
+ struct kstatfs *buf = (struct kstatfs *)b;
+ struct gfs_statfs_change_host sc, *m_sc = &sdp->sd_statfs_master;
+ struct gfs_statfs_change_host *l_sc = &sdp->sd_statfs_local;
+
+ spin_lock(&sdp->sd_statfs_spin);
+
+ sc.sc_total = m_sc->sc_total + l_sc->sc_total;
+ sc.sc_free = m_sc->sc_free + l_sc->sc_free;
+ sc.sc_dinodes = m_sc->sc_dinodes + l_sc->sc_dinodes;
+ spin_unlock(&sdp->sd_statfs_spin);
+
+ if (sc.sc_free < 0)
+ sc.sc_free = 0;
+ if (sc.sc_free > sc.sc_total)
+ sc.sc_free = sc.sc_total;
+ if (sc.sc_dinodes < 0)
+ sc.sc_dinodes = 0;
+
+ /* fill in the statistics */
+ memset(buf, 0, sizeof(struct kstatfs));
+
+ buf->f_type = GFS_MAGIC; buf->f_bsize = sdp->sd_sb.sb_bsize;
+ buf->f_blocks = sc.sc_total;
+ buf->f_bfree = sc.sc_free;
+ buf->f_bavail = sc.sc_free;
+ buf->f_files = sc.sc_dinodes + sc.sc_free;
+ buf->f_ffree = sc.sc_free;
+ buf->f_namelen = GFS_FNAMESIZE;
+
+ return 0;
+}
--- cluster/gfs-kernel/src/gfs/super.h 2005/01/04 10:07:12 1.2
+++ cluster/gfs-kernel/src/gfs/super.h 2007/03/13 21:21:21 1.2.2.1
@@ -42,6 +42,10 @@
int gfs_make_fs_rw(struct gfs_sbd *sdp);
int gfs_make_fs_ro(struct gfs_sbd *sdp);
+int gfs_statfs_init(struct gfs_sbd *sdp, int flag);
+int gfs_statfs_sync(struct gfs_sbd *sdp);
+int gfs_statfs_fast(struct gfs_sbd *sdp, void *buf);
+
struct gfs_stat_gfs {
uint64_t sg_total_blocks;
uint64_t sg_free;
More information about the Cluster-devel
mailing list