[Cluster-devel] GFS2: keep statfs info in sync on grows
Steven Whitehouse
swhiteho at redhat.com
Fri Jun 26 08:44:55 UTC 2009
Hi,
Now in the -nmw git tree. Thanks,
Steve.
On Thu, 2009-06-25 at 15:09 -0500, Benjamin Marzinski wrote:
> GFS2 wasn't syncing its statfs info on grows. This causes a problem
> when you grow the filesystem on multiple nodes. GFS2 would calculate
> the new space based on the resource groups (which are always current),
> and then assume that the filesystem had grown the from the existing
> statfs size. If you grew the filesystem on two different nodes in a
> short time, the second node wouldn't see the statfs size change from the
> first node, and would assume that it was grown by a larger amount than
> it was. When all these changes were synced out, the total fileystem
> size would be incorrect (the first grow would be counted twice).
>
> This patch syncs makes GFS2 read in the statfs changes from disk before
> a grow, and write them out after the grow, while the master statfs inode
> is locked.
>
> Signed-off-by: Benjamin Marzinski <bmarzins at redhat.com>
> ---
> fs/gfs2/aops.c | 39 +++++++++++++++++++++++++++++++++++++++
> fs/gfs2/super.c | 39 +++++++++++++++++++++++++--------------
> fs/gfs2/super.h | 4 ++++
> 3 files changed, 68 insertions(+), 14 deletions(-)
>
> Index: kernel-upstream/fs/gfs2/aops.c
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/aops.c
> +++ kernel-upstream/fs/gfs2/aops.c
> @@ -624,6 +624,7 @@ static int gfs2_write_begin(struct file
> {
> struct gfs2_inode *ip = GFS2_I(mapping->host);
> struct gfs2_sbd *sdp = GFS2_SB(mapping->host);
> + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> unsigned int data_blocks = 0, ind_blocks = 0, rblocks;
> int alloc_required;
> int error = 0;
> @@ -637,6 +638,14 @@ static int gfs2_write_begin(struct file
> error = gfs2_glock_nq(&ip->i_gh);
> if (unlikely(error))
> goto out_uninit;
> + if (&ip->i_inode == sdp->sd_rindex) {
> + error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
> + GL_NOCACHE, &m_ip->i_gh);
> + if (unlikely(error)) {
> + gfs2_glock_dq(&ip->i_gh);
> + goto out_uninit;
> + }
> + }
>
> error = gfs2_write_alloc_required(ip, pos, len, &alloc_required);
> if (error)
> @@ -667,6 +676,8 @@ static int gfs2_write_begin(struct file
> rblocks += data_blocks ? data_blocks : 1;
> if (ind_blocks || data_blocks)
> rblocks += RES_STATFS + RES_QUOTA;
> + if (&ip->i_inode == sdp->sd_rindex)
> + rblocks += 2 * RES_STATFS;
>
> error = gfs2_trans_begin(sdp, rblocks,
> PAGE_CACHE_SIZE/sdp->sd_sb.sb_bsize);
> @@ -712,6 +723,10 @@ out_alloc_put:
> gfs2_alloc_put(ip);
> }
> out_unlock:
> + if (&ip->i_inode == sdp->sd_rindex) {
> + gfs2_glock_dq(&m_ip->i_gh);
> + gfs2_holder_uninit(&m_ip->i_gh);
> + }
> gfs2_glock_dq(&ip->i_gh);
> out_uninit:
> gfs2_holder_uninit(&ip->i_gh);
> @@ -725,14 +740,21 @@ out_uninit:
> static void adjust_fs_space(struct inode *inode)
> {
> struct gfs2_sbd *sdp = inode->i_sb->s_fs_info;
> + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> + struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
> struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
> struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
> + struct buffer_head *m_bh, *l_bh;
> u64 fs_total, new_free;
>
> /* Total up the file system space, according to the latest rindex. */
> fs_total = gfs2_ri_total(sdp);
> + if (gfs2_meta_inode_buffer(m_ip, &m_bh) != 0)
> + return;
>
> spin_lock(&sdp->sd_statfs_spin);
> + gfs2_statfs_change_in(m_sc, m_bh->b_data +
> + sizeof(struct gfs2_dinode));
> if (fs_total > (m_sc->sc_total + l_sc->sc_total))
> new_free = fs_total - (m_sc->sc_total + l_sc->sc_total);
> else
> @@ -741,6 +763,13 @@ static void adjust_fs_space(struct inode
> fs_warn(sdp, "File system extended by %llu blocks.\n",
> (unsigned long long)new_free);
> gfs2_statfs_change(sdp, new_free, new_free, 0);
> +
> + if (gfs2_meta_inode_buffer(l_ip, &l_bh) != 0)
> + goto out;
> + update_statfs(sdp, m_bh, l_bh);
> + brelse(l_bh);
> +out:
> + brelse(m_bh);
> }
>
> /**
> @@ -763,6 +792,7 @@ static int gfs2_stuffed_write_end(struct
> {
> struct gfs2_inode *ip = GFS2_I(inode);
> struct gfs2_sbd *sdp = GFS2_SB(inode);
> + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> u64 to = pos + copied;
> void *kaddr;
> unsigned char *buf = dibh->b_data + sizeof(struct gfs2_dinode);
> @@ -794,6 +824,10 @@ static int gfs2_stuffed_write_end(struct
>
> brelse(dibh);
> gfs2_trans_end(sdp);
> + if (inode == sdp->sd_rindex) {
> + gfs2_glock_dq(&m_ip->i_gh);
> + gfs2_holder_uninit(&m_ip->i_gh);
> + }
> gfs2_glock_dq(&ip->i_gh);
> gfs2_holder_uninit(&ip->i_gh);
> return copied;
> @@ -823,6 +857,7 @@ static int gfs2_write_end(struct file *f
> struct inode *inode = page->mapping->host;
> struct gfs2_inode *ip = GFS2_I(inode);
> struct gfs2_sbd *sdp = GFS2_SB(inode);
> + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> struct buffer_head *dibh;
> struct gfs2_alloc *al = ip->i_alloc;
> unsigned int from = pos & (PAGE_CACHE_SIZE - 1);
> @@ -865,6 +900,10 @@ failed:
> gfs2_quota_unlock(ip);
> gfs2_alloc_put(ip);
> }
> + if (inode == sdp->sd_rindex) {
> + gfs2_glock_dq(&m_ip->i_gh);
> + gfs2_holder_uninit(&m_ip->i_gh);
> + }
> gfs2_glock_dq(&ip->i_gh);
> gfs2_holder_uninit(&ip->i_gh);
> return ret;
> Index: kernel-upstream/fs/gfs2/super.c
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/super.c
> +++ kernel-upstream/fs/gfs2/super.c
> @@ -353,7 +353,7 @@ fail:
> return error;
> }
>
> -static void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf)
> +void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf)
> {
> const struct gfs2_statfs_change *str = buf;
>
> @@ -441,6 +441,29 @@ void gfs2_statfs_change(struct gfs2_sbd
> brelse(l_bh);
> }
>
> +void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh,
> + struct buffer_head *l_bh)
> +{
> + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> + struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
> + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
> + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
> +
> + gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1);
> +
> + spin_lock(&sdp->sd_statfs_spin);
> + m_sc->sc_total += l_sc->sc_total;
> + m_sc->sc_free += l_sc->sc_free;
> + m_sc->sc_dinodes += l_sc->sc_dinodes;
> + memset(l_sc, 0, sizeof(struct gfs2_statfs_change));
> + memset(l_bh->b_data + sizeof(struct gfs2_dinode),
> + 0, sizeof(struct gfs2_statfs_change));
> + spin_unlock(&sdp->sd_statfs_spin);
> +
> + gfs2_trans_add_bh(m_ip->i_gl, m_bh, 1);
> + gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode));
> +}
> +
> int gfs2_statfs_sync(struct gfs2_sbd *sdp)
> {
> struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> @@ -477,19 +500,7 @@ int gfs2_statfs_sync(struct gfs2_sbd *sd
> if (error)
> goto out_bh2;
>
> - gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1);
> -
> - spin_lock(&sdp->sd_statfs_spin);
> - m_sc->sc_total += l_sc->sc_total;
> - m_sc->sc_free += l_sc->sc_free;
> - m_sc->sc_dinodes += l_sc->sc_dinodes;
> - memset(l_sc, 0, sizeof(struct gfs2_statfs_change));
> - memset(l_bh->b_data + sizeof(struct gfs2_dinode),
> - 0, sizeof(struct gfs2_statfs_change));
> - spin_unlock(&sdp->sd_statfs_spin);
> -
> - gfs2_trans_add_bh(m_ip->i_gl, m_bh, 1);
> - gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode));
> + update_statfs(sdp, m_bh, l_bh);
>
> gfs2_trans_end(sdp);
>
> Index: kernel-upstream/fs/gfs2/super.h
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/super.h
> +++ kernel-upstream/fs/gfs2/super.h
> @@ -40,6 +40,10 @@ extern int gfs2_make_fs_rw(struct gfs2_s
> extern int gfs2_statfs_init(struct gfs2_sbd *sdp);
> extern void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free,
> s64 dinodes);
> +extern void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc,
> + const void *buf);
> +extern void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh,
> + struct buffer_head *l_bh);
> extern int gfs2_statfs_sync(struct gfs2_sbd *sdp);
>
> extern int gfs2_freeze_fs(struct gfs2_sbd *sdp);
>
More information about the Cluster-devel
mailing list