[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [Cluster-devel] GFS2: keep statfs info in sync on grows



Hi,

Now in the -nmw git tree. Thanks,

Steve.

On Thu, 2009-06-25 at 15:09 -0500, Benjamin Marzinski wrote:
> GFS2 wasn't syncing its statfs info on grows.  This causes a problem
> when you grow the filesystem on multiple nodes.  GFS2 would calculate
> the new space based on the resource groups (which are always current),
> and then assume that the filesystem had grown the from the existing
> statfs size.  If you grew the filesystem on two different nodes in a
> short time, the second node wouldn't see the statfs size change from the
> first node, and would assume that it was grown by a larger amount than
> it was.  When all these changes were synced out, the total fileystem
> size would be incorrect (the first grow would be counted twice).
> 
> This patch syncs makes GFS2 read in the statfs changes from disk before
> a grow, and write them out after the grow, while the master statfs inode
> is locked.
> 
> Signed-off-by: Benjamin Marzinski <bmarzins redhat com>
> ---
>  fs/gfs2/aops.c  |   39 +++++++++++++++++++++++++++++++++++++++
>  fs/gfs2/super.c |   39 +++++++++++++++++++++++++--------------
>  fs/gfs2/super.h |    4 ++++
>  3 files changed, 68 insertions(+), 14 deletions(-)
> 
> Index: kernel-upstream/fs/gfs2/aops.c
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/aops.c
> +++ kernel-upstream/fs/gfs2/aops.c
> @@ -624,6 +624,7 @@ static int gfs2_write_begin(struct file 
>  {
>  	struct gfs2_inode *ip = GFS2_I(mapping->host);
>  	struct gfs2_sbd *sdp = GFS2_SB(mapping->host);
> +	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
>  	unsigned int data_blocks = 0, ind_blocks = 0, rblocks;
>  	int alloc_required;
>  	int error = 0;
> @@ -637,6 +638,14 @@ static int gfs2_write_begin(struct file 
>  	error = gfs2_glock_nq(&ip->i_gh);
>  	if (unlikely(error))
>  		goto out_uninit;
> +	if (&ip->i_inode == sdp->sd_rindex) {
> +		error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE,
> +					   GL_NOCACHE, &m_ip->i_gh);
> +		if (unlikely(error)) {
> +			gfs2_glock_dq(&ip->i_gh);
> +			goto out_uninit;
> +		}
> +	}
>  
>  	error = gfs2_write_alloc_required(ip, pos, len, &alloc_required);
>  	if (error)
> @@ -667,6 +676,8 @@ static int gfs2_write_begin(struct file 
>  		rblocks += data_blocks ? data_blocks : 1;
>  	if (ind_blocks || data_blocks)
>  		rblocks += RES_STATFS + RES_QUOTA;
> +	if (&ip->i_inode == sdp->sd_rindex)
> +		rblocks += 2 * RES_STATFS;
>  
>  	error = gfs2_trans_begin(sdp, rblocks,
>  				 PAGE_CACHE_SIZE/sdp->sd_sb.sb_bsize);
> @@ -712,6 +723,10 @@ out_alloc_put:
>  		gfs2_alloc_put(ip);
>  	}
>  out_unlock:
> +	if (&ip->i_inode == sdp->sd_rindex) {
> +		gfs2_glock_dq(&m_ip->i_gh);
> +		gfs2_holder_uninit(&m_ip->i_gh);
> +	}
>  	gfs2_glock_dq(&ip->i_gh);
>  out_uninit:
>  	gfs2_holder_uninit(&ip->i_gh);
> @@ -725,14 +740,21 @@ out_uninit:
>  static void adjust_fs_space(struct inode *inode)
>  {
>  	struct gfs2_sbd *sdp = inode->i_sb->s_fs_info;
> +	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> +	struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
>  	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
>  	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
> +	struct buffer_head *m_bh, *l_bh;
>  	u64 fs_total, new_free;
>  
>  	/* Total up the file system space, according to the latest rindex. */
>  	fs_total = gfs2_ri_total(sdp);
> +	if (gfs2_meta_inode_buffer(m_ip, &m_bh) != 0)
> +		return;
>  
>  	spin_lock(&sdp->sd_statfs_spin);
> +	gfs2_statfs_change_in(m_sc, m_bh->b_data +
> +			      sizeof(struct gfs2_dinode));
>  	if (fs_total > (m_sc->sc_total + l_sc->sc_total))
>  		new_free = fs_total - (m_sc->sc_total + l_sc->sc_total);
>  	else
> @@ -741,6 +763,13 @@ static void adjust_fs_space(struct inode
>  	fs_warn(sdp, "File system extended by %llu blocks.\n",
>  		(unsigned long long)new_free);
>  	gfs2_statfs_change(sdp, new_free, new_free, 0);
> +
> +	if (gfs2_meta_inode_buffer(l_ip, &l_bh) != 0)
> +		goto out;
> +	update_statfs(sdp, m_bh, l_bh);
> +	brelse(l_bh);
> +out:
> +	brelse(m_bh);
>  }
>  
>  /**
> @@ -763,6 +792,7 @@ static int gfs2_stuffed_write_end(struct
>  {
>  	struct gfs2_inode *ip = GFS2_I(inode);
>  	struct gfs2_sbd *sdp = GFS2_SB(inode);
> +	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
>  	u64 to = pos + copied;
>  	void *kaddr;
>  	unsigned char *buf = dibh->b_data + sizeof(struct gfs2_dinode);
> @@ -794,6 +824,10 @@ static int gfs2_stuffed_write_end(struct
>  
>  	brelse(dibh);
>  	gfs2_trans_end(sdp);
> +	if (inode == sdp->sd_rindex) {
> +		gfs2_glock_dq(&m_ip->i_gh);
> +		gfs2_holder_uninit(&m_ip->i_gh);
> +	}
>  	gfs2_glock_dq(&ip->i_gh);
>  	gfs2_holder_uninit(&ip->i_gh);
>  	return copied;
> @@ -823,6 +857,7 @@ static int gfs2_write_end(struct file *f
>  	struct inode *inode = page->mapping->host;
>  	struct gfs2_inode *ip = GFS2_I(inode);
>  	struct gfs2_sbd *sdp = GFS2_SB(inode);
> +	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
>  	struct buffer_head *dibh;
>  	struct gfs2_alloc *al = ip->i_alloc;
>  	unsigned int from = pos & (PAGE_CACHE_SIZE - 1);
> @@ -865,6 +900,10 @@ failed:
>  		gfs2_quota_unlock(ip);
>  		gfs2_alloc_put(ip);
>  	}
> +	if (inode == sdp->sd_rindex) {
> +		gfs2_glock_dq(&m_ip->i_gh);
> +		gfs2_holder_uninit(&m_ip->i_gh);
> +	}
>  	gfs2_glock_dq(&ip->i_gh);
>  	gfs2_holder_uninit(&ip->i_gh);
>  	return ret;
> Index: kernel-upstream/fs/gfs2/super.c
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/super.c
> +++ kernel-upstream/fs/gfs2/super.c
> @@ -353,7 +353,7 @@ fail:
>  	return error;
>  }
>  
> -static void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf)
> +void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf)
>  {
>  	const struct gfs2_statfs_change *str = buf;
>  
> @@ -441,6 +441,29 @@ void gfs2_statfs_change(struct gfs2_sbd 
>  	brelse(l_bh);
>  }
>  
> +void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh,
> +		   struct buffer_head *l_bh)
> +{
> +	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> +	struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
> +	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
> +	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
> +
> +	gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1);
> +
> +	spin_lock(&sdp->sd_statfs_spin);
> +	m_sc->sc_total += l_sc->sc_total;
> +	m_sc->sc_free += l_sc->sc_free;
> +	m_sc->sc_dinodes += l_sc->sc_dinodes;
> +	memset(l_sc, 0, sizeof(struct gfs2_statfs_change));
> +	memset(l_bh->b_data + sizeof(struct gfs2_dinode),
> +	       0, sizeof(struct gfs2_statfs_change));
> +	spin_unlock(&sdp->sd_statfs_spin);
> +
> +	gfs2_trans_add_bh(m_ip->i_gl, m_bh, 1);
> +	gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode));
> +}
> +
>  int gfs2_statfs_sync(struct gfs2_sbd *sdp)
>  {
>  	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
> @@ -477,19 +500,7 @@ int gfs2_statfs_sync(struct gfs2_sbd *sd
>  	if (error)
>  		goto out_bh2;
>  
> -	gfs2_trans_add_bh(l_ip->i_gl, l_bh, 1);
> -
> -	spin_lock(&sdp->sd_statfs_spin);
> -	m_sc->sc_total += l_sc->sc_total;
> -	m_sc->sc_free += l_sc->sc_free;
> -	m_sc->sc_dinodes += l_sc->sc_dinodes;
> -	memset(l_sc, 0, sizeof(struct gfs2_statfs_change));
> -	memset(l_bh->b_data + sizeof(struct gfs2_dinode),
> -	       0, sizeof(struct gfs2_statfs_change));
> -	spin_unlock(&sdp->sd_statfs_spin);
> -
> -	gfs2_trans_add_bh(m_ip->i_gl, m_bh, 1);
> -	gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode));
> +	update_statfs(sdp, m_bh, l_bh);
>  
>  	gfs2_trans_end(sdp);
>  
> Index: kernel-upstream/fs/gfs2/super.h
> ===================================================================
> --- kernel-upstream.orig/fs/gfs2/super.h
> +++ kernel-upstream/fs/gfs2/super.h
> @@ -40,6 +40,10 @@ extern int gfs2_make_fs_rw(struct gfs2_s
>  extern int gfs2_statfs_init(struct gfs2_sbd *sdp);
>  extern void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free,
>  			       s64 dinodes);
> +extern void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc,
> +				  const void *buf);
> +extern void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh,
> +			  struct buffer_head *l_bh);
>  extern int gfs2_statfs_sync(struct gfs2_sbd *sdp);
>  
>  extern int gfs2_freeze_fs(struct gfs2_sbd *sdp);
> 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]