RHEL5 - gfs-kernel: Bug 466645 - reproduceable gfs (dlm) hanger withsimple stresstest

[Abhijith Das <adas at fedoraproject dot org>]

Gitweb:        http://git.fedorahosted.org/git/cluster.git?p=cluster.git;a=commitdiff;h=39344881850283002c0578cc41f61c067948612a
Commit:        39344881850283002c0578cc41f61c067948612a
Parent:        ed809facf03ba7e16a789f2ce71d660d11b71cea
Author:        Abhijith Das <adas@redhat.com>
AuthorDate:    Thu Nov 6 14:58:14 2008 -0600
Committer:     Abhijith Das <adas@redhat.com>
CommitterDate: Thu Nov 6 14:58:14 2008 -0600

gfs-kernel: Bug 466645 - reproduceable gfs (dlm) hanger with simple stresstest

GFS used to attempt to prefetch inode/iopen locks in readdir
in the anticipation that stats will be called on the dirents
returned. Running a simple 'find' without stat, resulted in
wasteful prefetching and poor performance to the point that
it seemed like find was hanging the system.

This patch performs prefetch on a directory's inodes based
on a stat-rate. i.e the rate at which stats are performed on
the dirents returned by readdir. If there are a significant
number of stats being performed, we enable prefetching.
Otherwise, readdir is performed without prefetches.
---
 gfs-kernel/src/gfs/incore.h    |    3 +++
 gfs-kernel/src/gfs/inode.c     |    4 ++++
 gfs-kernel/src/gfs/ops_file.c  |   29 ++++++++++++++++++++++++++++-
 gfs-kernel/src/gfs/ops_inode.c |   13 +++++++++++--
 4 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/gfs-kernel/src/gfs/incore.h b/gfs-kernel/src/gfs/incore.h
index 633e9ca..fa6f539 100644
--- a/gfs-kernel/src/gfs/incore.h
+++ b/gfs-kernel/src/gfs/incore.h
@@ -617,6 +617,9 @@ struct gfs_inode {
 
 	unsigned int i_greedy; /* The amount of time to be greedy */
 	unsigned long i_last_pfault; /* The time of the last page fault */
+
+	struct timeval i_dir_stat_st; /* Start time of stat counter */
+	unsigned long i_dir_stats; /* stat counter for this directory */
 	struct address_space_operations gfs_file_aops;
 };
 
diff --git a/gfs-kernel/src/gfs/inode.c b/gfs-kernel/src/gfs/inode.c
index 6aeda25..3e2c601 100644
--- a/gfs-kernel/src/gfs/inode.c
+++ b/gfs-kernel/src/gfs/inode.c
@@ -347,6 +347,10 @@ inode_create(struct gfs_glock *i_gl, struct gfs_inum *inum,
 	gfs_glock_hold(i_gl);
 	set_gl2ip(i_gl, ip);
 
+	/* initialize stat counter and timestamp */
+	ip->i_dir_stats = 0;
+	do_gettimeofday(&ip->i_dir_stat_st);
+
 	atomic_inc(&sdp->sd_inode_count);
 
 	*ipp = ip;
diff --git a/gfs-kernel/src/gfs/ops_file.c b/gfs-kernel/src/gfs/ops_file.c
index 3cad78e..1e54e30 100644
--- a/gfs-kernel/src/gfs/ops_file.c
+++ b/gfs-kernel/src/gfs/ops_file.c
@@ -1131,6 +1131,29 @@ filldir_reg_func(void *opaque,
 	return 0;
 }
 
+static inline int
+high_stat_rate(struct gfs_inode *ip)
+{
+	struct timeval now, diff;
+	unsigned long milli = 0;
+	unsigned int rate = 0; /* stats per second in this dir */
+
+	do_gettimeofday(&now);
+	diff.tv_sec = now.tv_sec - ip->i_dir_stat_st.tv_sec;
+	diff.tv_usec = now.tv_usec - ip->i_dir_stat_st.tv_usec;
+	if (diff.tv_usec < 0) {
+		diff.tv_usec += 1000000;
+		diff.tv_sec--;
+	}
+	milli = (diff.tv_sec * 1000) + (diff.tv_usec / 1000);
+	if (milli)
+		rate = (ip->i_dir_stats * 1000) / milli;
+
+	if (rate > 10) /* More than 10 stats/sec */
+		return 1;
+	return 0;
+}
+
 /**
  * readdir_reg - Read directory entries from a directory
  * @file: The directory to read from
@@ -1150,10 +1173,14 @@ readdir_reg(struct file *file, void *dirent, filldir_t filldir)
 	int error;
 
 	fdr.fdr_sbd = dip->i_sbd;
-	fdr.fdr_prefetch = TRUE;
+	fdr.fdr_prefetch = high_stat_rate(dip) ? TRUE : FALSE;
 	fdr.fdr_filldir = filldir;
 	fdr.fdr_opaque = dirent;
 
+	/* reset stat counter and timestamp */
+	dip->i_dir_stats = 0;
+	do_gettimeofday(&dip->i_dir_stat_st);
+
 	gfs_holder_init(dip->i_gl, LM_ST_SHARED, GL_ATIME, &d_gh);
 	error = gfs_glock_nq_atime(&d_gh);
 	if (error) {
diff --git a/gfs-kernel/src/gfs/ops_inode.c b/gfs-kernel/src/gfs/ops_inode.c
index c63b16c..bff5aa0 100644
--- a/gfs-kernel/src/gfs/ops_inode.c
+++ b/gfs-kernel/src/gfs/ops_inode.c
@@ -1508,8 +1508,8 @@ gfs_setattr(struct dentry *dentry, struct iattr *attr)
 static int
 gfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
 {
-	struct inode *inode = dentry->d_inode;
-	struct gfs_inode *ip = get_v2ip(inode);
+	struct inode *inode = dentry->d_inode, *p_inode = NULL;
+	struct gfs_inode *ip = get_v2ip(inode), *pi = NULL;
 	struct gfs_holder gh;
 	int error;
 
@@ -1518,6 +1518,15 @@ gfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
 	error = gfs_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh);
 	if (!error) {
 		generic_fillattr(inode, stat);
+		if (S_ISREG(inode->i_mode) && dentry->d_parent 
+		    && dentry->d_parent->d_inode) {
+			p_inode = igrab(dentry->d_parent->d_inode);
+			if (p_inode) {
+				pi = get_v2ip(p_inode);
+				pi->i_dir_stats++;
+				iput(p_inode);
+			}
+		}
 		gfs_glock_dq_uninit(&gh);
 	}