From 789680d1ee9311cdf095241dc02bd9784d799cd1 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Fri, 7 Jan 2011 17:49:30 +1100 Subject: [PATCH] fs: dcache scale hash Add a new lock, dcache_hash_lock, to protect the dcache hash table from concurrent modification. d_hash is also protected by d_lock. Signed-off-by: Nick Piggin --- fs/dcache.c | 73 +++++++++++++++++++++++++++++++++++------- include/linux/dcache.h | 35 ++------------------ 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/fs/dcache.c b/fs/dcache.c index 61beb40dd6bf..1e124d4673c6 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -35,10 +35,24 @@ #include #include "internal.h" +/* + * Usage: + * dcache_hash_lock protects dcache hash table, s_anon lists + * + * Ordering: + * dcache_lock + * dentry->d_lock + * dcache_hash_lock + * + * if (dentry1 < dentry2) + * dentry1->d_lock + * dentry2->d_lock + */ int sysctl_vfs_cache_pressure __read_mostly = 100; EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); - __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock); +__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); EXPORT_SYMBOL(dcache_lock); @@ -196,6 +210,42 @@ static struct dentry *d_kill(struct dentry *dentry) return parent; } +/** + * d_drop - drop a dentry + * @dentry: dentry to drop + * + * d_drop() unhashes the entry from the parent dentry hashes, so that it won't + * be found through a VFS lookup any more. Note that this is different from + * deleting the dentry - d_delete will try to mark the dentry negative if + * possible, giving a successful _negative_ lookup, while d_drop will + * just make the cache lookup fail. + * + * d_drop() is used mainly for stuff that wants to invalidate a dentry for some + * reason (NFS timeouts or autofs deletes). + * + * __d_drop requires dentry->d_lock. + */ +void __d_drop(struct dentry *dentry) +{ + if (!(dentry->d_flags & DCACHE_UNHASHED)) { + dentry->d_flags |= DCACHE_UNHASHED; + spin_lock(&dcache_hash_lock); + hlist_del_rcu(&dentry->d_hash); + spin_unlock(&dcache_hash_lock); + } +} +EXPORT_SYMBOL(__d_drop); + +void d_drop(struct dentry *dentry) +{ + spin_lock(&dcache_lock); + spin_lock(&dentry->d_lock); + __d_drop(dentry); + spin_unlock(&dentry->d_lock); + spin_unlock(&dcache_lock); +} +EXPORT_SYMBOL(d_drop); + /* * This is dput * @@ -1199,7 +1249,9 @@ struct dentry *d_obtain_alias(struct inode *inode) tmp->d_flags |= DCACHE_DISCONNECTED; tmp->d_flags &= ~DCACHE_UNHASHED; list_add(&tmp->d_alias, &inode->i_dentry); + spin_lock(&dcache_hash_lock); hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon); + spin_unlock(&dcache_hash_lock); spin_unlock(&tmp->d_lock); spin_unlock(&dcache_lock); @@ -1585,7 +1637,9 @@ void d_rehash(struct dentry * entry) { spin_lock(&dcache_lock); spin_lock(&entry->d_lock); + spin_lock(&dcache_hash_lock); _d_rehash(entry); + spin_unlock(&dcache_hash_lock); spin_unlock(&entry->d_lock); spin_unlock(&dcache_lock); } @@ -1692,8 +1746,6 @@ static void switch_names(struct dentry *dentry, struct dentry *target) */ static void d_move_locked(struct dentry * dentry, struct dentry * target) { - struct hlist_head *list; - if (!dentry->d_inode) printk(KERN_WARNING "VFS: moving negative dcache entry\n"); @@ -1710,14 +1762,11 @@ static void d_move_locked(struct dentry * dentry, struct dentry * target) } /* Move the dentry to the target hash queue, if on different bucket */ - if (d_unhashed(dentry)) - goto already_unhashed; - - hlist_del_rcu(&dentry->d_hash); - -already_unhashed: - list = d_hash(target->d_parent, target->d_name.hash); - __d_rehash(dentry, list); + spin_lock(&dcache_hash_lock); + if (!d_unhashed(dentry)) + hlist_del_rcu(&dentry->d_hash); + __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash)); + spin_unlock(&dcache_hash_lock); /* Unhash the target: dput() will then get rid of it */ __d_drop(target); @@ -1914,7 +1963,9 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) found_lock: spin_lock(&actual->d_lock); found: + spin_lock(&dcache_hash_lock); _d_rehash(actual); + spin_unlock(&dcache_hash_lock); spin_unlock(&actual->d_lock); spin_unlock(&dcache_lock); out_nolock: diff --git a/include/linux/dcache.h b/include/linux/dcache.h index cea27dfca532..2feb624b67f1 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -184,39 +184,6 @@ struct dentry_operations { extern spinlock_t dcache_lock; extern seqlock_t rename_lock; -/** - * d_drop - drop a dentry - * @dentry: dentry to drop - * - * d_drop() unhashes the entry from the parent dentry hashes, so that it won't - * be found through a VFS lookup any more. Note that this is different from - * deleting the dentry - d_delete will try to mark the dentry negative if - * possible, giving a successful _negative_ lookup, while d_drop will - * just make the cache lookup fail. - * - * d_drop() is used mainly for stuff that wants to invalidate a dentry for some - * reason (NFS timeouts or autofs deletes). - * - * __d_drop requires dentry->d_lock. - */ - -static inline void __d_drop(struct dentry *dentry) -{ - if (!(dentry->d_flags & DCACHE_UNHASHED)) { - dentry->d_flags |= DCACHE_UNHASHED; - hlist_del_rcu(&dentry->d_hash); - } -} - -static inline void d_drop(struct dentry *dentry) -{ - spin_lock(&dcache_lock); - spin_lock(&dentry->d_lock); - __d_drop(dentry); - spin_unlock(&dentry->d_lock); - spin_unlock(&dcache_lock); -} - static inline int dname_external(struct dentry *dentry) { return dentry->d_name.name != dentry->d_iname; @@ -228,6 +195,8 @@ static inline int dname_external(struct dentry *dentry) extern void d_instantiate(struct dentry *, struct inode *); extern struct dentry * d_instantiate_unique(struct dentry *, struct inode *); extern struct dentry * d_materialise_unique(struct dentry *, struct inode *); +extern void __d_drop(struct dentry *dentry); +extern void d_drop(struct dentry *dentry); extern void d_delete(struct dentry *); /* allocate/de-allocate */