Skip to content

Commit f7f7e7a

Browse files
ukernelidryomov
ukernel
authored and
idryomov
committed
ceph: improve fscache revalidation
There are several issues in fscache revalidation code. - In ceph_revalidate_work(), fscache_invalidate() is called when fscache_check_consistency() return 0. This is complete wrong because 0 means cache is valid. - Handle_cap_grant() calls ceph_queue_revalidate() if client already has CAP_FILE_CACHE. This code is confusing. Client should revalidate the cache each time it got CAP_FILE_CACHE anew. - In Handle_cap_grant(), fscache_invalidate() is called if MDS revokes CAP_FILE_CACHE. This is inconsistency with the case that inode get evicted. In the later case, the cache is not discarded. Client may use the cache when inode is reloaded. This patch moves the fscache revalidation into ceph_get_caps(). Client revalidates the cache after it gets CAP_FILE_CACHE. i_rdcache_gen should keep constance while CAP_FILE_CACHE is used. If i_fscache_gen is not equal to i_rdcache_gen, client needs to check cache's consistency. Signed-off-by: Yan, Zheng <[email protected]>
1 parent 46b59b2 commit f7f7e7a

File tree

4 files changed

+41
-83
lines changed

4 files changed

+41
-83
lines changed

‎fs/ceph/cache.c

Lines changed: 17 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -69,15 +69,8 @@ int ceph_fscache_register_fs(struct ceph_fs_client* fsc)
6969
fsc->fscache = fscache_acquire_cookie(ceph_cache_netfs.primary_index,
7070
&ceph_fscache_fsid_object_def,
7171
fsc, true);
72-
73-
if (fsc->fscache == NULL) {
72+
if (!fsc->fscache)
7473
pr_err("Unable to resgister fsid: %p fscache cookie", fsc);
75-
return 0;
76-
}
77-
78-
fsc->revalidate_wq = alloc_workqueue("ceph-revalidate", 0, 1);
79-
if (fsc->revalidate_wq == NULL)
80-
return -ENOMEM;
8174

8275
return 0;
8376
}
@@ -260,8 +253,7 @@ static void ceph_vfs_readpage_complete_unlock(struct page *page, void *data, int
260253

261254
static inline bool cache_valid(struct ceph_inode_info *ci)
262255
{
263-
return ((ceph_caps_issued(ci) & CEPH_CAP_FILE_CACHE) &&
264-
(ci->i_fscache_gen == ci->i_rdcache_gen));
256+
return ci->i_fscache_gen == ci->i_rdcache_gen;
265257
}
266258

267259

@@ -354,69 +346,27 @@ void ceph_invalidate_fscache_page(struct inode* inode, struct page *page)
354346

355347
void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc)
356348
{
357-
if (fsc->revalidate_wq)
358-
destroy_workqueue(fsc->revalidate_wq);
359-
360349
fscache_relinquish_cookie(fsc->fscache, 0);
361350
fsc->fscache = NULL;
362351
}
363352

364-
static void ceph_revalidate_work(struct work_struct *work)
365-
{
366-
int issued;
367-
u32 orig_gen;
368-
struct ceph_inode_info *ci = container_of(work, struct ceph_inode_info,
369-
i_revalidate_work);
370-
struct inode *inode = &ci->vfs_inode;
371-
372-
spin_lock(&ci->i_ceph_lock);
373-
issued = __ceph_caps_issued(ci, NULL);
374-
orig_gen = ci->i_rdcache_gen;
375-
spin_unlock(&ci->i_ceph_lock);
376-
377-
if (!(issued & CEPH_CAP_FILE_CACHE)) {
378-
dout("revalidate_work lost cache before validation %p\n",
379-
inode);
380-
goto out;
381-
}
382-
383-
if (!fscache_check_consistency(ci->fscache))
384-
fscache_invalidate(ci->fscache);
385-
386-
spin_lock(&ci->i_ceph_lock);
387-
/* Update the new valid generation (backwards sanity check too) */
388-
if (orig_gen > ci->i_fscache_gen) {
389-
ci->i_fscache_gen = orig_gen;
390-
}
391-
spin_unlock(&ci->i_ceph_lock);
392-
393-
out:
394-
iput(&ci->vfs_inode);
395-
}
396-
397-
void ceph_queue_revalidate(struct inode *inode)
353+
/*
354+
* caller should hold CEPH_CAP_FILE_{RD,CACHE}
355+
*/
356+
void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci)
398357
{
399-
struct ceph_fs_client *fsc = ceph_sb_to_client(inode->i_sb);
400-
struct ceph_inode_info *ci = ceph_inode(inode);
401-
402-
if (fsc->revalidate_wq == NULL || ci->fscache == NULL)
358+
if (cache_valid(ci))
403359
return;
404360

405-
ihold(inode);
406-
407-
if (queue_work(ceph_sb_to_client(inode->i_sb)->revalidate_wq,
408-
&ci->i_revalidate_work)) {
409-
dout("ceph_queue_revalidate %p\n", inode);
410-
} else {
411-
dout("ceph_queue_revalidate %p failed\n)", inode);
412-
iput(inode);
361+
/* resue i_truncate_mutex. There should be no pending
362+
* truncate while the caller holds CEPH_CAP_FILE_RD */
363+
mutex_lock(&ci->i_truncate_mutex);
364+
if (!cache_valid(ci)) {
365+
if (fscache_check_consistency(ci->fscache))
366+
fscache_invalidate(ci->fscache);
367+
spin_lock(&ci->i_ceph_lock);
368+
ci->i_fscache_gen = ci->i_rdcache_gen;
369+
spin_unlock(&ci->i_ceph_lock);
413370
}
414-
}
415-
416-
void ceph_fscache_inode_init(struct ceph_inode_info *ci)
417-
{
418-
ci->fscache = NULL;
419-
/* The first load is verifed cookie open time */
420-
ci->i_fscache_gen = 1;
421-
INIT_WORK(&ci->i_revalidate_work, ceph_revalidate_work);
371+
mutex_unlock(&ci->i_truncate_mutex);
422372
}

‎fs/ceph/cache.h

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ void ceph_fscache_unregister(void);
3434
int ceph_fscache_register_fs(struct ceph_fs_client* fsc);
3535
void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc);
3636

37-
void ceph_fscache_inode_init(struct ceph_inode_info *ci);
3837
void ceph_fscache_register_inode_cookie(struct inode *inode);
3938
void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci);
4039
void ceph_fscache_file_set_cookie(struct inode *inode, struct file *filp);
40+
void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci);
4141

4242
int ceph_readpage_from_fscache(struct inode *inode, struct page *page);
4343
int ceph_readpages_from_fscache(struct inode *inode,
@@ -46,7 +46,12 @@ int ceph_readpages_from_fscache(struct inode *inode,
4646
unsigned *nr_pages);
4747
void ceph_readpage_to_fscache(struct inode *inode, struct page *page);
4848
void ceph_invalidate_fscache_page(struct inode* inode, struct page *page);
49-
void ceph_queue_revalidate(struct inode *inode);
49+
50+
static inline void ceph_fscache_inode_init(struct ceph_inode_info *ci)
51+
{
52+
ci->fscache = NULL;
53+
ci->i_fscache_gen = 0;
54+
}
5055

5156
static inline void ceph_fscache_invalidate(struct inode *inode)
5257
{
@@ -82,6 +87,11 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode,
8287
return fscache_readpages_cancel(ci->fscache, pages);
8388
}
8489

90+
static inline void ceph_disable_fscache_readpage(struct ceph_inode_info *ci)
91+
{
92+
ci->i_fscache_gen = ci->i_rdcache_gen - 1;
93+
}
94+
8595
#else
8696

8797
static inline int ceph_fscache_register(void)
@@ -119,6 +129,10 @@ static inline void ceph_fscache_file_set_cookie(struct inode *inode,
119129
{
120130
}
121131

132+
static inline void ceph_fscache_revalidate_cookie(struct ceph_inode_info *ci)
133+
{
134+
}
135+
122136
static inline void ceph_fscache_uncache_page(struct inode *inode,
123137
struct page *pages)
124138
{
@@ -167,7 +181,7 @@ static inline void ceph_fscache_readpages_cancel(struct inode *inode,
167181
{
168182
}
169183

170-
static inline void ceph_queue_revalidate(struct inode *inode)
184+
static inline void ceph_disable_fscache_readpage(struct ceph_inode_info *ci)
171185
{
172186
}
173187

‎fs/ceph/caps.c

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2393,6 +2393,9 @@ static int try_get_cap_refs(struct ceph_inode_info *ci, int need, int want,
23932393
snap_rwsem_locked = true;
23942394
}
23952395
*got = need | (have & want);
2396+
if ((need & CEPH_CAP_FILE_RD) &&
2397+
!(*got & CEPH_CAP_FILE_CACHE))
2398+
ceph_disable_fscache_readpage(ci);
23962399
__take_cap_refs(ci, *got, true);
23972400
ret = 1;
23982401
}
@@ -2554,6 +2557,9 @@ int ceph_get_caps(struct ceph_inode_info *ci, int need, int want,
25542557
break;
25552558
}
25562559

2560+
if ((_got & CEPH_CAP_FILE_RD) && (_got & CEPH_CAP_FILE_CACHE))
2561+
ceph_fscache_revalidate_cookie(ci);
2562+
25572563
*got = _got;
25582564
return 0;
25592565
}
@@ -2795,7 +2801,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
27952801
bool writeback = false;
27962802
bool queue_trunc = false;
27972803
bool queue_invalidate = false;
2798-
bool queue_revalidate = false;
27992804
bool deleted_inode = false;
28002805
bool fill_inline = false;
28012806

@@ -2837,8 +2842,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
28372842
ci->i_rdcache_revoking = ci->i_rdcache_gen;
28382843
}
28392844
}
2840-
2841-
ceph_fscache_invalidate(inode);
28422845
}
28432846

28442847
/* side effects now are allowed */
@@ -2880,11 +2883,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
28802883
}
28812884
}
28822885

2883-
/* Do we need to revalidate our fscache cookie. Don't bother on the
2884-
* first cache cap as we already validate at cookie creation time. */
2885-
if ((issued & CEPH_CAP_FILE_CACHE) && ci->i_rdcache_gen > 1)
2886-
queue_revalidate = true;
2887-
28882886
if (newcaps & CEPH_CAP_ANY_RD) {
28892887
/* ctime/mtime/atime? */
28902888
ceph_decode_timespec(&mtime, &grant->mtime);
@@ -2995,8 +2993,6 @@ static void handle_cap_grant(struct ceph_mds_client *mdsc,
29952993

29962994
if (queue_trunc)
29972995
ceph_queue_vmtruncate(inode);
2998-
else if (queue_revalidate)
2999-
ceph_queue_revalidate(inode);
30002996

30012997
if (writeback)
30022998
/*

‎fs/ceph/super.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,6 @@ struct ceph_fs_client {
103103

104104
#ifdef CONFIG_CEPH_FSCACHE
105105
struct fscache_cookie *fscache;
106-
struct workqueue_struct *revalidate_wq;
107106
#endif
108107
};
109108

@@ -360,8 +359,7 @@ struct ceph_inode_info {
360359

361360
#ifdef CONFIG_CEPH_FSCACHE
362361
struct fscache_cookie *fscache;
363-
u32 i_fscache_gen; /* sequence, for delayed fscache validate */
364-
struct work_struct i_revalidate_work;
362+
u32 i_fscache_gen;
365363
#endif
366364
struct inode vfs_inode; /* at end */
367365
};

0 commit comments

Comments
 (0)