421 lines
10 KiB
C
421 lines
10 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
/* kiocb-using read/write
|
|
*
|
|
* Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
|
|
* Written by David Howells (dhowells@redhat.com)
|
|
*/
|
|
|
|
#include <linux/mount.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/file.h>
|
|
#include <linux/uio.h>
|
|
#include <linux/sched/mm.h>
|
|
#include <linux/netfs.h>
|
|
#include "internal.h"
|
|
|
|
struct cachefiles_kiocb {
|
|
struct kiocb iocb;
|
|
refcount_t ki_refcnt;
|
|
loff_t start;
|
|
union {
|
|
size_t skipped;
|
|
size_t len;
|
|
};
|
|
netfs_io_terminated_t term_func;
|
|
void *term_func_priv;
|
|
bool was_async;
|
|
};
|
|
|
|
static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
|
|
{
|
|
if (refcount_dec_and_test(&ki->ki_refcnt)) {
|
|
fput(ki->iocb.ki_filp);
|
|
kfree(ki);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Handle completion of a read from the cache.
|
|
*/
|
|
static void cachefiles_read_complete(struct kiocb *iocb, long ret, long ret2)
|
|
{
|
|
struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
|
|
|
|
_enter("%ld,%ld", ret, ret2);
|
|
|
|
if (ki->term_func) {
|
|
if (ret >= 0)
|
|
ret += ki->skipped;
|
|
ki->term_func(ki->term_func_priv, ret, ki->was_async);
|
|
}
|
|
|
|
cachefiles_put_kiocb(ki);
|
|
}
|
|
|
|
/*
|
|
* Initiate a read from the cache.
|
|
*/
|
|
static int cachefiles_read(struct netfs_cache_resources *cres,
|
|
loff_t start_pos,
|
|
struct iov_iter *iter,
|
|
bool seek_data,
|
|
netfs_io_terminated_t term_func,
|
|
void *term_func_priv)
|
|
{
|
|
struct cachefiles_kiocb *ki;
|
|
struct file *file = cres->cache_priv2;
|
|
unsigned int old_nofs;
|
|
ssize_t ret = -ENOBUFS;
|
|
size_t len = iov_iter_count(iter), skipped = 0;
|
|
|
|
_enter("%pD,%li,%llx,%zx/%llx",
|
|
file, file_inode(file)->i_ino, start_pos, len,
|
|
i_size_read(file->f_inode));
|
|
|
|
/* If the caller asked us to seek for data before doing the read, then
|
|
* we should do that now. If we find a gap, we fill it with zeros.
|
|
*/
|
|
if (seek_data) {
|
|
loff_t off = start_pos, off2;
|
|
|
|
off2 = vfs_llseek(file, off, SEEK_DATA);
|
|
if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) {
|
|
skipped = 0;
|
|
ret = off2;
|
|
goto presubmission_error;
|
|
}
|
|
|
|
if (off2 == -ENXIO || off2 >= start_pos + len) {
|
|
/* The region is beyond the EOF or there's no more data
|
|
* in the region, so clear the rest of the buffer and
|
|
* return success.
|
|
*/
|
|
iov_iter_zero(len, iter);
|
|
skipped = len;
|
|
ret = 0;
|
|
goto presubmission_error;
|
|
}
|
|
|
|
skipped = off2 - off;
|
|
iov_iter_zero(skipped, iter);
|
|
}
|
|
|
|
ret = -ENOBUFS;
|
|
ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
|
|
if (!ki)
|
|
goto presubmission_error;
|
|
|
|
refcount_set(&ki->ki_refcnt, 2);
|
|
ki->iocb.ki_filp = file;
|
|
ki->iocb.ki_pos = start_pos + skipped;
|
|
ki->iocb.ki_flags = IOCB_DIRECT;
|
|
ki->iocb.ki_hint = ki_hint_validate(file_write_hint(file));
|
|
ki->iocb.ki_ioprio = get_current_ioprio();
|
|
ki->skipped = skipped;
|
|
ki->term_func = term_func;
|
|
ki->term_func_priv = term_func_priv;
|
|
ki->was_async = true;
|
|
|
|
if (ki->term_func)
|
|
ki->iocb.ki_complete = cachefiles_read_complete;
|
|
|
|
get_file(ki->iocb.ki_filp);
|
|
|
|
old_nofs = memalloc_nofs_save();
|
|
ret = vfs_iocb_iter_read(file, &ki->iocb, iter);
|
|
memalloc_nofs_restore(old_nofs);
|
|
switch (ret) {
|
|
case -EIOCBQUEUED:
|
|
goto in_progress;
|
|
|
|
case -ERESTARTSYS:
|
|
case -ERESTARTNOINTR:
|
|
case -ERESTARTNOHAND:
|
|
case -ERESTART_RESTARTBLOCK:
|
|
/* There's no easy way to restart the syscall since other AIO's
|
|
* may be already running. Just fail this IO with EINTR.
|
|
*/
|
|
ret = -EINTR;
|
|
fallthrough;
|
|
default:
|
|
ki->was_async = false;
|
|
cachefiles_read_complete(&ki->iocb, ret, 0);
|
|
if (ret > 0)
|
|
ret = 0;
|
|
break;
|
|
}
|
|
|
|
in_progress:
|
|
cachefiles_put_kiocb(ki);
|
|
_leave(" = %zd", ret);
|
|
return ret;
|
|
|
|
presubmission_error:
|
|
if (term_func)
|
|
term_func(term_func_priv, ret < 0 ? ret : skipped, false);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Handle completion of a write to the cache.
|
|
*/
|
|
static void cachefiles_write_complete(struct kiocb *iocb, long ret, long ret2)
|
|
{
|
|
struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
|
|
struct inode *inode = file_inode(ki->iocb.ki_filp);
|
|
|
|
_enter("%ld,%ld", ret, ret2);
|
|
|
|
/* Tell lockdep we inherited freeze protection from submission thread */
|
|
__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
|
|
__sb_end_write(inode->i_sb, SB_FREEZE_WRITE);
|
|
|
|
if (ki->term_func)
|
|
ki->term_func(ki->term_func_priv, ret, ki->was_async);
|
|
|
|
cachefiles_put_kiocb(ki);
|
|
}
|
|
|
|
/*
|
|
* Initiate a write to the cache.
|
|
*/
|
|
static int cachefiles_write(struct netfs_cache_resources *cres,
|
|
loff_t start_pos,
|
|
struct iov_iter *iter,
|
|
netfs_io_terminated_t term_func,
|
|
void *term_func_priv)
|
|
{
|
|
struct cachefiles_kiocb *ki;
|
|
struct inode *inode;
|
|
struct file *file = cres->cache_priv2;
|
|
unsigned int old_nofs;
|
|
ssize_t ret = -ENOBUFS;
|
|
size_t len = iov_iter_count(iter);
|
|
|
|
_enter("%pD,%li,%llx,%zx/%llx",
|
|
file, file_inode(file)->i_ino, start_pos, len,
|
|
i_size_read(file->f_inode));
|
|
|
|
ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);
|
|
if (!ki)
|
|
goto presubmission_error;
|
|
|
|
refcount_set(&ki->ki_refcnt, 2);
|
|
ki->iocb.ki_filp = file;
|
|
ki->iocb.ki_pos = start_pos;
|
|
ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE;
|
|
ki->iocb.ki_hint = ki_hint_validate(file_write_hint(file));
|
|
ki->iocb.ki_ioprio = get_current_ioprio();
|
|
ki->start = start_pos;
|
|
ki->len = len;
|
|
ki->term_func = term_func;
|
|
ki->term_func_priv = term_func_priv;
|
|
ki->was_async = true;
|
|
|
|
if (ki->term_func)
|
|
ki->iocb.ki_complete = cachefiles_write_complete;
|
|
|
|
/* Open-code file_start_write here to grab freeze protection, which
|
|
* will be released by another thread in aio_complete_rw(). Fool
|
|
* lockdep by telling it the lock got released so that it doesn't
|
|
* complain about the held lock when we return to userspace.
|
|
*/
|
|
inode = file_inode(file);
|
|
__sb_start_write(inode->i_sb, SB_FREEZE_WRITE);
|
|
__sb_writers_release(inode->i_sb, SB_FREEZE_WRITE);
|
|
|
|
get_file(ki->iocb.ki_filp);
|
|
|
|
old_nofs = memalloc_nofs_save();
|
|
ret = vfs_iocb_iter_write(file, &ki->iocb, iter);
|
|
memalloc_nofs_restore(old_nofs);
|
|
switch (ret) {
|
|
case -EIOCBQUEUED:
|
|
goto in_progress;
|
|
|
|
case -ERESTARTSYS:
|
|
case -ERESTARTNOINTR:
|
|
case -ERESTARTNOHAND:
|
|
case -ERESTART_RESTARTBLOCK:
|
|
/* There's no easy way to restart the syscall since other AIO's
|
|
* may be already running. Just fail this IO with EINTR.
|
|
*/
|
|
ret = -EINTR;
|
|
fallthrough;
|
|
default:
|
|
ki->was_async = false;
|
|
cachefiles_write_complete(&ki->iocb, ret, 0);
|
|
if (ret > 0)
|
|
ret = 0;
|
|
break;
|
|
}
|
|
|
|
in_progress:
|
|
cachefiles_put_kiocb(ki);
|
|
_leave(" = %zd", ret);
|
|
return ret;
|
|
|
|
presubmission_error:
|
|
if (term_func)
|
|
term_func(term_func_priv, -ENOMEM, false);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
/*
|
|
* Prepare a read operation, shortening it to a cached/uncached
|
|
* boundary as appropriate.
|
|
*/
|
|
static enum netfs_read_source cachefiles_prepare_read(struct netfs_read_subrequest *subreq,
|
|
loff_t i_size)
|
|
{
|
|
struct fscache_retrieval *op = subreq->rreq->cache_resources.cache_priv;
|
|
struct cachefiles_object *object;
|
|
struct cachefiles_cache *cache;
|
|
const struct cred *saved_cred;
|
|
struct file *file = subreq->rreq->cache_resources.cache_priv2;
|
|
loff_t off, to;
|
|
|
|
_enter("%zx @%llx/%llx", subreq->len, subreq->start, i_size);
|
|
|
|
object = container_of(op->op.object,
|
|
struct cachefiles_object, fscache);
|
|
cache = container_of(object->fscache.cache,
|
|
struct cachefiles_cache, cache);
|
|
|
|
if (!file)
|
|
goto cache_fail_nosec;
|
|
|
|
if (subreq->start >= i_size)
|
|
return NETFS_FILL_WITH_ZEROES;
|
|
|
|
cachefiles_begin_secure(cache, &saved_cred);
|
|
|
|
off = vfs_llseek(file, subreq->start, SEEK_DATA);
|
|
if (off < 0 && off >= (loff_t)-MAX_ERRNO) {
|
|
if (off == (loff_t)-ENXIO)
|
|
goto download_and_store;
|
|
goto cache_fail;
|
|
}
|
|
|
|
if (off >= subreq->start + subreq->len)
|
|
goto download_and_store;
|
|
|
|
if (off > subreq->start) {
|
|
off = round_up(off, cache->bsize);
|
|
subreq->len = off - subreq->start;
|
|
goto download_and_store;
|
|
}
|
|
|
|
to = vfs_llseek(file, subreq->start, SEEK_HOLE);
|
|
if (to < 0 && to >= (loff_t)-MAX_ERRNO)
|
|
goto cache_fail;
|
|
|
|
if (to < subreq->start + subreq->len) {
|
|
if (subreq->start + subreq->len >= i_size)
|
|
to = round_up(to, cache->bsize);
|
|
else
|
|
to = round_down(to, cache->bsize);
|
|
subreq->len = to - subreq->start;
|
|
}
|
|
|
|
cachefiles_end_secure(cache, saved_cred);
|
|
return NETFS_READ_FROM_CACHE;
|
|
|
|
download_and_store:
|
|
if (cachefiles_has_space(cache, 0, (subreq->len + PAGE_SIZE - 1) / PAGE_SIZE) == 0)
|
|
__set_bit(NETFS_SREQ_WRITE_TO_CACHE, &subreq->flags);
|
|
cache_fail:
|
|
cachefiles_end_secure(cache, saved_cred);
|
|
cache_fail_nosec:
|
|
return NETFS_DOWNLOAD_FROM_SERVER;
|
|
}
|
|
|
|
/*
|
|
* Prepare for a write to occur.
|
|
*/
|
|
static int cachefiles_prepare_write(struct netfs_cache_resources *cres,
|
|
loff_t *_start, size_t *_len, loff_t i_size)
|
|
{
|
|
loff_t start = *_start;
|
|
size_t len = *_len, down;
|
|
|
|
/* Round to DIO size */
|
|
down = start - round_down(start, PAGE_SIZE);
|
|
*_start = start - down;
|
|
*_len = round_up(down + len, PAGE_SIZE);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Clean up an operation.
|
|
*/
|
|
static void cachefiles_end_operation(struct netfs_cache_resources *cres)
|
|
{
|
|
struct fscache_retrieval *op = cres->cache_priv;
|
|
struct file *file = cres->cache_priv2;
|
|
|
|
_enter("");
|
|
|
|
if (file)
|
|
fput(file);
|
|
if (op) {
|
|
fscache_op_complete(&op->op, false);
|
|
fscache_put_retrieval(op);
|
|
}
|
|
|
|
_leave("");
|
|
}
|
|
|
|
static const struct netfs_cache_ops cachefiles_netfs_cache_ops = {
|
|
.end_operation = cachefiles_end_operation,
|
|
.read = cachefiles_read,
|
|
.write = cachefiles_write,
|
|
.prepare_read = cachefiles_prepare_read,
|
|
.prepare_write = cachefiles_prepare_write,
|
|
};
|
|
|
|
/*
|
|
* Open the cache file when beginning a cache operation.
|
|
*/
|
|
int cachefiles_begin_read_operation(struct netfs_read_request *rreq,
|
|
struct fscache_retrieval *op)
|
|
{
|
|
struct cachefiles_object *object;
|
|
struct cachefiles_cache *cache;
|
|
struct path path;
|
|
struct file *file;
|
|
|
|
_enter("");
|
|
|
|
object = container_of(op->op.object,
|
|
struct cachefiles_object, fscache);
|
|
cache = container_of(object->fscache.cache,
|
|
struct cachefiles_cache, cache);
|
|
|
|
path.mnt = cache->mnt;
|
|
path.dentry = object->backer;
|
|
file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT,
|
|
d_inode(object->backer), cache->cache_cred);
|
|
if (IS_ERR(file))
|
|
return PTR_ERR(file);
|
|
if (!S_ISREG(file_inode(file)->i_mode))
|
|
goto error_file;
|
|
if (unlikely(!file->f_op->read_iter) ||
|
|
unlikely(!file->f_op->write_iter)) {
|
|
pr_notice("Cache does not support read_iter and write_iter\n");
|
|
goto error_file;
|
|
}
|
|
|
|
fscache_get_retrieval(op);
|
|
rreq->cache_resources.cache_priv = op;
|
|
rreq->cache_resources.cache_priv2 = file;
|
|
rreq->cache_resources.ops = &cachefiles_netfs_cache_ops;
|
|
rreq->cookie_debug_id = object->fscache.debug_id;
|
|
_leave("");
|
|
return 0;
|
|
|
|
error_file:
|
|
fput(file);
|
|
return -EIO;
|
|
}
|