block: Convert drivers to immutable biovecs
Now that we've got a mechanism for immutable biovecs - bi_iter.bi_bvec_done - we need to convert drivers to use primitives that respect it instead of using the bvec array directly. Signed-off-by: Kent Overstreet <kmo@daterainc.com> Cc: Jens Axboe <axboe@kernel.dk> Cc: NeilBrown <neilb@suse.de> Cc: Alasdair Kergon <agk@redhat.com> Cc: dm-devel@redhat.com
This commit is contained in:
Kent Overstreet
parent
458b76ed2f
commit
003b5c5719
|
|
@ -108,8 +108,7 @@ struct cardinfo {
|
|||
* have been written
|
||||
*/
|
||||
struct bio *bio, *currentbio, **biotail;
|
||||
int current_idx;
|
||||
sector_t current_sector;
|
||||
struct bvec_iter current_iter;
|
||||
|
||||
struct request_queue *queue;
|
||||
|
||||
|
|
@ -118,7 +117,7 @@ struct cardinfo {
|
|||
struct mm_dma_desc *desc;
|
||||
int cnt, headcnt;
|
||||
struct bio *bio, **biotail;
|
||||
int idx;
|
||||
struct bvec_iter iter;
|
||||
} mm_pages[2];
|
||||
#define DESC_PER_PAGE ((PAGE_SIZE*2)/sizeof(struct mm_dma_desc))
|
||||
|
||||
|
|
@ -344,16 +343,13 @@ static int add_bio(struct cardinfo *card)
|
|||
dma_addr_t dma_handle;
|
||||
int offset;
|
||||
struct bio *bio;
|
||||
struct bio_vec *vec;
|
||||
int idx;
|
||||
struct bio_vec vec;
|
||||
int rw;
|
||||
int len;
|
||||
|
||||
bio = card->currentbio;
|
||||
if (!bio && card->bio) {
|
||||
card->currentbio = card->bio;
|
||||
card->current_idx = card->bio->bi_iter.bi_idx;
|
||||
card->current_sector = card->bio->bi_iter.bi_sector;
|
||||
card->current_iter = card->bio->bi_iter;
|
||||
card->bio = card->bio->bi_next;
|
||||
if (card->bio == NULL)
|
||||
card->biotail = &card->bio;
|
||||
|
|
@ -362,18 +358,17 @@ static int add_bio(struct cardinfo *card)
|
|||
}
|
||||
if (!bio)
|
||||
return 0;
|
||||
idx = card->current_idx;
|
||||
|
||||
rw = bio_rw(bio);
|
||||
if (card->mm_pages[card->Ready].cnt >= DESC_PER_PAGE)
|
||||
return 0;
|
||||
|
||||
vec = bio_iovec_idx(bio, idx);
|
||||
len = vec->bv_len;
|
||||
vec = bio_iter_iovec(bio, card->current_iter);
|
||||
|
||||
dma_handle = pci_map_page(card->dev,
|
||||
vec->bv_page,
|
||||
vec->bv_offset,
|
||||
len,
|
||||
vec.bv_page,
|
||||
vec.bv_offset,
|
||||
vec.bv_len,
|
||||
(rw == READ) ?
|
||||
PCI_DMA_FROMDEVICE : PCI_DMA_TODEVICE);
|
||||
|
||||
|
|
@ -381,7 +376,7 @@ static int add_bio(struct cardinfo *card)
|
|||
desc = &p->desc[p->cnt];
|
||||
p->cnt++;
|
||||
if (p->bio == NULL)
|
||||
p->idx = idx;
|
||||
p->iter = card->current_iter;
|
||||
if ((p->biotail) != &bio->bi_next) {
|
||||
*(p->biotail) = bio;
|
||||
p->biotail = &(bio->bi_next);
|
||||
|
|
@ -391,8 +386,8 @@ static int add_bio(struct cardinfo *card)
|
|||
desc->data_dma_handle = dma_handle;
|
||||
|
||||
desc->pci_addr = cpu_to_le64((u64)desc->data_dma_handle);
|
||||
desc->local_addr = cpu_to_le64(card->current_sector << 9);
|
||||
desc->transfer_size = cpu_to_le32(len);
|
||||
desc->local_addr = cpu_to_le64(card->current_iter.bi_sector << 9);
|
||||
desc->transfer_size = cpu_to_le32(vec.bv_len);
|
||||
offset = (((char *)&desc->sem_control_bits) - ((char *)p->desc));
|
||||
desc->sem_addr = cpu_to_le64((u64)(p->page_dma+offset));
|
||||
desc->zero1 = desc->zero2 = 0;
|
||||
|
|
@ -407,10 +402,9 @@ static int add_bio(struct cardinfo *card)
|
|||
desc->control_bits |= cpu_to_le32(DMASCR_TRANSFER_READ);
|
||||
desc->sem_control_bits = desc->control_bits;
|
||||
|
||||
card->current_sector += (len >> 9);
|
||||
idx++;
|
||||
card->current_idx = idx;
|
||||
if (idx >= bio->bi_vcnt)
|
||||
|
||||
bio_advance_iter(bio, &card->current_iter, vec.bv_len);
|
||||
if (!card->current_iter.bi_size)
|
||||
card->currentbio = NULL;
|
||||
|
||||
return 1;
|
||||
|
|
@ -439,23 +433,25 @@ static void process_page(unsigned long data)
|
|||
struct mm_dma_desc *desc = &page->desc[page->headcnt];
|
||||
int control = le32_to_cpu(desc->sem_control_bits);
|
||||
int last = 0;
|
||||
int idx;
|
||||
struct bio_vec vec;
|
||||
|
||||
if (!(control & DMASCR_DMA_COMPLETE)) {
|
||||
control = dma_status;
|
||||
last = 1;
|
||||
}
|
||||
|
||||
page->headcnt++;
|
||||
idx = page->idx;
|
||||
page->idx++;
|
||||
if (page->idx >= bio->bi_vcnt) {
|
||||
vec = bio_iter_iovec(bio, page->iter);
|
||||
bio_advance_iter(bio, &page->iter, vec.bv_len);
|
||||
|
||||
if (!page->iter.bi_size) {
|
||||
page->bio = bio->bi_next;
|
||||
if (page->bio)
|
||||
page->idx = page->bio->bi_iter.bi_idx;
|
||||
page->iter = page->bio->bi_iter;
|
||||
}
|
||||
|
||||
pci_unmap_page(card->dev, desc->data_dma_handle,
|
||||
bio_iovec_idx(bio, idx)->bv_len,
|
||||
vec.bv_len,
|
||||
(control & DMASCR_TRANSFER_READ) ?
|
||||
PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
|
||||
if (control & DMASCR_HARD_ERROR) {
|
||||
|
|
|
|||
|
|
@ -39,10 +39,8 @@ struct convert_context {
|
|||
struct completion restart;
|
||||
struct bio *bio_in;
|
||||
struct bio *bio_out;
|
||||
unsigned int offset_in;
|
||||
unsigned int offset_out;
|
||||
unsigned int idx_in;
|
||||
unsigned int idx_out;
|
||||
struct bvec_iter iter_in;
|
||||
struct bvec_iter iter_out;
|
||||
sector_t cc_sector;
|
||||
atomic_t cc_pending;
|
||||
};
|
||||
|
|
@ -826,10 +824,10 @@ static void crypt_convert_init(struct crypt_config *cc,
|
|||
{
|
||||
ctx->bio_in = bio_in;
|
||||
ctx->bio_out = bio_out;
|
||||
ctx->offset_in = 0;
|
||||
ctx->offset_out = 0;
|
||||
ctx->idx_in = bio_in ? bio_in->bi_iter.bi_idx : 0;
|
||||
ctx->idx_out = bio_out ? bio_out->bi_iter.bi_idx : 0;
|
||||
if (bio_in)
|
||||
ctx->iter_in = bio_in->bi_iter;
|
||||
if (bio_out)
|
||||
ctx->iter_out = bio_out->bi_iter;
|
||||
ctx->cc_sector = sector + cc->iv_offset;
|
||||
init_completion(&ctx->restart);
|
||||
}
|
||||
|
|
@ -857,8 +855,8 @@ static int crypt_convert_block(struct crypt_config *cc,
|
|||
struct convert_context *ctx,
|
||||
struct ablkcipher_request *req)
|
||||
{
|
||||
struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
|
||||
struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
|
||||
struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
|
||||
struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
|
||||
struct dm_crypt_request *dmreq;
|
||||
u8 *iv;
|
||||
int r;
|
||||
|
|
@ -869,24 +867,15 @@ static int crypt_convert_block(struct crypt_config *cc,
|
|||
dmreq->iv_sector = ctx->cc_sector;
|
||||
dmreq->ctx = ctx;
|
||||
sg_init_table(&dmreq->sg_in, 1);
|
||||
sg_set_page(&dmreq->sg_in, bv_in->bv_page, 1 << SECTOR_SHIFT,
|
||||
bv_in->bv_offset + ctx->offset_in);
|
||||
sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT,
|
||||
bv_in.bv_offset);
|
||||
|
||||
sg_init_table(&dmreq->sg_out, 1);
|
||||
sg_set_page(&dmreq->sg_out, bv_out->bv_page, 1 << SECTOR_SHIFT,
|
||||
bv_out->bv_offset + ctx->offset_out);
|
||||
sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT,
|
||||
bv_out.bv_offset);
|
||||
|
||||
ctx->offset_in += 1 << SECTOR_SHIFT;
|
||||
if (ctx->offset_in >= bv_in->bv_len) {
|
||||
ctx->offset_in = 0;
|
||||
ctx->idx_in++;
|
||||
}
|
||||
|
||||
ctx->offset_out += 1 << SECTOR_SHIFT;
|
||||
if (ctx->offset_out >= bv_out->bv_len) {
|
||||
ctx->offset_out = 0;
|
||||
ctx->idx_out++;
|
||||
}
|
||||
bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT);
|
||||
bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT);
|
||||
|
||||
if (cc->iv_gen_ops) {
|
||||
r = cc->iv_gen_ops->generator(cc, iv, dmreq);
|
||||
|
|
@ -937,8 +926,7 @@ static int crypt_convert(struct crypt_config *cc,
|
|||
|
||||
atomic_set(&ctx->cc_pending, 1);
|
||||
|
||||
while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
|
||||
ctx->idx_out < ctx->bio_out->bi_vcnt) {
|
||||
while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
|
||||
|
||||
crypt_alloc_req(cc, ctx);
|
||||
|
||||
|
|
@ -1207,7 +1195,7 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
|
|||
}
|
||||
|
||||
/* crypt_convert should have filled the clone bio */
|
||||
BUG_ON(io->ctx.idx_out < clone->bi_vcnt);
|
||||
BUG_ON(io->ctx.iter_out.bi_size);
|
||||
|
||||
clone->bi_iter.bi_sector = cc->start + io->sector;
|
||||
|
||||
|
|
@ -1246,7 +1234,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
|
|||
}
|
||||
|
||||
io->ctx.bio_out = clone;
|
||||
io->ctx.idx_out = 0;
|
||||
io->ctx.iter_out = clone->bi_iter;
|
||||
|
||||
remaining -= clone->bi_iter.bi_size;
|
||||
sector += bio_sectors(clone);
|
||||
|
|
@ -1290,8 +1278,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
|
|||
crypt_inc_pending(new_io);
|
||||
crypt_convert_init(cc, &new_io->ctx, NULL,
|
||||
io->base_bio, sector);
|
||||
new_io->ctx.idx_in = io->ctx.idx_in;
|
||||
new_io->ctx.offset_in = io->ctx.offset_in;
|
||||
new_io->ctx.iter_in = io->ctx.iter_in;
|
||||
|
||||
/*
|
||||
* Fragments after the first use the base_io
|
||||
|
|
|
|||
|
|
@ -201,26 +201,29 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offse
|
|||
/*
|
||||
* Functions for getting the pages from a bvec.
|
||||
*/
|
||||
static void bvec_get_page(struct dpages *dp,
|
||||
static void bio_get_page(struct dpages *dp,
|
||||
struct page **p, unsigned long *len, unsigned *offset)
|
||||
{
|
||||
struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
|
||||
*p = bvec->bv_page;
|
||||
*len = bvec->bv_len;
|
||||
*offset = bvec->bv_offset;
|
||||
struct bio *bio = dp->context_ptr;
|
||||
struct bio_vec bvec = bio_iovec(bio);
|
||||
*p = bvec.bv_page;
|
||||
*len = bvec.bv_len;
|
||||
*offset = bvec.bv_offset;
|
||||
}
|
||||
|
||||
static void bvec_next_page(struct dpages *dp)
|
||||
static void bio_next_page(struct dpages *dp)
|
||||
{
|
||||
struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
|
||||
dp->context_ptr = bvec + 1;
|
||||
struct bio *bio = dp->context_ptr;
|
||||
struct bio_vec bvec = bio_iovec(bio);
|
||||
|
||||
bio_advance(bio, bvec.bv_len);
|
||||
}
|
||||
|
||||
static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec)
|
||||
static void bio_dp_init(struct dpages *dp, struct bio *bio)
|
||||
{
|
||||
dp->get_page = bvec_get_page;
|
||||
dp->next_page = bvec_next_page;
|
||||
dp->context_ptr = bvec;
|
||||
dp->get_page = bio_get_page;
|
||||
dp->next_page = bio_next_page;
|
||||
dp->context_ptr = bio;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -457,8 +460,8 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp,
|
|||
list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset);
|
||||
break;
|
||||
|
||||
case DM_IO_BVEC:
|
||||
bvec_dp_init(dp, io_req->mem.ptr.bvec);
|
||||
case DM_IO_BIO:
|
||||
bio_dp_init(dp, io_req->mem.ptr.bio);
|
||||
break;
|
||||
|
||||
case DM_IO_VMA:
|
||||
|
|
|
|||
|
|
@ -526,8 +526,8 @@ static void read_async_bio(struct mirror *m, struct bio *bio)
|
|||
struct dm_io_region io;
|
||||
struct dm_io_request io_req = {
|
||||
.bi_rw = READ,
|
||||
.mem.type = DM_IO_BVEC,
|
||||
.mem.ptr.bvec = bio->bi_io_vec + bio->bi_iter.bi_idx,
|
||||
.mem.type = DM_IO_BIO,
|
||||
.mem.ptr.bio = bio,
|
||||
.notify.fn = read_callback,
|
||||
.notify.context = bio,
|
||||
.client = m->ms->io_client,
|
||||
|
|
@ -629,8 +629,8 @@ static void do_write(struct mirror_set *ms, struct bio *bio)
|
|||
struct mirror *m;
|
||||
struct dm_io_request io_req = {
|
||||
.bi_rw = WRITE | (bio->bi_rw & WRITE_FLUSH_FUA),
|
||||
.mem.type = DM_IO_BVEC,
|
||||
.mem.ptr.bvec = bio->bi_io_vec + bio->bi_iter.bi_idx,
|
||||
.mem.type = DM_IO_BIO,
|
||||
.mem.ptr.bio = bio,
|
||||
.notify.fn = write_callback,
|
||||
.notify.context = bio,
|
||||
.client = ms->io_client,
|
||||
|
|
|
|||
|
|
@ -73,15 +73,10 @@ struct dm_verity_io {
|
|||
sector_t block;
|
||||
unsigned n_blocks;
|
||||
|
||||
/* saved bio vector */
|
||||
struct bio_vec *io_vec;
|
||||
unsigned io_vec_size;
|
||||
struct bvec_iter iter;
|
||||
|
||||
struct work_struct work;
|
||||
|
||||
/* A space for short vectors; longer vectors are allocated separately. */
|
||||
struct bio_vec io_vec_inline[DM_VERITY_IO_VEC_INLINE];
|
||||
|
||||
/*
|
||||
* Three variably-size fields follow this struct:
|
||||
*
|
||||
|
|
@ -284,9 +279,10 @@ release_ret_r:
|
|||
static int verity_verify_io(struct dm_verity_io *io)
|
||||
{
|
||||
struct dm_verity *v = io->v;
|
||||
struct bio *bio = dm_bio_from_per_bio_data(io,
|
||||
v->ti->per_bio_data_size);
|
||||
unsigned b;
|
||||
int i;
|
||||
unsigned vector = 0, offset = 0;
|
||||
|
||||
for (b = 0; b < io->n_blocks; b++) {
|
||||
struct shash_desc *desc;
|
||||
|
|
@ -336,31 +332,22 @@ test_block_hash:
|
|||
}
|
||||
|
||||
todo = 1 << v->data_dev_block_bits;
|
||||
do {
|
||||
struct bio_vec *bv;
|
||||
while (io->iter.bi_size) {
|
||||
u8 *page;
|
||||
unsigned len;
|
||||
struct bio_vec bv = bio_iter_iovec(bio, io->iter);
|
||||
|
||||
BUG_ON(vector >= io->io_vec_size);
|
||||
bv = &io->io_vec[vector];
|
||||
page = kmap_atomic(bv->bv_page);
|
||||
len = bv->bv_len - offset;
|
||||
if (likely(len >= todo))
|
||||
len = todo;
|
||||
r = crypto_shash_update(desc,
|
||||
page + bv->bv_offset + offset, len);
|
||||
page = kmap_atomic(bv.bv_page);
|
||||
r = crypto_shash_update(desc, page + bv.bv_offset,
|
||||
bv.bv_len);
|
||||
kunmap_atomic(page);
|
||||
|
||||
if (r < 0) {
|
||||
DMERR("crypto_shash_update failed: %d", r);
|
||||
return r;
|
||||
}
|
||||
offset += len;
|
||||
if (likely(offset == bv->bv_len)) {
|
||||
offset = 0;
|
||||
vector++;
|
||||
}
|
||||
todo -= len;
|
||||
} while (todo);
|
||||
|
||||
bio_advance_iter(bio, &io->iter, bv.bv_len);
|
||||
}
|
||||
|
||||
if (!v->version) {
|
||||
r = crypto_shash_update(desc, v->salt, v->salt_size);
|
||||
|
|
@ -383,8 +370,6 @@ test_block_hash:
|
|||
return -EIO;
|
||||
}
|
||||
}
|
||||
BUG_ON(vector != io->io_vec_size);
|
||||
BUG_ON(offset);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -400,9 +385,6 @@ static void verity_finish_io(struct dm_verity_io *io, int error)
|
|||
bio->bi_end_io = io->orig_bi_end_io;
|
||||
bio->bi_private = io->orig_bi_private;
|
||||
|
||||
if (io->io_vec != io->io_vec_inline)
|
||||
mempool_free(io->io_vec, v->vec_mempool);
|
||||
|
||||
bio_endio(bio, error);
|
||||
}
|
||||
|
||||
|
|
@ -519,13 +501,7 @@ static int verity_map(struct dm_target *ti, struct bio *bio)
|
|||
|
||||
bio->bi_end_io = verity_end_io;
|
||||
bio->bi_private = io;
|
||||
io->io_vec_size = bio_segments(bio);
|
||||
if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
|
||||
io->io_vec = io->io_vec_inline;
|
||||
else
|
||||
io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
|
||||
memcpy(io->io_vec, __bio_iovec(bio),
|
||||
io->io_vec_size * sizeof(struct bio_vec));
|
||||
io->iter = bio->bi_iter;
|
||||
|
||||
verity_submit_prefetch(v, io);
|
||||
|
||||
|
|
|
|||
14
fs/bio.c
14
fs/bio.c
|
|
@ -525,8 +525,17 @@ EXPORT_SYMBOL(bio_phys_segments);
|
|||
*/
|
||||
void __bio_clone(struct bio *bio, struct bio *bio_src)
|
||||
{
|
||||
memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
|
||||
bio_src->bi_max_vecs * sizeof(struct bio_vec));
|
||||
if (bio_is_rw(bio_src)) {
|
||||
struct bio_vec bv;
|
||||
struct bvec_iter iter;
|
||||
|
||||
bio_for_each_segment(bv, bio_src, iter)
|
||||
bio->bi_io_vec[bio->bi_vcnt++] = bv;
|
||||
} else if (bio_has_data(bio_src)) {
|
||||
memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
|
||||
bio_src->bi_max_vecs * sizeof(struct bio_vec));
|
||||
bio->bi_vcnt = bio_src->bi_vcnt;
|
||||
}
|
||||
|
||||
/*
|
||||
* most users will be overriding ->bi_bdev with a new target,
|
||||
|
|
@ -535,7 +544,6 @@ void __bio_clone(struct bio *bio, struct bio *bio_src)
|
|||
bio->bi_bdev = bio_src->bi_bdev;
|
||||
bio->bi_flags |= 1 << BIO_CLONED;
|
||||
bio->bi_rw = bio_src->bi_rw;
|
||||
bio->bi_vcnt = bio_src->bi_vcnt;
|
||||
bio->bi_iter = bio_src->bi_iter;
|
||||
}
|
||||
EXPORT_SYMBOL(__bio_clone);
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ typedef void (*io_notify_fn)(unsigned long error, void *context);
|
|||
|
||||
enum dm_io_mem_type {
|
||||
DM_IO_PAGE_LIST,/* Page list */
|
||||
DM_IO_BVEC, /* Bio vector */
|
||||
DM_IO_BIO, /* Bio vector */
|
||||
DM_IO_VMA, /* Virtual memory area */
|
||||
DM_IO_KMEM, /* Kernel memory */
|
||||
};
|
||||
|
|
@ -41,7 +41,7 @@ struct dm_io_memory {
|
|||
|
||||
union {
|
||||
struct page_list *pl;
|
||||
struct bio_vec *bvec;
|
||||
struct bio *bio;
|
||||
void *vma;
|
||||
void *addr;
|
||||
} ptr;
|
||||
|
|
|
|||
Loading…
Reference in New Issue