UbiquitousOS
/
OpenCloudOS-Kernel
mirror of https://gitee.com/OpenCloudOS/OpenCloudOS-Kernel.git
11
0
Fork 0
Code Issues Projects Releases Wiki Activity

block: pass struct queue_limits to the bio splitting helpers 

Allow using the splitting helpers on just a queue_limits instead of
a full request_queue structure.  This will eventually allow file systems
or remapping drivers to split REQ_OP_ZONE_APPEND bios based on limits
calculated as the minimum common capabilities over multiple devices.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <damien.lemoal@opensource.wdc.com>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Link: https://lore.kernel.org/r/20220727162300.3089193-7-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Christoph Hellwig 2022-07-27 12:23:00 -04:00 committed by Jens Axboe
parent b6dc6198eb
commit c55ddd9082
5 changed files with 68 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
block/bio-integrity.c
View File

@ -134,7 +134,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
iv = bip->bip_vec + bip->bip_vcnt; iv = bip->bip_vec + bip->bip_vcnt;
if (bip->bip_vcnt && if (bip->bip_vcnt &&
bvec_gap_to_prev(bdev_get_queue(bio->bi_bdev), bvec_gap_to_prev(&bdev_get_queue(bio->bi_bdev)->limits,
&bip->bip_vec[bip->bip_vcnt - 1], offset)) &bip->bip_vec[bip->bip_vcnt - 1], offset))
return 0; return 0;

2
block/bio.c
View File

@ -965,7 +965,7 @@ int bio_add_hw_page(struct request_queue *q, struct bio *bio,
* would create a gap, disallow it. * would create a gap, disallow it.
*/ */
bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
if (bvec_gap_to_prev(q, bvec, offset)) if (bvec_gap_to_prev(&q->limits, bvec, offset))
return 0; return 0;
} }

107
block/blk-merge.c
View File

@ -82,7 +82,7 @@ static inline bool bio_will_gap(struct request_queue *q,
bio_get_first_bvec(next, &nb); bio_get_first_bvec(next, &nb);
if (biovec_phys_mergeable(q, &pb, &nb)) if (biovec_phys_mergeable(q, &pb, &nb))
return false; return false;
return __bvec_gap_to_prev(q, &pb, nb.bv_offset); return __bvec_gap_to_prev(&q->limits, &pb, nb.bv_offset);
} }
static inline bool req_gap_back_merge(struct request *req, struct bio *bio) static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
@ -100,26 +100,25 @@ static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
* is defined as 'unsigned int', meantime it has to be aligned to with the * is defined as 'unsigned int', meantime it has to be aligned to with the
* logical block size, which is the minimum accepted unit by hardware. * logical block size, which is the minimum accepted unit by hardware.
*/ */
static unsigned int bio_allowed_max_sectors(struct request_queue *q) static unsigned int bio_allowed_max_sectors(struct queue_limits *lim)
{ {
return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9; return round_down(UINT_MAX, lim->logical_block_size) >> SECTOR_SHIFT;
} }
static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q, static struct bio *bio_split_discard(struct bio *bio, struct queue_limits *lim,
unsigned *nsegs, struct bio_set *bs) unsigned *nsegs, struct bio_set *bs)
{ {
unsigned int max_discard_sectors, granularity; unsigned int max_discard_sectors, granularity;
int alignment;
sector_t tmp; sector_t tmp;
unsigned split_sectors; unsigned split_sectors;
*nsegs = 1; *nsegs = 1;
/* Zero-sector (unknown) and one-sector granularities are the same. */ /* Zero-sector (unknown) and one-sector granularities are the same. */
granularity = max(q->limits.discard_granularity >> 9, 1U); granularity = max(lim->discard_granularity >> 9, 1U);
max_discard_sectors = min(q->limits.max_discard_sectors, max_discard_sectors =
bio_allowed_max_sectors(q)); min(lim->max_discard_sectors, bio_allowed_max_sectors(lim));
max_discard_sectors -= max_discard_sectors % granularity; max_discard_sectors -= max_discard_sectors % granularity;
if (unlikely(!max_discard_sectors)) { if (unlikely(!max_discard_sectors)) {
@ -136,9 +135,8 @@ static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
* If the next starting sector would be misaligned, stop the discard at * If the next starting sector would be misaligned, stop the discard at
* the previous aligned sector. * the previous aligned sector.
*/ */
alignment = (q->limits.discard_alignment >> 9) % granularity; tmp = bio->bi_iter.bi_sector + split_sectors -
((lim->discard_alignment >> 9) % granularity);
tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
tmp = sector_div(tmp, granularity); tmp = sector_div(tmp, granularity);
if (split_sectors > tmp) if (split_sectors > tmp)
@ -148,17 +146,14 @@ static struct bio *bio_split_discard(struct bio *bio, struct request_queue *q,
} }
static struct bio *bio_split_write_zeroes(struct bio *bio, static struct bio *bio_split_write_zeroes(struct bio *bio,
struct request_queue *q, unsigned *nsegs, struct bio_set *bs) struct queue_limits *lim, unsigned *nsegs, struct bio_set *bs)
{ {
*nsegs = 0; *nsegs = 0;
if (!lim->max_write_zeroes_sectors)
if (!q->limits.max_write_zeroes_sectors)
return NULL; return NULL;
if (bio_sectors(bio) <= lim->max_write_zeroes_sectors)
if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)
return NULL; return NULL;
return bio_split(bio, lim->max_write_zeroes_sectors, GFP_NOIO, bs);
return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);
} }
/* /*
@ -170,16 +165,16 @@ static struct bio *bio_split_write_zeroes(struct bio *bio,
* aligned to a physical block boundary. * aligned to a physical block boundary.
*/ */
static inline unsigned get_max_io_size(struct bio *bio, static inline unsigned get_max_io_size(struct bio *bio,
struct request_queue *q) struct queue_limits *lim)
{ {
unsigned pbs = queue_physical_block_size(q) >> SECTOR_SHIFT; unsigned pbs = lim->physical_block_size >> SECTOR_SHIFT;
unsigned lbs = queue_logical_block_size(q) >> SECTOR_SHIFT; unsigned lbs = lim->logical_block_size >> SECTOR_SHIFT;
unsigned max_sectors = queue_max_sectors(q), start, end; unsigned max_sectors = lim->max_sectors, start, end;
if (q->limits.chunk_sectors) { if (lim->chunk_sectors) {
max_sectors = min(max_sectors, max_sectors = min(max_sectors,
blk_chunk_sectors_left(bio->bi_iter.bi_sector, blk_chunk_sectors_left(bio->bi_iter.bi_sector,
q->limits.chunk_sectors)); lim->chunk_sectors));
} }
start = bio->bi_iter.bi_sector & (pbs - 1); start = bio->bi_iter.bi_sector & (pbs - 1);
@ -189,11 +184,10 @@ static inline unsigned get_max_io_size(struct bio *bio,
return max_sectors & ~(lbs - 1); return max_sectors & ~(lbs - 1);
} }
static inline unsigned get_max_segment_size(const struct request_queue *q, static inline unsigned get_max_segment_size(struct queue_limits *lim,
struct page *start_page, struct page *start_page, unsigned long offset)
unsigned long offset)
{ {
unsigned long mask = queue_segment_boundary(q); unsigned long mask = lim->seg_boundary_mask;
offset = mask & (page_to_phys(start_page) + offset); offset = mask & (page_to_phys(start_page) + offset);
@ -202,12 +196,12 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,
* on 32bit arch, use queue's max segment size when that happens. * on 32bit arch, use queue's max segment size when that happens.
*/ */
return min_not_zero(mask - offset + 1, return min_not_zero(mask - offset + 1,
(unsigned long)queue_max_segment_size(q)); (unsigned long)lim->max_segment_size);
} }
/** /**
* bvec_split_segs - verify whether or not a bvec should be split in the middle * bvec_split_segs - verify whether or not a bvec should be split in the middle
* @q: [in] request queue associated with the bio associated with @bv * @lim: [in] queue limits to split based on
* @bv: [in] bvec to examine * @bv: [in] bvec to examine
* @nsegs: [in,out] Number of segments in the bio being built. Incremented * @nsegs: [in,out] Number of segments in the bio being built. Incremented
* by the number of segments from @bv that may be appended to that * by the number of segments from @bv that may be appended to that
@ -225,10 +219,9 @@ static inline unsigned get_max_segment_size(const struct request_queue *q,
* *@nsegs segments and *@sectors sectors would make that bio unacceptable for * *@nsegs segments and *@sectors sectors would make that bio unacceptable for
* the block driver. * the block driver.
*/ */
static bool bvec_split_segs(const struct request_queue *q, static bool bvec_split_segs(struct queue_limits *lim, const struct bio_vec *bv,
const struct bio_vec *bv, unsigned *nsegs, unsigned *nsegs, unsigned *bytes, unsigned max_segs,
unsigned *bytes, unsigned max_segs, unsigned max_bytes)
unsigned max_bytes)
{ {
unsigned max_len = min(max_bytes, UINT_MAX) - *bytes; unsigned max_len = min(max_bytes, UINT_MAX) - *bytes;
unsigned len = min(bv->bv_len, max_len); unsigned len = min(bv->bv_len, max_len);
@ -236,7 +229,7 @@ static bool bvec_split_segs(const struct request_queue *q,
unsigned seg_size = 0; unsigned seg_size = 0;
while (len && *nsegs < max_segs) { while (len && *nsegs < max_segs) {
seg_size = get_max_segment_size(q, bv->bv_page, seg_size = get_max_segment_size(lim, bv->bv_page,
bv->bv_offset + total_len); bv->bv_offset + total_len);
seg_size = min(seg_size, len); seg_size = min(seg_size, len);
@ -244,7 +237,7 @@ static bool bvec_split_segs(const struct request_queue *q,
total_len += seg_size; total_len += seg_size;
len -= seg_size; len -= seg_size;
if ((bv->bv_offset + total_len) & queue_virt_boundary(q)) if ((bv->bv_offset + total_len) & lim->virt_boundary_mask)
break; break;
} }
@ -257,7 +250,7 @@ static bool bvec_split_segs(const struct request_queue *q,
/** /**
* bio_split_rw - split a bio in two bios * bio_split_rw - split a bio in two bios
* @bio: [in] bio to be split * @bio: [in] bio to be split
* @q: [in] request queue pointer * @lim: [in] queue limits to split based on
* @segs: [out] number of segments in the bio with the first half of the sectors * @segs: [out] number of segments in the bio with the first half of the sectors
* @bs: [in] bio set to allocate the clone from * @bs: [in] bio set to allocate the clone from
* @max_bytes: [in] maximum number of bytes per bio * @max_bytes: [in] maximum number of bytes per bio
@ -274,30 +267,30 @@ static bool bvec_split_segs(const struct request_queue *q,
* responsible for ensuring that @bs is only destroyed after processing of the * responsible for ensuring that @bs is only destroyed after processing of the
* split bio has finished. * split bio has finished.
*/ */
static struct bio *bio_split_rw(struct bio *bio, struct request_queue *q, static struct bio *bio_split_rw(struct bio *bio, struct queue_limits *lim,
unsigned *segs, struct bio_set *bs, unsigned max_bytes) unsigned *segs, struct bio_set *bs, unsigned max_bytes)
{ {
struct bio_vec bv, bvprv, *bvprvp = NULL; struct bio_vec bv, bvprv, *bvprvp = NULL;
struct bvec_iter iter; struct bvec_iter iter;
unsigned nsegs = 0, bytes = 0; unsigned nsegs = 0, bytes = 0;
const unsigned max_segs = queue_max_segments(q);
bio_for_each_bvec(bv, bio, iter) { bio_for_each_bvec(bv, bio, iter) {
/* /*
* If the queue doesn't support SG gaps and adding this * If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it. * offset would create a gap, disallow it.
*/ */
if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset))
goto split; goto split;
if (nsegs < max_segs && if (nsegs < lim->max_segments &&
bytes + bv.bv_len <= max_bytes && bytes + bv.bv_len <= max_bytes &&
bv.bv_offset + bv.bv_len <= PAGE_SIZE) { bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
nsegs++; nsegs++;
bytes += bv.bv_len; bytes += bv.bv_len;
} else if (bvec_split_segs(q, &bv, &nsegs, &bytes, max_segs, } else {
max_bytes)) { if (bvec_split_segs(lim, &bv, &nsegs, &bytes,
goto split; lim->max_segments, max_bytes))
goto split;
} }
bvprv = bv; bvprv = bv;
@ -314,7 +307,7 @@ split:
* split size so that each bio is properly block size aligned, even if * split size so that each bio is properly block size aligned, even if
* we do not use the full hardware limits. * we do not use the full hardware limits.
*/ */
bytes = ALIGN_DOWN(bytes, queue_logical_block_size(q)); bytes = ALIGN_DOWN(bytes, lim->logical_block_size);
/* /*
* Bio splitting may cause subtle trouble such as hang when doing sync * Bio splitting may cause subtle trouble such as hang when doing sync
@ -328,7 +321,7 @@ split:
/** /**
* __bio_split_to_limits - split a bio to fit the queue limits * __bio_split_to_limits - split a bio to fit the queue limits
* @bio: bio to be split * @bio: bio to be split
* @q: request_queue new bio is being queued at * @lim: queue limits to split based on
* @nr_segs: returns the number of segments in the returned bio * @nr_segs: returns the number of segments in the returned bio
* *
* Check if @bio needs splitting based on the queue limits, and if so split off * Check if @bio needs splitting based on the queue limits, and if so split off
@ -338,7 +331,7 @@ split:
* The split bio is allocated from @q->bio_split, which is provided by the * The split bio is allocated from @q->bio_split, which is provided by the
* block layer. * block layer.
*/ */
struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q, struct bio *__bio_split_to_limits(struct bio *bio, struct queue_limits *lim,
unsigned int *nr_segs) unsigned int *nr_segs)
{ {
struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split; struct bio_set *bs = &bio->bi_bdev->bd_disk->bio_split;
@ -347,14 +340,14 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
switch (bio_op(bio)) { switch (bio_op(bio)) {
case REQ_OP_DISCARD: case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE: case REQ_OP_SECURE_ERASE:
split = bio_split_discard(bio, q, nr_segs, bs); split = bio_split_discard(bio, lim, nr_segs, bs);
break; break;
case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE_ZEROES:
split = bio_split_write_zeroes(bio, q, nr_segs, bs); split = bio_split_write_zeroes(bio, lim, nr_segs, bs);
break; break;
default: default:
split = bio_split_rw(bio, q, nr_segs, bs, split = bio_split_rw(bio, lim, nr_segs, bs,
get_max_io_size(bio, q) << SECTOR_SHIFT); get_max_io_size(bio, lim) << SECTOR_SHIFT);
break; break;
} }
@ -384,11 +377,11 @@ struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q,
*/ */
struct bio *bio_split_to_limits(struct bio *bio) struct bio *bio_split_to_limits(struct bio *bio)
{ {
struct request_queue *q = bdev_get_queue(bio->bi_bdev); struct queue_limits *lim = &bdev_get_queue(bio->bi_bdev)->limits;
unsigned int nr_segs; unsigned int nr_segs;
if (bio_may_exceed_limits(bio, q)) if (bio_may_exceed_limits(bio, lim))
return __bio_split_to_limits(bio, q, &nr_segs); return __bio_split_to_limits(bio, lim, &nr_segs);
return bio; return bio;
} }
EXPORT_SYMBOL(bio_split_to_limits); EXPORT_SYMBOL(bio_split_to_limits);
@ -421,7 +414,7 @@ unsigned int blk_recalc_rq_segments(struct request *rq)
} }
rq_for_each_bvec(bv, rq, iter) rq_for_each_bvec(bv, rq, iter)
bvec_split_segs(rq->q, &bv, &nr_phys_segs, &bytes, bvec_split_segs(&rq->q->limits, &bv, &nr_phys_segs, &bytes,
UINT_MAX, UINT_MAX); UINT_MAX, UINT_MAX);
return nr_phys_segs; return nr_phys_segs;
} }
@ -452,8 +445,8 @@ static unsigned blk_bvec_map_sg(struct request_queue *q,
while (nbytes > 0) { while (nbytes > 0) {
unsigned offset = bvec->bv_offset + total; unsigned offset = bvec->bv_offset + total;
unsigned len = min(get_max_segment_size(q, bvec->bv_page, unsigned len = min(get_max_segment_size(&q->limits,
offset), nbytes); bvec->bv_page, offset), nbytes);
struct page *page = bvec->bv_page; struct page *page = bvec->bv_page;
/* /*

4
block/blk-mq.c
View File

@ -2816,8 +2816,8 @@ void blk_mq_submit_bio(struct bio *bio)
blk_status_t ret; blk_status_t ret;
bio = blk_queue_bounce(bio, q); bio = blk_queue_bounce(bio, q);
if (bio_may_exceed_limits(bio, q)) if (bio_may_exceed_limits(bio, &q->limits))
bio = __bio_split_to_limits(bio, q, &nr_segs); bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
if (!bio_integrity_prep(bio)) if (!bio_integrity_prep(bio))
return; return;

25
block/blk.h
View File

@ -97,23 +97,23 @@ static inline bool biovec_phys_mergeable(struct request_queue *q,
return true; return true;
} }
static inline bool __bvec_gap_to_prev(struct request_queue *q, static inline bool __bvec_gap_to_prev(struct queue_limits *lim,
struct bio_vec *bprv, unsigned int offset) struct bio_vec *bprv, unsigned int offset)
{ {
return (offset & queue_virt_boundary(q)) || return (offset & lim->virt_boundary_mask) ||
((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); ((bprv->bv_offset + bprv->bv_len) & lim->virt_boundary_mask);
} }
/* /*
* Check if adding a bio_vec after bprv with offset would create a gap in * Check if adding a bio_vec after bprv with offset would create a gap in
* the SG list. Most drivers don't care about this, but some do. * the SG list. Most drivers don't care about this, but some do.
*/ */
static inline bool bvec_gap_to_prev(struct request_queue *q, static inline bool bvec_gap_to_prev(struct queue_limits *lim,
struct bio_vec *bprv, unsigned int offset) struct bio_vec *bprv, unsigned int offset)
{ {
if (!queue_virt_boundary(q)) if (!lim->virt_boundary_mask)
return false; return false;
return __bvec_gap_to_prev(q, bprv, offset); return __bvec_gap_to_prev(lim, bprv, offset);
} }
static inline bool rq_mergeable(struct request *rq) static inline bool rq_mergeable(struct request *rq)
@ -189,7 +189,8 @@ static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio_integrity_payload *bip = bio_integrity(req->bio); struct bio_integrity_payload *bip = bio_integrity(req->bio);
struct bio_integrity_payload *bip_next = bio_integrity(next); struct bio_integrity_payload *bip_next = bio_integrity(next);
return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], return bvec_gap_to_prev(&req->q->limits,
&bip->bip_vec[bip->bip_vcnt - 1],
bip_next->bip_vec[0].bv_offset); bip_next->bip_vec[0].bv_offset);
} }
@ -199,7 +200,8 @@ static inline bool integrity_req_gap_front_merge(struct request *req,
struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_integrity_payload *bip = bio_integrity(bio);
struct bio_integrity_payload *bip_next = bio_integrity(req->bio); struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], return bvec_gap_to_prev(&req->q->limits,
&bip->bip_vec[bip->bip_vcnt - 1],
bip_next->bip_vec[0].bv_offset); bip_next->bip_vec[0].bv_offset);
} }
@ -288,7 +290,8 @@ ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
ssize_t part_timeout_store(struct device *, struct device_attribute *, ssize_t part_timeout_store(struct device *, struct device_attribute *,
const char *, size_t); const char *, size_t);
static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *q) static inline bool bio_may_exceed_limits(struct bio *bio,
struct queue_limits *lim)
{ {
switch (bio_op(bio)) { switch (bio_op(bio)) {
case REQ_OP_DISCARD: case REQ_OP_DISCARD:
@ -307,11 +310,11 @@ static inline bool bio_may_exceed_limits(struct bio *bio, struct request_queue *
* to the performance impact of cloned bios themselves the loop below * to the performance impact of cloned bios themselves the loop below
* doesn't matter anyway. * doesn't matter anyway.
*/ */
return q->limits.chunk_sectors || bio->bi_vcnt != 1 || return lim->chunk_sectors || bio->bi_vcnt != 1 ||
bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE; bio->bi_io_vec->bv_len + bio->bi_io_vec->bv_offset > PAGE_SIZE;
} }
struct bio *__bio_split_to_limits(struct bio *bio, struct request_queue *q, struct bio *__bio_split_to_limits(struct bio *bio, struct queue_limits *lim,
unsigned int *nr_segs); unsigned int *nr_segs);
int ll_back_merge_fn(struct request *req, struct bio *bio, int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs); unsigned int nr_segs);