Skip to content

Commit

Permalink
block: use bi_max_vecs to find the bvec pool
Browse files Browse the repository at this point in the history 
Instead of encoding of the bvec pool using magic bio flags, just use
a helper to find the pool based on the max_vecs value.

Signed-off-by: Christoph Hellwig <[email protected]>
Signed-off-by: Jens Axboe <[email protected]>
  • Loading branch information
@axboe
Christoph Hellwig authored and axboe committed Feb 8, 2021
1 parent 72b0436 commit 7a800a2
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 100 deletions.
11 changes: 4 additions & 7 deletions block/bio-integrity.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs,
if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
if (bip->bip_vec)
bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
bip->bip_slab);
bip->bip_max_vcnt);
mempool_free(bip, &bs->bio_integrity_pool);
} else {
kfree(bip);
Expand Down Expand Up @@ -70,14 +70,11 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
memset(bip, 0, sizeof(*bip));

if (nr_vecs > inline_vecs) {
unsigned long idx = 0;

bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
&bs->bvec_integrity_pool);
bip->bip_max_vcnt = nr_vecs;
bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
&bip->bip_max_vcnt, gfp_mask);
if (!bip->bip_vec)
goto err;
bip->bip_max_vcnt = bvec_nr_vecs(idx);
bip->bip_slab = idx;
} else {
bip->bip_vec = bip->bip_inline_vecs;
bip->bip_max_vcnt = inline_vecs;
Expand Down
104 changes: 43 additions & 61 deletions block/bio.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,24 @@ static struct biovec_slab {
{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
};

static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
{
switch (nr_vecs) {
/* smaller bios use inline vecs */
case 5 ... 16:
return &bvec_slabs[0];
case 17 ... 64:
return &bvec_slabs[1];
case 65 ... 128:
return &bvec_slabs[2];
case 129 ... BIO_MAX_PAGES:
return &bvec_slabs[3];
default:
BUG();
return NULL;
}
}

/*
* fs_bio_set is the bio_set containing bio and iovec memory pools used by
* IO code that does not need private memory pools.
Expand Down Expand Up @@ -131,26 +149,14 @@ static void bio_put_slab(struct bio_set *bs)
mutex_unlock(&bio_slab_lock);
}

unsigned int bvec_nr_vecs(unsigned short idx)
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)
{
return bvec_slabs[--idx].nr_vecs;
}
BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);

void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
{
if (!idx)
return;
idx--;

BIO_BUG_ON(idx >= BVEC_POOL_NR);

if (idx == BVEC_POOL_MAX) {
if (nr_vecs == BIO_MAX_PAGES)
mempool_free(bv, pool);
} else {
struct biovec_slab *bvs = bvec_slabs + idx;

kmem_cache_free(bvs->slab, bv);
}
else if (nr_vecs > BIO_INLINE_VECS)
kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
}

/*
Expand All @@ -163,48 +169,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
}

struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
mempool_t *pool)
struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
gfp_t gfp_mask)
{
struct biovec_slab *bvs = biovec_slab(*nr_vecs);

if (WARN_ON_ONCE(!bvs))
return NULL;

/*
* see comment near bvec_array define!
* Upgrade the nr_vecs request to take full advantage of the allocation.
* We also rely on this in the bvec_free path.
*/
switch (nr) {
/* smaller bios use inline vecs */
case 5 ... 16:
*idx = 2;
break;
case 17 ... 64:
*idx = 3;
break;
case 65 ... 128:
*idx = 4;
break;
case 129 ... BIO_MAX_PAGES:
*idx = 5;
break;
default:
return NULL;
}
*nr_vecs = bvs->nr_vecs;

/*
* Try a slab allocation first for all smaller allocations. If that
* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
* The mempool is sized to handle up to BIO_MAX_PAGES entries.
*/
if (*idx < BVEC_POOL_MAX) {
struct biovec_slab *bvs = bvec_slabs + *idx;
if (*nr_vecs < BIO_MAX_PAGES) {
struct bio_vec *bvl;

bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) {
(*idx)++;
if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))
return bvl;
}
*idx = BVEC_POOL_MAX;
*nr_vecs = BIO_MAX_PAGES;
}

(*idx)++;
return mempool_alloc(pool, gfp_mask);
}

Expand All @@ -231,7 +223,7 @@ static void bio_free(struct bio *bio)
bio_uninit(bio);

if (bs) {
bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);

/*
* If we have front padding, adjust the bio pointer before freeing
Expand Down Expand Up @@ -275,12 +267,8 @@ EXPORT_SYMBOL(bio_init);
*/
void bio_reset(struct bio *bio)
{
unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);

bio_uninit(bio);

memset(bio, 0, BIO_RESET_BYTES);
bio->bi_flags = flags;
atomic_set(&bio->__bi_remaining, 1);
}
EXPORT_SYMBOL(bio_reset);
Expand Down Expand Up @@ -453,22 +441,18 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,

bio = p + bs->front_pad;
if (nr_iovecs > BIO_INLINE_VECS) {
unsigned long idx = 0;
struct bio_vec *bvl = NULL;

bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
if (!bvl && gfp_mask != saved_gfp) {
punt_bios_to_rescuer(bs);
gfp_mask = saved_gfp;
bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx,
&bs->bvec_pool);
bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
}

if (unlikely(!bvl))
goto err_free;

bio_init(bio, bvl, bvec_nr_vecs(idx));
bio->bi_flags |= idx << BVEC_POOL_OFFSET;
bio_init(bio, bvl, nr_iovecs);
} else if (nr_iovecs) {
bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
} else {
Expand Down Expand Up @@ -644,7 +628,7 @@ EXPORT_SYMBOL(bio_put);
*/
void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
{
BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);

/*
* most users will be overriding ->bi_bdev with a new target,
Expand Down Expand Up @@ -934,7 +918,7 @@ EXPORT_SYMBOL_GPL(bio_release_pages);

static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)
{
WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);
WARN_ON_ONCE(bio->bi_max_vecs);

bio->bi_vcnt = iter->nr_segs;
bio->bi_io_vec = (struct bio_vec *)iter->bvec;
Expand Down Expand Up @@ -1495,7 +1479,7 @@ EXPORT_SYMBOL_GPL(bio_trim);
*/
int biovec_init_pool(mempool_t *pool, int pool_entries)
{
struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;

return mempool_init_slab_pool(pool, pool_entries, bp->slab);
}
Expand Down Expand Up @@ -1605,8 +1589,6 @@ static int __init init_bio(void)
{
int i;

BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);

bio_integrity_init();

for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {
Expand Down
6 changes: 3 additions & 3 deletions block/blk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ void blk_free_flush_queue(struct blk_flush_queue *q);
void blk_freeze_queue(struct request_queue *q);

#define BIO_INLINE_VECS 4
struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
unsigned int bvec_nr_vecs(unsigned short idx);
struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,
gfp_t gfp_mask);
void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);

static inline bool biovec_phys_mergeable(struct request_queue *q,
struct bio_vec *vec1, struct bio_vec *vec2)
Expand Down
1 change: 0 additions & 1 deletion include/linux/bio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,6 @@ struct bio_integrity_payload {

struct bvec_iter bip_iter;

unsigned short bip_slab; /* slab the bip came from */
unsigned short bip_vcnt; /* # of integrity bio_vecs */
unsigned short bip_max_vcnt; /* integrity bio_vec slots */
unsigned short bip_flags; /* control flags */
Expand Down
29 changes: 1 addition & 28 deletions include/linux/blk_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ struct bio {
* top bits REQ_OP. Use
* accessors.
*/
unsigned short bi_flags; /* status, etc and bvec pool number */
unsigned short bi_flags; /* BIO_* below */
unsigned short bi_ioprio;
unsigned short bi_write_hint;
blk_status_t bi_status;
Expand Down Expand Up @@ -307,33 +307,6 @@ enum {
BIO_FLAG_LAST
};

/* See BVEC_POOL_OFFSET below before adding new flags */

/*
* We support 6 different bvec pools, the last one is magic in that it
* is backed by a mempool.
*/
#define BVEC_POOL_NR 6
#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)

/*
* Top 3 bits of bio flags indicate the pool the bvecs came from. We add
* 1 to the actual index so that 0 indicates that there are no bvecs to be
* freed.
*/
#define BVEC_POOL_BITS (3)
#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
# error "BVEC_POOL_BITS is too small"
#endif

/*
* Flags starting here get preserved by bio_reset() - this includes
* only BVEC_POOL_IDX()
*/
#define BIO_RESET_BITS BVEC_POOL_OFFSET

typedef __u32 __bitwise blk_mq_req_flags_t;

/*
Expand Down

0 comments on commit 7a800a2

Please sign in to comment.