Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

block: use {alloc|free}_sched data methods 

The previous patch introduced ->alloc_sched_data and
->free_sched_data methods. This patch builds upon that
by now using these methods during elevator switch and
nr_hw_queue update.

It's also ensured that scheduler-specific data is
allocated and freed through the new callbacks outside
of the ->freeze_lock and ->elevator_lock locking contexts,
thereby preventing any dependency on pcpu_alloc_mutex.

Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Yu Kuai <yukuai@fnnas.com>
Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Nilay Shroff 2025-11-13 14:28:21 +05:30 committed by Jens Axboe
parent 61019afdf6
commit 0315476e78
4 changed files with 50 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
block/blk-mq-sched.c
View File

@ -428,12 +428,17 @@ void blk_mq_free_sched_tags(struct elevator_tags *et,
}
void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set)
{
if (res->et) {
blk_mq_free_sched_tags(res->et, set);
res->et = NULL;
}
if (res->data) {
blk_mq_free_sched_data(type, res->data);
res->data = NULL;
}
}
void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
@ -458,7 +463,7 @@ void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
WARN_ON_ONCE(1);
continue;
}
blk_mq_free_sched_res(&ctx->res, set);
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
}
}
}
@ -540,7 +545,9 @@ struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
}
int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_resources *res, unsigned int nr_hw_queues)
struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues)
{
struct blk_mq_tag_set *set = q->tag_set;
@ -549,6 +556,12 @@ int blk_mq_alloc_sched_res(struct request_queue *q,
if (!res->et)
return -ENOMEM;
res->data = blk_mq_alloc_sched_data(q, type);
if (IS_ERR(res->data)) {
blk_mq_free_sched_tags(res->et, set);
return -ENOMEM;
}
return 0;
}
@ -576,19 +589,21 @@ int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
goto out_unwind;
}
ret = blk_mq_alloc_sched_res(q, &ctx->res,
nr_hw_queues);
ret = blk_mq_alloc_sched_res(q, q->elevator->type,
&ctx->res, nr_hw_queues);
if (ret)
goto out_unwind;
}
}
return 0;
out_unwind:
list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) {
if (q->elevator) {
ctx = xa_load(elv_tbl, q->id);
if (ctx)
blk_mq_free_sched_res(&ctx->res, set);
blk_mq_free_sched_res(&ctx->res,
ctx->type, set);
}
}
return ret;
@ -605,7 +620,7 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
unsigned long i;
int ret;
eq = elevator_alloc(q, e, et);
eq = elevator_alloc(q, e, res);
if (!eq)
return -ENOMEM;

5
block/blk-mq-sched.h
View File

@ -26,7 +26,9 @@ void blk_mq_sched_free_rqs(struct request_queue *q);
struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
unsigned int nr_hw_queues, unsigned int nr_requests);
int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_resources *res, unsigned int nr_hw_queues);
struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues);
int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set, unsigned int nr_hw_queues);
int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl,
@ -35,6 +37,7 @@ void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl);
void blk_mq_free_sched_tags(struct elevator_tags *et,
struct blk_mq_tag_set *set);
void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set);
void blk_mq_free_sched_res_batch(struct xarray *et_table,
struct blk_mq_tag_set *set);

34
block/elevator.c
View File

@ -121,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name)
static const struct kobj_type elv_ktype;
struct elevator_queue *elevator_alloc(struct request_queue *q,
struct elevator_type *e, struct elevator_tags *et)
struct elevator_type *e, struct elevator_resources *res)
{
struct elevator_queue *eq;
@ -134,7 +134,8 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
kobject_init(&eq->kobj, &elv_ktype);
mutex_init(&eq->sysfs_lock);
hash_init(eq->hash);
eq->et = et;
eq->et = res->et;
eq->elevator_data = res->data;
return eq;
}
@ -617,7 +618,7 @@ static void elv_exit_and_release(struct elv_change_ctx *ctx,
mutex_unlock(&q->elevator_lock);
blk_mq_unfreeze_queue(q, memflags);
if (e) {
blk_mq_free_sched_res(&ctx->res, q->tag_set);
blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set);
kobject_put(&e->kobj);
}
}
@ -628,12 +629,15 @@ static int elevator_change_done(struct request_queue *q,
int ret = 0;
if (ctx->old) {
struct elevator_resources res = {.et = ctx->old->et};
struct elevator_resources res = {
.et = ctx->old->et,
.data = ctx->old->elevator_data
};
bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT,
&ctx->old->flags);
elv_unregister_queue(q, ctx->old);
blk_mq_free_sched_res(&res, q->tag_set);
blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set);
kobject_put(&ctx->old->kobj);
if (enable_wbt)
wbt_enable_default(q->disk);
@ -658,7 +662,8 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
lockdep_assert_held(&set->update_nr_hwq_lock);
if (strncmp(ctx->name, "none", 4)) {
ret = blk_mq_alloc_sched_res(q, &ctx->res, set->nr_hw_queues);
ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res,
set->nr_hw_queues);
if (ret)
return ret;
}
@ -681,11 +686,12 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
blk_mq_unfreeze_queue(q, memflags);
if (!ret)
ret = elevator_change_done(q, ctx);
/*
* Free sched resource if it's allocated but we couldn't switch elevator.
*/
if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, set);
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
return ret;
}
@ -711,11 +717,12 @@ void elv_update_nr_hw_queues(struct request_queue *q,
blk_mq_unfreeze_queue_nomemrestore(q);
if (!ret)
WARN_ON_ONCE(elevator_change_done(q, ctx));
/*
* Free sched resource if it's allocated but we couldn't switch elevator.
*/
if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, set);
blk_mq_free_sched_res(&ctx->res, ctx->type, set);
}
/*
@ -729,7 +736,6 @@ void elevator_set_default(struct request_queue *q)
.no_uevent = true,
};
int err;
struct elevator_type *e;
/* now we allow to switch elevator */
blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q);
@ -742,8 +748,8 @@ void elevator_set_default(struct request_queue *q)
* have multiple queues or mq-deadline is not available, default
* to "none".
*/
e = elevator_find_get(ctx.name);
if (!e)
ctx.type = elevator_find_get(ctx.name);
if (!ctx.type)
return;
if ((q->nr_hw_queues == 1 ||
@ -753,7 +759,7 @@ void elevator_set_default(struct request_queue *q)
pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n",
ctx.name, err);
}
elevator_put(e);
elevator_put(ctx.type);
}
void elevator_set_none(struct request_queue *q)
@ -802,6 +808,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ctx.name = strstrip(elevator_name);
elv_iosched_load_module(ctx.name);
ctx.type = elevator_find_get(ctx.name);
down_read(&set->update_nr_hwq_lock);
if (!blk_queue_no_elv_switch(q)) {
@ -812,6 +819,9 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ret = -ENOENT;
}
up_read(&set->update_nr_hwq_lock);
if (ctx.type)
elevator_put(ctx.type);
return ret;
}

4
block/elevator.h
View File

@ -33,6 +33,8 @@ struct elevator_tags {
};
struct elevator_resources {
/* holds elevator data */
void *data;
/* holds elevator tags */
struct elevator_tags *et;
};
@ -185,7 +187,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count);
extern bool elv_bio_merge_ok(struct request *, struct bio *);
struct elevator_queue *elevator_alloc(struct request_queue *,
struct elevator_type *, struct elevator_tags *);
struct elevator_type *, struct elevator_resources *);
/*
* Helper functions.