Gh0st
/
linux
mirror of https://github.com/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

block: use {alloc|free}_sched data methods 

The previous patch introduced ->alloc_sched_data and
->free_sched_data methods. This patch builds upon that
by now using these methods during elevator switch and
nr_hw_queue update.

It's also ensured that scheduler-specific data is
allocated and freed through the new callbacks outside
of the ->freeze_lock and ->elevator_lock locking contexts,
thereby preventing any dependency on pcpu_alloc_mutex.

Reviewed-by: Ming Lei <ming.lei@redhat.com>
Reviewed-by: Yu Kuai <yukuai@fnnas.com>
Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Nilay Shroff 2025-11-13 14:28:21 +05:30 committed by Jens Axboe
parent 61019afdf6
commit 0315476e78
4 changed files with 50 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
block/blk-mq-sched.c
View File

@ -428,12 +428,17 @@ void blk_mq_free_sched_tags(struct elevator_tags *et,
} }
void blk_mq_free_sched_res(struct elevator_resources *res, void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set) struct blk_mq_tag_set *set)
{ {
if (res->et) { if (res->et) {
blk_mq_free_sched_tags(res->et, set); blk_mq_free_sched_tags(res->et, set);
res->et = NULL; res->et = NULL;
} }
if (res->data) {
blk_mq_free_sched_data(type, res->data);
res->data = NULL;
}
} }
void blk_mq_free_sched_res_batch(struct xarray *elv_tbl, void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
@ -458,7 +463,7 @@ void blk_mq_free_sched_res_batch(struct xarray *elv_tbl,
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
continue; continue;
} }
blk_mq_free_sched_res(&ctx->res, set); blk_mq_free_sched_res(&ctx->res, ctx->type, set);
} }
} }
} }
@ -540,7 +545,9 @@ struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
} }
int blk_mq_alloc_sched_res(struct request_queue *q, int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_resources *res, unsigned int nr_hw_queues) struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues)
{ {
struct blk_mq_tag_set *set = q->tag_set; struct blk_mq_tag_set *set = q->tag_set;
@ -549,6 +556,12 @@ int blk_mq_alloc_sched_res(struct request_queue *q,
if (!res->et) if (!res->et)
return -ENOMEM; return -ENOMEM;
res->data = blk_mq_alloc_sched_data(q, type);
if (IS_ERR(res->data)) {
blk_mq_free_sched_tags(res->et, set);
return -ENOMEM;
}
return 0; return 0;
} }
@ -576,19 +589,21 @@ int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
goto out_unwind; goto out_unwind;
} }
ret = blk_mq_alloc_sched_res(q, &ctx->res, ret = blk_mq_alloc_sched_res(q, q->elevator->type,
nr_hw_queues); &ctx->res, nr_hw_queues);
if (ret) if (ret)
goto out_unwind; goto out_unwind;
} }
} }
return 0; return 0;
out_unwind: out_unwind:
list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) { list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) {
if (q->elevator) { if (q->elevator) {
ctx = xa_load(elv_tbl, q->id); ctx = xa_load(elv_tbl, q->id);
if (ctx) if (ctx)
blk_mq_free_sched_res(&ctx->res, set); blk_mq_free_sched_res(&ctx->res,
ctx->type, set);
} }
} }
return ret; return ret;
@ -605,7 +620,7 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e,
unsigned long i; unsigned long i;
int ret; int ret;
eq = elevator_alloc(q, e, et); eq = elevator_alloc(q, e, res);
if (!eq) if (!eq)
return -ENOMEM; return -ENOMEM;

5
block/blk-mq-sched.h
View File

@ -26,7 +26,9 @@ void blk_mq_sched_free_rqs(struct request_queue *q);
struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set,
unsigned int nr_hw_queues, unsigned int nr_requests); unsigned int nr_hw_queues, unsigned int nr_requests);
int blk_mq_alloc_sched_res(struct request_queue *q, int blk_mq_alloc_sched_res(struct request_queue *q,
struct elevator_resources *res, unsigned int nr_hw_queues); struct elevator_type *type,
struct elevator_resources *res,
unsigned int nr_hw_queues);
int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl, int blk_mq_alloc_sched_res_batch(struct xarray *elv_tbl,
struct blk_mq_tag_set *set, unsigned int nr_hw_queues); struct blk_mq_tag_set *set, unsigned int nr_hw_queues);
int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl, int blk_mq_alloc_sched_ctx_batch(struct xarray *elv_tbl,
@ -35,6 +37,7 @@ void blk_mq_free_sched_ctx_batch(struct xarray *elv_tbl);
void blk_mq_free_sched_tags(struct elevator_tags *et, void blk_mq_free_sched_tags(struct elevator_tags *et,
struct blk_mq_tag_set *set); struct blk_mq_tag_set *set);
void blk_mq_free_sched_res(struct elevator_resources *res, void blk_mq_free_sched_res(struct elevator_resources *res,
struct elevator_type *type,
struct blk_mq_tag_set *set); struct blk_mq_tag_set *set);
void blk_mq_free_sched_res_batch(struct xarray *et_table, void blk_mq_free_sched_res_batch(struct xarray *et_table,
struct blk_mq_tag_set *set); struct blk_mq_tag_set *set);

34
block/elevator.c
View File

@ -121,7 +121,7 @@ static struct elevator_type *elevator_find_get(const char *name)
static const struct kobj_type elv_ktype; static const struct kobj_type elv_ktype;
struct elevator_queue *elevator_alloc(struct request_queue *q, struct elevator_queue *elevator_alloc(struct request_queue *q,
struct elevator_type *e, struct elevator_tags *et) struct elevator_type *e, struct elevator_resources *res)
{ {
struct elevator_queue *eq; struct elevator_queue *eq;
@ -134,7 +134,8 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
kobject_init(&eq->kobj, &elv_ktype); kobject_init(&eq->kobj, &elv_ktype);
mutex_init(&eq->sysfs_lock); mutex_init(&eq->sysfs_lock);
hash_init(eq->hash); hash_init(eq->hash);
eq->et = et; eq->et = res->et;
eq->elevator_data = res->data;
return eq; return eq;
} }
@ -617,7 +618,7 @@ static void elv_exit_and_release(struct elv_change_ctx *ctx,
mutex_unlock(&q->elevator_lock); mutex_unlock(&q->elevator_lock);
blk_mq_unfreeze_queue(q, memflags); blk_mq_unfreeze_queue(q, memflags);
if (e) { if (e) {
blk_mq_free_sched_res(&ctx->res, q->tag_set); blk_mq_free_sched_res(&ctx->res, ctx->type, q->tag_set);
kobject_put(&e->kobj); kobject_put(&e->kobj);
} }
} }
@ -628,12 +629,15 @@ static int elevator_change_done(struct request_queue *q,
int ret = 0; int ret = 0;
if (ctx->old) { if (ctx->old) {
struct elevator_resources res = {.et = ctx->old->et}; struct elevator_resources res = {
.et = ctx->old->et,
.data = ctx->old->elevator_data
};
bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT, bool enable_wbt = test_bit(ELEVATOR_FLAG_ENABLE_WBT_ON_EXIT,
&ctx->old->flags); &ctx->old->flags);
elv_unregister_queue(q, ctx->old); elv_unregister_queue(q, ctx->old);
blk_mq_free_sched_res(&res, q->tag_set); blk_mq_free_sched_res(&res, ctx->old->type, q->tag_set);
kobject_put(&ctx->old->kobj); kobject_put(&ctx->old->kobj);
if (enable_wbt) if (enable_wbt)
wbt_enable_default(q->disk); wbt_enable_default(q->disk);
@ -658,7 +662,8 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
lockdep_assert_held(&set->update_nr_hwq_lock); lockdep_assert_held(&set->update_nr_hwq_lock);
if (strncmp(ctx->name, "none", 4)) { if (strncmp(ctx->name, "none", 4)) {
ret = blk_mq_alloc_sched_res(q, &ctx->res, set->nr_hw_queues); ret = blk_mq_alloc_sched_res(q, ctx->type, &ctx->res,
set->nr_hw_queues);
if (ret) if (ret)
return ret; return ret;
} }
@ -681,11 +686,12 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx)
blk_mq_unfreeze_queue(q, memflags); blk_mq_unfreeze_queue(q, memflags);
if (!ret) if (!ret)
ret = elevator_change_done(q, ctx); ret = elevator_change_done(q, ctx);
/* /*
* Free sched resource if it's allocated but we couldn't switch elevator. * Free sched resource if it's allocated but we couldn't switch elevator.
*/ */
if (!ctx->new) if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, set); blk_mq_free_sched_res(&ctx->res, ctx->type, set);
return ret; return ret;
} }
@ -711,11 +717,12 @@ void elv_update_nr_hw_queues(struct request_queue *q,
blk_mq_unfreeze_queue_nomemrestore(q); blk_mq_unfreeze_queue_nomemrestore(q);
if (!ret) if (!ret)
WARN_ON_ONCE(elevator_change_done(q, ctx)); WARN_ON_ONCE(elevator_change_done(q, ctx));
/* /*
* Free sched resource if it's allocated but we couldn't switch elevator. * Free sched resource if it's allocated but we couldn't switch elevator.
*/ */
if (!ctx->new) if (!ctx->new)
blk_mq_free_sched_res(&ctx->res, set); blk_mq_free_sched_res(&ctx->res, ctx->type, set);
} }
/* /*
@ -729,7 +736,6 @@ void elevator_set_default(struct request_queue *q)
.no_uevent = true, .no_uevent = true,
}; };
int err; int err;
struct elevator_type *e;
/* now we allow to switch elevator */ /* now we allow to switch elevator */
blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q); blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q);
@ -742,8 +748,8 @@ void elevator_set_default(struct request_queue *q)
* have multiple queues or mq-deadline is not available, default * have multiple queues or mq-deadline is not available, default
* to "none". * to "none".
*/ */
e = elevator_find_get(ctx.name); ctx.type = elevator_find_get(ctx.name);
if (!e) if (!ctx.type)
return; return;
if ((q->nr_hw_queues == 1 || if ((q->nr_hw_queues == 1 ||
@ -753,7 +759,7 @@ void elevator_set_default(struct request_queue *q)
pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n", pr_warn("\"%s\" elevator initialization, failed %d, falling back to \"none\"\n",
ctx.name, err); ctx.name, err);
} }
elevator_put(e); elevator_put(ctx.type);
} }
void elevator_set_none(struct request_queue *q) void elevator_set_none(struct request_queue *q)
@ -802,6 +808,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ctx.name = strstrip(elevator_name); ctx.name = strstrip(elevator_name);
elv_iosched_load_module(ctx.name); elv_iosched_load_module(ctx.name);
ctx.type = elevator_find_get(ctx.name);
down_read(&set->update_nr_hwq_lock); down_read(&set->update_nr_hwq_lock);
if (!blk_queue_no_elv_switch(q)) { if (!blk_queue_no_elv_switch(q)) {
@ -812,6 +819,9 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
ret = -ENOENT; ret = -ENOENT;
} }
up_read(&set->update_nr_hwq_lock); up_read(&set->update_nr_hwq_lock);
if (ctx.type)
elevator_put(ctx.type);
return ret; return ret;
} }

4
block/elevator.h
View File

@ -33,6 +33,8 @@ struct elevator_tags {
}; };
struct elevator_resources { struct elevator_resources {
/* holds elevator data */
void *data;
/* holds elevator tags */ /* holds elevator tags */
struct elevator_tags *et; struct elevator_tags *et;
}; };
@ -185,7 +187,7 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count);
extern bool elv_bio_merge_ok(struct request *, struct bio *); extern bool elv_bio_merge_ok(struct request *, struct bio *);
struct elevator_queue *elevator_alloc(struct request_queue *, struct elevator_queue *elevator_alloc(struct request_queue *,
struct elevator_type *, struct elevator_tags *); struct elevator_type *, struct elevator_resources *);
/* /*
* Helper functions. * Helper functions.