Proxmox-Port/linux-loongson
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson synced 2025-08-27 15:36:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ublk: implement ->queue_rqs()

Browse Source 
Implement ->queue_rqs() for improving perf in case of MQ.

In this way, we just need to call io_uring_cmd_complete_in_task() once for
whole IO batch, then both io_uring and ublk server can get exact batch from
ublk frontend.

Follows IOPS improvement:

- tests

	tools/testing/selftests/ublk/kublk add -t null -q 2 [-z]

	fio/t/io_uring -p0 /dev/ublkb0

- results:

	more than 10% IOPS boost observed

Pass all ublk selftests, especially the io dispatch order test.

Cc: Uday Shankar <ushankar@purestorage.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/20250327095123.179113-9-ming.lei@redhat.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei 2025-03-27 17:51:17 +08:00 committed by Jens Axboe
parent 1797020916
commit d796cea7b9
1 changed files with 111 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
drivers/block/ublk_drv.c
View File

@ -81,6 +81,20 @@ struct ublk_rq_data {
};
struct ublk_uring_cmd_pdu {
/*
* Store requests in same batch temporarily for queuing them to
* daemon context.
*
* It should have been stored to request payload, but we do want
* to avoid extra pre-allocation, and uring_cmd payload is always
* free for us
*/
struct request *req_list;
/*
* The following two are valid in this cmd whole lifetime, and
* setup in ublk uring_cmd handler
*/
struct ublk_queue *ubq;
u16 tag;
};
@ -1170,14 +1184,12 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,
blk_mq_end_request(rq, BLK_STS_IOERR);
}
static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
static void ublk_dispatch_req(struct ublk_queue *ubq,
struct io_uring_cmd *cmd,
struct request *req,
unsigned int issue_flags)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct ublk_queue *ubq = pdu->ubq;
int tag = pdu->tag;
struct request *req = blk_mq_tag_to_rq(
ubq->dev->tag_set.tags[ubq->q_id], tag);
int tag = req->tag;
struct ublk_io *io = &ubq->ios[tag];
unsigned int mapped_bytes;
@ -1252,6 +1264,18 @@ static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd,
ubq_complete_io_cmd(io, UBLK_IO_RES_OK, issue_flags);
}
static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct ublk_queue *ubq = pdu->ubq;
int tag = pdu->tag;
struct request *req = blk_mq_tag_to_rq(
ubq->dev->tag_set.tags[ubq->q_id], tag);
ublk_dispatch_req(ubq, cmd, req, issue_flags);
}
static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
{
struct ublk_io *io = &ubq->ios[rq->tag];
@ -1259,6 +1283,35 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq)
io_uring_cmd_complete_in_task(io->cmd, ublk_rq_task_work_cb);
}
static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd,
unsigned int issue_flags)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
struct request *rq = pdu->req_list;
struct ublk_queue *ubq = rq->mq_hctx->driver_data;
struct request *next;
while (rq) {
struct ublk_io *io = &ubq->ios[rq->tag];
next = rq->rq_next;
rq->rq_next = NULL;
ublk_dispatch_req(ubq, io->cmd, rq, issue_flags);
rq = next;
}
}
static void ublk_queue_cmd_list(struct ublk_queue *ubq, struct rq_list *l)
{
struct request *rq = rq_list_peek(l);
struct ublk_io *io = &ubq->ios[rq->tag];
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(io->cmd);
pdu->req_list = rq;
rq_list_init(l);
io_uring_cmd_complete_in_task(io->cmd, ublk_cmd_list_tw_cb);
}
static enum blk_eh_timer_return ublk_timeout(struct request *rq)
{
struct ublk_queue *ubq = rq->mq_hctx->driver_data;
@ -1297,21 +1350,12 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq)
return BLK_EH_RESET_TIMER;
}
static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq)
{
struct ublk_queue *ubq = hctx->driver_data;
struct request *rq = bd->rq;
blk_status_t res;
if (unlikely(ubq->fail_io)) {
if (unlikely(ubq->fail_io))
return BLK_STS_TARGET;
}
/* fill iod to slot in io cmd buffer */
res = ublk_setup_iod(ubq, rq);
if (unlikely(res != BLK_STS_OK))
return BLK_STS_IOERR;
/* With recovery feature enabled, force_abort is set in
* ublk_stop_dev() before calling del_gendisk(). We have to
@ -1325,6 +1369,29 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
if (ublk_nosrv_should_queue_io(ubq) && unlikely(ubq->force_abort))
return BLK_STS_IOERR;
if (unlikely(ubq->canceling))
return BLK_STS_IOERR;
/* fill iod to slot in io cmd buffer */
res = ublk_setup_iod(ubq, rq);
if (unlikely(res != BLK_STS_OK))
return BLK_STS_IOERR;
blk_mq_start_request(rq);
return BLK_STS_OK;
}
static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *bd)
{
struct ublk_queue *ubq = hctx->driver_data;
struct request *rq = bd->rq;
blk_status_t res;
res = ublk_prep_req(ubq, rq);
if (res != BLK_STS_OK)
return res;
/*
* ->canceling has to be handled after ->force_abort and ->fail_io
* is dealt with, otherwise this request may not be failed in case
@ -1335,12 +1402,35 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx,
return BLK_STS_OK;
}
blk_mq_start_request(bd->rq);
ublk_queue_cmd(ubq, rq);
return BLK_STS_OK;
}
static void ublk_queue_rqs(struct rq_list *rqlist)
{
struct rq_list requeue_list = { };
struct rq_list submit_list = { };
struct ublk_queue *ubq = NULL;
struct request *req;
while ((req = rq_list_pop(rqlist))) {
struct ublk_queue *this_q = req->mq_hctx->driver_data;
if (ubq && ubq != this_q && !rq_list_empty(&submit_list))
ublk_queue_cmd_list(ubq, &submit_list);
ubq = this_q;
if (ublk_prep_req(ubq, req) == BLK_STS_OK)
rq_list_add_tail(&submit_list, req);
else
rq_list_add_tail(&requeue_list, req);
}
if (ubq && !rq_list_empty(&submit_list))
ublk_queue_cmd_list(ubq, &submit_list);
*rqlist = requeue_list;
}
static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
unsigned int hctx_idx)
{
@ -1353,6 +1443,7 @@ static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
static const struct blk_mq_ops ublk_mq_ops = {
.queue_rq = ublk_queue_rq,
.queue_rqs = ublk_queue_rqs,
.init_hctx = ublk_init_hctx,
.timeout = ublk_timeout,
};