diff --git a/block/blk-core.c b/block/blk-core.c index ef15be79b51ff7d46fe6a670550a14162eb0cd92..cfe5c2b46e4692286428c9663c838df44b748dba 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -1882,7 +1882,6 @@ bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, * blk_attempt_plug_merge - try to merge with %current's plugged list * @q: request_queue new bio is being queued at * @bio: new bio being queued - * @request_count: out parameter for number of traversed plugged requests * @same_queue_rq: pointer to &struct request that gets filled in when * another request associated with @q is found on the plug list * (optional, may be %NULL) @@ -1901,7 +1900,6 @@ bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, * Caller must ensure !blk_queue_nomerges(q) beforehand. */ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, - unsigned int *request_count, struct request **same_queue_rq) { struct blk_plug *plug; @@ -1911,7 +1909,6 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, plug = current->plug; if (!plug) return false; - *request_count = 0; if (q->mq_ops) plug_list = &plug->mq_list; @@ -1921,15 +1918,13 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, list_for_each_entry_reverse(rq, plug_list, queuelist) { bool merged = false; - if (rq->q == q) { - (*request_count)++; + if (rq->q == q && same_queue_rq) { /* * Only blk-mq multiple hardware queues case checks the * rq in the same queue, there should be only one such * rq in a queue **/ - if (same_queue_rq) - *same_queue_rq = rq; + *same_queue_rq = rq; } if (rq->q != q || !blk_rq_merge_ok(rq, bio)) @@ -1956,30 +1951,6 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, return false; } -unsigned int blk_plug_queued_count(struct request_queue *q) -{ - struct blk_plug *plug; - struct request *rq; - struct list_head *plug_list; - unsigned int ret = 0; - - plug = current->plug; - if (!plug) - goto out; - - if (q->mq_ops) - plug_list = &plug->mq_list; - else - plug_list = &plug->list; - - list_for_each_entry(rq, plug_list, queuelist) { - if (rq->q == q) - ret++; - } -out: - return ret; -} - void blk_init_request_from_bio(struct request *req, struct bio *bio) { if (bio->bi_opf & REQ_RAHEAD) @@ -2000,7 +1971,6 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) struct blk_plug *plug; int where = ELEVATOR_INSERT_SORT; struct request *req, *free; - unsigned int request_count = 0; /* * low level driver can indicate that it wants pages above a @@ -2025,10 +1995,9 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) * any locks. */ if (!blk_queue_nomerges(q)) { - if (blk_attempt_plug_merge(q, bio, &request_count, NULL)) + if (blk_attempt_plug_merge(q, bio, NULL)) return BLK_QC_T_NONE; - } else - request_count = blk_plug_queued_count(q); + } spin_lock_irq(q->queue_lock); @@ -2092,6 +2061,8 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) plug = current->plug; if (plug) { + unsigned int request_count = plug->rq_count; + /* * If this is the first request added after a plug, fire * of a plug trace. @@ -3623,6 +3594,7 @@ void blk_start_plug(struct blk_plug *plug) INIT_LIST_HEAD(&plug->list); INIT_LIST_HEAD(&plug->mq_list); INIT_LIST_HEAD(&plug->cb_list); + plug->rq_count = 0; plug->multiple_queues = false; /* * Store ordering should not be needed here, since a potential diff --git a/block/blk-mq.c b/block/blk-mq.c index 55e9c25fc0e81d850965a112ddf658c6a12b3c70..1bf41f83737dcbad8d2c2d6e2d267d4ace452975 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1689,6 +1689,7 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) if (plug->multiple_queues) list_sort(NULL, &list, plug_rq_cmp); + plug->rq_count = 0; this_q = NULL; this_hctx = NULL; this_ctx = NULL; @@ -1890,6 +1891,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) { list_add_tail(&rq->queuelist, &plug->mq_list); + plug->rq_count++; if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) { struct request *tmp; @@ -1906,7 +1908,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) const int is_flush_fua = op_is_flush(bio->bi_opf); struct blk_mq_alloc_data data = { .flags = 0}; struct request *rq; - unsigned int request_count = 0; struct blk_plug *plug; struct request *same_queue_rq = NULL; blk_qc_t cookie; @@ -1919,7 +1920,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; if (!is_flush_fua && !blk_queue_nomerges(q) && - blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) + blk_attempt_plug_merge(q, bio, &same_queue_rq)) return BLK_QC_T_NONE; if (blk_mq_sched_bio_merge(q, bio)) @@ -1955,20 +1956,12 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) * Use plugging if we have a ->commit_rqs() hook as well, as * we know the driver uses bd->last in a smart fashion. */ + unsigned int request_count = plug->rq_count; struct request *last = NULL; blk_mq_put_ctx(data.ctx); blk_mq_bio_to_request(rq, bio); - /* - * @request_count may become stale because of schedule - * out, so check the list again. - */ - if (list_empty(&plug->mq_list)) - request_count = 0; - else if (blk_queue_nomerges(q)) - request_count = blk_plug_queued_count(q); - if (!request_count) trace_block_plug(q); else diff --git a/block/blk.h b/block/blk.h index 8817f8562bff024675a99f79e7bb85264b7a1148..8a07502c62267dd3e55499ecb421f565fece096d 100644 --- a/block/blk.h +++ b/block/blk.h @@ -188,9 +188,7 @@ bool bio_attempt_back_merge(struct request_queue *q, struct request *req, bool bio_attempt_discard_merge(struct request_queue *q, struct request *req, struct bio *bio); bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, - unsigned int *request_count, struct request **same_queue_rq); -unsigned int blk_plug_queued_count(struct request_queue *q); void blk_account_io_start(struct request *req, bool new_io); void blk_account_io_completion(struct request *req, unsigned int bytes); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 22243822b8362e0165c17a22a03300afc4309ee0..f8f4b2598141affed2f31443e49ef75ac72327b1 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1374,6 +1374,7 @@ struct blk_plug { struct list_head list; /* requests */ struct list_head mq_list; /* blk-mq requests */ struct list_head cb_list; /* md requires an unplug callback */ + unsigned short rq_count; bool multiple_queues; }; #define BLK_MAX_REQUEST_COUNT 16