linux - block/blk-mq.c

5313 lines · 133.9 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Block multiqueue core code
4	*
5	* Copyright (C) 2013-2014 Jens Axboe
6	* Copyright (C) 2013-2014 Christoph Hellwig
7	*/
8	#include <linux/kernel.h>
9	#include <linux/module.h>
10	#include <linux/backing-dev.h>
11	#include <linux/bio.h>
12	#include <linux/blkdev.h>
13	#include <linux/blk-integrity.h>
14	#include <linux/kmemleak.h>
15	#include <linux/mm.h>
16	#include <linux/init.h>
17	#include <linux/slab.h>
18	#include <linux/workqueue.h>
19	#include <linux/smp.h>
20	#include <linux/interrupt.h>
21	#include <linux/llist.h>
22	#include <linux/cpu.h>
23	#include <linux/cache.h>
24	#include <linux/sched/topology.h>
25	#include <linux/sched/signal.h>
26	#include <linux/suspend.h>
27	#include <linux/delay.h>
28	#include <linux/crash_dump.h>
29	#include <linux/prefetch.h>
30	#include <linux/blk-crypto.h>
31	#include <linux/part_stat.h>
32	#include <linux/sched/isolation.h>
33
34	#include <trace/events/block.h>
35
36	#include <linux/t10-pi.h>
37	#include "blk.h"
38	#include "blk-mq.h"
39	#include "blk-mq-debugfs.h"
40	#include "blk-pm.h"
41	#include "blk-stat.h"
42	#include "blk-mq-sched.h"
43	#include "blk-rq-qos.h"
44
45	static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
46	static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
47	static DEFINE_MUTEX(blk_mq_cpuhp_lock);
48
49	static void blk_mq_insert_request(struct request *rq, blk_insert_t flags);
50	static void blk_mq_request_bypass_insert(struct request *rq,
51	blk_insert_t flags);
52	static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
53	struct list_head *list);
54	static int blk_hctx_poll(struct request_queue q, struct blk_mq_hw_ctx hctx,
55	struct io_comp_batch *iob, unsigned int flags);
56
57	/*
58	* Check if any of the ctx, dispatch list or elevator
59	* have pending work in this hardware queue.
60	*/
61	static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
62	{
63	return !list_empty_careful(&hctx->dispatch) \|\|
64	sbitmap_any_bit_set(&hctx->ctx_map) \|\|
65	blk_mq_sched_has_work(hctx);
66	}
67
68	/*
69	* Mark this ctx as having pending work in this hardware queue
70	*/
71	static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
72	struct blk_mq_ctx *ctx)
73	{
74	const int bit = ctx->index_hw[hctx->type];
75
76	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
77	sbitmap_set_bit(&hctx->ctx_map, bit);
78	}
79
80	static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
81	struct blk_mq_ctx *ctx)
82	{
83	const int bit = ctx->index_hw[hctx->type];
84
85	sbitmap_clear_bit(&hctx->ctx_map, bit);
86	}
87
88	struct mq_inflight {
89	struct block_device *part;
90	unsigned int inflight[2];
91	};
92
93	static bool blk_mq_check_in_driver(struct request rq, void priv)
94	{
95	struct mq_inflight *mi = priv;
96
97	if (rq->rq_flags & RQF_IO_STAT &&
98	(!bdev_is_partition(mi->part) \|\| rq->part == mi->part) &&
99	blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
100	mi->inflight[rq_data_dir(rq)]++;
101
102	return true;
103	}
104
105	void blk_mq_in_driver_rw(struct block_device *part, unsigned int inflight[2])
106	{
107	struct mq_inflight mi = { .part = part };
108
109	blk_mq_queue_tag_busy_iter(bdev_get_queue(part), blk_mq_check_in_driver,
110	&mi);
111	inflight[READ] = mi.inflight[READ];
112	inflight[WRITE] = mi.inflight[WRITE];
113	}
114
115	#ifdef CONFIG_LOCKDEP
116	static bool blk_freeze_set_owner(struct request_queue *q,
117	struct task_struct *owner)
118	{
119	if (!owner)
120	return false;
121
122	if (!q->mq_freeze_depth) {
123	q->mq_freeze_owner = owner;
124	q->mq_freeze_owner_depth = 1;
125	q->mq_freeze_disk_dead = !q->disk \|\|
126	test_bit(GD_DEAD, &q->disk->state) \|\|
127	!blk_queue_registered(q);
128	q->mq_freeze_queue_dying = blk_queue_dying(q);
129	return true;
130	}
131
132	if (owner == q->mq_freeze_owner)
133	q->mq_freeze_owner_depth += 1;
134	return false;
135	}
136
137	/* verify the last unfreeze in owner context */
138	static bool blk_unfreeze_check_owner(struct request_queue *q)
139	{
140	if (q->mq_freeze_owner != current)
141	return false;
142	if (--q->mq_freeze_owner_depth == 0) {
143	q->mq_freeze_owner = NULL;
144	return true;
145	}
146	return false;
147	}
148
149	#else
150
151	static bool blk_freeze_set_owner(struct request_queue *q,
152	struct task_struct *owner)
153	{
154	return false;
155	}
156
157	static bool blk_unfreeze_check_owner(struct request_queue *q)
158	{
159	return false;
160	}
161	#endif
162
163	bool __blk_freeze_queue_start(struct request_queue *q,
164	struct task_struct *owner)
165	{
166	bool freeze;
167
168	mutex_lock(&q->mq_freeze_lock);
169	freeze = blk_freeze_set_owner(q, owner);
170	if (++q->mq_freeze_depth == 1) {
171	percpu_ref_kill(&q->q_usage_counter);
172	mutex_unlock(&q->mq_freeze_lock);
173	if (queue_is_mq(q))
174	blk_mq_run_hw_queues(q, false);
175	} else {
176	mutex_unlock(&q->mq_freeze_lock);
177	}
178
179	return freeze;
180	}
181
182	void blk_freeze_queue_start(struct request_queue *q)
183	{
184	if (__blk_freeze_queue_start(q, current))
185	blk_freeze_acquire_lock(q);
186	}
187	EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
188
189	void blk_mq_freeze_queue_wait(struct request_queue *q)
190	{
191	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
192	}
193	EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
194
195	int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
196	unsigned long timeout)
197	{
198	return wait_event_timeout(q->mq_freeze_wq,
199	percpu_ref_is_zero(&q->q_usage_counter),
200	timeout);
201	}
202	EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
203
204	void blk_mq_freeze_queue_nomemsave(struct request_queue *q)
205	{
206	blk_freeze_queue_start(q);
207	blk_mq_freeze_queue_wait(q);
208	}
209	EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_nomemsave);
210
211	bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
212	{
213	bool unfreeze;
214
215	mutex_lock(&q->mq_freeze_lock);
216	if (force_atomic)
217	q->q_usage_counter.data->force_atomic = true;
218	q->mq_freeze_depth--;
219	WARN_ON_ONCE(q->mq_freeze_depth < 0);
220	if (!q->mq_freeze_depth) {
221	percpu_ref_resurrect(&q->q_usage_counter);
222	wake_up_all(&q->mq_freeze_wq);
223	}
224	unfreeze = blk_unfreeze_check_owner(q);
225	mutex_unlock(&q->mq_freeze_lock);
226
227	return unfreeze;
228	}
229
230	void blk_mq_unfreeze_queue_nomemrestore(struct request_queue *q)
231	{
232	if (__blk_mq_unfreeze_queue(q, false))
233	blk_unfreeze_release_lock(q);
234	}
235	EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_nomemrestore);
236
237	/*
238	* non_owner variant of blk_freeze_queue_start
239	*
240	* Unlike blk_freeze_queue_start, the queue doesn't need to be unfrozen
241	* by the same task. This is fragile and should not be used if at all
242	* possible.
243	*/
244	void blk_freeze_queue_start_non_owner(struct request_queue *q)
245	{
246	__blk_freeze_queue_start(q, NULL);
247	}
248	EXPORT_SYMBOL_GPL(blk_freeze_queue_start_non_owner);
249
250	/* non_owner variant of blk_mq_unfreeze_queue */
251	void blk_mq_unfreeze_queue_non_owner(struct request_queue *q)
252	{
253	__blk_mq_unfreeze_queue(q, false);
254	}
255	EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue_non_owner);
256
257	/*
258	* FIXME: replace the scsi_internal_device_*block_nowait() calls in the
259	* mpt3sas driver such that this function can be removed.
260	*/
261	void blk_mq_quiesce_queue_nowait(struct request_queue *q)
262	{
263	unsigned long flags;
264
265	spin_lock_irqsave(&q->queue_lock, flags);
266	if (!q->quiesce_depth++)
267	blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
268	spin_unlock_irqrestore(&q->queue_lock, flags);
269	}
270	EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);
271
272	/**
273	* blk_mq_wait_quiesce_done() - wait until in-progress quiesce is done
274	* @set: tag_set to wait on
275	*
276	* Note: it is driver's responsibility for making sure that quiesce has
277	* been started on or more of the request_queues of the tag_set. This
278	* function only waits for the quiesce on those request_queues that had
279	* the quiesce flag set using blk_mq_quiesce_queue_nowait.
280	*/
281	void blk_mq_wait_quiesce_done(struct blk_mq_tag_set *set)
282	{
283	if (set->flags & BLK_MQ_F_BLOCKING)
284	synchronize_srcu(set->srcu);
285	else
286	synchronize_rcu();
287	}
288	EXPORT_SYMBOL_GPL(blk_mq_wait_quiesce_done);
289
290	/**
291	* blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
292	* @q: request queue.
293	*
294	* Note: this function does not prevent that the struct request end_io()
295	* callback function is invoked. Once this function is returned, we make
296	* sure no dispatch can happen until the queue is unquiesced via
297	* blk_mq_unquiesce_queue().
298	*/
299	void blk_mq_quiesce_queue(struct request_queue *q)
300	{
301	blk_mq_quiesce_queue_nowait(q);
302	/* nothing to wait for non-mq queues */
303	if (queue_is_mq(q))
304	blk_mq_wait_quiesce_done(q->tag_set);
305	}
306	EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);
307
308	/*
309	* blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
310	* @q: request queue.
311	*
312	* This function recovers queue into the state before quiescing
313	* which is done by blk_mq_quiesce_queue.
314	*/
315	void blk_mq_unquiesce_queue(struct request_queue *q)
316	{
317	unsigned long flags;
318	bool run_queue = false;
319
320	spin_lock_irqsave(&q->queue_lock, flags);
321	if (WARN_ON_ONCE(q->quiesce_depth <= 0)) {
322	;
323	} else if (!--q->quiesce_depth) {
324	blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
325	run_queue = true;
326	}
327	spin_unlock_irqrestore(&q->queue_lock, flags);
328
329	/* dispatch requests which are inserted during quiescing */
330	if (run_queue)
331	blk_mq_run_hw_queues(q, true);
332	}
333	EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
334
335	void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
336	{
337	struct request_queue *q;
338
339	rcu_read_lock();
340	list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
341	if (!blk_queue_skip_tagset_quiesce(q))
342	blk_mq_quiesce_queue_nowait(q);
343	}
344	rcu_read_unlock();
345
346	blk_mq_wait_quiesce_done(set);
347	}
348	EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
349
350	void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
351	{
352	struct request_queue *q;
353
354	rcu_read_lock();
355	list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
356	if (!blk_queue_skip_tagset_quiesce(q))
357	blk_mq_unquiesce_queue(q);
358	}
359	rcu_read_unlock();
360	}
361	EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
362
363	void blk_mq_wake_waiters(struct request_queue *q)
364	{
365	struct blk_mq_hw_ctx *hctx;
366	unsigned long i;
367
368	queue_for_each_hw_ctx(q, hctx, i)
369	if (blk_mq_hw_queue_mapped(hctx))
370	blk_mq_tag_wakeup_all(hctx->tags, true);
371	}
372
373	void blk_rq_init(struct request_queue q, struct request rq)
374	{
375	memset(rq, 0, sizeof(*rq));
376
377	INIT_LIST_HEAD(&rq->queuelist);
378	rq->q = q;
379	rq->__sector = (sector_t) -1;
380	rq->phys_gap_bit = 0;
381	INIT_HLIST_NODE(&rq->hash);
382	RB_CLEAR_NODE(&rq->rb_node);
383	rq->tag = BLK_MQ_NO_TAG;
384	rq->internal_tag = BLK_MQ_NO_TAG;
385	rq->start_time_ns = blk_time_get_ns();
386	blk_crypto_rq_set_defaults(rq);
387	}
388	EXPORT_SYMBOL(blk_rq_init);
389
390	/* Set start and alloc time when the allocated request is actually used */
391	static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
392	{
393	#ifdef CONFIG_BLK_RQ_ALLOC_TIME
394	if (blk_queue_rq_alloc_time(rq->q))
395	rq->alloc_time_ns = alloc_time_ns;
396	else
397	rq->alloc_time_ns = 0;
398	#endif
399	}
400
401	static inline void blk_mq_bio_issue_init(struct request_queue *q,
402	struct bio *bio)
403	{
404	#ifdef CONFIG_BLK_CGROUP
405	if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags))
406	bio->issue_time_ns = blk_time_get_ns();
407	#endif
408	}
409
410	static struct request blk_mq_rq_ctx_init(struct blk_mq_alloc_data data,
411	struct blk_mq_tags *tags, unsigned int tag)
412	{
413	struct blk_mq_ctx *ctx = data->ctx;
414	struct blk_mq_hw_ctx *hctx = data->hctx;
415	struct request_queue *q = data->q;
416	struct request *rq = tags->static_rqs[tag];
417
418	rq->q = q;
419	rq->mq_ctx = ctx;
420	rq->mq_hctx = hctx;
421	rq->cmd_flags = data->cmd_flags;
422
423	if (data->flags & BLK_MQ_REQ_PM)
424	data->rq_flags \|= RQF_PM;
425	rq->rq_flags = data->rq_flags;
426
427	if (data->rq_flags & RQF_SCHED_TAGS) {
428	rq->tag = BLK_MQ_NO_TAG;
429	rq->internal_tag = tag;
430	} else {
431	rq->tag = tag;
432	rq->internal_tag = BLK_MQ_NO_TAG;
433	}
434	rq->timeout = 0;
435
436	rq->part = NULL;
437	rq->io_start_time_ns = 0;
438	rq->stats_sectors = 0;
439	rq->nr_phys_segments = 0;
440	rq->nr_integrity_segments = 0;
441	rq->end_io = NULL;
442	rq->end_io_data = NULL;
443
444	blk_crypto_rq_set_defaults(rq);
445	INIT_LIST_HEAD(&rq->queuelist);
446	/* tag was already set */
447	WRITE_ONCE(rq->deadline, 0);
448	req_ref_set(rq, 1);
449
450	return rq;
451	}
452
453	static inline struct request *
454	__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
455	{
456	unsigned int tag, tag_offset;
457	struct blk_mq_tags *tags;
458	struct request *rq;
459	unsigned long tag_mask;
460	int i, nr = 0;
461
462	do {
463	tag_mask = blk_mq_get_tags(data, data->nr_tags - nr, &tag_offset);
464	if (unlikely(!tag_mask)) {
465	if (nr == 0)
466	return NULL;
467	break;
468	}
469	tags = blk_mq_tags_from_data(data);
470	for (i = 0; tag_mask; i++) {
471	if (!(tag_mask & (1UL << i)))
472	continue;
473	tag = tag_offset + i;
474	prefetch(tags->static_rqs[tag]);
475	tag_mask &= ~(1UL << i);
476	rq = blk_mq_rq_ctx_init(data, tags, tag);
477	rq_list_add_head(data->cached_rqs, rq);
478	nr++;
479	}
480	} while (data->nr_tags > nr);
481
482	if (!(data->rq_flags & RQF_SCHED_TAGS))
483	blk_mq_add_active_requests(data->hctx, nr);
484	/* caller already holds a reference, add for remainder */
485	percpu_ref_get_many(&data->q->q_usage_counter, nr - 1);
486	data->nr_tags -= nr;
487
488	return rq_list_pop(data->cached_rqs);
489	}
490
491	static bool blk_op_bypass_sched(blk_opf_t opf)
492	{
493	return (opf & REQ_OP_MASK) == REQ_OP_FLUSH \|\|
494	blk_op_is_passthrough(opf);
495	}
496
497	static void blk_mq_limit_depth(struct blk_mq_alloc_data *data)
498	{
499	struct elevator_mq_ops *ops;
500
501	/* If no I/O scheduler has been configured, don't limit requests */
502	if (!data->q->elevator) {
503	blk_mq_tag_busy(data->hctx);
504	return;
505	}
506
507	/*
508	* All requests use scheduler tags when an I/O scheduler is
509	* enabled for the queue.
510	*/
511	data->rq_flags \|= RQF_SCHED_TAGS;
512
513	/*
514	* Flush/passthrough requests are special and go directly to the
515	* dispatch list, they are not subject to the async_depth limit.
516	*/
517	if (blk_op_bypass_sched(data->cmd_flags))
518	return;
519
520	WARN_ON_ONCE(data->flags & BLK_MQ_REQ_RESERVED);
521
522	/*
523	* By default, sync requests have no limit, and async requests are
524	* limited to async_depth.
525	*/
526	ops = &data->q->elevator->type->ops;
527	if (ops->limit_depth)
528	ops->limit_depth(data->cmd_flags, data);
529	}
530
531	/*
532	* Finish initializing a request once it has been claimed for an operation.
533	* Cached requests are allocated before that operation is known.
534	*/
535	static void blk_mq_rq_late_init(struct request *rq, u64 alloc_time_ns)
536	{
537	struct elevator_queue *e;
538
539	blk_mq_rq_time_init(rq, alloc_time_ns);
540
541	if (!(rq->rq_flags & RQF_SCHED_TAGS) \|\| (rq->rq_flags & RQF_RESV) \|\|
542	blk_op_bypass_sched(rq->cmd_flags))
543	return;
544
545	rq->rq_flags \|= RQF_USE_SCHED;
546	INIT_HLIST_NODE(&rq->hash);
547	RB_CLEAR_NODE(&rq->rb_node);
548
549	e = rq->q->elevator;
550	if (e->type->ops.prepare_request)
551	e->type->ops.prepare_request(rq);
552	}
553
554	static struct request __blk_mq_alloc_requests(struct blk_mq_alloc_data data)
555	{
556	struct request_queue *q = data->q;
557	u64 alloc_time_ns = 0;
558	struct request *rq;
559	unsigned int tag;
560
561	/* alloc_time includes depth and tag waits */
562	if (blk_queue_rq_alloc_time(q))
563	alloc_time_ns = blk_time_get_ns();
564
565	if (data->cmd_flags & REQ_NOWAIT)
566	data->flags \|= BLK_MQ_REQ_NOWAIT;
567
568	retry:
569	data->ctx = blk_mq_get_ctx(q);
570	data->hctx = blk_mq_map_queue(data->cmd_flags, data->ctx);
571
572	blk_mq_limit_depth(data);
573	if (data->flags & BLK_MQ_REQ_RESERVED)
574	data->rq_flags \|= RQF_RESV;
575
576	/*
577	* Try batched alloc if we want more than 1 tag.
578	*/
579	if (data->nr_tags > 1) {
580	rq = __blk_mq_alloc_requests_batch(data);
581	if (rq) {
582	blk_mq_rq_late_init(rq, alloc_time_ns);
583	return rq;
584	}
585	data->nr_tags = 1;
586	}
587
588	/*
589	* Waiting allocations only fail because of an inactive hctx. In that
590	* case just retry the hctx assignment and tag allocation as CPU hotplug
591	* should have migrated us to an online CPU by now.
592	*/
593	tag = blk_mq_get_tag(data);
594	if (tag == BLK_MQ_NO_TAG) {
595	if (data->flags & BLK_MQ_REQ_NOWAIT)
596	return NULL;
597	/*
598	* Give up the CPU and sleep for a random short time to
599	* ensure that thread using a realtime scheduling class
600	* are migrated off the CPU, and thus off the hctx that
601	* is going away.
602	*/
603	msleep(3);
604	goto retry;
605	}
606
607	if (!(data->rq_flags & RQF_SCHED_TAGS))
608	blk_mq_inc_active_requests(data->hctx);
609	rq = blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag);
610	blk_mq_rq_late_init(rq, alloc_time_ns);
611	return rq;
612	}
613
614	static struct request blk_mq_rq_cache_fill(struct request_queue q,
615	struct blk_plug *plug,
616	blk_opf_t opf,
617	blk_mq_req_flags_t flags)
618	{
619	struct blk_mq_alloc_data data = {
620	.q = q,
621	.flags = flags,
622	.shallow_depth = 0,
623	.cmd_flags = opf,
624	.rq_flags = 0,
625	.nr_tags = plug->nr_ios,
626	.cached_rqs = &plug->cached_rqs,
627	.ctx = NULL,
628	.hctx = NULL
629	};
630	struct request *rq;
631
632	if (blk_queue_enter(q, flags))
633	return NULL;
634
635	plug->nr_ios = 1;
636
637	rq = __blk_mq_alloc_requests(&data);
638	if (unlikely(!rq))
639	blk_queue_exit(q);
640	return rq;
641	}
642
643	static struct request blk_mq_alloc_cached_request(struct request_queue q,
644	blk_opf_t opf,
645	blk_mq_req_flags_t flags)
646	{
647	struct blk_plug *plug = current->plug;
648	struct request *rq;
649
650	if (!plug)
651	return NULL;
652
653	if (rq_list_empty(&plug->cached_rqs)) {
654	if (plug->nr_ios == 1)
655	return NULL;
656	rq = blk_mq_rq_cache_fill(q, plug, opf, flags);
657	} else {
658	rq = rq_list_peek(&plug->cached_rqs);
659	if (!rq \|\| rq->q != q)
660	return NULL;
661
662	if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type)
663	return NULL;
664
665	rq_list_pop(&plug->cached_rqs);
666	rq->cmd_flags = opf;
667	INIT_LIST_HEAD(&rq->queuelist);
668	blk_mq_rq_late_init(rq, blk_time_get_ns());
669	}
670
671	return rq;
672	}
673
674	struct request blk_mq_alloc_request(struct request_queue q, blk_opf_t opf,
675	blk_mq_req_flags_t flags)
676	{
677	struct request *rq;
678
679	rq = blk_mq_alloc_cached_request(q, opf, flags);
680	if (!rq) {
681	struct blk_mq_alloc_data data = {
682	.q = q,
683	.flags = flags,
684	.shallow_depth = 0,
685	.cmd_flags = opf,
686	.rq_flags = 0,
687	.nr_tags = 1,
688	.cached_rqs = NULL,
689	.ctx = NULL,
690	.hctx = NULL
691	};
692	int ret;
693
694	ret = blk_queue_enter(q, flags);
695	if (ret)
696	return ERR_PTR(ret);
697
698	rq = __blk_mq_alloc_requests(&data);
699	if (!rq)
700	goto out_queue_exit;
701	}
702	rq->__data_len = 0;
703	rq->phys_gap_bit = 0;
704	rq->__sector = (sector_t) -1;
705	rq->bio = rq->biotail = NULL;
706	return rq;
707	out_queue_exit:
708	blk_queue_exit(q);
709	return ERR_PTR(-EWOULDBLOCK);
710	}
711	EXPORT_SYMBOL(blk_mq_alloc_request);
712
713	struct request blk_mq_alloc_request_hctx(struct request_queue q,
714	blk_opf_t opf, blk_mq_req_flags_t flags, unsigned int hctx_idx)
715	{
716	struct blk_mq_alloc_data data = {
717	.q = q,
718	.flags = flags,
719	.shallow_depth = 0,
720	.cmd_flags = opf,
721	.rq_flags = 0,
722	.nr_tags = 1,
723	.cached_rqs = NULL,
724	.ctx = NULL,
725	.hctx = NULL
726	};
727	u64 alloc_time_ns = 0;
728	struct request *rq;
729	unsigned int cpu;
730	unsigned int tag;
731	int ret;
732
733	/* alloc_time includes depth and tag waits */
734	if (blk_queue_rq_alloc_time(q))
735	alloc_time_ns = blk_time_get_ns();
736
737	/*
738	* If the tag allocator sleeps we could get an allocation for a
739	* different hardware context. No need to complicate the low level
740	* allocator for this for the rare use case of a command tied to
741	* a specific queue.
742	*/
743	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)) \|\|
744	WARN_ON_ONCE(!(flags & BLK_MQ_REQ_RESERVED)))
745	return ERR_PTR(-EINVAL);
746
747	if (hctx_idx >= q->nr_hw_queues)
748	return ERR_PTR(-EIO);
749
750	ret = blk_queue_enter(q, flags);
751	if (ret)
752	return ERR_PTR(ret);
753
754	/*
755	* Check if the hardware context is actually mapped to anything.
756	* If not tell the caller that it should skip this queue.
757	*/
758	ret = -EXDEV;
759	data.hctx = q->queue_hw_ctx[hctx_idx];
760	if (!blk_mq_hw_queue_mapped(data.hctx))
761	goto out_queue_exit;
762	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
763	if (cpu >= nr_cpu_ids)
764	goto out_queue_exit;
765	data.ctx = __blk_mq_get_ctx(q, cpu);
766
767	if (q->elevator)
768	data.rq_flags \|= RQF_SCHED_TAGS;
769	else
770	blk_mq_tag_busy(data.hctx);
771
772	if (flags & BLK_MQ_REQ_RESERVED)
773	data.rq_flags \|= RQF_RESV;
774
775	ret = -EWOULDBLOCK;
776	tag = blk_mq_get_tag(&data);
777	if (tag == BLK_MQ_NO_TAG)
778	goto out_queue_exit;
779	if (!(data.rq_flags & RQF_SCHED_TAGS))
780	blk_mq_inc_active_requests(data.hctx);
781	rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag);
782	blk_mq_rq_late_init(rq, alloc_time_ns);
783	rq->__data_len = 0;
784	rq->phys_gap_bit = 0;
785	rq->__sector = (sector_t) -1;
786	rq->bio = rq->biotail = NULL;
787	return rq;
788
789	out_queue_exit:
790	blk_queue_exit(q);
791	return ERR_PTR(ret);
792	}
793	EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);
794
795	static void blk_mq_finish_request(struct request *rq)
796	{
797	struct request_queue *q = rq->q;
798
799	blk_zone_finish_request(rq);
800
801	if (rq->rq_flags & RQF_USE_SCHED) {
802	q->elevator->type->ops.finish_request(rq);
803	/*
804	* For postflush request that may need to be
805	* completed twice, we should clear this flag
806	* to avoid double finish_request() on the rq.
807	*/
808	rq->rq_flags &= ~RQF_USE_SCHED;
809	}
810	}
811
812	static void __blk_mq_free_request(struct request *rq)
813	{
814	struct request_queue *q = rq->q;
815	struct blk_mq_ctx *ctx = rq->mq_ctx;
816	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
817	const int sched_tag = rq->internal_tag;
818
819	blk_crypto_free_request(rq);
820	blk_pm_mark_last_busy(rq);
821	rq->mq_hctx = NULL;
822
823	if (rq->tag != BLK_MQ_NO_TAG) {
824	blk_mq_dec_active_requests(hctx);
825	blk_mq_put_tag(hctx->tags, ctx, rq->tag);
826	}
827	if (sched_tag != BLK_MQ_NO_TAG)
828	blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
829	blk_mq_sched_restart(hctx);
830	blk_queue_exit(q);
831	}
832
833	void blk_mq_free_request(struct request *rq)
834	{
835	struct request_queue *q = rq->q;
836
837	blk_mq_finish_request(rq);
838
839	rq_qos_done(q, rq);
840
841	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
842	if (req_ref_put_and_test(rq))
843	__blk_mq_free_request(rq);
844	}
845	EXPORT_SYMBOL_GPL(blk_mq_free_request);
846
847	void blk_mq_free_plug_rqs(struct blk_plug *plug)
848	{
849	struct request *rq;
850
851	while ((rq = rq_list_pop(&plug->cached_rqs)) != NULL)
852	blk_mq_free_request(rq);
853	}
854
855	void blk_dump_rq_flags(struct request rq, char msg)
856	{
857	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
858	rq->q->disk ? rq->q->disk->disk_name : "?",
859	(__force unsigned long long) rq->cmd_flags);
860
861	printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
862	(unsigned long long)blk_rq_pos(rq),
863	blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
864	printk(KERN_INFO " bio %p, biotail %p, len %u\n",
865	rq->bio, rq->biotail, blk_rq_bytes(rq));
866	}
867	EXPORT_SYMBOL(blk_dump_rq_flags);
868
869	static void blk_account_io_completion(struct request *req, unsigned int bytes)
870	{
871	if (req->rq_flags & RQF_IO_STAT) {
872	const int sgrp = op_stat_group(req_op(req));
873
874	part_stat_lock();
875	part_stat_add(req->part, sectors[sgrp], bytes >> 9);
876	part_stat_unlock();
877	}
878	}
879
880	static void blk_print_req_error(struct request *req, blk_status_t status)
881	{
882	printk_ratelimited(KERN_ERR
883	"%s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
884	"phys_seg %u prio class %u\n",
885	blk_status_to_str(status),
886	req->q->disk ? req->q->disk->disk_name : "?",
887	blk_rq_pos(req), (__force u32)req_op(req),
888	blk_op_str(req_op(req)),
889	(__force u32)(req->cmd_flags & ~REQ_OP_MASK),
890	req->nr_phys_segments,
891	IOPRIO_PRIO_CLASS(req_get_ioprio(req)));
892	}
893
894	/*
895	* Fully end IO on a request. Does not support partial completions, or
896	* errors.
897	*/
898	static void blk_complete_request(struct request *req)
899	{
900	const bool is_flush = (req->rq_flags & RQF_FLUSH_SEQ) != 0;
901	int total_bytes = blk_rq_bytes(req);
902	struct bio *bio = req->bio;
903
904	trace_block_rq_complete(req, BLK_STS_OK, total_bytes);
905
906	if (!bio)
907	return;
908
909	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ)
910	blk_integrity_complete(req, total_bytes);
911
912	/*
913	* Upper layers may call blk_crypto_evict_key() anytime after the last
914	* bio_endio(). Therefore, the keyslot must be released before that.
915	*/
916	blk_crypto_rq_put_keyslot(req);
917
918	blk_account_io_completion(req, total_bytes);
919
920	do {
921	struct bio *next = bio->bi_next;
922
923	/* Completion has already been traced */
924	bio_clear_flag(bio, BIO_TRACE_COMPLETION);
925
926	if (blk_req_bio_is_zone_append(req, bio))
927	blk_zone_append_update_request_bio(req, bio);
928
929	if (!is_flush)
930	bio_endio(bio);
931	bio = next;
932	} while (bio);
933
934	/*
935	* Reset counters so that the request stacking driver
936	* can find how many bytes remain in the request
937	* later.
938	*/
939	if (!req->end_io) {
940	req->bio = NULL;
941	req->__data_len = 0;
942	}
943	}
944
945	/**
946	* blk_update_request - Complete multiple bytes without completing the request
947	* @req: the request being processed
948	* @error: block status code
949	* @nr_bytes: number of bytes to complete for @req
950	*
951	* Description:
952	* Ends I/O on a number of bytes attached to @req, but doesn't complete
953	* the request structure even if @req doesn't have leftover.
954	* If @req has leftover, sets it up for the next range of segments.
955	*
956	* Passing the result of blk_rq_bytes() as @nr_bytes guarantees
957	* %false return from this function.
958	*
959	* Note:
960	* The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
961	* except in the consistency check at the end of this function.
962	*
963	* Return:
964	* %false - this request doesn't have any more data
965	* %true - this request has more data
966	**/
967	bool blk_update_request(struct request *req, blk_status_t error,
968	unsigned int nr_bytes)
969	{
970	bool is_flush = req->rq_flags & RQF_FLUSH_SEQ;
971	bool quiet = req->rq_flags & RQF_QUIET;
972	int total_bytes;
973
974	trace_block_rq_complete(req, error, nr_bytes);
975
976	if (!req->bio)
977	return false;
978
979	if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
980	error == BLK_STS_OK)
981	blk_integrity_complete(req, nr_bytes);
982
983	/*
984	* Upper layers may call blk_crypto_evict_key() anytime after the last
985	* bio_endio(). Therefore, the keyslot must be released before that.
986	*/
987	if (blk_crypto_rq_has_keyslot(req) && nr_bytes >= blk_rq_bytes(req))
988	__blk_crypto_rq_put_keyslot(req);
989
990	if (unlikely(error && !blk_rq_is_passthrough(req) && !quiet) &&
991	!test_bit(GD_DEAD, &req->q->disk->state)) {
992	blk_print_req_error(req, error);
993	trace_block_rq_error(req, error, nr_bytes);
994	}
995
996	blk_account_io_completion(req, nr_bytes);
997
998	total_bytes = 0;
999	while (req->bio) {
1000	struct bio *bio = req->bio;
1001	unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
1002
1003	if (unlikely(error))
1004	bio->bi_status = error;
1005
1006	if (bio_bytes == bio->bi_iter.bi_size) {
1007	req->bio = bio->bi_next;
1008	} else if (bio_is_zone_append(bio) && error == BLK_STS_OK) {
1009	/*
1010	* Partial zone append completions cannot be supported
1011	* as the BIO fragments may end up not being written
1012	* sequentially.
1013	*/
1014	bio->bi_status = BLK_STS_IOERR;
1015	}
1016
1017	/* Completion has already been traced */
1018	bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1019	if (unlikely(quiet))
1020	bio_set_flag(bio, BIO_QUIET);
1021
1022	bio_advance(bio, bio_bytes);
1023
1024	/* Don't actually finish bio if it's part of flush sequence */
1025	if (!bio->bi_iter.bi_size) {
1026	if (blk_req_bio_is_zone_append(req, bio))
1027	blk_zone_append_update_request_bio(req, bio);
1028	if (!is_flush)
1029	bio_endio(bio);
1030	}
1031
1032	total_bytes += bio_bytes;
1033	nr_bytes -= bio_bytes;
1034
1035	if (!nr_bytes)
1036	break;
1037	}
1038
1039	/*
1040	* completely done
1041	*/
1042	if (!req->bio) {
1043	/*
1044	* Reset counters so that the request stacking driver
1045	* can find how many bytes remain in the request
1046	* later.
1047	*/
1048	req->__data_len = 0;
1049	return false;
1050	}
1051
1052	req->__data_len -= total_bytes;
1053
1054	/* update sector only for requests with clear definition of sector */
1055	if (!blk_rq_is_passthrough(req))
1056	req->__sector += total_bytes >> 9;
1057
1058	/* mixed attributes always follow the first bio */
1059	if (req->rq_flags & RQF_MIXED_MERGE) {
1060	req->cmd_flags &= ~REQ_FAILFAST_MASK;
1061	req->cmd_flags \|= req->bio->bi_opf & REQ_FAILFAST_MASK;
1062	}
1063
1064	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
1065	/*
1066	* If total number of sectors is less than the first segment
1067	* size, something has gone terribly wrong.
1068	*/
1069	if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
1070	blk_dump_rq_flags(req, "request botched");
1071	req->__data_len = blk_rq_cur_bytes(req);
1072	}
1073
1074	/* recalculate the number of segments */
1075	req->nr_phys_segments = blk_recalc_rq_segments(req);
1076	}
1077
1078	return true;
1079	}
1080	EXPORT_SYMBOL_GPL(blk_update_request);
1081
1082	static inline void blk_account_io_done(struct request *req, u64 now)
1083	{
1084	trace_block_io_done(req);
1085
1086	/*
1087	* Account IO completion. flush_rq isn't accounted as a
1088	* normal IO on queueing nor completion. Accounting the
1089	* containing request is enough.
1090	*/
1091	if ((req->rq_flags & (RQF_IO_STAT\|RQF_FLUSH_SEQ)) == RQF_IO_STAT) {
1092	const int sgrp = op_stat_group(req_op(req));
1093
1094	part_stat_lock();
1095	update_io_ticks(req->part, jiffies, true);
1096	part_stat_inc(req->part, ios[sgrp]);
1097	part_stat_add(req->part, nsecs[sgrp], now - req->start_time_ns);
1098	bdev_dec_in_flight(req->part, req_op(req));
1099	part_stat_unlock();
1100	}
1101	}
1102
1103	static inline void blk_account_io_start(struct request *req)
1104	{
1105	trace_block_io_start(req);
1106
1107	if (!blk_queue_io_stat(req->q))
1108	return;
1109	if (blk_rq_is_passthrough(req) && !blk_rq_passthrough_stats(req, req->q))
1110	return;
1111
1112	req->rq_flags \|= RQF_IO_STAT;
1113	req->start_time_ns = blk_time_get_ns();
1114
1115	/*
1116	* All non-passthrough requests are created from a bio with one
1117	* exception: when a flush command that is part of a flush sequence
1118	* generated by the state machine in blk-flush.c is cloned onto the
1119	* lower device by dm-multipath we can get here without a bio.
1120	*/
1121	if (req->bio)
1122	req->part = req->bio->bi_bdev;
1123	else
1124	req->part = req->q->disk->part0;
1125
1126	part_stat_lock();
1127	update_io_ticks(req->part, jiffies, false);
1128	bdev_inc_in_flight(req->part, req_op(req));
1129	part_stat_unlock();
1130	}
1131
1132	static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
1133	{
1134	if (rq->rq_flags & RQF_STATS)
1135	blk_stat_add(rq, now);
1136
1137	blk_mq_sched_completed_request(rq, now);
1138	blk_account_io_done(rq, now);
1139	}
1140
1141	inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
1142	{
1143	if (blk_mq_need_time_stamp(rq))
1144	__blk_mq_end_request_acct(rq, blk_time_get_ns());
1145
1146	blk_mq_finish_request(rq);
1147
1148	if (rq->end_io) {
1149	rq_qos_done(rq->q, rq);
1150	if (rq->end_io(rq, error, NULL) == RQ_END_IO_FREE)
1151	blk_mq_free_request(rq);
1152	} else {
1153	blk_mq_free_request(rq);
1154	}
1155	}
1156	EXPORT_SYMBOL(__blk_mq_end_request);
1157
1158	void blk_mq_end_request(struct request *rq, blk_status_t error)
1159	{
1160	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
1161	BUG();
1162	__blk_mq_end_request(rq, error);
1163	}
1164	EXPORT_SYMBOL(blk_mq_end_request);
1165
1166	#define TAG_COMP_BATCH 32
1167
1168	static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
1169	int *tag_array, int nr_tags)
1170	{
1171	struct request_queue *q = hctx->queue;
1172
1173	blk_mq_sub_active_requests(hctx, nr_tags);
1174
1175	blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
1176	percpu_ref_put_many(&q->q_usage_counter, nr_tags);
1177	}
1178
1179	void blk_mq_end_request_batch(struct io_comp_batch *iob)
1180	{
1181	int tags[TAG_COMP_BATCH], nr_tags = 0;
1182	struct blk_mq_hw_ctx *cur_hctx = NULL;
1183	struct request *rq;
1184	u64 now = 0;
1185
1186	if (iob->need_ts)
1187	now = blk_time_get_ns();
1188
1189	while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
1190	prefetch(rq->bio);
1191	prefetch(rq->rq_next);
1192
1193	blk_complete_request(rq);
1194	if (iob->need_ts)
1195	__blk_mq_end_request_acct(rq, now);
1196
1197	blk_mq_finish_request(rq);
1198
1199	rq_qos_done(rq->q, rq);
1200
1201	/*
1202	* If end_io handler returns NONE, then it still has
1203	* ownership of the request.
1204	*/
1205	if (rq->end_io && rq->end_io(rq, 0, iob) == RQ_END_IO_NONE)
1206	continue;
1207
1208	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1209	if (!req_ref_put_and_test(rq))
1210	continue;
1211
1212	blk_crypto_free_request(rq);
1213	blk_pm_mark_last_busy(rq);
1214
1215	if (nr_tags == TAG_COMP_BATCH \|\| cur_hctx != rq->mq_hctx) {
1216	if (cur_hctx)
1217	blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1218	nr_tags = 0;
1219	cur_hctx = rq->mq_hctx;
1220	}
1221	tags[nr_tags++] = rq->tag;
1222	}
1223
1224	if (nr_tags)
1225	blk_mq_flush_tag_batch(cur_hctx, tags, nr_tags);
1226	}
1227	EXPORT_SYMBOL_GPL(blk_mq_end_request_batch);
1228
1229	static void blk_complete_reqs(struct llist_head *list)
1230	{
1231	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
1232	struct request rq, next;
1233
1234	llist_for_each_entry_safe(rq, next, entry, ipi_list)
1235	rq->q->mq_ops->complete(rq);
1236	}
1237
1238	static __latent_entropy void blk_done_softirq(void)
1239	{
1240	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
1241	}
1242
1243	static int blk_softirq_cpu_dead(unsigned int cpu)
1244	{
1245	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
1246	return 0;
1247	}
1248
1249	static void __blk_mq_complete_request_remote(void *data)
1250	{
1251	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
1252	}
1253
1254	static inline bool blk_mq_complete_need_ipi(struct request *rq)
1255	{
1256	int cpu = raw_smp_processor_id();
1257
1258	if (!IS_ENABLED(CONFIG_SMP) \|\|
1259	!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
1260	return false;
1261	/*
1262	* With force threaded interrupts enabled, raising softirq from an SMP
1263	* function call will always result in waking the ksoftirqd thread.
1264	* This is probably worse than completing the request on a different
1265	* cache domain.
1266	*/
1267	if (force_irqthreads())
1268	return false;
1269
1270	/* same CPU or cache domain and capacity? Complete locally */
1271	if (cpu == rq->mq_ctx->cpu \|\|
1272	(!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
1273	cpus_share_cache(cpu, rq->mq_ctx->cpu) &&
1274	cpus_equal_capacity(cpu, rq->mq_ctx->cpu)))
1275	return false;
1276
1277	/* don't try to IPI to an offline CPU */
1278	return cpu_online(rq->mq_ctx->cpu);
1279	}
1280
1281	static void blk_mq_complete_send_ipi(struct request *rq)
1282	{
1283	unsigned int cpu;
1284
1285	cpu = rq->mq_ctx->cpu;
1286	if (llist_add(&rq->ipi_list, &per_cpu(blk_cpu_done, cpu)))
1287	smp_call_function_single_async(cpu, &per_cpu(blk_cpu_csd, cpu));
1288	}
1289
1290	static void blk_mq_raise_softirq(struct request *rq)
1291	{
1292	struct llist_head *list;
1293
1294	preempt_disable();
1295	list = this_cpu_ptr(&blk_cpu_done);
1296	if (llist_add(&rq->ipi_list, list))
1297	raise_softirq(BLOCK_SOFTIRQ);
1298	preempt_enable();
1299	}
1300
1301	bool blk_mq_complete_request_remote(struct request *rq)
1302	{
1303	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
1304
1305	/*
1306	* For request which hctx has only one ctx mapping,
1307	* or a polled request, always complete locally,
1308	* it's pointless to redirect the completion.
1309	*/
1310	if ((rq->mq_hctx->nr_ctx == 1 &&
1311	rq->mq_ctx->cpu == raw_smp_processor_id()) \|\|
1312	rq->cmd_flags & REQ_POLLED)
1313	return false;
1314
1315	if (blk_mq_complete_need_ipi(rq)) {
1316	blk_mq_complete_send_ipi(rq);
1317	return true;
1318	}
1319
1320	if (rq->q->nr_hw_queues == 1) {
1321	blk_mq_raise_softirq(rq);
1322	return true;
1323	}
1324	return false;
1325	}
1326	EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
1327
1328	/**
1329	* blk_mq_complete_request - end I/O on a request
1330	* @rq: the request being processed
1331	*
1332	* Description:
1333	* Complete a request by scheduling the ->complete_rq operation.
1334	**/
1335	void blk_mq_complete_request(struct request *rq)
1336	{
1337	if (!blk_mq_complete_request_remote(rq))
1338	rq->q->mq_ops->complete(rq);
1339	}
1340	EXPORT_SYMBOL(blk_mq_complete_request);
1341
1342	/**
1343	* blk_mq_start_request - Start processing a request
1344	* @rq: Pointer to request to be started
1345	*
1346	* Function used by device drivers to notify the block layer that a request
1347	* is going to be processed now, so blk layer can do proper initializations
1348	* such as starting the timeout timer.
1349	*/
1350	void blk_mq_start_request(struct request *rq)
1351	{
1352	struct request_queue *q = rq->q;
1353
1354	trace_block_rq_issue(rq);
1355
1356	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags) &&
1357	!blk_rq_is_passthrough(rq)) {
1358	rq->io_start_time_ns = blk_time_get_ns();
1359	rq->stats_sectors = blk_rq_sectors(rq);
1360	rq->rq_flags \|= RQF_STATS;
1361	rq_qos_issue(q, rq);
1362	}
1363
1364	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
1365
1366	blk_add_timer(rq);
1367	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
1368	rq->mq_hctx->tags->rqs[rq->tag] = rq;
1369
1370	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
1371	blk_integrity_prepare(rq);
1372
1373	if (rq->bio && rq->bio->bi_opf & REQ_POLLED)
1374	WRITE_ONCE(rq->bio->bi_cookie, rq->mq_hctx->queue_num);
1375	}
1376	EXPORT_SYMBOL(blk_mq_start_request);
1377
1378	/*
1379	* Allow 2x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
1380	* queues. This is important for md arrays to benefit from merging
1381	* requests.
1382	*/
1383	static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
1384	{
1385	if (plug->multiple_queues)
1386	return BLK_MAX_REQUEST_COUNT * 2;
1387	return BLK_MAX_REQUEST_COUNT;
1388	}
1389
1390	static void blk_add_rq_to_plug(struct blk_plug plug, struct request rq)
1391	{
1392	struct request *last = rq_list_peek(&plug->mq_list);
1393
1394	if (!plug->rq_count) {
1395	trace_block_plug(rq->q);
1396	} else if (plug->rq_count >= blk_plug_max_rq_count(plug) \|\|
1397	(!blk_queue_nomerges(rq->q) &&
1398	blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1399	blk_mq_flush_plug_list(plug, false);
1400	last = NULL;
1401	trace_block_plug(rq->q);
1402	}
1403
1404	if (!plug->multiple_queues && last && last->q != rq->q)
1405	plug->multiple_queues = true;
1406	/*
1407	* Any request allocated from sched tags can't be issued to
1408	* ->queue_rqs() directly
1409	*/
1410	if (!plug->has_elevator && (rq->rq_flags & RQF_SCHED_TAGS))
1411	plug->has_elevator = true;
1412	rq_list_add_tail(&plug->mq_list, rq);
1413	plug->rq_count++;
1414	}
1415
1416	/**
1417	* blk_execute_rq_nowait - insert a request to I/O scheduler for execution
1418	* @rq: request to insert
1419	* @at_head: insert request at head or tail of queue
1420	*
1421	* Description:
1422	* Insert a fully prepared request at the back of the I/O scheduler queue
1423	* for execution. Don't wait for completion.
1424	*
1425	* Note:
1426	* This function will invoke @done directly if the queue is dead.
1427	*/
1428	void blk_execute_rq_nowait(struct request *rq, bool at_head)
1429	{
1430	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1431
1432	WARN_ON(irqs_disabled());
1433	WARN_ON(!blk_rq_is_passthrough(rq));
1434
1435	blk_account_io_start(rq);
1436
1437	if (current->plug && !at_head) {
1438	blk_add_rq_to_plug(current->plug, rq);
1439	return;
1440	}
1441
1442	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1443	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
1444	}
1445	EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
1446
1447	struct blk_rq_wait {
1448	struct completion done;
1449	blk_status_t ret;
1450	};
1451
1452	static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret,
1453	const struct io_comp_batch *iob)
1454	{
1455	struct blk_rq_wait *wait = rq->end_io_data;
1456
1457	wait->ret = ret;
1458	complete(&wait->done);
1459	return RQ_END_IO_NONE;
1460	}
1461
1462	bool blk_rq_is_poll(struct request *rq)
1463	{
1464	if (!rq->mq_hctx)
1465	return false;
1466	if (rq->mq_hctx->type != HCTX_TYPE_POLL)
1467	return false;
1468	return true;
1469	}
1470	EXPORT_SYMBOL_GPL(blk_rq_is_poll);
1471
1472	static void blk_rq_poll_completion(struct request rq, struct completion wait)
1473	{
1474	do {
1475	blk_hctx_poll(rq->q, rq->mq_hctx, NULL, BLK_POLL_ONESHOT);
1476	cond_resched();
1477	} while (!completion_done(wait));
1478	}
1479
1480	/**
1481	* blk_execute_rq - insert a request into queue for execution
1482	* @rq: request to insert
1483	* @at_head: insert request at head or tail of queue
1484	*
1485	* Description:
1486	* Insert a fully prepared request at the back of the I/O scheduler queue
1487	* for execution and wait for completion.
1488	* Return: The blk_status_t result provided to blk_mq_end_request().
1489	*/
1490	blk_status_t blk_execute_rq(struct request *rq, bool at_head)
1491	{
1492	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1493	struct blk_rq_wait wait = {
1494	.done = COMPLETION_INITIALIZER_ONSTACK(wait.done),
1495	};
1496
1497	WARN_ON(irqs_disabled());
1498	WARN_ON(!blk_rq_is_passthrough(rq));
1499
1500	rq->end_io_data = &wait;
1501	rq->end_io = blk_end_sync_rq;
1502
1503	blk_account_io_start(rq);
1504	blk_mq_insert_request(rq, at_head ? BLK_MQ_INSERT_AT_HEAD : 0);
1505	blk_mq_run_hw_queue(hctx, false);
1506
1507	if (blk_rq_is_poll(rq))
1508	blk_rq_poll_completion(rq, &wait.done);
1509	else
1510	blk_wait_io(&wait.done);
1511
1512	return wait.ret;
1513	}
1514	EXPORT_SYMBOL(blk_execute_rq);
1515
1516	static void __blk_mq_requeue_request(struct request *rq)
1517	{
1518	struct request_queue *q = rq->q;
1519
1520	blk_mq_put_driver_tag(rq);
1521
1522	trace_block_rq_requeue(rq);
1523	rq_qos_requeue(q, rq);
1524
1525	if (blk_mq_request_started(rq)) {
1526	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
1527	rq->rq_flags &= ~RQF_TIMED_OUT;
1528	}
1529	}
1530
1531	void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
1532	{
1533	struct request_queue *q = rq->q;
1534	unsigned long flags;
1535
1536	__blk_mq_requeue_request(rq);
1537
1538	/* this request will be re-inserted to io scheduler queue */
1539	blk_mq_sched_requeue_request(rq);
1540
1541	spin_lock_irqsave(&q->requeue_lock, flags);
1542	list_add_tail(&rq->queuelist, &q->requeue_list);
1543	spin_unlock_irqrestore(&q->requeue_lock, flags);
1544
1545	if (kick_requeue_list)
1546	blk_mq_kick_requeue_list(q);
1547	}
1548	EXPORT_SYMBOL(blk_mq_requeue_request);
1549
1550	static void blk_mq_requeue_work(struct work_struct *work)
1551	{
1552	struct request_queue *q =
1553	container_of(work, struct request_queue, requeue_work.work);
1554	LIST_HEAD(rq_list);
1555	LIST_HEAD(flush_list);
1556	struct request *rq;
1557
1558	spin_lock_irq(&q->requeue_lock);
1559	list_splice_init(&q->requeue_list, &rq_list);
1560	list_splice_init(&q->flush_list, &flush_list);
1561	spin_unlock_irq(&q->requeue_lock);
1562
1563	while (!list_empty(&rq_list)) {
1564	rq = list_entry(rq_list.next, struct request, queuelist);
1565	list_del_init(&rq->queuelist);
1566	/*
1567	* If RQF_DONTPREP is set, the request has been started by the
1568	* driver already and might have driver-specific data allocated
1569	* already. Insert it into the hctx dispatch list to avoid
1570	* block layer merges for the request.
1571	*/
1572	if (rq->rq_flags & RQF_DONTPREP)
1573	blk_mq_request_bypass_insert(rq, 0);
1574	else
1575	blk_mq_insert_request(rq, BLK_MQ_INSERT_AT_HEAD);
1576	}
1577
1578	while (!list_empty(&flush_list)) {
1579	rq = list_entry(flush_list.next, struct request, queuelist);
1580	list_del_init(&rq->queuelist);
1581	blk_mq_insert_request(rq, 0);
1582	}
1583
1584	blk_mq_run_hw_queues(q, false);
1585	}
1586
1587	void blk_mq_kick_requeue_list(struct request_queue *q)
1588	{
1589	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
1590	}
1591	EXPORT_SYMBOL(blk_mq_kick_requeue_list);
1592
1593	void blk_mq_delay_kick_requeue_list(struct request_queue *q,
1594	unsigned long msecs)
1595	{
1596	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
1597	msecs_to_jiffies(msecs));
1598	}
1599	EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);
1600
1601	static bool blk_is_flush_data_rq(struct request *rq)
1602	{
1603	return (rq->rq_flags & RQF_FLUSH_SEQ) && !is_flush_rq(rq);
1604	}
1605
1606	static bool blk_mq_rq_inflight(struct request rq, void priv)
1607	{
1608	/*
1609	* If we find a request that isn't idle we know the queue is busy
1610	* as it's checked in the iter.
1611	* Return false to stop the iteration.
1612	*
1613	* In case of queue quiesce, if one flush data request is completed,
1614	* don't count it as inflight given the flush sequence is suspended,
1615	* and the original flush data request is invisible to driver, just
1616	* like other pending requests because of quiesce
1617	*/
1618	if (blk_mq_request_started(rq) && !(blk_queue_quiesced(rq->q) &&
1619	blk_is_flush_data_rq(rq) &&
1620	blk_mq_request_completed(rq))) {
1621	bool *busy = priv;
1622
1623	*busy = true;
1624	return false;
1625	}
1626
1627	return true;
1628	}
1629
1630	bool blk_mq_queue_inflight(struct request_queue *q)
1631	{
1632	bool busy = false;
1633
1634	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
1635	return busy;
1636	}
1637	EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
1638
1639	static void blk_mq_rq_timed_out(struct request *req)
1640	{
1641	req->rq_flags \|= RQF_TIMED_OUT;
1642	if (req->q->mq_ops->timeout) {
1643	enum blk_eh_timer_return ret;
1644
1645	ret = req->q->mq_ops->timeout(req);
1646	if (ret == BLK_EH_DONE)
1647	return;
1648	WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
1649	}
1650
1651	blk_add_timer(req);
1652	}
1653
1654	struct blk_expired_data {
1655	bool has_timedout_rq;
1656	unsigned long next;
1657	unsigned long timeout_start;
1658	};
1659
1660	static bool blk_mq_req_expired(struct request rq, struct blk_expired_data expired)
1661	{
1662	unsigned long deadline;
1663
1664	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
1665	return false;
1666	if (rq->rq_flags & RQF_TIMED_OUT)
1667	return false;
1668
1669	deadline = READ_ONCE(rq->deadline);
1670	if (time_after_eq(expired->timeout_start, deadline))
1671	return true;
1672
1673	if (expired->next == 0)
1674	expired->next = deadline;
1675	else if (time_after(expired->next, deadline))
1676	expired->next = deadline;
1677	return false;
1678	}
1679
1680	void blk_mq_put_rq_ref(struct request *rq)
1681	{
1682	if (is_flush_rq(rq)) {
1683	if (rq->end_io(rq, 0, NULL) == RQ_END_IO_FREE)
1684	blk_mq_free_request(rq);
1685	} else if (req_ref_put_and_test(rq)) {
1686	__blk_mq_free_request(rq);
1687	}
1688	}
1689
1690	static bool blk_mq_check_expired(struct request rq, void priv)
1691	{
1692	struct blk_expired_data *expired = priv;
1693
1694	/*
1695	* blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
1696	* be reallocated underneath the timeout handler's processing, then
1697	* the expire check is reliable. If the request is not expired, then
1698	* it was completed and reallocated as a new request after returning
1699	* from blk_mq_check_expired().
1700	*/
1701	if (blk_mq_req_expired(rq, expired)) {
1702	expired->has_timedout_rq = true;
1703	return false;
1704	}
1705	return true;
1706	}
1707
1708	static bool blk_mq_handle_expired(struct request rq, void priv)
1709	{
1710	struct blk_expired_data *expired = priv;
1711
1712	if (blk_mq_req_expired(rq, expired))
1713	blk_mq_rq_timed_out(rq);
1714	return true;
1715	}
1716
1717	static void blk_mq_timeout_work(struct work_struct *work)
1718	{
1719	struct request_queue *q =
1720	container_of(work, struct request_queue, timeout_work);
1721	struct blk_expired_data expired = {
1722	.timeout_start = jiffies,
1723	};
1724	struct blk_mq_hw_ctx *hctx;
1725	unsigned long i;
1726
1727	/* A deadlock might occur if a request is stuck requiring a
1728	* timeout at the same time a queue freeze is waiting
1729	* completion, since the timeout code would not be able to
1730	* acquire the queue reference here.
1731	*
1732	* That's why we don't use blk_queue_enter here; instead, we use
1733	* percpu_ref_tryget directly, because we need to be able to
1734	* obtain a reference even in the short window between the queue
1735	* starting to freeze, by dropping the first reference in
1736	* blk_freeze_queue_start, and the moment the last request is
1737	* consumed, marked by the instant q_usage_counter reaches
1738	* zero.
1739	*/
1740	if (!percpu_ref_tryget(&q->q_usage_counter))
1741	return;
1742
1743	/* check if there is any timed-out request */
1744	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &expired);
1745	if (expired.has_timedout_rq) {
1746	/*
1747	* Before walking tags, we must ensure any submit started
1748	* before the current time has finished. Since the submit
1749	* uses srcu or rcu, wait for a synchronization point to
1750	* ensure all running submits have finished
1751	*/
1752	blk_mq_wait_quiesce_done(q->tag_set);
1753
1754	expired.next = 0;
1755	blk_mq_queue_tag_busy_iter(q, blk_mq_handle_expired, &expired);
1756	}
1757
1758	if (expired.next != 0) {
1759	mod_timer(&q->timeout, expired.next);
1760	} else {
1761	/*
1762	* Request timeouts are handled as a forward rolling timer. If
1763	* we end up here it means that no requests are pending and
1764	* also that no request has been pending for a while. Mark
1765	* each hctx as idle.
1766	*/
1767	queue_for_each_hw_ctx(q, hctx, i) {
1768	/* the hctx may be unmapped, so check it here */
1769	if (blk_mq_hw_queue_mapped(hctx))
1770	blk_mq_tag_idle(hctx);
1771	}
1772	}
1773	blk_queue_exit(q);
1774	}
1775
1776	struct flush_busy_ctx_data {
1777	struct blk_mq_hw_ctx *hctx;
1778	struct list_head *list;
1779	};
1780
1781	static bool flush_busy_ctx(struct sbitmap sb, unsigned int bitnr, void data)
1782	{
1783	struct flush_busy_ctx_data *flush_data = data;
1784	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
1785	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1786	enum hctx_type type = hctx->type;
1787
1788	spin_lock(&ctx->lock);
1789	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1790	sbitmap_clear_bit(sb, bitnr);
1791	spin_unlock(&ctx->lock);
1792	return true;
1793	}
1794
1795	/*
1796	* Process software queues that have been marked busy, splicing them
1797	* to the for-dispatch
1798	*/
1799	void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx hctx, struct list_head list)
1800	{
1801	struct flush_busy_ctx_data data = {
1802	.hctx = hctx,
1803	.list = list,
1804	};
1805
1806	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1807	}
1808
1809	struct dispatch_rq_data {
1810	struct blk_mq_hw_ctx *hctx;
1811	struct request *rq;
1812	};
1813
1814	static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
1815	void *data)
1816	{
1817	struct dispatch_rq_data *dispatch_data = data;
1818	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
1819	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
1820	enum hctx_type type = hctx->type;
1821
1822	spin_lock(&ctx->lock);
1823	if (!list_empty(&ctx->rq_lists[type])) {
1824	dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1825	list_del_init(&dispatch_data->rq->queuelist);
1826	if (list_empty(&ctx->rq_lists[type]))
1827	sbitmap_clear_bit(sb, bitnr);
1828	}
1829	spin_unlock(&ctx->lock);
1830
1831	return !dispatch_data->rq;
1832	}
1833
1834	struct request blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx hctx,
1835	struct blk_mq_ctx *start)
1836	{
1837	unsigned off = start ? start->index_hw[hctx->type] : 0;
1838	struct dispatch_rq_data data = {
1839	.hctx = hctx,
1840	.rq = NULL,
1841	};
1842
1843	__sbitmap_for_each_set(&hctx->ctx_map, off,
1844	dispatch_rq_from_ctx, &data);
1845
1846	return data.rq;
1847	}
1848
1849	bool __blk_mq_alloc_driver_tag(struct request *rq)
1850	{
1851	struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags;
1852	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
1853	int tag;
1854
1855	blk_mq_tag_busy(rq->mq_hctx);
1856
1857	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1858	bt = &rq->mq_hctx->tags->breserved_tags;
1859	tag_offset = 0;
1860	} else {
1861	if (!hctx_may_queue(rq->mq_hctx, bt))
1862	return false;
1863	}
1864
1865	tag = __sbitmap_queue_get(bt);
1866	if (tag == BLK_MQ_NO_TAG)
1867	return false;
1868
1869	rq->tag = tag + tag_offset;
1870	blk_mq_inc_active_requests(rq->mq_hctx);
1871	return true;
1872	}
1873
1874	static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
1875	int flags, void *key)
1876	{
1877	struct blk_mq_hw_ctx *hctx;
1878
1879	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);
1880
1881	spin_lock(&hctx->dispatch_wait_lock);
1882	if (!list_empty(&wait->entry)) {
1883	struct sbitmap_queue *sbq;
1884
1885	list_del_init(&wait->entry);
1886	sbq = &hctx->tags->bitmap_tags;
1887	atomic_dec(&sbq->ws_active);
1888	}
1889	spin_unlock(&hctx->dispatch_wait_lock);
1890
1891	blk_mq_run_hw_queue(hctx, true);
1892	return 1;
1893	}
1894
1895	/*
1896	* Mark us waiting for a tag. For shared tags, this involves hooking us into
1897	* the tag wakeups. For non-shared tags, we can simply mark us needing a
1898	* restart. For both cases, take care to check the condition again after
1899	* marking us as waiting.
1900	*/
1901	static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1902	struct request *rq)
1903	{
1904	struct sbitmap_queue *sbq;
1905	struct wait_queue_head *wq;
1906	wait_queue_entry_t *wait;
1907	bool ret;
1908
1909	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1910	!(blk_mq_is_shared_tags(hctx->flags))) {
1911	blk_mq_sched_mark_restart_hctx(hctx);
1912
1913	/*
1914	* It's possible that a tag was freed in the window between the
1915	* allocation failure and adding the hardware queue to the wait
1916	* queue.
1917	*
1918	* Don't clear RESTART here, someone else could have set it.
1919	* At most this will cost an extra queue run.
1920	*/
1921	return blk_mq_get_driver_tag(rq);
1922	}
1923
1924	wait = &hctx->dispatch_wait;
1925	if (!list_empty_careful(&wait->entry))
1926	return false;
1927
1928	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag))
1929	sbq = &hctx->tags->breserved_tags;
1930	else
1931	sbq = &hctx->tags->bitmap_tags;
1932	wq = &bt_wait_ptr(sbq, hctx)->wait;
1933
1934	spin_lock_irq(&wq->lock);
1935	spin_lock(&hctx->dispatch_wait_lock);
1936	if (!list_empty(&wait->entry)) {
1937	spin_unlock(&hctx->dispatch_wait_lock);
1938	spin_unlock_irq(&wq->lock);
1939	return false;
1940	}
1941
1942	atomic_inc(&sbq->ws_active);
1943	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
1944	__add_wait_queue(wq, wait);
1945
1946	/*
1947	* Add one explicit barrier since blk_mq_get_driver_tag() may
1948	* not imply barrier in case of failure.
1949	*
1950	* Order adding us to wait queue and allocating driver tag.
1951	*
1952	* The pair is the one implied in sbitmap_queue_wake_up() which
1953	* orders clearing sbitmap tag bits and waitqueue_active() in
1954	* __sbitmap_queue_wake_up(), since waitqueue_active() is lockless
1955	*
1956	* Otherwise, re-order of adding wait queue and getting driver tag
1957	* may cause __sbitmap_queue_wake_up() to wake up nothing because
1958	* the waitqueue_active() may not observe us in wait queue.
1959	*/
1960	smp_mb();
1961
1962	/*
1963	* It's possible that a tag was freed in the window between the
1964	* allocation failure and adding the hardware queue to the wait
1965	* queue.
1966	*/
1967	ret = blk_mq_get_driver_tag(rq);
1968	if (!ret) {
1969	spin_unlock(&hctx->dispatch_wait_lock);
1970	spin_unlock_irq(&wq->lock);
1971	return false;
1972	}
1973
1974	/*
1975	* We got a tag, remove ourselves from the wait queue to ensure
1976	* someone else gets the wakeup.
1977	*/
1978	list_del_init(&wait->entry);
1979	atomic_dec(&sbq->ws_active);
1980	spin_unlock(&hctx->dispatch_wait_lock);
1981	spin_unlock_irq(&wq->lock);
1982
1983	return true;
1984	}
1985
1986	#define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT 8
1987	#define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR 4
1988	/*
1989	* Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
1990	* - EWMA is one simple way to compute running average value
1991	* - weight(7/8 and 1/8) is applied so that it can decrease exponentially
1992	* - take 4 as factor for avoiding to get too small(0) result, and this
1993	* factor doesn't matter because EWMA decreases exponentially
1994	*/
1995	static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
1996	{
1997	unsigned int ewma;
1998
1999	ewma = hctx->dispatch_busy;
2000
2001	if (!ewma && !busy)
2002	return;
2003
2004	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
2005	if (busy)
2006	ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
2007	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;
2008
2009	hctx->dispatch_busy = ewma;
2010	}
2011
2012	#define BLK_MQ_RESOURCE_DELAY 3 /* ms units */
2013
2014	static void blk_mq_handle_dev_resource(struct request *rq,
2015	struct list_head *list)
2016	{
2017	list_add(&rq->queuelist, list);
2018	__blk_mq_requeue_request(rq);
2019	}
2020
2021	enum prep_dispatch {
2022	PREP_DISPATCH_OK,
2023	PREP_DISPATCH_NO_TAG,
2024	PREP_DISPATCH_NO_BUDGET,
2025	};
2026
2027	static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
2028	bool need_budget)
2029	{
2030	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2031	int budget_token = -1;
2032
2033	if (need_budget) {
2034	budget_token = blk_mq_get_dispatch_budget(rq->q);
2035	if (budget_token < 0) {
2036	blk_mq_put_driver_tag(rq);
2037	return PREP_DISPATCH_NO_BUDGET;
2038	}
2039	blk_mq_set_rq_budget_token(rq, budget_token);
2040	}
2041
2042	if (!blk_mq_get_driver_tag(rq)) {
2043	/*
2044	* The initial allocation attempt failed, so we need to
2045	* rerun the hardware queue when a tag is freed. The
2046	* waitqueue takes care of that. If the queue is run
2047	* before we add this entry back on the dispatch list,
2048	* we'll re-run it below.
2049	*/
2050	if (!blk_mq_mark_tag_wait(hctx, rq)) {
2051	/*
2052	* All budgets not got from this function will be put
2053	* together during handling partial dispatch
2054	*/
2055	if (need_budget)
2056	blk_mq_put_dispatch_budget(rq->q, budget_token);
2057	return PREP_DISPATCH_NO_TAG;
2058	}
2059	}
2060
2061	return PREP_DISPATCH_OK;
2062	}
2063
2064	/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
2065	static void blk_mq_release_budgets(struct request_queue *q,
2066	struct list_head *list)
2067	{
2068	struct request *rq;
2069
2070	list_for_each_entry(rq, list, queuelist) {
2071	int budget_token = blk_mq_get_rq_budget_token(rq);
2072
2073	if (budget_token >= 0)
2074	blk_mq_put_dispatch_budget(q, budget_token);
2075	}
2076	}
2077
2078	/*
2079	* blk_mq_commit_rqs will notify driver using bd->last that there is no
2080	* more requests. (See comment in struct blk_mq_ops for commit_rqs for
2081	* details)
2082	* Attention, we should explicitly call this in unusual cases:
2083	* 1) did not queue everything initially scheduled to queue
2084	* 2) the last attempt to queue a request failed
2085	*/
2086	static void blk_mq_commit_rqs(struct blk_mq_hw_ctx *hctx, int queued,
2087	bool from_schedule)
2088	{
2089	if (hctx->queue->mq_ops->commit_rqs && queued) {
2090	trace_block_unplug(hctx->queue, queued, !from_schedule);
2091	hctx->queue->mq_ops->commit_rqs(hctx);
2092	}
2093	}
2094
2095	/*
2096	* Returns true if we did some work AND can potentially do more.
2097	*/
2098	bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx hctx, struct list_head list,
2099	bool get_budget)
2100	{
2101	enum prep_dispatch prep;
2102	struct request_queue *q = hctx->queue;
2103	struct request *rq;
2104	int queued;
2105	blk_status_t ret = BLK_STS_OK;
2106	bool needs_resource = false;
2107
2108	if (list_empty(list))
2109	return false;
2110
2111	/*
2112	* Now process all the entries, sending them to the driver.
2113	*/
2114	queued = 0;
2115	do {
2116	struct blk_mq_queue_data bd;
2117
2118	rq = list_first_entry(list, struct request, queuelist);
2119
2120	WARN_ON_ONCE(hctx != rq->mq_hctx);
2121	prep = blk_mq_prep_dispatch_rq(rq, get_budget);
2122	if (prep != PREP_DISPATCH_OK)
2123	break;
2124
2125	list_del_init(&rq->queuelist);
2126
2127	bd.rq = rq;
2128	bd.last = list_empty(list);
2129
2130	ret = q->mq_ops->queue_rq(hctx, &bd);
2131	switch (ret) {
2132	case BLK_STS_OK:
2133	queued++;
2134	break;
2135	case BLK_STS_RESOURCE:
2136	needs_resource = true;
2137	fallthrough;
2138	case BLK_STS_DEV_RESOURCE:
2139	blk_mq_handle_dev_resource(rq, list);
2140	goto out;
2141	default:
2142	blk_mq_end_request(rq, ret);
2143	}
2144	} while (!list_empty(list));
2145	out:
2146	/* If we didn't flush the entire list, we could have told the driver
2147	* there was more coming, but that turned out to be a lie.
2148	*/
2149	if (!list_empty(list) \|\| ret != BLK_STS_OK)
2150	blk_mq_commit_rqs(hctx, queued, false);
2151
2152	/*
2153	* Any items that need requeuing? Stuff them into hctx->dispatch,
2154	* that is where we will continue on next queue run.
2155	*/
2156	if (!list_empty(list)) {
2157	bool needs_restart;
2158	/* For non-shared tags, the RESTART check will suffice */
2159	bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
2160	((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) \|\|
2161	blk_mq_is_shared_tags(hctx->flags));
2162
2163	/*
2164	* If the caller allocated budgets, free the budgets of the
2165	* requests that have not yet been passed to the block driver.
2166	*/
2167	if (!get_budget)
2168	blk_mq_release_budgets(q, list);
2169
2170	spin_lock(&hctx->lock);
2171	list_splice_tail_init(list, &hctx->dispatch);
2172	spin_unlock(&hctx->lock);
2173
2174	/*
2175	* Order adding requests to hctx->dispatch and checking
2176	* SCHED_RESTART flag. The pair of this smp_mb() is the one
2177	* in blk_mq_sched_restart(). Avoid restart code path to
2178	* miss the new added requests to hctx->dispatch, meantime
2179	* SCHED_RESTART is observed here.
2180	*/
2181	smp_mb();
2182
2183	/*
2184	* If SCHED_RESTART was set by the caller of this function and
2185	* it is no longer set that means that it was cleared by another
2186	* thread and hence that a queue rerun is needed.
2187	*
2188	* If 'no_tag' is set, that means that we failed getting
2189	* a driver tag with an I/O scheduler attached. If our dispatch
2190	* waitqueue is no longer active, ensure that we run the queue
2191	* AFTER adding our entries back to the list.
2192	*
2193	* If no I/O scheduler has been configured it is possible that
2194	* the hardware queue got stopped and restarted before requests
2195	* were pushed back onto the dispatch list. Rerun the queue to
2196	* avoid starvation. Notes:
2197	* - blk_mq_run_hw_queue() checks whether or not a queue has
2198	* been stopped before rerunning a queue.
2199	* - Some but not all block drivers stop a queue before
2200	* returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
2201	* and dm-rq.
2202	*
2203	* If driver returns BLK_STS_RESOURCE and SCHED_RESTART
2204	* bit is set, run queue after a delay to avoid IO stalls
2205	* that could otherwise occur if the queue is idle. We'll do
2206	* similar if we couldn't get budget or couldn't lock a zone
2207	* and SCHED_RESTART is set.
2208	*/
2209	needs_restart = blk_mq_sched_needs_restart(hctx);
2210	if (prep == PREP_DISPATCH_NO_BUDGET)
2211	needs_resource = true;
2212	if (!needs_restart \|\|
2213	(no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
2214	blk_mq_run_hw_queue(hctx, true);
2215	else if (needs_resource)
2216	blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
2217
2218	blk_mq_update_dispatch_busy(hctx, true);
2219	return false;
2220	}
2221
2222	blk_mq_update_dispatch_busy(hctx, false);
2223	return true;
2224	}
2225
2226	static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
2227	{
2228	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);
2229
2230	if (cpu >= nr_cpu_ids)
2231	cpu = cpumask_first(hctx->cpumask);
2232	return cpu;
2233	}
2234
2235	/*
2236	* ->next_cpu is always calculated from hctx->cpumask, so simply use
2237	* it for speeding up the check
2238	*/
2239	static bool blk_mq_hctx_empty_cpumask(struct blk_mq_hw_ctx *hctx)
2240	{
2241	return hctx->next_cpu >= nr_cpu_ids;
2242	}
2243
2244	/*
2245	* It'd be great if the workqueue API had a way to pass
2246	* in a mask and had some smarts for more clever placement.
2247	* For now we just round-robin here, switching for every
2248	* BLK_MQ_CPU_WORK_BATCH queued items.
2249	*/
2250	static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
2251	{
2252	bool tried = false;
2253	int next_cpu = hctx->next_cpu;
2254
2255	/* Switch to unbound if no allowable CPUs in this hctx */
2256	if (hctx->queue->nr_hw_queues == 1 \|\| blk_mq_hctx_empty_cpumask(hctx))
2257	return WORK_CPU_UNBOUND;
2258
2259	if (--hctx->next_cpu_batch <= 0) {
2260	select_cpu:
2261	next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
2262	cpu_online_mask);
2263	if (next_cpu >= nr_cpu_ids)
2264	next_cpu = blk_mq_first_mapped_cpu(hctx);
2265	hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
2266	}
2267
2268	/*
2269	* Do unbound schedule if we can't find a online CPU for this hctx,
2270	* and it should only happen in the path of handling CPU DEAD.
2271	*/
2272	if (!cpu_online(next_cpu)) {
2273	if (!tried) {
2274	tried = true;
2275	goto select_cpu;
2276	}
2277
2278	/*
2279	* Make sure to re-select CPU next time once after CPUs
2280	* in hctx->cpumask become online again.
2281	*/
2282	hctx->next_cpu = next_cpu;
2283	hctx->next_cpu_batch = 1;
2284	return WORK_CPU_UNBOUND;
2285	}
2286
2287	hctx->next_cpu = next_cpu;
2288	return next_cpu;
2289	}
2290
2291	/**
2292	* blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
2293	* @hctx: Pointer to the hardware queue to run.
2294	* @msecs: Milliseconds of delay to wait before running the queue.
2295	*
2296	* Run a hardware queue asynchronously with a delay of @msecs.
2297	*/
2298	void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
2299	{
2300	if (unlikely(blk_mq_hctx_stopped(hctx)))
2301	return;
2302	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
2303	msecs_to_jiffies(msecs));
2304	}
2305	EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
2306
2307	static inline bool blk_mq_hw_queue_need_run(struct blk_mq_hw_ctx *hctx)
2308	{
2309	bool need_run;
2310
2311	/*
2312	* When queue is quiesced, we may be switching io scheduler, or
2313	* updating nr_hw_queues, or other things, and we can't run queue
2314	* any more, even blk_mq_hctx_has_pending() can't be called safely.
2315	*
2316	* And queue will be rerun in blk_mq_unquiesce_queue() if it is
2317	* quiesced.
2318	*/
2319	__blk_mq_run_dispatch_ops(hctx->queue, false,
2320	need_run = !blk_queue_quiesced(hctx->queue) &&
2321	blk_mq_hctx_has_pending(hctx));
2322	return need_run;
2323	}
2324
2325	/**
2326	* blk_mq_run_hw_queue - Start to run a hardware queue.
2327	* @hctx: Pointer to the hardware queue to run.
2328	* @async: If we want to run the queue asynchronously.
2329	*
2330	* Check if the request queue is not in a quiesced state and if there are
2331	* pending requests to be sent. If this is true, run the queue to send requests
2332	* to hardware.
2333	*/
2334	void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2335	{
2336	bool need_run;
2337
2338	/*
2339	* We can't run the queue inline with interrupts disabled.
2340	*/
2341	WARN_ON_ONCE(!async && in_interrupt());
2342
2343	might_sleep_if(!async && hctx->flags & BLK_MQ_F_BLOCKING);
2344
2345	need_run = blk_mq_hw_queue_need_run(hctx);
2346	if (!need_run) {
2347	unsigned long flags;
2348
2349	/*
2350	* Synchronize with blk_mq_unquiesce_queue(), because we check
2351	* if hw queue is quiesced locklessly above, we need the use
2352	* ->queue_lock to make sure we see the up-to-date status to
2353	* not miss rerunning the hw queue.
2354	*/
2355	spin_lock_irqsave(&hctx->queue->queue_lock, flags);
2356	need_run = blk_mq_hw_queue_need_run(hctx);
2357	spin_unlock_irqrestore(&hctx->queue->queue_lock, flags);
2358
2359	if (!need_run)
2360	return;
2361	}
2362
2363	if (async \|\| !cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask)) {
2364	blk_mq_delay_run_hw_queue(hctx, 0);
2365	return;
2366	}
2367
2368	blk_mq_run_dispatch_ops(hctx->queue,
2369	blk_mq_sched_dispatch_requests(hctx));
2370	}
2371	EXPORT_SYMBOL(blk_mq_run_hw_queue);
2372
2373	/*
2374	* Return prefered queue to dispatch from (if any) for non-mq aware IO
2375	* scheduler.
2376	*/
2377	static struct blk_mq_hw_ctx blk_mq_get_sq_hctx(struct request_queue q)
2378	{
2379	struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
2380	/*
2381	* If the IO scheduler does not respect hardware queues when
2382	* dispatching, we just don't bother with multiple HW queues and
2383	* dispatch from hctx for the current CPU since running multiple queues
2384	* just causes lock contention inside the scheduler and pointless cache
2385	* bouncing.
2386	*/
2387	struct blk_mq_hw_ctx *hctx = ctx->hctxs[HCTX_TYPE_DEFAULT];
2388
2389	if (!blk_mq_hctx_stopped(hctx))
2390	return hctx;
2391	return NULL;
2392	}
2393
2394	/**
2395	* blk_mq_run_hw_queues - Run all hardware queues in a request queue.
2396	* @q: Pointer to the request queue to run.
2397	* @async: If we want to run the queue asynchronously.
2398	*/
2399	void blk_mq_run_hw_queues(struct request_queue *q, bool async)
2400	{
2401	struct blk_mq_hw_ctx hctx, sq_hctx;
2402	unsigned long i;
2403
2404	sq_hctx = NULL;
2405	if (blk_queue_sq_sched(q))
2406	sq_hctx = blk_mq_get_sq_hctx(q);
2407	queue_for_each_hw_ctx(q, hctx, i) {
2408	if (blk_mq_hctx_stopped(hctx))
2409	continue;
2410	/*
2411	* Dispatch from this hctx either if there's no hctx preferred
2412	* by IO scheduler or if it has requests that bypass the
2413	* scheduler.
2414	*/
2415	if (!sq_hctx \|\| sq_hctx == hctx \|\|
2416	!list_empty_careful(&hctx->dispatch))
2417	blk_mq_run_hw_queue(hctx, async);
2418	}
2419	}
2420	EXPORT_SYMBOL(blk_mq_run_hw_queues);
2421
2422	/**
2423	* blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
2424	* @q: Pointer to the request queue to run.
2425	* @msecs: Milliseconds of delay to wait before running the queues.
2426	*/
2427	void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
2428	{
2429	struct blk_mq_hw_ctx hctx, sq_hctx;
2430	unsigned long i;
2431
2432	sq_hctx = NULL;
2433	if (blk_queue_sq_sched(q))
2434	sq_hctx = blk_mq_get_sq_hctx(q);
2435	queue_for_each_hw_ctx(q, hctx, i) {
2436	if (blk_mq_hctx_stopped(hctx))
2437	continue;
2438	/*
2439	* If there is already a run_work pending, leave the
2440	* pending delay untouched. Otherwise, a hctx can stall
2441	* if another hctx is re-delaying the other's work
2442	* before the work executes.
2443	*/
2444	if (delayed_work_pending(&hctx->run_work))
2445	continue;
2446	/*
2447	* Dispatch from this hctx either if there's no hctx preferred
2448	* by IO scheduler or if it has requests that bypass the
2449	* scheduler.
2450	*/
2451	if (!sq_hctx \|\| sq_hctx == hctx \|\|
2452	!list_empty_careful(&hctx->dispatch))
2453	blk_mq_delay_run_hw_queue(hctx, msecs);
2454	}
2455	}
2456	EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);
2457
2458	/*
2459	* This function is often used for pausing .queue_rq() by driver when
2460	* there isn't enough resource or some conditions aren't satisfied, and
2461	* BLK_STS_RESOURCE is usually returned.
2462	*
2463	* We do not guarantee that dispatch can be drained or blocked
2464	* after blk_mq_stop_hw_queue() returns. Please use
2465	* blk_mq_quiesce_queue() for that requirement.
2466	*/
2467	void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
2468	{
2469	cancel_delayed_work(&hctx->run_work);
2470
2471	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
2472	}
2473	EXPORT_SYMBOL(blk_mq_stop_hw_queue);
2474
2475	/*
2476	* This function is often used for pausing .queue_rq() by driver when
2477	* there isn't enough resource or some conditions aren't satisfied, and
2478	* BLK_STS_RESOURCE is usually returned.
2479	*
2480	* We do not guarantee that dispatch can be drained or blocked
2481	* after blk_mq_stop_hw_queues() returns. Please use
2482	* blk_mq_quiesce_queue() for that requirement.
2483	*/
2484	void blk_mq_stop_hw_queues(struct request_queue *q)
2485	{
2486	struct blk_mq_hw_ctx *hctx;
2487	unsigned long i;
2488
2489	queue_for_each_hw_ctx(q, hctx, i)
2490	blk_mq_stop_hw_queue(hctx);
2491	}
2492	EXPORT_SYMBOL(blk_mq_stop_hw_queues);
2493
2494	void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
2495	{
2496	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2497
2498	blk_mq_run_hw_queue(hctx, hctx->flags & BLK_MQ_F_BLOCKING);
2499	}
2500	EXPORT_SYMBOL(blk_mq_start_hw_queue);
2501
2502	void blk_mq_start_hw_queues(struct request_queue *q)
2503	{
2504	struct blk_mq_hw_ctx *hctx;
2505	unsigned long i;
2506
2507	queue_for_each_hw_ctx(q, hctx, i)
2508	blk_mq_start_hw_queue(hctx);
2509	}
2510	EXPORT_SYMBOL(blk_mq_start_hw_queues);
2511
2512	void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
2513	{
2514	if (!blk_mq_hctx_stopped(hctx))
2515	return;
2516
2517	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
2518	/*
2519	* Pairs with the smp_mb() in blk_mq_hctx_stopped() to order the
2520	* clearing of BLK_MQ_S_STOPPED above and the checking of dispatch
2521	* list in the subsequent routine.
2522	*/
2523	smp_mb__after_atomic();
2524	blk_mq_run_hw_queue(hctx, async);
2525	}
2526	EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);
2527
2528	void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
2529	{
2530	struct blk_mq_hw_ctx *hctx;
2531	unsigned long i;
2532
2533	queue_for_each_hw_ctx(q, hctx, i)
2534	blk_mq_start_stopped_hw_queue(hctx, async \|\|
2535	(hctx->flags & BLK_MQ_F_BLOCKING));
2536	}
2537	EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
2538
2539	static void blk_mq_run_work_fn(struct work_struct *work)
2540	{
2541	struct blk_mq_hw_ctx *hctx =
2542	container_of(work, struct blk_mq_hw_ctx, run_work.work);
2543
2544	blk_mq_run_dispatch_ops(hctx->queue,
2545	blk_mq_sched_dispatch_requests(hctx));
2546	}
2547
2548	/**
2549	* blk_mq_request_bypass_insert - Insert a request at dispatch list.
2550	* @rq: Pointer to request to be inserted.
2551	* @flags: BLK_MQ_INSERT_*
2552	*
2553	* Should only be used carefully, when the caller knows we want to
2554	* bypass a potential IO scheduler on the target device.
2555	*/
2556	static void blk_mq_request_bypass_insert(struct request *rq, blk_insert_t flags)
2557	{
2558	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2559
2560	spin_lock(&hctx->lock);
2561	if (flags & BLK_MQ_INSERT_AT_HEAD)
2562	list_add(&rq->queuelist, &hctx->dispatch);
2563	else
2564	list_add_tail(&rq->queuelist, &hctx->dispatch);
2565	spin_unlock(&hctx->lock);
2566	}
2567
2568	static void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx,
2569	struct blk_mq_ctx ctx, struct list_head list,
2570	bool run_queue_async)
2571	{
2572	struct request *rq;
2573	enum hctx_type type = hctx->type;
2574
2575	/*
2576	* Try to issue requests directly if the hw queue isn't busy to save an
2577	* extra enqueue & dequeue to the sw queue.
2578	*/
2579	if (!hctx->dispatch_busy && !run_queue_async) {
2580	blk_mq_run_dispatch_ops(hctx->queue,
2581	blk_mq_try_issue_list_directly(hctx, list));
2582	if (list_empty(list))
2583	goto out;
2584	}
2585
2586	/*
2587	* preemption doesn't flush plug list, so it's possible ctx->cpu is
2588	* offline now
2589	*/
2590	list_for_each_entry(rq, list, queuelist) {
2591	BUG_ON(rq->mq_ctx != ctx);
2592	trace_block_rq_insert(rq);
2593	if (rq->cmd_flags & REQ_NOWAIT)
2594	run_queue_async = true;
2595	}
2596
2597	spin_lock(&ctx->lock);
2598	list_splice_tail_init(list, &ctx->rq_lists[type]);
2599	blk_mq_hctx_mark_pending(hctx, ctx);
2600	spin_unlock(&ctx->lock);
2601	out:
2602	blk_mq_run_hw_queue(hctx, run_queue_async);
2603	}
2604
2605	static void blk_mq_insert_request(struct request *rq, blk_insert_t flags)
2606	{
2607	struct request_queue *q = rq->q;
2608	struct blk_mq_ctx *ctx = rq->mq_ctx;
2609	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2610
2611	if (blk_rq_is_passthrough(rq)) {
2612	/*
2613	* Passthrough request have to be added to hctx->dispatch
2614	* directly. The device may be in a situation where it can't
2615	* handle FS request, and always returns BLK_STS_RESOURCE for
2616	* them, which gets them added to hctx->dispatch.
2617	*
2618	* If a passthrough request is required to unblock the queues,
2619	* and it is added to the scheduler queue, there is no chance to
2620	* dispatch it given we prioritize requests in hctx->dispatch.
2621	*/
2622	blk_mq_request_bypass_insert(rq, flags);
2623	} else if (req_op(rq) == REQ_OP_FLUSH) {
2624	/*
2625	* Firstly normal IO request is inserted to scheduler queue or
2626	* sw queue, meantime we add flush request to dispatch queue(
2627	* hctx->dispatch) directly and there is at most one in-flight
2628	* flush request for each hw queue, so it doesn't matter to add
2629	* flush request to tail or front of the dispatch queue.
2630	*
2631	* Secondly in case of NCQ, flush request belongs to non-NCQ
2632	* command, and queueing it will fail when there is any
2633	* in-flight normal IO request(NCQ command). When adding flush
2634	* rq to the front of hctx->dispatch, it is easier to introduce
2635	* extra time to flush rq's latency because of S_SCHED_RESTART
2636	* compared with adding to the tail of dispatch queue, then
2637	* chance of flush merge is increased, and less flush requests
2638	* will be issued to controller. It is observed that ~10% time
2639	* is saved in blktests block/004 on disk attached to AHCI/NCQ
2640	* drive when adding flush rq to the front of hctx->dispatch.
2641	*
2642	* Simply queue flush rq to the front of hctx->dispatch so that
2643	* intensive flush workloads can benefit in case of NCQ HW.
2644	*/
2645	blk_mq_request_bypass_insert(rq, BLK_MQ_INSERT_AT_HEAD);
2646	} else if (q->elevator) {
2647	LIST_HEAD(list);
2648
2649	WARN_ON_ONCE(rq->tag != BLK_MQ_NO_TAG);
2650
2651	list_add(&rq->queuelist, &list);
2652	q->elevator->type->ops.insert_requests(hctx, &list, flags);
2653	} else {
2654	trace_block_rq_insert(rq);
2655
2656	spin_lock(&ctx->lock);
2657	if (flags & BLK_MQ_INSERT_AT_HEAD)
2658	list_add(&rq->queuelist, &ctx->rq_lists[hctx->type]);
2659	else
2660	list_add_tail(&rq->queuelist,
2661	&ctx->rq_lists[hctx->type]);
2662	blk_mq_hctx_mark_pending(hctx, ctx);
2663	spin_unlock(&ctx->lock);
2664	}
2665	}
2666
2667	static void blk_mq_bio_to_request(struct request rq, struct bio bio,
2668	unsigned int nr_segs)
2669	{
2670	int err;
2671
2672	if (bio->bi_opf & REQ_RAHEAD)
2673	rq->cmd_flags \|= REQ_FAILFAST_MASK;
2674
2675	rq->bio = rq->biotail = bio;
2676	rq->__sector = bio->bi_iter.bi_sector;
2677	rq->__data_len = bio->bi_iter.bi_size;
2678	rq->phys_gap_bit = bio->bi_bvec_gap_bit;
2679
2680	rq->nr_phys_segments = nr_segs;
2681	if (bio_integrity(bio))
2682	rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q,
2683	bio);
2684
2685	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
2686	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
2687	WARN_ON_ONCE(err);
2688
2689	blk_account_io_start(rq);
2690	}
2691
2692	static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
2693	struct request *rq, bool last)
2694	{
2695	struct request_queue *q = rq->q;
2696	struct blk_mq_queue_data bd = {
2697	.rq = rq,
2698	.last = last,
2699	};
2700	blk_status_t ret;
2701
2702	/*
2703	* For OK queue, we are done. For error, caller may kill it.
2704	* Any other error (busy), just add it to our list as we
2705	* previously would have done.
2706	*/
2707	ret = q->mq_ops->queue_rq(hctx, &bd);
2708	switch (ret) {
2709	case BLK_STS_OK:
2710	blk_mq_update_dispatch_busy(hctx, false);
2711	break;
2712	case BLK_STS_RESOURCE:
2713	case BLK_STS_DEV_RESOURCE:
2714	blk_mq_update_dispatch_busy(hctx, true);
2715	__blk_mq_requeue_request(rq);
2716	break;
2717	default:
2718	blk_mq_update_dispatch_busy(hctx, false);
2719	break;
2720	}
2721
2722	return ret;
2723	}
2724
2725	static bool blk_mq_get_budget_and_tag(struct request *rq)
2726	{
2727	int budget_token;
2728
2729	budget_token = blk_mq_get_dispatch_budget(rq->q);
2730	if (budget_token < 0)
2731	return false;
2732	blk_mq_set_rq_budget_token(rq, budget_token);
2733	if (!blk_mq_get_driver_tag(rq)) {
2734	blk_mq_put_dispatch_budget(rq->q, budget_token);
2735	return false;
2736	}
2737	return true;
2738	}
2739
2740	/**
2741	* blk_mq_try_issue_directly - Try to send a request directly to device driver.
2742	* @hctx: Pointer of the associated hardware queue.
2743	* @rq: Pointer to request to be sent.
2744	*
2745	* If the device has enough resources to accept a new request now, send the
2746	* request directly to device driver. Else, insert at hctx->dispatch queue, so
2747	* we can try send it another time in the future. Requests inserted at this
2748	* queue have higher priority.
2749	*/
2750	static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2751	struct request *rq)
2752	{
2753	blk_status_t ret;
2754
2755	if (blk_mq_hctx_stopped(hctx) \|\| blk_queue_quiesced(rq->q)) {
2756	blk_mq_insert_request(rq, 0);
2757	blk_mq_run_hw_queue(hctx, false);
2758	return;
2759	}
2760
2761	if ((rq->rq_flags & RQF_USE_SCHED) \|\| !blk_mq_get_budget_and_tag(rq)) {
2762	blk_mq_insert_request(rq, 0);
2763	blk_mq_run_hw_queue(hctx, rq->cmd_flags & REQ_NOWAIT);
2764	return;
2765	}
2766
2767	ret = __blk_mq_issue_directly(hctx, rq, true);
2768	switch (ret) {
2769	case BLK_STS_OK:
2770	break;
2771	case BLK_STS_RESOURCE:
2772	case BLK_STS_DEV_RESOURCE:
2773	blk_mq_request_bypass_insert(rq, 0);
2774	blk_mq_run_hw_queue(hctx, false);
2775	break;
2776	default:
2777	blk_mq_end_request(rq, ret);
2778	break;
2779	}
2780	}
2781
2782	static blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
2783	{
2784	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
2785
2786	if (blk_mq_hctx_stopped(hctx) \|\| blk_queue_quiesced(rq->q)) {
2787	blk_mq_insert_request(rq, 0);
2788	blk_mq_run_hw_queue(hctx, false);
2789	return BLK_STS_OK;
2790	}
2791
2792	if (!blk_mq_get_budget_and_tag(rq))
2793	return BLK_STS_RESOURCE;
2794	return __blk_mq_issue_directly(hctx, rq, last);
2795	}
2796
2797	static void blk_mq_issue_direct(struct rq_list *rqs)
2798	{
2799	struct blk_mq_hw_ctx *hctx = NULL;
2800	struct request *rq;
2801	int queued = 0;
2802	blk_status_t ret = BLK_STS_OK;
2803
2804	while ((rq = rq_list_pop(rqs))) {
2805	bool last = rq_list_empty(rqs);
2806
2807	if (hctx != rq->mq_hctx) {
2808	if (hctx) {
2809	blk_mq_commit_rqs(hctx, queued, false);
2810	queued = 0;
2811	}
2812	hctx = rq->mq_hctx;
2813	}
2814
2815	ret = blk_mq_request_issue_directly(rq, last);
2816	switch (ret) {
2817	case BLK_STS_OK:
2818	queued++;
2819	break;
2820	case BLK_STS_RESOURCE:
2821	case BLK_STS_DEV_RESOURCE:
2822	blk_mq_request_bypass_insert(rq, 0);
2823	blk_mq_run_hw_queue(hctx, false);
2824	goto out;
2825	default:
2826	blk_mq_end_request(rq, ret);
2827	break;
2828	}
2829	}
2830
2831	out:
2832	if (ret != BLK_STS_OK)
2833	blk_mq_commit_rqs(hctx, queued, false);
2834	}
2835
2836	static void __blk_mq_flush_list(struct request_queue q, struct rq_list rqs)
2837	{
2838	if (blk_queue_quiesced(q))
2839	return;
2840	q->mq_ops->queue_rqs(rqs);
2841	}
2842
2843	static unsigned blk_mq_extract_queue_requests(struct rq_list *rqs,
2844	struct rq_list *queue_rqs)
2845	{
2846	struct request *rq = rq_list_pop(rqs);
2847	struct request_queue *this_q = rq->q;
2848	struct request **prev = &rqs->head;
2849	struct rq_list matched_rqs = {};
2850	struct request *last = NULL;
2851	unsigned depth = 1;
2852
2853	rq_list_add_tail(&matched_rqs, rq);
2854	while ((rq = *prev)) {
2855	if (rq->q == this_q) {
2856	/* move rq from rqs to matched_rqs */
2857	*prev = rq->rq_next;
2858	rq_list_add_tail(&matched_rqs, rq);
2859	depth++;
2860	} else {
2861	/* leave rq in rqs */
2862	prev = &rq->rq_next;
2863	last = rq;
2864	}
2865	}
2866
2867	rqs->tail = last;
2868	*queue_rqs = matched_rqs;
2869	return depth;
2870	}
2871
2872	static void blk_mq_dispatch_queue_requests(struct rq_list *rqs, unsigned depth)
2873	{
2874	struct request_queue *q = rq_list_peek(rqs)->q;
2875
2876	trace_block_unplug(q, depth, true);
2877
2878	/*
2879	* Peek first request and see if we have a ->queue_rqs() hook.
2880	* If we do, we can dispatch the whole list in one go.
2881	* We already know at this point that all requests belong to the
2882	* same queue, caller must ensure that's the case.
2883	*/
2884	if (q->mq_ops->queue_rqs) {
2885	blk_mq_run_dispatch_ops(q, __blk_mq_flush_list(q, rqs));
2886	if (rq_list_empty(rqs))
2887	return;
2888	}
2889
2890	blk_mq_run_dispatch_ops(q, blk_mq_issue_direct(rqs));
2891	}
2892
2893	static void blk_mq_dispatch_list(struct rq_list *rqs, bool from_sched)
2894	{
2895	struct blk_mq_hw_ctx *this_hctx = NULL;
2896	struct blk_mq_ctx *this_ctx = NULL;
2897	struct rq_list requeue_list = {};
2898	unsigned int depth = 0;
2899	bool is_passthrough = false;
2900	LIST_HEAD(list);
2901
2902	do {
2903	struct request *rq = rq_list_pop(rqs);
2904
2905	if (!this_hctx) {
2906	this_hctx = rq->mq_hctx;
2907	this_ctx = rq->mq_ctx;
2908	is_passthrough = blk_rq_is_passthrough(rq);
2909	} else if (this_hctx != rq->mq_hctx \|\| this_ctx != rq->mq_ctx \|\|
2910	is_passthrough != blk_rq_is_passthrough(rq)) {
2911	rq_list_add_tail(&requeue_list, rq);
2912	continue;
2913	}
2914	list_add_tail(&rq->queuelist, &list);
2915	depth++;
2916	} while (!rq_list_empty(rqs));
2917
2918	*rqs = requeue_list;
2919	trace_block_unplug(this_hctx->queue, depth, !from_sched);
2920
2921	percpu_ref_get(&this_hctx->queue->q_usage_counter);
2922	/* passthrough requests should never be issued to the I/O scheduler */
2923	if (is_passthrough) {
2924	spin_lock(&this_hctx->lock);
2925	list_splice_tail_init(&list, &this_hctx->dispatch);
2926	spin_unlock(&this_hctx->lock);
2927	blk_mq_run_hw_queue(this_hctx, from_sched);
2928	} else if (this_hctx->queue->elevator) {
2929	this_hctx->queue->elevator->type->ops.insert_requests(this_hctx,
2930	&list, 0);
2931	blk_mq_run_hw_queue(this_hctx, from_sched);
2932	} else {
2933	blk_mq_insert_requests(this_hctx, this_ctx, &list, from_sched);
2934	}
2935	percpu_ref_put(&this_hctx->queue->q_usage_counter);
2936	}
2937
2938	static void blk_mq_dispatch_multiple_queue_requests(struct rq_list *rqs)
2939	{
2940	do {
2941	struct rq_list queue_rqs;
2942	unsigned depth;
2943
2944	depth = blk_mq_extract_queue_requests(rqs, &queue_rqs);
2945	blk_mq_dispatch_queue_requests(&queue_rqs, depth);
2946	while (!rq_list_empty(&queue_rqs))
2947	blk_mq_dispatch_list(&queue_rqs, false);
2948	} while (!rq_list_empty(rqs));
2949	}
2950
2951	void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2952	{
2953	unsigned int depth;
2954
2955	/*
2956	* We may have been called recursively midway through handling
2957	* plug->mq_list via a schedule() in the driver's queue_rq() callback.
2958	* To avoid mq_list changing under our feet, clear rq_count early and
2959	* bail out specifically if rq_count is 0 rather than checking
2960	* whether the mq_list is empty.
2961	*/
2962	if (plug->rq_count == 0)
2963	return;
2964	depth = plug->rq_count;
2965	plug->rq_count = 0;
2966
2967	if (!plug->has_elevator && !from_schedule) {
2968	if (plug->multiple_queues) {
2969	blk_mq_dispatch_multiple_queue_requests(&plug->mq_list);
2970	return;
2971	}
2972
2973	blk_mq_dispatch_queue_requests(&plug->mq_list, depth);
2974	if (rq_list_empty(&plug->mq_list))
2975	return;
2976	}
2977
2978	do {
2979	blk_mq_dispatch_list(&plug->mq_list, from_schedule);
2980	} while (!rq_list_empty(&plug->mq_list));
2981	}
2982
2983	static void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
2984	struct list_head *list)
2985	{
2986	int queued = 0;
2987	blk_status_t ret = BLK_STS_OK;
2988
2989	while (!list_empty(list)) {
2990	struct request *rq = list_first_entry(list, struct request,
2991	queuelist);
2992
2993	list_del_init(&rq->queuelist);
2994	ret = blk_mq_request_issue_directly(rq, list_empty(list));
2995	switch (ret) {
2996	case BLK_STS_OK:
2997	queued++;
2998	break;
2999	case BLK_STS_RESOURCE:
3000	case BLK_STS_DEV_RESOURCE:
3001	blk_mq_request_bypass_insert(rq, 0);
3002	if (list_empty(list))
3003	blk_mq_run_hw_queue(hctx, false);
3004	goto out;
3005	default:
3006	blk_mq_end_request(rq, ret);
3007	break;
3008	}
3009	}
3010
3011	out:
3012	if (ret != BLK_STS_OK)
3013	blk_mq_commit_rqs(hctx, queued, false);
3014	}
3015
3016	static bool blk_mq_attempt_bio_merge(struct request_queue *q,
3017	struct bio *bio, unsigned int nr_segs)
3018	{
3019	if (!blk_queue_nomerges(q) && bio_mergeable(bio)) {
3020	if (blk_attempt_plug_merge(q, bio, nr_segs))
3021	return true;
3022	if (blk_mq_sched_bio_merge(q, bio, nr_segs))
3023	return true;
3024	}
3025	return false;
3026	}
3027
3028	static struct request blk_mq_get_new_requests(struct request_queue q,
3029	struct blk_plug *plug,
3030	struct bio *bio)
3031	{
3032	struct blk_mq_alloc_data data = {
3033	.q = q,
3034	.flags = 0,
3035	.shallow_depth = 0,
3036	.cmd_flags = bio->bi_opf,
3037	.rq_flags = 0,
3038	.nr_tags = 1,
3039	.cached_rqs = NULL,
3040	.ctx = NULL,
3041	.hctx = NULL
3042	};
3043	struct request *rq;
3044
3045	rq_qos_throttle(q, bio);
3046
3047	if (plug) {
3048	data.nr_tags = plug->nr_ios;
3049	plug->nr_ios = 1;
3050	data.cached_rqs = &plug->cached_rqs;
3051	}
3052
3053	rq = __blk_mq_alloc_requests(&data);
3054	if (unlikely(!rq))
3055	rq_qos_cleanup(q, bio);
3056	return rq;
3057	}
3058
3059	/*
3060	* Check if there is a suitable cached request and return it.
3061	*/
3062	static struct request blk_mq_get_cached_request(struct blk_plug plug,
3063	struct request_queue *q, blk_opf_t opf)
3064	{
3065	enum hctx_type type = blk_mq_get_hctx_type(opf);
3066	struct request *rq;
3067
3068	if (!plug)
3069	return NULL;
3070	rq = rq_list_peek(&plug->cached_rqs);
3071	if (!rq \|\| rq->q != q)
3072	return NULL;
3073	if (type != rq->mq_hctx->type &&
3074	(type != HCTX_TYPE_READ \|\| rq->mq_hctx->type != HCTX_TYPE_DEFAULT))
3075	return NULL;
3076	rq_list_pop(&plug->cached_rqs);
3077	return rq;
3078	}
3079
3080	static bool bio_unaligned(const struct bio bio, struct request_queue q)
3081	{
3082	unsigned int bs_mask = queue_logical_block_size(q) - 1;
3083
3084	/* .bi_sector of any zero sized bio need to be initialized */
3085	if ((bio->bi_iter.bi_size & bs_mask) \|\|
3086	((bio->bi_iter.bi_sector << SECTOR_SHIFT) & bs_mask))
3087	return true;
3088	return false;
3089	}
3090
3091	/**
3092	* blk_mq_submit_bio - Create and send a request to block device.
3093	* @bio: Bio pointer.
3094	*
3095	* Builds up a request structure from @q and @bio and send to the device. The
3096	* request may not be queued directly to hardware if:
3097	* * This request can be merged with another one
3098	* * We want to place request at plug queue for possible future merging
3099	* * There is an IO scheduler active at this queue
3100	*
3101	* It will not queue the request if there is an error with the bio, or at the
3102	* request creation.
3103	*/
3104	void blk_mq_submit_bio(struct bio *bio)
3105	{
3106	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
3107	struct blk_plug *plug = current->plug;
3108	const int is_sync = op_is_sync(bio->bi_opf);
3109	unsigned int integrity_action;
3110	struct blk_mq_hw_ctx *hctx;
3111	unsigned int nr_segs;
3112	struct request *rq;
3113	blk_status_t ret;
3114
3115	/*
3116	* If the plug has a cached request for this queue, try to use it.
3117	*/
3118	rq = blk_mq_get_cached_request(plug, q, bio->bi_opf);
3119
3120	/*
3121	* A BIO that was released from a zone write plug has already been
3122	* through the preparation in this function, already holds a reference
3123	* on the queue usage counter, and is the only write BIO in-flight for
3124	* the target zone. Go straight to preparing a request for it.
3125	*/
3126	if (bio_zone_write_plugging(bio)) {
3127	nr_segs = bio->__bi_nr_segments;
3128	if (rq)
3129	blk_queue_exit(q);
3130	goto new_request;
3131	}
3132
3133	/*
3134	* The cached request already holds a q_usage_counter reference and we
3135	* don't have to acquire a new one if we use it.
3136	*/
3137	if (!rq) {
3138	if (unlikely(bio_queue_enter(bio)))
3139	return;
3140	}
3141
3142	/*
3143	* Device reconfiguration may change logical block size or reduce the
3144	* number of poll queues, so the checks for alignment and poll support
3145	* have to be done with queue usage counter held.
3146	*/
3147	if (unlikely(bio_unaligned(bio, q))) {
3148	bio_io_error(bio);
3149	goto queue_exit;
3150	}
3151
3152	if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) {
3153	bio_endio_status(bio, BLK_STS_NOTSUPP);
3154	goto queue_exit;
3155	}
3156
3157	bio = __bio_split_to_limits(bio, &q->limits, &nr_segs);
3158	if (!bio)
3159	goto queue_exit;
3160
3161	integrity_action = bio_integrity_action(bio);
3162	if (integrity_action)
3163	bio_integrity_prep(bio, integrity_action);
3164
3165	blk_mq_bio_issue_init(q, bio);
3166	if (blk_mq_attempt_bio_merge(q, bio, nr_segs))
3167	goto queue_exit;
3168
3169	if (bio_needs_zone_write_plugging(bio)) {
3170	if (blk_zone_plug_bio(bio, nr_segs))
3171	goto queue_exit;
3172	}
3173
3174	new_request:
3175	if (rq) {
3176	rq_qos_throttle(rq->q, bio);
3177	rq->cmd_flags = bio->bi_opf;
3178	INIT_LIST_HEAD(&rq->queuelist);
3179	blk_mq_rq_late_init(rq, blk_time_get_ns());
3180	} else {
3181	rq = blk_mq_get_new_requests(q, plug, bio);
3182	if (unlikely(!rq)) {
3183	if (bio->bi_opf & REQ_NOWAIT)
3184	bio_wouldblock_error(bio);
3185	goto queue_exit;
3186	}
3187	}
3188
3189	trace_block_getrq(bio);
3190
3191	rq_qos_track(q, rq, bio);
3192
3193	blk_mq_bio_to_request(rq, bio, nr_segs);
3194
3195	ret = blk_crypto_rq_get_keyslot(rq);
3196	if (ret != BLK_STS_OK) {
3197	bio_endio_status(bio, ret);
3198	blk_mq_free_request(rq);
3199	return;
3200	}
3201
3202	if (bio_zone_write_plugging(bio))
3203	blk_zone_write_plug_init_request(rq);
3204
3205	if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
3206	return;
3207
3208	if (plug) {
3209	blk_add_rq_to_plug(plug, rq);
3210	return;
3211	}
3212
3213	hctx = rq->mq_hctx;
3214	if ((rq->rq_flags & RQF_USE_SCHED) \|\|
3215	(hctx->dispatch_busy && (q->nr_hw_queues == 1 \|\| !is_sync))) {
3216	blk_mq_insert_request(rq, 0);
3217	blk_mq_run_hw_queue(hctx, true);
3218	} else {
3219	blk_mq_run_dispatch_ops(q, blk_mq_try_issue_directly(hctx, rq));
3220	}
3221	return;
3222
3223	queue_exit:
3224	if (!rq)
3225	blk_queue_exit(q);
3226	else
3227	rq_list_add_head(&plug->cached_rqs, rq);
3228	}
3229
3230	#ifdef CONFIG_BLK_MQ_STACKING
3231	/**
3232	* blk_insert_cloned_request - Helper for stacking drivers to submit a request
3233	* @rq: the request being queued
3234	*/
3235	blk_status_t blk_insert_cloned_request(struct request *rq)
3236	{
3237	struct request_queue *q = rq->q;
3238	unsigned int max_sectors = blk_queue_get_max_sectors(rq);
3239	unsigned int max_segments = blk_rq_get_max_segments(rq);
3240	blk_status_t ret;
3241
3242	if (blk_rq_sectors(rq) > max_sectors) {
3243	/*
3244	* SCSI device does not have a good way to return if
3245	* Write Same/Zero is actually supported. If a device rejects
3246	* a non-read/write command (discard, write same,etc.) the
3247	* low-level device driver will set the relevant queue limit to
3248	* 0 to prevent blk-lib from issuing more of the offending
3249	* operations. Commands queued prior to the queue limit being
3250	* reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
3251	* errors being propagated to upper layers.
3252	*/
3253	if (max_sectors == 0)
3254	return BLK_STS_NOTSUPP;
3255
3256	printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
3257	__func__, blk_rq_sectors(rq), max_sectors);
3258	return BLK_STS_IOERR;
3259	}
3260
3261	/*
3262	* The queue settings related to segment counting may differ from the
3263	* original queue.
3264	*/
3265	rq->nr_phys_segments = blk_recalc_rq_segments(rq);
3266	if (rq->nr_phys_segments > max_segments) {
3267	printk(KERN_ERR "%s: over max segments limit. (%u > %u)\n",
3268	__func__, rq->nr_phys_segments, max_segments);
3269	return BLK_STS_IOERR;
3270	}
3271
3272	/*
3273	* Integrity segment counting depends on the same queue limits
3274	* (virt_boundary_mask, seg_boundary_mask, max_segment_size) that
3275	* vary across stacked queues, so recompute against the bottom
3276	* queue just like nr_phys_segments above.
3277	*/
3278	if (blk_integrity_rq(rq) && rq->bio) {
3279	unsigned short max_int_segs = queue_max_integrity_segments(q);
3280
3281	rq->nr_integrity_segments =
3282	blk_rq_count_integrity_sg(rq->q, rq->bio);
3283	if (rq->nr_integrity_segments > max_int_segs) {
3284	printk(KERN_ERR "%s: over max integrity segments limit. (%u > %u)\n",
3285	__func__, rq->nr_integrity_segments,
3286	max_int_segs);
3287	return BLK_STS_IOERR;
3288	}
3289	}
3290
3291	if (q->disk && should_fail_request(q->disk->part0, blk_rq_bytes(rq)))
3292	return BLK_STS_IOERR;
3293
3294	ret = blk_crypto_rq_get_keyslot(rq);
3295	if (ret != BLK_STS_OK)
3296	return ret;
3297
3298	blk_account_io_start(rq);
3299
3300	/*
3301	* Since we have a scheduler attached on the top device,
3302	* bypass a potential scheduler on the bottom device for
3303	* insert.
3304	*/
3305	blk_mq_run_dispatch_ops(q,
3306	ret = blk_mq_request_issue_directly(rq, true));
3307	if (ret)
3308	blk_account_io_done(rq, blk_time_get_ns());
3309	return ret;
3310	}
3311	EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
3312
3313	/**
3314	* blk_rq_unprep_clone - Helper function to free all bios in a cloned request
3315	* @rq: the clone request to be cleaned up
3316	*
3317	* Description:
3318	* Free all bios in @rq for a cloned request.
3319	*/
3320	void blk_rq_unprep_clone(struct request *rq)
3321	{
3322	struct bio *bio;
3323
3324	while ((bio = rq->bio) != NULL) {
3325	rq->bio = bio->bi_next;
3326
3327	bio_put(bio);
3328	}
3329	}
3330	EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
3331
3332	/**
3333	* blk_rq_prep_clone - Helper function to setup clone request
3334	* @rq: the request to be setup
3335	* @rq_src: original request to be cloned
3336	* @bs: bio_set that bios for clone are allocated from
3337	* @gfp_mask: memory allocation mask for bio
3338	* @bio_ctr: setup function to be called for each clone bio.
3339	* Returns %0 for success, non %0 for failure.
3340	* @data: private data to be passed to @bio_ctr
3341	*
3342	* Description:
3343	* Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
3344	* Also, pages which the original bios are pointing to are not copied
3345	* and the cloned bios just point same pages.
3346	* So cloned bios must be completed before original bios, which means
3347	* the caller must complete @rq before @rq_src.
3348	*/
3349	int blk_rq_prep_clone(struct request rq, struct request rq_src,
3350	struct bio_set *bs, gfp_t gfp_mask,
3351	int (bio_ctr)(struct bio , struct bio , void ),
3352	void *data)
3353	{
3354	struct bio *bio_src;
3355
3356	if (!bs)
3357	bs = &fs_bio_set;
3358
3359	__rq_for_each_bio(bio_src, rq_src) {
3360	struct bio *bio = bio_alloc_clone(rq->q->disk->part0, bio_src,
3361	gfp_mask, bs);
3362	if (!bio)
3363	goto free_and_out;
3364
3365	if (bio_ctr && bio_ctr(bio, bio_src, data)) {
3366	bio_put(bio);
3367	goto free_and_out;
3368	}
3369
3370	if (rq->bio) {
3371	rq->biotail->bi_next = bio;
3372	rq->biotail = bio;
3373	} else {
3374	rq->bio = rq->biotail = bio;
3375	}
3376	}
3377
3378	/* Copy attributes of the original request to the clone request. */
3379	rq->__sector = blk_rq_pos(rq_src);
3380	rq->__data_len = blk_rq_bytes(rq_src);
3381	if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
3382	rq->rq_flags \|= RQF_SPECIAL_PAYLOAD;
3383	rq->special_vec = rq_src->special_vec;
3384	}
3385	rq->nr_phys_segments = rq_src->nr_phys_segments;
3386	rq->nr_integrity_segments = rq_src->nr_integrity_segments;
3387	rq->phys_gap_bit = rq_src->phys_gap_bit;
3388
3389	if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
3390	goto free_and_out;
3391
3392	return 0;
3393
3394	free_and_out:
3395	blk_rq_unprep_clone(rq);
3396
3397	return -ENOMEM;
3398	}
3399	EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
3400	#endif /* CONFIG_BLK_MQ_STACKING */
3401
3402	/*
3403	* Steal bios from a request and add them to a bio list.
3404	* The request must not have been partially completed before.
3405	*/
3406	void blk_steal_bios(struct bio_list list, struct request rq)
3407	{
3408	struct bio *bio;
3409
3410	for (bio = rq->bio; bio; bio = bio->bi_next) {
3411	if (bio->bi_opf & REQ_POLLED) {
3412	bio->bi_opf &= ~REQ_POLLED;
3413	bio->bi_cookie = BLK_QC_T_NONE;
3414	}
3415	/*
3416	* The alternate request queue that we may end up submitting
3417	* the bio to may be frozen temporarily, in this case REQ_NOWAIT
3418	* will fail the I/O immediately with EAGAIN to the issuer.
3419	* We are not in the issuer context which cannot block. Clear
3420	* the flag to avoid spurious EAGAIN I/O failures.
3421	*/
3422	bio->bi_opf &= ~REQ_NOWAIT;
3423	bio_clear_flag(bio, BIO_QOS_THROTTLED);
3424	bio_clear_flag(bio, BIO_QOS_MERGED);
3425	}
3426
3427	if (rq->bio) {
3428	if (list->tail)
3429	list->tail->bi_next = rq->bio;
3430	else
3431	list->head = rq->bio;
3432	list->tail = rq->biotail;
3433
3434	rq->bio = NULL;
3435	rq->biotail = NULL;
3436	}
3437
3438	rq->__data_len = 0;
3439	}
3440	EXPORT_SYMBOL_GPL(blk_steal_bios);
3441
3442	static size_t order_to_size(unsigned int order)
3443	{
3444	return (size_t)PAGE_SIZE << order;
3445	}
3446
3447	/* called before freeing request pool in @tags */
3448	static void blk_mq_clear_rq_mapping(struct blk_mq_tags *drv_tags,
3449	struct blk_mq_tags *tags)
3450	{
3451	struct page *page;
3452
3453	/*
3454	* There is no need to clear mapping if driver tags is not initialized
3455	* or the mapping belongs to the driver tags.
3456	*/
3457	if (!drv_tags \|\| drv_tags == tags)
3458	return;
3459
3460	list_for_each_entry(page, &tags->page_list, lru) {
3461	unsigned long start = (unsigned long)page_address(page);
3462	unsigned long end = start + order_to_size(page->private);
3463	int i;
3464
3465	for (i = 0; i < drv_tags->nr_tags; i++) {
3466	struct request *rq = drv_tags->rqs[i];
3467	unsigned long rq_addr = (unsigned long)rq;
3468
3469	if (rq_addr >= start && rq_addr < end) {
3470	WARN_ON_ONCE(req_ref_read(rq) != 0);
3471	cmpxchg(&drv_tags->rqs[i], rq, NULL);
3472	}
3473	}
3474	}
3475	}
3476
3477	void blk_mq_free_rqs(struct blk_mq_tag_set set, struct blk_mq_tags tags,
3478	unsigned int hctx_idx)
3479	{
3480	struct blk_mq_tags *drv_tags;
3481
3482	if (list_empty(&tags->page_list))
3483	return;
3484
3485	if (blk_mq_is_shared_tags(set->flags))
3486	drv_tags = set->shared_tags;
3487	else
3488	drv_tags = set->tags[hctx_idx];
3489
3490	if (tags->static_rqs && set->ops->exit_request) {
3491	int i;
3492
3493	for (i = 0; i < tags->nr_tags; i++) {
3494	struct request *rq = tags->static_rqs[i];
3495
3496	if (!rq)
3497	continue;
3498	set->ops->exit_request(set, rq, hctx_idx);
3499	tags->static_rqs[i] = NULL;
3500	}
3501	}
3502
3503	blk_mq_clear_rq_mapping(drv_tags, tags);
3504	/*
3505	* Free request pages in SRCU callback, which is called from
3506	* blk_mq_free_tags().
3507	*/
3508	}
3509
3510	void blk_mq_free_rq_map(struct blk_mq_tag_set set, struct blk_mq_tags tags)
3511	{
3512	kfree(tags->rqs);
3513	tags->rqs = NULL;
3514	kfree(tags->static_rqs);
3515	tags->static_rqs = NULL;
3516
3517	blk_mq_free_tags(set, tags);
3518	}
3519
3520	static enum hctx_type hctx_idx_to_type(struct blk_mq_tag_set *set,
3521	unsigned int hctx_idx)
3522	{
3523	int i;
3524
3525	for (i = 0; i < set->nr_maps; i++) {
3526	unsigned int start = set->map[i].queue_offset;
3527	unsigned int end = start + set->map[i].nr_queues;
3528
3529	if (hctx_idx >= start && hctx_idx < end)
3530	break;
3531	}
3532
3533	if (i >= set->nr_maps)
3534	i = HCTX_TYPE_DEFAULT;
3535
3536	return i;
3537	}
3538
3539	static int blk_mq_get_hctx_node(struct blk_mq_tag_set *set,
3540	unsigned int hctx_idx)
3541	{
3542	enum hctx_type type = hctx_idx_to_type(set, hctx_idx);
3543
3544	return blk_mq_hw_queue_to_node(&set->map[type], hctx_idx);
3545	}
3546
3547	static struct blk_mq_tags blk_mq_alloc_rq_map(struct blk_mq_tag_set set,
3548	unsigned int hctx_idx,
3549	unsigned int nr_tags,
3550	unsigned int reserved_tags)
3551	{
3552	int node = blk_mq_get_hctx_node(set, hctx_idx);
3553	struct blk_mq_tags *tags;
3554
3555	if (node == NUMA_NO_NODE)
3556	node = set->numa_node;
3557
3558	tags = blk_mq_init_tags(nr_tags, reserved_tags, set->flags, node);
3559	if (!tags)
3560	return NULL;
3561
3562	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3563	GFP_NOIO \| __GFP_NOWARN \| __GFP_NORETRY,
3564	node);
3565	if (!tags->rqs)
3566	goto err_free_tags;
3567
3568	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
3569	GFP_NOIO \| __GFP_NOWARN \| __GFP_NORETRY,
3570	node);
3571	if (!tags->static_rqs)
3572	goto err_free_rqs;
3573
3574	return tags;
3575
3576	err_free_rqs:
3577	kfree(tags->rqs);
3578	err_free_tags:
3579	blk_mq_free_tags(set, tags);
3580	return NULL;
3581	}
3582
3583	static int blk_mq_init_request(struct blk_mq_tag_set set, struct request rq,
3584	unsigned int hctx_idx, int node)
3585	{
3586	int ret;
3587
3588	if (set->ops->init_request) {
3589	ret = set->ops->init_request(set, rq, hctx_idx, node);
3590	if (ret)
3591	return ret;
3592	}
3593
3594	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
3595	return 0;
3596	}
3597
3598	static int blk_mq_alloc_rqs(struct blk_mq_tag_set *set,
3599	struct blk_mq_tags *tags,
3600	unsigned int hctx_idx, unsigned int depth)
3601	{
3602	unsigned int i, j, entries_per_page, max_order = 4;
3603	int node = blk_mq_get_hctx_node(set, hctx_idx);
3604	size_t rq_size, left;
3605
3606	if (node == NUMA_NO_NODE)
3607	node = set->numa_node;
3608
3609	/*
3610	* rq_size is the size of the request plus driver payload, rounded
3611	* to the cacheline size
3612	*/
3613	rq_size = round_up(sizeof(struct request) + set->cmd_size,
3614	cache_line_size());
3615	left = rq_size * depth;
3616
3617	for (i = 0; i < depth; ) {
3618	int this_order = max_order;
3619	struct page *page;
3620	int to_do;
3621	void *p;
3622
3623	while (this_order && left < order_to_size(this_order - 1))
3624	this_order--;
3625
3626	do {
3627	page = alloc_pages_node(node,
3628	GFP_NOIO \| __GFP_NOWARN \| __GFP_NORETRY \| __GFP_ZERO,
3629	this_order);
3630	if (page)
3631	break;
3632	if (!this_order--)
3633	break;
3634	if (order_to_size(this_order) < rq_size)
3635	break;
3636	} while (1);
3637
3638	if (!page)
3639	goto fail;
3640
3641	page->private = this_order;
3642	list_add_tail(&page->lru, &tags->page_list);
3643
3644	p = page_address(page);
3645	/*
3646	* Allow kmemleak to scan these pages as they contain pointers
3647	* to additional allocations like via ops->init_request().
3648	*/
3649	kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
3650	entries_per_page = order_to_size(this_order) / rq_size;
3651	to_do = min(entries_per_page, depth - i);
3652	left -= to_do * rq_size;
3653	for (j = 0; j < to_do; j++) {
3654	struct request *rq = p;
3655
3656	tags->static_rqs[i] = rq;
3657	if (blk_mq_init_request(set, rq, hctx_idx, node)) {
3658	tags->static_rqs[i] = NULL;
3659	goto fail;
3660	}
3661
3662	p += rq_size;
3663	i++;
3664	}
3665	}
3666	return 0;
3667
3668	fail:
3669	blk_mq_free_rqs(set, tags, hctx_idx);
3670	return -ENOMEM;
3671	}
3672
3673	struct rq_iter_data {
3674	struct blk_mq_hw_ctx *hctx;
3675	bool has_rq;
3676	};
3677
3678	static bool blk_mq_has_request(struct request rq, void data)
3679	{
3680	struct rq_iter_data *iter_data = data;
3681
3682	if (rq->mq_hctx != iter_data->hctx)
3683	return true;
3684	iter_data->has_rq = true;
3685	return false;
3686	}
3687
3688	static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
3689	{
3690	struct blk_mq_tags *tags = hctx->sched_tags ?
3691	hctx->sched_tags : hctx->tags;
3692	struct rq_iter_data data = {
3693	.hctx = hctx,
3694	};
3695	int srcu_idx;
3696
3697	srcu_idx = srcu_read_lock(&hctx->queue->tag_set->tags_srcu);
3698	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
3699	srcu_read_unlock(&hctx->queue->tag_set->tags_srcu, srcu_idx);
3700
3701	return data.has_rq;
3702	}
3703
3704	static bool blk_mq_hctx_has_online_cpu(struct blk_mq_hw_ctx *hctx,
3705	unsigned int this_cpu)
3706	{
3707	enum hctx_type type = hctx->type;
3708	int cpu;
3709
3710	/*
3711	* hctx->cpumask has to rule out isolated CPUs, but userspace still
3712	* might submit IOs on these isolated CPUs, so use the queue map to
3713	* check if all CPUs mapped to this hctx are offline
3714	*/
3715	for_each_online_cpu(cpu) {
3716	struct blk_mq_hw_ctx *h = blk_mq_map_queue_type(hctx->queue,
3717	type, cpu);
3718
3719	if (h != hctx)
3720	continue;
3721
3722	/* this hctx has at least one online CPU */
3723	if (this_cpu != cpu)
3724	return true;
3725	}
3726
3727	return false;
3728	}
3729
3730	static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
3731	{
3732	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3733	struct blk_mq_hw_ctx, cpuhp_online);
3734	int ret = 0;
3735
3736	if (!hctx->nr_ctx \|\| blk_mq_hctx_has_online_cpu(hctx, cpu))
3737	return 0;
3738
3739	/*
3740	* Prevent new request from being allocated on the current hctx.
3741	*
3742	* The smp_mb__after_atomic() Pairs with the implied barrier in
3743	* test_and_set_bit_lock in sbitmap_get(). Ensures the inactive flag is
3744	* seen once we return from the tag allocator.
3745	*/
3746	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3747	smp_mb__after_atomic();
3748
3749	/*
3750	* Try to grab a reference to the queue and wait for any outstanding
3751	* requests. If we could not grab a reference the queue has been
3752	* frozen and there are no requests.
3753	*/
3754	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
3755	while (blk_mq_hctx_has_requests(hctx)) {
3756	/*
3757	* The wakeup capable IRQ handler of block device is
3758	* not called during suspend. Skip the loop by checking
3759	* pm_wakeup_pending to prevent the deadlock and improve
3760	* suspend latency.
3761	*/
3762	if (pm_wakeup_pending()) {
3763	clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3764	ret = -EBUSY;
3765	break;
3766	}
3767	msleep(5);
3768	}
3769	percpu_ref_put(&hctx->queue->q_usage_counter);
3770	}
3771
3772	return ret;
3773	}
3774
3775	/*
3776	* Check if one CPU is mapped to the specified hctx
3777	*
3778	* Isolated CPUs have been ruled out from hctx->cpumask, which is supposed
3779	* to be used for scheduling kworker only. For other usage, please call this
3780	* helper for checking if one CPU belongs to the specified hctx
3781	*/
3782	static bool blk_mq_cpu_mapped_to_hctx(unsigned int cpu,
3783	const struct blk_mq_hw_ctx *hctx)
3784	{
3785	struct blk_mq_hw_ctx *mapped_hctx = blk_mq_map_queue_type(hctx->queue,
3786	hctx->type, cpu);
3787
3788	return mapped_hctx == hctx;
3789	}
3790
3791	static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
3792	{
3793	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
3794	struct blk_mq_hw_ctx, cpuhp_online);
3795
3796	if (blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3797	clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
3798	return 0;
3799	}
3800
3801	/*
3802	* 'cpu' is going away. splice any existing rq_list entries from this
3803	* software queue to the hw queue dispatch list, and ensure that it
3804	* gets run.
3805	*/
3806	static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
3807	{
3808	struct blk_mq_hw_ctx *hctx;
3809	struct blk_mq_ctx *ctx;
3810	LIST_HEAD(tmp);
3811	enum hctx_type type;
3812
3813	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
3814	if (!blk_mq_cpu_mapped_to_hctx(cpu, hctx))
3815	return 0;
3816
3817	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
3818	type = hctx->type;
3819
3820	spin_lock(&ctx->lock);
3821	if (!list_empty(&ctx->rq_lists[type])) {
3822	list_splice_init(&ctx->rq_lists[type], &tmp);
3823	blk_mq_hctx_clear_pending(hctx, ctx);
3824	}
3825	spin_unlock(&ctx->lock);
3826
3827	if (list_empty(&tmp))
3828	return 0;
3829
3830	spin_lock(&hctx->lock);
3831	list_splice_tail_init(&tmp, &hctx->dispatch);
3832	spin_unlock(&hctx->lock);
3833
3834	blk_mq_run_hw_queue(hctx, true);
3835	return 0;
3836	}
3837
3838	static void __blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3839	{
3840	lockdep_assert_held(&blk_mq_cpuhp_lock);
3841
3842	if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3843	!hlist_unhashed(&hctx->cpuhp_online)) {
3844	cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3845	&hctx->cpuhp_online);
3846	INIT_HLIST_NODE(&hctx->cpuhp_online);
3847	}
3848
3849	if (!hlist_unhashed(&hctx->cpuhp_dead)) {
3850	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3851	&hctx->cpuhp_dead);
3852	INIT_HLIST_NODE(&hctx->cpuhp_dead);
3853	}
3854	}
3855
3856	static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
3857	{
3858	mutex_lock(&blk_mq_cpuhp_lock);
3859	__blk_mq_remove_cpuhp(hctx);
3860	mutex_unlock(&blk_mq_cpuhp_lock);
3861	}
3862
3863	static void __blk_mq_add_cpuhp(struct blk_mq_hw_ctx *hctx)
3864	{
3865	lockdep_assert_held(&blk_mq_cpuhp_lock);
3866
3867	if (!(hctx->flags & BLK_MQ_F_STACKING) &&
3868	hlist_unhashed(&hctx->cpuhp_online))
3869	cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
3870	&hctx->cpuhp_online);
3871
3872	if (hlist_unhashed(&hctx->cpuhp_dead))
3873	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD,
3874	&hctx->cpuhp_dead);
3875	}
3876
3877	static void __blk_mq_remove_cpuhp_list(struct list_head *head)
3878	{
3879	struct blk_mq_hw_ctx *hctx;
3880
3881	lockdep_assert_held(&blk_mq_cpuhp_lock);
3882
3883	list_for_each_entry(hctx, head, hctx_list)
3884	__blk_mq_remove_cpuhp(hctx);
3885	}
3886
3887	/*
3888	* Unregister cpuhp callbacks from exited hw queues
3889	*
3890	* Safe to call if this `request_queue` is live
3891	*/
3892	static void blk_mq_remove_hw_queues_cpuhp(struct request_queue *q)
3893	{
3894	LIST_HEAD(hctx_list);
3895
3896	spin_lock(&q->unused_hctx_lock);
3897	list_splice_init(&q->unused_hctx_list, &hctx_list);
3898	spin_unlock(&q->unused_hctx_lock);
3899
3900	mutex_lock(&blk_mq_cpuhp_lock);
3901	__blk_mq_remove_cpuhp_list(&hctx_list);
3902	mutex_unlock(&blk_mq_cpuhp_lock);
3903
3904	spin_lock(&q->unused_hctx_lock);
3905	list_splice(&hctx_list, &q->unused_hctx_list);
3906	spin_unlock(&q->unused_hctx_lock);
3907	}
3908
3909	/*
3910	* Register cpuhp callbacks from all hw queues
3911	*
3912	* Safe to call if this `request_queue` is live
3913	*/
3914	static void blk_mq_add_hw_queues_cpuhp(struct request_queue *q)
3915	{
3916	struct blk_mq_hw_ctx *hctx;
3917	unsigned long i;
3918
3919	mutex_lock(&blk_mq_cpuhp_lock);
3920	queue_for_each_hw_ctx(q, hctx, i)
3921	__blk_mq_add_cpuhp(hctx);
3922	mutex_unlock(&blk_mq_cpuhp_lock);
3923	}
3924
3925	/*
3926	* Before freeing hw queue, clearing the flush request reference in
3927	* tags->rqs[] for avoiding potential UAF.
3928	*/
3929	static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
3930	unsigned int queue_depth, struct request *flush_rq)
3931	{
3932	int i;
3933
3934	/* The hw queue may not be mapped yet */
3935	if (!tags)
3936	return;
3937
3938	WARN_ON_ONCE(req_ref_read(flush_rq) != 0);
3939
3940	for (i = 0; i < queue_depth; i++)
3941	cmpxchg(&tags->rqs[i], flush_rq, NULL);
3942	}
3943
3944	static void blk_free_flush_queue_callback(struct rcu_head *head)
3945	{
3946	struct blk_flush_queue *fq =
3947	container_of(head, struct blk_flush_queue, rcu_head);
3948
3949	blk_free_flush_queue(fq);
3950	}
3951
3952	/* hctx->ctxs will be freed in queue's release handler */
3953	static void blk_mq_exit_hctx(struct request_queue *q,
3954	struct blk_mq_tag_set *set,
3955	struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
3956	{
3957	struct request *flush_rq = hctx->fq->flush_rq;
3958
3959	if (blk_mq_hw_queue_mapped(hctx))
3960	blk_mq_tag_idle(hctx);
3961
3962	if (blk_queue_init_done(q))
3963	blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
3964	set->queue_depth, flush_rq);
3965	if (set->ops->exit_request)
3966	set->ops->exit_request(set, flush_rq, hctx_idx);
3967
3968	if (set->ops->exit_hctx)
3969	set->ops->exit_hctx(hctx, hctx_idx);
3970
3971	call_srcu(&set->tags_srcu, &hctx->fq->rcu_head,
3972	blk_free_flush_queue_callback);
3973	hctx->fq = NULL;
3974
3975	spin_lock(&q->unused_hctx_lock);
3976	list_add(&hctx->hctx_list, &q->unused_hctx_list);
3977	spin_unlock(&q->unused_hctx_lock);
3978	}
3979
3980	static void blk_mq_exit_hw_queues(struct request_queue *q,
3981	struct blk_mq_tag_set *set, int nr_queue)
3982	{
3983	struct blk_mq_hw_ctx *hctx;
3984	unsigned long i;
3985
3986	queue_for_each_hw_ctx(q, hctx, i) {
3987	if (i == nr_queue)
3988	break;
3989	blk_mq_remove_cpuhp(hctx);
3990	blk_mq_exit_hctx(q, set, hctx, i);
3991	}
3992	}
3993
3994	static int blk_mq_init_hctx(struct request_queue *q,
3995	struct blk_mq_tag_set *set,
3996	struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
3997	{
3998	gfp_t gfp = GFP_NOIO \| __GFP_NOWARN \| __GFP_NORETRY;
3999
4000	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
4001	if (!hctx->fq)
4002	goto fail;
4003
4004	hctx->queue_num = hctx_idx;
4005
4006	hctx->tags = set->tags[hctx_idx];
4007
4008	if (set->ops->init_hctx &&
4009	set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
4010	goto fail_free_fq;
4011
4012	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
4013	hctx->numa_node))
4014	goto exit_hctx;
4015
4016	return 0;
4017
4018	exit_hctx:
4019	if (set->ops->exit_hctx)
4020	set->ops->exit_hctx(hctx, hctx_idx);
4021	fail_free_fq:
4022	blk_free_flush_queue(hctx->fq);
4023	hctx->fq = NULL;
4024	fail:
4025	return -1;
4026	}
4027
4028	static struct blk_mq_hw_ctx *
4029	blk_mq_alloc_hctx(struct request_queue q, struct blk_mq_tag_set set,
4030	int node)
4031	{
4032	struct blk_mq_hw_ctx *hctx;
4033	gfp_t gfp = GFP_NOIO \| __GFP_NOWARN \| __GFP_NORETRY;
4034
4035	hctx = kzalloc_node(sizeof(struct blk_mq_hw_ctx), gfp, node);
4036	if (!hctx)
4037	goto fail_alloc_hctx;
4038
4039	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
4040	goto free_hctx;
4041
4042	atomic_set(&hctx->nr_active, 0);
4043	if (node == NUMA_NO_NODE)
4044	node = set->numa_node;
4045	hctx->numa_node = node;
4046
4047	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
4048	spin_lock_init(&hctx->lock);
4049	INIT_LIST_HEAD(&hctx->dispatch);
4050	INIT_HLIST_NODE(&hctx->cpuhp_dead);
4051	INIT_HLIST_NODE(&hctx->cpuhp_online);
4052	hctx->queue = q;
4053	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
4054
4055	INIT_LIST_HEAD(&hctx->hctx_list);
4056
4057	/*
4058	* Allocate space for all possible cpus to avoid allocation at
4059	* runtime
4060	*/
4061	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
4062	gfp, node);
4063	if (!hctx->ctxs)
4064	goto free_cpumask;
4065
4066	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
4067	gfp, node, false, false))
4068	goto free_ctxs;
4069	hctx->nr_ctx = 0;
4070
4071	spin_lock_init(&hctx->dispatch_wait_lock);
4072	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
4073	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);
4074
4075	blk_mq_hctx_kobj_init(hctx);
4076
4077	return hctx;
4078
4079	free_ctxs:
4080	kfree(hctx->ctxs);
4081	free_cpumask:
4082	free_cpumask_var(hctx->cpumask);
4083	free_hctx:
4084	kfree(hctx);
4085	fail_alloc_hctx:
4086	return NULL;
4087	}
4088
4089	static void blk_mq_init_cpu_queues(struct request_queue *q,
4090	unsigned int nr_hw_queues)
4091	{
4092	struct blk_mq_tag_set *set = q->tag_set;
4093	unsigned int i, j;
4094
4095	for_each_possible_cpu(i) {
4096	struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
4097	struct blk_mq_hw_ctx *hctx;
4098	int k;
4099
4100	__ctx->cpu = i;
4101	spin_lock_init(&__ctx->lock);
4102	for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
4103	INIT_LIST_HEAD(&__ctx->rq_lists[k]);
4104
4105	__ctx->queue = q;
4106
4107	/*
4108	* Set local node, IFF we have more than one hw queue. If
4109	* not, we remain on the home node of the device
4110	*/
4111	for (j = 0; j < set->nr_maps; j++) {
4112	hctx = blk_mq_map_queue_type(q, j, i);
4113	if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
4114	hctx->numa_node = cpu_to_node(i);
4115	}
4116	}
4117	}
4118
4119	struct blk_mq_tags blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set set,
4120	unsigned int hctx_idx,
4121	unsigned int depth)
4122	{
4123	struct blk_mq_tags *tags;
4124	int ret;
4125
4126	tags = blk_mq_alloc_rq_map(set, hctx_idx, depth, set->reserved_tags);
4127	if (!tags)
4128	return NULL;
4129
4130	ret = blk_mq_alloc_rqs(set, tags, hctx_idx, depth);
4131	if (ret) {
4132	blk_mq_free_rq_map(set, tags);
4133	return NULL;
4134	}
4135
4136	return tags;
4137	}
4138
4139	static bool __blk_mq_alloc_map_and_rqs(struct blk_mq_tag_set *set,
4140	int hctx_idx)
4141	{
4142	if (blk_mq_is_shared_tags(set->flags)) {
4143	set->tags[hctx_idx] = set->shared_tags;
4144
4145	return true;
4146	}
4147
4148	set->tags[hctx_idx] = blk_mq_alloc_map_and_rqs(set, hctx_idx,
4149	set->queue_depth);
4150
4151	return set->tags[hctx_idx];
4152	}
4153
4154	void blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4155	struct blk_mq_tags *tags,
4156	unsigned int hctx_idx)
4157	{
4158	if (tags) {
4159	blk_mq_free_rqs(set, tags, hctx_idx);
4160	blk_mq_free_rq_map(set, tags);
4161	}
4162	}
4163
4164	static void __blk_mq_free_map_and_rqs(struct blk_mq_tag_set *set,
4165	unsigned int hctx_idx)
4166	{
4167	if (!blk_mq_is_shared_tags(set->flags))
4168	blk_mq_free_map_and_rqs(set, set->tags[hctx_idx], hctx_idx);
4169
4170	set->tags[hctx_idx] = NULL;
4171	}
4172
4173	static void blk_mq_map_swqueue(struct request_queue *q)
4174	{
4175	unsigned int j, hctx_idx;
4176	unsigned long i;
4177	struct blk_mq_hw_ctx *hctx;
4178	struct blk_mq_ctx *ctx;
4179	struct blk_mq_tag_set *set = q->tag_set;
4180
4181	queue_for_each_hw_ctx(q, hctx, i) {
4182	cpumask_clear(hctx->cpumask);
4183	hctx->nr_ctx = 0;
4184	hctx->dispatch_from = NULL;
4185	}
4186
4187	/*
4188	* Map software to hardware queues.
4189	*
4190	* If the cpu isn't present, the cpu is mapped to first hctx.
4191	*/
4192	for_each_possible_cpu(i) {
4193
4194	ctx = per_cpu_ptr(q->queue_ctx, i);
4195	for (j = 0; j < set->nr_maps; j++) {
4196	if (!set->map[j].nr_queues) {
4197	ctx->hctxs[j] = blk_mq_map_queue_type(q,
4198	HCTX_TYPE_DEFAULT, i);
4199	continue;
4200	}
4201	hctx_idx = set->map[j].mq_map[i];
4202	/* unmapped hw queue can be remapped after CPU topo changed */
4203	if (!set->tags[hctx_idx] &&
4204	!__blk_mq_alloc_map_and_rqs(set, hctx_idx)) {
4205	/*
4206	* If tags initialization fail for some hctx,
4207	* that hctx won't be brought online. In this
4208	* case, remap the current ctx to hctx[0] which
4209	* is guaranteed to always have tags allocated
4210	*/
4211	set->map[j].mq_map[i] = 0;
4212	}
4213
4214	hctx = blk_mq_map_queue_type(q, j, i);
4215	ctx->hctxs[j] = hctx;
4216	/*
4217	* If the CPU is already set in the mask, then we've
4218	* mapped this one already. This can happen if
4219	* devices share queues across queue maps.
4220	*/
4221	if (cpumask_test_cpu(i, hctx->cpumask))
4222	continue;
4223
4224	cpumask_set_cpu(i, hctx->cpumask);
4225	hctx->type = j;
4226	ctx->index_hw[hctx->type] = hctx->nr_ctx;
4227	hctx->ctxs[hctx->nr_ctx++] = ctx;
4228
4229	/*
4230	* If the nr_ctx type overflows, we have exceeded the
4231	* amount of sw queues we can support.
4232	*/
4233	BUG_ON(!hctx->nr_ctx);
4234	}
4235
4236	for (; j < HCTX_MAX_TYPES; j++)
4237	ctx->hctxs[j] = blk_mq_map_queue_type(q,
4238	HCTX_TYPE_DEFAULT, i);
4239	}
4240
4241	queue_for_each_hw_ctx(q, hctx, i) {
4242	int cpu;
4243
4244	/*
4245	* If no software queues are mapped to this hardware queue,
4246	* disable it and free the request entries.
4247	*/
4248	if (!hctx->nr_ctx) {
4249	/* Never unmap queue 0. We need it as a
4250	* fallback in case of a new remap fails
4251	* allocation
4252	*/
4253	if (i)
4254	__blk_mq_free_map_and_rqs(set, i);
4255
4256	hctx->tags = NULL;
4257	continue;
4258	}
4259
4260	hctx->tags = set->tags[i];
4261	WARN_ON(!hctx->tags);
4262
4263	/*
4264	* Set the map size to the number of mapped software queues.
4265	* This is more accurate and more efficient than looping
4266	* over all possibly mapped software queues.
4267	*/
4268	sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
4269
4270	/*
4271	* Rule out isolated CPUs from hctx->cpumask to avoid
4272	* running block kworker on isolated CPUs.
4273	* FIXME: cpuset should propagate further changes to isolated CPUs
4274	* here.
4275	*/
4276	rcu_read_lock();
4277	for_each_cpu(cpu, hctx->cpumask) {
4278	if (cpu_is_isolated(cpu))
4279	cpumask_clear_cpu(cpu, hctx->cpumask);
4280	}
4281	rcu_read_unlock();
4282
4283	/*
4284	* Initialize batch roundrobin counts
4285	*/
4286	hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
4287	hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
4288	}
4289	}
4290
4291	/*
4292	* Caller needs to ensure that we're either frozen/quiesced, or that
4293	* the queue isn't live yet.
4294	*/
4295	static void queue_set_hctx_shared(struct request_queue *q, bool shared)
4296	{
4297	struct blk_mq_hw_ctx *hctx;
4298	unsigned long i;
4299
4300	queue_for_each_hw_ctx(q, hctx, i) {
4301	if (shared) {
4302	hctx->flags \|= BLK_MQ_F_TAG_QUEUE_SHARED;
4303	} else {
4304	blk_mq_tag_idle(hctx);
4305	hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4306	}
4307	}
4308	}
4309
4310	static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
4311	bool shared)
4312	{
4313	struct request_queue *q;
4314	unsigned int memflags;
4315
4316	lockdep_assert_held(&set->tag_list_lock);
4317
4318	list_for_each_entry(q, &set->tag_list, tag_set_list) {
4319	memflags = blk_mq_freeze_queue(q);
4320	queue_set_hctx_shared(q, shared);
4321	blk_mq_unfreeze_queue(q, memflags);
4322	}
4323	}
4324
4325	static void blk_mq_del_queue_tag_set(struct request_queue *q)
4326	{
4327	struct blk_mq_tag_set *set = q->tag_set;
4328
4329	mutex_lock(&set->tag_list_lock);
4330	list_del_rcu(&q->tag_set_list);
4331	if (list_is_singular(&set->tag_list)) {
4332	/* just transitioned to unshared */
4333	set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
4334	/* update existing queue */
4335	blk_mq_update_tag_set_shared(set, false);
4336	}
4337	mutex_unlock(&set->tag_list_lock);
4338	}
4339
4340	static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
4341	struct request_queue *q)
4342	{
4343	mutex_lock(&set->tag_list_lock);
4344
4345	/*
4346	* Check to see if we're transitioning to shared (from 1 to 2 queues).
4347	*/
4348	if (!list_empty(&set->tag_list) &&
4349	!(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
4350	set->flags \|= BLK_MQ_F_TAG_QUEUE_SHARED;
4351	/* update existing queue */
4352	blk_mq_update_tag_set_shared(set, true);
4353	}
4354	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
4355	queue_set_hctx_shared(q, true);
4356	list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
4357
4358	mutex_unlock(&set->tag_list_lock);
4359	}
4360
4361	/* All allocations will be freed in release handler of q->mq_kobj */
4362	static int blk_mq_alloc_ctxs(struct request_queue *q)
4363	{
4364	struct blk_mq_ctxs *ctxs;
4365	int cpu;
4366
4367	ctxs = kzalloc_obj(*ctxs);
4368	if (!ctxs)
4369	return -ENOMEM;
4370
4371	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
4372	if (!ctxs->queue_ctx)
4373	goto fail;
4374
4375	for_each_possible_cpu(cpu) {
4376	struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
4377	ctx->ctxs = ctxs;
4378	}
4379
4380	q->mq_kobj = &ctxs->kobj;
4381	q->queue_ctx = ctxs->queue_ctx;
4382
4383	return 0;
4384	fail:
4385	kfree(ctxs);
4386	return -ENOMEM;
4387	}
4388
4389	/*
4390	* It is the actual release handler for mq, but we do it from
4391	* request queue's release handler for avoiding use-after-free
4392	* and headache because q->mq_kobj shouldn't have been introduced,
4393	* but we can't group ctx/kctx kobj without it.
4394	*/
4395	void blk_mq_release(struct request_queue *q)
4396	{
4397	struct blk_mq_hw_ctx hctx, next;
4398	unsigned long i;
4399
4400	queue_for_each_hw_ctx(q, hctx, i)
4401	WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));
4402
4403	/* all hctx are in .unused_hctx_list now */
4404	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
4405	list_del_init(&hctx->hctx_list);
4406	kobject_put(&hctx->kobj);
4407	}
4408
4409	kfree(q->queue_hw_ctx);
4410
4411	/*
4412	* release .mq_kobj and sw queue's kobject now because
4413	* both share lifetime with request queue.
4414	*/
4415	blk_mq_sysfs_deinit(q);
4416	}
4417
4418	struct request_queue blk_mq_alloc_queue(struct blk_mq_tag_set set,
4419	struct queue_limits lim, void queuedata)
4420	{
4421	struct queue_limits default_lim = { };
4422	struct request_queue *q;
4423	int ret;
4424
4425	if (!lim)
4426	lim = &default_lim;
4427	lim->features \|= BLK_FEAT_IO_STAT \| BLK_FEAT_NOWAIT;
4428	if (set->nr_maps > HCTX_TYPE_POLL)
4429	lim->features \|= BLK_FEAT_POLL;
4430
4431	q = blk_alloc_queue(lim, set->numa_node);
4432	if (IS_ERR(q))
4433	return q;
4434	q->queuedata = queuedata;
4435	ret = blk_mq_init_allocated_queue(set, q);
4436	if (ret) {
4437	blk_put_queue(q);
4438	return ERR_PTR(ret);
4439	}
4440	return q;
4441	}
4442	EXPORT_SYMBOL(blk_mq_alloc_queue);
4443
4444	/**
4445	* blk_mq_destroy_queue - shutdown a request queue
4446	* @q: request queue to shutdown
4447	*
4448	* This shuts down a request queue allocated by blk_mq_alloc_queue(). All future
4449	* requests will be failed with -ENODEV. The caller is responsible for dropping
4450	* the reference from blk_mq_alloc_queue() by calling blk_put_queue().
4451	*
4452	* Context: can sleep
4453	*/
4454	void blk_mq_destroy_queue(struct request_queue *q)
4455	{
4456	WARN_ON_ONCE(!queue_is_mq(q));
4457	WARN_ON_ONCE(blk_queue_registered(q));
4458
4459	might_sleep();
4460
4461	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
4462	blk_queue_start_drain(q);
4463	blk_mq_freeze_queue_wait(q);
4464
4465	blk_sync_queue(q);
4466	blk_mq_cancel_work_sync(q);
4467	blk_mq_exit_queue(q);
4468	}
4469	EXPORT_SYMBOL(blk_mq_destroy_queue);
4470
4471	struct gendisk __blk_mq_alloc_disk(struct blk_mq_tag_set set,
4472	struct queue_limits lim, void queuedata,
4473	struct lock_class_key *lkclass)
4474	{
4475	struct request_queue *q;
4476	struct gendisk *disk;
4477
4478	q = blk_mq_alloc_queue(set, lim, queuedata);
4479	if (IS_ERR(q))
4480	return ERR_CAST(q);
4481
4482	disk = __alloc_disk_node(q, set->numa_node, lkclass);
4483	if (!disk) {
4484	blk_mq_destroy_queue(q);
4485	blk_put_queue(q);
4486	return ERR_PTR(-ENOMEM);
4487	}
4488	set_bit(GD_OWNS_QUEUE, &disk->state);
4489	return disk;
4490	}
4491	EXPORT_SYMBOL(__blk_mq_alloc_disk);
4492
4493	struct gendisk blk_mq_alloc_disk_for_queue(struct request_queue q,
4494	struct lock_class_key *lkclass)
4495	{
4496	struct gendisk *disk;
4497
4498	if (!blk_get_queue(q))
4499	return NULL;
4500	disk = __alloc_disk_node(q, NUMA_NO_NODE, lkclass);
4501	if (!disk)
4502	blk_put_queue(q);
4503	return disk;
4504	}
4505	EXPORT_SYMBOL(blk_mq_alloc_disk_for_queue);
4506
4507	/*
4508	* Only hctx removed from cpuhp list can be reused
4509	*/
4510	static bool blk_mq_hctx_is_reusable(struct blk_mq_hw_ctx *hctx)
4511	{
4512	return hlist_unhashed(&hctx->cpuhp_online) &&
4513	hlist_unhashed(&hctx->cpuhp_dead);
4514	}
4515
4516	static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
4517	struct blk_mq_tag_set set, struct request_queue q,
4518	int hctx_idx, int node)
4519	{
4520	struct blk_mq_hw_ctx hctx = NULL, tmp;
4521
4522	/* reuse dead hctx first */
4523	spin_lock(&q->unused_hctx_lock);
4524	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
4525	if (tmp->numa_node == node && blk_mq_hctx_is_reusable(tmp)) {
4526	hctx = tmp;
4527	break;
4528	}
4529	}
4530	if (hctx)
4531	list_del_init(&hctx->hctx_list);
4532	spin_unlock(&q->unused_hctx_lock);
4533
4534	if (!hctx)
4535	hctx = blk_mq_alloc_hctx(q, set, node);
4536	if (!hctx)
4537	goto fail;
4538
4539	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
4540	goto free_hctx;
4541
4542	return hctx;
4543
4544	free_hctx:
4545	kobject_put(&hctx->kobj);
4546	fail:
4547	return NULL;
4548	}
4549
4550	static void __blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4551	struct request_queue *q)
4552	{
4553	int i, j, end;
4554	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
4555
4556	if (q->nr_hw_queues < set->nr_hw_queues) {
4557	struct blk_mq_hw_ctx **new_hctxs;
4558
4559	new_hctxs = kcalloc_node(set->nr_hw_queues,
4560	sizeof(*new_hctxs), GFP_KERNEL,
4561	set->numa_node);
4562	if (!new_hctxs)
4563	return;
4564	if (hctxs)
4565	memcpy(new_hctxs, hctxs, q->nr_hw_queues *
4566	sizeof(*hctxs));
4567	rcu_assign_pointer(q->queue_hw_ctx, new_hctxs);
4568	/*
4569	* Make sure reading the old queue_hw_ctx from other
4570	* context concurrently won't trigger uaf.
4571	*/
4572	kfree_rcu_mightsleep(hctxs);
4573	hctxs = new_hctxs;
4574	}
4575
4576	for (i = 0; i < set->nr_hw_queues; i++) {
4577	int old_node;
4578	int node = blk_mq_get_hctx_node(set, i);
4579	struct blk_mq_hw_ctx *old_hctx = hctxs[i];
4580
4581	if (old_hctx) {
4582	old_node = old_hctx->numa_node;
4583	blk_mq_exit_hctx(q, set, old_hctx, i);
4584	}
4585
4586	hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i, node);
4587	if (!hctxs[i]) {
4588	if (!old_hctx)
4589	break;
4590	pr_warn("Allocate new hctx on node %d fails, fallback to previous one on node %d\n",
4591	node, old_node);
4592	hctxs[i] = blk_mq_alloc_and_init_hctx(set, q, i,
4593	old_node);
4594	WARN_ON_ONCE(!hctxs[i]);
4595	}
4596	}
4597	/*
4598	* Increasing nr_hw_queues fails. Free the newly allocated
4599	* hctxs and keep the previous q->nr_hw_queues.
4600	*/
4601	if (i != set->nr_hw_queues) {
4602	j = q->nr_hw_queues;
4603	end = i;
4604	} else {
4605	j = i;
4606	end = q->nr_hw_queues;
4607	q->nr_hw_queues = set->nr_hw_queues;
4608	}
4609
4610	for (; j < end; j++) {
4611	struct blk_mq_hw_ctx *hctx = hctxs[j];
4612
4613	if (hctx) {
4614	blk_mq_exit_hctx(q, set, hctx, j);
4615	hctxs[j] = NULL;
4616	}
4617	}
4618	}
4619
4620	static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
4621	struct request_queue *q)
4622	{
4623	__blk_mq_realloc_hw_ctxs(set, q);
4624
4625	/* unregister cpuhp callbacks for exited hctxs */
4626	blk_mq_remove_hw_queues_cpuhp(q);
4627
4628	/* register cpuhp for new initialized hctxs */
4629	blk_mq_add_hw_queues_cpuhp(q);
4630	}
4631
4632	int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
4633	struct request_queue *q)
4634	{
4635	/* mark the queue as mq asap */
4636	q->mq_ops = set->ops;
4637
4638	/*
4639	* ->tag_set has to be setup before initialize hctx, which cpuphp
4640	* handler needs it for checking queue mapping
4641	*/
4642	q->tag_set = set;
4643
4644	if (blk_mq_alloc_ctxs(q))
4645	goto err_exit;
4646
4647	/* init q->mq_kobj and sw queues' kobjects */
4648	blk_mq_sysfs_init(q);
4649
4650	INIT_LIST_HEAD(&q->unused_hctx_list);
4651	spin_lock_init(&q->unused_hctx_lock);
4652
4653	blk_mq_realloc_hw_ctxs(set, q);
4654	if (!q->nr_hw_queues)
4655	goto err_hctxs;
4656
4657	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
4658	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
4659
4660	q->queue_flags \|= QUEUE_FLAG_MQ_DEFAULT;
4661
4662	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
4663	INIT_LIST_HEAD(&q->flush_list);
4664	INIT_LIST_HEAD(&q->requeue_list);
4665	spin_lock_init(&q->requeue_lock);
4666
4667	q->nr_requests = set->queue_depth;
4668	q->async_depth = set->queue_depth;
4669
4670	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
4671	blk_mq_map_swqueue(q);
4672	blk_mq_add_queue_tag_set(set, q);
4673	return 0;
4674
4675	err_hctxs:
4676	blk_mq_release(q);
4677	err_exit:
4678	q->mq_ops = NULL;
4679	return -ENOMEM;
4680	}
4681	EXPORT_SYMBOL(blk_mq_init_allocated_queue);
4682
4683	/* tags can _not_ be used after returning from blk_mq_exit_queue */
4684	void blk_mq_exit_queue(struct request_queue *q)
4685	{
4686	struct blk_mq_tag_set *set = q->tag_set;
4687
4688	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
4689	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
4690	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
4691	blk_mq_del_queue_tag_set(q);
4692	}
4693
4694	static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
4695	{
4696	int i;
4697
4698	if (blk_mq_is_shared_tags(set->flags)) {
4699	set->shared_tags = blk_mq_alloc_map_and_rqs(set,
4700	BLK_MQ_NO_HCTX_IDX,
4701	set->queue_depth);
4702	if (!set->shared_tags)
4703	return -ENOMEM;
4704	}
4705
4706	for (i = 0; i < set->nr_hw_queues; i++) {
4707	if (!__blk_mq_alloc_map_and_rqs(set, i))
4708	goto out_unwind;
4709	cond_resched();
4710	}
4711
4712	return 0;
4713
4714	out_unwind:
4715	while (--i >= 0)
4716	__blk_mq_free_map_and_rqs(set, i);
4717
4718	if (blk_mq_is_shared_tags(set->flags)) {
4719	blk_mq_free_map_and_rqs(set, set->shared_tags,
4720	BLK_MQ_NO_HCTX_IDX);
4721	}
4722
4723	return -ENOMEM;
4724	}
4725
4726	/*
4727	* Allocate the request maps associated with this tag_set. Note that this
4728	* may reduce the depth asked for, if memory is tight. set->queue_depth
4729	* will be updated to reflect the allocated depth.
4730	*/
4731	static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set)
4732	{
4733	unsigned int depth;
4734	int err;
4735
4736	depth = set->queue_depth;
4737	do {
4738	err = __blk_mq_alloc_rq_maps(set);
4739	if (!err)
4740	break;
4741
4742	set->queue_depth >>= 1;
4743	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
4744	err = -ENOMEM;
4745	break;
4746	}
4747	} while (set->queue_depth);
4748
4749	if (!set->queue_depth \|\| err) {
4750	pr_err("blk-mq: failed to allocate request map\n");
4751	return -ENOMEM;
4752	}
4753
4754	if (depth != set->queue_depth)
4755	pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
4756	depth, set->queue_depth);
4757
4758	return 0;
4759	}
4760
4761	static void blk_mq_update_queue_map(struct blk_mq_tag_set *set)
4762	{
4763	/*
4764	* blk_mq_map_queues() and multiple .map_queues() implementations
4765	* expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
4766	* number of hardware queues.
4767	*/
4768	if (set->nr_maps == 1)
4769	set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;
4770
4771	if (set->ops->map_queues) {
4772	int i;
4773
4774	/*
4775	* transport .map_queues is usually done in the following
4776	* way:
4777	*
4778	* for (queue = 0; queue < set->nr_hw_queues; queue++) {
4779	* mask = get_cpu_mask(queue)
4780	* for_each_cpu(cpu, mask)
4781	* set->map[x].mq_map[cpu] = queue;
4782	* }
4783	*
4784	* When we need to remap, the table has to be cleared for
4785	* killing stale mapping since one CPU may not be mapped
4786	* to any hw queue.
4787	*/
4788	for (i = 0; i < set->nr_maps; i++)
4789	blk_mq_clear_mq_map(&set->map[i]);
4790
4791	set->ops->map_queues(set);
4792	} else {
4793	BUG_ON(set->nr_maps > 1);
4794	blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
4795	}
4796	}
4797
4798	static struct blk_mq_tags **blk_mq_prealloc_tag_set_tags(
4799	struct blk_mq_tag_set *set,
4800	int new_nr_hw_queues)
4801	{
4802	struct blk_mq_tags **new_tags;
4803	int i;
4804
4805	if (set->nr_hw_queues >= new_nr_hw_queues)
4806	return NULL;
4807
4808	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
4809	GFP_KERNEL, set->numa_node);
4810	if (!new_tags)
4811	return ERR_PTR(-ENOMEM);
4812
4813	if (set->tags)
4814	memcpy(new_tags, set->tags, set->nr_hw_queues *
4815	sizeof(*set->tags));
4816
4817	for (i = set->nr_hw_queues; i < new_nr_hw_queues; i++) {
4818	if (blk_mq_is_shared_tags(set->flags)) {
4819	new_tags[i] = set->shared_tags;
4820	} else {
4821	new_tags[i] = blk_mq_alloc_map_and_rqs(set, i,
4822	set->queue_depth);
4823	if (!new_tags[i])
4824	goto out_unwind;
4825	}
4826	cond_resched();
4827	}
4828
4829	return new_tags;
4830	out_unwind:
4831	while (--i >= set->nr_hw_queues) {
4832	if (!blk_mq_is_shared_tags(set->flags))
4833	blk_mq_free_map_and_rqs(set, new_tags[i], i);
4834	}
4835	kfree(new_tags);
4836	return ERR_PTR(-ENOMEM);
4837	}
4838
4839	/*
4840	* Alloc a tag set to be associated with one or more request queues.
4841	* May fail with EINVAL for various error conditions. May adjust the
4842	* requested depth down, if it's too large. In that case, the set
4843	* value will be stored in set->queue_depth.
4844	*/
4845	int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
4846	{
4847	int i, ret;
4848
4849	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);
4850
4851	if (!set->nr_hw_queues)
4852	return -EINVAL;
4853	if (!set->queue_depth)
4854	return -EINVAL;
4855	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
4856	return -EINVAL;
4857
4858	if (!set->ops->queue_rq)
4859	return -EINVAL;
4860
4861	if (!set->ops->get_budget ^ !set->ops->put_budget)
4862	return -EINVAL;
4863
4864	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
4865	pr_info("blk-mq: reduced tag depth to %u\n",
4866	BLK_MQ_MAX_DEPTH);
4867	set->queue_depth = BLK_MQ_MAX_DEPTH;
4868	}
4869
4870	if (!set->nr_maps)
4871	set->nr_maps = 1;
4872	else if (set->nr_maps > HCTX_MAX_TYPES)
4873	return -EINVAL;
4874
4875	/*
4876	* If a crashdump is active, then we are potentially in a very
4877	* memory constrained environment. Limit us to 64 tags to prevent
4878	* using too much memory.
4879	*/
4880	if (is_kdump_kernel())
4881	set->queue_depth = min(64U, set->queue_depth);
4882
4883	/*
4884	* There is no use for more h/w queues than cpus if we just have
4885	* a single map
4886	*/
4887	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
4888	set->nr_hw_queues = nr_cpu_ids;
4889
4890	if (set->flags & BLK_MQ_F_BLOCKING) {
4891	set->srcu = kmalloc_obj(*set->srcu);
4892	if (!set->srcu)
4893	return -ENOMEM;
4894	ret = init_srcu_struct(set->srcu);
4895	if (ret)
4896	goto out_free_srcu;
4897	}
4898	ret = init_srcu_struct(&set->tags_srcu);
4899	if (ret)
4900	goto out_cleanup_srcu;
4901
4902	init_rwsem(&set->update_nr_hwq_lock);
4903
4904	ret = -ENOMEM;
4905	set->tags = kcalloc_node(set->nr_hw_queues,
4906	sizeof(struct blk_mq_tags *), GFP_KERNEL,
4907	set->numa_node);
4908	if (!set->tags)
4909	goto out_cleanup_tags_srcu;
4910
4911	for (i = 0; i < set->nr_maps; i++) {
4912	set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
4913	sizeof(set->map[i].mq_map[0]),
4914	GFP_KERNEL, set->numa_node);
4915	if (!set->map[i].mq_map)
4916	goto out_free_mq_map;
4917	set->map[i].nr_queues = set->nr_hw_queues;
4918	}
4919
4920	blk_mq_update_queue_map(set);
4921
4922	ret = blk_mq_alloc_set_map_and_rqs(set);
4923	if (ret)
4924	goto out_free_mq_map;
4925
4926	mutex_init(&set->tag_list_lock);
4927	INIT_LIST_HEAD(&set->tag_list);
4928
4929	return 0;
4930
4931	out_free_mq_map:
4932	for (i = 0; i < set->nr_maps; i++) {
4933	kfree(set->map[i].mq_map);
4934	set->map[i].mq_map = NULL;
4935	}
4936	kfree(set->tags);
4937	set->tags = NULL;
4938	out_cleanup_tags_srcu:
4939	cleanup_srcu_struct(&set->tags_srcu);
4940	out_cleanup_srcu:
4941	if (set->flags & BLK_MQ_F_BLOCKING)
4942	cleanup_srcu_struct(set->srcu);
4943	out_free_srcu:
4944	if (set->flags & BLK_MQ_F_BLOCKING)
4945	kfree(set->srcu);
4946	return ret;
4947	}
4948	EXPORT_SYMBOL(blk_mq_alloc_tag_set);
4949
4950	/* allocate and initialize a tagset for a simple single-queue device */
4951	int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
4952	const struct blk_mq_ops *ops, unsigned int queue_depth,
4953	unsigned int set_flags)
4954	{
4955	memset(set, 0, sizeof(*set));
4956	set->ops = ops;
4957	set->nr_hw_queues = 1;
4958	set->nr_maps = 1;
4959	set->queue_depth = queue_depth;
4960	set->numa_node = NUMA_NO_NODE;
4961	set->flags = set_flags;
4962	return blk_mq_alloc_tag_set(set);
4963	}
4964	EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);
4965
4966	void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
4967	{
4968	int i, j;
4969
4970	for (i = 0; i < set->nr_hw_queues; i++)
4971	__blk_mq_free_map_and_rqs(set, i);
4972
4973	if (blk_mq_is_shared_tags(set->flags)) {
4974	blk_mq_free_map_and_rqs(set, set->shared_tags,
4975	BLK_MQ_NO_HCTX_IDX);
4976	}
4977
4978	for (j = 0; j < set->nr_maps; j++) {
4979	kfree(set->map[j].mq_map);
4980	set->map[j].mq_map = NULL;
4981	}
4982
4983	kfree(set->tags);
4984	set->tags = NULL;
4985
4986	srcu_barrier(&set->tags_srcu);
4987	cleanup_srcu_struct(&set->tags_srcu);
4988	if (set->flags & BLK_MQ_F_BLOCKING) {
4989	srcu_barrier(set->srcu);
4990	cleanup_srcu_struct(set->srcu);
4991	kfree(set->srcu);
4992	}
4993	}
4994	EXPORT_SYMBOL(blk_mq_free_tag_set);
4995
4996	struct elevator_tags blk_mq_update_nr_requests(struct request_queue q,
4997	struct elevator_tags *et,
4998	unsigned int nr)
4999	{
5000	struct blk_mq_tag_set *set = q->tag_set;
5001	struct elevator_tags *old_et = NULL;
5002	struct blk_mq_hw_ctx *hctx;
5003	unsigned long i;
5004
5005	blk_mq_quiesce_queue(q);
5006
5007	if (blk_mq_is_shared_tags(set->flags)) {
5008	/*
5009	* Shared tags, for sched tags, we allocate max initially hence
5010	* tags can't grow, see blk_mq_alloc_sched_tags().
5011	*/
5012	if (q->elevator)
5013	blk_mq_tag_update_sched_shared_tags(q, nr);
5014	else
5015	blk_mq_tag_resize_shared_tags(set, nr);
5016	} else if (!q->elevator) {
5017	/*
5018	* Non-shared hardware tags, nr is already checked from
5019	* queue_requests_store() and tags can't grow.
5020	*/
5021	queue_for_each_hw_ctx(q, hctx, i) {
5022	if (!hctx->tags)
5023	continue;
5024	sbitmap_queue_resize(&hctx->tags->bitmap_tags,
5025	nr - hctx->tags->nr_reserved_tags);
5026	}
5027	} else if (nr <= q->elevator->et->nr_requests) {
5028	/* Non-shared sched tags, and tags don't grow. */
5029	queue_for_each_hw_ctx(q, hctx, i) {
5030	if (!hctx->sched_tags)
5031	continue;
5032	sbitmap_queue_resize(&hctx->sched_tags->bitmap_tags,
5033	nr - hctx->sched_tags->nr_reserved_tags);
5034	}
5035	} else {
5036	/* Non-shared sched tags, and tags grow */
5037	queue_for_each_hw_ctx(q, hctx, i)
5038	hctx->sched_tags = et->tags[i];
5039	old_et = q->elevator->et;
5040	q->elevator->et = et;
5041	}
5042
5043	/*
5044	* Preserve relative value, both nr and async_depth are at most 16 bit
5045	* value, no need to worry about overflow.
5046	*/
5047	q->async_depth = max(q->async_depth * nr / q->nr_requests, 1);
5048	q->nr_requests = nr;
5049	if (q->elevator && q->elevator->type->ops.depth_updated)
5050	q->elevator->type->ops.depth_updated(q);
5051
5052	blk_mq_unquiesce_queue(q);
5053	return old_et;
5054	}
5055
5056	/*
5057	* Switch back to the elevator type stored in the xarray.
5058	*/
5059	static void blk_mq_elv_switch_back(struct request_queue *q,
5060	struct xarray *elv_tbl)
5061	{
5062	struct elv_change_ctx *ctx = xa_load(elv_tbl, q->id);
5063
5064	if (WARN_ON_ONCE(!ctx))
5065	return;
5066
5067	/* The elv_update_nr_hw_queues unfreezes the queue. */
5068	elv_update_nr_hw_queues(q, ctx);
5069
5070	/* Drop the reference acquired in blk_mq_elv_switch_none. */
5071	if (ctx->type)
5072	elevator_put(ctx->type);
5073	}
5074
5075	/*
5076	* Stores elevator name and type in ctx and set current elevator to none.
5077	*/
5078	static int blk_mq_elv_switch_none(struct request_queue *q,
5079	struct xarray *elv_tbl)
5080	{
5081	struct elv_change_ctx *ctx;
5082
5083	lockdep_assert_held_write(&q->tag_set->update_nr_hwq_lock);
5084
5085	/*
5086	* Accessing q->elevator without holding q->elevator_lock is safe here
5087	* because we're called from nr_hw_queue update which is protected by
5088	* set->update_nr_hwq_lock in the writer context. So, scheduler update/
5089	* switch code (which acquires the same lock in the reader context)
5090	* can't run concurrently.
5091	*/
5092	if (q->elevator) {
5093	ctx = xa_load(elv_tbl, q->id);
5094	if (WARN_ON_ONCE(!ctx))
5095	return -ENOENT;
5096
5097	ctx->name = q->elevator->type->elevator_name;
5098
5099	/*
5100	* Before we switch elevator to 'none', take a reference to
5101	* the elevator module so that while nr_hw_queue update is
5102	* running, no one can remove elevator module. We'd put the
5103	* reference to elevator module later when we switch back
5104	* elevator.
5105	*/
5106	__elevator_get(q->elevator->type);
5107
5108	/*
5109	* Store elevator type so that we can release the reference
5110	* taken above later.
5111	*/
5112	ctx->type = q->elevator->type;
5113	elevator_set_none(q);
5114	}
5115	return 0;
5116	}
5117
5118	static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
5119	int nr_hw_queues)
5120	{
5121	struct request_queue *q;
5122	int prev_nr_hw_queues = set->nr_hw_queues;
5123	unsigned int memflags;
5124	int i;
5125	struct xarray elv_tbl;
5126	struct blk_mq_tags **new_tags;
5127	bool queues_frozen = false;
5128
5129	lockdep_assert_held(&set->tag_list_lock);
5130
5131	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
5132	nr_hw_queues = nr_cpu_ids;
5133	if (nr_hw_queues < 1)
5134	return;
5135	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
5136	return;
5137
5138	memflags = memalloc_noio_save();
5139
5140	xa_init(&elv_tbl);
5141	if (blk_mq_alloc_sched_ctx_batch(&elv_tbl, set) < 0)
5142	goto out_free_ctx;
5143
5144	if (blk_mq_alloc_sched_res_batch(&elv_tbl, set, nr_hw_queues) < 0)
5145	goto out_free_ctx;
5146
5147	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5148	blk_mq_debugfs_unregister_hctxs(q);
5149	blk_mq_sysfs_unregister_hctxs(q);
5150	}
5151
5152	/*
5153	* Switch IO scheduler to 'none', cleaning up the data associated
5154	* with the previous scheduler. We will switch back once we are done
5155	* updating the new sw to hw queue mappings.
5156	*/
5157	list_for_each_entry(q, &set->tag_list, tag_set_list)
5158	if (blk_mq_elv_switch_none(q, &elv_tbl))
5159	goto switch_back;
5160
5161	new_tags = blk_mq_prealloc_tag_set_tags(set, nr_hw_queues);
5162	if (IS_ERR(new_tags))
5163	goto switch_back;
5164
5165	list_for_each_entry(q, &set->tag_list, tag_set_list)
5166	blk_mq_freeze_queue_nomemsave(q);
5167	queues_frozen = true;
5168	if (new_tags) {
5169	kfree(set->tags);
5170	set->tags = new_tags;
5171	}
5172	set->nr_hw_queues = nr_hw_queues;
5173
5174	fallback:
5175	blk_mq_update_queue_map(set);
5176	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5177	__blk_mq_realloc_hw_ctxs(set, q);
5178
5179	if (q->nr_hw_queues != set->nr_hw_queues) {
5180	int i = prev_nr_hw_queues;
5181
5182	pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
5183	nr_hw_queues, prev_nr_hw_queues);
5184	for (; i < set->nr_hw_queues; i++)
5185	__blk_mq_free_map_and_rqs(set, i);
5186
5187	set->nr_hw_queues = prev_nr_hw_queues;
5188	goto fallback;
5189	}
5190	blk_mq_map_swqueue(q);
5191	}
5192	switch_back:
5193	/* The blk_mq_elv_switch_back unfreezes queue for us. */
5194	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5195	/* switch_back expects queue to be frozen */
5196	if (!queues_frozen)
5197	blk_mq_freeze_queue_nomemsave(q);
5198	blk_mq_elv_switch_back(q, &elv_tbl);
5199	}
5200
5201	list_for_each_entry(q, &set->tag_list, tag_set_list) {
5202	blk_mq_sysfs_register_hctxs(q);
5203	blk_mq_debugfs_register_hctxs(q);
5204
5205	blk_mq_remove_hw_queues_cpuhp(q);
5206	blk_mq_add_hw_queues_cpuhp(q);
5207	}
5208
5209	out_free_ctx:
5210	blk_mq_free_sched_ctx_batch(&elv_tbl);
5211	xa_destroy(&elv_tbl);
5212	memalloc_noio_restore(memflags);
5213
5214	/* Free the excess tags when nr_hw_queues shrink. */
5215	for (i = set->nr_hw_queues; i < prev_nr_hw_queues; i++)
5216	__blk_mq_free_map_and_rqs(set, i);
5217	}
5218
5219	void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
5220	{
5221	down_write(&set->update_nr_hwq_lock);
5222	mutex_lock(&set->tag_list_lock);
5223	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
5224	mutex_unlock(&set->tag_list_lock);
5225	up_write(&set->update_nr_hwq_lock);
5226	}
5227	EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);
5228
5229	static int blk_hctx_poll(struct request_queue q, struct blk_mq_hw_ctx hctx,
5230	struct io_comp_batch *iob, unsigned int flags)
5231	{
5232	int ret;
5233	unsigned long timeout = jiffies + 2;
5234
5235	do {
5236	ret = q->mq_ops->poll(hctx, iob);
5237	if (ret > 0)
5238	return ret;
5239	if (task_sigpending(current))
5240	return 1;
5241	if (ret < 0 \|\| (flags & BLK_POLL_ONESHOT))
5242	break;
5243	cpu_relax();
5244	} while (!need_resched() && time_before(jiffies, timeout));
5245
5246	return 0;
5247	}
5248
5249	int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
5250	struct io_comp_batch *iob, unsigned int flags)
5251	{
5252	if (!blk_mq_can_poll(q))
5253	return 0;
5254	return blk_hctx_poll(q, q->queue_hw_ctx[cookie], iob, flags);
5255	}
5256
5257	int blk_rq_poll(struct request rq, struct io_comp_batch iob,
5258	unsigned int poll_flags)
5259	{
5260	struct request_queue *q = rq->q;
5261	int ret;
5262
5263	if (!blk_rq_is_poll(rq))
5264	return 0;
5265	if (!percpu_ref_tryget(&q->q_usage_counter))
5266	return 0;
5267
5268	ret = blk_hctx_poll(q, rq->mq_hctx, iob, poll_flags);
5269	blk_queue_exit(q);
5270
5271	return ret;
5272	}
5273	EXPORT_SYMBOL_GPL(blk_rq_poll);
5274
5275	unsigned int blk_mq_rq_cpu(struct request *rq)
5276	{
5277	return rq->mq_ctx->cpu;
5278	}
5279	EXPORT_SYMBOL(blk_mq_rq_cpu);
5280
5281	void blk_mq_cancel_work_sync(struct request_queue *q)
5282	{
5283	struct blk_mq_hw_ctx *hctx;
5284	unsigned long i;
5285
5286	cancel_delayed_work_sync(&q->requeue_work);
5287
5288	queue_for_each_hw_ctx(q, hctx, i)
5289	cancel_delayed_work_sync(&hctx->run_work);
5290	}
5291
5292	static int __init blk_mq_init(void)
5293	{
5294	int i;
5295
5296	for_each_possible_cpu(i)
5297	init_llist_head(&per_cpu(blk_cpu_done, i));
5298	for_each_possible_cpu(i)
5299	INIT_CSD(&per_cpu(blk_cpu_csd, i),
5300	__blk_mq_complete_request_remote, NULL);
5301	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);
5302
5303	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
5304	"block/softirq:dead", NULL,
5305	blk_softirq_cpu_dead);
5306	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
5307	blk_mq_hctx_notify_dead);
5308	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
5309	blk_mq_hctx_notify_online,
5310	blk_mq_hctx_notify_offline);
5311	return 0;
5312	}
5313	subsys_initcall(blk_mq_init);