projects/linux·blob assoc-array-fixes · refs

829 lines · 20.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 Facebook
4 */
5
6#include <linux/kernel.h>
7#include <linux/blkdev.h>
8#include <linux/build_bug.h>
9#include <linux/debugfs.h>
10
11#include "blk.h"
12#include "blk-mq.h"
13#include "blk-mq-debugfs.h"
14#include "blk-mq-sched.h"
15#include "blk-rq-qos.h"
16
17static int queue_poll_stat_show(void *data, struct seq_file *m)
18{
19 return 0;
20}
21
22#define TO_REQUEST_QUEUE(m) ((struct request_queue *)m->private)
23
24static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos)
25 __acquires(&TO_REQUEST_QUEUE(m)->requeue_lock)
26{
27 struct request_queue *q = m->private;
28
29 spin_lock_irq(&q->requeue_lock);
30 return seq_list_start(&q->requeue_list, *pos);
31}
32
33static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos)
34{
35 struct request_queue *q = m->private;
36
37 return seq_list_next(v, &q->requeue_list, pos);
38}
39
40static void queue_requeue_list_stop(struct seq_file *m, void *v)
41 __releases(&TO_REQUEST_QUEUE(m)->requeue_lock)
42{
43 struct request_queue *q = m->private;
44
45 spin_unlock_irq(&q->requeue_lock);
46}
47
48#undef TO_REQUEST_QUEUE
49
50static const struct seq_operations queue_requeue_list_seq_ops = {
51 .start = queue_requeue_list_start,
52 .next = queue_requeue_list_next,
53 .stop = queue_requeue_list_stop,
54 .show = blk_mq_debugfs_rq_show,
55};
56
57static int blk_flags_show(struct seq_file *m, const unsigned long flags,
58 const char *const *flag_name, int flag_name_count)
59{
60 bool sep = false;
61 int i;
62
63 for (i = 0; i < sizeof(flags) * BITS_PER_BYTE; i++) {
64 if (!(flags & BIT(i)))
65 continue;
66 if (sep)
67 seq_puts(m, "|");
68 sep = true;
69 if (i < flag_name_count && flag_name[i])
70 seq_puts(m, flag_name[i]);
71 else
72 seq_printf(m, "%d", i);
73 }
74 return 0;
75}
76
77static int queue_pm_only_show(void *data, struct seq_file *m)
78{
79 struct request_queue *q = data;
80
81 seq_printf(m, "%d\n", atomic_read(&q->pm_only));
82 return 0;
83}
84
85#define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name
86static const char *const blk_queue_flag_name[] = {
87 QUEUE_FLAG_NAME(DYING),
88 QUEUE_FLAG_NAME(NOMERGES),
89 QUEUE_FLAG_NAME(SAME_COMP),
90 QUEUE_FLAG_NAME(FAIL_IO),
91 QUEUE_FLAG_NAME(NOXMERGES),
92 QUEUE_FLAG_NAME(SAME_FORCE),
93 QUEUE_FLAG_NAME(INIT_DONE),
94 QUEUE_FLAG_NAME(STATS),
95 QUEUE_FLAG_NAME(REGISTERED),
96 QUEUE_FLAG_NAME(QUIESCED),
97 QUEUE_FLAG_NAME(RQ_ALLOC_TIME),
98 QUEUE_FLAG_NAME(HCTX_ACTIVE),
99 QUEUE_FLAG_NAME(SQ_SCHED),
100 QUEUE_FLAG_NAME(DISABLE_WBT_DEF),
101 QUEUE_FLAG_NAME(NO_ELV_SWITCH),
102 QUEUE_FLAG_NAME(QOS_ENABLED),
103 QUEUE_FLAG_NAME(BIO_ISSUE_TIME),
104 QUEUE_FLAG_NAME(ZONED_QD1_WRITES),
105};
106#undef QUEUE_FLAG_NAME
107
108static int queue_state_show(void *data, struct seq_file *m)
109{
110 struct request_queue *q = data;
111
112 BUILD_BUG_ON(ARRAY_SIZE(blk_queue_flag_name) != QUEUE_FLAG_MAX);
113 blk_flags_show(m, q->queue_flags, blk_queue_flag_name,
114 ARRAY_SIZE(blk_queue_flag_name));
115 seq_puts(m, "\n");
116 return 0;
117}
118
119static ssize_t queue_state_write(void *data, const char __user *buf,
120 size_t count, loff_t *ppos)
121{
122 struct request_queue *q = data;
123 char opbuf[16] = { }, *op;
124
125 /*
126 * The "state" attribute is removed when the queue is removed. Don't
127 * allow setting the state on a dying queue to avoid a use-after-free.
128 */
129 if (blk_queue_dying(q))
130 return -ENOENT;
131
132 if (count >= sizeof(opbuf)) {
133 pr_err("%s: operation too long\n", __func__);
134 goto inval;
135 }
136
137 if (copy_from_user(opbuf, buf, count))
138 return -EFAULT;
139 op = strstrip(opbuf);
140 if (strcmp(op, "run") == 0) {
141 blk_mq_run_hw_queues(q, true);
142 } else if (strcmp(op, "start") == 0) {
143 blk_mq_start_stopped_hw_queues(q, true);
144 } else if (strcmp(op, "kick") == 0) {
145 blk_mq_kick_requeue_list(q);
146 } else {
147 pr_err("%s: unsupported operation '%s'\n", __func__, op);
148inval:
149 pr_err("%s: use 'run', 'start' or 'kick'\n", __func__);
150 return -EINVAL;
151 }
152 return count;
153}
154
155static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = {
156 { "poll_stat", 0400, queue_poll_stat_show },
157 { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops },
158 { "pm_only", 0600, queue_pm_only_show, NULL },
159 { "state", 0600, queue_state_show, queue_state_write },
160 { "zone_wplugs", 0400, queue_zone_wplugs_show, NULL },
161 { },
162};
163
164#define HCTX_STATE_NAME(name) [BLK_MQ_S_##name] = #name
165static const char *const hctx_state_name[] = {
166 HCTX_STATE_NAME(STOPPED),
167 HCTX_STATE_NAME(TAG_ACTIVE),
168 HCTX_STATE_NAME(SCHED_RESTART),
169 HCTX_STATE_NAME(INACTIVE),
170};
171#undef HCTX_STATE_NAME
172
173static int hctx_state_show(void *data, struct seq_file *m)
174{
175 struct blk_mq_hw_ctx *hctx = data;
176
177 BUILD_BUG_ON(ARRAY_SIZE(hctx_state_name) != BLK_MQ_S_MAX);
178 blk_flags_show(m, hctx->state, hctx_state_name,
179 ARRAY_SIZE(hctx_state_name));
180 seq_puts(m, "\n");
181 return 0;
182}
183
184#define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name
185static const char *const hctx_flag_name[] = {
186 HCTX_FLAG_NAME(TAG_QUEUE_SHARED),
187 HCTX_FLAG_NAME(STACKING),
188 HCTX_FLAG_NAME(TAG_HCTX_SHARED),
189 HCTX_FLAG_NAME(BLOCKING),
190 HCTX_FLAG_NAME(TAG_RR),
191 HCTX_FLAG_NAME(NO_SCHED_BY_DEFAULT),
192};
193#undef HCTX_FLAG_NAME
194
195static int hctx_flags_show(void *data, struct seq_file *m)
196{
197 struct blk_mq_hw_ctx *hctx = data;
198
199 BUILD_BUG_ON(ARRAY_SIZE(hctx_flag_name) != ilog2(BLK_MQ_F_MAX));
200
201 blk_flags_show(m, hctx->flags, hctx_flag_name,
202 ARRAY_SIZE(hctx_flag_name));
203 seq_puts(m, "\n");
204 return 0;
205}
206
207#define CMD_FLAG_NAME(name) [__REQ_##name] = #name
208static const char *const cmd_flag_name[] = {
209 CMD_FLAG_NAME(FAILFAST_DEV),
210 CMD_FLAG_NAME(FAILFAST_TRANSPORT),
211 CMD_FLAG_NAME(FAILFAST_DRIVER),
212 CMD_FLAG_NAME(SYNC),
213 CMD_FLAG_NAME(META),
214 CMD_FLAG_NAME(PRIO),
215 CMD_FLAG_NAME(NOMERGE),
216 CMD_FLAG_NAME(IDLE),
217 CMD_FLAG_NAME(INTEGRITY),
218 CMD_FLAG_NAME(FUA),
219 CMD_FLAG_NAME(PREFLUSH),
220 CMD_FLAG_NAME(RAHEAD),
221 CMD_FLAG_NAME(BACKGROUND),
222 CMD_FLAG_NAME(NOWAIT),
223 CMD_FLAG_NAME(POLLED),
224 CMD_FLAG_NAME(ALLOC_CACHE),
225 CMD_FLAG_NAME(SWAP),
226 CMD_FLAG_NAME(DRV),
227 CMD_FLAG_NAME(FS_PRIVATE),
228 CMD_FLAG_NAME(ATOMIC),
229 CMD_FLAG_NAME(NOUNMAP),
230};
231#undef CMD_FLAG_NAME
232
233#define RQF_NAME(name) [__RQF_##name] = #name
234static const char *const rqf_name[] = {
235 RQF_NAME(STARTED),
236 RQF_NAME(FLUSH_SEQ),
237 RQF_NAME(MIXED_MERGE),
238 RQF_NAME(DONTPREP),
239 RQF_NAME(SCHED_TAGS),
240 RQF_NAME(USE_SCHED),
241 RQF_NAME(FAILED),
242 RQF_NAME(QUIET),
243 RQF_NAME(IO_STAT),
244 RQF_NAME(PM),
245 RQF_NAME(HASHED),
246 RQF_NAME(STATS),
247 RQF_NAME(SPECIAL_PAYLOAD),
248 RQF_NAME(ZONE_WRITE_PLUGGING),
249 RQF_NAME(TIMED_OUT),
250 RQF_NAME(RESV),
251};
252#undef RQF_NAME
253
254static const char *const blk_mq_rq_state_name_array[] = {
255 [MQ_RQ_IDLE] = "idle",
256 [MQ_RQ_IN_FLIGHT] = "in_flight",
257 [MQ_RQ_COMPLETE] = "complete",
258};
259
260static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state)
261{
262 if (WARN_ON_ONCE((unsigned int)rq_state >=
263 ARRAY_SIZE(blk_mq_rq_state_name_array)))
264 return "(?)";
265 return blk_mq_rq_state_name_array[rq_state];
266}
267
268int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
269{
270 const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
271 const enum req_op op = req_op(rq);
272 const char *op_str = blk_op_str(op);
273
274 BUILD_BUG_ON(ARRAY_SIZE(cmd_flag_name) != __REQ_NR_BITS);
275 BUILD_BUG_ON(ARRAY_SIZE(rqf_name) != __RQF_BITS);
276
277 seq_printf(m, "%p {.op=", rq);
278 if (strcmp(op_str, "UNKNOWN") == 0)
279 seq_printf(m, "%u", op);
280 else
281 seq_printf(m, "%s", op_str);
282 seq_puts(m, ", .cmd_flags=");
283 blk_flags_show(m, (__force unsigned int)(rq->cmd_flags & ~REQ_OP_MASK),
284 cmd_flag_name, ARRAY_SIZE(cmd_flag_name));
285 seq_puts(m, ", .rq_flags=");
286 blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name,
287 ARRAY_SIZE(rqf_name));
288 seq_printf(m, ", .state=%s", blk_mq_rq_state_name(blk_mq_rq_state(rq)));
289 seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag,
290 rq->internal_tag);
291 if (mq_ops->show_rq)
292 mq_ops->show_rq(m, rq);
293 seq_puts(m, "}\n");
294 return 0;
295}
296EXPORT_SYMBOL_GPL(__blk_mq_debugfs_rq_show);
297
298int blk_mq_debugfs_rq_show(struct seq_file *m, void *v)
299{
300 return __blk_mq_debugfs_rq_show(m, list_entry_rq(v));
301}
302EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show);
303
304#define TO_HCTX(m) ((struct blk_mq_hw_ctx *)m->private)
305
306static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos)
307 __acquires(&TO_HCTX(m)->lock)
308{
309 struct blk_mq_hw_ctx *hctx = m->private;
310
311 spin_lock(&hctx->lock);
312 return seq_list_start(&hctx->dispatch, *pos);
313}
314
315static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos)
316{
317 struct blk_mq_hw_ctx *hctx = m->private;
318
319 return seq_list_next(v, &hctx->dispatch, pos);
320}
321
322static void hctx_dispatch_stop(struct seq_file *m, void *v)
323 __releases(&TO_HCTX(m)->lock)
324{
325 struct blk_mq_hw_ctx *hctx = m->private;
326
327 spin_unlock(&hctx->lock);
328}
329
330#undef TO_HCTX
331
332static const struct seq_operations hctx_dispatch_seq_ops = {
333 .start = hctx_dispatch_start,
334 .next = hctx_dispatch_next,
335 .stop = hctx_dispatch_stop,
336 .show = blk_mq_debugfs_rq_show,
337};
338
339struct show_busy_params {
340 struct seq_file *m;
341 struct blk_mq_hw_ctx *hctx;
342};
343
344/*
345 * Note: the state of a request may change while this function is in progress,
346 * e.g. due to a concurrent blk_mq_finish_request() call. Returns true to
347 * keep iterating requests.
348 */
349static bool hctx_show_busy_rq(struct request *rq, void *data)
350{
351 const struct show_busy_params *params = data;
352
353 if (rq->mq_hctx == params->hctx)
354 __blk_mq_debugfs_rq_show(params->m, rq);
355
356 return true;
357}
358
359static int hctx_busy_show(void *data, struct seq_file *m)
360{
361 struct blk_mq_hw_ctx *hctx = data;
362 struct show_busy_params params = { .m = m, .hctx = hctx };
363 int res;
364
365 res = mutex_lock_interruptible(&hctx->queue->elevator_lock);
366 if (res)
367 return res;
368 blk_mq_tagset_busy_iter(hctx->queue->tag_set, hctx_show_busy_rq,
369 &params);
370 mutex_unlock(&hctx->queue->elevator_lock);
371
372 return 0;
373}
374
375static const char *const hctx_types[] = {
376 [HCTX_TYPE_DEFAULT] = "default",
377 [HCTX_TYPE_READ] = "read",
378 [HCTX_TYPE_POLL] = "poll",
379};
380
381static int hctx_type_show(void *data, struct seq_file *m)
382{
383 struct blk_mq_hw_ctx *hctx = data;
384
385 BUILD_BUG_ON(ARRAY_SIZE(hctx_types) != HCTX_MAX_TYPES);
386 seq_printf(m, "%s\n", hctx_types[hctx->type]);
387 return 0;
388}
389
390static int hctx_ctx_map_show(void *data, struct seq_file *m)
391{
392 struct blk_mq_hw_ctx *hctx = data;
393
394 sbitmap_bitmap_show(&hctx->ctx_map, m);
395 return 0;
396}
397
398static void blk_mq_debugfs_tags_show(struct seq_file *m,
399 struct blk_mq_tags *tags)
400{
401 seq_printf(m, "nr_tags=%u\n", tags->nr_tags);
402 seq_printf(m, "nr_reserved_tags=%u\n", tags->nr_reserved_tags);
403 seq_printf(m, "active_queues=%d\n",
404 READ_ONCE(tags->active_queues));
405
406 seq_puts(m, "\nbitmap_tags:\n");
407 sbitmap_queue_show(&tags->bitmap_tags, m);
408
409 if (tags->nr_reserved_tags) {
410 seq_puts(m, "\nbreserved_tags:\n");
411 sbitmap_queue_show(&tags->breserved_tags, m);
412 }
413}
414
415static int hctx_tags_show(void *data, struct seq_file *m)
416{
417 struct blk_mq_hw_ctx *hctx = data;
418 struct request_queue *q = hctx->queue;
419 int res;
420
421 res = mutex_lock_interruptible(&q->elevator_lock);
422 if (res)
423 return res;
424 if (hctx->tags)
425 blk_mq_debugfs_tags_show(m, hctx->tags);
426 mutex_unlock(&q->elevator_lock);
427
428 return 0;
429}
430
431static int hctx_tags_bitmap_show(void *data, struct seq_file *m)
432{
433 struct blk_mq_hw_ctx *hctx = data;
434 struct request_queue *q = hctx->queue;
435 int res;
436
437 res = mutex_lock_interruptible(&q->elevator_lock);
438 if (res)
439 return res;
440 if (hctx->tags)
441 sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
442 mutex_unlock(&q->elevator_lock);
443
444 return 0;
445}
446
447static int hctx_sched_tags_show(void *data, struct seq_file *m)
448{
449 struct blk_mq_hw_ctx *hctx = data;
450 struct request_queue *q = hctx->queue;
451 int res;
452
453 res = mutex_lock_interruptible(&q->elevator_lock);
454 if (res)
455 return res;
456 if (hctx->sched_tags)
457 blk_mq_debugfs_tags_show(m, hctx->sched_tags);
458 mutex_unlock(&q->elevator_lock);
459
460 return 0;
461}
462
463static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m)
464{
465 struct blk_mq_hw_ctx *hctx = data;
466 struct request_queue *q = hctx->queue;
467 int res;
468
469 res = mutex_lock_interruptible(&q->elevator_lock);
470 if (res)
471 return res;
472 if (hctx->sched_tags)
473 sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
474 mutex_unlock(&q->elevator_lock);
475
476 return 0;
477}
478
479static int hctx_active_show(void *data, struct seq_file *m)
480{
481 struct blk_mq_hw_ctx *hctx = data;
482
483 seq_printf(m, "%d\n", __blk_mq_active_requests(hctx));
484 return 0;
485}
486
487static int hctx_dispatch_busy_show(void *data, struct seq_file *m)
488{
489 struct blk_mq_hw_ctx *hctx = data;
490
491 seq_printf(m, "%u\n", hctx->dispatch_busy);
492 return 0;
493}
494
495#define TO_CTX(m) ((struct blk_mq_ctx *)m->private)
496
497#define CTX_RQ_SEQ_OPS(name, type) \
498static void *ctx_##name##_rq_list_start(struct seq_file *m, loff_t *pos) \
499 __acquires(&TO_CTX(m)->lock) \
500{ \
501 struct blk_mq_ctx *ctx = m->private; \
502 \
503 spin_lock(&ctx->lock); \
504 return seq_list_start(&ctx->rq_lists[type], *pos); \
505} \
506 \
507static void *ctx_##name##_rq_list_next(struct seq_file *m, void *v, \
508 loff_t *pos) \
509{ \
510 struct blk_mq_ctx *ctx = m->private; \
511 \
512 return seq_list_next(v, &ctx->rq_lists[type], pos); \
513} \
514 \
515static void ctx_##name##_rq_list_stop(struct seq_file *m, void *v) \
516 __releases(&TO_CTX(m)->lock) \
517{ \
518 struct blk_mq_ctx *ctx = m->private; \
519 \
520 spin_unlock(&ctx->lock); \
521} \
522 \
523static const struct seq_operations ctx_##name##_rq_list_seq_ops = { \
524 .start = ctx_##name##_rq_list_start, \
525 .next = ctx_##name##_rq_list_next, \
526 .stop = ctx_##name##_rq_list_stop, \
527 .show = blk_mq_debugfs_rq_show, \
528}
529
530CTX_RQ_SEQ_OPS(default, HCTX_TYPE_DEFAULT);
531CTX_RQ_SEQ_OPS(read, HCTX_TYPE_READ);
532CTX_RQ_SEQ_OPS(poll, HCTX_TYPE_POLL);
533
534#undef TO_CTX
535
536static int blk_mq_debugfs_show(struct seq_file *m, void *v)
537{
538 const struct blk_mq_debugfs_attr *attr = m->private;
539 void *data = debugfs_get_aux(m->file);
540
541 return attr->show(data, m);
542}
543
544static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf,
545 size_t count, loff_t *ppos)
546{
547 struct seq_file *m = file->private_data;
548 const struct blk_mq_debugfs_attr *attr = m->private;
549 void *data = debugfs_get_aux(file);
550
551 /*
552 * Attributes that only implement .seq_ops are read-only and 'attr' is
553 * the same with 'data' in this case.
554 */
555 if (attr == data || !attr->write)
556 return -EPERM;
557
558 return attr->write(data, buf, count, ppos);
559}
560
561static int blk_mq_debugfs_open(struct inode *inode, struct file *file)
562{
563 const struct blk_mq_debugfs_attr *attr = inode->i_private;
564 void *data = debugfs_get_aux(file);
565 struct seq_file *m;
566 int ret;
567
568 if (attr->seq_ops) {
569 ret = seq_open(file, attr->seq_ops);
570 if (!ret) {
571 m = file->private_data;
572 m->private = data;
573 }
574 return ret;
575 }
576
577 if (WARN_ON_ONCE(!attr->show))
578 return -EPERM;
579
580 return single_open(file, blk_mq_debugfs_show, inode->i_private);
581}
582
583static int blk_mq_debugfs_release(struct inode *inode, struct file *file)
584{
585 const struct blk_mq_debugfs_attr *attr = inode->i_private;
586
587 if (attr->show)
588 return single_release(inode, file);
589
590 return seq_release(inode, file);
591}
592
593static const struct file_operations blk_mq_debugfs_fops = {
594 .open = blk_mq_debugfs_open,
595 .read = seq_read,
596 .write = blk_mq_debugfs_write,
597 .llseek = seq_lseek,
598 .release = blk_mq_debugfs_release,
599};
600
601static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
602 {"state", 0400, hctx_state_show},
603 {"flags", 0400, hctx_flags_show},
604 {"dispatch", 0400, .seq_ops = &hctx_dispatch_seq_ops},
605 {"busy", 0400, hctx_busy_show},
606 {"ctx_map", 0400, hctx_ctx_map_show},
607 {"tags", 0400, hctx_tags_show},
608 {"tags_bitmap", 0400, hctx_tags_bitmap_show},
609 {"sched_tags", 0400, hctx_sched_tags_show},
610 {"sched_tags_bitmap", 0400, hctx_sched_tags_bitmap_show},
611 {"active", 0400, hctx_active_show},
612 {"dispatch_busy", 0400, hctx_dispatch_busy_show},
613 {"type", 0400, hctx_type_show},
614 {},
615};
616
617static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = {
618 {"default_rq_list", 0400, .seq_ops = &ctx_default_rq_list_seq_ops},
619 {"read_rq_list", 0400, .seq_ops = &ctx_read_rq_list_seq_ops},
620 {"poll_rq_list", 0400, .seq_ops = &ctx_poll_rq_list_seq_ops},
621 {},
622};
623
624static void debugfs_create_files(struct request_queue *q, struct dentry *parent,
625 void *data,
626 const struct blk_mq_debugfs_attr *attr)
627{
628 lockdep_assert_held(&q->debugfs_mutex);
629 /*
630 * debugfs_mutex should not be nested under other locks that can be
631 * grabbed while queue is frozen.
632 */
633 lockdep_assert_not_held(&q->elevator_lock);
634 lockdep_assert_not_held(&q->rq_qos_mutex);
635
636 if (IS_ERR_OR_NULL(parent))
637 return;
638
639 for (; attr->name; attr++)
640 debugfs_create_file_aux(attr->name, attr->mode, parent,
641 (void *)attr, data, &blk_mq_debugfs_fops);
642}
643
644void blk_mq_debugfs_register(struct request_queue *q)
645{
646 struct blk_mq_hw_ctx *hctx;
647 unsigned long i;
648
649 debugfs_create_files(q, q->debugfs_dir, q, blk_mq_debugfs_queue_attrs);
650
651 queue_for_each_hw_ctx(q, hctx, i) {
652 if (!hctx->debugfs_dir)
653 blk_mq_debugfs_register_hctx(q, hctx);
654 }
655
656 blk_mq_debugfs_register_rq_qos(q);
657}
658
659static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx,
660 struct blk_mq_ctx *ctx)
661{
662 struct dentry *ctx_dir;
663 char name[20];
664
665 snprintf(name, sizeof(name), "cpu%u", ctx->cpu);
666 ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir);
667
668 debugfs_create_files(hctx->queue, ctx_dir, ctx,
669 blk_mq_debugfs_ctx_attrs);
670}
671
672void blk_mq_debugfs_register_hctx(struct request_queue *q,
673 struct blk_mq_hw_ctx *hctx)
674{
675 struct blk_mq_ctx *ctx;
676 char name[20];
677 int i;
678
679 if (!q->debugfs_dir)
680 return;
681
682 snprintf(name, sizeof(name), "hctx%u", hctx->queue_num);
683 hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir);
684
685 debugfs_create_files(q, hctx->debugfs_dir, hctx,
686 blk_mq_debugfs_hctx_attrs);
687
688 hctx_for_each_ctx(hctx, ctx, i)
689 blk_mq_debugfs_register_ctx(hctx, ctx);
690}
691
692void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx)
693{
694 if (!hctx->queue->debugfs_dir)
695 return;
696 debugfs_remove_recursive(hctx->debugfs_dir);
697 hctx->sched_debugfs_dir = NULL;
698 hctx->debugfs_dir = NULL;
699}
700
701void blk_mq_debugfs_register_hctxs(struct request_queue *q)
702{
703 struct blk_mq_hw_ctx *hctx;
704 unsigned int memflags;
705 unsigned long i;
706
707 memflags = blk_debugfs_lock(q);
708 queue_for_each_hw_ctx(q, hctx, i)
709 blk_mq_debugfs_register_hctx(q, hctx);
710 blk_debugfs_unlock(q, memflags);
711}
712
713void blk_mq_debugfs_unregister_hctxs(struct request_queue *q)
714{
715 struct blk_mq_hw_ctx *hctx;
716 unsigned long i;
717
718 queue_for_each_hw_ctx(q, hctx, i)
719 blk_mq_debugfs_unregister_hctx(hctx);
720}
721
722void blk_mq_debugfs_register_sched(struct request_queue *q)
723{
724 struct elevator_type *e = q->elevator->type;
725
726 lockdep_assert_held(&q->debugfs_mutex);
727
728 /*
729 * If the parent directory has not been created yet, return, we will be
730 * called again later on and the directory/files will be created then.
731 */
732 if (!q->debugfs_dir)
733 return;
734
735 if (!e->queue_debugfs_attrs)
736 return;
737
738 q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir);
739
740 debugfs_create_files(q, q->sched_debugfs_dir, q, e->queue_debugfs_attrs);
741}
742
743void blk_mq_debugfs_unregister_sched(struct request_queue *q)
744{
745 lockdep_assert_held(&q->debugfs_mutex);
746
747 debugfs_remove_recursive(q->sched_debugfs_dir);
748 q->sched_debugfs_dir = NULL;
749}
750
751static const char *rq_qos_id_to_name(enum rq_qos_id id)
752{
753 switch (id) {
754 case RQ_QOS_WBT:
755 return "wbt";
756 case RQ_QOS_LATENCY:
757 return "latency";
758 case RQ_QOS_COST:
759 return "cost";
760 }
761 return "unknown";
762}
763
764static void blk_mq_debugfs_register_rqos(struct rq_qos *rqos)
765{
766 struct request_queue *q = rqos->disk->queue;
767 const char *dir_name = rq_qos_id_to_name(rqos->id);
768
769 lockdep_assert_held(&q->debugfs_mutex);
770
771 if (rqos->debugfs_dir || !rqos->ops->debugfs_attrs)
772 return;
773
774 if (!q->rqos_debugfs_dir)
775 q->rqos_debugfs_dir = debugfs_create_dir("rqos",
776 q->debugfs_dir);
777
778 rqos->debugfs_dir = debugfs_create_dir(dir_name, q->rqos_debugfs_dir);
779 debugfs_create_files(q, rqos->debugfs_dir, rqos,
780 rqos->ops->debugfs_attrs);
781}
782
783void blk_mq_debugfs_register_rq_qos(struct request_queue *q)
784{
785 lockdep_assert_held(&q->debugfs_mutex);
786
787 if (q->rq_qos) {
788 struct rq_qos *rqos = q->rq_qos;
789
790 while (rqos) {
791 blk_mq_debugfs_register_rqos(rqos);
792 rqos = rqos->next;
793 }
794 }
795}
796
797void blk_mq_debugfs_register_sched_hctx(struct request_queue *q,
798 struct blk_mq_hw_ctx *hctx)
799{
800 struct elevator_type *e = q->elevator->type;
801
802 lockdep_assert_held(&q->debugfs_mutex);
803
804 /*
805 * If the parent debugfs directory has not been created yet, return;
806 * We will be called again later on with appropriate parent debugfs
807 * directory from blk_register_queue()
808 */
809 if (!hctx->debugfs_dir)
810 return;
811
812 if (!e->hctx_debugfs_attrs)
813 return;
814
815 hctx->sched_debugfs_dir = debugfs_create_dir("sched",
816 hctx->debugfs_dir);
817 debugfs_create_files(q, hctx->sched_debugfs_dir, hctx,
818 e->hctx_debugfs_attrs);
819}
820
821void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx)
822{
823 lockdep_assert_held(&hctx->queue->debugfs_mutex);
824
825 if (!hctx->queue->debugfs_dir)
826 return;
827 debugfs_remove_recursive(hctx->sched_debugfs_dir);
828 hctx->sched_debugfs_dir = NULL;
829}