projects/linux·blob master · refs

2263 lines · 59.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common Block IO controller cgroup interface
4 *
5 * Based on ideas and code from CFQ, CFS and BFQ:
6 * Copyright (C) 2003 Jens Axboe <[email protected]>
7 *
8 * Copyright (C) 2008 Fabio Checconi <[email protected]>
9 * Paolo Valente <[email protected]>
10 *
11 * Copyright (C) 2009 Vivek Goyal <[email protected]>
12 * Nauman Rafique <[email protected]>
13 *
14 * For policy-specific per-blkcg data:
15 * Copyright (C) 2015 Paolo Valente <[email protected]>
16 * Arianna Avanzini <[email protected]>
17 */
18#include <linux/ioprio.h>
19#include <linux/kdev_t.h>
20#include <linux/module.h>
21#include <linux/sched/signal.h>
22#include <linux/err.h>
23#include <linux/blkdev.h>
24#include <linux/backing-dev.h>
25#include <linux/slab.h>
26#include <linux/delay.h>
27#include <linux/wait_bit.h>
28#include <linux/atomic.h>
29#include <linux/ctype.h>
30#include <linux/resume_user_mode.h>
31#include <linux/psi.h>
32#include <linux/part_stat.h>
33#include "blk.h"
34#include "blk-cgroup.h"
35#include "blk-ioprio.h"
36#include "blk-throttle.h"
37
38static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
39
40/*
41 * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
42 * blkcg_pol_register_mutex nests outside of it and synchronizes entire
43 * policy [un]register operations including cgroup file additions /
44 * removals. Putting cgroup file registration outside blkcg_pol_mutex
45 * allows grabbing it from cgroup callbacks.
46 */
47static DEFINE_MUTEX(blkcg_pol_register_mutex);
48static DEFINE_MUTEX(blkcg_pol_mutex);
49
50struct blkcg blkcg_root;
51EXPORT_SYMBOL_GPL(blkcg_root);
52
53struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
54EXPORT_SYMBOL_GPL(blkcg_root_css);
55
56/* number of blkcgs with a non-zero congestion_count */
57atomic_t blkcg_nr_congested __read_mostly = ATOMIC_INIT(0);
58
59static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
60
61static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
62
63bool blkcg_debug_stats = false;
64
65static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
66
67#define BLKG_DESTROY_BATCH_SIZE 64
68
69/*
70 * Lockless lists for tracking IO stats update
71 *
72 * New IO stats are stored in the percpu iostat_cpu within blkcg_gq (blkg).
73 * There are multiple blkg's (one for each block device) attached to each
74 * blkcg. The rstat code keeps track of which cpu has IO stats updated,
75 * but it doesn't know which blkg has the updated stats. If there are many
76 * block devices in a system, the cost of iterating all the blkg's to flush
77 * out the IO stats can be high. To reduce such overhead, a set of percpu
78 * lockless lists (lhead) per blkcg are used to track the set of recently
79 * updated iostat_cpu's since the last flush. An iostat_cpu will be put
80 * onto the lockless list on the update side [blk_cgroup_bio_start()] if
81 * not there yet and then removed when being flushed [blkcg_rstat_flush()].
82 * References to blkg are gotten and then put back in the process to
83 * protect against blkg removal.
84 *
85 * Return: 0 if successful or -ENOMEM if allocation fails.
86 */
87static int init_blkcg_llists(struct blkcg *blkcg)
88{
89 int cpu;
90
91 blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);
92 if (!blkcg->lhead)
93 return -ENOMEM;
94
95 for_each_possible_cpu(cpu)
96 init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));
97 return 0;
98}
99
100/**
101 * blkcg_css - find the current css
102 *
103 * Find the css associated with either the kthread or the current task.
104 * This may return a dying css, so it is up to the caller to use tryget logic
105 * to confirm it is alive and well.
106 */
107static struct cgroup_subsys_state *blkcg_css(void)
108{
109 struct cgroup_subsys_state *css;
110
111 css = kthread_blkcg();
112 if (css)
113 return css;
114 return task_css(current, io_cgrp_id);
115}
116
117static void blkg_free_workfn(struct work_struct *work)
118{
119 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
120 free_work);
121 struct request_queue *q = blkg->q;
122 int i;
123
124 /*
125 * pd_free_fn() can also be called from blkcg_deactivate_policy(),
126 * in order to make sure pd_free_fn() is called in order, the deletion
127 * of the list blkg->q_node is delayed to here from blkg_destroy(), and
128 * blkcg_mutex is used to synchronize blkg_free_workfn() and
129 * blkcg_deactivate_policy().
130 */
131 mutex_lock(&q->blkcg_mutex);
132 for (i = 0; i < BLKCG_MAX_POLS; i++)
133 if (blkg->pd[i])
134 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
135 if (blkg->parent)
136 blkg_put(blkg->parent);
137 spin_lock_irq(&q->queue_lock);
138 list_del_init(&blkg->q_node);
139 spin_unlock_irq(&q->queue_lock);
140 mutex_unlock(&q->blkcg_mutex);
141
142 /*
143 * Release blkcg css ref only after blkg is removed from q->blkg_list,
144 * so concurrent iterators won't see a blkg with a freed blkcg.
145 */
146 css_put(&blkg->blkcg->css);
147 blk_put_queue(q);
148 free_percpu(blkg->iostat_cpu);
149 percpu_ref_exit(&blkg->refcnt);
150 kfree(blkg);
151}
152
153/**
154 * blkg_free - free a blkg
155 * @blkg: blkg to free
156 *
157 * Free @blkg which may be partially allocated.
158 */
159static void blkg_free(struct blkcg_gq *blkg)
160{
161 if (!blkg)
162 return;
163
164 /*
165 * Both ->pd_free_fn() and request queue's release handler may
166 * sleep, so free us by scheduling one work func
167 */
168 INIT_WORK(&blkg->free_work, blkg_free_workfn);
169 schedule_work(&blkg->free_work);
170}
171
172static void __blkg_release(struct rcu_head *rcu)
173{
174 struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
175
176#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
177 WARN_ON(!bio_list_empty(&blkg->async_bios));
178#endif
179
180 blkg_free(blkg);
181}
182
183/*
184 * A group is RCU protected, but having an rcu lock does not mean that one
185 * can access all the fields of blkg and assume these are valid. For
186 * example, don't try to follow throtl_data and request queue links.
187 *
188 * Having a reference to blkg under an rcu allows accesses to only values
189 * local to groups like group stats and group rate limits.
190 */
191static void blkg_release(struct percpu_ref *ref)
192{
193 struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
194 struct blkcg *blkcg = blkg->blkcg;
195 int cpu;
196
197 /*
198 * Flush all the non-empty percpu lockless lists before releasing
199 * us, given these stat belongs to us.
200 *
201 * blkg_stat_lock is for serializing blkg stat update
202 */
203 for_each_possible_cpu(cpu)
204 __blkcg_rstat_flush(blkcg, cpu);
205
206 call_rcu(&blkg->rcu_head, __blkg_release);
207}
208
209#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
210static struct workqueue_struct *blkcg_punt_bio_wq;
211
212static void blkg_async_bio_workfn(struct work_struct *work)
213{
214 struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
215 async_bio_work);
216 struct bio_list bios = BIO_EMPTY_LIST;
217 struct bio *bio;
218 struct blk_plug plug;
219 bool need_plug = false;
220
221 /* as long as there are pending bios, @blkg can't go away */
222 spin_lock(&blkg->async_bio_lock);
223 bio_list_merge_init(&bios, &blkg->async_bios);
224 spin_unlock(&blkg->async_bio_lock);
225
226 /* start plug only when bio_list contains at least 2 bios */
227 if (bios.head && bios.head->bi_next) {
228 need_plug = true;
229 blk_start_plug(&plug);
230 }
231 while ((bio = bio_list_pop(&bios)))
232 submit_bio(bio);
233 if (need_plug)
234 blk_finish_plug(&plug);
235}
236
237/*
238 * When a shared kthread issues a bio for a cgroup, doing so synchronously can
239 * lead to priority inversions as the kthread can be trapped waiting for that
240 * cgroup. Use this helper instead of submit_bio to punt the actual issuing to
241 * a dedicated per-blkcg work item to avoid such priority inversions.
242 */
243void blkcg_punt_bio_submit(struct bio *bio)
244{
245 struct blkcg_gq *blkg = bio->bi_blkg;
246
247 if (blkg->parent) {
248 spin_lock(&blkg->async_bio_lock);
249 bio_list_add(&blkg->async_bios, bio);
250 spin_unlock(&blkg->async_bio_lock);
251 queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
252 } else {
253 /* never bounce for the root cgroup */
254 submit_bio(bio);
255 }
256}
257EXPORT_SYMBOL_GPL(blkcg_punt_bio_submit);
258
259static int __init blkcg_punt_bio_init(void)
260{
261 blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
262 WQ_MEM_RECLAIM | WQ_FREEZABLE |
263 WQ_UNBOUND | WQ_SYSFS, 0);
264 if (!blkcg_punt_bio_wq)
265 return -ENOMEM;
266 return 0;
267}
268subsys_initcall(blkcg_punt_bio_init);
269#endif /* CONFIG_BLK_CGROUP_PUNT_BIO */
270
271/**
272 * bio_blkcg_css - return the blkcg CSS associated with a bio
273 * @bio: target bio
274 *
275 * This returns the CSS for the blkcg associated with a bio, or %NULL if not
276 * associated. Callers are expected to either handle %NULL or know association
277 * has been done prior to calling this.
278 */
279struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio)
280{
281 if (!bio || !bio->bi_blkg)
282 return NULL;
283 return &bio->bi_blkg->blkcg->css;
284}
285EXPORT_SYMBOL_GPL(bio_blkcg_css);
286
287/**
288 * blkcg_parent - get the parent of a blkcg
289 * @blkcg: blkcg of interest
290 *
291 * Return the parent blkcg of @blkcg. Can be called anytime.
292 */
293static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
294{
295 return css_to_blkcg(blkcg->css.parent);
296}
297
298/**
299 * blkg_alloc - allocate a blkg
300 * @blkcg: block cgroup the new blkg is associated with
301 * @disk: gendisk the new blkg is associated with
302 * @gfp_mask: allocation mask to use
303 *
304 * Allocate a new blkg associating @blkcg and @disk.
305 */
306static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
307 gfp_t gfp_mask)
308{
309 struct blkcg_gq *blkg;
310 int i, cpu;
311
312 /* alloc and init base part */
313 blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node);
314 if (!blkg)
315 return NULL;
316 if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
317 goto out_free_blkg;
318 blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
319 if (!blkg->iostat_cpu)
320 goto out_exit_refcnt;
321 if (!blk_get_queue(disk->queue))
322 goto out_free_iostat;
323 /* blkg holds a reference to blkcg */
324 if (!css_tryget_online(&blkcg->css))
325 goto out_put_queue;
326
327 blkg->q = disk->queue;
328 INIT_LIST_HEAD(&blkg->q_node);
329 blkg->blkcg = blkcg;
330 blkg->iostat.blkg = blkg;
331#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
332 spin_lock_init(&blkg->async_bio_lock);
333 bio_list_init(&blkg->async_bios);
334 INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
335#endif
336
337 u64_stats_init(&blkg->iostat.sync);
338 for_each_possible_cpu(cpu) {
339 u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
340 per_cpu_ptr(blkg->iostat_cpu, cpu)->blkg = blkg;
341 }
342
343 for (i = 0; i < BLKCG_MAX_POLS; i++) {
344 struct blkcg_policy *pol = blkcg_policy[i];
345 struct blkg_policy_data *pd;
346
347 if (!blkcg_policy_enabled(disk->queue, pol))
348 continue;
349
350 /* alloc per-policy data and attach it to blkg */
351 pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);
352 if (!pd)
353 goto out_free_pds;
354 blkg->pd[i] = pd;
355 pd->blkg = blkg;
356 pd->plid = i;
357 pd->online = false;
358 }
359
360 return blkg;
361
362out_free_pds:
363 while (--i >= 0)
364 if (blkg->pd[i])
365 blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
366 css_put(&blkcg->css);
367out_put_queue:
368 blk_put_queue(disk->queue);
369out_free_iostat:
370 free_percpu(blkg->iostat_cpu);
371out_exit_refcnt:
372 percpu_ref_exit(&blkg->refcnt);
373out_free_blkg:
374 kfree(blkg);
375 return NULL;
376}
377
378/*
379 * If @new_blkg is %NULL, this function tries to allocate a new one as
380 * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
381 */
382static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
383 struct blkcg_gq *new_blkg)
384{
385 struct blkcg_gq *blkg;
386 int i, ret;
387
388 lockdep_assert_held(&disk->queue->queue_lock);
389
390 /* request_queue is dying, do not create/recreate a blkg */
391 if (blk_queue_dying(disk->queue)) {
392 ret = -ENODEV;
393 goto err_free_blkg;
394 }
395
396 /* allocate */
397 if (!new_blkg) {
398 new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT);
399 if (unlikely(!new_blkg)) {
400 ret = -ENOMEM;
401 goto err_free_blkg;
402 }
403 }
404 blkg = new_blkg;
405
406 /* link parent */
407 if (blkcg_parent(blkcg)) {
408 blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);
409 if (WARN_ON_ONCE(!blkg->parent)) {
410 ret = -ENODEV;
411 goto err_free_blkg;
412 }
413 blkg_get(blkg->parent);
414 }
415
416 /* invoke per-policy init */
417 for (i = 0; i < BLKCG_MAX_POLS; i++) {
418 struct blkcg_policy *pol = blkcg_policy[i];
419
420 if (blkg->pd[i] && pol->pd_init_fn)
421 pol->pd_init_fn(blkg->pd[i]);
422 }
423
424 /* insert */
425 spin_lock(&blkcg->lock);
426 ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);
427 if (likely(!ret)) {
428 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
429 list_add(&blkg->q_node, &disk->queue->blkg_list);
430
431 for (i = 0; i < BLKCG_MAX_POLS; i++) {
432 struct blkcg_policy *pol = blkcg_policy[i];
433
434 if (blkg->pd[i]) {
435 if (pol->pd_online_fn)
436 pol->pd_online_fn(blkg->pd[i]);
437 blkg->pd[i]->online = true;
438 }
439 }
440 blkg->online = true;
441 }
442 spin_unlock(&blkcg->lock);
443
444 if (!ret)
445 return blkg;
446
447 /* @blkg failed fully initialized, use the usual release path */
448 percpu_ref_kill(&blkg->refcnt);
449 return ERR_PTR(ret);
450
451err_free_blkg:
452 if (new_blkg)
453 blkg_free(new_blkg);
454 return ERR_PTR(ret);
455}
456
457/**
458 * blkg_lookup_create - lookup blkg, try to create one if not there
459 * @blkcg: blkcg of interest
460 * @disk: gendisk of interest
461 *
462 * Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to
463 * create one. blkg creation is performed recursively from blkcg_root such
464 * that all non-root blkg's have access to the parent blkg. This function
465 * should be called under RCU read lock and takes @disk->queue->queue_lock.
466 *
467 * Returns the blkg or the closest blkg if blkg_create() fails as it walks
468 * down from root.
469 */
470static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
471 struct gendisk *disk)
472{
473 struct request_queue *q = disk->queue;
474 struct blkcg_gq *blkg;
475
476 rcu_read_lock();
477 blkg = blkg_lookup(blkcg, q);
478 if (blkg) {
479 if (blkcg != &blkcg_root &&
480 blkg != rcu_dereference(blkcg->blkg_hint))
481 rcu_assign_pointer(blkcg->blkg_hint, blkg);
482 rcu_read_unlock();
483 return blkg;
484 }
485 rcu_read_unlock();
486
487 /*
488 * Create blkgs walking down from blkcg_root to @blkcg, so that all
489 * non-root blkgs have access to their parents. Returns the closest
490 * blkg to the intended blkg should blkg_create() fail.
491 */
492 while (true) {
493 struct blkcg *pos = blkcg;
494 struct blkcg *parent = blkcg_parent(blkcg);
495 struct blkcg_gq *ret_blkg = q->root_blkg;
496
497 while (parent) {
498 blkg = blkg_lookup(parent, q);
499 if (blkg) {
500 /* remember closest blkg */
501 ret_blkg = blkg;
502 break;
503 }
504 pos = parent;
505 parent = blkcg_parent(parent);
506 }
507
508 blkg = blkg_create(pos, disk, NULL);
509 if (IS_ERR(blkg)) {
510 blkg = ret_blkg;
511 break;
512 }
513 if (pos == blkcg)
514 break;
515 }
516
517 return blkg;
518}
519
520static void blkg_destroy(struct blkcg_gq *blkg)
521{
522 struct blkcg *blkcg = blkg->blkcg;
523 int i;
524
525 lockdep_assert_held(&blkg->q->queue_lock);
526 lockdep_assert_held(&blkcg->lock);
527
528 /*
529 * blkg stays on the queue list until blkg_free_workfn(), see details in
530 * blkg_free_workfn(), hence this function can be called from
531 * blkcg_destroy_blkgs() first and again from blkg_destroy_all() before
532 * blkg_free_workfn().
533 */
534 if (hlist_unhashed(&blkg->blkcg_node))
535 return;
536
537 for (i = 0; i < BLKCG_MAX_POLS; i++) {
538 struct blkcg_policy *pol = blkcg_policy[i];
539
540 if (blkg->pd[i] && blkg->pd[i]->online) {
541 blkg->pd[i]->online = false;
542 if (pol->pd_offline_fn)
543 pol->pd_offline_fn(blkg->pd[i]);
544 }
545 }
546
547 blkg->online = false;
548
549 radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
550 hlist_del_init_rcu(&blkg->blkcg_node);
551
552 /*
553 * Both setting lookup hint to and clearing it from @blkg are done
554 * under queue_lock. If it's not pointing to @blkg now, it never
555 * will. Hint assignment itself can race safely.
556 */
557 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
558 rcu_assign_pointer(blkcg->blkg_hint, NULL);
559
560 /*
561 * Put the reference taken at the time of creation so that when all
562 * queues are gone, group can be destroyed.
563 */
564 percpu_ref_kill(&blkg->refcnt);
565}
566
567static void blkg_destroy_all(struct gendisk *disk)
568{
569 struct request_queue *q = disk->queue;
570 struct blkcg_gq *blkg;
571 int count = BLKG_DESTROY_BATCH_SIZE;
572 int i;
573
574restart:
575 mutex_lock(&q->blkcg_mutex);
576 spin_lock_irq(&q->queue_lock);
577 list_for_each_entry(blkg, &q->blkg_list, q_node) {
578 struct blkcg *blkcg = blkg->blkcg;
579
580 if (hlist_unhashed(&blkg->blkcg_node))
581 continue;
582
583 spin_lock(&blkcg->lock);
584 blkg_destroy(blkg);
585 spin_unlock(&blkcg->lock);
586
587 /*
588 * in order to avoid holding the spin lock for too long, release
589 * it when a batch of blkgs are destroyed.
590 */
591 if (!(--count)) {
592 count = BLKG_DESTROY_BATCH_SIZE;
593 spin_unlock_irq(&q->queue_lock);
594 mutex_unlock(&q->blkcg_mutex);
595 cond_resched();
596 goto restart;
597 }
598 }
599
600 /*
601 * Mark policy deactivated since policy offline has been done, and
602 * the free is scheduled, so future blkcg_deactivate_policy() can
603 * be bypassed
604 */
605 for (i = 0; i < BLKCG_MAX_POLS; i++) {
606 struct blkcg_policy *pol = blkcg_policy[i];
607
608 if (pol)
609 __clear_bit(pol->plid, q->blkcg_pols);
610 }
611
612 q->root_blkg = NULL;
613 spin_unlock_irq(&q->queue_lock);
614 mutex_unlock(&q->blkcg_mutex);
615
616 wake_up_var(&q->root_blkg);
617}
618
619static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
620{
621 int i;
622
623 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
624 dst->bytes[i] = src->bytes[i];
625 dst->ios[i] = src->ios[i];
626 }
627}
628
629static void __blkg_clear_stat(struct blkg_iostat_set *bis)
630{
631 struct blkg_iostat cur = {0};
632 unsigned long flags;
633
634 flags = u64_stats_update_begin_irqsave(&bis->sync);
635 blkg_iostat_set(&bis->cur, &cur);
636 blkg_iostat_set(&bis->last, &cur);
637 u64_stats_update_end_irqrestore(&bis->sync, flags);
638}
639
640static void blkg_clear_stat(struct blkcg_gq *blkg)
641{
642 int cpu;
643
644 for_each_possible_cpu(cpu) {
645 struct blkg_iostat_set *s = per_cpu_ptr(blkg->iostat_cpu, cpu);
646
647 __blkg_clear_stat(s);
648 }
649 __blkg_clear_stat(&blkg->iostat);
650}
651
652static int blkcg_reset_stats(struct cgroup_subsys_state *css,
653 struct cftype *cftype, u64 val)
654{
655 struct blkcg *blkcg = css_to_blkcg(css);
656 struct blkcg_gq *blkg;
657 int i;
658
659 pr_info_once("blkio.%s is deprecated\n", cftype->name);
660 mutex_lock(&blkcg_pol_mutex);
661 spin_lock_irq(&blkcg->lock);
662
663 /*
664 * Note that stat reset is racy - it doesn't synchronize against
665 * stat updates. This is a debug feature which shouldn't exist
666 * anyway. If you get hit by a race, retry.
667 */
668 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
669 blkg_clear_stat(blkg);
670 for (i = 0; i < BLKCG_MAX_POLS; i++) {
671 struct blkcg_policy *pol = blkcg_policy[i];
672
673 if (blkg->pd[i] && pol->pd_reset_stats_fn)
674 pol->pd_reset_stats_fn(blkg->pd[i]);
675 }
676 }
677
678 spin_unlock_irq(&blkcg->lock);
679 mutex_unlock(&blkcg_pol_mutex);
680 return 0;
681}
682
683const char *blkg_dev_name(struct blkcg_gq *blkg)
684{
685 if (!blkg->q->disk)
686 return NULL;
687 return bdi_dev_name(blkg->q->disk->bdi);
688}
689
690/**
691 * blkcg_print_blkgs - helper for printing per-blkg data
692 * @sf: seq_file to print to
693 * @blkcg: blkcg of interest
694 * @prfill: fill function to print out a blkg
695 * @pol: policy in question
696 * @data: data to be passed to @prfill
697 * @show_total: to print out sum of prfill return values or not
698 *
699 * This function invokes @prfill on each blkg of @blkcg if pd for the
700 * policy specified by @pol exists. @prfill is invoked with @sf, the
701 * policy data and @data under RCU read lock. If @show_total is %true, the
702 * sum of the return values from @prfill is printed with "Total" label at the
703 * end.
704 *
705 * This is to be used to construct print functions for
706 * cftype->read_seq_string method.
707 */
708void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
709 u64 (*prfill)(struct seq_file *,
710 struct blkg_policy_data *, int),
711 const struct blkcg_policy *pol, int data,
712 bool show_total)
713{
714 struct blkcg_gq *blkg;
715 u64 total = 0;
716
717 rcu_read_lock();
718 hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
719 struct blkg_policy_data *pd;
720
721 if (!blkcg_policy_enabled(blkg->q, pol))
722 continue;
723
724 pd = blkg_to_pd(blkg, pol);
725 if (pd)
726 total += prfill(sf, pd, data);
727 }
728 rcu_read_unlock();
729
730 if (show_total)
731 seq_printf(sf, "Total %llu\n", (unsigned long long)total);
732}
733EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
734
735/**
736 * __blkg_prfill_u64 - prfill helper for a single u64 value
737 * @sf: seq_file to print to
738 * @pd: policy private data of interest
739 * @v: value to print
740 *
741 * Print @v to @sf for the device associated with @pd.
742 */
743u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
744{
745 const char *dname = blkg_dev_name(pd->blkg);
746
747 if (!dname)
748 return 0;
749
750 seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
751 return v;
752}
753EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
754
755/**
756 * blkg_conf_init - initialize a blkg_conf_ctx
757 * @ctx: blkg_conf_ctx to initialize
758 * @input: input string
759 *
760 * Initialize @ctx which can be used to parse blkg config input string @input.
761 * Once initialized, @ctx can be used with blkg_conf_open_bdev() and
762 * blkg_conf_prep().
763 */
764void blkg_conf_init(struct blkg_conf_ctx *ctx, char *input)
765{
766 *ctx = (struct blkg_conf_ctx){ .input = input };
767}
768EXPORT_SYMBOL_GPL(blkg_conf_init);
769
770/**
771 * blkg_conf_open_bdev - parse and open bdev for per-blkg config update
772 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
773 *
774 * Parse the device node prefix part, MAJ:MIN, of per-blkg config update from
775 * @ctx->input and get and store the matching bdev in @ctx->bdev. @ctx->body is
776 * set to point past the device node prefix.
777 *
778 * Returns: -errno on error.
779 */
780int blkg_conf_open_bdev(struct blkg_conf_ctx *ctx)
781{
782 char *input = ctx->input;
783 unsigned int major, minor;
784 struct block_device *bdev;
785 int key_len;
786
787 if (WARN_ON_ONCE(ctx->bdev))
788 return -EINVAL;
789
790 if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
791 return -EINVAL;
792
793 input += key_len;
794 if (!isspace(*input))
795 return -EINVAL;
796 input = skip_spaces(input);
797
798 bdev = blkdev_get_no_open(MKDEV(major, minor), false);
799 if (!bdev)
800 return -ENODEV;
801 if (bdev_is_partition(bdev)) {
802 blkdev_put_no_open(bdev);
803 return -ENODEV;
804 }
805
806 mutex_lock(&bdev->bd_queue->rq_qos_mutex);
807 if (!disk_live(bdev->bd_disk)) {
808 blkdev_put_no_open(bdev);
809 mutex_unlock(&bdev->bd_queue->rq_qos_mutex);
810 return -ENODEV;
811 }
812
813 ctx->body = input;
814 ctx->bdev = bdev;
815 return 0;
816}
817EXPORT_SYMBOL_GPL(blkg_conf_open_bdev);
818
819/**
820 * blkg_conf_prep - parse and prepare for per-blkg config update
821 * @blkcg: target block cgroup
822 * @pol: target policy
823 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
824 *
825 * Parse per-blkg config update from @ctx->input and initialize @ctx
826 * accordingly. On success, @ctx->body points to the part of @ctx->input
827 * following MAJ:MIN, @ctx->bdev points to the target block device and
828 * @ctx->blkg to the blkg being configured.
829 *
830 * blkg_conf_open_bdev() must be called on @ctx beforehand. On success, this
831 * function returns with queue lock held and must be followed by
832 * blkg_conf_close_bdev().
833 */
834int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
835 struct blkg_conf_ctx *ctx)
836{
837 struct gendisk *disk;
838 struct request_queue *q;
839 struct blkcg_gq *blkg;
840 int ret;
841
842 if (WARN_ON_ONCE(!ctx->bdev))
843 return -EINVAL;
844
845 disk = ctx->bdev->bd_disk;
846 q = disk->queue;
847
848 /* Prevent concurrent with blkcg_deactivate_policy() */
849 mutex_lock(&q->blkcg_mutex);
850 spin_lock_irq(&q->queue_lock);
851
852 if (!blkcg_policy_enabled(q, pol)) {
853 ret = -EOPNOTSUPP;
854 goto fail_unlock;
855 }
856
857 blkg = blkg_lookup(blkcg, q);
858 if (blkg)
859 goto success;
860
861 /*
862 * Create blkgs walking down from blkcg_root to @blkcg, so that all
863 * non-root blkgs have access to their parents.
864 */
865 while (true) {
866 struct blkcg *pos = blkcg;
867 struct blkcg *parent;
868 struct blkcg_gq *new_blkg;
869
870 parent = blkcg_parent(blkcg);
871 while (parent && !blkg_lookup(parent, q)) {
872 pos = parent;
873 parent = blkcg_parent(parent);
874 }
875
876 /* Drop locks to do new blkg allocation with GFP_KERNEL. */
877 spin_unlock_irq(&q->queue_lock);
878
879 new_blkg = blkg_alloc(pos, disk, GFP_NOIO);
880 if (unlikely(!new_blkg)) {
881 ret = -ENOMEM;
882 goto fail_exit;
883 }
884
885 if (radix_tree_preload(GFP_KERNEL)) {
886 blkg_free(new_blkg);
887 ret = -ENOMEM;
888 goto fail_exit;
889 }
890
891 spin_lock_irq(&q->queue_lock);
892
893 if (!blkcg_policy_enabled(q, pol)) {
894 blkg_free(new_blkg);
895 ret = -EOPNOTSUPP;
896 goto fail_preloaded;
897 }
898
899 blkg = blkg_lookup(pos, q);
900 if (blkg) {
901 blkg_free(new_blkg);
902 } else {
903 blkg = blkg_create(pos, disk, new_blkg);
904 if (IS_ERR(blkg)) {
905 ret = PTR_ERR(blkg);
906 goto fail_preloaded;
907 }
908 }
909
910 radix_tree_preload_end();
911
912 if (pos == blkcg)
913 goto success;
914 }
915success:
916 mutex_unlock(&q->blkcg_mutex);
917 ctx->blkg = blkg;
918 return 0;
919
920fail_preloaded:
921 radix_tree_preload_end();
922fail_unlock:
923 spin_unlock_irq(&q->queue_lock);
924fail_exit:
925 mutex_unlock(&q->blkcg_mutex);
926 /*
927 * If queue was bypassing, we should retry. Do so after a
928 * short msleep(). It isn't strictly necessary but queue
929 * can be bypassing for some time and it's always nice to
930 * avoid busy looping.
931 */
932 if (ret == -EBUSY) {
933 msleep(10);
934 ret = restart_syscall();
935 }
936 return ret;
937}
938EXPORT_SYMBOL_GPL(blkg_conf_prep);
939
940/**
941 * blkg_conf_unprep - counterpart of blkg_conf_prep()
942 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
943 */
944void blkg_conf_unprep(struct blkg_conf_ctx *ctx)
945{
946 WARN_ON_ONCE(!ctx->blkg);
947 spin_unlock_irq(&ctx->bdev->bd_disk->queue->queue_lock);
948 ctx->blkg = NULL;
949}
950EXPORT_SYMBOL_GPL(blkg_conf_unprep);
951
952/**
953 * blkg_conf_close_bdev - counterpart of blkg_conf_open_bdev()
954 * @ctx: blkg_conf_ctx initialized with blkg_conf_init()
955 */
956void blkg_conf_close_bdev(struct blkg_conf_ctx *ctx)
957{
958 mutex_unlock(&ctx->bdev->bd_queue->rq_qos_mutex);
959 blkdev_put_no_open(ctx->bdev);
960 ctx->body = NULL;
961 ctx->bdev = NULL;
962}
963EXPORT_SYMBOL_GPL(blkg_conf_close_bdev);
964
965static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
966{
967 int i;
968
969 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
970 dst->bytes[i] += src->bytes[i];
971 dst->ios[i] += src->ios[i];
972 }
973}
974
975static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
976{
977 int i;
978
979 for (i = 0; i < BLKG_IOSTAT_NR; i++) {
980 dst->bytes[i] -= src->bytes[i];
981 dst->ios[i] -= src->ios[i];
982 }
983}
984
985static void blkcg_iostat_update(struct blkcg_gq *blkg, struct blkg_iostat *cur,
986 struct blkg_iostat *last)
987{
988 struct blkg_iostat delta;
989 unsigned long flags;
990
991 /* propagate percpu delta to global */
992 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
993 blkg_iostat_set(&delta, cur);
994 blkg_iostat_sub(&delta, last);
995 blkg_iostat_add(&blkg->iostat.cur, &delta);
996 blkg_iostat_add(last, &delta);
997 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
998}
999
1000static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
1001{
1002 struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
1003 struct llist_node *lnode;
1004 struct blkg_iostat_set *bisc, *next_bisc;
1005 unsigned long flags;
1006
1007 rcu_read_lock();
1008
1009 lnode = llist_del_all(lhead);
1010 if (!lnode)
1011 goto out;
1012
1013 /*
1014 * For covering concurrent parent blkg update from blkg_release().
1015 *
1016 * When flushing from cgroup, the subsystem rstat lock is always held,
1017 * so this lock won't cause contention most of time.
1018 */
1019 raw_spin_lock_irqsave(&blkg_stat_lock, flags);
1020
1021 /*
1022 * Iterate only the iostat_cpu's queued in the lockless list.
1023 */
1024 llist_for_each_entry_safe(bisc, next_bisc, lnode, lnode) {
1025 struct blkcg_gq *blkg = bisc->blkg;
1026 struct blkcg_gq *parent = blkg->parent;
1027 struct blkg_iostat cur;
1028 unsigned int seq;
1029
1030 /*
1031 * Order assignment of `next_bisc` from `bisc->lnode.next` in
1032 * llist_for_each_entry_safe and clearing `bisc->lqueued` for
1033 * avoiding to assign `next_bisc` with new next pointer added
1034 * in blk_cgroup_bio_start() in case of re-ordering.
1035 *
1036 * The pair barrier is implied in llist_add() in blk_cgroup_bio_start().
1037 */
1038 smp_mb();
1039
1040 WRITE_ONCE(bisc->lqueued, false);
1041 if (bisc == &blkg->iostat)
1042 goto propagate_up; /* propagate up to parent only */
1043
1044 /* fetch the current per-cpu values */
1045 do {
1046 seq = u64_stats_fetch_begin(&bisc->sync);
1047 blkg_iostat_set(&cur, &bisc->cur);
1048 } while (u64_stats_fetch_retry(&bisc->sync, seq));
1049
1050 blkcg_iostat_update(blkg, &cur, &bisc->last);
1051
1052propagate_up:
1053 /* propagate global delta to parent (unless that's root) */
1054 if (parent && parent->parent) {
1055 blkcg_iostat_update(parent, &blkg->iostat.cur,
1056 &blkg->iostat.last);
1057 /*
1058 * Queue parent->iostat to its blkcg's lockless
1059 * list to propagate up to the grandparent if the
1060 * iostat hasn't been queued yet.
1061 */
1062 if (!parent->iostat.lqueued) {
1063 struct llist_head *plhead;
1064
1065 plhead = per_cpu_ptr(parent->blkcg->lhead, cpu);
1066 llist_add(&parent->iostat.lnode, plhead);
1067 parent->iostat.lqueued = true;
1068 }
1069 }
1070 }
1071 raw_spin_unlock_irqrestore(&blkg_stat_lock, flags);
1072out:
1073 rcu_read_unlock();
1074}
1075
1076static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
1077{
1078 /* Root-level stats are sourced from system-wide IO stats */
1079 if (cgroup_parent(css->cgroup))
1080 __blkcg_rstat_flush(css_to_blkcg(css), cpu);
1081}
1082
1083/*
1084 * We source root cgroup stats from the system-wide stats to avoid
1085 * tracking the same information twice and incurring overhead when no
1086 * cgroups are defined. For that reason, css_rstat_flush in
1087 * blkcg_print_stat does not actually fill out the iostat in the root
1088 * cgroup's blkcg_gq.
1089 *
1090 * However, we would like to re-use the printing code between the root and
1091 * non-root cgroups to the extent possible. For that reason, we simulate
1092 * flushing the root cgroup's stats by explicitly filling in the iostat
1093 * with disk level statistics.
1094 */
1095static void blkcg_fill_root_iostats(void)
1096{
1097 struct class_dev_iter iter;
1098 struct device *dev;
1099
1100 class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
1101 while ((dev = class_dev_iter_next(&iter))) {
1102 struct block_device *bdev = dev_to_bdev(dev);
1103 struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg;
1104 struct blkg_iostat tmp;
1105 int cpu;
1106 unsigned long flags;
1107
1108 memset(&tmp, 0, sizeof(tmp));
1109 for_each_possible_cpu(cpu) {
1110 struct disk_stats *cpu_dkstats;
1111
1112 cpu_dkstats = per_cpu_ptr(bdev->bd_stats, cpu);
1113 tmp.ios[BLKG_IOSTAT_READ] +=
1114 cpu_dkstats->ios[STAT_READ];
1115 tmp.ios[BLKG_IOSTAT_WRITE] +=
1116 cpu_dkstats->ios[STAT_WRITE];
1117 tmp.ios[BLKG_IOSTAT_DISCARD] +=
1118 cpu_dkstats->ios[STAT_DISCARD];
1119 // convert sectors to bytes
1120 tmp.bytes[BLKG_IOSTAT_READ] +=
1121 cpu_dkstats->sectors[STAT_READ] << 9;
1122 tmp.bytes[BLKG_IOSTAT_WRITE] +=
1123 cpu_dkstats->sectors[STAT_WRITE] << 9;
1124 tmp.bytes[BLKG_IOSTAT_DISCARD] +=
1125 cpu_dkstats->sectors[STAT_DISCARD] << 9;
1126 }
1127
1128 flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
1129 blkg_iostat_set(&blkg->iostat.cur, &tmp);
1130 u64_stats_update_end_irqrestore(&blkg->iostat.sync, flags);
1131 }
1132 class_dev_iter_exit(&iter);
1133}
1134
1135static void blkcg_print_one_stat(struct blkcg_gq *blkg, struct seq_file *s)
1136{
1137 struct blkg_iostat_set *bis = &blkg->iostat;
1138 u64 rbytes, wbytes, rios, wios, dbytes, dios;
1139 const char *dname;
1140 unsigned seq;
1141 int i;
1142
1143 if (!blkg->online)
1144 return;
1145
1146 dname = blkg_dev_name(blkg);
1147 if (!dname)
1148 return;
1149
1150 seq_printf(s, "%s ", dname);
1151
1152 do {
1153 seq = u64_stats_fetch_begin(&bis->sync);
1154
1155 rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
1156 wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
1157 dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
1158 rios = bis->cur.ios[BLKG_IOSTAT_READ];
1159 wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
1160 dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
1161 } while (u64_stats_fetch_retry(&bis->sync, seq));
1162
1163 if (rbytes || wbytes || rios || wios) {
1164 seq_printf(s, "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
1165 rbytes, wbytes, rios, wios,
1166 dbytes, dios);
1167 }
1168
1169 if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
1170 seq_printf(s, " use_delay=%d delay_nsec=%llu",
1171 atomic_read(&blkg->use_delay),
1172 atomic64_read(&blkg->delay_nsec));
1173 }
1174
1175 for (i = 0; i < BLKCG_MAX_POLS; i++) {
1176 struct blkcg_policy *pol = blkcg_policy[i];
1177
1178 if (!blkg->pd[i] || !pol->pd_stat_fn)
1179 continue;
1180
1181 pol->pd_stat_fn(blkg->pd[i], s);
1182 }
1183
1184 seq_puts(s, "\n");
1185}
1186
1187static int blkcg_print_stat(struct seq_file *sf, void *v)
1188{
1189 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
1190 struct blkcg_gq *blkg;
1191
1192 if (!seq_css(sf)->parent)
1193 blkcg_fill_root_iostats();
1194 else
1195 css_rstat_flush(&blkcg->css);
1196
1197 guard(spinlock_irq)(&blkcg->lock);
1198 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node)
1199 blkcg_print_one_stat(blkg, sf);
1200
1201 return 0;
1202}
1203
1204static struct cftype blkcg_files[] = {
1205 {
1206 .name = "stat",
1207 .seq_show = blkcg_print_stat,
1208 },
1209 { } /* terminate */
1210};
1211
1212static struct cftype blkcg_legacy_files[] = {
1213 {
1214 .name = "reset_stats",
1215 .write_u64 = blkcg_reset_stats,
1216 },
1217 { } /* terminate */
1218};
1219
1220#ifdef CONFIG_CGROUP_WRITEBACK
1221struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css)
1222{
1223 return &css_to_blkcg(css)->cgwb_list;
1224}
1225#endif
1226
1227/*
1228 * blkcg destruction is a three-stage process.
1229 *
1230 * 1. Destruction starts. The blkcg_css_offline() callback is invoked
1231 * which offlines writeback. Here we tie the next stage of blkg destruction
1232 * to the completion of writeback associated with the blkcg. This lets us
1233 * avoid punting potentially large amounts of outstanding writeback to root
1234 * while maintaining any ongoing policies. The next stage is triggered when
1235 * the nr_cgwbs count goes to zero.
1236 *
1237 * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
1238 * and handles the destruction of blkgs. Here the css reference held by
1239 * the blkg is put back eventually allowing blkcg_css_free() to be called.
1240 * This work may occur in cgwb_release_workfn() on the cgwb_release
1241 * workqueue. Any submitted ios that fail to get the blkg ref will be
1242 * punted to the root_blkg.
1243 *
1244 * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
1245 * This finally frees the blkcg.
1246 */
1247
1248static struct blkcg_gq *blkcg_get_first_blkg(struct blkcg *blkcg)
1249{
1250 struct blkcg_gq *blkg = NULL;
1251
1252 spin_lock_irq(&blkcg->lock);
1253 if (!hlist_empty(&blkcg->blkg_list)) {
1254 blkg = hlist_entry(blkcg->blkg_list.first, struct blkcg_gq,
1255 blkcg_node);
1256 blkg_get(blkg);
1257 }
1258 spin_unlock_irq(&blkcg->lock);
1259
1260 return blkg;
1261}
1262
1263/**
1264 * blkcg_destroy_blkgs - responsible for shooting down blkgs
1265 * @blkcg: blkcg of interest
1266 *
1267 * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
1268 * is nested inside q lock, this function performs reverse double lock dancing.
1269 * Destroying the blkgs releases the reference held on the blkcg's css allowing
1270 * blkcg_css_free to eventually be called.
1271 *
1272 * This is the blkcg counterpart of ioc_release_fn().
1273 */
1274static void blkcg_destroy_blkgs(struct blkcg *blkcg)
1275{
1276 struct blkcg_gq *blkg;
1277
1278 might_sleep();
1279
1280 while ((blkg = blkcg_get_first_blkg(blkcg))) {
1281 struct request_queue *q = blkg->q;
1282
1283 spin_lock_irq(&q->queue_lock);
1284 spin_lock(&blkcg->lock);
1285
1286 blkg_destroy(blkg);
1287
1288 spin_unlock(&blkcg->lock);
1289 spin_unlock_irq(&q->queue_lock);
1290
1291 blkg_put(blkg);
1292 cond_resched();
1293 }
1294}
1295
1296/**
1297 * blkcg_pin_online - pin online state
1298 * @blkcg_css: blkcg of interest
1299 *
1300 * While pinned, a blkcg is kept online. This is primarily used to
1301 * impedance-match blkg and cgwb lifetimes so that blkg doesn't go offline
1302 * while an associated cgwb is still active.
1303 */
1304void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css)
1305{
1306 refcount_inc(&css_to_blkcg(blkcg_css)->online_pin);
1307}
1308
1309/**
1310 * blkcg_unpin_online - unpin online state
1311 * @blkcg_css: blkcg of interest
1312 *
1313 * This is primarily used to impedance-match blkg and cgwb lifetimes so
1314 * that blkg doesn't go offline while an associated cgwb is still active.
1315 * When this count goes to zero, all active cgwbs have finished so the
1316 * blkcg can continue destruction by calling blkcg_destroy_blkgs().
1317 */
1318void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css)
1319{
1320 struct blkcg *blkcg = css_to_blkcg(blkcg_css);
1321
1322 do {
1323 struct blkcg *parent;
1324
1325 if (!refcount_dec_and_test(&blkcg->online_pin))
1326 break;
1327
1328 parent = blkcg_parent(blkcg);
1329 blkcg_destroy_blkgs(blkcg);
1330 blkcg = parent;
1331 } while (blkcg);
1332}
1333
1334/**
1335 * blkcg_css_offline - cgroup css_offline callback
1336 * @css: css of interest
1337 *
1338 * This function is called when @css is about to go away. Here the cgwbs are
1339 * offlined first and only once writeback associated with the blkcg has
1340 * finished do we start step 2 (see above).
1341 */
1342static void blkcg_css_offline(struct cgroup_subsys_state *css)
1343{
1344 /* this prevents anyone from attaching or migrating to this blkcg */
1345 wb_blkcg_offline(css);
1346
1347 /* put the base online pin allowing step 2 to be triggered */
1348 blkcg_unpin_online(css);
1349}
1350
1351static void blkcg_css_free(struct cgroup_subsys_state *css)
1352{
1353 struct blkcg *blkcg = css_to_blkcg(css);
1354 int i;
1355
1356 /*
1357 * Every blkg holds a reference on this css and drops any delay it
1358 * still has from pd_free_fn(), so this is expected to be zero. Should
1359 * a policy ever leave one behind, drop it here rather than let it pin
1360 * blkcg_nr_congested and disable the fast path for the rest of the
1361 * boot. Nothing can race with us at this point.
1362 */
1363 if (WARN_ON_ONCE(atomic_xchg(&blkcg->congestion_count, 0) > 0))
1364 atomic_dec(&blkcg_nr_congested);
1365
1366 mutex_lock(&blkcg_pol_mutex);
1367
1368 list_del(&blkcg->all_blkcgs_node);
1369
1370 for (i = 0; i < BLKCG_MAX_POLS; i++)
1371 if (blkcg->cpd[i])
1372 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1373
1374 mutex_unlock(&blkcg_pol_mutex);
1375
1376 free_percpu(blkcg->lhead);
1377 kfree(blkcg);
1378}
1379
1380static struct cgroup_subsys_state *
1381blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
1382{
1383 struct blkcg *blkcg;
1384 int i;
1385
1386 mutex_lock(&blkcg_pol_mutex);
1387
1388 if (!parent_css) {
1389 blkcg = &blkcg_root;
1390 } else {
1391 blkcg = kzalloc_obj(*blkcg);
1392 if (!blkcg)
1393 goto unlock;
1394 }
1395
1396 if (init_blkcg_llists(blkcg))
1397 goto free_blkcg;
1398
1399 for (i = 0; i < BLKCG_MAX_POLS ; i++) {
1400 struct blkcg_policy *pol = blkcg_policy[i];
1401 struct blkcg_policy_data *cpd;
1402
1403 /*
1404 * If the policy hasn't been attached yet, wait for it
1405 * to be attached before doing anything else. Otherwise,
1406 * check if the policy requires any specific per-cgroup
1407 * data: if it does, allocate and initialize it.
1408 */
1409 if (!pol || !pol->cpd_alloc_fn)
1410 continue;
1411
1412 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1413 if (!cpd)
1414 goto free_pd_blkcg;
1415
1416 blkcg->cpd[i] = cpd;
1417 cpd->blkcg = blkcg;
1418 cpd->plid = i;
1419 }
1420
1421 spin_lock_init(&blkcg->lock);
1422 refcount_set(&blkcg->online_pin, 1);
1423 INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);
1424 INIT_HLIST_HEAD(&blkcg->blkg_list);
1425#ifdef CONFIG_CGROUP_WRITEBACK
1426 INIT_LIST_HEAD(&blkcg->cgwb_list);
1427#endif
1428 list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
1429
1430 mutex_unlock(&blkcg_pol_mutex);
1431 return &blkcg->css;
1432
1433free_pd_blkcg:
1434 for (i--; i >= 0; i--)
1435 if (blkcg->cpd[i])
1436 blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
1437 free_percpu(blkcg->lhead);
1438free_blkcg:
1439 if (blkcg != &blkcg_root)
1440 kfree(blkcg);
1441unlock:
1442 mutex_unlock(&blkcg_pol_mutex);
1443 return ERR_PTR(-ENOMEM);
1444}
1445
1446static int blkcg_css_online(struct cgroup_subsys_state *css)
1447{
1448 struct blkcg *parent = blkcg_parent(css_to_blkcg(css));
1449
1450 /*
1451 * blkcg_pin_online() is used to delay blkcg offline so that blkgs
1452 * don't go offline while cgwbs are still active on them. Pin the
1453 * parent so that offline always happens towards the root.
1454 */
1455 if (parent)
1456 blkcg_pin_online(&parent->css);
1457 return 0;
1458}
1459
1460void blkg_init_queue(struct request_queue *q)
1461{
1462 INIT_LIST_HEAD(&q->blkg_list);
1463 mutex_init(&q->blkcg_mutex);
1464}
1465
1466int blkcg_init_disk(struct gendisk *disk)
1467{
1468 struct request_queue *q = disk->queue;
1469 struct blkcg_gq *new_blkg, *blkg;
1470 bool preloaded;
1471
1472 /*
1473 * If the queue is shared across disk rebind (e.g., SCSI), the
1474 * previous disk's blkcg state is cleaned up asynchronously via
1475 * disk_release() -> blkcg_exit_disk(). Wait for that cleanup to
1476 * finish (indicated by root_blkg becoming NULL) before setting up
1477 * new blkcg state. Otherwise, we may overwrite q->root_blkg while
1478 * the old one is still alive, and radix_tree_insert() in
1479 * blkg_create() will fail with -EEXIST because the old entries
1480 * still occupy the same queue id slot in blkcg->blkg_tree.
1481 */
1482 wait_var_event(&q->root_blkg, !READ_ONCE(q->root_blkg));
1483
1484 new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL);
1485 if (!new_blkg)
1486 return -ENOMEM;
1487
1488 preloaded = !radix_tree_preload(GFP_KERNEL);
1489
1490 /* Make sure the root blkg exists. */
1491 /* spin_lock_irq can serve as RCU read-side critical section. */
1492 spin_lock_irq(&q->queue_lock);
1493 blkg = blkg_create(&blkcg_root, disk, new_blkg);
1494 if (IS_ERR(blkg))
1495 goto err_unlock;
1496 q->root_blkg = blkg;
1497 spin_unlock_irq(&q->queue_lock);
1498
1499 if (preloaded)
1500 radix_tree_preload_end();
1501
1502 return 0;
1503
1504err_unlock:
1505 spin_unlock_irq(&q->queue_lock);
1506 if (preloaded)
1507 radix_tree_preload_end();
1508 return PTR_ERR(blkg);
1509}
1510
1511void blkcg_exit_disk(struct gendisk *disk)
1512{
1513 blkg_destroy_all(disk);
1514 blk_throtl_exit(disk);
1515}
1516
1517static void blkcg_exit(struct task_struct *tsk)
1518{
1519 if (tsk->throttle_disk)
1520 put_disk(tsk->throttle_disk);
1521 tsk->throttle_disk = NULL;
1522}
1523
1524struct cgroup_subsys io_cgrp_subsys = {
1525 .css_alloc = blkcg_css_alloc,
1526 .css_online = blkcg_css_online,
1527 .css_offline = blkcg_css_offline,
1528 .css_free = blkcg_css_free,
1529 .css_rstat_flush = blkcg_rstat_flush,
1530 .dfl_cftypes = blkcg_files,
1531 .legacy_cftypes = blkcg_legacy_files,
1532 .legacy_name = "blkio",
1533 .exit = blkcg_exit,
1534#ifdef CONFIG_MEMCG
1535 /*
1536 * This ensures that, if available, memcg is automatically enabled
1537 * together on the default hierarchy so that the owner cgroup can
1538 * be retrieved from writeback pages.
1539 */
1540 .depends_on = 1 << memory_cgrp_id,
1541#endif
1542};
1543EXPORT_SYMBOL_GPL(io_cgrp_subsys);
1544
1545/*
1546 * Tear down per-blkg policy data for @pol on @q.
1547 */
1548static void blkcg_policy_teardown_pds(struct request_queue *q,
1549 const struct blkcg_policy *pol)
1550{
1551 struct blkcg_gq *blkg;
1552
1553 list_for_each_entry(blkg, &q->blkg_list, q_node) {
1554 struct blkcg *blkcg = blkg->blkcg;
1555 struct blkg_policy_data *pd;
1556
1557 spin_lock(&blkcg->lock);
1558 pd = blkg->pd[pol->plid];
1559 if (pd) {
1560 if (pd->online && pol->pd_offline_fn)
1561 pol->pd_offline_fn(pd);
1562 pd->online = false;
1563 pol->pd_free_fn(pd);
1564 WRITE_ONCE(blkg->pd[pol->plid], NULL);
1565 }
1566 spin_unlock(&blkcg->lock);
1567 }
1568}
1569
1570/**
1571 * blkcg_activate_policy - activate a blkcg policy on a gendisk
1572 * @disk: gendisk of interest
1573 * @pol: blkcg policy to activate
1574 *
1575 * Activate @pol on @disk. Requires %GFP_KERNEL context. @disk goes through
1576 * bypass mode to populate its blkgs with policy_data for @pol.
1577 *
1578 * Activation happens with @disk bypassed, so nobody would be accessing blkgs
1579 * from IO path. Update of each blkg is protected by both queue and blkcg
1580 * locks so that holding either lock and testing blkcg_policy_enabled() is
1581 * always enough for dereferencing policy data.
1582 *
1583 * The caller is responsible for synchronizing [de]activations and policy
1584 * [un]registerations. Returns 0 on success, -errno on failure.
1585 */
1586int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol)
1587{
1588 struct request_queue *q = disk->queue;
1589 struct blkg_policy_data *pd_prealloc = NULL;
1590 struct blkcg_gq *blkg, *pinned_blkg = NULL;
1591 unsigned int memflags;
1592 int ret;
1593
1594 if (blkcg_policy_enabled(q, pol))
1595 return 0;
1596
1597 /*
1598 * Policy is allowed to be registered without pd_alloc_fn/pd_free_fn,
1599 * for example, ioprio. Such policy will work on blkcg level, not disk
1600 * level, and don't need to be activated.
1601 */
1602 if (WARN_ON_ONCE(!pol->pd_alloc_fn || !pol->pd_free_fn))
1603 return -EINVAL;
1604
1605 if (queue_is_mq(q))
1606 memflags = blk_mq_freeze_queue(q);
1607
1608 mutex_lock(&q->blkcg_mutex);
1609retry:
1610 spin_lock_irq(&q->queue_lock);
1611
1612 /* blkg_list is pushed at the head, reverse walk to initialize parents first */
1613 list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
1614 struct blkg_policy_data *pd;
1615
1616 if (blkg->pd[pol->plid])
1617 continue;
1618 if (hlist_unhashed(&blkg->blkcg_node))
1619 continue;
1620
1621 /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
1622 if (blkg == pinned_blkg) {
1623 pd = pd_prealloc;
1624 pd_prealloc = NULL;
1625 } else {
1626 pd = pol->pd_alloc_fn(disk, blkg->blkcg,
1627 GFP_NOWAIT);
1628 }
1629
1630 if (!pd) {
1631 /*
1632 * GFP_NOWAIT failed. Free the existing one and
1633 * prealloc for @blkg w/ GFP_KERNEL.
1634 */
1635 if (pinned_blkg)
1636 blkg_put(pinned_blkg);
1637 blkg_get(blkg);
1638 pinned_blkg = blkg;
1639
1640 spin_unlock_irq(&q->queue_lock);
1641
1642 if (pd_prealloc)
1643 pol->pd_free_fn(pd_prealloc);
1644 pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,
1645 GFP_KERNEL);
1646 if (pd_prealloc)
1647 goto retry;
1648 else
1649 goto enomem;
1650 }
1651
1652 spin_lock(&blkg->blkcg->lock);
1653
1654 pd->blkg = blkg;
1655 pd->plid = pol->plid;
1656 WRITE_ONCE(blkg->pd[pol->plid], pd);
1657
1658 if (pol->pd_init_fn)
1659 pol->pd_init_fn(pd);
1660
1661 if (pol->pd_online_fn)
1662 pol->pd_online_fn(pd);
1663 pd->online = true;
1664
1665 spin_unlock(&blkg->blkcg->lock);
1666 }
1667
1668 __set_bit(pol->plid, q->blkcg_pols);
1669 ret = 0;
1670
1671 spin_unlock_irq(&q->queue_lock);
1672out:
1673 mutex_unlock(&q->blkcg_mutex);
1674 if (queue_is_mq(q))
1675 blk_mq_unfreeze_queue(q, memflags);
1676 if (pinned_blkg)
1677 blkg_put(pinned_blkg);
1678 if (pd_prealloc)
1679 pol->pd_free_fn(pd_prealloc);
1680 return ret;
1681
1682enomem:
1683 /* alloc failed, take down everything */
1684 spin_lock_irq(&q->queue_lock);
1685 blkcg_policy_teardown_pds(q, pol);
1686 spin_unlock_irq(&q->queue_lock);
1687 ret = -ENOMEM;
1688 goto out;
1689}
1690EXPORT_SYMBOL_GPL(blkcg_activate_policy);
1691
1692/**
1693 * blkcg_deactivate_policy - deactivate a blkcg policy on a gendisk
1694 * @disk: gendisk of interest
1695 * @pol: blkcg policy to deactivate
1696 *
1697 * Deactivate @pol on @disk. Follows the same synchronization rules as
1698 * blkcg_activate_policy().
1699 */
1700void blkcg_deactivate_policy(struct gendisk *disk,
1701 const struct blkcg_policy *pol)
1702{
1703 struct request_queue *q = disk->queue;
1704 unsigned int memflags;
1705
1706 if (!blkcg_policy_enabled(q, pol))
1707 return;
1708
1709 if (queue_is_mq(q))
1710 memflags = blk_mq_freeze_queue(q);
1711
1712 mutex_lock(&q->blkcg_mutex);
1713 spin_lock_irq(&q->queue_lock);
1714
1715 __clear_bit(pol->plid, q->blkcg_pols);
1716 blkcg_policy_teardown_pds(q, pol);
1717 spin_unlock_irq(&q->queue_lock);
1718 mutex_unlock(&q->blkcg_mutex);
1719
1720 if (queue_is_mq(q))
1721 blk_mq_unfreeze_queue(q, memflags);
1722}
1723EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
1724
1725static void blkcg_free_all_cpd(struct blkcg_policy *pol)
1726{
1727 struct blkcg *blkcg;
1728
1729 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1730 if (blkcg->cpd[pol->plid]) {
1731 pol->cpd_free_fn(blkcg->cpd[pol->plid]);
1732 blkcg->cpd[pol->plid] = NULL;
1733 }
1734 }
1735}
1736
1737/**
1738 * blkcg_policy_register - register a blkcg policy
1739 * @pol: blkcg policy to register
1740 *
1741 * Register @pol with blkcg core. Might sleep and @pol may be modified on
1742 * successful registration. Returns 0 on success and -errno on failure.
1743 */
1744int blkcg_policy_register(struct blkcg_policy *pol)
1745{
1746 struct blkcg *blkcg;
1747 int i, ret;
1748
1749 /*
1750 * Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs, and policy
1751 * without pd_alloc_fn/pd_free_fn can't be activated.
1752 */
1753 if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
1754 (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
1755 return -EINVAL;
1756
1757 mutex_lock(&blkcg_pol_register_mutex);
1758 mutex_lock(&blkcg_pol_mutex);
1759
1760 /* find an empty slot */
1761 for (i = 0; i < BLKCG_MAX_POLS; i++)
1762 if (!blkcg_policy[i])
1763 break;
1764 if (i >= BLKCG_MAX_POLS) {
1765 pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
1766 ret = -ENOSPC;
1767 goto err_unlock;
1768 }
1769
1770 /* register @pol */
1771 pol->plid = i;
1772 blkcg_policy[pol->plid] = pol;
1773
1774 /* allocate and install cpd's */
1775 if (pol->cpd_alloc_fn) {
1776 list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
1777 struct blkcg_policy_data *cpd;
1778
1779 cpd = pol->cpd_alloc_fn(GFP_KERNEL);
1780 if (!cpd) {
1781 ret = -ENOMEM;
1782 goto err_free_cpds;
1783 }
1784
1785 blkcg->cpd[pol->plid] = cpd;
1786 cpd->blkcg = blkcg;
1787 cpd->plid = pol->plid;
1788 }
1789 }
1790
1791 mutex_unlock(&blkcg_pol_mutex);
1792
1793 /* everything is in place, add intf files for the new policy */
1794 if (pol->dfl_cftypes == pol->legacy_cftypes) {
1795 WARN_ON(cgroup_add_cftypes(&io_cgrp_subsys,
1796 pol->dfl_cftypes));
1797 } else {
1798 WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
1799 pol->dfl_cftypes));
1800 WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
1801 pol->legacy_cftypes));
1802 }
1803 mutex_unlock(&blkcg_pol_register_mutex);
1804 return 0;
1805
1806err_free_cpds:
1807 if (pol->cpd_free_fn)
1808 blkcg_free_all_cpd(pol);
1809
1810 blkcg_policy[pol->plid] = NULL;
1811err_unlock:
1812 mutex_unlock(&blkcg_pol_mutex);
1813 mutex_unlock(&blkcg_pol_register_mutex);
1814 return ret;
1815}
1816EXPORT_SYMBOL_GPL(blkcg_policy_register);
1817
1818/**
1819 * blkcg_policy_unregister - unregister a blkcg policy
1820 * @pol: blkcg policy to unregister
1821 *
1822 * Undo blkcg_policy_register(@pol). Might sleep.
1823 */
1824void blkcg_policy_unregister(struct blkcg_policy *pol)
1825{
1826 mutex_lock(&blkcg_pol_register_mutex);
1827
1828 if (WARN_ON(blkcg_policy[pol->plid] != pol))
1829 goto out_unlock;
1830
1831 /* kill the intf files first */
1832 if (pol->dfl_cftypes)
1833 cgroup_rm_cftypes(pol->dfl_cftypes);
1834 if (pol->legacy_cftypes)
1835 cgroup_rm_cftypes(pol->legacy_cftypes);
1836
1837 /* remove cpds and unregister */
1838 mutex_lock(&blkcg_pol_mutex);
1839
1840 if (pol->cpd_free_fn)
1841 blkcg_free_all_cpd(pol);
1842
1843 blkcg_policy[pol->plid] = NULL;
1844
1845 mutex_unlock(&blkcg_pol_mutex);
1846out_unlock:
1847 mutex_unlock(&blkcg_pol_register_mutex);
1848}
1849EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
1850
1851/*
1852 * Scale the accumulated delay based on how long it has been since we updated
1853 * the delay. We only call this when we are adding delay, in case it's been a
1854 * while since we added delay, and when we are checking to see if we need to
1855 * delay a task, to account for any delays that may have occurred.
1856 */
1857static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
1858{
1859 u64 old = atomic64_read(&blkg->delay_start);
1860
1861 /* negative use_delay means no scaling, see blkcg_set_delay() */
1862 if (atomic_read(&blkg->use_delay) < 0)
1863 return;
1864
1865 /*
1866 * We only want to scale down every second. The idea here is that we
1867 * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
1868 * time window. We only want to throttle tasks for recent delay that
1869 * has occurred, in 1 second time windows since that's the maximum
1870 * things can be throttled. We save the current delay window in
1871 * blkg->last_delay so we know what amount is still left to be charged
1872 * to the blkg from this point onward. blkg->last_use keeps track of
1873 * the use_delay counter. The idea is if we're unthrottling the blkg we
1874 * are ok with whatever is happening now, and we can take away more of
1875 * the accumulated delay as we've already throttled enough that
1876 * everybody is happy with their IO latencies.
1877 */
1878 if (time_before64(old + NSEC_PER_SEC, now) &&
1879 atomic64_try_cmpxchg(&blkg->delay_start, &old, now)) {
1880 u64 cur = atomic64_read(&blkg->delay_nsec);
1881 u64 sub = min_t(u64, blkg->last_delay, now - old);
1882 int cur_use = atomic_read(&blkg->use_delay);
1883
1884 /*
1885 * We've been unthrottled, subtract a larger chunk of our
1886 * accumulated delay.
1887 */
1888 if (cur_use < blkg->last_use)
1889 sub = max_t(u64, sub, blkg->last_delay >> 1);
1890
1891 /*
1892 * This shouldn't happen, but handle it anyway. Our delay_nsec
1893 * should only ever be growing except here where we subtract out
1894 * min(last_delay, 1 second), but lord knows bugs happen and I'd
1895 * rather not end up with negative numbers.
1896 */
1897 if (unlikely(cur < sub)) {
1898 atomic64_set(&blkg->delay_nsec, 0);
1899 blkg->last_delay = 0;
1900 } else {
1901 atomic64_sub(sub, &blkg->delay_nsec);
1902 blkg->last_delay = cur - sub;
1903 }
1904 blkg->last_use = cur_use;
1905 }
1906}
1907
1908/*
1909 * This is called when we want to actually walk up the hierarchy and check to
1910 * see if we need to throttle, and then actually throttle if there is some
1911 * accumulated delay. This should only be called upon return to user space so
1912 * we're not holding some lock that would induce a priority inversion.
1913 */
1914static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
1915{
1916 unsigned long pflags;
1917 bool clamp;
1918 u64 now = blk_time_get_ns();
1919 u64 exp;
1920 u64 delay_nsec = 0;
1921 int tok;
1922
1923 while (blkg->parent) {
1924 int use_delay = atomic_read(&blkg->use_delay);
1925
1926 if (use_delay) {
1927 u64 this_delay;
1928
1929 blkcg_scale_delay(blkg, now);
1930 this_delay = atomic64_read(&blkg->delay_nsec);
1931 if (this_delay > delay_nsec) {
1932 delay_nsec = this_delay;
1933 clamp = use_delay > 0;
1934 }
1935 }
1936 blkg = blkg->parent;
1937 }
1938
1939 if (!delay_nsec)
1940 return;
1941
1942 /*
1943 * Let's not sleep for all eternity if we've amassed a huge delay.
1944 * Swapping or metadata IO can accumulate 10's of seconds worth of
1945 * delay, and we want userspace to be able to do _something_ so cap the
1946 * delays at 0.25s. If there's 10's of seconds worth of delay then the
1947 * tasks will be delayed for 0.25 second for every syscall. If
1948 * blkcg_set_delay() was used as indicated by negative use_delay, the
1949 * caller is responsible for regulating the range.
1950 */
1951 if (clamp)
1952 delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
1953
1954 if (use_memdelay)
1955 psi_memstall_enter(&pflags);
1956
1957 exp = ktime_add_ns(now, delay_nsec);
1958 tok = io_schedule_prepare();
1959 do {
1960 __set_current_state(TASK_KILLABLE);
1961 if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
1962 break;
1963 } while (!fatal_signal_pending(current));
1964 io_schedule_finish(tok);
1965
1966 if (use_memdelay)
1967 psi_memstall_leave(&pflags);
1968}
1969
1970/**
1971 * blkcg_maybe_throttle_current - throttle the current task if it has been marked
1972 *
1973 * This is only called if we've been marked with set_notify_resume(). Obviously
1974 * we can be set_notify_resume() for reasons other than blkcg throttling, so we
1975 * check to see if current->throttle_disk is set and if not this doesn't do
1976 * anything. This should only ever be called by the resume code, it's not meant
1977 * to be called by people willy-nilly as it will actually do the work to
1978 * throttle the task if it is setup for throttling.
1979 */
1980void blkcg_maybe_throttle_current(void)
1981{
1982 struct gendisk *disk = current->throttle_disk;
1983 struct blkcg *blkcg;
1984 struct blkcg_gq *blkg;
1985 bool use_memdelay = current->use_memdelay;
1986
1987 if (!disk)
1988 return;
1989
1990 current->throttle_disk = NULL;
1991 current->use_memdelay = false;
1992
1993 rcu_read_lock();
1994 blkcg = css_to_blkcg(blkcg_css());
1995 if (!blkcg)
1996 goto out;
1997 blkg = blkg_lookup(blkcg, disk->queue);
1998 if (!blkg)
1999 goto out;
2000 if (!blkg_tryget(blkg))
2001 goto out;
2002 rcu_read_unlock();
2003
2004 blkcg_maybe_throttle_blkg(blkg, use_memdelay);
2005 blkg_put(blkg);
2006 put_disk(disk);
2007 return;
2008out:
2009 rcu_read_unlock();
2010 put_disk(disk);
2011}
2012
2013/**
2014 * blkcg_schedule_throttle - this task needs to check for throttling
2015 * @disk: disk to throttle
2016 * @use_memdelay: do we charge this to memory delay for PSI
2017 *
2018 * This is called by the IO controller when we know there's delay accumulated
2019 * for the blkg for this task. We do not pass the blkg because there are places
2020 * we call this that may not have that information, the swapping code for
2021 * instance will only have a block_device at that point. This set's the
2022 * notify_resume for the task to check and see if it requires throttling before
2023 * returning to user space.
2024 *
2025 * We will only schedule once per syscall. You can call this over and over
2026 * again and it will only do the check once upon return to user space, and only
2027 * throttle once. If the task needs to be throttled again it'll need to be
2028 * re-set at the next time we see the task.
2029 */
2030void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay)
2031{
2032 if (unlikely(current->flags & PF_KTHREAD))
2033 return;
2034
2035 if (current->throttle_disk != disk) {
2036 if (test_bit(GD_DEAD, &disk->state))
2037 return;
2038 get_device(disk_to_dev(disk));
2039
2040 if (current->throttle_disk)
2041 put_disk(current->throttle_disk);
2042 current->throttle_disk = disk;
2043 }
2044
2045 if (use_memdelay)
2046 current->use_memdelay = use_memdelay;
2047 set_notify_resume(current);
2048}
2049
2050/**
2051 * blkcg_add_delay - add delay to this blkg
2052 * @blkg: blkg of interest
2053 * @now: the current time in nanoseconds
2054 * @delta: how many nanoseconds of delay to add
2055 *
2056 * Charge @delta to the blkg's current delay accumulation. This is used to
2057 * throttle tasks if an IO controller thinks we need more throttling.
2058 */
2059void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
2060{
2061 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
2062 return;
2063 blkcg_scale_delay(blkg, now);
2064 atomic64_add(delta, &blkg->delay_nsec);
2065}
2066
2067static inline struct blkcg_gq *blkg_lookup_tryget(struct blkcg_gq *blkg)
2068{
2069retry:
2070 if (blkg_tryget(blkg))
2071 return blkg;
2072
2073 blkg = blkg->parent;
2074 if (blkg)
2075 goto retry;
2076
2077 return NULL;
2078}
2079/**
2080 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
2081 * @bio: target bio
2082 * @css: target css
2083 *
2084 * As the failure mode here is to walk up the blkg tree, this ensure that the
2085 * blkg->parent pointers are always valid. This returns the blkg that it ended
2086 * up taking a reference on or %NULL if no reference was taken.
2087 */
2088static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
2089 struct cgroup_subsys_state *css)
2090{
2091 struct request_queue *q = bio->bi_bdev->bd_queue;
2092 struct blkcg *blkcg = css_to_blkcg(css);
2093 struct blkcg_gq *blkg;
2094 unsigned long flags;
2095
2096 rcu_read_lock();
2097 blkg = blkg_lookup(blkcg, q);
2098 if (likely(blkg))
2099 blkg = blkg_lookup_tryget(blkg);
2100 rcu_read_unlock();
2101
2102 if (blkg)
2103 return blkg;
2104
2105 /*
2106 * Fast path failed, we're probably issuing IO in this cgroup the first
2107 * time, hold lock to create new blkg.
2108 */
2109 spin_lock_irqsave(&q->queue_lock, flags);
2110 blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk);
2111 if (blkg)
2112 blkg = blkg_lookup_tryget(blkg);
2113 spin_unlock_irqrestore(&q->queue_lock, flags);
2114
2115 return blkg;
2116}
2117
2118/**
2119 * bio_associate_blkg_from_css - associate a bio with a specified css
2120 * @bio: target bio
2121 * @css: target css
2122 *
2123 * Associate @bio with the blkg found by combining the css's blkg and the
2124 * request_queue of the @bio. An association failure is handled by walking up
2125 * the blkg tree. Therefore, the blkg associated can be anything between @blkg
2126 * and q->root_blkg. This situation only happens when a cgroup is dying and
2127 * then the remaining bios will spill to the closest alive blkg.
2128 *
2129 * A reference will be taken on the blkg and will be released when @bio is
2130 * freed.
2131 */
2132void bio_associate_blkg_from_css(struct bio *bio,
2133 struct cgroup_subsys_state *css)
2134{
2135 if (bio->bi_blkg)
2136 blkg_put(bio->bi_blkg);
2137
2138 if (css && css->parent) {
2139 bio->bi_blkg = blkg_tryget_closest(bio, css);
2140 } else {
2141 blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
2142 bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
2143 }
2144}
2145EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
2146
2147/**
2148 * bio_associate_blkg - associate a bio with a blkg
2149 * @bio: target bio
2150 *
2151 * Associate @bio with the blkg found from the bio's css and request_queue.
2152 * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
2153 * already associated, the css is reused and association redone as the
2154 * request_queue may have changed.
2155 */
2156void bio_associate_blkg(struct bio *bio)
2157{
2158 struct cgroup_subsys_state *css;
2159
2160 if (blk_op_is_passthrough(bio->bi_opf))
2161 return;
2162
2163 if (bio->bi_blkg) {
2164 css = bio_blkcg_css(bio);
2165 bio_associate_blkg_from_css(bio, css);
2166 } else {
2167 rcu_read_lock();
2168 css = blkcg_css();
2169 if (!css_tryget_online(css))
2170 css = NULL;
2171 rcu_read_unlock();
2172
2173 bio_associate_blkg_from_css(bio, css);
2174 if (css)
2175 css_put(css);
2176 }
2177}
2178EXPORT_SYMBOL_GPL(bio_associate_blkg);
2179
2180/**
2181 * bio_clone_blkg_association - clone blkg association from src to dst bio
2182 * @dst: destination bio
2183 * @src: source bio
2184 */
2185void bio_clone_blkg_association(struct bio *dst, struct bio *src)
2186{
2187 if (src->bi_blkg)
2188 bio_associate_blkg_from_css(dst, bio_blkcg_css(src));
2189}
2190EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
2191
2192static int blk_cgroup_io_type(struct bio *bio)
2193{
2194 if (op_is_discard(bio->bi_opf))
2195 return BLKG_IOSTAT_DISCARD;
2196 if (op_is_write(bio->bi_opf))
2197 return BLKG_IOSTAT_WRITE;
2198 return BLKG_IOSTAT_READ;
2199}
2200
2201void blk_cgroup_bio_start(struct bio *bio)
2202{
2203 struct blkcg *blkcg = bio->bi_blkg->blkcg;
2204 int rwd = blk_cgroup_io_type(bio), cpu;
2205 struct blkg_iostat_set *bis;
2206 unsigned long flags;
2207
2208 if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
2209 return;
2210
2211 /* Root-level stats are sourced from system-wide IO stats */
2212 if (!cgroup_parent(blkcg->css.cgroup))
2213 return;
2214
2215 cpu = get_cpu();
2216 bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
2217 flags = u64_stats_update_begin_irqsave(&bis->sync);
2218
2219 /*
2220 * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
2221 * bio and we would have already accounted for the size of the bio.
2222 */
2223 if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
2224 bio_set_flag(bio, BIO_CGROUP_ACCT);
2225 bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
2226 }
2227 bis->cur.ios[rwd]++;
2228
2229 /*
2230 * If the iostat_cpu isn't in a lockless list, put it into the
2231 * list to indicate that a stat update is pending.
2232 */
2233 if (!READ_ONCE(bis->lqueued)) {
2234 struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
2235
2236 llist_add(&bis->lnode, lhead);
2237 WRITE_ONCE(bis->lqueued, true);
2238 }
2239
2240 u64_stats_update_end_irqrestore(&bis->sync, flags);
2241 __css_rstat_updated(&blkcg->css, cpu);
2242 put_cpu();
2243}
2244
2245bool __blk_cgroup_congested(void)
2246{
2247 struct blkcg *blkcg;
2248 bool ret = false;
2249
2250 rcu_read_lock();
2251 for (blkcg = css_to_blkcg(blkcg_css()); blkcg;
2252 blkcg = blkcg_parent(blkcg)) {
2253 if (atomic_read(&blkcg->congestion_count)) {
2254 ret = true;
2255 break;
2256 }
2257 }
2258 rcu_read_unlock();
2259 return ret;
2260}
2261
2262module_param(blkcg_debug_stats, bool, 0644);
2263MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");