projects/linux·blob master · refs

1468 lines · 37.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * cgroups support for the BFQ I/O scheduler.
4 */
5#include <linux/module.h>
6#include <linux/slab.h>
7#include <linux/blkdev.h>
8#include <linux/cgroup.h>
9#include <linux/ktime.h>
10#include <linux/rbtree.h>
11#include <linux/ioprio.h>
12#include <linux/sbitmap.h>
13#include <linux/delay.h>
14
15#include "elevator.h"
16#include "bfq-iosched.h"
17
18#ifdef CONFIG_BFQ_CGROUP_DEBUG
19static int bfq_stat_init(struct bfq_stat *stat, gfp_t gfp)
20{
21 int ret;
22
23 ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
24 if (ret)
25 return ret;
26
27 atomic64_set(&stat->aux_cnt, 0);
28 return 0;
29}
30
31static void bfq_stat_exit(struct bfq_stat *stat)
32{
33 percpu_counter_destroy(&stat->cpu_cnt);
34}
35
36/**
37 * bfq_stat_add - add a value to a bfq_stat
38 * @stat: target bfq_stat
39 * @val: value to add
40 *
41 * Add @val to @stat. The caller must ensure that IRQ on the same CPU
42 * don't re-enter this function for the same counter.
43 */
44static inline void bfq_stat_add(struct bfq_stat *stat, uint64_t val)
45{
46 percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
47}
48
49/**
50 * bfq_stat_read - read the current value of a bfq_stat
51 * @stat: bfq_stat to read
52 */
53static inline uint64_t bfq_stat_read(struct bfq_stat *stat)
54{
55 return percpu_counter_sum_positive(&stat->cpu_cnt);
56}
57
58/**
59 * bfq_stat_reset - reset a bfq_stat
60 * @stat: bfq_stat to reset
61 */
62static inline void bfq_stat_reset(struct bfq_stat *stat)
63{
64 percpu_counter_set(&stat->cpu_cnt, 0);
65 atomic64_set(&stat->aux_cnt, 0);
66}
67
68/**
69 * bfq_stat_add_aux - add a bfq_stat into another's aux count
70 * @to: the destination bfq_stat
71 * @from: the source
72 *
73 * Add @from's count including the aux one to @to's aux count.
74 */
75static inline void bfq_stat_add_aux(struct bfq_stat *to,
76 struct bfq_stat *from)
77{
78 atomic64_add(bfq_stat_read(from) + atomic64_read(&from->aux_cnt),
79 &to->aux_cnt);
80}
81
82/**
83 * blkg_prfill_stat - prfill callback for bfq_stat
84 * @sf: seq_file to print to
85 * @pd: policy private data of interest
86 * @off: offset to the bfq_stat in @pd
87 *
88 * prfill callback for printing a bfq_stat.
89 */
90static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
91 int off)
92{
93 return __blkg_prfill_u64(sf, pd, bfq_stat_read((void *)pd + off));
94}
95
96/* bfqg stats flags */
97enum bfqg_stats_flags {
98 BFQG_stats_waiting = 0,
99 BFQG_stats_idling,
100 BFQG_stats_empty,
101};
102
103#define BFQG_FLAG_FNS(name) \
104static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
105{ \
106 stats->flags |= (1 << BFQG_stats_##name); \
107} \
108static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
109{ \
110 stats->flags &= ~(1 << BFQG_stats_##name); \
111} \
112static int bfqg_stats_##name(struct bfqg_stats *stats) \
113{ \
114 return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
115} \
116
117BFQG_FLAG_FNS(waiting)
118BFQG_FLAG_FNS(idling)
119BFQG_FLAG_FNS(empty)
120#undef BFQG_FLAG_FNS
121
122/* This should be called with the scheduler lock held. */
123static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
124{
125 u64 now;
126
127 if (!bfqg_stats_waiting(stats))
128 return;
129
130 now = blk_time_get_ns();
131 if (now > stats->start_group_wait_time)
132 bfq_stat_add(&stats->group_wait_time,
133 now - stats->start_group_wait_time);
134 bfqg_stats_clear_waiting(stats);
135}
136
137/* This should be called with the scheduler lock held. */
138static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
139 struct bfq_group *curr_bfqg)
140{
141 struct bfqg_stats *stats = &bfqg->stats;
142
143 if (bfqg_stats_waiting(stats))
144 return;
145 if (bfqg == curr_bfqg)
146 return;
147 stats->start_group_wait_time = blk_time_get_ns();
148 bfqg_stats_mark_waiting(stats);
149}
150
151/* This should be called with the scheduler lock held. */
152static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
153{
154 u64 now;
155
156 if (!bfqg_stats_empty(stats))
157 return;
158
159 now = blk_time_get_ns();
160 if (now > stats->start_empty_time)
161 bfq_stat_add(&stats->empty_time,
162 now - stats->start_empty_time);
163 bfqg_stats_clear_empty(stats);
164}
165
166void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
167{
168 bfq_stat_add(&bfqg->stats.dequeue, 1);
169}
170
171void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
172{
173 struct bfqg_stats *stats = &bfqg->stats;
174
175 if (blkg_rwstat_total(&stats->queued))
176 return;
177
178 /*
179 * group is already marked empty. This can happen if bfqq got new
180 * request in parent group and moved to this group while being added
181 * to service tree. Just ignore the event and move on.
182 */
183 if (bfqg_stats_empty(stats))
184 return;
185
186 stats->start_empty_time = blk_time_get_ns();
187 bfqg_stats_mark_empty(stats);
188}
189
190void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
191{
192 struct bfqg_stats *stats = &bfqg->stats;
193
194 if (bfqg_stats_idling(stats)) {
195 u64 now = blk_time_get_ns();
196
197 if (now > stats->start_idle_time)
198 bfq_stat_add(&stats->idle_time,
199 now - stats->start_idle_time);
200 bfqg_stats_clear_idling(stats);
201 }
202}
203
204void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
205{
206 struct bfqg_stats *stats = &bfqg->stats;
207
208 stats->start_idle_time = blk_time_get_ns();
209 bfqg_stats_mark_idling(stats);
210}
211
212void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
213{
214 struct bfqg_stats *stats = &bfqg->stats;
215
216 bfq_stat_add(&stats->avg_queue_size_sum,
217 blkg_rwstat_total(&stats->queued));
218 bfq_stat_add(&stats->avg_queue_size_samples, 1);
219 bfqg_stats_update_group_wait_time(stats);
220}
221
222void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
223 blk_opf_t opf)
224{
225 blkg_rwstat_add(&bfqg->stats.queued, opf, 1);
226 bfqg_stats_end_empty_time(&bfqg->stats);
227 if (!(bfqq == bfqg->bfqd->in_service_queue))
228 bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
229}
230
231void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)
232{
233 blkg_rwstat_add(&bfqg->stats.queued, opf, -1);
234}
235
236void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)
237{
238 blkg_rwstat_add(&bfqg->stats.merged, opf, 1);
239}
240
241void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
242 u64 io_start_time_ns, blk_opf_t opf)
243{
244 struct bfqg_stats *stats = &bfqg->stats;
245 u64 now = blk_time_get_ns();
246
247 if (now > io_start_time_ns)
248 blkg_rwstat_add(&stats->service_time, opf,
249 now - io_start_time_ns);
250 if (io_start_time_ns > start_time_ns)
251 blkg_rwstat_add(&stats->wait_time, opf,
252 io_start_time_ns - start_time_ns);
253}
254
255#else /* CONFIG_BFQ_CGROUP_DEBUG */
256
257void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }
258void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }
259void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
260 u64 io_start_time_ns, blk_opf_t opf) { }
261void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
262void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
263
264#endif /* CONFIG_BFQ_CGROUP_DEBUG */
265
266#ifdef CONFIG_BFQ_GROUP_IOSCHED
267
268/*
269 * blk-cgroup policy-related handlers
270 * The following functions help in converting between blk-cgroup
271 * internal structures and BFQ-specific structures.
272 */
273
274static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)
275{
276 return pd ? container_of(pd, struct bfq_group, pd) : NULL;
277}
278
279struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
280{
281 return pd_to_blkg(&bfqg->pd);
282}
283
284static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
285{
286 return pd_to_bfqg(blkg_to_pd(blkg, &blkcg_policy_bfq));
287}
288
289/*
290 * bfq_group handlers
291 * The following functions help in navigating the bfq_group hierarchy
292 * by allowing to find the parent of a bfq_group or the bfq_group
293 * associated to a bfq_queue.
294 */
295
296static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)
297{
298 struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;
299
300 return pblkg ? blkg_to_bfqg(pblkg) : NULL;
301}
302
303static void bfqg_stats_exit(struct bfqg_stats *stats)
304{
305 blkg_rwstat_exit(&stats->bytes);
306 blkg_rwstat_exit(&stats->ios);
307#ifdef CONFIG_BFQ_CGROUP_DEBUG
308 blkg_rwstat_exit(&stats->merged);
309 blkg_rwstat_exit(&stats->service_time);
310 blkg_rwstat_exit(&stats->wait_time);
311 blkg_rwstat_exit(&stats->queued);
312 bfq_stat_exit(&stats->time);
313 bfq_stat_exit(&stats->avg_queue_size_sum);
314 bfq_stat_exit(&stats->avg_queue_size_samples);
315 bfq_stat_exit(&stats->dequeue);
316 bfq_stat_exit(&stats->group_wait_time);
317 bfq_stat_exit(&stats->idle_time);
318 bfq_stat_exit(&stats->empty_time);
319#endif
320}
321
322struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
323{
324 struct bfq_entity *group_entity = bfqq->entity.parent;
325
326 return group_entity ? container_of(group_entity, struct bfq_group,
327 entity) :
328 bfqq->bfqd->root_group;
329}
330
331/*
332 * The following two functions handle get and put of a bfq_group by
333 * wrapping the related blk-cgroup hooks.
334 */
335
336static void bfqg_get(struct bfq_group *bfqg)
337{
338 refcount_inc(&bfqg->ref);
339}
340
341static void bfqg_put(struct bfq_group *bfqg)
342{
343 if (refcount_dec_and_test(&bfqg->ref)) {
344 bfqg_stats_exit(&bfqg->stats);
345 kfree(bfqg);
346 }
347}
348
349static void bfqg_and_blkg_get(struct bfq_group *bfqg)
350{
351 /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
352 bfqg_get(bfqg);
353
354 blkg_get(bfqg_to_blkg(bfqg));
355}
356
357void bfqg_and_blkg_put(struct bfq_group *bfqg)
358{
359 blkg_put(bfqg_to_blkg(bfqg));
360
361 bfqg_put(bfqg);
362}
363
364void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq)
365{
366 struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);
367
368 if (!bfqg)
369 return;
370
371 blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
372 blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);
373}
374
375/* @stats = 0 */
376static void bfqg_stats_reset(struct bfqg_stats *stats)
377{
378#ifdef CONFIG_BFQ_CGROUP_DEBUG
379 /* queued stats shouldn't be cleared */
380 blkg_rwstat_reset(&stats->merged);
381 blkg_rwstat_reset(&stats->service_time);
382 blkg_rwstat_reset(&stats->wait_time);
383 bfq_stat_reset(&stats->time);
384 bfq_stat_reset(&stats->avg_queue_size_sum);
385 bfq_stat_reset(&stats->avg_queue_size_samples);
386 bfq_stat_reset(&stats->dequeue);
387 bfq_stat_reset(&stats->group_wait_time);
388 bfq_stat_reset(&stats->idle_time);
389 bfq_stat_reset(&stats->empty_time);
390#endif
391}
392
393/* @to += @from */
394static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from)
395{
396 if (!to || !from)
397 return;
398
399#ifdef CONFIG_BFQ_CGROUP_DEBUG
400 /* queued stats shouldn't be cleared */
401 blkg_rwstat_add_aux(&to->merged, &from->merged);
402 blkg_rwstat_add_aux(&to->service_time, &from->service_time);
403 blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
404 bfq_stat_add_aux(&to->time, &from->time);
405 bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
406 bfq_stat_add_aux(&to->avg_queue_size_samples,
407 &from->avg_queue_size_samples);
408 bfq_stat_add_aux(&to->dequeue, &from->dequeue);
409 bfq_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
410 bfq_stat_add_aux(&to->idle_time, &from->idle_time);
411 bfq_stat_add_aux(&to->empty_time, &from->empty_time);
412#endif
413}
414
415/*
416 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
417 * recursive stats can still account for the amount used by this bfqg after
418 * it's gone.
419 */
420static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
421{
422 struct bfq_group *parent;
423
424 if (!bfqg) /* root_group */
425 return;
426
427 parent = bfqg_parent(bfqg);
428
429 lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);
430
431 if (unlikely(!parent))
432 return;
433
434 bfqg_stats_add_aux(&parent->stats, &bfqg->stats);
435 bfqg_stats_reset(&bfqg->stats);
436}
437
438void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
439{
440 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
441
442 entity->weight = entity->new_weight;
443 entity->orig_weight = entity->new_weight;
444 if (bfqq) {
445 bfqq->ioprio = bfqq->new_ioprio;
446 bfqq->ioprio_class = bfqq->new_ioprio_class;
447 /*
448 * Make sure that bfqg and its associated blkg do not
449 * disappear before entity.
450 */
451 bfqg_and_blkg_get(bfqg);
452 }
453 entity->parent = bfqg->my_entity; /* NULL for root group */
454 entity->sched_data = &bfqg->sched_data;
455}
456
457static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
458{
459 if (blkg_rwstat_init(&stats->bytes, gfp) ||
460 blkg_rwstat_init(&stats->ios, gfp))
461 goto error;
462
463#ifdef CONFIG_BFQ_CGROUP_DEBUG
464 if (blkg_rwstat_init(&stats->merged, gfp) ||
465 blkg_rwstat_init(&stats->service_time, gfp) ||
466 blkg_rwstat_init(&stats->wait_time, gfp) ||
467 blkg_rwstat_init(&stats->queued, gfp) ||
468 bfq_stat_init(&stats->time, gfp) ||
469 bfq_stat_init(&stats->avg_queue_size_sum, gfp) ||
470 bfq_stat_init(&stats->avg_queue_size_samples, gfp) ||
471 bfq_stat_init(&stats->dequeue, gfp) ||
472 bfq_stat_init(&stats->group_wait_time, gfp) ||
473 bfq_stat_init(&stats->idle_time, gfp) ||
474 bfq_stat_init(&stats->empty_time, gfp))
475 goto error;
476#endif
477
478 return 0;
479
480error:
481 bfqg_stats_exit(stats);
482 return -ENOMEM;
483}
484
485static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
486{
487 return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
488}
489
490static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
491{
492 return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
493}
494
495static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
496{
497 struct bfq_group_data *bgd;
498
499 bgd = kzalloc_obj(*bgd, gfp);
500 if (!bgd)
501 return NULL;
502
503 bgd->weight = CGROUP_WEIGHT_DFL;
504 return &bgd->pd;
505}
506
507static void bfq_cpd_free(struct blkcg_policy_data *cpd)
508{
509 kfree(cpd_to_bfqgd(cpd));
510}
511
512static struct blkg_policy_data *bfq_pd_alloc(struct gendisk *disk,
513 struct blkcg *blkcg, gfp_t gfp)
514{
515 struct bfq_group *bfqg;
516
517 bfqg = kzalloc_node(sizeof(*bfqg), gfp, disk->node_id);
518 if (!bfqg)
519 return NULL;
520
521 if (bfqg_stats_init(&bfqg->stats, gfp)) {
522 kfree(bfqg);
523 return NULL;
524 }
525
526 /* see comments in bfq_bic_update_cgroup for why refcounting */
527 refcount_set(&bfqg->ref, 1);
528 return &bfqg->pd;
529}
530
531static void bfq_pd_init(struct blkg_policy_data *pd)
532{
533 struct blkcg_gq *blkg = pd_to_blkg(pd);
534 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
535 struct bfq_data *bfqd = blkg->q->elevator->elevator_data;
536 struct bfq_entity *entity = &bfqg->entity;
537 struct bfq_group_data *d = blkcg_to_bfqgd(blkg->blkcg);
538
539 entity->orig_weight = entity->weight = entity->new_weight = d->weight;
540 entity->my_sched_data = &bfqg->sched_data;
541 entity->last_bfqq_created = NULL;
542
543 bfqg->my_entity = entity; /*
544 * the root_group's will be set to NULL
545 * in bfq_init_queue()
546 */
547 bfqg->bfqd = bfqd;
548 bfqg->active_entities = 0;
549 bfqg->num_queues_with_pending_reqs = 0;
550 bfqg->rq_pos_tree = RB_ROOT;
551}
552
553static void bfqg_release(struct rcu_head *rcu)
554{
555 struct blkg_policy_data *pd =
556 container_of(rcu, struct blkg_policy_data, rcu_head);
557 struct bfq_group *bfqg = pd_to_bfqg(pd);
558
559 bfqg_put(bfqg);
560}
561
562static void bfq_pd_free(struct blkg_policy_data *pd)
563{
564 call_rcu(&pd->rcu_head, bfqg_release);
565}
566
567static void bfq_pd_reset_stats(struct blkg_policy_data *pd)
568{
569 struct bfq_group *bfqg = pd_to_bfqg(pd);
570
571 bfqg_stats_reset(&bfqg->stats);
572}
573
574static void bfq_group_set_parent(struct bfq_group *bfqg,
575 struct bfq_group *parent)
576{
577 struct bfq_entity *entity;
578
579 entity = &bfqg->entity;
580 entity->parent = parent->my_entity;
581 entity->sched_data = &parent->sched_data;
582}
583
584static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)
585{
586 struct bfq_group *parent;
587 struct bfq_entity *entity;
588
589 /*
590 * Update chain of bfq_groups as we might be handling a leaf group
591 * which, along with some of its relatives, has not been hooked yet
592 * to the private hierarchy of BFQ.
593 */
594 entity = &bfqg->entity;
595 for_each_entity(entity) {
596 struct bfq_group *curr_bfqg = container_of(entity,
597 struct bfq_group, entity);
598 if (curr_bfqg != bfqd->root_group) {
599 parent = bfqg_parent(curr_bfqg);
600 if (!parent)
601 parent = bfqd->root_group;
602 bfq_group_set_parent(curr_bfqg, parent);
603 }
604 }
605}
606
607struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)
608{
609 struct blkcg_gq *blkg = bio->bi_blkg;
610 struct bfq_group *bfqg;
611
612 while (blkg) {
613 if (!data_race(blkg->online)) {
614 blkg = blkg->parent;
615 continue;
616 }
617 bfqg = blkg_to_bfqg(blkg);
618 if (bfqg->pd.online) {
619 bio_associate_blkg_from_css(bio, &blkg->blkcg->css);
620 return bfqg;
621 }
622 blkg = blkg->parent;
623 }
624 bio_associate_blkg_from_css(bio,
625 &bfqg_to_blkg(bfqd->root_group)->blkcg->css);
626 return bfqd->root_group;
627}
628
629/**
630 * bfq_bfqq_move - migrate @bfqq to @bfqg.
631 * @bfqd: queue descriptor.
632 * @bfqq: the queue to move.
633 * @bfqg: the group to move to.
634 *
635 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
636 * it on the new one. Avoid putting the entity on the old group idle tree.
637 *
638 * Must be called under the scheduler lock, to make sure that the blkg
639 * owning @bfqg does not disappear (see comments in
640 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
641 * objects).
642 */
643void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
644 struct bfq_group *bfqg)
645{
646 struct bfq_entity *entity = &bfqq->entity;
647 struct bfq_group *old_parent = bfqq_group(bfqq);
648 bool has_pending_reqs = false;
649
650 /*
651 * No point to move bfqq to the same group, which can happen when
652 * root group is offlined
653 */
654 if (old_parent == bfqg)
655 return;
656
657 /*
658 * oom_bfqq is not allowed to move, oom_bfqq will hold ref to root_group
659 * until elevator exit.
660 */
661 if (bfqq == &bfqd->oom_bfqq)
662 return;
663 /*
664 * Get extra reference to prevent bfqq from being freed in
665 * next possible expire or deactivate.
666 */
667 bfqq->ref++;
668
669 if (entity->in_groups_with_pending_reqs) {
670 has_pending_reqs = true;
671 bfq_del_bfqq_in_groups_with_pending_reqs(bfqq);
672 }
673
674 /* If bfqq is empty, then bfq_bfqq_expire also invokes
675 * bfq_del_bfqq_busy, thereby removing bfqq and its entity
676 * from data structures related to current group. Otherwise we
677 * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
678 * we do below.
679 */
680 if (bfqq == bfqd->in_service_queue)
681 bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
682 false, BFQQE_PREEMPTED);
683
684 if (bfq_bfqq_busy(bfqq))
685 bfq_deactivate_bfqq(bfqd, bfqq, false, false);
686 else if (entity->on_st_or_in_serv)
687 bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
688 bfqg_and_blkg_put(old_parent);
689
690 bfq_reassign_last_bfqq(bfqq, NULL);
691 entity->parent = bfqg->my_entity;
692 entity->sched_data = &bfqg->sched_data;
693 /* pin down bfqg and its associated blkg */
694 bfqg_and_blkg_get(bfqg);
695
696 if (has_pending_reqs)
697 bfq_add_bfqq_in_groups_with_pending_reqs(bfqq);
698
699 if (bfq_bfqq_busy(bfqq)) {
700 if (unlikely(!bfqd->nonrot_with_queueing))
701 bfq_pos_tree_add_move(bfqd, bfqq);
702 bfq_activate_bfqq(bfqd, bfqq);
703 }
704
705 if (!bfqd->in_service_queue && !bfqd->tot_rq_in_driver)
706 bfq_schedule_dispatch(bfqd);
707 /* release extra ref taken above, bfqq may happen to be freed now */
708 bfq_put_queue(bfqq);
709}
710
711static void bfq_sync_bfqq_move(struct bfq_data *bfqd,
712 struct bfq_queue *sync_bfqq,
713 struct bfq_io_cq *bic,
714 struct bfq_group *bfqg,
715 unsigned int act_idx)
716{
717 struct bfq_queue *bfqq;
718
719 if (!sync_bfqq->new_bfqq && !bfq_bfqq_coop(sync_bfqq)) {
720 /* We are the only user of this bfqq, just move it */
721 if (sync_bfqq->entity.sched_data != &bfqg->sched_data)
722 bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
723 return;
724 }
725
726 /*
727 * The queue was merged to a different queue. Check
728 * that the merge chain still belongs to the same
729 * cgroup.
730 */
731 for (bfqq = sync_bfqq; bfqq; bfqq = bfqq->new_bfqq)
732 if (bfqq->entity.sched_data != &bfqg->sched_data)
733 break;
734 if (bfqq) {
735 /*
736 * Some queue changed cgroup so the merge is not valid
737 * anymore. We cannot easily just cancel the merge (by
738 * clearing new_bfqq) as there may be other processes
739 * using this queue and holding refs to all queues
740 * below sync_bfqq->new_bfqq. Similarly if the merge
741 * already happened, we need to detach from bfqq now
742 * so that we cannot merge bio to a request from the
743 * old cgroup.
744 */
745 bfq_put_cooperator(sync_bfqq);
746 bic_set_bfqq(bic, NULL, true, act_idx);
747 bfq_release_process_ref(bfqd, sync_bfqq);
748 }
749}
750
751/**
752 * __bfq_bic_change_cgroup - move @bic to @bfqg.
753 * @bfqd: the queue descriptor.
754 * @bic: the bic to move.
755 * @bfqg: the group to move to.
756 *
757 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
758 * sure that the reference to cgroup is valid across the call (see
759 * comments in bfq_bic_update_cgroup on this issue)
760 */
761static void __bfq_bic_change_cgroup(struct bfq_data *bfqd,
762 struct bfq_io_cq *bic,
763 struct bfq_group *bfqg)
764{
765 unsigned int act_idx;
766
767 for (act_idx = 0; act_idx < bfqd->num_actuators; act_idx++) {
768 struct bfq_queue *async_bfqq = bic_to_bfqq(bic, false, act_idx);
769 struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, true, act_idx);
770
771 if (async_bfqq &&
772 async_bfqq->entity.sched_data != &bfqg->sched_data) {
773 bic_set_bfqq(bic, NULL, false, act_idx);
774 bfq_release_process_ref(bfqd, async_bfqq);
775 }
776
777 if (sync_bfqq)
778 bfq_sync_bfqq_move(bfqd, sync_bfqq, bic, bfqg, act_idx);
779 }
780}
781
782void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
783{
784 struct bfq_data *bfqd = bic_to_bfqd(bic);
785 struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);
786 uint64_t serial_nr;
787
788 serial_nr = bfqg_to_blkg(bfqg)->blkcg->css.serial_nr;
789
790 /*
791 * Check whether blkcg has changed. The condition may trigger
792 * spuriously on a newly created cic but there's no harm.
793 */
794 if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))
795 return;
796
797 /*
798 * New cgroup for this process. Make sure it is linked to bfq internal
799 * cgroup hierarchy.
800 */
801 bfq_link_bfqg(bfqd, bfqg);
802 __bfq_bic_change_cgroup(bfqd, bic, bfqg);
803 bic->blkcg_serial_nr = serial_nr;
804}
805
806/**
807 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
808 * @st: the service tree being flushed.
809 */
810static void bfq_flush_idle_tree(struct bfq_service_tree *st)
811{
812 struct bfq_entity *entity = st->first_idle;
813
814 for (; entity ; entity = st->first_idle)
815 __bfq_deactivate_entity(entity, false);
816}
817
818/**
819 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
820 * @bfqd: the device data structure with the root group.
821 * @entity: the entity to move, if entity is a leaf; or the parent entity
822 * of an active leaf entity to move, if entity is not a leaf.
823 * @ioprio_class: I/O priority class to reparent.
824 */
825static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
826 struct bfq_entity *entity,
827 int ioprio_class)
828{
829 struct bfq_queue *bfqq;
830 struct bfq_entity *child_entity = entity;
831
832 while (child_entity->my_sched_data) { /* leaf not reached yet */
833 struct bfq_sched_data *child_sd = child_entity->my_sched_data;
834 struct bfq_service_tree *child_st = child_sd->service_tree +
835 ioprio_class;
836 struct rb_root *child_active = &child_st->active;
837
838 child_entity = bfq_entity_of(rb_first(child_active));
839
840 if (!child_entity)
841 child_entity = child_sd->in_service_entity;
842 }
843
844 bfqq = bfq_entity_to_bfqq(child_entity);
845 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
846}
847
848/**
849 * bfq_reparent_active_queues - move to the root group all active queues.
850 * @bfqd: the device data structure with the root group.
851 * @bfqg: the group to move from.
852 * @st: the service tree to start the search from.
853 * @ioprio_class: I/O priority class to reparent.
854 */
855static void bfq_reparent_active_queues(struct bfq_data *bfqd,
856 struct bfq_group *bfqg,
857 struct bfq_service_tree *st,
858 int ioprio_class)
859{
860 struct rb_root *active = &st->active;
861 struct bfq_entity *entity;
862
863 while ((entity = bfq_entity_of(rb_first(active))))
864 bfq_reparent_leaf_entity(bfqd, entity, ioprio_class);
865
866 if (bfqg->sched_data.in_service_entity)
867 bfq_reparent_leaf_entity(bfqd,
868 bfqg->sched_data.in_service_entity,
869 ioprio_class);
870}
871
872/**
873 * bfq_pd_offline - deactivate the entity associated with @pd,
874 * and reparent its children entities.
875 * @pd: descriptor of the policy going offline.
876 *
877 * blkio already grabs the queue_lock for us, so no need to use
878 * RCU-based magic
879 */
880static void bfq_pd_offline(struct blkg_policy_data *pd)
881{
882 struct bfq_service_tree *st;
883 struct bfq_group *bfqg = pd_to_bfqg(pd);
884 struct bfq_data *bfqd = bfqg->bfqd;
885 struct bfq_entity *entity = bfqg->my_entity;
886 unsigned long flags;
887 int i;
888
889 spin_lock_irqsave(&bfqd->lock, flags);
890
891 if (!entity) /* root group */
892 goto put_async_queues;
893
894 /*
895 * Empty all service_trees belonging to this group before
896 * deactivating the group itself.
897 */
898 for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
899 st = bfqg->sched_data.service_tree + i;
900
901 /*
902 * It may happen that some queues are still active
903 * (busy) upon group destruction (if the corresponding
904 * processes have been forced to terminate). We move
905 * all the leaf entities corresponding to these queues
906 * to the root_group.
907 * Also, it may happen that the group has an entity
908 * in service, which is disconnected from the active
909 * tree: it must be moved, too.
910 * There is no need to put the sync queues, as the
911 * scheduler has taken no reference.
912 */
913 bfq_reparent_active_queues(bfqd, bfqg, st, i);
914
915 /*
916 * The idle tree may still contain bfq_queues
917 * belonging to exited task because they never
918 * migrated to a different cgroup from the one being
919 * destroyed now. In addition, even
920 * bfq_reparent_active_queues() may happen to add some
921 * entities to the idle tree. It happens if, in some
922 * of the calls to bfq_bfqq_move() performed by
923 * bfq_reparent_active_queues(), the queue to move is
924 * empty and gets expired.
925 */
926 bfq_flush_idle_tree(st);
927 }
928
929 __bfq_deactivate_entity(entity, false);
930
931put_async_queues:
932 bfq_put_async_queues(bfqd, bfqg);
933
934 spin_unlock_irqrestore(&bfqd->lock, flags);
935 /*
936 * @blkg is going offline and will be ignored by
937 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
938 * that they don't get lost. If IOs complete after this point, the
939 * stats for them will be lost. Oh well...
940 */
941 bfqg_stats_xfer_dead(bfqg);
942}
943
944void bfq_end_wr_async(struct bfq_data *bfqd)
945{
946 struct request_queue *q = bfqd->queue;
947 struct blkcg_gq *blkg;
948
949 mutex_lock(&q->blkcg_mutex);
950 spin_lock_irq(&q->queue_lock);
951 spin_lock(&bfqd->lock);
952
953 list_for_each_entry(blkg, &q->blkg_list, q_node) {
954 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
955
956 bfq_end_wr_async_queues(bfqd, bfqg);
957 }
958 bfq_end_wr_async_queues(bfqd, bfqd->root_group);
959
960 spin_unlock(&bfqd->lock);
961 spin_unlock_irq(&q->queue_lock);
962 mutex_unlock(&q->blkcg_mutex);
963}
964
965static int bfq_io_show_weight_legacy(struct seq_file *sf, void *v)
966{
967 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
968 struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
969 unsigned int val = 0;
970
971 if (bfqgd)
972 val = bfqgd->weight;
973
974 seq_printf(sf, "%u\n", val);
975
976 return 0;
977}
978
979static u64 bfqg_prfill_weight_device(struct seq_file *sf,
980 struct blkg_policy_data *pd, int off)
981{
982 struct bfq_group *bfqg = pd_to_bfqg(pd);
983
984 if (!bfqg->entity.dev_weight)
985 return 0;
986 return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);
987}
988
989static int bfq_io_show_weight(struct seq_file *sf, void *v)
990{
991 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
992 struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
993
994 seq_printf(sf, "default %u\n", bfqgd->weight);
995 blkcg_print_blkgs(sf, blkcg, bfqg_prfill_weight_device,
996 &blkcg_policy_bfq, 0, false);
997 return 0;
998}
999
1000static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)
1001{
1002 weight = dev_weight ?: weight;
1003
1004 bfqg->entity.dev_weight = dev_weight;
1005 /*
1006 * Setting the prio_changed flag of the entity
1007 * to 1 with new_weight == weight would re-set
1008 * the value of the weight to its ioprio mapping.
1009 * Set the flag only if necessary.
1010 */
1011 if ((unsigned short)weight != bfqg->entity.new_weight) {
1012 bfqg->entity.new_weight = (unsigned short)weight;
1013 /*
1014 * Make sure that the above new value has been
1015 * stored in bfqg->entity.new_weight before
1016 * setting the prio_changed flag. In fact,
1017 * this flag may be read asynchronously (in
1018 * critical sections protected by a different
1019 * lock than that held here), and finding this
1020 * flag set may cause the execution of the code
1021 * for updating parameters whose value may
1022 * depend also on bfqg->entity.new_weight (in
1023 * __bfq_entity_update_weight_prio).
1024 * This barrier makes sure that the new value
1025 * of bfqg->entity.new_weight is correctly
1026 * seen in that code.
1027 */
1028 smp_wmb();
1029 bfqg->entity.prio_changed = 1;
1030 }
1031}
1032
1033static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css,
1034 struct cftype *cftype,
1035 u64 val)
1036{
1037 struct blkcg *blkcg = css_to_blkcg(css);
1038 struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
1039 struct blkcg_gq *blkg;
1040 int ret = -ERANGE;
1041
1042 if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT)
1043 return ret;
1044
1045 ret = 0;
1046 spin_lock_irq(&blkcg->lock);
1047 bfqgd->weight = (unsigned short)val;
1048 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
1049 struct bfq_group *bfqg = blkg_to_bfqg(blkg);
1050
1051 if (bfqg)
1052 bfq_group_set_weight(bfqg, val, 0);
1053 }
1054 spin_unlock_irq(&blkcg->lock);
1055
1056 return ret;
1057}
1058
1059static ssize_t bfq_io_set_device_weight(struct kernfs_open_file *of,
1060 char *buf, size_t nbytes,
1061 loff_t off)
1062{
1063 int ret;
1064 struct blkg_conf_ctx ctx;
1065 struct blkcg *blkcg = css_to_blkcg(of_css(of));
1066 struct bfq_group *bfqg;
1067 u64 v;
1068
1069 blkg_conf_init(&ctx, buf);
1070
1071 ret = blkg_conf_open_bdev(&ctx);
1072 if (ret)
1073 return ret;
1074
1075 ret = blkg_conf_prep(blkcg, &blkcg_policy_bfq, &ctx);
1076 if (ret)
1077 goto close_bdev;
1078
1079 if (sscanf(ctx.body, "%llu", &v) == 1) {
1080 /* require "default" on dfl */
1081 ret = -ERANGE;
1082 if (!v)
1083 goto out;
1084 } else if (!strcmp(strim(ctx.body), "default")) {
1085 v = 0;
1086 } else {
1087 ret = -EINVAL;
1088 goto out;
1089 }
1090
1091 bfqg = blkg_to_bfqg(ctx.blkg);
1092
1093 ret = -ERANGE;
1094 if (!v || (v >= BFQ_MIN_WEIGHT && v <= BFQ_MAX_WEIGHT)) {
1095 bfq_group_set_weight(bfqg, bfqg->entity.weight, v);
1096 ret = 0;
1097 }
1098
1099out:
1100 blkg_conf_unprep(&ctx);
1101close_bdev:
1102 blkg_conf_close_bdev(&ctx);
1103 return ret ?: nbytes;
1104}
1105
1106static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
1107 char *buf, size_t nbytes,
1108 loff_t off)
1109{
1110 char *endp;
1111 int ret;
1112 u64 v;
1113
1114 buf = strim(buf);
1115
1116 /* "WEIGHT" or "default WEIGHT" sets the default weight */
1117 v = simple_strtoull(buf, &endp, 0);
1118 if (*endp == '\0' || sscanf(buf, "default %llu", &v) == 1) {
1119 ret = bfq_io_set_weight_legacy(of_css(of), NULL, v);
1120 return ret ?: nbytes;
1121 }
1122
1123 return bfq_io_set_device_weight(of, buf, nbytes, off);
1124}
1125
1126static int bfqg_print_rwstat(struct seq_file *sf, void *v)
1127{
1128 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
1129 &blkcg_policy_bfq, seq_cft(sf)->private, true);
1130 return 0;
1131}
1132
1133static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
1134 struct blkg_policy_data *pd, int off)
1135{
1136 struct blkg_rwstat_sample sum;
1137
1138 blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
1139 return __blkg_prfill_rwstat(sf, pd, &sum);
1140}
1141
1142static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
1143{
1144 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1145 bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq,
1146 seq_cft(sf)->private, true);
1147 return 0;
1148}
1149
1150#ifdef CONFIG_BFQ_CGROUP_DEBUG
1151static int bfqg_print_stat(struct seq_file *sf, void *v)
1152{
1153 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
1154 &blkcg_policy_bfq, seq_cft(sf)->private, false);
1155 return 0;
1156}
1157
1158static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,
1159 struct blkg_policy_data *pd, int off)
1160{
1161 struct blkcg_gq *blkg = pd_to_blkg(pd);
1162 struct blkcg_gq *pos_blkg;
1163 struct cgroup_subsys_state *pos_css;
1164 u64 sum = 0;
1165
1166 rcu_read_lock();
1167 blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
1168 struct blkg_policy_data *pd;
1169 struct bfq_stat *stat;
1170
1171 if (!data_race(pos_blkg->online))
1172 continue;
1173
1174 pd = blkg_to_pd(pos_blkg, &blkcg_policy_bfq);
1175 if (!pd)
1176 continue;
1177 stat = (void *)pd + off;
1178 sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt);
1179 }
1180 rcu_read_unlock();
1181
1182 return __blkg_prfill_u64(sf, pd, sum);
1183}
1184
1185static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)
1186{
1187 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1188 bfqg_prfill_stat_recursive, &blkcg_policy_bfq,
1189 seq_cft(sf)->private, false);
1190 return 0;
1191}
1192
1193static u64 bfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd,
1194 int off)
1195{
1196 struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);
1197 u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);
1198
1199 return __blkg_prfill_u64(sf, pd, sum >> 9);
1200}
1201
1202static int bfqg_print_stat_sectors(struct seq_file *sf, void *v)
1203{
1204 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1205 bfqg_prfill_sectors, &blkcg_policy_bfq, 0, false);
1206 return 0;
1207}
1208
1209static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf,
1210 struct blkg_policy_data *pd, int off)
1211{
1212 struct blkg_rwstat_sample tmp;
1213
1214 blkg_rwstat_recursive_sum(pd->blkg, &blkcg_policy_bfq,
1215 offsetof(struct bfq_group, stats.bytes), &tmp);
1216
1217 return __blkg_prfill_u64(sf, pd,
1218 (tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);
1219}
1220
1221static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)
1222{
1223 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1224 bfqg_prfill_sectors_recursive, &blkcg_policy_bfq, 0,
1225 false);
1226 return 0;
1227}
1228
1229static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf,
1230 struct blkg_policy_data *pd, int off)
1231{
1232 struct bfq_group *bfqg = pd_to_bfqg(pd);
1233 u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);
1234 u64 v = 0;
1235
1236 if (samples) {
1237 v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);
1238 v = div64_u64(v, samples);
1239 }
1240 __blkg_prfill_u64(sf, pd, v);
1241 return 0;
1242}
1243
1244/* print avg_queue_size */
1245static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v)
1246{
1247 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1248 bfqg_prfill_avg_queue_size, &blkcg_policy_bfq,
1249 0, false);
1250 return 0;
1251}
1252#endif /* CONFIG_BFQ_CGROUP_DEBUG */
1253
1254struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
1255{
1256 int ret;
1257
1258 ret = blkcg_activate_policy(bfqd->queue->disk, &blkcg_policy_bfq);
1259 if (ret)
1260 return NULL;
1261
1262 return blkg_to_bfqg(bfqd->queue->root_blkg);
1263}
1264
1265struct blkcg_policy blkcg_policy_bfq = {
1266 .dfl_cftypes = bfq_blkg_files,
1267 .legacy_cftypes = bfq_blkcg_legacy_files,
1268
1269 .cpd_alloc_fn = bfq_cpd_alloc,
1270 .cpd_free_fn = bfq_cpd_free,
1271
1272 .pd_alloc_fn = bfq_pd_alloc,
1273 .pd_init_fn = bfq_pd_init,
1274 .pd_offline_fn = bfq_pd_offline,
1275 .pd_free_fn = bfq_pd_free,
1276 .pd_reset_stats_fn = bfq_pd_reset_stats,
1277};
1278
1279struct cftype bfq_blkcg_legacy_files[] = {
1280 {
1281 .name = "bfq.weight",
1282 .flags = CFTYPE_NOT_ON_ROOT,
1283 .seq_show = bfq_io_show_weight_legacy,
1284 .write_u64 = bfq_io_set_weight_legacy,
1285 },
1286 {
1287 .name = "bfq.weight_device",
1288 .flags = CFTYPE_NOT_ON_ROOT,
1289 .seq_show = bfq_io_show_weight,
1290 .write = bfq_io_set_weight,
1291 },
1292
1293 /* statistics, covers only the tasks in the bfqg */
1294 {
1295 .name = "bfq.io_service_bytes",
1296 .private = offsetof(struct bfq_group, stats.bytes),
1297 .seq_show = bfqg_print_rwstat,
1298 },
1299 {
1300 .name = "bfq.io_serviced",
1301 .private = offsetof(struct bfq_group, stats.ios),
1302 .seq_show = bfqg_print_rwstat,
1303 },
1304#ifdef CONFIG_BFQ_CGROUP_DEBUG
1305 {
1306 .name = "bfq.time",
1307 .private = offsetof(struct bfq_group, stats.time),
1308 .seq_show = bfqg_print_stat,
1309 },
1310 {
1311 .name = "bfq.sectors",
1312 .seq_show = bfqg_print_stat_sectors,
1313 },
1314 {
1315 .name = "bfq.io_service_time",
1316 .private = offsetof(struct bfq_group, stats.service_time),
1317 .seq_show = bfqg_print_rwstat,
1318 },
1319 {
1320 .name = "bfq.io_wait_time",
1321 .private = offsetof(struct bfq_group, stats.wait_time),
1322 .seq_show = bfqg_print_rwstat,
1323 },
1324 {
1325 .name = "bfq.io_merged",
1326 .private = offsetof(struct bfq_group, stats.merged),
1327 .seq_show = bfqg_print_rwstat,
1328 },
1329 {
1330 .name = "bfq.io_queued",
1331 .private = offsetof(struct bfq_group, stats.queued),
1332 .seq_show = bfqg_print_rwstat,
1333 },
1334#endif /* CONFIG_BFQ_CGROUP_DEBUG */
1335
1336 /* the same statistics which cover the bfqg and its descendants */
1337 {
1338 .name = "bfq.io_service_bytes_recursive",
1339 .private = offsetof(struct bfq_group, stats.bytes),
1340 .seq_show = bfqg_print_rwstat_recursive,
1341 },
1342 {
1343 .name = "bfq.io_serviced_recursive",
1344 .private = offsetof(struct bfq_group, stats.ios),
1345 .seq_show = bfqg_print_rwstat_recursive,
1346 },
1347#ifdef CONFIG_BFQ_CGROUP_DEBUG
1348 {
1349 .name = "bfq.time_recursive",
1350 .private = offsetof(struct bfq_group, stats.time),
1351 .seq_show = bfqg_print_stat_recursive,
1352 },
1353 {
1354 .name = "bfq.sectors_recursive",
1355 .seq_show = bfqg_print_stat_sectors_recursive,
1356 },
1357 {
1358 .name = "bfq.io_service_time_recursive",
1359 .private = offsetof(struct bfq_group, stats.service_time),
1360 .seq_show = bfqg_print_rwstat_recursive,
1361 },
1362 {
1363 .name = "bfq.io_wait_time_recursive",
1364 .private = offsetof(struct bfq_group, stats.wait_time),
1365 .seq_show = bfqg_print_rwstat_recursive,
1366 },
1367 {
1368 .name = "bfq.io_merged_recursive",
1369 .private = offsetof(struct bfq_group, stats.merged),
1370 .seq_show = bfqg_print_rwstat_recursive,
1371 },
1372 {
1373 .name = "bfq.io_queued_recursive",
1374 .private = offsetof(struct bfq_group, stats.queued),
1375 .seq_show = bfqg_print_rwstat_recursive,
1376 },
1377 {
1378 .name = "bfq.avg_queue_size",
1379 .seq_show = bfqg_print_avg_queue_size,
1380 },
1381 {
1382 .name = "bfq.group_wait_time",
1383 .private = offsetof(struct bfq_group, stats.group_wait_time),
1384 .seq_show = bfqg_print_stat,
1385 },
1386 {
1387 .name = "bfq.idle_time",
1388 .private = offsetof(struct bfq_group, stats.idle_time),
1389 .seq_show = bfqg_print_stat,
1390 },
1391 {
1392 .name = "bfq.empty_time",
1393 .private = offsetof(struct bfq_group, stats.empty_time),
1394 .seq_show = bfqg_print_stat,
1395 },
1396 {
1397 .name = "bfq.dequeue",
1398 .private = offsetof(struct bfq_group, stats.dequeue),
1399 .seq_show = bfqg_print_stat,
1400 },
1401#endif /* CONFIG_BFQ_CGROUP_DEBUG */
1402 { } /* terminate */
1403};
1404
1405struct cftype bfq_blkg_files[] = {
1406 {
1407 .name = "bfq.weight",
1408 .flags = CFTYPE_NOT_ON_ROOT,
1409 .seq_show = bfq_io_show_weight,
1410 .write = bfq_io_set_weight,
1411 },
1412 {} /* terminate */
1413};
1414
1415#else /* CONFIG_BFQ_GROUP_IOSCHED */
1416
1417void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
1418 struct bfq_group *bfqg) {}
1419
1420void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
1421{
1422 struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
1423
1424 entity->weight = entity->new_weight;
1425 entity->orig_weight = entity->new_weight;
1426 if (bfqq) {
1427 bfqq->ioprio = bfqq->new_ioprio;
1428 bfqq->ioprio_class = bfqq->new_ioprio_class;
1429 }
1430 entity->sched_data = &bfqg->sched_data;
1431}
1432
1433void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {}
1434
1435void bfq_end_wr_async(struct bfq_data *bfqd)
1436{
1437 spin_lock_irq(&bfqd->lock);
1438 bfq_end_wr_async_queues(bfqd, bfqd->root_group);
1439 spin_unlock_irq(&bfqd->lock);
1440}
1441
1442struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)
1443{
1444 return bfqd->root_group;
1445}
1446
1447struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
1448{
1449 return bfqq->bfqd->root_group;
1450}
1451
1452void bfqg_and_blkg_put(struct bfq_group *bfqg) {}
1453
1454struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
1455{
1456 struct bfq_group *bfqg;
1457 int i;
1458
1459 bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
1460 if (!bfqg)
1461 return NULL;
1462
1463 for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
1464 bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
1465
1466 return bfqg;
1467}
1468#endif /* CONFIG_BFQ_GROUP_IOSCHED */