projects/linux·blob master · refs

1045 lines · 23.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * buffered writeback throttling. loosely based on CoDel. We can't drop
4 * packets for IO scheduling, so the logic is something like this:
5 *
6 * - Monitor latencies in a defined window of time.
7 * - If the minimum latency in the above window exceeds some target, increment
8 * scaling step and scale down queue depth by a factor of 2x. The monitoring
9 * window is then shrunk to 100 / sqrt(scaling step + 1).
10 * - For any window where we don't have solid data on what the latencies
11 * look like, retain status quo.
12 * - If latencies look good, decrement scaling step.
13 * - If we're only doing writes, allow the scaling step to go negative. This
14 * will temporarily boost write performance, snapping back to a stable
15 * scaling step of 0 if reads show up or the heavy writers finish. Unlike
16 * positive scaling steps where we shrink the monitoring window, a negative
17 * scaling step retains the default step==0 window size.
18 *
19 * Copyright (C) 2016 Jens Axboe
20 *
21 */
22#include <linux/kernel.h>
23#include <linux/blk_types.h>
24#include <linux/slab.h>
25#include <linux/backing-dev.h>
26#include <linux/swap.h>
27
28#include "blk-stat.h"
29#include "blk-wbt.h"
30#include "blk-rq-qos.h"
31#include "elevator.h"
32#include "blk.h"
33
34#define CREATE_TRACE_POINTS
35#include <trace/events/wbt.h>
36
37enum wbt_flags {
38 WBT_TRACKED = 1, /* write, tracked for throttling */
39 WBT_READ = 2, /* read */
40 WBT_SWAP = 4, /* write, from swap_writeout() */
41 WBT_DISCARD = 8, /* discard */
42
43 WBT_NR_BITS = 4, /* number of bits */
44};
45
46enum {
47 WBT_RWQ_BG = 0,
48 WBT_RWQ_SWAP,
49 WBT_RWQ_DISCARD,
50 WBT_NUM_RWQ,
51};
52
53/*
54 * If current state is WBT_STATE_ON/OFF_DEFAULT, it can be covered to any other
55 * state, if current state is WBT_STATE_ON/OFF_MANUAL, it can only be covered
56 * to WBT_STATE_OFF/ON_MANUAL.
57 */
58enum {
59 WBT_STATE_ON_DEFAULT = 1, /* on by default */
60 WBT_STATE_ON_MANUAL = 2, /* on manually by sysfs */
61 WBT_STATE_OFF_DEFAULT = 3, /* off by default */
62 WBT_STATE_OFF_MANUAL = 4, /* off manually by sysfs */
63};
64
65struct rq_wb {
66 /*
67 * Settings that govern how we throttle
68 */
69 unsigned int wb_background; /* background writeback */
70 unsigned int wb_normal; /* normal writeback */
71
72 short enable_state; /* WBT_STATE_* */
73
74 /*
75 * Number of consecutive periods where we don't have enough
76 * information to make a firm scale up/down decision.
77 */
78 unsigned int unknown_cnt;
79
80 u64 win_nsec; /* default window size */
81 u64 cur_win_nsec; /* current window size */
82
83 struct blk_stat_callback *cb;
84
85 u64 sync_issue;
86 void *sync_cookie;
87
88 unsigned long last_issue; /* issue time of last read rq */
89 unsigned long last_comp; /* completion time of last read rq */
90 unsigned long min_lat_nsec;
91 struct rq_qos rqos;
92 struct rq_wait rq_wait[WBT_NUM_RWQ];
93 struct rq_depth rq_depth;
94};
95
96static int wbt_init(struct gendisk *disk, struct rq_wb *rwb);
97
98static inline struct rq_wb *RQWB(struct rq_qos *rqos)
99{
100 return container_of(rqos, struct rq_wb, rqos);
101}
102
103static inline void wbt_clear_state(struct request *rq)
104{
105 rq->wbt_flags = 0;
106}
107
108static inline enum wbt_flags wbt_flags(struct request *rq)
109{
110 return rq->wbt_flags;
111}
112
113static inline bool wbt_is_tracked(struct request *rq)
114{
115 return rq->wbt_flags & WBT_TRACKED;
116}
117
118static inline bool wbt_is_read(struct request *rq)
119{
120 return rq->wbt_flags & WBT_READ;
121}
122
123enum {
124 /*
125 * Default setting, we'll scale up (to 75% of QD max) or down (min 1)
126 * from here depending on device stats
127 */
128 RWB_DEF_DEPTH = 16,
129
130 /*
131 * 100msec window
132 */
133 RWB_WINDOW_NSEC = 100 * 1000 * 1000ULL,
134
135 /*
136 * Disregard stats, if we don't meet this minimum
137 */
138 RWB_MIN_WRITE_SAMPLES = 3,
139
140 /*
141 * If we have this number of consecutive windows without enough
142 * information to scale up or down, slowly return to center state
143 * (step == 0).
144 */
145 RWB_UNKNOWN_BUMP = 5,
146};
147
148static inline bool rwb_enabled(struct rq_wb *rwb)
149{
150 return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT &&
151 rwb->enable_state != WBT_STATE_OFF_MANUAL;
152}
153
154static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)
155{
156 if (rwb_enabled(rwb)) {
157 const unsigned long cur = jiffies;
158
159 if (cur != *var)
160 *var = cur;
161 }
162}
163
164/*
165 * If a task was rate throttled in balance_dirty_pages() within the last
166 * second or so, use that to indicate a higher cleaning rate.
167 */
168static bool wb_recent_wait(struct rq_wb *rwb)
169{
170 struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
171
172 return time_before(jiffies, bdi->last_bdp_sleep + HZ);
173}
174
175static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,
176 enum wbt_flags wb_acct)
177{
178 if (wb_acct & WBT_SWAP)
179 return &rwb->rq_wait[WBT_RWQ_SWAP];
180 else if (wb_acct & WBT_DISCARD)
181 return &rwb->rq_wait[WBT_RWQ_DISCARD];
182
183 return &rwb->rq_wait[WBT_RWQ_BG];
184}
185
186static void rwb_wake_all(struct rq_wb *rwb)
187{
188 int i;
189
190 for (i = 0; i < WBT_NUM_RWQ; i++) {
191 struct rq_wait *rqw = &rwb->rq_wait[i];
192
193 if (wq_has_sleeper(&rqw->wait))
194 wake_up_all(&rqw->wait);
195 }
196}
197
198static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,
199 enum wbt_flags wb_acct)
200{
201 int inflight, limit;
202
203 inflight = atomic_dec_return(&rqw->inflight);
204
205 /*
206 * For discards, our limit is always the background. For writes, if
207 * the device does write back caching, drop further down before we
208 * wake people up.
209 */
210 if (wb_acct & WBT_DISCARD)
211 limit = rwb->wb_background;
212 else if (blk_queue_write_cache(rwb->rqos.disk->queue) &&
213 !wb_recent_wait(rwb))
214 limit = 0;
215 else
216 limit = rwb->wb_normal;
217
218 /*
219 * Don't wake anyone up if we are above the normal limit.
220 */
221 if (inflight && inflight >= limit)
222 return;
223
224 if (wq_has_sleeper(&rqw->wait)) {
225 int diff = limit - inflight;
226
227 if (!inflight || diff >= rwb->wb_background / 2)
228 wake_up_all(&rqw->wait);
229 }
230}
231
232static void __wbt_done(struct rq_qos *rqos, enum wbt_flags wb_acct)
233{
234 struct rq_wb *rwb = RQWB(rqos);
235 struct rq_wait *rqw;
236
237 if (!(wb_acct & WBT_TRACKED))
238 return;
239
240 rqw = get_rq_wait(rwb, wb_acct);
241 wbt_rqw_done(rwb, rqw, wb_acct);
242}
243
244/*
245 * Called on completion of a request. Note that it's also called when
246 * a request is merged, when the request gets freed.
247 */
248static void wbt_done(struct rq_qos *rqos, struct request *rq)
249{
250 struct rq_wb *rwb = RQWB(rqos);
251
252 if (!wbt_is_tracked(rq)) {
253 if (wbt_is_read(rq)) {
254 if (rwb->sync_cookie == rq) {
255 rwb->sync_issue = 0;
256 rwb->sync_cookie = NULL;
257 }
258
259 wb_timestamp(rwb, &rwb->last_comp);
260 }
261 } else {
262 WARN_ON_ONCE(rq == rwb->sync_cookie);
263 __wbt_done(rqos, wbt_flags(rq));
264 }
265 wbt_clear_state(rq);
266}
267
268static inline bool stat_sample_valid(struct blk_rq_stat *stat)
269{
270 /*
271 * We need at least one read sample, and a minimum of
272 * RWB_MIN_WRITE_SAMPLES. We require some write samples to know
273 * that it's writes impacting us, and not just some sole read on
274 * a device that is in a lower power state.
275 */
276 return (stat[READ].nr_samples >= 1 &&
277 stat[WRITE].nr_samples >= RWB_MIN_WRITE_SAMPLES);
278}
279
280static u64 rwb_sync_issue_lat(struct rq_wb *rwb)
281{
282 u64 issue = READ_ONCE(rwb->sync_issue);
283
284 if (!issue || !rwb->sync_cookie)
285 return 0;
286
287 return blk_time_get_ns() - issue;
288}
289
290static inline unsigned int wbt_inflight(struct rq_wb *rwb)
291{
292 unsigned int i, ret = 0;
293
294 for (i = 0; i < WBT_NUM_RWQ; i++)
295 ret += atomic_read(&rwb->rq_wait[i].inflight);
296
297 return ret;
298}
299
300enum {
301 LAT_OK = 1,
302 LAT_UNKNOWN,
303 LAT_UNKNOWN_WRITES,
304 LAT_EXCEEDED,
305};
306
307static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)
308{
309 struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
310 struct rq_depth *rqd = &rwb->rq_depth;
311 u64 thislat;
312
313 /*
314 * If our stored sync issue exceeds the window size, or it
315 * exceeds our min target AND we haven't logged any entries,
316 * flag the latency as exceeded. wbt works off completion latencies,
317 * but for a flooded device, a single sync IO can take a long time
318 * to complete after being issued. If this time exceeds our
319 * monitoring window AND we didn't see any other completions in that
320 * window, then count that sync IO as a violation of the latency.
321 */
322 thislat = rwb_sync_issue_lat(rwb);
323 if (thislat > rwb->cur_win_nsec ||
324 (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {
325 trace_wbt_lat(bdi, thislat);
326 return LAT_EXCEEDED;
327 }
328
329 /*
330 * No read/write mix, if stat isn't valid
331 */
332 if (!stat_sample_valid(stat)) {
333 /*
334 * If we had writes in this stat window and the window is
335 * current, we're only doing writes. If a task recently
336 * waited or still has writes in flights, consider us doing
337 * just writes as well.
338 */
339 if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||
340 wbt_inflight(rwb))
341 return LAT_UNKNOWN_WRITES;
342 return LAT_UNKNOWN;
343 }
344
345 /*
346 * If the 'min' latency exceeds our target, step down.
347 */
348 if (stat[READ].min > rwb->min_lat_nsec) {
349 trace_wbt_lat(bdi, stat[READ].min);
350 trace_wbt_stat(bdi, stat);
351 return LAT_EXCEEDED;
352 }
353
354 if (rqd->scale_step)
355 trace_wbt_stat(bdi, stat);
356
357 return LAT_OK;
358}
359
360static void rwb_trace_step(struct rq_wb *rwb, const char *msg)
361{
362 struct backing_dev_info *bdi = rwb->rqos.disk->bdi;
363 struct rq_depth *rqd = &rwb->rq_depth;
364
365 trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,
366 rwb->wb_background, rwb->wb_normal, rqd->max_depth);
367}
368
369static void calc_wb_limits(struct rq_wb *rwb)
370{
371 if (rwb->min_lat_nsec == 0) {
372 rwb->wb_normal = rwb->wb_background = 0;
373 } else if (rwb->rq_depth.max_depth <= 2) {
374 rwb->wb_normal = rwb->rq_depth.max_depth;
375 rwb->wb_background = 1;
376 } else {
377 rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;
378 rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;
379 }
380}
381
382static void scale_up(struct rq_wb *rwb)
383{
384 if (!rq_depth_scale_up(&rwb->rq_depth))
385 return;
386 calc_wb_limits(rwb);
387 rwb->unknown_cnt = 0;
388 rwb_wake_all(rwb);
389 rwb_trace_step(rwb, tracepoint_string("scale up"));
390}
391
392static void scale_down(struct rq_wb *rwb, bool hard_throttle)
393{
394 if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))
395 return;
396 calc_wb_limits(rwb);
397 rwb->unknown_cnt = 0;
398 rwb_trace_step(rwb, tracepoint_string("scale down"));
399}
400
401static void rwb_arm_timer(struct rq_wb *rwb)
402{
403 struct rq_depth *rqd = &rwb->rq_depth;
404
405 if (rqd->scale_step > 0) {
406 /*
407 * We should speed this up, using some variant of a fast
408 * integer inverse square root calculation. Since we only do
409 * this for every window expiration, it's not a huge deal,
410 * though.
411 */
412 rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,
413 int_sqrt((rqd->scale_step + 1) << 8));
414 } else {
415 /*
416 * For step < 0, we don't want to increase/decrease the
417 * window size.
418 */
419 rwb->cur_win_nsec = rwb->win_nsec;
420 }
421
422 blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);
423}
424
425static void wb_timer_fn(struct blk_stat_callback *cb)
426{
427 struct rq_wb *rwb = cb->data;
428 struct rq_depth *rqd = &rwb->rq_depth;
429 unsigned int inflight = wbt_inflight(rwb);
430 int status;
431
432 if (!rwb->rqos.disk)
433 return;
434
435 status = latency_exceeded(rwb, cb->stat);
436
437 trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight);
438
439 /*
440 * If we exceeded the latency target, step down. If we did not,
441 * step one level up. If we don't know enough to say either exceeded
442 * or ok, then don't do anything.
443 */
444 switch (status) {
445 case LAT_EXCEEDED:
446 scale_down(rwb, true);
447 break;
448 case LAT_OK:
449 scale_up(rwb);
450 break;
451 case LAT_UNKNOWN_WRITES:
452 /*
453 * We don't have a valid read/write sample, but we do have
454 * writes going on. Allow step to go negative, to increase
455 * write performance.
456 */
457 scale_up(rwb);
458 break;
459 case LAT_UNKNOWN:
460 if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)
461 break;
462 /*
463 * We get here when previously scaled reduced depth, and we
464 * currently don't have a valid read/write sample. For that
465 * case, slowly return to center state (step == 0).
466 */
467 if (rqd->scale_step > 0)
468 scale_up(rwb);
469 else if (rqd->scale_step < 0)
470 scale_down(rwb, false);
471 break;
472 default:
473 break;
474 }
475
476 /*
477 * Re-arm timer, if we have IO in flight
478 */
479 if (rqd->scale_step || inflight)
480 rwb_arm_timer(rwb);
481}
482
483static void wbt_update_limits(struct rq_wb *rwb)
484{
485 struct rq_depth *rqd = &rwb->rq_depth;
486
487 rqd->scale_step = 0;
488 rqd->scaled_max = false;
489
490 rq_depth_calc_max_depth(rqd);
491 calc_wb_limits(rwb);
492
493 rwb_wake_all(rwb);
494}
495
496bool wbt_disabled(struct request_queue *q)
497{
498 struct rq_qos *rqos = wbt_rq_qos(q);
499
500 return !rqos || !rwb_enabled(RQWB(rqos));
501}
502
503u64 wbt_get_min_lat(struct request_queue *q)
504{
505 struct rq_qos *rqos = wbt_rq_qos(q);
506 if (!rqos)
507 return 0;
508 return RQWB(rqos)->min_lat_nsec;
509}
510
511static void wbt_set_min_lat(struct request_queue *q, u64 val)
512{
513 struct rq_qos *rqos = wbt_rq_qos(q);
514 if (!rqos)
515 return;
516
517 RQWB(rqos)->min_lat_nsec = val;
518 if (val)
519 RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL;
520 else
521 RQWB(rqos)->enable_state = WBT_STATE_OFF_MANUAL;
522
523 wbt_update_limits(RQWB(rqos));
524}
525
526
527static bool close_io(struct rq_wb *rwb)
528{
529 const unsigned long now = jiffies;
530
531 return time_before(now, rwb->last_issue + HZ / 10) ||
532 time_before(now, rwb->last_comp + HZ / 10);
533}
534
535#define REQ_HIPRIO (REQ_SYNC | REQ_META | REQ_PRIO | REQ_SWAP)
536
537static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf)
538{
539 unsigned int limit;
540
541 if ((opf & REQ_OP_MASK) == REQ_OP_DISCARD)
542 return rwb->wb_background;
543
544 /*
545 * At this point we know it's a buffered write. If this is
546 * swap trying to free memory, or REQ_SYNC is set, then
547 * it's WB_SYNC_ALL writeback, and we'll use the max limit for
548 * that. If the write is marked as a background write, then use
549 * the idle limit, or go to normal if we haven't had competing
550 * IO for a bit.
551 */
552 if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb))
553 limit = rwb->rq_depth.max_depth;
554 else if ((opf & REQ_BACKGROUND) || close_io(rwb)) {
555 /*
556 * If less than 100ms since we completed unrelated IO,
557 * limit us to half the depth for background writeback.
558 */
559 limit = rwb->wb_background;
560 } else
561 limit = rwb->wb_normal;
562
563 return limit;
564}
565
566struct wbt_wait_data {
567 struct rq_wb *rwb;
568 enum wbt_flags wb_acct;
569 blk_opf_t opf;
570};
571
572static bool wbt_inflight_cb(struct rq_wait *rqw, void *private_data)
573{
574 struct wbt_wait_data *data = private_data;
575 return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf));
576}
577
578static void wbt_cleanup_cb(struct rq_wait *rqw, void *private_data)
579{
580 struct wbt_wait_data *data = private_data;
581 wbt_rqw_done(data->rwb, rqw, data->wb_acct);
582}
583
584/*
585 * Block if we will exceed our limit, or if we are currently waiting for
586 * the timer to kick off queuing again.
587 */
588static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,
589 blk_opf_t opf)
590{
591 struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);
592 struct wbt_wait_data data = {
593 .rwb = rwb,
594 .wb_acct = wb_acct,
595 .opf = opf,
596 };
597
598 rq_qos_wait(rqw, &data, wbt_inflight_cb, wbt_cleanup_cb);
599}
600
601static inline bool wbt_should_throttle(struct bio *bio)
602{
603 switch (bio_op(bio)) {
604 case REQ_OP_WRITE:
605 /*
606 * Don't throttle WRITE_ODIRECT
607 */
608 if ((bio->bi_opf & (REQ_SYNC | REQ_IDLE)) ==
609 (REQ_SYNC | REQ_IDLE))
610 return false;
611 fallthrough;
612 case REQ_OP_DISCARD:
613 return true;
614 default:
615 return false;
616 }
617}
618
619static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)
620{
621 enum wbt_flags flags = 0;
622
623 if (!rwb_enabled(rwb))
624 return 0;
625
626 if (bio_op(bio) == REQ_OP_READ) {
627 flags = WBT_READ;
628 } else if (wbt_should_throttle(bio)) {
629 if (bio->bi_opf & REQ_SWAP)
630 flags |= WBT_SWAP;
631 if (bio_op(bio) == REQ_OP_DISCARD)
632 flags |= WBT_DISCARD;
633 flags |= WBT_TRACKED;
634 }
635 return flags;
636}
637
638static void wbt_cleanup(struct rq_qos *rqos, struct bio *bio)
639{
640 struct rq_wb *rwb = RQWB(rqos);
641 enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);
642 __wbt_done(rqos, flags);
643}
644
645/* May sleep, if we have exceeded the writeback limits. */
646static void wbt_wait(struct rq_qos *rqos, struct bio *bio)
647{
648 struct rq_wb *rwb = RQWB(rqos);
649 enum wbt_flags flags;
650
651 flags = bio_to_wbt_flags(rwb, bio);
652 if (!(flags & WBT_TRACKED)) {
653 if (flags & WBT_READ)
654 wb_timestamp(rwb, &rwb->last_issue);
655 return;
656 }
657
658 __wbt_wait(rwb, flags, bio->bi_opf);
659
660 if (!blk_stat_is_active(rwb->cb))
661 rwb_arm_timer(rwb);
662}
663
664static void wbt_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
665{
666 struct rq_wb *rwb = RQWB(rqos);
667 rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);
668}
669
670static void wbt_issue(struct rq_qos *rqos, struct request *rq)
671{
672 struct rq_wb *rwb = RQWB(rqos);
673
674 if (!rwb_enabled(rwb))
675 return;
676
677 /*
678 * Track sync issue, in case it takes a long time to complete. Allows us
679 * to react quicker, if a sync IO takes a long time to complete. Note
680 * that this is just a hint. The request can go away when it completes,
681 * so it's important we never dereference it. We only use the address to
682 * compare with, which is why we store the sync_issue time locally.
683 */
684 if (wbt_is_read(rq) && !rwb->sync_issue) {
685 rwb->sync_cookie = rq;
686 rwb->sync_issue = rq->io_start_time_ns;
687 }
688}
689
690static void wbt_requeue(struct rq_qos *rqos, struct request *rq)
691{
692 struct rq_wb *rwb = RQWB(rqos);
693 if (!rwb_enabled(rwb))
694 return;
695 if (rq == rwb->sync_cookie) {
696 rwb->sync_issue = 0;
697 rwb->sync_cookie = NULL;
698 }
699}
700
701static int wbt_data_dir(const struct request *rq)
702{
703 const enum req_op op = req_op(rq);
704
705 if (op == REQ_OP_READ)
706 return READ;
707 else if (op_is_write(op))
708 return WRITE;
709
710 /* don't account */
711 return -1;
712}
713
714static struct rq_wb *wbt_alloc(void)
715{
716 struct rq_wb *rwb = kzalloc_obj(*rwb);
717
718 if (!rwb)
719 return NULL;
720
721 rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);
722 if (!rwb->cb) {
723 kfree(rwb);
724 return NULL;
725 }
726
727 return rwb;
728}
729
730static void wbt_free(struct rq_wb *rwb)
731{
732 blk_stat_free_callback(rwb->cb);
733 kfree(rwb);
734}
735
736/*
737 * Enable wbt if defaults are configured that way
738 */
739static bool __wbt_enable_default(struct gendisk *disk)
740{
741 struct request_queue *q = disk->queue;
742 struct rq_qos *rqos;
743 bool enable = IS_ENABLED(CONFIG_BLK_WBT_MQ);
744
745 mutex_lock(&disk->rqos_state_mutex);
746
747 if (blk_queue_disable_wbt(q))
748 enable = false;
749
750 /* Throttling already enabled? */
751 rqos = wbt_rq_qos(q);
752 if (rqos) {
753 if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
754 RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
755 mutex_unlock(&disk->rqos_state_mutex);
756 return false;
757 }
758 mutex_unlock(&disk->rqos_state_mutex);
759
760 /* Queue not registered? Maybe shutting down... */
761 if (!blk_queue_registered(q))
762 return false;
763
764 if (queue_is_mq(q) && enable)
765 return true;
766 return false;
767}
768
769void wbt_enable_default(struct gendisk *disk)
770{
771 __wbt_enable_default(disk);
772}
773EXPORT_SYMBOL_GPL(wbt_enable_default);
774
775void wbt_init_enable_default(struct gendisk *disk)
776{
777 struct request_queue *q = disk->queue;
778 struct rq_wb *rwb;
779 unsigned int memflags;
780
781 if (!__wbt_enable_default(disk))
782 return;
783
784 rwb = wbt_alloc();
785 if (!rwb)
786 return;
787
788 if (wbt_init(disk, rwb)) {
789 pr_warn("%s: failed to enable wbt\n", disk->disk_name);
790 wbt_free(rwb);
791 return;
792 }
793
794 memflags = blk_debugfs_lock(q);
795 blk_mq_debugfs_register_rq_qos(q);
796 blk_debugfs_unlock(q, memflags);
797}
798
799static u64 wbt_default_latency_nsec(struct request_queue *q)
800{
801 /*
802 * We default to 2msec for non-rotational storage, and 75msec
803 * for rotational storage.
804 */
805 if (blk_queue_rot(q))
806 return 75000000ULL;
807 return 2000000ULL;
808}
809
810static void wbt_queue_depth_changed(struct rq_qos *rqos)
811{
812 RQWB(rqos)->rq_depth.queue_depth = blk_queue_depth(rqos->disk->queue);
813 wbt_update_limits(RQWB(rqos));
814}
815
816static bool wbt_set_lat_changed(struct request_queue *q, u64 val)
817{
818 struct rq_qos *rqos = wbt_rq_qos(q);
819 struct rq_wb *rwb;
820
821 if (!rqos)
822 return true;
823
824 rwb = RQWB(rqos);
825 if (rwb->min_lat_nsec != val)
826 return true;
827
828 return rwb_enabled(rwb) != !!val;
829}
830
831static void wbt_exit(struct rq_qos *rqos)
832{
833 struct rq_wb *rwb = RQWB(rqos);
834
835 blk_stat_remove_callback(rqos->disk->queue, rwb->cb);
836 wbt_free(rwb);
837}
838
839/*
840 * Disable wbt, if enabled by default.
841 */
842void wbt_disable_default(struct gendisk *disk)
843{
844 struct rq_qos *rqos = wbt_rq_qos(disk->queue);
845 struct rq_wb *rwb;
846 if (!rqos)
847 return;
848 mutex_lock(&disk->rqos_state_mutex);
849 rwb = RQWB(rqos);
850 if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {
851 blk_stat_deactivate(rwb->cb);
852 rwb->enable_state = WBT_STATE_OFF_DEFAULT;
853 }
854 mutex_unlock(&disk->rqos_state_mutex);
855}
856EXPORT_SYMBOL_GPL(wbt_disable_default);
857
858#ifdef CONFIG_BLK_DEBUG_FS
859static int wbt_curr_win_nsec_show(void *data, struct seq_file *m)
860{
861 struct rq_qos *rqos = data;
862 struct rq_wb *rwb = RQWB(rqos);
863
864 seq_printf(m, "%llu\n", rwb->cur_win_nsec);
865 return 0;
866}
867
868static int wbt_enabled_show(void *data, struct seq_file *m)
869{
870 struct rq_qos *rqos = data;
871 struct rq_wb *rwb = RQWB(rqos);
872
873 seq_printf(m, "%d\n", rwb->enable_state);
874 return 0;
875}
876
877static int wbt_id_show(void *data, struct seq_file *m)
878{
879 struct rq_qos *rqos = data;
880
881 seq_printf(m, "%u\n", rqos->id);
882 return 0;
883}
884
885static int wbt_inflight_show(void *data, struct seq_file *m)
886{
887 struct rq_qos *rqos = data;
888 struct rq_wb *rwb = RQWB(rqos);
889 int i;
890
891 for (i = 0; i < WBT_NUM_RWQ; i++)
892 seq_printf(m, "%d: inflight %d\n", i,
893 atomic_read(&rwb->rq_wait[i].inflight));
894 return 0;
895}
896
897static int wbt_min_lat_nsec_show(void *data, struct seq_file *m)
898{
899 struct rq_qos *rqos = data;
900 struct rq_wb *rwb = RQWB(rqos);
901
902 seq_printf(m, "%lu\n", rwb->min_lat_nsec);
903 return 0;
904}
905
906static int wbt_unknown_cnt_show(void *data, struct seq_file *m)
907{
908 struct rq_qos *rqos = data;
909 struct rq_wb *rwb = RQWB(rqos);
910
911 seq_printf(m, "%u\n", rwb->unknown_cnt);
912 return 0;
913}
914
915static int wbt_normal_show(void *data, struct seq_file *m)
916{
917 struct rq_qos *rqos = data;
918 struct rq_wb *rwb = RQWB(rqos);
919
920 seq_printf(m, "%u\n", rwb->wb_normal);
921 return 0;
922}
923
924static int wbt_background_show(void *data, struct seq_file *m)
925{
926 struct rq_qos *rqos = data;
927 struct rq_wb *rwb = RQWB(rqos);
928
929 seq_printf(m, "%u\n", rwb->wb_background);
930 return 0;
931}
932
933static const struct blk_mq_debugfs_attr wbt_debugfs_attrs[] = {
934 {"curr_win_nsec", 0400, wbt_curr_win_nsec_show},
935 {"enabled", 0400, wbt_enabled_show},
936 {"id", 0400, wbt_id_show},
937 {"inflight", 0400, wbt_inflight_show},
938 {"min_lat_nsec", 0400, wbt_min_lat_nsec_show},
939 {"unknown_cnt", 0400, wbt_unknown_cnt_show},
940 {"wb_normal", 0400, wbt_normal_show},
941 {"wb_background", 0400, wbt_background_show},
942 {},
943};
944#endif
945
946static const struct rq_qos_ops wbt_rqos_ops = {
947 .throttle = wbt_wait,
948 .issue = wbt_issue,
949 .track = wbt_track,
950 .requeue = wbt_requeue,
951 .done = wbt_done,
952 .cleanup = wbt_cleanup,
953 .queue_depth_changed = wbt_queue_depth_changed,
954 .exit = wbt_exit,
955#ifdef CONFIG_BLK_DEBUG_FS
956 .debugfs_attrs = wbt_debugfs_attrs,
957#endif
958};
959
960static int wbt_init(struct gendisk *disk, struct rq_wb *rwb)
961{
962 struct request_queue *q = disk->queue;
963 int ret;
964 int i;
965
966 for (i = 0; i < WBT_NUM_RWQ; i++)
967 rq_wait_init(&rwb->rq_wait[i]);
968
969 rwb->last_comp = rwb->last_issue = jiffies;
970 rwb->win_nsec = RWB_WINDOW_NSEC;
971 rwb->enable_state = WBT_STATE_ON_DEFAULT;
972 rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
973 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
974 rwb->rq_depth.queue_depth = blk_queue_depth(q);
975 wbt_update_limits(rwb);
976
977 /*
978 * Assign rwb and add the stats callback.
979 */
980 mutex_lock(&q->rq_qos_mutex);
981 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);
982 mutex_unlock(&q->rq_qos_mutex);
983 if (ret)
984 return ret;
985
986 blk_stat_add_callback(q, rwb->cb);
987 return 0;
988}
989
990int wbt_set_lat(struct gendisk *disk, s64 val)
991{
992 struct request_queue *q = disk->queue;
993 struct rq_qos *rqos = wbt_rq_qos(q);
994 struct rq_wb *rwb = NULL;
995 unsigned int memflags;
996 int ret = 0;
997
998 if (!rqos) {
999 rwb = wbt_alloc();
1000 if (!rwb)
1001 return -ENOMEM;
1002 }
1003
1004 /*
1005 * Ensure that the queue is idled, in case the latency update
1006 * ends up either enabling or disabling wbt completely. We can't
1007 * have IO inflight if that happens.
1008 */
1009 memflags = blk_mq_freeze_queue(q);
1010 if (!rqos) {
1011 ret = wbt_init(disk, rwb);
1012 if (ret) {
1013 wbt_free(rwb);
1014 goto out;
1015 }
1016 }
1017
1018 if (val == -1)
1019 val = wbt_default_latency_nsec(q);
1020 else if (val >= 0)
1021 val *= 1000ULL;
1022
1023 mutex_lock(&disk->rqos_state_mutex);
1024 if (!wbt_set_lat_changed(q, val)) {
1025 mutex_unlock(&disk->rqos_state_mutex);
1026 goto out;
1027 }
1028 mutex_unlock(&disk->rqos_state_mutex);
1029
1030 blk_mq_quiesce_queue(q);
1031
1032 mutex_lock(&disk->rqos_state_mutex);
1033 wbt_set_min_lat(q, val);
1034 mutex_unlock(&disk->rqos_state_mutex);
1035
1036 blk_mq_unquiesce_queue(q);
1037out:
1038 blk_mq_unfreeze_queue(q, memflags);
1039
1040 memflags = blk_debugfs_lock(q);
1041 blk_mq_debugfs_register_rq_qos(q);
1042 blk_debugfs_unlock(q, memflags);
1043
1044 return ret;
1045}