projects/linux·blob master · refs

1327 lines · 32.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Simple CPU accounting cgroup controller
4 */
5#include <linux/sched/clock.h>
6#include <linux/sched/cputime.h>
7#include <linux/tsacct_kern.h>
8#include "sched.h"
9
10#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
11 #include <asm/cputime.h>
12#endif
13
14#ifdef CONFIG_IRQ_TIME_ACCOUNTING
15
16DEFINE_STATIC_KEY_FALSE(sched_clock_irqtime);
17
18/*
19 * There are no locks covering percpu hardirq/softirq time.
20 * They are only modified in vtime_account, on corresponding CPU
21 * with interrupts disabled. So, writes are safe.
22 * They are read and saved off onto struct rq in update_rq_clock().
23 * This may result in other CPU reading this CPU's IRQ time and can
24 * race with irq/vtime_account on this CPU. We would either get old
25 * or new value with a side effect of accounting a slice of IRQ time to wrong
26 * task when IRQ is in progress while we read rq->clock. That is a worthy
27 * compromise in place of having locks on each IRQ in account_system_time.
28 */
29DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
30
31void enable_sched_clock_irqtime(void)
32{
33 static_branch_enable(&sched_clock_irqtime);
34}
35
36void disable_sched_clock_irqtime(void)
37{
38 if (irqtime_enabled())
39 static_branch_disable(&sched_clock_irqtime);
40}
41
42static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
43 enum cpu_usage_stat idx)
44{
45 u64 *cpustat = kcpustat_this_cpu->cpustat;
46
47 u64_stats_update_begin(&irqtime->sync);
48 cpustat[idx] += delta;
49 irqtime->total += delta;
50 if (!kcpustat_idle_dyntick())
51 irqtime->tick_delta += delta;
52 u64_stats_update_end(&irqtime->sync);
53}
54
55/*
56 * Called after incrementing preempt_count on {soft,}irq_enter
57 * and before decrementing preempt_count on {soft,}irq_exit.
58 */
59void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
60{
61 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
62 unsigned int pc;
63 s64 delta;
64 int cpu;
65
66 if (!irqtime_enabled())
67 return;
68
69 cpu = smp_processor_id();
70 delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
71 irqtime->irq_start_time += delta;
72 pc = irq_count() - offset;
73
74 /*
75 * We do not account for softirq time from ksoftirqd here.
76 * We want to continue accounting softirq time to ksoftirqd thread
77 * in that case, so as not to confuse scheduler with a special task
78 * that do not consume any time, but still wants to run.
79 */
80 if (pc & HARDIRQ_MASK)
81 irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
82 else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())
83 irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
84}
85
86static u64 irqtime_tick_accounted(u64 maxtime)
87{
88 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
89 u64 delta;
90
91 delta = min(irqtime->tick_delta, maxtime);
92 irqtime->tick_delta -= delta;
93
94 return delta;
95}
96
97#else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
98
99static u64 irqtime_tick_accounted(u64 dummy)
100{
101 return 0;
102}
103
104#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
105
106static inline void task_group_account_field(struct task_struct *p, int index,
107 u64 tmp)
108{
109 /*
110 * Since all updates are sure to touch the root cgroup, we
111 * get ourselves ahead and touch it first. If the root cgroup
112 * is the only cgroup, then nothing else should be necessary.
113 *
114 */
115 __this_cpu_add(kernel_cpustat.cpustat[index], tmp);
116
117 cgroup_account_cputime_field(p, index, tmp);
118}
119
120/*
121 * Account user CPU time to a process.
122 * @p: the process that the CPU time gets accounted to
123 * @cputime: the CPU time spent in user space since the last update
124 */
125void account_user_time(struct task_struct *p, u64 cputime)
126{
127 int index;
128
129 /* Add user time to process. */
130 p->utime += cputime;
131 account_group_user_time(p, cputime);
132
133 index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
134
135 /* Add user time to cpustat. */
136 task_group_account_field(p, index, cputime);
137
138 /* Account for user time used */
139 acct_account_cputime(p);
140}
141
142/*
143 * Account guest CPU time to a process.
144 * @p: the process that the CPU time gets accounted to
145 * @cputime: the CPU time spent in virtual machine since the last update
146 */
147void account_guest_time(struct task_struct *p, u64 cputime)
148{
149 u64 *cpustat = kcpustat_this_cpu->cpustat;
150
151 /* Add guest time to process. */
152 p->utime += cputime;
153 account_group_user_time(p, cputime);
154 p->gtime += cputime;
155
156 /* Add guest time to cpustat. */
157 if (task_nice(p) > 0) {
158 task_group_account_field(p, CPUTIME_NICE, cputime);
159 cpustat[CPUTIME_GUEST_NICE] += cputime;
160 } else {
161 task_group_account_field(p, CPUTIME_USER, cputime);
162 cpustat[CPUTIME_GUEST] += cputime;
163 }
164}
165
166/*
167 * Account system CPU time to a process and desired cpustat field
168 * @p: the process that the CPU time gets accounted to
169 * @cputime: the CPU time spent in kernel space since the last update
170 * @index: pointer to cpustat field that has to be updated
171 */
172void account_system_index_time(struct task_struct *p,
173 u64 cputime, enum cpu_usage_stat index)
174{
175 /* Add system time to process. */
176 p->stime += cputime;
177 account_group_system_time(p, cputime);
178
179 /* Add system time to cpustat. */
180 task_group_account_field(p, index, cputime);
181
182 /* Account for system time used */
183 acct_account_cputime(p);
184}
185
186/*
187 * Account system CPU time to a process.
188 * @p: the process that the CPU time gets accounted to
189 * @hardirq_offset: the offset to subtract from hardirq_count()
190 * @cputime: the CPU time spent in kernel space since the last update
191 */
192void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
193{
194 int index;
195
196 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
197 account_guest_time(p, cputime);
198 return;
199 }
200
201 if (hardirq_count() - hardirq_offset)
202 index = CPUTIME_IRQ;
203 else if (in_serving_softirq())
204 index = CPUTIME_SOFTIRQ;
205 else
206 index = CPUTIME_SYSTEM;
207
208 account_system_index_time(p, cputime, index);
209}
210
211/*
212 * Account for involuntary wait time.
213 * @cputime: the CPU time spent in involuntary wait
214 */
215void account_steal_time(u64 cputime)
216{
217 u64 *cpustat = kcpustat_this_cpu->cpustat;
218
219 cpustat[CPUTIME_STEAL] += cputime;
220}
221
222/*
223 * Account for idle time.
224 * @cputime: the CPU time spent in idle wait
225 */
226void account_idle_time(u64 cputime)
227{
228 u64 *cpustat = kcpustat_this_cpu->cpustat;
229
230 if (cpu_has_iowait(smp_processor_id()))
231 cpustat[CPUTIME_IOWAIT] += cputime;
232 else
233 cpustat[CPUTIME_IDLE] += cputime;
234}
235
236
237#ifdef CONFIG_SCHED_CORE
238/*
239 * Account for forceidle time due to core scheduling.
240 *
241 * REQUIRES: schedstat is enabled.
242 */
243void __account_forceidle_time(struct task_struct *p, u64 delta)
244{
245 __schedstat_add(p->stats.core_forceidle_sum, delta);
246
247 task_group_account_field(p, CPUTIME_FORCEIDLE, delta);
248}
249#endif /* CONFIG_SCHED_CORE */
250
251/*
252 * When a guest is interrupted for a longer amount of time, missed clock
253 * ticks are not redelivered later. Due to that, this function may on
254 * occasion account more time than the calling functions think elapsed.
255 */
256#ifdef CONFIG_PARAVIRT
257DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled);
258
259#ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN
260static u64 native_steal_clock(int cpu)
261{
262 return 0;
263}
264
265DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
266#endif
267#endif
268
269static __always_inline u64 steal_account_process_time(u64 maxtime)
270{
271#ifdef CONFIG_PARAVIRT
272 if (static_branch_unlikely(&paravirt_steal_enabled)) {
273 u64 steal;
274
275 steal = paravirt_steal_clock(smp_processor_id());
276 steal -= this_rq()->prev_steal_time;
277 steal = min(steal, maxtime);
278 account_steal_time(steal);
279 this_rq()->prev_steal_time += steal;
280
281 return steal;
282 }
283#endif /* CONFIG_PARAVIRT */
284 return 0;
285}
286
287/*
288 * Account how much elapsed time was spent in steal, IRQ, or softirq time.
289 */
290static inline u64 account_other_time(u64 max)
291{
292 u64 accounted;
293
294 lockdep_assert_irqs_disabled();
295
296 accounted = steal_account_process_time(max);
297
298 if (accounted < max)
299 accounted += irqtime_tick_accounted(max - accounted);
300
301 return accounted;
302}
303
304#ifdef CONFIG_64BIT
305static inline u64 read_sum_exec_runtime(struct task_struct *t)
306{
307 return t->se.sum_exec_runtime;
308}
309#else /* !CONFIG_64BIT: */
310static u64 read_sum_exec_runtime(struct task_struct *t)
311{
312 u64 ns;
313 struct rq_flags rf;
314 struct rq *rq;
315
316 rq = task_rq_lock(t, &rf);
317 ns = t->se.sum_exec_runtime;
318 task_rq_unlock(rq, t, &rf);
319
320 return ns;
321}
322#endif /* !CONFIG_64BIT */
323
324/*
325 * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
326 * tasks (sum on group iteration) belonging to @tsk's group.
327 */
328void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
329{
330 struct signal_struct *sig = tsk->signal;
331 struct task_struct *t;
332 u64 utime, stime;
333
334 /*
335 * Update current task runtime to account pending time since last
336 * scheduler action or thread_group_cputime() call. This thread group
337 * might have other running tasks on different CPUs, but updating
338 * their runtime can affect syscall performance, so we skip account
339 * those pending times and rely only on values updated on tick or
340 * other scheduler action.
341 */
342 if (same_thread_group(current, tsk))
343 (void) task_sched_runtime(current);
344
345 guard(rcu)();
346 scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) {
347 times->utime = sig->utime;
348 times->stime = sig->stime;
349 times->sum_exec_runtime = sig->sum_sched_runtime;
350
351 __for_each_thread(sig, t) {
352 task_cputime(t, &utime, &stime);
353 times->utime += utime;
354 times->stime += stime;
355 times->sum_exec_runtime += read_sum_exec_runtime(t);
356 }
357 }
358}
359
360#ifdef CONFIG_IRQ_TIME_ACCOUNTING
361/*
362 * Account a tick to a process and cpustat
363 * @p: the process that the CPU time gets accounted to
364 * @user_tick: is the tick from userspace
365 * @rq: the pointer to rq
366 *
367 * Tick demultiplexing follows the order
368 * - pending hardirq update
369 * - pending softirq update
370 * - user_time
371 * - idle_time
372 * - system time
373 * - check for guest_time
374 * - else account as system_time
375 *
376 * Check for hardirq is done both for system and user time as there is
377 * no timer going off while we are on hardirq and hence we may never get an
378 * opportunity to update it solely in system time.
379 * p->stime and friends are only updated on system time and not on IRQ
380 * softirq as those do not count in task exec_runtime any more.
381 */
382static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
383 u64 cputime)
384{
385 u64 other;
386
387 /*
388 * When returning from idle, many ticks can get accounted at
389 * once, including some ticks of steal, IRQ, and softirq time.
390 * Subtract those ticks from the amount of time accounted to
391 * idle, or potentially user or system time. Due to rounding,
392 * other time can exceed ticks occasionally.
393 */
394 other = account_other_time(ULONG_MAX);
395 if (other >= cputime)
396 return;
397
398 cputime -= other;
399
400 if (this_cpu_ksoftirqd() == p) {
401 /*
402 * ksoftirqd time do not get accounted in cpu_softirq_time.
403 * So, we have to handle it separately here.
404 * Also, p->stime needs to be updated for ksoftirqd.
405 */
406 account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
407 } else if (user_tick) {
408 account_user_time(p, cputime);
409 } else if (p == this_rq()->idle) {
410 account_idle_time(cputime);
411 } else if (p->flags & PF_VCPU) { /* System time or guest time */
412 account_guest_time(p, cputime);
413 } else {
414 account_system_index_time(p, cputime, CPUTIME_SYSTEM);
415 }
416}
417
418#else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
419static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
420 u64 cputime) { }
421#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
422
423#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE)
424static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now)
425{
426 u64 *cpustat = kc->cpustat;
427 u64 delta, steal, steal_delta;
428 int iowait;
429
430 if (!kc->idle_elapse)
431 return;
432
433 iowait = cpu_has_iowait(smp_processor_id());
434 delta = now - kc->idle_entrytime;
435 steal = steal_account_process_time(delta);
436
437 /*
438 * Record the idle time after substracting the steal time from
439 * previous update sequence. Don't substract the steal time from
440 * the current update sequence to avoid readers moving backward.
441 */
442 write_seqcount_begin(&kc->idle_sleeptime_seq);
443 steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta);
444 delta -= steal_delta;
445 kc->idle_stealtime[iowait] -= steal_delta;
446
447 if (iowait)
448 cpustat[CPUTIME_IOWAIT] += delta;
449 else
450 cpustat[CPUTIME_IDLE] += delta;
451
452 kc->idle_stealtime[iowait] += steal;
453 kc->idle_entrytime = now;
454 kc->idle_elapse = false;
455 write_seqcount_end(&kc->idle_sleeptime_seq);
456}
457
458static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now)
459{
460 /* Irqtime accounting might have been enabled in the middle of the IRQ */
461 if (kc->idle_elapse)
462 return;
463
464 write_seqcount_begin(&kc->idle_sleeptime_seq);
465 kc->idle_entrytime = now;
466 kc->idle_elapse = true;
467 write_seqcount_end(&kc->idle_sleeptime_seq);
468}
469
470void kcpustat_dyntick_stop(u64 now, u64 tick_start)
471{
472 struct kernel_cpustat *kc = kcpustat_this_cpu;
473
474 if (!vtime_generic_enabled_this_cpu()) {
475 WARN_ON_ONCE(!kc->idle_dyntick);
476 if (kc->idle_tick_period != tick_start) {
477 kc->idle_tick_period = tick_start;
478 kc->idle_tick_overlap = 0;
479 }
480 tick_start = max(tick_start, kc->idle_dyntick_entry);
481 if (!WARN_ON_ONCE(now < tick_start))
482 kc->idle_tick_overlap += now - tick_start;
483 kcpustat_idle_stop(kc, now);
484 kc->idle_dyntick = false;
485 vtime_dyntick_stop();
486 }
487}
488
489/*
490 * The first tick after dyntick-idle covers a period that the dyntick-idle
491 * accounting may already have accounted in part.
492 */
493static inline u64 kcpustat_tick_overlap(void)
494{
495 struct kernel_cpustat *kc = kcpustat_this_cpu;
496 u64 overlap = kc->idle_tick_overlap;
497
498 kc->idle_tick_overlap = 0;
499 return overlap;
500}
501
502void kcpustat_dyntick_start(u64 now)
503{
504 struct kernel_cpustat *kc = kcpustat_this_cpu;
505
506 if (!vtime_generic_enabled_this_cpu()) {
507 vtime_dyntick_start();
508 kc->idle_dyntick = true;
509 kc->idle_dyntick_entry = now;
510 kcpustat_idle_start(kc, now);
511 }
512}
513
514void kcpustat_irq_enter(u64 now)
515{
516 struct kernel_cpustat *kc = kcpustat_this_cpu;
517
518 if (!vtime_generic_enabled_this_cpu() &&
519 (irqtime_enabled() || vtime_accounting_enabled_this_cpu()))
520 kcpustat_idle_stop(kc, now);
521}
522
523void kcpustat_irq_exit(u64 now)
524{
525 struct kernel_cpustat *kc = kcpustat_this_cpu;
526
527 /*
528 * Generic vtime already does its own idle accounting.
529 * But irqtime accounting or arch vtime which also accounts IRQs
530 * need to pause nohz accounting. Resume nohz accounting as long
531 * as the irqtime config is enabled to handle case where irqtime
532 * accounting got runtime disabled in the middle of an IRQ.
533 */
534 if (!vtime_generic_enabled_this_cpu() &&
535 (IS_ENABLED(CONFIG_IRQ_TIME_ACCOUNTING) || vtime_accounting_enabled_this_cpu()))
536 kcpustat_idle_start(kc, now);
537}
538
539static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
540 bool compute_delta, u64 now)
541{
542 struct kernel_cpustat *kc = &kcpustat_cpu(cpu);
543 int iowait = idx == CPUTIME_IOWAIT;
544 u64 *cpustat = kc->cpustat;
545 unsigned int seq;
546 u64 idle;
547
548 do {
549 seq = read_seqcount_begin(&kc->idle_sleeptime_seq);
550
551 idle = cpustat[idx];
552
553 if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) {
554 u64 delta = now - kc->idle_entrytime;
555
556 delta -= min_t(u64, kc->idle_stealtime[iowait], delta);
557 idle += delta;
558 }
559 } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq));
560
561 return idle;
562}
563
564u64 kcpustat_field_idle(int cpu)
565{
566 return kcpustat_field_dyntick(cpu, CPUTIME_IDLE,
567 !cpu_has_iowait(cpu), ktime_get());
568}
569EXPORT_SYMBOL_GPL(kcpustat_field_idle);
570
571u64 kcpustat_field_iowait(int cpu)
572{
573 return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT,
574 cpu_has_iowait(cpu), ktime_get());
575}
576EXPORT_SYMBOL_GPL(kcpustat_field_iowait);
577#else
578static inline u64 kcpustat_tick_overlap(void)
579{
580 return 0;
581}
582
583static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
584 bool compute_delta, ktime_t now)
585{
586 return kcpustat_cpu(cpu).cpustat[idx];
587}
588#endif /* CONFIG_NO_HZ_COMMON && !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */
589
590static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx,
591 bool compute_delta, u64 *last_update_time)
592{
593 ktime_t now = ktime_get();
594 u64 res;
595
596 if (vtime_generic_enabled_cpu(cpu))
597 res = kcpustat_field(idx, cpu);
598 else
599 res = kcpustat_field_dyntick(cpu, idx, compute_delta, now);
600
601 do_div(res, NSEC_PER_USEC);
602
603 if (last_update_time)
604 *last_update_time = ktime_to_us(now);
605
606 return res;
607}
608
609/**
610 * get_cpu_idle_time_us - get the total idle time of a CPU
611 * @cpu: CPU number to query
612 * @last_update_time: variable to store update time in. Do not update
613 * counters if NULL.
614 *
615 * Return the cumulative idle time (since boot) for a given
616 * CPU, in microseconds. Note that this is partially broken due to
617 * the counter of iowait tasks that can be remotely updated without
618 * any synchronization. Therefore it is possible to observe backward
619 * values within two consecutive reads.
620 *
621 * This time is measured via accounting rather than sampling,
622 * and is as accurate as ktime_get() is.
623 *
624 * Return: total idle time of the @cpu
625 */
626u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
627{
628 return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE,
629 !cpu_has_iowait(cpu), last_update_time);
630}
631EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
632
633/**
634 * get_cpu_iowait_time_us - get the total iowait time of a CPU
635 * @cpu: CPU number to query
636 * @last_update_time: variable to store update time in. Do not update
637 * counters if NULL.
638 *
639 * Return the cumulative iowait time (since boot) for a given
640 * CPU, in microseconds. Note this is partially broken due to
641 * the counter of iowait tasks that can be remotely updated without
642 * any synchronization. Therefore it is possible to observe backward
643 * values within two consecutive reads.
644 *
645 * This time is measured via accounting rather than sampling,
646 * and is as accurate as ktime_get() is.
647 *
648 * Return: total iowait time of @cpu
649 */
650u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
651{
652 return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT,
653 cpu_has_iowait(cpu), last_update_time);
654}
655EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
656
657/*
658 * Use precise platform statistics if available:
659 */
660#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
661
662void vtime_account_irq(struct task_struct *tsk, unsigned int offset)
663{
664 unsigned int pc = irq_count() - offset;
665
666 if (pc & HARDIRQ_OFFSET) {
667 vtime_account_hardirq(tsk);
668 } else if (pc & SOFTIRQ_OFFSET) {
669 vtime_account_softirq(tsk);
670 } else if (!kcpustat_idle_dyntick()) {
671 if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
672 is_idle_task(tsk)) {
673 vtime_account_idle(tsk);
674 } else {
675 vtime_account_kernel(tsk);
676 }
677 } else {
678 vtime_reset();
679 }
680}
681
682void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
683 u64 *ut, u64 *st)
684{
685 *ut = curr->utime;
686 *st = curr->stime;
687}
688
689void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
690{
691 *ut = p->utime;
692 *st = p->stime;
693}
694EXPORT_SYMBOL_GPL(task_cputime_adjusted);
695
696void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
697{
698 struct task_cputime cputime;
699
700 thread_group_cputime(p, &cputime);
701
702 *ut = cputime.utime;
703 *st = cputime.stime;
704}
705
706#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
707
708/*
709 * Account a single tick of CPU time.
710 * @p: the process that the CPU time gets accounted to
711 * @user_tick: indicates if the tick is a user or a system tick
712 */
713void account_process_tick(struct task_struct *p, int user_tick)
714{
715 u64 cputime, steal;
716
717 if (vtime_accounting_enabled_this_cpu())
718 return;
719
720 if (kcpustat_idle_dyntick())
721 return;
722
723 cputime = TICK_NSEC - kcpustat_tick_overlap();
724
725 if (irqtime_enabled()) {
726 irqtime_account_process_tick(p, user_tick, cputime);
727 return;
728 }
729
730 steal = steal_account_process_time(ULONG_MAX);
731
732 if (steal >= cputime)
733 return;
734
735 cputime -= steal;
736
737 if (user_tick)
738 account_user_time(p, cputime);
739 else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))
740 account_system_time(p, HARDIRQ_OFFSET, cputime);
741 else
742 account_idle_time(cputime);
743}
744
745/*
746 * Adjust tick based cputime random precision against scheduler runtime
747 * accounting.
748 *
749 * Tick based cputime accounting depend on random scheduling timeslices of a
750 * task to be interrupted or not by the timer. Depending on these
751 * circumstances, the number of these interrupts may be over or
752 * under-optimistic, matching the real user and system cputime with a variable
753 * precision.
754 *
755 * Fix this by scaling these tick based values against the total runtime
756 * accounted by the CFS scheduler.
757 *
758 * This code provides the following guarantees:
759 *
760 * stime + utime == rtime
761 * stime_i+1 >= stime_i, utime_i+1 >= utime_i
762 *
763 * Assuming that rtime_i+1 >= rtime_i.
764 */
765void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
766 u64 *ut, u64 *st)
767{
768 u64 rtime, stime, utime;
769 unsigned long flags;
770
771 /* Serialize concurrent callers such that we can honour our guarantees */
772 raw_spin_lock_irqsave(&prev->lock, flags);
773 rtime = curr->sum_exec_runtime;
774
775 /*
776 * This is possible under two circumstances:
777 * - rtime isn't monotonic after all (a bug);
778 * - we got reordered by the lock.
779 *
780 * In both cases this acts as a filter such that the rest of the code
781 * can assume it is monotonic regardless of anything else.
782 */
783 if (prev->stime + prev->utime >= rtime)
784 goto out;
785
786 stime = curr->stime;
787 utime = curr->utime;
788
789 /*
790 * If either stime or utime are 0, assume all runtime is userspace.
791 * Once a task gets some ticks, the monotonicity code at 'update:'
792 * will ensure things converge to the observed ratio.
793 */
794 if (stime == 0) {
795 utime = rtime;
796 goto update;
797 }
798
799 if (utime == 0) {
800 stime = rtime;
801 goto update;
802 }
803
804 stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
805
806update:
807 /*
808 * Make sure stime doesn't go backwards; this preserves monotonicity
809 * for utime because rtime is monotonic.
810 *
811 * utime_i+1 = rtime_i+1 - stime_i
812 * = rtime_i+1 - (rtime_i - utime_i)
813 * = (rtime_i+1 - rtime_i) + utime_i
814 * >= utime_i
815 */
816 if (stime < prev->stime)
817 stime = prev->stime;
818 utime = rtime - stime;
819
820 /*
821 * Make sure utime doesn't go backwards; this still preserves
822 * monotonicity for stime, analogous argument to above.
823 */
824 if (utime < prev->utime) {
825 utime = prev->utime;
826 stime = rtime - utime;
827 }
828
829 prev->stime = stime;
830 prev->utime = utime;
831out:
832 *ut = prev->utime;
833 *st = prev->stime;
834 raw_spin_unlock_irqrestore(&prev->lock, flags);
835}
836
837void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
838{
839 struct task_cputime cputime = {
840 .sum_exec_runtime = p->se.sum_exec_runtime,
841 };
842
843 if (task_cputime(p, &cputime.utime, &cputime.stime))
844 cputime.sum_exec_runtime = task_sched_runtime(p);
845 cputime_adjust(&cputime, &p->prev_cputime, ut, st);
846}
847EXPORT_SYMBOL_GPL(task_cputime_adjusted);
848
849void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
850{
851 struct task_cputime cputime;
852
853 thread_group_cputime(p, &cputime);
854 cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
855}
856#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
857
858#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
859static u64 vtime_delta(struct vtime *vtime)
860{
861 unsigned long long clock;
862
863 clock = sched_clock();
864 if (clock < vtime->starttime)
865 return 0;
866
867 return clock - vtime->starttime;
868}
869
870static u64 get_vtime_delta(struct vtime *vtime)
871{
872 u64 delta = vtime_delta(vtime);
873 u64 other;
874
875 /*
876 * Unlike tick based timing, vtime based timing never has lost
877 * ticks, and no need for steal time accounting to make up for
878 * lost ticks. Vtime accounts a rounded version of actual
879 * elapsed time. Limit account_other_time to prevent rounding
880 * errors from causing elapsed vtime to go negative.
881 */
882 other = account_other_time(delta);
883 WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
884 vtime->starttime += delta;
885
886 return delta - other;
887}
888
889static void vtime_account_system(struct task_struct *tsk,
890 struct vtime *vtime)
891{
892 vtime->stime += get_vtime_delta(vtime);
893 if (vtime->stime >= TICK_NSEC) {
894 account_system_time(tsk, irq_count(), vtime->stime);
895 vtime->stime = 0;
896 }
897}
898
899static void vtime_account_guest(struct task_struct *tsk,
900 struct vtime *vtime)
901{
902 vtime->gtime += get_vtime_delta(vtime);
903 if (vtime->gtime >= TICK_NSEC) {
904 account_guest_time(tsk, vtime->gtime);
905 vtime->gtime = 0;
906 }
907}
908
909static void __vtime_account_kernel(struct task_struct *tsk,
910 struct vtime *vtime)
911{
912 /* We might have scheduled out from guest path */
913 if (vtime->state == VTIME_GUEST)
914 vtime_account_guest(tsk, vtime);
915 else
916 vtime_account_system(tsk, vtime);
917}
918
919void vtime_account_kernel(struct task_struct *tsk)
920{
921 struct vtime *vtime = &tsk->vtime;
922
923 if (!vtime_delta(vtime))
924 return;
925
926 write_seqcount_begin(&vtime->seqcount);
927 __vtime_account_kernel(tsk, vtime);
928 write_seqcount_end(&vtime->seqcount);
929}
930
931void vtime_user_enter(struct task_struct *tsk)
932{
933 struct vtime *vtime = &tsk->vtime;
934
935 write_seqcount_begin(&vtime->seqcount);
936 vtime_account_system(tsk, vtime);
937 vtime->state = VTIME_USER;
938 write_seqcount_end(&vtime->seqcount);
939}
940
941void vtime_user_exit(struct task_struct *tsk)
942{
943 struct vtime *vtime = &tsk->vtime;
944
945 write_seqcount_begin(&vtime->seqcount);
946 vtime->utime += get_vtime_delta(vtime);
947 if (vtime->utime >= TICK_NSEC) {
948 account_user_time(tsk, vtime->utime);
949 vtime->utime = 0;
950 }
951 vtime->state = VTIME_SYS;
952 write_seqcount_end(&vtime->seqcount);
953}
954
955void vtime_guest_enter(struct task_struct *tsk)
956{
957 struct vtime *vtime = &tsk->vtime;
958 /*
959 * The flags must be updated under the lock with
960 * the vtime_starttime flush and update.
961 * That enforces a right ordering and update sequence
962 * synchronization against the reader (task_gtime())
963 * that can thus safely catch up with a tickless delta.
964 */
965 write_seqcount_begin(&vtime->seqcount);
966 vtime_account_system(tsk, vtime);
967 tsk->flags |= PF_VCPU;
968 vtime->state = VTIME_GUEST;
969 write_seqcount_end(&vtime->seqcount);
970}
971EXPORT_SYMBOL_GPL(vtime_guest_enter);
972
973void vtime_guest_exit(struct task_struct *tsk)
974{
975 struct vtime *vtime = &tsk->vtime;
976
977 write_seqcount_begin(&vtime->seqcount);
978 vtime_account_guest(tsk, vtime);
979 tsk->flags &= ~PF_VCPU;
980 vtime->state = VTIME_SYS;
981 write_seqcount_end(&vtime->seqcount);
982}
983EXPORT_SYMBOL_GPL(vtime_guest_exit);
984
985static void __vtime_account_idle(struct vtime *vtime)
986{
987 account_idle_time(get_vtime_delta(vtime));
988}
989
990void vtime_task_switch_generic(struct task_struct *prev)
991{
992 struct vtime *vtime = &prev->vtime;
993
994 write_seqcount_begin(&vtime->seqcount);
995 if (vtime->state == VTIME_IDLE)
996 __vtime_account_idle(vtime);
997 else
998 __vtime_account_kernel(prev, vtime);
999 vtime->state = VTIME_INACTIVE;
1000 vtime->cpu = -1;
1001 write_seqcount_end(&vtime->seqcount);
1002
1003 vtime = &current->vtime;
1004
1005 write_seqcount_begin(&vtime->seqcount);
1006 if (is_idle_task(current))
1007 vtime->state = VTIME_IDLE;
1008 else if (current->flags & PF_VCPU)
1009 vtime->state = VTIME_GUEST;
1010 else
1011 vtime->state = VTIME_SYS;
1012 vtime->starttime = sched_clock();
1013 vtime->cpu = smp_processor_id();
1014 write_seqcount_end(&vtime->seqcount);
1015}
1016
1017void vtime_init_idle(struct task_struct *t, int cpu)
1018{
1019 struct vtime *vtime = &t->vtime;
1020 unsigned long flags;
1021
1022 local_irq_save(flags);
1023 write_seqcount_begin(&vtime->seqcount);
1024 vtime->state = VTIME_IDLE;
1025 vtime->starttime = sched_clock();
1026 vtime->cpu = cpu;
1027 write_seqcount_end(&vtime->seqcount);
1028 local_irq_restore(flags);
1029}
1030
1031u64 task_gtime(struct task_struct *t)
1032{
1033 struct vtime *vtime = &t->vtime;
1034 unsigned int seq;
1035 u64 gtime;
1036
1037 if (!vtime_accounting_enabled())
1038 return t->gtime;
1039
1040 do {
1041 seq = read_seqcount_begin(&vtime->seqcount);
1042
1043 gtime = t->gtime;
1044 if (vtime->state == VTIME_GUEST)
1045 gtime += vtime->gtime + vtime_delta(vtime);
1046
1047 } while (read_seqcount_retry(&vtime->seqcount, seq));
1048
1049 return gtime;
1050}
1051
1052/*
1053 * Fetch cputime raw values from fields of task_struct and
1054 * add up the pending nohz execution time since the last
1055 * cputime snapshot.
1056 */
1057bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
1058{
1059 struct vtime *vtime = &t->vtime;
1060 unsigned int seq;
1061 u64 delta;
1062 int ret;
1063
1064 if (!vtime_accounting_enabled()) {
1065 *utime = t->utime;
1066 *stime = t->stime;
1067 return false;
1068 }
1069
1070 do {
1071 ret = false;
1072 seq = read_seqcount_begin(&vtime->seqcount);
1073
1074 *utime = t->utime;
1075 *stime = t->stime;
1076
1077 /* Task is sleeping or idle, nothing to add */
1078 if (vtime->state < VTIME_SYS)
1079 continue;
1080
1081 ret = true;
1082 delta = vtime_delta(vtime);
1083
1084 /*
1085 * Task runs either in user (including guest) or kernel space,
1086 * add pending nohz time to the right place.
1087 */
1088 if (vtime->state == VTIME_SYS)
1089 *stime += vtime->stime + delta;
1090 else
1091 *utime += vtime->utime + delta;
1092 } while (read_seqcount_retry(&vtime->seqcount, seq));
1093
1094 return ret;
1095}
1096
1097static int vtime_state_fetch(struct vtime *vtime, int cpu)
1098{
1099 int state = READ_ONCE(vtime->state);
1100
1101 /*
1102 * We raced against a context switch, fetch the
1103 * kcpustat task again.
1104 */
1105 if (vtime->cpu != cpu && vtime->cpu != -1)
1106 return -EAGAIN;
1107
1108 /*
1109 * Two possible things here:
1110 * 1) We are seeing the scheduling out task (prev) or any past one.
1111 * 2) We are seeing the scheduling in task (next) but it hasn't
1112 * passed though vtime_task_switch() yet so the pending
1113 * cputime of the prev task may not be flushed yet.
1114 *
1115 * Case 1) is ok but 2) is not. So wait for a safe VTIME state.
1116 */
1117 if (state == VTIME_INACTIVE)
1118 return -EAGAIN;
1119
1120 return state;
1121}
1122
1123static u64 kcpustat_user_vtime(struct vtime *vtime)
1124{
1125 if (vtime->state == VTIME_USER)
1126 return vtime->utime + vtime_delta(vtime);
1127 else if (vtime->state == VTIME_GUEST)
1128 return vtime->gtime + vtime_delta(vtime);
1129 return 0;
1130}
1131
1132static int kcpustat_field_vtime(u64 *cpustat,
1133 struct task_struct *tsk,
1134 enum cpu_usage_stat usage,
1135 int cpu, u64 *val)
1136{
1137 struct vtime *vtime = &tsk->vtime;
1138 unsigned int seq;
1139
1140 do {
1141 int state;
1142
1143 seq = read_seqcount_begin(&vtime->seqcount);
1144
1145 state = vtime_state_fetch(vtime, cpu);
1146 if (state < 0)
1147 return state;
1148
1149 *val = cpustat[usage];
1150
1151 /*
1152 * Nice VS unnice cputime accounting may be inaccurate if
1153 * the nice value has changed since the last vtime update.
1154 * But proper fix would involve interrupting target on nice
1155 * updates which is a no go on nohz_full (although the scheduler
1156 * may still interrupt the target if rescheduling is needed...)
1157 */
1158 switch (usage) {
1159 case CPUTIME_SYSTEM:
1160 if (state == VTIME_SYS)
1161 *val += vtime->stime + vtime_delta(vtime);
1162 break;
1163 case CPUTIME_USER:
1164 if (task_nice(tsk) <= 0)
1165 *val += kcpustat_user_vtime(vtime);
1166 break;
1167 case CPUTIME_NICE:
1168 if (task_nice(tsk) > 0)
1169 *val += kcpustat_user_vtime(vtime);
1170 break;
1171 case CPUTIME_GUEST:
1172 if (state == VTIME_GUEST && task_nice(tsk) <= 0)
1173 *val += vtime->gtime + vtime_delta(vtime);
1174 break;
1175 case CPUTIME_GUEST_NICE:
1176 if (state == VTIME_GUEST && task_nice(tsk) > 0)
1177 *val += vtime->gtime + vtime_delta(vtime);
1178 break;
1179 case CPUTIME_IDLE:
1180 if (state == VTIME_IDLE && !cpu_has_iowait(cpu))
1181 *val += vtime_delta(vtime);
1182 break;
1183 case CPUTIME_IOWAIT:
1184 if (state == VTIME_IDLE && cpu_has_iowait(cpu))
1185 *val += vtime_delta(vtime);
1186 break;
1187 default:
1188 break;
1189 }
1190 } while (read_seqcount_retry(&vtime->seqcount, seq));
1191
1192 return 0;
1193}
1194
1195u64 kcpustat_field(enum cpu_usage_stat usage, int cpu)
1196{
1197 u64 *cpustat = kcpustat_cpu(cpu).cpustat;
1198 u64 val = cpustat[usage];
1199 struct rq *rq;
1200 int err;
1201
1202 if (!vtime_generic_enabled_cpu(cpu))
1203 return kcpustat_field_default(usage, cpu);
1204
1205 rq = cpu_rq(cpu);
1206
1207 for (;;) {
1208 struct task_struct *curr;
1209
1210 rcu_read_lock();
1211 curr = rcu_dereference(rq->curr);
1212 if (WARN_ON_ONCE(!curr)) {
1213 rcu_read_unlock();
1214 return cpustat[usage];
1215 }
1216
1217 err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val);
1218 rcu_read_unlock();
1219
1220 if (!err)
1221 return val;
1222
1223 cpu_relax();
1224 }
1225}
1226EXPORT_SYMBOL_GPL(kcpustat_field);
1227
1228static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
1229 const struct kernel_cpustat *src,
1230 struct task_struct *tsk, int cpu)
1231{
1232 struct vtime *vtime = &tsk->vtime;
1233 unsigned int seq;
1234
1235 do {
1236 u64 *cpustat;
1237 u64 delta;
1238 int state;
1239
1240 seq = read_seqcount_begin(&vtime->seqcount);
1241
1242 state = vtime_state_fetch(vtime, cpu);
1243 if (state < 0)
1244 return state;
1245
1246 *dst = *src;
1247 cpustat = dst->cpustat;
1248
1249 /* Task is sleeping or dead, nothing to add */
1250 if (state < VTIME_IDLE)
1251 continue;
1252
1253 delta = vtime_delta(vtime);
1254
1255 /*
1256 * Task runs either in user (including guest) or kernel space,
1257 * add pending nohz time to the right place.
1258 */
1259 switch (state) {
1260 case VTIME_SYS:
1261 cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
1262 break;
1263 case VTIME_USER:
1264 if (task_nice(tsk) > 0)
1265 cpustat[CPUTIME_NICE] += vtime->utime + delta;
1266 else
1267 cpustat[CPUTIME_USER] += vtime->utime + delta;
1268 break;
1269 case VTIME_GUEST:
1270 if (task_nice(tsk) > 0) {
1271 cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
1272 cpustat[CPUTIME_NICE] += vtime->gtime + delta;
1273 } else {
1274 cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
1275 cpustat[CPUTIME_USER] += vtime->gtime + delta;
1276 }
1277 break;
1278 case VTIME_IDLE:
1279 if (cpu_has_iowait(cpu))
1280 cpustat[CPUTIME_IOWAIT] += delta;
1281 else
1282 cpustat[CPUTIME_IDLE] += delta;
1283 break;
1284 default:
1285 WARN_ON_ONCE(1);
1286 }
1287 } while (read_seqcount_retry(&vtime->seqcount, seq));
1288
1289 return 0;
1290}
1291
1292void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
1293{
1294 const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
1295 struct rq *rq;
1296 int err;
1297
1298 if (!vtime_generic_enabled_cpu(cpu)) {
1299 kcpustat_cpu_fetch_default(dst, cpu);
1300 return;
1301 }
1302
1303 rq = cpu_rq(cpu);
1304
1305 for (;;) {
1306 struct task_struct *curr;
1307
1308 rcu_read_lock();
1309 curr = rcu_dereference(rq->curr);
1310 if (WARN_ON_ONCE(!curr)) {
1311 rcu_read_unlock();
1312 kcpustat_cpu_fetch_default(dst, cpu);
1313 return;
1314 }
1315
1316 err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
1317 rcu_read_unlock();
1318
1319 if (!err)
1320 return;
1321
1322 cpu_relax();
1323 }
1324}
1325EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
1326
1327#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */