projects/linux·blob master · refs

242 lines · 6.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Virtual CPU time accounting
4 *
5 * Copyright IBM Corp. 2004, 2012
6 * Author(s): Jan Glauber <[email protected]>
7 */
8
9#include <linux/kernel_stat.h>
10#include <linux/export.h>
11#include <linux/kernel.h>
12#include <linux/timex.h>
13#include <linux/types.h>
14#include <linux/time.h>
15#include <asm/alternative.h>
16#include <asm/cputime.h>
17#include <asm/vtime.h>
18#include <asm/cpu_mf.h>
19#include <asm/idle.h>
20#include <asm/smp.h>
21
22#include "entry.h"
23
24#define CPU_TIMER_MAX 0x7fffffffffffffffUL
25
26DEFINE_PER_CPU(u64, mt_cycles[8]);
27static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
28static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
29static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies);
30
31static inline void cpu_timer_init(u64 value)
32{
33 struct lowcore *lc = get_lowcore();
34 u64 timer;
35
36 asm volatile(
37 " stpt %0\n" /* Store current cpu timer value */
38 " spt %1" /* Set new value imm. afterwards */
39 : "=Q" (timer) : "Q" (value));
40 lc->system_timer += lc->last_update_timer - timer;
41 lc->last_update_timer = value;
42}
43
44static void update_mt_scaling(void)
45{
46 u64 cycles_new[8], *cycles_old;
47 u64 delta, fac, mult, div;
48 int i;
49
50 stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
51 cycles_old = this_cpu_ptr(mt_cycles);
52 fac = 1;
53 mult = div = 0;
54 for (i = 0; i <= smp_cpu_mtid; i++) {
55 delta = cycles_new[i] - cycles_old[i];
56 div += delta;
57 mult *= i + 1;
58 mult += delta * fac;
59 fac *= i + 1;
60 }
61 div *= fac;
62 if (div > 0) {
63 /* Update scaling factor */
64 __this_cpu_write(mt_scaling_mult, mult);
65 __this_cpu_write(mt_scaling_div, div);
66 memcpy(cycles_old, cycles_new,
67 sizeof(u64) * (smp_cpu_mtid + 1));
68 }
69 __this_cpu_write(mt_scaling_jiffies, jiffies);
70}
71
72static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
73{
74 u64 delta;
75
76 delta = new - *tsk_vtime;
77 *tsk_vtime = new;
78 return delta;
79}
80
81
82static inline u64 scale_vtime(u64 vtime)
83{
84 u64 mult = __this_cpu_read(mt_scaling_mult);
85 u64 div = __this_cpu_read(mt_scaling_div);
86
87 if (smp_cpu_mtid)
88 return vtime * mult / div;
89 return vtime;
90}
91
92static void account_system_index_scaled(struct task_struct *p, u64 cputime,
93 enum cpu_usage_stat index)
94{
95 p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
96 account_system_index_time(p, cputime_to_nsecs(cputime), index);
97}
98
99static inline void vtime_reset_last_update(struct lowcore *lc)
100{
101 asm volatile(
102 " stpt %0\n" /* Store current cpu timer value */
103 " stckf %1" /* Store current tod clock value */
104 : "=Q" (lc->last_update_timer),
105 "=Q" (lc->last_update_clock)
106 : : "cc");
107}
108
109/*
110 * Update process times based on virtual cpu times stored by entry.S
111 * to the lowcore fields user_timer, system_timer & steal_clock.
112 */
113static void do_account_vtime(struct task_struct *tsk)
114{
115 u64 timer, clock, user, guest, system, hardirq, softirq;
116 struct lowcore *lc = get_lowcore();
117
118 timer = lc->last_update_timer;
119 clock = lc->last_update_clock;
120
121 vtime_reset_last_update(lc);
122
123 clock = lc->last_update_clock - clock;
124 timer -= lc->last_update_timer;
125
126 if (hardirq_count())
127 lc->hardirq_timer += timer;
128 else
129 lc->system_timer += timer;
130
131 /* Update MT utilization calculation */
132 if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies)))
133 update_mt_scaling();
134
135 /* Calculate cputime delta */
136 user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer);
137 guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer);
138 system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer);
139 hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer);
140 softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer);
141 lc->steal_timer += clock - user - guest - system - hardirq - softirq;
142
143 /* Push account value */
144 if (user) {
145 account_user_time(tsk, cputime_to_nsecs(user));
146 tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
147 }
148
149 if (guest) {
150 account_guest_time(tsk, cputime_to_nsecs(guest));
151 tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
152 }
153
154 if (system)
155 account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
156 if (hardirq)
157 account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
158 if (softirq)
159 account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
160}
161
162void vtime_task_switch(struct task_struct *prev)
163{
164 struct lowcore *lc = get_lowcore();
165
166 do_account_vtime(prev);
167 prev->thread.user_timer = lc->user_timer;
168 prev->thread.guest_timer = lc->guest_timer;
169 prev->thread.system_timer = lc->system_timer;
170 prev->thread.hardirq_timer = lc->hardirq_timer;
171 prev->thread.softirq_timer = lc->softirq_timer;
172 lc->user_timer = current->thread.user_timer;
173 lc->guest_timer = current->thread.guest_timer;
174 lc->system_timer = current->thread.system_timer;
175 lc->hardirq_timer = current->thread.hardirq_timer;
176 lc->softirq_timer = current->thread.softirq_timer;
177}
178
179/*
180 * In s390, accounting pending user time also implies
181 * accounting system time in order to correctly compute
182 * the stolen time accounting.
183 */
184void vtime_flush(struct task_struct *tsk)
185{
186 struct lowcore *lc = get_lowcore();
187 u64 steal, avg_steal;
188
189 do_account_vtime(tsk);
190
191 steal = lc->steal_timer;
192 avg_steal = lc->avg_steal_timer;
193 if ((s64) steal > 0) {
194 lc->steal_timer = 0;
195 account_steal_time(cputime_to_nsecs(steal));
196 avg_steal += steal;
197 }
198 lc->avg_steal_timer = avg_steal / 2;
199}
200
201static u64 vtime_delta(void)
202{
203 struct lowcore *lc = get_lowcore();
204 u64 timer = lc->last_update_timer;
205
206 lc->last_update_timer = get_cpu_timer();
207 return timer - lc->last_update_timer;
208}
209
210void vtime_account_kernel(struct task_struct *tsk)
211{
212 struct lowcore *lc = get_lowcore();
213 u64 delta = vtime_delta();
214
215 if (tsk->flags & PF_VCPU)
216 lc->guest_timer += delta;
217 else
218 lc->system_timer += delta;
219}
220EXPORT_SYMBOL_GPL(vtime_account_kernel);
221
222void vtime_account_softirq(struct task_struct *tsk)
223{
224 get_lowcore()->softirq_timer += vtime_delta();
225}
226
227void vtime_account_hardirq(struct task_struct *tsk)
228{
229 get_lowcore()->hardirq_timer += vtime_delta();
230}
231
232void vtime_init(void)
233{
234 cpu_timer_init(CPU_TIMER_MAX);
235 /* Setup initial MT scaling values */
236 if (smp_cpu_mtid) {
237 __this_cpu_write(mt_scaling_jiffies, jiffies);
238 __this_cpu_write(mt_scaling_mult, 1);
239 __this_cpu_write(mt_scaling_div, 1);
240 stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
241 }
242}