projects/linux·blob master · refs

386 lines · 9.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
4 *
5 * Authors:
6 * Atish Patra <[email protected]>
7 */
8
9#include <linux/errno.h>
10#include <linux/err.h>
11#include <linux/kvm_host.h>
12#include <linux/uaccess.h>
13#include <clocksource/timer-riscv.h>
14#include <asm/delay.h>
15#include <asm/kvm_isa.h>
16#include <asm/kvm_nacl.h>
17#include <asm/kvm_vcpu_timer.h>
18
19static u64 kvm_riscv_current_cycles(struct kvm_guest_timer *gt)
20{
21 return get_cycles64() + gt->time_delta;
22}
23
24static u64 kvm_riscv_delta_cycles2ns(u64 cycles,
25 struct kvm_guest_timer *gt,
26 struct kvm_vcpu_timer *t)
27{
28 unsigned long flags;
29 u64 cycles_now, cycles_delta, delta_ns;
30
31 local_irq_save(flags);
32 cycles_now = kvm_riscv_current_cycles(gt);
33 if (cycles_now < cycles)
34 cycles_delta = cycles - cycles_now;
35 else
36 cycles_delta = 0;
37 delta_ns = (cycles_delta * gt->nsec_mult) >> gt->nsec_shift;
38 local_irq_restore(flags);
39
40 return delta_ns;
41}
42
43static enum hrtimer_restart kvm_riscv_vcpu_hrtimer_expired(struct hrtimer *h)
44{
45 u64 delta_ns;
46 struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt);
47 struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer);
48 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
49
50 if (kvm_riscv_current_cycles(gt) < t->next_cycles) {
51 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
52 hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns));
53 return HRTIMER_RESTART;
54 }
55
56 t->next_set = false;
57 kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_TIMER);
58
59 return HRTIMER_NORESTART;
60}
61
62static int kvm_riscv_vcpu_timer_cancel(struct kvm_vcpu_timer *t)
63{
64 if (!t->init_done)
65 return -EINVAL;
66
67 hrtimer_cancel(&t->hrt);
68
69 if (!t->next_set)
70 return -EINVAL;
71 t->next_set = false;
72
73 return 0;
74}
75
76static int kvm_riscv_vcpu_update_vstimecmp(struct kvm_vcpu *vcpu, u64 ncycles)
77{
78#if defined(CONFIG_32BIT)
79 ncsr_write(CSR_VSTIMECMP, ULONG_MAX);
80 ncsr_write(CSR_VSTIMECMPH, ncycles >> 32);
81 ncsr_write(CSR_VSTIMECMP, (u32)ncycles);
82#else
83 ncsr_write(CSR_VSTIMECMP, ncycles);
84#endif
85 return 0;
86}
87
88static int kvm_riscv_vcpu_update_hrtimer(struct kvm_vcpu *vcpu, u64 ncycles)
89{
90 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
91 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
92 u64 delta_ns;
93
94 if (!t->init_done)
95 return -EINVAL;
96
97 kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_TIMER);
98
99 delta_ns = kvm_riscv_delta_cycles2ns(ncycles, gt, t);
100 t->next_cycles = ncycles;
101 hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL);
102 t->next_set = true;
103
104 return 0;
105}
106
107int kvm_riscv_vcpu_timer_next_event(struct kvm_vcpu *vcpu, u64 ncycles)
108{
109 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
110
111 return t->timer_next_event(vcpu, ncycles);
112}
113
114static enum hrtimer_restart kvm_riscv_vcpu_vstimer_expired(struct hrtimer *h)
115{
116 u64 delta_ns;
117 struct kvm_vcpu_timer *t = container_of(h, struct kvm_vcpu_timer, hrt);
118 struct kvm_vcpu *vcpu = container_of(t, struct kvm_vcpu, arch.timer);
119 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
120
121 if (kvm_riscv_current_cycles(gt) < t->next_cycles) {
122 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
123 hrtimer_forward_now(&t->hrt, ktime_set(0, delta_ns));
124 return HRTIMER_RESTART;
125 }
126
127 t->next_set = false;
128 kvm_vcpu_kick(vcpu);
129
130 return HRTIMER_NORESTART;
131}
132
133bool kvm_riscv_vcpu_timer_pending(struct kvm_vcpu *vcpu)
134{
135 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
136 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
137
138 if (!kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t) ||
139 kvm_riscv_vcpu_has_interrupts(vcpu, 1UL << IRQ_VS_TIMER))
140 return true;
141 else
142 return false;
143}
144
145static void kvm_riscv_vcpu_timer_blocking(struct kvm_vcpu *vcpu)
146{
147 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
148 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
149 u64 delta_ns;
150
151 if (!t->init_done)
152 return;
153
154 delta_ns = kvm_riscv_delta_cycles2ns(t->next_cycles, gt, t);
155 hrtimer_start(&t->hrt, ktime_set(0, delta_ns), HRTIMER_MODE_REL);
156 t->next_set = true;
157}
158
159static void kvm_riscv_vcpu_timer_unblocking(struct kvm_vcpu *vcpu)
160{
161 kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
162}
163
164int kvm_riscv_vcpu_get_reg_timer(struct kvm_vcpu *vcpu,
165 const struct kvm_one_reg *reg)
166{
167 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
168 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
169 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
170 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
171 KVM_REG_SIZE_MASK |
172 KVM_REG_RISCV_TIMER);
173 u64 reg_val;
174
175 if (KVM_REG_SIZE(reg->id) != sizeof(u64))
176 return -EINVAL;
177 if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64))
178 return -ENOENT;
179
180 switch (reg_num) {
181 case KVM_REG_RISCV_TIMER_REG(frequency):
182 reg_val = riscv_timebase;
183 break;
184 case KVM_REG_RISCV_TIMER_REG(time):
185 reg_val = kvm_riscv_current_cycles(gt);
186 break;
187 case KVM_REG_RISCV_TIMER_REG(compare):
188 reg_val = t->next_cycles;
189 break;
190 case KVM_REG_RISCV_TIMER_REG(state):
191 reg_val = (t->next_set) ? KVM_RISCV_TIMER_STATE_ON :
192 KVM_RISCV_TIMER_STATE_OFF;
193 break;
194 default:
195 return -ENOENT;
196 }
197
198 if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
199 return -EFAULT;
200
201 return 0;
202}
203
204int kvm_riscv_vcpu_set_reg_timer(struct kvm_vcpu *vcpu,
205 const struct kvm_one_reg *reg)
206{
207 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
208 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
209 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
210 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
211 KVM_REG_SIZE_MASK |
212 KVM_REG_RISCV_TIMER);
213 u64 reg_val;
214 int ret = 0;
215
216 if (KVM_REG_SIZE(reg->id) != sizeof(u64))
217 return -EINVAL;
218 if (reg_num >= sizeof(struct kvm_riscv_timer) / sizeof(u64))
219 return -ENOENT;
220
221 if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
222 return -EFAULT;
223
224 switch (reg_num) {
225 case KVM_REG_RISCV_TIMER_REG(frequency):
226 if (reg_val != riscv_timebase)
227 return -EINVAL;
228 break;
229 case KVM_REG_RISCV_TIMER_REG(time):
230 gt->time_delta = reg_val - get_cycles64();
231 break;
232 case KVM_REG_RISCV_TIMER_REG(compare):
233 t->next_cycles = reg_val;
234 break;
235 case KVM_REG_RISCV_TIMER_REG(state):
236 if (reg_val == KVM_RISCV_TIMER_STATE_ON)
237 ret = kvm_riscv_vcpu_timer_next_event(vcpu, t->next_cycles);
238 else
239 ret = kvm_riscv_vcpu_timer_cancel(t);
240 break;
241 default:
242 ret = -ENOENT;
243 break;
244 }
245
246 return ret;
247}
248
249int kvm_riscv_vcpu_timer_init(struct kvm_vcpu *vcpu)
250{
251 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
252
253 if (t->init_done)
254 return -EINVAL;
255
256 t->init_done = true;
257 t->next_set = false;
258
259 /* Enable sstc for every vcpu if available in hardware */
260 if (!kvm_riscv_isa_check_host(SSTC)) {
261 t->sstc_enabled = true;
262 hrtimer_setup(&t->hrt, kvm_riscv_vcpu_vstimer_expired, CLOCK_MONOTONIC,
263 HRTIMER_MODE_REL);
264 t->timer_next_event = kvm_riscv_vcpu_update_vstimecmp;
265 } else {
266 t->sstc_enabled = false;
267 hrtimer_setup(&t->hrt, kvm_riscv_vcpu_hrtimer_expired, CLOCK_MONOTONIC,
268 HRTIMER_MODE_REL);
269 t->timer_next_event = kvm_riscv_vcpu_update_hrtimer;
270 }
271
272 return 0;
273}
274
275int kvm_riscv_vcpu_timer_deinit(struct kvm_vcpu *vcpu)
276{
277 int ret;
278
279 ret = kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
280 vcpu->arch.timer.init_done = false;
281
282 return ret;
283}
284
285int kvm_riscv_vcpu_timer_reset(struct kvm_vcpu *vcpu)
286{
287 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
288
289 t->next_cycles = -1ULL;
290 return kvm_riscv_vcpu_timer_cancel(&vcpu->arch.timer);
291}
292
293static void kvm_riscv_vcpu_update_timedelta(struct kvm_vcpu *vcpu)
294{
295 struct kvm_guest_timer *gt = &vcpu->kvm->arch.timer;
296
297#if defined(CONFIG_32BIT)
298 ncsr_write(CSR_HTIMEDELTA, (u32)(gt->time_delta));
299 ncsr_write(CSR_HTIMEDELTAH, (u32)(gt->time_delta >> 32));
300#else
301 ncsr_write(CSR_HTIMEDELTA, gt->time_delta);
302#endif
303}
304
305void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu)
306{
307 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
308
309 kvm_riscv_vcpu_update_timedelta(vcpu);
310
311 if (!t->sstc_enabled)
312 return;
313
314#if defined(CONFIG_32BIT)
315 ncsr_write(CSR_VSTIMECMP, ULONG_MAX);
316 ncsr_write(CSR_VSTIMECMPH, (u32)(t->next_cycles >> 32));
317 ncsr_write(CSR_VSTIMECMP, (u32)(t->next_cycles));
318#else
319 ncsr_write(CSR_VSTIMECMP, t->next_cycles);
320#endif
321
322 /* timer should be enabled for the remaining operations */
323 if (unlikely(!t->init_done))
324 return;
325
326 kvm_riscv_vcpu_timer_unblocking(vcpu);
327}
328
329void kvm_riscv_vcpu_timer_sync(struct kvm_vcpu *vcpu)
330{
331 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
332
333 if (!t->sstc_enabled)
334 return;
335
336#if defined(CONFIG_32BIT)
337 t->next_cycles = ncsr_read(CSR_VSTIMECMP);
338 t->next_cycles |= (u64)ncsr_read(CSR_VSTIMECMPH) << 32;
339#else
340 t->next_cycles = ncsr_read(CSR_VSTIMECMP);
341#endif
342}
343
344void kvm_riscv_vcpu_timer_save(struct kvm_vcpu *vcpu)
345{
346 struct kvm_vcpu_timer *t = &vcpu->arch.timer;
347
348 if (!t->sstc_enabled)
349 return;
350
351 /*
352 * The vstimecmp CSRs are saved by kvm_riscv_vcpu_timer_sync()
353 * upon every VM exit so no need to save here.
354 *
355 * If VS-timer expires when no VCPU running on a host CPU then
356 * WFI executed by such host CPU will be effective NOP resulting
357 * in no power savings. This is because as-per RISC-V Privileged
358 * specificaiton: "WFI is also required to resume execution for
359 * locally enabled interrupts pending at any privilege level,
360 * regardless of the global interrupt enable at each privilege
361 * level."
362 *
363 * To address the above issue, vstimecmp CSR must be set to -1UL
364 * over here when VCPU is scheduled-out or exits to user space.
365 */
366
367 csr_write(CSR_VSTIMECMP, -1UL);
368#if defined(CONFIG_32BIT)
369 csr_write(CSR_VSTIMECMPH, -1UL);
370#endif
371
372 /* timer should be enabled for the remaining operations */
373 if (unlikely(!t->init_done))
374 return;
375
376 if (kvm_vcpu_is_blocking(vcpu))
377 kvm_riscv_vcpu_timer_blocking(vcpu);
378}
379
380void kvm_riscv_guest_timer_init(struct kvm *kvm)
381{
382 struct kvm_guest_timer *gt = &kvm->arch.timer;
383
384 riscv_cs_get_mult_shift(&gt->nsec_mult, &gt->nsec_shift);
385 gt->time_delta = -get_cycles64();
386}