| 1 | |
| 2 | |
| 3 | * Copyright (C) 2012 ARM Ltd.
|
| 4 | * Author: Marc Zyngier <[email protected]>
|
| 5 |
|
| 6 |
|
| 7 | #include <linux/cpu.h>
|
| 8 | #include <linux/kvm.h>
|
| 9 | #include <linux/kvm_host.h>
|
| 10 | #include <linux/interrupt.h>
|
| 11 | #include <linux/irq.h>
|
| 12 | #include <linux/irqdomain.h>
|
| 13 | #include <linux/uaccess.h>
|
| 14 |
|
| 15 | #include <clocksource/arm_arch_timer.h>
|
| 16 | #include <asm/arch_timer.h>
|
| 17 | #include <asm/kvm_emulate.h>
|
| 18 | #include <asm/kvm_hyp.h>
|
| 19 | #include <asm/kvm_nested.h>
|
| 20 |
|
| 21 | #include <kvm/arm_vgic.h>
|
| 22 | #include <kvm/arm_arch_timer.h>
|
| 23 |
|
| 24 | #include "trace.h"
|
| 25 |
|
| 26 | static struct timecounter *timecounter;
|
| 27 | static unsigned int host_vtimer_irq;
|
| 28 | static unsigned int host_ptimer_irq;
|
| 29 | static u32 host_vtimer_irq_flags;
|
| 30 | static u32 host_ptimer_irq_flags;
|
| 31 |
|
| 32 | static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
|
| 33 | DEFINE_STATIC_KEY_FALSE(broken_cntvoff_key);
|
| 34 |
|
| 35 | static const u8 default_ppi[] = {
|
| 36 | [TIMER_PTIMER] = 30,
|
| 37 | [TIMER_VTIMER] = 27,
|
| 38 | [TIMER_HPTIMER] = 26,
|
| 39 | [TIMER_HVTIMER] = 28,
|
| 40 | };
|
| 41 |
|
| 42 | static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
|
| 43 | struct arch_timer_context *timer_ctx);
|
| 44 | static bool kvm_timer_pending(struct arch_timer_context *timer_ctx);
|
| 45 | static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
|
| 46 | struct arch_timer_context *timer,
|
| 47 | enum kvm_arch_timer_regs treg,
|
| 48 | u64 val);
|
| 49 | static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
|
| 50 | struct arch_timer_context *timer,
|
| 51 | enum kvm_arch_timer_regs treg);
|
| 52 | static bool kvm_arch_timer_get_input_level(int vintid);
|
| 53 |
|
| 54 | static unsigned long kvm_arch_timer_get_irq_flags(void)
|
| 55 | {
|
| 56 | return kvm_vgic_global_state.no_hw_deactivation ? VGIC_IRQ_SW_RESAMPLE : 0;
|
| 57 | }
|
| 58 |
|
| 59 | static const struct irq_ops arch_timer_irq_ops = {
|
| 60 | .get_flags = kvm_arch_timer_get_irq_flags,
|
| 61 | .get_input_level = kvm_arch_timer_get_input_level,
|
| 62 | };
|
| 63 |
|
| 64 | static const struct irq_ops arch_timer_irq_ops_vgic_v5 = {
|
| 65 | .get_input_level = kvm_arch_timer_get_input_level,
|
| 66 | .queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock,
|
| 67 | .set_direct_injection = vgic_v5_set_ppi_dvi,
|
| 68 | };
|
| 69 |
|
| 70 | static int nr_timers(struct kvm_vcpu *vcpu)
|
| 71 | {
|
| 72 | if (!vcpu_has_nv(vcpu))
|
| 73 | return NR_KVM_EL0_TIMERS;
|
| 74 |
|
| 75 | return NR_KVM_TIMERS;
|
| 76 | }
|
| 77 |
|
| 78 | u32 timer_get_ctl(struct arch_timer_context *ctxt)
|
| 79 | {
|
| 80 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
|
| 81 |
|
| 82 | switch(arch_timer_ctx_index(ctxt)) {
|
| 83 | case TIMER_VTIMER:
|
| 84 | return __vcpu_sys_reg(vcpu, CNTV_CTL_EL0);
|
| 85 | case TIMER_PTIMER:
|
| 86 | return __vcpu_sys_reg(vcpu, CNTP_CTL_EL0);
|
| 87 | case TIMER_HVTIMER:
|
| 88 | return __vcpu_sys_reg(vcpu, CNTHV_CTL_EL2);
|
| 89 | case TIMER_HPTIMER:
|
| 90 | return __vcpu_sys_reg(vcpu, CNTHP_CTL_EL2);
|
| 91 | default:
|
| 92 | WARN_ON(1);
|
| 93 | return 0;
|
| 94 | }
|
| 95 | }
|
| 96 |
|
| 97 | u64 timer_get_cval(struct arch_timer_context *ctxt)
|
| 98 | {
|
| 99 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
|
| 100 |
|
| 101 | switch(arch_timer_ctx_index(ctxt)) {
|
| 102 | case TIMER_VTIMER:
|
| 103 | return __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
|
| 104 | case TIMER_PTIMER:
|
| 105 | return __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
|
| 106 | case TIMER_HVTIMER:
|
| 107 | return __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2);
|
| 108 | case TIMER_HPTIMER:
|
| 109 | return __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2);
|
| 110 | default:
|
| 111 | WARN_ON(1);
|
| 112 | return 0;
|
| 113 | }
|
| 114 | }
|
| 115 |
|
| 116 | static void timer_set_ctl(struct arch_timer_context *ctxt, u32 ctl)
|
| 117 | {
|
| 118 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
|
| 119 |
|
| 120 | switch(arch_timer_ctx_index(ctxt)) {
|
| 121 | case TIMER_VTIMER:
|
| 122 | __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, ctl);
|
| 123 | break;
|
| 124 | case TIMER_PTIMER:
|
| 125 | __vcpu_assign_sys_reg(vcpu, CNTP_CTL_EL0, ctl);
|
| 126 | break;
|
| 127 | case TIMER_HVTIMER:
|
| 128 | __vcpu_assign_sys_reg(vcpu, CNTHV_CTL_EL2, ctl);
|
| 129 | break;
|
| 130 | case TIMER_HPTIMER:
|
| 131 | __vcpu_assign_sys_reg(vcpu, CNTHP_CTL_EL2, ctl);
|
| 132 | break;
|
| 133 | default:
|
| 134 | WARN_ON(1);
|
| 135 | }
|
| 136 | }
|
| 137 |
|
| 138 | static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval)
|
| 139 | {
|
| 140 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctxt);
|
| 141 |
|
| 142 | switch(arch_timer_ctx_index(ctxt)) {
|
| 143 | case TIMER_VTIMER:
|
| 144 | __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, cval);
|
| 145 | break;
|
| 146 | case TIMER_PTIMER:
|
| 147 | __vcpu_assign_sys_reg(vcpu, CNTP_CVAL_EL0, cval);
|
| 148 | break;
|
| 149 | case TIMER_HVTIMER:
|
| 150 | __vcpu_assign_sys_reg(vcpu, CNTHV_CVAL_EL2, cval);
|
| 151 | break;
|
| 152 | case TIMER_HPTIMER:
|
| 153 | __vcpu_assign_sys_reg(vcpu, CNTHP_CVAL_EL2, cval);
|
| 154 | break;
|
| 155 | default:
|
| 156 | WARN_ON(1);
|
| 157 | }
|
| 158 | }
|
| 159 |
|
| 160 | u64 kvm_phys_timer_read(void)
|
| 161 | {
|
| 162 | return timecounter->cc->read(timecounter->cc);
|
| 163 | }
|
| 164 |
|
| 165 | void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
|
| 166 | {
|
| 167 | if (vcpu_has_nv(vcpu)) {
|
| 168 | if (is_hyp_ctxt(vcpu)) {
|
| 169 | map->direct_vtimer = vcpu_hvtimer(vcpu);
|
| 170 | map->direct_ptimer = vcpu_hptimer(vcpu);
|
| 171 | map->emul_vtimer = vcpu_vtimer(vcpu);
|
| 172 | map->emul_ptimer = vcpu_ptimer(vcpu);
|
| 173 | } else {
|
| 174 | map->direct_vtimer = vcpu_vtimer(vcpu);
|
| 175 | map->direct_ptimer = vcpu_ptimer(vcpu);
|
| 176 | map->emul_vtimer = vcpu_hvtimer(vcpu);
|
| 177 | map->emul_ptimer = vcpu_hptimer(vcpu);
|
| 178 | }
|
| 179 | } else if (has_vhe()) {
|
| 180 | map->direct_vtimer = vcpu_vtimer(vcpu);
|
| 181 | map->direct_ptimer = vcpu_ptimer(vcpu);
|
| 182 | map->emul_vtimer = NULL;
|
| 183 | map->emul_ptimer = NULL;
|
| 184 | } else {
|
| 185 | map->direct_vtimer = vcpu_vtimer(vcpu);
|
| 186 | map->direct_ptimer = NULL;
|
| 187 | map->emul_vtimer = NULL;
|
| 188 | map->emul_ptimer = vcpu_ptimer(vcpu);
|
| 189 | }
|
| 190 |
|
| 191 | trace_kvm_get_timer_map(vcpu->vcpu_id, map);
|
| 192 | }
|
| 193 |
|
| 194 | static inline bool userspace_irqchip(struct kvm *kvm)
|
| 195 | {
|
| 196 | return unlikely(!irqchip_in_kernel(kvm));
|
| 197 | }
|
| 198 |
|
| 199 | static void soft_timer_start(struct hrtimer *hrt, u64 ns)
|
| 200 | {
|
| 201 | hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
|
| 202 | HRTIMER_MODE_ABS_HARD);
|
| 203 | }
|
| 204 |
|
| 205 | static void soft_timer_cancel(struct hrtimer *hrt)
|
| 206 | {
|
| 207 | hrtimer_cancel(hrt);
|
| 208 | }
|
| 209 |
|
| 210 | static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
|
| 211 | {
|
| 212 | struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
|
| 213 | struct arch_timer_context *ctx;
|
| 214 | struct timer_map map;
|
| 215 |
|
| 216 | |
| 217 | * We may see a timer interrupt after vcpu_put() has been called which
|
| 218 | * sets the CPU's vcpu pointer to NULL, because even though the timer
|
| 219 | * has been disabled in timer_save_state(), the hardware interrupt
|
| 220 | * signal may not have been retired from the interrupt controller yet.
|
| 221 |
|
| 222 | if (!vcpu)
|
| 223 | return IRQ_HANDLED;
|
| 224 |
|
| 225 | get_timer_map(vcpu, &map);
|
| 226 |
|
| 227 | if (irq == host_vtimer_irq)
|
| 228 | ctx = map.direct_vtimer;
|
| 229 | else
|
| 230 | ctx = map.direct_ptimer;
|
| 231 |
|
| 232 | if (kvm_timer_pending(ctx))
|
| 233 | kvm_timer_update_irq(vcpu, true, ctx);
|
| 234 |
|
| 235 | if (userspace_irqchip(vcpu->kvm) &&
|
| 236 | !static_branch_unlikely(&has_gic_active_state))
|
| 237 | disable_percpu_irq(host_vtimer_irq);
|
| 238 |
|
| 239 | return IRQ_HANDLED;
|
| 240 | }
|
| 241 |
|
| 242 | static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx,
|
| 243 | u64 val)
|
| 244 | {
|
| 245 | u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
|
| 246 |
|
| 247 | if (now < val) {
|
| 248 | u64 ns;
|
| 249 |
|
| 250 | ns = cyclecounter_cyc2ns(timecounter->cc,
|
| 251 | val - now,
|
| 252 | timecounter->mask,
|
| 253 | &timer_ctx->ns_frac);
|
| 254 | return ns;
|
| 255 | }
|
| 256 |
|
| 257 | return 0;
|
| 258 | }
|
| 259 |
|
| 260 | static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
|
| 261 | {
|
| 262 | return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));
|
| 263 | }
|
| 264 |
|
| 265 | static bool kvm_timer_enabled(struct arch_timer_context *timer_ctx)
|
| 266 | {
|
| 267 | WARN_ON(timer_ctx && timer_ctx->loaded);
|
| 268 | return timer_ctx &&
|
| 269 | ((timer_get_ctl(timer_ctx) &
|
| 270 | (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE);
|
| 271 | }
|
| 272 |
|
| 273 | static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu)
|
| 274 | {
|
| 275 | return (cpus_have_final_cap(ARM64_HAS_WFXT) &&
|
| 276 | vcpu_get_flag(vcpu, IN_WFIT));
|
| 277 | }
|
| 278 |
|
| 279 | static u64 wfit_delay_ns(struct kvm_vcpu *vcpu)
|
| 280 | {
|
| 281 | u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu));
|
| 282 | struct arch_timer_context *ctx;
|
| 283 |
|
| 284 | ctx = is_hyp_ctxt(vcpu) ? vcpu_hvtimer(vcpu) : vcpu_vtimer(vcpu);
|
| 285 |
|
| 286 | return kvm_counter_compute_delta(ctx, val);
|
| 287 | }
|
| 288 |
|
| 289 | |
| 290 | * Returns the earliest expiration time in ns among guest timers.
|
| 291 | * Note that it will return 0 if none of timers can fire.
|
| 292 |
|
| 293 | static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
|
| 294 | {
|
| 295 | u64 min_delta = ULLONG_MAX;
|
| 296 | int i;
|
| 297 |
|
| 298 | for (i = 0; i < nr_timers(vcpu); i++) {
|
| 299 | struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
|
| 300 |
|
| 301 | WARN(ctx->loaded, "timer %d loaded\n", i);
|
| 302 | if (kvm_timer_enabled(ctx))
|
| 303 | min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
|
| 304 | }
|
| 305 |
|
| 306 | if (vcpu_has_wfit_active(vcpu))
|
| 307 | min_delta = min(min_delta, wfit_delay_ns(vcpu));
|
| 308 |
|
| 309 |
|
| 310 | if (min_delta == ULLONG_MAX)
|
| 311 | return 0;
|
| 312 |
|
| 313 | return min_delta;
|
| 314 | }
|
| 315 |
|
| 316 | static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
|
| 317 | {
|
| 318 | struct arch_timer_cpu *timer;
|
| 319 | struct kvm_vcpu *vcpu;
|
| 320 | u64 ns;
|
| 321 |
|
| 322 | timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
|
| 323 | vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
|
| 324 |
|
| 325 | |
| 326 | * Check that the timer has really expired from the guest's
|
| 327 | * PoV (NTP on the host may have forced it to expire
|
| 328 | * early). If we should have slept longer, restart it.
|
| 329 |
|
| 330 | ns = kvm_timer_earliest_exp(vcpu);
|
| 331 | if (unlikely(ns)) {
|
| 332 | hrtimer_forward_now(hrt, ns_to_ktime(ns));
|
| 333 | return HRTIMER_RESTART;
|
| 334 | }
|
| 335 |
|
| 336 | kvm_vcpu_wake_up(vcpu);
|
| 337 | return HRTIMER_NORESTART;
|
| 338 | }
|
| 339 |
|
| 340 | static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
|
| 341 | {
|
| 342 | struct arch_timer_context *ctx;
|
| 343 | struct kvm_vcpu *vcpu;
|
| 344 | u64 ns;
|
| 345 |
|
| 346 | ctx = container_of(hrt, struct arch_timer_context, hrtimer);
|
| 347 | vcpu = timer_context_to_vcpu(ctx);
|
| 348 |
|
| 349 | trace_kvm_timer_hrtimer_expire(ctx);
|
| 350 |
|
| 351 | |
| 352 | * Check that the timer has really expired from the guest's
|
| 353 | * PoV (NTP on the host may have forced it to expire
|
| 354 | * early). If not ready, schedule for a later time.
|
| 355 |
|
| 356 | ns = kvm_timer_compute_delta(ctx);
|
| 357 | if (unlikely(ns)) {
|
| 358 | hrtimer_forward_now(hrt, ns_to_ktime(ns));
|
| 359 | return HRTIMER_RESTART;
|
| 360 | }
|
| 361 |
|
| 362 | kvm_timer_update_irq(vcpu, true, ctx);
|
| 363 | return HRTIMER_NORESTART;
|
| 364 | }
|
| 365 |
|
| 366 | static bool kvm_timer_pending(struct arch_timer_context *timer_ctx)
|
| 367 | {
|
| 368 | enum kvm_arch_timers index;
|
| 369 | u64 cval, now;
|
| 370 |
|
| 371 | if (!timer_ctx)
|
| 372 | return false;
|
| 373 |
|
| 374 | index = arch_timer_ctx_index(timer_ctx);
|
| 375 |
|
| 376 | if (timer_ctx->loaded) {
|
| 377 | u32 cnt_ctl = 0;
|
| 378 |
|
| 379 | switch (index) {
|
| 380 | case TIMER_VTIMER:
|
| 381 | case TIMER_HVTIMER:
|
| 382 | cnt_ctl = read_sysreg_el0(SYS_CNTV_CTL);
|
| 383 | break;
|
| 384 | case TIMER_PTIMER:
|
| 385 | case TIMER_HPTIMER:
|
| 386 | cnt_ctl = read_sysreg_el0(SYS_CNTP_CTL);
|
| 387 | break;
|
| 388 | case NR_KVM_TIMERS:
|
| 389 |
|
| 390 | cnt_ctl = 0;
|
| 391 | break;
|
| 392 | }
|
| 393 |
|
| 394 | return (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
|
| 395 | (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
|
| 396 | !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
|
| 397 | }
|
| 398 |
|
| 399 | if (!kvm_timer_enabled(timer_ctx))
|
| 400 | return false;
|
| 401 |
|
| 402 | cval = timer_get_cval(timer_ctx);
|
| 403 | now = kvm_phys_timer_read() - timer_get_offset(timer_ctx);
|
| 404 |
|
| 405 | return cval <= now;
|
| 406 | }
|
| 407 |
|
| 408 | int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
|
| 409 | {
|
| 410 | return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
|
| 411 | }
|
| 412 |
|
| 413 | static u64 kvm_timer_needs_notify(struct kvm_vcpu *vcpu)
|
| 414 | {
|
| 415 | u64 v = vcpu->run->s.regs.device_irq_level;
|
| 416 |
|
| 417 | v ^= kvm_timer_pending(vcpu_vtimer(vcpu)) ? KVM_ARM_DEV_EL1_VTIMER : 0;
|
| 418 | v ^= kvm_timer_pending(vcpu_ptimer(vcpu)) ? KVM_ARM_DEV_EL1_PTIMER : 0;
|
| 419 |
|
| 420 | return v & (KVM_ARM_DEV_EL1_VTIMER | KVM_ARM_DEV_EL1_PTIMER);
|
| 421 | }
|
| 422 |
|
| 423 | bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
|
| 424 | {
|
| 425 | return !!kvm_timer_needs_notify(vcpu);
|
| 426 | }
|
| 427 |
|
| 428 | |
| 429 | * Reflect the timer output level into the kvm_run structure
|
| 430 |
|
| 431 | bool kvm_timer_update_run(struct kvm_vcpu *vcpu)
|
| 432 | {
|
| 433 | u64 mask = kvm_timer_needs_notify(vcpu);
|
| 434 | if (mask)
|
| 435 | vcpu->run->s.regs.device_irq_level ^= mask;
|
| 436 | return !!mask;
|
| 437 | }
|
| 438 |
|
| 439 | static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level)
|
| 440 | {
|
| 441 | |
| 442 | * Paper over NV2 brokenness by publishing the interrupt status
|
| 443 | * bit. This still results in a poor quality of emulation (guest
|
| 444 | * writes will have no effect until the next exit).
|
| 445 | *
|
| 446 | * But hey, it's fast, right?
|
| 447 |
|
| 448 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
|
| 449 | if (is_hyp_ctxt(vcpu) &&
|
| 450 | (ctx == vcpu_vtimer(vcpu) || ctx == vcpu_ptimer(vcpu))) {
|
| 451 | unsigned long val = timer_get_ctl(ctx);
|
| 452 | __assign_bit(__ffs(ARCH_TIMER_CTRL_IT_STAT), &val, level);
|
| 453 | timer_set_ctl(ctx, val);
|
| 454 | }
|
| 455 | }
|
| 456 |
|
| 457 | static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
|
| 458 | struct arch_timer_context *timer_ctx)
|
| 459 | {
|
| 460 | kvm_timer_update_status(timer_ctx, new_level);
|
| 461 |
|
| 462 | trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx),
|
| 463 | new_level);
|
| 464 |
|
| 465 | if (userspace_irqchip(vcpu->kvm))
|
| 466 | return;
|
| 467 |
|
| 468 |
|
| 469 | if (vgic_is_v5(vcpu->kvm)) {
|
| 470 | struct timer_map map;
|
| 471 |
|
| 472 | get_timer_map(vcpu, &map);
|
| 473 |
|
| 474 | if (map.direct_ptimer == timer_ctx ||
|
| 475 | map.direct_vtimer == timer_ctx)
|
| 476 | return;
|
| 477 | }
|
| 478 |
|
| 479 | kvm_vgic_inject_irq(vcpu->kvm, vcpu,
|
| 480 | timer_irq(timer_ctx),
|
| 481 | new_level,
|
| 482 | timer_ctx);
|
| 483 | }
|
| 484 |
|
| 485 |
|
| 486 | static void timer_emulate(struct arch_timer_context *ctx)
|
| 487 | {
|
| 488 | bool pending = kvm_timer_pending(ctx);
|
| 489 |
|
| 490 | trace_kvm_timer_emulate(ctx, pending);
|
| 491 |
|
| 492 | kvm_timer_update_irq(timer_context_to_vcpu(ctx), pending, ctx);
|
| 493 |
|
| 494 | |
| 495 | * If the timer is pending, we don't need to have a soft timer
|
| 496 | * scheduled for the future. If the timer is disabled, then
|
| 497 | * we don't need a soft timer either.
|
| 498 |
|
| 499 | if (pending || !kvm_timer_enabled(ctx))
|
| 500 | return;
|
| 501 |
|
| 502 | soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
|
| 503 | }
|
| 504 |
|
| 505 | static void set_cntvoff(u64 cntvoff)
|
| 506 | {
|
| 507 | kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff);
|
| 508 | }
|
| 509 |
|
| 510 | static void set_cntpoff(u64 cntpoff)
|
| 511 | {
|
| 512 | if (has_cntpoff())
|
| 513 | write_sysreg_s(cntpoff, SYS_CNTPOFF_EL2);
|
| 514 | }
|
| 515 |
|
| 516 | static void timer_save_state(struct arch_timer_context *ctx)
|
| 517 | {
|
| 518 | struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
|
| 519 | enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
|
| 520 | unsigned long flags;
|
| 521 |
|
| 522 | if (!timer->enabled)
|
| 523 | return;
|
| 524 |
|
| 525 | local_irq_save(flags);
|
| 526 |
|
| 527 | if (!ctx->loaded)
|
| 528 | goto out;
|
| 529 |
|
| 530 | switch (index) {
|
| 531 | u64 cval;
|
| 532 |
|
| 533 | case TIMER_VTIMER:
|
| 534 | case TIMER_HVTIMER:
|
| 535 | timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL));
|
| 536 | cval = read_sysreg_el0(SYS_CNTV_CVAL);
|
| 537 |
|
| 538 | if (has_broken_cntvoff())
|
| 539 | cval -= timer_get_offset(ctx);
|
| 540 |
|
| 541 | timer_set_cval(ctx, cval);
|
| 542 |
|
| 543 |
|
| 544 | write_sysreg_el0(0, SYS_CNTV_CTL);
|
| 545 | isb();
|
| 546 |
|
| 547 | |
| 548 | * The kernel may decide to run userspace after
|
| 549 | * calling vcpu_put, so we reset cntvoff to 0 to
|
| 550 | * ensure a consistent read between user accesses to
|
| 551 | * the virtual counter and kernel access to the
|
| 552 | * physical counter of non-VHE case.
|
| 553 | *
|
| 554 | * For VHE, the virtual counter uses a fixed virtual
|
| 555 | * offset of zero, so no need to zero CNTVOFF_EL2
|
| 556 | * register, but this is actually useful when switching
|
| 557 | * between EL1/vEL2 with NV.
|
| 558 | *
|
| 559 | * Do it unconditionally, as this is either unavoidable
|
| 560 | * or dirt cheap.
|
| 561 |
|
| 562 | set_cntvoff(0);
|
| 563 | break;
|
| 564 | case TIMER_PTIMER:
|
| 565 | case TIMER_HPTIMER:
|
| 566 | timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL));
|
| 567 | cval = read_sysreg_el0(SYS_CNTP_CVAL);
|
| 568 |
|
| 569 | cval -= timer_get_offset(ctx);
|
| 570 |
|
| 571 | timer_set_cval(ctx, cval);
|
| 572 |
|
| 573 |
|
| 574 | write_sysreg_el0(0, SYS_CNTP_CTL);
|
| 575 | isb();
|
| 576 |
|
| 577 | set_cntpoff(0);
|
| 578 | break;
|
| 579 | case NR_KVM_TIMERS:
|
| 580 | BUG();
|
| 581 | }
|
| 582 |
|
| 583 | trace_kvm_timer_save_state(ctx);
|
| 584 |
|
| 585 | ctx->loaded = false;
|
| 586 | out:
|
| 587 | local_irq_restore(flags);
|
| 588 | }
|
| 589 |
|
| 590 | |
| 591 | * Schedule the background timer before calling kvm_vcpu_halt, so that this
|
| 592 | * thread is removed from its waitqueue and made runnable when there's a timer
|
| 593 | * interrupt to handle.
|
| 594 |
|
| 595 | static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
|
| 596 | {
|
| 597 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 598 | struct timer_map map;
|
| 599 |
|
| 600 | get_timer_map(vcpu, &map);
|
| 601 |
|
| 602 | |
| 603 | * If no timers are capable of raising interrupts (disabled or
|
| 604 | * masked), then there's no more work for us to do.
|
| 605 |
|
| 606 | if (!kvm_timer_enabled(map.direct_vtimer) &&
|
| 607 | !kvm_timer_enabled(map.direct_ptimer) &&
|
| 608 | !kvm_timer_enabled(map.emul_vtimer) &&
|
| 609 | !kvm_timer_enabled(map.emul_ptimer) &&
|
| 610 | !vcpu_has_wfit_active(vcpu))
|
| 611 | return;
|
| 612 |
|
| 613 | |
| 614 | * At least one guest time will expire. Schedule a background timer.
|
| 615 | * Set the earliest expiration time among the guest timers.
|
| 616 |
|
| 617 | soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
|
| 618 | }
|
| 619 |
|
| 620 | static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
|
| 621 | {
|
| 622 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 623 |
|
| 624 | soft_timer_cancel(&timer->bg_timer);
|
| 625 | }
|
| 626 |
|
| 627 | static void timer_restore_state(struct arch_timer_context *ctx)
|
| 628 | {
|
| 629 | struct arch_timer_cpu *timer = vcpu_timer(timer_context_to_vcpu(ctx));
|
| 630 | enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
|
| 631 | unsigned long flags;
|
| 632 |
|
| 633 | if (!timer->enabled)
|
| 634 | return;
|
| 635 |
|
| 636 | local_irq_save(flags);
|
| 637 |
|
| 638 | if (ctx->loaded)
|
| 639 | goto out;
|
| 640 |
|
| 641 | switch (index) {
|
| 642 | u64 cval, offset;
|
| 643 |
|
| 644 | case TIMER_VTIMER:
|
| 645 | case TIMER_HVTIMER:
|
| 646 | cval = timer_get_cval(ctx);
|
| 647 | offset = timer_get_offset(ctx);
|
| 648 | if (has_broken_cntvoff()) {
|
| 649 | set_cntvoff(0);
|
| 650 | cval += offset;
|
| 651 | } else {
|
| 652 | set_cntvoff(offset);
|
| 653 | }
|
| 654 | write_sysreg_el0(cval, SYS_CNTV_CVAL);
|
| 655 | isb();
|
| 656 | write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTV_CTL);
|
| 657 | break;
|
| 658 | case TIMER_PTIMER:
|
| 659 | case TIMER_HPTIMER:
|
| 660 | cval = timer_get_cval(ctx);
|
| 661 | offset = timer_get_offset(ctx);
|
| 662 | set_cntpoff(offset);
|
| 663 | cval += offset;
|
| 664 | write_sysreg_el0(cval, SYS_CNTP_CVAL);
|
| 665 | isb();
|
| 666 | write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL);
|
| 667 | break;
|
| 668 | case NR_KVM_TIMERS:
|
| 669 | BUG();
|
| 670 | }
|
| 671 |
|
| 672 | trace_kvm_timer_restore_state(ctx);
|
| 673 |
|
| 674 | ctx->loaded = true;
|
| 675 | out:
|
| 676 | local_irq_restore(flags);
|
| 677 | }
|
| 678 |
|
| 679 | static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
|
| 680 | {
|
| 681 | int r;
|
| 682 | r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
|
| 683 | WARN_ON(r);
|
| 684 | }
|
| 685 |
|
| 686 | static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
|
| 687 | {
|
| 688 | struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
|
| 689 | bool pending = kvm_timer_pending(ctx);
|
| 690 | bool phys_active = false;
|
| 691 |
|
| 692 | |
| 693 | * Update the timer output so that it is likely to match the
|
| 694 | * state we're about to restore. If the timer expires between
|
| 695 | * this point and the register restoration, we'll take the
|
| 696 | * interrupt anyway.
|
| 697 |
|
| 698 | kvm_timer_update_irq(vcpu, pending, ctx);
|
| 699 |
|
| 700 | if (irqchip_in_kernel(vcpu->kvm))
|
| 701 | phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
|
| 702 |
|
| 703 | phys_active |= pending;
|
| 704 | phys_active |= vgic_is_v5(vcpu->kvm);
|
| 705 |
|
| 706 | set_timer_irq_phys_active(ctx, phys_active);
|
| 707 | }
|
| 708 |
|
| 709 | static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
|
| 710 | {
|
| 711 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
|
| 712 | bool pending = kvm_timer_pending(vtimer);
|
| 713 |
|
| 714 | |
| 715 | * Update the timer output so that it is likely to match the
|
| 716 | * state we're about to restore. If the timer expires between
|
| 717 | * this point and the register restoration, we'll take the
|
| 718 | * interrupt anyway.
|
| 719 |
|
| 720 | kvm_timer_update_irq(vcpu, pending, vtimer);
|
| 721 |
|
| 722 | |
| 723 | * When using a userspace irqchip with the architected timers and a
|
| 724 | * host interrupt controller that doesn't support an active state, we
|
| 725 | * must still prevent continuously exiting from the guest, and
|
| 726 | * therefore mask the physical interrupt by disabling it on the host
|
| 727 | * interrupt controller when the virtual level is high, such that the
|
| 728 | * guest can make forward progress. Once we detect the output level
|
| 729 | * being de-asserted, we unmask the interrupt again so that we exit
|
| 730 | * from the guest when the timer fires.
|
| 731 |
|
| 732 | if (pending)
|
| 733 | disable_percpu_irq(host_vtimer_irq);
|
| 734 | else
|
| 735 | enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
|
| 736 | }
|
| 737 |
|
| 738 |
|
| 739 | #define assign_clear_set_bit(_pred, _bit, _clr, _set) \
|
| 740 | do { \
|
| 741 | if (_pred) \
|
| 742 | (_set) |= (_bit); \
|
| 743 | else \
|
| 744 | (_clr) |= (_bit); \
|
| 745 | } while (0)
|
| 746 |
|
| 747 | static void kvm_timer_vcpu_load_nested_switch(struct kvm_vcpu *vcpu,
|
| 748 | struct timer_map *map)
|
| 749 | {
|
| 750 | int hw, ret;
|
| 751 |
|
| 752 | if (!irqchip_in_kernel(vcpu->kvm))
|
| 753 | return;
|
| 754 |
|
| 755 | |
| 756 | * We only ever unmap the vtimer irq on a VHE system that runs nested
|
| 757 | * virtualization, in which case we have both a valid emul_vtimer,
|
| 758 | * emul_ptimer, direct_vtimer, and direct_ptimer.
|
| 759 | *
|
| 760 | * Since this is called from kvm_timer_vcpu_load(), a change between
|
| 761 | * vEL2 and vEL1/0 will have just happened, and the timer_map will
|
| 762 | * represent this, and therefore we switch the emul/direct mappings
|
| 763 | * below.
|
| 764 |
|
| 765 | hw = kvm_vgic_get_map(vcpu, timer_irq(map->direct_vtimer));
|
| 766 | if (hw < 0) {
|
| 767 | kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_vtimer));
|
| 768 | kvm_vgic_unmap_phys_irq(vcpu, timer_irq(map->emul_ptimer));
|
| 769 |
|
| 770 | ret = kvm_vgic_map_phys_irq(vcpu,
|
| 771 | map->direct_vtimer->host_timer_irq,
|
| 772 | timer_irq(map->direct_vtimer));
|
| 773 | WARN_ON_ONCE(ret);
|
| 774 | ret = kvm_vgic_map_phys_irq(vcpu,
|
| 775 | map->direct_ptimer->host_timer_irq,
|
| 776 | timer_irq(map->direct_ptimer));
|
| 777 | WARN_ON_ONCE(ret);
|
| 778 | }
|
| 779 | }
|
| 780 |
|
| 781 | static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map)
|
| 782 | {
|
| 783 | bool tvt, tpt, tvc, tpc, tvt02, tpt02;
|
| 784 | u64 clr, set;
|
| 785 |
|
| 786 | |
| 787 | * No trapping gets configured here with nVHE. See
|
| 788 | * __timer_enable_traps(), which is where the stuff happens.
|
| 789 |
|
| 790 | if (!has_vhe())
|
| 791 | return;
|
| 792 |
|
| 793 | |
| 794 | * Our default policy is not to trap anything. As we progress
|
| 795 | * within this function, reality kicks in and we start adding
|
| 796 | * traps based on emulation requirements.
|
| 797 |
|
| 798 | tvt = tpt = tvc = tpc = false;
|
| 799 | tvt02 = tpt02 = false;
|
| 800 |
|
| 801 | |
| 802 | * NV2 badly breaks the timer semantics by redirecting accesses to
|
| 803 | * the EL1 timer state to memory, so let's call ECV to the rescue if
|
| 804 | * available: we trap all CNT{P,V}_{CTL,CVAL,TVAL}_EL0 accesses.
|
| 805 | *
|
| 806 | * The treatment slightly varies depending whether we run a nVHE or
|
| 807 | * VHE guest: nVHE will use the _EL0 registers directly, while VHE
|
| 808 | * will use the _EL02 accessors. This translates in different trap
|
| 809 | * bits.
|
| 810 | *
|
| 811 | * None of the trapping is required when running in non-HYP context,
|
| 812 | * unless required by the L1 hypervisor settings once we advertise
|
| 813 | * ECV+NV in the guest, or that we need trapping for other reasons.
|
| 814 |
|
| 815 | if (cpus_have_final_cap(ARM64_HAS_ECV) && is_hyp_ctxt(vcpu)) {
|
| 816 | if (vcpu_el2_e2h_is_set(vcpu))
|
| 817 | tvt02 = tpt02 = true;
|
| 818 | else
|
| 819 | tvt = tpt = true;
|
| 820 | }
|
| 821 |
|
| 822 | |
| 823 | * We have two possibility to deal with a physical offset:
|
| 824 | *
|
| 825 | * - Either we have CNTPOFF (yay!) or the offset is 0:
|
| 826 | * we let the guest freely access the HW
|
| 827 | *
|
| 828 | * - or neither of these condition apply:
|
| 829 | * we trap accesses to the HW, but still use it
|
| 830 | * after correcting the physical offset
|
| 831 |
|
| 832 | if (!has_cntpoff() && timer_get_offset(map->direct_ptimer))
|
| 833 | tpt = tpc = true;
|
| 834 |
|
| 835 | |
| 836 | * For the poor sods that could not correctly subtract one value
|
| 837 | * from another, trap the full virtual timer and counter.
|
| 838 |
|
| 839 | if (has_broken_cntvoff() && timer_get_offset(map->direct_vtimer))
|
| 840 | tvt = tvc = true;
|
| 841 |
|
| 842 | |
| 843 | * Apply the enable bits that the guest hypervisor has requested for
|
| 844 | * its own guest. We can only add traps that wouldn't have been set
|
| 845 | * above.
|
| 846 | * Implementation choices: we do not support NV when E2H=0 in the
|
| 847 | * guest, and we don't support configuration where E2H is writable
|
| 848 | * by the guest (either FEAT_VHE or FEAT_E2H0 is implemented, but
|
| 849 | * not both). This simplifies the handling of the EL1NV* bits.
|
| 850 |
|
| 851 | if (is_nested_ctxt(vcpu)) {
|
| 852 | u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
|
| 853 |
|
| 854 |
|
| 855 | if (!vcpu_el2_e2h_is_set(vcpu))
|
| 856 | val = (val & (CNTHCTL_EL1PCEN | CNTHCTL_EL1PCTEN)) << 10;
|
| 857 |
|
| 858 | tpt |= !(val & (CNTHCTL_EL1PCEN << 10));
|
| 859 | tpc |= !(val & (CNTHCTL_EL1PCTEN << 10));
|
| 860 |
|
| 861 | tpt02 |= (val & CNTHCTL_EL1NVPCT);
|
| 862 | tvt02 |= (val & CNTHCTL_EL1NVVCT);
|
| 863 | }
|
| 864 |
|
| 865 | |
| 866 | * Now that we have collected our requirements, compute the
|
| 867 | * trap and enable bits.
|
| 868 |
|
| 869 | set = 0;
|
| 870 | clr = 0;
|
| 871 |
|
| 872 | assign_clear_set_bit(tpt, CNTHCTL_EL1PCEN << 10, set, clr);
|
| 873 | assign_clear_set_bit(tpc, CNTHCTL_EL1PCTEN << 10, set, clr);
|
| 874 | assign_clear_set_bit(tvt, CNTHCTL_EL1TVT, clr, set);
|
| 875 | assign_clear_set_bit(tvc, CNTHCTL_EL1TVCT, clr, set);
|
| 876 | assign_clear_set_bit(tvt02, CNTHCTL_EL1NVVCT, clr, set);
|
| 877 | assign_clear_set_bit(tpt02, CNTHCTL_EL1NVPCT, clr, set);
|
| 878 |
|
| 879 | |
| 880 | * This only happens on VHE, so use the CNTHCTL_EL2 accessor, unless
|
| 881 | * we are sure CNTKCTL_EL1 is completely stateful with FEAT_NV2p1.
|
| 882 |
|
| 883 | if (!cpus_have_final_cap(ARM64_HAS_NV2P1))
|
| 884 | sysreg_clear_set(cnthctl_el2, clr, set);
|
| 885 | else
|
| 886 | sysreg_clear_set(cntkctl_el1, clr, set);
|
| 887 | }
|
| 888 |
|
| 889 | void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
|
| 890 | {
|
| 891 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 892 | struct timer_map map;
|
| 893 |
|
| 894 | if (unlikely(!timer->enabled))
|
| 895 | return;
|
| 896 |
|
| 897 | get_timer_map(vcpu, &map);
|
| 898 |
|
| 899 | if (static_branch_likely(&has_gic_active_state)) {
|
| 900 |
|
| 901 | if (vcpu_has_nv(vcpu) && !vgic_is_v5(vcpu->kvm))
|
| 902 | kvm_timer_vcpu_load_nested_switch(vcpu, &map);
|
| 903 |
|
| 904 | kvm_timer_vcpu_load_gic(map.direct_vtimer);
|
| 905 | if (map.direct_ptimer)
|
| 906 | kvm_timer_vcpu_load_gic(map.direct_ptimer);
|
| 907 | } else {
|
| 908 | kvm_timer_vcpu_load_nogic(vcpu);
|
| 909 | }
|
| 910 |
|
| 911 | kvm_timer_unblocking(vcpu);
|
| 912 |
|
| 913 | timer_restore_state(map.direct_vtimer);
|
| 914 | if (map.direct_ptimer)
|
| 915 | timer_restore_state(map.direct_ptimer);
|
| 916 | if (map.emul_vtimer)
|
| 917 | timer_emulate(map.emul_vtimer);
|
| 918 | if (map.emul_ptimer)
|
| 919 | timer_emulate(map.emul_ptimer);
|
| 920 |
|
| 921 | timer_set_traps(vcpu, &map);
|
| 922 | }
|
| 923 |
|
| 924 | void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
|
| 925 | {
|
| 926 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 927 | struct timer_map map;
|
| 928 |
|
| 929 | if (unlikely(!timer->enabled))
|
| 930 | return;
|
| 931 |
|
| 932 | get_timer_map(vcpu, &map);
|
| 933 |
|
| 934 | timer_save_state(map.direct_vtimer);
|
| 935 | if (map.direct_ptimer)
|
| 936 | timer_save_state(map.direct_ptimer);
|
| 937 |
|
| 938 | |
| 939 | * Cancel soft timer emulation, because the only case where we
|
| 940 | * need it after a vcpu_put is in the context of a sleeping VCPU, and
|
| 941 | * in that case we already factor in the deadline for the physical
|
| 942 | * timer when scheduling the bg_timer.
|
| 943 | *
|
| 944 | * In any case, we re-schedule the hrtimer for the physical timer when
|
| 945 | * coming back to the VCPU thread in kvm_timer_vcpu_load().
|
| 946 |
|
| 947 | if (map.emul_vtimer)
|
| 948 | soft_timer_cancel(&map.emul_vtimer->hrtimer);
|
| 949 | if (map.emul_ptimer)
|
| 950 | soft_timer_cancel(&map.emul_ptimer->hrtimer);
|
| 951 |
|
| 952 | if (kvm_vcpu_is_blocking(vcpu))
|
| 953 | kvm_timer_blocking(vcpu);
|
| 954 |
|
| 955 | if (vgic_is_v5(vcpu->kvm)) {
|
| 956 | set_timer_irq_phys_active(map.direct_vtimer, false);
|
| 957 | if (map.direct_ptimer)
|
| 958 | set_timer_irq_phys_active(map.direct_ptimer, false);
|
| 959 | }
|
| 960 | }
|
| 961 |
|
| 962 | void kvm_timer_sync_nested(struct kvm_vcpu *vcpu)
|
| 963 | {
|
| 964 | |
| 965 | * When NV2 is on, guest hypervisors have their EL1 timer register
|
| 966 | * accesses redirected to the VNCR page. Any guest action taken on
|
| 967 | * the timer is postponed until the next exit, leading to a very
|
| 968 | * poor quality of emulation.
|
| 969 | *
|
| 970 | * This is an unmitigated disaster, only papered over by FEAT_ECV,
|
| 971 | * which allows trapping of the timer registers even with NV2.
|
| 972 | * Still, this is still worse than FEAT_NV on its own. Meh.
|
| 973 |
|
| 974 | if (!cpus_have_final_cap(ARM64_HAS_ECV)) {
|
| 975 | |
| 976 | * For a VHE guest hypervisor, the EL2 state is directly
|
| 977 | * stored in the host EL1 timers, while the emulated EL1
|
| 978 | * state is stored in the VNCR page. The latter could have
|
| 979 | * been updated behind our back, and we must reset the
|
| 980 | * emulation of the timers.
|
| 981 | *
|
| 982 | * A non-VHE guest hypervisor doesn't have any direct access
|
| 983 | * to its timers: the EL2 registers trap despite being
|
| 984 | * notionally direct (we use the EL1 HW, as for VHE), while
|
| 985 | * the EL1 registers access memory.
|
| 986 | *
|
| 987 | * In both cases, process the emulated timers on each guest
|
| 988 | * exit. Boo.
|
| 989 |
|
| 990 | struct timer_map map;
|
| 991 | get_timer_map(vcpu, &map);
|
| 992 |
|
| 993 | soft_timer_cancel(&map.emul_vtimer->hrtimer);
|
| 994 | soft_timer_cancel(&map.emul_ptimer->hrtimer);
|
| 995 | timer_emulate(map.emul_vtimer);
|
| 996 | timer_emulate(map.emul_ptimer);
|
| 997 | }
|
| 998 | }
|
| 999 |
|
| 1000 | |
| 1001 | * With a userspace irqchip we have to check if the guest de-asserted the
|
| 1002 | * timer and if so, unmask the timer irq signal on the host interrupt
|
| 1003 | * controller to ensure that we see future timer signals.
|
| 1004 |
|
| 1005 | static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
|
| 1006 | {
|
| 1007 | struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
|
| 1008 |
|
| 1009 | if (!kvm_timer_pending(vtimer)) {
|
| 1010 | kvm_timer_update_irq(vcpu, false, vtimer);
|
| 1011 | if (static_branch_likely(&has_gic_active_state))
|
| 1012 | set_timer_irq_phys_active(vtimer, false);
|
| 1013 | else
|
| 1014 | enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
|
| 1015 | }
|
| 1016 | }
|
| 1017 |
|
| 1018 | void kvm_timer_sync_user(struct kvm_vcpu *vcpu)
|
| 1019 | {
|
| 1020 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 1021 |
|
| 1022 | if (unlikely(!timer->enabled))
|
| 1023 | return;
|
| 1024 |
|
| 1025 | if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
|
| 1026 | unmask_vtimer_irq_user(vcpu);
|
| 1027 | }
|
| 1028 |
|
| 1029 | void kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
|
| 1030 | {
|
| 1031 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 1032 | struct timer_map map;
|
| 1033 |
|
| 1034 | get_timer_map(vcpu, &map);
|
| 1035 |
|
| 1036 | |
| 1037 | * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
|
| 1038 | * and to 0 for ARMv7. We provide an implementation that always
|
| 1039 | * resets the timer to be disabled and unmasked and is compliant with
|
| 1040 | * the ARMv7 architecture.
|
| 1041 |
|
| 1042 | for (int i = 0; i < nr_timers(vcpu); i++)
|
| 1043 | timer_set_ctl(vcpu_get_timer(vcpu, i), 0);
|
| 1044 |
|
| 1045 | |
| 1046 | * A vcpu running at EL2 is in charge of the offset applied to
|
| 1047 | * the virtual timer, so use the physical VM offset, and point
|
| 1048 | * the vcpu offset to CNTVOFF_EL2.
|
| 1049 |
|
| 1050 | if (vcpu_has_nv(vcpu)) {
|
| 1051 | struct arch_timer_offset *offs = &vcpu_vtimer(vcpu)->offset;
|
| 1052 |
|
| 1053 | offs->vcpu_offset = __ctxt_sys_reg(&vcpu->arch.ctxt, CNTVOFF_EL2);
|
| 1054 | offs->vm_offset = &vcpu->kvm->arch.timer_data.poffset;
|
| 1055 | }
|
| 1056 |
|
| 1057 | if (timer->enabled) {
|
| 1058 | for (int i = 0; i < nr_timers(vcpu); i++)
|
| 1059 | kvm_timer_update_irq(vcpu, false,
|
| 1060 | vcpu_get_timer(vcpu, i));
|
| 1061 |
|
| 1062 | if (irqchip_in_kernel(vcpu->kvm)) {
|
| 1063 | kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_vtimer));
|
| 1064 | if (map.direct_ptimer)
|
| 1065 | kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_ptimer));
|
| 1066 | }
|
| 1067 | }
|
| 1068 |
|
| 1069 | if (map.emul_vtimer)
|
| 1070 | soft_timer_cancel(&map.emul_vtimer->hrtimer);
|
| 1071 | if (map.emul_ptimer)
|
| 1072 | soft_timer_cancel(&map.emul_ptimer->hrtimer);
|
| 1073 | }
|
| 1074 |
|
| 1075 | static void timer_context_init(struct kvm_vcpu *vcpu, int timerid)
|
| 1076 | {
|
| 1077 | struct arch_timer_context *ctxt = vcpu_get_timer(vcpu, timerid);
|
| 1078 | struct kvm *kvm = vcpu->kvm;
|
| 1079 |
|
| 1080 | ctxt->timer_id = timerid;
|
| 1081 |
|
| 1082 | if (!kvm_vm_is_protected(vcpu->kvm)) {
|
| 1083 | if (timerid == TIMER_VTIMER)
|
| 1084 | ctxt->offset.vm_offset = &kvm->arch.timer_data.voffset;
|
| 1085 | else
|
| 1086 | ctxt->offset.vm_offset = &kvm->arch.timer_data.poffset;
|
| 1087 | } else {
|
| 1088 | ctxt->offset.vm_offset = NULL;
|
| 1089 | }
|
| 1090 |
|
| 1091 | hrtimer_setup(&ctxt->hrtimer, kvm_hrtimer_expire, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
|
| 1092 |
|
| 1093 | switch (timerid) {
|
| 1094 | case TIMER_PTIMER:
|
| 1095 | case TIMER_HPTIMER:
|
| 1096 | ctxt->host_timer_irq = host_ptimer_irq;
|
| 1097 | break;
|
| 1098 | case TIMER_VTIMER:
|
| 1099 | case TIMER_HVTIMER:
|
| 1100 | ctxt->host_timer_irq = host_vtimer_irq;
|
| 1101 | break;
|
| 1102 | }
|
| 1103 | }
|
| 1104 |
|
| 1105 | void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
|
| 1106 | {
|
| 1107 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 1108 |
|
| 1109 | for (int i = 0; i < NR_KVM_TIMERS; i++)
|
| 1110 | timer_context_init(vcpu, i);
|
| 1111 |
|
| 1112 |
|
| 1113 | if (!vcpu_is_protected(vcpu) &&
|
| 1114 | !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
|
| 1115 | timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read());
|
| 1116 | timer_set_offset(vcpu_ptimer(vcpu), 0);
|
| 1117 | }
|
| 1118 |
|
| 1119 | hrtimer_setup(&timer->bg_timer, kvm_bg_timer_expire, CLOCK_MONOTONIC,
|
| 1120 | HRTIMER_MODE_ABS_HARD);
|
| 1121 | }
|
| 1122 |
|
| 1123 | |
| 1124 | * This is always called during kvm_arch_init_vm, but will also be
|
| 1125 | * called from kvm_vgic_create if we have a vGICv5.
|
| 1126 |
|
| 1127 | void kvm_timer_init_vm(struct kvm *kvm)
|
| 1128 | {
|
| 1129 | |
| 1130 | * Set up the default PPIs - note that we adjust them based on
|
| 1131 | * the model of the GIC as GICv5 uses a different way to
|
| 1132 | * describing interrupts.
|
| 1133 |
|
| 1134 | for (int i = 0; i < NR_KVM_TIMERS; i++)
|
| 1135 | kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);
|
| 1136 | }
|
| 1137 |
|
| 1138 | void kvm_timer_cpu_up(void)
|
| 1139 | {
|
| 1140 | enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
|
| 1141 | if (host_ptimer_irq)
|
| 1142 | enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
|
| 1143 | }
|
| 1144 |
|
| 1145 | void kvm_timer_cpu_down(void)
|
| 1146 | {
|
| 1147 | disable_percpu_irq(host_vtimer_irq);
|
| 1148 | if (host_ptimer_irq)
|
| 1149 | disable_percpu_irq(host_ptimer_irq);
|
| 1150 | }
|
| 1151 |
|
| 1152 | static u64 read_timer_ctl(struct arch_timer_context *timer)
|
| 1153 | {
|
| 1154 | |
| 1155 | * Set ISTATUS bit if it's expired.
|
| 1156 | * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
|
| 1157 | * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
|
| 1158 | * regardless of ENABLE bit for our implementation convenience.
|
| 1159 |
|
| 1160 | u32 ctl = timer_get_ctl(timer);
|
| 1161 |
|
| 1162 | if (!kvm_timer_compute_delta(timer))
|
| 1163 | ctl |= ARCH_TIMER_CTRL_IT_STAT;
|
| 1164 |
|
| 1165 | return ctl;
|
| 1166 | }
|
| 1167 |
|
| 1168 | static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
|
| 1169 | struct arch_timer_context *timer,
|
| 1170 | enum kvm_arch_timer_regs treg)
|
| 1171 | {
|
| 1172 | u64 val;
|
| 1173 |
|
| 1174 | switch (treg) {
|
| 1175 | case TIMER_REG_TVAL:
|
| 1176 | val = timer_get_cval(timer) - kvm_phys_timer_read() + timer_get_offset(timer);
|
| 1177 | val = lower_32_bits(val);
|
| 1178 | break;
|
| 1179 |
|
| 1180 | case TIMER_REG_CTL:
|
| 1181 | val = read_timer_ctl(timer);
|
| 1182 | break;
|
| 1183 |
|
| 1184 | case TIMER_REG_CVAL:
|
| 1185 | val = timer_get_cval(timer);
|
| 1186 | break;
|
| 1187 |
|
| 1188 | case TIMER_REG_CNT:
|
| 1189 | val = kvm_phys_timer_read() - timer_get_offset(timer);
|
| 1190 | break;
|
| 1191 |
|
| 1192 | case TIMER_REG_VOFF:
|
| 1193 | val = *timer->offset.vcpu_offset;
|
| 1194 | break;
|
| 1195 |
|
| 1196 | default:
|
| 1197 | BUG();
|
| 1198 | }
|
| 1199 |
|
| 1200 | return val;
|
| 1201 | }
|
| 1202 |
|
| 1203 | u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
|
| 1204 | enum kvm_arch_timers tmr,
|
| 1205 | enum kvm_arch_timer_regs treg)
|
| 1206 | {
|
| 1207 | struct arch_timer_context *timer;
|
| 1208 | struct timer_map map;
|
| 1209 | u64 val;
|
| 1210 |
|
| 1211 | get_timer_map(vcpu, &map);
|
| 1212 | timer = vcpu_get_timer(vcpu, tmr);
|
| 1213 |
|
| 1214 | if (timer == map.emul_vtimer || timer == map.emul_ptimer)
|
| 1215 | return kvm_arm_timer_read(vcpu, timer, treg);
|
| 1216 |
|
| 1217 | preempt_disable();
|
| 1218 | timer_save_state(timer);
|
| 1219 |
|
| 1220 | val = kvm_arm_timer_read(vcpu, timer, treg);
|
| 1221 |
|
| 1222 | timer_restore_state(timer);
|
| 1223 | preempt_enable();
|
| 1224 |
|
| 1225 | return val;
|
| 1226 | }
|
| 1227 |
|
| 1228 | static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
|
| 1229 | struct arch_timer_context *timer,
|
| 1230 | enum kvm_arch_timer_regs treg,
|
| 1231 | u64 val)
|
| 1232 | {
|
| 1233 | switch (treg) {
|
| 1234 | case TIMER_REG_TVAL:
|
| 1235 | timer_set_cval(timer, kvm_phys_timer_read() - timer_get_offset(timer) + (s32)val);
|
| 1236 | break;
|
| 1237 |
|
| 1238 | case TIMER_REG_CTL:
|
| 1239 | timer_set_ctl(timer, val & ~ARCH_TIMER_CTRL_IT_STAT);
|
| 1240 | break;
|
| 1241 |
|
| 1242 | case TIMER_REG_CVAL:
|
| 1243 | timer_set_cval(timer, val);
|
| 1244 | break;
|
| 1245 |
|
| 1246 | case TIMER_REG_VOFF:
|
| 1247 | *timer->offset.vcpu_offset = val;
|
| 1248 | break;
|
| 1249 |
|
| 1250 | default:
|
| 1251 | BUG();
|
| 1252 | }
|
| 1253 | }
|
| 1254 |
|
| 1255 | void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
|
| 1256 | enum kvm_arch_timers tmr,
|
| 1257 | enum kvm_arch_timer_regs treg,
|
| 1258 | u64 val)
|
| 1259 | {
|
| 1260 | struct arch_timer_context *timer;
|
| 1261 | struct timer_map map;
|
| 1262 |
|
| 1263 | get_timer_map(vcpu, &map);
|
| 1264 | timer = vcpu_get_timer(vcpu, tmr);
|
| 1265 | if (timer == map.emul_vtimer || timer == map.emul_ptimer) {
|
| 1266 | soft_timer_cancel(&timer->hrtimer);
|
| 1267 | kvm_arm_timer_write(vcpu, timer, treg, val);
|
| 1268 | timer_emulate(timer);
|
| 1269 | } else {
|
| 1270 | preempt_disable();
|
| 1271 | timer_save_state(timer);
|
| 1272 | kvm_arm_timer_write(vcpu, timer, treg, val);
|
| 1273 | timer_restore_state(timer);
|
| 1274 | preempt_enable();
|
| 1275 | }
|
| 1276 | }
|
| 1277 |
|
| 1278 | static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
|
| 1279 | {
|
| 1280 | if (vcpu)
|
| 1281 | irqd_set_forwarded_to_vcpu(d);
|
| 1282 | else
|
| 1283 | irqd_clr_forwarded_to_vcpu(d);
|
| 1284 |
|
| 1285 | return 0;
|
| 1286 | }
|
| 1287 |
|
| 1288 | static int timer_irq_set_irqchip_state(struct irq_data *d,
|
| 1289 | enum irqchip_irq_state which, bool val)
|
| 1290 | {
|
| 1291 | bool passthrough = which != IRQCHIP_STATE_ACTIVE ||
|
| 1292 | !irqd_is_forwarded_to_vcpu(d) ||
|
| 1293 | (kvm_vgic_global_state.type == VGIC_V5 &&
|
| 1294 | vgic_is_v3(kvm_get_running_vcpu()->kvm));
|
| 1295 |
|
| 1296 | if (passthrough)
|
| 1297 | return irq_chip_set_parent_state(d, which, val);
|
| 1298 |
|
| 1299 | if (val)
|
| 1300 | irq_chip_mask_parent(d);
|
| 1301 | else
|
| 1302 | irq_chip_unmask_parent(d);
|
| 1303 |
|
| 1304 | return 0;
|
| 1305 | }
|
| 1306 |
|
| 1307 | static void timer_irq_eoi(struct irq_data *d)
|
| 1308 | {
|
| 1309 | if (!irqd_is_forwarded_to_vcpu(d))
|
| 1310 | irq_chip_eoi_parent(d);
|
| 1311 | }
|
| 1312 |
|
| 1313 | static void timer_irq_ack(struct irq_data *d)
|
| 1314 | {
|
| 1315 | d = d->parent_data;
|
| 1316 | if (d->chip->irq_ack)
|
| 1317 | d->chip->irq_ack(d);
|
| 1318 | }
|
| 1319 |
|
| 1320 | static struct irq_chip timer_chip = {
|
| 1321 | .name = "KVM",
|
| 1322 | .irq_ack = timer_irq_ack,
|
| 1323 | .irq_mask = irq_chip_mask_parent,
|
| 1324 | .irq_unmask = irq_chip_unmask_parent,
|
| 1325 | .irq_eoi = timer_irq_eoi,
|
| 1326 | .irq_set_type = irq_chip_set_type_parent,
|
| 1327 | .irq_set_vcpu_affinity = timer_irq_set_vcpu_affinity,
|
| 1328 | .irq_set_irqchip_state = timer_irq_set_irqchip_state,
|
| 1329 | };
|
| 1330 |
|
| 1331 | static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
|
| 1332 | unsigned int nr_irqs, void *arg)
|
| 1333 | {
|
| 1334 | irq_hw_number_t hwirq = (uintptr_t)arg;
|
| 1335 |
|
| 1336 | return irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
|
| 1337 | &timer_chip, NULL);
|
| 1338 | }
|
| 1339 |
|
| 1340 | static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq,
|
| 1341 | unsigned int nr_irqs)
|
| 1342 | {
|
| 1343 | }
|
| 1344 |
|
| 1345 | static const struct irq_domain_ops timer_domain_ops = {
|
| 1346 | .alloc = timer_irq_domain_alloc,
|
| 1347 | .free = timer_irq_domain_free,
|
| 1348 | };
|
| 1349 |
|
| 1350 | static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags)
|
| 1351 | {
|
| 1352 | *flags = irq_get_trigger_type(virq);
|
| 1353 | if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) {
|
| 1354 | kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n",
|
| 1355 | virq);
|
| 1356 | *flags = IRQF_TRIGGER_LOW;
|
| 1357 | }
|
| 1358 | }
|
| 1359 |
|
| 1360 | static int kvm_irq_init(struct arch_timer_kvm_info *info)
|
| 1361 | {
|
| 1362 | struct irq_domain *domain = NULL;
|
| 1363 |
|
| 1364 | if (info->virtual_irq <= 0) {
|
| 1365 | kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
|
| 1366 | info->virtual_irq);
|
| 1367 | return -ENODEV;
|
| 1368 | }
|
| 1369 |
|
| 1370 | host_vtimer_irq = info->virtual_irq;
|
| 1371 | kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags);
|
| 1372 |
|
| 1373 | if (kvm_vgic_global_state.no_hw_deactivation ||
|
| 1374 | kvm_vgic_global_state.type == VGIC_V5) {
|
| 1375 | struct fwnode_handle *fwnode;
|
| 1376 | struct irq_data *data;
|
| 1377 |
|
| 1378 | fwnode = irq_domain_alloc_named_fwnode("kvm-timer");
|
| 1379 | if (!fwnode)
|
| 1380 | return -ENOMEM;
|
| 1381 |
|
| 1382 |
|
| 1383 | data = irq_get_irq_data(host_vtimer_irq);
|
| 1384 | domain = irq_domain_create_hierarchy(data->domain, 0,
|
| 1385 | NR_KVM_TIMERS, fwnode,
|
| 1386 | &timer_domain_ops, NULL);
|
| 1387 | if (!domain) {
|
| 1388 | irq_domain_free_fwnode(fwnode);
|
| 1389 | return -ENOMEM;
|
| 1390 | }
|
| 1391 |
|
| 1392 | WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq,
|
| 1393 | (void *)TIMER_VTIMER));
|
| 1394 | }
|
| 1395 |
|
| 1396 | if (info->physical_irq > 0) {
|
| 1397 | host_ptimer_irq = info->physical_irq;
|
| 1398 | kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags);
|
| 1399 |
|
| 1400 | if (domain)
|
| 1401 | WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq,
|
| 1402 | (void *)TIMER_PTIMER));
|
| 1403 | }
|
| 1404 |
|
| 1405 | return 0;
|
| 1406 | }
|
| 1407 |
|
| 1408 | static void kvm_timer_handle_errata(void)
|
| 1409 | {
|
| 1410 | u64 mmfr0, mmfr1, mmfr4;
|
| 1411 |
|
| 1412 | |
| 1413 | * CNTVOFF_EL2 is broken on some implementations. For those, we trap
|
| 1414 | * all virtual timer/counter accesses, requiring FEAT_ECV.
|
| 1415 | *
|
| 1416 | * However, a hypervisor supporting nesting is likely to mitigate the
|
| 1417 | * erratum at L0, and not require other levels to mitigate it (which
|
| 1418 | * would otherwise be a terrible performance sink due to trap
|
| 1419 | * amplification).
|
| 1420 | *
|
| 1421 | * Given that the affected HW implements both FEAT_VHE and FEAT_E2H0,
|
| 1422 | * and that NV is likely not to (because of limitations of the
|
| 1423 | * architecture), only enable the workaround when FEAT_VHE and
|
| 1424 | * FEAT_E2H0 are both detected. Time will tell if this actually holds.
|
| 1425 |
|
| 1426 | mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
|
| 1427 | mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
|
| 1428 | mmfr4 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR4_EL1);
|
| 1429 | if (SYS_FIELD_GET(ID_AA64MMFR1_EL1, VH, mmfr1) &&
|
| 1430 | !SYS_FIELD_GET(ID_AA64MMFR4_EL1, E2H0, mmfr4) &&
|
| 1431 | SYS_FIELD_GET(ID_AA64MMFR0_EL1, ECV, mmfr0) &&
|
| 1432 | (has_vhe() || has_hvhe()) &&
|
| 1433 | cpus_have_final_cap(ARM64_WORKAROUND_QCOM_ORYON_CNTVOFF)) {
|
| 1434 | static_branch_enable(&broken_cntvoff_key);
|
| 1435 | kvm_info("Broken CNTVOFF_EL2, trapping virtual timer\n");
|
| 1436 | }
|
| 1437 | }
|
| 1438 |
|
| 1439 | int __init kvm_timer_hyp_init(bool has_gic)
|
| 1440 | {
|
| 1441 | struct arch_timer_kvm_info *info;
|
| 1442 | int err;
|
| 1443 |
|
| 1444 | info = arch_timer_get_kvm_info();
|
| 1445 | timecounter = &info->timecounter;
|
| 1446 |
|
| 1447 | if (!timecounter->cc) {
|
| 1448 | kvm_err("kvm_arch_timer: uninitialized timecounter\n");
|
| 1449 | return -ENODEV;
|
| 1450 | }
|
| 1451 |
|
| 1452 | err = kvm_irq_init(info);
|
| 1453 | if (err)
|
| 1454 | return err;
|
| 1455 |
|
| 1456 |
|
| 1457 |
|
| 1458 | err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
|
| 1459 | "kvm guest vtimer", kvm_get_running_vcpus());
|
| 1460 | if (err) {
|
| 1461 | kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
|
| 1462 | host_vtimer_irq, err);
|
| 1463 | return err;
|
| 1464 | }
|
| 1465 |
|
| 1466 | if (has_gic) {
|
| 1467 | err = irq_set_vcpu_affinity(host_vtimer_irq,
|
| 1468 | kvm_get_running_vcpus());
|
| 1469 | if (err) {
|
| 1470 | kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
|
| 1471 | goto out_free_vtimer_irq;
|
| 1472 | }
|
| 1473 |
|
| 1474 | static_branch_enable(&has_gic_active_state);
|
| 1475 | }
|
| 1476 |
|
| 1477 | kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
|
| 1478 |
|
| 1479 |
|
| 1480 |
|
| 1481 | if (info->physical_irq > 0) {
|
| 1482 | err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
|
| 1483 | "kvm guest ptimer", kvm_get_running_vcpus());
|
| 1484 | if (err) {
|
| 1485 | kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
|
| 1486 | host_ptimer_irq, err);
|
| 1487 | goto out_free_vtimer_irq;
|
| 1488 | }
|
| 1489 |
|
| 1490 | if (has_gic) {
|
| 1491 | err = irq_set_vcpu_affinity(host_ptimer_irq,
|
| 1492 | kvm_get_running_vcpus());
|
| 1493 | if (err) {
|
| 1494 | kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
|
| 1495 | goto out_free_ptimer_irq;
|
| 1496 | }
|
| 1497 | }
|
| 1498 |
|
| 1499 | kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
|
| 1500 | } else if (has_vhe()) {
|
| 1501 | kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
|
| 1502 | info->physical_irq);
|
| 1503 | err = -ENODEV;
|
| 1504 | goto out_free_vtimer_irq;
|
| 1505 | }
|
| 1506 |
|
| 1507 | kvm_timer_handle_errata();
|
| 1508 | return 0;
|
| 1509 |
|
| 1510 | out_free_ptimer_irq:
|
| 1511 | if (info->physical_irq > 0)
|
| 1512 | free_percpu_irq(host_ptimer_irq, kvm_get_running_vcpus());
|
| 1513 | out_free_vtimer_irq:
|
| 1514 | free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
|
| 1515 | return err;
|
| 1516 | }
|
| 1517 |
|
| 1518 | void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
|
| 1519 | {
|
| 1520 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 1521 |
|
| 1522 | soft_timer_cancel(&timer->bg_timer);
|
| 1523 | }
|
| 1524 |
|
| 1525 | static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
|
| 1526 | {
|
| 1527 | u32 ppis = 0;
|
| 1528 | bool valid;
|
| 1529 |
|
| 1530 | mutex_lock(&vcpu->kvm->arch.config_lock);
|
| 1531 |
|
| 1532 | for (int i = 0; i < nr_timers(vcpu); i++) {
|
| 1533 | struct arch_timer_context *ctx;
|
| 1534 | int irq;
|
| 1535 |
|
| 1536 | ctx = vcpu_get_timer(vcpu, i);
|
| 1537 | irq = timer_irq(ctx);
|
| 1538 |
|
| 1539 |
|
| 1540 | if (vgic_is_v5(vcpu->kvm) && irq != get_vgic_ppi(vcpu->kvm, default_ppi[i]))
|
| 1541 | break;
|
| 1542 |
|
| 1543 | if (kvm_vgic_set_owner(vcpu, irq, ctx))
|
| 1544 | break;
|
| 1545 |
|
| 1546 | |
| 1547 | * We know by construction that we only have PPIs, so all values
|
| 1548 | * are less than 32 for non-GICv5 VGICs. On GICv5, they are
|
| 1549 | * architecturally defined to be under 32 too. However, we mask
|
| 1550 | * off most of the bits as we might be presented with a GICv5
|
| 1551 | * style PPI where the type is encoded in the top-bits.
|
| 1552 |
|
| 1553 | ppis |= BIT(irq & 0x1f);
|
| 1554 | }
|
| 1555 |
|
| 1556 | valid = hweight32(ppis) == nr_timers(vcpu);
|
| 1557 |
|
| 1558 | if (valid)
|
| 1559 | set_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, &vcpu->kvm->arch.flags);
|
| 1560 |
|
| 1561 | mutex_unlock(&vcpu->kvm->arch.config_lock);
|
| 1562 |
|
| 1563 | return valid;
|
| 1564 | }
|
| 1565 |
|
| 1566 | static bool kvm_arch_timer_get_input_level(int vintid)
|
| 1567 | {
|
| 1568 | struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
|
| 1569 |
|
| 1570 | if (WARN(!vcpu, "No vcpu context!\n"))
|
| 1571 | return false;
|
| 1572 |
|
| 1573 | for (int i = 0; i < nr_timers(vcpu); i++) {
|
| 1574 | struct arch_timer_context *ctx;
|
| 1575 |
|
| 1576 | ctx = vcpu_get_timer(vcpu, i);
|
| 1577 | if (timer_irq(ctx) == vintid)
|
| 1578 | return kvm_timer_pending(ctx);
|
| 1579 | }
|
| 1580 |
|
| 1581 |
|
| 1582 | WARN_RATELIMIT(1, "timer INTID%d unknown\n", vintid);
|
| 1583 |
|
| 1584 | return false;
|
| 1585 | }
|
| 1586 |
|
| 1587 | int kvm_timer_enable(struct kvm_vcpu *vcpu)
|
| 1588 | {
|
| 1589 | struct arch_timer_cpu *timer = vcpu_timer(vcpu);
|
| 1590 | const struct irq_ops *ops;
|
| 1591 | struct timer_map map;
|
| 1592 | int ret;
|
| 1593 |
|
| 1594 | if (timer->enabled)
|
| 1595 | return 0;
|
| 1596 |
|
| 1597 |
|
| 1598 | if (!irqchip_in_kernel(vcpu->kvm))
|
| 1599 | goto no_vgic;
|
| 1600 |
|
| 1601 | |
| 1602 | * At this stage, we have the guarantee that the vgic is both
|
| 1603 | * available and initialized.
|
| 1604 |
|
| 1605 | if (!timer_irqs_are_valid(vcpu)) {
|
| 1606 | kvm_debug("incorrectly configured timer irqs\n");
|
| 1607 | return -EINVAL;
|
| 1608 | }
|
| 1609 |
|
| 1610 | get_timer_map(vcpu, &map);
|
| 1611 |
|
| 1612 | ops = vgic_is_v5(vcpu->kvm) ? &arch_timer_irq_ops_vgic_v5 :
|
| 1613 | &arch_timer_irq_ops;
|
| 1614 |
|
| 1615 | for (int i = 0; i < nr_timers(vcpu); i++)
|
| 1616 | kvm_vgic_set_irq_ops(vcpu, timer_irq(vcpu_get_timer(vcpu, i)), ops);
|
| 1617 |
|
| 1618 | ret = kvm_vgic_map_phys_irq(vcpu,
|
| 1619 | map.direct_vtimer->host_timer_irq,
|
| 1620 | timer_irq(map.direct_vtimer));
|
| 1621 | if (ret)
|
| 1622 | return ret;
|
| 1623 |
|
| 1624 | if (map.direct_ptimer)
|
| 1625 | ret = kvm_vgic_map_phys_irq(vcpu,
|
| 1626 | map.direct_ptimer->host_timer_irq,
|
| 1627 | timer_irq(map.direct_ptimer));
|
| 1628 | if (ret)
|
| 1629 | return ret;
|
| 1630 |
|
| 1631 | no_vgic:
|
| 1632 | timer->enabled = 1;
|
| 1633 | return 0;
|
| 1634 | }
|
| 1635 |
|
| 1636 |
|
| 1637 | void kvm_timer_init_vhe(void)
|
| 1638 | {
|
| 1639 | if (cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF))
|
| 1640 | sysreg_clear_set(cnthctl_el2, 0, CNTHCTL_ECV);
|
| 1641 | }
|
| 1642 |
|
| 1643 | int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
|
| 1644 | {
|
| 1645 | int __user *uaddr = (int __user *)(long)attr->addr;
|
| 1646 | int irq, idx, ret = 0;
|
| 1647 |
|
| 1648 | if (!irqchip_in_kernel(vcpu->kvm))
|
| 1649 | return -EINVAL;
|
| 1650 |
|
| 1651 | if (get_user(irq, uaddr))
|
| 1652 | return -EFAULT;
|
| 1653 |
|
| 1654 | if (!(irq_is_ppi(vcpu->kvm, irq)))
|
| 1655 | return -EINVAL;
|
| 1656 |
|
| 1657 | guard(mutex)(&vcpu->kvm->arch.config_lock);
|
| 1658 |
|
| 1659 | if (test_bit(KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE,
|
| 1660 | &vcpu->kvm->arch.flags)) {
|
| 1661 | return -EBUSY;
|
| 1662 | }
|
| 1663 |
|
| 1664 | switch (attr->attr) {
|
| 1665 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
|
| 1666 | idx = TIMER_VTIMER;
|
| 1667 | break;
|
| 1668 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
|
| 1669 | idx = TIMER_PTIMER;
|
| 1670 | break;
|
| 1671 | case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
|
| 1672 | idx = TIMER_HVTIMER;
|
| 1673 | break;
|
| 1674 | case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
|
| 1675 | idx = TIMER_HPTIMER;
|
| 1676 | break;
|
| 1677 | default:
|
| 1678 | return -ENXIO;
|
| 1679 | }
|
| 1680 |
|
| 1681 | |
| 1682 | * We cannot validate the IRQ unicity before we run, so take it at
|
| 1683 | * face value. The verdict will be given on first vcpu run, for each
|
| 1684 | * vcpu. Yes this is late. Blame it on the stupid API.
|
| 1685 |
|
| 1686 | vcpu->kvm->arch.timer_data.ppi[idx] = irq;
|
| 1687 |
|
| 1688 | return ret;
|
| 1689 | }
|
| 1690 |
|
| 1691 | int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
|
| 1692 | {
|
| 1693 | int __user *uaddr = (int __user *)(long)attr->addr;
|
| 1694 | struct arch_timer_context *timer;
|
| 1695 | int irq;
|
| 1696 |
|
| 1697 | switch (attr->attr) {
|
| 1698 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
|
| 1699 | timer = vcpu_vtimer(vcpu);
|
| 1700 | break;
|
| 1701 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
|
| 1702 | timer = vcpu_ptimer(vcpu);
|
| 1703 | break;
|
| 1704 | case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
|
| 1705 | timer = vcpu_hvtimer(vcpu);
|
| 1706 | break;
|
| 1707 | case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
|
| 1708 | timer = vcpu_hptimer(vcpu);
|
| 1709 | break;
|
| 1710 | default:
|
| 1711 | return -ENXIO;
|
| 1712 | }
|
| 1713 |
|
| 1714 | irq = timer_irq(timer);
|
| 1715 | return put_user(irq, uaddr);
|
| 1716 | }
|
| 1717 |
|
| 1718 | int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
|
| 1719 | {
|
| 1720 | switch (attr->attr) {
|
| 1721 | case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
|
| 1722 | case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
|
| 1723 | case KVM_ARM_VCPU_TIMER_IRQ_HVTIMER:
|
| 1724 | case KVM_ARM_VCPU_TIMER_IRQ_HPTIMER:
|
| 1725 | return 0;
|
| 1726 | }
|
| 1727 |
|
| 1728 | return -ENXIO;
|
| 1729 | }
|
| 1730 |
|
| 1731 | int kvm_vm_ioctl_set_counter_offset(struct kvm *kvm,
|
| 1732 | struct kvm_arm_counter_offset *offset)
|
| 1733 | {
|
| 1734 | int ret = 0;
|
| 1735 |
|
| 1736 | if (offset->reserved)
|
| 1737 | return -EINVAL;
|
| 1738 |
|
| 1739 | if (kvm_vm_is_protected(kvm))
|
| 1740 | return -EINVAL;
|
| 1741 |
|
| 1742 | mutex_lock(&kvm->lock);
|
| 1743 |
|
| 1744 | if (!kvm_trylock_all_vcpus(kvm)) {
|
| 1745 | set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
|
| 1746 |
|
| 1747 | |
| 1748 | * If userspace decides to set the offset using this
|
| 1749 | * API rather than merely restoring the counter
|
| 1750 | * values, the offset applies to both the virtual and
|
| 1751 | * physical views.
|
| 1752 |
|
| 1753 | kvm->arch.timer_data.voffset = offset->counter_offset;
|
| 1754 | kvm->arch.timer_data.poffset = offset->counter_offset;
|
| 1755 |
|
| 1756 | kvm_unlock_all_vcpus(kvm);
|
| 1757 | } else {
|
| 1758 | ret = -EBUSY;
|
| 1759 | }
|
| 1760 |
|
| 1761 | mutex_unlock(&kvm->lock);
|
| 1762 |
|
| 1763 | return ret;
|
| 1764 | }
|