projects/linux·blob master · refs

1217 lines · 29.8 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * KVM paravirt_ops implementation
4 *
5 * Copyright (C) 2007, Red Hat, Inc., Ingo Molnar <[email protected]>
6 * Copyright IBM Corporation, 2007
7 * Authors: Anthony Liguori <[email protected]>
8 */
9
10#define pr_fmt(fmt) "kvm-guest: " fmt
11
12#include <linux/context_tracking.h>
13#include <linux/init.h>
14#include <linux/irq.h>
15#include <linux/kernel.h>
16#include <linux/kvm_para.h>
17#include <linux/cpu.h>
18#include <linux/mm.h>
19#include <linux/highmem.h>
20#include <linux/hardirq.h>
21#include <linux/notifier.h>
22#include <linux/reboot.h>
23#include <linux/hash.h>
24#include <linux/sched.h>
25#include <linux/slab.h>
26#include <linux/kprobes.h>
27#include <linux/nmi.h>
28#include <linux/swait.h>
29#include <linux/syscore_ops.h>
30#include <linux/cc_platform.h>
31#include <linux/efi.h>
32#include <linux/kvm_types.h>
33#include <linux/sched/cputime.h>
34#include <asm/timer.h>
35#include <asm/cpu.h>
36#include <asm/traps.h>
37#include <asm/desc.h>
38#include <asm/tlbflush.h>
39#include <asm/apic.h>
40#include <asm/apicdef.h>
41#include <asm/hypervisor.h>
42#include <asm/mtrr.h>
43#include <asm/tlb.h>
44#include <asm/cpuid/api.h>
45#include <asm/cpuidle_haltpoll.h>
46#include <asm/msr.h>
47#include <asm/ptrace.h>
48#include <asm/reboot.h>
49#include <asm/svm.h>
50#include <asm/e820/api.h>
51
52DEFINE_STATIC_KEY_FALSE_RO(kvm_async_pf_enabled);
53
54static int kvmapf = 1;
55
56static int __init parse_no_kvmapf(char *arg)
57{
58 kvmapf = 0;
59 return 0;
60}
61
62early_param("no-kvmapf", parse_no_kvmapf);
63
64static int steal_acc = 1;
65static int __init parse_no_stealacc(char *arg)
66{
67 steal_acc = 0;
68 return 0;
69}
70
71early_param("no-steal-acc", parse_no_stealacc);
72
73static DEFINE_PER_CPU_READ_MOSTLY(bool, async_pf_enabled);
74static DEFINE_PER_CPU_DECRYPTED(struct kvm_vcpu_pv_apf_data, apf_reason) __aligned(64);
75DEFINE_PER_CPU_DECRYPTED(struct kvm_steal_time, steal_time) __aligned(64) __visible;
76static int has_steal_clock = 0;
77
78static int has_guest_poll = 0;
79
80#define KVM_TASK_SLEEP_HASHBITS 8
81#define KVM_TASK_SLEEP_HASHSIZE (1<<KVM_TASK_SLEEP_HASHBITS)
82
83struct kvm_task_sleep_node {
84 struct hlist_node link;
85 struct swait_queue_head wq;
86 u32 token;
87 int cpu;
88 bool dummy;
89};
90
91static struct kvm_task_sleep_head {
92 raw_spinlock_t lock;
93 struct hlist_head list;
94} async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
95
96static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b,
97 u32 token)
98{
99 struct hlist_node *p;
100
101 hlist_for_each(p, &b->list) {
102 struct kvm_task_sleep_node *n =
103 hlist_entry(p, typeof(*n), link);
104 if (n->token == token)
105 return n;
106 }
107
108 return NULL;
109}
110
111static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n)
112{
113 u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
114 struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
115 struct kvm_task_sleep_node *e;
116
117 raw_spin_lock(&b->lock);
118 e = _find_apf_task(b, token);
119 if (e) {
120 struct kvm_task_sleep_node *dummy = NULL;
121
122 /*
123 * The entry can either be a 'dummy' entry (which is put on the
124 * list when wake-up happens ahead of APF handling completion)
125 * or a token from another task which should not be touched.
126 */
127 if (e->dummy) {
128 hlist_del(&e->link);
129 dummy = e;
130 }
131
132 raw_spin_unlock(&b->lock);
133 kfree(dummy);
134 return false;
135 }
136
137 n->token = token;
138 n->cpu = smp_processor_id();
139 n->dummy = false;
140 init_swait_queue_head(&n->wq);
141 hlist_add_head(&n->link, &b->list);
142 raw_spin_unlock(&b->lock);
143 return true;
144}
145
146/*
147 * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled
148 * @token: Token to identify the sleep node entry
149 *
150 * Invoked from the async pagefault handling code or from the VM exit page
151 * fault handler. In both cases RCU is watching.
152 */
153void kvm_async_pf_task_wait_schedule(u32 token)
154{
155 struct kvm_task_sleep_node n;
156 DECLARE_SWAITQUEUE(wait);
157
158 lockdep_assert_irqs_disabled();
159
160 if (!kvm_async_pf_queue_task(token, &n))
161 return;
162
163 for (;;) {
164 prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
165 if (hlist_unhashed(&n.link))
166 break;
167
168 local_irq_enable();
169 schedule();
170 local_irq_disable();
171 }
172 finish_swait(&n.wq, &wait);
173}
174EXPORT_SYMBOL_FOR_KVM(kvm_async_pf_task_wait_schedule);
175
176static void apf_task_wake_one(struct kvm_task_sleep_node *n)
177{
178 hlist_del_init(&n->link);
179 if (swq_has_sleeper(&n->wq))
180 swake_up_one(&n->wq);
181}
182
183static void apf_task_wake_all(void)
184{
185 int i;
186
187 for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
188 struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
189 struct kvm_task_sleep_node *n;
190 struct hlist_node *p, *next;
191
192 raw_spin_lock(&b->lock);
193 hlist_for_each_safe(p, next, &b->list) {
194 n = hlist_entry(p, typeof(*n), link);
195 if (n->cpu == smp_processor_id())
196 apf_task_wake_one(n);
197 }
198 raw_spin_unlock(&b->lock);
199 }
200}
201
202static void kvm_async_pf_task_wake(u32 token)
203{
204 u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
205 struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
206 struct kvm_task_sleep_node *n, *dummy = NULL;
207
208 if (token == ~0) {
209 apf_task_wake_all();
210 return;
211 }
212
213again:
214 raw_spin_lock(&b->lock);
215 n = _find_apf_task(b, token);
216 if (!n) {
217 /*
218 * Async #PF not yet handled, add a dummy entry for the token.
219 * Allocating the token must be down outside of the raw lock
220 * as the allocator is preemptible on PREEMPT_RT kernels.
221 */
222 if (!dummy) {
223 raw_spin_unlock(&b->lock);
224 dummy = kzalloc_obj(*dummy, GFP_ATOMIC);
225
226 /*
227 * Continue looping on allocation failure, eventually
228 * the async #PF will be handled and allocating a new
229 * node will be unnecessary.
230 */
231 if (!dummy)
232 cpu_relax();
233
234 /*
235 * Recheck for async #PF completion before enqueueing
236 * the dummy token to avoid duplicate list entries.
237 */
238 goto again;
239 }
240 dummy->token = token;
241 dummy->cpu = smp_processor_id();
242 dummy->dummy = true;
243 init_swait_queue_head(&dummy->wq);
244 hlist_add_head(&dummy->link, &b->list);
245 dummy = NULL;
246 } else {
247 apf_task_wake_one(n);
248 }
249 raw_spin_unlock(&b->lock);
250
251 /* A dummy token might be allocated and ultimately not used. */
252 kfree(dummy);
253}
254
255noinstr u32 kvm_read_and_reset_apf_flags(void)
256{
257 u32 flags = 0;
258
259 if (__this_cpu_read(async_pf_enabled)) {
260 flags = __this_cpu_read(apf_reason.flags);
261 __this_cpu_write(apf_reason.flags, 0);
262 }
263
264 return flags;
265}
266EXPORT_SYMBOL_FOR_KVM(kvm_read_and_reset_apf_flags);
267
268static __always_inline bool kvm_apf_reason_pending(void)
269{
270 return __this_cpu_read(async_pf_enabled) &&
271 __this_cpu_read(apf_reason.flags);
272}
273
274noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
275{
276 irqentry_state_t state;
277 u32 flags;
278
279 if (!kvm_apf_reason_pending())
280 return false;
281
282 /*
283 * The guest does not enable KVM_ASYNC_PF_SEND_ALWAYS, so the host
284 * delivers async #PF only to user mode (or as a #PF VM-exit, which KVM
285 * reads itself). A #PF from kernel mode can still arrive between that
286 * delivery and the read of the reason flags, inside an NMI that
287 * interrupted it, for instance perf reading user memory for a call
288 * chain. Such a fault is not async, so leave the async PF flags
289 * untouched for the outer handler. Outside NMI context, on the other
290 * hand, async PF flags on a kernel-mode fault mean the host is broken.
291 */
292 if (!user_mode(regs)) {
293 if (likely(in_nmi()))
294 return false;
295 state = irqentry_enter(regs);
296 instrumentation_begin();
297 panic("Host injected async #PF in kernel mode\n");
298 }
299
300 flags = kvm_read_and_reset_apf_flags();
301 if (!flags)
302 return false;
303
304 state = irqentry_enter(regs);
305 instrumentation_begin();
306
307 /*
308 * If the host managed to inject an async #PF into an interrupt
309 * disabled region, then die hard as this is not going to end well
310 * and the host side is seriously broken.
311 */
312 if (unlikely(!(regs->flags & X86_EFLAGS_IF)))
313 panic("Host injected async #PF in interrupt disabled region\n");
314
315 if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
316 /* Page is swapped out by the host. */
317 kvm_async_pf_task_wait_schedule(token);
318 } else {
319 WARN_ONCE(1, "Unexpected async PF flags: %x\n", flags);
320 }
321
322 instrumentation_end();
323 irqentry_exit(regs, state);
324 return true;
325}
326
327DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt)
328{
329 struct pt_regs *old_regs = set_irq_regs(regs);
330 u32 token;
331
332 apic_eoi();
333
334 inc_irq_stat(HYPERVISOR_CALLBACK);
335
336 if (__this_cpu_read(async_pf_enabled)) {
337 token = __this_cpu_read(apf_reason.token);
338 kvm_async_pf_task_wake(token);
339 __this_cpu_write(apf_reason.token, 0);
340 wrmsrq(MSR_KVM_ASYNC_PF_ACK, 1);
341 }
342
343 set_irq_regs(old_regs);
344}
345
346static void __init paravirt_ops_setup(void)
347{
348 pv_info.name = "KVM";
349
350 if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
351 pv_info.io_delay = false;
352
353#ifdef CONFIG_X86_IO_APIC
354 no_timer_check = 1;
355#endif
356}
357
358static void kvm_register_steal_time(void)
359{
360 int cpu = smp_processor_id();
361 struct kvm_steal_time *st = &per_cpu(steal_time, cpu);
362
363 if (!has_steal_clock)
364 return;
365
366 wrmsrq(MSR_KVM_STEAL_TIME, (slow_virt_to_phys(st) | KVM_MSR_ENABLED));
367 pr_debug("stealtime: cpu %d, msr %llx\n", cpu,
368 (unsigned long long) slow_virt_to_phys(st));
369}
370
371static DEFINE_PER_CPU_DECRYPTED(unsigned long, kvm_apic_eoi) = KVM_PV_EOI_DISABLED;
372
373static notrace __maybe_unused void kvm_guest_apic_eoi_write(void)
374{
375 /**
376 * This relies on __test_and_clear_bit to modify the memory
377 * in a way that is atomic with respect to the local CPU.
378 * The hypervisor only accesses this memory from the local CPU so
379 * there's no need for lock or memory barriers.
380 * An optimization barrier is implied in apic write.
381 */
382 if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
383 return;
384 apic_native_eoi();
385}
386
387static void kvm_guest_cpu_init(void)
388{
389 if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) {
390 u64 pa;
391
392 WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled));
393
394 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));
395 pa |= KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
396
397 if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
398 pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
399
400 wrmsrq(MSR_KVM_ASYNC_PF_INT, HYPERVISOR_CALLBACK_VECTOR);
401
402 wrmsrq(MSR_KVM_ASYNC_PF_EN, pa);
403 __this_cpu_write(async_pf_enabled, true);
404 pr_debug("setup async PF for cpu %d\n", smp_processor_id());
405 }
406
407 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
408 unsigned long pa;
409
410 /* Size alignment is implied but just to make it explicit. */
411 BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
412 __this_cpu_write(kvm_apic_eoi, 0);
413 pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
414 | KVM_MSR_ENABLED;
415 wrmsrq(MSR_KVM_PV_EOI_EN, pa);
416 }
417
418 if (has_steal_clock)
419 kvm_register_steal_time();
420}
421
422static void kvm_pv_disable_apf(void)
423{
424 if (!__this_cpu_read(async_pf_enabled))
425 return;
426
427 wrmsrq(MSR_KVM_ASYNC_PF_EN, 0);
428 __this_cpu_write(async_pf_enabled, false);
429
430 pr_debug("disable async PF for cpu %d\n", smp_processor_id());
431}
432
433static void kvm_disable_steal_time(void)
434{
435 if (!has_steal_clock)
436 return;
437
438 wrmsrq(MSR_KVM_STEAL_TIME, 0);
439}
440
441static u64 kvm_steal_clock(int cpu)
442{
443 u64 steal;
444 struct kvm_steal_time *src;
445 int version;
446
447 src = &per_cpu(steal_time, cpu);
448 do {
449 version = src->version;
450 virt_rmb();
451 steal = src->steal;
452 virt_rmb();
453 } while ((version & 1) || (version != src->version));
454
455 return steal;
456}
457
458static inline __init void __set_percpu_decrypted(void *ptr, unsigned long size)
459{
460 early_set_memory_decrypted((unsigned long) ptr, size);
461}
462
463/*
464 * Iterate through all possible CPUs and map the memory region pointed
465 * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once.
466 *
467 * Note: we iterate through all possible CPUs to ensure that CPUs
468 * hotplugged will have their per-cpu variable already mapped as
469 * decrypted.
470 */
471static void __init sev_map_percpu_data(void)
472{
473 int cpu;
474
475 if (cc_vendor != CC_VENDOR_AMD ||
476 !cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
477 return;
478
479 for_each_possible_cpu(cpu) {
480 __set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason));
481 __set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time));
482 __set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi));
483 }
484}
485
486static void kvm_guest_cpu_offline(bool shutdown)
487{
488 kvm_disable_steal_time();
489 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
490 wrmsrq(MSR_KVM_PV_EOI_EN, 0);
491 if (kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL))
492 wrmsrq(MSR_KVM_MIGRATION_CONTROL, 0);
493 kvm_pv_disable_apf();
494 if (!shutdown)
495 apf_task_wake_all();
496 kvmclock_disable();
497}
498
499static int kvm_cpu_online(unsigned int cpu)
500{
501 unsigned long flags;
502
503 local_irq_save(flags);
504 kvm_guest_cpu_init();
505 local_irq_restore(flags);
506 return 0;
507}
508
509#ifdef CONFIG_SMP
510
511static DEFINE_PER_CPU(cpumask_var_t, __pv_cpu_mask);
512
513static bool pv_tlb_flush_supported(void)
514{
515 return (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
516 !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
517 kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
518 !boot_cpu_has(X86_FEATURE_MWAIT) &&
519 (num_possible_cpus() != 1));
520}
521
522static bool pv_ipi_supported(void)
523{
524 return (kvm_para_has_feature(KVM_FEATURE_PV_SEND_IPI) &&
525 (num_possible_cpus() != 1));
526}
527
528static bool pv_sched_yield_supported(void)
529{
530 return (kvm_para_has_feature(KVM_FEATURE_PV_SCHED_YIELD) &&
531 !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
532 kvm_para_has_feature(KVM_FEATURE_STEAL_TIME) &&
533 !boot_cpu_has(X86_FEATURE_MWAIT) &&
534 (num_possible_cpus() != 1));
535}
536
537#define KVM_IPI_CLUSTER_SIZE (2 * BITS_PER_LONG)
538
539static void __send_ipi_mask(const struct cpumask *mask, int vector)
540{
541 unsigned long flags;
542 int cpu, min = 0, max = 0;
543#ifdef CONFIG_X86_64
544 __uint128_t ipi_bitmap = 0;
545#else
546 u64 ipi_bitmap = 0;
547#endif
548 u32 apic_id, icr;
549 long ret;
550
551 if (cpumask_empty(mask))
552 return;
553
554 local_irq_save(flags);
555
556 switch (vector) {
557 default:
558 icr = APIC_DM_FIXED | vector;
559 break;
560 case NMI_VECTOR:
561 icr = APIC_DM_NMI;
562 break;
563 }
564
565 for_each_cpu(cpu, mask) {
566 apic_id = per_cpu(x86_cpu_to_apicid, cpu);
567 if (!ipi_bitmap) {
568 min = max = apic_id;
569 } else if (apic_id < min && max - apic_id < KVM_IPI_CLUSTER_SIZE) {
570 ipi_bitmap <<= min - apic_id;
571 min = apic_id;
572 } else if (apic_id > min && apic_id < min + KVM_IPI_CLUSTER_SIZE) {
573 max = apic_id < max ? max : apic_id;
574 } else {
575 ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
576 (unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
577 WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld",
578 ret);
579 min = max = apic_id;
580 ipi_bitmap = 0;
581 }
582 __set_bit(apic_id - min, (unsigned long *)&ipi_bitmap);
583 }
584
585 if (ipi_bitmap) {
586 ret = kvm_hypercall4(KVM_HC_SEND_IPI, (unsigned long)ipi_bitmap,
587 (unsigned long)(ipi_bitmap >> BITS_PER_LONG), min, icr);
588 WARN_ONCE(ret < 0, "kvm-guest: failed to send PV IPI: %ld",
589 ret);
590 }
591
592 local_irq_restore(flags);
593}
594
595static void kvm_send_ipi_mask(const struct cpumask *mask, int vector)
596{
597 __send_ipi_mask(mask, vector);
598}
599
600static void kvm_send_ipi_mask_allbutself(const struct cpumask *mask, int vector)
601{
602 unsigned int this_cpu = smp_processor_id();
603 struct cpumask *new_mask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
604 const struct cpumask *local_mask;
605
606 cpumask_copy(new_mask, mask);
607 cpumask_clear_cpu(this_cpu, new_mask);
608 local_mask = new_mask;
609 __send_ipi_mask(local_mask, vector);
610}
611
612static int __init setup_efi_kvm_sev_migration(void)
613{
614 efi_char16_t efi_sev_live_migration_enabled[] = L"SevLiveMigrationEnabled";
615 efi_guid_t efi_variable_guid = AMD_SEV_MEM_ENCRYPT_GUID;
616 efi_status_t status;
617 unsigned long size;
618 bool enabled;
619
620 if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) ||
621 !kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL))
622 return 0;
623
624 if (!efi_enabled(EFI_BOOT))
625 return 0;
626
627 if (!efi_enabled(EFI_RUNTIME_SERVICES)) {
628 pr_info("%s : EFI runtime services are not enabled\n", __func__);
629 return 0;
630 }
631
632 size = sizeof(enabled);
633
634 /* Get variable contents into buffer */
635 status = efi.get_variable(efi_sev_live_migration_enabled,
636 &efi_variable_guid, NULL, &size, &enabled);
637
638 if (status == EFI_NOT_FOUND) {
639 pr_info("%s : EFI live migration variable not found\n", __func__);
640 return 0;
641 }
642
643 if (status != EFI_SUCCESS) {
644 pr_info("%s : EFI variable retrieval failed\n", __func__);
645 return 0;
646 }
647
648 if (enabled == 0) {
649 pr_info("%s: live migration disabled in EFI\n", __func__);
650 return 0;
651 }
652
653 pr_info("%s : live migration enabled in EFI\n", __func__);
654 wrmsrq(MSR_KVM_MIGRATION_CONTROL, KVM_MIGRATION_READY);
655
656 return 1;
657}
658
659late_initcall(setup_efi_kvm_sev_migration);
660
661/*
662 * Set the IPI entry points
663 */
664static __init void kvm_setup_pv_ipi(void)
665{
666 apic_update_callback(send_IPI_mask, kvm_send_ipi_mask);
667 apic_update_callback(send_IPI_mask_allbutself, kvm_send_ipi_mask_allbutself);
668 pr_info("setup PV IPIs\n");
669}
670
671static void kvm_smp_send_call_func_ipi(const struct cpumask *mask)
672{
673 int cpu;
674
675 native_send_call_func_ipi(mask);
676
677 /* Make sure other vCPUs get a chance to run if they need to. */
678 for_each_cpu(cpu, mask) {
679 if (!idle_cpu(cpu) && vcpu_is_preempted(cpu)) {
680 kvm_hypercall1(KVM_HC_SCHED_YIELD, per_cpu(x86_cpu_to_apicid, cpu));
681 break;
682 }
683 }
684}
685
686static void kvm_flush_tlb_multi(const struct cpumask *cpumask,
687 const struct flush_tlb_info *info)
688{
689 u8 state;
690 int cpu;
691 struct kvm_steal_time *src;
692 struct cpumask *flushmask;
693
694 guard(preempt)();
695 flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask);
696 cpumask_copy(flushmask, cpumask);
697 /*
698 * We have to call flush only on online vCPUs. And
699 * queue flush_on_enter for pre-empted vCPUs
700 */
701 for_each_cpu(cpu, flushmask) {
702 /*
703 * The local vCPU is never preempted, so we do not explicitly
704 * skip check for local vCPU - it will never be cleared from
705 * flushmask.
706 */
707 src = &per_cpu(steal_time, cpu);
708 state = READ_ONCE(src->preempted);
709 if ((state & KVM_VCPU_PREEMPTED)) {
710 if (try_cmpxchg(&src->preempted, &state,
711 state | KVM_VCPU_FLUSH_TLB))
712 __cpumask_clear_cpu(cpu, flushmask);
713 }
714 }
715
716 native_flush_tlb_multi(flushmask, info);
717}
718
719static __init int kvm_alloc_cpumask(void)
720{
721 int cpu;
722
723 if (!kvm_para_available() || nopv)
724 return 0;
725
726 if (pv_tlb_flush_supported() || pv_ipi_supported())
727 for_each_possible_cpu(cpu) {
728 zalloc_cpumask_var_node(per_cpu_ptr(&__pv_cpu_mask, cpu),
729 GFP_KERNEL, cpu_to_node(cpu));
730 }
731
732 return 0;
733}
734arch_initcall(kvm_alloc_cpumask);
735
736static void __init kvm_smp_prepare_boot_cpu(void)
737{
738 /*
739 * Map the per-cpu variables as decrypted before kvm_guest_cpu_init()
740 * shares the guest physical address with the hypervisor.
741 */
742 sev_map_percpu_data();
743
744 kvm_guest_cpu_init();
745 native_smp_prepare_boot_cpu();
746 kvm_spinlock_init();
747}
748
749static int kvm_cpu_down_prepare(unsigned int cpu)
750{
751 unsigned long flags;
752
753 local_irq_save(flags);
754 kvm_guest_cpu_offline(false);
755 local_irq_restore(flags);
756 return 0;
757}
758
759#endif
760
761static int kvm_suspend(void *data)
762{
763 u64 val = 0;
764
765 kvm_guest_cpu_offline(false);
766
767#ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
768 if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
769 rdmsrq(MSR_KVM_POLL_CONTROL, val);
770 has_guest_poll = !(val & 1);
771#endif
772 return 0;
773}
774
775static void kvm_resume(void *data)
776{
777 kvm_cpu_online(raw_smp_processor_id());
778
779#ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
780 if (kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL) && has_guest_poll)
781 wrmsrq(MSR_KVM_POLL_CONTROL, 0);
782#endif
783}
784
785static const struct syscore_ops kvm_syscore_ops = {
786 .suspend = kvm_suspend,
787 .resume = kvm_resume,
788};
789
790static struct syscore kvm_syscore = {
791 .ops = &kvm_syscore_ops,
792};
793
794static void kvm_pv_guest_cpu_reboot(void *unused)
795{
796 kvm_guest_cpu_offline(true);
797}
798
799static int kvm_pv_reboot_notify(struct notifier_block *nb,
800 unsigned long code, void *unused)
801{
802 if (code == SYS_RESTART)
803 on_each_cpu(kvm_pv_guest_cpu_reboot, NULL, 1);
804 return NOTIFY_DONE;
805}
806
807static struct notifier_block kvm_pv_reboot_nb = {
808 .notifier_call = kvm_pv_reboot_notify,
809};
810
811/*
812 * After a PV feature is registered, the host will keep writing to the
813 * registered memory location. If the guest happens to shutdown, this memory
814 * won't be valid. In cases like kexec, in which you install a new kernel, this
815 * means a random memory location will be kept being written.
816 */
817#ifdef CONFIG_CRASH_DUMP
818static void kvm_crash_shutdown(struct pt_regs *regs)
819{
820 kvm_guest_cpu_offline(true);
821 native_machine_crash_shutdown(regs);
822}
823#endif
824
825#if defined(CONFIG_X86_32) || !defined(CONFIG_SMP)
826bool __kvm_vcpu_is_preempted(long cpu);
827
828__visible bool __kvm_vcpu_is_preempted(long cpu)
829{
830 struct kvm_steal_time *src = &per_cpu(steal_time, cpu);
831
832 return !!(src->preempted & KVM_VCPU_PREEMPTED);
833}
834PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted);
835
836#else
837
838#include <asm/asm-offsets.h>
839
840extern bool __raw_callee_save___kvm_vcpu_is_preempted(long);
841
842/*
843 * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and
844 * restoring to/from the stack.
845 */
846#define PV_VCPU_PREEMPTED_ASM \
847 "movq __per_cpu_offset(,%rdi,8), %rax\n\t" \
848 "cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax)\n\t" \
849 "setne %al\n\t"
850
851DEFINE_ASM_FUNC(__raw_callee_save___kvm_vcpu_is_preempted,
852 PV_VCPU_PREEMPTED_ASM, .text);
853#endif
854
855static void __init kvm_guest_init(void)
856{
857 int i;
858
859 paravirt_ops_setup();
860 register_reboot_notifier(&kvm_pv_reboot_nb);
861 for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
862 raw_spin_lock_init(&async_pf_sleepers[i].lock);
863
864 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
865 has_steal_clock = 1;
866 static_call_update(pv_steal_clock, kvm_steal_clock);
867
868#ifdef CONFIG_PARAVIRT_SPINLOCKS
869 pv_ops_lock.vcpu_is_preempted =
870 PV_CALLEE_SAVE(__kvm_vcpu_is_preempted);
871#endif
872 }
873
874 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
875 apic_update_callback(eoi, kvm_guest_apic_eoi_write);
876
877 if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_INT) && kvmapf) {
878 static_branch_enable(&kvm_async_pf_enabled);
879 sysvec_install(HYPERVISOR_CALLBACK_VECTOR, sysvec_kvm_asyncpf_interrupt);
880 }
881
882#ifdef CONFIG_SMP
883 if (pv_tlb_flush_supported()) {
884 pv_ops.mmu.flush_tlb_multi = kvm_flush_tlb_multi;
885 pr_info("KVM setup pv remote TLB flush\n");
886 }
887
888 smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
889 if (pv_sched_yield_supported()) {
890 smp_ops.send_call_func_ipi = kvm_smp_send_call_func_ipi;
891 pr_info("setup PV sched yield\n");
892 }
893 if (cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "x86/kvm:online",
894 kvm_cpu_online, kvm_cpu_down_prepare) < 0)
895 pr_err("failed to install cpu hotplug callbacks\n");
896#else
897 sev_map_percpu_data();
898 kvm_guest_cpu_init();
899#endif
900
901#ifdef CONFIG_CRASH_DUMP
902 machine_ops.crash_shutdown = kvm_crash_shutdown;
903#endif
904
905 register_syscore(&kvm_syscore);
906
907 /*
908 * Hard lockup detection is enabled by default. Disable it, as guests
909 * can get false positives too easily, for example if the host is
910 * overcommitted.
911 */
912 hardlockup_detector_disable();
913}
914
915static noinline uint32_t __kvm_cpuid_base(void)
916{
917 if (boot_cpu_data.cpuid_level < 0)
918 return 0; /* So we don't blow up on old processors */
919
920 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
921 return cpuid_base_hypervisor(KVM_SIGNATURE, 0);
922
923 return 0;
924}
925
926static inline uint32_t kvm_cpuid_base(void)
927{
928 static int kvm_cpuid_base = -1;
929
930 if (kvm_cpuid_base == -1)
931 kvm_cpuid_base = __kvm_cpuid_base();
932
933 return kvm_cpuid_base;
934}
935
936bool kvm_para_available(void)
937{
938 return kvm_cpuid_base() != 0;
939}
940EXPORT_SYMBOL_GPL(kvm_para_available);
941
942unsigned int kvm_arch_para_features(void)
943{
944 return cpuid_eax(kvm_cpuid_base() | KVM_CPUID_FEATURES);
945}
946
947unsigned int kvm_arch_para_hints(void)
948{
949 return cpuid_edx(kvm_cpuid_base() | KVM_CPUID_FEATURES);
950}
951EXPORT_SYMBOL_GPL(kvm_arch_para_hints);
952
953static uint32_t __init kvm_detect(void)
954{
955 return kvm_cpuid_base();
956}
957
958static void __init kvm_apic_init(void)
959{
960#ifdef CONFIG_SMP
961 if (pv_ipi_supported())
962 kvm_setup_pv_ipi();
963#endif
964}
965
966static bool __init kvm_msi_ext_dest_id(void)
967{
968 return kvm_para_has_feature(KVM_FEATURE_MSI_EXT_DEST_ID);
969}
970
971static void kvm_sev_hc_page_enc_status(unsigned long pfn, int npages, bool enc)
972{
973 kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, pfn << PAGE_SHIFT, npages,
974 KVM_MAP_GPA_RANGE_ENC_STAT(enc) | KVM_MAP_GPA_RANGE_PAGE_SZ_4K);
975}
976
977static void __init kvm_init_platform(void)
978{
979 u64 tolud = PFN_PHYS(e820__end_of_low_ram_pfn());
980 /*
981 * Note, hardware requires variable MTRR ranges to be power-of-2 sized
982 * and naturally aligned. But when forcing guest MTRR state, Linux
983 * doesn't program the forced ranges into hardware. Don't bother doing
984 * the math to generate a technically-legal range.
985 */
986 struct mtrr_var_range pci_hole = {
987 .base_lo = tolud | X86_MEMTYPE_UC,
988 .mask_lo = (u32)(~(SZ_4G - tolud - 1)) | MTRR_PHYSMASK_V,
989 .mask_hi = (BIT_ULL(boot_cpu_data.x86_phys_bits) - 1) >> 32,
990 };
991
992 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) &&
993 kvm_para_has_feature(KVM_FEATURE_MIGRATION_CONTROL)) {
994 unsigned long nr_pages;
995 int i;
996
997 pv_ops.mmu.notify_page_enc_status_changed =
998 kvm_sev_hc_page_enc_status;
999
1000 /*
1001 * Reset the host's shared pages list related to kernel
1002 * specific page encryption status settings before we load a
1003 * new kernel by kexec. Reset the page encryption status
1004 * during early boot instead of just before kexec to avoid SMP
1005 * races during kvm_pv_guest_cpu_reboot().
1006 * NOTE: We cannot reset the complete shared pages list
1007 * here as we need to retain the UEFI/OVMF firmware
1008 * specific settings.
1009 */
1010
1011 for (i = 0; i < e820_table->nr_entries; i++) {
1012 struct e820_entry *entry = &e820_table->entries[i];
1013
1014 if (entry->type != E820_TYPE_RAM)
1015 continue;
1016
1017 nr_pages = DIV_ROUND_UP(entry->size, PAGE_SIZE);
1018
1019 kvm_sev_hypercall3(KVM_HC_MAP_GPA_RANGE, entry->addr,
1020 nr_pages,
1021 KVM_MAP_GPA_RANGE_ENCRYPTED | KVM_MAP_GPA_RANGE_PAGE_SZ_4K);
1022 }
1023
1024 /*
1025 * Ensure that _bss_decrypted section is marked as decrypted in the
1026 * shared pages list.
1027 */
1028 early_set_mem_enc_dec_hypercall((unsigned long)__start_bss_decrypted,
1029 __end_bss_decrypted - __start_bss_decrypted, 0);
1030
1031 /*
1032 * If not booted using EFI, enable Live migration support.
1033 */
1034 if (!efi_enabled(EFI_BOOT))
1035 wrmsrq(MSR_KVM_MIGRATION_CONTROL,
1036 KVM_MIGRATION_READY);
1037 }
1038 kvmclock_init();
1039 x86_platform.apic_post_init = kvm_apic_init;
1040
1041 /*
1042 * Set WB as the default cache mode for SEV-SNP and TDX, with a single
1043 * UC range for the legacy PCI hole, e.g. so that devices that expect
1044 * to get UC/WC mappings don't get surprised with WB.
1045 */
1046 guest_force_mtrr_state(&pci_hole, 1, MTRR_TYPE_WRBACK);
1047}
1048
1049#if defined(CONFIG_AMD_MEM_ENCRYPT)
1050static void kvm_sev_es_hcall_prepare(struct ghcb *ghcb, struct pt_regs *regs)
1051{
1052 /* RAX and CPL are already in the GHCB */
1053 ghcb_set_rbx(ghcb, regs->bx);
1054 ghcb_set_rcx(ghcb, regs->cx);
1055 ghcb_set_rdx(ghcb, regs->dx);
1056 ghcb_set_rsi(ghcb, regs->si);
1057}
1058
1059static bool kvm_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs)
1060{
1061 /* No checking of the return state needed */
1062 return true;
1063}
1064#endif
1065
1066const __initconst struct hypervisor_x86 x86_hyper_kvm = {
1067 .name = "KVM",
1068 .detect = kvm_detect,
1069 .type = X86_HYPER_KVM,
1070 .init.guest_late_init = kvm_guest_init,
1071 .init.x2apic_available = kvm_para_available,
1072 .init.msi_ext_dest_id = kvm_msi_ext_dest_id,
1073 .init.init_platform = kvm_init_platform,
1074#if defined(CONFIG_AMD_MEM_ENCRYPT)
1075 .runtime.sev_es_hcall_prepare = kvm_sev_es_hcall_prepare,
1076 .runtime.sev_es_hcall_finish = kvm_sev_es_hcall_finish,
1077#endif
1078};
1079
1080static __init int activate_jump_labels(void)
1081{
1082 if (has_steal_clock) {
1083 static_branch_inc(&paravirt_steal_enabled);
1084 if (steal_acc)
1085 static_branch_inc(&paravirt_steal_rq_enabled);
1086 }
1087
1088 return 0;
1089}
1090arch_initcall(activate_jump_labels);
1091
1092#ifdef CONFIG_PARAVIRT_SPINLOCKS
1093
1094/* Kick a cpu by its apicid. Used to wake up a halted vcpu */
1095static void kvm_kick_cpu(int cpu)
1096{
1097 unsigned long flags = 0;
1098 u32 apicid;
1099
1100 apicid = per_cpu(x86_cpu_to_apicid, cpu);
1101 kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
1102}
1103
1104#include <asm/qspinlock.h>
1105
1106static void kvm_wait(u8 *ptr, u8 val)
1107{
1108 if (in_nmi())
1109 return;
1110
1111 /*
1112 * halt until it's our turn and kicked. Note that we do safe halt
1113 * for irq enabled case to avoid hang when lock info is overwritten
1114 * in irq spinlock slowpath and no spurious interrupt occur to save us.
1115 */
1116 if (irqs_disabled()) {
1117 if (READ_ONCE(*ptr) == val)
1118 halt();
1119 } else {
1120 local_irq_disable();
1121
1122 /* safe_halt() will enable IRQ */
1123 if (READ_ONCE(*ptr) == val)
1124 safe_halt();
1125 else
1126 local_irq_enable();
1127 }
1128}
1129
1130/*
1131 * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
1132 */
1133void __init kvm_spinlock_init(void)
1134{
1135 /*
1136 * Disable PV spinlocks and use native qspinlock when dedicated pCPUs
1137 * are available.
1138 */
1139 if (kvm_para_has_hint(KVM_HINTS_REALTIME)) {
1140 pr_info("PV spinlocks disabled with KVM_HINTS_REALTIME hints\n");
1141 goto out;
1142 }
1143
1144 if (num_possible_cpus() == 1) {
1145 pr_info("PV spinlocks disabled, single CPU\n");
1146 goto out;
1147 }
1148
1149 if (nopvspin) {
1150 pr_info("PV spinlocks disabled, forced by \"nopvspin\" parameter\n");
1151 goto out;
1152 }
1153
1154 /*
1155 * In case host doesn't support KVM_FEATURE_PV_UNHALT there is still an
1156 * advantage of keeping virt_spin_lock_key enabled: virt_spin_lock() is
1157 * preferred over native qspinlock when vCPU is preempted.
1158 */
1159 if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT)) {
1160 pr_info("PV spinlocks disabled, no host support\n");
1161 return;
1162 }
1163
1164 pr_info("PV spinlocks enabled\n");
1165
1166 __pv_init_lock_hash();
1167 static_call_update(queued_spin_lock_slowpath, __pv_queued_spin_lock_slowpath);
1168 static_call_update(queued_spin_unlock, __raw_callee_save___pv_queued_spin_unlock);
1169 pv_ops_lock.wait = kvm_wait;
1170 pv_ops_lock.kick = kvm_kick_cpu;
1171
1172 /*
1173 * When PV spinlock is enabled which is preferred over
1174 * virt_spin_lock(), virt_spin_lock_key's value is meaningless.
1175 * Just disable it anyway.
1176 */
1177out:
1178 static_branch_disable(&virt_spin_lock_key);
1179}
1180
1181#endif /* CONFIG_PARAVIRT_SPINLOCKS */
1182
1183#ifdef CONFIG_ARCH_CPUIDLE_HALTPOLL
1184
1185static void kvm_disable_host_haltpoll(void *i)
1186{
1187 wrmsrq(MSR_KVM_POLL_CONTROL, 0);
1188}
1189
1190static void kvm_enable_host_haltpoll(void *i)
1191{
1192 wrmsrq(MSR_KVM_POLL_CONTROL, 1);
1193}
1194
1195void arch_haltpoll_enable(unsigned int cpu)
1196{
1197 if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL)) {
1198 pr_err_once("host does not support poll control\n");
1199 pr_err_once("host upgrade recommended\n");
1200 return;
1201 }
1202
1203 /* Enable guest halt poll disables host halt poll */
1204 smp_call_function_single(cpu, kvm_disable_host_haltpoll, NULL, 1);
1205}
1206EXPORT_SYMBOL_GPL(arch_haltpoll_enable);
1207
1208void arch_haltpoll_disable(unsigned int cpu)
1209{
1210 if (!kvm_para_has_feature(KVM_FEATURE_POLL_CONTROL))
1211 return;
1212
1213 /* Disable guest halt poll enables host halt poll */
1214 smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1);
1215}
1216EXPORT_SYMBOL_GPL(arch_haltpoll_disable);
1217#endif