projects/linux·blob master · refs

1490 lines · 39.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2018, Google LLC.
4 */
5
6#include "linux/bitmap.h"
7#include "test_util.h"
8#include "kvm_util.h"
9#include "pmu.h"
10#include "processor.h"
11#include "smm.h"
12#include "svm_util.h"
13#include "sev.h"
14#include "vmx.h"
15
16#ifndef NUM_INTERRUPTS
17#define NUM_INTERRUPTS 256
18#endif
19
20#define KERNEL_CS 0x8
21#define KERNEL_DS 0x10
22#define KERNEL_TSS 0x18
23
24gva_t exception_handlers;
25bool host_cpu_is_amd;
26bool host_cpu_is_intel;
27bool host_cpu_is_hygon;
28bool host_cpu_is_amd_compatible;
29bool is_forced_emulation_enabled;
30u64 guest_tsc_khz;
31struct kvm_mmu guest_mmu;
32
33struct guest_regs guest_regs;
34
35const char *ex_str(int vector)
36{
37 switch (vector) {
38#define VEC_STR(v) case v##_VECTOR: return "#" #v
39 case DE_VECTOR: return "no exception";
40 case KVM_MAGIC_DE_VECTOR: return "#DE";
41 VEC_STR(DB);
42 VEC_STR(NMI);
43 VEC_STR(BP);
44 VEC_STR(OF);
45 VEC_STR(BR);
46 VEC_STR(UD);
47 VEC_STR(NM);
48 VEC_STR(DF);
49 VEC_STR(TS);
50 VEC_STR(NP);
51 VEC_STR(SS);
52 VEC_STR(GP);
53 VEC_STR(PF);
54 VEC_STR(MF);
55 VEC_STR(AC);
56 VEC_STR(MC);
57 VEC_STR(XM);
58 VEC_STR(VE);
59 VEC_STR(CP);
60 VEC_STR(HV);
61 VEC_STR(VC);
62 VEC_STR(SX);
63 default: return "#??";
64#undef VEC_STR
65 }
66}
67
68static void regs_dump(FILE *stream, struct kvm_regs *regs, u8 indent)
69{
70 fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
71 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
72 indent, "",
73 regs->rax, regs->rbx, regs->rcx, regs->rdx);
74 fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
75 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
76 indent, "",
77 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
78 fprintf(stream, "%*sr8: 0x%.16llx r9: 0x%.16llx "
79 "r10: 0x%.16llx r11: 0x%.16llx\n",
80 indent, "",
81 regs->r8, regs->r9, regs->r10, regs->r11);
82 fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
83 "r14: 0x%.16llx r15: 0x%.16llx\n",
84 indent, "",
85 regs->r12, regs->r13, regs->r14, regs->r15);
86 fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
87 indent, "",
88 regs->rip, regs->rflags);
89}
90
91static void segment_dump(FILE *stream, struct kvm_segment *segment,
92 u8 indent)
93{
94 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
95 "selector: 0x%.4x type: 0x%.2x\n",
96 indent, "", segment->base, segment->limit,
97 segment->selector, segment->type);
98 fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
99 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
100 indent, "", segment->present, segment->dpl,
101 segment->db, segment->s, segment->l);
102 fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
103 "unusable: 0x%.2x padding: 0x%.2x\n",
104 indent, "", segment->g, segment->avl,
105 segment->unusable, segment->padding);
106}
107
108static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
109 u8 indent)
110{
111 fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
112 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
113 indent, "", dtable->base, dtable->limit,
114 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
115}
116
117static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent)
118{
119 unsigned int i;
120
121 fprintf(stream, "%*scs:\n", indent, "");
122 segment_dump(stream, &sregs->cs, indent + 2);
123 fprintf(stream, "%*sds:\n", indent, "");
124 segment_dump(stream, &sregs->ds, indent + 2);
125 fprintf(stream, "%*ses:\n", indent, "");
126 segment_dump(stream, &sregs->es, indent + 2);
127 fprintf(stream, "%*sfs:\n", indent, "");
128 segment_dump(stream, &sregs->fs, indent + 2);
129 fprintf(stream, "%*sgs:\n", indent, "");
130 segment_dump(stream, &sregs->gs, indent + 2);
131 fprintf(stream, "%*sss:\n", indent, "");
132 segment_dump(stream, &sregs->ss, indent + 2);
133 fprintf(stream, "%*str:\n", indent, "");
134 segment_dump(stream, &sregs->tr, indent + 2);
135 fprintf(stream, "%*sldt:\n", indent, "");
136 segment_dump(stream, &sregs->ldt, indent + 2);
137
138 fprintf(stream, "%*sgdt:\n", indent, "");
139 dtable_dump(stream, &sregs->gdt, indent + 2);
140 fprintf(stream, "%*sidt:\n", indent, "");
141 dtable_dump(stream, &sregs->idt, indent + 2);
142
143 fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
144 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
145 indent, "",
146 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
147 fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
148 "apic_base: 0x%.16llx\n",
149 indent, "",
150 sregs->cr8, sregs->efer, sregs->apic_base);
151
152 fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
153 for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
154 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
155 sregs->interrupt_bitmap[i]);
156 }
157}
158
159bool kvm_is_tdp_enabled(void)
160{
161 if (host_cpu_is_intel)
162 return get_kvm_intel_param_bool("ept");
163 else
164 return get_kvm_amd_param_bool("npt");
165}
166
167static void virt_mmu_init(struct kvm_vm *vm, struct kvm_mmu *mmu,
168 struct pte_masks *pte_masks)
169{
170 /* If needed, create the top-level page table. */
171 if (!mmu->pgd_created) {
172 mmu->pgd = vm_alloc_page_table(vm);
173 mmu->pgd_created = true;
174 mmu->arch.pte_masks = *pte_masks;
175 }
176
177 TEST_ASSERT(mmu->pgtable_levels == 4 || mmu->pgtable_levels == 5,
178 "Selftests MMU only supports 4-level and 5-level paging, not %u-level paging",
179 mmu->pgtable_levels);
180}
181
182void virt_arch_pgd_alloc(struct kvm_vm *vm)
183{
184 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
185 "Unknown or unsupported guest mode: 0x%x", vm->mode);
186
187 struct pte_masks pte_masks = (struct pte_masks){
188 .present = BIT_ULL(0),
189 .writable = BIT_ULL(1),
190 .user = BIT_ULL(2),
191 .accessed = BIT_ULL(5),
192 .dirty = BIT_ULL(6),
193 .huge = BIT_ULL(7),
194 .nx = BIT_ULL(63),
195 .executable = 0,
196 .c = vm->arch.c_bit,
197 .s = vm->arch.s_bit,
198 };
199
200 virt_mmu_init(vm, &vm->mmu, &pte_masks);
201}
202
203void tdp_mmu_init(struct kvm_vm *vm, int pgtable_levels,
204 struct pte_masks *pte_masks)
205{
206 TEST_ASSERT(!vm->stage2_mmu.pgtable_levels, "TDP MMU already initialized");
207
208 vm->stage2_mmu.pgtable_levels = pgtable_levels;
209 virt_mmu_init(vm, &vm->stage2_mmu, pte_masks);
210}
211
212static void *virt_get_pte(struct kvm_vm *vm, struct kvm_mmu *mmu,
213 u64 *parent_pte, gva_t gva, int level)
214{
215 u64 pt_gpa = PTE_GET_PA(*parent_pte);
216 u64 *page_table = addr_gpa2hva(vm, pt_gpa);
217 int index = (gva >> PG_LEVEL_SHIFT(level)) & 0x1ffu;
218
219 TEST_ASSERT((*parent_pte == mmu->pgd) || is_present_pte(mmu, parent_pte),
220 "Parent PTE (level %d) not PRESENT for gva: 0x%08lx",
221 level + 1, gva);
222
223 return &page_table[index];
224}
225
226static u64 *virt_create_upper_pte(struct kvm_vm *vm,
227 struct kvm_mmu *mmu,
228 u64 *parent_pte,
229 gva_t gva,
230 gpa_t gpa,
231 int current_level,
232 int target_level)
233{
234 u64 *pte = virt_get_pte(vm, mmu, parent_pte, gva, current_level);
235
236 gpa = vm_untag_gpa(vm, gpa);
237
238 if (!is_present_pte(mmu, pte)) {
239 *pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |
240 PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |
241 PTE_ALWAYS_SET_MASK(mmu);
242 if (current_level == target_level)
243 *pte |= PTE_HUGE_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);
244 else
245 *pte |= vm_alloc_page_table(vm) & PHYSICAL_PAGE_MASK;
246 } else {
247 /*
248 * Entry already present. Assert that the caller doesn't want
249 * a hugepage at this level, and that there isn't a hugepage at
250 * this level.
251 */
252 TEST_ASSERT(current_level != target_level,
253 "Cannot create hugepage at level: %u, gva: 0x%lx",
254 current_level, gva);
255 TEST_ASSERT(!is_huge_pte(mmu, pte),
256 "Cannot create page table at level: %u, gva: 0x%lx",
257 current_level, gva);
258 }
259 return pte;
260}
261
262void __virt_pg_map(struct kvm_vm *vm, struct kvm_mmu *mmu, gva_t gva,
263 gpa_t gpa, int level)
264{
265 const u64 pg_size = PG_LEVEL_SIZE(level);
266 u64 *pte = &mmu->pgd;
267 int current_level;
268
269 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
270 "Unknown or unsupported guest mode: 0x%x", vm->mode);
271
272 TEST_ASSERT((gva % pg_size) == 0,
273 "Virtual address not aligned,\n"
274 "gva: 0x%lx page size: 0x%lx", gva, pg_size);
275 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
276 "Invalid virtual address, gva: 0x%lx", gva);
277 TEST_ASSERT((gpa % pg_size) == 0,
278 "Physical address not aligned,\n"
279 " gpa: 0x%lx page size: 0x%lx", gpa, pg_size);
280 TEST_ASSERT((gpa >> vm->page_shift) <= vm->max_gfn,
281 "Physical address beyond maximum supported,\n"
282 " gpa: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
283 gpa, vm->max_gfn, vm->page_size);
284 TEST_ASSERT(vm_untag_gpa(vm, gpa) == gpa,
285 "Unexpected bits in gpa: %lx", gpa);
286
287 TEST_ASSERT(!PTE_EXECUTABLE_MASK(mmu) || !PTE_NX_MASK(mmu),
288 "X and NX bit masks cannot be used simultaneously");
289
290 /*
291 * Allocate upper level page tables, if not already present. Return
292 * early if a hugepage was created.
293 */
294 for (current_level = mmu->pgtable_levels;
295 current_level > PG_LEVEL_4K;
296 current_level--) {
297 pte = virt_create_upper_pte(vm, mmu, pte, gva, gpa,
298 current_level, level);
299 if (is_huge_pte(mmu, pte))
300 return;
301 }
302
303 /* Fill in page table entry. */
304 pte = virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);
305 TEST_ASSERT(!is_present_pte(mmu, pte),
306 "PTE already present for 4k page at gva: 0x%lx", gva);
307 *pte = PTE_PRESENT_MASK(mmu) | PTE_READABLE_MASK(mmu) |
308 PTE_WRITABLE_MASK(mmu) | PTE_EXECUTABLE_MASK(mmu) |
309 PTE_ALWAYS_SET_MASK(mmu) | (gpa & PHYSICAL_PAGE_MASK);
310
311 /*
312 * Neither SEV nor TDX supports shared page tables, so only the final
313 * leaf PTE needs manually set the C/S-bit.
314 */
315 if (vm_is_gpa_protected(vm, gpa))
316 *pte |= PTE_C_BIT_MASK(mmu);
317 else
318 *pte |= PTE_S_BIT_MASK(mmu);
319}
320
321void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa)
322{
323 __virt_pg_map(vm, &vm->mmu, gva, gpa, PG_LEVEL_4K);
324}
325
326void virt_map_level(struct kvm_vm *vm, gva_t gva, gpa_t gpa,
327 u64 nr_bytes, int level)
328{
329 u64 pg_size = PG_LEVEL_SIZE(level);
330 u64 nr_pages = nr_bytes / pg_size;
331 int i;
332
333 TEST_ASSERT(nr_bytes % pg_size == 0,
334 "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx",
335 nr_bytes, pg_size);
336
337 for (i = 0; i < nr_pages; i++) {
338 __virt_pg_map(vm, &vm->mmu, gva, gpa, level);
339 sparsebit_set_num(vm->vpages_mapped, gva >> vm->page_shift,
340 nr_bytes / PAGE_SIZE);
341
342 gva += pg_size;
343 gpa += pg_size;
344 }
345}
346
347static bool vm_is_target_pte(struct kvm_mmu *mmu, u64 *pte,
348 int *level, int current_level)
349{
350 if (is_huge_pte(mmu, pte)) {
351 TEST_ASSERT(*level == PG_LEVEL_NONE ||
352 *level == current_level,
353 "Unexpected hugepage at level %d", current_level);
354 *level = current_level;
355 }
356
357 return *level == current_level;
358}
359
360static u64 *__vm_get_page_table_entry(struct kvm_vm *vm,
361 struct kvm_mmu *mmu,
362 gva_t gva,
363 int *level)
364{
365 int va_width = 12 + (mmu->pgtable_levels) * 9;
366 u64 *pte = &mmu->pgd;
367 int current_level;
368
369 TEST_ASSERT(!vm->arch.is_pt_protected,
370 "Walking page tables of protected guests is impossible");
371
372 TEST_ASSERT(*level >= PG_LEVEL_NONE && *level <= mmu->pgtable_levels,
373 "Invalid PG_LEVEL_* '%d'", *level);
374
375 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
376 "Unknown or unsupported guest mode: 0x%x", vm->mode);
377 TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (gva >> vm->page_shift)),
378 "Invalid virtual address, gva: 0x%lx", gva);
379 /*
380 * Check that the gva is a sign-extended va_width value.
381 */
382 TEST_ASSERT(gva == (((s64)gva << (64 - va_width) >> (64 - va_width))),
383 "Canonical check failed. The virtual address is invalid.");
384
385 for (current_level = mmu->pgtable_levels;
386 current_level > PG_LEVEL_4K;
387 current_level--) {
388 pte = virt_get_pte(vm, mmu, pte, gva, current_level);
389 if (vm_is_target_pte(mmu, pte, level, current_level))
390 return pte;
391 }
392
393 return virt_get_pte(vm, mmu, pte, gva, PG_LEVEL_4K);
394}
395
396u64 *tdp_get_pte(struct kvm_vm *vm, u64 l2_gpa)
397{
398 int level = PG_LEVEL_4K;
399
400 return __vm_get_page_table_entry(vm, &vm->stage2_mmu, l2_gpa, &level);
401}
402
403u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva)
404{
405 int level = PG_LEVEL_4K;
406
407 return __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);
408}
409
410void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent)
411{
412 struct kvm_mmu *mmu = &vm->mmu;
413 u64 *pml4e, *pml4e_start;
414 u64 *pdpe, *pdpe_start;
415 u64 *pde, *pde_start;
416 u64 *pte, *pte_start;
417
418 if (!mmu->pgd_created)
419 return;
420
421 fprintf(stream, "%*s "
422 " no\n", indent, "");
423 fprintf(stream, "%*s index hvaddr gpaddr "
424 "addr w exec dirty\n",
425 indent, "");
426 pml4e_start = (u64 *)addr_gpa2hva(vm, mmu->pgd);
427 for (u16 n1 = 0; n1 <= 0x1ffu; n1++) {
428 pml4e = &pml4e_start[n1];
429 if (!is_present_pte(mmu, pml4e))
430 continue;
431 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10llx %u "
432 " %u\n",
433 indent, "",
434 pml4e - pml4e_start, pml4e,
435 addr_hva2gpa(vm, pml4e), PTE_GET_PFN(*pml4e),
436 is_writable_pte(mmu, pml4e), is_nx_pte(mmu, pml4e));
437
438 pdpe_start = addr_gpa2hva(vm, *pml4e & PHYSICAL_PAGE_MASK);
439 for (u16 n2 = 0; n2 <= 0x1ffu; n2++) {
440 pdpe = &pdpe_start[n2];
441 if (!is_present_pte(mmu, pdpe))
442 continue;
443 fprintf(stream, "%*spdpe 0x%-3zx %p 0x%-12lx 0x%-10llx "
444 "%u %u\n",
445 indent, "",
446 pdpe - pdpe_start, pdpe,
447 addr_hva2gpa(vm, pdpe),
448 PTE_GET_PFN(*pdpe), is_writable_pte(mmu, pdpe),
449 is_nx_pte(mmu, pdpe));
450
451 pde_start = addr_gpa2hva(vm, *pdpe & PHYSICAL_PAGE_MASK);
452 for (u16 n3 = 0; n3 <= 0x1ffu; n3++) {
453 pde = &pde_start[n3];
454 if (!is_present_pte(mmu, pde))
455 continue;
456 fprintf(stream, "%*spde 0x%-3zx %p "
457 "0x%-12lx 0x%-10llx %u %u\n",
458 indent, "", pde - pde_start, pde,
459 addr_hva2gpa(vm, pde),
460 PTE_GET_PFN(*pde), is_writable_pte(mmu, pde),
461 is_nx_pte(mmu, pde));
462
463 pte_start = addr_gpa2hva(vm, *pde & PHYSICAL_PAGE_MASK);
464 for (u16 n4 = 0; n4 <= 0x1ffu; n4++) {
465 pte = &pte_start[n4];
466 if (!is_present_pte(mmu, pte))
467 continue;
468 fprintf(stream, "%*spte 0x%-3zx %p "
469 "0x%-12lx 0x%-10llx %u %u "
470 " %u 0x%-10lx\n",
471 indent, "",
472 pte - pte_start, pte,
473 addr_hva2gpa(vm, pte),
474 PTE_GET_PFN(*pte),
475 is_writable_pte(mmu, pte),
476 is_nx_pte(mmu, pte),
477 is_dirty_pte(mmu, pte),
478 ((u64)n1 << 27)
479 | ((u64)n2 << 18)
480 | ((u64)n3 << 9)
481 | ((u64)n4));
482 }
483 }
484 }
485 }
486}
487
488void vm_enable_tdp(struct kvm_vm *vm)
489{
490 if (kvm_cpu_has(X86_FEATURE_VMX))
491 vm_enable_ept(vm);
492 else
493 vm_enable_npt(vm);
494}
495
496bool kvm_cpu_has_tdp(void)
497{
498 return kvm_cpu_has_ept() || kvm_cpu_has_npt();
499}
500
501void __tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size, int level)
502{
503 size_t page_size = PG_LEVEL_SIZE(level);
504 size_t npages = size / page_size;
505
506 TEST_ASSERT(l2_gpa + size > l2_gpa, "L2 GPA overflow");
507 TEST_ASSERT(gpa + size > gpa, "GPA overflow");
508
509 while (npages--) {
510 __virt_pg_map(vm, &vm->stage2_mmu, l2_gpa, gpa, level);
511 l2_gpa += page_size;
512 gpa += page_size;
513 }
514}
515
516void tdp_map(struct kvm_vm *vm, gpa_t l2_gpa, gpa_t gpa, u64 size)
517{
518 __tdp_map(vm, l2_gpa, gpa, size, PG_LEVEL_4K);
519}
520
521/* Prepare an identity extended page table that maps all the
522 * physical pages in VM.
523 */
524void tdp_identity_map_default_memslots(struct kvm_vm *vm)
525{
526 u32 s, memslot = 0;
527 sparsebit_idx_t i, last;
528 struct userspace_mem_region *region = memslot2region(vm, memslot);
529
530 /* Only memslot 0 is mapped here, ensure it's the only one being used */
531 for (s = 0; s < NR_MEM_REGIONS; s++)
532 TEST_ASSERT_EQ(vm->memslots[s], 0);
533
534 i = (region->region.guest_phys_addr >> vm->page_shift) - 1;
535 last = i + (region->region.memory_size >> vm->page_shift);
536 for (;;) {
537 i = sparsebit_next_clear(region->unused_phy_pages, i);
538 if (i > last)
539 break;
540
541 tdp_map(vm, (u64)i << vm->page_shift,
542 (u64)i << vm->page_shift, 1 << vm->page_shift);
543 }
544}
545
546/* Identity map a region with 1GiB Pages. */
547void tdp_identity_map_1g(struct kvm_vm *vm, u64 addr, u64 size)
548{
549 __tdp_map(vm, addr, addr, size, PG_LEVEL_1G);
550}
551
552/*
553 * Set Unusable Segment
554 *
555 * Input Args: None
556 *
557 * Output Args:
558 * segp - Pointer to segment register
559 *
560 * Return: None
561 *
562 * Sets the segment register pointed to by @segp to an unusable state.
563 */
564static void kvm_seg_set_unusable(struct kvm_segment *segp)
565{
566 memset(segp, 0, sizeof(*segp));
567 segp->unusable = true;
568}
569
570static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
571{
572 void *gdt = addr_gva2hva(vm, vm->arch.gdt);
573 struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
574
575 desc->limit0 = segp->limit & 0xFFFF;
576 desc->base0 = segp->base & 0xFFFF;
577 desc->base1 = segp->base >> 16;
578 desc->type = segp->type;
579 desc->s = segp->s;
580 desc->dpl = segp->dpl;
581 desc->p = segp->present;
582 desc->limit1 = segp->limit >> 16;
583 desc->avl = segp->avl;
584 desc->l = segp->l;
585 desc->db = segp->db;
586 desc->g = segp->g;
587 desc->base2 = segp->base >> 24;
588 if (!segp->s)
589 desc->base3 = segp->base >> 32;
590}
591
592static void kvm_seg_set_kernel_code_64bit(struct kvm_segment *segp)
593{
594 memset(segp, 0, sizeof(*segp));
595 segp->selector = KERNEL_CS;
596 segp->limit = 0xFFFFFFFFu;
597 segp->s = 0x1; /* kTypeCodeData */
598 segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
599 * | kFlagCodeReadable
600 */
601 segp->g = true;
602 segp->l = true;
603 segp->present = 1;
604}
605
606static void kvm_seg_set_kernel_data_64bit(struct kvm_segment *segp)
607{
608 memset(segp, 0, sizeof(*segp));
609 segp->selector = KERNEL_DS;
610 segp->limit = 0xFFFFFFFFu;
611 segp->s = 0x1; /* kTypeCodeData */
612 segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
613 * | kFlagDataWritable
614 */
615 segp->g = true;
616 segp->present = true;
617}
618
619gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva)
620{
621 int level = PG_LEVEL_NONE;
622 u64 *pte = __vm_get_page_table_entry(vm, &vm->mmu, gva, &level);
623
624 TEST_ASSERT(is_present_pte(&vm->mmu, pte),
625 "Leaf PTE not PRESENT for gva: 0x%08lx", gva);
626
627 /*
628 * No need for a hugepage mask on the PTE, x86-64 requires the "unused"
629 * address bits to be zero.
630 */
631 return vm_untag_gpa(vm, PTE_GET_PA(*pte)) | (gva & ~HUGEPAGE_MASK(level));
632}
633
634static void kvm_seg_set_tss_64bit(gva_t base, struct kvm_segment *segp)
635{
636 memset(segp, 0, sizeof(*segp));
637 segp->base = base;
638 segp->limit = 0x67;
639 segp->selector = KERNEL_TSS;
640 segp->type = 0xb;
641 segp->present = 1;
642}
643
644static void vcpu_init_sregs(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
645{
646 struct kvm_sregs sregs;
647
648 TEST_ASSERT(vm->mode == VM_MODE_PXXVYY_4K,
649 "Unknown or unsupported guest mode: 0x%x", vm->mode);
650
651 /* Set mode specific system register values. */
652 vcpu_sregs_get(vcpu, &sregs);
653
654 sregs.idt.base = vm->arch.idt;
655 sregs.idt.limit = NUM_INTERRUPTS * sizeof(struct idt_entry) - 1;
656 sregs.gdt.base = vm->arch.gdt;
657 sregs.gdt.limit = getpagesize() - 1;
658
659 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
660 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
661 if (kvm_cpu_has(X86_FEATURE_XSAVE))
662 sregs.cr4 |= X86_CR4_OSXSAVE;
663 if (vm->mmu.pgtable_levels == 5)
664 sregs.cr4 |= X86_CR4_LA57;
665 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
666
667 kvm_seg_set_unusable(&sregs.ldt);
668 kvm_seg_set_kernel_code_64bit(&sregs.cs);
669 kvm_seg_set_kernel_data_64bit(&sregs.ds);
670 kvm_seg_set_kernel_data_64bit(&sregs.es);
671 kvm_seg_set_kernel_data_64bit(&sregs.gs);
672 kvm_seg_set_tss_64bit(vm->arch.tss, &sregs.tr);
673
674 sregs.cr3 = vm->mmu.pgd;
675 vcpu_sregs_set(vcpu, &sregs);
676}
677
678static void vcpu_init_xcrs(struct kvm_vm *vm, struct kvm_vcpu *vcpu)
679{
680 struct kvm_xcrs xcrs = {
681 .nr_xcrs = 1,
682 .xcrs[0].xcr = 0,
683 .xcrs[0].value = kvm_cpu_supported_xcr0(),
684 };
685
686 if (!kvm_cpu_has(X86_FEATURE_XSAVE))
687 return;
688
689 vcpu_xcrs_set(vcpu, &xcrs);
690}
691
692static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr,
693 int dpl, unsigned short selector)
694{
695 struct idt_entry *base =
696 (struct idt_entry *)addr_gva2hva(vm, vm->arch.idt);
697 struct idt_entry *e = &base[vector];
698
699 memset(e, 0, sizeof(*e));
700 e->offset0 = addr;
701 e->selector = selector;
702 e->ist = 0;
703 e->type = 14;
704 e->dpl = dpl;
705 e->p = 1;
706 e->offset1 = addr >> 16;
707 e->offset2 = addr >> 32;
708}
709
710static bool kvm_fixup_exception(struct ex_regs *regs)
711{
712 if (regs->r9 != KVM_EXCEPTION_MAGIC || regs->rip != regs->r10)
713 return false;
714
715 if (regs->vector == DE_VECTOR)
716 regs->vector = KVM_MAGIC_DE_VECTOR;
717
718 regs->rip = regs->r11;
719 regs->r9 = regs->vector;
720 regs->r10 = regs->error_code;
721 return true;
722}
723
724void route_exception(struct ex_regs *regs)
725{
726 typedef void(*handler)(struct ex_regs *);
727 handler *handlers = (handler *)exception_handlers;
728
729 if (handlers && handlers[regs->vector]) {
730 handlers[regs->vector](regs);
731 return;
732 }
733
734 if (kvm_fixup_exception(regs))
735 return;
736
737 GUEST_FAIL("Unhandled exception '0x%lx' at guest RIP '0x%lx'",
738 regs->vector, regs->rip);
739}
740
741static void vm_init_descriptor_tables(struct kvm_vm *vm)
742{
743 extern void *idt_handlers;
744 struct kvm_segment seg;
745 int i;
746
747 vm->arch.gdt = __vm_alloc_page(vm, MEM_REGION_DATA);
748 vm->arch.idt = __vm_alloc_page(vm, MEM_REGION_DATA);
749 vm->handlers = __vm_alloc_page(vm, MEM_REGION_DATA);
750 vm->arch.tss = __vm_alloc_page(vm, MEM_REGION_DATA);
751
752 /* Handlers have the same address in both address spaces.*/
753 for (i = 0; i < NUM_INTERRUPTS; i++)
754 set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0, KERNEL_CS);
755
756 *(gva_t *)addr_gva2hva(vm, (gva_t)(&exception_handlers)) = vm->handlers;
757
758 kvm_seg_set_kernel_code_64bit(&seg);
759 kvm_seg_fill_gdt_64bit(vm, &seg);
760
761 kvm_seg_set_kernel_data_64bit(&seg);
762 kvm_seg_fill_gdt_64bit(vm, &seg);
763
764 kvm_seg_set_tss_64bit(vm->arch.tss, &seg);
765 kvm_seg_fill_gdt_64bit(vm, &seg);
766}
767
768void vm_install_exception_handler(struct kvm_vm *vm, int vector,
769 void (*handler)(struct ex_regs *))
770{
771 gva_t *handlers = (gva_t *)addr_gva2hva(vm, vm->handlers);
772
773 handlers[vector] = (gva_t)handler;
774}
775
776void assert_on_unhandled_exception(struct kvm_vcpu *vcpu)
777{
778 struct ucall uc;
779
780 if (get_ucall(vcpu, &uc) == UCALL_ABORT)
781 REPORT_GUEST_ASSERT(uc);
782}
783
784gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages)
785{
786 int size = nr_pages * getpagesize();
787 gva_t stack_gva;
788
789 stack_gva = __vm_alloc(vm, size, DEFAULT_GUEST_STACK_VADDR_MIN, MEM_REGION_DATA);
790 stack_gva += size;
791
792 /*
793 * Align stack to match calling sequence requirements in section "The
794 * Stack Frame" of the System V ABI AMD64 Architecture Processor
795 * Supplement, which requires the value (%rsp + 8) to be a multiple of
796 * 16 when control is transferred to the function entry point.
797 *
798 * If this code is ever used to launch a vCPU with 32-bit entry point it
799 * may need to subtract 4 bytes instead of 8 bytes.
800 */
801 TEST_ASSERT(IS_ALIGNED(stack_gva, PAGE_SIZE),
802 "__vm_alloc() did not provide a page-aligned address");
803 stack_gva -= 8;
804
805 return stack_gva;
806}
807
808void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus)
809{
810 int r;
811
812 TEST_ASSERT(kvm_has_cap(KVM_CAP_GET_TSC_KHZ),
813 "Require KVM_GET_TSC_KHZ to provide udelay() to guest.");
814
815 vm_create_irqchip(vm);
816 vm_init_descriptor_tables(vm);
817
818 sync_global_to_guest(vm, host_cpu_is_intel);
819 sync_global_to_guest(vm, host_cpu_is_amd);
820 sync_global_to_guest(vm, host_cpu_is_hygon);
821 sync_global_to_guest(vm, host_cpu_is_amd_compatible);
822 sync_global_to_guest(vm, is_forced_emulation_enabled);
823 sync_global_to_guest(vm, pmu_errata_mask);
824
825 if (is_sev_vm(vm)) {
826 struct kvm_sev_init init = { 0 };
827
828 vm_sev_ioctl(vm, KVM_SEV_INIT2, &init);
829 }
830
831 r = __vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL);
832 TEST_ASSERT(r > 0, "KVM_GET_TSC_KHZ did not provide a valid TSC frequency.");
833 guest_tsc_khz = r;
834 sync_global_to_guest(vm, guest_tsc_khz);
835
836 /*
837 * The guest MMU is just a placeholder to provide access to PTE masks
838 * (for now). The guest does not have mappings for its own page tables
839 * by default, so any meaningful use of guest page tables requires
840 * explicit setup by the test. Zero the PGD to make it obvious the guest
841 * page tables are not immediately usable by guest code.
842 */
843 guest_mmu = vm->mmu;
844 guest_mmu.pgd = 0;
845 sync_global_to_guest(vm, guest_mmu);
846}
847
848void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code)
849{
850 struct kvm_regs regs;
851
852 vcpu_regs_get(vcpu, &regs);
853 regs.rip = (unsigned long) guest_code;
854 vcpu_regs_set(vcpu, &regs);
855}
856
857struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id)
858{
859 struct kvm_mp_state mp_state;
860 struct kvm_regs regs;
861 struct kvm_vcpu *vcpu;
862
863 vcpu = __vm_vcpu_add(vm, vcpu_id);
864 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
865 vcpu_init_sregs(vm, vcpu);
866 vcpu_init_xcrs(vm, vcpu);
867
868 /* Setup guest general purpose registers */
869 vcpu_regs_get(vcpu, &regs);
870 regs.rflags = regs.rflags | X86_EFLAGS_FIXED;
871 regs.rsp = vm_alloc_stack(vm, DEFAULT_STACK_PGS);
872 vcpu_regs_set(vcpu, &regs);
873
874 /* Setup the MP state */
875 mp_state.mp_state = 0;
876 vcpu_mp_state_set(vcpu, &mp_state);
877
878 /*
879 * Refresh CPUID after setting SREGS and XCR0, so that KVM's "runtime"
880 * updates to guest CPUID, e.g. for OSXSAVE and XSAVE state size, are
881 * reflected into selftests' vCPU CPUID cache, i.e. so that the cache
882 * is consistent with vCPU state.
883 */
884 vcpu_get_cpuid(vcpu);
885 return vcpu;
886}
887
888struct kvm_vcpu *vm_arch_vcpu_recreate(struct kvm_vm *vm, u32 vcpu_id)
889{
890 struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
891
892 vcpu_init_cpuid(vcpu, kvm_get_supported_cpuid());
893
894 return vcpu;
895}
896
897void vcpu_arch_free(struct kvm_vcpu *vcpu)
898{
899 if (vcpu->cpuid)
900 free(vcpu->cpuid);
901}
902
903/* Do not use kvm_supported_cpuid directly except for validity checks. */
904static void *kvm_supported_cpuid;
905
906const struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
907{
908 int kvm_fd;
909
910 if (kvm_supported_cpuid)
911 return kvm_supported_cpuid;
912
913 kvm_supported_cpuid = allocate_kvm_cpuid2(MAX_NR_CPUID_ENTRIES);
914 kvm_fd = open_kvm_dev_path_or_exit();
915
916 kvm_ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID,
917 (struct kvm_cpuid2 *)kvm_supported_cpuid);
918
919 close(kvm_fd);
920 return kvm_supported_cpuid;
921}
922
923static u32 __kvm_cpu_has(const struct kvm_cpuid2 *cpuid,
924 u32 function, u32 index,
925 u8 reg, u8 lo, u8 hi)
926{
927 const struct kvm_cpuid_entry2 *entry;
928 int i;
929
930 for (i = 0; i < cpuid->nent; i++) {
931 entry = &cpuid->entries[i];
932
933 /*
934 * The output registers in kvm_cpuid_entry2 are in alphabetical
935 * order, but kvm_x86_cpu_feature matches that mess, so yay
936 * pointer shenanigans!
937 */
938 if (entry->function == function && entry->index == index)
939 return ((&entry->eax)[reg] & GENMASK(hi, lo)) >> lo;
940 }
941
942 return 0;
943}
944
945bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
946 struct kvm_x86_cpu_feature feature)
947{
948 return __kvm_cpu_has(cpuid, feature.function, feature.index,
949 feature.reg, feature.bit, feature.bit);
950}
951
952u32 kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
953 struct kvm_x86_cpu_property property)
954{
955 return __kvm_cpu_has(cpuid, property.function, property.index,
956 property.reg, property.lo_bit, property.hi_bit);
957}
958
959u64 kvm_get_feature_msr(u64 msr_index)
960{
961 struct {
962 struct kvm_msrs header;
963 struct kvm_msr_entry entry;
964 } buffer = {};
965 int r, kvm_fd;
966
967 buffer.header.nmsrs = 1;
968 buffer.entry.index = msr_index;
969 kvm_fd = open_kvm_dev_path_or_exit();
970
971 r = __kvm_ioctl(kvm_fd, KVM_GET_MSRS, &buffer.header);
972 TEST_ASSERT(r == 1, KVM_IOCTL_ERROR(KVM_GET_MSRS, r));
973
974 close(kvm_fd);
975 return buffer.entry.data;
976}
977
978void __vm_xsave_require_permission(u64 xfeature, const char *name)
979{
980 int kvm_fd;
981 u64 bitmask;
982 long rc;
983 struct kvm_device_attr attr = {
984 .group = 0,
985 .attr = KVM_X86_XCOMP_GUEST_SUPP,
986 .addr = (unsigned long) &bitmask,
987 };
988
989 TEST_ASSERT(!kvm_supported_cpuid,
990 "kvm_get_supported_cpuid() cannot be used before ARCH_REQ_XCOMP_GUEST_PERM");
991
992 TEST_ASSERT(is_power_of_2(xfeature),
993 "Dynamic XFeatures must be enabled one at a time");
994
995 kvm_fd = open_kvm_dev_path_or_exit();
996 rc = __kvm_ioctl(kvm_fd, KVM_GET_DEVICE_ATTR, &attr);
997 close(kvm_fd);
998
999 if (rc == -1 && (errno == ENXIO || errno == EINVAL))
1000 __TEST_REQUIRE(0, "KVM_X86_XCOMP_GUEST_SUPP not supported");
1001
1002 TEST_ASSERT(rc == 0, "KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) error: %ld", rc);
1003
1004 __TEST_REQUIRE(bitmask & xfeature,
1005 "Required XSAVE feature '%s' not supported", name);
1006
1007 TEST_REQUIRE(!syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, ilog2(xfeature)));
1008
1009 rc = syscall(SYS_arch_prctl, ARCH_GET_XCOMP_GUEST_PERM, &bitmask);
1010 TEST_ASSERT(rc == 0, "prctl(ARCH_GET_XCOMP_GUEST_PERM) error: %ld", rc);
1011 TEST_ASSERT(bitmask & xfeature,
1012 "'%s' (0x%lx) not permitted after prctl(ARCH_REQ_XCOMP_GUEST_PERM) permitted=0x%lx",
1013 name, xfeature, bitmask);
1014}
1015
1016void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid)
1017{
1018 TEST_ASSERT(cpuid != vcpu->cpuid, "@cpuid can't be the vCPU's CPUID");
1019
1020 /* Allow overriding the default CPUID. */
1021 if (vcpu->cpuid && vcpu->cpuid->nent < cpuid->nent) {
1022 free(vcpu->cpuid);
1023 vcpu->cpuid = NULL;
1024 }
1025
1026 if (!vcpu->cpuid)
1027 vcpu->cpuid = allocate_kvm_cpuid2(cpuid->nent);
1028
1029 memcpy(vcpu->cpuid, cpuid, kvm_cpuid2_size(cpuid->nent));
1030 vcpu_set_cpuid(vcpu);
1031}
1032
1033void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
1034 struct kvm_x86_cpu_property property,
1035 u32 value)
1036{
1037 struct kvm_cpuid_entry2 *entry;
1038
1039 entry = __vcpu_get_cpuid_entry(vcpu, property.function, property.index);
1040
1041 (&entry->eax)[property.reg] &= ~GENMASK(property.hi_bit, property.lo_bit);
1042 (&entry->eax)[property.reg] |= value << property.lo_bit;
1043
1044 vcpu_set_cpuid(vcpu);
1045
1046 /* Sanity check that @value doesn't exceed the bounds in any way. */
1047 TEST_ASSERT_EQ(kvm_cpuid_property(vcpu->cpuid, property), value);
1048}
1049
1050void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function)
1051{
1052 struct kvm_cpuid_entry2 *entry = vcpu_get_cpuid_entry(vcpu, function);
1053
1054 entry->eax = 0;
1055 entry->ebx = 0;
1056 entry->ecx = 0;
1057 entry->edx = 0;
1058 vcpu_set_cpuid(vcpu);
1059}
1060
1061void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
1062 struct kvm_x86_cpu_feature feature,
1063 bool set)
1064{
1065 struct kvm_cpuid_entry2 *entry;
1066 u32 *reg;
1067
1068 entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index);
1069 reg = (&entry->eax) + feature.reg;
1070
1071 if (set)
1072 *reg |= BIT(feature.bit);
1073 else
1074 *reg &= ~BIT(feature.bit);
1075
1076 vcpu_set_cpuid(vcpu);
1077}
1078
1079u64 vcpu_get_msr(struct kvm_vcpu *vcpu, u64 msr_index)
1080{
1081 struct {
1082 struct kvm_msrs header;
1083 struct kvm_msr_entry entry;
1084 } buffer = {};
1085
1086 buffer.header.nmsrs = 1;
1087 buffer.entry.index = msr_index;
1088
1089 vcpu_msrs_get(vcpu, &buffer.header);
1090
1091 return buffer.entry.data;
1092}
1093
1094int _vcpu_set_msr(struct kvm_vcpu *vcpu, u64 msr_index, u64 msr_value)
1095{
1096 struct {
1097 struct kvm_msrs header;
1098 struct kvm_msr_entry entry;
1099 } buffer = {};
1100
1101 memset(&buffer, 0, sizeof(buffer));
1102 buffer.header.nmsrs = 1;
1103 buffer.entry.index = msr_index;
1104 buffer.entry.data = msr_value;
1105
1106 return __vcpu_ioctl(vcpu, KVM_SET_MSRS, &buffer.header);
1107}
1108
1109void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...)
1110{
1111 va_list ap;
1112 struct kvm_regs regs;
1113
1114 TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
1115 " num: %u",
1116 num);
1117
1118 va_start(ap, num);
1119 vcpu_regs_get(vcpu, &regs);
1120
1121 if (num >= 1)
1122 regs.rdi = va_arg(ap, u64);
1123
1124 if (num >= 2)
1125 regs.rsi = va_arg(ap, u64);
1126
1127 if (num >= 3)
1128 regs.rdx = va_arg(ap, u64);
1129
1130 if (num >= 4)
1131 regs.rcx = va_arg(ap, u64);
1132
1133 if (num >= 5)
1134 regs.r8 = va_arg(ap, u64);
1135
1136 if (num >= 6)
1137 regs.r9 = va_arg(ap, u64);
1138
1139 vcpu_regs_set(vcpu, &regs);
1140 va_end(ap);
1141}
1142
1143void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent)
1144{
1145 struct kvm_regs regs;
1146 struct kvm_sregs sregs;
1147
1148 fprintf(stream, "%*svCPU ID: %u\n", indent, "", vcpu->id);
1149
1150 fprintf(stream, "%*sregs:\n", indent + 2, "");
1151 vcpu_regs_get(vcpu, &regs);
1152 regs_dump(stream, &regs, indent + 4);
1153
1154 fprintf(stream, "%*ssregs:\n", indent + 2, "");
1155 vcpu_sregs_get(vcpu, &sregs);
1156 sregs_dump(stream, &sregs, indent + 4);
1157}
1158
1159static struct kvm_msr_list *__kvm_get_msr_index_list(bool feature_msrs)
1160{
1161 struct kvm_msr_list *list;
1162 struct kvm_msr_list nmsrs;
1163 int kvm_fd, r;
1164
1165 kvm_fd = open_kvm_dev_path_or_exit();
1166
1167 nmsrs.nmsrs = 0;
1168 if (!feature_msrs)
1169 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
1170 else
1171 r = __kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, &nmsrs);
1172
1173 TEST_ASSERT(r == -1 && errno == E2BIG,
1174 "Expected -E2BIG, got rc: %i errno: %i (%s)",
1175 r, errno, strerror(errno));
1176
1177 list = malloc(sizeof(*list) + nmsrs.nmsrs * sizeof(list->indices[0]));
1178 TEST_ASSERT(list, "-ENOMEM when allocating MSR index list");
1179 list->nmsrs = nmsrs.nmsrs;
1180
1181 if (!feature_msrs)
1182 kvm_ioctl(kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1183 else
1184 kvm_ioctl(kvm_fd, KVM_GET_MSR_FEATURE_INDEX_LIST, list);
1185 close(kvm_fd);
1186
1187 TEST_ASSERT(list->nmsrs == nmsrs.nmsrs,
1188 "Number of MSRs in list changed, was %d, now %d",
1189 nmsrs.nmsrs, list->nmsrs);
1190 return list;
1191}
1192
1193const struct kvm_msr_list *kvm_get_msr_index_list(void)
1194{
1195 static const struct kvm_msr_list *list;
1196
1197 if (!list)
1198 list = __kvm_get_msr_index_list(false);
1199 return list;
1200}
1201
1202
1203const struct kvm_msr_list *kvm_get_feature_msr_index_list(void)
1204{
1205 static const struct kvm_msr_list *list;
1206
1207 if (!list)
1208 list = __kvm_get_msr_index_list(true);
1209 return list;
1210}
1211
1212bool kvm_msr_is_in_save_restore_list(u32 msr_index)
1213{
1214 const struct kvm_msr_list *list = kvm_get_msr_index_list();
1215 int i;
1216
1217 for (i = 0; i < list->nmsrs; ++i) {
1218 if (list->indices[i] == msr_index)
1219 return true;
1220 }
1221
1222 return false;
1223}
1224
1225static void vcpu_save_xsave_state(struct kvm_vcpu *vcpu,
1226 struct kvm_x86_state *state)
1227{
1228 int size = vm_check_cap(vcpu->vm, KVM_CAP_XSAVE2);
1229
1230 if (size) {
1231 state->xsave = malloc(size);
1232 vcpu_xsave2_get(vcpu, state->xsave);
1233 } else {
1234 state->xsave = malloc(sizeof(struct kvm_xsave));
1235 vcpu_xsave_get(vcpu, state->xsave);
1236 }
1237}
1238
1239struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu)
1240{
1241 const struct kvm_msr_list *msr_list = kvm_get_msr_index_list();
1242 struct kvm_x86_state *state;
1243 int i;
1244
1245 static int nested_size = -1;
1246
1247 if (nested_size == -1) {
1248 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
1249 TEST_ASSERT(nested_size <= sizeof(state->nested_),
1250 "Nested state size too big, %i > %zi",
1251 nested_size, sizeof(state->nested_));
1252 }
1253
1254 /*
1255 * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
1256 * guest state is consistent only after userspace re-enters the
1257 * kernel with KVM_RUN. Complete IO prior to migrating state
1258 * to a new VM.
1259 */
1260 vcpu_run_complete_io(vcpu);
1261
1262 state = malloc(sizeof(*state) + msr_list->nmsrs * sizeof(state->msrs.entries[0]));
1263 TEST_ASSERT(state, "-ENOMEM when allocating kvm state");
1264
1265 vcpu_events_get(vcpu, &state->events);
1266 vcpu_mp_state_get(vcpu, &state->mp_state);
1267 vcpu_regs_get(vcpu, &state->regs);
1268 vcpu_save_xsave_state(vcpu, state);
1269
1270 if (kvm_has_cap(KVM_CAP_XCRS))
1271 vcpu_xcrs_get(vcpu, &state->xcrs);
1272
1273 vcpu_sregs_get(vcpu, &state->sregs);
1274
1275 if (nested_size) {
1276 state->nested.size = sizeof(state->nested_);
1277
1278 vcpu_nested_state_get(vcpu, &state->nested);
1279 TEST_ASSERT(state->nested.size <= nested_size,
1280 "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1281 state->nested.size, nested_size);
1282 } else {
1283 state->nested.size = 0;
1284 }
1285
1286 state->msrs.nmsrs = msr_list->nmsrs;
1287 for (i = 0; i < msr_list->nmsrs; i++)
1288 state->msrs.entries[i].index = msr_list->indices[i];
1289 vcpu_msrs_get(vcpu, &state->msrs);
1290
1291 vcpu_debugregs_get(vcpu, &state->debugregs);
1292
1293 return state;
1294}
1295
1296void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state)
1297{
1298 vcpu_sregs_set(vcpu, &state->sregs);
1299 vcpu_msrs_set(vcpu, &state->msrs);
1300
1301 if (kvm_has_cap(KVM_CAP_XCRS))
1302 vcpu_xcrs_set(vcpu, &state->xcrs);
1303
1304 vcpu_xsave_set(vcpu, state->xsave);
1305 vcpu_events_set(vcpu, &state->events);
1306 vcpu_mp_state_set(vcpu, &state->mp_state);
1307 vcpu_debugregs_set(vcpu, &state->debugregs);
1308 vcpu_regs_set(vcpu, &state->regs);
1309
1310 if (state->nested.size)
1311 vcpu_nested_state_set(vcpu, &state->nested);
1312}
1313
1314void kvm_x86_state_cleanup(struct kvm_x86_state *state)
1315{
1316 free(state->xsave);
1317 free(state);
1318}
1319
1320void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits)
1321{
1322 if (!kvm_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR)) {
1323 *pa_bits = kvm_cpu_has(X86_FEATURE_PAE) ? 36 : 32;
1324 *va_bits = 32;
1325 } else {
1326 *pa_bits = kvm_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1327 *va_bits = kvm_cpu_property(X86_PROPERTY_MAX_VIRT_ADDR);
1328 }
1329}
1330
1331void kvm_init_vm_address_properties(struct kvm_vm *vm)
1332{
1333 if (is_sev_vm(vm)) {
1334 vm->arch.sev_fd = open_sev_dev_path_or_exit();
1335 vm->arch.c_bit = BIT_ULL(this_cpu_property(X86_PROPERTY_SEV_C_BIT));
1336 vm->gpa_tag_mask = vm->arch.c_bit;
1337 } else {
1338 vm->arch.sev_fd = -1;
1339 }
1340}
1341
1342const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
1343 u32 function, u32 index)
1344{
1345 int i;
1346
1347 for (i = 0; i < cpuid->nent; i++) {
1348 if (cpuid->entries[i].function == function &&
1349 cpuid->entries[i].index == index)
1350 return &cpuid->entries[i];
1351 }
1352
1353 TEST_FAIL("CPUID function 0x%x index 0x%x not found ", function, index);
1354
1355 return NULL;
1356}
1357
1358#define X86_HYPERCALL(inputs...) \
1359({ \
1360 u64 r; \
1361 \
1362 asm volatile("test %[use_vmmcall], %[use_vmmcall]\n\t" \
1363 "jnz 1f\n\t" \
1364 "vmcall\n\t" \
1365 "jmp 2f\n\t" \
1366 "1: vmmcall\n\t" \
1367 "2:" \
1368 : "=a"(r) \
1369 : [use_vmmcall] "r" (host_cpu_is_amd_compatible), \
1370 inputs); \
1371 \
1372 r; \
1373})
1374
1375u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3)
1376{
1377 return X86_HYPERCALL("a"(nr), "b"(a0), "c"(a1), "d"(a2), "S"(a3));
1378}
1379
1380u64 __xen_hypercall(u64 nr, u64 a0, void *a1)
1381{
1382 return X86_HYPERCALL("a"(nr), "D"(a0), "S"(a1));
1383}
1384
1385void xen_hypercall(u64 nr, u64 a0, void *a1)
1386{
1387 GUEST_ASSERT(!__xen_hypercall(nr, a0, a1));
1388}
1389
1390unsigned long vm_compute_max_gfn(struct kvm_vm *vm)
1391{
1392 const unsigned long num_ht_pages = 12 << (30 - vm->page_shift); /* 12 GiB */
1393 unsigned long ht_gfn, max_gfn, max_pfn;
1394 u8 maxphyaddr, guest_maxphyaddr;
1395
1396 /*
1397 * Use "guest MAXPHYADDR" from KVM if it's available. Guest MAXPHYADDR
1398 * enumerates the max _mappable_ GPA, which can be less than the raw
1399 * MAXPHYADDR, e.g. if MAXPHYADDR=52, KVM is using TDP, and the CPU
1400 * doesn't support 5-level TDP.
1401 */
1402 guest_maxphyaddr = kvm_cpu_property(X86_PROPERTY_GUEST_MAX_PHY_ADDR);
1403 guest_maxphyaddr = guest_maxphyaddr ?: vm->pa_bits;
1404 TEST_ASSERT(guest_maxphyaddr <= vm->pa_bits,
1405 "Guest MAXPHYADDR should never be greater than raw MAXPHYADDR");
1406
1407 max_gfn = (1ULL << (guest_maxphyaddr - vm->page_shift)) - 1;
1408
1409 /* Avoid reserved HyperTransport region on AMD or Hygon processors. */
1410 if (!host_cpu_is_amd_compatible)
1411 return max_gfn;
1412
1413 /* On parts with <40 physical address bits, the area is fully hidden */
1414 if (vm->pa_bits < 40)
1415 return max_gfn;
1416
1417 /* Before family 17h, the HyperTransport area is just below 1T. */
1418 ht_gfn = (1 << 28) - num_ht_pages;
1419 if (this_cpu_family() < 0x17)
1420 goto done;
1421
1422 /*
1423 * Otherwise it's at the top of the physical address space, possibly
1424 * reduced due to SME or CSV by bits 11:6 of CPUID[0x8000001f].EBX. Use
1425 * the old conservative value if MAXPHYADDR is not enumerated.
1426 */
1427 if (!this_cpu_has_p(X86_PROPERTY_MAX_PHY_ADDR))
1428 goto done;
1429
1430 maxphyaddr = this_cpu_property(X86_PROPERTY_MAX_PHY_ADDR);
1431 max_pfn = (1ULL << (maxphyaddr - vm->page_shift)) - 1;
1432
1433 if (this_cpu_has_p(X86_PROPERTY_PHYS_ADDR_REDUCTION))
1434 max_pfn >>= this_cpu_property(X86_PROPERTY_PHYS_ADDR_REDUCTION);
1435
1436 ht_gfn = max_pfn - num_ht_pages;
1437done:
1438 return min(max_gfn, ht_gfn - 1);
1439}
1440
1441void kvm_selftest_arch_init(void)
1442{
1443 host_cpu_is_intel = this_cpu_is_intel();
1444 host_cpu_is_amd = this_cpu_is_amd();
1445 host_cpu_is_hygon = this_cpu_is_hygon();
1446 host_cpu_is_amd_compatible = host_cpu_is_amd || host_cpu_is_hygon;
1447 is_forced_emulation_enabled = kvm_is_forced_emulation_enabled();
1448
1449 kvm_init_pmu_errata();
1450}
1451
1452bool sys_clocksource_is_based_on_tsc(void)
1453{
1454 char *clk_name = sys_get_cur_clocksource();
1455 bool ret = !strcmp(clk_name, "tsc\n") ||
1456 !strcmp(clk_name, "hyperv_clocksource_tsc_page\n");
1457
1458 free(clk_name);
1459
1460 return ret;
1461}
1462
1463bool kvm_arch_has_default_irqchip(void)
1464{
1465 return true;
1466}
1467
1468void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa,
1469 const void *smi_handler, size_t handler_size)
1470{
1471 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa,
1472 SMRAM_MEMSLOT, SMRAM_PAGES, 0);
1473 TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa,
1474 SMRAM_MEMSLOT) == smram_gpa,
1475 "Could not allocate guest physical addresses for SMRAM");
1476
1477 memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE);
1478 memcpy(addr_gpa2hva(vm, smram_gpa) + 0x8000, smi_handler, handler_size);
1479 vcpu_set_msr(vcpu, MSR_IA32_SMBASE, smram_gpa);
1480}
1481
1482void inject_smi(struct kvm_vcpu *vcpu)
1483{
1484 struct kvm_vcpu_events events;
1485
1486 vcpu_events_get(vcpu, &events);
1487 events.smi.pending = 1;
1488 events.flags |= KVM_VCPUEVENT_VALID_SMM;
1489 vcpu_events_set(vcpu, &events);
1490}