projects/linux·blob master · refs

441 lines · 10.0 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ARM64_KVM_NESTED_H
3#define __ARM64_KVM_NESTED_H
4
5#include <linux/bitfield.h>
6#include <linux/kvm_host.h>
7#include <asm/kvm_emulate.h>
8#include <asm/kvm_pgtable.h>
9
10static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
11{
12 return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
13 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
14 vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
15}
16
17/* Translation helpers from non-VHE EL2 to EL1 */
18static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
19{
20 return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
21}
22
23static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
24{
25 return TCR_EPD1_MASK | /* disable TTBR1_EL1 */
26 ((tcr & TCR_EL2_DS) ? TCR_DS : 0) |
27 ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
28 tcr_el2_ps_to_tcr_el1_ips(tcr) |
29 (tcr & TCR_EL2_TG0_MASK) |
30 (tcr & TCR_EL2_ORGN0_MASK) |
31 (tcr & TCR_EL2_IRGN0_MASK) |
32 (tcr & TCR_EL2_T0SZ_MASK);
33}
34
35static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
36{
37 u64 cpacr_el1 = CPACR_EL1_RES1;
38
39 if (cptr_el2 & CPTR_EL2_TTA)
40 cpacr_el1 |= CPACR_EL1_TTA;
41 if (!(cptr_el2 & CPTR_EL2_TFP))
42 cpacr_el1 |= CPACR_EL1_FPEN;
43 if (!(cptr_el2 & CPTR_EL2_TZ))
44 cpacr_el1 |= CPACR_EL1_ZEN;
45
46 cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);
47
48 return cpacr_el1;
49}
50
51static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
52{
53 /* Only preserve the minimal set of bits we support */
54 val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
55 SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
56 val |= SCTLR_EL1_RES1;
57
58 return val;
59}
60
61static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
62{
63 /* Clear the ASID field */
64 return ttbr0 & ~GENMASK_ULL(63, 48);
65}
66
67extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
68extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
69extern int kvm_init_nested(struct kvm *kvm);
70extern void kvm_destroy_nested(struct kvm *kvm);
71extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
72extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
73extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);
74
75union tlbi_info;
76
77extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
78 const union tlbi_info *info,
79 void (*)(struct kvm_s2_mmu *,
80 const union tlbi_info *));
81extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
82extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
83
84extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
85extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
86extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
87
88extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu);
89
90struct kvm_s2_trans {
91 phys_addr_t output;
92 unsigned long block_size;
93 bool writable;
94 bool readable;
95 int level;
96 u32 esr;
97 u64 desc;
98};
99
100static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)
101{
102 return trans->output;
103}
104
105static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)
106{
107 return trans->block_size;
108}
109
110static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)
111{
112 return trans->esr;
113}
114
115static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)
116{
117 return trans->readable;
118}
119
120static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)
121{
122 return trans->writable;
123}
124
125static inline bool kvm_has_xnx(struct kvm *kvm)
126{
127 return cpus_have_final_cap(ARM64_HAS_XNX) &&
128 kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP);
129}
130
131static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans)
132{
133 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
134
135 if (!kvm_has_xnx(kvm))
136 xn &= 0b10;
137
138 switch (xn) {
139 case 0b00:
140 case 0b01:
141 return true;
142 default:
143 return false;
144 }
145}
146
147static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans)
148{
149 u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
150
151 if (!kvm_has_xnx(kvm))
152 xn &= 0b10;
153
154 switch (xn) {
155 case 0b00:
156 case 0b11:
157 return true;
158 default:
159 return false;
160 }
161}
162
163extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
164 struct kvm_s2_trans *result);
165extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
166 struct kvm_s2_trans *trans);
167extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
168extern void kvm_nested_s2_wp(struct kvm *kvm);
169extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
170extern void kvm_nested_s2_flush(struct kvm *kvm);
171
172unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);
173
174static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)
175{
176 struct kvm *kvm = vpcu->kvm;
177 u8 CRm = sys_reg_CRm(instr);
178
179 if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
180 sys_reg_Op1(instr) == TLBI_Op1_EL1))
181 return false;
182
183 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
184 (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
185 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
186 return false;
187
188 if (CRm == TLBI_CRm_nROS &&
189 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
190 return false;
191
192 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
193 CRm == TLBI_CRm_RNS) &&
194 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
195 return false;
196
197 return true;
198}
199
200static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)
201{
202 struct kvm *kvm = vpcu->kvm;
203 u8 CRm = sys_reg_CRm(instr);
204
205 if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
206 sys_reg_Op1(instr) == TLBI_Op1_EL2))
207 return false;
208
209 if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
210 (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
211 kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
212 return false;
213
214 if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)
215 return false;
216
217 if (CRm == TLBI_CRm_nROS &&
218 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
219 return false;
220
221 if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
222 CRm == TLBI_CRm_RNS) &&
223 !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
224 return false;
225
226 return true;
227}
228
229int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu);
230u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val);
231
232#ifdef CONFIG_ARM64_PTR_AUTH
233bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr);
234#else
235static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr)
236{
237 /* We really should never execute this... */
238 WARN_ON_ONCE(1);
239 *elr = 0xbad9acc0debadbad;
240 return false;
241}
242#endif
243
244#define KVM_NV_GUEST_MAP_SZ (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)
245
246static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)
247{
248 return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);
249}
250
251/* Adjust alignment for the contiguous bit as per StageOA() */
252#define contiguous_bit_shift(d, wi, l) \
253 ({ \
254 u8 shift = 0; \
255 \
256 if ((d) & PTE_CONT) { \
257 switch (BIT((wi)->pgshift)) { \
258 case SZ_4K: \
259 shift = 4; \
260 break; \
261 case SZ_16K: \
262 shift = (l) == 2 ? 5 : 7; \
263 break; \
264 case SZ_64K: \
265 shift = 5; \
266 break; \
267 } \
268 } \
269 \
270 shift; \
271 })
272
273static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
274{
275 u64 base, tg, num, scale;
276 int shift;
277
278 tg = FIELD_GET(GENMASK(47, 46), val);
279
280 switch(tg) {
281 case 1:
282 shift = 12;
283 break;
284 case 2:
285 shift = 14;
286 break;
287 case 3:
288 default: /* IMPDEF: handle tg==0 as 64k */
289 shift = 16;
290 break;
291 }
292
293 base = (val & GENMASK(36, 0)) << shift;
294
295 /*
296 * We only deal with at most 48bit VA/IPA, so 48 is where we
297 * sign-extend from. Should we support FEAT_L{VP}A* at some point,
298 * this will need to be revisited.
299 */
300 base = (u64)sign_extend64(base, 48);
301
302 if (asid)
303 *asid = FIELD_GET(TLBIR_ASID_MASK, val);
304
305 scale = FIELD_GET(GENMASK(45, 44), val);
306 num = FIELD_GET(GENMASK(43, 39), val);
307 *range = __TLBI_RANGE_PAGES(num, scale) << shift;
308
309 /* Cap the range to the correct half of the address space */
310 if (!(base & BIT(48)))
311 *range = min(*range, (BIT(48) - base));
312 else
313 *range = min(*range, ~base + 1);
314
315 return base;
316}
317
318static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit)
319{
320 switch (ps) {
321 case 0: return 32;
322 case 1: return 36;
323 case 2: return 40;
324 case 3: return 42;
325 case 4: return 44;
326 case 5: return 48;
327 case 6: if (pa52bit)
328 return 52;
329 fallthrough;
330 default:
331 return 48;
332 }
333}
334
335enum trans_regime {
336 TR_EL10,
337 TR_EL20,
338 TR_EL2,
339};
340
341struct s1_walk_info;
342
343struct s1_walk_context {
344 struct s1_walk_info *wi;
345 u64 table_ipa;
346 int level;
347};
348
349struct s1_walk_filter {
350 int (*fn)(struct s1_walk_context *, void *);
351 void *priv;
352};
353
354struct s1_walk_info {
355 struct s1_walk_filter *filter;
356 u64 baddr;
357 enum trans_regime regime;
358 unsigned int max_oa_bits;
359 unsigned int pgshift;
360 unsigned int txsz;
361 int sl;
362 u8 sh;
363 bool as_el0;
364 bool hpd;
365 bool e0poe;
366 bool poe;
367 bool pan;
368 bool be;
369 bool s2;
370 bool pa52bit;
371 bool ha;
372};
373
374struct s1_walk_result {
375 union {
376 struct {
377 u64 desc;
378 u64 pa;
379 s8 level;
380 u8 APTable;
381 bool nG;
382 u16 asid;
383 bool UXNTable;
384 bool PXNTable;
385 bool uwxn;
386 bool uov;
387 bool ur;
388 bool uw;
389 bool ux;
390 bool pwxn;
391 bool pov;
392 bool pr;
393 bool pw;
394 bool px;
395 };
396 struct {
397 u8 fst;
398 bool ptw;
399 bool s2;
400 };
401 };
402 bool failed;
403};
404
405#define S1_MMU_DISABLED (-127)
406
407static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw)
408{
409 wr->fst = fst;
410 wr->ptw = s1ptw;
411 wr->s2 = s1ptw;
412 wr->failed = true;
413}
414
415static inline bool s1_walk_translated(struct s1_walk_result *wr)
416{
417 return wr->level != S1_MMU_DISABLED;
418}
419
420int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
421 struct s1_walk_result *wr, u64 va);
422int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa,
423 int *level);
424
425/* VNCR management */
426int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
427int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
428void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);
429
430u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime);
431
432#define vncr_fixmap(c) \
433 ({ \
434 u32 __c = (c); \
435 BUG_ON(__c >= NR_CPUS); \
436 (FIX_VNCR - __c); \
437 })
438
439int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new);
440
441#endif /* __ARM64_KVM_NESTED_H */