projects/linux·blob master · refs

2440 lines · 61.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2024, Microsoft Corporation.
4 *
5 * The main part of the mshv_root module, providing APIs to create
6 * and manage guest partitions.
7 *
8 * Authors: Microsoft Linux virtualization team
9 */
10
11#include <linux/entry-virt.h>
12#include <linux/kernel.h>
13#include <linux/module.h>
14#include <linux/fs.h>
15#include <linux/miscdevice.h>
16#include <linux/slab.h>
17#include <linux/file.h>
18#include <linux/anon_inodes.h>
19#include <linux/mm.h>
20#include <linux/io.h>
21#include <linux/cpuhotplug.h>
22#include <linux/random.h>
23#include <asm/mshyperv.h>
24#include <linux/hyperv.h>
25#include <linux/notifier.h>
26#include <linux/reboot.h>
27#include <linux/kexec.h>
28#include <linux/page-flags.h>
29#include <linux/crash_dump.h>
30#include <linux/panic_notifier.h>
31#include <linux/vmalloc.h>
32#include <linux/rseq.h>
33
34#include "mshv_eventfd.h"
35#include "mshv.h"
36#include "mshv_root.h"
37
38MODULE_AUTHOR("Microsoft");
39MODULE_LICENSE("GPL");
40MODULE_DESCRIPTION("Microsoft Hyper-V root partition VMM interface /dev/mshv");
41
42/* HV_THREAD_COUNTER */
43#if defined(CONFIG_X86_64)
44#define HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED 202
45#elif defined(CONFIG_ARM64)
46#define HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED 95
47#endif
48
49struct mshv_root mshv_root;
50
51enum hv_scheduler_type hv_scheduler_type;
52
53/* Once we implement the fast extended hypercall ABI they can go away. */
54static void * __percpu *root_scheduler_input;
55static void * __percpu *root_scheduler_output;
56
57static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
58static int mshv_dev_open(struct inode *inode, struct file *filp);
59static int mshv_dev_release(struct inode *inode, struct file *filp);
60static int mshv_vp_release(struct inode *inode, struct file *filp);
61static long mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
62static int mshv_partition_release(struct inode *inode, struct file *filp);
63static long mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg);
64static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma);
65static vm_fault_t mshv_vp_fault(struct vm_fault *vmf);
66static int mshv_init_async_handler(struct mshv_partition *partition);
67static void mshv_async_hvcall_handler(void *data, u64 *status);
68
69static const union hv_input_vtl input_vtl_zero;
70static const union hv_input_vtl input_vtl_normal = {
71 .target_vtl = HV_NORMAL_VTL,
72 .use_target_vtl = 1,
73};
74
75static const struct vm_operations_struct mshv_vp_vm_ops = {
76 .fault = mshv_vp_fault,
77};
78
79static const struct file_operations mshv_vp_fops = {
80 .owner = THIS_MODULE,
81 .release = mshv_vp_release,
82 .unlocked_ioctl = mshv_vp_ioctl,
83 .llseek = noop_llseek,
84 .mmap = mshv_vp_mmap,
85};
86
87static const struct file_operations mshv_partition_fops = {
88 .owner = THIS_MODULE,
89 .release = mshv_partition_release,
90 .unlocked_ioctl = mshv_partition_ioctl,
91 .llseek = noop_llseek,
92};
93
94static const struct file_operations mshv_dev_fops = {
95 .owner = THIS_MODULE,
96 .open = mshv_dev_open,
97 .release = mshv_dev_release,
98 .unlocked_ioctl = mshv_dev_ioctl,
99 .llseek = noop_llseek,
100};
101
102static struct miscdevice mshv_dev = {
103 .minor = MISC_DYNAMIC_MINOR,
104 .name = "mshv",
105 .fops = &mshv_dev_fops,
106 .mode = 0600,
107};
108
109/*
110 * Only allow hypercalls that have a u64 partition id as the first member of
111 * the input structure.
112 * These are sorted by value.
113 */
114static u16 mshv_passthru_hvcalls[] = {
115 HVCALL_GET_PARTITION_PROPERTY,
116 HVCALL_GET_PARTITION_PROPERTY_EX,
117 HVCALL_SET_PARTITION_PROPERTY,
118 HVCALL_INSTALL_INTERCEPT,
119 HVCALL_GET_VP_REGISTERS,
120 HVCALL_SET_VP_REGISTERS,
121 HVCALL_TRANSLATE_VIRTUAL_ADDRESS,
122 HVCALL_CLEAR_VIRTUAL_INTERRUPT,
123 HVCALL_REGISTER_INTERCEPT_RESULT,
124 HVCALL_ASSERT_VIRTUAL_INTERRUPT,
125 HVCALL_GET_GPA_PAGES_ACCESS_STATES,
126 HVCALL_SIGNAL_EVENT_DIRECT,
127 HVCALL_POST_MESSAGE_DIRECT,
128 HVCALL_GET_VP_CPUID_VALUES,
129};
130
131/*
132 * Only allow hypercalls that are safe to be called by the VMM with the host
133 * partition as target (i.e. HV_PARTITION_ID_SELF). Carefully audit that a
134 * hypercall cannot be misused by the VMM before adding it to this list.
135 */
136static u16 mshv_self_passthru_hvcalls[] = {
137 HVCALL_GET_PARTITION_PROPERTY,
138 HVCALL_GET_PARTITION_PROPERTY_EX,
139};
140
141static bool mshv_hvcall_is_async(u16 code)
142{
143 switch (code) {
144 case HVCALL_SET_PARTITION_PROPERTY:
145 return true;
146 default:
147 break;
148 }
149 return false;
150}
151
152static bool mshv_passthru_hvcall_allowed(u16 code, u64 pt_id)
153{
154 int i;
155 int n = ARRAY_SIZE(mshv_passthru_hvcalls);
156 u16 *allowed_hvcalls = mshv_passthru_hvcalls;
157
158 if (pt_id == HV_PARTITION_ID_SELF) {
159 n = ARRAY_SIZE(mshv_self_passthru_hvcalls);
160 allowed_hvcalls = mshv_self_passthru_hvcalls;
161 }
162
163 for (i = 0; i < n; ++i)
164 if (allowed_hvcalls[i] == code)
165 return true;
166
167 return false;
168}
169
170static int mshv_ioctl_passthru_hvcall(struct mshv_partition *partition,
171 bool partition_locked,
172 void __user *user_args)
173{
174 u64 status;
175 int ret = 0;
176 bool is_async;
177 struct mshv_root_hvcall args;
178 struct page *page;
179 unsigned int pages_order;
180 void *input_pg = NULL;
181 void *output_pg = NULL;
182 u16 reps_completed;
183 u64 pt_id = partition ? partition->pt_id : HV_PARTITION_ID_SELF;
184
185 if (copy_from_user(&args, user_args, sizeof(args)))
186 return -EFAULT;
187
188 if (args.status || !args.in_ptr || args.in_sz < sizeof(u64) ||
189 mshv_field_nonzero(args, rsvd) || args.in_sz > HV_HYP_PAGE_SIZE)
190 return -EINVAL;
191
192 if (args.out_ptr && (!args.out_sz || args.out_sz > HV_HYP_PAGE_SIZE))
193 return -EINVAL;
194
195 if (!mshv_passthru_hvcall_allowed(args.code, pt_id))
196 return -EINVAL;
197
198 is_async = mshv_hvcall_is_async(args.code);
199 if (is_async) {
200 /* async hypercalls can only be called from partition fd */
201 if (!partition || !partition_locked)
202 return -EINVAL;
203 ret = mshv_init_async_handler(partition);
204 if (ret)
205 return ret;
206 }
207
208 pages_order = args.out_ptr ? 1 : 0;
209 page = alloc_pages(GFP_KERNEL, pages_order);
210 if (!page)
211 return -ENOMEM;
212 input_pg = page_address(page);
213
214 if (args.out_ptr)
215 output_pg = (char *)input_pg + PAGE_SIZE;
216 else
217 output_pg = NULL;
218
219 if (copy_from_user(input_pg, (void __user *)args.in_ptr,
220 args.in_sz)) {
221 ret = -EFAULT;
222 goto free_pages_out;
223 }
224
225 /*
226 * NOTE: This only works because all the allowed hypercalls' input
227 * structs begin with a u64 partition_id field.
228 */
229 *(u64 *)input_pg = pt_id;
230
231 reps_completed = 0;
232 do {
233 if (args.reps) {
234 status = hv_do_rep_hypercall_ex(args.code, args.reps,
235 0, reps_completed,
236 input_pg, output_pg);
237 reps_completed = hv_repcomp(status);
238 } else {
239 status = hv_do_hypercall(args.code, input_pg, output_pg);
240 }
241
242 if (hv_result(status) == HV_STATUS_CALL_PENDING) {
243 if (is_async) {
244 mshv_async_hvcall_handler(partition, &status);
245 } else { /* Paranoia check. This shouldn't happen! */
246 ret = -EBADFD;
247 goto free_pages_out;
248 }
249 }
250
251 if (hv_result_success(status))
252 break;
253
254 if (!hv_result_needs_memory(status))
255 ret = hv_result_to_errno(status);
256 else
257 ret = hv_deposit_memory(pt_id, status);
258 } while (!ret);
259
260 args.status = hv_result(status);
261 args.reps = reps_completed;
262 if (copy_to_user(user_args, &args, sizeof(args)))
263 ret = -EFAULT;
264
265 if (!ret && output_pg &&
266 copy_to_user((void __user *)args.out_ptr, output_pg, args.out_sz))
267 ret = -EFAULT;
268
269free_pages_out:
270 free_pages((unsigned long)input_pg, pages_order);
271
272 return ret;
273}
274
275static inline bool is_ghcb_mapping_available(void)
276{
277#if IS_ENABLED(CONFIG_X86_64)
278 return ms_hyperv.ext_features & HV_VP_GHCB_ROOT_MAPPING_AVAILABLE;
279#else
280 return 0;
281#endif
282}
283
284static int mshv_get_vp_registers(u32 vp_index, u64 partition_id, u16 count,
285 struct hv_register_assoc *registers)
286{
287 return hv_call_get_vp_registers(vp_index, partition_id,
288 count, input_vtl_zero, registers);
289}
290
291static int mshv_set_vp_registers(u32 vp_index, u64 partition_id, u16 count,
292 struct hv_register_assoc *registers)
293{
294 return hv_call_set_vp_registers(vp_index, partition_id,
295 count, input_vtl_zero, registers);
296}
297
298/*
299 * Explicit guest vCPU suspend is asynchronous by nature (as it is requested by
300 * dom0 vCPU for guest vCPU) and thus it can race with "intercept" suspend,
301 * done by the hypervisor.
302 * "Intercept" suspend leads to asynchronous message delivery to dom0 which
303 * should be awaited to keep the VP loop consistent (i.e. no message pending
304 * upon VP resume).
305 * VP intercept suspend can't be done when the VP is explicitly suspended
306 * already, and thus can be only two possible race scenarios:
307 * 1. implicit suspend bit set -> explicit suspend bit set -> message sent
308 * 2. implicit suspend bit set -> message sent -> explicit suspend bit set
309 * Checking for implicit suspend bit set after explicit suspend request has
310 * succeeded in either case allows us to reliably identify, if there is a
311 * message to receive and deliver to VMM.
312 */
313static int
314mshv_suspend_vp(const struct mshv_vp *vp, bool *message_in_flight)
315{
316 struct hv_register_assoc explicit_suspend = {
317 .name = HV_REGISTER_EXPLICIT_SUSPEND
318 };
319 struct hv_register_assoc intercept_suspend = {
320 .name = HV_REGISTER_INTERCEPT_SUSPEND
321 };
322 union hv_explicit_suspend_register *es =
323 &explicit_suspend.value.explicit_suspend;
324 union hv_intercept_suspend_register *is =
325 &intercept_suspend.value.intercept_suspend;
326 int ret;
327
328 es->suspended = 1;
329
330 ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
331 1, &explicit_suspend);
332 if (ret) {
333 vp_err(vp, "Failed to explicitly suspend vCPU\n");
334 return ret;
335 }
336
337 ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
338 1, &intercept_suspend);
339 if (ret) {
340 vp_err(vp, "Failed to get intercept suspend state\n");
341 return ret;
342 }
343
344 *message_in_flight = is->suspended;
345
346 return 0;
347}
348
349/*
350 * This function is used when VPs are scheduled by the hypervisor's
351 * scheduler.
352 *
353 * Caller has to make sure the registers contain cleared
354 * HV_REGISTER_INTERCEPT_SUSPEND and HV_REGISTER_EXPLICIT_SUSPEND registers
355 * exactly in this order (the hypervisor clears them sequentially) to avoid
356 * potential invalid clearing a newly arrived HV_REGISTER_INTERCEPT_SUSPEND
357 * after VP is released from HV_REGISTER_EXPLICIT_SUSPEND in case of the
358 * opposite order.
359 */
360static long mshv_run_vp_with_hyp_scheduler(struct mshv_vp *vp)
361{
362 long ret;
363 struct hv_register_assoc suspend_regs[2] = {
364 { .name = HV_REGISTER_INTERCEPT_SUSPEND },
365 { .name = HV_REGISTER_EXPLICIT_SUSPEND }
366 };
367 size_t count = ARRAY_SIZE(suspend_regs);
368
369 /* Resume VP execution */
370 ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
371 count, suspend_regs);
372 if (ret) {
373 vp_err(vp, "Failed to resume vp execution. %lx\n", ret);
374 return ret;
375 }
376
377 ret = wait_event_interruptible(vp->run.vp_suspend_queue,
378 vp->run.kicked_by_hv == 1);
379 if (ret) {
380 bool message_in_flight;
381
382 /*
383 * Otherwise the waiting was interrupted by a signal: suspend
384 * the vCPU explicitly and copy message in flight (if any).
385 */
386 ret = mshv_suspend_vp(vp, &message_in_flight);
387 if (ret)
388 return ret;
389
390 /* Return if no message in flight */
391 if (!message_in_flight)
392 return -EINTR;
393
394 /* Wait for the message in flight. */
395 wait_event(vp->run.vp_suspend_queue, vp->run.kicked_by_hv == 1);
396 }
397
398 /*
399 * Reset the flag to make the wait_event call above work
400 * next time.
401 */
402 vp->run.kicked_by_hv = 0;
403
404 return 0;
405}
406
407static int
408mshv_vp_dispatch(struct mshv_vp *vp, u32 flags,
409 struct hv_output_dispatch_vp *res)
410{
411 struct hv_input_dispatch_vp *input;
412 struct hv_output_dispatch_vp *output;
413 u64 status;
414
415 preempt_disable();
416 input = *this_cpu_ptr(root_scheduler_input);
417 output = *this_cpu_ptr(root_scheduler_output);
418
419 memset(input, 0, sizeof(*input));
420 memset(output, 0, sizeof(*output));
421
422 input->partition_id = vp->vp_partition->pt_id;
423 input->vp_index = vp->vp_index;
424 input->time_slice = 0; /* Run forever until something happens */
425 input->spec_ctrl = 0; /* TODO: set sensible flags */
426 input->flags = flags;
427
428 vp->run.flags.root_sched_dispatched = 1;
429 status = hv_do_hypercall(HVCALL_DISPATCH_VP, input, output);
430 vp->run.flags.root_sched_dispatched = 0;
431
432 trace_mshv_hvcall_dispatch_vp(vp->vp_partition->pt_id,
433 vp->vp_index, flags,
434 output->dispatch_state,
435 output->dispatch_event,
436#if defined(CONFIG_X86_64)
437 vp->vp_register_page->interrupt_vectors.as_uint64,
438#else
439 0,
440#endif
441 status);
442
443 *res = *output;
444 preempt_enable();
445
446 if (!hv_result_success(status))
447 vp_err(vp, "%s: status %s\n", __func__,
448 hv_result_to_string(status));
449
450 return hv_result_to_errno(status);
451}
452
453static int
454mshv_vp_clear_explicit_suspend(struct mshv_vp *vp)
455{
456 struct hv_register_assoc explicit_suspend = {
457 .name = HV_REGISTER_EXPLICIT_SUSPEND,
458 .value.explicit_suspend.suspended = 0,
459 };
460 int ret;
461
462 ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
463 1, &explicit_suspend);
464
465 trace_mshv_vp_clear_explicit_suspend(vp->vp_partition->pt_id,
466 vp->vp_index, ret);
467
468 if (ret)
469 vp_err(vp, "Failed to unsuspend\n");
470
471 return ret;
472}
473
474#if IS_ENABLED(CONFIG_X86_64)
475static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp)
476{
477 if (!vp->vp_register_page)
478 return 0;
479 return vp->vp_register_page->interrupt_vectors.as_uint64;
480}
481#else
482static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp)
483{
484 return 0;
485}
486#endif
487
488static bool mshv_vp_dispatch_thread_blocked(struct mshv_vp *vp)
489{
490 struct hv_stats_page **stats = vp->vp_stats_pages;
491 u64 *self_vp_cntrs = stats[HV_STATS_AREA_SELF]->data;
492 u64 *parent_vp_cntrs = stats[HV_STATS_AREA_PARENT]->data;
493
494 return parent_vp_cntrs[HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED] ||
495 self_vp_cntrs[HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED];
496}
497
498static int
499mshv_vp_wait_for_hv_kick(struct mshv_vp *vp)
500{
501 int ret;
502
503 ret = wait_event_interruptible(vp->run.vp_suspend_queue,
504 (vp->run.kicked_by_hv == 1 &&
505 !mshv_vp_dispatch_thread_blocked(vp)) ||
506 mshv_vp_interrupt_pending(vp));
507 if (ret)
508 return -EINTR;
509
510 trace_mshv_vp_wait_for_hv_kick(vp->vp_partition->pt_id,
511 vp->vp_index,
512 vp->run.kicked_by_hv,
513 mshv_vp_dispatch_thread_blocked(vp),
514 mshv_vp_interrupt_pending(vp));
515
516 vp->run.flags.root_sched_blocked = 0;
517 vp->run.kicked_by_hv = 0;
518
519 return 0;
520}
521
522/* Must be called with interrupts enabled */
523static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp)
524{
525 long ret;
526
527 if (vp->run.flags.root_sched_blocked) {
528 /*
529 * Dispatch state of this VP is blocked. Need to wait
530 * for the hypervisor to clear the blocked state before
531 * dispatching it.
532 */
533 ret = mshv_vp_wait_for_hv_kick(vp);
534 if (ret)
535 return ret;
536 }
537
538 do {
539 u32 flags = 0;
540 struct hv_output_dispatch_vp output;
541
542 if (__xfer_to_guest_mode_work_pending()) {
543 ret = xfer_to_guest_mode_handle_work();
544
545 trace_mshv_xfer_to_guest_mode_work(vp->vp_partition->pt_id,
546 vp->vp_index,
547 read_thread_flags(),
548 ret);
549
550 if (ret)
551 break;
552 }
553
554 if (vp->run.flags.intercept_suspend)
555 flags |= HV_DISPATCH_VP_FLAG_CLEAR_INTERCEPT_SUSPEND;
556
557 if (mshv_vp_interrupt_pending(vp))
558 flags |= HV_DISPATCH_VP_FLAG_SCAN_INTERRUPT_INJECTION;
559
560 ret = mshv_vp_dispatch(vp, flags, &output);
561 if (ret)
562 break;
563
564 vp->run.flags.intercept_suspend = 0;
565
566 if (output.dispatch_state == HV_VP_DISPATCH_STATE_BLOCKED) {
567 if (output.dispatch_event ==
568 HV_VP_DISPATCH_EVENT_SUSPEND) {
569 /*
570 * TODO: remove the warning once VP canceling
571 * is supported
572 */
573 WARN_ONCE(atomic64_read(&vp->run.vp_signaled_count),
574 "%s: vp#%d: unexpected explicit suspend\n",
575 __func__, vp->vp_index);
576 /*
577 * Need to clear explicit suspend before
578 * dispatching.
579 * Explicit suspend is either:
580 * - set right after the first VP dispatch or
581 * - set explicitly via hypercall
582 * Since the latter case is not yet supported,
583 * simply clear it here.
584 */
585 ret = mshv_vp_clear_explicit_suspend(vp);
586 if (ret)
587 break;
588
589 ret = mshv_vp_wait_for_hv_kick(vp);
590 if (ret)
591 break;
592 } else {
593 vp->run.flags.root_sched_blocked = 1;
594 ret = mshv_vp_wait_for_hv_kick(vp);
595 if (ret)
596 break;
597 }
598 } else {
599 /* HV_VP_DISPATCH_STATE_READY */
600 if (output.dispatch_event ==
601 HV_VP_DISPATCH_EVENT_INTERCEPT)
602 vp->run.flags.intercept_suspend = 1;
603 }
604 } while (!vp->run.flags.intercept_suspend);
605
606 rseq_virt_userspace_exit();
607
608 return ret;
609}
610
611static_assert(sizeof(struct hv_message) <= MSHV_RUN_VP_BUF_SZ,
612 "sizeof(struct hv_message) must not exceed MSHV_RUN_VP_BUF_SZ");
613
614static struct mshv_mem_region *
615mshv_partition_region_by_gfn(struct mshv_partition *partition, u64 gfn)
616{
617 struct mshv_mem_region *region;
618
619 hlist_for_each_entry(region, &partition->pt_mem_regions, hnode) {
620 if (gfn >= region->start_gfn &&
621 gfn < region->start_gfn + region->nr_pages)
622 return region;
623 }
624
625 return NULL;
626}
627
628static struct mshv_mem_region *
629mshv_partition_region_by_gfn_get(struct mshv_partition *p, u64 gfn)
630{
631 struct mshv_mem_region *region;
632
633 spin_lock(&p->pt_mem_regions_lock);
634 region = mshv_partition_region_by_gfn(p, gfn);
635 if (!region || !mshv_region_get(region)) {
636 spin_unlock(&p->pt_mem_regions_lock);
637 return NULL;
638 }
639 spin_unlock(&p->pt_mem_regions_lock);
640
641 return region;
642}
643
644/**
645 * mshv_handle_gpa_intercept - Handle GPA (Guest Physical Address) intercepts.
646 * @vp: Pointer to the virtual processor structure.
647 *
648 * This function processes GPA intercepts by identifying the memory region
649 * corresponding to the intercepted GPA, aligning the page offset, and
650 * mapping the required pages. It ensures that the region is valid and
651 * handles faults efficiently by mapping multiple pages at once.
652 *
653 * Return: true if the intercept was handled successfully, false otherwise.
654 */
655static bool mshv_handle_gpa_intercept(struct mshv_vp *vp)
656{
657 struct mshv_partition *p = vp->vp_partition;
658 struct mshv_mem_region *region;
659 bool ret = false;
660 u64 gfn;
661#if defined(CONFIG_X86_64)
662 struct hv_x64_memory_intercept_message *msg =
663 (struct hv_x64_memory_intercept_message *)
664 vp->vp_intercept_msg_page->u.payload;
665#elif defined(CONFIG_ARM64)
666 struct hv_arm64_memory_intercept_message *msg =
667 (struct hv_arm64_memory_intercept_message *)
668 vp->vp_intercept_msg_page->u.payload;
669#endif
670 enum hv_intercept_access_type access_type =
671 msg->header.intercept_access_type;
672
673 gfn = HVPFN_DOWN(msg->guest_physical_address);
674
675 region = mshv_partition_region_by_gfn_get(p, gfn);
676 if (!region)
677 goto out;
678
679 if (access_type == HV_INTERCEPT_ACCESS_WRITE &&
680 !(region->hv_map_flags & HV_MAP_GPA_WRITABLE))
681 goto put_region;
682
683 if (access_type == HV_INTERCEPT_ACCESS_EXECUTE &&
684 !(region->hv_map_flags & HV_MAP_GPA_EXECUTABLE))
685 goto put_region;
686
687 /* Only movable memory ranges are supported for GPA intercepts */
688 if (region->mreg_type == MSHV_REGION_TYPE_MEM_MOVABLE)
689 ret = mshv_region_handle_gfn_fault(region, gfn);
690
691put_region:
692 mshv_region_put(region);
693out:
694 trace_mshv_handle_gpa_intercept(p->pt_id, vp->vp_index, gfn,
695 access_type, ret);
696 return ret;
697}
698
699static bool mshv_vp_handle_intercept(struct mshv_vp *vp)
700{
701 switch (vp->vp_intercept_msg_page->header.message_type) {
702 case HVMSG_GPA_INTERCEPT:
703 return mshv_handle_gpa_intercept(vp);
704 }
705 return false;
706}
707
708static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg)
709{
710 long rc;
711
712 trace_mshv_run_vp_entry(vp->vp_partition->pt_id, vp->vp_index);
713
714 do {
715 if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
716 rc = mshv_run_vp_with_root_scheduler(vp);
717 else
718 rc = mshv_run_vp_with_hyp_scheduler(vp);
719 } while (rc == 0 && mshv_vp_handle_intercept(vp));
720
721 trace_mshv_run_vp_exit(vp->vp_partition->pt_id, vp->vp_index,
722 vp->vp_intercept_msg_page->header.message_type,
723 rc);
724
725 if (rc)
726 return rc;
727
728 if (copy_to_user(ret_msg, vp->vp_intercept_msg_page,
729 sizeof(struct hv_message)))
730 rc = -EFAULT;
731
732 return rc;
733}
734
735static int
736mshv_vp_ioctl_get_set_state_pfn(struct mshv_vp *vp,
737 struct hv_vp_state_data state_data,
738 unsigned long user_pfn, size_t page_count,
739 bool is_set)
740{
741 int completed, ret = 0;
742 unsigned long check;
743 struct page **pages;
744
745 if (page_count > INT_MAX)
746 return -EINVAL;
747 /*
748 * Check the arithmetic for wraparound/overflow.
749 * The last page address in the buffer is:
750 * (user_pfn + (page_count - 1)) * PAGE_SIZE
751 */
752 if (check_add_overflow(user_pfn, (page_count - 1), &check))
753 return -EOVERFLOW;
754 if (check_mul_overflow(check, PAGE_SIZE, &check))
755 return -EOVERFLOW;
756
757 /* Pin user pages so hypervisor can copy directly to them */
758 pages = kzalloc_objs(struct page *, page_count);
759 if (!pages)
760 return -ENOMEM;
761
762 for (completed = 0; completed < page_count; completed += ret) {
763 unsigned long user_addr = (user_pfn + completed) * PAGE_SIZE;
764 int remaining = page_count - completed;
765
766 ret = pin_user_pages_fast(user_addr, remaining, FOLL_WRITE,
767 &pages[completed]);
768 if (ret < 0) {
769 vp_err(vp, "%s: Failed to pin user pages error %i\n",
770 __func__, ret);
771 goto unpin_pages;
772 }
773 }
774
775 if (is_set)
776 ret = hv_call_set_vp_state(vp->vp_index,
777 vp->vp_partition->pt_id,
778 state_data, page_count, pages,
779 0, NULL);
780 else
781 ret = hv_call_get_vp_state(vp->vp_index,
782 vp->vp_partition->pt_id,
783 state_data, page_count, pages,
784 NULL);
785
786unpin_pages:
787 unpin_user_pages(pages, completed);
788 kfree(pages);
789 return ret;
790}
791
792static long
793mshv_vp_ioctl_get_set_state(struct mshv_vp *vp,
794 struct mshv_get_set_vp_state __user *user_args,
795 bool is_set)
796{
797 struct mshv_get_set_vp_state args;
798 long ret = 0;
799 union hv_output_get_vp_state vp_state;
800 u32 data_sz;
801 struct hv_vp_state_data state_data = {};
802
803 if (copy_from_user(&args, user_args, sizeof(args)))
804 return -EFAULT;
805
806 if (args.type >= MSHV_VP_STATE_COUNT || mshv_field_nonzero(args, rsvd) ||
807 !args.buf_sz || !PAGE_ALIGNED(args.buf_sz) ||
808 !PAGE_ALIGNED(args.buf_ptr))
809 return -EINVAL;
810
811 if (!access_ok((void __user *)args.buf_ptr, args.buf_sz))
812 return -EFAULT;
813
814 switch (args.type) {
815 case MSHV_VP_STATE_LAPIC:
816 state_data.type = HV_GET_SET_VP_STATE_LAPIC_STATE;
817 data_sz = HV_HYP_PAGE_SIZE;
818 break;
819 case MSHV_VP_STATE_XSAVE:
820 {
821 u64 data_sz_64;
822
823 ret = hv_call_get_partition_property(vp->vp_partition->pt_id,
824 HV_PARTITION_PROPERTY_XSAVE_STATES,
825 &state_data.xsave.states.as_uint64);
826 if (ret)
827 return ret;
828
829 ret = hv_call_get_partition_property(vp->vp_partition->pt_id,
830 HV_PARTITION_PROPERTY_MAX_XSAVE_DATA_SIZE,
831 &data_sz_64);
832 if (ret)
833 return ret;
834
835 data_sz = (u32)data_sz_64;
836 state_data.xsave.flags = 0;
837 /* Always request legacy states */
838 state_data.xsave.states.legacy_x87 = 1;
839 state_data.xsave.states.legacy_sse = 1;
840 state_data.type = HV_GET_SET_VP_STATE_XSAVE;
841 break;
842 }
843 case MSHV_VP_STATE_SIMP:
844 state_data.type = HV_GET_SET_VP_STATE_SIM_PAGE;
845 data_sz = HV_HYP_PAGE_SIZE;
846 break;
847 case MSHV_VP_STATE_SIEFP:
848 state_data.type = HV_GET_SET_VP_STATE_SIEF_PAGE;
849 data_sz = HV_HYP_PAGE_SIZE;
850 break;
851 case MSHV_VP_STATE_SYNTHETIC_TIMERS:
852 state_data.type = HV_GET_SET_VP_STATE_SYNTHETIC_TIMERS;
853 data_sz = sizeof(vp_state.synthetic_timers_state);
854 break;
855 default:
856 return -EINVAL;
857 }
858
859 if (copy_to_user(&user_args->buf_sz, &data_sz, sizeof(user_args->buf_sz)))
860 return -EFAULT;
861
862 if (data_sz > args.buf_sz)
863 return -EINVAL;
864
865 /* If the data is transmitted via pfns, delegate to helper */
866 if (state_data.type & HV_GET_SET_VP_STATE_TYPE_PFN) {
867 unsigned long user_pfn = PFN_DOWN(args.buf_ptr);
868 size_t page_count = PFN_DOWN(args.buf_sz);
869
870 return mshv_vp_ioctl_get_set_state_pfn(vp, state_data, user_pfn,
871 page_count, is_set);
872 }
873
874 /* Paranoia check - this shouldn't happen! */
875 if (data_sz > sizeof(vp_state)) {
876 vp_err(vp, "Invalid vp state data size!\n");
877 return -EINVAL;
878 }
879
880 if (is_set) {
881 if (copy_from_user(&vp_state, (__user void *)args.buf_ptr, data_sz))
882 return -EFAULT;
883
884 return hv_call_set_vp_state(vp->vp_index,
885 vp->vp_partition->pt_id,
886 state_data, 0, NULL,
887 sizeof(vp_state), (u8 *)&vp_state);
888 }
889
890 ret = hv_call_get_vp_state(vp->vp_index, vp->vp_partition->pt_id,
891 state_data, 0, NULL, &vp_state);
892 if (ret)
893 return ret;
894
895 if (copy_to_user((void __user *)args.buf_ptr, &vp_state, data_sz))
896 return -EFAULT;
897
898 return 0;
899}
900
901static long
902mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
903{
904 struct mshv_vp *vp = filp->private_data;
905 long r = -ENOTTY;
906
907 if (mutex_lock_killable(&vp->vp_mutex))
908 return -EINTR;
909
910 switch (ioctl) {
911 case MSHV_RUN_VP:
912 r = mshv_vp_ioctl_run_vp(vp, (void __user *)arg);
913 break;
914 case MSHV_GET_VP_STATE:
915 r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, false);
916 break;
917 case MSHV_SET_VP_STATE:
918 r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, true);
919 break;
920 case MSHV_ROOT_HVCALL:
921 r = mshv_ioctl_passthru_hvcall(vp->vp_partition, false,
922 (void __user *)arg);
923 break;
924 default:
925 vp_warn(vp, "Invalid ioctl: %#x\n", ioctl);
926 break;
927 }
928 mutex_unlock(&vp->vp_mutex);
929
930 return r;
931}
932
933static vm_fault_t mshv_vp_fault(struct vm_fault *vmf)
934{
935 struct mshv_vp *vp = vmf->vma->vm_file->private_data;
936
937 switch (vmf->vma->vm_pgoff) {
938 case MSHV_VP_MMAP_OFFSET_REGISTERS:
939 vmf->page = virt_to_page(vp->vp_register_page);
940 break;
941 case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE:
942 vmf->page = virt_to_page(vp->vp_intercept_msg_page);
943 break;
944 case MSHV_VP_MMAP_OFFSET_GHCB:
945 vmf->page = virt_to_page(vp->vp_ghcb_page);
946 break;
947 default:
948 return VM_FAULT_SIGBUS;
949 }
950
951 get_page(vmf->page);
952
953 return 0;
954}
955
956static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma)
957{
958 struct mshv_vp *vp = file->private_data;
959
960 switch (vma->vm_pgoff) {
961 case MSHV_VP_MMAP_OFFSET_REGISTERS:
962 if (!vp->vp_register_page)
963 return -ENODEV;
964 break;
965 case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE:
966 if (!vp->vp_intercept_msg_page)
967 return -ENODEV;
968 break;
969 case MSHV_VP_MMAP_OFFSET_GHCB:
970 if (!vp->vp_ghcb_page)
971 return -ENODEV;
972 break;
973 default:
974 return -EINVAL;
975 }
976
977 vma->vm_ops = &mshv_vp_vm_ops;
978 return 0;
979}
980
981static int
982mshv_vp_release(struct inode *inode, struct file *filp)
983{
984 struct mshv_vp *vp = filp->private_data;
985
986 trace_mshv_vp_release(vp->vp_partition->pt_id, vp->vp_index);
987
988 /* Rest of VP cleanup happens in destroy_partition() */
989 mshv_partition_put(vp->vp_partition);
990 return 0;
991}
992
993void mshv_vp_stats_unmap(u64 partition_id, u32 vp_index,
994 struct hv_stats_page *stats_pages[])
995{
996 union hv_stats_object_identity identity = {
997 .vp.partition_id = partition_id,
998 .vp.vp_index = vp_index,
999 };
1000 int err;
1001
1002 identity.vp.stats_area_type = HV_STATS_AREA_SELF;
1003 err = hv_unmap_stats_page(HV_STATS_OBJECT_VP,
1004 stats_pages[HV_STATS_AREA_SELF],
1005 &identity);
1006 if (err)
1007 pr_err("%s: failed to unmap partition %llu vp %u self stats, err: %d\n",
1008 __func__, partition_id, vp_index, err);
1009
1010 if (stats_pages[HV_STATS_AREA_PARENT] != stats_pages[HV_STATS_AREA_SELF]) {
1011 identity.vp.stats_area_type = HV_STATS_AREA_PARENT;
1012 err = hv_unmap_stats_page(HV_STATS_OBJECT_VP,
1013 stats_pages[HV_STATS_AREA_PARENT],
1014 &identity);
1015 if (err)
1016 pr_err("%s: failed to unmap partition %llu vp %u parent stats, err: %d\n",
1017 __func__, partition_id, vp_index, err);
1018 }
1019}
1020
1021int mshv_vp_stats_map(u64 partition_id, u32 vp_index,
1022 struct hv_stats_page *stats_pages[])
1023{
1024 union hv_stats_object_identity identity = {
1025 .vp.partition_id = partition_id,
1026 .vp.vp_index = vp_index,
1027 };
1028 int err;
1029
1030 identity.vp.stats_area_type = HV_STATS_AREA_SELF;
1031 err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity,
1032 &stats_pages[HV_STATS_AREA_SELF]);
1033 if (err) {
1034 pr_err("%s: failed to map partition %llu vp %u self stats, err: %d\n",
1035 __func__, partition_id, vp_index, err);
1036 return err;
1037 }
1038
1039 /*
1040 * L1VH partition cannot access its vp stats in parent area.
1041 */
1042 if (is_l1vh_parent(partition_id)) {
1043 stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF];
1044 } else {
1045 identity.vp.stats_area_type = HV_STATS_AREA_PARENT;
1046 err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity,
1047 &stats_pages[HV_STATS_AREA_PARENT]);
1048 if (err) {
1049 pr_err("%s: failed to map partition %llu vp %u parent stats, err: %d\n",
1050 __func__, partition_id, vp_index, err);
1051 goto unmap_self;
1052 }
1053 if (!stats_pages[HV_STATS_AREA_PARENT])
1054 stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF];
1055 }
1056
1057 return 0;
1058
1059unmap_self:
1060 identity.vp.stats_area_type = HV_STATS_AREA_SELF;
1061 hv_unmap_stats_page(HV_STATS_OBJECT_VP,
1062 stats_pages[HV_STATS_AREA_SELF],
1063 &identity);
1064 return err;
1065}
1066
1067static long
1068mshv_partition_ioctl_create_vp(struct mshv_partition *partition,
1069 void __user *arg)
1070{
1071 struct mshv_create_vp args;
1072 struct mshv_vp *vp;
1073 struct page *intercept_msg_page, *register_page, *ghcb_page;
1074 struct hv_stats_page *stats_pages[2];
1075 struct file *file;
1076 int fd;
1077 long ret;
1078
1079 if (copy_from_user(&args, arg, sizeof(args)))
1080 return -EFAULT;
1081
1082 if (args.vp_index >= MSHV_MAX_VPS)
1083 return -EINVAL;
1084
1085 if (partition->pt_vp_array[args.vp_index])
1086 return -EEXIST;
1087
1088 ret = hv_call_create_vp(NUMA_NO_NODE, partition->pt_id, args.vp_index,
1089 0 /* Only valid for root partition VPs */);
1090 if (ret)
1091 return ret;
1092
1093 ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1094 HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1095 input_vtl_zero, &intercept_msg_page);
1096 if (ret)
1097 goto destroy_vp;
1098
1099 if (!mshv_partition_encrypted(partition)) {
1100 ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1101 HV_VP_STATE_PAGE_REGISTERS,
1102 input_vtl_zero, &register_page);
1103 if (ret)
1104 goto unmap_intercept_message_page;
1105 }
1106
1107 if (mshv_partition_encrypted(partition) &&
1108 is_ghcb_mapping_available()) {
1109 ret = hv_map_vp_state_page(partition->pt_id, args.vp_index,
1110 HV_VP_STATE_PAGE_GHCB,
1111 input_vtl_normal, &ghcb_page);
1112 if (ret)
1113 goto unmap_register_page;
1114 }
1115
1116 ret = mshv_vp_stats_map(partition->pt_id, args.vp_index,
1117 stats_pages);
1118 if (ret)
1119 goto unmap_ghcb_page;
1120
1121 vp = kzalloc_obj(*vp);
1122 if (!vp) {
1123 ret = -ENOMEM;
1124 goto unmap_stats_pages;
1125 }
1126
1127 vp->vp_partition = mshv_partition_get(partition);
1128 if (!vp->vp_partition) {
1129 ret = -EBADF;
1130 goto free_vp;
1131 }
1132
1133 mutex_init(&vp->vp_mutex);
1134 init_waitqueue_head(&vp->run.vp_suspend_queue);
1135 atomic64_set(&vp->run.vp_signaled_count, 0);
1136
1137 vp->vp_index = args.vp_index;
1138 vp->vp_intercept_msg_page = page_to_virt(intercept_msg_page);
1139 if (!mshv_partition_encrypted(partition))
1140 vp->vp_register_page = page_to_virt(register_page);
1141
1142 if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available())
1143 vp->vp_ghcb_page = page_to_virt(ghcb_page);
1144
1145 memcpy(vp->vp_stats_pages, stats_pages, sizeof(stats_pages));
1146
1147 ret = mshv_debugfs_vp_create(vp);
1148 if (ret)
1149 goto put_partition;
1150
1151 fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
1152 if (fd < 0) {
1153 ret = fd;
1154 goto remove_debugfs_vp;
1155 }
1156
1157 file = anon_inode_getfile("mshv_vp", &mshv_vp_fops, vp,
1158 O_RDWR | O_CLOEXEC);
1159 if (IS_ERR(file)) {
1160 ret = PTR_ERR(file);
1161 goto put_unused_vp_fd;
1162 }
1163
1164 /* already exclusive with the partition mutex for all ioctls */
1165 partition->pt_vp_count++;
1166 /*
1167 * Pairs with smp_load_acquire() in mshv_try_assert_irq_fast(), which
1168 * can run concurrently from an irqfd waker without holding pt_mutex.
1169 * The release ensures the VP's initialising stores are visible to any
1170 * reader that observes a non-NULL pointer in pt_vp_array.
1171 */
1172 smp_store_release(&partition->pt_vp_array[args.vp_index], vp);
1173
1174 /*
1175 * fd_install() is the userspace-visibility commit point. Must be the
1176 * last operation that can fail or be observed.
1177 */
1178 fd_install(fd, file);
1179 ret = fd;
1180
1181 goto out;
1182
1183put_unused_vp_fd:
1184 put_unused_fd(fd);
1185remove_debugfs_vp:
1186 mshv_debugfs_vp_remove(vp);
1187put_partition:
1188 mshv_partition_put(partition);
1189free_vp:
1190 kfree(vp);
1191unmap_stats_pages:
1192 mshv_vp_stats_unmap(partition->pt_id, args.vp_index, stats_pages);
1193unmap_ghcb_page:
1194 if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available())
1195 hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1196 HV_VP_STATE_PAGE_GHCB, ghcb_page,
1197 input_vtl_normal);
1198unmap_register_page:
1199 if (!mshv_partition_encrypted(partition))
1200 hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1201 HV_VP_STATE_PAGE_REGISTERS,
1202 register_page, input_vtl_zero);
1203unmap_intercept_message_page:
1204 hv_unmap_vp_state_page(partition->pt_id, args.vp_index,
1205 HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1206 intercept_msg_page, input_vtl_zero);
1207destroy_vp:
1208 hv_call_delete_vp(partition->pt_id, args.vp_index);
1209out:
1210 trace_mshv_create_vp(partition->pt_id, args.vp_index, ret);
1211 return ret;
1212}
1213
1214static int mshv_init_async_handler(struct mshv_partition *partition)
1215{
1216 if (completion_done(&partition->async_hypercall)) {
1217 pt_err(partition,
1218 "Cannot issue async hypercall while another one in progress!\n");
1219 return -EPERM;
1220 }
1221
1222 reinit_completion(&partition->async_hypercall);
1223 return 0;
1224}
1225
1226static void mshv_async_hvcall_handler(void *data, u64 *status)
1227{
1228 struct mshv_partition *partition = data;
1229
1230 wait_for_completion(&partition->async_hypercall);
1231 pt_dbg(partition, "Async hypercall completed!\n");
1232
1233 *status = partition->async_hypercall_status;
1234}
1235
1236/*
1237 * NB: caller checks and makes sure mem->size is page aligned
1238 * Returns: 0 with regionpp updated on success, or -errno
1239 */
1240static int mshv_partition_create_region(struct mshv_partition *partition,
1241 struct mshv_user_mem_region *mem,
1242 struct mshv_mem_region **regionpp,
1243 bool is_mmio)
1244{
1245 struct mshv_mem_region *rg;
1246 u64 nr_pages = HVPFN_DOWN(mem->size);
1247
1248 /* Reject overlapping regions */
1249 spin_lock(&partition->pt_mem_regions_lock);
1250 hlist_for_each_entry(rg, &partition->pt_mem_regions, hnode) {
1251 if (mem->guest_pfn + nr_pages <= rg->start_gfn ||
1252 rg->start_gfn + rg->nr_pages <= mem->guest_pfn)
1253 continue;
1254 spin_unlock(&partition->pt_mem_regions_lock);
1255 return -EEXIST;
1256 }
1257 spin_unlock(&partition->pt_mem_regions_lock);
1258
1259 rg = mshv_region_create(mem->guest_pfn, nr_pages,
1260 mem->userspace_addr, mem->flags);
1261 if (IS_ERR(rg))
1262 return PTR_ERR(rg);
1263
1264 if (is_mmio)
1265 rg->mreg_type = MSHV_REGION_TYPE_MMIO;
1266 else if (mshv_partition_encrypted(partition) ||
1267 !mshv_region_movable_init(rg))
1268 rg->mreg_type = MSHV_REGION_TYPE_MEM_PINNED;
1269 else
1270 rg->mreg_type = MSHV_REGION_TYPE_MEM_MOVABLE;
1271
1272 rg->partition = partition;
1273
1274 *regionpp = rg;
1275
1276 return 0;
1277}
1278
1279/**
1280 * mshv_prepare_pinned_region - Pin and map memory regions
1281 * @region: Pointer to the memory region structure
1282 *
1283 * This function processes memory regions that are explicitly marked as pinned.
1284 * Pinned regions are preallocated, mapped upfront, and do not rely on fault-based
1285 * population. The function ensures the region is properly populated, handles
1286 * encryption requirements for SNP partitions if applicable, maps the region,
1287 * and performs necessary sharing or eviction operations based on the mapping
1288 * result.
1289 *
1290 * Return: 0 on success, negative error code on failure.
1291 */
1292static int mshv_prepare_pinned_region(struct mshv_mem_region *region)
1293{
1294 struct mshv_partition *partition = region->partition;
1295 int ret;
1296
1297 ret = mshv_region_pin(region);
1298 if (ret) {
1299 pt_err(partition, "Failed to pin memory region: %d\n",
1300 ret);
1301 goto err_out;
1302 }
1303
1304 /*
1305 * For an SNP partition it is a requirement that for every memory region
1306 * that we are going to map for this partition we should make sure that
1307 * host access to that region is released. This is ensured by doing an
1308 * additional hypercall which will update the SLAT to release host
1309 * access to guest memory regions.
1310 */
1311 if (mshv_partition_encrypted(partition)) {
1312 ret = mshv_region_unshare(region);
1313 if (ret) {
1314 pt_err(partition,
1315 "Failed to unshare memory region (guest_pfn: %llu): %d\n",
1316 region->start_gfn, ret);
1317 goto invalidate_region;
1318 }
1319 }
1320
1321 ret = mshv_region_map(region);
1322 if (ret && mshv_partition_encrypted(partition)) {
1323 int shrc;
1324
1325 shrc = mshv_region_share(region);
1326 if (!shrc)
1327 goto invalidate_region;
1328
1329 pt_err(partition,
1330 "Failed to share memory region (guest_pfn: %llu): %d\n",
1331 region->start_gfn, shrc);
1332 /*
1333 * Don't unpin if marking shared failed because pages are no
1334 * longer mapped in the host, ie root, anymore.
1335 */
1336 goto err_out;
1337 }
1338
1339 return 0;
1340
1341invalidate_region:
1342 mshv_region_invalidate(region);
1343err_out:
1344 return ret;
1345}
1346
1347/*
1348 * This maps two things: guest RAM and for pci passthru mmio space.
1349 *
1350 * mmio:
1351 * - vfio overloads vm_pgoff to store the mmio start pfn/spa.
1352 * - Two things need to happen for mapping mmio range:
1353 * 1. mapped in the uaddr so VMM can access it.
1354 * 2. mapped in the hwpt (gfn <-> mmio phys addr) so guest can access it.
1355 *
1356 * This function takes care of the second. The first one is managed by vfio,
1357 * and hence is taken care of via vfio_pci_mmap_fault().
1358 */
1359static long
1360mshv_map_user_memory(struct mshv_partition *partition,
1361 struct mshv_user_mem_region *mem)
1362{
1363 struct mshv_mem_region *region;
1364 struct vm_area_struct *vma;
1365 bool is_mmio;
1366 ulong mmio_pfn;
1367 long ret;
1368
1369 if (mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP) ||
1370 !access_ok((const void __user *)mem->userspace_addr, mem->size))
1371 return -EINVAL;
1372
1373 mmap_read_lock(current->mm);
1374 vma = vma_lookup(current->mm, mem->userspace_addr);
1375 is_mmio = vma ? !!(vma->vm_flags & (VM_IO | VM_PFNMAP)) : 0;
1376 mmio_pfn = is_mmio ? vma->vm_pgoff : 0;
1377 mmap_read_unlock(current->mm);
1378
1379 if (!vma)
1380 return -EINVAL;
1381
1382 ret = mshv_partition_create_region(partition, mem, &region,
1383 is_mmio);
1384 if (ret)
1385 return ret;
1386
1387 switch (region->mreg_type) {
1388 case MSHV_REGION_TYPE_MEM_PINNED:
1389 ret = mshv_prepare_pinned_region(region);
1390 break;
1391 case MSHV_REGION_TYPE_MEM_MOVABLE:
1392 /*
1393 * For movable memory regions, remap with no access to let
1394 * the hypervisor track dirty pages, enabling pre-copy live
1395 * migration.
1396 */
1397 ret = hv_call_map_gpa_pages(partition->pt_id,
1398 region->start_gfn,
1399 region->nr_pages,
1400 HV_MAP_GPA_NO_ACCESS, NULL);
1401 break;
1402 case MSHV_REGION_TYPE_MMIO:
1403 ret = hv_call_map_mmio_pages(partition->pt_id,
1404 region->start_gfn,
1405 mmio_pfn,
1406 region->nr_pages);
1407 break;
1408 }
1409
1410 trace_mshv_map_user_memory(partition->pt_id, region->start_uaddr,
1411 region->start_gfn, region->nr_pages,
1412 region->hv_map_flags, ret);
1413
1414 if (ret)
1415 goto errout;
1416
1417 spin_lock(&partition->pt_mem_regions_lock);
1418 hlist_add_head(&region->hnode, &partition->pt_mem_regions);
1419 spin_unlock(&partition->pt_mem_regions_lock);
1420
1421 return 0;
1422
1423errout:
1424 mshv_region_put(region);
1425 return ret;
1426}
1427
1428/* Called for unmapping both the guest ram and the mmio space */
1429static long
1430mshv_unmap_user_memory(struct mshv_partition *partition,
1431 struct mshv_user_mem_region *mem)
1432{
1433 struct mshv_mem_region *region;
1434
1435 if (!(mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP)))
1436 return -EINVAL;
1437
1438 spin_lock(&partition->pt_mem_regions_lock);
1439
1440 region = mshv_partition_region_by_gfn(partition, mem->guest_pfn);
1441 if (!region) {
1442 spin_unlock(&partition->pt_mem_regions_lock);
1443 return -ENOENT;
1444 }
1445
1446 /* Paranoia check */
1447 if (region->start_uaddr != mem->userspace_addr ||
1448 region->start_gfn != mem->guest_pfn ||
1449 region->nr_pages != HVPFN_DOWN(mem->size)) {
1450 spin_unlock(&partition->pt_mem_regions_lock);
1451 return -EINVAL;
1452 }
1453
1454 hlist_del(&region->hnode);
1455
1456 spin_unlock(&partition->pt_mem_regions_lock);
1457
1458 mshv_region_put(region);
1459
1460 return 0;
1461}
1462
1463static long
1464mshv_partition_ioctl_set_memory(struct mshv_partition *partition,
1465 struct mshv_user_mem_region __user *user_mem)
1466{
1467 struct mshv_user_mem_region mem;
1468
1469 if (copy_from_user(&mem, user_mem, sizeof(mem)))
1470 return -EFAULT;
1471
1472 if (!mem.size ||
1473 !PAGE_ALIGNED(mem.size) ||
1474 !PAGE_ALIGNED(mem.userspace_addr) ||
1475 (mem.flags & ~MSHV_SET_MEM_FLAGS_MASK) ||
1476 mshv_field_nonzero(mem, rsvd))
1477 return -EINVAL;
1478
1479 if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP))
1480 return mshv_unmap_user_memory(partition, &mem);
1481
1482 return mshv_map_user_memory(partition, &mem);
1483}
1484
1485static long
1486mshv_partition_ioctl_ioeventfd(struct mshv_partition *partition,
1487 void __user *user_args)
1488{
1489 struct mshv_user_ioeventfd args;
1490
1491 if (copy_from_user(&args, user_args, sizeof(args)))
1492 return -EFAULT;
1493
1494 return mshv_set_unset_ioeventfd(partition, &args);
1495}
1496
1497static long
1498mshv_partition_ioctl_irqfd(struct mshv_partition *partition,
1499 void __user *user_args)
1500{
1501 struct mshv_user_irqfd args;
1502
1503 if (copy_from_user(&args, user_args, sizeof(args)))
1504 return -EFAULT;
1505
1506 return mshv_set_unset_irqfd(partition, &args);
1507}
1508
1509static long
1510mshv_partition_ioctl_get_gpap_access_bitmap(struct mshv_partition *partition,
1511 void __user *user_args)
1512{
1513 struct mshv_gpap_access_bitmap args;
1514 union hv_gpa_page_access_state *states;
1515 long ret, i;
1516 union hv_gpa_page_access_state_flags hv_flags = {};
1517 u8 hv_type_mask;
1518 ulong bitmap_buf_sz, states_buf_sz;
1519 int written = 0;
1520
1521 if (copy_from_user(&args, user_args, sizeof(args)))
1522 return -EFAULT;
1523
1524 if (args.access_type >= MSHV_GPAP_ACCESS_TYPE_COUNT ||
1525 args.access_op >= MSHV_GPAP_ACCESS_OP_COUNT ||
1526 mshv_field_nonzero(args, rsvd) || !args.page_count ||
1527 !args.bitmap_ptr)
1528 return -EINVAL;
1529
1530 if (check_mul_overflow(args.page_count, sizeof(*states), &states_buf_sz))
1531 return -E2BIG;
1532
1533 /* Num bytes needed to store bitmap; one bit per page rounded up */
1534 bitmap_buf_sz = DIV_ROUND_UP(args.page_count, 8);
1535
1536 /* Sanity check */
1537 if (bitmap_buf_sz > states_buf_sz)
1538 return -EBADFD;
1539
1540 switch (args.access_type) {
1541 case MSHV_GPAP_ACCESS_TYPE_ACCESSED:
1542 hv_type_mask = 1;
1543 if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) {
1544 hv_flags.clear_accessed = 1;
1545 /* not accessed implies not dirty */
1546 hv_flags.clear_dirty = 1;
1547 } else { /* MSHV_GPAP_ACCESS_OP_SET */
1548 hv_flags.set_accessed = 1;
1549 }
1550 break;
1551 case MSHV_GPAP_ACCESS_TYPE_DIRTY:
1552 hv_type_mask = 2;
1553 if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) {
1554 hv_flags.clear_dirty = 1;
1555 } else { /* MSHV_GPAP_ACCESS_OP_SET */
1556 hv_flags.set_dirty = 1;
1557 /* dirty implies accessed */
1558 hv_flags.set_accessed = 1;
1559 }
1560 break;
1561 }
1562
1563 states = vzalloc(states_buf_sz);
1564 if (!states)
1565 return -ENOMEM;
1566
1567 ret = hv_call_get_gpa_access_states(partition->pt_id, args.page_count,
1568 args.gpap_base, hv_flags, &written,
1569 states);
1570 if (ret)
1571 goto free_return;
1572
1573 /*
1574 * Overwrite states buffer with bitmap - the bits in hv_type_mask
1575 * correspond to bitfields in hv_gpa_page_access_state
1576 */
1577 for (i = 0; i < written; ++i)
1578 __assign_bit(i, (ulong *)states,
1579 states[i].as_uint8 & hv_type_mask);
1580
1581 /* zero the unused bits in the last byte(s) of the returned bitmap */
1582 for (i = written; i < bitmap_buf_sz * 8; ++i)
1583 __clear_bit(i, (ulong *)states);
1584
1585 if (copy_to_user((void __user *)args.bitmap_ptr, states, bitmap_buf_sz))
1586 ret = -EFAULT;
1587
1588free_return:
1589 vfree(states);
1590 return ret;
1591}
1592
1593static long
1594mshv_partition_ioctl_set_msi_routing(struct mshv_partition *partition,
1595 void __user *user_args)
1596{
1597 struct mshv_user_irq_entry *entries = NULL;
1598 struct mshv_user_irq_table args;
1599 long ret;
1600
1601 if (copy_from_user(&args, user_args, sizeof(args)))
1602 return -EFAULT;
1603
1604 if (args.nr > MSHV_MAX_GUEST_IRQS ||
1605 mshv_field_nonzero(args, rsvd))
1606 return -EINVAL;
1607
1608 if (args.nr) {
1609 struct mshv_user_irq_table __user *urouting = user_args;
1610
1611 entries = vmemdup_user(urouting->entries,
1612 array_size(sizeof(*entries),
1613 args.nr));
1614 if (IS_ERR(entries))
1615 return PTR_ERR(entries);
1616 }
1617 ret = mshv_update_routing_table(partition, entries, args.nr);
1618 kvfree(entries);
1619
1620 return ret;
1621}
1622
1623static long
1624mshv_partition_ioctl_initialize(struct mshv_partition *partition)
1625{
1626 long ret;
1627
1628 if (partition->pt_initialized)
1629 return 0;
1630
1631 ret = hv_call_initialize_partition(partition->pt_id);
1632 if (ret)
1633 goto withdraw_mem;
1634
1635 ret = mshv_debugfs_partition_create(partition);
1636 if (ret)
1637 goto finalize_partition;
1638
1639 partition->pt_initialized = true;
1640
1641 return 0;
1642
1643finalize_partition:
1644 hv_call_finalize_partition(partition->pt_id);
1645withdraw_mem:
1646 hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id);
1647
1648 return ret;
1649}
1650
1651static long
1652mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1653{
1654 struct mshv_partition *partition = filp->private_data;
1655 long ret;
1656 void __user *uarg = (void __user *)arg;
1657
1658 if (mutex_lock_killable(&partition->pt_mutex))
1659 return -EINTR;
1660
1661 switch (ioctl) {
1662 case MSHV_INITIALIZE_PARTITION:
1663 ret = mshv_partition_ioctl_initialize(partition);
1664 break;
1665 case MSHV_SET_GUEST_MEMORY:
1666 ret = mshv_partition_ioctl_set_memory(partition, uarg);
1667 break;
1668 case MSHV_CREATE_VP:
1669 ret = mshv_partition_ioctl_create_vp(partition, uarg);
1670 break;
1671 case MSHV_IRQFD:
1672 ret = mshv_partition_ioctl_irqfd(partition, uarg);
1673 break;
1674 case MSHV_IOEVENTFD:
1675 ret = mshv_partition_ioctl_ioeventfd(partition, uarg);
1676 break;
1677 case MSHV_SET_MSI_ROUTING:
1678 ret = mshv_partition_ioctl_set_msi_routing(partition, uarg);
1679 break;
1680 case MSHV_GET_GPAP_ACCESS_BITMAP:
1681 ret = mshv_partition_ioctl_get_gpap_access_bitmap(partition,
1682 uarg);
1683 break;
1684 case MSHV_ROOT_HVCALL:
1685 ret = mshv_ioctl_passthru_hvcall(partition, true, uarg);
1686 break;
1687 default:
1688 ret = -ENOTTY;
1689 }
1690
1691 mutex_unlock(&partition->pt_mutex);
1692 return ret;
1693}
1694
1695static int
1696disable_vp_dispatch(struct mshv_vp *vp)
1697{
1698 int ret;
1699 struct hv_register_assoc dispatch_suspend = {
1700 .name = HV_REGISTER_DISPATCH_SUSPEND,
1701 .value.dispatch_suspend.suspended = 1,
1702 };
1703
1704 ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
1705 1, &dispatch_suspend);
1706 if (ret)
1707 vp_err(vp, "failed to suspend\n");
1708
1709 trace_mshv_disable_vp_dispatch(vp->vp_partition->pt_id,
1710 vp->vp_index, ret);
1711
1712 return ret;
1713}
1714
1715static int
1716get_vp_signaled_count(struct mshv_vp *vp, u64 *count)
1717{
1718 int ret;
1719 struct hv_register_assoc root_signal_count = {
1720 .name = HV_REGISTER_VP_ROOT_SIGNAL_COUNT,
1721 };
1722
1723 ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id,
1724 1, &root_signal_count);
1725
1726 if (ret) {
1727 vp_err(vp, "Failed to get root signal count");
1728 *count = 0;
1729 return ret;
1730 }
1731
1732 *count = root_signal_count.value.reg64;
1733
1734 return ret;
1735}
1736
1737static void
1738drain_vp_signals(struct mshv_vp *vp)
1739{
1740 u64 hv_signal_count;
1741 u64 vp_signal_count;
1742
1743 get_vp_signaled_count(vp, &hv_signal_count);
1744
1745 vp_signal_count = atomic64_read(&vp->run.vp_signaled_count);
1746
1747 /*
1748 * There should be at most 1 outstanding notification, but be extra
1749 * careful anyway.
1750 */
1751 while (hv_signal_count != vp_signal_count) {
1752 WARN_ON(hv_signal_count - vp_signal_count != 1);
1753
1754 if (wait_event_interruptible(vp->run.vp_suspend_queue,
1755 vp->run.kicked_by_hv == 1))
1756 break;
1757 vp->run.kicked_by_hv = 0;
1758 vp_signal_count = atomic64_read(&vp->run.vp_signaled_count);
1759 }
1760
1761 trace_mshv_drain_vp_signals(vp->vp_partition->pt_id, vp->vp_index);
1762}
1763
1764static void drain_all_vps(const struct mshv_partition *partition)
1765{
1766 int i;
1767 struct mshv_vp *vp;
1768
1769 /*
1770 * VPs are reachable from ISR. It is safe to not take the partition
1771 * lock because nobody else can enter this function and drop the
1772 * partition from the list.
1773 */
1774 for (i = 0; i < MSHV_MAX_VPS; i++) {
1775 vp = partition->pt_vp_array[i];
1776 if (!vp)
1777 continue;
1778 /*
1779 * Disable dispatching of the VP in the hypervisor. After this
1780 * the hypervisor guarantees it won't generate any signals for
1781 * the VP and the hypervisor's VP signal count won't change.
1782 */
1783 disable_vp_dispatch(vp);
1784 drain_vp_signals(vp);
1785 }
1786}
1787
1788static void
1789remove_partition(struct mshv_partition *partition)
1790{
1791 spin_lock(&mshv_root.pt_ht_lock);
1792 hlist_del_rcu(&partition->pt_hnode);
1793 spin_unlock(&mshv_root.pt_ht_lock);
1794
1795 synchronize_rcu();
1796}
1797
1798/*
1799 * Tear down a partition and remove it from the list.
1800 * Partition's refcount must be 0
1801 */
1802static void destroy_partition(struct mshv_partition *partition)
1803{
1804 struct mshv_vp *vp;
1805 struct mshv_mem_region *region;
1806 struct hlist_node *n;
1807 int i;
1808
1809 if (refcount_read(&partition->pt_ref_count)) {
1810 pt_err(partition,
1811 "Attempt to destroy partition but refcount > 0\n");
1812 return;
1813 }
1814
1815 trace_mshv_destroy_partition(partition->pt_id);
1816
1817 if (partition->pt_initialized) {
1818 /*
1819 * We only need to drain signals for root scheduler. This should be
1820 * done before removing the partition from the partition list.
1821 */
1822 if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT)
1823 drain_all_vps(partition);
1824
1825 /* Remove vps */
1826 for (i = 0; i < MSHV_MAX_VPS; ++i) {
1827 vp = partition->pt_vp_array[i];
1828 if (!vp)
1829 continue;
1830
1831 mshv_debugfs_vp_remove(vp);
1832 mshv_vp_stats_unmap(partition->pt_id, vp->vp_index,
1833 vp->vp_stats_pages);
1834
1835 if (vp->vp_register_page) {
1836 (void)hv_unmap_vp_state_page(partition->pt_id,
1837 vp->vp_index,
1838 HV_VP_STATE_PAGE_REGISTERS,
1839 virt_to_page(vp->vp_register_page),
1840 input_vtl_zero);
1841 vp->vp_register_page = NULL;
1842 }
1843
1844 (void)hv_unmap_vp_state_page(partition->pt_id,
1845 vp->vp_index,
1846 HV_VP_STATE_PAGE_INTERCEPT_MESSAGE,
1847 virt_to_page(vp->vp_intercept_msg_page),
1848 input_vtl_zero);
1849 vp->vp_intercept_msg_page = NULL;
1850
1851 if (vp->vp_ghcb_page) {
1852 (void)hv_unmap_vp_state_page(partition->pt_id,
1853 vp->vp_index,
1854 HV_VP_STATE_PAGE_GHCB,
1855 virt_to_page(vp->vp_ghcb_page),
1856 input_vtl_normal);
1857 vp->vp_ghcb_page = NULL;
1858 }
1859
1860 kfree(vp);
1861
1862 partition->pt_vp_array[i] = NULL;
1863 }
1864
1865 mshv_debugfs_partition_remove(partition);
1866
1867 /* Deallocates and unmaps everything including vcpus, GPA mappings etc */
1868 hv_call_finalize_partition(partition->pt_id);
1869
1870 partition->pt_initialized = false;
1871 }
1872
1873 remove_partition(partition);
1874
1875 hlist_for_each_entry_safe(region, n, &partition->pt_mem_regions,
1876 hnode) {
1877 hlist_del(&region->hnode);
1878 mshv_region_put(region);
1879 }
1880
1881 /* Withdraw and free all pages we deposited */
1882 hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id);
1883 hv_call_delete_partition(partition->pt_id);
1884
1885 mshv_free_routing_table(partition);
1886 kfree(partition);
1887}
1888
1889struct
1890mshv_partition *mshv_partition_get(struct mshv_partition *partition)
1891{
1892 if (refcount_inc_not_zero(&partition->pt_ref_count))
1893 return partition;
1894 return NULL;
1895}
1896
1897struct
1898mshv_partition *mshv_partition_find(u64 partition_id)
1899 __must_hold(RCU)
1900{
1901 struct mshv_partition *p;
1902
1903 hash_for_each_possible_rcu(mshv_root.pt_htable, p, pt_hnode,
1904 partition_id)
1905 if (p->pt_id == partition_id)
1906 return p;
1907
1908 return NULL;
1909}
1910
1911void
1912mshv_partition_put(struct mshv_partition *partition)
1913{
1914 if (refcount_dec_and_test(&partition->pt_ref_count))
1915 destroy_partition(partition);
1916}
1917
1918static int
1919mshv_partition_release(struct inode *inode, struct file *filp)
1920{
1921 struct mshv_partition *partition = filp->private_data;
1922
1923 trace_mshv_partition_release(partition->pt_id);
1924
1925 mshv_eventfd_release(partition);
1926
1927 cleanup_srcu_struct(&partition->pt_irq_srcu);
1928
1929 mshv_partition_put(partition);
1930
1931 return 0;
1932}
1933
1934static int
1935add_partition(struct mshv_partition *partition)
1936{
1937 spin_lock(&mshv_root.pt_ht_lock);
1938
1939 hash_add_rcu(mshv_root.pt_htable, &partition->pt_hnode,
1940 partition->pt_id);
1941
1942 spin_unlock(&mshv_root.pt_ht_lock);
1943
1944 return 0;
1945}
1946
1947static_assert(MSHV_NUM_CPU_FEATURES_BANKS ==
1948 HV_PARTITION_PROCESSOR_FEATURES_BANKS);
1949
1950static long mshv_ioctl_process_pt_flags(void __user *user_arg, u64 *pt_flags,
1951 struct hv_partition_creation_properties *cr_props,
1952 union hv_partition_isolation_properties *isol_props)
1953{
1954 int i;
1955 struct mshv_create_partition_v2 args;
1956 union hv_partition_processor_features *disabled_procs;
1957 union hv_partition_processor_xsave_features *disabled_xsave;
1958
1959 /* First, copy v1 struct in case user is on previous versions */
1960 if (copy_from_user(&args, user_arg,
1961 sizeof(struct mshv_create_partition)))
1962 return -EFAULT;
1963
1964 if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) ||
1965 args.pt_isolation >= MSHV_PT_ISOLATION_COUNT)
1966 return -EINVAL;
1967
1968 disabled_procs = &cr_props->disabled_processor_features;
1969 disabled_xsave = &cr_props->disabled_processor_xsave_features;
1970
1971 /* Check if user provided newer struct with feature fields */
1972 if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_CPU_AND_XSAVE_FEATURES)) {
1973 if (copy_from_user(&args, user_arg, sizeof(args)))
1974 return -EFAULT;
1975
1976 /* Re-validate v1 fields after second copy_from_user() */
1977 if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) ||
1978 args.pt_isolation >= MSHV_PT_ISOLATION_COUNT)
1979 return -EINVAL;
1980
1981 if (args.pt_num_cpu_fbanks != MSHV_NUM_CPU_FEATURES_BANKS ||
1982 mshv_field_nonzero(args, pt_rsvd) ||
1983 mshv_field_nonzero(args, pt_rsvd1))
1984 return -EINVAL;
1985
1986 /*
1987 * Note this assumes MSHV_NUM_CPU_FEATURES_BANKS will never
1988 * change and equals HV_PARTITION_PROCESSOR_FEATURES_BANKS
1989 * (i.e. 2).
1990 *
1991 * Further banks (index >= 2) will be modifiable as 'early'
1992 * properties via the set partition property hypercall.
1993 */
1994 for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++)
1995 disabled_procs->as_uint64[i] = args.pt_cpu_fbanks[i];
1996
1997#if IS_ENABLED(CONFIG_X86_64)
1998 disabled_xsave->as_uint64 = args.pt_disabled_xsave;
1999#else
2000 /*
2001 * In practice this field is ignored on arm64, but safer to
2002 * zero it in case it is ever used.
2003 */
2004 disabled_xsave->as_uint64 = 0;
2005
2006 if (mshv_field_nonzero(args, pt_rsvd2))
2007 return -EINVAL;
2008#endif
2009 } else {
2010 /*
2011 * v1 behavior: try to enable everything. The hypervisor will
2012 * disable features that are not supported. The banks can be
2013 * queried via the get partition property hypercall.
2014 */
2015 for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++)
2016 disabled_procs->as_uint64[i] = 0;
2017
2018 disabled_xsave->as_uint64 = 0;
2019 }
2020
2021 /* Only support EXO partitions */
2022 *pt_flags = HV_PARTITION_CREATION_FLAG_EXO_PARTITION |
2023 HV_PARTITION_CREATION_FLAG_INTERCEPT_MESSAGE_PAGE_ENABLED;
2024
2025 if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_LAPIC))
2026 *pt_flags |= HV_PARTITION_CREATION_FLAG_LAPIC_ENABLED;
2027 if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_X2APIC))
2028 *pt_flags |= HV_PARTITION_CREATION_FLAG_X2APIC_CAPABLE;
2029 if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_GPA_SUPER_PAGES))
2030 *pt_flags |= HV_PARTITION_CREATION_FLAG_GPA_SUPER_PAGES_ENABLED;
2031 if (args.pt_flags & BIT(MSHV_PT_BIT_NESTED_VIRTUALIZATION))
2032 *pt_flags |= HV_PARTITION_CREATION_FLAG_NESTED_VIRTUALIZATION_CAPABLE;
2033 if (args.pt_flags & BIT(MSHV_PT_BIT_SMT_ENABLED_GUEST))
2034 *pt_flags |= HV_PARTITION_CREATION_FLAG_SMT_ENABLED_GUEST;
2035
2036 isol_props->as_uint64 = 0;
2037
2038 switch (args.pt_isolation) {
2039 case MSHV_PT_ISOLATION_NONE:
2040 isol_props->isolation_type = HV_PARTITION_ISOLATION_TYPE_NONE;
2041 break;
2042 }
2043
2044 return 0;
2045}
2046
2047static long
2048mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev)
2049{
2050 u64 creation_flags;
2051 struct hv_partition_creation_properties creation_properties;
2052 union hv_partition_isolation_properties isolation_properties;
2053 struct mshv_partition *partition;
2054 u64 pt_id = -1;
2055 long ret;
2056
2057 ret = mshv_ioctl_process_pt_flags(user_arg, &creation_flags,
2058 &creation_properties,
2059 &isolation_properties);
2060 if (ret)
2061 return ret;
2062
2063 partition = kzalloc_obj(*partition);
2064 if (!partition)
2065 return -ENOMEM;
2066
2067 partition->pt_module_dev = module_dev;
2068 partition->isolation_type = isolation_properties.isolation_type;
2069
2070 refcount_set(&partition->pt_ref_count, 1);
2071
2072 mutex_init(&partition->pt_mutex);
2073
2074 mutex_init(&partition->pt_irq_lock);
2075
2076 init_completion(&partition->async_hypercall);
2077
2078 INIT_HLIST_HEAD(&partition->irq_ack_notifier_list);
2079
2080 INIT_HLIST_HEAD(&partition->pt_devices);
2081
2082 spin_lock_init(&partition->pt_mem_regions_lock);
2083 INIT_HLIST_HEAD(&partition->pt_mem_regions);
2084
2085 mshv_eventfd_init(partition);
2086
2087 ret = init_srcu_struct(&partition->pt_irq_srcu);
2088 if (ret)
2089 goto free_partition;
2090
2091 ret = hv_call_create_partition(creation_flags,
2092 creation_properties,
2093 isolation_properties,
2094 &pt_id);
2095 if (ret)
2096 goto cleanup_irq_srcu;
2097
2098 partition->pt_id = pt_id;
2099
2100 ret = add_partition(partition);
2101 if (ret)
2102 goto delete_partition;
2103
2104 ret = mshv_init_async_handler(partition);
2105 if (ret)
2106 goto remove_partition;
2107
2108 ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition",
2109 &mshv_partition_fops,
2110 partition, O_RDWR));
2111 if (ret < 0)
2112 goto remove_partition;
2113
2114 goto out;
2115
2116remove_partition:
2117 remove_partition(partition);
2118delete_partition:
2119 hv_call_delete_partition(partition->pt_id);
2120cleanup_irq_srcu:
2121 cleanup_srcu_struct(&partition->pt_irq_srcu);
2122free_partition:
2123 kfree(partition);
2124out:
2125 trace_mshv_create_partition(pt_id, ret);
2126 return ret;
2127}
2128
2129static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl,
2130 unsigned long arg)
2131{
2132 struct miscdevice *misc = filp->private_data;
2133
2134 switch (ioctl) {
2135 case MSHV_CREATE_PARTITION:
2136 return mshv_ioctl_create_partition((void __user *)arg,
2137 misc->this_device);
2138 case MSHV_ROOT_HVCALL:
2139 return mshv_ioctl_passthru_hvcall(NULL, false,
2140 (void __user *)arg);
2141 }
2142
2143 return -ENOTTY;
2144}
2145
2146static int
2147mshv_dev_open(struct inode *inode, struct file *filp)
2148{
2149 return 0;
2150}
2151
2152static int
2153mshv_dev_release(struct inode *inode, struct file *filp)
2154{
2155 return 0;
2156}
2157
2158static int mshv_root_sched_online;
2159
2160static const char *scheduler_type_to_string(enum hv_scheduler_type type)
2161{
2162 switch (type) {
2163 case HV_SCHEDULER_TYPE_LP:
2164 return "classic scheduler without SMT";
2165 case HV_SCHEDULER_TYPE_LP_SMT:
2166 return "classic scheduler with SMT";
2167 case HV_SCHEDULER_TYPE_CORE_SMT:
2168 return "core scheduler";
2169 case HV_SCHEDULER_TYPE_ROOT:
2170 return "root scheduler";
2171 default:
2172 return "unknown scheduler";
2173 };
2174}
2175
2176static int __init l1vh_retrieve_scheduler_type(enum hv_scheduler_type *out)
2177{
2178 u64 integrated_sched_enabled;
2179 int ret;
2180
2181 *out = HV_SCHEDULER_TYPE_CORE_SMT;
2182
2183 if (!mshv_root.vmm_caps.vmm_enable_integrated_scheduler)
2184 return 0;
2185
2186 ret = hv_call_get_partition_property_ex(HV_PARTITION_ID_SELF,
2187 HV_PARTITION_PROPERTY_INTEGRATED_SCHEDULER_ENABLED,
2188 0, &integrated_sched_enabled,
2189 sizeof(integrated_sched_enabled));
2190 if (ret)
2191 return ret;
2192
2193 if (integrated_sched_enabled)
2194 *out = HV_SCHEDULER_TYPE_ROOT;
2195
2196 return 0;
2197}
2198
2199/* TODO move this to hv_common.c when needed outside */
2200static int __init hv_retrieve_scheduler_type(enum hv_scheduler_type *out)
2201{
2202 struct hv_input_get_system_property *input;
2203 struct hv_output_get_system_property *output;
2204 unsigned long flags;
2205 u64 status;
2206
2207 local_irq_save(flags);
2208 input = *this_cpu_ptr(hyperv_pcpu_input_arg);
2209 output = *this_cpu_ptr(hyperv_pcpu_output_arg);
2210
2211 memset(input, 0, sizeof(*input));
2212 memset(output, 0, sizeof(*output));
2213 input->property_id = HV_SYSTEM_PROPERTY_SCHEDULER_TYPE;
2214
2215 status = hv_do_hypercall(HVCALL_GET_SYSTEM_PROPERTY, input, output);
2216 if (!hv_result_success(status)) {
2217 local_irq_restore(flags);
2218 pr_err("%s: %s\n", __func__, hv_result_to_string(status));
2219 return hv_result_to_errno(status);
2220 }
2221
2222 *out = output->scheduler_type;
2223 local_irq_restore(flags);
2224
2225 return 0;
2226}
2227
2228/* Retrieve and stash the supported scheduler type */
2229static int __init mshv_retrieve_scheduler_type(struct device *dev)
2230{
2231 int ret;
2232
2233 if (hv_l1vh_partition())
2234 ret = l1vh_retrieve_scheduler_type(&hv_scheduler_type);
2235 else
2236 ret = hv_retrieve_scheduler_type(&hv_scheduler_type);
2237 if (ret)
2238 return ret;
2239
2240 dev_info(dev, "Hypervisor using %s\n",
2241 scheduler_type_to_string(hv_scheduler_type));
2242
2243 switch (hv_scheduler_type) {
2244 case HV_SCHEDULER_TYPE_CORE_SMT:
2245 case HV_SCHEDULER_TYPE_LP_SMT:
2246 case HV_SCHEDULER_TYPE_ROOT:
2247 case HV_SCHEDULER_TYPE_LP:
2248 /* Supported scheduler, nothing to do */
2249 break;
2250 default:
2251 dev_err(dev, "unsupported scheduler 0x%x, bailing.\n",
2252 hv_scheduler_type);
2253 return -EOPNOTSUPP;
2254 }
2255
2256 return 0;
2257}
2258
2259static int mshv_root_scheduler_init(unsigned int cpu)
2260{
2261 void **inputarg, **outputarg, *p;
2262
2263 inputarg = (void **)this_cpu_ptr(root_scheduler_input);
2264 outputarg = (void **)this_cpu_ptr(root_scheduler_output);
2265
2266 /* Allocate two consecutive pages. One for input, one for output. */
2267 p = kmalloc_array(2, HV_HYP_PAGE_SIZE, GFP_KERNEL);
2268 if (!p)
2269 return -ENOMEM;
2270
2271 *inputarg = p;
2272 *outputarg = (char *)p + HV_HYP_PAGE_SIZE;
2273
2274 return 0;
2275}
2276
2277static int mshv_root_scheduler_cleanup(unsigned int cpu)
2278{
2279 void *p, **inputarg, **outputarg;
2280
2281 inputarg = (void **)this_cpu_ptr(root_scheduler_input);
2282 outputarg = (void **)this_cpu_ptr(root_scheduler_output);
2283
2284 p = *inputarg;
2285
2286 *inputarg = NULL;
2287 *outputarg = NULL;
2288
2289 kfree(p);
2290
2291 return 0;
2292}
2293
2294/* Must be called after retrieving the scheduler type */
2295static int
2296root_scheduler_init(struct device *dev)
2297{
2298 int ret;
2299
2300 if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT)
2301 return 0;
2302
2303 root_scheduler_input = alloc_percpu(void *);
2304 root_scheduler_output = alloc_percpu(void *);
2305
2306 if (!root_scheduler_input || !root_scheduler_output) {
2307 dev_err(dev, "Failed to allocate root scheduler buffers\n");
2308 ret = -ENOMEM;
2309 goto out;
2310 }
2311
2312 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_root_sched",
2313 mshv_root_scheduler_init,
2314 mshv_root_scheduler_cleanup);
2315
2316 if (ret < 0) {
2317 dev_err(dev, "Failed to setup root scheduler state: %i\n", ret);
2318 goto out;
2319 }
2320
2321 mshv_root_sched_online = ret;
2322
2323 return 0;
2324
2325out:
2326 free_percpu(root_scheduler_input);
2327 free_percpu(root_scheduler_output);
2328 return ret;
2329}
2330
2331static void
2332root_scheduler_deinit(void)
2333{
2334 if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT)
2335 return;
2336
2337 cpuhp_remove_state(mshv_root_sched_online);
2338 free_percpu(root_scheduler_input);
2339 free_percpu(root_scheduler_output);
2340}
2341
2342static int __init mshv_init_vmm_caps(struct device *dev)
2343{
2344 int ret;
2345
2346 ret = hv_call_get_partition_property_ex(HV_PARTITION_ID_SELF,
2347 HV_PARTITION_PROPERTY_VMM_CAPABILITIES,
2348 0, &mshv_root.vmm_caps,
2349 sizeof(mshv_root.vmm_caps));
2350 if (ret && hv_l1vh_partition()) {
2351 dev_err(dev, "Failed to get VMM capabilities: %d\n", ret);
2352 return ret;
2353 }
2354
2355 dev_dbg(dev, "vmm_caps = %#llx\n", mshv_root.vmm_caps.as_uint64[0]);
2356
2357 return 0;
2358}
2359
2360static int __init mshv_parent_partition_init(void)
2361{
2362 int ret;
2363 struct device *dev;
2364 union hv_hypervisor_version_info version_info;
2365
2366 if (!hv_parent_partition() || is_kdump_kernel())
2367 return -ENODEV;
2368
2369 if (hv_get_hypervisor_version(&version_info))
2370 return -ENODEV;
2371
2372 ret = misc_register(&mshv_dev);
2373 if (ret)
2374 return ret;
2375
2376 dev = mshv_dev.this_device;
2377
2378 if (version_info.build_number < MSHV_HV_MIN_VERSION ||
2379 version_info.build_number > MSHV_HV_MAX_VERSION) {
2380 dev_err(dev, "Running on unvalidated Hyper-V version\n");
2381 dev_err(dev, "Versions: current: %u min: %u max: %u\n",
2382 version_info.build_number, MSHV_HV_MIN_VERSION,
2383 MSHV_HV_MAX_VERSION);
2384 }
2385
2386 ret = mshv_synic_init(dev);
2387 if (ret)
2388 goto device_deregister;
2389
2390 ret = mshv_init_vmm_caps(dev);
2391 if (ret)
2392 goto synic_cleanup;
2393
2394 ret = mshv_retrieve_scheduler_type(dev);
2395 if (ret)
2396 goto synic_cleanup;
2397
2398 ret = root_scheduler_init(dev);
2399 if (ret)
2400 goto synic_cleanup;
2401
2402 ret = mshv_debugfs_init();
2403 if (ret)
2404 goto deinit_root_scheduler;
2405
2406 ret = mshv_irqfd_wq_init();
2407 if (ret)
2408 goto exit_debugfs;
2409
2410 spin_lock_init(&mshv_root.pt_ht_lock);
2411 hash_init(mshv_root.pt_htable);
2412
2413 hv_setup_mshv_handler(mshv_isr);
2414
2415 return 0;
2416
2417exit_debugfs:
2418 mshv_debugfs_exit();
2419deinit_root_scheduler:
2420 root_scheduler_deinit();
2421synic_cleanup:
2422 mshv_synic_exit();
2423device_deregister:
2424 misc_deregister(&mshv_dev);
2425 return ret;
2426}
2427
2428static void __exit mshv_parent_partition_exit(void)
2429{
2430 hv_setup_mshv_handler(NULL);
2431 mshv_port_table_fini();
2432 mshv_debugfs_exit();
2433 misc_deregister(&mshv_dev);
2434 mshv_irqfd_wq_cleanup();
2435 root_scheduler_deinit();
2436 mshv_synic_exit();
2437}
2438
2439module_init(mshv_parent_partition_init);
2440module_exit(mshv_parent_partition_exit);