projects/linux·blob master · refs

1050 lines · 25.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kvm eventfd support - use eventfd objects to signal various KVM events
4 *
5 * Copyright 2009 Novell. All Rights Reserved.
6 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
7 *
8 * Author:
9 * Gregory Haskins <[email protected]>
10 */
11
12#include <linux/kvm_host.h>
13#include <linux/kvm.h>
14#include <linux/kvm_irqfd.h>
15#include <linux/workqueue.h>
16#include <linux/syscalls.h>
17#include <linux/wait.h>
18#include <linux/poll.h>
19#include <linux/file.h>
20#include <linux/list.h>
21#include <linux/eventfd.h>
22#include <linux/kernel.h>
23#include <linux/srcu.h>
24#include <linux/slab.h>
25#include <linux/seqlock.h>
26#include <linux/irqbypass.h>
27#include <linux/unaligned.h>
28#include <trace/events/kvm.h>
29
30#include <kvm/iodev.h>
31
32#ifdef CONFIG_HAVE_KVM_IRQCHIP
33
34static struct workqueue_struct *irqfd_cleanup_wq;
35
36bool __attribute__((weak))
37kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args)
38{
39 return true;
40}
41
42static void
43irqfd_inject(struct work_struct *work)
44{
45 struct kvm_kernel_irqfd *irqfd =
46 container_of(work, struct kvm_kernel_irqfd, inject);
47 struct kvm *kvm = irqfd->kvm;
48
49 if (!irqfd->resampler) {
50 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,
51 false);
52 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,
53 false);
54 } else
55 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
56 irqfd->gsi, 1, false);
57}
58
59static void irqfd_resampler_notify(struct kvm_kernel_irqfd_resampler *resampler)
60{
61 struct kvm_kernel_irqfd *irqfd;
62
63 list_for_each_entry_srcu(irqfd, &resampler->list, resampler_link,
64 srcu_read_lock_held(&resampler->kvm->irq_srcu))
65 eventfd_signal(irqfd->resamplefd);
66}
67
68/*
69 * Since resampler irqfds share an IRQ source ID, we de-assert once
70 * then notify all of the resampler irqfds using this GSI. We can't
71 * do multiple de-asserts or we risk racing with incoming re-asserts.
72 */
73static void
74irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian)
75{
76 struct kvm_kernel_irqfd_resampler *resampler;
77 struct kvm *kvm;
78 int idx;
79
80 resampler = container_of(kian,
81 struct kvm_kernel_irqfd_resampler, notifier);
82 kvm = resampler->kvm;
83
84 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
85 resampler->notifier.gsi, 0, false);
86
87 idx = srcu_read_lock(&kvm->irq_srcu);
88 irqfd_resampler_notify(resampler);
89 srcu_read_unlock(&kvm->irq_srcu, idx);
90}
91
92static void
93irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)
94{
95 struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;
96 struct kvm *kvm = resampler->kvm;
97
98 mutex_lock(&kvm->irqfds.resampler_lock);
99
100 list_del_rcu(&irqfd->resampler_link);
101
102 if (list_empty(&resampler->list)) {
103 list_del_rcu(&resampler->link);
104 kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier);
105 /*
106 * synchronize_srcu_expedited(&kvm->irq_srcu) already called
107 * in kvm_unregister_irq_ack_notifier().
108 */
109 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
110 resampler->notifier.gsi, 0, false);
111 kfree(resampler);
112 } else {
113 synchronize_srcu_expedited(&kvm->irq_srcu);
114 }
115
116 mutex_unlock(&kvm->irqfds.resampler_lock);
117}
118
119/*
120 * Race-free decouple logic (ordering is critical)
121 */
122static void
123irqfd_shutdown(struct work_struct *work)
124{
125 struct kvm_kernel_irqfd *irqfd =
126 container_of(work, struct kvm_kernel_irqfd, shutdown);
127 struct kvm *kvm = irqfd->kvm;
128 u64 cnt;
129
130 /* Make sure irqfd has been initialized in assign path. */
131 synchronize_srcu_expedited(&kvm->irq_srcu);
132
133 /*
134 * Synchronize with the wait-queue and unhook ourselves to prevent
135 * further events.
136 */
137 eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);
138
139 /*
140 * We know no new events will be scheduled at this point, so block
141 * until all previously outstanding events have completed
142 */
143 flush_work(&irqfd->inject);
144
145 if (irqfd->resampler) {
146 irqfd_resampler_shutdown(irqfd);
147 eventfd_ctx_put(irqfd->resamplefd);
148 }
149
150 /*
151 * It is now safe to release the object's resources
152 */
153#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS)
154 irq_bypass_unregister_consumer(&irqfd->consumer);
155#endif
156 eventfd_ctx_put(irqfd->eventfd);
157 kfree(irqfd);
158}
159
160
161static bool irqfd_is_active(struct kvm_kernel_irqfd *irqfd)
162{
163 /*
164 * Assert that either irqfds.lock or SRCU is held, as irqfds.lock must
165 * be held to prevent false positives (on the irqfd being active), and
166 * while false negatives are impossible as irqfds are never added back
167 * to the list once they're deactivated, the caller must at least hold
168 * SRCU to guard against routing changes if the irqfd is deactivated.
169 */
170 lockdep_assert_once(lockdep_is_held(&irqfd->kvm->irqfds.lock) ||
171 srcu_read_lock_held(&irqfd->kvm->irq_srcu));
172
173 return list_empty(&irqfd->list) ? false : true;
174}
175
176/*
177 * Mark the irqfd as inactive and schedule it for removal
178 */
179static void irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)
180{
181 lockdep_assert_held(&irqfd->kvm->irqfds.lock);
182
183 BUG_ON(!irqfd_is_active(irqfd));
184
185 list_del_init(&irqfd->list);
186
187 queue_work(irqfd_cleanup_wq, &irqfd->shutdown);
188}
189
190int __attribute__((weak)) kvm_arch_set_irq_inatomic(
191 struct kvm_kernel_irq_routing_entry *irq,
192 struct kvm *kvm, int irq_source_id,
193 int level,
194 bool line_status)
195{
196 return -EWOULDBLOCK;
197}
198
199/*
200 * Called with wqh->lock held and interrupts disabled
201 */
202static int
203irqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
204{
205 struct kvm_kernel_irqfd *irqfd =
206 container_of(wait, struct kvm_kernel_irqfd, wait);
207 __poll_t flags = key_to_poll(key);
208 struct kvm_kernel_irq_routing_entry irq;
209 struct kvm *kvm = irqfd->kvm;
210 unsigned seq;
211 int idx;
212 int ret = 0;
213
214 if (flags & EPOLLIN) {
215 /*
216 * WARNING: Do NOT take irqfds.lock in any path except EPOLLHUP,
217 * as KVM holds irqfds.lock when registering the irqfd with the
218 * eventfd.
219 */
220 u64 cnt;
221 eventfd_ctx_do_read(irqfd->eventfd, &cnt);
222
223 idx = srcu_read_lock(&kvm->irq_srcu);
224 do {
225 seq = read_seqcount_begin(&irqfd->irq_entry_sc);
226 irq = irqfd->irq_entry;
227 } while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));
228
229 /*
230 * An event has been signaled, inject an interrupt unless the
231 * irqfd is being deassigned (isn't active), in which case the
232 * routing information may be stale (once the irqfd is removed
233 * from the list, it will stop receiving routing updates).
234 */
235 if (unlikely(!irqfd_is_active(irqfd)) ||
236 kvm_arch_set_irq_inatomic(&irq, kvm,
237 KVM_USERSPACE_IRQ_SOURCE_ID, 1,
238 false) == -EWOULDBLOCK)
239 schedule_work(&irqfd->inject);
240 srcu_read_unlock(&kvm->irq_srcu, idx);
241 ret = 1;
242 }
243
244 if (flags & EPOLLHUP) {
245 /* The eventfd is closing, detach from KVM */
246 unsigned long iflags;
247
248 /*
249 * Taking irqfds.lock is safe here, as KVM holds a reference to
250 * the eventfd when registering the irqfd, i.e. this path can't
251 * be reached while kvm_irqfd_add() is running.
252 */
253 spin_lock_irqsave(&kvm->irqfds.lock, iflags);
254
255 /*
256 * We must check if someone deactivated the irqfd before
257 * we could acquire the irqfds.lock since the item is
258 * deactivated from the KVM side before it is unhooked from
259 * the wait-queue. If it is already deactivated, we can
260 * simply return knowing the other side will cleanup for us.
261 * We cannot race against the irqfd going away since the
262 * other side is required to acquire wqh->lock, which we hold
263 */
264 if (irqfd_is_active(irqfd))
265 irqfd_deactivate(irqfd);
266
267 spin_unlock_irqrestore(&kvm->irqfds.lock, iflags);
268 }
269
270 return ret;
271}
272
273static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)
274{
275 struct kvm_kernel_irq_routing_entry *e;
276 struct kvm_kernel_irq_routing_entry entries[KVM_NR_IRQCHIPS];
277 int n_entries;
278
279 lockdep_assert_held(&kvm->irqfds.lock);
280
281 n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);
282
283 write_seqcount_begin(&irqfd->irq_entry_sc);
284
285 e = entries;
286 if (n_entries == 1)
287 irqfd->irq_entry = *e;
288 else
289 irqfd->irq_entry.type = 0;
290
291 write_seqcount_end(&irqfd->irq_entry_sc);
292}
293
294struct kvm_irqfd_pt {
295 struct kvm_kernel_irqfd *irqfd;
296 struct kvm *kvm;
297 poll_table pt;
298 int ret;
299};
300
301static void kvm_irqfd_register(struct file *file, wait_queue_head_t *wqh,
302 poll_table *pt)
303{
304 struct kvm_irqfd_pt *p = container_of(pt, struct kvm_irqfd_pt, pt);
305 struct kvm_kernel_irqfd *irqfd = p->irqfd;
306 struct kvm *kvm = p->kvm;
307
308 /*
309 * Note, irqfds.lock protects the irqfd's irq_entry, i.e. its routing,
310 * and irqfds.items. It does NOT protect registering with the eventfd.
311 */
312 spin_lock_irq(&kvm->irqfds.lock);
313
314 /*
315 * Initialize the routing information prior to adding the irqfd to the
316 * eventfd's waitqueue, as irqfd_wakeup() can be invoked as soon as the
317 * irqfd is registered.
318 */
319 irqfd_update(kvm, irqfd);
320
321 /*
322 * Add the irqfd as a priority waiter on the eventfd, with a custom
323 * wake-up handler, so that KVM *and only KVM* is notified whenever the
324 * underlying eventfd is signaled.
325 */
326 init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);
327
328 /*
329 * Temporarily lie to lockdep about holding irqfds.lock to avoid a
330 * false positive regarding potential deadlock with irqfd_wakeup()
331 * (see irqfd_wakeup() for details).
332 *
333 * Adding to the wait queue will fail if there is already a priority
334 * waiter, i.e. if the eventfd is associated with another irqfd (in any
335 * VM). Note, kvm_irqfd_deassign() waits for all in-flight shutdown
336 * jobs to complete, i.e. ensures the irqfd has been removed from the
337 * eventfd's waitqueue before returning to userspace.
338 */
339 spin_release(&kvm->irqfds.lock.dep_map, _RET_IP_);
340 p->ret = add_wait_queue_priority_exclusive(wqh, &irqfd->wait);
341 spin_acquire(&kvm->irqfds.lock.dep_map, 0, 0, _RET_IP_);
342 if (p->ret)
343 goto out;
344
345 list_add_tail(&irqfd->list, &kvm->irqfds.items);
346
347out:
348 spin_unlock_irq(&kvm->irqfds.lock);
349}
350
351#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS)
352void __attribute__((weak)) kvm_arch_irq_bypass_stop(
353 struct irq_bypass_consumer *cons)
354{
355}
356
357void __attribute__((weak)) kvm_arch_irq_bypass_start(
358 struct irq_bypass_consumer *cons)
359{
360}
361
362void __weak kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
363 struct kvm_kernel_irq_routing_entry *old,
364 struct kvm_kernel_irq_routing_entry *new)
365{
366
367}
368#endif
369
370static int
371kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args)
372{
373 struct kvm_kernel_irqfd *irqfd;
374 struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL;
375 struct kvm_irqfd_pt irqfd_pt;
376 int ret;
377 __poll_t events;
378 int idx;
379
380 if (!kvm_arch_intc_initialized(kvm))
381 return -EAGAIN;
382
383 if (!kvm_arch_irqfd_allowed(kvm, args))
384 return -EINVAL;
385
386 irqfd = kzalloc_obj(*irqfd, GFP_KERNEL_ACCOUNT);
387 if (!irqfd)
388 return -ENOMEM;
389
390 irqfd->kvm = kvm;
391 irqfd->gsi = args->gsi;
392 INIT_LIST_HEAD(&irqfd->list);
393 INIT_WORK(&irqfd->inject, irqfd_inject);
394 INIT_WORK(&irqfd->shutdown, irqfd_shutdown);
395 seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);
396
397 CLASS(fd, f)(args->fd);
398 if (fd_empty(f)) {
399 ret = -EBADF;
400 goto out;
401 }
402
403 eventfd = eventfd_ctx_fileget(fd_file(f));
404 if (IS_ERR(eventfd)) {
405 ret = PTR_ERR(eventfd);
406 goto out;
407 }
408
409 irqfd->eventfd = eventfd;
410
411 if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) {
412 struct kvm_kernel_irqfd_resampler *resampler;
413
414 resamplefd = eventfd_ctx_fdget(args->resamplefd);
415 if (IS_ERR(resamplefd)) {
416 ret = PTR_ERR(resamplefd);
417 goto fail;
418 }
419
420 irqfd->resamplefd = resamplefd;
421 INIT_LIST_HEAD(&irqfd->resampler_link);
422
423 mutex_lock(&kvm->irqfds.resampler_lock);
424
425 list_for_each_entry(resampler,
426 &kvm->irqfds.resampler_list, link) {
427 if (resampler->notifier.gsi == irqfd->gsi) {
428 irqfd->resampler = resampler;
429 break;
430 }
431 }
432
433 if (!irqfd->resampler) {
434 resampler = kzalloc_obj(*resampler, GFP_KERNEL_ACCOUNT);
435 if (!resampler) {
436 ret = -ENOMEM;
437 mutex_unlock(&kvm->irqfds.resampler_lock);
438 goto fail;
439 }
440
441 resampler->kvm = kvm;
442 INIT_LIST_HEAD(&resampler->list);
443 resampler->notifier.gsi = irqfd->gsi;
444 resampler->notifier.irq_acked = irqfd_resampler_ack;
445 INIT_LIST_HEAD(&resampler->link);
446
447 list_add_rcu(&resampler->link, &kvm->irqfds.resampler_list);
448 kvm_register_irq_ack_notifier(kvm,
449 &resampler->notifier);
450 irqfd->resampler = resampler;
451 }
452
453 list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);
454 synchronize_srcu_expedited(&kvm->irq_srcu);
455
456 mutex_unlock(&kvm->irqfds.resampler_lock);
457 }
458
459 /*
460 * Set the irqfd routing and add it to KVM's list before registering
461 * the irqfd with the eventfd, so that the routing information is valid
462 * and stays valid, e.g. if there are GSI routing changes, prior to
463 * making the irqfd visible, i.e. before it might be signaled.
464 *
465 * Note, holding SRCU ensures a stable read of routing information, and
466 * also prevents irqfd_shutdown() from freeing the irqfd before it's
467 * fully initialized.
468 */
469 idx = srcu_read_lock(&kvm->irq_srcu);
470
471 /*
472 * Register the irqfd with the eventfd by polling on the eventfd, and
473 * simultaneously and the irqfd to KVM's list. If there was en event
474 * pending on the eventfd prior to registering, manually trigger IRQ
475 * injection.
476 */
477 irqfd_pt.irqfd = irqfd;
478 irqfd_pt.kvm = kvm;
479 init_poll_funcptr(&irqfd_pt.pt, kvm_irqfd_register);
480
481 events = vfs_poll(fd_file(f), &irqfd_pt.pt);
482
483 ret = irqfd_pt.ret;
484 if (ret)
485 goto fail_poll;
486
487 if (events & EPOLLIN)
488 schedule_work(&irqfd->inject);
489
490#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS)
491 if (kvm_arch_has_irq_bypass()) {
492 irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;
493 irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;
494 irqfd->consumer.stop = kvm_arch_irq_bypass_stop;
495 irqfd->consumer.start = kvm_arch_irq_bypass_start;
496 ret = irq_bypass_register_consumer(&irqfd->consumer, irqfd->eventfd);
497 if (ret)
498 pr_info("irq bypass consumer (eventfd %p) registration fails: %d\n",
499 irqfd->eventfd, ret);
500 }
501#endif
502
503 srcu_read_unlock(&kvm->irq_srcu, idx);
504 return 0;
505
506fail_poll:
507 srcu_read_unlock(&kvm->irq_srcu, idx);
508fail:
509 if (irqfd->resampler)
510 irqfd_resampler_shutdown(irqfd);
511
512 if (resamplefd && !IS_ERR(resamplefd))
513 eventfd_ctx_put(resamplefd);
514
515 if (eventfd && !IS_ERR(eventfd))
516 eventfd_ctx_put(eventfd);
517
518out:
519 kfree(irqfd);
520 return ret;
521}
522
523bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin)
524{
525 struct kvm_irq_ack_notifier *kian;
526 int gsi, idx;
527
528 idx = srcu_read_lock(&kvm->irq_srcu);
529 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
530 if (gsi != -1)
531 hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
532 link, srcu_read_lock_held(&kvm->irq_srcu))
533 if (kian->gsi == gsi) {
534 srcu_read_unlock(&kvm->irq_srcu, idx);
535 return true;
536 }
537
538 srcu_read_unlock(&kvm->irq_srcu, idx);
539
540 return false;
541}
542EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_irq_has_notifier);
543
544void kvm_notify_acked_gsi(struct kvm *kvm, int gsi)
545{
546 struct kvm_irq_ack_notifier *kian;
547
548 hlist_for_each_entry_srcu(kian, &kvm->irq_ack_notifier_list,
549 link, srcu_read_lock_held(&kvm->irq_srcu))
550 if (kian->gsi == gsi)
551 kian->irq_acked(kian);
552}
553
554void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin)
555{
556 int gsi, idx;
557
558 trace_kvm_ack_irq(irqchip, pin);
559
560 idx = srcu_read_lock(&kvm->irq_srcu);
561 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
562 if (gsi != -1)
563 kvm_notify_acked_gsi(kvm, gsi);
564 srcu_read_unlock(&kvm->irq_srcu, idx);
565}
566
567void kvm_register_irq_ack_notifier(struct kvm *kvm,
568 struct kvm_irq_ack_notifier *kian)
569{
570 mutex_lock(&kvm->irq_lock);
571 hlist_add_head_rcu(&kian->link, &kvm->irq_ack_notifier_list);
572 mutex_unlock(&kvm->irq_lock);
573 kvm_arch_post_irq_ack_notifier_list_update(kvm);
574}
575
576void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
577 struct kvm_irq_ack_notifier *kian)
578{
579 mutex_lock(&kvm->irq_lock);
580 hlist_del_init_rcu(&kian->link);
581 mutex_unlock(&kvm->irq_lock);
582 synchronize_srcu_expedited(&kvm->irq_srcu);
583 kvm_arch_post_irq_ack_notifier_list_update(kvm);
584}
585
586/*
587 * shutdown any irqfd's that match fd+gsi
588 */
589static int
590kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args)
591{
592 struct kvm_kernel_irqfd *irqfd, *tmp;
593 struct eventfd_ctx *eventfd;
594
595 eventfd = eventfd_ctx_fdget(args->fd);
596 if (IS_ERR(eventfd))
597 return PTR_ERR(eventfd);
598
599 spin_lock_irq(&kvm->irqfds.lock);
600
601 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {
602 if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi)
603 irqfd_deactivate(irqfd);
604 }
605
606 spin_unlock_irq(&kvm->irqfds.lock);
607 eventfd_ctx_put(eventfd);
608
609 /*
610 * Block until we know all outstanding shutdown jobs have completed
611 * so that we guarantee there will not be any more interrupts on this
612 * gsi once this deassign function returns.
613 */
614 flush_workqueue(irqfd_cleanup_wq);
615
616 return 0;
617}
618
619int
620kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args)
621{
622 if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE))
623 return -EINVAL;
624
625 if (args->flags & KVM_IRQFD_FLAG_DEASSIGN)
626 return kvm_irqfd_deassign(kvm, args);
627
628 return kvm_irqfd_assign(kvm, args);
629}
630
631/*
632 * This function is called as the kvm VM fd is being released. Shutdown all
633 * irqfds that still remain open
634 */
635void
636kvm_irqfd_release(struct kvm *kvm)
637{
638 struct kvm_kernel_irqfd *irqfd, *tmp;
639
640 spin_lock_irq(&kvm->irqfds.lock);
641
642 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)
643 irqfd_deactivate(irqfd);
644
645 spin_unlock_irq(&kvm->irqfds.lock);
646
647 /*
648 * Block until we know all outstanding shutdown jobs have completed
649 * since we do not take a kvm* reference.
650 */
651 flush_workqueue(irqfd_cleanup_wq);
652
653}
654
655/*
656 * Take note of a change in irq routing.
657 * Caller must invoke synchronize_srcu_expedited(&kvm->irq_srcu) afterwards.
658 */
659void kvm_irq_routing_update(struct kvm *kvm)
660{
661 struct kvm_kernel_irqfd *irqfd;
662
663 spin_lock_irq(&kvm->irqfds.lock);
664
665 list_for_each_entry(irqfd, &kvm->irqfds.items, list) {
666#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS)
667 /* Under irqfds.lock, so can read irq_entry safely */
668 struct kvm_kernel_irq_routing_entry old = irqfd->irq_entry;
669#endif
670
671 irqfd_update(kvm, irqfd);
672
673#if IS_ENABLED(CONFIG_HAVE_KVM_IRQ_BYPASS)
674 if (irqfd->producer)
675 kvm_arch_update_irqfd_routing(irqfd, &old, &irqfd->irq_entry);
676#endif
677 }
678
679 spin_unlock_irq(&kvm->irqfds.lock);
680}
681
682bool kvm_notify_irqfd_resampler(struct kvm *kvm,
683 unsigned int irqchip,
684 unsigned int pin)
685{
686 struct kvm_kernel_irqfd_resampler *resampler;
687 int gsi, idx;
688
689 idx = srcu_read_lock(&kvm->irq_srcu);
690 gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin);
691 if (gsi != -1) {
692 list_for_each_entry_srcu(resampler,
693 &kvm->irqfds.resampler_list, link,
694 srcu_read_lock_held(&kvm->irq_srcu)) {
695 if (resampler->notifier.gsi == gsi) {
696 irqfd_resampler_notify(resampler);
697 srcu_read_unlock(&kvm->irq_srcu, idx);
698 return true;
699 }
700 }
701 }
702 srcu_read_unlock(&kvm->irq_srcu, idx);
703
704 return false;
705}
706
707/*
708 * create a host-wide workqueue for issuing deferred shutdown requests
709 * aggregated from all vm* instances. We need our own isolated
710 * queue to ease flushing work items when a VM exits.
711 */
712int kvm_irqfd_init(void)
713{
714 irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", WQ_PERCPU, 0);
715 if (!irqfd_cleanup_wq)
716 return -ENOMEM;
717
718 return 0;
719}
720
721void kvm_irqfd_exit(void)
722{
723 destroy_workqueue(irqfd_cleanup_wq);
724}
725#endif
726
727/*
728 * --------------------------------------------------------------------
729 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal.
730 *
731 * userspace can register a PIO/MMIO address with an eventfd for receiving
732 * notification when the memory has been touched.
733 * --------------------------------------------------------------------
734 */
735
736struct _ioeventfd {
737 struct list_head list;
738 u64 addr;
739 int length;
740 struct eventfd_ctx *eventfd;
741 u64 datamatch;
742 struct kvm_io_device dev;
743 u8 bus_idx;
744 bool wildcard;
745};
746
747static inline struct _ioeventfd *
748to_ioeventfd(struct kvm_io_device *dev)
749{
750 return container_of(dev, struct _ioeventfd, dev);
751}
752
753static void
754ioeventfd_release(struct _ioeventfd *p)
755{
756 eventfd_ctx_put(p->eventfd);
757 list_del(&p->list);
758 kfree(p);
759}
760
761static bool
762ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val)
763{
764 u64 _val;
765
766 if (addr != p->addr)
767 /* address must be precise for a hit */
768 return false;
769
770 if (!p->length)
771 /* length = 0 means only look at the address, so always a hit */
772 return true;
773
774 if (len != p->length)
775 /* address-range must be precise for a hit */
776 return false;
777
778 if (p->wildcard)
779 /* all else equal, wildcard is always a hit */
780 return true;
781
782 /* otherwise, we have to actually compare the data */
783 switch (len) {
784 case 1:
785 _val = get_unaligned((u8 *)val);
786 break;
787 case 2:
788 _val = get_unaligned((u16 *)val);
789 break;
790 case 4:
791 _val = get_unaligned((u32 *)val);
792 break;
793 case 8:
794 _val = get_unaligned((u64 *)val);
795 break;
796 default:
797 return false;
798 }
799
800 return _val == p->datamatch;
801}
802
803/* MMIO/PIO writes trigger an event if the addr/val match */
804static int
805ioeventfd_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this, gpa_t addr,
806 int len, const void *val)
807{
808 struct _ioeventfd *p = to_ioeventfd(this);
809
810 if (!ioeventfd_in_range(p, addr, len, val))
811 return -EOPNOTSUPP;
812
813 eventfd_signal(p->eventfd);
814 return 0;
815}
816
817/*
818 * This function is called as KVM is completely shutting down. We do not
819 * need to worry about locking just nuke anything we have as quickly as possible
820 */
821static void
822ioeventfd_destructor(struct kvm_io_device *this)
823{
824 struct _ioeventfd *p = to_ioeventfd(this);
825
826 ioeventfd_release(p);
827}
828
829static const struct kvm_io_device_ops ioeventfd_ops = {
830 .write = ioeventfd_write,
831 .destructor = ioeventfd_destructor,
832};
833
834/* assumes kvm->slots_lock held */
835static bool
836ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p)
837{
838 struct _ioeventfd *_p;
839
840 list_for_each_entry(_p, &kvm->ioeventfds, list)
841 if (_p->bus_idx == p->bus_idx &&
842 _p->addr == p->addr &&
843 (!_p->length || !p->length ||
844 (_p->length == p->length &&
845 (_p->wildcard || p->wildcard ||
846 _p->datamatch == p->datamatch))))
847 return true;
848
849 return false;
850}
851
852static enum kvm_bus ioeventfd_bus_from_flags(__u32 flags)
853{
854 if (flags & KVM_IOEVENTFD_FLAG_PIO)
855 return KVM_PIO_BUS;
856 if (flags & KVM_IOEVENTFD_FLAG_VIRTIO_CCW_NOTIFY)
857 return KVM_VIRTIO_CCW_NOTIFY_BUS;
858 return KVM_MMIO_BUS;
859}
860
861static int kvm_assign_ioeventfd_idx(struct kvm *kvm,
862 enum kvm_bus bus_idx,
863 struct kvm_ioeventfd *args)
864{
865
866 struct eventfd_ctx *eventfd;
867 struct _ioeventfd *p;
868 int ret;
869
870 eventfd = eventfd_ctx_fdget(args->fd);
871 if (IS_ERR(eventfd))
872 return PTR_ERR(eventfd);
873
874 p = kzalloc_obj(*p, GFP_KERNEL_ACCOUNT);
875 if (!p) {
876 ret = -ENOMEM;
877 goto fail;
878 }
879
880 INIT_LIST_HEAD(&p->list);
881 p->addr = args->addr;
882 p->bus_idx = bus_idx;
883 p->length = args->len;
884 p->eventfd = eventfd;
885
886 /* The datamatch feature is optional, otherwise this is a wildcard */
887 if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH)
888 p->datamatch = args->datamatch;
889 else
890 p->wildcard = true;
891
892 mutex_lock(&kvm->slots_lock);
893
894 /* Verify that there isn't a match already */
895 if (ioeventfd_check_collision(kvm, p)) {
896 ret = -EEXIST;
897 goto unlock_fail;
898 }
899
900 kvm_iodevice_init(&p->dev, &ioeventfd_ops);
901
902 ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length,
903 &p->dev);
904 if (ret < 0)
905 goto unlock_fail;
906
907 kvm_get_bus(kvm, bus_idx)->ioeventfd_count++;
908 list_add_tail(&p->list, &kvm->ioeventfds);
909
910 mutex_unlock(&kvm->slots_lock);
911
912 return 0;
913
914unlock_fail:
915 mutex_unlock(&kvm->slots_lock);
916 kfree(p);
917
918fail:
919 eventfd_ctx_put(eventfd);
920
921 return ret;
922}
923
924static int
925kvm_deassign_ioeventfd_idx(struct kvm *kvm, enum kvm_bus bus_idx,
926 struct kvm_ioeventfd *args)
927{
928 struct _ioeventfd *p;
929 struct eventfd_ctx *eventfd;
930 struct kvm_io_bus *bus;
931 int ret = -ENOENT;
932 bool wildcard;
933
934 eventfd = eventfd_ctx_fdget(args->fd);
935 if (IS_ERR(eventfd))
936 return PTR_ERR(eventfd);
937
938 wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);
939
940 mutex_lock(&kvm->slots_lock);
941
942 list_for_each_entry(p, &kvm->ioeventfds, list) {
943 if (p->bus_idx != bus_idx ||
944 p->eventfd != eventfd ||
945 p->addr != args->addr ||
946 p->length != args->len ||
947 p->wildcard != wildcard)
948 continue;
949
950 if (!p->wildcard && p->datamatch != args->datamatch)
951 continue;
952
953 kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
954 bus = kvm_get_bus(kvm, bus_idx);
955 if (bus)
956 bus->ioeventfd_count--;
957 ret = 0;
958 break;
959 }
960
961 mutex_unlock(&kvm->slots_lock);
962
963 eventfd_ctx_put(eventfd);
964
965 return ret;
966}
967
968static int kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
969{
970 enum kvm_bus bus_idx = ioeventfd_bus_from_flags(args->flags);
971 int ret = kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
972
973 if (!args->len && bus_idx == KVM_MMIO_BUS)
974 kvm_deassign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
975
976 return ret;
977}
978
979static int
980kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
981{
982 enum kvm_bus bus_idx;
983 int ret;
984
985 bus_idx = ioeventfd_bus_from_flags(args->flags);
986 /* must be natural-word sized, or 0 to ignore length */
987 switch (args->len) {
988 case 0:
989 case 1:
990 case 2:
991 case 4:
992 case 8:
993 break;
994 default:
995 return -EINVAL;
996 }
997
998 /* check for range overflow */
999 if (args->addr + args->len < args->addr)
1000 return -EINVAL;
1001
1002 /* check for extra flags that we don't understand */
1003 if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK)
1004 return -EINVAL;
1005
1006 /* ioeventfd with no length can't be combined with DATAMATCH */
1007 if (!args->len && (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH))
1008 return -EINVAL;
1009
1010 ret = kvm_assign_ioeventfd_idx(kvm, bus_idx, args);
1011 if (ret)
1012 goto fail;
1013
1014 /* When length is ignored, MMIO is also put on a separate bus, for
1015 * faster lookups.
1016 */
1017 if (!args->len && bus_idx == KVM_MMIO_BUS) {
1018 ret = kvm_assign_ioeventfd_idx(kvm, KVM_FAST_MMIO_BUS, args);
1019 if (ret < 0)
1020 goto fast_fail;
1021 }
1022
1023 return 0;
1024
1025fast_fail:
1026 kvm_deassign_ioeventfd_idx(kvm, bus_idx, args);
1027fail:
1028 return ret;
1029}
1030
1031int
1032kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
1033{
1034 if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN)
1035 return kvm_deassign_ioeventfd(kvm, args);
1036
1037 return kvm_assign_ioeventfd(kvm, args);
1038}
1039
1040void
1041kvm_eventfd_init(struct kvm *kvm)
1042{
1043#ifdef CONFIG_HAVE_KVM_IRQCHIP
1044 spin_lock_init(&kvm->irqfds.lock);
1045 INIT_LIST_HEAD(&kvm->irqfds.items);
1046 INIT_LIST_HEAD(&kvm->irqfds.resampler_list);
1047 mutex_init(&kvm->irqfds.resampler_lock);
1048#endif
1049 INIT_LIST_HEAD(&kvm->ioeventfds);
1050}