projects/linux·blob master · refs

214 lines · 5.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * IRQ offload/bypass manager
4 *
5 * Copyright (C) 2015 Red Hat, Inc.
6 * Copyright (c) 2015 Linaro Ltd.
7 *
8 * Various virtualization hardware acceleration techniques allow bypassing or
9 * offloading interrupts received from devices around the host kernel. Posted
10 * Interrupts on Intel VT-d systems can allow interrupts to be received
11 * directly by a virtual machine. ARM IRQ Forwarding allows forwarded physical
12 * interrupts to be directly deactivated by the guest. This manager allows
13 * interrupt producers and consumers to find each other to enable this sort of
14 * bypass.
15 */
16
17#include <linux/irqbypass.h>
18#include <linux/list.h>
19#include <linux/module.h>
20#include <linux/mutex.h>
21
22MODULE_LICENSE("GPL v2");
23MODULE_DESCRIPTION("IRQ bypass manager utility module");
24
25static DEFINE_XARRAY(producers);
26static DEFINE_XARRAY(consumers);
27static DEFINE_MUTEX(lock);
28
29/* @lock must be held when calling connect */
30static int __connect(struct irq_bypass_producer *prod,
31 struct irq_bypass_consumer *cons)
32{
33 int ret = 0;
34
35 if (prod->stop)
36 prod->stop(prod);
37 if (cons->stop)
38 cons->stop(cons);
39
40 if (prod->add_consumer)
41 ret = prod->add_consumer(prod, cons);
42
43 if (!ret) {
44 ret = cons->add_producer(cons, prod);
45 if (ret && prod->del_consumer)
46 prod->del_consumer(prod, cons);
47 }
48
49 if (cons->start)
50 cons->start(cons);
51 if (prod->start)
52 prod->start(prod);
53
54 if (!ret) {
55 prod->consumer = cons;
56 cons->producer = prod;
57 }
58 return ret;
59}
60
61/* @lock must be held when calling disconnect */
62static void __disconnect(struct irq_bypass_producer *prod,
63 struct irq_bypass_consumer *cons)
64{
65 if (prod->stop)
66 prod->stop(prod);
67 if (cons->stop)
68 cons->stop(cons);
69
70 cons->del_producer(cons, prod);
71
72 if (prod->del_consumer)
73 prod->del_consumer(prod, cons);
74
75 if (cons->start)
76 cons->start(cons);
77 if (prod->start)
78 prod->start(prod);
79
80 prod->consumer = NULL;
81 cons->producer = NULL;
82}
83
84/**
85 * irq_bypass_register_producer - register IRQ bypass producer
86 * @producer: pointer to producer structure
87 * @eventfd: pointer to the eventfd context associated with the producer
88 * @irq: Linux IRQ number of the underlying producer device
89 *
90 * Add the provided IRQ producer to the set of producers and connect with the
91 * consumer with a matching eventfd, if one exists.
92 */
93int irq_bypass_register_producer(struct irq_bypass_producer *producer,
94 struct eventfd_ctx *eventfd, int irq)
95{
96 unsigned long index = (unsigned long)eventfd;
97 struct irq_bypass_consumer *consumer;
98 int ret;
99
100 if (WARN_ON_ONCE(producer->eventfd))
101 return -EINVAL;
102
103 producer->irq = irq;
104
105 guard(mutex)(&lock);
106
107 ret = xa_insert(&producers, index, producer, GFP_KERNEL);
108 if (ret)
109 return ret;
110
111 consumer = xa_load(&consumers, index);
112 if (consumer) {
113 ret = __connect(producer, consumer);
114 if (ret) {
115 WARN_ON_ONCE(xa_erase(&producers, index) != producer);
116 return ret;
117 }
118 }
119
120 producer->eventfd = eventfd;
121 return 0;
122}
123EXPORT_SYMBOL_GPL(irq_bypass_register_producer);
124
125/**
126 * irq_bypass_unregister_producer - unregister IRQ bypass producer
127 * @producer: pointer to producer structure
128 *
129 * Remove a previously registered IRQ producer (note, it's safe to call this
130 * even if registration was unsuccessful). Disconnect from the associated
131 * consumer, if one exists.
132 */
133void irq_bypass_unregister_producer(struct irq_bypass_producer *producer)
134{
135 unsigned long index = (unsigned long)producer->eventfd;
136
137 if (!producer->eventfd)
138 return;
139
140 guard(mutex)(&lock);
141
142 if (producer->consumer)
143 __disconnect(producer, producer->consumer);
144
145 WARN_ON_ONCE(xa_erase(&producers, index) != producer);
146 producer->eventfd = NULL;
147}
148EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer);
149
150/**
151 * irq_bypass_register_consumer - register IRQ bypass consumer
152 * @consumer: pointer to consumer structure
153 * @eventfd: pointer to the eventfd context associated with the consumer
154 *
155 * Add the provided IRQ consumer to the set of consumers and connect with the
156 * producer with a matching eventfd, if one exists.
157 */
158int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer,
159 struct eventfd_ctx *eventfd)
160{
161 unsigned long index = (unsigned long)eventfd;
162 struct irq_bypass_producer *producer;
163 int ret;
164
165 if (WARN_ON_ONCE(consumer->eventfd))
166 return -EINVAL;
167
168 if (!consumer->add_producer || !consumer->del_producer)
169 return -EINVAL;
170
171 guard(mutex)(&lock);
172
173 ret = xa_insert(&consumers, index, consumer, GFP_KERNEL);
174 if (ret)
175 return ret;
176
177 producer = xa_load(&producers, index);
178 if (producer) {
179 ret = __connect(producer, consumer);
180 if (ret) {
181 WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
182 return ret;
183 }
184 }
185
186 consumer->eventfd = eventfd;
187 return 0;
188}
189EXPORT_SYMBOL_GPL(irq_bypass_register_consumer);
190
191/**
192 * irq_bypass_unregister_consumer - unregister IRQ bypass consumer
193 * @consumer: pointer to consumer structure
194 *
195 * Remove a previously registered IRQ consumer (note, it's safe to call this
196 * even if registration was unsuccessful). Disconnect from the associated
197 * producer, if one exists.
198 */
199void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer)
200{
201 unsigned long index = (unsigned long)consumer->eventfd;
202
203 if (!consumer->eventfd)
204 return;
205
206 guard(mutex)(&lock);
207
208 if (consumer->producer)
209 __disconnect(consumer->producer, consumer);
210
211 WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
212 consumer->eventfd = NULL;
213}
214EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer);