projects/linux·blob master · refs

728 lines · 16.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 - Cambridge Greys Ltd
4 * Copyright (C) 2011 - 2014 Cisco Systems Inc
5 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
6 * Derived (i.e. mostly copied) from arch/i386/kernel/irq.c:
7 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
8 */
9
10#include <linux/cpumask.h>
11#include <linux/hardirq.h>
12#include <linux/interrupt.h>
13#include <linux/kernel_stat.h>
14#include <linux/module.h>
15#include <linux/sched.h>
16#include <linux/seq_file.h>
17#include <linux/slab.h>
18#include <as-layout.h>
19#include <kern_util.h>
20#include <os.h>
21#include <irq_user.h>
22#include <irq_kern.h>
23#include <linux/time-internal.h>
24
25DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
26
27#define irq_stats(x) (&per_cpu(irq_stat, x))
28
29/* When epoll triggers we do not know why it did so
30 * we can also have different IRQs for read and write.
31 * This is why we keep a small irq_reg array for each fd -
32 * one entry per IRQ type
33 */
34struct irq_reg {
35 void *id;
36 int irq;
37 /* it's cheaper to store this than to query it */
38 int events;
39 bool active;
40 bool pending;
41 bool wakeup;
42#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
43 bool pending_event;
44 void (*timetravel_handler)(int, int, void *,
45 struct time_travel_event *);
46 struct time_travel_event event;
47#endif
48};
49
50struct irq_entry {
51 struct list_head list;
52 int fd;
53 struct irq_reg reg[NUM_IRQ_TYPES];
54 bool suspended;
55 bool sigio_workaround;
56};
57
58static DEFINE_RAW_SPINLOCK(irq_lock);
59static LIST_HEAD(active_fds);
60static DECLARE_BITMAP(irqs_allocated, UM_LAST_SIGNAL_IRQ);
61static bool irqs_suspended;
62#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
63static bool irqs_pending;
64#endif
65
66static void irq_io_loop(struct irq_reg *irq, struct uml_pt_regs *regs)
67{
68/*
69 * irq->active guards against reentry
70 * irq->pending accumulates pending requests
71 * if pending is raised the irq_handler is re-run
72 * until pending is cleared
73 */
74 if (irq->active) {
75 irq->active = false;
76
77 do {
78 irq->pending = false;
79 do_IRQ(irq->irq, regs);
80 } while (irq->pending);
81
82 irq->active = true;
83 } else {
84 irq->pending = true;
85 }
86}
87
88#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
89static void irq_event_handler(struct time_travel_event *ev)
90{
91 struct irq_reg *reg = container_of(ev, struct irq_reg, event);
92
93 /* do nothing if suspended; just cause a wakeup and mark as pending */
94 if (irqs_suspended) {
95 irqs_pending = true;
96 reg->pending_event = true;
97 return;
98 }
99
100 generic_handle_irq(reg->irq);
101}
102
103static bool irq_do_timetravel_handler(struct irq_entry *entry,
104 enum um_irq_type t)
105{
106 struct irq_reg *reg = &entry->reg[t];
107
108 if (!reg->timetravel_handler)
109 return false;
110
111 /*
112 * Handle all messages - we might get multiple even while
113 * interrupts are already suspended, due to suspend order
114 * etc. Note that time_travel_add_irq_event() will not add
115 * an event twice, if it's pending already "first wins".
116 */
117 reg->timetravel_handler(reg->irq, entry->fd, reg->id, &reg->event);
118
119 if (!reg->event.pending)
120 return false;
121
122 return true;
123}
124
125static void irq_do_pending_events(bool timetravel_handlers_only)
126{
127 struct irq_entry *entry;
128
129 if (!irqs_pending || timetravel_handlers_only)
130 return;
131
132 irqs_pending = false;
133
134 list_for_each_entry(entry, &active_fds, list) {
135 enum um_irq_type t;
136
137 for (t = 0; t < NUM_IRQ_TYPES; t++) {
138 struct irq_reg *reg = &entry->reg[t];
139
140 /*
141 * Any timetravel_handler was invoked already, just
142 * directly run the IRQ.
143 */
144 if (reg->pending_event) {
145 irq_enter();
146 generic_handle_irq(reg->irq);
147 irq_exit();
148 reg->pending_event = false;
149 }
150 }
151 }
152}
153#else
154static bool irq_do_timetravel_handler(struct irq_entry *entry,
155 enum um_irq_type t)
156{
157 return false;
158}
159
160static void irq_do_pending_events(bool timetravel_handlers_only)
161{
162}
163#endif
164
165static void sigio_reg_handler(int idx, struct irq_entry *entry, enum um_irq_type t,
166 struct uml_pt_regs *regs,
167 bool timetravel_handlers_only)
168{
169 struct irq_reg *reg = &entry->reg[t];
170
171 if (!reg->events)
172 return;
173
174 if (os_epoll_triggered(idx, reg->events) <= 0)
175 return;
176
177 if (irq_do_timetravel_handler(entry, t))
178 return;
179
180 /*
181 * If we're called to only run time-travel handlers then don't
182 * actually proceed but mark sigio as pending (if applicable).
183 * For suspend/resume, timetravel_handlers_only may be true
184 * despite time-travel not being configured and used.
185 */
186 if (timetravel_handlers_only) {
187#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
188 reg->pending_event = true;
189 irqs_pending = true;
190 mark_sigio_pending();
191#endif
192 return;
193 }
194
195 irq_io_loop(reg, regs);
196}
197
198static void _sigio_handler(struct uml_pt_regs *regs,
199 bool timetravel_handlers_only)
200{
201 struct irq_entry *irq_entry;
202 int n, i;
203
204 if (timetravel_handlers_only && !um_irq_timetravel_handler_used())
205 return;
206
207 /* Flush out pending events that were ignored due to time-travel. */
208 if (!irqs_suspended)
209 irq_do_pending_events(timetravel_handlers_only);
210
211 while (1) {
212 /* This is now lockless - epoll keeps back-referencesto the irqs
213 * which have trigger it so there is no need to walk the irq
214 * list and lock it every time. We avoid locking by turning off
215 * IO for a specific fd by executing os_del_epoll_fd(fd) before
216 * we do any changes to the actual data structures
217 */
218 n = os_waiting_for_events_epoll();
219
220 if (n <= 0) {
221 if (n == -EINTR)
222 continue;
223 else
224 break;
225 }
226
227 for (i = 0; i < n ; i++) {
228 enum um_irq_type t;
229
230 irq_entry = os_epoll_get_data_pointer(i);
231
232 for (t = 0; t < NUM_IRQ_TYPES; t++)
233 sigio_reg_handler(i, irq_entry, t, regs,
234 timetravel_handlers_only);
235 }
236 }
237
238 if (!timetravel_handlers_only)
239 free_irqs();
240}
241
242void sigio_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs,
243 void *mc)
244{
245 preempt_disable();
246 _sigio_handler(regs, irqs_suspended);
247 preempt_enable();
248}
249
250static struct irq_entry *get_irq_entry_by_fd(int fd)
251{
252 struct irq_entry *walk;
253
254 lockdep_assert_held(&irq_lock);
255
256 list_for_each_entry(walk, &active_fds, list) {
257 if (walk->fd == fd)
258 return walk;
259 }
260
261 return NULL;
262}
263
264static void remove_irq_entry(struct irq_entry *to_free, bool remove)
265{
266 if (!to_free)
267 return;
268
269 if (remove)
270 os_del_epoll_fd(to_free->fd);
271 list_del(&to_free->list);
272}
273
274static bool update_irq_entry(struct irq_entry *entry)
275{
276 enum um_irq_type i;
277 int events = 0;
278
279 for (i = 0; i < NUM_IRQ_TYPES; i++)
280 events |= entry->reg[i].events;
281
282 if (events) {
283 /* will modify (instead of add) if needed */
284 os_add_epoll_fd(events, entry->fd, entry);
285 return true;
286 }
287
288 os_del_epoll_fd(entry->fd);
289 return false;
290}
291
292static struct irq_entry *update_or_remove_irq_entry(struct irq_entry *entry)
293{
294 if (update_irq_entry(entry))
295 return NULL;
296 remove_irq_entry(entry, false);
297 return entry;
298}
299
300static int activate_fd(int irq, int fd, enum um_irq_type type, void *dev_id,
301 void (*timetravel_handler)(int, int, void *,
302 struct time_travel_event *))
303{
304 struct irq_entry *irq_entry, *to_free = NULL;
305 int err, events = os_event_mask(type);
306 unsigned long flags;
307
308 err = os_set_fd_async(fd);
309 if (err < 0)
310 goto out;
311
312 raw_spin_lock_irqsave(&irq_lock, flags);
313 irq_entry = get_irq_entry_by_fd(fd);
314 if (irq_entry) {
315already:
316 /* cannot register the same FD twice with the same type */
317 if (WARN_ON(irq_entry->reg[type].events)) {
318 err = -EALREADY;
319 goto out_unlock;
320 }
321
322 /* temporarily disable to avoid IRQ-side locking */
323 os_del_epoll_fd(fd);
324 } else {
325 struct irq_entry *new;
326
327 /* don't restore interrupts */
328 raw_spin_unlock(&irq_lock);
329 new = kzalloc_obj(*irq_entry, GFP_ATOMIC);
330 if (!new) {
331 local_irq_restore(flags);
332 return -ENOMEM;
333 }
334 raw_spin_lock(&irq_lock);
335 irq_entry = get_irq_entry_by_fd(fd);
336 if (irq_entry) {
337 to_free = new;
338 goto already;
339 }
340 irq_entry = new;
341 irq_entry->fd = fd;
342 list_add_tail(&irq_entry->list, &active_fds);
343 maybe_sigio_broken(fd);
344 }
345
346 irq_entry->reg[type].id = dev_id;
347 irq_entry->reg[type].irq = irq;
348 irq_entry->reg[type].active = true;
349 irq_entry->reg[type].events = events;
350
351#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
352 if (um_irq_timetravel_handler_used()) {
353 irq_entry->reg[type].timetravel_handler = timetravel_handler;
354 irq_entry->reg[type].event.fn = irq_event_handler;
355 }
356#endif
357
358 WARN_ON(!update_irq_entry(irq_entry));
359 err = 0;
360out_unlock:
361 raw_spin_unlock_irqrestore(&irq_lock, flags);
362out:
363 kfree(to_free);
364 return err;
365}
366
367/*
368 * Remove the entry or entries for a specific FD, if you
369 * don't want to remove all the possible entries then use
370 * um_free_irq() or deactivate_fd() instead.
371 */
372void free_irq_by_fd(int fd)
373{
374 struct irq_entry *to_free;
375 unsigned long flags;
376
377 raw_spin_lock_irqsave(&irq_lock, flags);
378 to_free = get_irq_entry_by_fd(fd);
379 remove_irq_entry(to_free, true);
380 raw_spin_unlock_irqrestore(&irq_lock, flags);
381 kfree(to_free);
382}
383EXPORT_SYMBOL(free_irq_by_fd);
384
385static void free_irq_by_irq_and_dev(unsigned int irq, void *dev)
386{
387 struct irq_entry *entry, *to_free = NULL;
388 unsigned long flags;
389
390 raw_spin_lock_irqsave(&irq_lock, flags);
391 list_for_each_entry(entry, &active_fds, list) {
392 enum um_irq_type i;
393
394 for (i = 0; i < NUM_IRQ_TYPES; i++) {
395 struct irq_reg *reg = &entry->reg[i];
396
397 if (!reg->events)
398 continue;
399 if (reg->irq != irq)
400 continue;
401 if (reg->id != dev)
402 continue;
403
404 os_del_epoll_fd(entry->fd);
405 reg->events = 0;
406 to_free = update_or_remove_irq_entry(entry);
407 goto out;
408 }
409 }
410out:
411 raw_spin_unlock_irqrestore(&irq_lock, flags);
412 kfree(to_free);
413}
414
415void deactivate_fd(int fd, int irqnum)
416{
417 struct irq_entry *entry;
418 unsigned long flags;
419 enum um_irq_type i;
420
421 os_del_epoll_fd(fd);
422
423 raw_spin_lock_irqsave(&irq_lock, flags);
424 entry = get_irq_entry_by_fd(fd);
425 if (!entry)
426 goto out;
427
428 for (i = 0; i < NUM_IRQ_TYPES; i++) {
429 if (!entry->reg[i].events)
430 continue;
431 if (entry->reg[i].irq == irqnum)
432 entry->reg[i].events = 0;
433 }
434
435 entry = update_or_remove_irq_entry(entry);
436out:
437 raw_spin_unlock_irqrestore(&irq_lock, flags);
438 kfree(entry);
439
440 ignore_sigio_fd(fd);
441}
442EXPORT_SYMBOL(deactivate_fd);
443
444/*
445 * Called just before shutdown in order to provide a clean exec
446 * environment in case the system is rebooting. No locking because
447 * that would cause a pointless shutdown hang if something hadn't
448 * released the lock.
449 */
450int deactivate_all_fds(void)
451{
452 struct irq_entry *entry;
453
454 /* Stop IO. The IRQ loop has no lock so this is our
455 * only way of making sure we are safe to dispose
456 * of all IRQ handlers
457 */
458 os_set_ioignore();
459
460 /* we can no longer call kfree() here so just deactivate */
461 list_for_each_entry(entry, &active_fds, list)
462 os_del_epoll_fd(entry->fd);
463 os_close_epoll_fd();
464 return 0;
465}
466
467/*
468 * do_IRQ handles all normal device IRQs (the special
469 * SMP cross-CPU interrupts have their own specific
470 * handlers).
471 */
472unsigned int do_IRQ(int irq, struct uml_pt_regs *regs)
473{
474 struct pt_regs *old_regs = set_irq_regs((struct pt_regs *)regs);
475 irq_enter();
476 generic_handle_irq(irq);
477 irq_exit();
478 set_irq_regs(old_regs);
479 return 1;
480}
481
482void um_free_irq(int irq, void *dev)
483{
484 if (WARN(irq < 0 || irq > UM_LAST_SIGNAL_IRQ,
485 "freeing invalid irq %d", irq))
486 return;
487
488 free_irq_by_irq_and_dev(irq, dev);
489 free_irq(irq, dev);
490 clear_bit(irq, irqs_allocated);
491}
492EXPORT_SYMBOL(um_free_irq);
493
494static int
495_um_request_irq(int irq, int fd, enum um_irq_type type,
496 irq_handler_t handler, unsigned long irqflags,
497 const char *devname, void *dev_id,
498 void (*timetravel_handler)(int, int, void *,
499 struct time_travel_event *))
500{
501 int err;
502
503 if (irq == UM_IRQ_ALLOC) {
504 int i;
505
506 for (i = UM_FIRST_DYN_IRQ; i < NR_IRQS; i++) {
507 if (!test_and_set_bit(i, irqs_allocated)) {
508 irq = i;
509 break;
510 }
511 }
512 }
513
514 if (irq < 0)
515 return -ENOSPC;
516
517 if (fd != -1) {
518 err = activate_fd(irq, fd, type, dev_id, timetravel_handler);
519 if (err)
520 goto error;
521 }
522
523 err = request_irq(irq, handler, irqflags, devname, dev_id);
524 if (err < 0)
525 goto error;
526
527 return irq;
528error:
529 clear_bit(irq, irqs_allocated);
530 return err;
531}
532
533int um_request_irq(int irq, int fd, enum um_irq_type type,
534 irq_handler_t handler, unsigned long irqflags,
535 const char *devname, void *dev_id)
536{
537 return _um_request_irq(irq, fd, type, handler, irqflags,
538 devname, dev_id, NULL);
539}
540EXPORT_SYMBOL(um_request_irq);
541
542#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
543int um_request_irq_tt(int irq, int fd, enum um_irq_type type,
544 irq_handler_t handler, unsigned long irqflags,
545 const char *devname, void *dev_id,
546 void (*timetravel_handler)(int, int, void *,
547 struct time_travel_event *))
548{
549 return _um_request_irq(irq, fd, type, handler, irqflags,
550 devname, dev_id, timetravel_handler);
551}
552EXPORT_SYMBOL(um_request_irq_tt);
553
554void sigio_run_timetravel_handlers(void)
555{
556 _sigio_handler(NULL, true);
557}
558#endif
559
560#ifdef CONFIG_PM_SLEEP
561void um_irqs_suspend(void)
562{
563 struct irq_entry *entry;
564 unsigned long flags;
565
566 irqs_suspended = true;
567
568 raw_spin_lock_irqsave(&irq_lock, flags);
569 list_for_each_entry(entry, &active_fds, list) {
570 enum um_irq_type t;
571 bool clear = true;
572
573 for (t = 0; t < NUM_IRQ_TYPES; t++) {
574 if (!entry->reg[t].events)
575 continue;
576
577 /*
578 * For the SIGIO_WRITE_IRQ, which is used to handle the
579 * SIGIO workaround thread, we need special handling:
580 * enable wake for it itself, but below we tell it about
581 * any FDs that should be suspended.
582 */
583 if (entry->reg[t].wakeup ||
584 entry->reg[t].irq == SIGIO_WRITE_IRQ
585#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
586 || entry->reg[t].timetravel_handler
587#endif
588 ) {
589 clear = false;
590 break;
591 }
592 }
593
594 if (clear) {
595 entry->suspended = true;
596 os_clear_fd_async(entry->fd);
597 entry->sigio_workaround =
598 !__ignore_sigio_fd(entry->fd);
599 }
600 }
601 raw_spin_unlock_irqrestore(&irq_lock, flags);
602}
603
604void um_irqs_resume(void)
605{
606 struct irq_entry *entry;
607 unsigned long flags;
608
609
610 raw_spin_lock_irqsave(&irq_lock, flags);
611 list_for_each_entry(entry, &active_fds, list) {
612 if (entry->suspended) {
613 int err = os_set_fd_async(entry->fd);
614
615 WARN(err < 0, "os_set_fd_async returned %d\n", err);
616 entry->suspended = false;
617
618 if (entry->sigio_workaround) {
619 err = __add_sigio_fd(entry->fd);
620 WARN(err < 0, "add_sigio_returned %d\n", err);
621 }
622 }
623 }
624 raw_spin_unlock_irqrestore(&irq_lock, flags);
625
626 irqs_suspended = false;
627 send_sigio_to_self();
628}
629
630static int normal_irq_set_wake(struct irq_data *d, unsigned int on)
631{
632 struct irq_entry *entry;
633 unsigned long flags;
634
635 raw_spin_lock_irqsave(&irq_lock, flags);
636 list_for_each_entry(entry, &active_fds, list) {
637 enum um_irq_type t;
638
639 for (t = 0; t < NUM_IRQ_TYPES; t++) {
640 if (!entry->reg[t].events)
641 continue;
642
643 if (entry->reg[t].irq != d->irq)
644 continue;
645 entry->reg[t].wakeup = on;
646 goto unlock;
647 }
648 }
649unlock:
650 raw_spin_unlock_irqrestore(&irq_lock, flags);
651 return 0;
652}
653#else
654#define normal_irq_set_wake NULL
655#endif
656
657/*
658 * irq_chip must define at least enable/disable and ack when
659 * the edge handler is used.
660 */
661static void dummy(struct irq_data *d)
662{
663}
664
665/* This is used for everything other than the timer. */
666static struct irq_chip normal_irq_type = {
667 .name = "SIGIO",
668 .irq_disable = dummy,
669 .irq_enable = dummy,
670 .irq_ack = dummy,
671 .irq_mask = dummy,
672 .irq_unmask = dummy,
673 .irq_set_wake = normal_irq_set_wake,
674};
675
676static struct irq_chip alarm_irq_type = {
677 .name = "SIGALRM",
678 .irq_disable = dummy,
679 .irq_enable = dummy,
680 .irq_ack = dummy,
681 .irq_mask = dummy,
682 .irq_unmask = dummy,
683};
684
685void __init init_IRQ(void)
686{
687 int i;
688
689 irq_set_chip_and_handler(TIMER_IRQ, &alarm_irq_type, handle_percpu_irq);
690
691 for (i = 1; i < UM_LAST_SIGNAL_IRQ; i++)
692 irq_set_chip_and_handler(i, &normal_irq_type, handle_edge_irq);
693 /* Initialize EPOLL Loop */
694 os_setup_epoll();
695}
696
697int __init arch_probe_nr_irqs(void)
698{
699 return NR_IRQS;
700}
701
702void sigchld_handler(int sig, struct siginfo *unused_si,
703 struct uml_pt_regs *regs, void *mc)
704{
705 do_IRQ(SIGCHLD_IRQ, regs);
706}
707
708/*
709 * /proc/interrupts printing for arch specific interrupts
710 */
711int arch_show_interrupts(struct seq_file *p, int prec)
712{
713#if IS_ENABLED(CONFIG_SMP)
714 int cpu;
715
716 seq_printf(p, "%*s: ", prec, "RES");
717 for_each_online_cpu(cpu)
718 seq_printf(p, "%10u ", irq_stats(cpu)->irq_resched_count);
719 seq_puts(p, " Rescheduling interrupts\n");
720
721 seq_printf(p, "%*s: ", prec, "CAL");
722 for_each_online_cpu(cpu)
723 seq_printf(p, "%10u ", irq_stats(cpu)->irq_call_count);
724 seq_puts(p, " Function call interrupts\n");
725#endif
726
727 return 0;
728}