projects/linux·blob master · refs

1583 lines · 41.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4 * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5 *
6 * This file contains the core interrupt handling code, for irq-chip based
7 * architectures. Detailed information is available in
8 * Documentation/core-api/genericirq.rst
9 */
10
11#include <linux/irq.h>
12#include <linux/msi.h>
13#include <linux/module.h>
14#include <linux/interrupt.h>
15#include <linux/kernel_stat.h>
16#include <linux/irqdomain.h>
17#include <linux/preempt.h>
18#include <linux/random.h>
19
20#include <trace/events/irq.h>
21
22#include "internals.h"
23
24static irqreturn_t bad_chained_irq(int irq, void *dev_id)
25{
26 WARN_ONCE(1, "Chained irq %d should not call an action\n", irq);
27 return IRQ_NONE;
28}
29
30/*
31 * Chained handlers should never call action on their IRQ. This default
32 * action will emit warning if such thing happens.
33 */
34struct irqaction chained_action = {
35 .handler = bad_chained_irq,
36};
37
38/**
39 * irq_set_chip - set the irq chip for an irq
40 * @irq: irq number
41 * @chip: pointer to irq chip description structure
42 */
43int irq_set_chip(unsigned int irq, const struct irq_chip *chip)
44{
45 int ret = -EINVAL;
46
47 scoped_irqdesc_get_and_lock(irq, 0) {
48 scoped_irqdesc->irq_data.chip = (struct irq_chip *)(chip ?: &no_irq_chip);
49 ret = 0;
50 }
51 if (!ret) {
52 /* For !CONFIG_SPARSE_IRQ make the irq show up in allocated_irqs. */
53 irq_mark_irq(irq);
54 irq_proc_update_chip(chip);
55 }
56 return ret;
57}
58EXPORT_SYMBOL(irq_set_chip);
59
60/**
61 * irq_set_irq_type - set the irq trigger type for an irq
62 * @irq: irq number
63 * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
64 */
65int irq_set_irq_type(unsigned int irq, unsigned int type)
66{
67 scoped_irqdesc_get_and_buslock(irq, IRQ_GET_DESC_CHECK_GLOBAL)
68 return __irq_set_trigger(scoped_irqdesc, type);
69 return -EINVAL;
70}
71EXPORT_SYMBOL(irq_set_irq_type);
72
73/**
74 * irq_set_handler_data - set irq handler data for an irq
75 * @irq: Interrupt number
76 * @data: Pointer to interrupt specific data
77 *
78 * Set the hardware irq controller data for an irq
79 */
80int irq_set_handler_data(unsigned int irq, void *data)
81{
82 scoped_irqdesc_get_and_lock(irq, 0) {
83 scoped_irqdesc->irq_common_data.handler_data = data;
84 return 0;
85 }
86 return -EINVAL;
87}
88EXPORT_SYMBOL(irq_set_handler_data);
89
90/**
91 * irq_set_msi_desc_off - set MSI descriptor data for an irq at offset
92 * @irq_base: Interrupt number base
93 * @irq_offset: Interrupt number offset
94 * @entry: Pointer to MSI descriptor data
95 *
96 * Set the MSI descriptor entry for an irq at offset
97 */
98int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset, struct msi_desc *entry)
99{
100 scoped_irqdesc_get_and_lock(irq_base + irq_offset, IRQ_GET_DESC_CHECK_GLOBAL) {
101 scoped_irqdesc->irq_common_data.msi_desc = entry;
102 if (entry && !irq_offset)
103 entry->irq = irq_base;
104 return 0;
105 }
106 return -EINVAL;
107}
108
109/**
110 * irq_set_msi_desc - set MSI descriptor data for an irq
111 * @irq: Interrupt number
112 * @entry: Pointer to MSI descriptor data
113 *
114 * Set the MSI descriptor entry for an irq
115 */
116int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry)
117{
118 return irq_set_msi_desc_off(irq, 0, entry);
119}
120
121/**
122 * irq_set_chip_data - set irq chip data for an irq
123 * @irq: Interrupt number
124 * @data: Pointer to chip specific data
125 *
126 * Set the hardware irq chip data for an irq
127 */
128int irq_set_chip_data(unsigned int irq, void *data)
129{
130 scoped_irqdesc_get_and_lock(irq, 0) {
131 scoped_irqdesc->irq_data.chip_data = data;
132 return 0;
133 }
134 return -EINVAL;
135}
136EXPORT_SYMBOL(irq_set_chip_data);
137
138struct irq_data *irq_get_irq_data(unsigned int irq)
139{
140 struct irq_desc *desc = irq_to_desc(irq);
141
142 return desc ? &desc->irq_data : NULL;
143}
144EXPORT_SYMBOL_GPL(irq_get_irq_data);
145
146static void irq_state_clr_disabled(struct irq_desc *desc)
147{
148 irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED);
149}
150
151static void irq_state_clr_masked(struct irq_desc *desc)
152{
153 irqd_clear(&desc->irq_data, IRQD_IRQ_MASKED);
154}
155
156static void irq_state_clr_started(struct irq_desc *desc)
157{
158 irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
159}
160
161static void irq_state_set_started(struct irq_desc *desc)
162{
163 irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
164}
165
166enum {
167 IRQ_STARTUP_NORMAL,
168 IRQ_STARTUP_MANAGED,
169 IRQ_STARTUP_ABORT,
170};
171
172#ifdef CONFIG_SMP
173static int
174__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
175 bool force)
176{
177 struct irq_data *d = irq_desc_get_irq_data(desc);
178
179 if (!irqd_affinity_is_managed(d))
180 return IRQ_STARTUP_NORMAL;
181
182 irqd_clr_managed_shutdown(d);
183
184 if (!cpumask_intersects(aff, cpu_online_mask)) {
185 /*
186 * Catch code which fiddles with enable_irq() on a managed
187 * and potentially shutdown IRQ. Chained interrupt
188 * installment or irq auto probing should not happen on
189 * managed irqs either.
190 */
191 if (WARN_ON_ONCE(force))
192 return IRQ_STARTUP_ABORT;
193 /*
194 * The interrupt was requested, but there is no online CPU
195 * in it's affinity mask. Put it into managed shutdown
196 * state and let the cpu hotplug mechanism start it up once
197 * a CPU in the mask becomes available.
198 */
199 return IRQ_STARTUP_ABORT;
200 }
201 /*
202 * Managed interrupts have reserved resources, so this should not
203 * happen.
204 */
205 if (WARN_ON(irq_domain_activate_irq(d, false)))
206 return IRQ_STARTUP_ABORT;
207 return IRQ_STARTUP_MANAGED;
208}
209
210void irq_startup_managed(struct irq_desc *desc)
211{
212 struct irq_data *d = irq_desc_get_irq_data(desc);
213
214 /*
215 * Clear managed-shutdown flag, so we don't repeat managed-startup for
216 * multiple hotplugs, and cause imbalanced disable depth.
217 */
218 irqd_clr_managed_shutdown(d);
219
220 /*
221 * Only start it up when the disable depth is 1, so that a disable,
222 * hotunplug, hotplug sequence does not end up enabling it during
223 * hotplug unconditionally.
224 */
225 desc->depth--;
226 if (!desc->depth)
227 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
228}
229
230#else
231static __always_inline int
232__irq_startup_managed(struct irq_desc *desc, const struct cpumask *aff,
233 bool force)
234{
235 return IRQ_STARTUP_NORMAL;
236}
237#endif
238
239static void irq_enable(struct irq_desc *desc)
240{
241 if (!irqd_irq_disabled(&desc->irq_data)) {
242 unmask_irq(desc);
243 } else {
244 irq_state_clr_disabled(desc);
245 if (desc->irq_data.chip->irq_enable) {
246 desc->irq_data.chip->irq_enable(&desc->irq_data);
247 irq_state_clr_masked(desc);
248 } else {
249 unmask_irq(desc);
250 }
251 }
252}
253
254static int __irq_startup(struct irq_desc *desc)
255{
256 struct irq_data *d = irq_desc_get_irq_data(desc);
257 int ret = 0;
258
259 /* Warn if this interrupt is not activated but try nevertheless */
260 WARN_ON_ONCE(!irqd_is_activated(d));
261
262 if (d->chip->irq_startup) {
263 ret = d->chip->irq_startup(d);
264 irq_state_clr_disabled(desc);
265 irq_state_clr_masked(desc);
266 } else {
267 irq_enable(desc);
268 }
269 irq_state_set_started(desc);
270 return ret;
271}
272
273int irq_startup(struct irq_desc *desc, bool resend, bool force)
274{
275 struct irq_data *d = irq_desc_get_irq_data(desc);
276 const struct cpumask *aff = irq_data_get_affinity_mask(d);
277 int ret = 0;
278
279 desc->depth = 0;
280
281 if (irqd_is_started(d)) {
282 irq_enable(desc);
283 } else {
284 switch (__irq_startup_managed(desc, aff, force)) {
285 case IRQ_STARTUP_NORMAL:
286 if (d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP)
287 irq_setup_affinity(desc);
288 ret = __irq_startup(desc);
289 if (!(d->chip->flags & IRQCHIP_AFFINITY_PRE_STARTUP))
290 irq_setup_affinity(desc);
291 break;
292 case IRQ_STARTUP_MANAGED:
293 irq_do_set_affinity(d, aff, false);
294 ret = __irq_startup(desc);
295 break;
296 case IRQ_STARTUP_ABORT:
297 desc->depth = 1;
298 irqd_set_managed_shutdown(d);
299 return 0;
300 }
301 }
302 if (resend)
303 check_irq_resend(desc, false);
304
305 return ret;
306}
307
308int irq_activate(struct irq_desc *desc)
309{
310 struct irq_data *d = irq_desc_get_irq_data(desc);
311
312 if (!irqd_affinity_is_managed(d))
313 return irq_domain_activate_irq(d, false);
314 return 0;
315}
316
317int irq_activate_and_startup(struct irq_desc *desc, bool resend)
318{
319 if (WARN_ON(irq_activate(desc)))
320 return 0;
321 return irq_startup(desc, resend, IRQ_START_FORCE);
322}
323
324static void __irq_disable(struct irq_desc *desc, bool mask);
325
326void irq_shutdown(struct irq_desc *desc)
327{
328 if (irqd_is_started(&desc->irq_data)) {
329 clear_irq_resend(desc);
330 /*
331 * Increment disable depth, so that a managed shutdown on
332 * CPU hotunplug preserves the actual disabled state when the
333 * CPU comes back online. See irq_startup_managed().
334 */
335 desc->depth++;
336
337 if (desc->irq_data.chip->irq_shutdown) {
338 desc->irq_data.chip->irq_shutdown(&desc->irq_data);
339 irq_state_set_disabled(desc);
340 irq_state_set_masked(desc);
341 } else {
342 __irq_disable(desc, true);
343 }
344 irq_state_clr_started(desc);
345 }
346}
347
348
349void irq_shutdown_and_deactivate(struct irq_desc *desc)
350{
351 irq_shutdown(desc);
352 /*
353 * This must be called even if the interrupt was never started up,
354 * because the activation can happen before the interrupt is
355 * available for request/startup. It has it's own state tracking so
356 * it's safe to call it unconditionally.
357 */
358 irq_domain_deactivate_irq(&desc->irq_data);
359}
360
361static void __irq_disable(struct irq_desc *desc, bool mask)
362{
363 if (irqd_irq_disabled(&desc->irq_data)) {
364 if (mask)
365 mask_irq(desc);
366 } else {
367 irq_state_set_disabled(desc);
368 if (desc->irq_data.chip->irq_disable) {
369 desc->irq_data.chip->irq_disable(&desc->irq_data);
370 irq_state_set_masked(desc);
371 } else if (mask) {
372 mask_irq(desc);
373 }
374 }
375}
376
377/**
378 * irq_disable - Mark interrupt disabled
379 * @desc: irq descriptor which should be disabled
380 *
381 * If the chip does not implement the irq_disable callback, we
382 * use a lazy disable approach. That means we mark the interrupt
383 * disabled, but leave the hardware unmasked. That's an
384 * optimization because we avoid the hardware access for the
385 * common case where no interrupt happens after we marked it
386 * disabled. If an interrupt happens, then the interrupt flow
387 * handler masks the line at the hardware level and marks it
388 * pending.
389 *
390 * If the interrupt chip does not implement the irq_disable callback,
391 * a driver can disable the lazy approach for a particular irq line by
392 * calling 'irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY)'. This can
393 * be used for devices which cannot disable the interrupt at the
394 * device level under certain circumstances and have to use
395 * disable_irq[_nosync] instead.
396 */
397void irq_disable(struct irq_desc *desc)
398{
399 __irq_disable(desc, irq_settings_disable_unlazy(desc));
400}
401
402void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
403{
404 if (desc->irq_data.chip->irq_enable)
405 desc->irq_data.chip->irq_enable(&desc->irq_data);
406 else
407 desc->irq_data.chip->irq_unmask(&desc->irq_data);
408 cpumask_set_cpu(cpu, desc->percpu_enabled);
409}
410
411void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
412{
413 if (desc->irq_data.chip->irq_disable)
414 desc->irq_data.chip->irq_disable(&desc->irq_data);
415 else
416 desc->irq_data.chip->irq_mask(&desc->irq_data);
417 cpumask_clear_cpu(cpu, desc->percpu_enabled);
418}
419
420static inline void mask_ack_irq(struct irq_desc *desc)
421{
422 if (desc->irq_data.chip->irq_mask_ack) {
423 desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
424 irq_state_set_masked(desc);
425 } else {
426 mask_irq(desc);
427 if (desc->irq_data.chip->irq_ack)
428 desc->irq_data.chip->irq_ack(&desc->irq_data);
429 }
430}
431
432void mask_irq(struct irq_desc *desc)
433{
434 if (irqd_irq_masked(&desc->irq_data))
435 return;
436
437 if (desc->irq_data.chip->irq_mask) {
438 desc->irq_data.chip->irq_mask(&desc->irq_data);
439 irq_state_set_masked(desc);
440 }
441}
442
443void unmask_irq(struct irq_desc *desc)
444{
445 if (!irqd_irq_masked(&desc->irq_data))
446 return;
447
448 if (desc->irq_data.chip->irq_unmask) {
449 desc->irq_data.chip->irq_unmask(&desc->irq_data);
450 irq_state_clr_masked(desc);
451 }
452}
453
454void unmask_threaded_irq(struct irq_desc *desc)
455{
456 struct irq_chip *chip = desc->irq_data.chip;
457
458 if (chip->flags & IRQCHIP_EOI_THREADED)
459 chip->irq_eoi(&desc->irq_data);
460
461 unmask_irq(desc);
462}
463
464/* Busy wait until INPROGRESS is cleared */
465static bool irq_wait_on_inprogress(struct irq_desc *desc)
466{
467 if (IS_ENABLED(CONFIG_SMP)) {
468 do {
469 raw_spin_unlock(&desc->lock);
470 while (irqd_irq_inprogress(&desc->irq_data))
471 cpu_relax();
472 raw_spin_lock(&desc->lock);
473 } while (irqd_irq_inprogress(&desc->irq_data));
474
475 /* Might have been disabled in meantime */
476 return !irqd_irq_disabled(&desc->irq_data) && desc->action;
477 }
478 return false;
479}
480
481static bool irq_can_handle_pm(struct irq_desc *desc)
482{
483 struct irq_data *irqd = &desc->irq_data;
484 const struct cpumask *aff;
485
486 /*
487 * If the interrupt is not in progress and is not an armed
488 * wakeup interrupt, proceed.
489 */
490 if (!irqd_has_set(irqd, IRQD_IRQ_INPROGRESS | IRQD_WAKEUP_ARMED))
491 return true;
492
493 /*
494 * If the interrupt is an armed wakeup source, mark it pending
495 * and suspended, disable it and notify the pm core about the
496 * event.
497 */
498 if (unlikely(irqd_has_set(irqd, IRQD_WAKEUP_ARMED))) {
499 irq_pm_handle_wakeup(desc);
500 return false;
501 }
502
503 /* Check whether the interrupt is polled on another CPU */
504 if (unlikely(desc->istate & IRQS_POLL_INPROGRESS)) {
505 if (WARN_ONCE(irq_poll_cpu == smp_processor_id(),
506 "irq poll in progress on cpu %d for irq %d\n",
507 smp_processor_id(), desc->irq_data.irq))
508 return false;
509 return irq_wait_on_inprogress(desc);
510 }
511
512 /* The below works only for single target interrupts */
513 if (!IS_ENABLED(CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK) ||
514 !irqd_is_single_target(irqd) || desc->handle_irq != handle_edge_irq)
515 return false;
516
517 /*
518 * If the interrupt affinity was moved to this CPU and the
519 * interrupt is currently handled on the previous target CPU, then
520 * busy wait for INPROGRESS to be cleared. Otherwise for edge type
521 * interrupts the handler might get stuck on the previous target:
522 *
523 * CPU 0 CPU 1 (new target)
524 * handle_edge_irq()
525 * repeat:
526 * handle_event() handle_edge_irq()
527 * if (INPROGESS) {
528 * set(PENDING);
529 * mask();
530 * return;
531 * }
532 * if (PENDING) {
533 * clear(PENDING);
534 * unmask();
535 * goto repeat;
536 * }
537 *
538 * This happens when the device raises interrupts with a high rate
539 * and always before handle_event() completes and the CPU0 handler
540 * can clear INPROGRESS. This has been observed in virtual machines.
541 */
542 aff = irq_data_get_effective_affinity_mask(irqd);
543 if (cpumask_first(aff) != smp_processor_id())
544 return false;
545 return irq_wait_on_inprogress(desc);
546}
547
548static inline bool irq_can_handle_actions(struct irq_desc *desc)
549{
550 desc->istate &= ~(IRQS_REPLAY | IRQS_WAITING);
551
552 if (unlikely(!desc->action || irqd_irq_disabled(&desc->irq_data))) {
553 desc->istate |= IRQS_PENDING;
554 return false;
555 }
556 return true;
557}
558
559static inline bool irq_can_handle(struct irq_desc *desc)
560{
561 if (!irq_can_handle_pm(desc))
562 return false;
563
564 return irq_can_handle_actions(desc);
565}
566
567/**
568 * handle_nested_irq - Handle a nested irq from a irq thread
569 * @irq: the interrupt number
570 *
571 * Handle interrupts which are nested into a threaded interrupt
572 * handler. The handler function is called inside the calling threads
573 * context.
574 */
575void handle_nested_irq(unsigned int irq)
576{
577 struct irq_desc *desc = irq_to_desc(irq);
578 struct irqaction *action;
579 irqreturn_t action_ret;
580
581 might_sleep();
582
583 scoped_guard(raw_spinlock_irq, &desc->lock) {
584 if (!irq_can_handle_actions(desc))
585 return;
586
587 action = desc->action;
588 kstat_incr_irqs_this_cpu(desc);
589 atomic_inc(&desc->threads_active);
590 }
591
592 action_ret = IRQ_NONE;
593 for_each_action_of_desc(desc, action)
594 action_ret |= action->thread_fn(action->irq, action->dev_id);
595
596 if (!irq_settings_no_debug(desc))
597 note_interrupt(desc, action_ret);
598
599 wake_threads_waitq(desc);
600}
601EXPORT_SYMBOL_GPL(handle_nested_irq);
602
603/**
604 * handle_simple_irq - Simple and software-decoded IRQs.
605 * @desc: the interrupt description structure for this irq
606 *
607 * Simple interrupts are either sent from a demultiplexing interrupt
608 * handler or come from hardware, where no interrupt hardware control is
609 * necessary.
610 *
611 * Note: The caller is expected to handle the ack, clear, mask and unmask
612 * issues if necessary.
613 */
614void handle_simple_irq(struct irq_desc *desc)
615{
616 guard(raw_spinlock)(&desc->lock);
617
618 if (!irq_can_handle_pm(desc)) {
619 if (irqd_needs_resend_when_in_progress(&desc->irq_data))
620 desc->istate |= IRQS_PENDING;
621 return;
622 }
623
624 if (!irq_can_handle_actions(desc))
625 return;
626
627 kstat_incr_irqs_this_cpu(desc);
628 handle_irq_event(desc);
629}
630EXPORT_SYMBOL_GPL(handle_simple_irq);
631
632/**
633 * handle_untracked_irq - Simple and software-decoded IRQs.
634 * @desc: the interrupt description structure for this irq
635 *
636 * Untracked interrupts are sent from a demultiplexing interrupt handler
637 * when the demultiplexer does not know which device it its multiplexed irq
638 * domain generated the interrupt. IRQ's handled through here are not
639 * subjected to stats tracking, randomness, or spurious interrupt
640 * detection.
641 *
642 * Note: Like handle_simple_irq, the caller is expected to handle the ack,
643 * clear, mask and unmask issues if necessary.
644 */
645void handle_untracked_irq(struct irq_desc *desc)
646{
647 scoped_guard(raw_spinlock, &desc->lock) {
648 if (!irq_can_handle(desc))
649 return;
650
651 desc->istate &= ~IRQS_PENDING;
652 irqd_set(&desc->irq_data, IRQD_IRQ_INPROGRESS);
653 }
654
655 __handle_irq_event_percpu(desc);
656
657 scoped_guard(raw_spinlock, &desc->lock)
658 irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
659}
660EXPORT_SYMBOL_GPL(handle_untracked_irq);
661
662/*
663 * Called unconditionally from handle_level_irq() and only for oneshot
664 * interrupts from handle_fasteoi_irq()
665 */
666static void cond_unmask_irq(struct irq_desc *desc)
667{
668 /*
669 * We need to unmask in the following cases:
670 * - Standard level irq (IRQF_ONESHOT is not set)
671 * - Oneshot irq which did not wake the thread (caused by a
672 * spurious interrupt or a primary handler handling it
673 * completely).
674 */
675 if (!irqd_irq_disabled(&desc->irq_data) &&
676 irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot)
677 unmask_irq(desc);
678}
679
680/**
681 * handle_level_irq - Level type irq handler
682 * @desc: the interrupt description structure for this irq
683 *
684 * Level type interrupts are active as long as the hardware line has the
685 * active level. This may require to mask the interrupt and unmask it after
686 * the associated handler has acknowledged the device, so the interrupt
687 * line is back to inactive.
688 */
689void handle_level_irq(struct irq_desc *desc)
690{
691 guard(raw_spinlock)(&desc->lock);
692 mask_ack_irq(desc);
693
694 if (!irq_can_handle(desc))
695 return;
696
697 kstat_incr_irqs_this_cpu(desc);
698 handle_irq_event(desc);
699
700 cond_unmask_irq(desc);
701}
702EXPORT_SYMBOL_GPL(handle_level_irq);
703
704static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip)
705{
706 if (!(desc->istate & IRQS_ONESHOT)) {
707 chip->irq_eoi(&desc->irq_data);
708 return;
709 }
710 /*
711 * We need to unmask in the following cases:
712 * - Oneshot irq which did not wake the thread (caused by a
713 * spurious interrupt or a primary handler handling it
714 * completely).
715 */
716 if (!irqd_irq_disabled(&desc->irq_data) &&
717 irqd_irq_masked(&desc->irq_data) && !desc->threads_oneshot) {
718 chip->irq_eoi(&desc->irq_data);
719 unmask_irq(desc);
720 } else if (!(chip->flags & IRQCHIP_EOI_THREADED)) {
721 chip->irq_eoi(&desc->irq_data);
722 }
723}
724
725static inline void cond_eoi_irq(struct irq_chip *chip, struct irq_data *data)
726{
727 if (!(chip->flags & IRQCHIP_EOI_IF_HANDLED))
728 chip->irq_eoi(data);
729}
730
731/**
732 * handle_fasteoi_irq - irq handler for transparent controllers
733 * @desc: the interrupt description structure for this irq
734 *
735 * Only a single callback will be issued to the chip: an ->eoi() call when
736 * the interrupt has been serviced. This enables support for modern forms
737 * of interrupt handlers, which handle the flow details in hardware,
738 * transparently.
739 */
740void handle_fasteoi_irq(struct irq_desc *desc)
741{
742 struct irq_chip *chip = desc->irq_data.chip;
743
744 guard(raw_spinlock)(&desc->lock);
745
746 /*
747 * When an affinity change races with IRQ handling, the next interrupt
748 * can arrive on the new CPU before the original CPU has completed
749 * handling the previous one - it may need to be resent.
750 */
751 if (!irq_can_handle_pm(desc)) {
752 if (irqd_needs_resend_when_in_progress(&desc->irq_data))
753 desc->istate |= IRQS_PENDING;
754 cond_eoi_irq(chip, &desc->irq_data);
755 return;
756 }
757
758 if (!irq_can_handle_actions(desc)) {
759 mask_irq(desc);
760 cond_eoi_irq(chip, &desc->irq_data);
761 return;
762 }
763
764 kstat_incr_irqs_this_cpu(desc);
765 if (desc->istate & IRQS_ONESHOT)
766 mask_irq(desc);
767
768 handle_irq_event(desc);
769
770 cond_unmask_eoi_irq(desc, chip);
771
772 /*
773 * When the race described above happens this will resend the interrupt.
774 */
775 if (unlikely(desc->istate & IRQS_PENDING))
776 check_irq_resend(desc, false);
777}
778EXPORT_SYMBOL_GPL(handle_fasteoi_irq);
779
780/**
781 * handle_fasteoi_nmi - irq handler for NMI interrupt lines
782 * @desc: the interrupt description structure for this irq
783 *
784 * A simple NMI-safe handler, considering the restrictions
785 * from request_nmi.
786 *
787 * Only a single callback will be issued to the chip: an ->eoi()
788 * call when the interrupt has been serviced. This enables support
789 * for modern forms of interrupt handlers, which handle the flow
790 * details in hardware, transparently.
791 */
792void handle_fasteoi_nmi(struct irq_desc *desc)
793{
794 struct irq_chip *chip = irq_desc_get_chip(desc);
795 struct irqaction *action = desc->action;
796 unsigned int irq = irq_desc_get_irq(desc);
797 irqreturn_t res;
798
799 __kstat_incr_irqs_this_cpu(desc);
800
801 trace_irq_handler_entry(irq, action);
802 /*
803 * NMIs cannot be shared, there is only one action.
804 */
805 res = action->handler(irq, action->dev_id);
806 trace_irq_handler_exit(irq, action, res);
807
808 if (chip->irq_eoi)
809 chip->irq_eoi(&desc->irq_data);
810}
811EXPORT_SYMBOL_GPL(handle_fasteoi_nmi);
812
813/**
814 * handle_edge_irq - edge type IRQ handler
815 * @desc: the interrupt description structure for this irq
816 *
817 * Interrupt occurs on the falling and/or rising edge of a hardware
818 * signal. The occurrence is latched into the irq controller hardware and
819 * must be acked in order to be reenabled. After the ack another interrupt
820 * can happen on the same source even before the first one is handled by
821 * the associated event handler. If this happens it might be necessary to
822 * disable (mask) the interrupt depending on the controller hardware. This
823 * requires to reenable the interrupt inside of the loop which handles the
824 * interrupts which have arrived while the handler was running. If all
825 * pending interrupts are handled, the loop is left.
826 */
827void handle_edge_irq(struct irq_desc *desc)
828{
829 guard(raw_spinlock)(&desc->lock);
830
831 if (!irq_can_handle(desc)) {
832 desc->istate |= IRQS_PENDING;
833 mask_ack_irq(desc);
834 return;
835 }
836
837 kstat_incr_irqs_this_cpu(desc);
838
839 /* Start handling the irq */
840 desc->irq_data.chip->irq_ack(&desc->irq_data);
841
842 do {
843 if (unlikely(!desc->action)) {
844 mask_irq(desc);
845 return;
846 }
847
848 /*
849 * When another irq arrived while we were handling
850 * one, we could have masked the irq.
851 * Reenable it, if it was not disabled in meantime.
852 */
853 if (unlikely(desc->istate & IRQS_PENDING)) {
854 if (!irqd_irq_disabled(&desc->irq_data) &&
855 irqd_irq_masked(&desc->irq_data))
856 unmask_irq(desc);
857 }
858
859 handle_irq_event(desc);
860
861 } while ((desc->istate & IRQS_PENDING) && !irqd_irq_disabled(&desc->irq_data));
862}
863EXPORT_SYMBOL(handle_edge_irq);
864
865/**
866 * handle_percpu_irq - Per CPU local irq handler
867 * @desc: the interrupt description structure for this irq
868 *
869 * Per CPU interrupts on SMP machines without locking requirements
870 */
871void handle_percpu_irq(struct irq_desc *desc)
872{
873 struct irq_chip *chip = irq_desc_get_chip(desc);
874
875 /*
876 * PER CPU interrupts are not serialized. Do not touch
877 * desc->tot_count.
878 */
879 __kstat_incr_irqs_this_cpu(desc);
880
881 if (chip->irq_ack)
882 chip->irq_ack(&desc->irq_data);
883
884 handle_irq_event_percpu(desc);
885
886 if (chip->irq_eoi)
887 chip->irq_eoi(&desc->irq_data);
888}
889
890/**
891 * handle_percpu_devid_irq - Per CPU local irq handler with per cpu dev ids
892 * @desc: the interrupt description structure for this irq
893 *
894 * Per CPU interrupts on SMP machines without locking requirements. Same as
895 * handle_percpu_irq() above but with the following extras:
896 *
897 * action->percpu_dev_id is a pointer to percpu variables which
898 * contain the real device id for the cpu on which this handler is
899 * called.
900 *
901 * May be used for NMI interrupt lines, and so may be called in IRQ or NMI
902 * context.
903 */
904void handle_percpu_devid_irq(struct irq_desc *desc)
905{
906 struct irq_chip *chip = irq_desc_get_chip(desc);
907 unsigned int irq = irq_desc_get_irq(desc);
908 unsigned int cpu = smp_processor_id();
909 struct irqaction *action;
910 irqreturn_t res;
911
912 /*
913 * PER CPU interrupts are not serialized. Do not touch
914 * desc->tot_count.
915 */
916 __kstat_incr_irqs_this_cpu(desc);
917
918 if (chip->irq_ack)
919 chip->irq_ack(&desc->irq_data);
920
921 for (action = desc->action; action; action = action->next)
922 if (cpumask_test_cpu(cpu, action->affinity))
923 break;
924
925 if (likely(action)) {
926 trace_irq_handler_entry(irq, action);
927 res = action->handler(irq, raw_cpu_ptr(action->percpu_dev_id));
928 trace_irq_handler_exit(irq, action, res);
929 } else {
930 bool enabled = cpumask_test_cpu(cpu, desc->percpu_enabled);
931
932 if (enabled)
933 irq_percpu_disable(desc, cpu);
934
935 pr_err_once("Spurious%s percpu IRQ%u on CPU%u\n",
936 enabled ? " and unmasked" : "", irq, cpu);
937 }
938
939 if (!in_nmi())
940 add_interrupt_randomness(irq);
941
942 if (chip->irq_eoi)
943 chip->irq_eoi(&desc->irq_data);
944}
945
946static void
947__irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
948 int is_chained, const char *name)
949{
950 if (!handle) {
951 handle = handle_bad_irq;
952 } else {
953 struct irq_data *irq_data = &desc->irq_data;
954#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
955 /*
956 * With hierarchical domains we might run into a
957 * situation where the outermost chip is not yet set
958 * up, but the inner chips are there. Instead of
959 * bailing we install the handler, but obviously we
960 * cannot enable/startup the interrupt at this point.
961 */
962 while (irq_data) {
963 if (irq_data->chip != &no_irq_chip)
964 break;
965 /*
966 * Bail out if the outer chip is not set up
967 * and the interrupt supposed to be started
968 * right away.
969 */
970 if (WARN_ON(is_chained))
971 return;
972 /* Try the parent */
973 irq_data = irq_data->parent_data;
974 }
975#endif
976 if (WARN_ON(!irq_data || irq_data->chip == &no_irq_chip))
977 return;
978 }
979
980 /* Uninstall? */
981 if (handle == handle_bad_irq) {
982 if (desc->irq_data.chip != &no_irq_chip)
983 mask_ack_irq(desc);
984 irq_state_set_disabled(desc);
985 if (is_chained) {
986 desc->action = NULL;
987 irq_chip_pm_put(irq_desc_get_irq_data(desc));
988 }
989 desc->depth = 1;
990 }
991 desc->handle_irq = handle;
992 desc->name = name;
993
994 if (handle != handle_bad_irq && is_chained) {
995 unsigned int type = irqd_get_trigger_type(&desc->irq_data);
996
997 /*
998 * We're about to start this interrupt immediately,
999 * hence the need to set the trigger configuration.
1000 * But the .set_type callback may have overridden the
1001 * flow handler, ignoring that we're dealing with a
1002 * chained interrupt. Reset it immediately because we
1003 * do know better.
1004 */
1005 if (type != IRQ_TYPE_NONE) {
1006 __irq_set_trigger(desc, type);
1007 desc->handle_irq = handle;
1008 }
1009
1010 irq_settings_set_noprobe(desc);
1011 irq_settings_set_norequest(desc);
1012 irq_settings_set_nothread(desc);
1013 desc->action = &chained_action;
1014 WARN_ON(irq_chip_pm_get(irq_desc_get_irq_data(desc)));
1015 irq_activate_and_startup(desc, IRQ_RESEND);
1016 }
1017 irq_proc_update_valid(desc);
1018}
1019
1020void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
1021 const char *name)
1022{
1023 scoped_irqdesc_get_and_buslock(irq, 0)
1024 __irq_do_set_handler(scoped_irqdesc, handle, is_chained, name);
1025}
1026EXPORT_SYMBOL_GPL(__irq_set_handler);
1027
1028void irq_set_chained_handler_and_data(unsigned int irq, irq_flow_handler_t handle,
1029 void *data)
1030{
1031 scoped_irqdesc_get_and_buslock(irq, 0) {
1032 struct irq_desc *desc = scoped_irqdesc;
1033
1034 desc->irq_common_data.handler_data = data;
1035 __irq_do_set_handler(desc, handle, 1, NULL);
1036 }
1037}
1038EXPORT_SYMBOL_GPL(irq_set_chained_handler_and_data);
1039
1040void
1041irq_set_chip_and_handler_name(unsigned int irq, const struct irq_chip *chip,
1042 irq_flow_handler_t handle, const char *name)
1043{
1044 irq_set_chip(irq, chip);
1045 __irq_set_handler(irq, handle, 0, name);
1046}
1047EXPORT_SYMBOL_GPL(irq_set_chip_and_handler_name);
1048
1049void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
1050{
1051 scoped_irqdesc_get_and_lock(irq, 0) {
1052 struct irq_desc *desc = scoped_irqdesc;
1053 unsigned long trigger, tmp;
1054 /*
1055 * Warn when a driver sets the no autoenable flag on an already
1056 * active interrupt.
1057 */
1058 WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1059
1060 irq_settings_clr_and_set(desc, clr, set);
1061
1062 trigger = irqd_get_trigger_type(&desc->irq_data);
1063
1064 irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
1065 IRQD_TRIGGER_MASK | IRQD_LEVEL);
1066 if (irq_settings_has_no_balance_set(desc))
1067 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1068 if (irq_settings_is_per_cpu(desc))
1069 irqd_set(&desc->irq_data, IRQD_PER_CPU);
1070 if (irq_settings_is_level(desc))
1071 irqd_set(&desc->irq_data, IRQD_LEVEL);
1072
1073 tmp = irq_settings_get_trigger_mask(desc);
1074 if (tmp != IRQ_TYPE_NONE)
1075 trigger = tmp;
1076
1077 irqd_set(&desc->irq_data, trigger);
1078 irq_proc_update_valid(desc);
1079 }
1080}
1081EXPORT_SYMBOL_GPL(irq_modify_status);
1082
1083#ifdef CONFIG_DEPRECATED_IRQ_CPU_ONOFFLINE
1084/**
1085 * irq_cpu_online - Invoke all irq_cpu_online functions.
1086 *
1087 * Iterate through all irqs and invoke the chip.irq_cpu_online()
1088 * for each.
1089 */
1090void irq_cpu_online(void)
1091{
1092 unsigned int irq;
1093
1094 for_each_active_irq(irq) {
1095 struct irq_desc *desc = irq_to_desc(irq);
1096 struct irq_chip *chip;
1097
1098 if (!desc)
1099 continue;
1100
1101 guard(raw_spinlock_irqsave)(&desc->lock);
1102 chip = irq_data_get_irq_chip(&desc->irq_data);
1103 if (chip && chip->irq_cpu_online &&
1104 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1105 !irqd_irq_disabled(&desc->irq_data)))
1106 chip->irq_cpu_online(&desc->irq_data);
1107 }
1108}
1109
1110/**
1111 * irq_cpu_offline - Invoke all irq_cpu_offline functions.
1112 *
1113 * Iterate through all irqs and invoke the chip.irq_cpu_offline()
1114 * for each.
1115 */
1116void irq_cpu_offline(void)
1117{
1118 unsigned int irq;
1119
1120 for_each_active_irq(irq) {
1121 struct irq_desc *desc = irq_to_desc(irq);
1122 struct irq_chip *chip;
1123
1124 if (!desc)
1125 continue;
1126
1127 guard(raw_spinlock_irqsave)(&desc->lock);
1128 chip = irq_data_get_irq_chip(&desc->irq_data);
1129 if (chip && chip->irq_cpu_offline &&
1130 (!(chip->flags & IRQCHIP_ONOFFLINE_ENABLED) ||
1131 !irqd_irq_disabled(&desc->irq_data)))
1132 chip->irq_cpu_offline(&desc->irq_data);
1133 }
1134}
1135#endif
1136
1137#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1138
1139#ifdef CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS
1140/**
1141 * handle_fasteoi_ack_irq - irq handler for edge hierarchy stacked on
1142 * transparent controllers
1143 *
1144 * @desc: the interrupt description structure for this irq
1145 *
1146 * Like handle_fasteoi_irq(), but for use with hierarchy where the irq_chip
1147 * also needs to have its ->irq_ack() function called.
1148 */
1149void handle_fasteoi_ack_irq(struct irq_desc *desc)
1150{
1151 struct irq_chip *chip = desc->irq_data.chip;
1152
1153 guard(raw_spinlock)(&desc->lock);
1154
1155 if (!irq_can_handle_pm(desc)) {
1156 cond_eoi_irq(chip, &desc->irq_data);
1157 return;
1158 }
1159
1160 if (unlikely(!irq_can_handle_actions(desc))) {
1161 mask_irq(desc);
1162 cond_eoi_irq(chip, &desc->irq_data);
1163 return;
1164 }
1165
1166 kstat_incr_irqs_this_cpu(desc);
1167 if (desc->istate & IRQS_ONESHOT)
1168 mask_irq(desc);
1169
1170 desc->irq_data.chip->irq_ack(&desc->irq_data);
1171
1172 handle_irq_event(desc);
1173
1174 cond_unmask_eoi_irq(desc, chip);
1175}
1176EXPORT_SYMBOL_GPL(handle_fasteoi_ack_irq);
1177
1178/**
1179 * handle_fasteoi_mask_irq - irq handler for level hierarchy stacked on
1180 * transparent controllers
1181 *
1182 * @desc: the interrupt description structure for this irq
1183 *
1184 * Like handle_fasteoi_irq(), but for use with hierarchy where the irq_chip
1185 * also needs to have its ->irq_mask_ack() function called.
1186 */
1187void handle_fasteoi_mask_irq(struct irq_desc *desc)
1188{
1189 struct irq_chip *chip = desc->irq_data.chip;
1190
1191 guard(raw_spinlock)(&desc->lock);
1192 mask_ack_irq(desc);
1193
1194 if (!irq_can_handle(desc)) {
1195 cond_eoi_irq(chip, &desc->irq_data);
1196 return;
1197 }
1198
1199 kstat_incr_irqs_this_cpu(desc);
1200
1201 handle_irq_event(desc);
1202
1203 cond_unmask_eoi_irq(desc, chip);
1204}
1205EXPORT_SYMBOL_GPL(handle_fasteoi_mask_irq);
1206
1207#endif /* CONFIG_IRQ_FASTEOI_HIERARCHY_HANDLERS */
1208
1209#ifdef CONFIG_SMP
1210void irq_chip_pre_redirect_parent(struct irq_data *data)
1211{
1212 data = data->parent_data;
1213 data->chip->irq_pre_redirect(data);
1214}
1215EXPORT_SYMBOL_GPL(irq_chip_pre_redirect_parent);
1216#endif
1217
1218/**
1219 * irq_chip_set_parent_state - set the state of a parent interrupt.
1220 *
1221 * @data: Pointer to interrupt specific data
1222 * @which: State to be restored (one of IRQCHIP_STATE_*)
1223 * @val: Value corresponding to @which
1224 *
1225 * Conditional success, if the underlying irqchip does not implement it.
1226 */
1227int irq_chip_set_parent_state(struct irq_data *data,
1228 enum irqchip_irq_state which,
1229 bool val)
1230{
1231 data = data->parent_data;
1232
1233 if (!data || !data->chip->irq_set_irqchip_state)
1234 return 0;
1235
1236 return data->chip->irq_set_irqchip_state(data, which, val);
1237}
1238EXPORT_SYMBOL_GPL(irq_chip_set_parent_state);
1239
1240/**
1241 * irq_chip_get_parent_state - get the state of a parent interrupt.
1242 *
1243 * @data: Pointer to interrupt specific data
1244 * @which: one of IRQCHIP_STATE_* the caller wants to know
1245 * @state: a pointer to a boolean where the state is to be stored
1246 *
1247 * Conditional success, if the underlying irqchip does not implement it.
1248 */
1249int irq_chip_get_parent_state(struct irq_data *data,
1250 enum irqchip_irq_state which,
1251 bool *state)
1252{
1253 data = data->parent_data;
1254
1255 if (!data || !data->chip->irq_get_irqchip_state)
1256 return 0;
1257
1258 return data->chip->irq_get_irqchip_state(data, which, state);
1259}
1260EXPORT_SYMBOL_GPL(irq_chip_get_parent_state);
1261
1262/**
1263 * irq_chip_shutdown_parent - Shutdown the parent interrupt
1264 * @data: Pointer to interrupt specific data
1265 *
1266 * Invokes the irq_shutdown() callback of the parent if available or falls
1267 * back to irq_chip_disable_parent().
1268 */
1269void irq_chip_shutdown_parent(struct irq_data *data)
1270{
1271 struct irq_data *parent = data->parent_data;
1272
1273 if (parent->chip->irq_shutdown)
1274 parent->chip->irq_shutdown(parent);
1275 else
1276 irq_chip_disable_parent(data);
1277}
1278EXPORT_SYMBOL_GPL(irq_chip_shutdown_parent);
1279
1280/**
1281 * irq_chip_startup_parent - Startup the parent interrupt
1282 * @data: Pointer to interrupt specific data
1283 *
1284 * Invokes the irq_startup() callback of the parent if available or falls
1285 * back to irq_chip_enable_parent().
1286 */
1287unsigned int irq_chip_startup_parent(struct irq_data *data)
1288{
1289 struct irq_data *parent = data->parent_data;
1290
1291 if (parent->chip->irq_startup)
1292 return parent->chip->irq_startup(parent);
1293
1294 irq_chip_enable_parent(data);
1295 return 0;
1296}
1297EXPORT_SYMBOL_GPL(irq_chip_startup_parent);
1298
1299/**
1300 * irq_chip_enable_parent - Enable the parent interrupt (defaults to unmask if
1301 * NULL)
1302 * @data: Pointer to interrupt specific data
1303 */
1304void irq_chip_enable_parent(struct irq_data *data)
1305{
1306 data = data->parent_data;
1307 if (data->chip->irq_enable)
1308 data->chip->irq_enable(data);
1309 else
1310 data->chip->irq_unmask(data);
1311}
1312EXPORT_SYMBOL_GPL(irq_chip_enable_parent);
1313
1314/**
1315 * irq_chip_disable_parent - Disable the parent interrupt (defaults to mask if
1316 * NULL)
1317 * @data: Pointer to interrupt specific data
1318 */
1319void irq_chip_disable_parent(struct irq_data *data)
1320{
1321 data = data->parent_data;
1322 if (data->chip->irq_disable)
1323 data->chip->irq_disable(data);
1324 else
1325 data->chip->irq_mask(data);
1326}
1327EXPORT_SYMBOL_GPL(irq_chip_disable_parent);
1328
1329/**
1330 * irq_chip_ack_parent - Acknowledge the parent interrupt
1331 * @data: Pointer to interrupt specific data
1332 */
1333void irq_chip_ack_parent(struct irq_data *data)
1334{
1335 data = data->parent_data;
1336 data->chip->irq_ack(data);
1337}
1338EXPORT_SYMBOL_GPL(irq_chip_ack_parent);
1339
1340/**
1341 * irq_chip_mask_parent - Mask the parent interrupt
1342 * @data: Pointer to interrupt specific data
1343 */
1344void irq_chip_mask_parent(struct irq_data *data)
1345{
1346 data = data->parent_data;
1347 data->chip->irq_mask(data);
1348}
1349EXPORT_SYMBOL_GPL(irq_chip_mask_parent);
1350
1351/**
1352 * irq_chip_mask_ack_parent - Mask and acknowledge the parent interrupt
1353 * @data: Pointer to interrupt specific data
1354 */
1355void irq_chip_mask_ack_parent(struct irq_data *data)
1356{
1357 data = data->parent_data;
1358 data->chip->irq_mask_ack(data);
1359}
1360EXPORT_SYMBOL_GPL(irq_chip_mask_ack_parent);
1361
1362/**
1363 * irq_chip_unmask_parent - Unmask the parent interrupt
1364 * @data: Pointer to interrupt specific data
1365 */
1366void irq_chip_unmask_parent(struct irq_data *data)
1367{
1368 data = data->parent_data;
1369 data->chip->irq_unmask(data);
1370}
1371EXPORT_SYMBOL_GPL(irq_chip_unmask_parent);
1372
1373/**
1374 * irq_chip_eoi_parent - Invoke EOI on the parent interrupt
1375 * @data: Pointer to interrupt specific data
1376 */
1377void irq_chip_eoi_parent(struct irq_data *data)
1378{
1379 data = data->parent_data;
1380 data->chip->irq_eoi(data);
1381}
1382EXPORT_SYMBOL_GPL(irq_chip_eoi_parent);
1383
1384/**
1385 * irq_chip_set_affinity_parent - Set affinity on the parent interrupt
1386 * @data: Pointer to interrupt specific data
1387 * @dest: The affinity mask to set
1388 * @force: Flag to enforce setting (disable online checks)
1389 *
1390 * Conditional, as the underlying parent chip might not implement it.
1391 */
1392int irq_chip_set_affinity_parent(struct irq_data *data,
1393 const struct cpumask *dest, bool force)
1394{
1395 data = data->parent_data;
1396 if (data->chip->irq_set_affinity)
1397 return data->chip->irq_set_affinity(data, dest, force);
1398
1399 return -ENOSYS;
1400}
1401EXPORT_SYMBOL_GPL(irq_chip_set_affinity_parent);
1402
1403/**
1404 * irq_chip_set_type_parent - Set IRQ type on the parent interrupt
1405 * @data: Pointer to interrupt specific data
1406 * @type: IRQ_TYPE_{LEVEL,EDGE}_* value - see include/linux/irq.h
1407 *
1408 * Conditional, as the underlying parent chip might not implement it.
1409 */
1410int irq_chip_set_type_parent(struct irq_data *data, unsigned int type)
1411{
1412 data = data->parent_data;
1413
1414 if (data->chip->irq_set_type)
1415 return data->chip->irq_set_type(data, type);
1416
1417 return -ENOSYS;
1418}
1419EXPORT_SYMBOL_GPL(irq_chip_set_type_parent);
1420
1421/**
1422 * irq_chip_retrigger_hierarchy - Retrigger an interrupt in hardware
1423 * @data: Pointer to interrupt specific data
1424 *
1425 * Iterate through the domain hierarchy of the interrupt and check
1426 * whether a hw retrigger function exists. If yes, invoke it.
1427 */
1428int irq_chip_retrigger_hierarchy(struct irq_data *data)
1429{
1430 for (data = data->parent_data; data; data = data->parent_data)
1431 if (data->chip && data->chip->irq_retrigger)
1432 return data->chip->irq_retrigger(data);
1433
1434 return 0;
1435}
1436EXPORT_SYMBOL_GPL(irq_chip_retrigger_hierarchy);
1437
1438/**
1439 * irq_chip_set_vcpu_affinity_parent - Set vcpu affinity on the parent interrupt
1440 * @data: Pointer to interrupt specific data
1441 * @vcpu_info: The vcpu affinity information
1442 */
1443int irq_chip_set_vcpu_affinity_parent(struct irq_data *data, void *vcpu_info)
1444{
1445 data = data->parent_data;
1446 if (data->chip->irq_set_vcpu_affinity)
1447 return data->chip->irq_set_vcpu_affinity(data, vcpu_info);
1448
1449 return -ENOSYS;
1450}
1451EXPORT_SYMBOL_GPL(irq_chip_set_vcpu_affinity_parent);
1452/**
1453 * irq_chip_set_wake_parent - Set/reset wake-up on the parent interrupt
1454 * @data: Pointer to interrupt specific data
1455 * @on: Whether to set or reset the wake-up capability of this irq
1456 *
1457 * Conditional, as the underlying parent chip might not implement it.
1458 */
1459int irq_chip_set_wake_parent(struct irq_data *data, unsigned int on)
1460{
1461 data = data->parent_data;
1462
1463 if (data->chip->flags & IRQCHIP_SKIP_SET_WAKE)
1464 return 0;
1465
1466 if (data->chip->irq_set_wake)
1467 return data->chip->irq_set_wake(data, on);
1468
1469 return -ENOSYS;
1470}
1471EXPORT_SYMBOL_GPL(irq_chip_set_wake_parent);
1472
1473/**
1474 * irq_chip_request_resources_parent - Request resources on the parent interrupt
1475 * @data: Pointer to interrupt specific data
1476 */
1477int irq_chip_request_resources_parent(struct irq_data *data)
1478{
1479 data = data->parent_data;
1480
1481 if (data->chip->irq_request_resources)
1482 return data->chip->irq_request_resources(data);
1483
1484 /* no error on missing optional irq_chip::irq_request_resources */
1485 return 0;
1486}
1487EXPORT_SYMBOL_GPL(irq_chip_request_resources_parent);
1488
1489/**
1490 * irq_chip_release_resources_parent - Release resources on the parent interrupt
1491 * @data: Pointer to interrupt specific data
1492 */
1493void irq_chip_release_resources_parent(struct irq_data *data)
1494{
1495 data = data->parent_data;
1496 if (data->chip->irq_release_resources)
1497 data->chip->irq_release_resources(data);
1498}
1499EXPORT_SYMBOL_GPL(irq_chip_release_resources_parent);
1500#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
1501
1502#ifdef CONFIG_SMP
1503int irq_chip_redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
1504{
1505 struct irq_redirect *redir = &irq_data_to_desc(data)->redirect;
1506
1507 WRITE_ONCE(redir->target_cpu, cpumask_first(dest));
1508 irq_data_update_effective_affinity(data, dest);
1509
1510 return IRQ_SET_MASK_OK_DONE;
1511}
1512EXPORT_SYMBOL_GPL(irq_chip_redirect_set_affinity);
1513#endif
1514
1515/**
1516 * irq_chip_compose_msi_msg - Compose msi message for a irq chip
1517 * @data: Pointer to interrupt specific data
1518 * @msg: Pointer to the MSI message
1519 *
1520 * For hierarchical domains we find the first chip in the hierarchy
1521 * which implements the irq_compose_msi_msg callback. For non
1522 * hierarchical we use the top level chip.
1523 */
1524int irq_chip_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1525{
1526 struct irq_data *pos;
1527
1528 for (pos = NULL; !pos && data; data = irqd_get_parent_data(data)) {
1529 if (data->chip && data->chip->irq_compose_msi_msg)
1530 pos = data;
1531 }
1532
1533 if (!pos)
1534 return -ENOSYS;
1535
1536 pos->chip->irq_compose_msi_msg(pos, msg);
1537 return 0;
1538}
1539
1540static struct device *irq_get_pm_device(struct irq_data *data)
1541{
1542 if (data->domain)
1543 return data->domain->pm_dev;
1544
1545 return NULL;
1546}
1547
1548/**
1549 * irq_chip_pm_get - Enable power for an IRQ chip
1550 * @data: Pointer to interrupt specific data
1551 *
1552 * Enable the power to the IRQ chip referenced by the interrupt data
1553 * structure.
1554 */
1555int irq_chip_pm_get(struct irq_data *data)
1556{
1557 struct device *dev = irq_get_pm_device(data);
1558 int retval = 0;
1559
1560 if (IS_ENABLED(CONFIG_PM) && dev)
1561 retval = pm_runtime_resume_and_get(dev);
1562
1563 return retval;
1564}
1565
1566/**
1567 * irq_chip_pm_put - Drop a PM reference on an IRQ chip
1568 * @data: Pointer to interrupt specific data
1569 *
1570 * Drop a power management reference, acquired via irq_chip_pm_get(), on the IRQ
1571 * chip represented by the interrupt data structure.
1572 *
1573 * Note that this will not disable power to the IRQ chip until this function
1574 * has been called for all IRQs that have called irq_chip_pm_get() and it may
1575 * not disable power at all (if user space prevents that, for example).
1576 */
1577void irq_chip_pm_put(struct irq_data *data)
1578{
1579 struct device *dev = irq_get_pm_device(data);
1580
1581 if (dev)
1582 pm_runtime_put(dev);
1583}