projects/linux·blob master · refs

397 lines · 11.7 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * IRQ subsystem internal functions and variables:
4 *
5 * Do not ever include this file from anything else than
6 * kernel/irq/. Do not even think about using any information outside
7 * of this file for your non core code.
8 */
9#include <linux/irqdesc.h>
10#include <linux/kernel_stat.h>
11#include <linux/pm_runtime.h>
12#include <linux/rcuref.h>
13#include <linux/sched/clock.h>
14
15#include "debugfs.h"
16#include "proc.h"
17
18#ifdef CONFIG_SPARSE_IRQ
19# define MAX_SPARSE_IRQS INT_MAX
20#else
21# define MAX_SPARSE_IRQS NR_IRQS
22#endif
23
24#define istate core_internal_state__do_not_mess_with_it
25
26extern bool noirqdebug;
27extern int irq_poll_cpu;
28extern unsigned int total_nr_irqs;
29
30extern struct irqaction chained_action;
31
32/*
33 * Bits used by threaded handlers:
34 * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run
35 * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
36 * IRQTF_AFFINITY - irq thread is requested to adjust affinity
37 * IRQTF_FORCED_THREAD - irq action is force threaded
38 * IRQTF_READY - signals that irq thread is ready
39 */
40enum {
41 IRQTF_RUNTHREAD,
42 IRQTF_WARNED,
43 IRQTF_AFFINITY,
44 IRQTF_FORCED_THREAD,
45 IRQTF_READY,
46};
47
48/*
49 * Bit masks for desc->core_internal_state__do_not_mess_with_it
50 *
51 * IRQS_AUTODETECT - autodetection in progress
52 * IRQS_SPURIOUS_DISABLED - was disabled due to spurious interrupt
53 * detection
54 * IRQS_POLL_INPROGRESS - polling in progress
55 * IRQS_ONESHOT - irq is not unmasked in primary handler
56 * IRQS_REPLAY - irq has been resent and will not be resent
57 * again until the handler has run and cleared
58 * this flag.
59 * IRQS_WAITING - irq is waiting
60 * IRQS_PENDING - irq needs to be resent and should be resent
61 * at the next available opportunity.
62 * IRQS_SUSPENDED - irq is suspended
63 * IRQS_NMI - irq line is used to deliver NMIs
64 * IRQS_SYSFS - descriptor has been added to sysfs
65 */
66enum {
67 IRQS_AUTODETECT = 0x00000001,
68 IRQS_SPURIOUS_DISABLED = 0x00000002,
69 IRQS_POLL_INPROGRESS = 0x00000008,
70 IRQS_ONESHOT = 0x00000020,
71 IRQS_REPLAY = 0x00000040,
72 IRQS_WAITING = 0x00000080,
73 IRQS_PENDING = 0x00000200,
74 IRQS_SUSPENDED = 0x00000800,
75 IRQS_TIMINGS = 0x00001000,
76 IRQS_NMI = 0x00002000,
77 IRQS_SYSFS = 0x00004000,
78};
79
80#include "debug.h"
81#include "settings.h"
82
83extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);
84extern void __disable_irq(struct irq_desc *desc);
85extern void __enable_irq(struct irq_desc *desc);
86
87#define IRQ_RESEND true
88#define IRQ_NORESEND false
89
90#define IRQ_START_FORCE true
91#define IRQ_START_COND false
92
93extern int irq_activate(struct irq_desc *desc);
94extern int irq_activate_and_startup(struct irq_desc *desc, bool resend);
95extern int irq_startup(struct irq_desc *desc, bool resend, bool force);
96extern void irq_startup_managed(struct irq_desc *desc);
97
98extern void irq_shutdown(struct irq_desc *desc);
99extern void irq_shutdown_and_deactivate(struct irq_desc *desc);
100extern void irq_disable(struct irq_desc *desc);
101extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu);
102extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu);
103extern void mask_irq(struct irq_desc *desc);
104extern void unmask_irq(struct irq_desc *desc);
105extern void unmask_threaded_irq(struct irq_desc *desc);
106
107#ifdef CONFIG_SPARSE_IRQ
108static __always_inline void irq_mark_irq(unsigned int irq) { }
109void irq_desc_free_rcu(struct irq_desc *desc);
110
111static __always_inline bool irq_desc_get_ref(struct irq_desc *desc)
112{
113 return rcuref_get(&desc->refcnt);
114}
115
116static __always_inline void irq_desc_put_ref(struct irq_desc *desc)
117{
118 if (rcuref_put(&desc->refcnt))
119 irq_desc_free_rcu(desc);
120}
121#else
122extern void irq_mark_irq(unsigned int irq);
123static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; }
124static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { }
125#endif
126
127irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc);
128irqreturn_t handle_irq_event_percpu(struct irq_desc *desc);
129irqreturn_t handle_irq_event(struct irq_desc *desc);
130
131/* Resending of interrupts :*/
132int check_irq_resend(struct irq_desc *desc, bool inject);
133void clear_irq_resend(struct irq_desc *desc);
134void irq_resend_init(struct irq_desc *desc);
135void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action);
136
137void wake_threads_waitq(struct irq_desc *desc);
138
139#ifdef CONFIG_PROC_FS
140extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
141extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc);
142extern void register_handler_proc(unsigned int irq, struct irqaction *action);
143extern void unregister_handler_proc(unsigned int irq, struct irqaction *action);
144void irq_proc_update_valid(struct irq_desc *desc);
145#else
146static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { }
147static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { }
148static inline void register_handler_proc(unsigned int irq,
149 struct irqaction *action) { }
150static inline void unregister_handler_proc(unsigned int irq,
151 struct irqaction *action) { }
152static inline void irq_proc_update_valid(struct irq_desc *desc) { }
153#endif
154
155struct irq_desc *irq_find_desc_at_or_after(unsigned int offset);
156
157extern bool irq_can_set_affinity_usr(unsigned int irq);
158
159extern int irq_do_set_affinity(struct irq_data *data,
160 const struct cpumask *dest, bool force);
161extern void irq_affinity_schedule_notify_work(struct irq_desc *desc);
162
163#ifdef CONFIG_SMP
164extern int irq_setup_affinity(struct irq_desc *desc);
165#else
166static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }
167#endif
168
169#define for_each_action_of_desc(desc, act) \
170 for (act = desc->action; act; act = act->next)
171
172/* Inline functions for support of irq chips on slow busses */
173static inline void chip_bus_lock(struct irq_desc *desc)
174{
175 if (unlikely(desc->irq_data.chip->irq_bus_lock))
176 desc->irq_data.chip->irq_bus_lock(&desc->irq_data);
177}
178
179static inline void chip_bus_sync_unlock(struct irq_desc *desc)
180{
181 if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock))
182 desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data);
183}
184
185#define _IRQ_DESC_CHECK (1 << 0)
186#define _IRQ_DESC_PERCPU (1 << 1)
187
188#define IRQ_GET_DESC_CHECK_GLOBAL (_IRQ_DESC_CHECK)
189#define IRQ_GET_DESC_CHECK_PERCPU (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU)
190
191struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
192 unsigned int check);
193void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus);
194
195__DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true);
196__DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc,
197 if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus),
198 unsigned long flags; bool bus);
199
200static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus,
201 unsigned int check)
202{
203 class_irqdesc_lock_t _t = { .bus = bus, };
204
205 _t.lock = __irq_get_desc_lock(irq, &_t.flags, bus, check);
206
207 return _t;
208}
209
210#define scoped_irqdesc_get_and_lock(_irq, _check) \
211 scoped_guard(irqdesc_lock, _irq, false, _check)
212
213#define scoped_irqdesc_get_and_buslock(_irq, _check) \
214 scoped_guard(irqdesc_lock, _irq, true, _check)
215
216#define scoped_irqdesc ((struct irq_desc *)(__guard_ptr(irqdesc_lock)(&scope)))
217
218#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
219
220static inline unsigned int irqd_get(struct irq_data *d)
221{
222 return __irqd_to_state(d);
223}
224
225/*
226 * Manipulation functions for irq_data.state
227 */
228static inline void irqd_set_move_pending(struct irq_data *d)
229{
230 __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING;
231}
232
233static inline void irqd_clr_move_pending(struct irq_data *d)
234{
235 __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;
236}
237
238static inline void irqd_set_managed_shutdown(struct irq_data *d)
239{
240 __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;
241}
242
243static inline void irqd_clr_managed_shutdown(struct irq_data *d)
244{
245 __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;
246}
247
248static inline void irqd_clear(struct irq_data *d, unsigned int mask)
249{
250 __irqd_to_state(d) &= ~mask;
251}
252
253static inline void irqd_set(struct irq_data *d, unsigned int mask)
254{
255 __irqd_to_state(d) |= mask;
256}
257
258static inline bool irqd_has_set(struct irq_data *d, unsigned int mask)
259{
260 return __irqd_to_state(d) & mask;
261}
262
263static inline void irq_state_set_disabled(struct irq_desc *desc)
264{
265 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
266}
267
268static inline void irq_state_set_masked(struct irq_desc *desc)
269{
270 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
271}
272
273#undef __irqd_to_state
274
275static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc)
276{
277 __this_cpu_inc(desc->kstat_irqs->cnt);
278 __this_cpu_inc(kstat.irqs_sum);
279}
280
281static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc)
282{
283 __kstat_incr_irqs_this_cpu(desc);
284 desc->tot_count++;
285}
286
287static inline int irq_desc_get_node(struct irq_desc *desc)
288{
289 return irq_common_data_get_node(&desc->irq_common_data);
290}
291
292static inline int irq_desc_is_chained(struct irq_desc *desc)
293{
294 return (desc->action && desc->action == &chained_action);
295}
296
297static inline bool irq_is_nmi(struct irq_desc *desc)
298{
299 return desc->istate & IRQS_NMI;
300}
301
302#ifdef CONFIG_PM_SLEEP
303void irq_pm_handle_wakeup(struct irq_desc *desc);
304void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action);
305void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action);
306#else
307static inline void irq_pm_handle_wakeup(struct irq_desc *desc) { }
308static inline void
309irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }
310static inline void
311irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
312#endif
313
314#ifdef CONFIG_GENERIC_IRQ_CHIP
315void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
316 int num_ct, unsigned int irq_base,
317 void __iomem *reg_base, irq_flow_handler_t handler);
318#else
319static inline void
320irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
321 int num_ct, unsigned int irq_base,
322 void __iomem *reg_base, irq_flow_handler_t handler) { }
323#endif /* CONFIG_GENERIC_IRQ_CHIP */
324
325#ifdef CONFIG_GENERIC_PENDING_IRQ
326static inline bool irq_can_move_pcntxt(struct irq_data *data)
327{
328 return !(data->chip->flags & IRQCHIP_MOVE_DEFERRED);
329}
330static inline bool irq_move_pending(struct irq_data *data)
331{
332 return irqd_is_setaffinity_pending(data);
333}
334static inline void
335irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
336{
337 cpumask_copy(desc->pending_mask, mask);
338}
339static inline void
340irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
341{
342 cpumask_copy(mask, desc->pending_mask);
343}
344static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
345{
346 return desc->pending_mask;
347}
348bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);
349void irq_force_complete_move(struct irq_desc *desc);
350#else /* CONFIG_GENERIC_PENDING_IRQ */
351static inline bool irq_can_move_pcntxt(struct irq_data *data)
352{
353 return true;
354}
355static inline bool irq_move_pending(struct irq_data *data)
356{
357 return false;
358}
359static inline void
360irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
361{
362}
363static inline void
364irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
365{
366}
367static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
368{
369 return NULL;
370}
371static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear)
372{
373 return false;
374}
375static inline void irq_force_complete_move(struct irq_desc *desc) { }
376#endif /* !CONFIG_GENERIC_PENDING_IRQ */
377
378#if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY)
379static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve)
380{
381 irqd_set_activated(data);
382 return 0;
383}
384static inline void irq_domain_deactivate_irq(struct irq_data *data)
385{
386 irqd_clr_activated(data);
387}
388#endif
389
390static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd)
391{
392#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
393 return irqd->parent_data;
394#else
395 return NULL;
396#endif
397}