projects/linux·blob master · refs

95 lines · 2.6 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __LINUX_INTERRUPT_RC_H
3#define __LINUX_INTERRUPT_RC_H
4
5/*
6 * include/linux/interrupt_rc.h - refcounted local processor interrupt
7 * management.
8 *
9 * Since the implementation of this API currently depends on
10 * local_irq_save()/local_irq_restore(), we split this into its own header to
11 * make it easier to include without hitting circular header dependencies.
12 */
13
14#include <linux/debug_locks.h>
15#include <linux/irqflags.h>
16#include <linux/preempt.h>
17#include <linux/processor.h>
18#include <linux/smp.h>
19
20#ifndef MODULE
21/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
22DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
23
24static __always_inline void __local_interrupt_save_state(unsigned long flags)
25{
26 raw_cpu_write(local_interrupt_disable_state, flags);
27}
28
29static __always_inline void __local_interrupt_enable(void)
30{
31 unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
32
33 local_irq_restore(flags);
34}
35
36#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
37static __always_inline void _local_interrupt_save_state(unsigned long flags)
38{
39 __local_interrupt_save_state(flags);
40}
41
42static __always_inline void _local_interrupt_enable(void)
43{
44 __local_interrupt_enable();
45}
46#else
47extern void _local_interrupt_save_state(unsigned long flags);
48extern void _local_interrupt_enable(void);
49#endif
50
51#else /* !MODULE */
52extern void _local_interrupt_save_state(unsigned long flags);
53extern void _local_interrupt_enable(void);
54#endif /* !MODULE */
55
56#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)
57#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
58
59static inline void local_interrupt_disable(void)
60{
61 int new_count;
62 unsigned long flags;
63
64 WARN_ON_ONCE(in_nmi());
65
66 local_irq_save(flags);
67 new_count = hardirq_disable_enter();
68
69 /* Is hardirq disable count overflow soon? */
70 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
71 DEBUG_LOCKS_WARN_ON(((new_count & HARDIRQ_DISABLE_MASK) +
72 (10 << HARDIRQ_DISABLE_SHIFT)) >=
73 HARDIRQ_DISABLE_MASK))
74 return;
75
76 if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
77 _local_interrupt_save_state(flags);
78}
79
80static inline void local_interrupt_enable(void)
81{
82 int new_count;
83
84 /* Unpaired local_interrupt_enable()? Warn and abort. */
85 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
86 DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
87 return;
88
89 new_count = hardirq_disable_exit();
90
91 if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
92 _local_interrupt_enable();
93}
94
95#endif /* !__LINUX_INTERRUPT_RC_H */