projects/linux·blob master · refs

105 lines · 2.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_PREEMPT_H
3#define __ASM_PREEMPT_H
4
5#include <linux/thread_info.h>
6
7#define PREEMPT_NEED_RESCHED BIT(32)
8#define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
9
10static inline int preempt_count(void)
11{
12 return READ_ONCE(current_thread_info()->preempt.count);
13}
14
15static inline void preempt_count_set(u64 pc)
16{
17 /* Preserve existing value of PREEMPT_NEED_RESCHED */
18 WRITE_ONCE(current_thread_info()->preempt.count, pc);
19}
20
21#define init_task_preempt_count(p) do { \
22 task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \
23} while (0)
24
25#define init_idle_preempt_count(p, cpu) do { \
26 task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \
27} while (0)
28
29static inline void set_preempt_need_resched(void)
30{
31 current_thread_info()->preempt.need_resched = 0;
32}
33
34static inline void clear_preempt_need_resched(void)
35{
36 current_thread_info()->preempt.need_resched = 1;
37}
38
39static inline bool test_preempt_need_resched(void)
40{
41 return !current_thread_info()->preempt.need_resched;
42}
43
44static inline void __preempt_count_add(int val)
45{
46 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
47 pc += val;
48 WRITE_ONCE(current_thread_info()->preempt.count, pc);
49}
50
51static inline void __preempt_count_sub(int val)
52{
53 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
54 pc -= val;
55 WRITE_ONCE(current_thread_info()->preempt.count, pc);
56}
57
58static inline int __preempt_count_add_return(int val)
59{
60 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
61
62 pc += val;
63 WRITE_ONCE(current_thread_info()->preempt.count, pc);
64
65 return pc;
66}
67
68static inline int __preempt_count_sub_return(int val)
69{
70 u32 pc = READ_ONCE(current_thread_info()->preempt.count);
71
72 pc -= val;
73 WRITE_ONCE(current_thread_info()->preempt.count, pc);
74
75 return pc;
76}
77
78static inline bool should_resched(int preempt_offset)
79{
80 u64 pc = READ_ONCE(current_thread_info()->preempt_count);
81 return unlikely(pc == preempt_offset);
82}
83
84static inline bool __preempt_count_dec_and_test(void)
85{
86 /*
87 * We must load the combined 'preempt_count' after decrementing
88 * 'preempt.count' as an interrupt could modify 'need_resched' before
89 * __preempt_count_sub() writes back to 'preempt.count'.
90 */
91 __preempt_count_sub(1);
92 return should_resched(0);
93}
94
95#ifdef CONFIG_PREEMPTION
96
97void preempt_schedule(void);
98void preempt_schedule_notrace(void);
99
100#define __preempt_schedule() preempt_schedule()
101#define __preempt_schedule_notrace() preempt_schedule_notrace()
102
103#endif /* CONFIG_PREEMPTION */
104
105#endif /* __ASM_PREEMPT_H */