projects/linux·blob master · refs

481 lines · 12.9 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _FUTEX_H
3#define _FUTEX_H
4
5#include <linux/futex.h>
6#include <linux/rtmutex.h>
7#include <linux/sched/wake_q.h>
8#include <linux/compat.h>
9#include <linux/uaccess.h>
10#include <linux/cleanup.h>
11
12#ifdef CONFIG_PREEMPT_RT
13#include <linux/rcuwait.h>
14#endif
15
16#include <asm/futex.h>
17
18/*
19 * Futex flags used to encode options to functions and preserve them across
20 * restarts.
21 */
22#define FLAGS_SIZE_8 0x0000
23#define FLAGS_SIZE_16 0x0001
24#define FLAGS_SIZE_32 0x0002
25#define FLAGS_SIZE_64 0x0003
26
27#define FLAGS_SIZE_MASK 0x0003
28
29#ifdef CONFIG_MMU
30# define FLAGS_SHARED 0x0010
31#else
32/*
33 * NOMMU does not have per process address space. Let the compiler optimize
34 * code away.
35 */
36# define FLAGS_SHARED 0x0000
37#endif
38#define FLAGS_CLOCKRT 0x0020
39#define FLAGS_HAS_TIMEOUT 0x0040
40#define FLAGS_NUMA 0x0080
41#define FLAGS_STRICT 0x0100
42#define FLAGS_MPOL 0x0200
43#define FLAGS_ROBUST_UNLOCK 0x0400
44#define FLAGS_ROBUST_LIST32 0x0800
45
46/* FUTEX_ to FLAGS_ */
47static inline unsigned int futex_to_flags(unsigned int op)
48{
49 unsigned int flags = FLAGS_SIZE_32;
50
51 if (!(op & FUTEX_PRIVATE_FLAG))
52 flags |= FLAGS_SHARED;
53
54 if (op & FUTEX_CLOCK_REALTIME)
55 flags |= FLAGS_CLOCKRT;
56
57 if (op & FUTEX_ROBUST_UNLOCK)
58 flags |= FLAGS_ROBUST_UNLOCK;
59
60 if (op & FUTEX_ROBUST_LIST32)
61 flags |= FLAGS_ROBUST_LIST32;
62
63 return flags;
64}
65
66#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_NUMA | FUTEX2_MPOL | FUTEX2_PRIVATE)
67
68/* FUTEX2_ to FLAGS_ */
69static inline unsigned int futex2_to_flags(unsigned int flags2)
70{
71 unsigned int flags = flags2 & FUTEX2_SIZE_MASK;
72
73 if (!(flags2 & FUTEX2_PRIVATE))
74 flags |= FLAGS_SHARED;
75
76 if (flags2 & FUTEX2_NUMA)
77 flags |= FLAGS_NUMA;
78
79 if (flags2 & FUTEX2_MPOL)
80 flags |= FLAGS_MPOL;
81
82 return flags;
83}
84
85static inline unsigned int futex_size(unsigned int flags)
86{
87 return 1 << (flags & FLAGS_SIZE_MASK);
88}
89
90static inline bool futex_flags_valid(unsigned int flags)
91{
92 /* Only 64bit futexes for 64bit code */
93 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) {
94 if ((flags & FLAGS_SIZE_MASK) == FLAGS_SIZE_64)
95 return false;
96 }
97
98 /* Only 32bit futexes are implemented -- for now */
99 if ((flags & FLAGS_SIZE_MASK) != FLAGS_SIZE_32)
100 return false;
101
102 /*
103 * Must be able to represent both FUTEX_NO_NODE and every valid nodeid
104 * in a futex word.
105 */
106 if (flags & FLAGS_NUMA) {
107 int bits = 8 * futex_size(flags);
108 u64 max = ~0ULL;
109
110 max >>= 64 - bits;
111 if (nr_node_ids >= max)
112 return false;
113 }
114
115 return true;
116}
117
118static inline bool futex_validate_input(unsigned int flags, u64 val)
119{
120 int bits = 8 * futex_size(flags);
121
122 if (bits < 64 && (val >> bits))
123 return false;
124
125 return true;
126}
127
128#ifdef CONFIG_FAIL_FUTEX
129extern bool should_fail_futex(bool fshared);
130#else
131static inline bool should_fail_futex(bool fshared)
132{
133 return false;
134}
135#endif
136
137static inline bool futex_key_is_private(union futex_key *key)
138{
139 /*
140 * Relies on get_futex_key() to set either bit for shared
141 * futexes -- see comment with union futex_key.
142 */
143 return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED));
144}
145
146/*
147 * Hash buckets are shared by all the futex_keys that hash to the same
148 * location. Each key may have multiple futex_q structures, one for each task
149 * waiting on a futex.
150 */
151struct futex_hash_bucket {
152 atomic_t waiters;
153 spinlock_t lock;
154 struct plist_head chain;
155} ____cacheline_aligned_in_smp;
156
157/*
158 * Priority Inheritance state:
159 */
160struct futex_pi_state {
161 /*
162 * list of 'owned' pi_state instances - these have to be
163 * cleaned up in do_exit() if the task exits prematurely:
164 */
165 struct list_head list;
166
167 /*
168 * The PI object:
169 */
170 struct rt_mutex_base pi_mutex;
171
172 struct task_struct *owner;
173 refcount_t refcount;
174
175 union futex_key key;
176} __randomize_layout;
177
178struct futex_q;
179typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q);
180
181/**
182 * struct futex_q - The hashed futex queue entry, one per waiting task
183 * @list: priority-sorted list of tasks waiting on this futex
184 * @task: the task waiting on the futex
185 * @lock_ptr: the hash bucket lock
186 * @wake: the wake handler for this queue
187 * @wake_data: data associated with the wake handler
188 * @key: the key the futex is hashed on
189 * @pi_state: optional priority inheritance state
190 * @rt_waiter: rt_waiter storage for use with requeue_pi
191 * @requeue_pi_key: the requeue_pi target futex key
192 * @bitset: bitset for the optional bitmasked wakeup
193 * @requeue_state: State field for futex_requeue_pi()
194 * @drop_fph: Waiter should drop the extra private hash reference when set
195 * @requeue_wait: RCU wait for futex_requeue_pi() (RT only)
196 *
197 * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
198 * we can wake only the relevant ones (hashed queues may be shared).
199 *
200 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
201 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
202 * The order of wakeup is always to make the first condition true, then
203 * the second.
204 *
205 * PI futexes are typically woken before they are removed from the hash list via
206 * the rt_mutex code. See futex_unqueue_pi().
207 */
208struct futex_q {
209 struct plist_node list;
210
211 struct task_struct *task;
212 spinlock_t *lock_ptr;
213 futex_wake_fn *wake;
214 void *wake_data;
215 union futex_key key;
216 struct futex_pi_state *pi_state;
217 struct rt_mutex_waiter *rt_waiter;
218 union futex_key *requeue_pi_key;
219 u32 bitset;
220 atomic_t requeue_state;
221 struct futex_private_hash *drop_fph;
222#ifdef CONFIG_PREEMPT_RT
223 struct rcuwait requeue_wait;
224#endif
225} __randomize_layout;
226
227extern const struct futex_q futex_q_init;
228
229enum futex_access {
230 FUTEX_READ,
231 FUTEX_WRITE
232};
233
234extern int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
235 enum futex_access rw);
236extern void futex_q_lockptr_lock(struct futex_q *q) __acquires(q->lock_ptr);
237extern struct hrtimer_sleeper *
238futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
239 int flags, u64 range_ns);
240
241struct futex_bucket_ref {
242 struct futex_hash_bucket *hb;
243 struct futex_private_hash *fph;
244};
245
246#ifdef CONFIG_FUTEX_PRIVATE_HASH
247extern struct futex_private_hash *futex_private_hash(struct mm_struct *mm);
248extern void futex_private_hash_put(struct futex_private_hash *fph);
249
250#else /* !CONFIG_FUTEX_PRIVATE_HASH */
251static inline struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { return NULL; }
252static inline void futex_private_hash_put(struct futex_private_hash *fph) { }
253#endif
254
255extern struct futex_bucket_ref futex_hash(union futex_key *key);
256
257DEFINE_CLASS(hbr, struct futex_bucket_ref,
258 if (_T.fph) futex_private_hash_put(_T.fph),
259 futex_hash(key), union futex_key *key);
260
261DEFINE_CLASS(private_hash, struct futex_private_hash *,
262 if (_T) futex_private_hash_put(_T),
263 futex_private_hash(mm), struct mm_struct *mm);
264
265/**
266 * futex_match - Check whether two futex keys are equal
267 * @key1: Pointer to key1
268 * @key2: Pointer to key2
269 *
270 * Return 1 if two futex_keys are equal, 0 otherwise.
271 */
272static inline int futex_match(union futex_key *key1, union futex_key *key2)
273{
274 return (key1 && key2
275 && key1->both.word == key2->both.word
276 && key1->both.ptr == key2->both.ptr
277 && key1->both.offset == key2->both.offset);
278}
279
280extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
281 struct futex_q *q, union futex_key *key2,
282 struct task_struct *task);
283extern void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout);
284extern bool __futex_wake_mark(struct futex_q *q);
285extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q);
286
287extern int fault_in_user_writeable(u32 __user *uaddr);
288extern struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key);
289
290static inline int futex_cmpxchg_value_locked(u32 *curval, u32 __user *uaddr, u32 uval, u32 newval)
291{
292 int ret;
293
294 pagefault_disable();
295 ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
296 pagefault_enable();
297
298 return ret;
299}
300
301/* Read from user memory with pagefaults disabled */
302static inline int futex_get_value_locked(u32 *dest, u32 __user *from)
303{
304 guard(pagefault)();
305 return get_user_inline(*dest, from);
306}
307
308extern void __futex_unqueue(struct futex_q *q);
309extern void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
310 struct task_struct *task);
311extern int futex_unqueue(struct futex_q *q);
312
313/**
314 * futex_queue() - Enqueue the futex_q on the futex_hash_bucket
315 * @q: The futex_q to enqueue
316 * @hb: The destination hash bucket
317 * @task: Task queueing this futex
318 *
319 * The hb->lock must be held by the caller, and is released here. A call to
320 * futex_queue() is typically paired with exactly one call to futex_unqueue(). The
321 * exceptions involve the PI related operations, which may use futex_unqueue_pi()
322 * or nothing if the unqueue is done as part of the wake process and the unqueue
323 * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
324 * an example).
325 *
326 * Note that @task may be NULL, for async usage of futexes.
327 */
328static inline void futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
329 struct task_struct *task)
330 __releases(&hb->lock)
331 __releases(q->lock_ptr)
332{
333 __futex_queue(q, hb, task);
334 spin_unlock(&hb->lock);
335 __release(q->lock_ptr);
336}
337
338extern void futex_unqueue_pi(struct futex_q *q);
339
340extern void wait_for_owner_exiting(int ret, struct task_struct *exiting);
341
342/*
343 * Reflects a new waiter being added to the waitqueue.
344 */
345static inline void futex_hb_waiters_inc(struct futex_hash_bucket *hb)
346{
347#ifdef CONFIG_SMP
348 atomic_inc(&hb->waiters);
349 /*
350 * Full barrier (A), see the ordering comment above.
351 */
352 smp_mb__after_atomic();
353#endif
354}
355
356/*
357 * Reflects a waiter being removed from the waitqueue by wakeup
358 * paths.
359 */
360static inline void futex_hb_waiters_dec(struct futex_hash_bucket *hb)
361{
362#ifdef CONFIG_SMP
363 atomic_dec(&hb->waiters);
364#endif
365}
366
367static inline int futex_hb_waiters_pending(struct futex_hash_bucket *hb)
368{
369#ifdef CONFIG_SMP
370 /*
371 * Full barrier (B), see the ordering comment above.
372 */
373 smp_mb();
374 return atomic_read(&hb->waiters);
375#else
376 return 1;
377#endif
378}
379
380extern void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb)
381 __acquires(&hb->lock)
382 __acquires(q->lock_ptr);
383
384extern void futex_q_unlock(struct futex_hash_bucket *hb)
385 __releases(&hb->lock);
386
387extern int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
388 union futex_key *key,
389 struct futex_pi_state **ps,
390 struct task_struct *task,
391 struct task_struct **exiting,
392 int set_waiters);
393
394extern int refill_pi_state_cache(void);
395extern void get_pi_state(struct futex_pi_state *pi_state);
396extern void put_pi_state(struct futex_pi_state *pi_state);
397extern int fixup_pi_owner(u32 __user *uaddr, struct futex_q *q, int locked);
398
399/*
400 * Express the locking dependencies for lockdep:
401 */
402static inline void
403double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
404 __acquires(&hb1->lock)
405 __acquires(&hb2->lock)
406 __no_context_analysis
407{
408 if (hb1 > hb2)
409 swap(hb1, hb2);
410
411 spin_lock(&hb1->lock);
412 if (hb1 != hb2)
413 spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
414}
415
416static inline void
417double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
418 __releases(&hb1->lock)
419 __releases(&hb2->lock)
420 __no_context_analysis
421{
422 spin_unlock(&hb1->lock);
423 if (hb1 != hb2)
424 spin_unlock(&hb2->lock);
425}
426
427/* syscalls */
428
429extern int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, u32
430 val, ktime_t *abs_time, u32 bitset, u32 __user
431 *uaddr2);
432
433extern int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
434 u32 __user *uaddr2, unsigned int flags2,
435 int nr_wake, int nr_requeue,
436 u32 *cmpval, int requeue_pi);
437
438extern int __futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
439 struct hrtimer_sleeper *to, u32 bitset);
440
441extern int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
442 ktime_t *abs_time, u32 bitset);
443
444/**
445 * struct futex_vector - Auxiliary struct for futex_waitv()
446 * @w: Userspace provided data
447 * @q: Kernel side data
448 *
449 * Struct used to build an array with all data need for futex_waitv()
450 */
451struct futex_vector {
452 struct futex_waitv w;
453 struct futex_q q;
454};
455
456extern int futex_parse_waitv(struct futex_vector *futexv,
457 struct futex_waitv __user *uwaitv,
458 unsigned int nr_futexes, futex_wake_fn *wake,
459 void *wake_data);
460
461extern int futex_wait_multiple_setup(struct futex_vector *vs, int count,
462 int *woken);
463
464extern int futex_unqueue_multiple(struct futex_vector *v, int count);
465
466extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
467 struct hrtimer_sleeper *to);
468
469extern int futex_wake(u32 __user *uaddr, unsigned int flags, void __user *pop,
470 int nr_wake, u32 bitset);
471
472extern int futex_wake_op(u32 __user *uaddr1, unsigned int flags,
473 u32 __user *uaddr2, int nr_wake, int nr_wake2, int op);
474
475extern int futex_unlock_pi(u32 __user *uaddr, unsigned int flags, void __user *pop);
476
477extern int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int trylock);
478
479bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags);
480
481#endif /* _FUTEX_H */