projects/linux·blob master · refs

2145 lines · 57.1 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Fast Userspace Mutexes (which I call "Futexes!").
4 * (C) Rusty Russell, IBM 2002
5 *
6 * Generalized futexes, futex requeueing, misc fixes by Ingo Molnar
7 * (C) Copyright 2003 Red Hat Inc, All Rights Reserved
8 *
9 * Removed page pinning, fix privately mapped COW pages and other cleanups
10 * (C) Copyright 2003, 2004 Jamie Lokier
11 *
12 * Robust futex support started by Ingo Molnar
13 * (C) Copyright 2006 Red Hat Inc, All Rights Reserved
14 * Thanks to Thomas Gleixner for suggestions, analysis and fixes.
15 *
16 * PI-futex support started by Ingo Molnar and Thomas Gleixner
17 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <[email protected]>
18 * Copyright (C) 2006 Timesys Corp., Thomas Gleixner <[email protected]>
19 *
20 * PRIVATE futexes by Eric Dumazet
21 * Copyright (C) 2007 Eric Dumazet <[email protected]>
22 *
23 * Requeue-PI support by Darren Hart <[email protected]>
24 * Copyright (C) IBM Corporation, 2009
25 * Thanks to Thomas Gleixner for conceptual design and careful reviews.
26 *
27 * Thanks to Ben LaHaise for yelling "hashed waitqueues" loudly
28 * enough at me, Linus for the original (flawed) idea, Matthew
29 * Kirkwood for proof-of-concept implementation.
30 *
31 * "The futexes are also cursed."
32 * "But they come in a choice of three flavours!"
33 */
34#include <linux/compat.h>
35#include <linux/debugfs.h>
36#include <linux/fault-inject.h>
37#include <linux/gfp.h>
38#include <linux/jhash.h>
39#include <linux/memblock.h>
40#include <linux/mempolicy.h>
41#include <linux/mmap_lock.h>
42#include <linux/pagemap.h>
43#include <linux/plist.h>
44#include <linux/prctl.h>
45#include <linux/rseq.h>
46#include <linux/slab.h>
47#include <linux/vmalloc.h>
48#include <linux/kmemleak.h>
49#include <linux/wait_bit.h>
50
51#include <vdso/futex.h>
52
53#include <asm/runtime-const.h>
54
55#include "futex.h"
56#include "../locking/rtmutex_common.h"
57
58static u32 __futex_mask __ro_after_init;
59static u32 __futex_shift __ro_after_init;
60static struct futex_hash_bucket **__futex_queues __ro_after_init;
61
62static __always_inline struct futex_hash_bucket **futex_queues(void)
63{
64 return runtime_const_ptr(__futex_queues);
65}
66
67struct futex_private_hash {
68 int state;
69 unsigned int hash_mask;
70 struct rcu_head rcu;
71 void *mm;
72 bool custom;
73 struct futex_hash_bucket queues[];
74};
75
76/*
77 * Fault injections for futexes.
78 */
79#ifdef CONFIG_FAIL_FUTEX
80
81static struct {
82 struct fault_attr attr;
83
84 bool ignore_private;
85} fail_futex = {
86 .attr = FAULT_ATTR_INITIALIZER,
87 .ignore_private = false,
88};
89
90static int __init setup_fail_futex(char *str)
91{
92 return setup_fault_attr(&fail_futex.attr, str);
93}
94__setup("fail_futex=", setup_fail_futex);
95
96bool should_fail_futex(bool fshared)
97{
98 if (fail_futex.ignore_private && !fshared)
99 return false;
100
101 return should_fail(&fail_futex.attr, 1);
102}
103
104#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
105
106static int __init fail_futex_debugfs(void)
107{
108 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
109 struct dentry *dir;
110
111 dir = fault_create_debugfs_attr("fail_futex", NULL,
112 &fail_futex.attr);
113 if (IS_ERR(dir))
114 return PTR_ERR(dir);
115
116 debugfs_create_bool("ignore-private", mode, dir,
117 &fail_futex.ignore_private);
118 return 0;
119}
120
121late_initcall(fail_futex_debugfs);
122
123#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
124
125#endif /* CONFIG_FAIL_FUTEX */
126
127static struct futex_hash_bucket *
128__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p);
129
130#ifdef CONFIG_FUTEX_PRIVATE_HASH
131static bool futex_ref_get(struct futex_private_hash *fph);
132static bool futex_ref_put(struct futex_private_hash *fph);
133static bool futex_ref_is_dead(struct futex_private_hash *fph);
134
135enum { FR_PERCPU = 0, FR_ATOMIC };
136
137static bool futex_private_hash_get(struct futex_private_hash *fph)
138{
139 return futex_ref_get(fph);
140}
141
142void futex_private_hash_put(struct futex_private_hash *fph)
143{
144 struct mm_struct *mm;
145
146 if (!fph)
147 return;
148
149 mm = fph->mm;
150 if (futex_ref_put(fph))
151 wake_up_var(mm);
152}
153
154static struct futex_hash_bucket *
155__futex_hash_private(union futex_key *key, struct futex_private_hash *fph)
156{
157 u32 hash;
158
159 hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4,
160 key->both.offset);
161
162 return &fph->queues[hash & fph->hash_mask];
163}
164
165static void futex_rehash_private(struct futex_private_hash *old,
166 struct futex_private_hash *new)
167{
168 struct futex_hash_bucket *hb_old, *hb_new;
169 unsigned int slots = old->hash_mask + 1;
170 unsigned int i;
171
172 for (i = 0; i < slots; i++) {
173 struct futex_q *this, *tmp;
174
175 hb_old = &old->queues[i];
176
177 spin_lock(&hb_old->lock);
178 plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) {
179 plist_del(&this->list, &hb_old->chain);
180 futex_hb_waiters_dec(hb_old);
181
182 WARN_ON_ONCE(this->lock_ptr != &hb_old->lock);
183
184 hb_new = __futex_hash(&this->key, new, NULL);
185 futex_hb_waiters_inc(hb_new);
186 /*
187 * The new pointer isn't published yet but an already
188 * moved user can be unqueued due to timeout or signal.
189 */
190 spin_lock_nested(&hb_new->lock, SINGLE_DEPTH_NESTING);
191 plist_add(&this->list, &hb_new->chain);
192 this->lock_ptr = &hb_new->lock;
193 spin_unlock(&hb_new->lock);
194 }
195 spin_unlock(&hb_old->lock);
196 }
197}
198
199static bool __futex_pivot_hash(struct mm_struct *mm, struct futex_private_hash *new)
200{
201 struct futex_mm_phash *mmph = &mm->futex.phash;
202 struct futex_private_hash *fph;
203
204 WARN_ON_ONCE(mmph->hash_new);
205
206 fph = rcu_dereference_protected(mmph->hash, lockdep_is_held(&mmph->lock));
207 if (fph) {
208 if (!futex_ref_is_dead(fph)) {
209 mmph->hash_new = new;
210 return false;
211 }
212
213 futex_rehash_private(fph, new);
214 }
215 new->state = FR_PERCPU;
216 rcu_assign_pointer(mmph->hash, new);
217 /*
218 * mmph->batches must reference a grace period which started after
219 * mmph->hash was assigned. See futex_ref_drop().
220 */
221 mmph->batches = get_state_synchronize_rcu();
222 kvfree_rcu(fph, rcu);
223 return true;
224}
225
226static void futex_pivot_hash(struct mm_struct *mm)
227{
228 scoped_guard(mutex, &mm->futex.phash.lock) {
229 struct futex_private_hash *fph;
230
231 fph = mm->futex.phash.hash_new;
232 if (fph) {
233 mm->futex.phash.hash_new = NULL;
234 __futex_pivot_hash(mm, fph);
235 }
236 }
237}
238
239struct futex_private_hash *futex_private_hash(struct mm_struct *mm)
240{
241 /*
242 * Ideally we don't loop. If there is a replacement in progress
243 * then a new private hash is already prepared and a reference can't be
244 * obtained once the last user dropped it's.
245 * In that case we block on mm_struct::futex_hash_lock and either have
246 * to perform the replacement or wait while someone else is doing the
247 * job. Eitherway, on the second iteration we acquire a reference on the
248 * new private hash or loop again because a new replacement has been
249 * requested.
250 */
251again:
252 scoped_guard(rcu) {
253 struct futex_private_hash *fph;
254
255 fph = rcu_dereference(mm->futex.phash.hash);
256 if (!fph)
257 return NULL;
258
259 if (futex_private_hash_get(fph))
260 return fph;
261 }
262 futex_pivot_hash(mm);
263 goto again;
264}
265
266struct futex_bucket_ref futex_hash(union futex_key *key)
267{
268again:
269 scoped_guard(rcu) {
270 struct futex_private_hash *fph = NULL;
271 struct futex_hash_bucket *hb;
272
273 hb = __futex_hash(key, NULL, &fph);
274
275 if (!fph || futex_private_hash_get(fph))
276 return (struct futex_bucket_ref){ .hb = hb, .fph = fph };
277 }
278 futex_pivot_hash(key->private.mm);
279 goto again;
280}
281
282#else /* !CONFIG_FUTEX_PRIVATE_HASH */
283
284struct futex_bucket_ref futex_hash(union futex_key *key)
285{
286 return (struct futex_bucket_ref){ .hb = __futex_hash(key, NULL, NULL), .fph = NULL };
287}
288
289#endif /* CONFIG_FUTEX_PRIVATE_HASH */
290
291#ifdef CONFIG_FUTEX_MPOL
292
293static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr)
294{
295 struct vm_area_struct *vma = vma_lookup(mm, addr);
296 struct mempolicy *mpol;
297 int node = FUTEX_NO_NODE;
298
299 if (!vma)
300 return FUTEX_NO_NODE;
301
302 mpol = READ_ONCE(vma->vm_policy);
303 if (!mpol)
304 return FUTEX_NO_NODE;
305
306 switch (mpol->mode) {
307 case MPOL_PREFERRED:
308 node = first_node(mpol->nodes);
309 break;
310 case MPOL_PREFERRED_MANY:
311 case MPOL_BIND:
312 if (mpol->home_node != NUMA_NO_NODE)
313 node = mpol->home_node;
314 break;
315 default:
316 break;
317 }
318
319 return node;
320}
321
322static int futex_key_to_node_opt(struct mm_struct *mm, unsigned long addr)
323{
324 int seq, node;
325
326 guard(rcu)();
327
328 if (!mmap_lock_speculate_try_begin(mm, &seq))
329 return -EBUSY;
330
331 node = __futex_key_to_node(mm, addr);
332
333 if (mmap_lock_speculate_retry(mm, seq))
334 return -EAGAIN;
335
336 return node;
337}
338
339static int futex_mpol(struct mm_struct *mm, unsigned long addr)
340{
341 int node;
342
343 node = futex_key_to_node_opt(mm, addr);
344 if (node >= FUTEX_NO_NODE)
345 return node;
346
347 guard(mmap_read_lock)(mm);
348 return __futex_key_to_node(mm, addr);
349}
350
351#else /* !CONFIG_FUTEX_MPOL */
352
353static int futex_mpol(struct mm_struct *mm, unsigned long addr)
354{
355 return FUTEX_NO_NODE;
356}
357
358#endif /* CONFIG_FUTEX_MPOL */
359
360/**
361 * __futex_hash - Return the hash bucket
362 * @key: Pointer to the futex key for which the hash is calculated
363 * @fph: Pointer to private hash if known
364 * @fph_p: Pointer to a private hash pointer; output for the private hash
365 * used when set.
366 *
367 * We hash on the keys returned from get_futex_key (see below) and return the
368 * corresponding hash bucket.
369 * If the FUTEX is PROCESS_PRIVATE then a per-process hash bucket (from the
370 * private hash) is returned if existing. Otherwise a hash bucket from the
371 * global hash is returned.
372 */
373static struct futex_hash_bucket *
374__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p)
375{
376 int node = key->both.node;
377 u32 hash;
378
379#ifdef CONFIG_FUTEX_PRIVATE_HASH
380 if (node == FUTEX_NO_NODE && futex_key_is_private(key)) {
381 if (!fph)
382 fph = rcu_dereference(key->private.mm->futex.phash.hash);
383 if (fph && fph->hash_mask) {
384 if (fph_p)
385 *fph_p = fph;
386 return __futex_hash_private(key, fph);
387 }
388 }
389#endif
390
391 hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32),
392 key->both.offset);
393
394 if (node == FUTEX_NO_NODE) {
395 /*
396 * In case of !FLAGS_NUMA, use some unused hash bits to pick a
397 * node -- this ensures regular futexes are interleaved across
398 * the nodes and avoids having to allocate multiple
399 * hash-tables.
400 *
401 * NOTE: this isn't perfectly uniform, but it is fast and
402 * handles sparse node masks.
403 */
404 node = runtime_const_shift_right_32(hash, __futex_shift) % nr_node_ids;
405 if (!node_possible(node)) {
406 node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node);
407 }
408 }
409
410 return &futex_queues()[node][runtime_const_mask_32(hash, __futex_mask)];
411}
412
413/**
414 * futex_setup_timer - set up the sleeping hrtimer.
415 * @time: ptr to the given timeout value
416 * @timeout: the hrtimer_sleeper structure to be set up
417 * @flags: futex flags
418 * @range_ns: optional range in ns
419 *
420 * Return: Initialized hrtimer_sleeper structure or NULL if no timeout
421 * value given
422 */
423struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
424 int flags, u64 range_ns)
425{
426 if (!time)
427 return NULL;
428
429 hrtimer_setup_sleeper_on_stack(timeout,
430 (flags & FLAGS_CLOCKRT) ? CLOCK_REALTIME : CLOCK_MONOTONIC,
431 HRTIMER_MODE_ABS);
432 /*
433 * If range_ns is 0, calling hrtimer_set_expires_range_ns() is
434 * effectively the same as calling hrtimer_set_expires().
435 */
436 hrtimer_set_expires_range_ns(&timeout->timer, *time, range_ns);
437
438 return timeout;
439}
440
441/*
442 * Generate a machine wide unique identifier for this object (inode or mm).
443 *
444 * This relies on u64 not wrapping in the life-time of the machine; which with
445 * 1ns resolution means almost 585 years.
446 */
447static u64 get_object_id(atomic64_t *object_seq)
448{
449 static atomic64_t last_assigned_seq;
450 u64 old;
451
452 /* Does the object already have a sequence number? */
453 old = atomic64_read(object_seq);
454 if (likely(old))
455 return old;
456
457 for (;;) {
458 u64 new = atomic64_inc_return(&last_assigned_seq);
459 if (WARN_ON_ONCE(!new))
460 continue;
461
462 old = 0;
463 if (!atomic64_try_cmpxchg_relaxed(object_seq, &old, new))
464 return old;
465 return new;
466 }
467}
468
469/*
470 * This relies on the fact that a well formed program will not unmap
471 * the file while it has a (shared) futex waiting on it. This mapping will have
472 * a file reference which pins the mount and inode.
473 *
474 * If for some reason an inode gets evicted and read back in again, it will get
475 * a new sequence number and will _NOT_ match, even though it is the exact same
476 * file.
477 *
478 * It is important that futex_match() will never have a false-positive, esp.
479 * for PI futexes that can mess up the state. The above argues that false-negatives
480 * are only possible for malformed programs.
481 */
482static u64 get_inode_sequence_number(struct inode *inode)
483{
484 return get_object_id(&inode->i_sequence);
485}
486
487/**
488 * get_futex_key() - Get parameters which are the keys for a futex
489 * @uaddr: virtual address of the futex
490 * @flags: FLAGS_*
491 * @key: address where result is stored.
492 * @rw: mapping needs to be read/write (values: FUTEX_READ,
493 * FUTEX_WRITE)
494 *
495 * Return: a negative error code or 0
496 *
497 * The key words are stored in @key on success.
498 *
499 * For shared mappings (when @fshared), the key is:
500 *
501 * ( inode->i_sequence, page offset within mapping, offset_within_page )
502 *
503 * [ also see get_inode_sequence_number() ]
504 *
505 * For private mappings (or when !@fshared), the key is:
506 *
507 * ( current->mm, address, 0 )
508 *
509 * This allows (cross process, where applicable) identification of the futex
510 * without keeping the page pinned for the duration of the FUTEX_WAIT.
511 *
512 * lock_page() might sleep, the caller should not hold a spinlock.
513 */
514int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key,
515 enum futex_access rw)
516{
517 unsigned long address = (unsigned long)uaddr;
518 struct mm_struct *mm = current->mm;
519 struct page *page;
520 struct folio *folio;
521 struct address_space *mapping;
522 int node, err, size, ro = 0;
523 bool node_updated = false;
524 bool fshared;
525
526 fshared = flags & FLAGS_SHARED;
527 size = futex_size(flags);
528 if (flags & FLAGS_NUMA)
529 size *= 2;
530
531 /*
532 * The futex address must be "naturally" aligned.
533 */
534 key->both.offset = address % PAGE_SIZE;
535 if (unlikely((address & (size-1)) != 0))
536 return -EINVAL;
537 address -= key->both.offset;
538
539 if (unlikely(!access_ok(uaddr, size)))
540 return -EFAULT;
541
542 if (unlikely(should_fail_futex(fshared)))
543 return -EFAULT;
544
545 node = FUTEX_NO_NODE;
546
547 if (flags & FLAGS_NUMA) {
548 u32 __user *naddr = (void *)uaddr + size / 2;
549
550 if (get_user_inline(node, naddr))
551 return -EFAULT;
552
553 if ((node != FUTEX_NO_NODE) &&
554 ((unsigned int)node >= MAX_NUMNODES || !node_possible(node)))
555 return -EINVAL;
556 }
557
558 if (node == FUTEX_NO_NODE && (flags & FLAGS_MPOL)) {
559 node = futex_mpol(mm, address);
560 node_updated = true;
561 }
562
563 if (flags & FLAGS_NUMA) {
564 u32 __user *naddr = (void *)uaddr + size / 2;
565
566 if (node == FUTEX_NO_NODE) {
567 node = numa_node_id();
568 node_updated = true;
569 }
570 if (node_updated && put_user_inline(node, naddr))
571 return -EFAULT;
572 }
573
574 key->both.node = node;
575
576 /*
577 * PROCESS_PRIVATE futexes are fast.
578 * As the mm cannot disappear under us and the 'key' only needs
579 * virtual address, we dont even have to find the underlying vma.
580 * Note : We do have to check 'uaddr' is a valid user address,
581 * but access_ok() should be faster than find_vma()
582 */
583 if (!fshared) {
584 /*
585 * On no-MMU, shared futexes are treated as private, therefore
586 * we must not include the current process in the key. Since
587 * there is only one address space, the address is a unique key
588 * on its own.
589 */
590 if (IS_ENABLED(CONFIG_MMU))
591 key->private.mm = mm;
592 else
593 key->private.mm = NULL;
594
595 key->private.address = address;
596 return 0;
597 }
598
599again:
600 /* Ignore any VERIFY_READ mapping (futex common case) */
601 if (unlikely(should_fail_futex(true)))
602 return -EFAULT;
603
604 err = get_user_pages_fast(address, 1, FOLL_WRITE, &page);
605 /*
606 * If write access is not required (eg. FUTEX_WAIT), try
607 * and get read-only access.
608 */
609 if (err == -EFAULT && rw == FUTEX_READ) {
610 err = get_user_pages_fast(address, 1, 0, &page);
611 ro = 1;
612 }
613 if (err < 0)
614 return err;
615 else
616 err = 0;
617
618 /*
619 * The treatment of mapping from this point on is critical. The folio
620 * lock protects many things but in this context the folio lock
621 * stabilizes mapping, prevents inode freeing in the shared
622 * file-backed region case and guards against movement to swap cache.
623 *
624 * Strictly speaking the folio lock is not needed in all cases being
625 * considered here and folio lock forces unnecessarily serialization.
626 * From this point on, mapping will be re-verified if necessary and
627 * folio lock will be acquired only if it is unavoidable
628 *
629 * Mapping checks require the folio so it is looked up now. For
630 * anonymous pages, it does not matter if the folio is split
631 * in the future as the key is based on the address. For
632 * filesystem-backed pages, the precise page is required as the
633 * index of the page determines the key.
634 */
635 folio = page_folio(page);
636 mapping = READ_ONCE(folio->mapping);
637
638 /*
639 * If folio->mapping is NULL, then it cannot be an anonymous
640 * page; but it might be the ZERO_PAGE or in the gate area or
641 * in a special mapping (all cases which we are happy to fail);
642 * or it may have been a good file page when get_user_pages_fast
643 * found it, but truncated or holepunched or subjected to
644 * invalidate_complete_page2 before we got the folio lock (also
645 * cases which we are happy to fail). And we hold a reference,
646 * so refcount care in invalidate_inode_page's remove_mapping
647 * prevents drop_caches from setting mapping to NULL beneath us.
648 *
649 * The case we do have to guard against is when memory pressure made
650 * shmem_writepage move it from filecache to swapcache beneath us:
651 * an unlikely race, but we do need to retry for folio->mapping.
652 */
653 if (unlikely(!mapping)) {
654 int shmem_swizzled;
655
656 /*
657 * Folio lock is required to identify which special case above
658 * applies. If this is really a shmem page then the folio lock
659 * will prevent unexpected transitions.
660 */
661 folio_lock(folio);
662 shmem_swizzled = folio_test_swapcache(folio) || folio->mapping;
663 folio_unlock(folio);
664 folio_put(folio);
665
666 if (shmem_swizzled)
667 goto again;
668
669 return -EFAULT;
670 }
671
672 /*
673 * Private mappings are handled in a simple way.
674 *
675 * If the futex key is stored in anonymous memory, then the associated
676 * object is the mm which is implicitly pinned by the calling process.
677 *
678 * NOTE: When userspace waits on a MAP_SHARED mapping, even if
679 * it's a read-only handle, it's expected that futexes attach to
680 * the object not the particular process.
681 */
682 if (folio_test_anon(folio)) {
683 /*
684 * A RO anonymous page will never change and thus doesn't make
685 * sense for futex operations.
686 */
687 if (unlikely(should_fail_futex(true)) || ro) {
688 err = -EFAULT;
689 goto out;
690 }
691
692 key->both.offset |= FUT_OFF_MMSHARED;
693 key->private.mm_seq = get_object_id(&mm->futex.unique_id);
694 key->private.address = address;
695
696 } else {
697 struct inode *inode;
698
699 /*
700 * The associated futex object in this case is the inode and
701 * the folio->mapping must be traversed. Ordinarily this should
702 * be stabilised under folio lock but it's not strictly
703 * necessary in this case as we just want to pin the inode, not
704 * update i_pages or anything like that.
705 *
706 * The RCU read lock is taken as the inode is finally freed
707 * under RCU. If the mapping still matches expectations then the
708 * mapping->host can be safely accessed as being a valid inode.
709 */
710 rcu_read_lock();
711
712 if (READ_ONCE(folio->mapping) != mapping) {
713 rcu_read_unlock();
714 folio_put(folio);
715
716 goto again;
717 }
718
719 inode = READ_ONCE(mapping->host);
720 if (!inode) {
721 rcu_read_unlock();
722 folio_put(folio);
723
724 goto again;
725 }
726
727 key->both.offset |= FUT_OFF_INODE; /* inode-based key */
728 key->shared.i_seq = get_inode_sequence_number(inode);
729 key->shared.pgoff = page_pgoff(folio, page);
730 rcu_read_unlock();
731 }
732
733out:
734 folio_put(folio);
735 return err;
736}
737
738/**
739 * fault_in_user_writeable() - Fault in user address and verify RW access
740 * @uaddr: pointer to faulting user space address
741 *
742 * Slow path to fixup the fault we just took in the atomic write
743 * access to @uaddr.
744 *
745 * We have no generic implementation of a non-destructive write to the
746 * user address. We know that we faulted in the atomic pagefault
747 * disabled section so we can as well avoid the #PF overhead by
748 * calling get_user_pages() right away.
749 */
750int fault_in_user_writeable(u32 __user *uaddr)
751{
752 struct mm_struct *mm = current->mm;
753 int ret;
754
755 mmap_read_lock(mm);
756 ret = fixup_user_fault(mm, (unsigned long)uaddr,
757 FAULT_FLAG_WRITE, NULL);
758 mmap_read_unlock(mm);
759
760 return ret < 0 ? ret : 0;
761}
762
763/**
764 * futex_top_waiter() - Return the highest priority waiter on a futex
765 * @hb: the hash bucket the futex_q's reside in
766 * @key: the futex key (to distinguish it from other futex futex_q's)
767 *
768 * Must be called with the hb lock held.
769 */
770struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb, union futex_key *key)
771{
772 struct futex_q *this;
773
774 plist_for_each_entry(this, &hb->chain, list) {
775 if (futex_match(&this->key, key))
776 return this;
777 }
778 return NULL;
779}
780
781/**
782 * wait_for_owner_exiting - Block until the owner has exited
783 * @ret: owner's current futex lock status
784 * @exiting: Pointer to the exiting task
785 *
786 * Caller must hold a refcount on @exiting.
787 */
788void wait_for_owner_exiting(int ret, struct task_struct *exiting)
789{
790 if (ret != -EBUSY) {
791 WARN_ON_ONCE(exiting);
792 return;
793 }
794
795 if (WARN_ON_ONCE(ret == -EBUSY && !exiting))
796 return;
797
798 mutex_lock(&exiting->futex.exit_mutex);
799 /*
800 * No point in doing state checking here. If the waiter got here
801 * while the task was in exec()->exec_futex_release() then it can
802 * have any FUTEX_STATE_* value when the waiter has acquired the
803 * mutex. OK, if running, EXITING or DEAD if it reached exit()
804 * already. Highly unlikely and not a problem. Just one more round
805 * through the futex maze.
806 */
807 mutex_unlock(&exiting->futex.exit_mutex);
808
809 put_task_struct(exiting);
810}
811
812/**
813 * __futex_unqueue() - Remove the futex_q from its futex_hash_bucket
814 * @q: The futex_q to unqueue
815 *
816 * The q->lock_ptr must not be NULL and must be held by the caller.
817 */
818void __futex_unqueue(struct futex_q *q)
819{
820 struct futex_hash_bucket *hb;
821
822 if (WARN_ON_SMP(!q->lock_ptr) || WARN_ON(plist_node_empty(&q->list)))
823 return;
824 lockdep_assert_held(q->lock_ptr);
825
826 hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
827 plist_del(&q->list, &hb->chain);
828 futex_hb_waiters_dec(hb);
829}
830
831/* The key must be already stored in q->key. */
832void futex_q_lock(struct futex_q *q, struct futex_hash_bucket *hb)
833{
834 /*
835 * Increment the counter before taking the lock so that
836 * a potential waker won't miss a to-be-slept task that is
837 * waiting for the spinlock. This is safe as all futex_q_lock()
838 * users end up calling futex_queue(). Similarly, for housekeeping,
839 * decrement the counter at futex_q_unlock() when some error has
840 * occurred and we don't end up adding the task to the list.
841 */
842 futex_hb_waiters_inc(hb); /* implies smp_mb(); (A) */
843
844 q->lock_ptr = &hb->lock;
845
846 spin_lock(&hb->lock);
847 __acquire(q->lock_ptr);
848}
849
850void futex_q_unlock(struct futex_hash_bucket *hb)
851{
852 futex_hb_waiters_dec(hb);
853 spin_unlock(&hb->lock);
854}
855
856void __futex_queue(struct futex_q *q, struct futex_hash_bucket *hb,
857 struct task_struct *task)
858{
859 int prio;
860
861 /*
862 * The priority used to register this element is
863 * - either the real thread-priority for the real-time threads
864 * (i.e. threads with a priority lower than MAX_RT_PRIO)
865 * - or MAX_RT_PRIO for non-RT threads.
866 * Thus, all RT-threads are woken first in priority order, and
867 * the others are woken last, in FIFO order.
868 */
869 prio = min(current->normal_prio, MAX_RT_PRIO);
870
871 plist_node_init(&q->list, prio);
872 plist_add(&q->list, &hb->chain);
873 q->task = task;
874}
875
876/**
877 * futex_unqueue() - Remove the futex_q from its futex_hash_bucket
878 * @q: The futex_q to unqueue
879 *
880 * The q->lock_ptr must not be held by the caller. A call to futex_unqueue() must
881 * be paired with exactly one earlier call to futex_queue().
882 *
883 * Return:
884 * - 1 - if the futex_q was still queued (and we removed unqueued it);
885 * - 0 - if the futex_q was already removed by the waking thread
886 */
887int futex_unqueue(struct futex_q *q)
888{
889 spinlock_t *lock_ptr;
890 int ret = 0;
891
892 /* RCU so lock_ptr is not going away during locking. */
893 guard(rcu)();
894 /* In the common case we don't take the spinlock, which is nice. */
895retry:
896 /*
897 * q->lock_ptr can change between this read and the following spin_lock.
898 * Use READ_ONCE to forbid the compiler from reloading q->lock_ptr and
899 * optimizing lock_ptr out of the logic below.
900 */
901 lock_ptr = READ_ONCE(q->lock_ptr);
902 if (lock_ptr != NULL) {
903 spin_lock(lock_ptr);
904 /*
905 * q->lock_ptr can change between reading it and
906 * spin_lock(), causing us to take the wrong lock. This
907 * corrects the race condition.
908 *
909 * Reasoning goes like this: if we have the wrong lock,
910 * q->lock_ptr must have changed (maybe several times)
911 * between reading it and the spin_lock(). It can
912 * change again after the spin_lock() but only if it was
913 * already changed before the spin_lock(). It cannot,
914 * however, change back to the original value. Therefore
915 * we can detect whether we acquired the correct lock.
916 */
917 if (unlikely(lock_ptr != q->lock_ptr)) {
918 spin_unlock(lock_ptr);
919 goto retry;
920 }
921 __futex_unqueue(q);
922
923 BUG_ON(q->pi_state);
924
925 spin_unlock(lock_ptr);
926 ret = 1;
927 }
928
929 return ret;
930}
931
932void futex_q_lockptr_lock(struct futex_q *q)
933{
934 spinlock_t *lock_ptr;
935
936 /*
937 * See futex_unqueue() why lock_ptr can change.
938 */
939 guard(rcu)();
940retry:
941 lock_ptr = READ_ONCE(q->lock_ptr);
942 spin_lock(lock_ptr);
943
944 if (unlikely(lock_ptr != q->lock_ptr)) {
945 spin_unlock(lock_ptr);
946 goto retry;
947 }
948}
949
950/*
951 * PI futexes can not be requeued and must remove themselves from the hash
952 * bucket. The hash bucket lock (i.e. lock_ptr) is held.
953 */
954void futex_unqueue_pi(struct futex_q *q)
955{
956 /*
957 * If the lock was not acquired (due to timeout or signal) then the
958 * rt_waiter is removed before futex_q is. If this is observed by
959 * an unlocker after dropping the rtmutex wait lock and before
960 * acquiring the hash bucket lock, then the unlocker dequeues the
961 * futex_q from the hash bucket list to guarantee consistent state
962 * vs. userspace. Therefore the dequeue here must be conditional.
963 */
964 if (!plist_node_empty(&q->list))
965 __futex_unqueue(q);
966
967 BUG_ON(!q->pi_state);
968 put_pi_state(q->pi_state);
969 q->pi_state = NULL;
970}
971
972/* Constants for the pending_op argument of handle_futex_death */
973#define HANDLE_DEATH_PENDING true
974#define HANDLE_DEATH_LIST false
975
976/*
977 * Process a futex-list entry, check whether it's owned by the
978 * dying task, and do notification if so:
979 */
980static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
981 unsigned int mod, bool pending_op)
982{
983 bool pi = !!(mod & FUTEX_ROBUST_MOD_PI);
984 u32 uval, nval, mval;
985 pid_t owner;
986 int err;
987
988 /* Futex address must be 32bit aligned */
989 if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)
990 return -1;
991
992retry:
993 if (get_user(uval, uaddr))
994 return -1;
995
996 /*
997 * Special case for regular (non PI) futexes. Ordinarily, we do
998 * not perform any processing here unless the current thread was
999 * the owner of the futex (by the TID check below).
1000 *
1001 * However, the unlock path has three race scenarios:
1002 *
1003 * 1. The unlock path releases the user space futex value and
1004 * before it can execute the futex() syscall to wake up
1005 * waiters it is killed.
1006 *
1007 * 2. A woken up waiter is killed before it can acquire the
1008 * futex in user space.
1009 *
1010 * 3. A woken up waiter is killed in user space after another
1011 * thread has acquired the futex, but before it can set
1012 * FUTEX_WAITERS.
1013 *
1014 * Note that, if userspace uses the FUTEX_ROBUST_UNLOCK flag, we
1015 * will not see case 1 here.
1016 *
1017 * In the second and third case, the wake up notification could
1018 * be generated from any of:
1019 *
1020 * i. An ordinary futex wakeup after unlock (with or
1021 * without FUTEX_ROBUST_UNLOCK)
1022 * ii. A robust wakeup from another thread's death
1023 * iii. A previous round through this special case
1024 *
1025 * As a result, the futex world will be in one of four states:
1026 *
1027 * A. The futex word is 0 (unlocked)
1028 * B. The futex word is owned by another thread
1029 * (FUTEX_WAITERS is not set)
1030 * C. The futex word is owned by another thread
1031 * (FUTEX_WAITERS set)
1032 * D. The futex's owner died and OWNER_DIED is set
1033 * (the owner part of the word is 0)
1034 *
1035 * The key issue is that the kernel usually (at least from
1036 * sources ii. and iii. or when so requested by userspace from
1037 * source i.) only ever wakes *one* waiter at a time. If this
1038 * waiter dies before acquiring the futex (or setting the
1039 * FUTEX_WAITERS bit), the kernel *must* still wake the next
1040 * waiter down the line to uphold the futex invariants and
1041 * avoid lost wakeups. Note we do not need to handle state C,
1042 * as it does not matter to us whether *we* successfully set
1043 * the bit or a third thread did so in the meantime.
1044 *
1045 * Therefore, in these cases we must issue an additional
1046 * futex_wake(). Note however that we *must not* set OWNER_DIED
1047 * here. Our thread is *not* the owner of the futex.
1048 *
1049 * Thus to summarize, the conditions for needing the additional
1050 * futex_wake() are:
1051 *
1052 * 1) @pending_op == true (the thread has not finished the
1053 * mutex operation)
1054 * 2) The futex word is in one of the states A, B or D
1055 * 3) Regular futex: @pi == false
1056 *
1057 * Note in particular that in all of the states A-D the owner
1058 * portion of the futex word differs from our thread's TID
1059 * (unless the actual owner has the same TID in another PID
1060 * namespace, but we cannot currently distinguish that
1061 * scenario), so this can be a special-case wakeup in the bail
1062 * path of the ordinary TID check.
1063 */
1064 owner = uval & FUTEX_TID_MASK;
1065
1066 if (owner != task_pid_vnr(curr)) {
1067 if (pending_op && !pi && (!owner || !(uval & FUTEX_WAITERS))) {
1068 futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
1069 FUTEX_BITSET_MATCH_ANY);
1070 }
1071 return 0;
1072 }
1073
1074 /*
1075 * Ok, this dying thread is truly holding a futex
1076 * of interest. Set the OWNER_DIED bit atomically
1077 * via cmpxchg, and if the value had FUTEX_WAITERS
1078 * set, wake up a waiter (if any). (We have to do a
1079 * futex_wake() even if OWNER_DIED is already set -
1080 * to handle the rare but possible case of recursive
1081 * thread-death.) The rest of the cleanup is done in
1082 * userspace.
1083 */
1084 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
1085
1086 /*
1087 * We are not holding a lock here, but we want to have
1088 * the pagefault_disable/enable() protection because
1089 * we want to handle the fault gracefully. If the
1090 * access fails we try to fault in the futex with R/W
1091 * verification via get_user_pages. get_user() above
1092 * does not guarantee R/W access. If that fails we
1093 * give up and leave the futex locked.
1094 */
1095 if ((err = futex_cmpxchg_value_locked(&nval, uaddr, uval, mval))) {
1096 switch (err) {
1097 case -EFAULT:
1098 if (fault_in_user_writeable(uaddr))
1099 return -1;
1100 goto retry;
1101
1102 case -EAGAIN:
1103 cond_resched();
1104 goto retry;
1105
1106 default:
1107 WARN_ON_ONCE(1);
1108 return err;
1109 }
1110 }
1111
1112 if (nval != uval)
1113 goto retry;
1114
1115 /*
1116 * Wake robust non-PI futexes here. The wakeup of
1117 * PI futexes happens in exit_pi_state():
1118 */
1119 if (!pi && (uval & FUTEX_WAITERS)) {
1120 futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
1121 FUTEX_BITSET_MATCH_ANY);
1122 }
1123
1124 return 0;
1125}
1126
1127/*
1128 * Fetch a robust-list pointer. Bit 0 signals PI futexes:
1129 */
1130static inline int fetch_robust_entry(struct robust_list __user **entry,
1131 struct robust_list __user * __user *head,
1132 unsigned int *mod)
1133{
1134 unsigned long uentry;
1135
1136 if (get_user(uentry, (unsigned long __user *)head))
1137 return -EFAULT;
1138
1139 *entry = (void __user *)(uentry & ~FUTEX_ROBUST_MOD_MASK);
1140 *mod = uentry & FUTEX_ROBUST_MOD_MASK;
1141
1142 return 0;
1143}
1144
1145/*
1146 * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
1147 * and mark any locks found there dead, and notify any waiters.
1148 *
1149 * We silently return on any sign of list-walking problem.
1150 */
1151static void exit_robust_list(struct task_struct *curr)
1152{
1153 struct robust_list_head __user *head = curr->futex.robust_list;
1154 unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod;
1155 struct robust_list __user *entry, *next_entry, *pending;
1156 unsigned long futex_offset;
1157 int rc;
1158
1159 /*
1160 * Fetch the list head (which was registered earlier, via
1161 * sys_set_robust_list()):
1162 */
1163 if (fetch_robust_entry(&entry, &head->list.next, &cur_mod))
1164 return;
1165 /*
1166 * Fetch the relative futex offset:
1167 */
1168 if (get_user(futex_offset, &head->futex_offset))
1169 return;
1170 /*
1171 * Fetch any possibly pending lock-add first, and handle it
1172 * if it exists:
1173 */
1174 if (fetch_robust_entry(&pending, &head->list_op_pending, &pend_mod))
1175 return;
1176
1177 next_entry = NULL; /* avoid warning with gcc */
1178 while (entry != &head->list) {
1179 /*
1180 * Fetch the next entry in the list before calling
1181 * handle_futex_death:
1182 */
1183 rc = fetch_robust_entry(&next_entry, &entry->next, &next_mod);
1184 /*
1185 * A pending lock might already be on the list, so
1186 * don't process it twice:
1187 */
1188 if (entry != pending) {
1189 if (handle_futex_death((void __user *)entry + futex_offset,
1190 curr, cur_mod, HANDLE_DEATH_LIST))
1191 return;
1192 }
1193 if (rc)
1194 return;
1195 entry = next_entry;
1196 cur_mod = next_mod;
1197 /*
1198 * Avoid excessively long or circular lists:
1199 */
1200 if (!--limit)
1201 break;
1202
1203 cond_resched();
1204 }
1205
1206 if (pending) {
1207 handle_futex_death((void __user *)pending + futex_offset,
1208 curr, pend_mod, HANDLE_DEATH_PENDING);
1209 }
1210}
1211
1212static bool robust_list_clear_pending(unsigned long __user *pop)
1213{
1214 struct robust_list_head __user *head = current->futex.robust_list;
1215
1216 if (!put_user(0UL, pop))
1217 return true;
1218
1219 /*
1220 * Just give up. The robust list head is usually part of TLS, so the
1221 * chance that this gets resolved is close to zero.
1222 *
1223 * If @pop_addr is the robust_list_head::list_op_pending pointer then
1224 * clear the robust list head pointer to prevent further damage when the
1225 * task exits. Better a few stale futexes than corrupted memory. But
1226 * that's mostly an academic exercise.
1227 */
1228 if (pop == (unsigned long __user *)&head->list_op_pending)
1229 current->futex.robust_list = NULL;
1230 return false;
1231}
1232
1233#ifdef CONFIG_COMPAT
1234static void __user *futex_uaddr(struct robust_list __user *entry,
1235 compat_long_t futex_offset)
1236{
1237 compat_uptr_t base = ptr_to_compat(entry);
1238 void __user *uaddr = compat_ptr(base + futex_offset);
1239
1240 return uaddr;
1241}
1242
1243/*
1244 * Fetch a robust-list pointer. Bit 0 signals PI futexes:
1245 */
1246static inline int
1247compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry,
1248 compat_uptr_t __user *head, unsigned int *pflags)
1249{
1250 if (get_user(*uentry, head))
1251 return -EFAULT;
1252
1253 *entry = compat_ptr((*uentry) & ~FUTEX_ROBUST_MOD_MASK);
1254 *pflags = (unsigned int)(*uentry) & FUTEX_ROBUST_MOD_MASK;
1255
1256 return 0;
1257}
1258
1259/*
1260 * Walk curr->futex.robust_list (very carefully, it's a userspace list!)
1261 * and mark any locks found there dead, and notify any waiters.
1262 *
1263 * We silently return on any sign of list-walking problem.
1264 */
1265static void compat_exit_robust_list(struct task_struct *curr)
1266{
1267 struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
1268 unsigned int limit = ROBUST_LIST_LIMIT, cur_mod, next_mod, pend_mod;
1269 struct robust_list __user *entry, *next_entry, *pending;
1270 compat_uptr_t uentry, next_uentry, upending;
1271 compat_long_t futex_offset;
1272 int rc;
1273
1274 /*
1275 * Fetch the list head (which was registered earlier, via
1276 * sys_set_robust_list()):
1277 */
1278 if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &cur_mod))
1279 return;
1280 /*
1281 * Fetch the relative futex offset:
1282 */
1283 if (get_user(futex_offset, &head->futex_offset))
1284 return;
1285 /*
1286 * Fetch any possibly pending lock-add first, and handle it
1287 * if it exists:
1288 */
1289 if (compat_fetch_robust_entry(&upending, &pending, &head->list_op_pending, &pend_mod))
1290 return;
1291
1292 next_entry = NULL; /* avoid warning with gcc */
1293 while (entry != (struct robust_list __user *) &head->list) {
1294 /*
1295 * Fetch the next entry in the list before calling
1296 * handle_futex_death:
1297 */
1298 rc = compat_fetch_robust_entry(&next_uentry, &next_entry,
1299 (compat_uptr_t __user *)&entry->next, &next_mod);
1300 /*
1301 * A pending lock might already be on the list, so
1302 * dont process it twice:
1303 */
1304 if (entry != pending) {
1305 void __user *uaddr = futex_uaddr(entry, futex_offset);
1306
1307 if (handle_futex_death(uaddr, curr, cur_mod, HANDLE_DEATH_LIST))
1308 return;
1309 }
1310 if (rc)
1311 return;
1312 uentry = next_uentry;
1313 entry = next_entry;
1314 cur_mod = next_mod;
1315 /*
1316 * Avoid excessively long or circular lists:
1317 */
1318 if (!--limit)
1319 break;
1320
1321 cond_resched();
1322 }
1323 if (pending) {
1324 void __user *uaddr = futex_uaddr(pending, futex_offset);
1325
1326 handle_futex_death(uaddr, curr, pend_mod, HANDLE_DEATH_PENDING);
1327 }
1328}
1329
1330static bool compat_robust_list_clear_pending(u32 __user *pop)
1331{
1332 struct compat_robust_list_head __user *head = current->futex.compat_robust_list;
1333
1334 if (!put_user(0U, pop))
1335 return true;
1336
1337 /* See comment in robust_list_clear_pending(). */
1338 if (pop == &head->list_op_pending)
1339 current->futex.compat_robust_list = NULL;
1340 return false;
1341}
1342#else
1343static bool compat_robust_list_clear_pending(u32 __user *pop_addr) { return false; }
1344#endif
1345
1346#ifdef CONFIG_FUTEX_PI
1347
1348/*
1349 * This task is holding PI mutexes at exit time => bad.
1350 * Kernel cleans up PI-state, but userspace is likely hosed.
1351 * (Robust-futex cleanup is separate and might save the day for userspace.)
1352 */
1353static void exit_pi_state_list(struct task_struct *curr)
1354{
1355 struct list_head *next, *head = &curr->futex.pi_state_list;
1356 struct futex_pi_state *pi_state;
1357 union futex_key key = FUTEX_KEY_INIT;
1358
1359 /*
1360 * The mutex mm_struct::futex_hash_lock might be acquired.
1361 */
1362 might_sleep();
1363 /*
1364 * Ensure the hash remains stable (no resize) during the while loop
1365 * below. The hb pointer is acquired under the pi_lock so we can't block
1366 * on the mutex.
1367 */
1368 WARN_ON(curr != current);
1369 guard(private_hash)(current->mm);
1370 /*
1371 * We are a ZOMBIE and nobody can enqueue itself on
1372 * pi_state_list anymore, but we have to be careful
1373 * versus waiters unqueueing themselves:
1374 */
1375 raw_spin_lock_irq(&curr->pi_lock);
1376 while (!list_empty(head)) {
1377 next = head->next;
1378 pi_state = list_entry(next, struct futex_pi_state, list);
1379 key = pi_state->key;
1380 if (1) {
1381 CLASS(hbr, hbr)(&key);
1382 auto hb = hbr.hb;
1383
1384 /*
1385 * We can race against put_pi_state() removing itself from the
1386 * list (a waiter going away). put_pi_state() will first
1387 * decrement the reference count and then modify the list, so
1388 * its possible to see the list entry but fail this reference
1389 * acquire.
1390 *
1391 * In that case; drop the locks to let put_pi_state() make
1392 * progress and retry the loop.
1393 */
1394 if (!refcount_inc_not_zero(&pi_state->refcount)) {
1395 raw_spin_unlock_irq(&curr->pi_lock);
1396 cpu_relax();
1397 raw_spin_lock_irq(&curr->pi_lock);
1398 continue;
1399 }
1400 raw_spin_unlock_irq(&curr->pi_lock);
1401
1402 spin_lock(&hb->lock);
1403 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1404 raw_spin_lock(&curr->pi_lock);
1405 /*
1406 * We dropped the pi-lock, so re-check whether this
1407 * task still owns the PI-state:
1408 */
1409 if (head->next != next) {
1410 /* retain curr->pi_lock for the loop invariant */
1411 raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
1412 spin_unlock(&hb->lock);
1413 put_pi_state(pi_state);
1414 continue;
1415 }
1416
1417 WARN_ON(pi_state->owner != curr);
1418 WARN_ON(list_empty(&pi_state->list));
1419 list_del_init(&pi_state->list);
1420 pi_state->owner = NULL;
1421
1422 raw_spin_unlock(&curr->pi_lock);
1423 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1424 spin_unlock(&hb->lock);
1425 }
1426
1427 rt_mutex_futex_unlock(&pi_state->pi_mutex);
1428 put_pi_state(pi_state);
1429
1430 raw_spin_lock_irq(&curr->pi_lock);
1431 }
1432 raw_spin_unlock_irq(&curr->pi_lock);
1433}
1434#else
1435static inline void exit_pi_state_list(struct task_struct *curr) { }
1436#endif
1437
1438bool futex_robust_list_clear_pending(void __user *pop, unsigned int flags)
1439{
1440 bool size32bit = !!(flags & FLAGS_ROBUST_LIST32);
1441
1442 if (!IS_ENABLED(CONFIG_64BIT) && !size32bit)
1443 return false;
1444
1445 if (IS_ENABLED(CONFIG_64BIT) && size32bit)
1446 return compat_robust_list_clear_pending(pop);
1447
1448 return robust_list_clear_pending(pop);
1449}
1450
1451#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
1452void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr)
1453{
1454 /*
1455 * arch_futex_robust_unlock_get_pop() returns the list pending op pointer from
1456 * @regs if the try_cmpxchg() succeeded.
1457 */
1458 void __user *pop = arch_futex_robust_unlock_get_pop(regs);
1459
1460 if (!pop)
1461 return;
1462
1463 futex_robust_list_clear_pending(pop, csr->pop_size32 ? FLAGS_ROBUST_LIST32 : 0);
1464}
1465#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */
1466
1467static void futex_cleanup(struct task_struct *tsk)
1468{
1469 if (unlikely(tsk->futex.robust_list)) {
1470 exit_robust_list(tsk);
1471 tsk->futex.robust_list = NULL;
1472 }
1473
1474#ifdef CONFIG_COMPAT
1475 if (unlikely(tsk->futex.compat_robust_list)) {
1476 compat_exit_robust_list(tsk);
1477 tsk->futex.compat_robust_list = NULL;
1478 }
1479#endif
1480
1481 if (unlikely(!list_empty(&tsk->futex.pi_state_list)))
1482 exit_pi_state_list(tsk);
1483}
1484
1485/**
1486 * futex_exit_recursive - Set the tasks futex state to FUTEX_STATE_DEAD
1487 * @tsk: task to set the state on
1488 *
1489 * Set the futex exit state of the task lockless. The futex waiter code
1490 * observes that state when a task is exiting and loops until the task has
1491 * actually finished the futex cleanup. The worst case for this is that the
1492 * waiter runs through the wait loop until the state becomes visible.
1493 *
1494 * This is called from the recursive fault handling path in make_task_dead().
1495 *
1496 * This is best effort. Either the futex exit code has run already or
1497 * not. If the OWNER_DIED bit has been set on the futex then the waiter can
1498 * take it over. If not, the problem is pushed back to user space. If the
1499 * futex exit code did not run yet, then an already queued waiter might
1500 * block forever, but there is nothing which can be done about that.
1501 */
1502void futex_exit_recursive(struct task_struct *tsk)
1503{
1504 /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */
1505 if (tsk->futex.state == FUTEX_STATE_EXITING) {
1506 __assume_ctx_lock(&tsk->futex.exit_mutex);
1507 mutex_unlock(&tsk->futex.exit_mutex);
1508 }
1509 tsk->futex.state = FUTEX_STATE_DEAD;
1510}
1511
1512static void futex_cleanup_begin(struct task_struct *tsk)
1513 __acquires(&tsk->futex.exit_mutex)
1514{
1515 /*
1516 * Prevent various race issues against a concurrent incoming waiter
1517 * including live locks by forcing the waiter to block on
1518 * tsk->futex.exit_mutex when it observes FUTEX_STATE_EXITING in
1519 * attach_to_pi_owner().
1520 */
1521 mutex_lock(&tsk->futex.exit_mutex);
1522
1523 /*
1524 * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock.
1525 *
1526 * This ensures that all subsequent checks of tsk->futex_state in
1527 * attach_to_pi_owner() must observe FUTEX_STATE_EXITING with
1528 * tsk->pi_lock held.
1529 *
1530 * It guarantees also that a pi_state which was queued right before
1531 * the state change under tsk->pi_lock by a concurrent waiter must
1532 * be observed in exit_pi_state_list().
1533 */
1534 raw_spin_lock_irq(&tsk->pi_lock);
1535 tsk->futex.state = FUTEX_STATE_EXITING;
1536 raw_spin_unlock_irq(&tsk->pi_lock);
1537}
1538
1539static void futex_cleanup_end(struct task_struct *tsk)
1540 __releases(&tsk->futex.exit_mutex)
1541{
1542 scoped_guard(raw_spinlock_irq, &tsk->pi_lock)
1543 tsk->futex.state = FUTEX_STATE_DEAD;
1544
1545 /*
1546 * Drop the exit protection. This unblocks waiters which observed
1547 * FUTEX_STATE_EXITING to reevaluate the state.
1548 */
1549 mutex_unlock(&tsk->futex.exit_mutex);
1550}
1551
1552/*
1553 * Invoked from mm_exit_exec_release() to cleanup the robust lists and pi state
1554 * of the outgoing task.
1555 *
1556 * exec() makes it interesting for futexes because the TID of the task stays the
1557 * same, but from a futex perspective the task has to be treated like an exiting
1558 * task. This is especially important for the sanity check for private futexes
1559 * in attach_to_pi_owner() which compares the owner's mm with the waiter's mm.
1560 *
1561 * That check would give the wrong answer if futex_cleanup_end() would
1562 * set the state to FUTEX_STATE_OK as long as the task still has the old
1563 * mm.
1564 *
1565 * After the task has switched to the new mm it sets it to
1566 * FUTEX_STATE_OK again in futex_exec_done().
1567 */
1568void futex_exit_exec_release(struct task_struct *tsk)
1569{
1570 futex_cleanup_begin(tsk);
1571 futex_cleanup(tsk);
1572 futex_cleanup_end(tsk);
1573}
1574
1575/*
1576 * exec() has switched to the new mm. Futex operations are safe again.
1577 */
1578void futex_exec_done(struct task_struct *tsk)
1579{
1580 /*
1581 * This store does not have to take tsk::futex::exit_mutex because the
1582 * phase where waiters block on it during state FUTEX_STATE_EXITING has
1583 * been finished when futex_cleanup_end() set the state to
1584 * FUTEX_STATE_DEAD.
1585 *
1586 * This transitions back from FUTEX_STATE_DEAD to FUTEX_STATE_OK. The
1587 * ordering guarantee required here is that the previous store to
1588 * tsk::mm in the calling code cannot be reordered against this store.
1589 */
1590 guard(raw_spinlock_irq)(&tsk->pi_lock);
1591 tsk->futex.state = FUTEX_STATE_OK;
1592}
1593
1594static void futex_hash_bucket_init(struct futex_hash_bucket *fhb)
1595{
1596 atomic_set(&fhb->waiters, 0);
1597 plist_head_init(&fhb->chain);
1598 spin_lock_init(&fhb->lock);
1599}
1600
1601#define FH_CUSTOM 0x01
1602
1603#ifdef CONFIG_FUTEX_PRIVATE_HASH
1604
1605/*
1606 * futex-ref
1607 *
1608 * Heavily inspired by percpu-rwsem/percpu-refcount; not reusing any of that
1609 * code because it just doesn't fit right.
1610 *
1611 * Dual counter, per-cpu / atomic approach like percpu-refcount, except it
1612 * re-initializes the state automatically, such that the fph swizzle is also a
1613 * transition back to per-cpu.
1614 */
1615
1616static void futex_ref_rcu(struct rcu_head *head);
1617
1618static void __futex_ref_atomic_begin(struct futex_private_hash *fph)
1619{
1620 struct mm_struct *mm = fph->mm;
1621
1622 /*
1623 * The counter we're about to switch to must have fully switched;
1624 * otherwise it would be impossible for it to have reported success
1625 * from futex_ref_is_dead().
1626 */
1627 WARN_ON_ONCE(atomic_long_read(&mm->futex.phash.atomic) != 0);
1628
1629 /*
1630 * Set the atomic to the bias value such that futex_ref_{get,put}()
1631 * will never observe 0. Will be fixed up in __futex_ref_atomic_end()
1632 * when folding in the percpu count.
1633 */
1634 atomic_long_set(&mm->futex.phash.atomic, LONG_MAX);
1635 smp_store_release(&fph->state, FR_ATOMIC);
1636
1637 call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
1638}
1639
1640static void __futex_ref_atomic_end(struct futex_private_hash *fph)
1641{
1642 struct mm_struct *mm = fph->mm;
1643 unsigned int count = 0;
1644 long ret;
1645 int cpu;
1646
1647 /*
1648 * Per __futex_ref_atomic_begin() the state of the fph must be ATOMIC
1649 * and per this RCU callback, everybody must now observe this state and
1650 * use the atomic variable.
1651 */
1652 WARN_ON_ONCE(fph->state != FR_ATOMIC);
1653
1654 /*
1655 * Therefore the per-cpu counter is now stable, sum and reset.
1656 */
1657 for_each_possible_cpu(cpu) {
1658 unsigned int *ptr = per_cpu_ptr(mm->futex.phash.ref, cpu);
1659 count += *ptr;
1660 *ptr = 0;
1661 }
1662
1663 /*
1664 * Re-init for the next cycle.
1665 */
1666 this_cpu_inc(*mm->futex.phash.ref); /* 0 -> 1 */
1667
1668 /*
1669 * Add actual count, subtract bias and initial refcount.
1670 *
1671 * The moment this atomic operation happens, futex_ref_is_dead() can
1672 * become true.
1673 */
1674 ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash.atomic);
1675 if (!ret)
1676 wake_up_var(mm);
1677
1678 WARN_ON_ONCE(ret < 0);
1679 mmput_async(mm);
1680}
1681
1682static void futex_ref_rcu(struct rcu_head *head)
1683{
1684 struct mm_struct *mm = container_of(head, struct mm_struct, futex.phash.rcu);
1685 struct futex_private_hash *fph = rcu_dereference_raw(mm->futex.phash.hash);
1686
1687 if (fph->state == FR_PERCPU) {
1688 /*
1689 * Per this extra grace-period, everybody must now observe
1690 * fph as the current fph and no previously observed fph's
1691 * are in-flight.
1692 *
1693 * Notably, nobody will now rely on the atomic
1694 * futex_ref_is_dead() state anymore so we can begin the
1695 * migration of the per-cpu counter into the atomic.
1696 */
1697 __futex_ref_atomic_begin(fph);
1698 return;
1699 }
1700
1701 __futex_ref_atomic_end(fph);
1702}
1703
1704/*
1705 * Drop the initial refcount and transition to atomics.
1706 */
1707static void futex_ref_drop(struct futex_private_hash *fph)
1708{
1709 struct mm_struct *mm = fph->mm;
1710
1711 /*
1712 * Can only transition the current fph;
1713 */
1714 WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash.hash) != fph);
1715 /*
1716 * We enqueue at least one RCU callback. Ensure mm stays if the task
1717 * exits before the transition is completed.
1718 */
1719 mmget(mm);
1720
1721 /*
1722 * In order to avoid the following scenario:
1723 *
1724 * futex_hash() __futex_pivot_hash()
1725 * guard(rcu); guard(mm->futex.phash.lock);
1726 * fph = mm->futex.phash.hash;
1727 * rcu_assign_pointer(&mm->futex.phash.hash, new);
1728 * futex_hash_allocate()
1729 * futex_ref_drop()
1730 * fph->state = FR_ATOMIC;
1731 * atomic_set(, BIAS);
1732 *
1733 * futex_private_hash_get(fph); // OOPS
1734 *
1735 * Where an old fph (which is FR_ATOMIC) and should fail on
1736 * inc_not_zero, will succeed because a new transition is started and
1737 * the atomic is bias'ed away from 0.
1738 *
1739 * There must be at least one full grace-period between publishing a
1740 * new fph and trying to replace it.
1741 */
1742 if (poll_state_synchronize_rcu(mm->futex.phash.batches)) {
1743 /*
1744 * There was a grace-period, we can begin now.
1745 */
1746 __futex_ref_atomic_begin(fph);
1747 return;
1748 }
1749
1750 call_rcu_hurry(&mm->futex.phash.rcu, futex_ref_rcu);
1751}
1752
1753static bool futex_ref_get(struct futex_private_hash *fph)
1754{
1755 struct mm_struct *mm = fph->mm;
1756
1757 guard(preempt)();
1758
1759 if (READ_ONCE(fph->state) == FR_PERCPU) {
1760 __this_cpu_inc(*mm->futex.phash.ref);
1761 return true;
1762 }
1763
1764 return atomic_long_inc_not_zero(&mm->futex.phash.atomic);
1765}
1766
1767static bool futex_ref_put(struct futex_private_hash *fph)
1768{
1769 struct mm_struct *mm = fph->mm;
1770
1771 guard(preempt)();
1772
1773 if (READ_ONCE(fph->state) == FR_PERCPU) {
1774 __this_cpu_dec(*mm->futex.phash.ref);
1775 return false;
1776 }
1777
1778 return atomic_long_dec_and_test(&mm->futex.phash.atomic);
1779}
1780
1781static bool futex_ref_is_dead(struct futex_private_hash *fph)
1782{
1783 struct mm_struct *mm = fph->mm;
1784
1785 guard(rcu)();
1786
1787 if (smp_load_acquire(&fph->state) == FR_PERCPU)
1788 return false;
1789
1790 return atomic_long_read(&mm->futex.phash.atomic) == 0;
1791}
1792
1793static void futex_hash_init_mm(struct futex_mm_data *fd)
1794{
1795 memset(&fd->phash, 0, sizeof(fd->phash));
1796 mutex_init(&fd->phash.lock);
1797 fd->phash.batches = get_state_synchronize_rcu();
1798}
1799
1800void futex_hash_free(struct mm_struct *mm)
1801{
1802 struct futex_private_hash *fph;
1803
1804 free_percpu(mm->futex.phash.ref);
1805 kvfree(mm->futex.phash.hash_new);
1806 fph = rcu_dereference_raw(mm->futex.phash.hash);
1807 kvfree(fph);
1808}
1809
1810static bool futex_pivot_pending(struct mm_struct *mm)
1811{
1812 struct futex_mm_phash *mmph = &mm->futex.phash;
1813 struct futex_private_hash *fph;
1814
1815 guard(mutex)(&mmph->lock);
1816
1817 if (!mmph->hash_new)
1818 return true;
1819
1820 fph = rcu_dereference_raw(mmph->hash);
1821 return futex_ref_is_dead(fph);
1822}
1823
1824static bool futex_hash_less(struct futex_private_hash *a,
1825 struct futex_private_hash *b)
1826{
1827 /* user provided always wins */
1828 if (!a->custom && b->custom)
1829 return true;
1830 if (a->custom && !b->custom)
1831 return false;
1832
1833 /* zero-sized hash wins */
1834 if (!b->hash_mask)
1835 return true;
1836 if (!a->hash_mask)
1837 return false;
1838
1839 /* keep the biggest */
1840 if (a->hash_mask < b->hash_mask)
1841 return true;
1842 if (a->hash_mask > b->hash_mask)
1843 return false;
1844
1845 return false; /* equal */
1846}
1847
1848static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags)
1849{
1850 struct mm_struct *mm = current->mm;
1851 struct futex_private_hash *fph;
1852 bool custom = flags & FH_CUSTOM;
1853 int i;
1854
1855 if (hash_slots && (hash_slots == 1 || !is_power_of_2(hash_slots)))
1856 return -EINVAL;
1857
1858 /*
1859 * Once we've disabled the global hash there is no way back.
1860 */
1861 scoped_guard(rcu) {
1862 fph = rcu_dereference(mm->futex.phash.hash);
1863 if (fph && !fph->hash_mask) {
1864 if (custom)
1865 return -EBUSY;
1866 return 0;
1867 }
1868 }
1869
1870 if (!mm->futex.phash.ref) {
1871 unsigned int __percpu *ref = alloc_percpu(unsigned int);
1872
1873 if (!ref)
1874 return -ENOMEM;
1875
1876 /*
1877 * Tasks sharing the mm can run this concurrently, so take the
1878 * initial reference before publishing the counter.
1879 */
1880 this_cpu_inc(*ref); /* 0 -> 1 */
1881 if (cmpxchg(&mm->futex.phash.ref, NULL, ref))
1882 free_percpu(ref);
1883 }
1884
1885 fph = kvzalloc_flex(*fph, queues, hash_slots,
1886 GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
1887 if (!fph)
1888 return -ENOMEM;
1889
1890 fph->hash_mask = hash_slots ? hash_slots - 1 : 0;
1891 fph->custom = custom;
1892 fph->mm = mm;
1893
1894 for (i = 0; i < hash_slots; i++)
1895 futex_hash_bucket_init(&fph->queues[i]);
1896
1897 if (custom) {
1898 struct wait_bit_queue_entry __wbq_entry;
1899 struct wait_queue_head *__wq_head;
1900
1901 /*
1902 * Only let prctl() wait / retry; don't unduly delay clone().
1903 */
1904again:
1905 __wq_head = __var_waitqueue(mm);
1906 init_wait_var_entry(&__wbq_entry, mm, 0);
1907 __wbq_entry.wq_entry.func = woken_wake_bit_function;
1908 add_wait_queue(__wq_head, &__wbq_entry.wq_entry);
1909
1910 /*
1911 * add_wait_queue() futex_ref_put()
1912 * MB (this) MB (implied)
1913 * futex_pivot_pending() wake_up_var()
1914 * waitqueue_active()
1915 *
1916 * Notably, it must not be possible to see
1917 * !futex_pivot_pending() && !waitqueue_active().
1918 */
1919 smp_mb();
1920
1921 while (!futex_pivot_pending(mm) &&
1922 wait_woken(&__wbq_entry.wq_entry, TASK_UNINTERRUPTIBLE,
1923 MAX_SCHEDULE_TIMEOUT))
1924 /* empty */;
1925
1926 remove_wait_queue(__wq_head, &__wbq_entry.wq_entry);
1927 }
1928
1929 scoped_guard(mutex, &mm->futex.phash.lock) {
1930 struct futex_private_hash *free __free(kvfree) = NULL;
1931 struct futex_private_hash *cur, *new;
1932
1933 cur = rcu_dereference_protected(mm->futex.phash.hash,
1934 lockdep_is_held(&mm->futex.phash.lock));
1935 new = mm->futex.phash.hash_new;
1936 mm->futex.phash.hash_new = NULL;
1937
1938 if (fph) {
1939 if (cur && !cur->hash_mask) {
1940 /*
1941 * If two threads simultaneously request the global
1942 * hash then the first one performs the switch,
1943 * the second one returns here.
1944 */
1945 free = fph;
1946 mm->futex.phash.hash_new = new;
1947 return -EBUSY;
1948 }
1949 if (cur && !new) {
1950 /*
1951 * If we have an existing hash, but do not yet have
1952 * allocated a replacement hash, drop the initial
1953 * reference on the existing hash.
1954 */
1955 futex_ref_drop(cur);
1956 }
1957
1958 if (new) {
1959 /*
1960 * Two updates raced; throw out the lesser one.
1961 */
1962 if (futex_hash_less(new, fph)) {
1963 free = new;
1964 new = fph;
1965 } else {
1966 free = fph;
1967 }
1968 } else {
1969 new = fph;
1970 }
1971 fph = NULL;
1972 }
1973
1974 if (new) {
1975 /*
1976 * Will set mm->futex.phash.new_hash on failure;
1977 * futex_private_hash_get() will try again.
1978 */
1979 if (!__futex_pivot_hash(mm, new) && custom)
1980 goto again;
1981 }
1982 }
1983 return 0;
1984}
1985
1986int futex_hash_allocate_default(void)
1987{
1988 unsigned int threads, buckets, current_buckets = 0;
1989 struct futex_private_hash *fph;
1990
1991 if (!current->mm)
1992 return 0;
1993
1994 scoped_guard(rcu) {
1995 threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus());
1996
1997 fph = rcu_dereference(current->mm->futex.phash.hash);
1998 if (fph) {
1999 if (fph->custom)
2000 return 0;
2001
2002 current_buckets = fph->hash_mask + 1;
2003 }
2004 }
2005
2006 /*
2007 * The default allocation will remain within
2008 * 16 <= threads * 4 <= global hash size
2009 */
2010 buckets = roundup_pow_of_two(4 * threads);
2011 buckets = clamp(buckets, 16, __futex_mask + 1);
2012
2013 if (current_buckets >= buckets)
2014 return 0;
2015
2016 return futex_hash_allocate(buckets, 0);
2017}
2018
2019static int futex_hash_get_slots(void)
2020{
2021 struct futex_private_hash *fph;
2022
2023 guard(rcu)();
2024 fph = rcu_dereference(current->mm->futex.phash.hash);
2025 if (fph && fph->hash_mask)
2026 return fph->hash_mask + 1;
2027 return 0;
2028}
2029#else /* CONFIG_FUTEX_PRIVATE_HASH */
2030static inline int futex_hash_allocate(unsigned int hslots, unsigned int flags) { return -EINVAL; }
2031static inline int futex_hash_get_slots(void) { return 0; }
2032static inline void futex_hash_init_mm(struct futex_mm_data *fd) { }
2033#endif /* !CONFIG_FUTEX_PRIVATE_HASH */
2034
2035#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
2036static void futex_invalidate_cs_ranges(struct futex_mm_data *fd)
2037{
2038 /*
2039 * Invalidate start_ip so that the quick check fails for ip >= start_ip
2040 * if VDSO is not mapped or the second slot is not available for compat
2041 * tasks as they use VDSO32 which does not provide the 64-bit pointer
2042 * variant.
2043 */
2044 for (int i = 0; i < FUTEX_ROBUST_MAX_CS_RANGES; i++)
2045 fd->unlock.cs_ranges[i].start_ip = ~0UL;
2046}
2047
2048void futex_reset_cs_ranges(struct futex_mm_data *fd)
2049{
2050 memset(fd->unlock.cs_ranges, 0, sizeof(fd->unlock.cs_ranges));
2051 futex_invalidate_cs_ranges(fd);
2052}
2053
2054static void futex_robust_unlock_init_mm(struct futex_mm_data *fd)
2055{
2056 /* mm_dup() preserves the range, mm_alloc() clears it */
2057 if (!fd->unlock.cs_ranges[0].start_ip)
2058 futex_invalidate_cs_ranges(fd);
2059}
2060#else /* CONFIG_FUTEX_ROBUST_UNLOCK */
2061static inline void futex_robust_unlock_init_mm(struct futex_mm_data *fd) { }
2062#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */
2063
2064void futex_mm_init(struct mm_struct *mm)
2065{
2066 atomic64_set(&mm->futex.unique_id, 0);
2067 futex_hash_init_mm(&mm->futex);
2068 futex_robust_unlock_init_mm(&mm->futex);
2069}
2070
2071int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
2072{
2073 unsigned int flags = FH_CUSTOM;
2074 int ret;
2075
2076 switch (arg2) {
2077 case PR_FUTEX_HASH_SET_SLOTS:
2078 if (arg4)
2079 return -EINVAL;
2080 ret = futex_hash_allocate(arg3, flags);
2081 break;
2082
2083 case PR_FUTEX_HASH_GET_SLOTS:
2084 ret = futex_hash_get_slots();
2085 break;
2086
2087 default:
2088 ret = -EINVAL;
2089 break;
2090 }
2091 return ret;
2092}
2093
2094static int __init futex_init(void)
2095{
2096 unsigned long hashsize, i;
2097 unsigned int order, n;
2098 unsigned long size;
2099
2100#ifdef CONFIG_BASE_SMALL
2101 hashsize = 16;
2102#else
2103 hashsize = 256 * num_possible_cpus();
2104 hashsize /= num_possible_nodes();
2105 hashsize = max(4, hashsize);
2106 hashsize = roundup_pow_of_two(hashsize);
2107#endif
2108 __futex_mask = hashsize - 1;
2109 __futex_shift = ilog2(hashsize);
2110 size = sizeof(struct futex_hash_bucket) * hashsize;
2111 order = get_order(size);
2112
2113 __futex_queues = kzalloc_objs(*__futex_queues, nr_node_ids);
2114 kmemleak_not_leak(__futex_queues);
2115
2116 runtime_const_init(shift, __futex_shift);
2117 runtime_const_init(mask, __futex_mask);
2118 runtime_const_init(ptr, __futex_queues);
2119
2120 barrier();
2121
2122 BUG_ON(!futex_queues());
2123
2124 for_each_node(n) {
2125 struct futex_hash_bucket *table;
2126
2127 if (order > MAX_PAGE_ORDER)
2128 table = vmalloc_huge_node(size, GFP_KERNEL, n);
2129 else
2130 table = alloc_pages_exact_nid(n, size, GFP_KERNEL);
2131
2132 BUG_ON(!table);
2133
2134 for (i = 0; i < hashsize; i++)
2135 futex_hash_bucket_init(&table[i]);
2136
2137 futex_queues()[n] = table;
2138 }
2139
2140 pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n",
2141 hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024,
2142 order > MAX_PAGE_ORDER ? "vmalloc" : "linear");
2143 return 0;
2144}
2145core_initcall(futex_init);