| 1 |
|
| 2 | |
| 3 |
|
| 4 |
|
| 5 | #ifndef __MMAP_UNLOCK_WORK_H__
|
| 6 | #define __MMAP_UNLOCK_WORK_H__
|
| 7 | #include <linux/atomic.h>
|
| 8 | #include <linux/err.h>
|
| 9 | #include <linux/irq_work.h>
|
| 10 |
|
| 11 |
|
| 12 | struct mmap_unlock_irq_work {
|
| 13 | struct irq_work irq_work;
|
| 14 | struct mm_struct *mm;
|
| 15 | atomic_t active;
|
| 16 | };
|
| 17 |
|
| 18 | DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
|
| 19 |
|
| 20 | |
| 21 | * We cannot do mmap_read_unlock() when the irq is disabled, because of
|
| 22 | * risk to deadlock with rq_lock. To look up vma when the irqs are
|
| 23 | * disabled, we need to run mmap_read_unlock() in irq_work. We use a
|
| 24 | * percpu variable to do the irq_work. The active flag reserves the slot
|
| 25 | * before mmap_read_trylock() and until the irq_work callback consumes mm.
|
| 26 |
|
| 27 | static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void)
|
| 28 | {
|
| 29 | struct mmap_unlock_irq_work *work;
|
| 30 |
|
| 31 | if (!irqs_disabled())
|
| 32 | return NULL;
|
| 33 |
|
| 34 | |
| 35 | * PREEMPT_RT does not allow to trylock mmap sem in interrupt
|
| 36 | * disabled context. Force the fallback code.
|
| 37 |
|
| 38 | if (IS_ENABLED(CONFIG_PREEMPT_RT))
|
| 39 | return ERR_PTR(-EBUSY);
|
| 40 |
|
| 41 | work = this_cpu_ptr(&mmap_unlock_work);
|
| 42 | if (irq_work_is_busy(&work->irq_work) ||
|
| 43 | atomic_cmpxchg_acquire(&work->active, 0, 1))
|
| 44 | return ERR_PTR(-EBUSY);
|
| 45 |
|
| 46 | return work;
|
| 47 | }
|
| 48 |
|
| 49 | static inline void
|
| 50 | bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work)
|
| 51 | {
|
| 52 | if (work)
|
| 53 | atomic_set_release(&work->active, 0);
|
| 54 | }
|
| 55 |
|
| 56 | static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
|
| 57 | {
|
| 58 | if (!work) {
|
| 59 | mmap_read_unlock(mm);
|
| 60 | } else {
|
| 61 | work->mm = mm;
|
| 62 |
|
| 63 | |
| 64 | * context. Tell lockdep that we've released it now so
|
| 65 | * it doesn't complain that we forgot to release it.
|
| 66 |
|
| 67 | rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_);
|
| 68 | irq_work_queue(&work->irq_work);
|
| 69 | }
|
| 70 | }
|
| 71 |
|
| 72 | #endif
|