| 1 |
|
| 2 | #ifndef _PROTO_MEMORY_H
|
| 3 | #define _PROTO_MEMORY_H
|
| 4 |
|
| 5 | #include <net/sock.h>
|
| 6 | #include <net/hotdata.h>
|
| 7 |
|
| 8 |
|
| 9 | #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
|
| 10 |
|
| 11 | static inline bool sk_has_memory_pressure(const struct sock *sk)
|
| 12 | {
|
| 13 | return sk->sk_prot->memory_pressure != NULL;
|
| 14 | }
|
| 15 |
|
| 16 | static inline bool
|
| 17 | proto_memory_pressure(const struct proto *prot)
|
| 18 | {
|
| 19 | if (!prot->memory_pressure)
|
| 20 | return false;
|
| 21 | return !!READ_ONCE(*prot->memory_pressure);
|
| 22 | }
|
| 23 |
|
| 24 | static inline bool sk_under_global_memory_pressure(const struct sock *sk)
|
| 25 | {
|
| 26 | return proto_memory_pressure(sk->sk_prot);
|
| 27 | }
|
| 28 |
|
| 29 | static inline bool sk_under_memory_pressure(const struct sock *sk)
|
| 30 | {
|
| 31 | if (!sk->sk_prot->memory_pressure)
|
| 32 | return false;
|
| 33 |
|
| 34 | if (mem_cgroup_sk_enabled(sk) &&
|
| 35 | mem_cgroup_sk_under_memory_pressure(sk))
|
| 36 | return true;
|
| 37 |
|
| 38 | if (sk->sk_bypass_prot_mem)
|
| 39 | return false;
|
| 40 |
|
| 41 | return !!READ_ONCE(*sk->sk_prot->memory_pressure);
|
| 42 | }
|
| 43 |
|
| 44 | static inline long
|
| 45 | proto_memory_allocated(const struct proto *prot)
|
| 46 | {
|
| 47 | return max(0L, atomic_long_read(prot->memory_allocated));
|
| 48 | }
|
| 49 |
|
| 50 | static inline long
|
| 51 | sk_memory_allocated(const struct sock *sk)
|
| 52 | {
|
| 53 | return proto_memory_allocated(sk->sk_prot);
|
| 54 | }
|
| 55 |
|
| 56 | static inline void proto_memory_pcpu_drain(struct proto *proto)
|
| 57 | {
|
| 58 | int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
|
| 59 |
|
| 60 | if (val)
|
| 61 | atomic_long_add(val, proto->memory_allocated);
|
| 62 | }
|
| 63 |
|
| 64 | static inline void
|
| 65 | sk_memory_allocated_add(const struct sock *sk, int val)
|
| 66 | {
|
| 67 | struct proto *proto = sk->sk_prot;
|
| 68 |
|
| 69 | val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
|
| 70 |
|
| 71 | if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
|
| 72 | proto_memory_pcpu_drain(proto);
|
| 73 | }
|
| 74 |
|
| 75 | static inline void
|
| 76 | sk_memory_allocated_sub(const struct sock *sk, int val)
|
| 77 | {
|
| 78 | struct proto *proto = sk->sk_prot;
|
| 79 |
|
| 80 | val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
|
| 81 |
|
| 82 | if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
|
| 83 | proto_memory_pcpu_drain(proto);
|
| 84 | }
|
| 85 |
|
| 86 | #endif
|