projects/linux·blob v4.16-rc4 · refs

277 lines · 8.0 KiB

1/*
2 * shadow.c - Shadow Variables
3 *
4 * Copyright (C) 2014 Josh Poimboeuf <[email protected]>
5 * Copyright (C) 2014 Seth Jennings <[email protected]>
6 * Copyright (C) 2017 Joe Lawrence <[email protected]>
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 2
11 * of the License, or (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
22/**
23 * DOC: Shadow variable API concurrency notes:
24 *
25 * The shadow variable API provides a simple relationship between an
26 * <obj, id> pair and a pointer value. It is the responsibility of the
27 * caller to provide any mutual exclusion required of the shadow data.
28 *
29 * Once a shadow variable is attached to its parent object via the
30 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
31 * call to klp_shadow_get() may then return the shadow variable's data
32 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
33 * accordingly.
34 *
35 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
36 * variable structures. Their implementation does not call kmalloc
37 * inside any spinlocks, but API callers should pass GFP flags according
38 * to their specific needs.
39 *
40 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
41 * concurrent klp_shadow_free() and klp_shadow_get() operations.
42 */
43
44#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
45
46#include <linux/hashtable.h>
47#include <linux/slab.h>
48#include <linux/livepatch.h>
49
50static DEFINE_HASHTABLE(klp_shadow_hash, 12);
51
52/*
53 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
54 * the shadow variables it references.
55 */
56static DEFINE_SPINLOCK(klp_shadow_lock);
57
58/**
59 * struct klp_shadow - shadow variable structure
60 * @node: klp_shadow_hash hash table node
61 * @rcu_head: RCU is used to safely free this structure
62 * @obj: pointer to parent object
63 * @id: data identifier
64 * @data: data area
65 */
66struct klp_shadow {
67 struct hlist_node node;
68 struct rcu_head rcu_head;
69 void *obj;
70 unsigned long id;
71 char data[];
72};
73
74/**
75 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
76 * @shadow: shadow variable to match
77 * @obj: pointer to parent object
78 * @id: data identifier
79 *
80 * Return: true if the shadow variable matches.
81 */
82static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
83 unsigned long id)
84{
85 return shadow->obj == obj && shadow->id == id;
86}
87
88/**
89 * klp_shadow_get() - retrieve a shadow variable data pointer
90 * @obj: pointer to parent object
91 * @id: data identifier
92 *
93 * Return: the shadow variable data element, NULL on failure.
94 */
95void *klp_shadow_get(void *obj, unsigned long id)
96{
97 struct klp_shadow *shadow;
98
99 rcu_read_lock();
100
101 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
102 (unsigned long)obj) {
103
104 if (klp_shadow_match(shadow, obj, id)) {
105 rcu_read_unlock();
106 return shadow->data;
107 }
108 }
109
110 rcu_read_unlock();
111
112 return NULL;
113}
114EXPORT_SYMBOL_GPL(klp_shadow_get);
115
116static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data,
117 size_t size, gfp_t gfp_flags, bool warn_on_exist)
118{
119 struct klp_shadow *new_shadow;
120 void *shadow_data;
121 unsigned long flags;
122
123 /* Check if the shadow variable already exists */
124 shadow_data = klp_shadow_get(obj, id);
125 if (shadow_data)
126 goto exists;
127
128 /* Allocate a new shadow variable for use inside the lock below */
129 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
130 if (!new_shadow)
131 return NULL;
132
133 new_shadow->obj = obj;
134 new_shadow->id = id;
135
136 /* Initialize the shadow variable if data provided */
137 if (data)
138 memcpy(new_shadow->data, data, size);
139
140 /* Look for <obj, id> again under the lock */
141 spin_lock_irqsave(&klp_shadow_lock, flags);
142 shadow_data = klp_shadow_get(obj, id);
143 if (unlikely(shadow_data)) {
144 /*
145 * Shadow variable was found, throw away speculative
146 * allocation.
147 */
148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow);
150 goto exists;
151 }
152
153 /* No <obj, id> found, so attach the newly allocated one */
154 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
155 (unsigned long)new_shadow->obj);
156 spin_unlock_irqrestore(&klp_shadow_lock, flags);
157
158 return new_shadow->data;
159
160exists:
161 if (warn_on_exist) {
162 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
163 return NULL;
164 }
165
166 return shadow_data;
167}
168
169/**
170 * klp_shadow_alloc() - allocate and add a new shadow variable
171 * @obj: pointer to parent object
172 * @id: data identifier
173 * @data: pointer to data to attach to parent
174 * @size: size of attached data
175 * @gfp_flags: GFP mask for allocation
176 *
177 * Allocates @size bytes for new shadow variable data using @gfp_flags
178 * and copies @size bytes from @data into the new shadow variable's own
179 * data space. If @data is NULL, @size bytes are still allocated, but
180 * no copy is performed. The new shadow variable is then added to the
181 * global hashtable.
182 *
183 * If an existing <obj, id> shadow variable can be found, this routine
184 * will issue a WARN, exit early and return NULL.
185 *
186 * Return: the shadow variable data element, NULL on duplicate or
187 * failure.
188 */
189void *klp_shadow_alloc(void *obj, unsigned long id, void *data,
190 size_t size, gfp_t gfp_flags)
191{
192 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, true);
193}
194EXPORT_SYMBOL_GPL(klp_shadow_alloc);
195
196/**
197 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
198 * @obj: pointer to parent object
199 * @id: data identifier
200 * @data: pointer to data to attach to parent
201 * @size: size of attached data
202 * @gfp_flags: GFP mask for allocation
203 *
204 * Returns a pointer to existing shadow data if an <obj, id> shadow
205 * variable is already present. Otherwise, it creates a new shadow
206 * variable like klp_shadow_alloc().
207 *
208 * This function guarantees that only one shadow variable exists with
209 * the given @id for the given @obj. It also guarantees that the shadow
210 * variable will be initialized by the given @data only when it did not
211 * exist before.
212 *
213 * Return: the shadow variable data element, NULL on failure.
214 */
215void *klp_shadow_get_or_alloc(void *obj, unsigned long id, void *data,
216 size_t size, gfp_t gfp_flags)
217{
218 return __klp_shadow_get_or_alloc(obj, id, data, size, gfp_flags, false);
219}
220EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
221
222/**
223 * klp_shadow_free() - detach and free a <obj, id> shadow variable
224 * @obj: pointer to parent object
225 * @id: data identifier
226 *
227 * This function releases the memory for this <obj, id> shadow variable
228 * instance, callers should stop referencing it accordingly.
229 */
230void klp_shadow_free(void *obj, unsigned long id)
231{
232 struct klp_shadow *shadow;
233 unsigned long flags;
234
235 spin_lock_irqsave(&klp_shadow_lock, flags);
236
237 /* Delete <obj, id> from hash */
238 hash_for_each_possible(klp_shadow_hash, shadow, node,
239 (unsigned long)obj) {
240
241 if (klp_shadow_match(shadow, obj, id)) {
242 hash_del_rcu(&shadow->node);
243 kfree_rcu(shadow, rcu_head);
244 break;
245 }
246 }
247
248 spin_unlock_irqrestore(&klp_shadow_lock, flags);
249}
250EXPORT_SYMBOL_GPL(klp_shadow_free);
251
252/**
253 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
254 * @id: data identifier
255 *
256 * This function releases the memory for all <*, id> shadow variable
257 * instances, callers should stop referencing them accordingly.
258 */
259void klp_shadow_free_all(unsigned long id)
260{
261 struct klp_shadow *shadow;
262 unsigned long flags;
263 int i;
264
265 spin_lock_irqsave(&klp_shadow_lock, flags);
266
267 /* Delete all <*, id> from hash */
268 hash_for_each(klp_shadow_hash, i, shadow, node) {
269 if (klp_shadow_match(shadow, shadow->obj, id)) {
270 hash_del_rcu(&shadow->node);
271 kfree_rcu(shadow, rcu_head);
272 }
273 }
274
275 spin_unlock_irqrestore(&klp_shadow_lock, flags);
276}
277EXPORT_SYMBOL_GPL(klp_shadow_free_all);