projects/linux·blob master · refs

320 lines · 9.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_RISCV_RUNTIME_CONST_H
3#define _ASM_RISCV_RUNTIME_CONST_H
4
5#ifdef MODULE
6 #error "Cannot use runtime-const infrastructure from modules"
7#endif
8
9#include <asm/asm.h>
10#include <asm/alternative.h>
11#include <asm/cacheflush.h>
12#include <asm/insn-def.h>
13#include <linux/memory.h>
14#include <asm/text-patching.h>
15
16#include <linux/uaccess.h>
17
18#define RUNTIME_MAGIC __ASM_STR(0x89ABCDEF)
19
20#ifdef CONFIG_32BIT
21#define runtime_const_ptr(sym) \
22({ \
23 typeof(sym) __ret; \
24 asm_inline(".option push\n\t" \
25 ".option norvc\n\t" \
26 ".option norelax\n\t" \
27 "1:\t" \
28 "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \
29 "addi %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \
30 ".option pop\n\t" \
31 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
32 ".long 1b - .\n\t" \
33 ".popsection" \
34 : [__ret] "=r" (__ret)); \
35 __ret; \
36})
37#else
38/*
39 * Loading 64-bit constants into a register from immediates is a non-trivial
40 * task on riscv64. To get it somewhat performant, load 32 bits into two
41 * different registers and then combine the results.
42 *
43 * If the processor supports the Zbkb extension, we can combine the final
44 * "slli,slli,srli,add" into the single "pack" instruction. If the processor
45 * doesn't support Zbkb but does support the Zbb extension, we can
46 * combine the final "slli,srli,add" into one instruction "add.uw".
47 */
48#define RISCV_RUNTIME_CONST_64_PREAMBLE \
49 ".option push\n\t" \
50 ".option norvc\n\t" \
51 ".option norelax\n\t" \
52 "1:\t" \
53 "lui %[__ret], %%hi(" RUNTIME_MAGIC ")\n\t" \
54 "lui %[__tmp], %%hi(" RUNTIME_MAGIC ")\n\t" \
55 "addiw %[__ret],%[__ret], %%lo(" RUNTIME_MAGIC ")\n\t" \
56 "addiw %[__tmp],%[__tmp], %%lo(" RUNTIME_MAGIC ")\n\t" \
57
58#define RISCV_RUNTIME_CONST_64_BASE \
59 "slli %[__tmp],%[__tmp],32\n\t" \
60 "slli %[__ret],%[__ret],32\n\t" \
61 "srli %[__ret],%[__ret],32\n\t" \
62 "add %[__ret],%[__ret],%[__tmp]\n\t" \
63
64#define RISCV_RUNTIME_CONST_64_ZBA \
65 ".option push\n\t" \
66 ".option arch,+zba\n\t" \
67 ".option norvc\n\t" \
68 "slli %[__tmp],%[__tmp],32\n\t" \
69 "add.uw %[__ret],%[__ret],%[__tmp]\n\t" \
70 "nop\n\t" \
71 "nop\n\t" \
72 ".option pop\n\t" \
73
74#define RISCV_RUNTIME_CONST_64_ZBKB \
75 ".option push\n\t" \
76 ".option arch,+zbkb\n\t" \
77 ".option norvc\n\t" \
78 "pack %[__ret],%[__ret],%[__tmp]\n\t" \
79 "nop\n\t" \
80 "nop\n\t" \
81 "nop\n\t" \
82 ".option pop\n\t" \
83
84#define RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
85 ".option pop\n\t" \
86 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
87 ".long 1b - .\n\t" \
88 ".popsection" \
89
90#if defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA) \
91 && defined(CONFIG_RISCV_ISA_ZBKB)
92#define runtime_const_ptr(sym) \
93({ \
94 typeof(sym) __ret, __tmp; \
95 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
96 ALTERNATIVE_2( \
97 RISCV_RUNTIME_CONST_64_BASE, \
98 RISCV_RUNTIME_CONST_64_ZBA, \
99 0, RISCV_ISA_EXT_ZBA, 1, \
100 RISCV_RUNTIME_CONST_64_ZBKB, \
101 0, RISCV_ISA_EXT_ZBKB, 1 \
102 ) \
103 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
104 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
105 __ret; \
106})
107#elif defined(CONFIG_RISCV_ISA_ZBA) && defined(CONFIG_TOOLCHAIN_HAS_ZBA)
108#define runtime_const_ptr(sym) \
109({ \
110 typeof(sym) __ret, __tmp; \
111 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
112 ALTERNATIVE( \
113 RISCV_RUNTIME_CONST_64_BASE, \
114 RISCV_RUNTIME_CONST_64_ZBA, \
115 0, RISCV_ISA_EXT_ZBA, 1 \
116 ) \
117 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
118 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
119 __ret; \
120})
121#elif defined(CONFIG_RISCV_ISA_ZBKB)
122#define runtime_const_ptr(sym) \
123({ \
124 typeof(sym) __ret, __tmp; \
125 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
126 ALTERNATIVE( \
127 RISCV_RUNTIME_CONST_64_BASE, \
128 RISCV_RUNTIME_CONST_64_ZBKB, \
129 0, RISCV_ISA_EXT_ZBKB, 1 \
130 ) \
131 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
132 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
133 __ret; \
134})
135#else
136#define runtime_const_ptr(sym) \
137({ \
138 typeof(sym) __ret, __tmp; \
139 asm_inline(RISCV_RUNTIME_CONST_64_PREAMBLE \
140 RISCV_RUNTIME_CONST_64_BASE \
141 RISCV_RUNTIME_CONST_64_POSTAMBLE(sym) \
142 : [__ret] "=r" (__ret), [__tmp] "=r" (__tmp)); \
143 __ret; \
144})
145#endif
146#endif
147
148#define runtime_const_shift_right_32(val, sym) \
149({ \
150 u32 __ret; \
151 asm_inline(".option push\n\t" \
152 ".option norvc\n\t" \
153 "1:\t" \
154 SRLI " %[__ret],%[__val],12\n\t" \
155 ".option pop\n\t" \
156 ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \
157 ".long 1b - .\n\t" \
158 ".popsection" \
159 : [__ret] "=r" (__ret) \
160 : [__val] "r" (val)); \
161 __ret; \
162})
163
164#define runtime_const_mask_32(val, sym) \
165({ \
166 u32 __ret; \
167 asm_inline(".option push\n\t" \
168 ".option norvc\n\t" \
169 "1:\t" \
170 SLLI " %[__ret],%[__val],12\n\t" \
171 SRLI " %[__ret],%[__ret],12\n\t" \
172 ".option pop\n\t" \
173 ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \
174 ".long 1b - .\n\t" \
175 ".popsection" \
176 : [__ret] "=r" (__ret) \
177 : [__val] "r" (val)); \
178 __ret; \
179})
180
181#define runtime_const_init(type, sym) do { \
182 extern s32 __start_runtime_##type##_##sym[]; \
183 extern s32 __stop_runtime_##type##_##sym[]; \
184 \
185 runtime_const_fixup(__runtime_fixup_##type, \
186 (unsigned long)(sym), \
187 __start_runtime_##type##_##sym, \
188 __stop_runtime_##type##_##sym); \
189} while (0)
190
191static inline void __runtime_fixup_caches(void *where, unsigned int insns)
192{
193 /* On riscv there are currently only cache-wide flushes so va is ignored. */
194 __always_unused uintptr_t va = (uintptr_t)where;
195
196 flush_icache_range(va, va + 4 * insns);
197}
198
199/*
200 * The 32-bit immediate is stored in a lui+addi pairing.
201 * lui holds the upper 20 bits of the immediate in the first 20 bits of the instruction.
202 * addi holds the lower 12 bits of the immediate in the first 12 bits of the instruction.
203 */
204static inline void __runtime_fixup_32(__le16 *lui_parcel, __le16 *addi_parcel, unsigned int val)
205{
206 unsigned int lower_immediate, upper_immediate;
207 u32 lui_insn, addi_insn, addi_insn_mask;
208 __le32 lui_res, addi_res;
209
210 /* Mask out upper 12 bit of addi */
211 addi_insn_mask = 0x000fffff;
212
213 lui_insn = (u32)le16_to_cpu(lui_parcel[0]) | (u32)le16_to_cpu(lui_parcel[1]) << 16;
214 addi_insn = (u32)le16_to_cpu(addi_parcel[0]) | (u32)le16_to_cpu(addi_parcel[1]) << 16;
215
216 lower_immediate = sign_extend32(val, 11);
217 upper_immediate = (val - lower_immediate);
218
219 if (upper_immediate & 0xfffff000) {
220 /* replace upper 20 bits of lui with upper immediate */
221 lui_insn &= 0x00000fff;
222 lui_insn |= upper_immediate & 0xfffff000;
223 } else {
224 /* replace lui with nop if immediate is small enough to fit in addi */
225 lui_insn = RISCV_INSN_NOP4;
226 /*
227 * lui is being skipped, so do a load instead of an add. A load
228 * is performed by adding with the x0 register. Setting rs to
229 * zero with the following mask will accomplish this goal.
230 */
231 addi_insn_mask &= 0x07fff;
232 }
233
234 if (lower_immediate & 0x00000fff || lui_insn == RISCV_INSN_NOP4) {
235 /* replace upper 12 bits of addi with lower 12 bits of val */
236 addi_insn &= addi_insn_mask;
237 addi_insn |= (lower_immediate & 0x00000fff) << 20;
238 } else {
239 /* replace addi with nop if lower_immediate is empty */
240 addi_insn = RISCV_INSN_NOP4;
241 }
242
243 addi_res = cpu_to_le32(addi_insn);
244 lui_res = cpu_to_le32(lui_insn);
245 mutex_lock(&text_mutex);
246 patch_insn_write(addi_parcel, &addi_res, sizeof(addi_res));
247 patch_insn_write(lui_parcel, &lui_res, sizeof(lui_res));
248 mutex_unlock(&text_mutex);
249}
250
251static inline void __runtime_fixup_ptr(void *where, unsigned long val)
252{
253#ifdef CONFIG_32BIT
254 __runtime_fixup_32(where, where + 4, val);
255 __runtime_fixup_caches(where, 2);
256#else
257 __runtime_fixup_32(where, where + 8, val);
258 __runtime_fixup_32(where + 4, where + 12, val >> 32);
259 __runtime_fixup_caches(where, 4);
260#endif
261}
262
263/*
264 * Replace the least significant 5 bits of the srli/srliw immediate that is
265 * located at bits 20-24
266 */
267static inline void __runtime_fixup_shift(void *where, unsigned long val)
268{
269 __le16 *parcel = where;
270 __le32 res;
271 u32 insn;
272
273 insn = (u32)le16_to_cpu(parcel[0]) | (u32)le16_to_cpu(parcel[1]) << 16;
274
275 insn &= 0xfe0fffff;
276 insn |= (val & 0b11111) << 20;
277
278 res = cpu_to_le32(insn);
279 mutex_lock(&text_mutex);
280 patch_text_nosync(where, &res, sizeof(insn));
281 mutex_unlock(&text_mutex);
282}
283
284static inline void __runtime_fixup_mask(void *where, unsigned long val)
285{
286 unsigned int width = (val) ? __fls(val) + 1 : 0;
287
288 /*
289 * XXX: Current implementation only supports patching masks of
290 * form GENMASK(width, 0) (width >= 0) using a SRLI + SLLI
291 * sequence instead of LUI + ADDI + AND sequence to improve
292 * performance, density, and covers all the current use-cases.
293 *
294 * When the need arises to support any generic mask, and this
295 * BUG_ON() is tripped, consider using a:
296 *
297 * lui %[__ret], #imm16
298 * addi %[__ret], #imm16
299 *
300 * sequence to load the 32bit const mask, and perform a logical
301 * and outside the asm block before returning the result. Fixup
302 * can simply reuse the existing __runtime_fixup_32() to patch
303 * the LUI + ADDI sequence.
304 */
305 BUG_ON(!val || width > 31 || (GENMASK(width - 1, 0) != val));
306
307 __runtime_fixup_shift(where, 32 - width);
308 __runtime_fixup_shift(where + 4, 32 - width);
309}
310
311static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
312 unsigned long val, s32 *start, s32 *end)
313{
314 while (start < end) {
315 fn(*start + (void *)start, val);
316 start++;
317 }
318}
319
320#endif /* _ASM_RISCV_RUNTIME_CONST_H */