projects/linux·blob master · refs

131 lines · 3.5 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_RUNTIME_CONST_H
3#define _ASM_RUNTIME_CONST_H
4
5#ifdef MODULE
6 #error "Cannot use runtime-const infrastructure from modules"
7#endif
8
9#include <asm/cacheflush.h>
10#include <asm/text-patching.h>
11
12/* Sigh. You can still run arm64 in BE mode */
13#include <asm/byteorder.h>
14
15#define runtime_const_ptr(sym) ({ \
16 typeof(sym) __ret; \
17 asm_inline("1:\t" \
18 "movz %0, #0xcdef\n\t" \
19 "movk %0, #0x89ab, lsl #16\n\t" \
20 "movk %0, #0x4567, lsl #32\n\t" \
21 "movk %0, #0x0123, lsl #48\n\t" \
22 ".pushsection runtime_ptr_" #sym ",\"a\"\n\t" \
23 ".long 1b - .\n\t" \
24 ".popsection" \
25 :"=r" (__ret)); \
26 __ret; })
27
28#define runtime_const_shift_right_32(val, sym) ({ \
29 unsigned long __ret; \
30 asm_inline("1:\t" \
31 "lsr %w0,%w1,#12\n\t" \
32 ".pushsection runtime_shift_" #sym ",\"a\"\n\t" \
33 ".long 1b - .\n\t" \
34 ".popsection" \
35 :"=r" (__ret) \
36 :"r" (0u+(val))); \
37 __ret; })
38
39#define runtime_const_mask_32(val, sym) ({ \
40 unsigned long __ret; \
41 asm_inline("1:\t" \
42 "ubfx %w0, %w1, #0, #32\n\t" \
43 ".pushsection runtime_mask_" #sym ",\"a\"\n\t" \
44 ".long 1b - .\n\t" \
45 ".popsection" \
46 :"=r" (__ret) \
47 :"r" (0u+(val))); \
48 __ret; })
49
50#define runtime_const_init(type, sym) do { \
51 extern s32 __start_runtime_##type##_##sym[]; \
52 extern s32 __stop_runtime_##type##_##sym[]; \
53 runtime_const_fixup(__runtime_fixup_##type, \
54 (unsigned long)(sym), \
55 __start_runtime_##type##_##sym, \
56 __stop_runtime_##type##_##sym); \
57} while (0)
58
59/* 16-bit immediate for wide move (movz and movk) in bits 5..20 */
60static inline void __runtime_fixup_16(__le32 *p, unsigned int val)
61{
62 u32 insn = le32_to_cpu(*p);
63 insn &= 0xffe0001f;
64 insn |= (val & 0xffff) << 5;
65 aarch64_insn_patch_text_nosync(p, insn);
66}
67
68static inline void __runtime_fixup_ptr(void *where, unsigned long val)
69{
70 __le32 *p = where;
71 __runtime_fixup_16(p, val);
72 __runtime_fixup_16(p+1, val >> 16);
73 __runtime_fixup_16(p+2, val >> 32);
74 __runtime_fixup_16(p+3, val >> 48);
75}
76
77/* Immediate value is 6 bits starting at bit #16 */
78static inline void __runtime_fixup_shift(void *where, unsigned long val)
79{
80 __le32 *p = where;
81 u32 insn = le32_to_cpu(*p);
82 insn &= 0xffc0ffff;
83 insn |= (val & 63) << 16;
84 aarch64_insn_patch_text_nosync(p, insn);
85}
86
87static inline void __runtime_fixup_mask(void *where, unsigned long val)
88{
89 unsigned int width = (val) ? __fls(val) + 1 : 0;
90 __le32 *p = where;
91 u32 insn;
92
93 /*
94 * XXX: Current implementation only supports patching masks of
95 * form GENMASK(n, 0) (n >= 0) using a single UBFX instruction
96 * to improve performance, density, and covers all the current
97 * use-cases.
98 *
99 * When the need arises to support any generic mask, and this
100 * BUG_ON() is tripped, consider using a:
101 *
102 * movz %w0, #imm16
103 * movk %w0, #imm16, lsl #16
104 *
105 * sequence to load the 32bit const mask, and perform a logical
106 * and outside the asm block before returning the result. Fixup
107 * can simply reuse the existing __runtime_fixup_16() to patch
108 * the individual mov instructions.
109 */
110 BUG_ON(!val || width > 32 || (GENMASK(width - 1, 0) != val));
111
112 /*
113 * The width of the mask is encoded as (width - 1) in imms
114 * which is 6 bits starting at bit #10.
115 */
116 insn = le32_to_cpu(*p);
117 insn &= 0xffff03ff;
118 insn |= ((width - 1) & 0x1f) << 10;
119 aarch64_insn_patch_text_nosync(p, insn);
120}
121
122static inline void runtime_const_fixup(void (*fn)(void *, unsigned long),
123 unsigned long val, s32 *start, s32 *end)
124{
125 while (start < end) {
126 fn(*start + (void *)start, val);
127 start++;
128 }
129}
130
131#endif