projects/linux·blob master · refs

443 lines · 9.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <linux/sizes.h>
6#include <linux/uaccess.h>
7#include <linux/set_memory.h>
8#include <linux/stop_machine.h>
9
10#include <asm/cacheflush.h>
11#include <asm/inst.h>
12
13static DEFINE_RAW_SPINLOCK(patch_lock);
14
15void simu_pc(struct pt_regs *regs, union loongarch_instruction insn)
16{
17 unsigned long pc = regs->csr_era;
18 unsigned int rd = insn.reg1i20_format.rd;
19 unsigned int imm = insn.reg1i20_format.immediate;
20
21 if (pc & 3) {
22 pr_warn("%s: invalid pc 0x%lx\n", __func__, pc);
23 return;
24 }
25
26 switch (insn.reg1i20_format.opcode) {
27 case pcaddi_op:
28 regs->regs[rd] = pc + sign_extend64(imm << 2, 21);
29 break;
30 case pcaddu12i_op:
31 regs->regs[rd] = pc + sign_extend64(imm << 12, 31);
32 break;
33 case pcaddu18i_op:
34 regs->regs[rd] = pc + sign_extend64(imm << 18, 37);
35 break;
36 case pcalau12i_op:
37 regs->regs[rd] = pc + sign_extend64(imm << 12, 31);
38 regs->regs[rd] &= ~((1 << 12) - 1);
39 break;
40 default:
41 pr_info("%s: unknown opcode\n", __func__);
42 return;
43 }
44
45 regs->csr_era += LOONGARCH_INSN_SIZE;
46}
47
48void simu_branch(struct pt_regs *regs, union loongarch_instruction insn)
49{
50 unsigned int imm, imm_l, imm_h, rd, rj;
51 unsigned long pc = regs->csr_era;
52
53 if (pc & 3) {
54 pr_warn("%s: invalid pc 0x%lx\n", __func__, pc);
55 return;
56 }
57
58 imm_l = insn.reg0i26_format.immediate_l;
59 imm_h = insn.reg0i26_format.immediate_h;
60 switch (insn.reg0i26_format.opcode) {
61 case b_op:
62 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27);
63 return;
64 case bl_op:
65 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27);
66 regs->regs[1] = pc + LOONGARCH_INSN_SIZE;
67 return;
68 }
69
70 imm_l = insn.reg1i21_format.immediate_l;
71 imm_h = insn.reg1i21_format.immediate_h;
72 rj = insn.reg1i21_format.rj;
73 switch (insn.reg1i21_format.opcode) {
74 case beqz_op:
75 if (regs->regs[rj] == 0)
76 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22);
77 else
78 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
79 return;
80 case bnez_op:
81 if (regs->regs[rj] != 0)
82 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22);
83 else
84 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
85 return;
86 }
87
88 imm = insn.reg2i16_format.immediate;
89 rj = insn.reg2i16_format.rj;
90 rd = insn.reg2i16_format.rd;
91 switch (insn.reg2i16_format.opcode) {
92 case beq_op:
93 if (regs->regs[rj] == regs->regs[rd])
94 regs->csr_era = pc + sign_extend64(imm << 2, 17);
95 else
96 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
97 break;
98 case bne_op:
99 if (regs->regs[rj] != regs->regs[rd])
100 regs->csr_era = pc + sign_extend64(imm << 2, 17);
101 else
102 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
103 break;
104 case blt_op:
105 if ((long)regs->regs[rj] < (long)regs->regs[rd])
106 regs->csr_era = pc + sign_extend64(imm << 2, 17);
107 else
108 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
109 break;
110 case bge_op:
111 if ((long)regs->regs[rj] >= (long)regs->regs[rd])
112 regs->csr_era = pc + sign_extend64(imm << 2, 17);
113 else
114 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
115 break;
116 case bltu_op:
117 if (regs->regs[rj] < regs->regs[rd])
118 regs->csr_era = pc + sign_extend64(imm << 2, 17);
119 else
120 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
121 break;
122 case bgeu_op:
123 if (regs->regs[rj] >= regs->regs[rd])
124 regs->csr_era = pc + sign_extend64(imm << 2, 17);
125 else
126 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
127 break;
128 case jirl_op:
129 regs->csr_era = regs->regs[rj] + sign_extend64(imm << 2, 17);
130 regs->regs[rd] = pc + LOONGARCH_INSN_SIZE;
131 break;
132 default:
133 pr_info("%s: unknown opcode\n", __func__);
134 return;
135 }
136}
137
138bool insns_not_supported(union loongarch_instruction insn)
139{
140 switch (insn.reg3_format.opcode) {
141 case amswapw_op ... ammindbdu_op:
142 pr_notice("atomic memory access instructions are not supported\n");
143 return true;
144 case scq_op:
145 pr_notice("sc.q instruction is not supported\n");
146 return true;
147 }
148
149 switch (insn.reg2i14_format.opcode) {
150 case llw_op:
151 case lld_op:
152 case scw_op:
153 case scd_op:
154 pr_notice("ll and sc instructions are not supported\n");
155 return true;
156 }
157
158 switch (insn.reg2_format.opcode) {
159 case llacqw_op:
160 case llacqd_op:
161 case screlw_op:
162 case screld_op:
163 pr_notice("llacq and screl instructions are not supported\n");
164 return true;
165 }
166
167 switch (insn.reg1i21_format.opcode) {
168 case bceqz_op:
169 pr_notice("bceqz and bcnez instructions are not supported\n");
170 return true;
171 }
172
173 return false;
174}
175
176bool insns_need_simulation(union loongarch_instruction insn)
177{
178 if (is_pc_ins(&insn))
179 return true;
180
181 if (is_branch_ins(&insn))
182 return true;
183
184 return false;
185}
186
187void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs)
188{
189 if (is_pc_ins(&insn))
190 simu_pc(regs, insn);
191 else if (is_branch_ins(&insn))
192 simu_branch(regs, insn);
193}
194
195int larch_insn_read(void *addr, u32 *insnp)
196{
197 int ret;
198 u32 val;
199
200 ret = copy_from_kernel_nofault(&val, addr, LOONGARCH_INSN_SIZE);
201 if (!ret)
202 *insnp = val;
203
204 return ret;
205}
206
207int larch_insn_write(void *addr, u32 insn)
208{
209 int ret;
210 unsigned long flags = 0;
211
212 if ((unsigned long)addr & 3)
213 return -EINVAL;
214
215 raw_spin_lock_irqsave(&patch_lock, flags);
216 ret = copy_to_kernel_nofault(addr, &insn, LOONGARCH_INSN_SIZE);
217 raw_spin_unlock_irqrestore(&patch_lock, flags);
218
219 return ret;
220}
221
222int larch_insn_patch_text(void *addr, u32 insn)
223{
224 int ret;
225 u32 *tp = addr;
226
227 ret = larch_insn_write(tp, insn);
228 if (!ret)
229 flush_icache_range((unsigned long)tp,
230 (unsigned long)tp + LOONGARCH_INSN_SIZE);
231
232 return ret;
233}
234
235struct insn_copy {
236 void *dst;
237 void *src;
238 size_t len;
239 unsigned int cpu;
240};
241
242static int text_copy_cb(void *data)
243{
244 int ret = 0;
245 struct insn_copy *copy = data;
246
247 if (smp_processor_id() == copy->cpu) {
248 ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len);
249 if (ret) {
250 pr_err("%s: operation failed\n", __func__);
251 return ret;
252 }
253 }
254
255 flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len);
256
257 return 0;
258}
259
260int larch_insn_text_copy(void *dst, void *src, size_t len)
261{
262 int ret = 0;
263 int err = 0;
264 size_t start, end;
265 struct insn_copy copy = {
266 .dst = dst,
267 .src = src,
268 .len = len,
269 .cpu = raw_smp_processor_id(),
270 };
271
272 /*
273 * Ensure copy.cpu won't be hot removed before stop_machine.
274 * If it is removed nobody will really update the text.
275 */
276 lockdep_assert_cpus_held();
277
278 start = round_down((size_t)dst, PAGE_SIZE);
279 end = round_up((size_t)dst + len, PAGE_SIZE);
280
281 err = set_memory_rw(start, (end - start) / PAGE_SIZE);
282 if (err) {
283 pr_info("%s: set_memory_rw() failed\n", __func__);
284 return err;
285 }
286
287 ret = stop_machine_cpuslocked(text_copy_cb, &copy, cpu_online_mask);
288
289 err = set_memory_rox(start, (end - start) / PAGE_SIZE);
290 if (err) {
291 pr_info("%s: set_memory_rox() failed\n", __func__);
292 return err;
293 }
294
295 return ret;
296}
297
298u32 larch_insn_gen_nop(void)
299{
300 return INSN_NOP;
301}
302
303u32 larch_insn_gen_b(unsigned long pc, unsigned long dest)
304{
305 long offset = dest - pc;
306 union loongarch_instruction insn;
307
308 if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) {
309 pr_warn("The generated b instruction is out of range.\n");
310 return INSN_BREAK;
311 }
312
313 emit_b(&insn, offset >> 2);
314
315 return insn.word;
316}
317
318u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest)
319{
320 long offset = dest - pc;
321 union loongarch_instruction insn;
322
323 if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) {
324 pr_warn("The generated bl instruction is out of range.\n");
325 return INSN_BREAK;
326 }
327
328 emit_bl(&insn, offset >> 2);
329
330 return insn.word;
331}
332
333u32 larch_insn_gen_break(int imm)
334{
335 union loongarch_instruction insn;
336
337 if (imm < 0 || imm >= SZ_32K) {
338 pr_warn("The generated break instruction is out of range.\n");
339 return INSN_BREAK;
340 }
341
342 emit_break(&insn, imm);
343
344 return insn.word;
345}
346
347u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)
348{
349 union loongarch_instruction insn;
350
351 emit_or(&insn, rd, rj, rk);
352
353 return insn.word;
354}
355
356u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj)
357{
358 return larch_insn_gen_or(rd, rj, 0);
359}
360
361u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm)
362{
363 union loongarch_instruction insn;
364
365 if (imm < -SZ_512K || imm >= SZ_512K) {
366 pr_warn("The generated lu12i.w instruction is out of range.\n");
367 return INSN_BREAK;
368 }
369
370 emit_lu12iw(&insn, rd, imm);
371
372 return insn.word;
373}
374
375u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm)
376{
377 union loongarch_instruction insn;
378
379 if (imm < -SZ_512K || imm >= SZ_512K) {
380 pr_warn("The generated lu32i.d instruction is out of range.\n");
381 return INSN_BREAK;
382 }
383
384 emit_lu32id(&insn, rd, imm);
385
386 return insn.word;
387}
388
389u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
390{
391 union loongarch_instruction insn;
392
393 if (imm < -SZ_2K || imm >= SZ_2K) {
394 pr_warn("The generated lu52i.d instruction is out of range.\n");
395 return INSN_BREAK;
396 }
397
398 emit_lu52id(&insn, rd, rj, imm);
399
400 return insn.word;
401}
402
403u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
404{
405 union loongarch_instruction insn;
406
407 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
408 pr_warn("The generated beq instruction is out of range.\n");
409 return INSN_BREAK;
410 }
411
412 emit_beq(&insn, rj, rd, imm >> 2);
413
414 return insn.word;
415}
416
417u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
418{
419 union loongarch_instruction insn;
420
421 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
422 pr_warn("The generated bne instruction is out of range.\n");
423 return INSN_BREAK;
424 }
425
426 emit_bne(&insn, rj, rd, imm >> 2);
427
428 return insn.word;
429}
430
431u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
432{
433 union loongarch_instruction insn;
434
435 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
436 pr_warn("The generated jirl instruction is out of range.\n");
437 return INSN_BREAK;
438 }
439
440 emit_jirl(&insn, rd, rj, imm >> 2);
441
442 return insn.word;
443}