projects/linux·blob uaccess-cleanups · refs

2365 lines · 56.5 KiB

1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/security.h>
30#include <linux/pm_runtime.h>
31
32#include "amdgpu.h"
33#include "amdgpu_pm.h"
34#include "amdgpu_dm_debugfs.h"
35#include "amdgpu_ras.h"
36#include "amdgpu_rap.h"
37#include "amdgpu_securedisplay.h"
38#include "amdgpu_fw_attestation.h"
39#include "amdgpu_umr.h"
40
41#include "amdgpu_reset.h"
42#include "amdgpu_psp_ta.h"
43#include "amdgpu_userq.h"
44
45#if defined(CONFIG_DEBUG_FS)
46
47/* Encode milliwatts in the raw Q24.8 sensor report format used by UMR. */
48#define AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(power_mw) \
49 DIV_ROUND_CLOSEST_ULL((u64)(power_mw) * BIT(8), \
50 MILLIWATT_PER_WATT)
51
52/**
53 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
54 *
55 * @read: True if reading
56 * @f: open file handle
57 * @buf: User buffer to write/read to
58 * @size: Number of bytes to write/read
59 * @pos: Offset to seek to
60 *
61 * This debugfs entry has special meaning on the offset being sought.
62 * Various bits have different meanings:
63 *
64 * Bit 62: Indicates a GRBM bank switch is needed
65 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
66 * zero)
67 * Bits 24..33: The SE or ME selector if needed
68 * Bits 34..43: The SH (or SA) or PIPE selector if needed
69 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
70 *
71 * Bit 23: Indicates that the PM power gating lock should be held
72 * This is necessary to read registers that might be
73 * unreliable during a power gating transistion.
74 *
75 * The lower bits are the BYTE offset of the register to read. This
76 * allows reading multiple registers in a single call and having
77 * the returned size reflect that.
78 */
79static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
80 char __user *buf, size_t size, loff_t *pos)
81{
82 struct amdgpu_device *adev = file_inode(f)->i_private;
83 ssize_t result = 0;
84 int r;
85 bool pm_pg_lock, use_bank, use_ring;
86 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
87
88 pm_pg_lock = use_bank = use_ring = false;
89 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
90
91 if (size & 0x3 || *pos & 0x3 ||
92 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
93 return -EINVAL;
94
95 /* are we reading registers for which a PG lock is necessary? */
96 pm_pg_lock = (*pos >> 23) & 1;
97
98 if (*pos & (1ULL << 62)) {
99 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
100 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
101 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
102
103 if (se_bank == 0x3FF)
104 se_bank = 0xFFFFFFFF;
105 if (sh_bank == 0x3FF)
106 sh_bank = 0xFFFFFFFF;
107 if (instance_bank == 0x3FF)
108 instance_bank = 0xFFFFFFFF;
109 use_bank = true;
110 } else if (*pos & (1ULL << 61)) {
111
112 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
113 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
114 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
115 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
116
117 use_ring = true;
118 } else {
119 use_bank = use_ring = false;
120 }
121
122 *pos &= (1UL << 22) - 1;
123
124 if (size > PAGE_SIZE)
125 return -EINVAL;
126
127 /*
128 * Access the user buffer only while none of the locks below are held:
129 * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest
130 * inside it.
131 */
132 u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL);
133 if (!data)
134 return -ENOMEM;
135 if (!read && copy_from_user(data, buf, size))
136 return -EFAULT;
137
138 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
139 if (r < 0) {
140 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
141 return r;
142 }
143
144 r = amdgpu_virt_enable_access_debugfs(adev);
145 if (r < 0) {
146 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
147 return r;
148 }
149
150 if (use_bank) {
151 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
152 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
153 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
154 amdgpu_virt_disable_access_debugfs(adev);
155 return -EINVAL;
156 }
157 mutex_lock(&adev->grbm_idx_mutex);
158 amdgpu_gfx_select_se_sh(adev, se_bank,
159 sh_bank, instance_bank, 0);
160 } else if (use_ring) {
161 mutex_lock(&adev->srbm_mutex);
162 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
163 }
164
165 if (pm_pg_lock)
166 mutex_lock(&adev->pm.mutex);
167
168 while (size) {
169 if (read)
170 data[result >> 2] = RREG32(*pos >> 2);
171 else
172 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, data[result >> 2], 0);
173
174 result += 4;
175 *pos += 4;
176 size -= 4;
177 }
178
179 if (use_bank) {
180 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
181 mutex_unlock(&adev->grbm_idx_mutex);
182 } else if (use_ring) {
183 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
184 mutex_unlock(&adev->srbm_mutex);
185 }
186
187 if (pm_pg_lock)
188 mutex_unlock(&adev->pm.mutex);
189
190 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
191
192 amdgpu_virt_disable_access_debugfs(adev);
193
194 if (read && copy_to_user(buf, data, result))
195 return -EFAULT;
196
197 return result;
198}
199
200/*
201 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
202 */
203static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
204 size_t size, loff_t *pos)
205{
206 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
207}
208
209/*
210 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
211 */
212static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
213 size_t size, loff_t *pos)
214{
215 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
216}
217
218static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
219{
220 struct amdgpu_debugfs_regs2_data *rd;
221
222 rd = kzalloc_obj(*rd);
223 if (!rd)
224 return -ENOMEM;
225 rd->adev = file_inode(file)->i_private;
226 file->private_data = rd;
227 mutex_init(&rd->lock);
228
229 return 0;
230}
231
232static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
233{
234 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
235
236 mutex_destroy(&rd->lock);
237 kfree(file->private_data);
238 return 0;
239}
240
241static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
242{
243 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
244 struct amdgpu_device *adev = rd->adev;
245 ssize_t result = 0;
246 int r;
247
248 if (size & 0x3 || offset & 0x3)
249 return -EINVAL;
250
251 if (size > PAGE_SIZE)
252 return -EINVAL;
253
254 /*
255 * Access the user buffer only while none of the locks below are held:
256 * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest
257 * inside it.
258 */
259 u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL);
260 if (!data)
261 return -ENOMEM;
262 if (write_en && copy_from_user(data, buf, size))
263 return -EFAULT;
264
265 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
266 if (r < 0) {
267 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
268 return r;
269 }
270
271 r = amdgpu_virt_enable_access_debugfs(adev);
272 if (r < 0) {
273 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
274 return r;
275 }
276
277 mutex_lock(&rd->lock);
278
279 if (rd->id.use_grbm) {
280 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
281 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
282 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
283 amdgpu_virt_disable_access_debugfs(adev);
284 mutex_unlock(&rd->lock);
285 return -EINVAL;
286 }
287 mutex_lock(&adev->grbm_idx_mutex);
288 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
289 rd->id.grbm.sh,
290 rd->id.grbm.instance, rd->id.xcc_id);
291 }
292
293 if (rd->id.use_srbm) {
294 mutex_lock(&adev->srbm_mutex);
295 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
296 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
297 }
298
299 if (rd->id.pg_lock)
300 mutex_lock(&adev->pm.mutex);
301
302 while (size) {
303 if (!write_en)
304 data[result >> 2] = RREG32(offset >> 2);
305 else
306 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, data[result >> 2],
307 rd->id.xcc_id);
308 offset += 4;
309 size -= 4;
310 result += 4;
311 }
312
313 if (rd->id.use_grbm) {
314 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
315 mutex_unlock(&adev->grbm_idx_mutex);
316 }
317
318 if (rd->id.use_srbm) {
319 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
320 mutex_unlock(&adev->srbm_mutex);
321 }
322
323 if (rd->id.pg_lock)
324 mutex_unlock(&adev->pm.mutex);
325
326 mutex_unlock(&rd->lock);
327
328 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
329
330 amdgpu_virt_disable_access_debugfs(adev);
331
332 if (!write_en && copy_to_user(buf, data, result))
333 return -EFAULT;
334
335 return result;
336}
337
338static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
339{
340 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
341 struct amdgpu_debugfs_regs2_iocdata v1_data;
342 int r;
343
344 mutex_lock(&rd->lock);
345
346 switch (cmd) {
347 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
348 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
349 sizeof(rd->id));
350 if (r)
351 r = -EINVAL;
352 goto done;
353 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
354 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
355 sizeof(v1_data));
356 if (r) {
357 r = -EINVAL;
358 goto done;
359 }
360 goto v1_copy;
361 default:
362 r = -EINVAL;
363 goto done;
364 }
365
366v1_copy:
367 rd->id.use_srbm = v1_data.use_srbm;
368 rd->id.use_grbm = v1_data.use_grbm;
369 rd->id.pg_lock = v1_data.pg_lock;
370 rd->id.grbm.se = v1_data.grbm.se;
371 rd->id.grbm.sh = v1_data.grbm.sh;
372 rd->id.grbm.instance = v1_data.grbm.instance;
373 rd->id.srbm.me = v1_data.srbm.me;
374 rd->id.srbm.pipe = v1_data.srbm.pipe;
375 rd->id.srbm.queue = v1_data.srbm.queue;
376 rd->id.xcc_id = 0;
377done:
378 mutex_unlock(&rd->lock);
379 return r;
380}
381
382static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
383{
384 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
385}
386
387static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
388{
389 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
390}
391
392static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
393{
394 struct amdgpu_debugfs_gprwave_data *rd;
395
396 rd = kzalloc_obj(*rd);
397 if (!rd)
398 return -ENOMEM;
399 rd->adev = file_inode(file)->i_private;
400 file->private_data = rd;
401 mutex_init(&rd->lock);
402
403 return 0;
404}
405
406static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
407{
408 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
409
410 mutex_destroy(&rd->lock);
411 kfree(file->private_data);
412 return 0;
413}
414
415static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
416{
417 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
418 struct amdgpu_device *adev = rd->adev;
419 ssize_t result = 0;
420 int r;
421 uint32_t *data, x;
422
423 if (size > 4096 || size & 0x3 || *pos & 0x3)
424 return -EINVAL;
425
426 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
427 if (r < 0) {
428 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
429 return r;
430 }
431
432 r = amdgpu_virt_enable_access_debugfs(adev);
433 if (r < 0) {
434 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
435 return r;
436 }
437
438 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
439 if (!data) {
440 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
441 amdgpu_virt_disable_access_debugfs(adev);
442 return -ENOMEM;
443 }
444
445 /* switch to the specific se/sh/cu */
446 mutex_lock(&adev->grbm_idx_mutex);
447 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
448
449 if (!rd->id.gpr_or_wave) {
450 x = 0;
451 if (adev->gfx.funcs->read_wave_data)
452 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
453 } else {
454 x = size >> 2;
455 if (rd->id.gpr.vpgr_or_sgpr) {
456 if (adev->gfx.funcs->read_wave_vgprs)
457 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
458 } else {
459 if (adev->gfx.funcs->read_wave_sgprs)
460 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
461 }
462 }
463
464 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
465 mutex_unlock(&adev->grbm_idx_mutex);
466
467 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
468
469 if (!x) {
470 result = -EINVAL;
471 goto done;
472 }
473
474 while (size && (*pos < x * 4)) {
475 uint32_t value;
476
477 value = data[*pos >> 2];
478 r = put_user(value, (uint32_t *)buf);
479 if (r) {
480 result = r;
481 goto done;
482 }
483
484 result += 4;
485 buf += 4;
486 *pos += 4;
487 size -= 4;
488 }
489
490done:
491 amdgpu_virt_disable_access_debugfs(adev);
492 kfree(data);
493 return result;
494}
495
496static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
497{
498 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
499 int r = 0;
500
501 mutex_lock(&rd->lock);
502
503 switch (cmd) {
504 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
505 if (copy_from_user(&rd->id,
506 (struct amdgpu_debugfs_gprwave_iocdata *)data,
507 sizeof(rd->id)))
508 r = -EFAULT;
509 goto done;
510 default:
511 r = -EINVAL;
512 goto done;
513 }
514
515done:
516 mutex_unlock(&rd->lock);
517 return r;
518}
519
520
521
522
523/**
524 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
525 *
526 * @f: open file handle
527 * @buf: User buffer to store read data in
528 * @size: Number of bytes to read
529 * @pos: Offset to seek to
530 *
531 * The lower bits are the BYTE offset of the register to read. This
532 * allows reading multiple registers in a single call and having
533 * the returned size reflect that.
534 */
535static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
536 size_t size, loff_t *pos)
537{
538 struct amdgpu_device *adev = file_inode(f)->i_private;
539 ssize_t result = 0;
540 int r;
541
542 if (size & 0x3 || *pos & 0x3)
543 return -EINVAL;
544
545 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
546 if (r < 0) {
547 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
548 return r;
549 }
550
551 r = amdgpu_virt_enable_access_debugfs(adev);
552 if (r < 0) {
553 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
554 return r;
555 }
556
557 while (size) {
558 uint32_t value;
559
560 if (upper_32_bits(*pos))
561 value = RREG32_PCIE_EXT(*pos);
562 else
563 value = RREG32_PCIE(*pos);
564
565 r = put_user(value, (uint32_t *)buf);
566 if (r)
567 goto out;
568
569 result += 4;
570 buf += 4;
571 *pos += 4;
572 size -= 4;
573 }
574
575 r = result;
576out:
577 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
578 amdgpu_virt_disable_access_debugfs(adev);
579 return r;
580}
581
582/**
583 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
584 *
585 * @f: open file handle
586 * @buf: User buffer to write data from
587 * @size: Number of bytes to write
588 * @pos: Offset to seek to
589 *
590 * The lower bits are the BYTE offset of the register to write. This
591 * allows writing multiple registers in a single call and having
592 * the returned size reflect that.
593 */
594static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
595 size_t size, loff_t *pos)
596{
597 struct amdgpu_device *adev = file_inode(f)->i_private;
598 ssize_t result = 0;
599 int r;
600
601 if (size & 0x3 || *pos & 0x3)
602 return -EINVAL;
603
604 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
605 if (r < 0) {
606 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
607 return r;
608 }
609
610 r = amdgpu_virt_enable_access_debugfs(adev);
611 if (r < 0) {
612 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
613 return r;
614 }
615
616 while (size) {
617 uint32_t value;
618
619 r = get_user(value, (uint32_t *)buf);
620 if (r)
621 goto out;
622
623 if (upper_32_bits(*pos))
624 WREG32_PCIE_EXT(*pos, value);
625 else
626 WREG32_PCIE(*pos, value);
627
628 result += 4;
629 buf += 4;
630 *pos += 4;
631 size -= 4;
632 }
633
634 r = result;
635out:
636 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
637 amdgpu_virt_disable_access_debugfs(adev);
638 return r;
639}
640
641/**
642 * amdgpu_debugfs_regs_pcie64_read - Read from a 64-bit PCIE register
643 *
644 * @f: open file handle
645 * @buf: User buffer to store read data in
646 * @size: Number of bytes to read
647 * @pos: Offset to seek to
648 */
649static ssize_t amdgpu_debugfs_regs_pcie64_read(struct file *f, char __user *buf,
650 size_t size, loff_t *pos)
651{
652 struct amdgpu_device *adev = file_inode(f)->i_private;
653 ssize_t result = 0;
654 int r;
655
656 if (size & 0x7 || *pos & 0x7)
657 return -EINVAL;
658
659 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
660 if (r < 0) {
661 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
662 return r;
663 }
664
665 r = amdgpu_virt_enable_access_debugfs(adev);
666 if (r < 0) {
667 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
668 return r;
669 }
670
671 while (size) {
672 uint64_t value;
673
674 value = RREG64_PCIE_EXT(*pos);
675
676 r = put_user(value, (uint64_t *)buf);
677 if (r)
678 goto out;
679
680 result += 8;
681 buf += 8;
682 *pos += 8;
683 size -= 8;
684 }
685
686 r = result;
687out:
688 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
689 amdgpu_virt_disable_access_debugfs(adev);
690 return r;
691}
692
693/**
694 * amdgpu_debugfs_regs_pcie64_write - Write to a 64-bit PCIE register
695 *
696 * @f: open file handle
697 * @buf: User buffer to write data from
698 * @size: Number of bytes to write
699 * @pos: Offset to seek to
700 */
701static ssize_t amdgpu_debugfs_regs_pcie64_write(struct file *f, const char __user *buf,
702 size_t size, loff_t *pos)
703{
704 struct amdgpu_device *adev = file_inode(f)->i_private;
705 ssize_t result = 0;
706 int r;
707
708 if (size & 0x7 || *pos & 0x7)
709 return -EINVAL;
710
711 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
712 if (r < 0) {
713 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
714 return r;
715 }
716
717 r = amdgpu_virt_enable_access_debugfs(adev);
718 if (r < 0) {
719 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
720 return r;
721 }
722
723 while (size) {
724 uint64_t value;
725
726 r = get_user(value, (uint64_t *)buf);
727 if (r)
728 goto out;
729
730 WREG64_PCIE_EXT(*pos, value);
731
732 result += 8;
733 buf += 8;
734 *pos += 8;
735 size -= 8;
736 }
737
738 r = result;
739out:
740 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
741 amdgpu_virt_disable_access_debugfs(adev);
742 return r;
743}
744
745/**
746 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
747 *
748 * @f: open file handle
749 * @buf: User buffer to store read data in
750 * @size: Number of bytes to read
751 * @pos: Offset to seek to
752 *
753 * The lower bits are the BYTE offset of the register to read. This
754 * allows reading multiple registers in a single call and having
755 * the returned size reflect that.
756 */
757static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
758 size_t size, loff_t *pos)
759{
760 struct amdgpu_device *adev = file_inode(f)->i_private;
761 ssize_t result = 0;
762 int r;
763
764 if (size & 0x3 || *pos & 0x3)
765 return -EINVAL;
766
767 if (!adev->reg.didt.rreg)
768 return -EOPNOTSUPP;
769
770 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
771 if (r < 0) {
772 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
773 return r;
774 }
775
776 r = amdgpu_virt_enable_access_debugfs(adev);
777 if (r < 0) {
778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
779 return r;
780 }
781
782 while (size) {
783 uint32_t value;
784
785 value = RREG32_DIDT(*pos >> 2);
786 r = put_user(value, (uint32_t *)buf);
787 if (r)
788 goto out;
789
790 result += 4;
791 buf += 4;
792 *pos += 4;
793 size -= 4;
794 }
795
796 r = result;
797out:
798 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
799 amdgpu_virt_disable_access_debugfs(adev);
800 return r;
801}
802
803/**
804 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
805 *
806 * @f: open file handle
807 * @buf: User buffer to write data from
808 * @size: Number of bytes to write
809 * @pos: Offset to seek to
810 *
811 * The lower bits are the BYTE offset of the register to write. This
812 * allows writing multiple registers in a single call and having
813 * the returned size reflect that.
814 */
815static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
816 size_t size, loff_t *pos)
817{
818 struct amdgpu_device *adev = file_inode(f)->i_private;
819 ssize_t result = 0;
820 int r;
821
822 if (size & 0x3 || *pos & 0x3)
823 return -EINVAL;
824
825 if (!adev->reg.didt.wreg)
826 return -EOPNOTSUPP;
827
828 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
829 if (r < 0) {
830 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
831 return r;
832 }
833
834 r = amdgpu_virt_enable_access_debugfs(adev);
835 if (r < 0) {
836 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
837 return r;
838 }
839
840 while (size) {
841 uint32_t value;
842
843 r = get_user(value, (uint32_t *)buf);
844 if (r)
845 goto out;
846
847 WREG32_DIDT(*pos >> 2, value);
848
849 result += 4;
850 buf += 4;
851 *pos += 4;
852 size -= 4;
853 }
854
855 r = result;
856out:
857 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
858 amdgpu_virt_disable_access_debugfs(adev);
859 return r;
860}
861
862/**
863 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
864 *
865 * @f: open file handle
866 * @buf: User buffer to store read data in
867 * @size: Number of bytes to read
868 * @pos: Offset to seek to
869 *
870 * The lower bits are the BYTE offset of the register to read. This
871 * allows reading multiple registers in a single call and having
872 * the returned size reflect that.
873 */
874static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
875 size_t size, loff_t *pos)
876{
877 struct amdgpu_device *adev = file_inode(f)->i_private;
878 ssize_t result = 0;
879 int r;
880
881 if (!adev->reg.smc.rreg)
882 return -EOPNOTSUPP;
883
884 if (size & 0x3 || *pos & 0x3)
885 return -EINVAL;
886
887 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
888 if (r < 0) {
889 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
890 return r;
891 }
892
893 r = amdgpu_virt_enable_access_debugfs(adev);
894 if (r < 0) {
895 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
896 return r;
897 }
898
899 while (size) {
900 uint32_t value;
901
902 value = RREG32_SMC(*pos);
903 r = put_user(value, (uint32_t *)buf);
904 if (r)
905 goto out;
906
907 result += 4;
908 buf += 4;
909 *pos += 4;
910 size -= 4;
911 }
912
913 r = result;
914out:
915 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
916 amdgpu_virt_disable_access_debugfs(adev);
917 return r;
918}
919
920/**
921 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
922 *
923 * @f: open file handle
924 * @buf: User buffer to write data from
925 * @size: Number of bytes to write
926 * @pos: Offset to seek to
927 *
928 * The lower bits are the BYTE offset of the register to write. This
929 * allows writing multiple registers in a single call and having
930 * the returned size reflect that.
931 */
932static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
933 size_t size, loff_t *pos)
934{
935 struct amdgpu_device *adev = file_inode(f)->i_private;
936 ssize_t result = 0;
937 int r;
938
939 if (!adev->reg.smc.wreg)
940 return -EOPNOTSUPP;
941
942 if (size & 0x3 || *pos & 0x3)
943 return -EINVAL;
944
945 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
946 if (r < 0) {
947 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
948 return r;
949 }
950
951 r = amdgpu_virt_enable_access_debugfs(adev);
952 if (r < 0) {
953 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
954 return r;
955 }
956
957 while (size) {
958 uint32_t value;
959
960 r = get_user(value, (uint32_t *)buf);
961 if (r)
962 goto out;
963
964 WREG32_SMC(*pos, value);
965
966 result += 4;
967 buf += 4;
968 *pos += 4;
969 size -= 4;
970 }
971
972 r = result;
973out:
974 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
975 amdgpu_virt_disable_access_debugfs(adev);
976 return r;
977}
978
979/**
980 * amdgpu_debugfs_gca_config_read - Read from gfx config data
981 *
982 * @f: open file handle
983 * @buf: User buffer to store read data in
984 * @size: Number of bytes to read
985 * @pos: Offset to seek to
986 *
987 * This file is used to access configuration data in a somewhat
988 * stable fashion. The format is a series of DWORDs with the first
989 * indicating which revision it is. New content is appended to the
990 * end so that older software can still read the data.
991 */
992
993static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
994 size_t size, loff_t *pos)
995{
996 struct amdgpu_device *adev = file_inode(f)->i_private;
997 ssize_t result = 0;
998 int r;
999 uint32_t *config, no_regs = 0;
1000
1001 if (size & 0x3 || *pos & 0x3)
1002 return -EINVAL;
1003
1004 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
1005 if (!config)
1006 return -ENOMEM;
1007
1008 /* version, increment each time something is added */
1009 config[no_regs++] = 5;
1010 config[no_regs++] = adev->gfx.config.max_shader_engines;
1011 config[no_regs++] = adev->gfx.config.max_tile_pipes;
1012 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
1013 config[no_regs++] = adev->gfx.config.max_sh_per_se;
1014 config[no_regs++] = adev->gfx.config.max_backends_per_se;
1015 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
1016 config[no_regs++] = adev->gfx.config.max_gprs;
1017 config[no_regs++] = adev->gfx.config.max_gs_threads;
1018 config[no_regs++] = adev->gfx.config.max_hw_contexts;
1019 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
1020 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
1021 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
1022 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
1023 config[no_regs++] = adev->gfx.config.num_tile_pipes;
1024 config[no_regs++] = adev->gfx.config.backend_enable_mask;
1025 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
1026 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
1027 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
1028 config[no_regs++] = adev->gfx.config.num_gpus;
1029 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
1030 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
1031 config[no_regs++] = adev->gfx.config.gb_addr_config;
1032 config[no_regs++] = adev->gfx.config.num_rbs;
1033
1034 /* rev==1 */
1035 config[no_regs++] = adev->rev_id;
1036 config[no_regs++] = adev->pg_flags;
1037 config[no_regs++] = lower_32_bits(adev->cg_flags);
1038
1039 /* rev==2 */
1040 config[no_regs++] = adev->family;
1041 config[no_regs++] = adev->external_rev_id;
1042
1043 /* rev==3 */
1044 config[no_regs++] = adev->pdev->device;
1045 config[no_regs++] = adev->pdev->revision;
1046 config[no_regs++] = adev->pdev->subsystem_device;
1047 config[no_regs++] = adev->pdev->subsystem_vendor;
1048
1049 /* rev==4 APU flag */
1050 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
1051
1052 /* rev==5 PG/CG flag upper 32bit */
1053 config[no_regs++] = 0;
1054 config[no_regs++] = upper_32_bits(adev->cg_flags);
1055
1056 while (size && (*pos < no_regs * 4)) {
1057 uint32_t value;
1058
1059 value = config[*pos >> 2];
1060 r = put_user(value, (uint32_t *)buf);
1061 if (r) {
1062 kfree(config);
1063 return r;
1064 }
1065
1066 result += 4;
1067 buf += 4;
1068 *pos += 4;
1069 size -= 4;
1070 }
1071
1072 kfree(config);
1073 return result;
1074}
1075
1076/**
1077 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
1078 *
1079 * @f: open file handle
1080 * @buf: User buffer to store read data in
1081 * @size: Number of bytes to read
1082 * @pos: Offset to seek to
1083 *
1084 * The offset is treated as the BYTE address of one of the sensors
1085 * enumerated in amd/include/kgd_pp_interface.h under the
1086 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
1087 * you would use the offset 3 * 4 = 12.
1088 */
1089static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
1090 size_t size, loff_t *pos)
1091{
1092 struct amdgpu_device *adev = file_inode(f)->i_private;
1093 int idx, x, outsize, r, valuesize;
1094 uint32_t values[16];
1095
1096 if (size & 3 || *pos & 0x3)
1097 return -EINVAL;
1098
1099 if (!adev->pm.dpm_enabled)
1100 return -EINVAL;
1101
1102 /* convert offset to sensor number */
1103 idx = *pos >> 2;
1104
1105 valuesize = sizeof(values);
1106
1107 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1108 if (r < 0) {
1109 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1110 return r;
1111 }
1112
1113 r = amdgpu_virt_enable_access_debugfs(adev);
1114 if (r < 0) {
1115 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1116 return r;
1117 }
1118
1119 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
1120
1121 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1122
1123 if (r) {
1124 amdgpu_virt_disable_access_debugfs(adev);
1125 return r;
1126 }
1127
1128 if (idx == AMDGPU_PP_SENSOR_GPU_AVG_POWER ||
1129 idx == AMDGPU_PP_SENSOR_GPU_INPUT_POWER)
1130 values[0] = AMDGPU_DEBUGFS_PWR_MW_TO_Q24_8(values[0]);
1131
1132 if (size > valuesize) {
1133 amdgpu_virt_disable_access_debugfs(adev);
1134 return -EINVAL;
1135 }
1136
1137 outsize = 0;
1138 x = 0;
1139 if (!r) {
1140 while (size) {
1141 r = put_user(values[x++], (int32_t *)buf);
1142 buf += 4;
1143 size -= 4;
1144 outsize += 4;
1145 }
1146 }
1147
1148 amdgpu_virt_disable_access_debugfs(adev);
1149 return !r ? outsize : r;
1150}
1151
1152/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1153 *
1154 * @f: open file handle
1155 * @buf: User buffer to store read data in
1156 * @size: Number of bytes to read
1157 * @pos: Offset to seek to
1158 *
1159 * The offset being sought changes which wave that the status data
1160 * will be returned for. The bits are used as follows:
1161 *
1162 * Bits 0..6: Byte offset into data
1163 * Bits 7..14: SE selector
1164 * Bits 15..22: SH/SA selector
1165 * Bits 23..30: CU/{WGP+SIMD} selector
1166 * Bits 31..36: WAVE ID selector
1167 * Bits 37..44: SIMD ID selector
1168 *
1169 * The returned data begins with one DWORD of version information
1170 * Followed by WAVE STATUS registers relevant to the GFX IP version
1171 * being used. See gfx_v8_0_read_wave_data() for an example output.
1172 */
1173static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1174 size_t size, loff_t *pos)
1175{
1176 struct amdgpu_device *adev = f->f_inode->i_private;
1177 int r, x;
1178 ssize_t result = 0;
1179 uint32_t offset, se, sh, cu, wave, simd, data[32];
1180
1181 if (size & 3 || *pos & 3)
1182 return -EINVAL;
1183
1184 /* decode offset */
1185 offset = (*pos & GENMASK_ULL(6, 0));
1186 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1187 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1188 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1189 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1190 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1191
1192 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1193 if (r < 0) {
1194 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1195 return r;
1196 }
1197
1198 r = amdgpu_virt_enable_access_debugfs(adev);
1199 if (r < 0) {
1200 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1201 return r;
1202 }
1203
1204 /* switch to the specific se/sh/cu */
1205 mutex_lock(&adev->grbm_idx_mutex);
1206 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1207
1208 x = 0;
1209 if (adev->gfx.funcs->read_wave_data)
1210 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1211
1212 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1213 mutex_unlock(&adev->grbm_idx_mutex);
1214
1215 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1216
1217 if (!x) {
1218 amdgpu_virt_disable_access_debugfs(adev);
1219 return -EINVAL;
1220 }
1221
1222 while (size && (offset < x * 4)) {
1223 uint32_t value;
1224
1225 value = data[offset >> 2];
1226 r = put_user(value, (uint32_t *)buf);
1227 if (r) {
1228 amdgpu_virt_disable_access_debugfs(adev);
1229 return r;
1230 }
1231
1232 result += 4;
1233 buf += 4;
1234 offset += 4;
1235 size -= 4;
1236 }
1237
1238 amdgpu_virt_disable_access_debugfs(adev);
1239 return result;
1240}
1241
1242/** amdgpu_debugfs_gpr_read - Read wave gprs
1243 *
1244 * @f: open file handle
1245 * @buf: User buffer to store read data in
1246 * @size: Number of bytes to read
1247 * @pos: Offset to seek to
1248 *
1249 * The offset being sought changes which wave that the status data
1250 * will be returned for. The bits are used as follows:
1251 *
1252 * Bits 0..11: Byte offset into data
1253 * Bits 12..19: SE selector
1254 * Bits 20..27: SH/SA selector
1255 * Bits 28..35: CU/{WGP+SIMD} selector
1256 * Bits 36..43: WAVE ID selector
1257 * Bits 37..44: SIMD ID selector
1258 * Bits 52..59: Thread selector
1259 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1260 *
1261 * The return data comes from the SGPR or VGPR register bank for
1262 * the selected operational unit.
1263 */
1264static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1265 size_t size, loff_t *pos)
1266{
1267 struct amdgpu_device *adev = f->f_inode->i_private;
1268 int r;
1269 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1270
1271 if (size > 4096 || size & 3 || *pos & 3)
1272 return -EINVAL;
1273
1274 /* decode offset */
1275 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1276 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1277 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1278 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1279 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1280 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1281 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1282 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1283
1284 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1285 if (!data)
1286 return -ENOMEM;
1287
1288 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1289 if (r < 0)
1290 goto err;
1291
1292 r = amdgpu_virt_enable_access_debugfs(adev);
1293 if (r < 0)
1294 goto err;
1295
1296 /* switch to the specific se/sh/cu */
1297 mutex_lock(&adev->grbm_idx_mutex);
1298 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1299
1300 if (bank == 0) {
1301 if (adev->gfx.funcs->read_wave_vgprs)
1302 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1303 } else {
1304 if (adev->gfx.funcs->read_wave_sgprs)
1305 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1306 }
1307
1308 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1309 mutex_unlock(&adev->grbm_idx_mutex);
1310
1311 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1312 amdgpu_virt_disable_access_debugfs(adev);
1313
1314 r = copy_to_user(buf, data, size) ? -EFAULT : size;
1315 kfree(data);
1316 return r;
1317
1318err:
1319 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1320 kfree(data);
1321 return r;
1322}
1323
1324/**
1325 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1326 *
1327 * @f: open file handle
1328 * @buf: User buffer to store read data in
1329 * @size: Number of bytes to read
1330 * @pos: Offset to seek to
1331 *
1332 * Read a live GFXOFF residency sample from firmware. One needs to start logging
1333 * before getting the current value.
1334 */
1335static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1336 size_t size, loff_t *pos)
1337{
1338 struct amdgpu_device *adev = file_inode(f)->i_private;
1339 ssize_t result = 0;
1340 int r;
1341
1342 if (size & 0x3 || *pos & 0x3)
1343 return -EINVAL;
1344
1345 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1346 if (r < 0) {
1347 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1348 return r;
1349 }
1350
1351 while (size) {
1352 uint32_t value;
1353
1354 r = amdgpu_get_gfx_off_residency(adev, &value);
1355 if (r)
1356 goto out;
1357
1358 r = put_user(value, (uint32_t *)buf);
1359 if (r)
1360 goto out;
1361
1362 result += 4;
1363 buf += 4;
1364 *pos += 4;
1365 size -= 4;
1366 }
1367
1368 r = result;
1369out:
1370 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1371
1372 return r;
1373}
1374
1375/**
1376 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1377 *
1378 * @f: open file handle
1379 * @buf: User buffer to write data from
1380 * @size: Number of bytes to write
1381 * @pos: Offset to seek to
1382 *
1383 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1384 */
1385static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1386 size_t size, loff_t *pos)
1387{
1388 struct amdgpu_device *adev = file_inode(f)->i_private;
1389 ssize_t result = 0;
1390 int r;
1391
1392 if (size & 0x3 || *pos & 0x3)
1393 return -EINVAL;
1394
1395 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1396 if (r < 0) {
1397 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1398 return r;
1399 }
1400
1401 while (size) {
1402 u32 value;
1403
1404 r = get_user(value, (uint32_t *)buf);
1405 if (r)
1406 goto out;
1407
1408 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1409
1410 result += 4;
1411 buf += 4;
1412 *pos += 4;
1413 size -= 4;
1414 }
1415
1416 r = result;
1417out:
1418 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1419
1420 return r;
1421}
1422
1423
1424/**
1425 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1426 *
1427 * @f: open file handle
1428 * @buf: User buffer to store read data in
1429 * @size: Number of bytes to read
1430 * @pos: Offset to seek to
1431 */
1432static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1433 size_t size, loff_t *pos)
1434{
1435 struct amdgpu_device *adev = file_inode(f)->i_private;
1436 ssize_t result = 0;
1437 int r;
1438
1439 if (size & 0x3 || *pos & 0x3)
1440 return -EINVAL;
1441
1442 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1443 if (r < 0) {
1444 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1445 return r;
1446 }
1447
1448 while (size) {
1449 u64 value = 0;
1450
1451 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1452 if (r)
1453 goto out;
1454
1455 r = put_user(value, (u64 *)buf);
1456 if (r)
1457 goto out;
1458
1459 result += 4;
1460 buf += 4;
1461 *pos += 4;
1462 size -= 4;
1463 }
1464
1465 r = result;
1466out:
1467 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1468
1469 return r;
1470}
1471
1472/**
1473 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1474 *
1475 * @f: open file handle
1476 * @buf: User buffer to write data from
1477 * @size: Number of bytes to write
1478 * @pos: Offset to seek to
1479 *
1480 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1481 */
1482static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1483 size_t size, loff_t *pos)
1484{
1485 struct amdgpu_device *adev = file_inode(f)->i_private;
1486 ssize_t result = 0;
1487 int r;
1488
1489 if (size & 0x3 || *pos & 0x3)
1490 return -EINVAL;
1491
1492 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1493 if (r < 0) {
1494 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1495 return r;
1496 }
1497
1498 while (size) {
1499 uint32_t value;
1500
1501 r = get_user(value, (uint32_t *)buf);
1502 if (r)
1503 goto out;
1504
1505 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1506
1507 result += 4;
1508 buf += 4;
1509 *pos += 4;
1510 size -= 4;
1511 }
1512
1513 r = result;
1514out:
1515 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1516
1517 return r;
1518}
1519
1520
1521/**
1522 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1523 *
1524 * @f: open file handle
1525 * @buf: User buffer to store read data in
1526 * @size: Number of bytes to read
1527 * @pos: Offset to seek to
1528 */
1529static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1530 size_t size, loff_t *pos)
1531{
1532 struct amdgpu_device *adev = file_inode(f)->i_private;
1533 ssize_t result = 0;
1534 int r;
1535
1536 if (size & 0x3 || *pos & 0x3)
1537 return -EINVAL;
1538
1539 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1540 if (r < 0) {
1541 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1542 return r;
1543 }
1544
1545 while (size) {
1546 u32 value = adev->gfx.gfx_off_state;
1547
1548 r = put_user(value, (u32 *)buf);
1549 if (r)
1550 goto out;
1551
1552 result += 4;
1553 buf += 4;
1554 *pos += 4;
1555 size -= 4;
1556 }
1557
1558 r = result;
1559out:
1560 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1561
1562 return r;
1563}
1564
1565static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1566 size_t size, loff_t *pos)
1567{
1568 struct amdgpu_device *adev = file_inode(f)->i_private;
1569 ssize_t result = 0;
1570 int r;
1571
1572 if (size & 0x3 || *pos & 0x3)
1573 return -EINVAL;
1574
1575 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1576 if (r < 0) {
1577 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1578 return r;
1579 }
1580
1581 while (size) {
1582 u32 value;
1583
1584 r = amdgpu_get_gfx_off_status(adev, &value);
1585 if (r)
1586 goto out;
1587
1588 r = put_user(value, (u32 *)buf);
1589 if (r)
1590 goto out;
1591
1592 result += 4;
1593 buf += 4;
1594 *pos += 4;
1595 size -= 4;
1596 }
1597
1598 r = result;
1599out:
1600 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1601
1602 return r;
1603}
1604
1605static const struct file_operations amdgpu_debugfs_regs2_fops = {
1606 .owner = THIS_MODULE,
1607 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1608 .read = amdgpu_debugfs_regs2_read,
1609 .write = amdgpu_debugfs_regs2_write,
1610 .open = amdgpu_debugfs_regs2_open,
1611 .release = amdgpu_debugfs_regs2_release,
1612 .llseek = default_llseek
1613};
1614
1615static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1616 .owner = THIS_MODULE,
1617 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1618 .read = amdgpu_debugfs_gprwave_read,
1619 .open = amdgpu_debugfs_gprwave_open,
1620 .release = amdgpu_debugfs_gprwave_release,
1621 .llseek = default_llseek
1622};
1623
1624static const struct file_operations amdgpu_debugfs_regs_fops = {
1625 .owner = THIS_MODULE,
1626 .read = amdgpu_debugfs_regs_read,
1627 .write = amdgpu_debugfs_regs_write,
1628 .llseek = default_llseek
1629};
1630static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1631 .owner = THIS_MODULE,
1632 .read = amdgpu_debugfs_regs_didt_read,
1633 .write = amdgpu_debugfs_regs_didt_write,
1634 .llseek = default_llseek
1635};
1636static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1637 .owner = THIS_MODULE,
1638 .read = amdgpu_debugfs_regs_pcie_read,
1639 .write = amdgpu_debugfs_regs_pcie_write,
1640 .llseek = default_llseek
1641};
1642static const struct file_operations amdgpu_debugfs_regs_pcie64_fops = {
1643 .owner = THIS_MODULE,
1644 .read = amdgpu_debugfs_regs_pcie64_read,
1645 .write = amdgpu_debugfs_regs_pcie64_write,
1646 .llseek = default_llseek
1647};
1648static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1649 .owner = THIS_MODULE,
1650 .read = amdgpu_debugfs_regs_smc_read,
1651 .write = amdgpu_debugfs_regs_smc_write,
1652 .llseek = default_llseek
1653};
1654
1655static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1656 .owner = THIS_MODULE,
1657 .read = amdgpu_debugfs_gca_config_read,
1658 .llseek = default_llseek
1659};
1660
1661static const struct file_operations amdgpu_debugfs_sensors_fops = {
1662 .owner = THIS_MODULE,
1663 .read = amdgpu_debugfs_sensor_read,
1664 .llseek = default_llseek
1665};
1666
1667static const struct file_operations amdgpu_debugfs_wave_fops = {
1668 .owner = THIS_MODULE,
1669 .read = amdgpu_debugfs_wave_read,
1670 .llseek = default_llseek
1671};
1672static const struct file_operations amdgpu_debugfs_gpr_fops = {
1673 .owner = THIS_MODULE,
1674 .read = amdgpu_debugfs_gpr_read,
1675 .llseek = default_llseek
1676};
1677
1678static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1679 .owner = THIS_MODULE,
1680 .read = amdgpu_debugfs_gfxoff_read,
1681 .write = amdgpu_debugfs_gfxoff_write,
1682 .llseek = default_llseek
1683};
1684
1685static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1686 .owner = THIS_MODULE,
1687 .read = amdgpu_debugfs_gfxoff_status_read,
1688 .llseek = default_llseek
1689};
1690
1691static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1692 .owner = THIS_MODULE,
1693 .read = amdgpu_debugfs_gfxoff_count_read,
1694 .llseek = default_llseek
1695};
1696
1697static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1698 .owner = THIS_MODULE,
1699 .read = amdgpu_debugfs_gfxoff_residency_read,
1700 .write = amdgpu_debugfs_gfxoff_residency_write,
1701 .llseek = default_llseek
1702};
1703
1704static const struct file_operations *debugfs_regs[] = {
1705 &amdgpu_debugfs_regs_fops,
1706 &amdgpu_debugfs_regs2_fops,
1707 &amdgpu_debugfs_gprwave_fops,
1708 &amdgpu_debugfs_regs_didt_fops,
1709 &amdgpu_debugfs_regs_pcie_fops,
1710 &amdgpu_debugfs_regs_pcie64_fops,
1711 &amdgpu_debugfs_regs_smc_fops,
1712 &amdgpu_debugfs_gca_config_fops,
1713 &amdgpu_debugfs_sensors_fops,
1714 &amdgpu_debugfs_wave_fops,
1715 &amdgpu_debugfs_gpr_fops,
1716 &amdgpu_debugfs_gfxoff_fops,
1717 &amdgpu_debugfs_gfxoff_status_fops,
1718 &amdgpu_debugfs_gfxoff_count_fops,
1719 &amdgpu_debugfs_gfxoff_residency_fops,
1720};
1721
1722static const char * const debugfs_regs_names[] = {
1723 "amdgpu_regs",
1724 "amdgpu_regs2",
1725 "amdgpu_gprwave",
1726 "amdgpu_regs_didt",
1727 "amdgpu_regs_pcie",
1728 "amdgpu_regs_pcie64",
1729 "amdgpu_regs_smc",
1730 "amdgpu_gca_config",
1731 "amdgpu_sensors",
1732 "amdgpu_wave",
1733 "amdgpu_gpr",
1734 "amdgpu_gfxoff",
1735 "amdgpu_gfxoff_status",
1736 "amdgpu_gfxoff_count",
1737 "amdgpu_gfxoff_residency",
1738};
1739
1740/**
1741 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1742 * register access.
1743 *
1744 * @adev: The device to attach the debugfs entries to
1745 */
1746int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1747{
1748 struct drm_minor *minor = adev_to_drm(adev)->primary;
1749 struct dentry *ent, *root = minor->debugfs_root;
1750 unsigned int i;
1751
1752 if (security_locked_down(LOCKDOWN_PCI_ACCESS)) {
1753 drm_info(adev_to_drm(adev),
1754 "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n");
1755 return 0;
1756 }
1757
1758 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1759 ent = debugfs_create_file(debugfs_regs_names[i],
1760 S_IFREG | 0400, root,
1761 adev, debugfs_regs[i]);
1762 if (!i && !IS_ERR_OR_NULL(ent))
1763 i_size_write(ent->d_inode, adev->rmmio_size);
1764 }
1765
1766 return 0;
1767}
1768
1769static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1770{
1771 struct amdgpu_device *adev = m->private;
1772 struct drm_device *dev = adev_to_drm(adev);
1773 int r = 0, i;
1774
1775 r = pm_runtime_get_sync(dev->dev);
1776 if (r < 0) {
1777 pm_runtime_put_autosuspend(dev->dev);
1778 return r;
1779 }
1780
1781 /* Avoid accidently unparking the sched thread during GPU reset */
1782 r = down_write_killable(&adev->reset_domain->sem);
1783 if (r) {
1784 pm_runtime_put_autosuspend(dev->dev);
1785 return r;
1786 }
1787
1788 /* hold on the scheduler */
1789 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1790 struct amdgpu_ring *ring = adev->rings[i];
1791
1792 if (!amdgpu_ring_sched_ready(ring))
1793 continue;
1794 drm_sched_wqueue_stop(&ring->sched);
1795 }
1796
1797 seq_puts(m, "run ib test:\n");
1798 r = amdgpu_ib_ring_tests(adev);
1799 if (r)
1800 seq_printf(m, "ib ring tests failed (%d).\n", r);
1801 else
1802 seq_puts(m, "ib ring tests passed.\n");
1803
1804 /* go on the scheduler */
1805 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1806 struct amdgpu_ring *ring = adev->rings[i];
1807
1808 if (!amdgpu_ring_sched_ready(ring))
1809 continue;
1810 drm_sched_wqueue_start(&ring->sched);
1811 }
1812
1813 up_write(&adev->reset_domain->sem);
1814
1815 pm_runtime_put_autosuspend(dev->dev);
1816
1817 return 0;
1818}
1819
1820static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1821{
1822 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1823 struct drm_device *dev = adev_to_drm(adev);
1824 int r;
1825
1826 r = pm_runtime_get_sync(dev->dev);
1827 if (r < 0) {
1828 pm_runtime_put_autosuspend(dev->dev);
1829 return r;
1830 }
1831
1832 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1833
1834 pm_runtime_put_autosuspend(dev->dev);
1835
1836 return 0;
1837}
1838
1839
1840static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1841{
1842 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1843 struct drm_device *dev = adev_to_drm(adev);
1844 int r;
1845
1846 r = pm_runtime_get_sync(dev->dev);
1847 if (r < 0) {
1848 pm_runtime_put_autosuspend(dev->dev);
1849 return r;
1850 }
1851
1852 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1853
1854 pm_runtime_put_autosuspend(dev->dev);
1855
1856 return 0;
1857}
1858
1859static int amdgpu_debugfs_benchmark(void *data, u64 val)
1860{
1861 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1862 struct drm_device *dev = adev_to_drm(adev);
1863 int r;
1864
1865 r = pm_runtime_get_sync(dev->dev);
1866 if (r < 0) {
1867 pm_runtime_put_autosuspend(dev->dev);
1868 return r;
1869 }
1870
1871 r = amdgpu_benchmark(adev, val);
1872
1873 pm_runtime_put_autosuspend(dev->dev);
1874
1875 return r;
1876}
1877
1878static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1879{
1880 struct amdgpu_device *adev = m->private;
1881 struct drm_device *dev = adev_to_drm(adev);
1882 struct drm_file *file;
1883 int r;
1884
1885 r = mutex_lock_interruptible(&dev->filelist_mutex);
1886 if (r)
1887 return r;
1888
1889 list_for_each_entry(file, &dev->filelist, lhead) {
1890 struct amdgpu_fpriv *fpriv = file->driver_priv;
1891 struct amdgpu_vm *vm = &fpriv->vm;
1892 struct amdgpu_task_info *ti;
1893
1894 ti = amdgpu_vm_get_task_info_vm(vm);
1895 if (ti) {
1896 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->task.pid, ti->process_name);
1897 amdgpu_vm_put_task_info(ti);
1898 }
1899
1900 r = amdgpu_bo_reserve(vm->root.bo, true);
1901 if (r)
1902 break;
1903 amdgpu_debugfs_vm_bo_info(vm, m);
1904 amdgpu_bo_unreserve(vm->root.bo);
1905 }
1906
1907 mutex_unlock(&dev->filelist_mutex);
1908
1909 return r;
1910}
1911
1912DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1913DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1914DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1915 NULL, "%lld\n");
1916DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1917 NULL, "%lld\n");
1918DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1919 "%lld\n");
1920
1921static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1922 struct dma_fence **fences)
1923{
1924 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1925 uint32_t sync_seq, last_seq;
1926
1927 last_seq = atomic_read(&ring->fence_drv.last_seq);
1928 sync_seq = ring->fence_drv.sync_seq;
1929
1930 last_seq &= drv->num_fences_mask;
1931 sync_seq &= drv->num_fences_mask;
1932
1933 do {
1934 struct dma_fence *fence, **ptr;
1935
1936 ++last_seq;
1937 last_seq &= drv->num_fences_mask;
1938 ptr = &drv->fences[last_seq];
1939
1940 fence = rcu_dereference_protected(*ptr, 1);
1941 RCU_INIT_POINTER(*ptr, NULL);
1942
1943 if (!fence)
1944 continue;
1945
1946 fences[last_seq] = fence;
1947
1948 } while (last_seq != sync_seq);
1949}
1950
1951static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1952 int length)
1953{
1954 int i;
1955 struct dma_fence *fence;
1956
1957 for (i = 0; i < length; i++) {
1958 fence = fences[i];
1959 if (!fence)
1960 continue;
1961 dma_fence_signal(fence);
1962 dma_fence_put(fence);
1963 }
1964}
1965
1966static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1967{
1968 struct drm_sched_job *s_job;
1969 struct dma_fence *fence;
1970
1971 spin_lock(&sched->job_list_lock);
1972 list_for_each_entry(s_job, &sched->pending_list, list) {
1973 fence = sched->ops->run_job(s_job);
1974 dma_fence_put(fence);
1975 }
1976 spin_unlock(&sched->job_list_lock);
1977}
1978
1979static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1980{
1981 struct amdgpu_job *job;
1982 struct drm_sched_job *s_job, *tmp;
1983 uint32_t preempt_seq;
1984 struct dma_fence *fence, **ptr;
1985 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1986 struct drm_gpu_scheduler *sched = &ring->sched;
1987 bool preempted = true;
1988
1989 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1990 return;
1991
1992 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1993 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1994 preempted = false;
1995 goto no_preempt;
1996 }
1997
1998 preempt_seq &= drv->num_fences_mask;
1999 ptr = &drv->fences[preempt_seq];
2000 fence = rcu_dereference_protected(*ptr, 1);
2001
2002no_preempt:
2003 spin_lock(&sched->job_list_lock);
2004 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
2005 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
2006 /* remove job from ring_mirror_list */
2007 list_del_init(&s_job->list);
2008 sched->ops->free_job(s_job);
2009 continue;
2010 }
2011 job = to_amdgpu_job(s_job);
2012 if (preempted && (&job->hw_fence->base) == fence)
2013 /* mark the job as preempted */
2014 job->preemption_status |= AMDGPU_IB_PREEMPTED;
2015 }
2016 spin_unlock(&sched->job_list_lock);
2017}
2018
2019static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
2020{
2021 int r, length;
2022 struct amdgpu_ring *ring;
2023 struct dma_fence **fences = NULL;
2024 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2025
2026 if (val >= AMDGPU_MAX_RINGS)
2027 return -EINVAL;
2028
2029 ring = adev->rings[val];
2030
2031 if (!amdgpu_ring_sched_ready(ring) ||
2032 !ring->funcs->preempt_ib)
2033 return -EINVAL;
2034
2035 /* the last preemption failed */
2036 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
2037 return -EBUSY;
2038
2039 length = ring->fence_drv.num_fences_mask + 1;
2040 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
2041 if (!fences)
2042 return -ENOMEM;
2043
2044 /* Avoid accidently unparking the sched thread during GPU reset */
2045 r = down_read_killable(&adev->reset_domain->sem);
2046 if (r)
2047 goto pro_end;
2048
2049 /* stop the scheduler */
2050 drm_sched_wqueue_stop(&ring->sched);
2051
2052 /* preempt the IB */
2053 r = amdgpu_ring_preempt_ib(ring);
2054 if (r) {
2055 drm_warn(adev_to_drm(adev), "failed to preempt ring %d\n", ring->idx);
2056 goto failure;
2057 }
2058
2059 amdgpu_fence_process(ring);
2060
2061 if (atomic_read(&ring->fence_drv.last_seq) !=
2062 ring->fence_drv.sync_seq) {
2063 drm_info(adev_to_drm(adev), "ring %d was preempted\n", ring->idx);
2064
2065 amdgpu_ib_preempt_mark_partial_job(ring);
2066
2067 /* swap out the old fences */
2068 amdgpu_ib_preempt_fences_swap(ring, fences);
2069
2070 amdgpu_fence_driver_force_completion(ring, NULL);
2071
2072 /* resubmit unfinished jobs */
2073 amdgpu_ib_preempt_job_recovery(&ring->sched);
2074
2075 /* wait for jobs finished */
2076 amdgpu_fence_wait_empty(ring);
2077
2078 /* signal the old fences */
2079 amdgpu_ib_preempt_signal_fences(fences, length);
2080 }
2081
2082failure:
2083 /* restart the scheduler */
2084 drm_sched_wqueue_start(&ring->sched);
2085
2086 up_read(&adev->reset_domain->sem);
2087
2088pro_end:
2089 kfree(fences);
2090
2091 return r;
2092}
2093
2094static int amdgpu_debugfs_sclk_set(void *data, u64 val)
2095{
2096 int ret = 0;
2097 uint32_t max_freq, min_freq;
2098 struct amdgpu_device *adev = (struct amdgpu_device *)data;
2099
2100 if (amdgpu_sriov_multi_vf_mode(adev))
2101 return -EINVAL;
2102
2103 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
2104 if (ret < 0) {
2105 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2106 return ret;
2107 }
2108
2109 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
2110 if (ret == -EOPNOTSUPP) {
2111 ret = 0;
2112 goto out;
2113 }
2114 if (ret || val > max_freq || val < min_freq) {
2115 ret = -EINVAL;
2116 goto out;
2117 }
2118
2119 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2120 if (ret)
2121 ret = -EINVAL;
2122
2123out:
2124 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2125
2126 return ret;
2127}
2128
2129DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2130 amdgpu_debugfs_ib_preempt, "%llu\n");
2131
2132DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2133 amdgpu_debugfs_sclk_set, "%llu\n");
2134
2135int amdgpu_debugfs_init(struct amdgpu_device *adev)
2136{
2137 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2138 struct dentry *ent;
2139 int r, i;
2140
2141 if (!debugfs_initialized())
2142 return 0;
2143
2144 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2145 &adev->pm.smu_debug_mask);
2146
2147 debugfs_create_x64("unique_id", 0444, root, &adev->unique_id);
2148 debugfs_create_x8("unitid", 0444, root, &adev->unitid);
2149
2150 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2151 &fops_ib_preempt);
2152 if (IS_ERR(ent)) {
2153 drm_err(adev_to_drm(adev),
2154 "unable to create amdgpu_preempt_ib debugsfs file\n");
2155 return PTR_ERR(ent);
2156 }
2157
2158 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2159 &fops_sclk_set);
2160 if (IS_ERR(ent)) {
2161 drm_err(adev_to_drm(adev),
2162 "unable to create amdgpu_set_sclk debugsfs file\n");
2163 return PTR_ERR(ent);
2164 }
2165
2166 /* Register debugfs entries for amdgpu_ttm */
2167 amdgpu_ttm_debugfs_init(adev);
2168 amdgpu_debugfs_pm_init(adev);
2169 amdgpu_debugfs_sa_init(adev);
2170 amdgpu_debugfs_fence_init(adev);
2171 amdgpu_debugfs_gem_init(adev);
2172
2173 r = amdgpu_debugfs_regs_init(adev);
2174 if (r)
2175 drm_err(adev_to_drm(adev), "registering register debugfs failed (%d).\n", r);
2176
2177 amdgpu_debugfs_firmware_init(adev);
2178 amdgpu_ta_if_debugfs_init(adev);
2179
2180 amdgpu_debugfs_mes_event_log_init(adev);
2181
2182#if defined(CONFIG_DRM_AMD_DC)
2183 if (adev->dc_enabled)
2184 dtn_debugfs_init(adev);
2185#endif
2186
2187 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2188 struct amdgpu_ring *ring = adev->rings[i];
2189
2190 if (!ring)
2191 continue;
2192 if (ring == &adev->cper.ring_buf && !adev->cper.enabled)
2193 continue;
2194
2195 amdgpu_debugfs_ring_init(adev, ring);
2196 }
2197
2198 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2199 if (!amdgpu_vcnfw_log)
2200 break;
2201
2202 if (adev->vcn.harvest_config & (1 << i))
2203 continue;
2204
2205 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2206 }
2207
2208 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog)
2209 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm);
2210
2211 amdgpu_debugfs_vcn_sched_mask_init(adev);
2212 amdgpu_debugfs_jpeg_sched_mask_init(adev);
2213 amdgpu_debugfs_gfx_sched_mask_init(adev);
2214 amdgpu_debugfs_compute_sched_mask_init(adev);
2215 amdgpu_debugfs_sdma_sched_mask_init(adev);
2216
2217 amdgpu_ras_debugfs_create_all(adev);
2218 amdgpu_rap_debugfs_init(adev);
2219 amdgpu_securedisplay_debugfs_init(adev);
2220 amdgpu_fw_attestation_debugfs_init(adev);
2221 amdgpu_psp_debugfs_init(adev);
2222
2223 debugfs_create_file("amdgpu_evict_vram", 0400, root, adev,
2224 &amdgpu_evict_vram_fops);
2225 debugfs_create_file("amdgpu_evict_gtt", 0400, root, adev,
2226 &amdgpu_evict_gtt_fops);
2227 debugfs_create_file("amdgpu_test_ib", 0400, root, adev,
2228 &amdgpu_debugfs_test_ib_fops);
2229 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2230 &amdgpu_debugfs_vm_info_fops);
2231 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2232 &amdgpu_benchmark_fops);
2233
2234 adev->debugfs_vbios_blob.data = adev->bios;
2235 adev->debugfs_vbios_blob.size = adev->bios_size;
2236 debugfs_create_blob("amdgpu_vbios", 0444, root,
2237 &adev->debugfs_vbios_blob);
2238
2239 if (adev->discovery.debugfs_blob.size)
2240 debugfs_create_blob("amdgpu_discovery", 0444, root,
2241 &adev->discovery.debugfs_blob);
2242
2243 return 0;
2244}
2245
2246static int amdgpu_pt_info_read(struct seq_file *m, void *unused)
2247{
2248 struct drm_file *file;
2249 struct amdgpu_fpriv *fpriv;
2250 struct amdgpu_bo *root_bo;
2251 struct amdgpu_device *adev;
2252 int r;
2253
2254 file = m->private;
2255 if (!file)
2256 return -EINVAL;
2257
2258 adev = drm_to_adev(file->minor->dev);
2259 fpriv = file->driver_priv;
2260 if (!fpriv || !fpriv->vm.root.bo)
2261 return -ENODEV;
2262
2263 root_bo = amdgpu_bo_ref(fpriv->vm.root.bo);
2264 r = amdgpu_bo_reserve(root_bo, true);
2265 if (r) {
2266 amdgpu_bo_unref(&root_bo);
2267 return -EINVAL;
2268 }
2269
2270 seq_printf(m, "pd_address: 0x%llx\n", amdgpu_gmc_pd_addr(fpriv->vm.root.bo));
2271 seq_printf(m, "max_pfn: 0x%llx\n", adev->vm_manager.max_pfn);
2272 seq_printf(m, "num_level: 0x%x\n", adev->vm_manager.num_level);
2273 seq_printf(m, "block_size: 0x%x\n", adev->vm_manager.block_size);
2274 seq_printf(m, "fragment_size: 0x%x\n", adev->vm_manager.fragment_size);
2275
2276 amdgpu_bo_unreserve(root_bo);
2277 amdgpu_bo_unref(&root_bo);
2278
2279 return 0;
2280}
2281
2282static int amdgpu_pt_info_open(struct inode *inode, struct file *file)
2283{
2284 return single_open(file, amdgpu_pt_info_read, inode->i_private);
2285}
2286
2287static const struct file_operations amdgpu_pt_info_fops = {
2288 .owner = THIS_MODULE,
2289 .open = amdgpu_pt_info_open,
2290 .read = seq_read,
2291 .llseek = seq_lseek,
2292 .release = single_release,
2293};
2294
2295static int amdgpu_mqd_info_read(struct seq_file *m, void *unused)
2296{
2297 struct amdgpu_usermode_queue *queue = m->private;
2298 struct amdgpu_bo *bo;
2299 int r;
2300
2301 if (!queue || !queue->mqd.obj)
2302 return -EINVAL;
2303
2304 bo = amdgpu_bo_ref(queue->mqd.obj);
2305 r = amdgpu_bo_reserve(bo, true);
2306 if (r) {
2307 amdgpu_bo_unref(&bo);
2308 return -EINVAL;
2309 }
2310
2311 seq_printf(m, "queue_type: %d\n", queue->queue_type);
2312 seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));
2313
2314 amdgpu_bo_unreserve(bo);
2315 amdgpu_bo_unref(&bo);
2316
2317 return 0;
2318}
2319
2320static int amdgpu_mqd_info_open(struct inode *inode, struct file *file)
2321{
2322 return single_open(file, amdgpu_mqd_info_read, inode->i_private);
2323}
2324
2325static const struct file_operations amdgpu_mqd_info_fops = {
2326 .owner = THIS_MODULE,
2327 .open = amdgpu_mqd_info_open,
2328 .read = seq_read,
2329 .llseek = seq_lseek,
2330 .release = single_release,
2331};
2332
2333void amdgpu_debugfs_userq_init(struct drm_file *file, struct amdgpu_usermode_queue *queue, int qid)
2334{
2335 char queue_name[32];
2336
2337 scnprintf(queue_name, sizeof(queue_name), "queue_%d", qid);
2338 queue->debugfs_queue = debugfs_create_dir(queue_name, file->debugfs_client);
2339 debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
2340}
2341
2342void amdgpu_debugfs_vm_init(struct drm_file *file)
2343{
2344 debugfs_create_file("vm_pagetable_info", 0444, file->debugfs_client, file,
2345 &amdgpu_pt_info_fops);
2346}
2347
2348#else
2349int amdgpu_debugfs_init(struct amdgpu_device *adev)
2350{
2351 return 0;
2352}
2353int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2354{
2355 return 0;
2356}
2357void amdgpu_debugfs_vm_init(struct drm_file *file)
2358{
2359}
2360void amdgpu_debugfs_userq_init(struct drm_file *file,
2361 struct amdgpu_usermode_queue *queue,
2362 int qid)
2363{
2364}
2365#endif