projects/linux·blob master · refs

982 lines · 27.0 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright 2025 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#include <linux/delay.h>
25
26#include "amdgpu.h"
27#include "amdgpu_reset.h"
28#include "amdgpu_trace.h"
29#include "amdgpu_virt.h"
30#include "amdgpu_reg_access.h"
31
32#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
33#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
34#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
35
36void amdgpu_reg_access_init(struct amdgpu_device *adev)
37{
38 spin_lock_init(&adev->reg.smc.lock);
39 adev->reg.smc.rreg = NULL;
40 adev->reg.smc.wreg = NULL;
41
42 spin_lock_init(&adev->reg.uvd_ctx.lock);
43 adev->reg.uvd_ctx.rreg = NULL;
44 adev->reg.uvd_ctx.wreg = NULL;
45
46 spin_lock_init(&adev->reg.didt.lock);
47 adev->reg.didt.rreg = NULL;
48 adev->reg.didt.wreg = NULL;
49
50 spin_lock_init(&adev->reg.gc_cac.lock);
51 adev->reg.gc_cac.rreg = NULL;
52 adev->reg.gc_cac.wreg = NULL;
53
54 spin_lock_init(&adev->reg.se_cac.lock);
55 adev->reg.se_cac.rreg = NULL;
56 adev->reg.se_cac.wreg = NULL;
57
58 spin_lock_init(&adev->reg.audio_endpt.lock);
59 adev->reg.audio_endpt.rreg = NULL;
60 adev->reg.audio_endpt.wreg = NULL;
61
62 spin_lock_init(&adev->reg.pcie.lock);
63 adev->reg.pcie.rreg = NULL;
64 adev->reg.pcie.wreg = NULL;
65 adev->reg.pcie.rreg_ext = NULL;
66 adev->reg.pcie.wreg_ext = NULL;
67 adev->reg.pcie.rreg64 = NULL;
68 adev->reg.pcie.wreg64 = NULL;
69 adev->reg.pcie.rreg64_ext = NULL;
70 adev->reg.pcie.wreg64_ext = NULL;
71 adev->reg.pcie.port_rreg = NULL;
72 adev->reg.pcie.port_wreg = NULL;
73}
74
75uint32_t amdgpu_reg_smc_rd32(struct amdgpu_device *adev, uint32_t reg)
76{
77 if (!adev->reg.smc.rreg) {
78 dev_err_once(adev->dev, "SMC register read not supported\n");
79 return 0;
80 }
81 return adev->reg.smc.rreg(adev, reg);
82}
83
84void amdgpu_reg_smc_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
85{
86 if (!adev->reg.smc.wreg) {
87 dev_err_once(adev->dev, "SMC register write not supported\n");
88 return;
89 }
90 adev->reg.smc.wreg(adev, reg, v);
91}
92
93uint32_t amdgpu_reg_uvd_ctx_rd32(struct amdgpu_device *adev, uint32_t reg)
94{
95 if (!adev->reg.uvd_ctx.rreg) {
96 dev_err_once(adev->dev,
97 "UVD_CTX register read not supported\n");
98 return 0;
99 }
100 return adev->reg.uvd_ctx.rreg(adev, reg);
101}
102
103void amdgpu_reg_uvd_ctx_wr32(struct amdgpu_device *adev, uint32_t reg,
104 uint32_t v)
105{
106 if (!adev->reg.uvd_ctx.wreg) {
107 dev_err_once(adev->dev,
108 "UVD_CTX register write not supported\n");
109 return;
110 }
111 adev->reg.uvd_ctx.wreg(adev, reg, v);
112}
113
114uint32_t amdgpu_reg_didt_rd32(struct amdgpu_device *adev, uint32_t reg)
115{
116 if (!adev->reg.didt.rreg) {
117 dev_err_once(adev->dev, "DIDT register read not supported\n");
118 return 0;
119 }
120 return adev->reg.didt.rreg(adev, reg);
121}
122
123void amdgpu_reg_didt_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
124{
125 if (!adev->reg.didt.wreg) {
126 dev_err_once(adev->dev, "DIDT register write not supported\n");
127 return;
128 }
129 adev->reg.didt.wreg(adev, reg, v);
130}
131
132uint32_t amdgpu_reg_gc_cac_rd32(struct amdgpu_device *adev, uint32_t reg)
133{
134 if (!adev->reg.gc_cac.rreg) {
135 dev_err_once(adev->dev, "GC_CAC register read not supported\n");
136 return 0;
137 }
138 return adev->reg.gc_cac.rreg(adev, reg);
139}
140
141void amdgpu_reg_gc_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
142 uint32_t v)
143{
144 if (!adev->reg.gc_cac.wreg) {
145 dev_err_once(adev->dev,
146 "GC_CAC register write not supported\n");
147 return;
148 }
149 adev->reg.gc_cac.wreg(adev, reg, v);
150}
151
152uint32_t amdgpu_reg_se_cac_rd32(struct amdgpu_device *adev, uint32_t reg)
153{
154 if (!adev->reg.se_cac.rreg) {
155 dev_err_once(adev->dev, "SE_CAC register read not supported\n");
156 return 0;
157 }
158 return adev->reg.se_cac.rreg(adev, reg);
159}
160
161void amdgpu_reg_se_cac_wr32(struct amdgpu_device *adev, uint32_t reg,
162 uint32_t v)
163{
164 if (!adev->reg.se_cac.wreg) {
165 dev_err_once(adev->dev,
166 "SE_CAC register write not supported\n");
167 return;
168 }
169 adev->reg.se_cac.wreg(adev, reg, v);
170}
171
172uint32_t amdgpu_reg_audio_endpt_rd32(struct amdgpu_device *adev, uint32_t block,
173 uint32_t reg)
174{
175 if (!adev->reg.audio_endpt.rreg) {
176 dev_err_once(adev->dev,
177 "AUDIO_ENDPT register read not supported\n");
178 return 0;
179 }
180 return adev->reg.audio_endpt.rreg(adev, block, reg);
181}
182
183void amdgpu_reg_audio_endpt_wr32(struct amdgpu_device *adev, uint32_t block,
184 uint32_t reg, uint32_t v)
185{
186 if (!adev->reg.audio_endpt.wreg) {
187 dev_err_once(adev->dev,
188 "AUDIO_ENDPT register write not supported\n");
189 return;
190 }
191 adev->reg.audio_endpt.wreg(adev, block, reg, v);
192}
193
194uint32_t amdgpu_reg_pcie_rd32(struct amdgpu_device *adev, uint32_t reg)
195{
196 if (!adev->reg.pcie.rreg) {
197 dev_err_once(adev->dev, "PCIE register read not supported\n");
198 return 0;
199 }
200 return adev->reg.pcie.rreg(adev, reg);
201}
202
203void amdgpu_reg_pcie_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
204{
205 if (!adev->reg.pcie.wreg) {
206 dev_err_once(adev->dev, "PCIE register write not supported\n");
207 return;
208 }
209 adev->reg.pcie.wreg(adev, reg, v);
210}
211
212uint32_t amdgpu_reg_pcie_ext_rd32(struct amdgpu_device *adev, uint64_t reg)
213{
214 if (!adev->reg.pcie.rreg_ext) {
215 dev_err_once(adev->dev, "PCIE EXT register read not supported\n");
216 return 0;
217 }
218 return adev->reg.pcie.rreg_ext(adev, reg);
219}
220
221void amdgpu_reg_pcie_ext_wr32(struct amdgpu_device *adev, uint64_t reg,
222 uint32_t v)
223{
224 if (!adev->reg.pcie.wreg_ext) {
225 dev_err_once(adev->dev, "PCIE EXT register write not supported\n");
226 return;
227 }
228 adev->reg.pcie.wreg_ext(adev, reg, v);
229}
230
231uint64_t amdgpu_reg_pcie_rd64(struct amdgpu_device *adev, uint32_t reg)
232{
233 if (!adev->reg.pcie.rreg64) {
234 dev_err_once(adev->dev, "PCIE 64-bit register read not supported\n");
235 return 0;
236 }
237 return adev->reg.pcie.rreg64(adev, reg);
238}
239
240void amdgpu_reg_pcie_wr64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
241{
242 if (!adev->reg.pcie.wreg64) {
243 dev_err_once(adev->dev, "PCIE 64-bit register write not supported\n");
244 return;
245 }
246 adev->reg.pcie.wreg64(adev, reg, v);
247}
248
249uint64_t amdgpu_reg_pcie_ext_rd64(struct amdgpu_device *adev, uint64_t reg)
250{
251 if (!adev->reg.pcie.rreg64_ext) {
252 dev_err_once(adev->dev, "PCIE EXT 64-bit register read not supported\n");
253 return 0;
254 }
255 return adev->reg.pcie.rreg64_ext(adev, reg);
256}
257
258void amdgpu_reg_pcie_ext_wr64(struct amdgpu_device *adev, uint64_t reg,
259 uint64_t v)
260{
261 if (!adev->reg.pcie.wreg64_ext) {
262 dev_err_once(adev->dev, "PCIE EXT 64-bit register write not supported\n");
263 return;
264 }
265 adev->reg.pcie.wreg64_ext(adev, reg, v);
266}
267
268uint32_t amdgpu_reg_pciep_rd32(struct amdgpu_device *adev, uint32_t reg)
269{
270 if (!adev->reg.pcie.port_rreg) {
271 dev_err_once(adev->dev, "PCIEP register read not supported\n");
272 return 0;
273 }
274 return adev->reg.pcie.port_rreg(adev, reg);
275}
276
277void amdgpu_reg_pciep_wr32(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
278{
279 if (!adev->reg.pcie.port_wreg) {
280 dev_err_once(adev->dev, "PCIEP register write not supported\n");
281 return;
282 }
283 adev->reg.pcie.port_wreg(adev, reg, v);
284}
285
286static int amdgpu_reg_get_smn_base_version(struct amdgpu_device *adev)
287{
288 struct pci_dev *pdev = adev->pdev;
289 int id;
290
291 if (amdgpu_sriov_vf(adev))
292 return -EOPNOTSUPP;
293
294 id = (pdev->device >> 4) & 0xFFFF;
295 if (id == 0x74A || id == 0x74B || id == 0x75A || id == 0x75B)
296 return 1;
297
298 return -EOPNOTSUPP;
299}
300
301uint64_t amdgpu_reg_get_smn_base64(struct amdgpu_device *adev,
302 enum amd_hw_ip_block_type block,
303 int die_inst)
304{
305 if (!adev->reg.smn.get_smn_base) {
306 int version = amdgpu_reg_get_smn_base_version(adev);
307 switch (version) {
308 case 1:
309 return amdgpu_reg_smn_v1_0_get_base(adev, block,
310 die_inst);
311 default:
312 dev_err_once(
313 adev->dev,
314 "SMN base address query not supported for this device\n");
315 return 0;
316 }
317 }
318 return adev->reg.smn.get_smn_base(adev, block, die_inst);
319}
320
321uint64_t amdgpu_reg_smn_v1_0_get_base(struct amdgpu_device *adev,
322 enum amd_hw_ip_block_type block,
323 int die_inst)
324{
325 uint64_t smn_base;
326
327 if (die_inst == 0)
328 return 0;
329
330 switch (block) {
331 case XGMI_HWIP:
332 case NBIO_HWIP:
333 case MP0_HWIP:
334 case UMC_HWIP:
335 case DF_HWIP:
336 smn_base = ((uint64_t)(die_inst & 0x3) << 32) | (1ULL << 34);
337 break;
338 default:
339 dev_warn_once(
340 adev->dev,
341 "SMN base address query not supported for this block %d\n",
342 block);
343 smn_base = 0;
344 break;
345 }
346
347 return smn_base;
348}
349
350/*
351 * register access helper functions.
352 */
353
354/**
355 * amdgpu_device_rreg - read a memory mapped IO or indirect register
356 *
357 * @adev: amdgpu_device pointer
358 * @reg: dword aligned register offset
359 * @acc_flags: access flags which require special behavior
360 *
361 * Returns the 32 bit value from the offset specified.
362 */
363uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, uint32_t reg,
364 uint32_t acc_flags)
365{
366 uint32_t ret;
367
368 if (amdgpu_device_skip_hw_access(adev))
369 return 0;
370
371 if ((reg * 4) < adev->rmmio_size) {
372 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
373 amdgpu_sriov_runtime(adev) &&
374 down_read_trylock(&adev->reset_domain->sem)) {
375 ret = amdgpu_kiq_rreg(adev, reg, 0);
376 up_read(&adev->reset_domain->sem);
377 } else {
378 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
379 }
380 } else {
381 ret = amdgpu_reg_pcie_rd32(adev, reg * 4);
382 }
383
384 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
385
386 return ret;
387}
388
389/*
390 * MMIO register read with bytes helper functions
391 * @offset:bytes offset from MMIO start
392 */
393
394/**
395 * amdgpu_mm_rreg8 - read a memory mapped IO register
396 *
397 * @adev: amdgpu_device pointer
398 * @offset: byte aligned register offset
399 *
400 * Returns the 8 bit value from the offset specified.
401 */
402uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
403{
404 if (amdgpu_device_skip_hw_access(adev))
405 return 0;
406
407 if (offset < adev->rmmio_size)
408 return (readb(adev->rmmio + offset));
409
410 dev_err(adev->dev, "invalid MMIO read offset 0x%x (rmmio size 0x%x)\n",
411 offset, (unsigned int)adev->rmmio_size);
412 return 0;
413}
414
415/**
416 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
417 *
418 * @adev: amdgpu_device pointer
419 * @reg: dword aligned register offset
420 * @acc_flags: access flags which require special behavior
421 * @xcc_id: xcc accelerated compute core id
422 *
423 * Returns the 32 bit value from the offset specified.
424 */
425uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev, uint32_t reg,
426 uint32_t acc_flags, uint32_t xcc_id)
427{
428 uint32_t ret, rlcg_flag;
429
430 if (amdgpu_device_skip_hw_access(adev))
431 return 0;
432
433 if ((reg * 4) < adev->rmmio_size) {
434 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) &&
435 adev->gfx.rlc.rlcg_reg_access_supported &&
436 amdgpu_virt_get_rlcg_reg_access_flag(
437 adev, acc_flags, GC_HWIP, false, &rlcg_flag)) {
438 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag,
439 GET_INST(GC, xcc_id));
440 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
441 amdgpu_sriov_runtime(adev) &&
442 down_read_trylock(&adev->reset_domain->sem)) {
443 ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
444 up_read(&adev->reset_domain->sem);
445 } else {
446 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
447 }
448 } else {
449 ret = amdgpu_reg_pcie_rd32(adev, reg * 4);
450 }
451
452 return ret;
453}
454
455/*
456 * MMIO register write with bytes helper functions
457 * @offset:bytes offset from MMIO start
458 * @value: the value want to be written to the register
459 */
460
461/**
462 * amdgpu_mm_wreg8 - read a memory mapped IO register
463 *
464 * @adev: amdgpu_device pointer
465 * @offset: byte aligned register offset
466 * @value: 8 bit value to write
467 *
468 * Writes the value specified to the offset specified.
469 */
470void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
471{
472 if (amdgpu_device_skip_hw_access(adev))
473 return;
474
475 if (offset < adev->rmmio_size) {
476 writeb(value, adev->rmmio + offset);
477 } else {
478 dev_err(adev->dev, "invalid MMIO write offset 0x%x (rmmio size 0x%x)\n",
479 offset, (unsigned int)adev->rmmio_size);
480 return;
481 }
482}
483
484/**
485 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
486 *
487 * @adev: amdgpu_device pointer
488 * @reg: dword aligned register offset
489 * @v: 32 bit value to write to the register
490 * @acc_flags: access flags which require special behavior
491 *
492 * Writes the value specified to the offset specified.
493 */
494void amdgpu_device_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
495 uint32_t acc_flags)
496{
497 if (amdgpu_device_skip_hw_access(adev))
498 return;
499
500 if ((reg * 4) < adev->rmmio_size) {
501 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
502 amdgpu_sriov_runtime(adev) &&
503 down_read_trylock(&adev->reset_domain->sem)) {
504 amdgpu_kiq_wreg(adev, reg, v, 0);
505 up_read(&adev->reset_domain->sem);
506 } else {
507 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
508 }
509 } else {
510 amdgpu_reg_pcie_wr32(adev, reg * 4, v);
511 }
512
513 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
514}
515
516/**
517 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
518 *
519 * @adev: amdgpu_device pointer
520 * @reg: mmio/rlc register
521 * @v: value to write
522 * @xcc_id: xcc accelerated compute core id
523 *
524 * this function is invoked only for the debugfs register access
525 */
526void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, uint32_t reg,
527 uint32_t v, uint32_t xcc_id)
528{
529 if (amdgpu_device_skip_hw_access(adev))
530 return;
531
532 if (amdgpu_sriov_fullaccess(adev) && adev->gfx.rlc.funcs &&
533 adev->gfx.rlc.funcs->is_rlcg_access_range) {
534 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
535 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
536 } else if ((reg * 4) >= adev->rmmio_size) {
537 amdgpu_reg_pcie_wr32(adev, reg * 4, v);
538 } else {
539 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
540 }
541}
542
543/**
544 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
545 *
546 * @adev: amdgpu_device pointer
547 * @reg: dword aligned register offset
548 * @v: 32 bit value to write to the register
549 * @acc_flags: access flags which require special behavior
550 * @xcc_id: xcc accelerated compute core id
551 *
552 * Writes the value specified to the offset specified.
553 */
554void amdgpu_device_xcc_wreg(struct amdgpu_device *adev, uint32_t reg,
555 uint32_t v, uint32_t acc_flags, uint32_t xcc_id)
556{
557 uint32_t rlcg_flag;
558
559 if (amdgpu_device_skip_hw_access(adev))
560 return;
561
562 if ((reg * 4) < adev->rmmio_size) {
563 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_runtime(adev) &&
564 adev->gfx.rlc.rlcg_reg_access_supported &&
565 amdgpu_virt_get_rlcg_reg_access_flag(
566 adev, acc_flags, GC_HWIP, true, &rlcg_flag)) {
567 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag,
568 GET_INST(GC, xcc_id));
569 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
570 amdgpu_sriov_runtime(adev) &&
571 down_read_trylock(&adev->reset_domain->sem)) {
572 amdgpu_kiq_wreg(adev, reg, v, xcc_id);
573 up_read(&adev->reset_domain->sem);
574 } else {
575 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
576 }
577 } else {
578 amdgpu_reg_pcie_wr32(adev, reg * 4, v);
579 }
580}
581
582/**
583 * amdgpu_device_indirect_rreg - read an indirect register
584 *
585 * @adev: amdgpu_device pointer
586 * @reg_addr: indirect register address to read from
587 *
588 * Returns the value of indirect register @reg_addr
589 */
590u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, u32 reg_addr)
591{
592 unsigned long flags, pcie_index, pcie_data;
593 void __iomem *pcie_index_offset;
594 void __iomem *pcie_data_offset;
595 u32 r;
596
597 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
598 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
599
600 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
601 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
602 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
603
604 writel(reg_addr, pcie_index_offset);
605 readl(pcie_index_offset);
606 r = readl(pcie_data_offset);
607 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
608
609 return r;
610}
611
612u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev, u64 reg_addr)
613{
614 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
615 u32 r;
616 void __iomem *pcie_index_offset;
617 void __iomem *pcie_index_hi_offset;
618 void __iomem *pcie_data_offset;
619
620 if (unlikely(!adev->nbio.funcs)) {
621 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
622 pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
623 } else {
624 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
625 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
626 }
627
628 if (reg_addr >> 32) {
629 if (unlikely(!adev->nbio.funcs))
630 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
631 else
632 pcie_index_hi =
633 adev->nbio.funcs->get_pcie_index_hi_offset(
634 adev);
635 } else {
636 pcie_index_hi = 0;
637 }
638
639 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
640 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
641 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
642 if (pcie_index_hi != 0)
643 pcie_index_hi_offset =
644 (void __iomem *)adev->rmmio + pcie_index_hi * 4;
645
646 writel(reg_addr, pcie_index_offset);
647 readl(pcie_index_offset);
648 if (pcie_index_hi != 0) {
649 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
650 readl(pcie_index_hi_offset);
651 }
652 r = readl(pcie_data_offset);
653
654 /* clear the high bits */
655 if (pcie_index_hi != 0) {
656 writel(0, pcie_index_hi_offset);
657 readl(pcie_index_hi_offset);
658 }
659
660 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
661
662 return r;
663}
664
665/**
666 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
667 *
668 * @adev: amdgpu_device pointer
669 * @reg_addr: indirect register address to read from
670 *
671 * Returns the value of indirect register @reg_addr
672 */
673u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, u32 reg_addr)
674{
675 unsigned long flags, pcie_index, pcie_data;
676 void __iomem *pcie_index_offset;
677 void __iomem *pcie_data_offset;
678 u64 r;
679
680 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
681 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
682
683 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
684 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
685 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
686
687 /* read low 32 bits */
688 writel(reg_addr, pcie_index_offset);
689 readl(pcie_index_offset);
690 r = readl(pcie_data_offset);
691 /* read high 32 bits */
692 writel(reg_addr + 4, pcie_index_offset);
693 readl(pcie_index_offset);
694 r |= ((u64)readl(pcie_data_offset) << 32);
695 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
696
697 return r;
698}
699
700u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev, u64 reg_addr)
701{
702 unsigned long flags, pcie_index, pcie_data;
703 unsigned long pcie_index_hi = 0;
704 void __iomem *pcie_index_offset;
705 void __iomem *pcie_index_hi_offset;
706 void __iomem *pcie_data_offset;
707 u64 r;
708
709 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
710 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
711 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
712 pcie_index_hi =
713 adev->nbio.funcs->get_pcie_index_hi_offset(adev);
714
715 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
716 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
717 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
718 if (pcie_index_hi != 0)
719 pcie_index_hi_offset =
720 (void __iomem *)adev->rmmio + pcie_index_hi * 4;
721
722 /* read low 32 bits */
723 writel(reg_addr, pcie_index_offset);
724 readl(pcie_index_offset);
725 if (pcie_index_hi != 0) {
726 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
727 readl(pcie_index_hi_offset);
728 }
729 r = readl(pcie_data_offset);
730 /* read high 32 bits */
731 writel(reg_addr + 4, pcie_index_offset);
732 readl(pcie_index_offset);
733 if (pcie_index_hi != 0) {
734 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
735 readl(pcie_index_hi_offset);
736 }
737 r |= ((u64)readl(pcie_data_offset) << 32);
738
739 /* clear the high bits */
740 if (pcie_index_hi != 0) {
741 writel(0, pcie_index_hi_offset);
742 readl(pcie_index_hi_offset);
743 }
744
745 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
746
747 return r;
748}
749
750/**
751 * amdgpu_device_indirect_wreg - write an indirect register address
752 *
753 * @adev: amdgpu_device pointer
754 * @reg_addr: indirect register offset
755 * @reg_data: indirect register data
756 *
757 */
758void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, u32 reg_addr,
759 u32 reg_data)
760{
761 unsigned long flags, pcie_index, pcie_data;
762 void __iomem *pcie_index_offset;
763 void __iomem *pcie_data_offset;
764
765 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
766 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
767
768 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
769 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
770 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
771
772 writel(reg_addr, pcie_index_offset);
773 readl(pcie_index_offset);
774 writel(reg_data, pcie_data_offset);
775 readl(pcie_data_offset);
776 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
777}
778
779void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev, u64 reg_addr,
780 u32 reg_data)
781{
782 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
783 void __iomem *pcie_index_offset;
784 void __iomem *pcie_index_hi_offset;
785 void __iomem *pcie_data_offset;
786
787 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
788 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
789 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
790 pcie_index_hi =
791 adev->nbio.funcs->get_pcie_index_hi_offset(adev);
792 else
793 pcie_index_hi = 0;
794
795 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
796 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
797 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
798 if (pcie_index_hi != 0)
799 pcie_index_hi_offset =
800 (void __iomem *)adev->rmmio + pcie_index_hi * 4;
801
802 writel(reg_addr, pcie_index_offset);
803 readl(pcie_index_offset);
804 if (pcie_index_hi != 0) {
805 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
806 readl(pcie_index_hi_offset);
807 }
808 writel(reg_data, pcie_data_offset);
809 readl(pcie_data_offset);
810
811 /* clear the high bits */
812 if (pcie_index_hi != 0) {
813 writel(0, pcie_index_hi_offset);
814 readl(pcie_index_hi_offset);
815 }
816
817 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
818}
819
820/**
821 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
822 *
823 * @adev: amdgpu_device pointer
824 * @reg_addr: indirect register offset
825 * @reg_data: indirect register data
826 *
827 */
828void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, u32 reg_addr,
829 u64 reg_data)
830{
831 unsigned long flags, pcie_index, pcie_data;
832 void __iomem *pcie_index_offset;
833 void __iomem *pcie_data_offset;
834
835 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
836 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
837
838 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
839 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
840 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
841
842 /* write low 32 bits */
843 writel(reg_addr, pcie_index_offset);
844 readl(pcie_index_offset);
845 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
846 readl(pcie_data_offset);
847 /* write high 32 bits */
848 writel(reg_addr + 4, pcie_index_offset);
849 readl(pcie_index_offset);
850 writel((u32)(reg_data >> 32), pcie_data_offset);
851 readl(pcie_data_offset);
852 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
853}
854
855void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev, u64 reg_addr,
856 u64 reg_data)
857{
858 unsigned long flags, pcie_index, pcie_data;
859 unsigned long pcie_index_hi = 0;
860 void __iomem *pcie_index_offset;
861 void __iomem *pcie_index_hi_offset;
862 void __iomem *pcie_data_offset;
863
864 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
865 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
866 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
867 pcie_index_hi =
868 adev->nbio.funcs->get_pcie_index_hi_offset(adev);
869
870 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
871 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
872 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
873 if (pcie_index_hi != 0)
874 pcie_index_hi_offset =
875 (void __iomem *)adev->rmmio + pcie_index_hi * 4;
876
877 /* write low 32 bits */
878 writel(reg_addr, pcie_index_offset);
879 readl(pcie_index_offset);
880 if (pcie_index_hi != 0) {
881 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
882 readl(pcie_index_hi_offset);
883 }
884 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
885 readl(pcie_data_offset);
886 /* write high 32 bits */
887 writel(reg_addr + 4, pcie_index_offset);
888 readl(pcie_index_offset);
889 if (pcie_index_hi != 0) {
890 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
891 readl(pcie_index_hi_offset);
892 }
893 writel((u32)(reg_data >> 32), pcie_data_offset);
894 readl(pcie_data_offset);
895
896 /* clear the high bits */
897 if (pcie_index_hi != 0) {
898 writel(0, pcie_index_hi_offset);
899 readl(pcie_index_hi_offset);
900 }
901
902 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
903}
904
905u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg)
906{
907 unsigned long flags, address, data;
908 u32 r;
909
910 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
911 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
912
913 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
914 WREG32(address, reg * 4);
915 (void)RREG32(address);
916 r = RREG32(data);
917 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
918 return r;
919}
920
921void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
922{
923 unsigned long flags, address, data;
924
925 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
926 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
927
928 spin_lock_irqsave(&adev->reg.pcie.lock, flags);
929 WREG32(address, reg * 4);
930 (void)RREG32(address);
931 WREG32(data, v);
932 (void)RREG32(data);
933 spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
934}
935
936uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
937 uint32_t reg_addr, char reg_name[],
938 uint32_t expected_value, uint32_t mask)
939{
940 uint32_t ret = 0;
941 uint32_t old_ = 0;
942 uint32_t tmp_ = RREG32(reg_addr);
943 uint32_t loop = adev->usec_timeout;
944
945 while ((tmp_ & (mask)) != (expected_value)) {
946 if (old_ != tmp_) {
947 loop = adev->usec_timeout;
948 old_ = tmp_;
949 } else
950 udelay(1);
951 tmp_ = RREG32(reg_addr);
952 loop--;
953 if (!loop) {
954 dev_warn(
955 adev->dev,
956 "Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
957 inst, reg_name, (uint32_t)expected_value,
958 (uint32_t)(tmp_ & (mask)));
959 ret = -ETIMEDOUT;
960 break;
961 }
962 }
963 return ret;
964}
965
966
967uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev,
968 u32 se_num, u32 sh_num, u32 reg_offset)
969{
970 uint32_t val;
971
972 mutex_lock(&adev->grbm_idx_mutex);
973 if (se_num != 0xffffffff || sh_num != 0xffffffff)
974 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
975
976 val = RREG32(reg_offset);
977
978 if (se_num != 0xffffffff || sh_num != 0xffffffff)
979 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
980 mutex_unlock(&adev->grbm_idx_mutex);
981 return val;
982}