projects/linux·blob master · refs

1129 lines · 28.0 KiB

1/*
2 * Copyright 2022 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include "amdgpu.h"
24#include "amdgpu_xcp.h"
25#include "amdgpu_drv.h"
26
27#include <drm/drm_drv.h>
28#include "../amdxcp/amdgpu_xcp_drv.h"
29
30static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr);
31static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr);
32
33static int __amdgpu_xcp_run(struct amdgpu_xcp_mgr *xcp_mgr,
34 struct amdgpu_xcp_ip *xcp_ip, int xcp_state)
35{
36 int (*run_func)(void *handle, uint32_t inst_mask);
37 int ret = 0;
38
39 if (!xcp_ip || !xcp_ip->valid || !xcp_ip->ip_funcs)
40 return 0;
41
42 run_func = NULL;
43
44 switch (xcp_state) {
45 case AMDGPU_XCP_PREPARE_SUSPEND:
46 run_func = xcp_ip->ip_funcs->prepare_suspend;
47 break;
48 case AMDGPU_XCP_SUSPEND:
49 run_func = xcp_ip->ip_funcs->suspend;
50 break;
51 case AMDGPU_XCP_PREPARE_RESUME:
52 run_func = xcp_ip->ip_funcs->prepare_resume;
53 break;
54 case AMDGPU_XCP_RESUME:
55 run_func = xcp_ip->ip_funcs->resume;
56 break;
57 }
58
59 if (run_func)
60 ret = run_func(xcp_mgr->adev, xcp_ip->inst_mask);
61
62 return ret;
63}
64
65static int amdgpu_xcp_run_transition(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
66 int state)
67{
68 struct amdgpu_xcp_ip *xcp_ip;
69 struct amdgpu_xcp *xcp;
70 int i, ret;
71
72 if (xcp_id >= MAX_XCP || !xcp_mgr->xcp[xcp_id].valid)
73 return -EINVAL;
74
75 xcp = &xcp_mgr->xcp[xcp_id];
76 for (i = 0; i < AMDGPU_XCP_MAX_BLOCKS; ++i) {
77 xcp_ip = &xcp->ip[i];
78 ret = __amdgpu_xcp_run(xcp_mgr, xcp_ip, state);
79 if (ret)
80 break;
81 }
82
83 return ret;
84}
85
86int amdgpu_xcp_prepare_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
87{
88 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
89 AMDGPU_XCP_PREPARE_SUSPEND);
90}
91
92int amdgpu_xcp_suspend(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
93{
94 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_SUSPEND);
95}
96
97int amdgpu_xcp_prepare_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
98{
99 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id,
100 AMDGPU_XCP_PREPARE_RESUME);
101}
102
103int amdgpu_xcp_resume(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id)
104{
105 return amdgpu_xcp_run_transition(xcp_mgr, xcp_id, AMDGPU_XCP_RESUME);
106}
107
108static void __amdgpu_xcp_add_block(struct amdgpu_xcp_mgr *xcp_mgr, int xcp_id,
109 struct amdgpu_xcp_ip *ip)
110{
111 struct amdgpu_xcp *xcp;
112
113 if (!ip)
114 return;
115
116 xcp = &xcp_mgr->xcp[xcp_id];
117 xcp->ip[ip->ip_id] = *ip;
118 xcp->ip[ip->ip_id].valid = true;
119
120 xcp->valid = true;
121}
122
123static void __amdgpu_xcp_set_unique_id(struct amdgpu_xcp_mgr *xcp_mgr,
124 int xcp_id)
125{
126 struct amdgpu_xcp *xcp = &xcp_mgr->xcp[xcp_id];
127 struct amdgpu_device *adev = xcp_mgr->adev;
128 uint32_t inst_mask;
129 uint64_t uid;
130 int i;
131
132 if (!amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask) &&
133 inst_mask) {
134 i = GET_INST(GC, (ffs(inst_mask) - 1));
135 uid = amdgpu_device_get_uid(xcp_mgr->adev->uid_info,
136 AMDGPU_UID_TYPE_XCD, i);
137 if (uid)
138 xcp->unique_id = uid;
139 }
140}
141
142int amdgpu_xcp_init(struct amdgpu_xcp_mgr *xcp_mgr, int num_xcps, int mode)
143{
144 struct amdgpu_device *adev = xcp_mgr->adev;
145 struct amdgpu_xcp_ip ip;
146 uint8_t mem_id;
147 int i, j, ret;
148
149 if (!num_xcps || num_xcps > MAX_XCP)
150 return -EINVAL;
151
152 xcp_mgr->mode = mode;
153
154 for (i = 0; i < MAX_XCP; ++i)
155 xcp_mgr->xcp[i].valid = false;
156
157 /* This is needed for figuring out memory id of xcp */
158 xcp_mgr->num_xcp_per_mem_partition = num_xcps / xcp_mgr->adev->gmc.num_mem_partitions;
159
160 for (i = 0; i < num_xcps; ++i) {
161 for (j = AMDGPU_XCP_GFXHUB; j < AMDGPU_XCP_MAX_BLOCKS; ++j) {
162 ret = xcp_mgr->funcs->get_ip_details(xcp_mgr, i, j,
163 &ip);
164 if (ret)
165 continue;
166
167 __amdgpu_xcp_add_block(xcp_mgr, i, &ip);
168 }
169
170 xcp_mgr->xcp[i].id = i;
171
172 if (xcp_mgr->funcs->get_xcp_mem_id) {
173 ret = xcp_mgr->funcs->get_xcp_mem_id(
174 xcp_mgr, &xcp_mgr->xcp[i], &mem_id);
175 if (ret)
176 continue;
177 else
178 xcp_mgr->xcp[i].mem_id = mem_id;
179 }
180 __amdgpu_xcp_set_unique_id(xcp_mgr, i);
181 }
182
183 xcp_mgr->num_xcps = num_xcps;
184 xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN;
185 amdgpu_xcp_update_partition_sched_list(adev);
186
187 return 0;
188}
189
190static int __amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
191 int mode)
192{
193 int ret, curr_mode, num_xcps = 0;
194
195 if (!xcp_mgr->funcs || !xcp_mgr->funcs->switch_partition_mode)
196 return 0;
197
198 mutex_lock(&xcp_mgr->xcp_lock);
199
200 curr_mode = xcp_mgr->mode;
201 /* State set to transient mode */
202 xcp_mgr->mode = AMDGPU_XCP_MODE_TRANS;
203
204 ret = xcp_mgr->funcs->switch_partition_mode(xcp_mgr, mode, &num_xcps);
205
206 if (ret) {
207 /* Failed, get whatever mode it's at now */
208 if (xcp_mgr->funcs->query_partition_mode)
209 xcp_mgr->mode = amdgpu_xcp_query_partition_mode(
210 xcp_mgr, AMDGPU_XCP_FL_LOCKED);
211 else
212 xcp_mgr->mode = curr_mode;
213
214 goto out;
215 }
216 amdgpu_xcp_sysfs_entries_update(xcp_mgr);
217out:
218 mutex_unlock(&xcp_mgr->xcp_lock);
219
220 return ret;
221}
222
223int amdgpu_xcp_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, int mode)
224{
225 if (!xcp_mgr || mode == AMDGPU_XCP_MODE_NONE)
226 return -EINVAL;
227
228 if (xcp_mgr->mode == mode)
229 return 0;
230
231 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, mode);
232}
233
234int amdgpu_xcp_restore_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr)
235{
236 if (!xcp_mgr || xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
237 return 0;
238
239 return __amdgpu_xcp_switch_partition_mode(xcp_mgr, xcp_mgr->mode);
240}
241
242static bool __amdgpu_xcp_is_cached_mode_valid(struct amdgpu_xcp_mgr *xcp_mgr)
243{
244 if (!xcp_mgr->funcs || !xcp_mgr->funcs->query_partition_mode)
245 return true;
246
247 if (!amdgpu_sriov_vf(xcp_mgr->adev) &&
248 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
249 return true;
250
251 if (xcp_mgr->mode != AMDGPU_XCP_MODE_NONE &&
252 xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS)
253 return true;
254
255 return false;
256}
257
258int amdgpu_xcp_query_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
259{
260 int mode;
261
262 if (__amdgpu_xcp_is_cached_mode_valid(xcp_mgr))
263 return xcp_mgr->mode;
264
265 if (!(flags & AMDGPU_XCP_FL_LOCKED))
266 mutex_lock(&xcp_mgr->xcp_lock);
267 mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
268
269 /* First time query for VF, set the mode here */
270 if (amdgpu_sriov_vf(xcp_mgr->adev) &&
271 xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
272 xcp_mgr->mode = mode;
273
274 if (xcp_mgr->mode != AMDGPU_XCP_MODE_TRANS && mode != xcp_mgr->mode)
275 dev_WARN(
276 xcp_mgr->adev->dev,
277 "Cached partition mode %d not matching with device mode %d",
278 xcp_mgr->mode, mode);
279
280 if (!(flags & AMDGPU_XCP_FL_LOCKED))
281 mutex_unlock(&xcp_mgr->xcp_lock);
282
283 return mode;
284}
285
286static int amdgpu_xcp_dev_alloc(struct amdgpu_device *adev)
287{
288 struct drm_device *p_ddev;
289 struct drm_device *ddev;
290 int i, ret;
291
292 ddev = adev_to_drm(adev);
293
294 /* xcp #0 shares drm device setting with adev */
295 adev->xcp_mgr->xcp->ddev = ddev;
296
297 for (i = 1; i < MAX_XCP; i++) {
298 ret = amdgpu_xcp_drm_dev_alloc(&p_ddev);
299 if (ret == -ENOSPC) {
300 dev_warn(adev->dev,
301 "Skip xcp node #%d when out of drm node resource.", i);
302 ret = 0;
303 goto out;
304 } else if (ret) {
305 goto out;
306 }
307
308 /* Redirect all IOCTLs to the primary device */
309 adev->xcp_mgr->xcp[i].rdev = p_ddev->render->dev;
310 adev->xcp_mgr->xcp[i].pdev = p_ddev->primary->dev;
311 adev->xcp_mgr->xcp[i].driver = (struct drm_driver *)p_ddev->driver;
312 adev->xcp_mgr->xcp[i].vma_offset_manager = p_ddev->vma_offset_manager;
313 p_ddev->render->dev = ddev;
314 p_ddev->primary->dev = ddev;
315 p_ddev->vma_offset_manager = ddev->vma_offset_manager;
316 p_ddev->driver = &amdgpu_partition_driver;
317 adev->xcp_mgr->xcp[i].ddev = p_ddev;
318
319 dev_set_drvdata(p_ddev->dev, &adev->xcp_mgr->xcp[i]);
320 }
321 ret = 0;
322out:
323 amdgpu_xcp_sysfs_entries_init(adev->xcp_mgr);
324
325 return ret;
326}
327
328int amdgpu_xcp_mgr_init(struct amdgpu_device *adev, int init_mode,
329 int init_num_xcps,
330 struct amdgpu_xcp_mgr_funcs *xcp_funcs)
331{
332 struct amdgpu_xcp_mgr *xcp_mgr;
333 int i;
334
335 if (!xcp_funcs || !xcp_funcs->get_ip_details)
336 return -EINVAL;
337
338 xcp_mgr = kzalloc_obj(*xcp_mgr);
339
340 if (!xcp_mgr)
341 return -ENOMEM;
342
343 xcp_mgr->adev = adev;
344 xcp_mgr->funcs = xcp_funcs;
345 xcp_mgr->mode = init_mode;
346 mutex_init(&xcp_mgr->xcp_lock);
347
348 if (init_mode != AMDGPU_XCP_MODE_NONE)
349 amdgpu_xcp_init(xcp_mgr, init_num_xcps, init_mode);
350
351 adev->xcp_mgr = xcp_mgr;
352 for (i = 0; i < MAX_XCP; ++i)
353 xcp_mgr->xcp[i].xcp_mgr = xcp_mgr;
354
355 return amdgpu_xcp_dev_alloc(adev);
356}
357
358int amdgpu_xcp_get_partition(struct amdgpu_xcp_mgr *xcp_mgr,
359 enum AMDGPU_XCP_IP_BLOCK ip, int instance)
360{
361 struct amdgpu_xcp *xcp;
362 int i, id_mask = 0;
363
364 if (ip >= AMDGPU_XCP_MAX_BLOCKS)
365 return -EINVAL;
366
367 for (i = 0; i < xcp_mgr->num_xcps; ++i) {
368 xcp = &xcp_mgr->xcp[i];
369 if ((xcp->valid) && (xcp->ip[ip].valid) &&
370 (xcp->ip[ip].inst_mask & BIT(instance)))
371 id_mask |= BIT(i);
372 }
373
374 if (!id_mask)
375 id_mask = -ENXIO;
376
377 return id_mask;
378}
379
380int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp,
381 enum AMDGPU_XCP_IP_BLOCK ip,
382 uint32_t *inst_mask)
383{
384 if (!xcp->valid || !inst_mask || ip >= AMDGPU_XCP_MAX_BLOCKS ||
385 !(xcp->ip[ip].valid))
386 return -EINVAL;
387
388 *inst_mask = xcp->ip[ip].inst_mask;
389
390 return 0;
391}
392
393int amdgpu_xcp_dev_register(struct amdgpu_device *adev,
394 const struct pci_device_id *ent)
395{
396 int i, ret;
397
398 if (!adev->xcp_mgr)
399 return 0;
400
401 for (i = 1; i < MAX_XCP; i++) {
402 if (!adev->xcp_mgr->xcp[i].ddev)
403 break;
404
405 ret = drm_dev_register(adev->xcp_mgr->xcp[i].ddev, ent->driver_data);
406 if (ret)
407 return ret;
408 }
409
410 return 0;
411}
412
413void amdgpu_xcp_dev_unplug(struct amdgpu_device *adev)
414{
415 struct drm_device *p_ddev;
416 int i;
417
418 if (!adev->xcp_mgr)
419 return;
420
421 for (i = 1; i < MAX_XCP; i++) {
422 if (!adev->xcp_mgr->xcp[i].ddev)
423 break;
424
425 p_ddev = adev->xcp_mgr->xcp[i].ddev;
426 drm_dev_unplug(p_ddev);
427 p_ddev->render->dev = adev->xcp_mgr->xcp[i].rdev;
428 p_ddev->primary->dev = adev->xcp_mgr->xcp[i].pdev;
429 p_ddev->driver = adev->xcp_mgr->xcp[i].driver;
430 p_ddev->vma_offset_manager = adev->xcp_mgr->xcp[i].vma_offset_manager;
431 amdgpu_xcp_drm_dev_free(p_ddev);
432 }
433}
434
435int amdgpu_xcp_open_device(struct amdgpu_device *adev,
436 struct amdgpu_fpriv *fpriv,
437 struct drm_file *file_priv)
438{
439 int i;
440
441 if (!adev->xcp_mgr)
442 return 0;
443
444 fpriv->xcp_id = AMDGPU_XCP_NO_PARTITION;
445 for (i = 0; i < MAX_XCP; ++i) {
446 if (!adev->xcp_mgr->xcp[i].ddev)
447 break;
448
449 if (file_priv->minor == adev->xcp_mgr->xcp[i].ddev->render) {
450 if (adev->xcp_mgr->xcp[i].valid == FALSE) {
451 dev_err(adev->dev, "renderD%d partition %d not valid!",
452 file_priv->minor->index, i);
453 return -ENOENT;
454 }
455 dev_dbg(adev->dev, "renderD%d partition %d opened!",
456 file_priv->minor->index, i);
457 fpriv->xcp_id = i;
458 break;
459 }
460 }
461
462 fpriv->vm.mem_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? -1 :
463 adev->xcp_mgr->xcp[fpriv->xcp_id].mem_id;
464 return 0;
465}
466
467void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
468 struct amdgpu_ctx_entity *entity)
469{
470 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
471 struct drm_gpu_scheduler *sched;
472
473 if (!xcp_mgr)
474 return;
475
476 sched = entity->entity.rq->sched;
477 if (drm_sched_wqueue_ready(sched)) {
478 struct amdgpu_ring *ring = to_amdgpu_ring(sched);
479
480 mutex_lock(&xcp_mgr->xcp_lock);
481 if (ring->xcp_id < xcp_mgr->num_xcps && xcp_mgr->xcp[ring->xcp_id].valid)
482 atomic_dec(&xcp_mgr->xcp[ring->xcp_id].ref_cnt);
483 mutex_unlock(&xcp_mgr->xcp_lock);
484 }
485}
486
487int amdgpu_xcp_select_scheds(struct amdgpu_device *adev,
488 u32 hw_ip, u32 hw_prio,
489 struct amdgpu_fpriv *fpriv,
490 unsigned int *num_scheds,
491 struct drm_gpu_scheduler ***scheds)
492{
493 u32 sel_xcp_id;
494 int i;
495 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
496 int r = 0;
497
498 mutex_lock(&xcp_mgr->xcp_lock);
499 if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) {
500 u32 least_ref_cnt = ~0;
501
502 fpriv->xcp_id = 0;
503 for (i = 0; i < xcp_mgr->num_xcps; i++) {
504 u32 total_ref_cnt;
505
506 total_ref_cnt = atomic_read(&xcp_mgr->xcp[i].ref_cnt);
507 if (total_ref_cnt < least_ref_cnt) {
508 fpriv->xcp_id = i;
509 least_ref_cnt = total_ref_cnt;
510 }
511 }
512 }
513 sel_xcp_id = fpriv->xcp_id;
514
515 if (sel_xcp_id >= xcp_mgr->num_xcps || !xcp_mgr->xcp[sel_xcp_id].valid) {
516 dev_err(adev->dev, "Selected partition #%d is not valid.", sel_xcp_id);
517 r = -ENODEV;
518 goto out;
519 }
520
521 if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) {
522 *num_scheds =
523 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds;
524 *scheds =
525 xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].sched;
526 atomic_inc(&xcp_mgr->xcp[sel_xcp_id].ref_cnt);
527 dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id);
528 } else {
529 dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id);
530 r = -ENOENT;
531 }
532
533out:
534 mutex_unlock(&xcp_mgr->xcp_lock);
535 return r;
536}
537
538static void amdgpu_set_xcp_id(struct amdgpu_device *adev,
539 uint32_t inst_idx,
540 struct amdgpu_ring *ring)
541{
542 int xcp_id;
543 enum AMDGPU_XCP_IP_BLOCK ip_blk;
544 uint32_t inst_mask;
545
546 ring->xcp_id = AMDGPU_XCP_NO_PARTITION;
547 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
548 adev->gfx.enforce_isolation[0].xcp_id = ring->xcp_id;
549 if ((adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE) ||
550 (ring->funcs->type == AMDGPU_RING_TYPE_CPER))
551 return;
552
553 inst_mask = 1 << inst_idx;
554
555 switch (ring->funcs->type) {
556 case AMDGPU_HW_IP_GFX:
557 case AMDGPU_RING_TYPE_COMPUTE:
558 case AMDGPU_RING_TYPE_KIQ:
559 case AMDGPU_RING_TYPE_MES:
560 ip_blk = AMDGPU_XCP_GFX;
561 break;
562 case AMDGPU_RING_TYPE_SDMA:
563 ip_blk = AMDGPU_XCP_SDMA;
564 break;
565 case AMDGPU_RING_TYPE_VCN_ENC:
566 case AMDGPU_RING_TYPE_VCN_JPEG:
567 ip_blk = AMDGPU_XCP_VCN;
568 break;
569 default:
570 dev_err(adev->dev, "Not support ring type %d!", ring->funcs->type);
571 return;
572 }
573
574 for (xcp_id = 0; xcp_id < adev->xcp_mgr->num_xcps; xcp_id++) {
575 if (adev->xcp_mgr->xcp[xcp_id].ip[ip_blk].inst_mask & inst_mask) {
576 ring->xcp_id = xcp_id;
577 dev_dbg(adev->dev, "ring:%s xcp_id :%u", ring->name,
578 ring->xcp_id);
579 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
580 adev->gfx.enforce_isolation[xcp_id].xcp_id = xcp_id;
581 break;
582 }
583 }
584}
585
586static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev,
587 struct amdgpu_ring *ring,
588 unsigned int sel_xcp_id)
589{
590 unsigned int *num_gpu_sched;
591
592 if (sel_xcp_id >= MAX_XCP || sel_xcp_id == AMDGPU_XCP_NO_PARTITION)
593 return;
594
595 num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id]
596 .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds;
597 adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio]
598 .sched[(*num_gpu_sched)++] = &ring->sched;
599 dev_dbg(adev->dev, "%s :[%d] gpu_sched[%d][%d] = %d",
600 ring->name, sel_xcp_id, ring->funcs->type,
601 ring->hw_prio, *num_gpu_sched);
602}
603
604static int amdgpu_xcp_sched_list_update(struct amdgpu_device *adev)
605{
606 struct amdgpu_ring *ring;
607 int i;
608
609 for (i = 0; i < MAX_XCP; i++) {
610 atomic_set(&adev->xcp_mgr->xcp[i].ref_cnt, 0);
611 memset(adev->xcp_mgr->xcp[i].gpu_sched, 0, sizeof(adev->xcp_mgr->xcp->gpu_sched));
612 }
613
614 if (adev->xcp_mgr->mode == AMDGPU_XCP_MODE_NONE)
615 return 0;
616
617 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
618 ring = adev->rings[i];
619 if (!ring || !ring->sched.ready || ring->no_scheduler)
620 continue;
621
622 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id);
623
624 /* VCN may be shared by two partitions under CPX MODE in certain
625 * configs.
626 */
627 if ((ring->funcs->type == AMDGPU_RING_TYPE_VCN_ENC ||
628 ring->funcs->type == AMDGPU_RING_TYPE_VCN_JPEG) &&
629 (adev->xcp_mgr->num_xcps > adev->vcn.num_vcn_inst))
630 amdgpu_xcp_gpu_sched_update(adev, ring, ring->xcp_id + 1);
631 }
632
633 return 0;
634}
635
636int amdgpu_xcp_update_partition_sched_list(struct amdgpu_device *adev)
637{
638 int i;
639
640 for (i = 0; i < adev->num_rings; i++) {
641 struct amdgpu_ring *ring = adev->rings[i];
642
643 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE ||
644 ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
645 amdgpu_set_xcp_id(adev, ring->xcc_id, ring);
646 else
647 amdgpu_set_xcp_id(adev, ring->me, ring);
648 }
649
650 return amdgpu_xcp_sched_list_update(adev);
651}
652
653void amdgpu_xcp_update_supported_modes(struct amdgpu_xcp_mgr *xcp_mgr)
654{
655 struct amdgpu_device *adev = xcp_mgr->adev;
656
657 xcp_mgr->supp_xcp_modes = 0;
658
659 switch (NUM_XCC(adev->gfx.xcc_mask)) {
660 case 8:
661 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
662 BIT(AMDGPU_DPX_PARTITION_MODE) |
663 BIT(AMDGPU_QPX_PARTITION_MODE) |
664 BIT(AMDGPU_CPX_PARTITION_MODE);
665 break;
666 case 6:
667 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
668 BIT(AMDGPU_TPX_PARTITION_MODE) |
669 BIT(AMDGPU_CPX_PARTITION_MODE);
670 break;
671 case 4:
672 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
673 BIT(AMDGPU_DPX_PARTITION_MODE) |
674 BIT(AMDGPU_CPX_PARTITION_MODE);
675 break;
676 case 2:
677 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
678 BIT(AMDGPU_CPX_PARTITION_MODE);
679 break;
680 case 1:
681 xcp_mgr->supp_xcp_modes = BIT(AMDGPU_SPX_PARTITION_MODE) |
682 BIT(AMDGPU_CPX_PARTITION_MODE);
683 break;
684
685 default:
686 break;
687 }
688}
689
690int amdgpu_xcp_pre_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
691{
692 /* TODO:
693 * Stop user queues and threads, and make sure GPU is empty of work.
694 */
695
696 if (flags & AMDGPU_XCP_OPS_KFD)
697 amdgpu_amdkfd_device_fini_sw(xcp_mgr->adev);
698
699 return 0;
700}
701
702int amdgpu_xcp_post_partition_switch(struct amdgpu_xcp_mgr *xcp_mgr, u32 flags)
703{
704 int ret = 0;
705
706 if (flags & AMDGPU_XCP_OPS_KFD) {
707 amdgpu_amdkfd_device_probe(xcp_mgr->adev);
708 amdgpu_amdkfd_device_init(xcp_mgr->adev);
709 /* If KFD init failed, return failure */
710 if (!xcp_mgr->adev->kfd.init_complete)
711 ret = -EIO;
712 }
713
714 return ret;
715}
716
717/*====================== xcp sysfs - configuration ======================*/
718#define XCP_CFG_SYSFS_RES_ATTR_SHOW(_name) \
719 static ssize_t amdgpu_xcp_res_sysfs_##_name##_show( \
720 struct amdgpu_xcp_res_details *xcp_res, char *buf) \
721 { \
722 return sysfs_emit(buf, "%d\n", xcp_res->_name); \
723 }
724
725struct amdgpu_xcp_res_sysfs_attribute {
726 struct attribute attr;
727 ssize_t (*show)(struct amdgpu_xcp_res_details *xcp_res, char *buf);
728};
729
730#define XCP_CFG_SYSFS_RES_ATTR(_name) \
731 struct amdgpu_xcp_res_sysfs_attribute xcp_res_sysfs_attr_##_name = { \
732 .attr = { .name = __stringify(_name), .mode = 0400 }, \
733 .show = amdgpu_xcp_res_sysfs_##_name##_show, \
734 }
735
736XCP_CFG_SYSFS_RES_ATTR_SHOW(num_inst)
737XCP_CFG_SYSFS_RES_ATTR(num_inst);
738XCP_CFG_SYSFS_RES_ATTR_SHOW(num_shared)
739XCP_CFG_SYSFS_RES_ATTR(num_shared);
740
741#define XCP_CFG_SYSFS_RES_ATTR_PTR(_name) xcp_res_sysfs_attr_##_name.attr
742
743static struct attribute *xcp_cfg_res_sysfs_attrs[] = {
744 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_inst),
745 &XCP_CFG_SYSFS_RES_ATTR_PTR(num_shared), NULL
746};
747
748static const char *xcp_desc[] = {
749 [AMDGPU_SPX_PARTITION_MODE] = "SPX",
750 [AMDGPU_DPX_PARTITION_MODE] = "DPX",
751 [AMDGPU_TPX_PARTITION_MODE] = "TPX",
752 [AMDGPU_QPX_PARTITION_MODE] = "QPX",
753 [AMDGPU_CPX_PARTITION_MODE] = "CPX",
754};
755
756static const char *nps_desc[] = {
757 [UNKNOWN_MEMORY_PARTITION_MODE] = "UNKNOWN",
758 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1",
759 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2",
760 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3",
761 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4",
762 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6",
763 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8",
764};
765
766ATTRIBUTE_GROUPS(xcp_cfg_res_sysfs);
767
768#define to_xcp_attr(x) \
769 container_of(x, struct amdgpu_xcp_res_sysfs_attribute, attr)
770#define to_xcp_res(x) container_of(x, struct amdgpu_xcp_res_details, kobj)
771
772static ssize_t xcp_cfg_res_sysfs_attr_show(struct kobject *kobj,
773 struct attribute *attr, char *buf)
774{
775 struct amdgpu_xcp_res_sysfs_attribute *attribute;
776 struct amdgpu_xcp_res_details *xcp_res;
777
778 attribute = to_xcp_attr(attr);
779 xcp_res = to_xcp_res(kobj);
780
781 if (!attribute->show)
782 return -EIO;
783
784 return attribute->show(xcp_res, buf);
785}
786
787static const struct sysfs_ops xcp_cfg_res_sysfs_ops = {
788 .show = xcp_cfg_res_sysfs_attr_show,
789};
790
791static const struct kobj_type xcp_cfg_res_sysfs_ktype = {
792 .sysfs_ops = &xcp_cfg_res_sysfs_ops,
793 .default_groups = xcp_cfg_res_sysfs_groups,
794};
795
796const char *xcp_res_names[] = {
797 [AMDGPU_XCP_RES_XCC] = "xcc",
798 [AMDGPU_XCP_RES_DMA] = "dma",
799 [AMDGPU_XCP_RES_DEC] = "dec",
800 [AMDGPU_XCP_RES_JPEG] = "jpeg",
801};
802
803static int amdgpu_xcp_get_res_info(struct amdgpu_xcp_mgr *xcp_mgr,
804 int mode,
805 struct amdgpu_xcp_cfg *xcp_cfg)
806{
807 if (xcp_mgr->funcs && xcp_mgr->funcs->get_xcp_res_info)
808 return xcp_mgr->funcs->get_xcp_res_info(xcp_mgr, mode, xcp_cfg);
809
810 return -EOPNOTSUPP;
811}
812
813#define to_xcp_cfg(x) container_of(x, struct amdgpu_xcp_cfg, kobj)
814static ssize_t supported_xcp_configs_show(struct kobject *kobj,
815 struct kobj_attribute *attr, char *buf)
816{
817 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj);
818 struct amdgpu_xcp_mgr *xcp_mgr = xcp_cfg->xcp_mgr;
819 int size = 0, mode;
820 char *sep = "";
821
822 if (!xcp_mgr || !xcp_mgr->supp_xcp_modes)
823 return sysfs_emit(buf, "Not supported\n");
824
825 for_each_inst(mode, xcp_mgr->supp_xcp_modes) {
826 size += sysfs_emit_at(buf, size, "%s%s", sep, xcp_desc[mode]);
827 sep = ", ";
828 }
829
830 size += sysfs_emit_at(buf, size, "\n");
831
832 return size;
833}
834
835static ssize_t supported_nps_configs_show(struct kobject *kobj,
836 struct kobj_attribute *attr, char *buf)
837{
838 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj);
839 int size = 0, mode;
840 char *sep = "";
841
842 if (!xcp_cfg || !xcp_cfg->compatible_nps_modes)
843 return sysfs_emit(buf, "Not supported\n");
844
845 for_each_inst(mode, xcp_cfg->compatible_nps_modes) {
846 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]);
847 sep = ", ";
848 }
849
850 size += sysfs_emit_at(buf, size, "\n");
851
852 return size;
853}
854
855static ssize_t xcp_config_show(struct kobject *kobj,
856 struct kobj_attribute *attr, char *buf)
857{
858 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj);
859
860 return sysfs_emit(buf, "%s\n",
861 amdgpu_gfx_compute_mode_desc(xcp_cfg->mode));
862}
863
864static ssize_t xcp_config_store(struct kobject *kobj,
865 struct kobj_attribute *attr,
866 const char *buf, size_t size)
867{
868 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj);
869 int mode, r;
870
871 if (!strncasecmp("SPX", buf, strlen("SPX")))
872 mode = AMDGPU_SPX_PARTITION_MODE;
873 else if (!strncasecmp("DPX", buf, strlen("DPX")))
874 mode = AMDGPU_DPX_PARTITION_MODE;
875 else if (!strncasecmp("TPX", buf, strlen("TPX")))
876 mode = AMDGPU_TPX_PARTITION_MODE;
877 else if (!strncasecmp("QPX", buf, strlen("QPX")))
878 mode = AMDGPU_QPX_PARTITION_MODE;
879 else if (!strncasecmp("CPX", buf, strlen("CPX")))
880 mode = AMDGPU_CPX_PARTITION_MODE;
881 else
882 return -EINVAL;
883
884 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg);
885
886 if (r)
887 return r;
888
889 xcp_cfg->mode = mode;
890 return size;
891}
892
893static struct kobj_attribute xcp_cfg_sysfs_mode =
894 __ATTR_RW_MODE(xcp_config, 0644);
895
896static void xcp_cfg_sysfs_release(struct kobject *kobj)
897{
898 struct amdgpu_xcp_cfg *xcp_cfg = to_xcp_cfg(kobj);
899
900 kfree(xcp_cfg);
901}
902
903static const struct kobj_type xcp_cfg_sysfs_ktype = {
904 .release = xcp_cfg_sysfs_release,
905 .sysfs_ops = &kobj_sysfs_ops,
906};
907
908static struct kobj_attribute supp_part_sysfs_mode =
909 __ATTR_RO(supported_xcp_configs);
910
911static struct kobj_attribute supp_nps_sysfs_mode =
912 __ATTR_RO(supported_nps_configs);
913
914static const struct attribute *xcp_attrs[] = {
915 &supp_part_sysfs_mode.attr,
916 &xcp_cfg_sysfs_mode.attr,
917 NULL,
918};
919
920static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev)
921{
922 struct amdgpu_xcp_res_details *xcp_res;
923 struct amdgpu_xcp_cfg *xcp_cfg;
924 int i, r, rid, mode;
925
926 if (!adev->xcp_mgr)
927 return;
928
929 xcp_cfg = kzalloc_obj(*xcp_cfg);
930 if (!xcp_cfg)
931 return;
932 xcp_cfg->xcp_mgr = adev->xcp_mgr;
933
934 r = kobject_init_and_add(&xcp_cfg->kobj, &xcp_cfg_sysfs_ktype,
935 &adev->dev->kobj, "compute_partition_config");
936 if (r)
937 goto err1;
938
939 r = sysfs_create_files(&xcp_cfg->kobj, xcp_attrs);
940 if (r)
941 goto err1;
942
943 if (adev->gmc.supported_nps_modes != 0) {
944 r = sysfs_create_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr);
945 if (r) {
946 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs);
947 goto err1;
948 }
949 }
950
951 mode = (xcp_cfg->xcp_mgr->mode ==
952 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) ?
953 AMDGPU_SPX_PARTITION_MODE :
954 xcp_cfg->xcp_mgr->mode;
955 r = amdgpu_xcp_get_res_info(xcp_cfg->xcp_mgr, mode, xcp_cfg);
956 if (r) {
957 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr);
958 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs);
959 goto err1;
960 }
961
962 xcp_cfg->mode = mode;
963 for (i = 0; i < xcp_cfg->num_res; i++) {
964 xcp_res = &xcp_cfg->xcp_res[i];
965 rid = xcp_res->id;
966 r = kobject_init_and_add(&xcp_res->kobj,
967 &xcp_cfg_res_sysfs_ktype,
968 &xcp_cfg->kobj, "%s",
969 xcp_res_names[rid]);
970 if (r) {
971 kobject_put(&xcp_res->kobj);
972 goto err;
973 }
974 }
975
976 adev->xcp_mgr->xcp_cfg = xcp_cfg;
977 return;
978err:
979 while (i--) {
980 xcp_res = &xcp_cfg->xcp_res[i];
981 kobject_put(&xcp_res->kobj);
982 }
983
984 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr);
985 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs);
986err1:
987 kobject_put(&xcp_cfg->kobj);
988}
989
990static void amdgpu_xcp_cfg_sysfs_fini(struct amdgpu_device *adev)
991{
992 struct amdgpu_xcp_res_details *xcp_res;
993 struct amdgpu_xcp_cfg *xcp_cfg;
994 int i;
995
996 if (!adev->xcp_mgr || !adev->xcp_mgr->xcp_cfg)
997 return;
998
999 xcp_cfg = adev->xcp_mgr->xcp_cfg;
1000 for (i = 0; i < xcp_cfg->num_res; i++) {
1001 xcp_res = &xcp_cfg->xcp_res[i];
1002 kobject_put(&xcp_res->kobj);
1003 }
1004
1005 sysfs_remove_file(&xcp_cfg->kobj, &supp_nps_sysfs_mode.attr);
1006 sysfs_remove_files(&xcp_cfg->kobj, xcp_attrs);
1007 kobject_put(&xcp_cfg->kobj);
1008}
1009
1010/*====================== xcp sysfs - data entries ======================*/
1011
1012#define to_xcp(x) container_of(x, struct amdgpu_xcp, kobj)
1013
1014static ssize_t xcp_metrics_show(struct kobject *kobj,
1015 struct kobj_attribute *attr, char *buf)
1016{
1017 struct amdgpu_xcp *xcp = to_xcp(kobj);
1018 struct amdgpu_xcp_mgr *xcp_mgr;
1019 ssize_t size;
1020
1021 xcp_mgr = xcp->xcp_mgr;
1022 size = amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, NULL);
1023 if (size <= 0)
1024 return size;
1025
1026 if (size > PAGE_SIZE)
1027 return -ENOSPC;
1028
1029 return amdgpu_dpm_get_xcp_metrics(xcp_mgr->adev, xcp->id, buf);
1030}
1031
1032static umode_t amdgpu_xcp_attrs_is_visible(struct kobject *kobj,
1033 struct attribute *attr, int n)
1034{
1035 struct amdgpu_xcp *xcp = to_xcp(kobj);
1036
1037 if (!xcp || !xcp->valid)
1038 return 0;
1039
1040 return attr->mode;
1041}
1042
1043static struct kobj_attribute xcp_sysfs_metrics = __ATTR_RO(xcp_metrics);
1044
1045static struct attribute *amdgpu_xcp_attrs[] = {
1046 &xcp_sysfs_metrics.attr,
1047 NULL,
1048};
1049
1050static const struct attribute_group amdgpu_xcp_attrs_group = {
1051 .attrs = amdgpu_xcp_attrs,
1052 .is_visible = amdgpu_xcp_attrs_is_visible
1053};
1054
1055static const struct kobj_type xcp_sysfs_ktype = {
1056 .sysfs_ops = &kobj_sysfs_ops,
1057};
1058
1059static void amdgpu_xcp_sysfs_entries_fini(struct amdgpu_xcp_mgr *xcp_mgr, int n)
1060{
1061 struct amdgpu_xcp *xcp;
1062
1063 for (n--; n >= 0; n--) {
1064 xcp = &xcp_mgr->xcp[n];
1065 if (!xcp->ddev || !xcp->valid)
1066 continue;
1067 sysfs_remove_group(&xcp->kobj, &amdgpu_xcp_attrs_group);
1068 kobject_put(&xcp->kobj);
1069 }
1070}
1071
1072static void amdgpu_xcp_sysfs_entries_init(struct amdgpu_xcp_mgr *xcp_mgr)
1073{
1074 struct amdgpu_xcp *xcp;
1075 int i, r;
1076
1077 for (i = 0; i < MAX_XCP; i++) {
1078 /* Redirect all IOCTLs to the primary device */
1079 xcp = &xcp_mgr->xcp[i];
1080 if (!xcp->ddev)
1081 break;
1082 r = kobject_init_and_add(&xcp->kobj, &xcp_sysfs_ktype,
1083 &xcp->ddev->dev->kobj, "xcp");
1084 if (r)
1085 goto out;
1086
1087 r = sysfs_create_group(&xcp->kobj, &amdgpu_xcp_attrs_group);
1088 if (r)
1089 goto out;
1090 }
1091
1092 return;
1093out:
1094 kobject_put(&xcp->kobj);
1095}
1096
1097static void amdgpu_xcp_sysfs_entries_update(struct amdgpu_xcp_mgr *xcp_mgr)
1098{
1099 struct amdgpu_xcp *xcp;
1100 int i;
1101
1102 for (i = 0; i < MAX_XCP; i++) {
1103 /* Redirect all IOCTLs to the primary device */
1104 xcp = &xcp_mgr->xcp[i];
1105 if (!xcp->ddev)
1106 continue;
1107 sysfs_update_group(&xcp->kobj, &amdgpu_xcp_attrs_group);
1108 }
1109
1110 return;
1111}
1112
1113void amdgpu_xcp_sysfs_init(struct amdgpu_device *adev)
1114{
1115 if (!adev->xcp_mgr)
1116 return;
1117
1118 amdgpu_xcp_cfg_sysfs_init(adev);
1119
1120 return;
1121}
1122
1123void amdgpu_xcp_sysfs_fini(struct amdgpu_device *adev)
1124{
1125 if (!adev->xcp_mgr)
1126 return;
1127 amdgpu_xcp_sysfs_entries_fini(adev->xcp_mgr, MAX_XCP);
1128 amdgpu_xcp_cfg_sysfs_fini(adev);
1129}