projects/linux·blob master · refs

1185 lines · 31.3 KiB

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/slab.h>
26#include <linux/list.h>
27#include "kfd_device_queue_manager.h"
28#include "kfd_priv.h"
29#include "kfd_kernel_queue.h"
30#include "amdgpu_amdkfd.h"
31#include "amdgpu_reset.h"
32
33static inline struct process_queue_node *get_queue_by_qid(
34 struct process_queue_manager *pqm, unsigned int qid)
35{
36 struct process_queue_node *pqn;
37
38 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
39 if ((pqn->q && pqn->q->properties.queue_id == qid) ||
40 (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
41 return pqn;
42 }
43
44 return NULL;
45}
46
47static int assign_queue_slot_by_qid(struct process_queue_manager *pqm,
48 unsigned int qid)
49{
50 if (qid >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
51 return -EINVAL;
52
53 if (__test_and_set_bit(qid, pqm->queue_slot_bitmap)) {
54 pr_err("Cannot create new queue because requested qid(%u) is in use\n", qid);
55 return -ENOSPC;
56 }
57
58 return 0;
59}
60
61static int find_available_queue_slot(struct process_queue_manager *pqm,
62 unsigned int *qid)
63{
64 unsigned long found;
65
66 found = find_first_zero_bit(pqm->queue_slot_bitmap,
67 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
68
69 pr_debug("The new slot id %lu\n", found);
70
71 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
72 pr_info("Cannot open more queues for process with pid %d\n",
73 pqm->process->lead_thread->pid);
74 return -ENOMEM;
75 }
76
77 set_bit(found, pqm->queue_slot_bitmap);
78 *qid = found;
79
80 return 0;
81}
82
83void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
84{
85 struct kfd_node *dev = pdd->dev;
86
87 if (pdd->already_dequeued)
88 return;
89 /* The MES context flush needs to filter out the case which the
90 * KFD process is created without setting up the MES context and
91 * queue for creating a compute queue.
92 */
93 dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
94 if (dev->kfd->shared_resources.enable_mes && !!pdd->proc_ctx_gpu_addr &&
95 down_read_trylock(&dev->adev->reset_domain->sem)) {
96 amdgpu_mes_flush_shader_debugger(dev->adev,
97 pdd->proc_ctx_gpu_addr,
98 ffs(pdd->dev->xcc_mask) - 1);
99 up_read(&dev->adev->reset_domain->sem);
100 }
101 pdd->already_dequeued = true;
102}
103
104int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
105 void *gws)
106{
107 struct mqd_update_info minfo = {0};
108 struct kfd_node *dev = NULL;
109 struct process_queue_node *pqn;
110 struct kfd_process_device *pdd;
111 struct kgd_mem *mem = NULL;
112 int ret;
113
114 pqn = get_queue_by_qid(pqm, qid);
115 if (!pqn) {
116 pr_err("Queue id does not match any known queue\n");
117 return -EINVAL;
118 }
119
120 if (pqn->q)
121 dev = pqn->q->device;
122 if (WARN_ON(!dev))
123 return -ENODEV;
124
125 pdd = kfd_get_process_device_data(dev, pqm->process);
126 if (!pdd) {
127 pr_err("Process device data doesn't exist\n");
128 return -EINVAL;
129 }
130
131 /* Only allow one queue per process can have GWS assigned */
132 if (gws && pdd->qpd.num_gws)
133 return -EBUSY;
134
135 if (!gws && pdd->qpd.num_gws == 0)
136 return -EINVAL;
137
138 if ((KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 3) &&
139 KFD_GC_VERSION(dev) != IP_VERSION(9, 4, 4) &&
140 KFD_GC_VERSION(dev) != IP_VERSION(9, 5, 0)) &&
141 !dev->kfd->shared_resources.enable_mes) {
142 if (gws)
143 ret = amdgpu_amdkfd_add_gws_to_process(pdd->process->kgd_process_info,
144 gws, &mem);
145 else
146 ret = amdgpu_amdkfd_remove_gws_from_process(pdd->process->kgd_process_info,
147 pqn->q->gws);
148 if (unlikely(ret))
149 return ret;
150 pqn->q->gws = mem;
151 } else {
152 /*
153 * Intentionally set GWS to a non-NULL value
154 * for devices that do not use GWS for global wave
155 * synchronization but require the formality
156 * of setting GWS for cooperative groups.
157 */
158 pqn->q->gws = gws ? ERR_PTR(-ENOMEM) : NULL;
159 }
160
161 pdd->qpd.num_gws = gws ? dev->adev->gds.gws_size : 0;
162 minfo.update_flag = gws ? UPDATE_FLAG_IS_GWS : 0;
163
164 return pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
165 pqn->q, &minfo);
166}
167
168void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
169{
170 int i;
171
172 for (i = 0; i < p->n_pdds; i++)
173 kfd_process_dequeue_from_device(p->pdds[i]);
174}
175
176int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
177{
178 INIT_LIST_HEAD(&pqm->queues);
179 pqm->queue_slot_bitmap = bitmap_zalloc(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
180 GFP_KERNEL);
181 if (!pqm->queue_slot_bitmap)
182 return -ENOMEM;
183 pqm->process = p;
184
185 return 0;
186}
187
188static void pqm_clean_queue_resource(struct process_queue_manager *pqm,
189 struct process_queue_node *pqn)
190{
191 struct kfd_node *dev;
192 struct kfd_process_device *pdd;
193
194 dev = pqn->q->device;
195
196 pdd = kfd_get_process_device_data(dev, pqm->process);
197 if (!pdd) {
198 pr_err("Process device data doesn't exist\n");
199 return;
200 }
201
202 if (pqn->q->gws) {
203 if (KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 3) &&
204 KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 4, 4) &&
205 KFD_GC_VERSION(pqn->q->device) != IP_VERSION(9, 5, 0) &&
206 !dev->kfd->shared_resources.enable_mes)
207 amdgpu_amdkfd_remove_gws_from_process(
208 pqm->process->kgd_process_info, pqn->q->gws);
209 pdd->qpd.num_gws = 0;
210 }
211
212 if (dev->kfd->shared_resources.enable_mes) {
213 amdgpu_mes_free_gang_ctx_index(&dev->adev->mes, pqn->q->gang_ctx_array_index);
214 amdgpu_amdkfd_free_kernel_mem(dev->adev, &pqn->q->gang_ctx_bo);
215 amdgpu_amdkfd_free_kernel_mem(dev->adev, (void **)&pqn->q->wptr_bo_gart);
216 }
217}
218
219void pqm_uninit(struct process_queue_manager *pqm)
220{
221 struct process_queue_node *pqn, *next;
222
223 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
224 if (pqn->q) {
225 struct kfd_process_device *pdd = kfd_get_process_device_data(pqn->q->device,
226 pqm->process);
227 if (pdd) {
228 kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);
229 kfd_queue_release_buffers(pdd, &pqn->q->properties);
230 } else {
231 WARN_ON(!pdd);
232 }
233 pqm_clean_queue_resource(pqm, pqn);
234 }
235
236 kfd_procfs_del_queue(pqn->q);
237 uninit_queue(pqn->q);
238 list_del(&pqn->process_queue_list);
239 kfree(pqn);
240 }
241
242 bitmap_free(pqm->queue_slot_bitmap);
243 pqm->queue_slot_bitmap = NULL;
244}
245
246static int init_user_queue(struct process_queue_manager *pqm,
247 struct kfd_node *dev, struct queue **q,
248 struct queue_properties *q_properties,
249 unsigned int qid)
250{
251 int retval;
252
253 /* Doorbell initialized in user space*/
254 q_properties->doorbell_ptr = NULL;
255 q_properties->exception_status = KFD_EC_MASK(EC_QUEUE_NEW);
256
257 /* let DQM handle it*/
258 q_properties->vmid = 0;
259 q_properties->queue_id = qid;
260
261 retval = init_queue(q, q_properties);
262 if (retval != 0)
263 return retval;
264
265 (*q)->device = dev;
266 (*q)->process = pqm->process;
267
268 if (dev->kfd->shared_resources.enable_mes) {
269 if (!q_properties->wptr_bo) {
270 pr_debug("Queue initialization with shared MES requires queue buffers to be initialized\n");
271 return -EINVAL;
272 }
273
274 retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev,
275 AMDGPU_MES_GANG_CTX_SIZE,
276 AMDGPU_GEM_DOMAIN_GTT,
277 &(*q)->gang_ctx_bo,
278 &(*q)->gang_ctx_gpu_addr,
279 &(*q)->gang_ctx_cpu_ptr,
280 false);
281 if (retval) {
282 pr_err("failed to allocate gang context bo\n");
283 goto cleanup;
284 }
285 memset((*q)->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE);
286 /* Bind one MES gang context slot per queue (gang). */
287 if (dev->adev->mes.use_rs64mem) {
288 retval = amdgpu_mes_alloc_gang_ctx_index(&dev->adev->mes,
289 &(*q)->gang_ctx_array_index);
290 if (retval) {
291 pr_err("failed to allocate gang context index slot\n");
292 goto cleanup;
293 }
294 }
295 /* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
296 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
297 */
298 if (dev->adev != amdgpu_ttm_adev(q_properties->wptr_bo->tbo.bdev)) {
299 pr_err("Queue memory allocated to wrong device\n");
300 retval = -EINVAL;
301 goto free_gang_ctx_bo;
302 }
303
304 retval = amdgpu_amdkfd_map_gtt_bo_to_gart(q_properties->wptr_bo,
305 &(*q)->wptr_bo_gart);
306 if (retval) {
307 pr_err("Failed to map wptr bo to GART\n");
308 goto free_gang_ctx_bo;
309 }
310 }
311
312 pr_debug("PQM After init queue");
313 return 0;
314
315free_gang_ctx_bo:
316 amdgpu_mes_free_gang_ctx_index(&dev->adev->mes, (*q)->gang_ctx_array_index);
317 amdgpu_amdkfd_free_kernel_mem(dev->adev, &(*q)->gang_ctx_bo);
318cleanup:
319 uninit_queue(*q);
320 *q = NULL;
321 return retval;
322}
323
324int pqm_create_queue(struct process_queue_manager *pqm,
325 struct kfd_node *dev,
326 struct queue_properties *properties,
327 unsigned int *qid,
328 const struct kfd_criu_queue_priv_data *q_data,
329 const void *restore_mqd,
330 const void *restore_ctl_stack,
331 uint32_t *p_doorbell_offset_in_process)
332{
333 int retval;
334 struct kfd_process_device *pdd;
335 struct queue *q;
336 struct process_queue_node *pqn;
337 struct kernel_queue *kq;
338 enum kfd_queue_type type = properties->type;
339 unsigned int max_queues = 127; /* HWS limit */
340
341 /*
342 * On GFX 9.4.3/9.5.0, increase the number of queues that
343 * can be created to 255. No HWS limit on GFX 9.4.3/9.5.0.
344 */
345 if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) ||
346 KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) ||
347 KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0))
348 max_queues = 255;
349
350 q = NULL;
351 kq = NULL;
352
353 pdd = kfd_get_process_device_data(dev, pqm->process);
354 if (!pdd) {
355 pr_err("Process device data doesn't exist\n");
356 return -1;
357 }
358
359 /*
360 * for debug process, verify that it is within the static queues limit
361 * currently limit is set to half of the total avail HQD slots
362 * If we are just about to create DIQ, the is_debug flag is not set yet
363 * Hence we also check the type as well
364 */
365 if (pdd->qpd.is_debug)
366 max_queues = dev->kfd->device_info.max_no_of_hqd/2;
367
368 if (pdd->qpd.queue_count >= max_queues)
369 return -ENOSPC;
370
371 if (q_data) {
372 retval = assign_queue_slot_by_qid(pqm, q_data->q_id);
373 *qid = q_data->q_id;
374 } else
375 retval = find_available_queue_slot(pqm, qid);
376
377 if (retval != 0)
378 return retval;
379
380 /* Register process if this is the first queue */
381 if (list_empty(&pdd->qpd.queues_list) &&
382 list_empty(&pdd->qpd.priv_queue_list))
383 dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
384
385 /* Allocate proc_ctx_bo only if MES is enabled and this is the first queue */
386 if (!pdd->proc_ctx_cpu_ptr && dev->kfd->shared_resources.enable_mes) {
387 retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev,
388 AMDGPU_MES_PROC_CTX_SIZE,
389 AMDGPU_GEM_DOMAIN_GTT,
390 &pdd->proc_ctx_bo,
391 &pdd->proc_ctx_gpu_addr,
392 &pdd->proc_ctx_cpu_ptr,
393 false);
394 if (retval) {
395 dev_err(dev->adev->dev, "failed to allocate process context bo\n");
396 goto err_allocate_pqn;
397 }
398 memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE);
399 /* Bind one MES process context slot to the whole process
400 * (per device); every queue of this process reuses it.
401 */
402 if (dev->adev->mes.use_rs64mem) {
403 retval = amdgpu_mes_alloc_proc_ctx_index(&dev->adev->mes,
404 &pdd->proc_ctx_array_index);
405 if (retval) {
406 dev_err(dev->adev->dev, "failed to allocate process context index\n");
407 goto err_allocate_pqn;
408 }
409 }
410 }
411
412 pqn = kzalloc_obj(*pqn);
413 if (!pqn) {
414 retval = -ENOMEM;
415 goto err_allocate_pqn;
416 }
417
418 switch (type) {
419 case KFD_QUEUE_TYPE_SDMA:
420 case KFD_QUEUE_TYPE_SDMA_XGMI:
421 case KFD_QUEUE_TYPE_SDMA_BY_ENG_ID:
422 /* SDMA queues are always allocated statically no matter
423 * which scheduler mode is used. We also do not need to
424 * check whether a SDMA queue can be allocated here, because
425 * allocate_sdma_queue() in create_queue() has the
426 * corresponding check logic.
427 */
428 retval = init_user_queue(pqm, dev, &q, properties, *qid);
429 if (retval != 0)
430 goto err_create_queue;
431 pqn->q = q;
432 pqn->kq = NULL;
433 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
434 restore_mqd, restore_ctl_stack);
435 print_queue(q);
436 break;
437
438 case KFD_QUEUE_TYPE_COMPUTE:
439 /* check if there is over subscription */
440 if ((dev->dqm->sched_policy ==
441 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
442 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
443 (dev->dqm->active_queue_count >= get_cp_queues_num(dev->dqm)))) {
444 pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
445 retval = -EPERM;
446 goto err_create_queue;
447 }
448
449 retval = init_user_queue(pqm, dev, &q, properties, *qid);
450 if (retval != 0)
451 goto err_create_queue;
452 pqn->q = q;
453 pqn->kq = NULL;
454 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd, q_data,
455 restore_mqd, restore_ctl_stack);
456 print_queue(q);
457 break;
458 default:
459 WARN(1, "Invalid queue type %d", type);
460 retval = -EINVAL;
461 }
462
463 if (retval != 0) {
464 if ((type == KFD_QUEUE_TYPE_SDMA ||
465 type == KFD_QUEUE_TYPE_SDMA_XGMI ||
466 type == KFD_QUEUE_TYPE_SDMA_BY_ENG_ID) &&
467 retval == -ENOMEM)
468 pr_warn("process pid %d DQM create queue type %d failed. ret %d\n",
469 pqm->process->lead_thread->pid, type, retval);
470 else
471 pr_err("process pid %d DQM create queue type %d failed. ret %d\n",
472 pqm->process->lead_thread->pid, type, retval);
473 goto err_create_queue;
474 }
475
476 if (q && p_doorbell_offset_in_process) {
477 /* Return the doorbell offset within the doorbell page
478 * to the caller so it can be passed up to user mode
479 * (in bytes).
480 * relative doorbell index = Absolute doorbell index -
481 * absolute index of first doorbell in the page.
482 */
483 uint32_t first_db_index = amdgpu_doorbell_index_on_bar(pdd->dev->adev,
484 pdd->qpd.proc_doorbells,
485 0,
486 pdd->dev->kfd->device_info.doorbell_size);
487
488 *p_doorbell_offset_in_process = (q->properties.doorbell_off
489 - first_db_index) * sizeof(uint32_t);
490 }
491
492 pr_debug("PQM After DQM create queue\n");
493
494 list_add(&pqn->process_queue_list, &pqm->queues);
495
496 if (q) {
497 pr_debug("PQM done creating queue\n");
498 kfd_procfs_add_queue(q);
499 print_queue_properties(&q->properties);
500 }
501
502 return retval;
503
504err_create_queue:
505 uninit_queue(q);
506 if (kq)
507 kernel_queue_uninit(kq);
508 kfree(pqn);
509err_allocate_pqn:
510 /* check if queues list is empty unregister process from device */
511 clear_bit(*qid, pqm->queue_slot_bitmap);
512 if (list_empty(&pdd->qpd.queues_list) &&
513 list_empty(&pdd->qpd.priv_queue_list))
514 dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
515 return retval;
516}
517
518int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
519{
520 struct process_queue_node *pqn;
521 struct kfd_process_device *pdd;
522 struct device_queue_manager *dqm;
523 struct kfd_node *dev;
524 int retval;
525
526 dqm = NULL;
527
528 retval = 0;
529
530 pqn = get_queue_by_qid(pqm, qid);
531 if (!pqn) {
532 pr_err("Queue id does not match any known queue\n");
533 return -EINVAL;
534 }
535
536 dev = NULL;
537 if (pqn->kq)
538 dev = pqn->kq->dev;
539 if (pqn->q)
540 dev = pqn->q->device;
541 if (WARN_ON(!dev))
542 return -ENODEV;
543
544 pdd = kfd_get_process_device_data(dev, pqm->process);
545 if (!pdd) {
546 pr_err("Process device data doesn't exist\n");
547 return -1;
548 }
549
550 if (pqn->kq) {
551 /* destroy kernel queue (DIQ) */
552 dqm = pqn->kq->dev->dqm;
553 dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
554 kernel_queue_uninit(pqn->kq);
555 }
556
557 if (pqn->q) {
558 retval = kfd_queue_unref_bo_vas(pdd, &pqn->q->properties);
559 if (retval)
560 goto err_destroy_queue;
561
562 dqm = pqn->q->device->dqm;
563 retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
564 if (retval) {
565 pr_err("Pasid 0x%x destroy queue %d failed, ret %d\n",
566 pdd->pasid,
567 pqn->q->properties.queue_id, retval);
568 if (retval != -ETIME && retval != -EIO)
569 goto err_destroy_queue;
570 }
571 kfd_procfs_del_queue(pqn->q);
572 kfd_queue_release_buffers(pdd, &pqn->q->properties);
573 pqm_clean_queue_resource(pqm, pqn);
574 uninit_queue(pqn->q);
575 }
576
577 list_del(&pqn->process_queue_list);
578 kfree(pqn);
579 clear_bit(qid, pqm->queue_slot_bitmap);
580
581 if (list_empty(&pdd->qpd.queues_list) &&
582 list_empty(&pdd->qpd.priv_queue_list))
583 dqm->ops.unregister_process(dqm, &pdd->qpd);
584
585err_destroy_queue:
586 return retval;
587}
588
589int pqm_update_queue_properties(struct process_queue_manager *pqm,
590 unsigned int qid, struct queue_properties *p)
591{
592 int retval;
593 struct process_queue_node *pqn;
594
595 pqn = get_queue_by_qid(pqm, qid);
596 if (!pqn || !pqn->q) {
597 pr_debug("No queue %d exists for update operation\n", qid);
598 return -EFAULT;
599 }
600
601 /*
602 * Update with NULL ring address is used to disable the queue
603 */
604 if (p->queue_address && p->queue_size) {
605 struct kfd_process_device *pdd;
606 struct amdgpu_vm *vm;
607 struct queue *q = pqn->q;
608 int err;
609
610 pdd = kfd_get_process_device_data(q->device, q->process);
611 if (!pdd)
612 return -ENODEV;
613 vm = drm_priv_to_vm(pdd->drm_priv);
614 err = amdgpu_bo_reserve(vm->root.bo, false);
615 if (err)
616 return err;
617
618 if (kfd_queue_buffer_get(vm, (void *)p->queue_address, &p->ring_bo,
619 p->queue_size +
620 pqn->q->properties.metadata_queue_size)) {
621 pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n",
622 p->queue_address, p->queue_size);
623 amdgpu_bo_unreserve(vm->root.bo);
624 return -EFAULT;
625 }
626
627 kfd_queue_unref_bo_va(vm, &pqn->q->properties.ring_bo);
628 kfd_queue_buffer_put(&pqn->q->properties.ring_bo);
629 amdgpu_bo_unreserve(vm->root.bo);
630
631 pqn->q->properties.ring_bo = p->ring_bo;
632 }
633
634 pqn->q->properties.queue_address = p->queue_address;
635 pqn->q->properties.queue_size = p->queue_size;
636 pqn->q->properties.queue_percent = p->queue_percent;
637 pqn->q->properties.priority = p->priority;
638 pqn->q->properties.pm4_target_xcc = p->pm4_target_xcc;
639
640 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
641 pqn->q, NULL);
642 if (retval != 0)
643 return retval;
644
645 return 0;
646}
647
648int pqm_update_mqd(struct process_queue_manager *pqm,
649 unsigned int qid, struct mqd_update_info *minfo)
650{
651 int retval;
652 struct process_queue_node *pqn;
653
654 pqn = get_queue_by_qid(pqm, qid);
655 if (!pqn) {
656 pr_debug("No queue %d exists for update operation\n", qid);
657 return -EFAULT;
658 }
659
660 /* CUs are masked for debugger requirements so deny user mask */
661 if (pqn->q->properties.is_dbg_wa && minfo && minfo->cu_mask.ptr)
662 return -EBUSY;
663
664 /* ASICs that have WGPs must enforce pairwise enabled mask checks. */
665 if (minfo && minfo->cu_mask.ptr &&
666 KFD_GC_VERSION(pqn->q->device) >= IP_VERSION(10, 0, 0)) {
667 int i;
668
669 for (i = 0; i < minfo->cu_mask.count; i += 2) {
670 uint32_t cu_pair = (minfo->cu_mask.ptr[i / 32] >> (i % 32)) & 0x3;
671
672 if (cu_pair && cu_pair != 0x3) {
673 pr_debug("CUs must be adjacent pairwise enabled.\n");
674 return -EINVAL;
675 }
676 }
677 }
678
679 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
680 pqn->q, minfo);
681 if (retval != 0)
682 return retval;
683
684 if (minfo && minfo->cu_mask.ptr)
685 pqn->q->properties.is_user_cu_masked = true;
686
687 return 0;
688}
689
690struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
691 unsigned int qid)
692{
693 struct process_queue_node *pqn;
694
695 pqn = get_queue_by_qid(pqm, qid);
696 return pqn ? pqn->q : NULL;
697}
698
699int pqm_get_wave_state(struct process_queue_manager *pqm,
700 unsigned int qid,
701 void __user *ctl_stack,
702 u32 *ctl_stack_used_size,
703 u32 *save_area_used_size)
704{
705 struct process_queue_node *pqn;
706
707 pqn = get_queue_by_qid(pqm, qid);
708 if (!pqn) {
709 pr_debug("amdkfd: No queue %d exists for operation\n",
710 qid);
711 return -EFAULT;
712 }
713
714 return pqn->q->device->dqm->ops.get_wave_state(pqn->q->device->dqm,
715 pqn->q,
716 ctl_stack,
717 ctl_stack_used_size,
718 save_area_used_size);
719}
720
721int pqm_get_queue_snapshot(struct process_queue_manager *pqm,
722 uint64_t exception_clear_mask,
723 void __user *buf,
724 int *num_qss_entries,
725 uint32_t *entry_size)
726{
727 struct process_queue_node *pqn;
728 struct kfd_queue_snapshot_entry src;
729 uint32_t tmp_entry_size = *entry_size, tmp_qss_entries = *num_qss_entries;
730 int r = 0;
731
732 *num_qss_entries = 0;
733 if (!(*entry_size))
734 return -EINVAL;
735
736 *entry_size = min_t(size_t, *entry_size, sizeof(struct kfd_queue_snapshot_entry));
737 mutex_lock(&pqm->process->event_mutex);
738
739 memset(&src, 0, sizeof(src));
740
741 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
742 if (!pqn->q)
743 continue;
744
745 if (*num_qss_entries < tmp_qss_entries) {
746 set_queue_snapshot_entry(pqn->q, exception_clear_mask, &src);
747
748 if (copy_to_user(buf, &src, *entry_size)) {
749 r = -EFAULT;
750 break;
751 }
752 buf += tmp_entry_size;
753 }
754 *num_qss_entries += 1;
755 }
756
757 mutex_unlock(&pqm->process->event_mutex);
758 return r;
759}
760
761static int get_queue_data_sizes(struct kfd_process_device *pdd,
762 struct queue *q,
763 uint32_t *mqd_size,
764 uint32_t *ctl_stack_size)
765{
766 int ret;
767
768 ret = pqm_get_queue_checkpoint_info(&pdd->process->pqm,
769 q->properties.queue_id,
770 mqd_size,
771 ctl_stack_size);
772 if (ret)
773 pr_err("Failed to get queue dump info (%d)\n", ret);
774
775 return ret;
776}
777
778int kfd_process_get_queue_info(struct kfd_process *p,
779 uint32_t *num_queues,
780 uint64_t *priv_data_sizes)
781{
782 uint32_t extra_data_sizes = 0;
783 struct queue *q;
784 int i;
785 int ret;
786
787 *num_queues = 0;
788
789 /* Run over all PDDs of the process */
790 for (i = 0; i < p->n_pdds; i++) {
791 struct kfd_process_device *pdd = p->pdds[i];
792
793 list_for_each_entry(q, &pdd->qpd.queues_list, list) {
794 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
795 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
796 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
797 uint32_t mqd_size, ctl_stack_size;
798
799 *num_queues = *num_queues + 1;
800
801 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
802 if (ret)
803 return ret;
804
805 extra_data_sizes += mqd_size + ctl_stack_size;
806 } else {
807 pr_err("Unsupported queue type (%d)\n", q->properties.type);
808 return -EOPNOTSUPP;
809 }
810 }
811 }
812 *priv_data_sizes = extra_data_sizes +
813 (*num_queues * sizeof(struct kfd_criu_queue_priv_data));
814
815 return 0;
816}
817
818static int pqm_checkpoint_mqd(struct process_queue_manager *pqm,
819 unsigned int qid,
820 void *mqd,
821 void *ctl_stack)
822{
823 struct process_queue_node *pqn;
824
825 pqn = get_queue_by_qid(pqm, qid);
826 if (!pqn) {
827 pr_debug("amdkfd: No queue %d exists for operation\n", qid);
828 return -EFAULT;
829 }
830
831 if (!pqn->q->device->dqm->ops.checkpoint_mqd) {
832 pr_err("amdkfd: queue dumping not supported on this device\n");
833 return -EOPNOTSUPP;
834 }
835
836 return pqn->q->device->dqm->ops.checkpoint_mqd(pqn->q->device->dqm,
837 pqn->q, mqd, ctl_stack);
838}
839
840static int criu_checkpoint_queue(struct kfd_process_device *pdd,
841 struct queue *q,
842 struct kfd_criu_queue_priv_data *q_data)
843{
844 uint8_t *mqd, *ctl_stack;
845 int ret;
846
847 mqd = (void *)(q_data + 1);
848 ctl_stack = mqd + q_data->mqd_size;
849
850 q_data->gpu_id = pdd->user_gpu_id;
851 q_data->type = q->properties.type;
852 q_data->format = q->properties.format;
853 q_data->q_id = q->properties.queue_id;
854 q_data->q_address = q->properties.queue_address;
855 q_data->q_size = q->properties.queue_size;
856 q_data->priority = q->properties.priority;
857 q_data->q_percent = q->properties.queue_percent;
858 q_data->read_ptr_addr = (uint64_t)q->properties.read_ptr;
859 q_data->write_ptr_addr = (uint64_t)q->properties.write_ptr;
860 q_data->doorbell_id = q->doorbell_id;
861
862 q_data->sdma_id = q->sdma_id;
863
864 q_data->eop_ring_buffer_address =
865 q->properties.eop_ring_buffer_address;
866
867 q_data->eop_ring_buffer_size = q->properties.eop_ring_buffer_size;
868
869 q_data->ctx_save_restore_area_address =
870 q->properties.ctx_save_restore_area_address;
871
872 q_data->ctx_save_restore_area_size =
873 q->properties.ctx_save_restore_area_size;
874
875 q_data->gws = !!q->gws;
876
877 ret = pqm_checkpoint_mqd(&pdd->process->pqm, q->properties.queue_id, mqd, ctl_stack);
878 if (ret) {
879 pr_err("Failed checkpoint queue_mqd (%d)\n", ret);
880 return ret;
881 }
882
883 pr_debug("Dumping Queue: gpu_id:%x queue_id:%u\n", q_data->gpu_id, q_data->q_id);
884 return ret;
885}
886
887static int criu_checkpoint_queues_device(struct kfd_process_device *pdd,
888 uint8_t __user *user_priv,
889 unsigned int *q_index,
890 uint64_t *queues_priv_data_offset)
891{
892 unsigned int q_private_data_size = 0;
893 uint8_t *q_private_data = NULL; /* Local buffer to store individual queue private data */
894 struct queue *q;
895 int ret = 0;
896
897 list_for_each_entry(q, &pdd->qpd.queues_list, list) {
898 struct kfd_criu_queue_priv_data *q_data;
899 uint64_t q_data_size;
900 uint32_t mqd_size;
901 uint32_t ctl_stack_size;
902
903 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE &&
904 q->properties.type != KFD_QUEUE_TYPE_SDMA &&
905 q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI) {
906
907 pr_err("Unsupported queue type (%d)\n", q->properties.type);
908 ret = -EOPNOTSUPP;
909 break;
910 }
911
912 ret = get_queue_data_sizes(pdd, q, &mqd_size, &ctl_stack_size);
913 if (ret)
914 break;
915
916 q_data_size = sizeof(*q_data) + mqd_size + ctl_stack_size;
917
918 /* Increase local buffer space if needed */
919 if (q_private_data_size < q_data_size) {
920 kfree(q_private_data);
921
922 q_private_data = kzalloc(q_data_size, GFP_KERNEL);
923 if (!q_private_data) {
924 ret = -ENOMEM;
925 break;
926 }
927 q_private_data_size = q_data_size;
928 }
929
930 q_data = (struct kfd_criu_queue_priv_data *)q_private_data;
931
932 /*
933 * data stored in this order:
934 * priv_data, mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]...
935 */
936 q_data->mqd_size = mqd_size;
937 q_data->ctl_stack_size = ctl_stack_size;
938
939 ret = criu_checkpoint_queue(pdd, q, q_data);
940 if (ret)
941 break;
942
943 q_data->object_type = KFD_CRIU_OBJECT_TYPE_QUEUE;
944
945 ret = copy_to_user(user_priv + *queues_priv_data_offset,
946 q_data, q_data_size);
947 if (ret) {
948 ret = -EFAULT;
949 break;
950 }
951 *queues_priv_data_offset += q_data_size;
952 *q_index = *q_index + 1;
953 }
954
955 kfree(q_private_data);
956
957 return ret;
958}
959
960int kfd_criu_checkpoint_queues(struct kfd_process *p,
961 uint8_t __user *user_priv_data,
962 uint64_t *priv_data_offset)
963{
964 int ret = 0, pdd_index, q_index = 0;
965
966 for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
967 struct kfd_process_device *pdd = p->pdds[pdd_index];
968
969 /*
970 * criu_checkpoint_queues_device will copy data to user and update q_index and
971 * queues_priv_data_offset
972 */
973 ret = criu_checkpoint_queues_device(pdd, user_priv_data, &q_index,
974 priv_data_offset);
975
976 if (ret)
977 break;
978 }
979
980 return ret;
981}
982
983static void set_queue_properties_from_criu(struct queue_properties *qp,
984 struct kfd_criu_queue_priv_data *q_data, uint32_t num_xcc)
985{
986 qp->is_interop = false;
987 qp->queue_percent = q_data->q_percent;
988 qp->priority = q_data->priority;
989 qp->queue_address = q_data->q_address;
990 qp->queue_size = q_data->q_size;
991 qp->read_ptr = (void __user *)q_data->read_ptr_addr;
992 qp->write_ptr = (void __user *)q_data->write_ptr_addr;
993 qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address;
994 qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size;
995 qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address;
996 qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size;
997 if (q_data->type == KFD_QUEUE_TYPE_COMPUTE)
998 qp->ctl_stack_size = q_data->ctl_stack_size / num_xcc;
999 else
1000 qp->ctl_stack_size = q_data->ctl_stack_size;
1001
1002 qp->type = q_data->type;
1003 qp->format = q_data->format;
1004}
1005
1006int kfd_criu_restore_queue(struct kfd_process *p,
1007 uint8_t __user *user_priv_ptr,
1008 uint64_t *priv_data_offset,
1009 uint64_t max_priv_data_size)
1010{
1011 uint8_t *mqd, *ctl_stack, *q_extra_data = NULL;
1012 struct kfd_criu_queue_priv_data *q_data;
1013 struct kfd_process_device *pdd;
1014 uint64_t q_extra_data_size;
1015 struct queue_properties qp;
1016 unsigned int queue_id;
1017 int ret = 0;
1018
1019 if (*priv_data_offset + sizeof(*q_data) > max_priv_data_size)
1020 return -EINVAL;
1021
1022 q_data = kmalloc_obj(*q_data);
1023 if (!q_data)
1024 return -ENOMEM;
1025
1026 ret = copy_from_user(q_data, user_priv_ptr + *priv_data_offset, sizeof(*q_data));
1027 if (ret) {
1028 ret = -EFAULT;
1029 goto exit;
1030 }
1031
1032 pdd = kfd_process_device_data_by_id(p, q_data->gpu_id);
1033 if (!pdd) {
1034 pr_err("Failed to get pdd\n");
1035 ret = -EINVAL;
1036 goto exit;
1037 }
1038
1039 if (q_data->type >= KFD_QUEUE_TYPE_MAX) {
1040 ret = -EINVAL;
1041 goto exit;
1042 }
1043
1044 if (q_data->mqd_size != mqd_size_from_queue_type(pdd->dev->dqm, q_data->type)) {
1045 ret = -EINVAL;
1046 goto exit;
1047 }
1048
1049 *priv_data_offset += sizeof(*q_data);
1050 q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size;
1051
1052 if (*priv_data_offset + q_extra_data_size > max_priv_data_size) {
1053 ret = -EINVAL;
1054 goto exit;
1055 }
1056
1057 q_extra_data = kmalloc(q_extra_data_size, GFP_KERNEL);
1058 if (!q_extra_data) {
1059 ret = -ENOMEM;
1060 goto exit;
1061 }
1062
1063 ret = copy_from_user(q_extra_data, user_priv_ptr + *priv_data_offset, q_extra_data_size);
1064 if (ret) {
1065 ret = -EFAULT;
1066 goto exit;
1067 }
1068
1069 *priv_data_offset += q_extra_data_size;
1070
1071 /*
1072 * data stored in this order:
1073 * mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]...
1074 */
1075 mqd = q_extra_data;
1076 ctl_stack = mqd + q_data->mqd_size;
1077
1078 memset(&qp, 0, sizeof(qp));
1079 set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->xcc_mask));
1080
1081 print_queue_properties(&qp);
1082
1083 ret = pqm_create_queue(&p->pqm, pdd->dev, &qp, &queue_id, q_data, mqd, ctl_stack, NULL);
1084 if (ret) {
1085 pr_err("Failed to create new queue err:%d\n", ret);
1086 goto exit;
1087 }
1088
1089 if (q_data->gws)
1090 ret = pqm_set_gws(&p->pqm, q_data->q_id, pdd->dev->gws);
1091
1092exit:
1093 if (ret)
1094 pr_err("Failed to restore queue (%d)\n", ret);
1095 else
1096 pr_debug("Queue id %d was restored successfully\n", queue_id);
1097
1098 kfree(q_data);
1099 kfree(q_extra_data);
1100
1101 return ret;
1102}
1103
1104int pqm_get_queue_checkpoint_info(struct process_queue_manager *pqm,
1105 unsigned int qid,
1106 uint32_t *mqd_size,
1107 uint32_t *ctl_stack_size)
1108{
1109 struct process_queue_node *pqn;
1110 int ret;
1111
1112 pqn = get_queue_by_qid(pqm, qid);
1113 if (!pqn) {
1114 pr_debug("amdkfd: No queue %d exists for operation\n", qid);
1115 return -EFAULT;
1116 }
1117
1118 if (!pqn->q->device->dqm->ops.get_queue_checkpoint_info) {
1119 pr_err("amdkfd: queue dumping not supported on this device\n");
1120 return -EOPNOTSUPP;
1121 }
1122
1123 ret = pqn->q->device->dqm->ops.get_queue_checkpoint_info(pqn->q->device->dqm,
1124 pqn->q, mqd_size,
1125 ctl_stack_size);
1126 if (ret) {
1127 pr_debug("amdkfd: Overflow while computing stack size for queue %d\n", qid);
1128 return ret;
1129 }
1130
1131 return 0;
1132}
1133
1134#if defined(CONFIG_DEBUG_FS)
1135
1136int pqm_debugfs_mqds(struct seq_file *m, void *data)
1137{
1138 struct process_queue_manager *pqm = data;
1139 struct process_queue_node *pqn;
1140 struct queue *q;
1141 enum KFD_MQD_TYPE mqd_type;
1142 struct mqd_manager *mqd_mgr;
1143 int r = 0, xcc, num_xccs = 1;
1144 void *mqd;
1145 uint64_t size = 0;
1146
1147 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
1148 if (pqn->q) {
1149 q = pqn->q;
1150 switch (q->properties.type) {
1151 case KFD_QUEUE_TYPE_SDMA:
1152 case KFD_QUEUE_TYPE_SDMA_XGMI:
1153 seq_printf(m, " SDMA queue on device %x\n",
1154 q->device->id);
1155 mqd_type = KFD_MQD_TYPE_SDMA;
1156 break;
1157 case KFD_QUEUE_TYPE_COMPUTE:
1158 seq_printf(m, " Compute queue on device %x\n",
1159 q->device->id);
1160 mqd_type = KFD_MQD_TYPE_CP;
1161 num_xccs = NUM_XCC(q->device->xcc_mask);
1162 break;
1163 default:
1164 seq_printf(m,
1165 " Queue node with bad user queue type %d on device %x\n",
1166 q->properties.type, q->device->id);
1167 continue;
1168 }
1169 mqd_mgr = q->device->dqm->mqd_mgrs[mqd_type];
1170 size = mqd_mgr->mqd_stride(mqd_mgr,
1171 &q->properties);
1172
1173 for (xcc = 0; xcc < num_xccs; xcc++) {
1174 mqd = q->mqd + size * xcc;
1175 r = mqd_mgr->debugfs_show_mqd(m, mqd);
1176 if (r != 0)
1177 break;
1178 }
1179 }
1180 }
1181
1182 return r;
1183}
1184
1185#endif