projects/linux·blob master · refs

2486 lines · 63.3 KiB

1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#include <linux/mutex.h>
25#include <linux/log2.h>
26#include <linux/sched.h>
27#include <linux/sched/mm.h>
28#include <linux/sched/task.h>
29#include <linux/mmu_context.h>
30#include <linux/slab.h>
31#include <linux/notifier.h>
32#include <linux/compat.h>
33#include <linux/mman.h>
34#include <linux/file.h>
35#include <linux/pm_runtime.h>
36#include <drm/ttm/ttm_bo.h>
37#include "amdgpu_amdkfd.h"
38#include "amdgpu.h"
39#include "amdgpu_reset.h"
40
41struct mm_struct;
42
43#include "kfd_priv.h"
44#include "kfd_device_queue_manager.h"
45#include "kfd_svm.h"
46#include "kfd_smi_events.h"
47#include "kfd_debug.h"
48
49/*
50 * List of struct kfd_process (field kfd_process).
51 * Unique/indexed by mm_struct*
52 */
53DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
54DEFINE_MUTEX(kfd_processes_mutex);
55
56DEFINE_SRCU(kfd_processes_srcu);
57
58/* For process termination handling */
59static struct workqueue_struct *kfd_process_wq;
60
61/* Ordered, single-threaded workqueue for restoring evicted
62 * processes. Restoring multiple processes concurrently under memory
63 * pressure can lead to processes blocking each other from validating
64 * their BOs and result in a live-lock situation where processes
65 * remain evicted indefinitely.
66 */
67static struct workqueue_struct *kfd_restore_wq;
68
69static struct kfd_process *find_process(const struct task_struct *thread,
70 bool ref);
71static void kfd_process_ref_release(struct kref *ref);
72
73static void evict_process_worker(struct work_struct *work);
74static void restore_process_worker(struct work_struct *work);
75
76static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77
78struct kfd_procfs_tree {
79 struct kobject *kobj;
80};
81
82static struct kfd_procfs_tree procfs;
83
84/*
85 * Structure for SDMA activity tracking
86 */
87struct kfd_sdma_activity_handler_workarea {
88 struct work_struct sdma_activity_work;
89 struct kfd_process_device *pdd;
90 uint64_t sdma_activity_counter;
91};
92
93struct temp_sdma_queue_list {
94 uint64_t __user *rptr;
95 uint64_t sdma_val;
96 unsigned int queue_id;
97 struct list_head list;
98};
99
100static void kfd_sdma_activity_worker(struct work_struct *work)
101{
102 struct kfd_sdma_activity_handler_workarea *workarea;
103 struct kfd_process_device *pdd;
104 uint64_t val;
105 struct mm_struct *mm;
106 struct queue *q;
107 struct qcm_process_device *qpd;
108 struct device_queue_manager *dqm;
109 int ret = 0;
110 struct temp_sdma_queue_list sdma_q_list;
111 struct temp_sdma_queue_list *sdma_q, *next;
112
113 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 sdma_activity_work);
115
116 pdd = workarea->pdd;
117 if (!pdd)
118 return;
119 dqm = pdd->dev->dqm;
120 qpd = &pdd->qpd;
121 if (!dqm || !qpd)
122 return;
123 /*
124 * Total SDMA activity is current SDMA activity + past SDMA activity
125 * Past SDMA count is stored in pdd.
126 * To get the current activity counters for all active SDMA queues,
127 * we loop over all SDMA queues and get their counts from user-space.
128 *
129 * We cannot call get_user() with dqm_lock held as it can cause
130 * a circular lock dependency situation. To read the SDMA stats,
131 * we need to do the following:
132 *
133 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 * with dqm_lock/dqm_unlock().
135 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 * Save the SDMA count for each node and also add the count to the total
137 * SDMA count counter.
138 * Its possible, during this step, a few SDMA queue nodes got deleted
139 * from the qpd->queues_list.
140 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 * If any node got deleted, its SDMA count would be captured in the sdma
142 * past activity counter. So subtract the SDMA counter stored in step 2
143 * for this node from the total SDMA count.
144 */
145 INIT_LIST_HEAD(&sdma_q_list.list);
146
147 /*
148 * Create the temp list of all SDMA queues
149 */
150 dqm_lock(dqm);
151
152 list_for_each_entry(q, &qpd->queues_list, list) {
153 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 continue;
156
157 if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) {
158 val = 0;
159 ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter(
160 dqm->dev->adev, q->mqd,
161 dqm->dev->kfd->device_info.num_sdma_queues_per_engine,
162 &val);
163
164 if (ret)
165 pr_debug("Failed to read SDMA queue active counter %i\n", ret);
166 else
167 workarea->sdma_activity_counter += val;
168
169 continue;
170 }
171
172 sdma_q = kzalloc_obj(struct temp_sdma_queue_list);
173 if (!sdma_q) {
174 dqm_unlock(dqm);
175 goto cleanup;
176 }
177
178 INIT_LIST_HEAD(&sdma_q->list);
179 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
180 sdma_q->queue_id = q->properties.queue_id;
181 list_add_tail(&sdma_q->list, &sdma_q_list.list);
182 }
183
184 /*
185 * If the temp list is empty, then no SDMA queues nodes were found in
186 * qpd->queues_list. Return the past activity count as the total sdma
187 * count
188 */
189 if (list_empty(&sdma_q_list.list)) {
190 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
191 dqm_unlock(dqm);
192 return;
193 }
194
195 dqm_unlock(dqm);
196
197 /*
198 * Get the usage count for each SDMA queue in temp_list.
199 */
200 mm = get_task_mm(pdd->process->lead_thread);
201 if (!mm)
202 goto cleanup;
203
204 kthread_use_mm(mm);
205
206 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
207 val = 0;
208 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
209 if (ret) {
210 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
211 sdma_q->queue_id);
212 } else {
213 sdma_q->sdma_val = val;
214 workarea->sdma_activity_counter += val;
215 }
216 }
217
218 kthread_unuse_mm(mm);
219 mmput(mm);
220
221 /*
222 * Do a second iteration over qpd_queues_list to check if any SDMA
223 * nodes got deleted while fetching SDMA counter.
224 */
225 dqm_lock(dqm);
226
227 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
228
229 list_for_each_entry(q, &qpd->queues_list, list) {
230 if (list_empty(&sdma_q_list.list))
231 break;
232
233 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
234 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
235 continue;
236
237 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
238 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
239 (sdma_q->queue_id == q->properties.queue_id)) {
240 list_del(&sdma_q->list);
241 kfree(sdma_q);
242 break;
243 }
244 }
245 }
246
247 dqm_unlock(dqm);
248
249 /*
250 * If temp list is not empty, it implies some queues got deleted
251 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
252 * count for each node from the total SDMA count.
253 */
254 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
255 workarea->sdma_activity_counter -= sdma_q->sdma_val;
256 list_del(&sdma_q->list);
257 kfree(sdma_q);
258 }
259
260 return;
261
262cleanup:
263 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
264 list_del(&sdma_q->list);
265 kfree(sdma_q);
266 }
267}
268
269/**
270 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
271 * by current process. Translates acquired wave count into number of compute units
272 * that are occupied.
273 *
274 * @attr: Handle of attribute that allows reporting of wave count. The attribute
275 * handle encapsulates GPU device it is associated with, thereby allowing collection
276 * of waves in flight, etc
277 * @buffer: Handle of user provided buffer updated with wave count
278 *
279 * Return: Number of bytes written to user buffer or an error value
280 */
281static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
282{
283 int cu_cnt;
284 int wave_cnt;
285 int max_waves_per_cu;
286 struct kfd_node *dev = NULL;
287 struct kfd_process *proc = NULL;
288 struct kfd_process_device *pdd = NULL;
289 int i;
290 struct kfd_cu_occupancy *cu_occupancy;
291 u32 queue_format;
292
293 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
294 dev = pdd->dev;
295 if (dev->kfd2kgd->get_cu_occupancy == NULL)
296 return -EINVAL;
297
298 cu_cnt = 0;
299 proc = pdd->process;
300 if (pdd->qpd.queue_count == 0) {
301 pr_debug("Gpu-Id: %d has no active queues for process pid %d\n",
302 dev->id, (int)proc->lead_thread->pid);
303 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
304 }
305
306 /* Collect wave count from device if it supports */
307 wave_cnt = 0;
308 max_waves_per_cu = 0;
309
310 cu_occupancy = kzalloc_objs(*cu_occupancy, AMDGPU_MAX_QUEUES);
311 if (!cu_occupancy)
312 return -ENOMEM;
313
314 /*
315 * For GFX 9.4.3, fetch the CU occupancy from the first XCC in the partition.
316 * For AQL queues, because of cooperative dispatch we multiply the wave count
317 * by number of XCCs in the partition to get the total wave counts across all
318 * XCCs in the partition.
319 * For PM4 queues, there is no cooperative dispatch so wave_cnt stay as it is.
320 */
321 dev->kfd2kgd->get_cu_occupancy(dev->adev, cu_occupancy,
322 &max_waves_per_cu, ffs(dev->xcc_mask) - 1);
323
324 for (i = 0; i < AMDGPU_MAX_QUEUES; i++) {
325 if (cu_occupancy[i].wave_cnt != 0 &&
326 kfd_dqm_is_queue_in_process(dev->dqm, &pdd->qpd,
327 cu_occupancy[i].doorbell_off,
328 &queue_format)) {
329 if (unlikely(queue_format == KFD_QUEUE_FORMAT_PM4))
330 wave_cnt += cu_occupancy[i].wave_cnt;
331 else
332 wave_cnt += (NUM_XCC(dev->xcc_mask) *
333 cu_occupancy[i].wave_cnt);
334 }
335 }
336
337 /* Translate wave count to number of compute units */
338 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
339 kfree(cu_occupancy);
340 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
341}
342
343static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
344 char *buffer)
345{
346 if (strcmp(attr->name, "pasid") == 0)
347 return snprintf(buffer, PAGE_SIZE, "%d\n", 0);
348 else if (strncmp(attr->name, "vram_", 5) == 0) {
349 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
350 attr_vram);
351 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
352 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
353 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
354 attr_sdma);
355 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
356
357 INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
358 kfd_sdma_activity_worker);
359
360 sdma_activity_work_handler.pdd = pdd;
361 sdma_activity_work_handler.sdma_activity_counter = 0;
362
363 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
364
365 flush_work(&sdma_activity_work_handler.sdma_activity_work);
366 destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
367
368 return snprintf(buffer, PAGE_SIZE, "%llu\n",
369 (sdma_activity_work_handler.sdma_activity_counter)/
370 SDMA_ACTIVITY_DIVISOR);
371 } else {
372 pr_err("Invalid attribute");
373 return -EINVAL;
374 }
375
376 return 0;
377}
378
379static void kfd_procfs_kobj_release(struct kobject *kobj)
380{
381 kfree(kobj);
382}
383
384static const struct sysfs_ops kfd_procfs_ops = {
385 .show = kfd_procfs_show,
386};
387
388static const struct kobj_type procfs_type = {
389 .release = kfd_procfs_kobj_release,
390 .sysfs_ops = &kfd_procfs_ops,
391};
392
393void kfd_procfs_init(void)
394{
395 int ret = 0;
396
397 procfs.kobj = kfd_alloc_struct(procfs.kobj);
398 if (!procfs.kobj)
399 return;
400
401 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
402 &kfd_device->kobj, "proc");
403 if (ret) {
404 pr_warn("Could not create procfs proc folder");
405 /* If we fail to create the procfs, clean up */
406 kfd_procfs_shutdown();
407 }
408}
409
410void kfd_procfs_shutdown(void)
411{
412 if (procfs.kobj) {
413 kobject_del(procfs.kobj);
414 kobject_put(procfs.kobj);
415 procfs.kobj = NULL;
416 }
417}
418
419static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
420 struct attribute *attr, char *buffer)
421{
422 struct queue *q = container_of(kobj, struct queue, kobj);
423
424 if (!strcmp(attr->name, "size"))
425 return snprintf(buffer, PAGE_SIZE, "%llu",
426 q->properties.queue_size);
427 else if (!strcmp(attr->name, "type"))
428 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
429 else if (!strcmp(attr->name, "gpuid"))
430 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
431 else
432 pr_err("Invalid attribute");
433
434 return 0;
435}
436
437static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
438 struct attribute *attr, char *buffer)
439{
440 if (strcmp(attr->name, "evicted_ms") == 0) {
441 struct kfd_process_device *pdd = container_of(attr,
442 struct kfd_process_device,
443 attr_evict);
444 uint64_t evict_jiffies;
445
446 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
447
448 return snprintf(buffer,
449 PAGE_SIZE,
450 "%llu\n",
451 jiffies64_to_msecs(evict_jiffies));
452
453 /* Sysfs handle that gets CU occupancy is per device */
454 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
455 return kfd_get_cu_occupancy(attr, buffer);
456 } else {
457 pr_err("Invalid attribute");
458 }
459
460 return 0;
461}
462
463static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
464 struct attribute *attr, char *buf)
465{
466 struct kfd_process_device *pdd;
467
468 if (!strcmp(attr->name, "faults")) {
469 pdd = container_of(attr, struct kfd_process_device,
470 attr_faults);
471 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
472 }
473 if (!strcmp(attr->name, "page_in")) {
474 pdd = container_of(attr, struct kfd_process_device,
475 attr_page_in);
476 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
477 }
478 if (!strcmp(attr->name, "page_out")) {
479 pdd = container_of(attr, struct kfd_process_device,
480 attr_page_out);
481 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
482 }
483 return 0;
484}
485
486static struct attribute attr_queue_size = {
487 .name = "size",
488 .mode = KFD_SYSFS_FILE_MODE
489};
490
491static struct attribute attr_queue_type = {
492 .name = "type",
493 .mode = KFD_SYSFS_FILE_MODE
494};
495
496static struct attribute attr_queue_gpuid = {
497 .name = "gpuid",
498 .mode = KFD_SYSFS_FILE_MODE
499};
500
501static struct attribute *procfs_queue_attrs[] = {
502 &attr_queue_size,
503 &attr_queue_type,
504 &attr_queue_gpuid,
505 NULL
506};
507ATTRIBUTE_GROUPS(procfs_queue);
508
509static const struct sysfs_ops procfs_queue_ops = {
510 .show = kfd_procfs_queue_show,
511};
512
513static const struct kobj_type procfs_queue_type = {
514 .sysfs_ops = &procfs_queue_ops,
515 .default_groups = procfs_queue_groups,
516};
517
518static const struct sysfs_ops procfs_stats_ops = {
519 .show = kfd_procfs_stats_show,
520};
521
522static const struct kobj_type procfs_stats_type = {
523 .sysfs_ops = &procfs_stats_ops,
524 .release = kfd_procfs_kobj_release,
525};
526
527static const struct sysfs_ops sysfs_counters_ops = {
528 .show = kfd_sysfs_counters_show,
529};
530
531static const struct kobj_type sysfs_counters_type = {
532 .sysfs_ops = &sysfs_counters_ops,
533 .release = kfd_procfs_kobj_release,
534};
535
536int kfd_procfs_add_queue(struct queue *q)
537{
538 struct kfd_process *proc;
539 int ret;
540
541 if (!q || !q->process)
542 return -EINVAL;
543 proc = q->process;
544
545 /* Create proc/<pid>/queues/<queue id> folder */
546 if (!proc->kobj_queues)
547 return -EFAULT;
548 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
549 proc->kobj_queues, "%u", q->properties.queue_id);
550 if (ret < 0) {
551 pr_warn("Creating proc/<pid>/queues/%u failed",
552 q->properties.queue_id);
553 kobject_put(&q->kobj);
554 return ret;
555 }
556
557 return 0;
558}
559
560static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
561 char *name)
562{
563 int ret;
564
565 if (!kobj || !attr || !name)
566 return;
567
568 attr->name = name;
569 attr->mode = KFD_SYSFS_FILE_MODE;
570 sysfs_attr_init(attr);
571
572 ret = sysfs_create_file(kobj, attr);
573 if (ret)
574 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
575}
576
577static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
578{
579 int ret;
580 int i;
581 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
582
583 if (!p || !p->kobj)
584 return;
585
586 /*
587 * Create sysfs files for each GPU:
588 * - proc/<pid>/stats_<gpuid>/
589 * - proc/<pid>/stats_<gpuid>/evicted_ms
590 * - proc/<pid>/stats_<gpuid>/cu_occupancy
591 */
592 for (i = 0; i < p->n_pdds; i++) {
593 struct kfd_process_device *pdd = p->pdds[i];
594
595 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
596 "stats_%u", pdd->dev->id);
597 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
598 if (!pdd->kobj_stats)
599 return;
600
601 ret = kobject_init_and_add(pdd->kobj_stats,
602 &procfs_stats_type,
603 p->kobj,
604 "%s", stats_dir_filename);
605
606 if (ret) {
607 pr_warn("Creating KFD proc/stats_%s folder failed",
608 stats_dir_filename);
609 kobject_put(pdd->kobj_stats);
610 pdd->kobj_stats = NULL;
611 return;
612 }
613
614 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
615 "evicted_ms");
616 /* Add sysfs file to report compute unit occupancy */
617 if (pdd->dev->kfd2kgd->get_cu_occupancy)
618 kfd_sysfs_create_file(pdd->kobj_stats,
619 &pdd->attr_cu_occupancy,
620 "cu_occupancy");
621 }
622}
623
624static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
625{
626 int ret = 0;
627 int i;
628 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
629
630 if (!p || !p->kobj)
631 return;
632
633 /*
634 * Create sysfs files for each GPU which supports SVM
635 * - proc/<pid>/counters_<gpuid>/
636 * - proc/<pid>/counters_<gpuid>/faults
637 * - proc/<pid>/counters_<gpuid>/page_in
638 * - proc/<pid>/counters_<gpuid>/page_out
639 */
640 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
641 struct kfd_process_device *pdd = p->pdds[i];
642 struct kobject *kobj_counters;
643
644 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
645 "counters_%u", pdd->dev->id);
646 kobj_counters = kfd_alloc_struct(kobj_counters);
647 if (!kobj_counters)
648 return;
649
650 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
651 p->kobj, "%s", counters_dir_filename);
652 if (ret) {
653 pr_warn("Creating KFD proc/%s folder failed",
654 counters_dir_filename);
655 kobject_put(kobj_counters);
656 return;
657 }
658
659 pdd->kobj_counters = kobj_counters;
660 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
661 "faults");
662 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
663 "page_in");
664 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
665 "page_out");
666 }
667}
668
669static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
670{
671 int i;
672
673 if (!p || !p->kobj)
674 return;
675
676 /*
677 * Create sysfs files for each GPU:
678 * - proc/<pid>/vram_<gpuid>
679 * - proc/<pid>/sdma_<gpuid>
680 */
681 for (i = 0; i < p->n_pdds; i++) {
682 struct kfd_process_device *pdd = p->pdds[i];
683
684 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
685 pdd->dev->id);
686 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
687 pdd->vram_filename);
688
689 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
690 pdd->dev->id);
691 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
692 pdd->sdma_filename);
693 }
694}
695
696void kfd_procfs_del_queue(struct queue *q)
697{
698 if (!q || !q->process->kobj)
699 return;
700
701 kobject_del(&q->kobj);
702 kobject_put(&q->kobj);
703}
704
705int kfd_process_create_wq(void)
706{
707 if (!kfd_process_wq)
708 kfd_process_wq = alloc_workqueue("kfd_process_wq", WQ_UNBOUND,
709 0);
710 if (!kfd_restore_wq)
711 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq",
712 WQ_FREEZABLE);
713
714 if (!kfd_process_wq || !kfd_restore_wq) {
715 kfd_process_destroy_wq();
716 return -ENOMEM;
717 }
718
719 return 0;
720}
721
722void kfd_process_destroy_wq(void)
723{
724 if (kfd_process_wq) {
725 destroy_workqueue(kfd_process_wq);
726 kfd_process_wq = NULL;
727 }
728 if (kfd_restore_wq) {
729 destroy_workqueue(kfd_restore_wq);
730 kfd_restore_wq = NULL;
731 }
732}
733
734static void kfd_process_free_gpuvm(struct kgd_mem *mem,
735 struct kfd_process_device *pdd, void **kptr)
736{
737 struct kfd_node *dev = pdd->dev;
738
739 if (kptr && *kptr) {
740 amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
741 *kptr = NULL;
742 }
743
744 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
745 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
746 NULL);
747}
748
749static void kfd_process_free_gpuvm_map(struct kgd_mem *mem,
750 struct kfd_process_device *pdd,
751 struct iosys_map *map)
752{
753 struct kfd_node *dev = pdd->dev;
754
755 if (map && !iosys_map_is_null(map)) {
756 amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
757 iosys_map_clear(map);
758 }
759
760 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
761 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
762 NULL);
763}
764
765/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
766 * This function should be only called right after the process
767 * is created and when kfd_processes_mutex is still being held
768 * to avoid concurrency. Because of that exclusiveness, we do
769 * not need to take p->mutex.
770 */
771static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
772 uint64_t gpu_va, uint32_t size,
773 uint32_t flags, struct kgd_mem **mem, void **kptr)
774{
775 struct kfd_node *kdev = pdd->dev;
776 int err;
777
778 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
779 pdd->drm_priv, mem, NULL,
780 flags, false);
781 if (err)
782 goto err_alloc_mem;
783
784 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
785 pdd->drm_priv);
786 if (err)
787 goto err_map_mem;
788
789 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
790 if (err) {
791 pr_debug("Sync memory failed, wait interrupted by user signal\n");
792 goto sync_memory_failed;
793 }
794
795 if (kptr) {
796 u32 domain;
797
798 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
799 domain = AMDGPU_GEM_DOMAIN_VRAM;
800 else
801 domain = AMDGPU_GEM_DOMAIN_GTT;
802 err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel((struct kgd_mem *)*mem,
803 kptr, NULL, domain);
804 if (err) {
805 pr_debug("Map BO to kernel failed err %d\n", err);
806 goto sync_memory_failed;
807 }
808 }
809
810 return err;
811
812sync_memory_failed:
813 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
814
815err_map_mem:
816 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
817 NULL);
818err_alloc_mem:
819 *mem = NULL;
820 *kptr = NULL;
821 return err;
822}
823
824/* kfd_process_device_reserve_ib_mem - Reserve memory inside the
825 * process for IB usage The memory reserved is for KFD to submit
826 * IB to AMDGPU from kernel. If the memory is reserved
827 * successfully, ib_kaddr will have the CPU/kernel
828 * address. Check ib_kaddr before accessing the memory.
829 */
830static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
831{
832 struct qcm_process_device *qpd = &pdd->qpd;
833 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
834 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
835 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
836 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
837 struct kgd_mem *mem;
838 void *kaddr;
839 int ret;
840
841 if (qpd->ib_kaddr || !qpd->ib_base)
842 return 0;
843
844 /* ib_base is only set for dGPU */
845 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
846 &mem, &kaddr);
847 if (ret)
848 return ret;
849
850 qpd->ib_mem = mem;
851 qpd->ib_kaddr = kaddr;
852
853 return 0;
854}
855
856static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
857{
858 struct qcm_process_device *qpd = &pdd->qpd;
859
860 if (!qpd->ib_kaddr || !qpd->ib_base)
861 return;
862
863 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
864}
865
866int kfd_create_process_sysfs(struct kfd_process *process)
867{
868 struct kfd_process *primary_process;
869 int ret;
870
871 if (process->kobj) {
872 pr_warn("kobject already exists for the kfd_process\n");
873 return -EINVAL;
874 }
875
876 process->kobj = kfd_alloc_struct(process->kobj);
877 if (!process->kobj) {
878 pr_warn("Creating procfs kobject failed");
879 return -ENOMEM;
880 }
881
882 if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
883 ret = kobject_init_and_add(process->kobj, &procfs_type,
884 procfs.kobj, "%d",
885 (int)process->lead_thread->pid);
886 else {
887 primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
888 if (!primary_process)
889 return -ESRCH;
890
891 ret = kobject_init_and_add(process->kobj, &procfs_type,
892 primary_process->kobj, "context_%u",
893 process->context_id);
894 kfd_unref_process(primary_process);
895 }
896
897 if (ret) {
898 pr_warn("Creating procfs pid directory failed");
899 kobject_put(process->kobj);
900 process->kobj = NULL;
901 return ret;
902 }
903
904 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
905 "pasid");
906
907 process->kobj_queues = kobject_create_and_add("queues",
908 process->kobj);
909 if (!process->kobj_queues)
910 pr_warn("Creating KFD proc/queues folder failed");
911
912 kfd_procfs_add_sysfs_stats(process);
913 kfd_procfs_add_sysfs_files(process);
914 kfd_procfs_add_sysfs_counters(process);
915
916 return 0;
917}
918
919static int kfd_process_alloc_id(struct kfd_process *process)
920{
921 int ret;
922 struct kfd_process *primary_process;
923
924 /* already assign 0xFFFF when create */
925 if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
926 return 0;
927
928 primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
929 if (!primary_process)
930 return -ESRCH;
931
932 /* id range: KFD_CONTEXT_ID_MIN to 0xFFFE */
933 ret = ida_alloc_range(&primary_process->id_table, KFD_CONTEXT_ID_MIN,
934 KFD_CONTEXT_ID_PRIMARY - 1, GFP_KERNEL);
935 if (ret < 0)
936 goto out;
937
938 process->context_id = ret;
939 ret = 0;
940
941out:
942 kfd_unref_process(primary_process);
943
944 return ret;
945}
946
947static void kfd_process_free_id(struct kfd_process *process)
948{
949 struct kfd_process *primary_process;
950
951 if (process->context_id == KFD_CONTEXT_ID_PRIMARY)
952 return;
953
954 primary_process = kfd_lookup_process_by_mm(process->lead_thread->mm);
955 if (!primary_process)
956 return;
957
958 ida_free(&primary_process->id_table, process->context_id);
959
960 kfd_unref_process(primary_process);
961}
962
963struct kfd_process *kfd_create_process(struct task_struct *thread)
964{
965 struct kfd_process *process;
966 int ret;
967
968 if (!(thread->mm && mmget_not_zero(thread->mm)))
969 return ERR_PTR(-EINVAL);
970
971 /* If the process just called exec(3), it is possible that the
972 * cleanup of the kfd_process (following the release of the mm
973 * of the old process image) is still in the cleanup work queue.
974 * Make sure to drain any job before trying to recreate any
975 * resource for this process.
976 */
977 flush_workqueue(kfd_process_wq);
978
979 /*
980 * take kfd processes mutex before starting of process creation
981 * so there won't be a case where two threads of the same process
982 * create two kfd_process structures
983 */
984 mutex_lock(&kfd_processes_mutex);
985
986 if (kfd_gpu_node_num() <= 0) {
987 pr_warn("no gpu node! Cannot create KFD process");
988 process = ERR_PTR(-EINVAL);
989 goto out;
990 }
991
992 if (kfd_is_locked(NULL)) {
993 pr_debug("KFD is locked! Cannot create process");
994 process = ERR_PTR(-EINVAL);
995 goto out;
996 }
997
998 /* A prior open of /dev/kfd could have already created the process.
999 * find_process will increase process kref in this case
1000 */
1001 process = find_process(thread, true);
1002 if (process) {
1003 pr_debug("Process already found\n");
1004 } else {
1005 process = create_process(thread, true);
1006 if (IS_ERR(process))
1007 goto out;
1008
1009 if (!procfs.kobj)
1010 goto out;
1011
1012 ret = kfd_create_process_sysfs(process);
1013 if (ret)
1014 pr_warn("Failed to create sysfs entry for the kfd_process");
1015
1016 ret = kfd_debugfs_add_process(process);
1017 if (ret)
1018 pr_warn("Failed to create debugfs entry for the kfd_process, ret = %d\n",
1019 ret);
1020
1021 init_waitqueue_head(&process->wait_irq_drain);
1022 }
1023out:
1024 mutex_unlock(&kfd_processes_mutex);
1025 mmput(thread->mm);
1026
1027 return process;
1028}
1029
1030/**
1031 * amdgpu_amdkfd_set_sigbus_delay - Set per-process KFD SIGBUS delay
1032 * @task: task in the target process
1033 * @ms: encoded delay value (0 = immediate, 0xFFFFFFFF = suppress,
1034 * otherwise delay in milliseconds)
1035 *
1036 * Stores the SIGBUS delivery option on the kfd_process associated with
1037 * @task. If the calling process has not opened /dev/kfd yet (no
1038 * kfd_process exists), this is a no-op - the option only applies to
1039 * processes that actually use KFD.
1040 */
1041int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms)
1042{
1043 struct kfd_process *p;
1044
1045 if (!task->mm)
1046 return -EINVAL;
1047
1048 p = kfd_lookup_process_by_mm(task->mm);
1049 if (!p)
1050 return 0;
1051
1052 atomic_set(&p->kfd_sigbus_delay_ms, ms);
1053 kfd_unref_process(p);
1054 return 0;
1055}
1056
1057static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
1058{
1059 struct kfd_process *process;
1060
1061 hash_for_each_possible_rcu(kfd_processes_table, process,
1062 kfd_processes, (uintptr_t)mm)
1063 if (process->mm == mm && process->context_id == KFD_CONTEXT_ID_PRIMARY)
1064 return process;
1065
1066 return NULL;
1067}
1068
1069static struct kfd_process *find_process(const struct task_struct *thread,
1070 bool ref)
1071{
1072 struct kfd_process *p;
1073 int idx;
1074
1075 idx = srcu_read_lock(&kfd_processes_srcu);
1076 p = find_process_by_mm(thread->mm);
1077 if (p && ref)
1078 kref_get(&p->ref);
1079 srcu_read_unlock(&kfd_processes_srcu, idx);
1080
1081 return p;
1082}
1083
1084void kfd_unref_process(struct kfd_process *p)
1085{
1086 kref_put(&p->ref, kfd_process_ref_release);
1087}
1088
1089/* This increments the process->ref counter. */
1090struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
1091{
1092 struct task_struct *task = NULL;
1093 struct kfd_process *p = NULL;
1094
1095 if (!pid) {
1096 task = current;
1097 get_task_struct(task);
1098 } else {
1099 task = get_pid_task(pid, PIDTYPE_PID);
1100 }
1101
1102 if (task) {
1103 p = find_process(task, true);
1104 put_task_struct(task);
1105 }
1106
1107 return p;
1108}
1109
1110static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
1111{
1112 struct kfd_process *p = pdd->process;
1113 void *mem;
1114 int id;
1115 int i;
1116
1117 /*
1118 * Remove all handles from idr and release appropriate
1119 * local memory object
1120 */
1121 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1122
1123 for (i = 0; i < p->n_pdds; i++) {
1124 struct kfd_process_device *peer_pdd = p->pdds[i];
1125
1126 if (!peer_pdd->drm_priv)
1127 continue;
1128 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1129 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
1130 }
1131
1132 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
1133 pdd->drm_priv, NULL);
1134 kfd_process_device_remove_obj_handle(pdd, id);
1135 }
1136}
1137
1138/*
1139 * Just kunmap and unpin signal BO here. It will be freed in
1140 * kfd_process_free_outstanding_kfd_bos()
1141 */
1142static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
1143{
1144 struct kfd_process_device *pdd;
1145 struct kfd_node *kdev;
1146 void *mem;
1147
1148 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
1149 if (!kdev)
1150 return;
1151
1152 mutex_lock(&p->mutex);
1153
1154 pdd = kfd_get_process_device_data(kdev, p);
1155 if (!pdd)
1156 goto out;
1157
1158 mem = kfd_process_device_translate_handle(
1159 pdd, GET_IDR_HANDLE(p->signal_handle));
1160 if (!mem)
1161 goto out;
1162
1163 amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem);
1164
1165out:
1166 mutex_unlock(&p->mutex);
1167}
1168
1169static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1170{
1171 int i;
1172
1173 for (i = 0; i < p->n_pdds; i++)
1174 kfd_process_device_free_bos(p->pdds[i]);
1175}
1176
1177static void kfd_process_profiler_release(struct kfd_process *p, struct kfd_process_device *pdd)
1178{
1179 mutex_lock(&pdd->dev->kfd->profiler_lock);
1180 if (pdd->dev->kfd->profiler_process == p) {
1181 pdd->qpd.dqm->ops.set_perfcount(pdd->qpd.dqm, 0);
1182 pdd->dev->kfd->profiler_process = NULL;
1183 }
1184 mutex_unlock(&pdd->dev->kfd->profiler_lock);
1185}
1186
1187static void kfd_process_destroy_pdds(struct kfd_process *p)
1188{
1189 int i;
1190
1191 for (i = 0; i < p->n_pdds; i++) {
1192 struct kfd_process_device *pdd = p->pdds[i];
1193
1194 kfd_smi_event_process(pdd, false);
1195
1196 pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
1197 pdd->dev->id, p->lead_thread->pid);
1198 kfd_process_profiler_release(p, pdd);
1199
1200 if (pdd->ptl_disable_req)
1201 kfd_ptl_disable_release(pdd, p);
1202
1203 kfd_process_device_destroy_cwsr_dgpu(pdd);
1204 kfd_process_device_destroy_ib_mem(pdd);
1205
1206 if (pdd->drm_file)
1207 fput(pdd->drm_file);
1208
1209 if (!iosys_map_is_null(&pdd->qpd.cwsr_map) && !pdd->qpd.cwsr_base)
1210 free_pages((unsigned long)pdd->qpd.cwsr_map.vaddr,
1211 get_order(KFD_CWSR_TBA_TMA_SIZE));
1212
1213 idr_destroy(&pdd->alloc_idr);
1214
1215 kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1216
1217 if (pdd->dev->kfd->shared_resources.enable_mes &&
1218 pdd->proc_ctx_cpu_ptr) {
1219 amdgpu_mes_free_proc_ctx_index(&pdd->dev->adev->mes,
1220 pdd->proc_ctx_array_index);
1221 amdgpu_amdkfd_free_kernel_mem(pdd->dev->adev,
1222 &pdd->proc_ctx_bo);
1223 }
1224 /*
1225 * before destroying pdd, make sure to report availability
1226 * for auto suspend
1227 */
1228 if (pdd->runtime_inuse) {
1229 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1230 pdd->runtime_inuse = false;
1231 }
1232
1233 atomic_dec(&pdd->dev->kfd->kfd_processes_count);
1234
1235 kfree(pdd);
1236 p->pdds[i] = NULL;
1237 }
1238 p->n_pdds = 0;
1239}
1240
1241static void kfd_process_remove_sysfs(struct kfd_process *p)
1242{
1243 struct kfd_process_device *pdd;
1244 int i;
1245
1246 if (!p->kobj)
1247 return;
1248
1249 if (p->kobj_queues) {
1250 sysfs_remove_file(p->kobj, &p->attr_pasid);
1251 kobject_del(p->kobj_queues);
1252 kobject_put(p->kobj_queues);
1253 p->kobj_queues = NULL;
1254 }
1255
1256 for (i = 0; i < p->n_pdds; i++) {
1257 pdd = p->pdds[i];
1258
1259 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1260 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1261
1262 if (pdd->kobj_stats) {
1263 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1264 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1265 sysfs_remove_file(pdd->kobj_stats,
1266 &pdd->attr_cu_occupancy);
1267 kobject_del(pdd->kobj_stats);
1268 kobject_put(pdd->kobj_stats);
1269 pdd->kobj_stats = NULL;
1270 }
1271 }
1272
1273 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1274 pdd = p->pdds[i];
1275 if (!pdd->kobj_counters)
1276 continue;
1277
1278 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1279 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1280 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1281 kobject_del(pdd->kobj_counters);
1282 kobject_put(pdd->kobj_counters);
1283 pdd->kobj_counters = NULL;
1284 }
1285
1286 kobject_del(p->kobj);
1287 kobject_put(p->kobj);
1288 p->kobj = NULL;
1289}
1290
1291/*
1292 * If any GPU is ongoing reset, wait for reset complete.
1293 */
1294static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1295{
1296 int i;
1297
1298 for (i = 0; i < p->n_pdds; i++)
1299 flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1300}
1301
1302/* No process locking is needed in this function, because the process
1303 * is not findable any more. We must assume that no other thread is
1304 * using it any more, otherwise we couldn't safely free the process
1305 * structure in the end.
1306 */
1307static void kfd_process_wq_release(struct work_struct *work)
1308{
1309 struct kfd_process *p = container_of(work, struct kfd_process,
1310 release_work);
1311 struct dma_fence *ef;
1312
1313 /*
1314 * If GPU in reset, user queues may still running, wait for reset complete.
1315 */
1316 kfd_process_wait_gpu_reset_complete(p);
1317
1318 /* Signal the eviction fence after user mode queues are
1319 * destroyed. This allows any BOs to be freed without
1320 * triggering pointless evictions or waiting for fences.
1321 */
1322 synchronize_rcu();
1323 ef = rcu_access_pointer(p->ef);
1324 if (ef)
1325 dma_fence_signal(ef);
1326
1327 if (p->context_id != KFD_CONTEXT_ID_PRIMARY)
1328 kfd_process_free_id(p);
1329 else
1330 ida_destroy(&p->id_table);
1331
1332 kfd_debugfs_remove_process(p);
1333
1334 /*
1335 * Remove the proc/sysfs entries before destroying PDDs. The removal path
1336 * walks the PDD array and sysfs callbacks dereference PDD fields, so the
1337 * backing data must remain valid until sysfs removal has completed.
1338 */
1339 kfd_process_remove_sysfs(p);
1340
1341 kfd_process_kunmap_signal_bo(p);
1342 kfd_process_free_outstanding_kfd_bos(p);
1343 svm_range_list_fini(p);
1344
1345 kfd_process_destroy_pdds(p);
1346 dma_fence_put(ef);
1347
1348 kfd_event_free_process(p);
1349
1350 mutex_destroy(&p->mutex);
1351
1352 put_task_struct(p->lead_thread);
1353
1354 kfree(p);
1355}
1356
1357static void kfd_process_ref_release(struct kref *ref)
1358{
1359 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1360
1361 INIT_WORK(&p->release_work, kfd_process_wq_release);
1362 queue_work(kfd_process_wq, &p->release_work);
1363}
1364
1365static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1366{
1367 /* This increments p->ref counter if kfd process p exists */
1368 struct kfd_process *p = kfd_lookup_process_by_mm(mm);
1369
1370 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1371}
1372
1373static void kfd_process_free_notifier(struct mmu_notifier *mn)
1374{
1375 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1376}
1377
1378static void kfd_process_table_remove(struct kfd_process *p)
1379{
1380 mutex_lock(&kfd_processes_mutex);
1381 /*
1382 * Do early return if table is empty.
1383 *
1384 * This could potentially happen if this function is called concurrently
1385 * by mmu_notifier and by kfd_cleanup_pocesses.
1386 *
1387 */
1388 if (hash_empty(kfd_processes_table)) {
1389 mutex_unlock(&kfd_processes_mutex);
1390 return;
1391 }
1392 hash_del_rcu(&p->kfd_processes);
1393 mutex_unlock(&kfd_processes_mutex);
1394 synchronize_srcu(&kfd_processes_srcu);
1395}
1396
1397void kfd_process_notifier_release_internal(struct kfd_process *p)
1398{
1399 int i;
1400
1401 kfd_process_table_remove(p);
1402 cancel_delayed_work_sync(&p->eviction_work);
1403 cancel_delayed_work_sync(&p->restore_work);
1404 /*
1405 * If work pending, cancel it and drop the extra ref
1406 */
1407 if (cancel_delayed_work_sync(&p->signal_work))
1408 kfd_unref_process(p);
1409
1410 /*
1411 * Dequeue and destroy user queues, it is not safe for GPU to access
1412 * system memory after mmu release notifier callback returns because
1413 * exit_mmap free process memory afterwards.
1414 */
1415 kfd_process_dequeue_from_all_devices(p);
1416 pqm_uninit(&p->pqm);
1417
1418 for (i = 0; i < p->n_pdds; i++) {
1419 struct kfd_process_device *pdd = p->pdds[i];
1420
1421 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1422 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1423 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1424 }
1425
1426 /* Indicate to other users that MM is no longer valid */
1427 p->mm = NULL;
1428 kfd_dbg_trap_disable(p);
1429
1430 if (atomic_read(&p->debugged_process_count) > 0) {
1431 struct kfd_process *target;
1432 unsigned int temp;
1433 int idx = srcu_read_lock(&kfd_processes_srcu);
1434
1435 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1436 if (target->debugger_process && target->debugger_process == p) {
1437 mutex_lock_nested(&target->mutex, 1);
1438 kfd_dbg_trap_disable(target);
1439 mutex_unlock(&target->mutex);
1440 if (atomic_read(&p->debugged_process_count) == 0)
1441 break;
1442 }
1443 }
1444
1445 srcu_read_unlock(&kfd_processes_srcu, idx);
1446 }
1447
1448 if (p->context_id == KFD_CONTEXT_ID_PRIMARY)
1449 mmu_notifier_put(&p->mmu_notifier);
1450}
1451
1452static void kfd_process_notifier_release(struct mmu_notifier *mn,
1453 struct mm_struct *mm)
1454{
1455 struct kfd_process *p;
1456
1457 /*
1458 * The kfd_process structure can not be free because the
1459 * mmu_notifier srcu is read locked
1460 */
1461 p = container_of(mn, struct kfd_process, mmu_notifier);
1462 if (WARN_ON(p->mm != mm))
1463 return;
1464
1465 kfd_process_notifier_release_internal(p);
1466}
1467
1468static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1469 .release = kfd_process_notifier_release,
1470 .alloc_notifier = kfd_process_alloc_notifier,
1471 .free_notifier = kfd_process_free_notifier,
1472};
1473
1474/*
1475 * This code handles the case when driver is being unloaded before all
1476 * mm_struct are released. We need to safely free the kfd_process and
1477 * avoid race conditions with mmu_notifier that might try to free them.
1478 *
1479 */
1480void kfd_cleanup_processes(void)
1481{
1482 struct kfd_process *p;
1483 struct hlist_node *p_temp;
1484 unsigned int temp;
1485 HLIST_HEAD(cleanup_list);
1486
1487 /*
1488 * Move all remaining kfd_process from the process table to a
1489 * temp list for processing. Once done, callback from mmu_notifier
1490 * release will not see the kfd_process in the table and do early return,
1491 * avoiding double free issues.
1492 */
1493 mutex_lock(&kfd_processes_mutex);
1494 hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1495 hash_del_rcu(&p->kfd_processes);
1496 synchronize_srcu(&kfd_processes_srcu);
1497 hlist_add_head(&p->kfd_processes, &cleanup_list);
1498 }
1499 mutex_unlock(&kfd_processes_mutex);
1500
1501 hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1502 kfd_process_notifier_release_internal(p);
1503
1504 /*
1505 * Ensures that all outstanding free_notifier get called, triggering
1506 * the release of the kfd_process struct.
1507 */
1508 mmu_notifier_synchronize();
1509}
1510
1511static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1512{
1513 struct kfd_node *dev = pdd->dev;
1514 struct qcm_process_device *qpd = &pdd->qpd;
1515 u32 flags = KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1516 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1517 struct kgd_mem *mem;
1518 void *kaddr;
1519 int ret;
1520
1521 if (!dev->kfd->cwsr_enabled || !iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
1522 return 0;
1523
1524 if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt)
1525 flags |= KFD_IOC_ALLOC_MEM_FLAGS_VRAM;
1526 else
1527 flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT;
1528
1529 /* Allocate CWSR TBA/TMA buffers */
1530 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1531 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1532 if (ret)
1533 return ret;
1534
1535 qpd->cwsr_mem = mem;
1536
1537 /* Set up iosys_map based on whether memory is MMIO or system memory */
1538 if (mem->bo->kmap.bo_kmap_type & TTM_BO_MAP_IOMEM_MASK)
1539 iosys_map_set_vaddr_iomem(&qpd->cwsr_map, kaddr);
1540 else
1541 iosys_map_set_vaddr(&qpd->cwsr_map, kaddr);
1542
1543 qpd->tba_addr = qpd->cwsr_base;
1544
1545 /* Copy CWSR ISA to buffer using appropriate accessor */
1546 iosys_map_memcpy_to(&qpd->cwsr_map, 0, dev->kfd->cwsr_isa,
1547 dev->kfd->cwsr_isa_size);
1548
1549 kfd_process_set_trap_debug_flag(&pdd->qpd,
1550 pdd->process->debug_trap_enabled);
1551
1552 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1553 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_map:%s at %p for pqm.\n",
1554 qpd->tba_addr, qpd->tma_addr,
1555 qpd->cwsr_map.is_iomem ? "iomem" : "system",
1556 qpd->cwsr_map.is_iomem ? (void *)qpd->cwsr_map.vaddr_iomem :
1557 qpd->cwsr_map.vaddr);
1558
1559 return 0;
1560}
1561
1562static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1563{
1564 struct kfd_node *dev = pdd->dev;
1565 struct qcm_process_device *qpd = &pdd->qpd;
1566
1567 if (!dev->kfd->cwsr_enabled || iosys_map_is_null(&qpd->cwsr_map) || !qpd->cwsr_base)
1568 return;
1569
1570 kfd_process_free_gpuvm_map(qpd->cwsr_mem, pdd, &qpd->cwsr_map);
1571}
1572
1573void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1574 uint64_t tba_addr,
1575 uint64_t tma_addr)
1576{
1577 if (!iosys_map_is_null(&qpd->cwsr_map)) {
1578 /* KFD trap handler is bound, record as second-level TBA/TMA
1579 * in first-level TMA. First-level trap will jump to second.
1580 */
1581 iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET,
1582 uint64_t, tba_addr);
1583 iosys_map_wr(&qpd->cwsr_map, KFD_CWSR_TMA_OFFSET + sizeof(uint64_t),
1584 uint64_t, tma_addr);
1585 } else {
1586 /* No trap handler bound, bind as first-level TBA/TMA. */
1587 qpd->tba_addr = tba_addr;
1588 qpd->tma_addr = tma_addr;
1589 }
1590}
1591
1592bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1593{
1594 int i;
1595
1596 /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1597 * boot time retry setting. Mixing processes with different
1598 * XNACK/retry settings can hang the GPU.
1599 *
1600 * Different GPUs can have different noretry settings depending
1601 * on HW bugs or limitations. We need to find at least one
1602 * XNACK mode for this process that's compatible with all GPUs.
1603 * Fortunately GPUs with retry enabled (noretry=0) can run code
1604 * built for XNACK-off. On GFXv9 it may perform slower.
1605 *
1606 * Therefore applications built for XNACK-off can always be
1607 * supported and will be our fallback if any GPU does not
1608 * support retry.
1609 */
1610 for (i = 0; i < p->n_pdds; i++) {
1611 struct kfd_node *dev = p->pdds[i]->dev;
1612
1613 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1614 * support the SVM APIs and don't need to be considered
1615 * for the XNACK mode selection.
1616 */
1617 if (!KFD_IS_SOC15(dev))
1618 continue;
1619 /* Aldebaran can always support XNACK because it can support
1620 * per-process XNACK mode selection. But let the dev->noretry
1621 * setting still influence the default XNACK mode.
1622 */
1623 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev)) {
1624 if (!amdgpu_sriov_xnack_support(dev->kfd->adev)) {
1625 pr_debug("SRIOV platform xnack not supported\n");
1626 return false;
1627 }
1628 continue;
1629 }
1630
1631 /* GFXv10 and later GPUs do not support shader preemption
1632 * during page faults. This can lead to poor QoS for queue
1633 * management and memory-manager-related preemptions or
1634 * even deadlocks.
1635 */
1636 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1) &&
1637 KFD_GC_VERSION(dev) < IP_VERSION(12, 1, 0))
1638 return false;
1639
1640 if (dev->kfd->noretry)
1641 return false;
1642 }
1643
1644 return true;
1645}
1646
1647void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1648 bool enabled)
1649{
1650 if (!iosys_map_is_null(&qpd->cwsr_map)) {
1651 iosys_map_wr(&qpd->cwsr_map,
1652 KFD_CWSR_TMA_OFFSET + 2 * sizeof(uint64_t),
1653 uint64_t, enabled);
1654 }
1655}
1656
1657/*
1658 * On return the kfd_process is fully operational and will be freed when the
1659 * mm is released
1660 */
1661struct kfd_process *create_process(const struct task_struct *thread, bool primary)
1662{
1663 struct kfd_process *process;
1664 struct mmu_notifier *mn;
1665 int err = -ENOMEM;
1666
1667 process = kzalloc_obj(*process);
1668 if (!process)
1669 goto err_alloc_process;
1670
1671 kref_init(&process->ref);
1672 mutex_init(&process->mutex);
1673 process->mm = thread->mm;
1674 process->lead_thread = thread->group_leader;
1675 process->n_pdds = 0;
1676 process->queues_paused = false;
1677
1678 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1679 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1680 INIT_DELAYED_WORK(&process->signal_work, kfd_signal_sigbus_delayed_fn);
1681 process->last_restore_timestamp = get_jiffies_64();
1682 err = kfd_event_init_process(process);
1683 if (err)
1684 goto err_event_init;
1685 process->is_32bit_user_mode = in_compat_syscall();
1686 process->debug_trap_enabled = false;
1687 process->debugger_process = NULL;
1688 process->exception_enable_mask = 0;
1689 atomic_set(&process->debugged_process_count, 0);
1690 sema_init(&process->runtime_enable_sema, 0);
1691
1692 err = pqm_init(&process->pqm, process);
1693 if (err != 0)
1694 goto err_process_pqm_init;
1695
1696 /* init process apertures*/
1697 err = kfd_init_apertures(process);
1698 if (err != 0)
1699 goto err_init_apertures;
1700
1701 /* Check XNACK support after PDDs are created in kfd_init_apertures */
1702 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1703
1704 err = svm_range_list_init(process);
1705 if (err)
1706 goto err_init_svm_range_list;
1707
1708 /* alloc_notifier needs to find the process in the hash table */
1709 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1710 (uintptr_t)process->mm);
1711
1712 /* Avoid free_notifier to start kfd_process_wq_release if
1713 * mmu_notifier_get failed because of pending signal.
1714 */
1715 kref_get(&process->ref);
1716
1717 /* MMU notifier registration must be the last call that can fail
1718 * because after this point we cannot unwind the process creation.
1719 * After this point, mmu_notifier_put will trigger the cleanup by
1720 * dropping the last process reference in the free_notifier.
1721 */
1722 if (primary) {
1723 process->context_id = KFD_CONTEXT_ID_PRIMARY;
1724 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1725 if (IS_ERR(mn)) {
1726 err = PTR_ERR(mn);
1727 goto err_register_notifier;
1728 }
1729 BUG_ON(mn != &process->mmu_notifier);
1730 ida_init(&process->id_table);
1731 }
1732
1733 err = kfd_process_alloc_id(process);
1734 if (err) {
1735 pr_err("Creating kfd process: failed to alloc an id\n");
1736 goto err_alloc_id;
1737 }
1738
1739 kfd_unref_process(process);
1740 get_task_struct(process->lead_thread);
1741
1742 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1743
1744 return process;
1745
1746err_alloc_id:
1747 kfd_process_free_id(process);
1748err_register_notifier:
1749 hash_del_rcu(&process->kfd_processes);
1750 svm_range_list_fini(process);
1751err_init_svm_range_list:
1752 kfd_process_free_outstanding_kfd_bos(process);
1753 kfd_process_destroy_pdds(process);
1754err_init_apertures:
1755 pqm_uninit(&process->pqm);
1756err_process_pqm_init:
1757 kfd_event_free_process(process);
1758err_event_init:
1759 mutex_destroy(&process->mutex);
1760 kfree(process);
1761err_alloc_process:
1762 return ERR_PTR(err);
1763}
1764
1765struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1766 struct kfd_process *p)
1767{
1768 int i;
1769
1770 for (i = 0; i < p->n_pdds; i++)
1771 if (p->pdds[i]->dev == dev)
1772 return p->pdds[i];
1773
1774 return NULL;
1775}
1776
1777struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1778 struct kfd_process *p)
1779{
1780 struct kfd_process_device *pdd = NULL;
1781
1782 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1783 return NULL;
1784 pdd = kzalloc_obj(*pdd);
1785 if (!pdd)
1786 return NULL;
1787
1788 pdd->dev = dev;
1789 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1790 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1791 pdd->qpd.dqm = dev->dqm;
1792 pdd->qpd.pqm = &p->pqm;
1793 pdd->qpd.evicted = 0;
1794 pdd->qpd.mapped_gws_queue = false;
1795 pdd->process = p;
1796 pdd->bound = PDD_UNBOUND;
1797 pdd->already_dequeued = false;
1798 pdd->runtime_inuse = false;
1799 atomic64_set(&pdd->vram_usage, 0);
1800 pdd->sdma_past_activity_counter = 0;
1801 pdd->user_gpu_id = dev->id;
1802 atomic64_set(&pdd->evict_duration_counter, 0);
1803
1804 p->pdds[p->n_pdds++] = pdd;
1805 if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1806 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1807 pdd->dev->adev,
1808 false,
1809 0);
1810
1811 /* Init idr used for memory handle translation */
1812 idr_init(&pdd->alloc_idr);
1813
1814 atomic_inc(&dev->kfd->kfd_processes_count);
1815
1816 return pdd;
1817}
1818
1819/**
1820 * kfd_process_device_init_vm - Initialize a VM for a process-device
1821 *
1822 * @pdd: The process-device
1823 * @drm_file: Optional pointer to a DRM file descriptor
1824 *
1825 * If @drm_file is specified, it will be used to acquire the VM from
1826 * that file descriptor. If successful, the @pdd takes ownership of
1827 * the file descriptor.
1828 *
1829 * If @drm_file is NULL, a new VM is created.
1830 *
1831 * Returns 0 on success, -errno on failure.
1832 */
1833int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1834 struct file *drm_file)
1835{
1836 struct amdgpu_fpriv *drv_priv;
1837 struct amdgpu_vm *avm;
1838 struct kfd_process *p;
1839 struct dma_fence *ef;
1840 struct kfd_node *dev;
1841 int ret;
1842
1843 if (pdd->drm_priv)
1844 return -EBUSY;
1845
1846 ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1847 if (ret)
1848 return ret;
1849 avm = &drv_priv->vm;
1850
1851 p = pdd->process;
1852 dev = pdd->dev;
1853
1854 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1855 &p->kgd_process_info,
1856 p->ef ? NULL : &ef);
1857 if (ret) {
1858 dev_err(dev->adev->dev, "Failed to create process VM object\n");
1859 return ret;
1860 }
1861
1862 if (!p->ef)
1863 RCU_INIT_POINTER(p->ef, ef);
1864
1865 pdd->drm_priv = drm_file->private_data;
1866
1867 ret = kfd_process_device_reserve_ib_mem(pdd);
1868 if (ret)
1869 goto err_reserve_ib_mem;
1870 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1871 if (ret)
1872 goto err_init_cwsr;
1873
1874 if (unlikely(!avm->pasid)) {
1875 dev_warn(pdd->dev->adev->dev, "WARN: vm %p has no pasid associated",
1876 avm);
1877 ret = -EINVAL;
1878 goto err_get_pasid;
1879 }
1880
1881 pdd->pasid = avm->pasid;
1882 pdd->drm_file = drm_file;
1883
1884 kfd_smi_event_process(pdd, true);
1885
1886 return 0;
1887
1888err_get_pasid:
1889 kfd_process_device_destroy_cwsr_dgpu(pdd);
1890err_init_cwsr:
1891 kfd_process_device_destroy_ib_mem(pdd);
1892err_reserve_ib_mem:
1893 pdd->drm_priv = NULL;
1894 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1895
1896 return ret;
1897}
1898
1899/*
1900 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1901 * to the device.
1902 * Unbinding occurs when the process dies or the device is removed.
1903 *
1904 * Assumes that the process lock is held.
1905 */
1906struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1907 struct kfd_process *p)
1908{
1909 struct kfd_process_device *pdd;
1910 int err;
1911
1912 pdd = kfd_get_process_device_data(dev, p);
1913 if (!pdd) {
1914 dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1915 return ERR_PTR(-ENOMEM);
1916 }
1917
1918 if (!pdd->drm_priv)
1919 return ERR_PTR(-ENODEV);
1920
1921 /*
1922 * signal runtime-pm system to auto resume and prevent
1923 * further runtime suspend once device pdd is created until
1924 * pdd is destroyed.
1925 */
1926 if (!pdd->runtime_inuse) {
1927 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1928 if (err < 0) {
1929 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1930 return ERR_PTR(err);
1931 }
1932 }
1933
1934 /*
1935 * make sure that runtime_usage counter is incremented just once
1936 * per pdd
1937 */
1938 pdd->runtime_inuse = true;
1939
1940 return pdd;
1941}
1942
1943/* Create specific handle mapped to mem from process local memory idr
1944 * Assumes that the process lock is held.
1945 */
1946int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1947 void *mem)
1948{
1949 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1950}
1951
1952/* Translate specific handle from process local memory idr
1953 * Assumes that the process lock is held.
1954 */
1955void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1956 int handle)
1957{
1958 if (handle < 0)
1959 return NULL;
1960
1961 return idr_find(&pdd->alloc_idr, handle);
1962}
1963
1964/* Remove specific handle from process local memory idr
1965 * Assumes that the process lock is held.
1966 */
1967void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1968 int handle)
1969{
1970 if (handle >= 0)
1971 idr_remove(&pdd->alloc_idr, handle);
1972}
1973
1974static struct kfd_process_device *kfd_lookup_process_device_by_pasid(u32 pasid)
1975{
1976 struct kfd_process_device *ret_p = NULL;
1977 struct kfd_process *p;
1978 unsigned int temp;
1979 int i;
1980
1981 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1982 for (i = 0; i < p->n_pdds; i++) {
1983 if (p->pdds[i]->pasid == pasid) {
1984 ret_p = p->pdds[i];
1985 break;
1986 }
1987 }
1988 if (ret_p)
1989 break;
1990 }
1991 return ret_p;
1992}
1993
1994/* This increments the process->ref counter. */
1995struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid,
1996 struct kfd_process_device **pdd)
1997{
1998 struct kfd_process_device *ret_p;
1999
2000 int idx = srcu_read_lock(&kfd_processes_srcu);
2001
2002 ret_p = kfd_lookup_process_device_by_pasid(pasid);
2003 if (ret_p) {
2004 if (pdd)
2005 *pdd = ret_p;
2006 kref_get(&ret_p->process->ref);
2007
2008 srcu_read_unlock(&kfd_processes_srcu, idx);
2009 return ret_p->process;
2010 }
2011
2012 srcu_read_unlock(&kfd_processes_srcu, idx);
2013
2014 if (pdd)
2015 *pdd = NULL;
2016
2017 return NULL;
2018}
2019
2020/* This increments the process->ref counter. */
2021struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
2022{
2023 struct kfd_process *p;
2024
2025 int idx = srcu_read_lock(&kfd_processes_srcu);
2026
2027 p = find_process_by_mm(mm);
2028 if (p)
2029 kref_get(&p->ref);
2030
2031 srcu_read_unlock(&kfd_processes_srcu, idx);
2032
2033 return p;
2034}
2035
2036/* This increments the process->ref counter. */
2037struct kfd_process *kfd_lookup_process_by_id(const struct mm_struct *mm, u16 id)
2038{
2039 struct kfd_process *p, *ret_p = NULL;
2040 unsigned int temp;
2041
2042 int idx = srcu_read_lock(&kfd_processes_srcu);
2043
2044 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2045 if (p->mm == mm && p->context_id == id) {
2046 kref_get(&p->ref);
2047 ret_p = p;
2048 break;
2049 }
2050 }
2051
2052 srcu_read_unlock(&kfd_processes_srcu, idx);
2053
2054 return ret_p;
2055}
2056
2057/* kfd_process_evict_queues - Evict all user queues of a process
2058 *
2059 * Eviction is reference-counted per process-device. This means multiple
2060 * evictions from different sources can be nested safely.
2061 */
2062int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
2063{
2064 int r = 0;
2065 int i;
2066 unsigned int n_evicted = 0;
2067
2068 for (i = 0; i < p->n_pdds; i++) {
2069 struct kfd_process_device *pdd = p->pdds[i];
2070 struct device *dev = pdd->dev->adev->dev;
2071
2072 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread,
2073 trigger);
2074
2075 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
2076 &pdd->qpd);
2077 /* evict return -EIO if HWS is hang or asic is resetting, in this case
2078 * we would like to set all the queues to be in evicted state to prevent
2079 * them been add back since they actually not be saved right now.
2080 */
2081 if (r && r != -EIO) {
2082 dev_err(dev, "Failed to evict process queues\n");
2083 goto fail;
2084 }
2085 n_evicted++;
2086
2087 pdd->dev->dqm->is_hws_hang = false;
2088 }
2089
2090 return r;
2091
2092fail:
2093 /* To keep state consistent, roll back partial eviction by
2094 * restoring queues
2095 */
2096 for (i = 0; i < p->n_pdds; i++) {
2097 struct kfd_process_device *pdd = p->pdds[i];
2098
2099 if (n_evicted == 0)
2100 break;
2101
2102 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread);
2103
2104 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
2105 &pdd->qpd))
2106 dev_err(pdd->dev->adev->dev,
2107 "Failed to restore queues\n");
2108
2109 n_evicted--;
2110 }
2111
2112 return r;
2113}
2114
2115/* kfd_process_restore_queues - Restore all user queues of a process */
2116int kfd_process_restore_queues(struct kfd_process *p)
2117{
2118 int r, ret = 0;
2119 int i;
2120
2121 for (i = 0; i < p->n_pdds; i++) {
2122 struct kfd_process_device *pdd = p->pdds[i];
2123 struct device *dev = pdd->dev->adev->dev;
2124
2125 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread);
2126
2127 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
2128 &pdd->qpd);
2129 if (r) {
2130 dev_err(dev, "Failed to restore process queues\n");
2131 if (!ret)
2132 ret = r;
2133 }
2134 }
2135
2136 return ret;
2137}
2138
2139int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
2140{
2141 int i;
2142
2143 for (i = 0; i < p->n_pdds; i++)
2144 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
2145 return i;
2146 return -EINVAL;
2147}
2148
2149int
2150kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
2151 uint32_t *gpuid, uint32_t *gpuidx)
2152{
2153 int i;
2154
2155 for (i = 0; i < p->n_pdds; i++)
2156 if (p->pdds[i] && p->pdds[i]->dev == node) {
2157 *gpuid = p->pdds[i]->user_gpu_id;
2158 *gpuidx = i;
2159 return 0;
2160 }
2161 return -EINVAL;
2162}
2163
2164static bool signal_eviction_fence(struct kfd_process *p)
2165{
2166 struct dma_fence *ef;
2167 bool ret;
2168
2169 rcu_read_lock();
2170 ef = dma_fence_get_rcu_safe(&p->ef);
2171 rcu_read_unlock();
2172 if (!ef)
2173 return true;
2174
2175 ret = dma_fence_check_and_signal(ef);
2176 dma_fence_put(ef);
2177
2178 return ret;
2179}
2180
2181static void evict_process_worker(struct work_struct *work)
2182{
2183 int ret;
2184 struct kfd_process *p;
2185 struct delayed_work *dwork;
2186
2187 dwork = to_delayed_work(work);
2188
2189 /* Process termination destroys this worker thread. So during the
2190 * lifetime of this thread, kfd_process p will be valid
2191 */
2192 p = container_of(dwork, struct kfd_process, eviction_work);
2193
2194 pr_debug("Started evicting process pid %d\n", p->lead_thread->pid);
2195 ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
2196 if (!ret) {
2197 /* If another thread already signaled the eviction fence,
2198 * they are responsible stopping the queues and scheduling
2199 * the restore work.
2200 */
2201 if (signal_eviction_fence(p) ||
2202 mod_delayed_work(kfd_restore_wq, &p->restore_work,
2203 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2204 kfd_process_restore_queues(p);
2205
2206 pr_debug("Finished evicting process pid %d\n", p->lead_thread->pid);
2207 } else
2208 pr_err("Failed to evict queues of process pid %d\n", p->lead_thread->pid);
2209}
2210
2211static int restore_process_helper(struct kfd_process *p)
2212{
2213 int ret = 0;
2214
2215 /* VMs may not have been acquired yet during debugging. */
2216 if (p->kgd_process_info) {
2217 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(
2218 p->kgd_process_info, &p->ef);
2219 if (ret)
2220 return ret;
2221 }
2222
2223 ret = kfd_process_restore_queues(p);
2224 if (!ret)
2225 pr_debug("Finished restoring process pid %d\n",
2226 p->lead_thread->pid);
2227 else
2228 pr_err("Failed to restore queues of process pid %d\n",
2229 p->lead_thread->pid);
2230
2231 return ret;
2232}
2233
2234static void restore_process_worker(struct work_struct *work)
2235{
2236 struct delayed_work *dwork;
2237 struct kfd_process *p;
2238 int ret = 0;
2239
2240 dwork = to_delayed_work(work);
2241
2242 /* Process termination destroys this worker thread. So during the
2243 * lifetime of this thread, kfd_process p will be valid
2244 */
2245 p = container_of(dwork, struct kfd_process, restore_work);
2246 pr_debug("Started restoring process pasid %d\n", (int)p->lead_thread->pid);
2247
2248 /* Setting last_restore_timestamp before successful restoration.
2249 * Otherwise this would have to be set by KGD (restore_process_bos)
2250 * before KFD BOs are unreserved. If not, the process can be evicted
2251 * again before the timestamp is set.
2252 * If restore fails, the timestamp will be set again in the next
2253 * attempt. This would mean that the minimum GPU quanta would be
2254 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
2255 * functions)
2256 */
2257
2258 p->last_restore_timestamp = get_jiffies_64();
2259
2260 ret = restore_process_helper(p);
2261 if (ret) {
2262 pr_debug("Failed to restore BOs of process pid %d, retry after %d ms\n",
2263 p->lead_thread->pid, PROCESS_BACK_OFF_TIME_MS);
2264 if (mod_delayed_work(kfd_restore_wq, &p->restore_work,
2265 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)))
2266 kfd_process_restore_queues(p);
2267 }
2268}
2269
2270void kfd_suspend_all_processes(void)
2271{
2272 struct kfd_process *p;
2273 unsigned int temp;
2274 int idx = srcu_read_lock(&kfd_processes_srcu);
2275
2276 WARN(debug_evictions, "Evicting all processes");
2277 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2278 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2279 pr_err("Failed to suspend process pid %d\n", p->lead_thread->pid);
2280 signal_eviction_fence(p);
2281 }
2282 srcu_read_unlock(&kfd_processes_srcu, idx);
2283}
2284
2285int kfd_resume_all_processes(void)
2286{
2287 struct kfd_process *p;
2288 unsigned int temp;
2289 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2290
2291 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2292 if (restore_process_helper(p)) {
2293 pr_err("Restore process pid %d failed during resume\n",
2294 p->lead_thread->pid);
2295 ret = -EFAULT;
2296 }
2297 }
2298 srcu_read_unlock(&kfd_processes_srcu, idx);
2299 return ret;
2300}
2301
2302/* assumes caller holds process lock. */
2303int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2304{
2305 uint32_t irq_drain_fence[8];
2306 uint8_t node_id = 0;
2307 int r = 0;
2308
2309 if (!KFD_IS_SOC15(pdd->dev))
2310 return 0;
2311
2312 pdd->process->irq_drain_is_open = true;
2313
2314 memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2315 irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2316 KFD_IRQ_FENCE_CLIENTID;
2317 irq_drain_fence[3] = pdd->pasid;
2318
2319 /*
2320 * For GFX 9.4.3/9.5.0, send the NodeId also in IH cookie DW[3]
2321 */
2322 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3) ||
2323 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 4) ||
2324 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 5, 0) ||
2325 KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(12, 1, 0)) {
2326 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2327 irq_drain_fence[3] |= node_id << 16;
2328 }
2329
2330 /* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2331 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2332 irq_drain_fence)) {
2333 pdd->process->irq_drain_is_open = false;
2334 return 0;
2335 }
2336
2337 r = wait_event_interruptible(pdd->process->wait_irq_drain,
2338 !READ_ONCE(pdd->process->irq_drain_is_open));
2339 if (r)
2340 pdd->process->irq_drain_is_open = false;
2341
2342 return r;
2343}
2344
2345void kfd_process_close_interrupt_drain(unsigned int pasid)
2346{
2347 struct kfd_process *p;
2348
2349 p = kfd_lookup_process_by_pasid(pasid, NULL);
2350
2351 if (!p)
2352 return;
2353
2354 WRITE_ONCE(p->irq_drain_is_open, false);
2355 wake_up_all(&p->wait_irq_drain);
2356 kfd_unref_process(p);
2357}
2358
2359struct send_exception_work_handler_workarea {
2360 struct work_struct work;
2361 struct kfd_process *p;
2362 unsigned int queue_id;
2363 uint64_t error_reason;
2364};
2365
2366static void send_exception_work_handler(struct work_struct *work)
2367{
2368 struct send_exception_work_handler_workarea *workarea;
2369 struct kfd_process *p;
2370 struct queue *q;
2371 struct mm_struct *mm;
2372 struct kfd_context_save_area_header __user *csa_header;
2373 uint64_t __user *err_payload_ptr;
2374 uint64_t cur_err;
2375 uint32_t ev_id;
2376
2377 workarea = container_of(work,
2378 struct send_exception_work_handler_workarea,
2379 work);
2380 p = workarea->p;
2381
2382 mm = get_task_mm(p->lead_thread);
2383
2384 if (!mm)
2385 return;
2386
2387 kthread_use_mm(mm);
2388
2389 q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2390
2391 if (!q)
2392 goto out;
2393
2394 csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2395
2396 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2397 get_user(cur_err, err_payload_ptr);
2398 cur_err |= workarea->error_reason;
2399 put_user(cur_err, err_payload_ptr);
2400 get_user(ev_id, &csa_header->err_event_id);
2401
2402 kfd_set_event(p, ev_id);
2403
2404out:
2405 kthread_unuse_mm(mm);
2406 mmput(mm);
2407}
2408
2409int kfd_send_exception_to_runtime(struct kfd_process *p,
2410 unsigned int queue_id,
2411 uint64_t error_reason)
2412{
2413 struct send_exception_work_handler_workarea worker;
2414
2415 INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2416
2417 worker.p = p;
2418 worker.queue_id = queue_id;
2419 worker.error_reason = error_reason;
2420
2421 schedule_work(&worker.work);
2422 flush_work(&worker.work);
2423 destroy_work_on_stack(&worker.work);
2424
2425 return 0;
2426}
2427
2428struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2429{
2430 int i;
2431
2432 if (gpu_id) {
2433 for (i = 0; i < p->n_pdds; i++) {
2434 struct kfd_process_device *pdd = p->pdds[i];
2435
2436 if (pdd->user_gpu_id == gpu_id)
2437 return pdd;
2438 }
2439 }
2440 return NULL;
2441}
2442
2443int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2444{
2445 int i;
2446
2447 if (!actual_gpu_id)
2448 return 0;
2449
2450 for (i = 0; i < p->n_pdds; i++) {
2451 struct kfd_process_device *pdd = p->pdds[i];
2452
2453 if (pdd->dev->id == actual_gpu_id)
2454 return pdd->user_gpu_id;
2455 }
2456 return -EINVAL;
2457}
2458
2459#if defined(CONFIG_DEBUG_FS)
2460
2461int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2462{
2463 struct kfd_process *p;
2464 unsigned int temp;
2465 int r = 0;
2466
2467 int idx = srcu_read_lock(&kfd_processes_srcu);
2468
2469 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2470 seq_printf(m, "Process %d PASID %d:\n",
2471 p->lead_thread->tgid, p->lead_thread->pid);
2472
2473 mutex_lock(&p->mutex);
2474 r = pqm_debugfs_mqds(m, &p->pqm);
2475 mutex_unlock(&p->mutex);
2476
2477 if (r)
2478 break;
2479 }
2480
2481 srcu_read_unlock(&kfd_processes_srcu, idx);
2482
2483 return r;
2484}
2485
2486#endif