projects/linux·blob master · refs

1037 lines · 27.6 KiB

1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christian König
23 */
24
25#include <linux/dma-mapping.h>
26#include <linux/cgroup_dmem.h>
27#include <linux/pm_runtime.h>
28#include <drm/ttm/ttm_range_manager.h>
29#include <drm/drm_drv.h>
30#include <drm/drm_buddy.h>
31
32#include "amdgpu.h"
33#include "amdgpu_vm.h"
34#include "amdgpu_res_cursor.h"
35#include "atom.h"
36
37#define AMDGPU_MAX_SG_SEGMENT_SIZE (2UL << 30)
38
39struct amdgpu_vram_reservation {
40 u64 start;
41 u64 size;
42 struct list_head allocated;
43 struct list_head blocks;
44};
45
46static inline struct amdgpu_vram_mgr *
47to_vram_mgr(struct ttm_resource_manager *man)
48{
49 return container_of(man, struct amdgpu_vram_mgr, manager);
50}
51
52static inline struct amdgpu_device *
53to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
54{
55 return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
56}
57
58static inline struct gpu_buddy_block *
59amdgpu_vram_mgr_first_block(struct list_head *list)
60{
61 return list_first_entry_or_null(list, struct gpu_buddy_block, link);
62}
63
64static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
65{
66 struct gpu_buddy_block *block;
67 u64 start, size;
68
69 block = amdgpu_vram_mgr_first_block(head);
70 if (!block)
71 return false;
72
73 while (head != block->link.next) {
74 start = amdgpu_vram_mgr_block_start(block);
75 size = amdgpu_vram_mgr_block_size(block);
76
77 block = list_entry(block->link.next, struct gpu_buddy_block, link);
78 if (start + size != amdgpu_vram_mgr_block_start(block))
79 return false;
80 }
81
82 return true;
83}
84
85static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head)
86{
87 struct gpu_buddy_block *block;
88 u64 size = 0;
89
90 list_for_each_entry(block, head, link)
91 size += amdgpu_vram_mgr_block_size(block);
92
93 return size;
94}
95
96/**
97 * DOC: mem_info_vram_total
98 *
99 * The amdgpu driver provides a sysfs API for reporting current total VRAM
100 * available on the device
101 * The file mem_info_vram_total is used for this and returns the total
102 * amount of VRAM in bytes
103 */
104static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
105 struct device_attribute *attr, char *buf)
106{
107 struct drm_device *ddev = dev_get_drvdata(dev);
108 struct amdgpu_device *adev = drm_to_adev(ddev);
109
110 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);
111}
112
113/**
114 * DOC: mem_info_vis_vram_total
115 *
116 * The amdgpu driver provides a sysfs API for reporting current total
117 * visible VRAM available on the device
118 * The file mem_info_vis_vram_total is used for this and returns the total
119 * amount of visible VRAM in bytes
120 */
121static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
122 struct device_attribute *attr, char *buf)
123{
124 struct drm_device *ddev = dev_get_drvdata(dev);
125 struct amdgpu_device *adev = drm_to_adev(ddev);
126
127 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);
128}
129
130/**
131 * DOC: mem_info_vram_used
132 *
133 * The amdgpu driver provides a sysfs API for reporting current total VRAM
134 * available on the device
135 * The file mem_info_vram_used is used for this and returns the total
136 * amount of currently used VRAM in bytes
137 */
138static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
139 struct device_attribute *attr,
140 char *buf)
141{
142 struct drm_device *ddev = dev_get_drvdata(dev);
143 struct amdgpu_device *adev = drm_to_adev(ddev);
144 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;
145
146 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man));
147}
148
149/**
150 * DOC: mem_info_vis_vram_used
151 *
152 * The amdgpu driver provides a sysfs API for reporting current total of
153 * used visible VRAM
154 * The file mem_info_vis_vram_used is used for this and returns the total
155 * amount of currently used visible VRAM in bytes
156 */
157static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
158 struct device_attribute *attr,
159 char *buf)
160{
161 struct drm_device *ddev = dev_get_drvdata(dev);
162 struct amdgpu_device *adev = drm_to_adev(ddev);
163
164 return sysfs_emit(buf, "%llu\n",
165 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));
166}
167
168/**
169 * DOC: mem_info_vram_vendor
170 *
171 * The amdgpu driver provides a sysfs API for reporting the vendor of the
172 * installed VRAM
173 * The file mem_info_vram_vendor is used for this and returns the name of the
174 * vendor.
175 */
176static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
177 struct device_attribute *attr,
178 char *buf)
179{
180 struct drm_device *ddev = dev_get_drvdata(dev);
181 struct amdgpu_device *adev = drm_to_adev(ddev);
182
183 switch (adev->gmc.vram_vendor) {
184 case SAMSUNG:
185 return sysfs_emit(buf, "samsung\n");
186 case INFINEON:
187 return sysfs_emit(buf, "infineon\n");
188 case ELPIDA:
189 return sysfs_emit(buf, "elpida\n");
190 case ETRON:
191 return sysfs_emit(buf, "etron\n");
192 case NANYA:
193 return sysfs_emit(buf, "nanya\n");
194 case HYNIX:
195 return sysfs_emit(buf, "hynix\n");
196 case MOSEL:
197 return sysfs_emit(buf, "mosel\n");
198 case WINBOND:
199 return sysfs_emit(buf, "winbond\n");
200 case ESMT:
201 return sysfs_emit(buf, "esmt\n");
202 case MICRON:
203 return sysfs_emit(buf, "micron\n");
204 default:
205 return sysfs_emit(buf, "unknown\n");
206 }
207}
208
209static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
210 amdgpu_mem_info_vram_total_show, NULL);
211static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
212 amdgpu_mem_info_vis_vram_total_show,NULL);
213static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
214 amdgpu_mem_info_vram_used_show, NULL);
215static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
216 amdgpu_mem_info_vis_vram_used_show, NULL);
217static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
218 amdgpu_mem_info_vram_vendor, NULL);
219
220static struct attribute *amdgpu_vram_mgr_attributes[] = {
221 &dev_attr_mem_info_vram_total.attr,
222 &dev_attr_mem_info_vis_vram_total.attr,
223 &dev_attr_mem_info_vram_used.attr,
224 &dev_attr_mem_info_vis_vram_used.attr,
225 &dev_attr_mem_info_vram_vendor.attr,
226 NULL
227};
228
229static umode_t amdgpu_vram_attrs_is_visible(struct kobject *kobj,
230 struct attribute *attr, int i)
231{
232 struct device *dev = kobj_to_dev(kobj);
233 struct drm_device *ddev = dev_get_drvdata(dev);
234 struct amdgpu_device *adev = drm_to_adev(ddev);
235
236 if (attr == &dev_attr_mem_info_vram_vendor.attr &&
237 !adev->gmc.vram_vendor)
238 return 0;
239
240 if (!ttm_resource_manager_used(&adev->mman.vram_mgr.manager))
241 return 0;
242
243 return attr->mode;
244}
245
246const struct attribute_group amdgpu_vram_mgr_attr_group = {
247 .attrs = amdgpu_vram_mgr_attributes,
248 .is_visible = amdgpu_vram_attrs_is_visible
249};
250
251/**
252 * amdgpu_vram_mgr_vis_size - Calculate visible block size
253 *
254 * @adev: amdgpu_device pointer
255 * @block: DRM BUDDY block structure
256 *
257 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
258 */
259static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
260 struct gpu_buddy_block *block)
261{
262 u64 start = amdgpu_vram_mgr_block_start(block);
263 u64 end = start + amdgpu_vram_mgr_block_size(block);
264
265 if (start >= adev->gmc.visible_vram_size)
266 return 0;
267
268 return (end > adev->gmc.visible_vram_size ?
269 adev->gmc.visible_vram_size : end) - start;
270}
271
272/**
273 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
274 *
275 * @bo: &amdgpu_bo buffer object (must be in VRAM)
276 *
277 * Returns:
278 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
279 */
280u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
281{
282 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
283 struct ttm_resource *res = bo->tbo.resource;
284 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
285 struct gpu_buddy_block *block;
286 u64 usage = 0;
287
288 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
289 return amdgpu_bo_size(bo);
290
291 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
292 return 0;
293
294 list_for_each_entry(block, &vres->blocks, link)
295 usage += amdgpu_vram_mgr_vis_size(adev, block);
296
297 return usage;
298}
299
300/* Commit the reservation of VRAM pages */
301static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
302{
303 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
304 struct amdgpu_device *adev = to_amdgpu_device(mgr);
305 struct gpu_buddy *mm = &mgr->mm;
306 struct amdgpu_vram_reservation *rsv, *temp;
307 struct gpu_buddy_block *block;
308 uint64_t vis_usage;
309
310 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
311 if (gpu_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
312 rsv->size, mm->chunk_size, &rsv->allocated,
313 GPU_BUDDY_RANGE_ALLOCATION))
314 continue;
315
316 block = amdgpu_vram_mgr_first_block(&rsv->allocated);
317 if (!block)
318 continue;
319
320 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
321 rsv->start, rsv->size);
322
323 vis_usage = amdgpu_vram_mgr_vis_size(adev, block);
324 atomic64_add(vis_usage, &mgr->vis_usage);
325 spin_lock(&man->bdev->lru_lock);
326 man->usage += rsv->size;
327 spin_unlock(&man->bdev->lru_lock);
328 list_move(&rsv->blocks, &mgr->reserved_pages);
329 }
330}
331
332/**
333 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
334 *
335 * @mgr: amdgpu_vram_mgr pointer
336 * @start: start address of the range in VRAM
337 * @size: size of the range
338 *
339 * Reserve memory from start address with the specified size in VRAM
340 */
341int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
342 uint64_t start, uint64_t size)
343{
344 struct amdgpu_vram_reservation *rsv;
345
346 rsv = kzalloc_obj(*rsv);
347 if (!rsv)
348 return -ENOMEM;
349
350 INIT_LIST_HEAD(&rsv->allocated);
351 INIT_LIST_HEAD(&rsv->blocks);
352
353 rsv->start = start;
354 rsv->size = size;
355
356 mutex_lock(&mgr->lock);
357 list_add_tail(&rsv->blocks, &mgr->reservations_pending);
358 amdgpu_vram_mgr_do_reserve(&mgr->manager);
359 mutex_unlock(&mgr->lock);
360
361 return 0;
362}
363
364/**
365 * amdgpu_vram_mgr_query_page_status - query the reservation status
366 *
367 * @mgr: amdgpu_vram_mgr pointer
368 * @start: start address of a page in VRAM
369 *
370 * Returns:
371 * -EBUSY: the page is still hold and in pending list
372 * 0: the page has been reserved
373 * -ENOENT: the input page is not a reservation
374 */
375int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
376 uint64_t start)
377{
378 struct amdgpu_vram_reservation *rsv;
379 int ret;
380
381 mutex_lock(&mgr->lock);
382
383 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {
384 if (rsv->start <= start &&
385 (start < (rsv->start + rsv->size))) {
386 ret = -EBUSY;
387 goto out;
388 }
389 }
390
391 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {
392 if (rsv->start <= start &&
393 (start < (rsv->start + rsv->size))) {
394 ret = 0;
395 goto out;
396 }
397 }
398
399 ret = -ENOENT;
400out:
401 mutex_unlock(&mgr->lock);
402 return ret;
403}
404
405int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr,
406 uint64_t address, struct amdgpu_vram_block_info *info)
407{
408 struct amdgpu_vram_mgr_resource *vres;
409 struct gpu_buddy_block *block;
410 u64 start, size;
411 int ret = -ENOENT;
412
413 mutex_lock(&mgr->lock);
414 list_for_each_entry(vres, &mgr->allocated_vres_list, vres_node) {
415 list_for_each_entry(block, &vres->blocks, link) {
416 start = amdgpu_vram_mgr_block_start(block);
417 size = amdgpu_vram_mgr_block_size(block);
418 if ((start <= address) && (address < (start + size))) {
419 info->start = start;
420 info->size = size;
421 memcpy(&info->task, &vres->task, sizeof(vres->task));
422 ret = 0;
423 goto out;
424 }
425 }
426 }
427
428out:
429 mutex_unlock(&mgr->lock);
430
431 return ret;
432}
433
434/**
435 * amdgpu_vram_mgr_new - allocate new ranges
436 *
437 * @man: TTM memory type manager
438 * @tbo: TTM BO we need this range for
439 * @place: placement flags and restrictions
440 * @res: the resulting mem object
441 *
442 * Allocate VRAM for the given BO.
443 */
444static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
445 struct ttm_buffer_object *tbo,
446 const struct ttm_place *place,
447 struct ttm_resource **res)
448{
449 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
450 struct amdgpu_device *adev = to_amdgpu_device(mgr);
451 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
452 u64 vis_usage = 0, max_bytes, min_block_size;
453 struct amdgpu_vram_mgr_resource *vres;
454 u64 size, remaining_size, lpfn, fpfn;
455 unsigned int adjust_dcc_size = 0;
456 struct gpu_buddy *mm = &mgr->mm;
457 struct gpu_buddy_block *block;
458 unsigned long pages_per_block;
459 int r;
460
461 lpfn = (u64)place->lpfn << PAGE_SHIFT;
462 if (!lpfn || lpfn > man->size)
463 lpfn = man->size;
464
465 fpfn = (u64)place->fpfn << PAGE_SHIFT;
466
467 max_bytes = adev->gmc.mc_vram_size;
468 if (tbo->type != ttm_bo_type_kernel)
469 max_bytes -= AMDGPU_VM_RESERVED_VRAM;
470
471 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) {
472 pages_per_block = ~0ul;
473 } else {
474#ifdef CONFIG_TRANSPARENT_HUGEPAGE
475 pages_per_block = HPAGE_PMD_NR;
476#else
477 /* default to 2MB */
478 pages_per_block = 2UL << (20UL - PAGE_SHIFT);
479#endif
480 pages_per_block = max_t(u32, pages_per_block,
481 tbo->page_alignment);
482 }
483
484 vres = kzalloc_obj(*vres);
485 if (!vres)
486 return -ENOMEM;
487
488 ttm_resource_init(tbo, place, &vres->base);
489
490 /* bail out quickly if there's likely not enough VRAM for this BO */
491 if (ttm_resource_manager_usage(man) > max_bytes) {
492 r = -ENOSPC;
493 goto error_fini;
494 }
495
496 INIT_LIST_HEAD(&vres->blocks);
497
498 if (place->flags & TTM_PL_FLAG_TOPDOWN)
499 vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION;
500
501 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
502 vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION;
503
504 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED)
505 vres->flags |= GPU_BUDDY_CLEAR_ALLOCATION;
506
507 if (fpfn || lpfn != mgr->mm.size)
508 /* Allocate blocks in desired range */
509 vres->flags |= GPU_BUDDY_RANGE_ALLOCATION;
510
511 if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC &&
512 adev->gmc.gmc_funcs->get_dcc_alignment)
513 adjust_dcc_size = amdgpu_gmc_get_dcc_alignment(adev);
514
515 remaining_size = (u64)vres->base.size;
516 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {
517 unsigned int dcc_size;
518
519 dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size);
520 remaining_size = (u64)dcc_size;
521
522 vres->flags |= GPU_BUDDY_TRIM_DISABLE;
523 }
524
525 mutex_lock(&mgr->lock);
526 while (remaining_size) {
527 if (tbo->page_alignment)
528 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT;
529 else
530 min_block_size = mgr->default_page_size;
531
532 size = remaining_size;
533
534 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size)
535 min_block_size = size;
536 else if ((size >= (u64)pages_per_block << PAGE_SHIFT) &&
537 !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1)))
538 min_block_size = (u64)pages_per_block << PAGE_SHIFT;
539
540 BUG_ON(min_block_size < mm->chunk_size);
541
542 r = gpu_buddy_alloc_blocks(mm, fpfn,
543 lpfn,
544 size,
545 min_block_size,
546 &vres->blocks,
547 vres->flags);
548
549 if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul &&
550 !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) {
551 vres->flags &= ~GPU_BUDDY_CONTIGUOUS_ALLOCATION;
552 pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT),
553 tbo->page_alignment);
554
555 continue;
556 }
557
558 if (unlikely(r))
559 goto error_free_blocks;
560
561 if (size > remaining_size)
562 remaining_size = 0;
563 else
564 remaining_size -= size;
565 }
566
567 vres->task.pid = task_pid_nr(current);
568 get_task_comm(vres->task.comm, current);
569 list_add_tail(&vres->vres_node, &mgr->allocated_vres_list);
570
571 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) {
572 struct gpu_buddy_block *dcc_block;
573 unsigned long dcc_start;
574 u64 trim_start;
575
576 dcc_block = amdgpu_vram_mgr_first_block(&vres->blocks);
577 /* Adjust the start address for DCC buffers only */
578 dcc_start =
579 roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block),
580 adjust_dcc_size);
581 trim_start = (u64)dcc_start;
582 gpu_buddy_block_trim(mm, &trim_start,
583 (u64)vres->base.size,
584 &vres->blocks);
585 }
586 mutex_unlock(&mgr->lock);
587
588 vres->base.start = 0;
589 size = max_t(u64, amdgpu_vram_mgr_blocks_size(&vres->blocks),
590 vres->base.size);
591 list_for_each_entry(block, &vres->blocks, link) {
592 unsigned long start;
593
594 start = amdgpu_vram_mgr_block_start(block) +
595 amdgpu_vram_mgr_block_size(block);
596 start >>= PAGE_SHIFT;
597
598 if (start > PFN_UP(size))
599 start -= PFN_UP(size);
600 else
601 start = 0;
602 vres->base.start = max(vres->base.start, start);
603
604 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
605 }
606
607 if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks))
608 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
609
610 if (adev->gmc.xgmi.connected_to_cpu)
611 vres->base.bus.caching = ttm_cached;
612 else
613 vres->base.bus.caching = ttm_write_combined;
614
615 atomic64_add(vis_usage, &mgr->vis_usage);
616 *res = &vres->base;
617 return 0;
618
619error_free_blocks:
620 gpu_buddy_free_list(mm, &vres->blocks, 0);
621 mutex_unlock(&mgr->lock);
622error_fini:
623 ttm_resource_fini(man, &vres->base);
624 kfree(vres);
625
626 return r;
627}
628
629/**
630 * amdgpu_vram_mgr_del - free ranges
631 *
632 * @man: TTM memory type manager
633 * @res: TTM memory object
634 *
635 * Free the allocated VRAM again.
636 */
637static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
638 struct ttm_resource *res)
639{
640 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
641 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
642 struct amdgpu_device *adev = to_amdgpu_device(mgr);
643 struct gpu_buddy *mm = &mgr->mm;
644 struct gpu_buddy_block *block;
645 uint64_t vis_usage = 0;
646
647 mutex_lock(&mgr->lock);
648
649 list_del(&vres->vres_node);
650 memset(&vres->task, 0, sizeof(vres->task));
651
652 list_for_each_entry(block, &vres->blocks, link)
653 vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
654
655 gpu_buddy_free_list(mm, &vres->blocks, vres->flags);
656 amdgpu_vram_mgr_do_reserve(man);
657 mutex_unlock(&mgr->lock);
658
659 atomic64_sub(vis_usage, &mgr->vis_usage);
660
661 ttm_resource_fini(man, res);
662 kfree(vres);
663}
664
665/**
666 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
667 *
668 * @adev: amdgpu device pointer
669 * @res: TTM memory object
670 * @offset: byte offset from the base of VRAM BO
671 * @length: number of bytes to export in sg_table
672 * @dev: the other device
673 * @dir: dma direction
674 * @sgt: resulting sg table
675 *
676 * Allocate and fill a sg table from a VRAM allocation.
677 */
678int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
679 struct ttm_resource *res,
680 u64 offset, u64 length,
681 struct device *dev,
682 enum dma_data_direction dir,
683 struct sg_table **sgt)
684{
685 struct amdgpu_res_cursor cursor;
686 struct scatterlist *sg;
687 int num_entries = 0;
688 int i, r;
689
690 *sgt = kmalloc_obj(**sgt);
691 if (!*sgt)
692 return -ENOMEM;
693
694 /* Determine the number of GPU_BUDDY blocks to export */
695 amdgpu_res_first(res, offset, length, &cursor);
696 while (cursor.remaining) {
697 num_entries++;
698 amdgpu_res_next(&cursor, min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE));
699 }
700
701 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
702 if (r)
703 goto error_free;
704
705 /* Initialize scatterlist nodes of sg_table */
706 for_each_sgtable_sg((*sgt), sg, i)
707 sg->length = 0;
708
709 /*
710 * Walk down GPU_BUDDY blocks to populate scatterlist nodes
711 * @note: Use iterator api to get first the GPU_BUDDY block
712 * and the number of bytes from it. Access the following
713 * GPU_BUDDY block(s) if more buffer needs to exported
714 */
715 amdgpu_res_first(res, offset, length, &cursor);
716 for_each_sgtable_sg((*sgt), sg, i) {
717 phys_addr_t phys = cursor.start + adev->gmc.aper_base;
718 unsigned long size = min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE);
719 dma_addr_t addr;
720
721 addr = dma_map_resource(dev, phys, size, dir,
722 DMA_ATTR_SKIP_CPU_SYNC);
723 r = dma_mapping_error(dev, addr);
724 if (r)
725 goto error_unmap;
726
727 sg_set_page(sg, NULL, size, 0);
728 sg_dma_address(sg) = addr;
729 sg_dma_len(sg) = size;
730
731 amdgpu_res_next(&cursor, size);
732 }
733
734 return 0;
735
736error_unmap:
737 for_each_sgtable_sg((*sgt), sg, i) {
738 if (!sg->length)
739 continue;
740
741 dma_unmap_resource(dev, sg->dma_address,
742 sg->length, dir,
743 DMA_ATTR_SKIP_CPU_SYNC);
744 }
745 sg_free_table(*sgt);
746
747error_free:
748 kfree(*sgt);
749 return r;
750}
751
752/**
753 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
754 *
755 * @dev: device pointer
756 * @dir: data direction of resource to unmap
757 * @sgt: sg table to free
758 *
759 * Free a previously allocate sg table.
760 */
761void amdgpu_vram_mgr_free_sgt(struct device *dev,
762 enum dma_data_direction dir,
763 struct sg_table *sgt)
764{
765 struct scatterlist *sg;
766 int i;
767
768 for_each_sgtable_sg(sgt, sg, i)
769 dma_unmap_resource(dev, sg->dma_address,
770 sg->length, dir,
771 DMA_ATTR_SKIP_CPU_SYNC);
772 sg_free_table(sgt);
773 kfree(sgt);
774}
775
776/**
777 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
778 *
779 * @mgr: amdgpu_vram_mgr pointer
780 *
781 * Returns how many bytes are used in the visible part of VRAM
782 */
783uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
784{
785 return atomic64_read(&mgr->vis_usage);
786}
787
788/**
789 * amdgpu_vram_mgr_clear_reset_blocks - reset clear blocks
790 *
791 * @adev: amdgpu device pointer
792 *
793 * Reset the cleared drm buddy blocks.
794 */
795void amdgpu_vram_mgr_clear_reset_blocks(struct amdgpu_device *adev)
796{
797 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
798 struct gpu_buddy *mm = &mgr->mm;
799
800 mutex_lock(&mgr->lock);
801 gpu_buddy_reset_clear(mm, false);
802 mutex_unlock(&mgr->lock);
803}
804
805/**
806 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection
807 *
808 * @man: TTM memory type manager
809 * @res: The resource to test
810 * @place: The place to test against
811 * @size: Size of the new allocation
812 *
813 * Test each drm buddy block for intersection for eviction decision.
814 */
815static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
816 struct ttm_resource *res,
817 const struct ttm_place *place,
818 size_t size)
819{
820 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
821 struct gpu_buddy_block *block;
822
823 /* Check each drm buddy block individually */
824 list_for_each_entry(block, &mgr->blocks, link) {
825 unsigned long fpfn =
826 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
827 unsigned long lpfn = fpfn +
828 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
829
830 if (place->fpfn < lpfn &&
831 (!place->lpfn || place->lpfn > fpfn))
832 return true;
833 }
834
835 return false;
836}
837
838/**
839 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility
840 *
841 * @man: TTM memory type manager
842 * @res: The resource to test
843 * @place: The place to test against
844 * @size: Size of the new allocation
845 *
846 * Test each drm buddy block for placement compatibility.
847 */
848static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
849 struct ttm_resource *res,
850 const struct ttm_place *place,
851 size_t size)
852{
853 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
854 struct gpu_buddy_block *block;
855
856 /* Check each drm buddy block individually */
857 list_for_each_entry(block, &mgr->blocks, link) {
858 unsigned long fpfn =
859 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
860 unsigned long lpfn = fpfn +
861 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
862
863 if (fpfn < place->fpfn ||
864 (place->lpfn && lpfn > place->lpfn))
865 return false;
866 }
867
868 return true;
869}
870
871/**
872 * amdgpu_vram_mgr_debug - dump VRAM table
873 *
874 * @man: TTM memory type manager
875 * @printer: DRM printer to use
876 *
877 * Dump the table content using printk.
878 */
879static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
880 struct drm_printer *printer)
881{
882 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
883 struct gpu_buddy *mm = &mgr->mm;
884 struct amdgpu_vram_reservation *rsv;
885
886 drm_printf(printer, " vis usage:%llu\n",
887 amdgpu_vram_mgr_vis_usage(mgr));
888
889 mutex_lock(&mgr->lock);
890 drm_printf(printer, "default_page_size: %lluKiB\n",
891 mgr->default_page_size >> 10);
892
893 drm_buddy_print(mm, printer);
894
895 drm_printf(printer, "reserved:\n");
896 list_for_each_entry(rsv, &mgr->reserved_pages, blocks)
897 drm_printf(printer, "%#018llx-%#018llx: %llu\n",
898 rsv->start, rsv->start + rsv->size, rsv->size);
899 mutex_unlock(&mgr->lock);
900}
901
902static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
903 .alloc = amdgpu_vram_mgr_new,
904 .free = amdgpu_vram_mgr_del,
905 .intersects = amdgpu_vram_mgr_intersects,
906 .compatible = amdgpu_vram_mgr_compatible,
907 .debug = amdgpu_vram_mgr_debug
908};
909
910static const struct dmem_cgroup_ops amdgpu_vram_mgr_dmem_ops;
911
912static int amdgpu_vram_mgr_dmem_reclaim(struct dmem_cgroup_pool_state *pool,
913 u64 target_bytes, void *priv)
914{
915 struct ttm_resource_manager *man = priv;
916 struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
917 int ret, idx;
918
919 if (!drm_dev_enter(adev_to_drm(adev), &idx))
920 return -ENODEV;
921
922 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
923 if (ret < 0) {
924 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
925 goto out;
926 }
927
928 ret = ttm_resource_manager_dmem_reclaim(pool, target_bytes, priv);
929
930 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
931 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
932
933out:
934 drm_dev_exit(idx);
935 return ret;
936}
937
938static const struct dmem_cgroup_ops amdgpu_vram_mgr_dmem_ops = {
939 .reclaim = amdgpu_vram_mgr_dmem_reclaim,
940};
941
942/**
943 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
944 *
945 * @adev: amdgpu_device pointer
946 *
947 * Allocate and initialize the VRAM manager.
948 */
949int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
950{
951 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
952 struct ttm_resource_manager *man = &mgr->manager;
953 struct dmem_cgroup_region *cg;
954 int err;
955
956 ttm_resource_manager_init(man, &adev->mman.bdev,
957 adev->gmc.real_vram_size);
958
959 mutex_init(&mgr->lock);
960 INIT_LIST_HEAD(&mgr->reservations_pending);
961 INIT_LIST_HEAD(&mgr->reserved_pages);
962 INIT_LIST_HEAD(&mgr->allocated_vres_list);
963 mgr->default_page_size = PAGE_SIZE;
964
965 man->func = &amdgpu_vram_mgr_func;
966 err = gpu_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
967 if (err)
968 return err;
969
970 cg = dmem_cgroup_register_region(&(struct dmem_cgroup_init){
971 .size = adev->gmc.real_vram_size,
972 .ops = &amdgpu_vram_mgr_dmem_ops,
973 .reclaim_priv = man,
974 },
975 "drm/%s/vram", adev_to_drm(adev)->unique);
976 if (IS_ERR(cg)) {
977 gpu_buddy_fini(&mgr->mm);
978 return PTR_ERR(cg);
979 }
980
981 mgr->cg_region = cg;
982 ttm_resource_manager_set_dmem_region(man, cg);
983
984 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
985 ttm_resource_manager_set_used(man, true);
986 return 0;
987}
988
989/**
990 * amdgpu_vram_mgr_fini - free and destroy VRAM manager
991 *
992 * @adev: amdgpu_device pointer
993 *
994 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
995 * allocated inside it.
996 */
997void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
998{
999 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
1000 struct ttm_resource_manager *man = &mgr->manager;
1001 int ret;
1002 struct amdgpu_vram_reservation *rsv, *temp;
1003
1004 ttm_resource_manager_set_used(man, false);
1005
1006 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
1007
1008 /*
1009 * Unregister the dmem cgroup region regardless of the evict_all()
1010 * result and before any further teardown. This drains in-flight
1011 * reclaim callbacks and blocks new ones, so no reclaim can reference
1012 * the manager once we start freeing it. It must run after evict_all()
1013 * so that ttm_resource_free() can still uncharge via man->cg during
1014 * eviction. Clear man->cg afterwards.
1015 */
1016 dmem_cgroup_unregister_region(mgr->cg_region);
1017 mgr->cg_region = NULL;
1018 ttm_resource_manager_set_dmem_region(man, NULL);
1019
1020 if (ret)
1021 return;
1022
1023 mutex_lock(&mgr->lock);
1024 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)
1025 kfree(rsv);
1026
1027 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
1028 gpu_buddy_free_list(&mgr->mm, &rsv->allocated, 0);
1029 kfree(rsv);
1030 }
1031 if (!adev->gmc.is_app_apu)
1032 gpu_buddy_fini(&mgr->mm);
1033 mutex_unlock(&mgr->lock);
1034
1035 ttm_resource_manager_cleanup(man);
1036 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
1037}