projects/linux·blob master · refs

1709 lines · 43.8 KiB

1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <[email protected]>
29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 * Dave Airlie
31 */
32#include <linux/list.h>
33#include <linux/slab.h>
34#include <linux/dma-buf.h>
35#include <linux/export.h>
36
37#include <drm/drm_drv.h>
38#include <drm/amdgpu_drm.h>
39#include <drm/drm_cache.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_amdkfd.h"
43#include "amdgpu_vram_mgr.h"
44#include "amdgpu_vm.h"
45#include "amdgpu_dma_buf.h"
46#include "kfd_svm.h"
47
48/**
49 * DOC: amdgpu_object
50 *
51 * This defines the interfaces to operate on an &amdgpu_bo buffer object which
52 * represents memory used by driver (VRAM, system memory, etc.). The driver
53 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
54 * to create/destroy/set buffer object which are then managed by the kernel TTM
55 * memory manager.
56 * The interfaces are also used internally by kernel clients, including gfx,
57 * uvd, etc. for kernel managed allocations used by the GPU.
58 *
59 */
60
61static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
62{
63 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
64
65 amdgpu_bo_kunmap(bo);
66
67 if (drm_gem_is_imported(&bo->tbo.base))
68 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
69 drm_gem_object_release(&bo->tbo.base);
70 amdgpu_bo_unref(&bo->parent);
71 kvfree(bo);
72}
73
74static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
75{
76 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
77 struct amdgpu_bo_user *ubo;
78
79 ubo = to_amdgpu_bo_user(bo);
80 kfree(ubo->metadata);
81 amdgpu_bo_destroy(tbo);
82}
83
84/**
85 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
86 * @bo: buffer object to be checked
87 *
88 * Uses destroy function associated with the object to determine if this is
89 * an &amdgpu_bo.
90 *
91 * Returns:
92 * true if the object belongs to &amdgpu_bo, false if not.
93 */
94bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
95{
96 if (bo->destroy == &amdgpu_bo_destroy ||
97 bo->destroy == &amdgpu_bo_user_destroy ||
98 bo->destroy == &svm_range_bo_destroy)
99 return true;
100
101 return false;
102}
103
104/**
105 * amdgpu_bo_placement_from_domain - set buffer's placement
106 * @abo: &amdgpu_bo buffer object whose placement is to be set
107 * @domain: requested domain
108 *
109 * Sets buffer's placement according to requested domain and the buffer's
110 * flags.
111 */
112void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
113{
114 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
115 struct ttm_placement *placement = &abo->placement;
116 struct ttm_place *places = abo->placements;
117 u64 flags = abo->flags;
118 u32 c = 0;
119
120 if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
121 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
122 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
123
124 if (adev->gmc.mem_partitions && mem_id >= 0) {
125 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn;
126 /*
127 * memory partition range lpfn is inclusive start + size - 1
128 * TTM place lpfn is exclusive start + size
129 */
130 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1;
131 } else {
132 places[c].fpfn = 0;
133 places[c].lpfn = 0;
134 }
135 places[c].mem_type = TTM_PL_VRAM;
136 places[c].flags = 0;
137
138 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
139 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn);
140 else
141 places[c].flags |= TTM_PL_FLAG_TOPDOWN;
142
143 if (abo->tbo.type == ttm_bo_type_kernel &&
144 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
145 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
146
147 c++;
148 }
149
150 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) {
151 places[c].fpfn = 0;
152 places[c].lpfn = 0;
153 places[c].mem_type = AMDGPU_PL_DOORBELL;
154 places[c].flags = 0;
155 c++;
156 }
157
158 if (domain & AMDGPU_GEM_DOMAIN_GTT) {
159 places[c].fpfn = 0;
160 places[c].lpfn = 0;
161 places[c].mem_type =
162 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
163 AMDGPU_PL_PREEMPT : TTM_PL_TT;
164 places[c].flags = 0;
165 /*
166 * When GTT is just an alternative to VRAM make sure that we
167 * only use it as fallback and still try to fill up VRAM first.
168 */
169 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) &&
170 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)
171 places[c].flags |= TTM_PL_FLAG_FALLBACK;
172 c++;
173 }
174
175 if (domain & AMDGPU_GEM_DOMAIN_CPU) {
176 places[c].fpfn = 0;
177 places[c].lpfn = 0;
178 places[c].mem_type = TTM_PL_SYSTEM;
179 places[c].flags = 0;
180 c++;
181 }
182
183 if (domain & AMDGPU_GEM_DOMAIN_GDS) {
184 places[c].fpfn = 0;
185 places[c].lpfn = 0;
186 places[c].mem_type = AMDGPU_PL_GDS;
187 places[c].flags = 0;
188 c++;
189 }
190
191 if (domain & AMDGPU_GEM_DOMAIN_GWS) {
192 places[c].fpfn = 0;
193 places[c].lpfn = 0;
194 places[c].mem_type = AMDGPU_PL_GWS;
195 places[c].flags = 0;
196 c++;
197 }
198
199 if (domain & AMDGPU_GEM_DOMAIN_OA) {
200 places[c].fpfn = 0;
201 places[c].lpfn = 0;
202 places[c].mem_type = AMDGPU_PL_OA;
203 places[c].flags = 0;
204 c++;
205 }
206
207 if (!c) {
208 places[c].fpfn = 0;
209 places[c].lpfn = 0;
210 places[c].mem_type = TTM_PL_SYSTEM;
211 places[c].flags = 0;
212 c++;
213 }
214
215 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
216
217 placement->num_placement = c;
218 placement->placement = places;
219}
220
221/**
222 * amdgpu_bo_create_reserved - create reserved BO for kernel use
223 *
224 * @adev: amdgpu device object
225 * @size: size for the new BO
226 * @align: alignment for the new BO
227 * @domain: where to place it
228 * @bo_ptr: used to initialize BOs in structures
229 * @gpu_addr: GPU addr of the pinned BO
230 * @cpu_addr: optional CPU address mapping
231 *
232 * Allocates and pins a BO for kernel internal use, and returns it still
233 * reserved.
234 *
235 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
236 *
237 * Returns:
238 * 0 on success, negative error code otherwise.
239 */
240int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
241 unsigned long size, int align,
242 u32 domain, struct amdgpu_bo **bo_ptr,
243 u64 *gpu_addr, void **cpu_addr)
244{
245 struct amdgpu_bo_param bp;
246 bool free = false;
247 int r;
248
249 if (!size) {
250 amdgpu_bo_unref(bo_ptr);
251 return 0;
252 }
253
254 memset(&bp, 0, sizeof(bp));
255 bp.size = size;
256 bp.byte_align = align;
257 bp.domain = domain;
258 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
259 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
260 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
261 bp.type = ttm_bo_type_kernel;
262 bp.resv = NULL;
263 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
264
265 if (!*bo_ptr) {
266 r = amdgpu_bo_create(adev, &bp, bo_ptr);
267 if (r) {
268 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
269 r);
270 return r;
271 }
272 free = true;
273 }
274
275 r = amdgpu_bo_reserve(*bo_ptr, false);
276 if (r) {
277 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
278 goto error_free;
279 }
280
281 if (free) {
282 r = amdgpu_bo_pin(*bo_ptr, domain);
283 if (r) {
284 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
285 goto error_unreserve;
286 }
287 }
288
289 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
290 if (r) {
291 dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
292 goto error_unpin;
293 }
294
295 if (gpu_addr)
296 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
297
298 if (cpu_addr) {
299 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
300 if (r) {
301 dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
302 goto error_unpin;
303 }
304 }
305
306 return 0;
307
308error_unpin:
309 if (free)
310 amdgpu_bo_unpin(*bo_ptr);
311error_unreserve:
312 amdgpu_bo_unreserve(*bo_ptr);
313
314error_free:
315 if (free)
316 amdgpu_bo_unref(bo_ptr);
317
318 return r;
319}
320
321/**
322 * amdgpu_bo_create_kernel - create BO for kernel use
323 *
324 * @adev: amdgpu device object
325 * @size: size for the new BO
326 * @align: alignment for the new BO
327 * @domain: where to place it
328 * @bo_ptr: used to initialize BOs in structures
329 * @gpu_addr: GPU addr of the pinned BO
330 * @cpu_addr: optional CPU address mapping
331 *
332 * Allocates and pins a BO for kernel internal use.
333 *
334 * This function is exported to allow the V4L2 isp device
335 * external to drm device to create and access the kernel BO.
336 *
337 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
338 *
339 * Returns:
340 * 0 on success, negative error code otherwise.
341 */
342int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
343 unsigned long size, int align,
344 u32 domain, struct amdgpu_bo **bo_ptr,
345 u64 *gpu_addr, void **cpu_addr)
346{
347 int r;
348
349 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
350 gpu_addr, cpu_addr);
351
352 if (r)
353 return r;
354
355 if (*bo_ptr)
356 amdgpu_bo_unreserve(*bo_ptr);
357
358 return 0;
359}
360
361/**
362 * amdgpu_bo_create_isp_user - create user BO for isp
363 *
364 * @adev: amdgpu device object
365 * @dma_buf: DMABUF handle for isp buffer
366 * @domain: where to place it
367 * @bo: used to initialize BOs in structures
368 * @gpu_addr: GPU addr of the pinned BO
369 *
370 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does
371 * GART alloc to generate gpu_addr for BO to make it accessible through the
372 * GART aperture for ISP HW.
373 *
374 * This function is exported to allow the V4L2 isp device external to drm device
375 * to create and access the isp user BO.
376 *
377 * Returns:
378 * 0 on success, negative error code otherwise.
379 */
380int amdgpu_bo_create_isp_user(struct amdgpu_device *adev,
381 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo,
382 u64 *gpu_addr)
383
384{
385 struct drm_gem_object *gem_obj;
386 int r;
387
388 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf);
389 *bo = gem_to_amdgpu_bo(gem_obj);
390 if (!(*bo)) {
391 dev_err(adev->dev, "failed to get valid isp user bo\n");
392 return -EINVAL;
393 }
394
395 r = amdgpu_bo_reserve(*bo, false);
396 if (r) {
397 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r);
398 return r;
399 }
400
401 r = amdgpu_bo_pin(*bo, domain);
402 if (r) {
403 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r);
404 goto error_unreserve;
405 }
406
407 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo);
408 if (r) {
409 dev_err(adev->dev, "%p bind failed\n", *bo);
410 goto error_unpin;
411 }
412
413 if (!WARN_ON(!gpu_addr))
414 *gpu_addr = amdgpu_bo_gpu_offset(*bo);
415
416 amdgpu_bo_unreserve(*bo);
417
418 return 0;
419
420error_unpin:
421 amdgpu_bo_unpin(*bo);
422error_unreserve:
423 amdgpu_bo_unreserve(*bo);
424 amdgpu_bo_unref(bo);
425
426 return r;
427}
428
429/**
430 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
431 *
432 * @adev: amdgpu device object
433 * @offset: offset of the BO
434 * @size: size of the BO
435 * @bo_ptr: used to initialize BOs in structures
436 * @cpu_addr: optional CPU address mapping
437 *
438 * Creates a kernel BO at a specific offset in VRAM.
439 *
440 * Returns:
441 * 0 on success, negative error code otherwise.
442 */
443int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
444 uint64_t offset, uint64_t size,
445 struct amdgpu_bo **bo_ptr, void **cpu_addr)
446{
447 struct ttm_operation_ctx ctx = { false, false };
448 unsigned int i;
449 int r;
450
451 offset &= PAGE_MASK;
452 size = ALIGN(size, PAGE_SIZE);
453
454 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE,
455 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL,
456 cpu_addr);
457 if (r)
458 return r;
459
460 if ((*bo_ptr) == NULL)
461 return 0;
462
463 /*
464 * Remove the original mem node and create a new one at the request
465 * position.
466 */
467 if (cpu_addr)
468 amdgpu_bo_kunmap(*bo_ptr);
469
470 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
471
472 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
473 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
474 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
475 }
476 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
477 &(*bo_ptr)->tbo.resource, &ctx);
478 if (r)
479 goto error;
480
481 if (cpu_addr) {
482 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
483 if (r)
484 goto error;
485 }
486
487 amdgpu_bo_unreserve(*bo_ptr);
488 return 0;
489
490error:
491 amdgpu_bo_unreserve(*bo_ptr);
492 amdgpu_bo_unref(bo_ptr);
493 return r;
494}
495
496/**
497 * amdgpu_bo_free_kernel - free BO for kernel use
498 *
499 * @bo: amdgpu BO to free
500 * @gpu_addr: pointer to where the BO's GPU memory space address was stored
501 * @cpu_addr: pointer to where the BO's CPU memory space address was stored
502 *
503 * unmaps and unpin a BO for kernel internal use.
504 *
505 * This function is exported to allow the V4L2 isp device
506 * external to drm device to free the kernel BO.
507 */
508void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
509 void **cpu_addr)
510{
511 if (*bo == NULL)
512 return;
513
514 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend);
515
516 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
517 if (cpu_addr)
518 amdgpu_bo_kunmap(*bo);
519
520 amdgpu_bo_unpin(*bo);
521 amdgpu_bo_unreserve(*bo);
522 }
523 amdgpu_bo_unref(bo);
524
525 if (gpu_addr)
526 *gpu_addr = 0;
527
528 if (cpu_addr)
529 *cpu_addr = NULL;
530}
531
532/**
533 * amdgpu_bo_free_isp_user - free BO for isp use
534 *
535 * @bo: amdgpu isp user BO to free
536 *
537 * unpin and unref BO for isp internal use.
538 *
539 * This function is exported to allow the V4L2 isp device
540 * external to drm device to free the isp user BO.
541 */
542void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo)
543{
544 if (bo == NULL)
545 return;
546
547 if (amdgpu_bo_reserve(bo, true) == 0) {
548 amdgpu_bo_unpin(bo);
549 amdgpu_bo_unreserve(bo);
550 }
551 amdgpu_bo_unref(&bo);
552}
553
554/* Validate bo size is bit bigger than the request domain */
555static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
556 unsigned long size, u32 domain)
557{
558 struct ttm_resource_manager *man = NULL;
559
560 /*
561 * If GTT is part of requested domains the check must succeed to
562 * allow fall back to GTT.
563 */
564 if (domain & AMDGPU_GEM_DOMAIN_GTT)
565 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
566 else if (domain & AMDGPU_GEM_DOMAIN_VRAM)
567 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
568 else
569 return true;
570
571 if (!man) {
572 if (domain & AMDGPU_GEM_DOMAIN_GTT)
573 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized");
574 return false;
575 }
576
577 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */
578 if (size < man->size)
579 return true;
580
581 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size);
582 return false;
583}
584
585bool amdgpu_bo_support_uswc(u64 bo_flags)
586{
587
588#ifdef CONFIG_X86_32
589 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
590 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
591 */
592 return false;
593#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
594 /* Don't try to enable write-combining when it can't work, or things
595 * may be slow
596 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
597 */
598
599#ifndef CONFIG_COMPILE_TEST
600#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
601 thanks to write-combining
602#endif
603
604 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
605 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
606 "better performance thanks to write-combining\n");
607 return false;
608#else
609 /* For architectures that don't support WC memory,
610 * mask out the WC flag from the BO
611 */
612 if (!drm_arch_can_wc_memory())
613 return false;
614
615 return true;
616#endif
617}
618
619/**
620 * amdgpu_bo_create - create an &amdgpu_bo buffer object
621 * @adev: amdgpu device object
622 * @bp: parameters to be used for the buffer object
623 * @bo_ptr: pointer to the buffer object pointer
624 *
625 * Creates an &amdgpu_bo buffer object.
626 *
627 * Returns:
628 * 0 for success or a negative error code on failure.
629 */
630int amdgpu_bo_create(struct amdgpu_device *adev,
631 struct amdgpu_bo_param *bp,
632 struct amdgpu_bo **bo_ptr)
633{
634 struct ttm_operation_ctx ctx = {
635 .interruptible = (bp->type != ttm_bo_type_kernel),
636 .no_wait_gpu = bp->no_wait_gpu,
637 /* We opt to avoid OOM on system pages allocations */
638 .gfp_retry_mayfail = true,
639 .allow_res_evict = bp->type != ttm_bo_type_kernel,
640 .resv = bp->resv
641 };
642 struct amdgpu_bo *bo;
643 unsigned long page_align, size = bp->size;
644 int r;
645
646 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
647 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
648 /* GWS and OA don't need any alignment. */
649 page_align = bp->byte_align;
650 size <<= PAGE_SHIFT;
651
652 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
653 /* Both size and alignment must be a multiple of 4. */
654 page_align = ALIGN(bp->byte_align, 4);
655 size = ALIGN(size, 4) << PAGE_SHIFT;
656 } else {
657 /* Memory should be aligned at least to a page size. */
658 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
659 size = ALIGN(size, PAGE_SIZE);
660 }
661
662 if (!amdgpu_bo_validate_size(adev, size, bp->domain))
663 return -ENOMEM;
664
665 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
666
667 *bo_ptr = NULL;
668 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
669 if (bo == NULL)
670 return -ENOMEM;
671 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
672 bo->tbo.base.funcs = &amdgpu_gem_object_funcs;
673 bo->vm_bo = NULL;
674 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
675 bp->domain;
676 bo->allowed_domains = bo->preferred_domains;
677 if (bp->type != ttm_bo_type_kernel &&
678 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
679 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
680 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
681
682 bo->flags = bp->flags;
683
684 if (adev->gmc.mem_partitions)
685 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */
686 bo->xcp_id = bp->xcp_id_plus1 - 1;
687 else
688 /* For GPUs without spatial partitioning */
689 bo->xcp_id = 0;
690
691 if (!amdgpu_bo_support_uswc(bo->flags))
692 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
693
694 bo->tbo.bdev = &adev->mman.bdev;
695 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
696 AMDGPU_GEM_DOMAIN_GDS))
697 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
698 else
699 amdgpu_bo_placement_from_domain(bo, bp->domain);
700 if (bp->type == ttm_bo_type_kernel)
701 bo->tbo.priority = 2;
702 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE))
703 bo->tbo.priority = 1;
704
705 if (!bp->destroy)
706 bp->destroy = &amdgpu_bo_destroy;
707
708 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
709 &bo->placement, page_align, &ctx, NULL,
710 bp->resv, bp->destroy);
711 if (unlikely(r != 0))
712 return r;
713
714 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
715 amdgpu_res_cpu_visible(adev, bo->tbo.resource))
716 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
717 ctx.bytes_moved);
718 else
719 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
720
721 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
722 bo->tbo.resource->mem_type == TTM_PL_VRAM) {
723 struct dma_fence *fence;
724
725 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
726 bo, bo->tbo.base.resv, &fence,
727 true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
728 if (unlikely(r))
729 goto fail_unreserve;
730
731 if (fence) {
732 dma_resv_add_fence(bo->tbo.base.resv, fence,
733 DMA_RESV_USAGE_KERNEL);
734 dma_fence_put(fence);
735 }
736 }
737 if (!bp->resv)
738 amdgpu_bo_unreserve(bo);
739 *bo_ptr = bo;
740
741 trace_amdgpu_bo_create(bo);
742
743 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
744 if (bp->type == ttm_bo_type_device)
745 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
746
747 return 0;
748
749fail_unreserve:
750 if (!bp->resv)
751 dma_resv_unlock(bo->tbo.base.resv);
752 amdgpu_bo_unref(&bo);
753 return r;
754}
755
756/**
757 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
758 * @adev: amdgpu device object
759 * @bp: parameters to be used for the buffer object
760 * @ubo_ptr: pointer to the buffer object pointer
761 *
762 * Create a BO to be used by user application;
763 *
764 * Returns:
765 * 0 for success or a negative error code on failure.
766 */
767
768int amdgpu_bo_create_user(struct amdgpu_device *adev,
769 struct amdgpu_bo_param *bp,
770 struct amdgpu_bo_user **ubo_ptr)
771{
772 struct amdgpu_bo *bo_ptr;
773 int r;
774
775 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
776 bp->destroy = &amdgpu_bo_user_destroy;
777 r = amdgpu_bo_create(adev, bp, &bo_ptr);
778 if (r)
779 return r;
780
781 *ubo_ptr = to_amdgpu_bo_user(bo_ptr);
782 return r;
783}
784
785/**
786 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
787 * @adev: amdgpu device object
788 * @bp: parameters to be used for the buffer object
789 * @vmbo_ptr: pointer to the buffer object pointer
790 *
791 * Create a BO to be for GPUVM.
792 *
793 * Returns:
794 * 0 for success or a negative error code on failure.
795 */
796
797int amdgpu_bo_create_vm(struct amdgpu_device *adev,
798 struct amdgpu_bo_param *bp,
799 struct amdgpu_bo_vm **vmbo_ptr)
800{
801 struct amdgpu_bo *bo_ptr;
802 int r;
803
804 /* bo_ptr_size will be determined by the caller and it depends on
805 * num of amdgpu_vm_pt entries.
806 */
807 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
808 r = amdgpu_bo_create(adev, bp, &bo_ptr);
809 if (r)
810 return r;
811
812 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
813 return r;
814}
815
816/**
817 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
818 * @bo: &amdgpu_bo buffer object to be mapped
819 * @ptr: kernel virtual address to be returned
820 *
821 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
822 * amdgpu_bo_kptr() to get the kernel virtual address.
823 *
824 * Returns:
825 * 0 for success or a negative error code on failure.
826 */
827int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
828{
829 void *kptr;
830 long r;
831
832 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
833 return -EPERM;
834
835 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
836 false, MAX_SCHEDULE_TIMEOUT);
837 if (r < 0)
838 return r;
839
840 kptr = amdgpu_bo_kptr(bo);
841 if (kptr) {
842 if (ptr)
843 *ptr = kptr;
844 return 0;
845 }
846
847 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap);
848 if (r)
849 return r;
850
851 if (ptr)
852 *ptr = amdgpu_bo_kptr(bo);
853
854 return 0;
855}
856
857/**
858 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
859 * @bo: &amdgpu_bo buffer object
860 *
861 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
862 *
863 * Returns:
864 * the virtual address of a buffer object area.
865 */
866void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
867{
868 bool is_iomem;
869
870 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
871}
872
873/**
874 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
875 * @bo: &amdgpu_bo buffer object to be unmapped
876 *
877 * Unmaps a kernel map set up by amdgpu_bo_kmap().
878 */
879void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
880{
881 if (bo->kmap.bo)
882 ttm_bo_kunmap(&bo->kmap);
883}
884
885/**
886 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
887 * @bo: &amdgpu_bo buffer object
888 *
889 * References the contained &ttm_buffer_object.
890 *
891 * Returns:
892 * a refcounted pointer to the &amdgpu_bo buffer object.
893 */
894struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
895{
896 if (bo == NULL)
897 return NULL;
898
899 drm_gem_object_get(&bo->tbo.base);
900 return bo;
901}
902
903/**
904 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
905 * @bo: &amdgpu_bo buffer object
906 *
907 * Unreferences the contained &ttm_buffer_object and clear the pointer
908 */
909void amdgpu_bo_unref(struct amdgpu_bo **bo)
910{
911 if ((*bo) == NULL)
912 return;
913
914 drm_gem_object_put(&(*bo)->tbo.base);
915 *bo = NULL;
916}
917
918/**
919 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
920 * @bo: &amdgpu_bo buffer object to be pinned
921 * @domain: domain to be pinned to
922 *
923 * Pins the buffer object according to requested domain. If the memory is
924 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and
925 * pin_size accordingly.
926 *
927 * Pinning means to lock pages in memory along with keeping them at a fixed
928 * offset. It is required when a buffer can not be moved, for example, when
929 * a display buffer is being scanned out.
930 *
931 * Returns:
932 * 0 for success or a negative error code on failure.
933 */
934int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
935{
936 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
937 struct ttm_operation_ctx ctx = { false, false };
938 int r, i;
939
940 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
941 return -EPERM;
942
943 /* Check domain to be pinned to against preferred domains */
944 if (bo->preferred_domains & domain)
945 domain = bo->preferred_domains & domain;
946
947 /* A shared bo cannot be migrated to VRAM */
948 if (drm_gem_is_imported(&bo->tbo.base)) {
949 if (domain & AMDGPU_GEM_DOMAIN_GTT)
950 domain = AMDGPU_GEM_DOMAIN_GTT;
951 else
952 return -EINVAL;
953 }
954
955 if (bo->tbo.pin_count) {
956 uint32_t mem_type = bo->tbo.resource->mem_type;
957 uint32_t mem_flags = bo->tbo.resource->placement;
958
959 if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
960 return -EINVAL;
961
962 if ((mem_type == TTM_PL_VRAM) &&
963 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
964 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
965 return -EINVAL;
966
967 ttm_bo_pin(&bo->tbo);
968 return 0;
969 }
970
971 /* This assumes only APU display buffers are pinned with (VRAM|GTT).
972 * See function amdgpu_display_supported_domains()
973 */
974 domain = amdgpu_bo_get_preferred_domain(adev, domain);
975
976 if (drm_gem_is_imported(&bo->tbo.base))
977 dma_buf_pin(bo->tbo.base.import_attach);
978
979 /* force to pin into visible video ram */
980 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
981 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
982 amdgpu_bo_placement_from_domain(bo, domain);
983 for (i = 0; i < bo->placement.num_placement; i++) {
984 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS &&
985 bo->placements[i].mem_type == TTM_PL_VRAM)
986 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
987 }
988
989 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
990 if (unlikely(r)) {
991 dev_err(adev->dev, "%p pin failed\n", bo);
992 goto error;
993 }
994
995 ttm_bo_pin(&bo->tbo);
996
997 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
998 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
999 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
1000 &adev->visible_pin_size);
1001 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1002 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
1003 }
1004
1005error:
1006 return r;
1007}
1008
1009/**
1010 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
1011 * @bo: &amdgpu_bo buffer object to be unpinned
1012 *
1013 * Decreases the pin_count, and clears the flags if pin_count reaches 0.
1014 * Changes placement and pin size accordingly.
1015 *
1016 * Returns:
1017 * 0 for success or a negative error code on failure.
1018 */
1019void amdgpu_bo_unpin(struct amdgpu_bo *bo)
1020{
1021 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1022
1023 ttm_bo_unpin(&bo->tbo);
1024 if (bo->tbo.pin_count)
1025 return;
1026
1027 if (drm_gem_is_imported(&bo->tbo.base))
1028 dma_buf_unpin(bo->tbo.base.import_attach);
1029
1030 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1031 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1032 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1033 &adev->visible_pin_size);
1034 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1035 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1036 }
1037
1038}
1039
1040static const char * const amdgpu_vram_names[] = {
1041 "UNKNOWN",
1042 "GDDR1",
1043 "DDR2",
1044 "GDDR3",
1045 "GDDR4",
1046 "GDDR5",
1047 "HBM",
1048 "DDR3",
1049 "DDR4",
1050 "GDDR6",
1051 "DDR5",
1052 "LPDDR4",
1053 "LPDDR5",
1054 "HBM3E",
1055 "HBM4"
1056};
1057
1058/**
1059 * amdgpu_bo_init - initialize memory manager
1060 * @adev: amdgpu device object
1061 *
1062 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1063 *
1064 * Returns:
1065 * 0 for success or a negative error code on failure.
1066 */
1067int amdgpu_bo_init(struct amdgpu_device *adev)
1068{
1069 /* On A+A platform, VRAM can be mapped as WB */
1070 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1071 /* reserve PAT memory space to WC for VRAM */
1072 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1073 adev->gmc.aper_size);
1074
1075 if (r) {
1076 DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1077 return r;
1078 }
1079
1080 /* Add an MTRR for the VRAM */
1081 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1082 adev->gmc.aper_size);
1083 }
1084
1085 drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n",
1086 adev->gmc.mc_vram_size >> 20,
1087 (unsigned long long)adev->gmc.aper_size >> 20);
1088 drm_info(adev_to_drm(adev), "RAM width %dbits %s\n",
1089 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1090 return amdgpu_ttm_init(adev);
1091}
1092
1093/**
1094 * amdgpu_bo_fini - tear down memory manager
1095 * @adev: amdgpu device object
1096 *
1097 * Reverses amdgpu_bo_init() to tear down memory manager.
1098 */
1099void amdgpu_bo_fini(struct amdgpu_device *adev)
1100{
1101 int idx;
1102
1103 amdgpu_ttm_fini(adev);
1104
1105 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1106 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
1107 arch_phys_wc_del(adev->gmc.vram_mtrr);
1108 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1109 }
1110 drm_dev_exit(idx);
1111 }
1112}
1113
1114/**
1115 * amdgpu_bo_set_tiling_flags - set tiling flags
1116 * @bo: &amdgpu_bo buffer object
1117 * @tiling_flags: new flags
1118 *
1119 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1120 * kernel driver to set the tiling flags on a buffer.
1121 *
1122 * Returns:
1123 * 0 for success or a negative error code on failure.
1124 */
1125int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1126{
1127 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1128 struct amdgpu_bo_user *ubo;
1129
1130 /* MMIO_REMAP is BAR I/O space; tiling should never be used here. */
1131 WARN_ON_ONCE(bo->tbo.resource &&
1132 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1133
1134 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1135 if (adev->family <= AMDGPU_FAMILY_CZ &&
1136 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1137 return -EINVAL;
1138
1139 ubo = to_amdgpu_bo_user(bo);
1140 ubo->tiling_flags = tiling_flags;
1141 return 0;
1142}
1143
1144/**
1145 * amdgpu_bo_get_tiling_flags - get tiling flags
1146 * @bo: &amdgpu_bo buffer object
1147 * @tiling_flags: returned flags
1148 *
1149 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1150 * set the tiling flags on a buffer.
1151 */
1152void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1153{
1154 struct amdgpu_bo_user *ubo;
1155
1156 /*
1157 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window.
1158 * They should never go through GEM tiling helpers.
1159 */
1160 WARN_ON_ONCE(bo->tbo.resource &&
1161 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP);
1162
1163 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1164 dma_resv_assert_held(bo->tbo.base.resv);
1165 ubo = to_amdgpu_bo_user(bo);
1166
1167 if (tiling_flags)
1168 *tiling_flags = ubo->tiling_flags;
1169}
1170
1171/**
1172 * amdgpu_bo_set_metadata - set metadata
1173 * @bo: &amdgpu_bo buffer object
1174 * @metadata: new metadata
1175 * @metadata_size: size of the new metadata
1176 * @flags: flags of the new metadata
1177 *
1178 * Sets buffer object's metadata, its size and flags.
1179 * Used via GEM ioctl.
1180 *
1181 * Returns:
1182 * 0 for success or a negative error code on failure.
1183 */
1184int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata,
1185 u32 metadata_size, uint64_t flags)
1186{
1187 struct amdgpu_bo_user *ubo;
1188 void *buffer;
1189
1190 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1191 ubo = to_amdgpu_bo_user(bo);
1192 if (!metadata_size) {
1193 if (ubo->metadata_size) {
1194 kfree(ubo->metadata);
1195 ubo->metadata = NULL;
1196 ubo->metadata_size = 0;
1197 }
1198 return 0;
1199 }
1200
1201 if (metadata == NULL)
1202 return -EINVAL;
1203
1204 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1205 if (buffer == NULL)
1206 return -ENOMEM;
1207
1208 kfree(ubo->metadata);
1209 ubo->metadata_flags = flags;
1210 ubo->metadata = buffer;
1211 ubo->metadata_size = metadata_size;
1212
1213 return 0;
1214}
1215
1216/**
1217 * amdgpu_bo_get_metadata - get metadata
1218 * @bo: &amdgpu_bo buffer object
1219 * @buffer: returned metadata
1220 * @buffer_size: size of the buffer
1221 * @metadata_size: size of the returned metadata
1222 * @flags: flags of the returned metadata
1223 *
1224 * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1225 * less than metadata_size.
1226 * Used via GEM ioctl.
1227 *
1228 * Returns:
1229 * 0 for success or a negative error code on failure.
1230 */
1231int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1232 size_t buffer_size, uint32_t *metadata_size,
1233 uint64_t *flags)
1234{
1235 struct amdgpu_bo_user *ubo;
1236
1237 if (!buffer && !metadata_size)
1238 return -EINVAL;
1239
1240 BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1241 ubo = to_amdgpu_bo_user(bo);
1242 if (metadata_size)
1243 *metadata_size = ubo->metadata_size;
1244
1245 if (buffer) {
1246 if (buffer_size < ubo->metadata_size)
1247 return -EINVAL;
1248
1249 if (ubo->metadata_size)
1250 memcpy(buffer, ubo->metadata, ubo->metadata_size);
1251 }
1252
1253 if (flags)
1254 *flags = ubo->metadata_flags;
1255
1256 return 0;
1257}
1258
1259/**
1260 * amdgpu_bo_move_notify - notification about a memory move
1261 * @bo: pointer to a buffer object
1262 * @evict: if this move is evicting the buffer from the graphics address space
1263 * @new_mem: new resource for backing the BO
1264 *
1265 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1266 * bookkeeping.
1267 * TTM driver callback which is called when ttm moves a buffer.
1268 */
1269void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1270 bool evict,
1271 struct ttm_resource *new_mem)
1272{
1273 struct ttm_resource *old_mem = bo->resource;
1274 struct amdgpu_bo *abo;
1275
1276 if (!amdgpu_bo_is_amdgpu_bo(bo))
1277 return;
1278
1279 abo = ttm_to_amdgpu_bo(bo);
1280 amdgpu_vm_bo_move(abo, new_mem, evict);
1281
1282 amdgpu_bo_kunmap(abo);
1283
1284 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) &&
1285 old_mem && old_mem->mem_type != TTM_PL_SYSTEM)
1286 dma_buf_invalidate_mappings(abo->tbo.base.dma_buf);
1287
1288 /* move_notify is called before move happens */
1289 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1,
1290 old_mem ? old_mem->mem_type : -1);
1291}
1292
1293/**
1294 * amdgpu_bo_release_notify - notification about a BO being released
1295 * @bo: pointer to a buffer object
1296 *
1297 * Wipes VRAM buffers whose contents should not be leaked before the
1298 * memory is released.
1299 */
1300void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1301{
1302 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1303 struct dma_fence *fence = NULL;
1304 struct amdgpu_bo *abo;
1305 int r;
1306
1307 if (!amdgpu_bo_is_amdgpu_bo(bo))
1308 return;
1309
1310 abo = ttm_to_amdgpu_bo(bo);
1311
1312 WARN_ON(abo->vm_bo);
1313
1314 if (abo->kfd_bo)
1315 amdgpu_amdkfd_release_notify(abo);
1316
1317 /*
1318 * We lock the private dma_resv object here and since the BO is about to
1319 * be released nobody else should have a pointer to it.
1320 * So when this locking here fails something is wrong with the reference
1321 * counting.
1322 */
1323 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv)))
1324 return;
1325
1326 amdgpu_amdkfd_remove_all_eviction_fences(abo);
1327
1328 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
1329 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1330 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev)))
1331 goto out;
1332
1333 r = dma_resv_reserve_fences(&bo->base._resv, 1);
1334 if (r)
1335 goto out;
1336
1337 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
1338 abo, &bo->base._resv, &fence,
1339 false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
1340 if (WARN_ON(r))
1341 goto out;
1342
1343 amdgpu_vram_mgr_set_cleared(bo->resource);
1344 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL);
1345 dma_fence_put(fence);
1346
1347out:
1348 dma_resv_unlock(&bo->base._resv);
1349}
1350
1351/**
1352 * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1353 * @bo: pointer to a buffer object
1354 *
1355 * Notifies the driver we are taking a fault on this BO and have reserved it,
1356 * also performs bookkeeping.
1357 * TTM driver callback for dealing with vm faults.
1358 *
1359 * Returns:
1360 * 0 for success or a negative error code on failure.
1361 */
1362vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1363{
1364 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1365 struct ttm_operation_ctx ctx = { false, false };
1366 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1367 int r;
1368
1369 /* Remember that this BO was accessed by the CPU */
1370 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1371
1372 if (amdgpu_res_cpu_visible(adev, bo->resource))
1373 return 0;
1374
1375 /* Can't move a pinned BO to visible VRAM */
1376 if (abo->tbo.pin_count > 0)
1377 return VM_FAULT_SIGBUS;
1378
1379 /* hurrah the memory is not visible ! */
1380 atomic64_inc(&adev->num_vram_cpu_page_faults);
1381 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1382 AMDGPU_GEM_DOMAIN_GTT);
1383
1384 /* Avoid costly evictions; only set GTT as a busy placement */
1385 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
1386
1387 r = ttm_bo_validate(bo, &abo->placement, &ctx);
1388 if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1389 return VM_FAULT_NOPAGE;
1390 else if (unlikely(r))
1391 return VM_FAULT_SIGBUS;
1392
1393 /* this should never happen */
1394 if (bo->resource->mem_type == TTM_PL_VRAM &&
1395 !amdgpu_res_cpu_visible(adev, bo->resource))
1396 return VM_FAULT_SIGBUS;
1397
1398 ttm_bo_move_to_lru_tail_unlocked(bo);
1399 return 0;
1400}
1401
1402/**
1403 * amdgpu_bo_fence - add fence to buffer object
1404 *
1405 * @bo: buffer object in question
1406 * @fence: fence to add
1407 * @shared: true if fence should be added shared
1408 *
1409 */
1410void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1411 bool shared)
1412{
1413 struct dma_resv *resv = bo->tbo.base.resv;
1414 int r;
1415
1416 r = dma_resv_reserve_fences(resv, 1);
1417 if (r) {
1418 /* As last resort on OOM we block for the fence */
1419 dma_fence_wait(fence, false);
1420 return;
1421 }
1422
1423 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1424 DMA_RESV_USAGE_WRITE);
1425}
1426
1427/**
1428 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1429 *
1430 * @adev: amdgpu device pointer
1431 * @resv: reservation object to sync to
1432 * @sync_mode: synchronization mode
1433 * @owner: fence owner
1434 * @intr: Whether the wait is interruptible
1435 *
1436 * Extract the fences from the reservation object and waits for them to finish.
1437 *
1438 * Returns:
1439 * 0 on success, errno otherwise.
1440 */
1441int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1442 enum amdgpu_sync_mode sync_mode, void *owner,
1443 bool intr)
1444{
1445 struct amdgpu_sync sync;
1446 int r;
1447
1448 amdgpu_sync_create(&sync);
1449 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1450 r = amdgpu_sync_wait(&sync, intr);
1451 amdgpu_sync_free(&sync);
1452 return r;
1453}
1454
1455/**
1456 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1457 * @bo: buffer object to wait for
1458 * @owner: fence owner
1459 * @intr: Whether the wait is interruptible
1460 *
1461 * Wrapper to wait for fences in a BO.
1462 * Returns:
1463 * 0 on success, errno otherwise.
1464 */
1465int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1466{
1467 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1468
1469 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1470 AMDGPU_SYNC_NE_OWNER, owner, intr);
1471}
1472
1473/**
1474 * amdgpu_bo_gpu_offset - return GPU offset of bo
1475 * @bo: amdgpu object for which we query the offset
1476 *
1477 * Note: object should either be pinned or reserved when calling this
1478 * function, it might be useful to add check for this for debugging.
1479 *
1480 * Returns:
1481 * current GPU offset of the object.
1482 */
1483u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1484{
1485 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1486 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1487 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1488 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1489 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1490 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1491
1492 return amdgpu_bo_gpu_offset_no_check(bo);
1493}
1494
1495/**
1496 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo
1497 * @bo: amdgpu VRAM buffer object for which we query the offset
1498 *
1499 * Returns:
1500 * current FB aperture GPU offset of the object.
1501 */
1502u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo)
1503{
1504 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1505 uint64_t offset, fb_base;
1506
1507 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM);
1508
1509 fb_base = adev->gmc.fb_start;
1510 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1511 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base;
1512 return amdgpu_gmc_sign_extend(offset);
1513}
1514
1515/**
1516 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1517 * @bo: amdgpu object for which we query the offset
1518 *
1519 * Returns:
1520 * current GPU offset of the object without raising warnings.
1521 */
1522u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1523{
1524 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1525 uint64_t offset = AMDGPU_BO_INVALID_OFFSET;
1526
1527 if (bo->tbo.resource->mem_type == TTM_PL_TT)
1528 offset = amdgpu_gmc_agp_addr(&bo->tbo);
1529
1530 if (offset == AMDGPU_BO_INVALID_OFFSET)
1531 offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1532 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1533
1534 return amdgpu_gmc_sign_extend(offset);
1535}
1536
1537/**
1538 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting
1539 * @bo: the buffer object we should look at
1540 *
1541 * BO can have multiple preferred placements, to avoid double counting we want
1542 * to file it under a single placement for memory stats.
1543 * Luckily, if we take the highest set bit in preferred_domains the result is
1544 * quite sensible.
1545 *
1546 * Returns:
1547 * Which of the placements should the BO be accounted under.
1548 */
1549uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo)
1550{
1551 u32 domain;
1552
1553 /*
1554 * MMIO_REMAP is internal now, so it no longer maps from a userspace
1555 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual
1556 * TTM placement.
1557 */
1558 if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP)
1559 return AMDGPU_PL_MMIO_REMAP;
1560
1561 domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK;
1562 if (!domain)
1563 return TTM_PL_SYSTEM;
1564
1565 switch (rounddown_pow_of_two(domain)) {
1566 case AMDGPU_GEM_DOMAIN_CPU:
1567 return TTM_PL_SYSTEM;
1568 case AMDGPU_GEM_DOMAIN_GTT:
1569 return TTM_PL_TT;
1570 case AMDGPU_GEM_DOMAIN_VRAM:
1571 return TTM_PL_VRAM;
1572 case AMDGPU_GEM_DOMAIN_GDS:
1573 return AMDGPU_PL_GDS;
1574 case AMDGPU_GEM_DOMAIN_GWS:
1575 return AMDGPU_PL_GWS;
1576 case AMDGPU_GEM_DOMAIN_OA:
1577 return AMDGPU_PL_OA;
1578 case AMDGPU_GEM_DOMAIN_DOORBELL:
1579 return AMDGPU_PL_DOORBELL;
1580 default:
1581 return TTM_PL_SYSTEM;
1582 }
1583}
1584
1585/**
1586 * amdgpu_bo_get_preferred_domain - get preferred domain
1587 * @adev: amdgpu device object
1588 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1589 *
1590 * Returns:
1591 * Which of the allowed domains is preferred for allocating the BO.
1592 */
1593uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1594 uint32_t domain)
1595{
1596 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) &&
1597 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) {
1598 domain = AMDGPU_GEM_DOMAIN_VRAM;
1599 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1600 domain = AMDGPU_GEM_DOMAIN_GTT;
1601 }
1602 return domain;
1603}
1604
1605#if defined(CONFIG_DEBUG_FS)
1606#define amdgpu_bo_print_flag(m, bo, flag) \
1607 do { \
1608 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1609 seq_printf((m), " " #flag); \
1610 } \
1611 } while (0)
1612
1613/**
1614 * amdgpu_bo_print_info - print BO info in debugfs file
1615 *
1616 * @id: Index or Id of the BO
1617 * @bo: Requested BO for printing info
1618 * @m: debugfs file
1619 *
1620 * Print BO information in debugfs file
1621 *
1622 * Returns:
1623 * Size of the BO in bytes.
1624 */
1625u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1626{
1627 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1628 struct dma_buf_attachment *attachment;
1629 struct dma_buf *dma_buf;
1630 const char *placement;
1631 unsigned int pin_count;
1632 u64 size;
1633
1634 if (dma_resv_trylock(bo->tbo.base.resv)) {
1635 if (!bo->tbo.resource) {
1636 placement = "NONE";
1637 } else {
1638 switch (bo->tbo.resource->mem_type) {
1639 case TTM_PL_VRAM:
1640 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource))
1641 placement = "VRAM VISIBLE";
1642 else
1643 placement = "VRAM";
1644 break;
1645 case TTM_PL_TT:
1646 placement = "GTT";
1647 break;
1648 case AMDGPU_PL_GDS:
1649 placement = "GDS";
1650 break;
1651 case AMDGPU_PL_GWS:
1652 placement = "GWS";
1653 break;
1654 case AMDGPU_PL_OA:
1655 placement = "OA";
1656 break;
1657 case AMDGPU_PL_PREEMPT:
1658 placement = "PREEMPTIBLE";
1659 break;
1660 case AMDGPU_PL_DOORBELL:
1661 placement = "DOORBELL";
1662 break;
1663 case AMDGPU_PL_MMIO_REMAP:
1664 placement = "MMIO REMAP";
1665 break;
1666 case TTM_PL_SYSTEM:
1667 default:
1668 placement = "CPU";
1669 break;
1670 }
1671 }
1672 dma_resv_unlock(bo->tbo.base.resv);
1673 } else {
1674 placement = "UNKNOWN";
1675 }
1676
1677 size = amdgpu_bo_size(bo);
1678 seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1679 id, size, placement);
1680
1681 pin_count = READ_ONCE(bo->tbo.pin_count);
1682 if (pin_count)
1683 seq_printf(m, " pin count %d", pin_count);
1684
1685 dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1686 attachment = READ_ONCE(bo->tbo.base.import_attach);
1687
1688 if (attachment)
1689 seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino);
1690 else if (dma_buf)
1691 seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino);
1692
1693 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1694 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1695 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1696 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1697 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1698 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1699 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1700 /* Add the gem obj resv fence dump*/
1701 if (dma_resv_trylock(bo->tbo.base.resv)) {
1702 dma_resv_describe(bo->tbo.base.resv, m);
1703 dma_resv_unlock(bo->tbo.base.resv);
1704 }
1705 seq_puts(m, "\n");
1706
1707 return size;
1708}
1709#endif