| 1 | |
| 2 | * Copyright 2009 Jerome Glisse.
|
| 3 | * All Rights Reserved.
|
| 4 | *
|
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a
|
| 6 | * copy of this software and associated documentation files (the
|
| 7 | * "Software"), to deal in the Software without restriction, including
|
| 8 | * without limitation the rights to use, copy, modify, merge, publish,
|
| 9 | * distribute, sub license, and/or sell copies of the Software, and to
|
| 10 | * permit persons to whom the Software is furnished to do so, subject to
|
| 11 | * the following conditions:
|
| 12 | *
|
| 13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
| 16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
| 17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
| 18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
| 19 | * USE OR OTHER DEALINGS IN THE SOFTWARE.
|
| 20 | *
|
| 21 | * The above copyright notice and this permission notice (including the
|
| 22 | * next paragraph) shall be included in all copies or substantial portions
|
| 23 | * of the Software.
|
| 24 | *
|
| 25 |
|
| 26 | |
| 27 | * Authors:
|
| 28 | * Jerome Glisse <[email protected]>
|
| 29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
|
| 30 | * Dave Airlie
|
| 31 |
|
| 32 |
|
| 33 | #include <linux/dma-mapping.h>
|
| 34 | #include <linux/iommu.h>
|
| 35 | #include <linux/pagemap.h>
|
| 36 | #include <linux/sched/task.h>
|
| 37 | #include <linux/sched/mm.h>
|
| 38 | #include <linux/seq_file.h>
|
| 39 | #include <linux/slab.h>
|
| 40 | #include <linux/swap.h>
|
| 41 | #include <linux/dma-buf.h>
|
| 42 | #include <linux/sizes.h>
|
| 43 | #include <linux/module.h>
|
| 44 |
|
| 45 | #include <drm/drm_drv.h>
|
| 46 | #include <drm/ttm/ttm_bo.h>
|
| 47 | #include <drm/ttm/ttm_placement.h>
|
| 48 | #include <drm/ttm/ttm_range_manager.h>
|
| 49 | #include <drm/ttm/ttm_tt.h>
|
| 50 |
|
| 51 | #include <drm/amdgpu_drm.h>
|
| 52 |
|
| 53 | #include "amdgpu.h"
|
| 54 | #include "amdgpu_object.h"
|
| 55 | #include "amdgpu_trace.h"
|
| 56 | #include "amdgpu_amdkfd.h"
|
| 57 | #include "amdgpu_sdma.h"
|
| 58 | #include "amdgpu_ras.h"
|
| 59 | #include "amdgpu_hmm.h"
|
| 60 | #include "amdgpu_atomfirmware.h"
|
| 61 | #include "amdgpu_res_cursor.h"
|
| 62 | #include "bif/bif_4_1_d.h"
|
| 63 | #include "kfd_svm.h"
|
| 64 |
|
| 65 | MODULE_IMPORT_NS("DMA_BUF");
|
| 66 |
|
| 67 | #define AMDGPU_TTM_VRAM_MAX_DW_READ ((size_t)128)
|
| 68 |
|
| 69 | static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
|
| 70 | struct ttm_tt *ttm,
|
| 71 | struct ttm_resource *bo_mem);
|
| 72 | static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
|
| 73 | struct ttm_tt *ttm);
|
| 74 |
|
| 75 | static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
|
| 76 | unsigned int type,
|
| 77 | uint64_t size_in_page)
|
| 78 | {
|
| 79 | if (!size_in_page)
|
| 80 | return 0;
|
| 81 |
|
| 82 | return ttm_range_man_init(&adev->mman.bdev, type,
|
| 83 | false, size_in_page);
|
| 84 | }
|
| 85 |
|
| 86 | |
| 87 | * amdgpu_evict_flags - Compute placement flags
|
| 88 | *
|
| 89 | * @bo: The buffer object to evict
|
| 90 | * @placement: Possible destination(s) for evicted BO
|
| 91 | *
|
| 92 | * Fill in placement data when ttm_bo_evict() is called
|
| 93 |
|
| 94 | static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
|
| 95 | struct ttm_placement *placement)
|
| 96 | {
|
| 97 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 98 | struct amdgpu_bo *abo;
|
| 99 | static const struct ttm_place placements = {
|
| 100 | .fpfn = 0,
|
| 101 | .lpfn = 0,
|
| 102 | .mem_type = TTM_PL_SYSTEM,
|
| 103 | .flags = 0
|
| 104 | };
|
| 105 |
|
| 106 |
|
| 107 | if (bo->type == ttm_bo_type_sg) {
|
| 108 | placement->num_placement = 0;
|
| 109 | return;
|
| 110 | }
|
| 111 |
|
| 112 |
|
| 113 | if (!amdgpu_bo_is_amdgpu_bo(bo)) {
|
| 114 | placement->placement = &placements;
|
| 115 | placement->num_placement = 1;
|
| 116 | return;
|
| 117 | }
|
| 118 |
|
| 119 | abo = ttm_to_amdgpu_bo(bo);
|
| 120 | if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) {
|
| 121 | placement->num_placement = 0;
|
| 122 | return;
|
| 123 | }
|
| 124 |
|
| 125 | switch (bo->resource->mem_type) {
|
| 126 | case AMDGPU_PL_GDS:
|
| 127 | case AMDGPU_PL_GWS:
|
| 128 | case AMDGPU_PL_OA:
|
| 129 | case AMDGPU_PL_DOORBELL:
|
| 130 | case AMDGPU_PL_MMIO_REMAP:
|
| 131 | placement->num_placement = 0;
|
| 132 | return;
|
| 133 |
|
| 134 | case TTM_PL_VRAM:
|
| 135 | if (!adev->mman.buffer_funcs_enabled) {
|
| 136 |
|
| 137 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
|
| 138 |
|
| 139 | } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
|
| 140 | !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
|
| 141 | amdgpu_res_cpu_visible(adev, bo->resource)) {
|
| 142 |
|
| 143 | |
| 144 | * first, but only set GTT as busy placement, so this
|
| 145 | * BO will be evicted to GTT rather than causing other
|
| 146 | * BOs to be evicted from VRAM
|
| 147 |
|
| 148 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
|
| 149 | AMDGPU_GEM_DOMAIN_GTT |
|
| 150 | AMDGPU_GEM_DOMAIN_CPU);
|
| 151 | abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
|
| 152 | abo->placements[0].lpfn = 0;
|
| 153 | abo->placements[0].flags |= TTM_PL_FLAG_DESIRED;
|
| 154 | } else {
|
| 155 |
|
| 156 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT |
|
| 157 | AMDGPU_GEM_DOMAIN_CPU);
|
| 158 | }
|
| 159 | break;
|
| 160 | case TTM_PL_TT:
|
| 161 | case AMDGPU_PL_PREEMPT:
|
| 162 | default:
|
| 163 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
|
| 164 | break;
|
| 165 | }
|
| 166 | *placement = abo->placement;
|
| 167 | }
|
| 168 |
|
| 169 | static struct dma_fence *
|
| 170 | amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entity *entity,
|
| 171 | struct amdgpu_job *job, u32 num_dw)
|
| 172 | {
|
| 173 | struct amdgpu_ring *ring;
|
| 174 |
|
| 175 | ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
|
| 176 | amdgpu_ring_pad_ib(ring, &job->ibs[0]);
|
| 177 | WARN_ON(job->ibs[0].length_dw > num_dw);
|
| 178 |
|
| 179 | lockdep_assert_held(&entity->lock);
|
| 180 |
|
| 181 | return amdgpu_job_submit(job);
|
| 182 | }
|
| 183 |
|
| 184 | |
| 185 | * amdgpu_ttm_map_buffer - Map memory into the GART windows
|
| 186 | * @entity: entity to run the window setup job
|
| 187 | * @bo: buffer object to map
|
| 188 | * @mem: memory object to map
|
| 189 | * @mm_cur: range to map
|
| 190 | * @window: which GART window to use
|
| 191 | * @tmz: if we should setup a TMZ enabled mapping
|
| 192 | * @size: in number of bytes to map, out number of bytes mapped
|
| 193 | * @addr: resulting address inside the MC address space
|
| 194 | * @vm_needs_flush: out, set true if a GART window was programmed (VMID 0 flush
|
| 195 | * needed) or false for a direct address
|
| 196 | *
|
| 197 | * Setup one of the GART windows to access a specific piece of memory or return
|
| 198 | * the physical address for local memory.
|
| 199 |
|
| 200 | static int amdgpu_ttm_map_buffer(struct amdgpu_ttm_buffer_entity *entity,
|
| 201 | struct ttm_buffer_object *bo,
|
| 202 | struct ttm_resource *mem,
|
| 203 | struct amdgpu_res_cursor *mm_cur,
|
| 204 | unsigned int window,
|
| 205 | bool tmz, uint64_t *size, uint64_t *addr,
|
| 206 | bool *vm_needs_flush)
|
| 207 | {
|
| 208 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 209 | unsigned int offset, num_pages, num_dw, num_bytes;
|
| 210 | uint64_t src_addr, dst_addr;
|
| 211 | struct amdgpu_job *job;
|
| 212 | void *cpu_addr;
|
| 213 | uint64_t flags;
|
| 214 | int r;
|
| 215 | const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
|
| 216 |
|
| 217 | BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
|
| 218 | GTT_MAX_PAGES * AMDGPU_GPU_PAGES_IN_CPU_PAGE * 8);
|
| 219 |
|
| 220 | if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT))
|
| 221 | return -EINVAL;
|
| 222 |
|
| 223 |
|
| 224 | if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) {
|
| 225 | *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) +
|
| 226 | mm_cur->start;
|
| 227 | *vm_needs_flush = false;
|
| 228 | return 0;
|
| 229 | }
|
| 230 |
|
| 231 |
|
| 232 | *vm_needs_flush = true;
|
| 233 |
|
| 234 | |
| 235 | * If start begins at an offset inside the page, then adjust the size
|
| 236 | * and addr accordingly
|
| 237 |
|
| 238 | offset = mm_cur->start & ~PAGE_MASK;
|
| 239 |
|
| 240 | num_pages = PFN_UP(*size + offset);
|
| 241 | num_pages = min_t(uint32_t, num_pages, GTT_MAX_PAGES);
|
| 242 |
|
| 243 | *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset);
|
| 244 |
|
| 245 | *addr = amdgpu_compute_gart_address(&adev->gmc, entity, window);
|
| 246 | *addr += offset;
|
| 247 |
|
| 248 | num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
|
| 249 | num_bytes = num_pages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
|
| 250 |
|
| 251 | r = amdgpu_job_alloc_with_ib(adev, &entity->base,
|
| 252 | AMDGPU_FENCE_OWNER_UNDEFINED,
|
| 253 | num_dw * 4 + num_bytes,
|
| 254 | AMDGPU_IB_POOL_DELAYED,
|
| 255 | AMDGPU_KERNEL_JOB_ID_TTM_MAP_BUFFER,
|
| 256 | &job);
|
| 257 | if (r)
|
| 258 | return r;
|
| 259 |
|
| 260 | src_addr = num_dw * 4;
|
| 261 | src_addr += job->ibs[0].gpu_addr;
|
| 262 |
|
| 263 | dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
|
| 264 | dst_addr += (entity->gart_window_offs[window] >> AMDGPU_GPU_PAGE_SHIFT) * 8;
|
| 265 | amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
|
| 266 | dst_addr, num_bytes, 0);
|
| 267 |
|
| 268 | flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem);
|
| 269 | if (tmz)
|
| 270 | flags |= AMDGPU_PTE_TMZ;
|
| 271 |
|
| 272 | cpu_addr = &job->ibs[0].ptr[num_dw];
|
| 273 |
|
| 274 | if (mem->mem_type == TTM_PL_TT) {
|
| 275 | dma_addr_t *dma_addr;
|
| 276 |
|
| 277 | dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT];
|
| 278 | amdgpu_gart_map(adev, 0, num_pages, dma_addr, flags, cpu_addr);
|
| 279 | } else {
|
| 280 | u64 pa = mm_cur->start + adev->vm_manager.vram_base_offset;
|
| 281 |
|
| 282 | amdgpu_gart_map_vram_range(adev, pa, 0, num_pages, flags, cpu_addr);
|
| 283 | }
|
| 284 |
|
| 285 | dma_fence_put(amdgpu_ttm_job_submit(adev, entity, job, num_dw));
|
| 286 | return 0;
|
| 287 | }
|
| 288 |
|
| 289 | |
| 290 | * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
|
| 291 | * @adev: amdgpu device
|
| 292 | * @entity: entity to run the jobs
|
| 293 | * @src: buffer/address where to read from
|
| 294 | * @dst: buffer/address where to write to
|
| 295 | * @size: number of bytes to copy
|
| 296 | * @tmz: if a secure copy should be used
|
| 297 | * @resv: resv object to sync to
|
| 298 | * @f: Returns the last fence if multiple jobs are submitted.
|
| 299 | *
|
| 300 | * The function copies @size bytes from {src->mem + src->offset} to
|
| 301 | * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
|
| 302 | * move and different for a BO to BO copy.
|
| 303 | *
|
| 304 |
|
| 305 | __attribute__((nonnull))
|
| 306 | static int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
|
| 307 | struct amdgpu_ttm_buffer_entity *entity,
|
| 308 | const struct amdgpu_copy_mem *src,
|
| 309 | const struct amdgpu_copy_mem *dst,
|
| 310 | uint64_t size, bool tmz,
|
| 311 | struct dma_resv *resv,
|
| 312 | struct dma_fence **f)
|
| 313 | {
|
| 314 | struct amdgpu_res_cursor src_mm, dst_mm;
|
| 315 | struct dma_fence *fence = NULL;
|
| 316 | int r = 0;
|
| 317 | uint32_t copy_flags = 0;
|
| 318 | struct amdgpu_bo *abo_src, *abo_dst;
|
| 319 |
|
| 320 | if (!adev->mman.buffer_funcs_enabled) {
|
| 321 | dev_err(adev->dev,
|
| 322 | "Trying to move memory with ring turned off.\n");
|
| 323 | return -EINVAL;
|
| 324 | }
|
| 325 |
|
| 326 | amdgpu_res_first(src->mem, src->offset, size, &src_mm);
|
| 327 | amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm);
|
| 328 |
|
| 329 | mutex_lock(&entity->lock);
|
| 330 | while (src_mm.remaining) {
|
| 331 | uint64_t from, to, cur_size, tiling_flags;
|
| 332 | uint32_t num_type, data_format, max_com, write_compress_disable;
|
| 333 | bool src_vm_flush, dst_vm_flush;
|
| 334 | struct dma_fence *next;
|
| 335 |
|
| 336 |
|
| 337 | cur_size = min3(src_mm.size, dst_mm.size, 256ULL << 20);
|
| 338 |
|
| 339 |
|
| 340 | r = amdgpu_ttm_map_buffer(entity, src->bo, src->mem, &src_mm,
|
| 341 | 0, tmz, &cur_size, &from, &src_vm_flush);
|
| 342 | if (r)
|
| 343 | goto error;
|
| 344 |
|
| 345 | r = amdgpu_ttm_map_buffer(entity, dst->bo, dst->mem, &dst_mm,
|
| 346 | 1, tmz, &cur_size, &to, &dst_vm_flush);
|
| 347 | if (r)
|
| 348 | goto error;
|
| 349 |
|
| 350 | abo_src = ttm_to_amdgpu_bo(src->bo);
|
| 351 | abo_dst = ttm_to_amdgpu_bo(dst->bo);
|
| 352 | if (tmz)
|
| 353 | copy_flags |= AMDGPU_COPY_FLAGS_TMZ;
|
| 354 | if ((abo_src->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
|
| 355 | (abo_src->tbo.resource->mem_type == TTM_PL_VRAM))
|
| 356 | copy_flags |= AMDGPU_COPY_FLAGS_READ_DECOMPRESSED;
|
| 357 | if ((abo_dst->flags & AMDGPU_GEM_CREATE_GFX12_DCC) &&
|
| 358 | (dst->mem->mem_type == TTM_PL_VRAM)) {
|
| 359 | copy_flags |= AMDGPU_COPY_FLAGS_WRITE_COMPRESSED;
|
| 360 | amdgpu_bo_get_tiling_flags(abo_dst, &tiling_flags);
|
| 361 | max_com = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_MAX_COMPRESSED_BLOCK);
|
| 362 | num_type = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_NUMBER_TYPE);
|
| 363 | data_format = AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_DATA_FORMAT);
|
| 364 | write_compress_disable =
|
| 365 | AMDGPU_TILING_GET(tiling_flags, GFX12_DCC_WRITE_COMPRESS_DISABLE);
|
| 366 | copy_flags |= (AMDGPU_COPY_FLAGS_SET(MAX_COMPRESSED, max_com) |
|
| 367 | AMDGPU_COPY_FLAGS_SET(NUMBER_TYPE, num_type) |
|
| 368 | AMDGPU_COPY_FLAGS_SET(DATA_FORMAT, data_format) |
|
| 369 | AMDGPU_COPY_FLAGS_SET(WRITE_COMPRESS_DISABLE,
|
| 370 | write_compress_disable));
|
| 371 | }
|
| 372 |
|
| 373 | r = amdgpu_copy_buffer(adev, entity, from, to, cur_size, resv,
|
| 374 | &next, src_vm_flush || dst_vm_flush, copy_flags);
|
| 375 | if (r)
|
| 376 | goto error;
|
| 377 |
|
| 378 | dma_fence_put(fence);
|
| 379 | fence = next;
|
| 380 |
|
| 381 | amdgpu_res_next(&src_mm, cur_size);
|
| 382 | amdgpu_res_next(&dst_mm, cur_size);
|
| 383 | }
|
| 384 | error:
|
| 385 | mutex_unlock(&entity->lock);
|
| 386 | *f = fence;
|
| 387 | return r;
|
| 388 | }
|
| 389 |
|
| 390 | |
| 391 | * amdgpu_move_blit - Copy an entire buffer to another buffer
|
| 392 | *
|
| 393 | * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to
|
| 394 | * help move buffers to and from VRAM.
|
| 395 |
|
| 396 | static int amdgpu_move_blit(struct ttm_buffer_object *bo,
|
| 397 | bool evict,
|
| 398 | struct ttm_resource *new_mem,
|
| 399 | struct ttm_resource *old_mem)
|
| 400 | {
|
| 401 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 402 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
|
| 403 | struct amdgpu_ttm_buffer_entity *entity;
|
| 404 | struct amdgpu_copy_mem src, dst;
|
| 405 | struct dma_fence *fence = NULL;
|
| 406 | int r;
|
| 407 | u32 e;
|
| 408 |
|
| 409 | src.bo = bo;
|
| 410 | dst.bo = bo;
|
| 411 | src.mem = old_mem;
|
| 412 | dst.mem = new_mem;
|
| 413 | src.offset = 0;
|
| 414 | dst.offset = 0;
|
| 415 |
|
| 416 | e = atomic_inc_return(&adev->mman.next_move_entity) %
|
| 417 | adev->mman.num_move_entities;
|
| 418 | entity = &adev->mman.move_entities[e];
|
| 419 |
|
| 420 | r = amdgpu_ttm_copy_mem_to_mem(adev,
|
| 421 | entity,
|
| 422 | &src, &dst,
|
| 423 | new_mem->size,
|
| 424 | amdgpu_bo_encrypted(abo),
|
| 425 | bo->base.resv, &fence);
|
| 426 | if (r)
|
| 427 | goto error;
|
| 428 |
|
| 429 |
|
| 430 | if (old_mem->mem_type == TTM_PL_VRAM &&
|
| 431 | (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
|
| 432 | struct dma_fence *wipe_fence = NULL;
|
| 433 | r = amdgpu_ttm_clear_buffer(entity, abo, NULL, &wipe_fence,
|
| 434 | false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
|
| 435 | if (r) {
|
| 436 | goto error;
|
| 437 | } else if (wipe_fence) {
|
| 438 | amdgpu_vram_mgr_set_cleared(bo->resource);
|
| 439 | dma_fence_put(fence);
|
| 440 | fence = wipe_fence;
|
| 441 | }
|
| 442 | }
|
| 443 |
|
| 444 |
|
| 445 | if (bo->type == ttm_bo_type_kernel)
|
| 446 | r = ttm_bo_move_accel_cleanup(bo, fence, true, false, new_mem);
|
| 447 | else
|
| 448 | r = ttm_bo_move_accel_cleanup(bo, fence, evict, true, new_mem);
|
| 449 | dma_fence_put(fence);
|
| 450 | return r;
|
| 451 |
|
| 452 | error:
|
| 453 | if (fence)
|
| 454 | dma_fence_wait(fence, false);
|
| 455 | dma_fence_put(fence);
|
| 456 | return r;
|
| 457 | }
|
| 458 |
|
| 459 | |
| 460 | * amdgpu_res_cpu_visible - Check that resource can be accessed by CPU
|
| 461 | * @adev: amdgpu device
|
| 462 | * @res: the resource to check
|
| 463 | *
|
| 464 | * Returns: true if the full resource is CPU visible, false otherwise.
|
| 465 |
|
| 466 | bool amdgpu_res_cpu_visible(struct amdgpu_device *adev,
|
| 467 | struct ttm_resource *res)
|
| 468 | {
|
| 469 | struct amdgpu_res_cursor cursor;
|
| 470 |
|
| 471 | if (!res)
|
| 472 | return false;
|
| 473 |
|
| 474 | if (res->mem_type == TTM_PL_SYSTEM || res->mem_type == TTM_PL_TT ||
|
| 475 | res->mem_type == AMDGPU_PL_PREEMPT || res->mem_type == AMDGPU_PL_DOORBELL ||
|
| 476 | res->mem_type == AMDGPU_PL_MMIO_REMAP)
|
| 477 | return true;
|
| 478 |
|
| 479 | if (res->mem_type != TTM_PL_VRAM)
|
| 480 | return false;
|
| 481 |
|
| 482 | amdgpu_res_first(res, 0, res->size, &cursor);
|
| 483 | while (cursor.remaining) {
|
| 484 | if ((cursor.start + cursor.size) > adev->gmc.visible_vram_size)
|
| 485 | return false;
|
| 486 | amdgpu_res_next(&cursor, cursor.size);
|
| 487 | }
|
| 488 |
|
| 489 | return true;
|
| 490 | }
|
| 491 |
|
| 492 | |
| 493 | * amdgpu_res_copyable - Check that memory can be accessed by ttm_bo_move_memcpy
|
| 494 | *
|
| 495 | * Called by amdgpu_bo_move()
|
| 496 |
|
| 497 | static bool amdgpu_res_copyable(struct amdgpu_device *adev,
|
| 498 | struct ttm_resource *mem)
|
| 499 | {
|
| 500 | if (!amdgpu_res_cpu_visible(adev, mem))
|
| 501 | return false;
|
| 502 |
|
| 503 |
|
| 504 | if (mem->mem_type == TTM_PL_VRAM &&
|
| 505 | !(mem->placement & TTM_PL_FLAG_CONTIGUOUS))
|
| 506 | return false;
|
| 507 |
|
| 508 | return true;
|
| 509 | }
|
| 510 |
|
| 511 | |
| 512 | * amdgpu_bo_move - Move a buffer object to a new memory location
|
| 513 | *
|
| 514 | * Called by ttm_bo_handle_move_mem()
|
| 515 |
|
| 516 | static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
|
| 517 | struct ttm_operation_ctx *ctx,
|
| 518 | struct ttm_resource *new_mem,
|
| 519 | struct ttm_place *hop)
|
| 520 | {
|
| 521 | struct amdgpu_device *adev;
|
| 522 | struct amdgpu_bo *abo;
|
| 523 | struct ttm_resource *old_mem = bo->resource;
|
| 524 | int r;
|
| 525 |
|
| 526 | if (new_mem->mem_type == TTM_PL_TT ||
|
| 527 | new_mem->mem_type == AMDGPU_PL_PREEMPT) {
|
| 528 | if (old_mem && (old_mem->mem_type == TTM_PL_TT ||
|
| 529 | old_mem->mem_type == AMDGPU_PL_PREEMPT)) {
|
| 530 | r = ttm_bo_wait_ctx(bo, ctx);
|
| 531 | if (r)
|
| 532 | return r;
|
| 533 |
|
| 534 | amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
|
| 535 | }
|
| 536 |
|
| 537 | r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem);
|
| 538 | if (r)
|
| 539 | return r;
|
| 540 | }
|
| 541 |
|
| 542 | abo = ttm_to_amdgpu_bo(bo);
|
| 543 | adev = amdgpu_ttm_adev(bo->bdev);
|
| 544 |
|
| 545 | if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
|
| 546 | bo->ttm == NULL)) {
|
| 547 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 548 | ttm_bo_move_null(bo, new_mem);
|
| 549 | return 0;
|
| 550 | }
|
| 551 | if (old_mem->mem_type == TTM_PL_SYSTEM &&
|
| 552 | (new_mem->mem_type == TTM_PL_TT ||
|
| 553 | new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
|
| 554 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 555 | ttm_bo_move_null(bo, new_mem);
|
| 556 | return 0;
|
| 557 | }
|
| 558 | if ((old_mem->mem_type == TTM_PL_TT ||
|
| 559 | old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
|
| 560 | new_mem->mem_type == TTM_PL_SYSTEM) {
|
| 561 | r = ttm_bo_wait_ctx(bo, ctx);
|
| 562 | if (r)
|
| 563 | return r;
|
| 564 |
|
| 565 | amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm);
|
| 566 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 567 | ttm_resource_free(bo, &bo->resource);
|
| 568 | ttm_bo_assign_mem(bo, new_mem);
|
| 569 | return 0;
|
| 570 | }
|
| 571 | if ((old_mem->mem_type == TTM_PL_TT ||
|
| 572 | old_mem->mem_type == AMDGPU_PL_PREEMPT) &&
|
| 573 | (new_mem->mem_type == TTM_PL_TT ||
|
| 574 | new_mem->mem_type == AMDGPU_PL_PREEMPT)) {
|
| 575 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 576 | ttm_resource_free(bo, &bo->resource);
|
| 577 | ttm_bo_assign_mem(bo, new_mem);
|
| 578 | return 0;
|
| 579 | }
|
| 580 |
|
| 581 | if (old_mem->mem_type == AMDGPU_PL_GDS ||
|
| 582 | old_mem->mem_type == AMDGPU_PL_GWS ||
|
| 583 | old_mem->mem_type == AMDGPU_PL_OA ||
|
| 584 | old_mem->mem_type == AMDGPU_PL_DOORBELL ||
|
| 585 | old_mem->mem_type == AMDGPU_PL_MMIO_REMAP ||
|
| 586 | new_mem->mem_type == AMDGPU_PL_GDS ||
|
| 587 | new_mem->mem_type == AMDGPU_PL_GWS ||
|
| 588 | new_mem->mem_type == AMDGPU_PL_OA ||
|
| 589 | new_mem->mem_type == AMDGPU_PL_DOORBELL ||
|
| 590 | new_mem->mem_type == AMDGPU_PL_MMIO_REMAP) {
|
| 591 |
|
| 592 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 593 | ttm_bo_move_null(bo, new_mem);
|
| 594 | return 0;
|
| 595 | }
|
| 596 |
|
| 597 | if (bo->type == ttm_bo_type_device &&
|
| 598 | new_mem->mem_type == TTM_PL_VRAM &&
|
| 599 | old_mem->mem_type != TTM_PL_VRAM) {
|
| 600 | |
| 601 | * accesses the BO after it's moved.
|
| 602 |
|
| 603 | abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
|
| 604 | }
|
| 605 |
|
| 606 | if (adev->mman.buffer_funcs_enabled &&
|
| 607 | ((old_mem->mem_type == TTM_PL_SYSTEM &&
|
| 608 | new_mem->mem_type == TTM_PL_VRAM) ||
|
| 609 | (old_mem->mem_type == TTM_PL_VRAM &&
|
| 610 | new_mem->mem_type == TTM_PL_SYSTEM))) {
|
| 611 | hop->fpfn = 0;
|
| 612 | hop->lpfn = 0;
|
| 613 | hop->mem_type = TTM_PL_TT;
|
| 614 | hop->flags = TTM_PL_FLAG_TEMPORARY;
|
| 615 | return -EMULTIHOP;
|
| 616 | }
|
| 617 |
|
| 618 | amdgpu_bo_move_notify(bo, evict, new_mem);
|
| 619 | if (adev->mman.buffer_funcs_enabled)
|
| 620 | r = amdgpu_move_blit(bo, evict, new_mem, old_mem);
|
| 621 | else
|
| 622 | r = -ENODEV;
|
| 623 |
|
| 624 | if (r) {
|
| 625 |
|
| 626 | if (!amdgpu_res_copyable(adev, old_mem) ||
|
| 627 | !amdgpu_res_copyable(adev, new_mem)) {
|
| 628 | pr_err("Move buffer fallback to memcpy unavailable\n");
|
| 629 | return r;
|
| 630 | }
|
| 631 |
|
| 632 | r = ttm_bo_move_memcpy(bo, ctx, new_mem);
|
| 633 | if (r)
|
| 634 | return r;
|
| 635 | }
|
| 636 |
|
| 637 |
|
| 638 | if (evict)
|
| 639 | atomic64_inc(&adev->num_evictions);
|
| 640 | atomic64_add(bo->base.size, &adev->num_bytes_moved);
|
| 641 | return 0;
|
| 642 | }
|
| 643 |
|
| 644 | |
| 645 | * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
|
| 646 | *
|
| 647 | * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
|
| 648 |
|
| 649 | static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev,
|
| 650 | struct ttm_resource *mem)
|
| 651 | {
|
| 652 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 653 |
|
| 654 | switch (mem->mem_type) {
|
| 655 | case TTM_PL_SYSTEM:
|
| 656 |
|
| 657 | return 0;
|
| 658 | case TTM_PL_TT:
|
| 659 | case AMDGPU_PL_PREEMPT:
|
| 660 | break;
|
| 661 | case TTM_PL_VRAM:
|
| 662 | mem->bus.offset = mem->start << PAGE_SHIFT;
|
| 663 |
|
| 664 | if (adev->mman.aper_base_kaddr &&
|
| 665 | mem->placement & TTM_PL_FLAG_CONTIGUOUS)
|
| 666 | mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
|
| 667 | mem->bus.offset;
|
| 668 |
|
| 669 | mem->bus.offset += adev->gmc.aper_base;
|
| 670 | mem->bus.is_iomem = true;
|
| 671 | break;
|
| 672 | case AMDGPU_PL_DOORBELL:
|
| 673 | mem->bus.offset = mem->start << PAGE_SHIFT;
|
| 674 | mem->bus.offset += adev->doorbell.base;
|
| 675 | mem->bus.is_iomem = true;
|
| 676 | mem->bus.caching = ttm_uncached;
|
| 677 | break;
|
| 678 | case AMDGPU_PL_MMIO_REMAP:
|
| 679 | mem->bus.offset = mem->start << PAGE_SHIFT;
|
| 680 | mem->bus.offset += adev->rmmio_remap.bus_addr;
|
| 681 | mem->bus.is_iomem = true;
|
| 682 | mem->bus.caching = ttm_uncached;
|
| 683 | break;
|
| 684 | default:
|
| 685 | return -EINVAL;
|
| 686 | }
|
| 687 | return 0;
|
| 688 | }
|
| 689 |
|
| 690 | static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
|
| 691 | unsigned long page_offset)
|
| 692 | {
|
| 693 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 694 | struct amdgpu_res_cursor cursor;
|
| 695 |
|
| 696 | amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0,
|
| 697 | &cursor);
|
| 698 |
|
| 699 | if (bo->resource->mem_type == AMDGPU_PL_DOORBELL)
|
| 700 | return ((uint64_t)(adev->doorbell.base + cursor.start)) >> PAGE_SHIFT;
|
| 701 | else if (bo->resource->mem_type == AMDGPU_PL_MMIO_REMAP)
|
| 702 | return ((uint64_t)(adev->rmmio_remap.bus_addr + cursor.start)) >> PAGE_SHIFT;
|
| 703 |
|
| 704 | return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT;
|
| 705 | }
|
| 706 |
|
| 707 | |
| 708 | * amdgpu_ttm_domain_start - Returns GPU start address
|
| 709 | * @adev: amdgpu device object
|
| 710 | * @type: type of the memory
|
| 711 | *
|
| 712 | * Returns:
|
| 713 | * GPU start address of a memory domain
|
| 714 |
|
| 715 |
|
| 716 | uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type)
|
| 717 | {
|
| 718 | switch (type) {
|
| 719 | case TTM_PL_TT:
|
| 720 | return adev->gmc.gart_start;
|
| 721 | case TTM_PL_VRAM:
|
| 722 | return adev->gmc.vram_start;
|
| 723 | }
|
| 724 |
|
| 725 | return 0;
|
| 726 | }
|
| 727 |
|
| 728 | |
| 729 | * TTM backend functions.
|
| 730 |
|
| 731 | struct amdgpu_ttm_tt {
|
| 732 | struct ttm_tt ttm;
|
| 733 | struct drm_gem_object *gobj;
|
| 734 | u64 offset;
|
| 735 | uint64_t userptr;
|
| 736 | struct task_struct *usertask;
|
| 737 | uint32_t userflags;
|
| 738 | bool bound;
|
| 739 | int32_t pool_id;
|
| 740 | };
|
| 741 |
|
| 742 | #define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm)
|
| 743 |
|
| 744 | #ifdef CONFIG_DRM_AMDGPU_USERPTR
|
| 745 | |
| 746 | * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user
|
| 747 | * memory and start HMM tracking CPU page table update
|
| 748 | *
|
| 749 | * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
|
| 750 | * once afterwards to stop HMM tracking. Its the caller responsibility to ensure
|
| 751 | * that range is a valid memory and it is freed too.
|
| 752 |
|
| 753 | int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
|
| 754 | struct amdgpu_hmm_range *range)
|
| 755 | {
|
| 756 | struct ttm_tt *ttm = bo->tbo.ttm;
|
| 757 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 758 | unsigned long start = gtt->userptr;
|
| 759 | struct vm_area_struct *vma;
|
| 760 | struct mm_struct *mm;
|
| 761 | bool readonly;
|
| 762 | int r = 0;
|
| 763 |
|
| 764 | mm = bo->notifier.mm;
|
| 765 | if (unlikely(!mm)) {
|
| 766 | DRM_DEBUG_DRIVER("BO is not registered?\n");
|
| 767 | return -EFAULT;
|
| 768 | }
|
| 769 |
|
| 770 | if (!mmget_not_zero(mm))
|
| 771 | return -ESRCH;
|
| 772 |
|
| 773 | mmap_read_lock(mm);
|
| 774 | vma = vma_lookup(mm, start);
|
| 775 | if (unlikely(!vma)) {
|
| 776 | r = -EFAULT;
|
| 777 | goto out_unlock;
|
| 778 | }
|
| 779 | if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
|
| 780 | vma->vm_file)) {
|
| 781 | r = -EPERM;
|
| 782 | goto out_unlock;
|
| 783 | }
|
| 784 |
|
| 785 | readonly = amdgpu_ttm_tt_is_readonly(ttm);
|
| 786 | r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages,
|
| 787 | readonly, NULL, range);
|
| 788 | out_unlock:
|
| 789 | mmap_read_unlock(mm);
|
| 790 | if (r)
|
| 791 | pr_debug("failed %d to get user pages 0x%lx\n", r, start);
|
| 792 |
|
| 793 | mmput(mm);
|
| 794 |
|
| 795 | return r;
|
| 796 | }
|
| 797 |
|
| 798 | #endif
|
| 799 |
|
| 800 | |
| 801 | * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary.
|
| 802 | *
|
| 803 | * Called by amdgpu_cs_list_validate(). This creates the page list
|
| 804 | * that backs user memory and will ultimately be mapped into the device
|
| 805 | * address space.
|
| 806 |
|
| 807 | void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct amdgpu_hmm_range *range)
|
| 808 | {
|
| 809 | unsigned long i;
|
| 810 |
|
| 811 | for (i = 0; i < ttm->num_pages; ++i)
|
| 812 | ttm->pages[i] = range ? hmm_pfn_to_page(range->hmm_range.hmm_pfns[i]) : NULL;
|
| 813 | }
|
| 814 |
|
| 815 | |
| 816 | * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages
|
| 817 | *
|
| 818 | * Called by amdgpu_ttm_backend_bind()
|
| 819 | *
|
| 820 | static int amdgpu_ttm_tt_pin_userptr(struct ttm_device *bdev,
|
| 821 | struct ttm_tt *ttm)
|
| 822 | {
|
| 823 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 824 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 825 | int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
|
| 826 | enum dma_data_direction direction = write ?
|
| 827 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
|
| 828 | int r;
|
| 829 |
|
| 830 |
|
| 831 | r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
|
| 832 | (u64)ttm->num_pages << PAGE_SHIFT,
|
| 833 | GFP_KERNEL);
|
| 834 | if (r)
|
| 835 | goto release_sg;
|
| 836 |
|
| 837 |
|
| 838 | r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
|
| 839 | if (r)
|
| 840 | goto release_sg_table;
|
| 841 |
|
| 842 |
|
| 843 | drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
|
| 844 | ttm->num_pages);
|
| 845 |
|
| 846 | return 0;
|
| 847 |
|
| 848 | release_sg_table:
|
| 849 | sg_free_table(ttm->sg);
|
| 850 | release_sg:
|
| 851 | kfree(ttm->sg);
|
| 852 | ttm->sg = NULL;
|
| 853 | return r;
|
| 854 | }
|
| 855 |
|
| 856 | |
| 857 | * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
|
| 858 |
|
| 859 | static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device *bdev,
|
| 860 | struct ttm_tt *ttm)
|
| 861 | {
|
| 862 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 863 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 864 | int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
|
| 865 | enum dma_data_direction direction = write ?
|
| 866 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
|
| 867 |
|
| 868 |
|
| 869 | if (!ttm->sg || !ttm->sg->sgl)
|
| 870 | return;
|
| 871 |
|
| 872 |
|
| 873 | dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
|
| 874 | sg_free_table(ttm->sg);
|
| 875 | }
|
| 876 |
|
| 877 | |
| 878 | * total_pages is constructed as MQD0+CtrlStack0 + MQD1+CtrlStack1 + ...
|
| 879 | * MQDn+CtrlStackn where n is the number of XCCs per partition.
|
| 880 | * pages_per_xcc is the size of one MQD+CtrlStack. The first page is MQD
|
| 881 | * and uses memory type default, UC. The rest of pages_per_xcc are
|
| 882 | * Ctrl stack and modify their memory type to NC.
|
| 883 |
|
| 884 | static void amdgpu_ttm_gart_bind_gfx9_mqd(struct amdgpu_device *adev,
|
| 885 | struct ttm_tt *ttm, uint64_t flags)
|
| 886 | {
|
| 887 | struct amdgpu_ttm_tt *gtt = (void *)ttm;
|
| 888 | uint64_t total_pages = ttm->num_pages;
|
| 889 | int num_xcc = max(1U, adev->gfx.num_xcc_per_xcp);
|
| 890 | uint64_t page_idx, pages_per_xcc;
|
| 891 | int i;
|
| 892 |
|
| 893 | pages_per_xcc = total_pages;
|
| 894 | do_div(pages_per_xcc, num_xcc);
|
| 895 |
|
| 896 | for (i = 0, page_idx = 0; i < num_xcc; i++, page_idx += pages_per_xcc) {
|
| 897 | amdgpu_gart_map_gfx9_mqd(adev,
|
| 898 | gtt->offset + (page_idx << PAGE_SHIFT),
|
| 899 | pages_per_xcc, >t->ttm.dma_address[page_idx],
|
| 900 | flags);
|
| 901 | }
|
| 902 | }
|
| 903 |
|
| 904 | static void amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
|
| 905 | struct ttm_buffer_object *tbo,
|
| 906 | uint64_t flags)
|
| 907 | {
|
| 908 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
|
| 909 | struct ttm_tt *ttm = tbo->ttm;
|
| 910 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 911 |
|
| 912 | if (amdgpu_bo_encrypted(abo))
|
| 913 | flags |= AMDGPU_PTE_TMZ;
|
| 914 |
|
| 915 | if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) {
|
| 916 | amdgpu_ttm_gart_bind_gfx9_mqd(adev, ttm, flags);
|
| 917 | } else {
|
| 918 | amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
|
| 919 | gtt->ttm.dma_address, flags);
|
| 920 | }
|
| 921 | gtt->bound = true;
|
| 922 | }
|
| 923 |
|
| 924 | |
| 925 | * amdgpu_ttm_backend_bind - Bind GTT memory
|
| 926 | *
|
| 927 | * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
|
| 928 | * This handles binding GTT memory to the device address space.
|
| 929 |
|
| 930 | static int amdgpu_ttm_backend_bind(struct ttm_device *bdev,
|
| 931 | struct ttm_tt *ttm,
|
| 932 | struct ttm_resource *bo_mem)
|
| 933 | {
|
| 934 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 935 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 936 | uint64_t flags;
|
| 937 | int r;
|
| 938 |
|
| 939 | if (!bo_mem)
|
| 940 | return -EINVAL;
|
| 941 |
|
| 942 | if (gtt->bound)
|
| 943 | return 0;
|
| 944 |
|
| 945 | if (gtt->userptr) {
|
| 946 | r = amdgpu_ttm_tt_pin_userptr(bdev, ttm);
|
| 947 | if (r) {
|
| 948 | dev_err(adev->dev, "failed to pin userptr\n");
|
| 949 | return r;
|
| 950 | }
|
| 951 | } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) {
|
| 952 | if (!ttm->sg) {
|
| 953 | struct dma_buf_attachment *attach;
|
| 954 | struct sg_table *sgt;
|
| 955 |
|
| 956 | attach = gtt->gobj->import_attach;
|
| 957 | sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
|
| 958 | if (IS_ERR(sgt))
|
| 959 | return PTR_ERR(sgt);
|
| 960 |
|
| 961 | ttm->sg = sgt;
|
| 962 | }
|
| 963 |
|
| 964 | drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address,
|
| 965 | ttm->num_pages);
|
| 966 | }
|
| 967 |
|
| 968 | if (!ttm->num_pages) {
|
| 969 | WARN(1, "nothing to bind %u pages for mreg %p back %p!\n",
|
| 970 | ttm->num_pages, bo_mem, ttm);
|
| 971 | }
|
| 972 |
|
| 973 | if (bo_mem->mem_type != TTM_PL_TT ||
|
| 974 | !amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
|
| 975 | gtt->offset = AMDGPU_BO_INVALID_OFFSET;
|
| 976 | return 0;
|
| 977 | }
|
| 978 |
|
| 979 |
|
| 980 | flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
|
| 981 |
|
| 982 |
|
| 983 | gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
|
| 984 | amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
|
| 985 | gtt->ttm.dma_address, flags);
|
| 986 | gtt->bound = true;
|
| 987 | return 0;
|
| 988 | }
|
| 989 |
|
| 990 | |
| 991 | * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either
|
| 992 | * through AGP or GART aperture.
|
| 993 | *
|
| 994 | * If bo is accessible through AGP aperture, then use AGP aperture
|
| 995 | * to access bo; otherwise allocate logical space in GART aperture
|
| 996 | * and map bo to GART aperture.
|
| 997 |
|
| 998 | int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
|
| 999 | {
|
| 1000 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 1001 | struct ttm_operation_ctx ctx = { false, false };
|
| 1002 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
|
| 1003 | struct ttm_placement placement;
|
| 1004 | struct ttm_place placements;
|
| 1005 | struct ttm_resource *tmp;
|
| 1006 | uint64_t addr, flags;
|
| 1007 | int r;
|
| 1008 |
|
| 1009 | if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET)
|
| 1010 | return 0;
|
| 1011 |
|
| 1012 | addr = amdgpu_gmc_agp_addr(bo);
|
| 1013 | if (addr != AMDGPU_BO_INVALID_OFFSET)
|
| 1014 | return 0;
|
| 1015 |
|
| 1016 |
|
| 1017 | placement.num_placement = 1;
|
| 1018 | placement.placement = &placements;
|
| 1019 | placements.fpfn = 0;
|
| 1020 | placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
|
| 1021 | placements.mem_type = TTM_PL_TT;
|
| 1022 | placements.flags = bo->resource->placement;
|
| 1023 |
|
| 1024 | r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
|
| 1025 | if (unlikely(r))
|
| 1026 | return r;
|
| 1027 |
|
| 1028 |
|
| 1029 | flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp);
|
| 1030 |
|
| 1031 |
|
| 1032 | gtt->offset = (u64)tmp->start << PAGE_SHIFT;
|
| 1033 | amdgpu_ttm_gart_bind(adev, bo, flags);
|
| 1034 | amdgpu_gart_invalidate_tlb(adev);
|
| 1035 | ttm_resource_free(bo, &bo->resource);
|
| 1036 | ttm_bo_assign_mem(bo, tmp);
|
| 1037 |
|
| 1038 | return 0;
|
| 1039 | }
|
| 1040 |
|
| 1041 | |
| 1042 | * amdgpu_ttm_recover_gart - Rebind GTT pages
|
| 1043 | *
|
| 1044 | * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
|
| 1045 | * rebind GTT pages during a GPU reset.
|
| 1046 |
|
| 1047 | void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
|
| 1048 | {
|
| 1049 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
|
| 1050 | uint64_t flags;
|
| 1051 |
|
| 1052 | if (!tbo->ttm)
|
| 1053 | return;
|
| 1054 |
|
| 1055 | flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource);
|
| 1056 | amdgpu_ttm_gart_bind(adev, tbo, flags);
|
| 1057 | }
|
| 1058 |
|
| 1059 | |
| 1060 | * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
|
| 1061 | *
|
| 1062 | * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
|
| 1063 | * ttm_tt_destroy().
|
| 1064 |
|
| 1065 | static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev,
|
| 1066 | struct ttm_tt *ttm)
|
| 1067 | {
|
| 1068 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 1069 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1070 |
|
| 1071 |
|
| 1072 | if (gtt->userptr) {
|
| 1073 | amdgpu_ttm_tt_unpin_userptr(bdev, ttm);
|
| 1074 | } else if (ttm->sg && drm_gem_is_imported(gtt->gobj)) {
|
| 1075 | struct dma_buf_attachment *attach;
|
| 1076 |
|
| 1077 | attach = gtt->gobj->import_attach;
|
| 1078 | dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL);
|
| 1079 | ttm->sg = NULL;
|
| 1080 | }
|
| 1081 |
|
| 1082 | if (!gtt->bound)
|
| 1083 | return;
|
| 1084 |
|
| 1085 | if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
|
| 1086 | return;
|
| 1087 |
|
| 1088 |
|
| 1089 | amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
|
| 1090 | gtt->bound = false;
|
| 1091 | }
|
| 1092 |
|
| 1093 | static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev,
|
| 1094 | struct ttm_tt *ttm)
|
| 1095 | {
|
| 1096 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1097 |
|
| 1098 | if (gtt->usertask)
|
| 1099 | put_task_struct(gtt->usertask);
|
| 1100 |
|
| 1101 | ttm_tt_fini(>t->ttm);
|
| 1102 | kfree(gtt);
|
| 1103 | }
|
| 1104 |
|
| 1105 | |
| 1106 | * amdgpu_ttm_mmio_remap_alloc_sgt - build an sg_table for MMIO_REMAP I/O aperture
|
| 1107 | * @adev: amdgpu device providing the remap BAR base (adev->rmmio_remap.bus_addr)
|
| 1108 | * @res: TTM resource of the BO to export; expected to live in AMDGPU_PL_MMIO_REMAP
|
| 1109 | * @dev: importing device to map for (typically @attach->dev in dma-buf paths)
|
| 1110 | * @dir: DMA data direction for the importer (passed to dma_map_resource())
|
| 1111 | * @sgt: output; on success, set to a newly allocated sg_table describing the I/O span
|
| 1112 | *
|
| 1113 | * The HDP flush page (AMDGPU_PL_MMIO_REMAP) is a fixed hardware I/O window in a PCI
|
| 1114 | * BAR—there are no struct pages to back it. Importers still need a DMA address list,
|
| 1115 | * so we synthesize a minimal sg_table and populate it from dma_map_resource(), not
|
| 1116 | * from pages. Using the common amdgpu_res_cursor walker keeps the offset/size math
|
| 1117 | * consistent with other TTM/manager users.
|
| 1118 | *
|
| 1119 | * - @res is assumed to be a small, contiguous I/O region (typically a single 4 KiB
|
| 1120 | * page) in AMDGPU_PL_MMIO_REMAP. Callers should validate placement before calling.
|
| 1121 | * - The sg entry is created with sg_set_page(sg, NULL, …) to reflect I/O space.
|
| 1122 | * - The mapping uses DMA_ATTR_SKIP_CPU_SYNC because this is MMIO, not cacheable RAM.
|
| 1123 | * - Peer reachability / p2pdma policy checks must be done by the caller.
|
| 1124 | *
|
| 1125 | * Return:
|
| 1126 | * * 0 on success, with *@sgt set to a valid table that must be freed via
|
| 1127 | * amdgpu_ttm_mmio_remap_free_sgt().
|
| 1128 | * * -ENOMEM if allocation of the sg_table fails.
|
| 1129 | * * -EIO if dma_map_resource() fails.
|
| 1130 | *
|
| 1131 |
|
| 1132 | int amdgpu_ttm_mmio_remap_alloc_sgt(struct amdgpu_device *adev,
|
| 1133 | struct ttm_resource *res,
|
| 1134 | struct device *dev,
|
| 1135 | enum dma_data_direction dir,
|
| 1136 | struct sg_table **sgt)
|
| 1137 | {
|
| 1138 | struct amdgpu_res_cursor cur;
|
| 1139 | dma_addr_t dma;
|
| 1140 | resource_size_t phys;
|
| 1141 | struct scatterlist *sg;
|
| 1142 | int r;
|
| 1143 |
|
| 1144 |
|
| 1145 | amdgpu_res_first(res, 0, res->size, &cur);
|
| 1146 |
|
| 1147 |
|
| 1148 | phys = adev->rmmio_remap.bus_addr + cur.start;
|
| 1149 |
|
| 1150 |
|
| 1151 | *sgt = kzalloc_obj(**sgt);
|
| 1152 | if (!*sgt)
|
| 1153 | return -ENOMEM;
|
| 1154 | r = sg_alloc_table(*sgt, 1, GFP_KERNEL);
|
| 1155 | if (r) {
|
| 1156 | kfree(*sgt);
|
| 1157 | return r;
|
| 1158 | }
|
| 1159 | sg = (*sgt)->sgl;
|
| 1160 | sg_set_page(sg, NULL, cur.size, 0);
|
| 1161 |
|
| 1162 | dma = dma_map_resource(dev, phys, cur.size, dir, DMA_ATTR_SKIP_CPU_SYNC);
|
| 1163 | if (dma_mapping_error(dev, dma)) {
|
| 1164 | sg_free_table(*sgt);
|
| 1165 | kfree(*sgt);
|
| 1166 | return -EIO;
|
| 1167 | }
|
| 1168 | sg_dma_address(sg) = dma;
|
| 1169 | sg_dma_len(sg) = cur.size;
|
| 1170 | return 0;
|
| 1171 | }
|
| 1172 |
|
| 1173 | void amdgpu_ttm_mmio_remap_free_sgt(struct device *dev,
|
| 1174 | enum dma_data_direction dir,
|
| 1175 | struct sg_table *sgt)
|
| 1176 | {
|
| 1177 | struct scatterlist *sg = sgt->sgl;
|
| 1178 |
|
| 1179 | dma_unmap_resource(dev, sg_dma_address(sg), sg_dma_len(sg),
|
| 1180 | dir, DMA_ATTR_SKIP_CPU_SYNC);
|
| 1181 | sg_free_table(sgt);
|
| 1182 | kfree(sgt);
|
| 1183 | }
|
| 1184 |
|
| 1185 | |
| 1186 | * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
|
| 1187 | *
|
| 1188 | * @bo: The buffer object to create a GTT ttm_tt object around
|
| 1189 | * @page_flags: Page flags to be added to the ttm_tt object
|
| 1190 | *
|
| 1191 | * Called by ttm_tt_create().
|
| 1192 |
|
| 1193 | static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
|
| 1194 | uint32_t page_flags)
|
| 1195 | {
|
| 1196 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
|
| 1197 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
|
| 1198 | struct amdgpu_ttm_tt *gtt;
|
| 1199 | enum ttm_caching caching;
|
| 1200 |
|
| 1201 | gtt = kzalloc_obj(struct amdgpu_ttm_tt);
|
| 1202 | if (!gtt)
|
| 1203 | return NULL;
|
| 1204 |
|
| 1205 | gtt->gobj = &bo->base;
|
| 1206 | if (adev->gmc.mem_partitions && abo->xcp_id >= 0)
|
| 1207 | gtt->pool_id = KFD_XCP_MEM_ID(adev, abo->xcp_id);
|
| 1208 | else
|
| 1209 | gtt->pool_id = abo->xcp_id;
|
| 1210 |
|
| 1211 | if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
|
| 1212 | caching = ttm_write_combined;
|
| 1213 | else
|
| 1214 | caching = ttm_cached;
|
| 1215 |
|
| 1216 |
|
| 1217 | if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) {
|
| 1218 | kfree(gtt);
|
| 1219 | return NULL;
|
| 1220 | }
|
| 1221 | return >t->ttm;
|
| 1222 | }
|
| 1223 |
|
| 1224 | |
| 1225 | * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
|
| 1226 | *
|
| 1227 | * Map the pages of a ttm_tt object to an address space visible
|
| 1228 | * to the underlying device.
|
| 1229 |
|
| 1230 | static int amdgpu_ttm_tt_populate(struct ttm_device *bdev,
|
| 1231 | struct ttm_tt *ttm,
|
| 1232 | struct ttm_operation_ctx *ctx)
|
| 1233 | {
|
| 1234 | struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
|
| 1235 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1236 | struct ttm_pool *pool;
|
| 1237 | pgoff_t i;
|
| 1238 | int ret;
|
| 1239 |
|
| 1240 |
|
| 1241 | if (gtt->userptr) {
|
| 1242 | ttm->sg = kzalloc_obj(struct sg_table);
|
| 1243 | if (!ttm->sg)
|
| 1244 | return -ENOMEM;
|
| 1245 | return 0;
|
| 1246 | }
|
| 1247 |
|
| 1248 | if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
|
| 1249 | return 0;
|
| 1250 |
|
| 1251 | if (adev->mman.ttm_pools && gtt->pool_id >= 0)
|
| 1252 | pool = &adev->mman.ttm_pools[gtt->pool_id];
|
| 1253 | else
|
| 1254 | pool = &adev->mman.bdev.pool;
|
| 1255 | ret = ttm_pool_alloc(pool, ttm, ctx);
|
| 1256 | if (ret)
|
| 1257 | return ret;
|
| 1258 |
|
| 1259 | for (i = 0; i < ttm->num_pages; ++i)
|
| 1260 | ttm->pages[i]->mapping = bdev->dev_mapping;
|
| 1261 |
|
| 1262 | return 0;
|
| 1263 | }
|
| 1264 |
|
| 1265 | |
| 1266 | * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
|
| 1267 | *
|
| 1268 | * Unmaps pages of a ttm_tt object from the device address space and
|
| 1269 | * unpopulates the page array backing it.
|
| 1270 |
|
| 1271 | static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev,
|
| 1272 | struct ttm_tt *ttm)
|
| 1273 | {
|
| 1274 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1275 | struct amdgpu_device *adev;
|
| 1276 | struct ttm_pool *pool;
|
| 1277 | pgoff_t i;
|
| 1278 |
|
| 1279 | amdgpu_ttm_backend_unbind(bdev, ttm);
|
| 1280 |
|
| 1281 | if (gtt->userptr) {
|
| 1282 | amdgpu_ttm_tt_set_user_pages(ttm, NULL);
|
| 1283 | kfree(ttm->sg);
|
| 1284 | ttm->sg = NULL;
|
| 1285 | return;
|
| 1286 | }
|
| 1287 |
|
| 1288 | if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL)
|
| 1289 | return;
|
| 1290 |
|
| 1291 | for (i = 0; i < ttm->num_pages; ++i)
|
| 1292 | ttm->pages[i]->mapping = NULL;
|
| 1293 |
|
| 1294 | adev = amdgpu_ttm_adev(bdev);
|
| 1295 |
|
| 1296 | if (adev->mman.ttm_pools && gtt->pool_id >= 0)
|
| 1297 | pool = &adev->mman.ttm_pools[gtt->pool_id];
|
| 1298 | else
|
| 1299 | pool = &adev->mman.bdev.pool;
|
| 1300 |
|
| 1301 | return ttm_pool_free(pool, ttm);
|
| 1302 | }
|
| 1303 |
|
| 1304 | |
| 1305 | * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current
|
| 1306 | * task
|
| 1307 | *
|
| 1308 | * @tbo: The ttm_buffer_object that contains the userptr
|
| 1309 | * @user_addr: The returned value
|
| 1310 |
|
| 1311 | int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
|
| 1312 | uint64_t *user_addr)
|
| 1313 | {
|
| 1314 | struct amdgpu_ttm_tt *gtt;
|
| 1315 |
|
| 1316 | if (!tbo->ttm)
|
| 1317 | return -EINVAL;
|
| 1318 |
|
| 1319 | gtt = (void *)tbo->ttm;
|
| 1320 | *user_addr = gtt->userptr;
|
| 1321 | return 0;
|
| 1322 | }
|
| 1323 |
|
| 1324 | |
| 1325 | * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
|
| 1326 | * task
|
| 1327 | *
|
| 1328 | * @bo: The ttm_buffer_object to bind this userptr to
|
| 1329 | * @addr: The address in the current tasks VM space to use
|
| 1330 | * @flags: Requirements of userptr object.
|
| 1331 | *
|
| 1332 | * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to
|
| 1333 | * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to
|
| 1334 | * initialize GPU VM for a KFD process.
|
| 1335 |
|
| 1336 | int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
|
| 1337 | uint64_t addr, uint32_t flags)
|
| 1338 | {
|
| 1339 | struct amdgpu_ttm_tt *gtt;
|
| 1340 |
|
| 1341 | if (!bo->ttm) {
|
| 1342 |
|
| 1343 | bo->ttm = amdgpu_ttm_tt_create(bo, 0);
|
| 1344 | if (bo->ttm == NULL)
|
| 1345 | return -ENOMEM;
|
| 1346 | }
|
| 1347 |
|
| 1348 |
|
| 1349 | bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL;
|
| 1350 |
|
| 1351 | gtt = ttm_to_amdgpu_ttm_tt(bo->ttm);
|
| 1352 | gtt->userptr = addr;
|
| 1353 | gtt->userflags = flags;
|
| 1354 |
|
| 1355 | if (gtt->usertask)
|
| 1356 | put_task_struct(gtt->usertask);
|
| 1357 | gtt->usertask = current->group_leader;
|
| 1358 | get_task_struct(gtt->usertask);
|
| 1359 |
|
| 1360 | return 0;
|
| 1361 | }
|
| 1362 |
|
| 1363 | |
| 1364 | * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
|
| 1365 |
|
| 1366 | struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
|
| 1367 | {
|
| 1368 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1369 |
|
| 1370 | if (gtt == NULL)
|
| 1371 | return NULL;
|
| 1372 |
|
| 1373 | if (gtt->usertask == NULL)
|
| 1374 | return NULL;
|
| 1375 |
|
| 1376 | return gtt->usertask->mm;
|
| 1377 | }
|
| 1378 |
|
| 1379 | |
| 1380 | * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an
|
| 1381 | * address range for the current task.
|
| 1382 | *
|
| 1383 |
|
| 1384 | bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
|
| 1385 | unsigned long end, unsigned long *userptr)
|
| 1386 | {
|
| 1387 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1388 | unsigned long size;
|
| 1389 |
|
| 1390 | if (gtt == NULL || !gtt->userptr)
|
| 1391 | return false;
|
| 1392 |
|
| 1393 | |
| 1394 | * the range
|
| 1395 |
|
| 1396 | size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE;
|
| 1397 | if (gtt->userptr > end || gtt->userptr + size <= start)
|
| 1398 | return false;
|
| 1399 |
|
| 1400 | if (userptr)
|
| 1401 | *userptr = gtt->userptr;
|
| 1402 | return true;
|
| 1403 | }
|
| 1404 |
|
| 1405 | |
| 1406 | * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr?
|
| 1407 |
|
| 1408 | bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm)
|
| 1409 | {
|
| 1410 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1411 |
|
| 1412 | if (gtt == NULL || !gtt->userptr)
|
| 1413 | return false;
|
| 1414 |
|
| 1415 | return true;
|
| 1416 | }
|
| 1417 |
|
| 1418 | |
| 1419 | * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
|
| 1420 |
|
| 1421 | bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
|
| 1422 | {
|
| 1423 | struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm);
|
| 1424 |
|
| 1425 | if (gtt == NULL)
|
| 1426 | return false;
|
| 1427 |
|
| 1428 | return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
|
| 1429 | }
|
| 1430 |
|
| 1431 | |
| 1432 | * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object
|
| 1433 | *
|
| 1434 | * @ttm: The ttm_tt object to compute the flags for
|
| 1435 | * @mem: The memory registry backing this ttm_tt object
|
| 1436 | *
|
| 1437 | * Figure out the flags to use for a VM PDE (Page Directory Entry).
|
| 1438 |
|
| 1439 | uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)
|
| 1440 | {
|
| 1441 | uint64_t flags = 0;
|
| 1442 |
|
| 1443 | if (mem && mem->mem_type != TTM_PL_SYSTEM)
|
| 1444 | flags |= AMDGPU_PTE_VALID;
|
| 1445 |
|
| 1446 | if (mem && (mem->mem_type == TTM_PL_TT ||
|
| 1447 | mem->mem_type == AMDGPU_PL_DOORBELL ||
|
| 1448 | mem->mem_type == AMDGPU_PL_PREEMPT ||
|
| 1449 | mem->mem_type == AMDGPU_PL_MMIO_REMAP)) {
|
| 1450 | flags |= AMDGPU_PTE_SYSTEM;
|
| 1451 |
|
| 1452 | if (ttm && ttm->caching == ttm_cached)
|
| 1453 | flags |= AMDGPU_PTE_SNOOPED;
|
| 1454 | }
|
| 1455 |
|
| 1456 | if (mem && mem->mem_type == TTM_PL_VRAM &&
|
| 1457 | mem->bus.caching == ttm_cached)
|
| 1458 | flags |= AMDGPU_PTE_SNOOPED;
|
| 1459 |
|
| 1460 | return flags;
|
| 1461 | }
|
| 1462 |
|
| 1463 | |
| 1464 | * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
|
| 1465 | *
|
| 1466 | * @adev: amdgpu_device pointer
|
| 1467 | * @ttm: The ttm_tt object to compute the flags for
|
| 1468 | * @mem: The memory registry backing this ttm_tt object
|
| 1469 | *
|
| 1470 | * Figure out the flags to use for a VM PTE (Page Table Entry).
|
| 1471 |
|
| 1472 | uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
|
| 1473 | struct ttm_resource *mem)
|
| 1474 | {
|
| 1475 | uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
|
| 1476 |
|
| 1477 | flags |= adev->gart.gart_pte_flags;
|
| 1478 | flags |= AMDGPU_PTE_READABLE;
|
| 1479 |
|
| 1480 | if (!amdgpu_ttm_tt_is_readonly(ttm))
|
| 1481 | flags |= AMDGPU_PTE_WRITEABLE;
|
| 1482 |
|
| 1483 | return flags;
|
| 1484 | }
|
| 1485 |
|
| 1486 | |
| 1487 | * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer
|
| 1488 | * object.
|
| 1489 | *
|
| 1490 | * Return true if eviction is sensible. Called by ttm_mem_evict_first() on
|
| 1491 | * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until
|
| 1492 | * it can find space for a new object and by ttm_bo_force_list_clean() which is
|
| 1493 | * used to clean out a memory space.
|
| 1494 |
|
| 1495 | static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
|
| 1496 | const struct ttm_place *place)
|
| 1497 | {
|
| 1498 | struct dma_resv_iter resv_cursor;
|
| 1499 | struct amdgpu_bo *abo;
|
| 1500 | struct dma_fence *f;
|
| 1501 |
|
| 1502 | if (!amdgpu_bo_is_amdgpu_bo(bo))
|
| 1503 | return ttm_bo_eviction_valuable(bo, place);
|
| 1504 |
|
| 1505 |
|
| 1506 | if (bo->resource->mem_type == TTM_PL_SYSTEM)
|
| 1507 | return true;
|
| 1508 |
|
| 1509 | abo = ttm_to_amdgpu_bo(bo);
|
| 1510 | if ((abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
|
| 1511 | bo->destroy == &svm_range_bo_destroy) {
|
| 1512 | |
| 1513 | * SVM BOs are migrated to system memory synchronously in this
|
| 1514 | * TTM eviction context. The migration needs the owning
|
| 1515 | * process's mmap lock, but the normal lock order is
|
| 1516 | * mmap_lock -> BO reservation and the BO is already reserved
|
| 1517 | * here. svm_range_evict_svm_bo() only trylocks the mmap lock;
|
| 1518 | * if the eviction fails for any reason, we return false so TTM
|
| 1519 | * skips this BO instead of risking a deadlock.
|
| 1520 |
|
| 1521 | if (svm_range_evict_svm_bo(abo) < 0)
|
| 1522 | return false;
|
| 1523 | }
|
| 1524 |
|
| 1525 | if (bo->type == ttm_bo_type_kernel &&
|
| 1526 | !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo)))
|
| 1527 | return false;
|
| 1528 |
|
| 1529 | |
| 1530 | * If true, then return false as any KFD process needs all its BOs to
|
| 1531 | * be resident to run successfully
|
| 1532 |
|
| 1533 | dma_resv_for_each_fence(&resv_cursor, bo->base.resv,
|
| 1534 | DMA_RESV_USAGE_BOOKKEEP, f) {
|
| 1535 | if (amdkfd_fence_check_mm(f, current->mm) &&
|
| 1536 | !(place && (place->flags & TTM_PL_FLAG_CONTIGUOUS)))
|
| 1537 | return false;
|
| 1538 | }
|
| 1539 |
|
| 1540 | |
| 1541 | * driver (pages, VRAM, GART space). They point to resources
|
| 1542 | * owned by someone else (e.g. pageable memory in user mode
|
| 1543 | * or a DMABuf). They are used in a preemptible context so we
|
| 1544 | * can guarantee no deadlocks and good QoS in case of MMU
|
| 1545 | * notifiers or DMABuf move notifiers from the resource owner.
|
| 1546 |
|
| 1547 | if (bo->resource->mem_type == AMDGPU_PL_PREEMPT)
|
| 1548 | return false;
|
| 1549 |
|
| 1550 | if (bo->resource->mem_type == TTM_PL_TT &&
|
| 1551 | amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo)))
|
| 1552 | return false;
|
| 1553 |
|
| 1554 | return ttm_bo_eviction_valuable(bo, place);
|
| 1555 | }
|
| 1556 |
|
| 1557 | static void amdgpu_ttm_vram_mm_access(struct amdgpu_device *adev, loff_t pos,
|
| 1558 | void *buf, size_t size, bool write)
|
| 1559 | {
|
| 1560 | while (size) {
|
| 1561 | uint64_t aligned_pos = ALIGN_DOWN(pos, 4);
|
| 1562 | uint64_t bytes = 4 - (pos & 0x3);
|
| 1563 | uint32_t shift = (pos & 0x3) * 8;
|
| 1564 | uint32_t mask = 0xffffffff << shift;
|
| 1565 | uint32_t value = 0;
|
| 1566 |
|
| 1567 | if (size < bytes) {
|
| 1568 | mask &= 0xffffffff >> (bytes - size) * 8;
|
| 1569 | bytes = size;
|
| 1570 | }
|
| 1571 |
|
| 1572 | if (mask != 0xffffffff) {
|
| 1573 | amdgpu_device_mm_access(adev, aligned_pos, &value, 4, false);
|
| 1574 | if (write) {
|
| 1575 | value &= ~mask;
|
| 1576 | value |= (*(uint32_t *)buf << shift) & mask;
|
| 1577 | amdgpu_device_mm_access(adev, aligned_pos, &value, 4, true);
|
| 1578 | } else {
|
| 1579 | value = (value & mask) >> shift;
|
| 1580 | memcpy(buf, &value, bytes);
|
| 1581 | }
|
| 1582 | } else {
|
| 1583 | amdgpu_device_mm_access(adev, aligned_pos, buf, 4, write);
|
| 1584 | }
|
| 1585 |
|
| 1586 | pos += bytes;
|
| 1587 | buf += bytes;
|
| 1588 | size -= bytes;
|
| 1589 | }
|
| 1590 | }
|
| 1591 |
|
| 1592 | static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
|
| 1593 | unsigned long offset, void *buf,
|
| 1594 | int len, int write)
|
| 1595 | {
|
| 1596 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
|
| 1597 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
|
| 1598 | struct amdgpu_res_cursor src_mm;
|
| 1599 | struct amdgpu_job *job;
|
| 1600 | struct dma_fence *fence;
|
| 1601 | uint64_t src_addr, dst_addr;
|
| 1602 | unsigned int num_dw;
|
| 1603 | int r, idx;
|
| 1604 |
|
| 1605 | if (len != PAGE_SIZE)
|
| 1606 | return -EINVAL;
|
| 1607 |
|
| 1608 | if (!adev->mman.sdma_access_ptr)
|
| 1609 | return -EACCES;
|
| 1610 |
|
| 1611 | if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx))
|
| 1612 | return -ENODEV;
|
| 1613 |
|
| 1614 | if (write)
|
| 1615 | memcpy(adev->mman.sdma_access_ptr, buf, len);
|
| 1616 |
|
| 1617 | num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
|
| 1618 | r = amdgpu_job_alloc_with_ib(adev, &adev->mman.default_entity.base,
|
| 1619 | AMDGPU_FENCE_OWNER_UNDEFINED,
|
| 1620 | num_dw * 4, AMDGPU_IB_POOL_DELAYED,
|
| 1621 | AMDGPU_KERNEL_JOB_ID_TTM_ACCESS_MEMORY_SDMA,
|
| 1622 | &job);
|
| 1623 | if (r)
|
| 1624 | goto out;
|
| 1625 |
|
| 1626 | mutex_lock(&adev->mman.default_entity.lock);
|
| 1627 | amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm);
|
| 1628 | src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) +
|
| 1629 | src_mm.start;
|
| 1630 | dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo);
|
| 1631 | if (write)
|
| 1632 | swap(src_addr, dst_addr);
|
| 1633 |
|
| 1634 | amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr,
|
| 1635 | PAGE_SIZE, 0);
|
| 1636 |
|
| 1637 | fence = amdgpu_ttm_job_submit(adev, &adev->mman.default_entity, job, num_dw);
|
| 1638 | mutex_unlock(&adev->mman.default_entity.lock);
|
| 1639 |
|
| 1640 | if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout))
|
| 1641 | r = -ETIMEDOUT;
|
| 1642 | dma_fence_put(fence);
|
| 1643 |
|
| 1644 | if (!(r || write))
|
| 1645 | memcpy(buf, adev->mman.sdma_access_ptr, len);
|
| 1646 | out:
|
| 1647 | drm_dev_exit(idx);
|
| 1648 | return r;
|
| 1649 | }
|
| 1650 |
|
| 1651 | |
| 1652 | * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object.
|
| 1653 | *
|
| 1654 | * @bo: The buffer object to read/write
|
| 1655 | * @offset: Offset into buffer object
|
| 1656 | * @buf: Secondary buffer to write/read from
|
| 1657 | * @len: Length in bytes of access
|
| 1658 | * @write: true if writing
|
| 1659 | *
|
| 1660 | * This is used to access VRAM that backs a buffer object via MMIO
|
| 1661 | * access for debugging purposes.
|
| 1662 |
|
| 1663 | static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
|
| 1664 | unsigned long offset, void *buf, int len,
|
| 1665 | int write)
|
| 1666 | {
|
| 1667 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
|
| 1668 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
|
| 1669 | struct amdgpu_res_cursor cursor;
|
| 1670 | int ret = 0;
|
| 1671 |
|
| 1672 | if (bo->resource->mem_type != TTM_PL_VRAM)
|
| 1673 | return -EIO;
|
| 1674 |
|
| 1675 | if (amdgpu_device_has_timeouts_enabled(adev) &&
|
| 1676 | !amdgpu_ttm_access_memory_sdma(bo, offset, buf, len, write))
|
| 1677 | return len;
|
| 1678 |
|
| 1679 | amdgpu_res_first(bo->resource, offset, len, &cursor);
|
| 1680 | while (cursor.remaining) {
|
| 1681 | size_t count, size = cursor.size;
|
| 1682 | loff_t pos = cursor.start;
|
| 1683 |
|
| 1684 | count = amdgpu_device_aper_access(adev, pos, buf, size, write);
|
| 1685 | size -= count;
|
| 1686 | if (size) {
|
| 1687 |
|
| 1688 | pos += count;
|
| 1689 | buf += count;
|
| 1690 | amdgpu_ttm_vram_mm_access(adev, pos, buf, size, write);
|
| 1691 | }
|
| 1692 |
|
| 1693 | ret += cursor.size;
|
| 1694 | buf += cursor.size;
|
| 1695 | amdgpu_res_next(&cursor, cursor.size);
|
| 1696 | }
|
| 1697 |
|
| 1698 | return ret;
|
| 1699 | }
|
| 1700 |
|
| 1701 | static void
|
| 1702 | amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo)
|
| 1703 | {
|
| 1704 | if (bo->resource && bo->resource->mem_type == TTM_PL_TT)
|
| 1705 | amdgpu_gtt_mgr_mark_bo_teardown(bo);
|
| 1706 |
|
| 1707 | amdgpu_bo_move_notify(bo, false, NULL);
|
| 1708 | }
|
| 1709 |
|
| 1710 | static struct ttm_device_funcs amdgpu_bo_driver = {
|
| 1711 | .ttm_tt_create = &amdgpu_ttm_tt_create,
|
| 1712 | .ttm_tt_populate = &amdgpu_ttm_tt_populate,
|
| 1713 | .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
|
| 1714 | .ttm_tt_destroy = &amdgpu_ttm_backend_destroy,
|
| 1715 | .eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
|
| 1716 | .evict_flags = &amdgpu_evict_flags,
|
| 1717 | .move = &amdgpu_bo_move,
|
| 1718 | .delete_mem_notify = &amdgpu_bo_delete_mem_notify,
|
| 1719 | .release_notify = &amdgpu_bo_release_notify,
|
| 1720 | .io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
|
| 1721 | .io_mem_pfn = amdgpu_ttm_io_mem_pfn,
|
| 1722 | .access_memory = &amdgpu_ttm_access_memory,
|
| 1723 | };
|
| 1724 |
|
| 1725 | void amdgpu_ttm_init_vram_resv(struct amdgpu_device *adev,
|
| 1726 | enum amdgpu_resv_region_id id,
|
| 1727 | uint64_t offset, uint64_t size,
|
| 1728 | bool needs_cpu_map)
|
| 1729 | {
|
| 1730 | struct amdgpu_vram_resv *resv;
|
| 1731 |
|
| 1732 | if (id >= AMDGPU_RESV_MAX)
|
| 1733 | return;
|
| 1734 |
|
| 1735 | resv = &adev->mman.resv_region[id];
|
| 1736 | resv->offset = offset;
|
| 1737 | resv->size = size;
|
| 1738 | resv->needs_cpu_map = needs_cpu_map;
|
| 1739 | }
|
| 1740 |
|
| 1741 | static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev)
|
| 1742 | {
|
| 1743 | uint32_t reserve_size = 0;
|
| 1744 |
|
| 1745 | if (!adev->discovery.reserve_tmr)
|
| 1746 | return;
|
| 1747 |
|
| 1748 | |
| 1749 | * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all
|
| 1750 | * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc)
|
| 1751 | *
|
| 1752 | * Otherwise, fallback to legacy approach to check and reserve tmr block for ip
|
| 1753 | * discovery data and G6 memory training data respectively
|
| 1754 |
|
| 1755 | if (adev->bios)
|
| 1756 | reserve_size =
|
| 1757 | amdgpu_atomfirmware_get_fw_reserved_fb_size(adev);
|
| 1758 |
|
| 1759 | if (!adev->bios &&
|
| 1760 | (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
|
| 1761 | amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) ||
|
| 1762 | amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)))
|
| 1763 | reserve_size = max(reserve_size, (uint32_t)280 << 20);
|
| 1764 | else if (!adev->bios &&
|
| 1765 | amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
|
| 1766 | reserve_size = max(reserve_size, (uint32_t)150 << 20);
|
| 1767 | } else if (!reserve_size)
|
| 1768 | reserve_size = DISCOVERY_TMR_OFFSET;
|
| 1769 |
|
| 1770 | amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_FW,
|
| 1771 | adev->gmc.real_vram_size - reserve_size,
|
| 1772 | reserve_size, false);
|
| 1773 | }
|
| 1774 |
|
| 1775 | static void amdgpu_ttm_init_mem_train_resv_region(struct amdgpu_device *adev)
|
| 1776 | {
|
| 1777 | uint64_t reserve_size;
|
| 1778 | uint64_t offset;
|
| 1779 |
|
| 1780 | if (!adev->discovery.reserve_tmr)
|
| 1781 | return;
|
| 1782 |
|
| 1783 | if (!adev->bios || amdgpu_sriov_vf(adev))
|
| 1784 | return;
|
| 1785 |
|
| 1786 | if (!amdgpu_atomfirmware_mem_training_supported(adev))
|
| 1787 | return;
|
| 1788 |
|
| 1789 | reserve_size = adev->mman.resv_region[AMDGPU_RESV_FW].size;
|
| 1790 | offset = ALIGN((adev->gmc.mc_vram_size - reserve_size - SZ_1M), SZ_1M);
|
| 1791 | amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_MEM_TRAIN,
|
| 1792 | offset,
|
| 1793 | GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES,
|
| 1794 | false);
|
| 1795 | }
|
| 1796 |
|
| 1797 | static void amdgpu_ttm_init_vram_resv_regions(struct amdgpu_device *adev)
|
| 1798 | {
|
| 1799 | uint64_t vram_size = adev->gmc.visible_vram_size;
|
| 1800 |
|
| 1801 | |
| 1802 | * This is used for VGA emulation and pre-OS scanout buffers to
|
| 1803 | * avoid display artifacts while transitioning between pre-OS
|
| 1804 | * and driver.
|
| 1805 |
|
| 1806 | amdgpu_gmc_init_vga_resv_regions(adev);
|
| 1807 | amdgpu_ttm_init_fw_resv_region(adev);
|
| 1808 | amdgpu_ttm_init_mem_train_resv_region(adev);
|
| 1809 |
|
| 1810 | if (adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size > vram_size)
|
| 1811 | adev->mman.resv_region[AMDGPU_RESV_FW_VRAM_USAGE].size = 0;
|
| 1812 |
|
| 1813 | if (adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].size > vram_size)
|
| 1814 | adev->mman.resv_region[AMDGPU_RESV_DRV_VRAM_USAGE].size = 0;
|
| 1815 | }
|
| 1816 |
|
| 1817 | int amdgpu_ttm_mark_vram_reserved(struct amdgpu_device *adev,
|
| 1818 | enum amdgpu_resv_region_id id)
|
| 1819 | {
|
| 1820 | struct amdgpu_vram_resv *resv;
|
| 1821 | int ret;
|
| 1822 |
|
| 1823 | if (id >= AMDGPU_RESV_MAX)
|
| 1824 | return -EINVAL;
|
| 1825 |
|
| 1826 | resv = &adev->mman.resv_region[id];
|
| 1827 | if (!resv->size)
|
| 1828 | return 0;
|
| 1829 |
|
| 1830 | ret = amdgpu_bo_create_kernel_at(adev, resv->offset, resv->size,
|
| 1831 | &resv->bo,
|
| 1832 | resv->needs_cpu_map ? &resv->cpu_ptr : NULL);
|
| 1833 | if (ret) {
|
| 1834 | dev_err(adev->dev,
|
| 1835 | "reserve vram failed: id=%d offset=0x%llx size=0x%llx ret=%d\n",
|
| 1836 | id, resv->offset, resv->size, ret);
|
| 1837 | memset(resv, 0, sizeof(*resv));
|
| 1838 | }
|
| 1839 |
|
| 1840 | return ret;
|
| 1841 | }
|
| 1842 |
|
| 1843 | void amdgpu_ttm_unmark_vram_reserved(struct amdgpu_device *adev,
|
| 1844 | enum amdgpu_resv_region_id id)
|
| 1845 | {
|
| 1846 | struct amdgpu_vram_resv *resv;
|
| 1847 |
|
| 1848 | if (id >= AMDGPU_RESV_MAX)
|
| 1849 | return;
|
| 1850 |
|
| 1851 | resv = &adev->mman.resv_region[id];
|
| 1852 | if (!resv->bo)
|
| 1853 | return;
|
| 1854 |
|
| 1855 | amdgpu_bo_free_kernel(&resv->bo, NULL,
|
| 1856 | resv->needs_cpu_map ? &resv->cpu_ptr : NULL);
|
| 1857 | memset(resv, 0, sizeof(*resv));
|
| 1858 | }
|
| 1859 |
|
| 1860 | |
| 1861 | * Reserve all regions with non-zero size. Regions whose info is not
|
| 1862 | * yet available (e.g., fw extended region) may still be reserved
|
| 1863 | * during runtime.
|
| 1864 |
|
| 1865 | static int amdgpu_ttm_alloc_vram_resv_regions(struct amdgpu_device *adev)
|
| 1866 | {
|
| 1867 | int i, r;
|
| 1868 |
|
| 1869 | for (i = 0; i < AMDGPU_RESV_MAX; i++) {
|
| 1870 | r = amdgpu_ttm_mark_vram_reserved(adev, i);
|
| 1871 | if (r)
|
| 1872 | return r;
|
| 1873 | }
|
| 1874 |
|
| 1875 | return 0;
|
| 1876 | }
|
| 1877 |
|
| 1878 | |
| 1879 | * Memoy training reservation functions
|
| 1880 |
|
| 1881 |
|
| 1882 | |
| 1883 | * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram
|
| 1884 | *
|
| 1885 | * @adev: amdgpu_device pointer
|
| 1886 | *
|
| 1887 | * free memory training reserved vram if it has been reserved.
|
| 1888 |
|
| 1889 | static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev)
|
| 1890 | {
|
| 1891 | struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
|
| 1892 |
|
| 1893 | ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
|
| 1894 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_MEM_TRAIN);
|
| 1895 |
|
| 1896 | return 0;
|
| 1897 | }
|
| 1898 |
|
| 1899 | static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev)
|
| 1900 | {
|
| 1901 | struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx;
|
| 1902 | struct amdgpu_vram_resv *resv =
|
| 1903 | &adev->mman.resv_region[AMDGPU_RESV_MEM_TRAIN];
|
| 1904 |
|
| 1905 | memset(ctx, 0, sizeof(*ctx));
|
| 1906 |
|
| 1907 | ctx->c2p_train_data_offset = resv->offset;
|
| 1908 | ctx->p2c_train_data_offset =
|
| 1909 | (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET);
|
| 1910 | ctx->train_data_size = resv->size;
|
| 1911 |
|
| 1912 | DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
|
| 1913 | ctx->train_data_size,
|
| 1914 | ctx->p2c_train_data_offset,
|
| 1915 | ctx->c2p_train_data_offset);
|
| 1916 | }
|
| 1917 |
|
| 1918 | static int amdgpu_ttm_pools_init(struct amdgpu_device *adev)
|
| 1919 | {
|
| 1920 | int i;
|
| 1921 |
|
| 1922 | if (!adev->gmc.is_app_apu || !adev->gmc.num_mem_partitions)
|
| 1923 | return 0;
|
| 1924 |
|
| 1925 | adev->mman.ttm_pools = kzalloc_objs(*adev->mman.ttm_pools,
|
| 1926 | adev->gmc.num_mem_partitions);
|
| 1927 | if (!adev->mman.ttm_pools)
|
| 1928 | return -ENOMEM;
|
| 1929 |
|
| 1930 | for (i = 0; i < adev->gmc.num_mem_partitions; i++) {
|
| 1931 | ttm_pool_init(&adev->mman.ttm_pools[i], adev->dev,
|
| 1932 | adev->gmc.mem_partitions[i].numa.node,
|
| 1933 | TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
|
| 1934 | }
|
| 1935 | return 0;
|
| 1936 | }
|
| 1937 |
|
| 1938 | static void amdgpu_ttm_pools_fini(struct amdgpu_device *adev)
|
| 1939 | {
|
| 1940 | int i;
|
| 1941 |
|
| 1942 | if (!adev->gmc.is_app_apu || !adev->mman.ttm_pools)
|
| 1943 | return;
|
| 1944 |
|
| 1945 | for (i = 0; i < adev->gmc.num_mem_partitions; i++)
|
| 1946 | ttm_pool_fini(&adev->mman.ttm_pools[i]);
|
| 1947 |
|
| 1948 | kfree(adev->mman.ttm_pools);
|
| 1949 | adev->mman.ttm_pools = NULL;
|
| 1950 | }
|
| 1951 |
|
| 1952 | |
| 1953 | * amdgpu_ttm_alloc_mmio_remap_bo - Allocate the singleton MMIO_REMAP BO
|
| 1954 | * @adev: amdgpu device
|
| 1955 | *
|
| 1956 | * Allocates a global BO with backing AMDGPU_PL_MMIO_REMAP when the
|
| 1957 | * hardware exposes a remap base (adev->rmmio_remap.bus_addr) and the host
|
| 1958 | * PAGE_SIZE is <= AMDGPU_GPU_PAGE_SIZE (4K). The BO is created as a regular
|
| 1959 | * GEM object (amdgpu_bo_create).
|
| 1960 | *
|
| 1961 | * Return:
|
| 1962 | * * 0 on success or intentional skip (feature not present/unsupported)
|
| 1963 | * * negative errno on allocation failure
|
| 1964 |
|
| 1965 | static int amdgpu_ttm_alloc_mmio_remap_bo(struct amdgpu_device *adev)
|
| 1966 | {
|
| 1967 | struct ttm_operation_ctx ctx = { false, false };
|
| 1968 | struct ttm_placement placement;
|
| 1969 | struct ttm_buffer_object *tbo;
|
| 1970 | struct ttm_place placements;
|
| 1971 | struct amdgpu_bo_param bp;
|
| 1972 | struct ttm_resource *tmp;
|
| 1973 | int r;
|
| 1974 |
|
| 1975 |
|
| 1976 | if (!adev->rmmio_remap.bus_addr || PAGE_SIZE > AMDGPU_GPU_PAGE_SIZE)
|
| 1977 | return 0;
|
| 1978 |
|
| 1979 | |
| 1980 | * Allocate a BO first and then move it to AMDGPU_PL_MMIO_REMAP.
|
| 1981 | * The initial TTM resource assigned by amdgpu_bo_create() is
|
| 1982 | * replaced below with a fixed MMIO_REMAP placement.
|
| 1983 |
|
| 1984 | memset(&bp, 0, sizeof(bp));
|
| 1985 | bp.type = ttm_bo_type_device;
|
| 1986 | bp.size = AMDGPU_GPU_PAGE_SIZE;
|
| 1987 | bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
|
| 1988 | bp.domain = 0;
|
| 1989 | bp.flags = 0;
|
| 1990 | bp.resv = NULL;
|
| 1991 | bp.bo_ptr_size = sizeof(struct amdgpu_bo);
|
| 1992 | r = amdgpu_bo_create(adev, &bp, &adev->rmmio_remap.bo);
|
| 1993 | if (r)
|
| 1994 | return r;
|
| 1995 |
|
| 1996 | r = amdgpu_bo_reserve(adev->rmmio_remap.bo, true);
|
| 1997 | if (r)
|
| 1998 | goto err_unref;
|
| 1999 |
|
| 2000 | tbo = &adev->rmmio_remap.bo->tbo;
|
| 2001 |
|
| 2002 | |
| 2003 | * MMIO_REMAP is a fixed I/O placement (AMDGPU_PL_MMIO_REMAP).
|
| 2004 |
|
| 2005 | placement.num_placement = 1;
|
| 2006 | placement.placement = &placements;
|
| 2007 | placements.fpfn = 0;
|
| 2008 | placements.lpfn = 0;
|
| 2009 | placements.mem_type = AMDGPU_PL_MMIO_REMAP;
|
| 2010 | placements.flags = 0;
|
| 2011 |
|
| 2012 | r = ttm_bo_mem_space(tbo, &placement, &tmp, &ctx);
|
| 2013 | if (unlikely(r))
|
| 2014 | goto err_unlock;
|
| 2015 |
|
| 2016 | ttm_resource_free(tbo, &tbo->resource);
|
| 2017 | ttm_bo_assign_mem(tbo, tmp);
|
| 2018 | ttm_bo_pin(tbo);
|
| 2019 |
|
| 2020 | amdgpu_bo_unreserve(adev->rmmio_remap.bo);
|
| 2021 | return 0;
|
| 2022 |
|
| 2023 | err_unlock:
|
| 2024 | amdgpu_bo_unreserve(adev->rmmio_remap.bo);
|
| 2025 |
|
| 2026 | err_unref:
|
| 2027 | amdgpu_bo_unref(&adev->rmmio_remap.bo);
|
| 2028 | adev->rmmio_remap.bo = NULL;
|
| 2029 | return r;
|
| 2030 | }
|
| 2031 |
|
| 2032 | |
| 2033 | * amdgpu_ttm_free_mmio_remap_bo - Free the singleton MMIO_REMAP BO
|
| 2034 | * @adev: amdgpu device
|
| 2035 | *
|
| 2036 | * Frees the kernel-owned MMIO_REMAP BO if it was allocated by
|
| 2037 | * amdgpu_ttm_mmio_remap_bo_init().
|
| 2038 |
|
| 2039 | static void amdgpu_ttm_free_mmio_remap_bo(struct amdgpu_device *adev)
|
| 2040 | {
|
| 2041 | if (!adev->rmmio_remap.bo)
|
| 2042 | return;
|
| 2043 |
|
| 2044 | if (!amdgpu_bo_reserve(adev->rmmio_remap.bo, true)) {
|
| 2045 | ttm_bo_unpin(&adev->rmmio_remap.bo->tbo);
|
| 2046 | amdgpu_bo_unreserve(adev->rmmio_remap.bo);
|
| 2047 | }
|
| 2048 |
|
| 2049 | |
| 2050 | * At this point we rely on normal DRM teardown ordering:
|
| 2051 | * no new user ioctls can access the global MMIO_REMAP BO
|
| 2052 | * once TTM teardown begins.
|
| 2053 |
|
| 2054 | amdgpu_bo_unref(&adev->rmmio_remap.bo);
|
| 2055 | adev->rmmio_remap.bo = NULL;
|
| 2056 | }
|
| 2057 |
|
| 2058 | static int amdgpu_ttm_buffer_entity_init(struct amdgpu_gtt_mgr *mgr,
|
| 2059 | struct amdgpu_ttm_buffer_entity *entity,
|
| 2060 | enum drm_sched_priority prio,
|
| 2061 | struct drm_gpu_scheduler **scheds,
|
| 2062 | int num_schedulers,
|
| 2063 | u32 num_gart_windows)
|
| 2064 | {
|
| 2065 | int i, r, num_pages;
|
| 2066 | const u64 GTT_MAX_PAGES = (AMDGPU_GTT_MAX_TRANSFER_SIZE >> PAGE_SHIFT);
|
| 2067 |
|
| 2068 | if (ARRAY_SIZE(entity->gart_window_offs) < num_gart_windows)
|
| 2069 | return -EINVAL;
|
| 2070 |
|
| 2071 | r = drm_sched_entity_init(&entity->base, prio, scheds, num_schedulers, NULL);
|
| 2072 | if (r)
|
| 2073 | return r;
|
| 2074 |
|
| 2075 | mutex_init(&entity->lock);
|
| 2076 |
|
| 2077 | if (num_gart_windows == 0)
|
| 2078 | return 0;
|
| 2079 |
|
| 2080 | num_pages = num_gart_windows * GTT_MAX_PAGES;
|
| 2081 | r = amdgpu_gtt_mgr_alloc_entries(mgr, &entity->gart_node, num_pages,
|
| 2082 | DRM_MM_INSERT_BEST);
|
| 2083 | if (r) {
|
| 2084 | mutex_destroy(&entity->lock);
|
| 2085 | drm_sched_entity_destroy(&entity->base);
|
| 2086 | return r;
|
| 2087 | }
|
| 2088 |
|
| 2089 | for (i = 0; i < num_gart_windows; i++) {
|
| 2090 | entity->gart_window_offs[i] =
|
| 2091 | amdgpu_gtt_node_to_byte_offset(&entity->gart_node) +
|
| 2092 | i * GTT_MAX_PAGES * PAGE_SIZE;
|
| 2093 | }
|
| 2094 |
|
| 2095 | return 0;
|
| 2096 | }
|
| 2097 |
|
| 2098 | static void amdgpu_ttm_buffer_entity_fini(struct amdgpu_gtt_mgr *mgr,
|
| 2099 | struct amdgpu_ttm_buffer_entity *entity)
|
| 2100 | {
|
| 2101 | amdgpu_gtt_mgr_free_entries(mgr, &entity->gart_node);
|
| 2102 | mutex_destroy(&entity->lock);
|
| 2103 | drm_sched_entity_destroy(&entity->base);
|
| 2104 | }
|
| 2105 |
|
| 2106 | |
| 2107 | * amdgpu_ttm_init - Init the memory management (ttm) as well as various
|
| 2108 | * gtt/vram related fields.
|
| 2109 | *
|
| 2110 | * This initializes all of the memory space pools that the TTM layer
|
| 2111 | * will need such as the GTT space (system memory mapped to the device),
|
| 2112 | * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
|
| 2113 | * can be mapped per VMID.
|
| 2114 |
|
| 2115 | int amdgpu_ttm_init(struct amdgpu_device *adev)
|
| 2116 | {
|
| 2117 | uint64_t gtt_size;
|
| 2118 | int r;
|
| 2119 |
|
| 2120 | dma_set_max_seg_size(adev->dev, UINT_MAX);
|
| 2121 |
|
| 2122 | r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev,
|
| 2123 | adev_to_drm(adev)->anon_inode->i_mapping,
|
| 2124 | adev_to_drm(adev)->vma_offset_manager,
|
| 2125 | (adev->need_swiotlb ?
|
| 2126 | TTM_ALLOCATION_POOL_USE_DMA_ALLOC : 0) |
|
| 2127 | (dma_addressing_limited(adev->dev) ?
|
| 2128 | TTM_ALLOCATION_POOL_USE_DMA32 : 0) |
|
| 2129 | TTM_ALLOCATION_POOL_BENEFICIAL_ORDER(get_order(SZ_2M)));
|
| 2130 | if (r) {
|
| 2131 | dev_err(adev->dev,
|
| 2132 | "failed initializing buffer object driver(%d).\n", r);
|
| 2133 | return r;
|
| 2134 | }
|
| 2135 |
|
| 2136 | r = amdgpu_ttm_pools_init(adev);
|
| 2137 | if (r) {
|
| 2138 | dev_err(adev->dev, "failed to init ttm pools(%d).\n", r);
|
| 2139 | return r;
|
| 2140 | }
|
| 2141 | adev->mman.initialized = true;
|
| 2142 |
|
| 2143 | if (!adev->gmc.is_app_apu) {
|
| 2144 |
|
| 2145 | r = amdgpu_vram_mgr_init(adev);
|
| 2146 | if (r) {
|
| 2147 | dev_err(adev->dev, "Failed initializing VRAM heap.\n");
|
| 2148 | return r;
|
| 2149 | }
|
| 2150 | }
|
| 2151 |
|
| 2152 |
|
| 2153 | amdgpu_ttm_disable_buffer_funcs(adev);
|
| 2154 | #ifdef CONFIG_64BIT
|
| 2155 | #ifdef CONFIG_X86
|
| 2156 | if (adev->gmc.xgmi.connected_to_cpu)
|
| 2157 | adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base,
|
| 2158 | adev->gmc.visible_vram_size);
|
| 2159 | else if (adev->gmc.is_app_apu)
|
| 2160 | DRM_DEBUG_DRIVER(
|
| 2161 | "No need to ioremap when real vram size is 0\n");
|
| 2162 | else
|
| 2163 | #endif
|
| 2164 | adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
|
| 2165 | adev->gmc.visible_vram_size);
|
| 2166 | #endif
|
| 2167 |
|
| 2168 | amdgpu_ttm_init_vram_resv_regions(adev);
|
| 2169 |
|
| 2170 | r = amdgpu_ttm_alloc_vram_resv_regions(adev);
|
| 2171 | if (r)
|
| 2172 | return r;
|
| 2173 |
|
| 2174 | if (adev->mman.resv_region[AMDGPU_RESV_MEM_TRAIN].size) {
|
| 2175 | struct psp_memory_training_context *ctx =
|
| 2176 | &adev->psp.mem_train_ctx;
|
| 2177 |
|
| 2178 | amdgpu_ttm_training_data_block_init(adev);
|
| 2179 | ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS;
|
| 2180 | }
|
| 2181 |
|
| 2182 | dev_info(adev->dev, " %uM of VRAM memory ready\n",
|
| 2183 | (unsigned int)(adev->gmc.real_vram_size / (1024 * 1024)));
|
| 2184 |
|
| 2185 | |
| 2186 | * or whatever the user passed on module init.
|
| 2187 |
|
| 2188 | gtt_size = ttm_tt_pages_limit() << PAGE_SHIFT;
|
| 2189 | if (amdgpu_gtt_size != -1) {
|
| 2190 | uint64_t configured_size = (uint64_t)amdgpu_gtt_size << 20;
|
| 2191 |
|
| 2192 | drm_warn(&adev->ddev,
|
| 2193 | "Configuring gttsize via module parameter is deprecated, please use ttm.pages_limit\n");
|
| 2194 | if (gtt_size != configured_size)
|
| 2195 | drm_warn(&adev->ddev,
|
| 2196 | "GTT size has been set as %llu but TTM size has been set as %llu, this is unusual\n",
|
| 2197 | configured_size, gtt_size);
|
| 2198 |
|
| 2199 | gtt_size = configured_size;
|
| 2200 | }
|
| 2201 |
|
| 2202 |
|
| 2203 | if (adev->flags & AMD_IS_APU) {
|
| 2204 | u64 phys_ram = (u64)totalram_pages() << PAGE_SHIFT;
|
| 2205 |
|
| 2206 | if (gtt_size > phys_ram) {
|
| 2207 | gtt_size = phys_ram;
|
| 2208 | dev_info(adev->dev,
|
| 2209 | "Capping GTT to %uM to not exceed available system memory\n",
|
| 2210 | (unsigned int)(gtt_size / (1024 * 1024)));
|
| 2211 | }
|
| 2212 | }
|
| 2213 |
|
| 2214 |
|
| 2215 | r = amdgpu_gtt_mgr_init(adev, gtt_size);
|
| 2216 | if (r) {
|
| 2217 | dev_err(adev->dev, "Failed initializing GTT heap.\n");
|
| 2218 | return r;
|
| 2219 | }
|
| 2220 | dev_info(adev->dev, " %uM of GTT memory ready.\n",
|
| 2221 | (unsigned int)(gtt_size / (1024 * 1024)));
|
| 2222 |
|
| 2223 | if (adev->flags & AMD_IS_APU) {
|
| 2224 | if (adev->gmc.real_vram_size < gtt_size)
|
| 2225 | adev->apu_prefer_gtt = true;
|
| 2226 | }
|
| 2227 |
|
| 2228 |
|
| 2229 | r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_DOORBELL, adev->doorbell.size / PAGE_SIZE);
|
| 2230 | if (r) {
|
| 2231 | dev_err(adev->dev, "Failed initializing doorbell heap.\n");
|
| 2232 | return r;
|
| 2233 | }
|
| 2234 |
|
| 2235 |
|
| 2236 | r = amdgpu_doorbell_create_kernel_doorbells(adev);
|
| 2237 | if (r) {
|
| 2238 | dev_err(adev->dev, "Failed to initialize kernel doorbells.\n");
|
| 2239 | return r;
|
| 2240 | }
|
| 2241 |
|
| 2242 |
|
| 2243 | r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_MMIO_REMAP, 1);
|
| 2244 | if (r) {
|
| 2245 | dev_err(adev->dev, "Failed initializing MMIO-remap heap.\n");
|
| 2246 | return r;
|
| 2247 | }
|
| 2248 |
|
| 2249 |
|
| 2250 | r = amdgpu_ttm_alloc_mmio_remap_bo(adev);
|
| 2251 | if (r)
|
| 2252 | return r;
|
| 2253 |
|
| 2254 |
|
| 2255 | r = amdgpu_preempt_mgr_init(adev);
|
| 2256 | if (r) {
|
| 2257 | dev_err(adev->dev, "Failed initializing PREEMPT heap.\n");
|
| 2258 | return r;
|
| 2259 | }
|
| 2260 |
|
| 2261 |
|
| 2262 | r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size);
|
| 2263 | if (r) {
|
| 2264 | dev_err(adev->dev, "Failed initializing GDS heap.\n");
|
| 2265 | return r;
|
| 2266 | }
|
| 2267 |
|
| 2268 | r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size);
|
| 2269 | if (r) {
|
| 2270 | dev_err(adev->dev, "Failed initializing gws heap.\n");
|
| 2271 | return r;
|
| 2272 | }
|
| 2273 |
|
| 2274 | r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size);
|
| 2275 | if (r) {
|
| 2276 | dev_err(adev->dev, "Failed initializing oa heap.\n");
|
| 2277 | return r;
|
| 2278 | }
|
| 2279 | if (amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
|
| 2280 | AMDGPU_GEM_DOMAIN_GTT,
|
| 2281 | &adev->mman.sdma_access_bo, NULL,
|
| 2282 | &adev->mman.sdma_access_ptr))
|
| 2283 | drm_warn(adev_to_drm(adev),
|
| 2284 | "Debug VRAM access will use slowpath MM access\n");
|
| 2285 |
|
| 2286 | return 0;
|
| 2287 | }
|
| 2288 |
|
| 2289 | |
| 2290 | * amdgpu_ttm_fini - De-initialize the TTM memory pools
|
| 2291 |
|
| 2292 | void amdgpu_ttm_fini(struct amdgpu_device *adev)
|
| 2293 | {
|
| 2294 | if (!adev->mman.initialized)
|
| 2295 | return;
|
| 2296 |
|
| 2297 | amdgpu_ttm_pools_fini(adev);
|
| 2298 |
|
| 2299 | amdgpu_ttm_training_reserve_vram_fini(adev);
|
| 2300 |
|
| 2301 | if (!adev->gmc.is_app_apu) {
|
| 2302 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_VGA);
|
| 2303 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_EXTENDED);
|
| 2304 |
|
| 2305 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_FW);
|
| 2306 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_FW_EXTEND);
|
| 2307 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_STOLEN_RESERVED);
|
| 2308 | }
|
| 2309 | amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL,
|
| 2310 | &adev->mman.sdma_access_ptr);
|
| 2311 |
|
| 2312 | amdgpu_ttm_free_mmio_remap_bo(adev);
|
| 2313 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_FW_VRAM_USAGE);
|
| 2314 | amdgpu_ttm_unmark_vram_reserved(adev, AMDGPU_RESV_DRV_VRAM_USAGE);
|
| 2315 |
|
| 2316 | if (adev->mman.aper_base_kaddr) {
|
| 2317 | iounmap(adev->mman.aper_base_kaddr);
|
| 2318 | adev->mman.aper_base_kaddr = NULL;
|
| 2319 | }
|
| 2320 |
|
| 2321 | if (!adev->gmc.is_app_apu)
|
| 2322 | amdgpu_vram_mgr_fini(adev);
|
| 2323 | amdgpu_gtt_mgr_fini(adev);
|
| 2324 | amdgpu_preempt_mgr_fini(adev);
|
| 2325 | amdgpu_doorbell_fini(adev);
|
| 2326 |
|
| 2327 | ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);
|
| 2328 | ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);
|
| 2329 | ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);
|
| 2330 | ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_DOORBELL);
|
| 2331 | ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_MMIO_REMAP);
|
| 2332 | ttm_device_fini(&adev->mman.bdev);
|
| 2333 | adev->mman.initialized = false;
|
| 2334 | dev_info(adev->dev, " ttm finalized\n");
|
| 2335 | }
|
| 2336 |
|
| 2337 | |
| 2338 | * amdgpu_ttm_enable_buffer_funcs - enable use of buffer functions
|
| 2339 | *
|
| 2340 | * @adev: amdgpu_device pointer
|
| 2341 | *
|
| 2342 | * Enable use of buffer functions during suspend/resume. This should
|
| 2343 | * only be called at bootup or when userspace isn't running.
|
| 2344 |
|
| 2345 | void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev)
|
| 2346 | {
|
| 2347 | struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
|
| 2348 | u32 num_clear_entities, num_move_entities, sdma_ip_version;
|
| 2349 | int r, i, j;
|
| 2350 |
|
| 2351 | if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
|
| 2352 | adev->mman.buffer_funcs_enabled || adev->gmc.is_app_apu)
|
| 2353 | return;
|
| 2354 |
|
| 2355 | if (!adev->mman.num_buffer_funcs_scheds) {
|
| 2356 | dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
|
| 2357 | return;
|
| 2358 | }
|
| 2359 |
|
| 2360 |
|
| 2361 | r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
|
| 2362 | &adev->mman.default_entity,
|
| 2363 | DRM_SCHED_PRIORITY_KERNEL,
|
| 2364 | adev->mman.buffer_funcs_scheds, 1, 0);
|
| 2365 | if (r < 0) {
|
| 2366 | dev_err(adev->dev,
|
| 2367 | "Failed setting up TTM entity (%d)\n", r);
|
| 2368 | return;
|
| 2369 | }
|
| 2370 |
|
| 2371 | num_clear_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
|
| 2372 | num_move_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
|
| 2373 | |
| 2374 | * the same time: use a single queue until the root cause is identified and fixed.
|
| 2375 |
|
| 2376 | sdma_ip_version = amdgpu_ip_version(adev, SDMA0_HWIP, 0);
|
| 2377 | if (sdma_ip_version == IP_VERSION(7, 0, 0) || sdma_ip_version == IP_VERSION(7, 0, 1))
|
| 2378 | num_move_entities = 1;
|
| 2379 |
|
| 2380 | adev->mman.clear_entities = kzalloc_objs(struct amdgpu_ttm_buffer_entity,
|
| 2381 | num_clear_entities);
|
| 2382 | atomic_set(&adev->mman.next_clear_entity, 0);
|
| 2383 | if (!adev->mman.clear_entities)
|
| 2384 | goto error_free_default_entity;
|
| 2385 |
|
| 2386 | adev->mman.num_clear_entities = num_clear_entities;
|
| 2387 |
|
| 2388 | for (i = 0; i < num_clear_entities; i++) {
|
| 2389 | r = amdgpu_ttm_buffer_entity_init(
|
| 2390 | &adev->mman.gtt_mgr,
|
| 2391 | &adev->mman.clear_entities[i],
|
| 2392 | DRM_SCHED_PRIORITY_KERNEL,
|
| 2393 | adev->mman.buffer_funcs_scheds,
|
| 2394 | adev->mman.num_buffer_funcs_scheds, 1);
|
| 2395 |
|
| 2396 | if (r < 0) {
|
| 2397 | for (j = 0; j < i; j++)
|
| 2398 | amdgpu_ttm_buffer_entity_fini(
|
| 2399 | &adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);
|
| 2400 | adev->mman.num_clear_entities = 0;
|
| 2401 | kfree(adev->mman.clear_entities);
|
| 2402 | goto error_free_default_entity;
|
| 2403 | }
|
| 2404 | }
|
| 2405 |
|
| 2406 | adev->mman.num_move_entities = num_move_entities;
|
| 2407 | atomic_set(&adev->mman.next_move_entity, 0);
|
| 2408 | for (i = 0; i < num_move_entities; i++) {
|
| 2409 | r = amdgpu_ttm_buffer_entity_init(
|
| 2410 | &adev->mman.gtt_mgr,
|
| 2411 | &adev->mman.move_entities[i],
|
| 2412 | DRM_SCHED_PRIORITY_KERNEL,
|
| 2413 | adev->mman.buffer_funcs_scheds,
|
| 2414 | adev->mman.num_buffer_funcs_scheds, 2);
|
| 2415 |
|
| 2416 | if (r < 0) {
|
| 2417 | for (j = 0; j < i; j++)
|
| 2418 | amdgpu_ttm_buffer_entity_fini(
|
| 2419 | &adev->mman.gtt_mgr,
|
| 2420 | &adev->mman.move_entities[j]);
|
| 2421 | adev->mman.num_move_entities = 0;
|
| 2422 | goto error_free_clear_entities;
|
| 2423 | }
|
| 2424 | }
|
| 2425 |
|
| 2426 |
|
| 2427 | man->size = adev->gmc.real_vram_size;
|
| 2428 | adev->mman.buffer_funcs_enabled = true;
|
| 2429 |
|
| 2430 | return;
|
| 2431 |
|
| 2432 | error_free_clear_entities:
|
| 2433 | for (i = 0; i < adev->mman.num_clear_entities; i++)
|
| 2434 | amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
|
| 2435 | &adev->mman.clear_entities[i]);
|
| 2436 | kfree(adev->mman.clear_entities);
|
| 2437 | adev->mman.clear_entities = NULL;
|
| 2438 | adev->mman.num_clear_entities = 0;
|
| 2439 | error_free_default_entity:
|
| 2440 | amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
|
| 2441 | &adev->mman.default_entity);
|
| 2442 | }
|
| 2443 |
|
| 2444 | |
| 2445 | * amdgpu_ttm_disable_buffer_funcs - disable use of buffer functions
|
| 2446 | *
|
| 2447 | * @adev: amdgpu_device pointer
|
| 2448 |
|
| 2449 | void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev)
|
| 2450 | {
|
| 2451 | struct ttm_resource_manager *man =
|
| 2452 | ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
|
| 2453 | int i;
|
| 2454 |
|
| 2455 | if (!adev->mman.buffer_funcs_enabled || amdgpu_in_reset(adev))
|
| 2456 | return;
|
| 2457 |
|
| 2458 | amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
|
| 2459 | &adev->mman.default_entity);
|
| 2460 | for (i = 0; i < adev->mman.num_move_entities; i++)
|
| 2461 | amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
|
| 2462 | &adev->mman.move_entities[i]);
|
| 2463 | for (i = 0; i < adev->mman.num_clear_entities; i++)
|
| 2464 | amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
|
| 2465 | &adev->mman.clear_entities[i]);
|
| 2466 | |
| 2467 | * will allocate new contexts.
|
| 2468 |
|
| 2469 | ttm_resource_manager_cleanup(man);
|
| 2470 |
|
| 2471 | kfree(adev->mman.clear_entities);
|
| 2472 | adev->mman.clear_entities = NULL;
|
| 2473 | adev->mman.num_clear_entities = 0;
|
| 2474 | adev->mman.num_move_entities = 0;
|
| 2475 |
|
| 2476 | man->size = adev->gmc.visible_vram_size;
|
| 2477 | adev->mman.buffer_funcs_enabled = false;
|
| 2478 | }
|
| 2479 |
|
| 2480 | static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
|
| 2481 | struct amdgpu_ttm_buffer_entity *entity,
|
| 2482 | unsigned int num_dw,
|
| 2483 | struct dma_resv *resv,
|
| 2484 | bool vm_needs_flush,
|
| 2485 | struct amdgpu_job **job,
|
| 2486 | u64 k_job_id)
|
| 2487 | {
|
| 2488 | enum amdgpu_ib_pool_type pool = AMDGPU_IB_POOL_DELAYED;
|
| 2489 | int r;
|
| 2490 | r = amdgpu_job_alloc_with_ib(adev, &entity->base,
|
| 2491 | AMDGPU_FENCE_OWNER_UNDEFINED,
|
| 2492 | num_dw * 4, pool, k_job_id, job);
|
| 2493 | if (r)
|
| 2494 | return r;
|
| 2495 |
|
| 2496 | if (vm_needs_flush) {
|
| 2497 | (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ?
|
| 2498 | adev->gmc.pdb0_bo :
|
| 2499 | adev->gart.bo);
|
| 2500 | (*job)->vm_needs_flush = true;
|
| 2501 | }
|
| 2502 | if (!resv)
|
| 2503 | return 0;
|
| 2504 |
|
| 2505 | return drm_sched_job_add_resv_dependencies(&(*job)->base, resv,
|
| 2506 | DMA_RESV_USAGE_BOOKKEEP);
|
| 2507 | }
|
| 2508 |
|
| 2509 | static int amdgpu_calc_bytes_per_packet(u32 max_bytes_per_packet,
|
| 2510 | u32 byte_count)
|
| 2511 | {
|
| 2512 |
|
| 2513 | if (!(byte_count & 0x3) && byte_count <= max_bytes_per_packet)
|
| 2514 | return max_bytes_per_packet;
|
| 2515 |
|
| 2516 | |
| 2517 | * Align down maximum byte count to 256 bytes so that
|
| 2518 | * the copy optimally uses all memory channels and
|
| 2519 | * also to ensure that SDMA can use its dword mode, which
|
| 2520 | * is faster.
|
| 2521 | *
|
| 2522 | * This assumes that the starting addresses of BOs are always
|
| 2523 | * dword aligned, which should be the case for every copy
|
| 2524 | * operation in the kernel, because the kernel always copies
|
| 2525 | * pages.
|
| 2526 |
|
| 2527 | return ALIGN_DOWN(max_bytes_per_packet, SZ_256);
|
| 2528 | }
|
| 2529 |
|
| 2530 | int amdgpu_copy_buffer(struct amdgpu_device *adev,
|
| 2531 | struct amdgpu_ttm_buffer_entity *entity,
|
| 2532 | uint64_t src_offset,
|
| 2533 | uint64_t dst_offset, uint32_t byte_count,
|
| 2534 | struct dma_resv *resv,
|
| 2535 | struct dma_fence **fence,
|
| 2536 | bool vm_needs_flush, uint32_t copy_flags)
|
| 2537 | {
|
| 2538 | unsigned int num_loops, num_dw;
|
| 2539 | struct amdgpu_ring *ring;
|
| 2540 | struct amdgpu_job *job;
|
| 2541 | uint32_t max_bytes;
|
| 2542 | unsigned int i;
|
| 2543 | int r;
|
| 2544 |
|
| 2545 | ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
|
| 2546 |
|
| 2547 | if (!ring->sched.ready) {
|
| 2548 | dev_err(adev->dev,
|
| 2549 | "Trying to move memory with ring turned off.\n");
|
| 2550 | return -EINVAL;
|
| 2551 | }
|
| 2552 |
|
| 2553 | max_bytes = amdgpu_calc_bytes_per_packet(adev->mman.buffer_funcs->copy_max_bytes,
|
| 2554 | byte_count);
|
| 2555 | num_loops = DIV_ROUND_UP(byte_count, max_bytes);
|
| 2556 | num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8);
|
| 2557 | r = amdgpu_ttm_prepare_job(adev, entity, num_dw,
|
| 2558 | resv, vm_needs_flush, &job,
|
| 2559 | AMDGPU_KERNEL_JOB_ID_TTM_COPY_BUFFER);
|
| 2560 | if (r)
|
| 2561 | goto error_free;
|
| 2562 |
|
| 2563 | for (i = 0; i < num_loops; i++) {
|
| 2564 | uint32_t cur_size_in_bytes = min(byte_count, max_bytes);
|
| 2565 |
|
| 2566 | amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
|
| 2567 | dst_offset, cur_size_in_bytes, copy_flags);
|
| 2568 | src_offset += cur_size_in_bytes;
|
| 2569 | dst_offset += cur_size_in_bytes;
|
| 2570 | byte_count -= cur_size_in_bytes;
|
| 2571 | }
|
| 2572 |
|
| 2573 | *fence = amdgpu_ttm_job_submit(adev, entity, job, num_dw);
|
| 2574 |
|
| 2575 | return 0;
|
| 2576 |
|
| 2577 | error_free:
|
| 2578 | amdgpu_job_free(job);
|
| 2579 | dev_err(adev->dev, "Error scheduling IBs (%d)\n", r);
|
| 2580 | return r;
|
| 2581 | }
|
| 2582 |
|
| 2583 | static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
|
| 2584 | struct amdgpu_ttm_buffer_entity *entity,
|
| 2585 | uint32_t src_data,
|
| 2586 | uint64_t dst_addr, uint32_t byte_count,
|
| 2587 | struct dma_resv *resv,
|
| 2588 | struct dma_fence **fence,
|
| 2589 | bool vm_needs_flush,
|
| 2590 | u64 k_job_id)
|
| 2591 | {
|
| 2592 | unsigned int num_loops, num_dw;
|
| 2593 | struct amdgpu_job *job;
|
| 2594 | uint32_t max_bytes;
|
| 2595 | unsigned int i;
|
| 2596 | int r;
|
| 2597 |
|
| 2598 | max_bytes = amdgpu_calc_bytes_per_packet(adev->mman.buffer_funcs->fill_max_bytes,
|
| 2599 | byte_count);
|
| 2600 | num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes);
|
| 2601 | num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8);
|
| 2602 | r = amdgpu_ttm_prepare_job(adev, entity, num_dw, resv,
|
| 2603 | vm_needs_flush, &job, k_job_id);
|
| 2604 | if (r)
|
| 2605 | return r;
|
| 2606 |
|
| 2607 | for (i = 0; i < num_loops; i++) {
|
| 2608 | uint32_t cur_size = min(byte_count, max_bytes);
|
| 2609 |
|
| 2610 | amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr,
|
| 2611 | cur_size);
|
| 2612 |
|
| 2613 | dst_addr += cur_size;
|
| 2614 | byte_count -= cur_size;
|
| 2615 | }
|
| 2616 |
|
| 2617 | *fence = amdgpu_ttm_job_submit(adev, entity, job, num_dw);
|
| 2618 | return 0;
|
| 2619 | }
|
| 2620 |
|
| 2621 | |
| 2622 | * amdgpu_ttm_clear_buffer - fill a buffer with 0
|
| 2623 | * @entity: entity to use
|
| 2624 | * @bo: the bo to fill
|
| 2625 | * @resv: fences contained in this reservation will be used as dependencies.
|
| 2626 | * @out_fence: the fence from the last clear will be stored here. It might be
|
| 2627 | * NULL if no job was run.
|
| 2628 | * @consider_clear_status: true if region reported as cleared by amdgpu_res_cleared()
|
| 2629 | * are skipped.
|
| 2630 | * @k_job_id: trace id
|
| 2631 | *
|
| 2632 |
|
| 2633 | int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
|
| 2634 | struct amdgpu_bo *bo,
|
| 2635 | struct dma_resv *resv,
|
| 2636 | struct dma_fence **out_fence,
|
| 2637 | bool consider_clear_status,
|
| 2638 | u64 k_job_id)
|
| 2639 | {
|
| 2640 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
|
| 2641 | struct dma_fence *fence = NULL;
|
| 2642 | struct amdgpu_res_cursor dst;
|
| 2643 | bool vm_needs_flush = false;
|
| 2644 | int r;
|
| 2645 |
|
| 2646 | if (!entity)
|
| 2647 | return -EINVAL;
|
| 2648 |
|
| 2649 | amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst);
|
| 2650 |
|
| 2651 | mutex_lock(&entity->lock);
|
| 2652 | while (dst.remaining) {
|
| 2653 | struct dma_fence *next;
|
| 2654 | uint64_t cur_size, to;
|
| 2655 |
|
| 2656 | if (consider_clear_status && amdgpu_res_cleared(&dst)) {
|
| 2657 | amdgpu_res_next(&dst, dst.size);
|
| 2658 | continue;
|
| 2659 | }
|
| 2660 |
|
| 2661 |
|
| 2662 | cur_size = min(dst.size, 256ULL << 20);
|
| 2663 |
|
| 2664 | r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &dst,
|
| 2665 | 0, false, &cur_size, &to, &vm_needs_flush);
|
| 2666 | if (r)
|
| 2667 | goto error;
|
| 2668 |
|
| 2669 | r = amdgpu_ttm_fill_mem(adev, entity,
|
| 2670 | 0, to, cur_size, resv,
|
| 2671 | &next, vm_needs_flush, k_job_id);
|
| 2672 | if (r)
|
| 2673 | goto error;
|
| 2674 |
|
| 2675 | dma_fence_put(fence);
|
| 2676 | fence = next;
|
| 2677 |
|
| 2678 | amdgpu_res_next(&dst, cur_size);
|
| 2679 | }
|
| 2680 | error:
|
| 2681 | mutex_unlock(&entity->lock);
|
| 2682 | *out_fence = fence;
|
| 2683 | return r;
|
| 2684 | }
|
| 2685 |
|
| 2686 | struct amdgpu_ttm_buffer_entity *
|
| 2687 | amdgpu_ttm_next_clear_entity(struct amdgpu_device *adev)
|
| 2688 | {
|
| 2689 | struct amdgpu_mman *mman = &adev->mman;
|
| 2690 | u32 i;
|
| 2691 |
|
| 2692 | if (mman->num_clear_entities == 0)
|
| 2693 | return NULL;
|
| 2694 |
|
| 2695 | i = atomic_inc_return(&mman->next_clear_entity) %
|
| 2696 | mman->num_clear_entities;
|
| 2697 | return &mman->clear_entities[i];
|
| 2698 | }
|
| 2699 |
|
| 2700 | |
| 2701 | * amdgpu_ttm_evict_resources - evict memory buffers
|
| 2702 | * @adev: amdgpu device object
|
| 2703 | * @mem_type: evicted BO's memory type
|
| 2704 | *
|
| 2705 | * Evicts all @mem_type buffers on the lru list of the memory type.
|
| 2706 | *
|
| 2707 | * Returns:
|
| 2708 | * 0 for success or a negative error code on failure.
|
| 2709 |
|
| 2710 | int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
|
| 2711 | {
|
| 2712 | struct ttm_resource_manager *man;
|
| 2713 |
|
| 2714 | switch (mem_type) {
|
| 2715 | case TTM_PL_VRAM:
|
| 2716 | case TTM_PL_TT:
|
| 2717 | case AMDGPU_PL_GWS:
|
| 2718 | case AMDGPU_PL_GDS:
|
| 2719 | case AMDGPU_PL_OA:
|
| 2720 | man = ttm_manager_type(&adev->mman.bdev, mem_type);
|
| 2721 | break;
|
| 2722 | default:
|
| 2723 | dev_err(adev->dev, "Trying to evict invalid memory type\n");
|
| 2724 | return -EINVAL;
|
| 2725 | }
|
| 2726 |
|
| 2727 | return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
|
| 2728 | }
|
| 2729 |
|
| 2730 | void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
|
| 2731 | const struct amdgpu_buffer_funcs *buffer_funcs)
|
| 2732 | {
|
| 2733 | struct drm_gpu_scheduler *sched;
|
| 2734 | struct amdgpu_vmhub *hub;
|
| 2735 | int i, n;
|
| 2736 |
|
| 2737 | adev->mman.buffer_funcs = buffer_funcs;
|
| 2738 |
|
| 2739 | for (i = 0, n = 0; i < adev->sdma.num_instances; i++) {
|
| 2740 | if (adev->sdma.has_page_queue)
|
| 2741 | sched = &adev->sdma.instance[i].page.sched;
|
| 2742 | else
|
| 2743 | sched = &adev->sdma.instance[i].ring.sched;
|
| 2744 |
|
| 2745 | if (!sched->ready)
|
| 2746 | continue;
|
| 2747 |
|
| 2748 | adev->mman.buffer_funcs_scheds[n++] = sched;
|
| 2749 | }
|
| 2750 |
|
| 2751 | if (n == 0) {
|
| 2752 | adev->mman.num_buffer_funcs_scheds = 0;
|
| 2753 | drm_warn(&adev->ddev, "No working sdma ring available\n");
|
| 2754 | return;
|
| 2755 | }
|
| 2756 |
|
| 2757 | hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
|
| 2758 |
|
| 2759 | |
| 2760 | * Allow using multiple SDMA schedulers only on GPUs where
|
| 2761 | * we are allowed to do concurrent VM flushes.
|
| 2762 | * This consideration is necessary because all GART windows
|
| 2763 | * are mapped in VMID 0 (the kernel VMID) so each buffer
|
| 2764 | * entity would flush VMID 0 concurrently.
|
| 2765 | *
|
| 2766 | * Also consider the SDMA invalidation workaround on
|
| 2767 | * Navi 1x GPUs, which also prevents us from using
|
| 2768 | * multiple SDMA engines on VMID 0 at the same time.
|
| 2769 |
|
| 2770 | adev->mman.num_buffer_funcs_scheds =
|
| 2771 | (adev->vm_manager.concurrent_flush &&
|
| 2772 | !hub->sdma_invalidation_workaround) ? n : 1;
|
| 2773 | }
|
| 2774 |
|
| 2775 | #if defined(CONFIG_DEBUG_FS)
|
| 2776 |
|
| 2777 | static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)
|
| 2778 | {
|
| 2779 | struct amdgpu_device *adev = m->private;
|
| 2780 |
|
| 2781 | return ttm_pool_debugfs(&adev->mman.bdev.pool, m);
|
| 2782 | }
|
| 2783 |
|
| 2784 | DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool);
|
| 2785 |
|
| 2786 | |
| 2787 | * amdgpu_ttm_vram_read - Linear read access to VRAM
|
| 2788 | *
|
| 2789 | * Accesses VRAM via MMIO for debugging purposes.
|
| 2790 |
|
| 2791 | static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
|
| 2792 | size_t size, loff_t *pos)
|
| 2793 | {
|
| 2794 | struct amdgpu_device *adev = file_inode(f)->i_private;
|
| 2795 | ssize_t result = 0;
|
| 2796 |
|
| 2797 | if (size & 0x3 || *pos & 0x3)
|
| 2798 | return -EINVAL;
|
| 2799 |
|
| 2800 | if (*pos >= adev->gmc.mc_vram_size)
|
| 2801 | return -ENXIO;
|
| 2802 |
|
| 2803 | size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos));
|
| 2804 | while (size) {
|
| 2805 | size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4);
|
| 2806 | uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ];
|
| 2807 |
|
| 2808 | amdgpu_device_vram_access(adev, *pos, value, bytes, false);
|
| 2809 | if (copy_to_user(buf, value, bytes))
|
| 2810 | return -EFAULT;
|
| 2811 |
|
| 2812 | result += bytes;
|
| 2813 | buf += bytes;
|
| 2814 | *pos += bytes;
|
| 2815 | size -= bytes;
|
| 2816 | }
|
| 2817 |
|
| 2818 | return result;
|
| 2819 | }
|
| 2820 |
|
| 2821 | |
| 2822 | * amdgpu_ttm_vram_write - Linear write access to VRAM
|
| 2823 | *
|
| 2824 | * Accesses VRAM via MMIO for debugging purposes.
|
| 2825 |
|
| 2826 | static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
|
| 2827 | size_t size, loff_t *pos)
|
| 2828 | {
|
| 2829 | struct amdgpu_device *adev = file_inode(f)->i_private;
|
| 2830 | ssize_t result = 0;
|
| 2831 | int r;
|
| 2832 |
|
| 2833 | if (size & 0x3 || *pos & 0x3)
|
| 2834 | return -EINVAL;
|
| 2835 |
|
| 2836 | if (*pos >= adev->gmc.mc_vram_size)
|
| 2837 | return -ENXIO;
|
| 2838 |
|
| 2839 | while (size) {
|
| 2840 | uint32_t value;
|
| 2841 |
|
| 2842 | if (*pos >= adev->gmc.mc_vram_size)
|
| 2843 | return result;
|
| 2844 |
|
| 2845 | r = get_user(value, (uint32_t *)buf);
|
| 2846 | if (r)
|
| 2847 | return r;
|
| 2848 |
|
| 2849 | amdgpu_device_mm_access(adev, *pos, &value, 4, true);
|
| 2850 |
|
| 2851 | result += 4;
|
| 2852 | buf += 4;
|
| 2853 | *pos += 4;
|
| 2854 | size -= 4;
|
| 2855 | }
|
| 2856 |
|
| 2857 | return result;
|
| 2858 | }
|
| 2859 |
|
| 2860 | static const struct file_operations amdgpu_ttm_vram_fops = {
|
| 2861 | .owner = THIS_MODULE,
|
| 2862 | .read = amdgpu_ttm_vram_read,
|
| 2863 | .write = amdgpu_ttm_vram_write,
|
| 2864 | .llseek = default_llseek,
|
| 2865 | };
|
| 2866 |
|
| 2867 | |
| 2868 | * amdgpu_iomem_read - Virtual read access to GPU mapped memory
|
| 2869 | *
|
| 2870 | * This function is used to read memory that has been mapped to the
|
| 2871 | * GPU and the known addresses are not physical addresses but instead
|
| 2872 | * bus addresses (e.g., what you'd put in an IB or ring buffer).
|
| 2873 |
|
| 2874 | static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
|
| 2875 | size_t size, loff_t *pos)
|
| 2876 | {
|
| 2877 | struct amdgpu_device *adev = file_inode(f)->i_private;
|
| 2878 | struct iommu_domain *dom;
|
| 2879 | ssize_t result = 0;
|
| 2880 | int r;
|
| 2881 |
|
| 2882 |
|
| 2883 | dom = iommu_get_domain_for_dev(adev->dev);
|
| 2884 |
|
| 2885 | while (size) {
|
| 2886 | phys_addr_t addr = *pos & PAGE_MASK;
|
| 2887 | loff_t off = *pos & ~PAGE_MASK;
|
| 2888 | size_t bytes = PAGE_SIZE - off;
|
| 2889 | unsigned long pfn;
|
| 2890 | struct page *p;
|
| 2891 | void *ptr;
|
| 2892 |
|
| 2893 | bytes = min(bytes, size);
|
| 2894 |
|
| 2895 | |
| 2896 | * the domain is NULL it means there is no IOMMU active
|
| 2897 | * and the address translation is the identity
|
| 2898 |
|
| 2899 | addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
|
| 2900 |
|
| 2901 | pfn = addr >> PAGE_SHIFT;
|
| 2902 | if (!pfn_valid(pfn))
|
| 2903 | return -EPERM;
|
| 2904 |
|
| 2905 | p = pfn_to_page(pfn);
|
| 2906 | if (p->mapping != adev->mman.bdev.dev_mapping)
|
| 2907 | return -EPERM;
|
| 2908 |
|
| 2909 | ptr = kmap_local_page(p);
|
| 2910 | r = copy_to_user(buf, ptr + off, bytes);
|
| 2911 | kunmap_local(ptr);
|
| 2912 | if (r)
|
| 2913 | return -EFAULT;
|
| 2914 |
|
| 2915 | size -= bytes;
|
| 2916 | *pos += bytes;
|
| 2917 | result += bytes;
|
| 2918 | }
|
| 2919 |
|
| 2920 | return result;
|
| 2921 | }
|
| 2922 |
|
| 2923 | |
| 2924 | * amdgpu_iomem_write - Virtual write access to GPU mapped memory
|
| 2925 | *
|
| 2926 | * This function is used to write memory that has been mapped to the
|
| 2927 | * GPU and the known addresses are not physical addresses but instead
|
| 2928 | * bus addresses (e.g., what you'd put in an IB or ring buffer).
|
| 2929 |
|
| 2930 | static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
|
| 2931 | size_t size, loff_t *pos)
|
| 2932 | {
|
| 2933 | struct amdgpu_device *adev = file_inode(f)->i_private;
|
| 2934 | struct iommu_domain *dom;
|
| 2935 | ssize_t result = 0;
|
| 2936 | int r;
|
| 2937 |
|
| 2938 | dom = iommu_get_domain_for_dev(adev->dev);
|
| 2939 |
|
| 2940 | while (size) {
|
| 2941 | phys_addr_t addr = *pos & PAGE_MASK;
|
| 2942 | loff_t off = *pos & ~PAGE_MASK;
|
| 2943 | size_t bytes = PAGE_SIZE - off;
|
| 2944 | unsigned long pfn;
|
| 2945 | struct page *p;
|
| 2946 | void *ptr;
|
| 2947 |
|
| 2948 | bytes = min(bytes, size);
|
| 2949 |
|
| 2950 | addr = dom ? iommu_iova_to_phys(dom, addr) : addr;
|
| 2951 |
|
| 2952 | pfn = addr >> PAGE_SHIFT;
|
| 2953 | if (!pfn_valid(pfn))
|
| 2954 | return -EPERM;
|
| 2955 |
|
| 2956 | p = pfn_to_page(pfn);
|
| 2957 | if (p->mapping != adev->mman.bdev.dev_mapping)
|
| 2958 | return -EPERM;
|
| 2959 |
|
| 2960 | ptr = kmap_local_page(p);
|
| 2961 | r = copy_from_user(ptr + off, buf, bytes);
|
| 2962 | kunmap_local(ptr);
|
| 2963 | if (r)
|
| 2964 | return -EFAULT;
|
| 2965 |
|
| 2966 | size -= bytes;
|
| 2967 | *pos += bytes;
|
| 2968 | result += bytes;
|
| 2969 | }
|
| 2970 |
|
| 2971 | return result;
|
| 2972 | }
|
| 2973 |
|
| 2974 | static const struct file_operations amdgpu_ttm_iomem_fops = {
|
| 2975 | .owner = THIS_MODULE,
|
| 2976 | .read = amdgpu_iomem_read,
|
| 2977 | .write = amdgpu_iomem_write,
|
| 2978 | .llseek = default_llseek
|
| 2979 | };
|
| 2980 |
|
| 2981 | #endif
|
| 2982 |
|
| 2983 | void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
|
| 2984 | {
|
| 2985 | #if defined(CONFIG_DEBUG_FS)
|
| 2986 | struct drm_minor *minor = adev_to_drm(adev)->primary;
|
| 2987 | struct dentry *root = minor->debugfs_root;
|
| 2988 |
|
| 2989 | debugfs_create_file_size("amdgpu_vram", 0444, root, adev,
|
| 2990 | &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size);
|
| 2991 | debugfs_create_file("amdgpu_iomem", 0444, root, adev,
|
| 2992 | &amdgpu_ttm_iomem_fops);
|
| 2993 | debugfs_create_file("ttm_page_pool", 0444, root, adev,
|
| 2994 | &amdgpu_ttm_page_pool_fops);
|
| 2995 | ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
|
| 2996 | TTM_PL_VRAM),
|
| 2997 | root, "amdgpu_vram_mm");
|
| 2998 | ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
|
| 2999 | TTM_PL_TT),
|
| 3000 | root, "amdgpu_gtt_mm");
|
| 3001 | ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
|
| 3002 | AMDGPU_PL_GDS),
|
| 3003 | root, "amdgpu_gds_mm");
|
| 3004 | ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
|
| 3005 | AMDGPU_PL_GWS),
|
| 3006 | root, "amdgpu_gws_mm");
|
| 3007 | ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev,
|
| 3008 | AMDGPU_PL_OA),
|
| 3009 | root, "amdgpu_oa_mm");
|
| 3010 |
|
| 3011 | #endif
|
| 3012 | }
|