| 1 | |
| 2 | |
| 3 | * NVIDIA Tegra DRM GEM helper functions
|
| 4 | *
|
| 5 | * Copyright (C) 2012 Sascha Hauer, Pengutronix
|
| 6 | * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved.
|
| 7 | *
|
| 8 | * Based on the GEM/CMA helpers
|
| 9 | *
|
| 10 | * Copyright (c) 2011 Samsung Electronics Co., Ltd.
|
| 11 |
|
| 12 |
|
| 13 | #include <linux/dma-buf.h>
|
| 14 | #include <linux/iommu.h>
|
| 15 | #include <linux/module.h>
|
| 16 | #include <linux/pagemap.h>
|
| 17 | #include <linux/vmalloc.h>
|
| 18 |
|
| 19 | #include <drm/drm_drv.h>
|
| 20 | #include <drm/drm_dumb_buffers.h>
|
| 21 | #include <drm/drm_prime.h>
|
| 22 |
|
| 23 | #include "drm.h"
|
| 24 | #include "gem.h"
|
| 25 |
|
| 26 | MODULE_IMPORT_NS("DMA_BUF");
|
| 27 |
|
| 28 | static unsigned int sg_dma_count_chunks(struct scatterlist *sgl, unsigned int nents)
|
| 29 | {
|
| 30 | dma_addr_t next = ~(dma_addr_t)0;
|
| 31 | unsigned int count = 0, i;
|
| 32 | struct scatterlist *s;
|
| 33 |
|
| 34 | for_each_sg(sgl, s, nents, i) {
|
| 35 |
|
| 36 | if (!sg_dma_len(s))
|
| 37 | continue;
|
| 38 |
|
| 39 | if (sg_dma_address(s) != next) {
|
| 40 | next = sg_dma_address(s) + sg_dma_len(s);
|
| 41 | count++;
|
| 42 | }
|
| 43 | }
|
| 44 |
|
| 45 | return count;
|
| 46 | }
|
| 47 |
|
| 48 | static inline unsigned int sgt_dma_count_chunks(struct sg_table *sgt)
|
| 49 | {
|
| 50 | return sg_dma_count_chunks(sgt->sgl, sgt->nents);
|
| 51 | }
|
| 52 |
|
| 53 | static void tegra_bo_put(struct host1x_bo *bo)
|
| 54 | {
|
| 55 | struct tegra_bo *obj = host1x_to_tegra_bo(bo);
|
| 56 |
|
| 57 | drm_gem_object_put(&obj->gem);
|
| 58 | }
|
| 59 |
|
| 60 | static struct host1x_bo_mapping *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,
|
| 61 | enum dma_data_direction direction)
|
| 62 | {
|
| 63 | struct tegra_bo *obj = host1x_to_tegra_bo(bo);
|
| 64 | struct drm_gem_object *gem = &obj->gem;
|
| 65 | struct host1x_bo_mapping *map;
|
| 66 | int err;
|
| 67 |
|
| 68 | map = kzalloc_obj(*map);
|
| 69 | if (!map)
|
| 70 | return ERR_PTR(-ENOMEM);
|
| 71 |
|
| 72 | kref_init(&map->ref);
|
| 73 | map->bo = bo;
|
| 74 | map->direction = direction;
|
| 75 | map->dev = dev;
|
| 76 |
|
| 77 | |
| 78 | * Imported buffers need special treatment to satisfy the semantics of DMA-BUF.
|
| 79 |
|
| 80 | if (obj->dma_buf) {
|
| 81 | struct dma_buf *buf = obj->dma_buf;
|
| 82 |
|
| 83 | map->attach = dma_buf_attach(buf, dev);
|
| 84 | if (IS_ERR(map->attach)) {
|
| 85 | err = PTR_ERR(map->attach);
|
| 86 | goto free;
|
| 87 | }
|
| 88 |
|
| 89 | map->sgt = dma_buf_map_attachment_unlocked(map->attach, direction);
|
| 90 | if (IS_ERR(map->sgt)) {
|
| 91 | dma_buf_detach(buf, map->attach);
|
| 92 | err = PTR_ERR(map->sgt);
|
| 93 | map->sgt = NULL;
|
| 94 | goto free;
|
| 95 | }
|
| 96 |
|
| 97 | err = sgt_dma_count_chunks(map->sgt);
|
| 98 | map->size = gem->size;
|
| 99 |
|
| 100 | goto out;
|
| 101 | }
|
| 102 |
|
| 103 | |
| 104 | * If we don't have a mapping for this buffer yet, return an SG table
|
| 105 | * so that host1x can do the mapping for us via the DMA API.
|
| 106 |
|
| 107 | map->sgt = kzalloc_obj(*map->sgt);
|
| 108 | if (!map->sgt) {
|
| 109 | err = -ENOMEM;
|
| 110 | goto free;
|
| 111 | }
|
| 112 |
|
| 113 | if (obj->pages) {
|
| 114 | |
| 115 | * If the buffer object was allocated from the explicit IOMMU
|
| 116 | * API code paths, construct an SG table from the pages.
|
| 117 |
|
| 118 | err = sg_alloc_table_from_pages(map->sgt, obj->pages, obj->num_pages, 0, gem->size,
|
| 119 | GFP_KERNEL);
|
| 120 | if (err < 0)
|
| 121 | goto free;
|
| 122 | } else {
|
| 123 | |
| 124 | * If the buffer object had no pages allocated and if it was
|
| 125 | * not imported, it had to be allocated with the DMA API, so
|
| 126 | * the DMA API helper can be used.
|
| 127 |
|
| 128 | err = dma_get_sgtable(dev, map->sgt, obj->vaddr, obj->iova, gem->size);
|
| 129 | if (err < 0)
|
| 130 | goto free;
|
| 131 | }
|
| 132 |
|
| 133 | err = dma_map_sgtable(dev, map->sgt, direction, 0);
|
| 134 | if (err)
|
| 135 | goto free_sgt;
|
| 136 |
|
| 137 | out:
|
| 138 | |
| 139 | * If we've manually mapped the buffer object through the IOMMU, make sure to return the
|
| 140 | * existing IOVA address of our mapping.
|
| 141 |
|
| 142 | if (!obj->mm) {
|
| 143 | map->phys = sg_dma_address(map->sgt->sgl);
|
| 144 | map->chunks = err;
|
| 145 | } else {
|
| 146 | map->phys = obj->iova;
|
| 147 | map->chunks = 1;
|
| 148 | }
|
| 149 |
|
| 150 | map->size = gem->size;
|
| 151 |
|
| 152 | return map;
|
| 153 |
|
| 154 | free_sgt:
|
| 155 | sg_free_table(map->sgt);
|
| 156 | free:
|
| 157 | kfree(map->sgt);
|
| 158 | kfree(map);
|
| 159 | return ERR_PTR(err);
|
| 160 | }
|
| 161 |
|
| 162 | static void tegra_bo_unpin(struct host1x_bo_mapping *map)
|
| 163 | {
|
| 164 | if (map->attach) {
|
| 165 | dma_buf_unmap_attachment_unlocked(map->attach, map->sgt,
|
| 166 | map->direction);
|
| 167 | dma_buf_detach(map->attach->dmabuf, map->attach);
|
| 168 | } else {
|
| 169 | dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
|
| 170 | sg_free_table(map->sgt);
|
| 171 | kfree(map->sgt);
|
| 172 | }
|
| 173 |
|
| 174 | kfree(map);
|
| 175 | }
|
| 176 |
|
| 177 | static void *tegra_bo_mmap(struct host1x_bo *bo)
|
| 178 | {
|
| 179 | struct tegra_bo *obj = host1x_to_tegra_bo(bo);
|
| 180 | struct iosys_map map = { 0 };
|
| 181 | void *vaddr;
|
| 182 | int ret;
|
| 183 |
|
| 184 | if (obj->vaddr)
|
| 185 | return obj->vaddr;
|
| 186 |
|
| 187 | if (obj->dma_buf) {
|
| 188 | ret = dma_buf_vmap_unlocked(obj->dma_buf, &map);
|
| 189 | if (ret < 0)
|
| 190 | return ERR_PTR(ret);
|
| 191 |
|
| 192 | return map.vaddr;
|
| 193 | }
|
| 194 |
|
| 195 | vaddr = vmap(obj->pages, obj->num_pages, VM_MAP,
|
| 196 | pgprot_writecombine(PAGE_KERNEL));
|
| 197 | if (!vaddr)
|
| 198 | return ERR_PTR(-ENOMEM);
|
| 199 |
|
| 200 | return vaddr;
|
| 201 | }
|
| 202 |
|
| 203 | static void tegra_bo_munmap(struct host1x_bo *bo, void *addr)
|
| 204 | {
|
| 205 | struct tegra_bo *obj = host1x_to_tegra_bo(bo);
|
| 206 | struct iosys_map map = IOSYS_MAP_INIT_VADDR(addr);
|
| 207 |
|
| 208 | if (obj->vaddr)
|
| 209 | return;
|
| 210 |
|
| 211 | if (obj->dma_buf)
|
| 212 | return dma_buf_vunmap_unlocked(obj->dma_buf, &map);
|
| 213 |
|
| 214 | vunmap(addr);
|
| 215 | }
|
| 216 |
|
| 217 | static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
|
| 218 | {
|
| 219 | struct tegra_bo *obj = host1x_to_tegra_bo(bo);
|
| 220 |
|
| 221 | drm_gem_object_get(&obj->gem);
|
| 222 |
|
| 223 | return bo;
|
| 224 | }
|
| 225 |
|
| 226 | static const struct host1x_bo_ops tegra_bo_ops = {
|
| 227 | .get = tegra_bo_get,
|
| 228 | .put = tegra_bo_put,
|
| 229 | .pin = tegra_bo_pin,
|
| 230 | .unpin = tegra_bo_unpin,
|
| 231 | .mmap = tegra_bo_mmap,
|
| 232 | .munmap = tegra_bo_munmap,
|
| 233 | };
|
| 234 |
|
| 235 | static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo)
|
| 236 | {
|
| 237 | int prot = IOMMU_READ | IOMMU_WRITE;
|
| 238 | ssize_t size;
|
| 239 | int err;
|
| 240 |
|
| 241 | if (bo->mm)
|
| 242 | return -EBUSY;
|
| 243 |
|
| 244 | bo->mm = kzalloc_obj(*bo->mm);
|
| 245 | if (!bo->mm)
|
| 246 | return -ENOMEM;
|
| 247 |
|
| 248 | mutex_lock(&tegra->mm_lock);
|
| 249 |
|
| 250 | err = drm_mm_insert_node_generic(&tegra->mm,
|
| 251 | bo->mm, bo->gem.size, PAGE_SIZE, 0, 0);
|
| 252 | if (err < 0) {
|
| 253 | dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n",
|
| 254 | err);
|
| 255 | goto unlock;
|
| 256 | }
|
| 257 |
|
| 258 | bo->iova = bo->mm->start;
|
| 259 |
|
| 260 | size = iommu_map_sgtable(tegra->domain, bo->iova, bo->sgt, prot);
|
| 261 | if (size < 0) {
|
| 262 | dev_err(tegra->drm->dev, "failed to map buffer\n");
|
| 263 | err = size;
|
| 264 | goto remove;
|
| 265 | }
|
| 266 |
|
| 267 | bo->size = size;
|
| 268 |
|
| 269 | mutex_unlock(&tegra->mm_lock);
|
| 270 |
|
| 271 | return 0;
|
| 272 |
|
| 273 | remove:
|
| 274 | drm_mm_remove_node(bo->mm);
|
| 275 | unlock:
|
| 276 | mutex_unlock(&tegra->mm_lock);
|
| 277 | kfree(bo->mm);
|
| 278 | return err;
|
| 279 | }
|
| 280 |
|
| 281 | static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo)
|
| 282 | {
|
| 283 | if (!bo->mm)
|
| 284 | return 0;
|
| 285 |
|
| 286 | mutex_lock(&tegra->mm_lock);
|
| 287 | iommu_unmap(tegra->domain, bo->iova, bo->size);
|
| 288 | drm_mm_remove_node(bo->mm);
|
| 289 | mutex_unlock(&tegra->mm_lock);
|
| 290 |
|
| 291 | kfree(bo->mm);
|
| 292 |
|
| 293 | return 0;
|
| 294 | }
|
| 295 |
|
| 296 | static const struct drm_gem_object_funcs tegra_gem_object_funcs = {
|
| 297 | .free = tegra_bo_free_object,
|
| 298 | .export = tegra_gem_prime_export,
|
| 299 | .vm_ops = &tegra_bo_vm_ops,
|
| 300 | };
|
| 301 |
|
| 302 | static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm,
|
| 303 | size_t size)
|
| 304 | {
|
| 305 | struct tegra_bo *bo;
|
| 306 | int err;
|
| 307 |
|
| 308 | bo = kzalloc_obj(*bo);
|
| 309 | if (!bo)
|
| 310 | return ERR_PTR(-ENOMEM);
|
| 311 |
|
| 312 | bo->gem.funcs = &tegra_gem_object_funcs;
|
| 313 |
|
| 314 | host1x_bo_init(&bo->base, &tegra_bo_ops);
|
| 315 | size = round_up(size, PAGE_SIZE);
|
| 316 |
|
| 317 | err = drm_gem_object_init(drm, &bo->gem, size);
|
| 318 | if (err < 0)
|
| 319 | goto free;
|
| 320 |
|
| 321 | err = drm_gem_create_mmap_offset(&bo->gem);
|
| 322 | if (err < 0)
|
| 323 | goto release;
|
| 324 |
|
| 325 | return bo;
|
| 326 |
|
| 327 | release:
|
| 328 | drm_gem_object_release(&bo->gem);
|
| 329 | free:
|
| 330 | kfree(bo);
|
| 331 | return ERR_PTR(err);
|
| 332 | }
|
| 333 |
|
| 334 | static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo)
|
| 335 | {
|
| 336 | if (bo->pages) {
|
| 337 | dma_unmap_sgtable(drm->dev, bo->sgt, DMA_FROM_DEVICE, 0);
|
| 338 | drm_gem_put_pages(&bo->gem, bo->pages, true, true);
|
| 339 | sg_free_table(bo->sgt);
|
| 340 | kfree(bo->sgt);
|
| 341 | } else if (bo->vaddr) {
|
| 342 | dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->iova);
|
| 343 | }
|
| 344 | }
|
| 345 |
|
| 346 | static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo)
|
| 347 | {
|
| 348 | int err;
|
| 349 |
|
| 350 | bo->pages = drm_gem_get_pages(&bo->gem);
|
| 351 | if (IS_ERR(bo->pages))
|
| 352 | return PTR_ERR(bo->pages);
|
| 353 |
|
| 354 | bo->num_pages = bo->gem.size >> PAGE_SHIFT;
|
| 355 |
|
| 356 | bo->sgt = drm_prime_pages_to_sg(bo->gem.dev, bo->pages, bo->num_pages);
|
| 357 | if (IS_ERR(bo->sgt)) {
|
| 358 | err = PTR_ERR(bo->sgt);
|
| 359 | goto put_pages;
|
| 360 | }
|
| 361 |
|
| 362 | err = dma_map_sgtable(drm->dev, bo->sgt, DMA_FROM_DEVICE, 0);
|
| 363 | if (err)
|
| 364 | goto free_sgt;
|
| 365 |
|
| 366 | return 0;
|
| 367 |
|
| 368 | free_sgt:
|
| 369 | sg_free_table(bo->sgt);
|
| 370 | kfree(bo->sgt);
|
| 371 | put_pages:
|
| 372 | drm_gem_put_pages(&bo->gem, bo->pages, false, false);
|
| 373 | return err;
|
| 374 | }
|
| 375 |
|
| 376 | static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo)
|
| 377 | {
|
| 378 | struct tegra_drm *tegra = drm->dev_private;
|
| 379 | int err;
|
| 380 |
|
| 381 | if (tegra->domain) {
|
| 382 | err = tegra_bo_get_pages(drm, bo);
|
| 383 | if (err < 0)
|
| 384 | return err;
|
| 385 |
|
| 386 | err = tegra_bo_iommu_map(tegra, bo);
|
| 387 | if (err < 0) {
|
| 388 | tegra_bo_free(drm, bo);
|
| 389 | return err;
|
| 390 | }
|
| 391 | } else {
|
| 392 | size_t size = bo->gem.size;
|
| 393 |
|
| 394 | bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->iova,
|
| 395 | GFP_KERNEL | __GFP_NOWARN);
|
| 396 | if (!bo->vaddr) {
|
| 397 | dev_err(drm->dev,
|
| 398 | "failed to allocate buffer of size %zu\n",
|
| 399 | size);
|
| 400 | return -ENOMEM;
|
| 401 | }
|
| 402 | }
|
| 403 |
|
| 404 | return 0;
|
| 405 | }
|
| 406 |
|
| 407 | struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size,
|
| 408 | unsigned long flags)
|
| 409 | {
|
| 410 | struct tegra_bo *bo;
|
| 411 | int err;
|
| 412 |
|
| 413 | bo = tegra_bo_alloc_object(drm, size);
|
| 414 | if (IS_ERR(bo))
|
| 415 | return bo;
|
| 416 |
|
| 417 | err = tegra_bo_alloc(drm, bo);
|
| 418 | if (err < 0)
|
| 419 | goto release;
|
| 420 |
|
| 421 | if (flags & DRM_TEGRA_GEM_CREATE_TILED)
|
| 422 | bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED;
|
| 423 |
|
| 424 | if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP)
|
| 425 | bo->flags |= TEGRA_BO_BOTTOM_UP;
|
| 426 |
|
| 427 | return bo;
|
| 428 |
|
| 429 | release:
|
| 430 | drm_gem_object_release(&bo->gem);
|
| 431 | kfree(bo);
|
| 432 | return ERR_PTR(err);
|
| 433 | }
|
| 434 |
|
| 435 | struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
|
| 436 | struct drm_device *drm,
|
| 437 | size_t size,
|
| 438 | unsigned long flags,
|
| 439 | u32 *handle)
|
| 440 | {
|
| 441 | struct tegra_bo *bo;
|
| 442 | int err;
|
| 443 |
|
| 444 | bo = tegra_bo_create(drm, size, flags);
|
| 445 | if (IS_ERR(bo))
|
| 446 | return bo;
|
| 447 |
|
| 448 | err = drm_gem_handle_create(file, &bo->gem, handle);
|
| 449 | if (err) {
|
| 450 | tegra_bo_free_object(&bo->gem);
|
| 451 | return ERR_PTR(err);
|
| 452 | }
|
| 453 |
|
| 454 | drm_gem_object_put(&bo->gem);
|
| 455 |
|
| 456 | return bo;
|
| 457 | }
|
| 458 |
|
| 459 | static struct tegra_bo *tegra_bo_import(struct drm_device *drm,
|
| 460 | struct dma_buf *buf)
|
| 461 | {
|
| 462 | struct tegra_drm *tegra = drm->dev_private;
|
| 463 | struct dma_buf_attachment *attach;
|
| 464 | struct tegra_bo *bo;
|
| 465 | int err;
|
| 466 |
|
| 467 | bo = tegra_bo_alloc_object(drm, buf->size);
|
| 468 | if (IS_ERR(bo))
|
| 469 | return bo;
|
| 470 |
|
| 471 | |
| 472 | * If we need to use IOMMU API to map the dma-buf into the internally managed
|
| 473 | * domain, map it first to the DRM device to get an sgt.
|
| 474 |
|
| 475 | if (tegra->domain) {
|
| 476 | attach = dma_buf_attach(buf, drm->dev);
|
| 477 | if (IS_ERR(attach)) {
|
| 478 | err = PTR_ERR(attach);
|
| 479 | goto free;
|
| 480 | }
|
| 481 |
|
| 482 | bo->sgt = dma_buf_map_attachment_unlocked(attach, DMA_TO_DEVICE);
|
| 483 | if (IS_ERR(bo->sgt)) {
|
| 484 | err = PTR_ERR(bo->sgt);
|
| 485 | goto detach;
|
| 486 | }
|
| 487 |
|
| 488 | err = tegra_bo_iommu_map(tegra, bo);
|
| 489 | if (err < 0)
|
| 490 | goto detach;
|
| 491 |
|
| 492 | bo->gem.import_attach = attach;
|
| 493 | }
|
| 494 |
|
| 495 | get_dma_buf(buf);
|
| 496 | bo->dma_buf = buf;
|
| 497 |
|
| 498 | return bo;
|
| 499 |
|
| 500 | detach:
|
| 501 | if (!IS_ERR_OR_NULL(bo->sgt))
|
| 502 | dma_buf_unmap_attachment_unlocked(attach, bo->sgt, DMA_TO_DEVICE);
|
| 503 |
|
| 504 | dma_buf_detach(buf, attach);
|
| 505 | dma_buf_put(buf);
|
| 506 | free:
|
| 507 | drm_gem_object_release(&bo->gem);
|
| 508 | kfree(bo);
|
| 509 | return ERR_PTR(err);
|
| 510 | }
|
| 511 |
|
| 512 | void tegra_bo_free_object(struct drm_gem_object *gem)
|
| 513 | {
|
| 514 | struct tegra_drm *tegra = gem->dev->dev_private;
|
| 515 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 516 |
|
| 517 | host1x_bo_clear_cached_mappings(&bo->base);
|
| 518 |
|
| 519 | if (tegra->domain) {
|
| 520 | tegra_bo_iommu_unmap(tegra, bo);
|
| 521 |
|
| 522 | if (drm_gem_is_imported(gem)) {
|
| 523 | dma_buf_unmap_attachment_unlocked(gem->import_attach, bo->sgt,
|
| 524 | DMA_TO_DEVICE);
|
| 525 | dma_buf_detach(gem->import_attach->dmabuf, gem->import_attach);
|
| 526 | }
|
| 527 | }
|
| 528 |
|
| 529 | tegra_bo_free(gem->dev, bo);
|
| 530 |
|
| 531 | if (bo->dma_buf)
|
| 532 | dma_buf_put(bo->dma_buf);
|
| 533 |
|
| 534 | drm_gem_object_release(gem);
|
| 535 | kfree(bo);
|
| 536 | }
|
| 537 |
|
| 538 | int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
|
| 539 | struct drm_mode_create_dumb *args)
|
| 540 | {
|
| 541 | struct tegra_drm *tegra = drm->dev_private;
|
| 542 | struct tegra_bo *bo;
|
| 543 | int ret;
|
| 544 |
|
| 545 | ret = drm_mode_size_dumb(drm, args, tegra->pitch_align, 0);
|
| 546 | if (ret)
|
| 547 | return ret;
|
| 548 |
|
| 549 | bo = tegra_bo_create_with_handle(file, drm, args->size, 0,
|
| 550 | &args->handle);
|
| 551 | if (IS_ERR(bo))
|
| 552 | return PTR_ERR(bo);
|
| 553 |
|
| 554 | return 0;
|
| 555 | }
|
| 556 |
|
| 557 | static vm_fault_t tegra_bo_fault(struct vm_fault *vmf)
|
| 558 | {
|
| 559 | struct vm_area_struct *vma = vmf->vma;
|
| 560 | struct drm_gem_object *gem = vma->vm_private_data;
|
| 561 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 562 | struct page *page;
|
| 563 | pgoff_t offset;
|
| 564 |
|
| 565 | if (!bo->pages)
|
| 566 | return VM_FAULT_SIGBUS;
|
| 567 |
|
| 568 | offset = linear_page_delta(vma, vmf->address);
|
| 569 | page = bo->pages[offset];
|
| 570 |
|
| 571 | return vmf_insert_page(vma, vmf->address, page);
|
| 572 | }
|
| 573 |
|
| 574 | const struct vm_operations_struct tegra_bo_vm_ops = {
|
| 575 | .fault = tegra_bo_fault,
|
| 576 | .open = drm_gem_vm_open,
|
| 577 | .close = drm_gem_vm_close,
|
| 578 | };
|
| 579 |
|
| 580 | int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma)
|
| 581 | {
|
| 582 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 583 |
|
| 584 | if (!bo->pages) {
|
| 585 | unsigned long vm_pgoff = vma->vm_pgoff;
|
| 586 | int err;
|
| 587 |
|
| 588 | |
| 589 | * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(),
|
| 590 | * and set the vm_pgoff (used as a fake buffer offset by DRM)
|
| 591 | * to 0 as we want to map the whole buffer.
|
| 592 |
|
| 593 | vm_flags_clear(vma, VM_PFNMAP);
|
| 594 | vma->vm_pgoff = 0;
|
| 595 |
|
| 596 | err = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->iova,
|
| 597 | gem->size);
|
| 598 | if (err < 0) {
|
| 599 | drm_gem_vm_close(vma);
|
| 600 | return err;
|
| 601 | }
|
| 602 |
|
| 603 | vma->vm_pgoff = vm_pgoff;
|
| 604 | } else {
|
| 605 | pgprot_t prot = vma_get_page_prot(vma);
|
| 606 |
|
| 607 | vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
|
| 608 |
|
| 609 | vma->vm_page_prot = pgprot_writecombine(prot);
|
| 610 | }
|
| 611 |
|
| 612 | return 0;
|
| 613 | }
|
| 614 |
|
| 615 | int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
|
| 616 | {
|
| 617 | struct drm_gem_object *gem;
|
| 618 | int err;
|
| 619 |
|
| 620 | err = drm_gem_mmap(file, vma);
|
| 621 | if (err < 0)
|
| 622 | return err;
|
| 623 |
|
| 624 | gem = vma->vm_private_data;
|
| 625 |
|
| 626 | return __tegra_gem_mmap(gem, vma);
|
| 627 | }
|
| 628 |
|
| 629 | static struct sg_table *
|
| 630 | tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
|
| 631 | enum dma_data_direction dir)
|
| 632 | {
|
| 633 | struct drm_gem_object *gem = attach->dmabuf->priv;
|
| 634 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 635 | struct sg_table *sgt;
|
| 636 |
|
| 637 | sgt = kmalloc_obj(*sgt);
|
| 638 | if (!sgt)
|
| 639 | return NULL;
|
| 640 |
|
| 641 | if (bo->pages) {
|
| 642 | if (sg_alloc_table_from_pages(sgt, bo->pages, bo->num_pages,
|
| 643 | 0, gem->size, GFP_KERNEL) < 0)
|
| 644 | goto free;
|
| 645 | } else {
|
| 646 | if (dma_get_sgtable(attach->dev, sgt, bo->vaddr, bo->iova,
|
| 647 | gem->size) < 0)
|
| 648 | goto free;
|
| 649 | }
|
| 650 |
|
| 651 | if (dma_map_sgtable(attach->dev, sgt, dir, 0))
|
| 652 | goto free;
|
| 653 |
|
| 654 | return sgt;
|
| 655 |
|
| 656 | free:
|
| 657 | sg_free_table(sgt);
|
| 658 | kfree(sgt);
|
| 659 | return NULL;
|
| 660 | }
|
| 661 |
|
| 662 | static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
|
| 663 | struct sg_table *sgt,
|
| 664 | enum dma_data_direction dir)
|
| 665 | {
|
| 666 | struct drm_gem_object *gem = attach->dmabuf->priv;
|
| 667 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 668 |
|
| 669 | if (bo->pages)
|
| 670 | dma_unmap_sgtable(attach->dev, sgt, dir, 0);
|
| 671 |
|
| 672 | sg_free_table(sgt);
|
| 673 | kfree(sgt);
|
| 674 | }
|
| 675 |
|
| 676 | static void tegra_gem_prime_release(struct dma_buf *buf)
|
| 677 | {
|
| 678 | drm_gem_dmabuf_release(buf);
|
| 679 | }
|
| 680 |
|
| 681 | static int tegra_gem_prime_begin_cpu_access(struct dma_buf *buf,
|
| 682 | enum dma_data_direction direction)
|
| 683 | {
|
| 684 | struct drm_gem_object *gem = buf->priv;
|
| 685 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 686 | struct drm_device *drm = gem->dev;
|
| 687 |
|
| 688 | if (bo->pages)
|
| 689 | dma_sync_sgtable_for_cpu(drm->dev, bo->sgt, DMA_FROM_DEVICE);
|
| 690 |
|
| 691 | return 0;
|
| 692 | }
|
| 693 |
|
| 694 | static int tegra_gem_prime_end_cpu_access(struct dma_buf *buf,
|
| 695 | enum dma_data_direction direction)
|
| 696 | {
|
| 697 | struct drm_gem_object *gem = buf->priv;
|
| 698 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 699 | struct drm_device *drm = gem->dev;
|
| 700 |
|
| 701 | if (bo->pages)
|
| 702 | dma_sync_sgtable_for_device(drm->dev, bo->sgt, DMA_TO_DEVICE);
|
| 703 |
|
| 704 | return 0;
|
| 705 | }
|
| 706 |
|
| 707 | static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
|
| 708 | {
|
| 709 | struct drm_gem_object *gem = buf->priv;
|
| 710 | int err;
|
| 711 |
|
| 712 | err = drm_gem_mmap_obj(gem, gem->size, vma);
|
| 713 | if (err < 0)
|
| 714 | return err;
|
| 715 |
|
| 716 | return __tegra_gem_mmap(gem, vma);
|
| 717 | }
|
| 718 |
|
| 719 | static int tegra_gem_prime_vmap(struct dma_buf *buf, struct iosys_map *map)
|
| 720 | {
|
| 721 | struct drm_gem_object *gem = buf->priv;
|
| 722 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 723 | void *vaddr;
|
| 724 |
|
| 725 | vaddr = tegra_bo_mmap(&bo->base);
|
| 726 | if (IS_ERR(vaddr))
|
| 727 | return PTR_ERR(vaddr);
|
| 728 |
|
| 729 | iosys_map_set_vaddr(map, vaddr);
|
| 730 |
|
| 731 | return 0;
|
| 732 | }
|
| 733 |
|
| 734 | static void tegra_gem_prime_vunmap(struct dma_buf *buf, struct iosys_map *map)
|
| 735 | {
|
| 736 | struct drm_gem_object *gem = buf->priv;
|
| 737 | struct tegra_bo *bo = to_tegra_bo(gem);
|
| 738 |
|
| 739 | tegra_bo_munmap(&bo->base, map->vaddr);
|
| 740 | }
|
| 741 |
|
| 742 | static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = {
|
| 743 | .map_dma_buf = tegra_gem_prime_map_dma_buf,
|
| 744 | .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf,
|
| 745 | .release = tegra_gem_prime_release,
|
| 746 | .begin_cpu_access = tegra_gem_prime_begin_cpu_access,
|
| 747 | .end_cpu_access = tegra_gem_prime_end_cpu_access,
|
| 748 | .mmap = tegra_gem_prime_mmap,
|
| 749 | .vmap = tegra_gem_prime_vmap,
|
| 750 | .vunmap = tegra_gem_prime_vunmap,
|
| 751 | };
|
| 752 |
|
| 753 | struct dma_buf *tegra_gem_prime_export(struct drm_gem_object *gem,
|
| 754 | int flags)
|
| 755 | {
|
| 756 | DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
|
| 757 |
|
| 758 | exp_info.exp_name = KBUILD_MODNAME;
|
| 759 | exp_info.owner = gem->dev->driver->fops->owner;
|
| 760 | exp_info.ops = &tegra_gem_prime_dmabuf_ops;
|
| 761 | exp_info.size = gem->size;
|
| 762 | exp_info.flags = flags;
|
| 763 | exp_info.priv = gem;
|
| 764 |
|
| 765 | return drm_gem_dmabuf_export(gem->dev, &exp_info);
|
| 766 | }
|
| 767 |
|
| 768 | struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm,
|
| 769 | struct dma_buf *buf)
|
| 770 | {
|
| 771 | struct tegra_bo *bo;
|
| 772 |
|
| 773 | if (buf->ops == &tegra_gem_prime_dmabuf_ops) {
|
| 774 | struct drm_gem_object *gem = buf->priv;
|
| 775 |
|
| 776 | if (gem->dev == drm) {
|
| 777 | drm_gem_object_get(gem);
|
| 778 | return gem;
|
| 779 | }
|
| 780 | }
|
| 781 |
|
| 782 | bo = tegra_bo_import(drm, buf);
|
| 783 | if (IS_ERR(bo))
|
| 784 | return ERR_CAST(bo);
|
| 785 |
|
| 786 | return &bo->gem;
|
| 787 | }
|
| 788 |
|
| 789 | struct host1x_bo *tegra_gem_lookup(struct drm_file *file, u32 handle)
|
| 790 | {
|
| 791 | struct drm_gem_object *gem;
|
| 792 | struct tegra_bo *bo;
|
| 793 |
|
| 794 | gem = drm_gem_object_lookup(file, handle);
|
| 795 | if (!gem)
|
| 796 | return NULL;
|
| 797 |
|
| 798 | bo = to_tegra_bo(gem);
|
| 799 | return &bo->base;
|
| 800 | }
|