| 1 | |
| 2 | |
| 3 | * Copyright (c) 2022 Red Hat.
|
| 4 | *
|
| 5 | * Permission is hereby granted, free of charge, to any person obtaining a
|
| 6 | * copy of this software and associated documentation files (the "Software"),
|
| 7 | * to deal in the Software without restriction, including without limitation
|
| 8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
| 9 | * and/or sell copies of the Software, and to permit persons to whom the
|
| 10 | * Software is furnished to do so, subject to the following conditions:
|
| 11 | *
|
| 12 | * The above copyright notice and this permission notice shall be included in
|
| 13 | * all copies or substantial portions of the Software.
|
| 14 | *
|
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
| 18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
| 19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
| 20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
| 21 | * OTHER DEALINGS IN THE SOFTWARE.
|
| 22 | *
|
| 23 | * Authors:
|
| 24 | * Danilo Krummrich <[email protected]>
|
| 25 | *
|
| 26 |
|
| 27 |
|
| 28 | #include <drm/drm_drv.h>
|
| 29 | #include <drm/drm_gpuvm.h>
|
| 30 | #include <drm/drm_print.h>
|
| 31 |
|
| 32 | #include <linux/export.h>
|
| 33 | #include <linux/interval_tree_generic.h>
|
| 34 | #include <linux/mm.h>
|
| 35 |
|
| 36 | |
| 37 | * DOC: Overview
|
| 38 | *
|
| 39 | * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a
|
| 40 | * GPU's virtual address (VA) space and manages the corresponding virtual
|
| 41 | * mappings represented by &drm_gpuva objects. It also keeps track of the
|
| 42 | * mapping's backing &drm_gem_object buffers.
|
| 43 | *
|
| 44 | * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing
|
| 45 | * all existing GPU VA mappings using this &drm_gem_object as backing buffer.
|
| 46 | *
|
| 47 | * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also
|
| 48 | * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'.
|
| 49 | *
|
| 50 | * The GPU VA manager internally uses a rb-tree to manage the
|
| 51 | * &drm_gpuva mappings within a GPU's virtual address space.
|
| 52 | *
|
| 53 | * The &drm_gpuvm structure contains a special &drm_gpuva representing the
|
| 54 | * portion of VA space reserved by the kernel. This node is initialized together
|
| 55 | * with the GPU VA manager instance and removed when the GPU VA manager is
|
| 56 | * destroyed.
|
| 57 | *
|
| 58 | * In a typical application drivers would embed struct drm_gpuvm and
|
| 59 | * struct drm_gpuva within their own driver specific structures, there won't be
|
| 60 | * any memory allocations of its own nor memory allocations of &drm_gpuva
|
| 61 | * entries.
|
| 62 | *
|
| 63 | * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are
|
| 64 | * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva
|
| 65 | * entries from within dma-fence signalling critical sections it is enough to
|
| 66 | * pre-allocate the &drm_gpuva structures.
|
| 67 | *
|
| 68 | * &drm_gem_objects which are private to a single VM can share a common
|
| 69 | * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).
|
| 70 | * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in
|
| 71 | * the following called 'resv object', which serves as the container of the
|
| 72 | * GPUVM's shared &dma_resv. This resv object can be a driver specific
|
| 73 | * &drm_gem_object, such as the &drm_gem_object containing the root page table,
|
| 74 | * but it can also be a 'dummy' object, which can be allocated with
|
| 75 | * drm_gpuvm_resv_object_alloc().
|
| 76 | *
|
| 77 | * In order to connect a struct drm_gpuva to its backing &drm_gem_object each
|
| 78 | * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each
|
| 79 | * &drm_gpuvm_bo contains a list of &drm_gpuva structures.
|
| 80 | *
|
| 81 | * A &drm_gpuvm_bo is an abstraction that represents a combination of a
|
| 82 | * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.
|
| 83 | * This is ensured by the API through drm_gpuvm_bo_obtain() and
|
| 84 | * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding
|
| 85 | * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
|
| 86 | * particular combination. If not present, a new instance is created and linked
|
| 87 | * to the &drm_gem_object.
|
| 88 | *
|
| 89 | * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used
|
| 90 | * as entry for the &drm_gpuvm's lists of external and evicted objects. Those
|
| 91 | * lists are maintained in order to accelerate locking of dma-resv locks and
|
| 92 | * validation of evicted objects bound in a &drm_gpuvm. For instance, all
|
| 93 | * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling
|
| 94 | * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in
|
| 95 | * order to validate all evicted &drm_gem_objects. It is also possible to lock
|
| 96 | * additional &drm_gem_objects by providing the corresponding parameters to
|
| 97 | * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making
|
| 98 | * use of helper functions such as drm_gpuvm_prepare_range() or
|
| 99 | * drm_gpuvm_prepare_objects().
|
| 100 | *
|
| 101 | * Every bound &drm_gem_object is treated as external object when its &dma_resv
|
| 102 | * structure is different than the &drm_gpuvm's common &dma_resv structure.
|
| 103 |
|
| 104 |
|
| 105 | |
| 106 | * DOC: Split and Merge
|
| 107 | *
|
| 108 | * Besides its capability to manage and represent a GPU VA space, the
|
| 109 | * GPU VA manager also provides functions to let the &drm_gpuvm calculate a
|
| 110 | * sequence of operations to satisfy a given map or unmap request.
|
| 111 | *
|
| 112 | * Therefore the DRM GPU VA manager provides an algorithm implementing splitting
|
| 113 | * and merging of existing GPU VA mappings with the ones that are requested to
|
| 114 | * be mapped or unmapped. This feature is required by the Vulkan API to
|
| 115 | * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this
|
| 116 | * as VM BIND.
|
| 117 | *
|
| 118 | * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks
|
| 119 | * containing map, unmap and remap operations for a given newly requested
|
| 120 | * mapping. The sequence of callbacks represents the set of operations to
|
| 121 | * execute in order to integrate the new mapping cleanly into the current state
|
| 122 | * of the GPU VA space.
|
| 123 | *
|
| 124 | * Depending on how the new GPU VA mapping intersects with the existing mappings
|
| 125 | * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount
|
| 126 | * of unmap operations, a maximum of two remap operations and a single map
|
| 127 | * operation. The caller might receive no callback at all if no operation is
|
| 128 | * required, e.g. if the requested mapping already exists in the exact same way.
|
| 129 | *
|
| 130 | * The single map operation represents the original map operation requested by
|
| 131 | * the caller.
|
| 132 | *
|
| 133 | * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the
|
| 134 | * &drm_gpuva to unmap is physically contiguous with the original mapping
|
| 135 | * request. Optionally, if 'keep' is set, drivers may keep the actual page table
|
| 136 | * entries for this &drm_gpuva, adding the missing page table entries only and
|
| 137 | * update the &drm_gpuvm's view of things accordingly.
|
| 138 | *
|
| 139 | * Drivers may do the same optimization, namely delta page table updates, also
|
| 140 | * for remap operations. This is possible since &drm_gpuva_op_remap consists of
|
| 141 | * one unmap operation and one or two map operations, such that drivers can
|
| 142 | * derive the page table update delta accordingly.
|
| 143 | *
|
| 144 | * Note that there can't be more than two existing mappings to split up, one at
|
| 145 | * the beginning and one at the end of the new mapping, hence there is a
|
| 146 | * maximum of two remap operations.
|
| 147 | *
|
| 148 | * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to
|
| 149 | * call back into the driver in order to unmap a range of GPU VA space. The
|
| 150 | * logic behind this function is way simpler though: For all existing mappings
|
| 151 | * enclosed by the given range unmap operations are created. For mappings which
|
| 152 | * are only partially located within the given range, remap operations are
|
| 153 | * created such that those mappings are split up and re-mapped partially.
|
| 154 | *
|
| 155 | * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(),
|
| 156 | * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used
|
| 157 | * to directly obtain an instance of struct drm_gpuva_ops containing a list of
|
| 158 | * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list
|
| 159 | * contains the &drm_gpuva_ops analogous to the callbacks one would receive when
|
| 160 | * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires
|
| 161 | * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to
|
| 162 | * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory
|
| 163 | * allocations are possible (e.g. to allocate GPU page tables) and once in the
|
| 164 | * dma-fence signalling critical path.
|
| 165 | *
|
| 166 | * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and
|
| 167 | * drm_gpuva_remove() may be used. These functions can safely be used from
|
| 168 | * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or
|
| 169 | * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the
|
| 170 | * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and
|
| 171 | * drm_gpuva_unmap() instead.
|
| 172 | *
|
| 173 | * The following diagram depicts the basic relationships of existing GPU VA
|
| 174 | * mappings, a newly requested mapping and the resulting mappings as implemented
|
| 175 | * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these.
|
| 176 | *
|
| 177 | * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs
|
| 178 | * could be kept.
|
| 179 | *
|
| 180 | * ::
|
| 181 | *
|
| 182 | * 0 a 1
|
| 183 | * old: |-----------| (bo_offset=n)
|
| 184 | *
|
| 185 | * 0 a 1
|
| 186 | * req: |-----------| (bo_offset=n)
|
| 187 | *
|
| 188 | * 0 a 1
|
| 189 | * new: |-----------| (bo_offset=n)
|
| 190 | *
|
| 191 | *
|
| 192 | * 2) Requested mapping is identical, except for the BO offset, hence replace
|
| 193 | * the mapping.
|
| 194 | *
|
| 195 | * ::
|
| 196 | *
|
| 197 | * 0 a 1
|
| 198 | * old: |-----------| (bo_offset=n)
|
| 199 | *
|
| 200 | * 0 a 1
|
| 201 | * req: |-----------| (bo_offset=m)
|
| 202 | *
|
| 203 | * 0 a 1
|
| 204 | * new: |-----------| (bo_offset=m)
|
| 205 | *
|
| 206 | *
|
| 207 | * 3) Requested mapping is identical, except for the backing BO, hence replace
|
| 208 | * the mapping.
|
| 209 | *
|
| 210 | * ::
|
| 211 | *
|
| 212 | * 0 a 1
|
| 213 | * old: |-----------| (bo_offset=n)
|
| 214 | *
|
| 215 | * 0 b 1
|
| 216 | * req: |-----------| (bo_offset=n)
|
| 217 | *
|
| 218 | * 0 b 1
|
| 219 | * new: |-----------| (bo_offset=n)
|
| 220 | *
|
| 221 | *
|
| 222 | * 4) Existent mapping is a left aligned subset of the requested one, hence
|
| 223 | * replace the existing one.
|
| 224 | *
|
| 225 | * ::
|
| 226 | *
|
| 227 | * 0 a 1
|
| 228 | * old: |-----| (bo_offset=n)
|
| 229 | *
|
| 230 | * 0 a 2
|
| 231 | * req: |-----------| (bo_offset=n)
|
| 232 | *
|
| 233 | * 0 a 2
|
| 234 | * new: |-----------| (bo_offset=n)
|
| 235 | *
|
| 236 | * .. note::
|
| 237 | * We expect to see the same result for a request with a different BO
|
| 238 | * and/or non-contiguous BO offset.
|
| 239 | *
|
| 240 | *
|
| 241 | * 5) Requested mapping's range is a left aligned subset of the existing one,
|
| 242 | * but backed by a different BO. Hence, map the requested mapping and split
|
| 243 | * the existing one adjusting its BO offset.
|
| 244 | *
|
| 245 | * ::
|
| 246 | *
|
| 247 | * 0 a 2
|
| 248 | * old: |-----------| (bo_offset=n)
|
| 249 | *
|
| 250 | * 0 b 1
|
| 251 | * req: |-----| (bo_offset=n)
|
| 252 | *
|
| 253 | * 0 b 1 a' 2
|
| 254 | * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1)
|
| 255 | *
|
| 256 | * .. note::
|
| 257 | * We expect to see the same result for a request with a different BO
|
| 258 | * and/or non-contiguous BO offset.
|
| 259 | *
|
| 260 | *
|
| 261 | * 6) Existent mapping is a superset of the requested mapping. Split it up, but
|
| 262 | * indicate that the backing PTEs could be kept.
|
| 263 | *
|
| 264 | * ::
|
| 265 | *
|
| 266 | * 0 a 2
|
| 267 | * old: |-----------| (bo_offset=n)
|
| 268 | *
|
| 269 | * 0 a 1
|
| 270 | * req: |-----| (bo_offset=n)
|
| 271 | *
|
| 272 | * 0 a 1 a' 2
|
| 273 | * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1)
|
| 274 | *
|
| 275 | *
|
| 276 | * 7) Requested mapping's range is a right aligned subset of the existing one,
|
| 277 | * but backed by a different BO. Hence, map the requested mapping and split
|
| 278 | * the existing one, without adjusting the BO offset.
|
| 279 | *
|
| 280 | * ::
|
| 281 | *
|
| 282 | * 0 a 2
|
| 283 | * old: |-----------| (bo_offset=n)
|
| 284 | *
|
| 285 | * 1 b 2
|
| 286 | * req: |-----| (bo_offset=m)
|
| 287 | *
|
| 288 | * 0 a 1 b 2
|
| 289 | * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m)
|
| 290 | *
|
| 291 | *
|
| 292 | * 8) Existent mapping is a superset of the requested mapping. Split it up, but
|
| 293 | * indicate that the backing PTEs could be kept.
|
| 294 | *
|
| 295 | * ::
|
| 296 | *
|
| 297 | * 0 a 2
|
| 298 | * old: |-----------| (bo_offset=n)
|
| 299 | *
|
| 300 | * 1 a 2
|
| 301 | * req: |-----| (bo_offset=n+1)
|
| 302 | *
|
| 303 | * 0 a' 1 a 2
|
| 304 | * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1)
|
| 305 | *
|
| 306 | *
|
| 307 | * 9) Existent mapping is overlapped at the end by the requested mapping backed
|
| 308 | * by a different BO. Hence, map the requested mapping and split up the
|
| 309 | * existing one, without adjusting the BO offset.
|
| 310 | *
|
| 311 | * ::
|
| 312 | *
|
| 313 | * 0 a 2
|
| 314 | * old: |-----------| (bo_offset=n)
|
| 315 | *
|
| 316 | * 1 b 3
|
| 317 | * req: |-----------| (bo_offset=m)
|
| 318 | *
|
| 319 | * 0 a 1 b 3
|
| 320 | * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m)
|
| 321 | *
|
| 322 | *
|
| 323 | * 10) Existent mapping is overlapped by the requested mapping, both having the
|
| 324 | * same backing BO with a contiguous offset. Indicate the backing PTEs of
|
| 325 | * the old mapping could be kept.
|
| 326 | *
|
| 327 | * ::
|
| 328 | *
|
| 329 | * 0 a 2
|
| 330 | * old: |-----------| (bo_offset=n)
|
| 331 | *
|
| 332 | * 1 a 3
|
| 333 | * req: |-----------| (bo_offset=n+1)
|
| 334 | *
|
| 335 | * 0 a' 1 a 3
|
| 336 | * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1)
|
| 337 | *
|
| 338 | *
|
| 339 | * 11) Requested mapping's range is a centered subset of the existing one
|
| 340 | * having a different backing BO. Hence, map the requested mapping and split
|
| 341 | * up the existing one in two mappings, adjusting the BO offset of the right
|
| 342 | * one accordingly.
|
| 343 | *
|
| 344 | * ::
|
| 345 | *
|
| 346 | * 0 a 3
|
| 347 | * old: |-----------------| (bo_offset=n)
|
| 348 | *
|
| 349 | * 1 b 2
|
| 350 | * req: |-----| (bo_offset=m)
|
| 351 | *
|
| 352 | * 0 a 1 b 2 a' 3
|
| 353 | * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2)
|
| 354 | *
|
| 355 | *
|
| 356 | * 12) Requested mapping is a contiguous subset of the existing one. Split it
|
| 357 | * up, but indicate that the backing PTEs could be kept.
|
| 358 | *
|
| 359 | * ::
|
| 360 | *
|
| 361 | * 0 a 3
|
| 362 | * old: |-----------------| (bo_offset=n)
|
| 363 | *
|
| 364 | * 1 a 2
|
| 365 | * req: |-----| (bo_offset=n+1)
|
| 366 | *
|
| 367 | * 0 a' 1 a 2 a'' 3
|
| 368 | * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2)
|
| 369 | *
|
| 370 | *
|
| 371 | * 13) Existent mapping is a right aligned subset of the requested one, hence
|
| 372 | * replace the existing one.
|
| 373 | *
|
| 374 | * ::
|
| 375 | *
|
| 376 | * 1 a 2
|
| 377 | * old: |-----| (bo_offset=n+1)
|
| 378 | *
|
| 379 | * 0 a 2
|
| 380 | * req: |-----------| (bo_offset=n)
|
| 381 | *
|
| 382 | * 0 a 2
|
| 383 | * new: |-----------| (bo_offset=n)
|
| 384 | *
|
| 385 | * .. note::
|
| 386 | * We expect to see the same result for a request with a different bo
|
| 387 | * and/or non-contiguous bo_offset.
|
| 388 | *
|
| 389 | *
|
| 390 | * 14) Existent mapping is a centered subset of the requested one, hence
|
| 391 | * replace the existing one.
|
| 392 | *
|
| 393 | * ::
|
| 394 | *
|
| 395 | * 1 a 2
|
| 396 | * old: |-----| (bo_offset=n+1)
|
| 397 | *
|
| 398 | * 0 a 3
|
| 399 | * req: |----------------| (bo_offset=n)
|
| 400 | *
|
| 401 | * 0 a 3
|
| 402 | * new: |----------------| (bo_offset=n)
|
| 403 | *
|
| 404 | * .. note::
|
| 405 | * We expect to see the same result for a request with a different bo
|
| 406 | * and/or non-contiguous bo_offset.
|
| 407 | *
|
| 408 | *
|
| 409 | * 15) Existent mappings is overlapped at the beginning by the requested mapping
|
| 410 | * backed by a different BO. Hence, map the requested mapping and split up
|
| 411 | * the existing one, adjusting its BO offset accordingly.
|
| 412 | *
|
| 413 | * ::
|
| 414 | *
|
| 415 | * 1 a 3
|
| 416 | * old: |-----------| (bo_offset=n)
|
| 417 | *
|
| 418 | * 0 b 2
|
| 419 | * req: |-----------| (bo_offset=m)
|
| 420 | *
|
| 421 | * 0 b 2 a' 3
|
| 422 | * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2)
|
| 423 |
|
| 424 |
|
| 425 | |
| 426 | * DOC: Madvise Logic - Splitting and Traversal
|
| 427 | *
|
| 428 | * This logic handles GPU VA range updates by generating remap and map operations
|
| 429 | * without performing unmaps or merging existing mappings.
|
| 430 | *
|
| 431 | * 1) The requested range lies entirely within a single drm_gpuva. The logic splits
|
| 432 | * the existing mapping at the start and end boundaries and inserts a new map.
|
| 433 | *
|
| 434 | * ::
|
| 435 | * a start end b
|
| 436 | * pre: |-----------------------|
|
| 437 | * drm_gpuva1
|
| 438 | *
|
| 439 | * a start end b
|
| 440 | * new: |-----|=========|-------|
|
| 441 | * remap map remap
|
| 442 | *
|
| 443 | * one REMAP and one MAP : Same behaviour as SPLIT and MERGE
|
| 444 | *
|
| 445 | * 2) The requested range spans multiple drm_gpuva regions. The logic traverses
|
| 446 | * across boundaries, remapping the start and end segments, and inserting two
|
| 447 | * map operations to cover the full range.
|
| 448 | *
|
| 449 | * :: a start b c end d
|
| 450 | * pre: |------------------|--------------|------------------|
|
| 451 | * drm_gpuva1 drm_gpuva2 drm_gpuva3
|
| 452 | *
|
| 453 | * a start b c end d
|
| 454 | * new: |-------|==========|--------------|========|---------|
|
| 455 | * remap1 map1 drm_gpuva2 map2 remap2
|
| 456 | *
|
| 457 | * two REMAPS and two MAPS
|
| 458 | *
|
| 459 | * 3) Either start or end lies within a drm_gpuva. A single remap and map operation
|
| 460 | * are generated to update the affected portion.
|
| 461 | *
|
| 462 | *
|
| 463 | * :: a/start b c end d
|
| 464 | * pre: |------------------|--------------|------------------|
|
| 465 | * drm_gpuva1 drm_gpuva2 drm_gpuva3
|
| 466 | *
|
| 467 | * a/start b c end d
|
| 468 | * new: |------------------|--------------|========|---------|
|
| 469 | * drm_gpuva1 drm_gpuva2 map1 remap1
|
| 470 | *
|
| 471 | * :: a start b c/end d
|
| 472 | * pre: |------------------|--------------|------------------|
|
| 473 | * drm_gpuva1 drm_gpuva2 drm_gpuva3
|
| 474 | *
|
| 475 | * a start b c/end d
|
| 476 | * new: |-------|==========|--------------|------------------|
|
| 477 | * remap1 map1 drm_gpuva2 drm_gpuva3
|
| 478 | *
|
| 479 | * one REMAP and one MAP
|
| 480 | *
|
| 481 | * 4) Both start and end align with existing drm_gpuva boundaries. No operations
|
| 482 | * are needed as the range is already covered.
|
| 483 | *
|
| 484 | * 5) No existing drm_gpuvas. No operations.
|
| 485 | *
|
| 486 | * Unlike drm_gpuvm_sm_map_ops_create, this logic avoids unmaps and merging,
|
| 487 | * focusing solely on remap and map operations for efficient traversal and update.
|
| 488 |
|
| 489 |
|
| 490 | |
| 491 | * DOC: Locking
|
| 492 | *
|
| 493 | * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of
|
| 494 | * locking itself, it is the drivers responsibility to take care about locking.
|
| 495 | * Drivers might want to protect the following operations: inserting, removing
|
| 496 | * and iterating &drm_gpuva objects as well as generating all kinds of
|
| 497 | * operations, such as split / merge or prefetch.
|
| 498 | *
|
| 499 | * DRM GPUVM also does not take care of the locking of the backing
|
| 500 | * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by
|
| 501 | * itself; drivers are responsible to enforce mutual exclusion using either the
|
| 502 | * GEMs dma_resv lock or the GEMs gpuva.lock mutex.
|
| 503 | *
|
| 504 | * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold
|
| 505 | * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed
|
| 506 | * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also
|
| 507 | * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().
|
| 508 | *
|
| 509 | * The latter is required since on creation and destruction of a &drm_gpuvm_bo
|
| 510 | * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.
|
| 511 | * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
|
| 512 | * &drm_gem_object must be able to observe previous creations and destructions
|
| 513 | * of &drm_gpuvm_bos in order to keep instances unique.
|
| 514 | *
|
| 515 | * The &drm_gpuvm's lists for keeping track of external and evicted objects are
|
| 516 | * protected against concurrent insertion / removal and iteration internally.
|
| 517 | *
|
| 518 | * However, drivers still need ensure to protect concurrent calls to functions
|
| 519 | * iterating those lists, namely drm_gpuvm_prepare_objects() and
|
| 520 | * drm_gpuvm_validate().
|
| 521 | *
|
| 522 | * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate
|
| 523 | * that the corresponding &dma_resv locks are held in order to protect the
|
| 524 | * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and
|
| 525 | * the corresponding lockdep checks are enabled. This is an optimization for
|
| 526 | * drivers which are capable of taking the corresponding &dma_resv locks and
|
| 527 | * hence do not require internal locking.
|
| 528 |
|
| 529 |
|
| 530 | |
| 531 | * DOC: Examples
|
| 532 | *
|
| 533 | * This section gives two examples on how to let the DRM GPUVA Manager generate
|
| 534 | * &drm_gpuva_op in order to satisfy a given map or unmap request and how to
|
| 535 | * make use of them.
|
| 536 | *
|
| 537 | * The below code is strictly limited to illustrate the generic usage pattern.
|
| 538 | * To maintain simplicity, it doesn't make use of any abstractions for common
|
| 539 | * code, different (asynchronous) stages with fence signalling critical paths,
|
| 540 | * any other helpers or error handling in terms of freeing memory and dropping
|
| 541 | * previously taken locks.
|
| 542 | *
|
| 543 | * 1) Obtain a list of &drm_gpuva_op to create a new mapping::
|
| 544 | *
|
| 545 | * // Allocates a new &drm_gpuva.
|
| 546 | * struct drm_gpuva * driver_gpuva_alloc(void);
|
| 547 | *
|
| 548 | * // Typically drivers would embed the &drm_gpuvm and &drm_gpuva
|
| 549 | * // structure in individual driver structures and lock the dma-resv with
|
| 550 | * // drm_exec or similar helpers.
|
| 551 | * int driver_mapping_create(struct drm_gpuvm *gpuvm,
|
| 552 | * u64 addr, u64 range,
|
| 553 | * struct drm_gem_object *obj, u64 offset)
|
| 554 | * {
|
| 555 | * struct drm_gpuvm_map_req map_req = {
|
| 556 | * .map.va.addr = addr,
|
| 557 | * .map.va.range = range,
|
| 558 | * .map.gem.obj = obj,
|
| 559 | * .map.gem.offset = offset,
|
| 560 | * };
|
| 561 | * struct drm_gpuva_ops *ops;
|
| 562 | * struct drm_gpuva_op *op
|
| 563 | * struct drm_gpuvm_bo *vm_bo;
|
| 564 | *
|
| 565 | * driver_lock_va_space();
|
| 566 | * ops = drm_gpuvm_sm_map_ops_create(gpuvm, &map_req);
|
| 567 | * if (IS_ERR(ops))
|
| 568 | * return PTR_ERR(ops);
|
| 569 | *
|
| 570 | * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);
|
| 571 | * if (IS_ERR(vm_bo))
|
| 572 | * return PTR_ERR(vm_bo);
|
| 573 | *
|
| 574 | * drm_gpuva_for_each_op(op, ops) {
|
| 575 | * struct drm_gpuva *va;
|
| 576 | *
|
| 577 | * switch (op->op) {
|
| 578 | * case DRM_GPUVA_OP_MAP:
|
| 579 | * va = driver_gpuva_alloc();
|
| 580 | * if (!va)
|
| 581 | * ; // unwind previous VA space updates,
|
| 582 | * // free memory and unlock
|
| 583 | *
|
| 584 | * driver_vm_map();
|
| 585 | * drm_gpuva_map(gpuvm, va, &op->map);
|
| 586 | * drm_gpuva_link(va, vm_bo);
|
| 587 | *
|
| 588 | * break;
|
| 589 | * case DRM_GPUVA_OP_REMAP: {
|
| 590 | * struct drm_gpuva *prev = NULL, *next = NULL;
|
| 591 | *
|
| 592 | * va = op->remap.unmap->va;
|
| 593 | *
|
| 594 | * if (op->remap.prev) {
|
| 595 | * prev = driver_gpuva_alloc();
|
| 596 | * if (!prev)
|
| 597 | * ; // unwind previous VA space
|
| 598 | * // updates, free memory and
|
| 599 | * // unlock
|
| 600 | * }
|
| 601 | *
|
| 602 | * if (op->remap.next) {
|
| 603 | * next = driver_gpuva_alloc();
|
| 604 | * if (!next)
|
| 605 | * ; // unwind previous VA space
|
| 606 | * // updates, free memory and
|
| 607 | * // unlock
|
| 608 | * }
|
| 609 | *
|
| 610 | * driver_vm_remap();
|
| 611 | * drm_gpuva_remap(prev, next, &op->remap);
|
| 612 | *
|
| 613 | * if (prev)
|
| 614 | * drm_gpuva_link(prev, va->vm_bo);
|
| 615 | * if (next)
|
| 616 | * drm_gpuva_link(next, va->vm_bo);
|
| 617 | * drm_gpuva_unlink(va);
|
| 618 | *
|
| 619 | * break;
|
| 620 | * }
|
| 621 | * case DRM_GPUVA_OP_UNMAP:
|
| 622 | * va = op->unmap->va;
|
| 623 | *
|
| 624 | * driver_vm_unmap();
|
| 625 | * drm_gpuva_unlink(va);
|
| 626 | * drm_gpuva_unmap(&op->unmap);
|
| 627 | *
|
| 628 | * break;
|
| 629 | * default:
|
| 630 | * break;
|
| 631 | * }
|
| 632 | * }
|
| 633 | * drm_gpuvm_bo_put(vm_bo);
|
| 634 | * driver_unlock_va_space();
|
| 635 | *
|
| 636 | * return 0;
|
| 637 | * }
|
| 638 | *
|
| 639 | * 2) Receive a callback for each &drm_gpuva_op to create a new mapping::
|
| 640 | *
|
| 641 | * struct driver_context {
|
| 642 | * struct drm_gpuvm *gpuvm;
|
| 643 | * struct drm_gpuvm_bo *vm_bo;
|
| 644 | * struct drm_gpuva *new_va;
|
| 645 | * struct drm_gpuva *prev_va;
|
| 646 | * struct drm_gpuva *next_va;
|
| 647 | * };
|
| 648 | *
|
| 649 | * // ops to pass to drm_gpuvm_init()
|
| 650 | * static const struct drm_gpuvm_ops driver_gpuvm_ops = {
|
| 651 | * .sm_step_map = driver_gpuva_map,
|
| 652 | * .sm_step_remap = driver_gpuva_remap,
|
| 653 | * .sm_step_unmap = driver_gpuva_unmap,
|
| 654 | * };
|
| 655 | *
|
| 656 | * // Typically drivers would embed the &drm_gpuvm and &drm_gpuva
|
| 657 | * // structure in individual driver structures and lock the dma-resv with
|
| 658 | * // drm_exec or similar helpers.
|
| 659 | * int driver_mapping_create(struct drm_gpuvm *gpuvm,
|
| 660 | * u64 addr, u64 range,
|
| 661 | * struct drm_gem_object *obj, u64 offset)
|
| 662 | * {
|
| 663 | * struct driver_context ctx;
|
| 664 | * struct drm_gpuvm_bo *vm_bo;
|
| 665 | * struct drm_gpuva_ops *ops;
|
| 666 | * struct drm_gpuva_op *op;
|
| 667 | * int ret = 0;
|
| 668 | *
|
| 669 | * ctx.gpuvm = gpuvm;
|
| 670 | *
|
| 671 | * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
|
| 672 | * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
|
| 673 | * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
|
| 674 | * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
|
| 675 | * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {
|
| 676 | * ret = -ENOMEM;
|
| 677 | * goto out;
|
| 678 | * }
|
| 679 | *
|
| 680 | * // Typically protected with a driver specific GEM gpuva lock
|
| 681 | * // used in the fence signaling path for drm_gpuva_link() and
|
| 682 | * // drm_gpuva_unlink(), hence pre-allocate.
|
| 683 | * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);
|
| 684 | *
|
| 685 | * driver_lock_va_space();
|
| 686 | * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);
|
| 687 | * driver_unlock_va_space();
|
| 688 | *
|
| 689 | * out:
|
| 690 | * drm_gpuvm_bo_put(ctx.vm_bo);
|
| 691 | * kfree(ctx.new_va);
|
| 692 | * kfree(ctx.prev_va);
|
| 693 | * kfree(ctx.next_va);
|
| 694 | * return ret;
|
| 695 | * }
|
| 696 | *
|
| 697 | * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx)
|
| 698 | * {
|
| 699 | * struct driver_context *ctx = __ctx;
|
| 700 | *
|
| 701 | * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);
|
| 702 | *
|
| 703 | * drm_gpuva_link(ctx->new_va, ctx->vm_bo);
|
| 704 | *
|
| 705 | * // prevent the new GPUVA from being freed in
|
| 706 | * // driver_mapping_create()
|
| 707 | * ctx->new_va = NULL;
|
| 708 | *
|
| 709 | * return 0;
|
| 710 | * }
|
| 711 | *
|
| 712 | * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
|
| 713 | * {
|
| 714 | * struct driver_context *ctx = __ctx;
|
| 715 | * struct drm_gpuva *va = op->remap.unmap->va;
|
| 716 | *
|
| 717 | * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
|
| 718 | *
|
| 719 | * if (op->remap.prev) {
|
| 720 | * drm_gpuva_link(ctx->prev_va, va->vm_bo);
|
| 721 | * ctx->prev_va = NULL;
|
| 722 | * }
|
| 723 | *
|
| 724 | * if (op->remap.next) {
|
| 725 | * drm_gpuva_link(ctx->next_va, va->vm_bo);
|
| 726 | * ctx->next_va = NULL;
|
| 727 | * }
|
| 728 | *
|
| 729 | * drm_gpuva_unlink(va);
|
| 730 | * kfree(va);
|
| 731 | *
|
| 732 | * return 0;
|
| 733 | * }
|
| 734 | *
|
| 735 | * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx)
|
| 736 | * {
|
| 737 | * drm_gpuva_unlink(op->unmap.va);
|
| 738 | * drm_gpuva_unmap(&op->unmap);
|
| 739 | * kfree(op->unmap.va);
|
| 740 | *
|
| 741 | * return 0;
|
| 742 | * }
|
| 743 |
|
| 744 |
|
| 745 | |
| 746 | * get_next_vm_bo_from_list() - get the next vm_bo element
|
| 747 | * @__gpuvm: the &drm_gpuvm
|
| 748 | * @__list_name: the name of the list we're iterating on
|
| 749 | * @__local_list: a pointer to the local list used to store already iterated items
|
| 750 | * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()
|
| 751 | *
|
| 752 | * This helper is here to provide lockless list iteration. Lockless as in, the
|
| 753 | * iterator releases the lock immediately after picking the first element from
|
| 754 | * the list, so list insertion and deletion can happen concurrently.
|
| 755 | *
|
| 756 | * Elements popped from the original list are kept in a local list, so removal
|
| 757 | * and is_empty checks can still happen while we're iterating the list.
|
| 758 |
|
| 759 | #define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \
|
| 760 | ({ \
|
| 761 | struct drm_gpuvm_bo *__vm_bo = NULL; \
|
| 762 | \
|
| 763 | drm_gpuvm_bo_put(__prev_vm_bo); \
|
| 764 | \
|
| 765 | spin_lock(&(__gpuvm)->__list_name.lock); \
|
| 766 | if (!(__gpuvm)->__list_name.local_list) \
|
| 767 | (__gpuvm)->__list_name.local_list = __local_list; \
|
| 768 | else \
|
| 769 | drm_WARN_ON((__gpuvm)->drm, \
|
| 770 | (__gpuvm)->__list_name.local_list != __local_list); \
|
| 771 | \
|
| 772 | while (!list_empty(&(__gpuvm)->__list_name.list)) { \
|
| 773 | __vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \
|
| 774 | struct drm_gpuvm_bo, \
|
| 775 | list.entry.__list_name); \
|
| 776 | if (kref_get_unless_zero(&__vm_bo->kref)) { \
|
| 777 | list_move_tail(&(__vm_bo)->list.entry.__list_name, \
|
| 778 | __local_list); \
|
| 779 | break; \
|
| 780 | } else { \
|
| 781 | list_del_init(&(__vm_bo)->list.entry.__list_name); \
|
| 782 | __vm_bo = NULL; \
|
| 783 | } \
|
| 784 | } \
|
| 785 | spin_unlock(&(__gpuvm)->__list_name.lock); \
|
| 786 | \
|
| 787 | __vm_bo; \
|
| 788 | })
|
| 789 |
|
| 790 | |
| 791 | * for_each_vm_bo_in_list() - internal vm_bo list iterator
|
| 792 | * @__gpuvm: the &drm_gpuvm
|
| 793 | * @__list_name: the name of the list we're iterating on
|
| 794 | * @__local_list: a pointer to the local list used to store already iterated items
|
| 795 | * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step
|
| 796 | *
|
| 797 | * This helper is here to provide lockless list iteration. Lockless as in, the
|
| 798 | * iterator releases the lock immediately after picking the first element from the
|
| 799 | * list, hence list insertion and deletion can happen concurrently.
|
| 800 | *
|
| 801 | * It is not allowed to re-assign the vm_bo pointer from inside this loop.
|
| 802 | *
|
| 803 | * Typical use:
|
| 804 | *
|
| 805 | * struct drm_gpuvm_bo *vm_bo;
|
| 806 | * LIST_HEAD(my_local_list);
|
| 807 | *
|
| 808 | * ret = 0;
|
| 809 | * for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {
|
| 810 | * ret = do_something_with_vm_bo(..., vm_bo);
|
| 811 | * if (ret)
|
| 812 | * break;
|
| 813 | * }
|
| 814 | * // Drop ref in case we break out of the loop.
|
| 815 | * drm_gpuvm_bo_put(vm_bo);
|
| 816 | * restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);
|
| 817 | *
|
| 818 | *
|
| 819 | * Only used for internal list iterations, not meant to be exposed to the outside
|
| 820 | * world.
|
| 821 |
|
| 822 | #define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \
|
| 823 | for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
|
| 824 | __local_list, NULL); \
|
| 825 | __vm_bo; \
|
| 826 | __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \
|
| 827 | __local_list, __vm_bo))
|
| 828 |
|
| 829 | static void
|
| 830 | __restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,
|
| 831 | struct list_head *list, struct list_head **local_list)
|
| 832 | {
|
| 833 | |
| 834 | * head to preserve previous ordering, in case it matters.
|
| 835 |
|
| 836 | spin_lock(lock);
|
| 837 | if (*local_list) {
|
| 838 | list_splice(*local_list, list);
|
| 839 | *local_list = NULL;
|
| 840 | }
|
| 841 | spin_unlock(lock);
|
| 842 | }
|
| 843 |
|
| 844 | |
| 845 | * restore_vm_bo_list() - move vm_bo elements back to their original list
|
| 846 | * @__gpuvm: the &drm_gpuvm
|
| 847 | * @__list_name: the name of the list we're iterating on
|
| 848 | *
|
| 849 | * When we're done iterating a vm_bo list, we should call restore_vm_bo_list()
|
| 850 | * to restore the original state and let new iterations take place.
|
| 851 |
|
| 852 | #define restore_vm_bo_list(__gpuvm, __list_name) \
|
| 853 | __restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \
|
| 854 | &(__gpuvm)->__list_name.list, \
|
| 855 | &(__gpuvm)->__list_name.local_list)
|
| 856 |
|
| 857 | static void
|
| 858 | cond_spin_lock(spinlock_t *lock, bool cond)
|
| 859 | {
|
| 860 | if (cond)
|
| 861 | spin_lock(lock);
|
| 862 | }
|
| 863 |
|
| 864 | static void
|
| 865 | cond_spin_unlock(spinlock_t *lock, bool cond)
|
| 866 | {
|
| 867 | if (cond)
|
| 868 | spin_unlock(lock);
|
| 869 | }
|
| 870 |
|
| 871 | static void
|
| 872 | __drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,
|
| 873 | struct list_head *entry, struct list_head *list)
|
| 874 | {
|
| 875 | cond_spin_lock(lock, !!lock);
|
| 876 | if (list_empty(entry))
|
| 877 | list_add_tail(entry, list);
|
| 878 | cond_spin_unlock(lock, !!lock);
|
| 879 | }
|
| 880 |
|
| 881 | |
| 882 | * drm_gpuvm_bo_is_zombie() - check whether this vm_bo is scheduled for cleanup
|
| 883 | * @vm_bo: the &drm_gpuvm_bo
|
| 884 | *
|
| 885 | * When a vm_bo is scheduled for cleanup using the bo_defer list, it is not
|
| 886 | * immediately removed from the evict and extobj lists. Therefore, anyone
|
| 887 | * iterating these lists should skip entries that are being destroyed.
|
| 888 | *
|
| 889 | * Checking the refcount without incrementing it is okay as long as the lock
|
| 890 | * protecting the evict/extobj list is held for as long as you are using the
|
| 891 | * vm_bo, because even if the refcount hits zero while you are using it, freeing
|
| 892 | * the vm_bo requires taking the list's lock.
|
| 893 | *
|
| 894 | * Zombie entries can be observed on the evict and extobj lists regardless of
|
| 895 | * whether DRM_GPUVM_RESV_PROTECTED is used, but they remain on the lists for a
|
| 896 | * longer time when the resv lock is used because we can't take the resv lock
|
| 897 | * during run_job() in immediate mode, meaning that they need to remain on the
|
| 898 | * lists until drm_gpuvm_bo_deferred_cleanup() is called.
|
| 899 |
|
| 900 | static bool
|
| 901 | drm_gpuvm_bo_is_zombie(struct drm_gpuvm_bo *vm_bo)
|
| 902 | {
|
| 903 | return !kref_read(&vm_bo->kref);
|
| 904 | }
|
| 905 |
|
| 906 | |
| 907 | * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list
|
| 908 | * @__vm_bo: the &drm_gpuvm_bo
|
| 909 | * @__list_name: the name of the list to insert into
|
| 910 | * @__lock: whether to lock with the internal spinlock
|
| 911 | *
|
| 912 | * Inserts the given @__vm_bo into the list specified by @__list_name.
|
| 913 |
|
| 914 | #define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \
|
| 915 | __drm_gpuvm_bo_list_add((__vm_bo)->vm, \
|
| 916 | __lock ? &(__vm_bo)->vm->__list_name.lock : \
|
| 917 | NULL, \
|
| 918 | &(__vm_bo)->list.entry.__list_name, \
|
| 919 | &(__vm_bo)->vm->__list_name.list)
|
| 920 |
|
| 921 | static void
|
| 922 | __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
|
| 923 | struct list_head *entry, bool init)
|
| 924 | {
|
| 925 | cond_spin_lock(lock, !!lock);
|
| 926 | if (init) {
|
| 927 | if (!list_empty(entry))
|
| 928 | list_del_init(entry);
|
| 929 | } else {
|
| 930 | list_del(entry);
|
| 931 | }
|
| 932 | cond_spin_unlock(lock, !!lock);
|
| 933 | }
|
| 934 |
|
| 935 | |
| 936 | * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list
|
| 937 | * @__vm_bo: the &drm_gpuvm_bo
|
| 938 | * @__list_name: the name of the list to insert into
|
| 939 | * @__lock: whether to lock with the internal spinlock
|
| 940 | *
|
| 941 | * Removes the given @__vm_bo from the list specified by @__list_name.
|
| 942 |
|
| 943 | #define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \
|
| 944 | __drm_gpuvm_bo_list_del((__vm_bo)->vm, \
|
| 945 | __lock ? &(__vm_bo)->vm->__list_name.lock : \
|
| 946 | NULL, \
|
| 947 | &(__vm_bo)->list.entry.__list_name, \
|
| 948 | true)
|
| 949 |
|
| 950 | |
| 951 | * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list
|
| 952 | * @__vm_bo: the &drm_gpuvm_bo
|
| 953 | * @__list_name: the name of the list to insert into
|
| 954 | * @__lock: whether to lock with the internal spinlock
|
| 955 | *
|
| 956 | * Removes the given @__vm_bo from the list specified by @__list_name.
|
| 957 |
|
| 958 | #define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \
|
| 959 | __drm_gpuvm_bo_list_del((__vm_bo)->vm, \
|
| 960 | __lock ? &(__vm_bo)->vm->__list_name.lock : \
|
| 961 | NULL, \
|
| 962 | &(__vm_bo)->list.entry.__list_name, \
|
| 963 | false)
|
| 964 |
|
| 965 | #define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node)
|
| 966 |
|
| 967 | #define GPUVA_START(node) ((node)->va.addr)
|
| 968 | #define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
|
| 969 |
|
| 970 | |
| 971 | * about this.
|
| 972 |
|
| 973 | INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last,
|
| 974 | GPUVA_START, GPUVA_LAST, static __maybe_unused,
|
| 975 | drm_gpuva_it)
|
| 976 |
|
| 977 | static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
|
| 978 | struct drm_gpuva *va);
|
| 979 | static void __drm_gpuva_remove(struct drm_gpuva *va);
|
| 980 |
|
| 981 | static bool
|
| 982 | drm_gpuvm_check_overflow(u64 addr, u64 range)
|
| 983 | {
|
| 984 | u64 end;
|
| 985 |
|
| 986 | return check_add_overflow(addr, range, &end);
|
| 987 | }
|
| 988 |
|
| 989 | static bool
|
| 990 | drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
|
| 991 | {
|
| 992 | return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),
|
| 993 | "GPUVA address limited to %zu bytes.\n", sizeof(addr));
|
| 994 | }
|
| 995 |
|
| 996 | static bool
|
| 997 | drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
|
| 998 | {
|
| 999 | u64 end = addr + range;
|
| 1000 | u64 mm_start = gpuvm->mm_start;
|
| 1001 | u64 mm_end = mm_start + gpuvm->mm_range;
|
| 1002 |
|
| 1003 | return addr >= mm_start && end <= mm_end;
|
| 1004 | }
|
| 1005 |
|
| 1006 | static bool
|
| 1007 | drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
|
| 1008 | {
|
| 1009 | u64 end = addr + range;
|
| 1010 | u64 kstart = gpuvm->kernel_alloc_node.va.addr;
|
| 1011 | u64 krange = gpuvm->kernel_alloc_node.va.range;
|
| 1012 | u64 kend = kstart + krange;
|
| 1013 |
|
| 1014 | return krange && addr < kend && kstart < end;
|
| 1015 | }
|
| 1016 |
|
| 1017 | |
| 1018 | * drm_gpuvm_range_valid() - checks whether the given range is valid for the
|
| 1019 | * given &drm_gpuvm
|
| 1020 | * @gpuvm: the GPUVM to check the range for
|
| 1021 | * @addr: the base address
|
| 1022 | * @range: the range starting from the base address
|
| 1023 | *
|
| 1024 | * Checks whether the range is within the GPUVM's managed boundaries.
|
| 1025 | *
|
| 1026 | * Returns: true for a valid range, false otherwise
|
| 1027 |
|
| 1028 | bool
|
| 1029 | drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
|
| 1030 | u64 addr, u64 range)
|
| 1031 | {
|
| 1032 | return !drm_gpuvm_check_overflow(addr, range) &&
|
| 1033 | drm_gpuvm_in_mm_range(gpuvm, addr, range) &&
|
| 1034 | !drm_gpuvm_in_kernel_node(gpuvm, addr, range);
|
| 1035 | }
|
| 1036 | EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);
|
| 1037 |
|
| 1038 | static void
|
| 1039 | drm_gpuvm_gem_object_free(struct drm_gem_object *obj)
|
| 1040 | {
|
| 1041 | drm_gem_object_release(obj);
|
| 1042 | kfree(obj);
|
| 1043 | }
|
| 1044 |
|
| 1045 | static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {
|
| 1046 | .free = drm_gpuvm_gem_object_free,
|
| 1047 | };
|
| 1048 |
|
| 1049 | |
| 1050 | * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object
|
| 1051 | * @drm: the drivers &drm_device
|
| 1052 | *
|
| 1053 | * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in
|
| 1054 | * order to serve as root GEM object providing the &drm_resv shared across
|
| 1055 | * &drm_gem_objects local to a single GPUVM.
|
| 1056 | *
|
| 1057 | * Returns: the &drm_gem_object on success, NULL on failure
|
| 1058 |
|
| 1059 | struct drm_gem_object *
|
| 1060 | drm_gpuvm_resv_object_alloc(struct drm_device *drm)
|
| 1061 | {
|
| 1062 | struct drm_gem_object *obj;
|
| 1063 |
|
| 1064 | obj = kzalloc_obj(*obj);
|
| 1065 | if (!obj)
|
| 1066 | return NULL;
|
| 1067 |
|
| 1068 | obj->funcs = &drm_gpuvm_object_funcs;
|
| 1069 | drm_gem_private_object_init(drm, obj, 0);
|
| 1070 |
|
| 1071 | return obj;
|
| 1072 | }
|
| 1073 | EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
|
| 1074 |
|
| 1075 | |
| 1076 | * drm_gpuvm_init() - initialize a &drm_gpuvm
|
| 1077 | * @gpuvm: pointer to the &drm_gpuvm to initialize
|
| 1078 | * @name: the name of the GPU VA space
|
| 1079 | * @flags: the &drm_gpuvm_flags for this GPUVM
|
| 1080 | * @drm: the &drm_device this VM resides in
|
| 1081 | * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
|
| 1082 | * @start_offset: the start offset of the GPU VA space
|
| 1083 | * @range: the size of the GPU VA space
|
| 1084 | * @reserve_offset: the start of the kernel reserved GPU VA area
|
| 1085 | * @reserve_range: the size of the kernel reserved GPU VA area
|
| 1086 | * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap
|
| 1087 | *
|
| 1088 | * The &drm_gpuvm must be initialized with this function before use.
|
| 1089 | *
|
| 1090 | * Note that @gpuvm must be cleared to 0 before calling this function. The given
|
| 1091 | * &name is expected to be managed by the surrounding driver structures.
|
| 1092 |
|
| 1093 | void
|
| 1094 | drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
|
| 1095 | enum drm_gpuvm_flags flags,
|
| 1096 | struct drm_device *drm,
|
| 1097 | struct drm_gem_object *r_obj,
|
| 1098 | u64 start_offset, u64 range,
|
| 1099 | u64 reserve_offset, u64 reserve_range,
|
| 1100 | const struct drm_gpuvm_ops *ops)
|
| 1101 | {
|
| 1102 | gpuvm->rb.tree = RB_ROOT_CACHED;
|
| 1103 | INIT_LIST_HEAD(&gpuvm->rb.list);
|
| 1104 |
|
| 1105 | INIT_LIST_HEAD(&gpuvm->extobj.list);
|
| 1106 | spin_lock_init(&gpuvm->extobj.lock);
|
| 1107 |
|
| 1108 | INIT_LIST_HEAD(&gpuvm->evict.list);
|
| 1109 | spin_lock_init(&gpuvm->evict.lock);
|
| 1110 |
|
| 1111 | init_llist_head(&gpuvm->bo_defer);
|
| 1112 |
|
| 1113 | kref_init(&gpuvm->kref);
|
| 1114 |
|
| 1115 | gpuvm->name = name ? name : "unknown";
|
| 1116 | gpuvm->flags = flags;
|
| 1117 | gpuvm->ops = ops;
|
| 1118 | gpuvm->drm = drm;
|
| 1119 | gpuvm->r_obj = r_obj;
|
| 1120 |
|
| 1121 | drm_dev_get(drm);
|
| 1122 | drm_gem_object_get(r_obj);
|
| 1123 |
|
| 1124 | drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);
|
| 1125 | gpuvm->mm_start = start_offset;
|
| 1126 | gpuvm->mm_range = range;
|
| 1127 |
|
| 1128 | memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
|
| 1129 | if (reserve_range) {
|
| 1130 | gpuvm->kernel_alloc_node.va.addr = reserve_offset;
|
| 1131 | gpuvm->kernel_alloc_node.va.range = reserve_range;
|
| 1132 |
|
| 1133 | if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,
|
| 1134 | reserve_range)))
|
| 1135 | __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node);
|
| 1136 | }
|
| 1137 | }
|
| 1138 | EXPORT_SYMBOL_GPL(drm_gpuvm_init);
|
| 1139 |
|
| 1140 | static void
|
| 1141 | drm_gpuvm_fini(struct drm_gpuvm *gpuvm)
|
| 1142 | {
|
| 1143 | gpuvm->name = NULL;
|
| 1144 |
|
| 1145 | if (gpuvm->kernel_alloc_node.va.range)
|
| 1146 | __drm_gpuva_remove(&gpuvm->kernel_alloc_node);
|
| 1147 |
|
| 1148 | drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
|
| 1149 | "GPUVA tree is not empty, potentially leaking memory.\n");
|
| 1150 |
|
| 1151 | drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),
|
| 1152 | "Extobj list should be empty.\n");
|
| 1153 | drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),
|
| 1154 | "Evict list should be empty.\n");
|
| 1155 | drm_WARN(gpuvm->drm, !llist_empty(&gpuvm->bo_defer),
|
| 1156 | "VM BO cleanup list should be empty.\n");
|
| 1157 |
|
| 1158 | drm_gem_object_put(gpuvm->r_obj);
|
| 1159 | }
|
| 1160 |
|
| 1161 | static void
|
| 1162 | drm_gpuvm_free(struct kref *kref)
|
| 1163 | {
|
| 1164 | struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);
|
| 1165 | struct drm_device *drm = gpuvm->drm;
|
| 1166 |
|
| 1167 | drm_gpuvm_fini(gpuvm);
|
| 1168 |
|
| 1169 | if (drm_WARN_ON(drm, !gpuvm->ops->vm_free))
|
| 1170 | return;
|
| 1171 |
|
| 1172 | gpuvm->ops->vm_free(gpuvm);
|
| 1173 | drm_dev_put(drm);
|
| 1174 | }
|
| 1175 |
|
| 1176 | |
| 1177 | * drm_gpuvm_put() - drop a struct drm_gpuvm reference
|
| 1178 | * @gpuvm: the &drm_gpuvm to release the reference of
|
| 1179 | *
|
| 1180 | * This releases a reference to @gpuvm.
|
| 1181 | *
|
| 1182 | * This function may be called from atomic context.
|
| 1183 |
|
| 1184 | void
|
| 1185 | drm_gpuvm_put(struct drm_gpuvm *gpuvm)
|
| 1186 | {
|
| 1187 | if (gpuvm)
|
| 1188 | kref_put(&gpuvm->kref, drm_gpuvm_free);
|
| 1189 | }
|
| 1190 | EXPORT_SYMBOL_GPL(drm_gpuvm_put);
|
| 1191 |
|
| 1192 | static int
|
| 1193 | exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
|
| 1194 | unsigned int num_fences)
|
| 1195 | {
|
| 1196 | return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) :
|
| 1197 | drm_exec_lock_obj(exec, obj);
|
| 1198 | }
|
| 1199 |
|
| 1200 | |
| 1201 | * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv
|
| 1202 | * @gpuvm: the &drm_gpuvm
|
| 1203 | * @exec: the &drm_exec context
|
| 1204 | * @num_fences: the amount of &dma_fences to reserve
|
| 1205 | *
|
| 1206 | * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if
|
| 1207 | * @num_fences is zero drm_exec_lock_obj() is called instead.
|
| 1208 | *
|
| 1209 | * Using this function directly, it is the drivers responsibility to call
|
| 1210 | * drm_exec_init() and drm_exec_fini() accordingly.
|
| 1211 | *
|
| 1212 | * Returns: 0 on success, negative error code on failure.
|
| 1213 |
|
| 1214 | int
|
| 1215 | drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
|
| 1216 | struct drm_exec *exec,
|
| 1217 | unsigned int num_fences)
|
| 1218 | {
|
| 1219 | return exec_prepare_obj(exec, gpuvm->r_obj, num_fences);
|
| 1220 | }
|
| 1221 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm);
|
| 1222 |
|
| 1223 | static int
|
| 1224 | __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
|
| 1225 | struct drm_exec *exec,
|
| 1226 | unsigned int num_fences)
|
| 1227 | {
|
| 1228 | struct drm_gpuvm_bo *vm_bo;
|
| 1229 | LIST_HEAD(extobjs);
|
| 1230 | int ret = 0;
|
| 1231 |
|
| 1232 | for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {
|
| 1233 | ret = exec_prepare_obj(exec, vm_bo->obj, num_fences);
|
| 1234 | if (ret)
|
| 1235 | break;
|
| 1236 | }
|
| 1237 |
|
| 1238 | drm_gpuvm_bo_put(vm_bo);
|
| 1239 | restore_vm_bo_list(gpuvm, extobj);
|
| 1240 |
|
| 1241 | return ret;
|
| 1242 | }
|
| 1243 |
|
| 1244 | static int
|
| 1245 | drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,
|
| 1246 | struct drm_exec *exec,
|
| 1247 | unsigned int num_fences)
|
| 1248 | {
|
| 1249 | struct drm_gpuvm_bo *vm_bo;
|
| 1250 | int ret = 0;
|
| 1251 |
|
| 1252 | drm_gpuvm_resv_assert_held(gpuvm);
|
| 1253 | list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {
|
| 1254 | if (drm_gpuvm_bo_is_zombie(vm_bo))
|
| 1255 | continue;
|
| 1256 |
|
| 1257 | ret = exec_prepare_obj(exec, vm_bo->obj, num_fences);
|
| 1258 | if (ret)
|
| 1259 | break;
|
| 1260 |
|
| 1261 | if (vm_bo->evicted)
|
| 1262 | drm_gpuvm_bo_list_add(vm_bo, evict, false);
|
| 1263 | }
|
| 1264 |
|
| 1265 | return ret;
|
| 1266 | }
|
| 1267 |
|
| 1268 | |
| 1269 | * drm_gpuvm_prepare_objects() - prepare all associated BOs
|
| 1270 | * @gpuvm: the &drm_gpuvm
|
| 1271 | * @exec: the &drm_exec locking context
|
| 1272 | * @num_fences: the amount of &dma_fences to reserve
|
| 1273 | *
|
| 1274 | * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given
|
| 1275 | * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj()
|
| 1276 | * is called instead.
|
| 1277 | *
|
| 1278 | * Using this function directly, it is the drivers responsibility to call
|
| 1279 | * drm_exec_init() and drm_exec_fini() accordingly.
|
| 1280 | *
|
| 1281 | * Note: This function is safe against concurrent insertion and removal of
|
| 1282 | * external objects, however it is not safe against concurrent usage itself.
|
| 1283 | *
|
| 1284 | * Drivers need to make sure to protect this case with either an outer VM lock
|
| 1285 | * or by calling drm_gpuvm_prepare_vm() before this function within the
|
| 1286 | * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures
|
| 1287 | * mutual exclusion.
|
| 1288 | *
|
| 1289 | * Returns: 0 on success, negative error code on failure.
|
| 1290 |
|
| 1291 | int
|
| 1292 | drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
|
| 1293 | struct drm_exec *exec,
|
| 1294 | unsigned int num_fences)
|
| 1295 | {
|
| 1296 | if (drm_gpuvm_resv_protected(gpuvm))
|
| 1297 | return drm_gpuvm_prepare_objects_locked(gpuvm, exec,
|
| 1298 | num_fences);
|
| 1299 | else
|
| 1300 | return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
|
| 1301 | }
|
| 1302 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
|
| 1303 |
|
| 1304 | |
| 1305 | * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range
|
| 1306 | * @gpuvm: the &drm_gpuvm
|
| 1307 | * @exec: the &drm_exec locking context
|
| 1308 | * @addr: the start address within the VA space
|
| 1309 | * @range: the range to iterate within the VA space
|
| 1310 | * @num_fences: the amount of &dma_fences to reserve
|
| 1311 | *
|
| 1312 | * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr
|
| 1313 | * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called
|
| 1314 | * instead.
|
| 1315 | *
|
| 1316 | * Returns: 0 on success, negative error code on failure.
|
| 1317 |
|
| 1318 | int
|
| 1319 | drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
|
| 1320 | u64 addr, u64 range, unsigned int num_fences)
|
| 1321 | {
|
| 1322 | struct drm_gpuva *va;
|
| 1323 | u64 end = addr + range;
|
| 1324 | int ret;
|
| 1325 |
|
| 1326 | drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
|
| 1327 | struct drm_gem_object *obj = va->gem.obj;
|
| 1328 |
|
| 1329 | if (unlikely(!obj))
|
| 1330 | continue;
|
| 1331 |
|
| 1332 | ret = exec_prepare_obj(exec, obj, num_fences);
|
| 1333 | if (ret)
|
| 1334 | return ret;
|
| 1335 | }
|
| 1336 |
|
| 1337 | return 0;
|
| 1338 | }
|
| 1339 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
|
| 1340 |
|
| 1341 | |
| 1342 | * drm_gpuvm_exec_lock() - lock all dma-resv of all associated BOs
|
| 1343 | * @vm_exec: the &drm_gpuvm_exec wrapper
|
| 1344 | *
|
| 1345 | * Acquires all dma-resv locks of all &drm_gem_objects the given
|
| 1346 | * &drm_gpuvm contains mappings of.
|
| 1347 | *
|
| 1348 | * Additionally, when calling this function with struct drm_gpuvm_exec::extra
|
| 1349 | * being set the driver receives the given @fn callback to lock additional
|
| 1350 | * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers
|
| 1351 | * would call drm_exec_prepare_obj() from within this callback.
|
| 1352 | *
|
| 1353 | * Returns: 0 on success, negative error code on failure.
|
| 1354 |
|
| 1355 | int
|
| 1356 | drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)
|
| 1357 | {
|
| 1358 | struct drm_gpuvm *gpuvm = vm_exec->vm;
|
| 1359 | struct drm_exec *exec = &vm_exec->exec;
|
| 1360 | unsigned int num_fences = vm_exec->num_fences;
|
| 1361 | int ret;
|
| 1362 |
|
| 1363 | drm_exec_init(exec, vm_exec->flags, 0);
|
| 1364 |
|
| 1365 | drm_exec_until_all_locked(exec) {
|
| 1366 | ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);
|
| 1367 | drm_exec_retry_on_contention(exec);
|
| 1368 | if (ret)
|
| 1369 | goto err;
|
| 1370 |
|
| 1371 | ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
|
| 1372 | drm_exec_retry_on_contention(exec);
|
| 1373 | if (ret)
|
| 1374 | goto err;
|
| 1375 |
|
| 1376 | if (vm_exec->extra.fn) {
|
| 1377 | ret = vm_exec->extra.fn(vm_exec);
|
| 1378 | drm_exec_retry_on_contention(exec);
|
| 1379 | if (ret)
|
| 1380 | goto err;
|
| 1381 | }
|
| 1382 | }
|
| 1383 |
|
| 1384 | return 0;
|
| 1385 |
|
| 1386 | err:
|
| 1387 | drm_exec_fini(exec);
|
| 1388 | return ret;
|
| 1389 | }
|
| 1390 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
|
| 1391 |
|
| 1392 | static int
|
| 1393 | fn_lock_array(struct drm_gpuvm_exec *vm_exec)
|
| 1394 | {
|
| 1395 | struct {
|
| 1396 | struct drm_gem_object **objs;
|
| 1397 | unsigned int num_objs;
|
| 1398 | } *args = vm_exec->extra.priv;
|
| 1399 |
|
| 1400 | return drm_exec_prepare_array(&vm_exec->exec, args->objs,
|
| 1401 | args->num_objs, vm_exec->num_fences);
|
| 1402 | }
|
| 1403 |
|
| 1404 | |
| 1405 | * drm_gpuvm_exec_lock_array() - lock all dma-resv of all associated BOs
|
| 1406 | * @vm_exec: the &drm_gpuvm_exec wrapper
|
| 1407 | * @objs: additional &drm_gem_objects to lock
|
| 1408 | * @num_objs: the number of additional &drm_gem_objects to lock
|
| 1409 | *
|
| 1410 | * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm
|
| 1411 | * contains mappings of, plus the ones given through @objs.
|
| 1412 | *
|
| 1413 | * Returns: 0 on success, negative error code on failure.
|
| 1414 |
|
| 1415 | int
|
| 1416 | drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
|
| 1417 | struct drm_gem_object **objs,
|
| 1418 | unsigned int num_objs)
|
| 1419 | {
|
| 1420 | struct {
|
| 1421 | struct drm_gem_object **objs;
|
| 1422 | unsigned int num_objs;
|
| 1423 | } args;
|
| 1424 |
|
| 1425 | args.objs = objs;
|
| 1426 | args.num_objs = num_objs;
|
| 1427 |
|
| 1428 | vm_exec->extra.fn = fn_lock_array;
|
| 1429 | vm_exec->extra.priv = &args;
|
| 1430 |
|
| 1431 | return drm_gpuvm_exec_lock(vm_exec);
|
| 1432 | }
|
| 1433 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
|
| 1434 |
|
| 1435 | |
| 1436 | * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range
|
| 1437 | * @vm_exec: the &drm_gpuvm_exec wrapper
|
| 1438 | * @addr: the start address within the VA space
|
| 1439 | * @range: the range to iterate within the VA space
|
| 1440 | *
|
| 1441 | * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and
|
| 1442 | * @addr + @range.
|
| 1443 | *
|
| 1444 | * Returns: 0 on success, negative error code on failure.
|
| 1445 |
|
| 1446 | int
|
| 1447 | drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
|
| 1448 | u64 addr, u64 range)
|
| 1449 | {
|
| 1450 | struct drm_gpuvm *gpuvm = vm_exec->vm;
|
| 1451 | struct drm_exec *exec = &vm_exec->exec;
|
| 1452 | int ret;
|
| 1453 |
|
| 1454 | drm_exec_init(exec, vm_exec->flags, 0);
|
| 1455 |
|
| 1456 | drm_exec_until_all_locked(exec) {
|
| 1457 | ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,
|
| 1458 | vm_exec->num_fences);
|
| 1459 | drm_exec_retry_on_contention(exec);
|
| 1460 | if (ret)
|
| 1461 | goto err;
|
| 1462 | }
|
| 1463 |
|
| 1464 | return ret;
|
| 1465 |
|
| 1466 | err:
|
| 1467 | drm_exec_fini(exec);
|
| 1468 | return ret;
|
| 1469 | }
|
| 1470 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
|
| 1471 |
|
| 1472 | static int
|
| 1473 | __drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
|
| 1474 | {
|
| 1475 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 1476 | struct drm_gpuvm_bo *vm_bo;
|
| 1477 | LIST_HEAD(evict);
|
| 1478 | int ret = 0;
|
| 1479 |
|
| 1480 | for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {
|
| 1481 | ret = ops->vm_bo_validate(vm_bo, exec);
|
| 1482 | if (ret)
|
| 1483 | break;
|
| 1484 | }
|
| 1485 |
|
| 1486 | drm_gpuvm_bo_put(vm_bo);
|
| 1487 | restore_vm_bo_list(gpuvm, evict);
|
| 1488 |
|
| 1489 | return ret;
|
| 1490 | }
|
| 1491 |
|
| 1492 | static int
|
| 1493 | drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
|
| 1494 | {
|
| 1495 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 1496 | struct drm_gpuvm_bo *vm_bo, *next;
|
| 1497 | int ret = 0;
|
| 1498 |
|
| 1499 | drm_gpuvm_resv_assert_held(gpuvm);
|
| 1500 |
|
| 1501 | list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,
|
| 1502 | list.entry.evict) {
|
| 1503 | if (drm_gpuvm_bo_is_zombie(vm_bo))
|
| 1504 | continue;
|
| 1505 |
|
| 1506 | ret = ops->vm_bo_validate(vm_bo, exec);
|
| 1507 | if (ret)
|
| 1508 | break;
|
| 1509 |
|
| 1510 | dma_resv_assert_held(vm_bo->obj->resv);
|
| 1511 | if (!vm_bo->evicted)
|
| 1512 | drm_gpuvm_bo_list_del_init(vm_bo, evict, false);
|
| 1513 | }
|
| 1514 |
|
| 1515 | return ret;
|
| 1516 | }
|
| 1517 |
|
| 1518 | |
| 1519 | * drm_gpuvm_validate() - validate all BOs marked as evicted
|
| 1520 | * @gpuvm: the &drm_gpuvm to validate evicted BOs
|
| 1521 | * @exec: the &drm_exec instance used for locking the GPUVM
|
| 1522 | *
|
| 1523 | * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer
|
| 1524 | * objects being mapped in the given &drm_gpuvm.
|
| 1525 | *
|
| 1526 | * Returns: 0 on success, negative error code on failure.
|
| 1527 |
|
| 1528 | int
|
| 1529 | drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
|
| 1530 | {
|
| 1531 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 1532 |
|
| 1533 | if (unlikely(!ops || !ops->vm_bo_validate))
|
| 1534 | return -EOPNOTSUPP;
|
| 1535 |
|
| 1536 | if (drm_gpuvm_resv_protected(gpuvm))
|
| 1537 | return drm_gpuvm_validate_locked(gpuvm, exec);
|
| 1538 | else
|
| 1539 | return __drm_gpuvm_validate(gpuvm, exec);
|
| 1540 | }
|
| 1541 | EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
|
| 1542 |
|
| 1543 | |
| 1544 | * drm_gpuvm_resv_add_fence - add fence to private and all extobj
|
| 1545 | * dma-resv
|
| 1546 | * @gpuvm: the &drm_gpuvm to add a fence to
|
| 1547 | * @exec: the &drm_exec locking context
|
| 1548 | * @fence: fence to add
|
| 1549 | * @private_usage: private dma-resv usage
|
| 1550 | * @extobj_usage: extobj dma-resv usage
|
| 1551 |
|
| 1552 | void
|
| 1553 | drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
|
| 1554 | struct drm_exec *exec,
|
| 1555 | struct dma_fence *fence,
|
| 1556 | enum dma_resv_usage private_usage,
|
| 1557 | enum dma_resv_usage extobj_usage)
|
| 1558 | {
|
| 1559 | struct drm_gem_object *obj;
|
| 1560 |
|
| 1561 | drm_exec_for_each_locked_object(exec, obj) {
|
| 1562 | dma_resv_assert_held(obj->resv);
|
| 1563 | dma_resv_add_fence(obj->resv, fence,
|
| 1564 | drm_gpuvm_is_extobj(gpuvm, obj) ?
|
| 1565 | extobj_usage : private_usage);
|
| 1566 | }
|
| 1567 | }
|
| 1568 | EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
|
| 1569 |
|
| 1570 | |
| 1571 | * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
|
| 1572 | * @gpuvm: The &drm_gpuvm the @obj is mapped in.
|
| 1573 | * @obj: The &drm_gem_object being mapped in the @gpuvm.
|
| 1574 | *
|
| 1575 | * If provided by the driver, this function uses the &drm_gpuvm_ops
|
| 1576 | * vm_bo_alloc() callback to allocate.
|
| 1577 | *
|
| 1578 | * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
|
| 1579 |
|
| 1580 | struct drm_gpuvm_bo *
|
| 1581 | drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
|
| 1582 | struct drm_gem_object *obj)
|
| 1583 | {
|
| 1584 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 1585 | struct drm_gpuvm_bo *vm_bo;
|
| 1586 |
|
| 1587 | if (ops && ops->vm_bo_alloc)
|
| 1588 | vm_bo = ops->vm_bo_alloc();
|
| 1589 | else
|
| 1590 | vm_bo = kzalloc_obj(*vm_bo);
|
| 1591 |
|
| 1592 | if (unlikely(!vm_bo))
|
| 1593 | return NULL;
|
| 1594 |
|
| 1595 | vm_bo->vm = drm_gpuvm_get(gpuvm);
|
| 1596 | vm_bo->obj = obj;
|
| 1597 | drm_gem_object_get(obj);
|
| 1598 |
|
| 1599 | kref_init(&vm_bo->kref);
|
| 1600 | INIT_LIST_HEAD(&vm_bo->list.gpuva);
|
| 1601 | INIT_LIST_HEAD(&vm_bo->list.entry.gem);
|
| 1602 |
|
| 1603 | INIT_LIST_HEAD(&vm_bo->list.entry.extobj);
|
| 1604 | INIT_LIST_HEAD(&vm_bo->list.entry.evict);
|
| 1605 | init_llist_node(&vm_bo->list.entry.bo_defer);
|
| 1606 |
|
| 1607 | return vm_bo;
|
| 1608 | }
|
| 1609 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
|
| 1610 |
|
| 1611 | |
| 1612 | * drm_gpuvm_bo_destroy_not_in_lists() - final part of drm_gpuvm_bo cleanup
|
| 1613 | * @vm_bo: the &drm_gpuvm_bo to destroy
|
| 1614 | *
|
| 1615 | * It is illegal to call this method if the @vm_bo is present in the GEMs gpuva
|
| 1616 | * list, the extobj list, or the evicted list.
|
| 1617 | *
|
| 1618 | * Note that this puts a refcount on the GEM object, which may destroy the GEM
|
| 1619 | * object if the refcount reaches zero. It's illegal for this to happen if the
|
| 1620 | * caller holds the GEMs gpuva mutex because it would free the mutex.
|
| 1621 |
|
| 1622 | static void
|
| 1623 | drm_gpuvm_bo_destroy_not_in_lists(struct drm_gpuvm_bo *vm_bo)
|
| 1624 | {
|
| 1625 | struct drm_gpuvm *gpuvm = vm_bo->vm;
|
| 1626 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 1627 | struct drm_gem_object *obj = vm_bo->obj;
|
| 1628 |
|
| 1629 | if (ops && ops->vm_bo_free)
|
| 1630 | ops->vm_bo_free(vm_bo);
|
| 1631 | else
|
| 1632 | kfree(vm_bo);
|
| 1633 |
|
| 1634 | drm_gpuvm_put(gpuvm);
|
| 1635 | drm_gem_object_put(obj);
|
| 1636 | }
|
| 1637 |
|
| 1638 | static void
|
| 1639 | drm_gpuvm_bo_destroy_not_in_lists_kref(struct kref *kref)
|
| 1640 | {
|
| 1641 | struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
|
| 1642 | kref);
|
| 1643 |
|
| 1644 | drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
|
| 1645 | }
|
| 1646 |
|
| 1647 | static void
|
| 1648 | drm_gpuvm_bo_destroy(struct kref *kref)
|
| 1649 | {
|
| 1650 | struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
|
| 1651 | kref);
|
| 1652 | struct drm_gpuvm *gpuvm = vm_bo->vm;
|
| 1653 | bool lock = !drm_gpuvm_resv_protected(gpuvm);
|
| 1654 |
|
| 1655 | if (!lock)
|
| 1656 | drm_gpuvm_resv_assert_held(gpuvm);
|
| 1657 |
|
| 1658 | drm_gpuvm_bo_list_del(vm_bo, extobj, lock);
|
| 1659 | drm_gpuvm_bo_list_del(vm_bo, evict, lock);
|
| 1660 |
|
| 1661 | drm_gem_gpuva_assert_lock_held(gpuvm, vm_bo->obj);
|
| 1662 | list_del(&vm_bo->list.entry.gem);
|
| 1663 |
|
| 1664 | drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
|
| 1665 | }
|
| 1666 |
|
| 1667 | |
| 1668 | * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference
|
| 1669 | * @vm_bo: the &drm_gpuvm_bo to release the reference of
|
| 1670 | *
|
| 1671 | * This releases a reference to @vm_bo.
|
| 1672 | *
|
| 1673 | * If the reference count drops to zero, the &gpuvm_bo is destroyed, which
|
| 1674 | * includes removing it from the GEMs gpuva list. Hence, if a call to this
|
| 1675 | * function can potentially let the reference count drop to zero the caller must
|
| 1676 | * hold the lock that the GEM uses for its gpuva list (either the GEM's
|
| 1677 | * dma-resv or gpuva.lock mutex).
|
| 1678 | *
|
| 1679 | * This function may only be called from non-atomic context.
|
| 1680 | *
|
| 1681 | * Returns: true if vm_bo was destroyed, false otherwise.
|
| 1682 |
|
| 1683 | bool
|
| 1684 | drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)
|
| 1685 | {
|
| 1686 | might_sleep();
|
| 1687 |
|
| 1688 | if (vm_bo)
|
| 1689 | return !!kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy);
|
| 1690 |
|
| 1691 | return false;
|
| 1692 | }
|
| 1693 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
|
| 1694 |
|
| 1695 | |
| 1696 | * drm_gpuvm_bo_into_zombie() - called when the vm_bo becomes a zombie due to
|
| 1697 | * deferred cleanup
|
| 1698 | *
|
| 1699 | * If deferred cleanup is used, then this must be called right after the vm_bo
|
| 1700 | * refcount drops to zero. Must be called with GEM mutex held. After releasing
|
| 1701 | * the GEM mutex, drm_gpuvm_bo_defer_zombie_cleanup() must be called.
|
| 1702 |
|
| 1703 | static void
|
| 1704 | drm_gpuvm_bo_into_zombie(struct kref *kref)
|
| 1705 | {
|
| 1706 | struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
|
| 1707 | kref);
|
| 1708 |
|
| 1709 | if (!drm_gpuvm_resv_protected(vm_bo->vm)) {
|
| 1710 | drm_gpuvm_bo_list_del(vm_bo, extobj, true);
|
| 1711 | drm_gpuvm_bo_list_del(vm_bo, evict, true);
|
| 1712 | }
|
| 1713 |
|
| 1714 | list_del(&vm_bo->list.entry.gem);
|
| 1715 | }
|
| 1716 |
|
| 1717 | |
| 1718 | * drm_gpuvm_bo_defer_zombie_cleanup() - adds a new zombie vm_bo to the
|
| 1719 | * bo_defer list
|
| 1720 | *
|
| 1721 | * Called after drm_gpuvm_bo_into_zombie(). GEM mutex must not be held.
|
| 1722 | *
|
| 1723 | * It's important that the GEM stays alive for the duration in which we hold
|
| 1724 | * the mutex, but the instant we add the vm_bo to bo_defer, another thread
|
| 1725 | * might call drm_gpuvm_bo_deferred_cleanup() and put the GEM. Therefore, to
|
| 1726 | * avoid kfreeing a mutex we are holding, the GEM mutex must be released
|
| 1727 | * *before* calling this function.
|
| 1728 |
|
| 1729 | static void
|
| 1730 | drm_gpuvm_bo_defer_zombie_cleanup(struct drm_gpuvm_bo *vm_bo)
|
| 1731 | {
|
| 1732 | llist_add(&vm_bo->list.entry.bo_defer, &vm_bo->vm->bo_defer);
|
| 1733 | }
|
| 1734 |
|
| 1735 | static void
|
| 1736 | drm_gpuvm_bo_defer_free(struct kref *kref)
|
| 1737 | {
|
| 1738 | struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
|
| 1739 | kref);
|
| 1740 |
|
| 1741 | drm_gpuvm_bo_into_zombie(kref);
|
| 1742 | mutex_unlock(&vm_bo->obj->gpuva.lock);
|
| 1743 | drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);
|
| 1744 | }
|
| 1745 |
|
| 1746 | |
| 1747 | * drm_gpuvm_bo_put_deferred() - drop a struct drm_gpuvm_bo reference with
|
| 1748 | * deferred cleanup
|
| 1749 | * @vm_bo: the &drm_gpuvm_bo to release the reference of
|
| 1750 | *
|
| 1751 | * This releases a reference to @vm_bo.
|
| 1752 | *
|
| 1753 | * This might take and release the GEMs GPUVA lock. You should call
|
| 1754 | * drm_gpuvm_bo_deferred_cleanup() later to complete the cleanup process.
|
| 1755 | *
|
| 1756 | * Returns: true if vm_bo is being destroyed, false otherwise.
|
| 1757 |
|
| 1758 | bool
|
| 1759 | drm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo)
|
| 1760 | {
|
| 1761 | if (!vm_bo)
|
| 1762 | return false;
|
| 1763 |
|
| 1764 | drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
|
| 1765 |
|
| 1766 | return !!kref_put_mutex(&vm_bo->kref,
|
| 1767 | drm_gpuvm_bo_defer_free,
|
| 1768 | &vm_bo->obj->gpuva.lock);
|
| 1769 | }
|
| 1770 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put_deferred);
|
| 1771 |
|
| 1772 | |
| 1773 | * drm_gpuvm_bo_deferred_cleanup() - clean up BOs in the deferred list
|
| 1774 | * deferred cleanup
|
| 1775 | * @gpuvm: the VM to clean up
|
| 1776 | *
|
| 1777 | * Cleans up &drm_gpuvm_bo instances in the deferred cleanup list.
|
| 1778 |
|
| 1779 | void
|
| 1780 | drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm)
|
| 1781 | {
|
| 1782 | struct drm_gpuvm_bo *vm_bo;
|
| 1783 | struct llist_node *bo_defer;
|
| 1784 |
|
| 1785 | bo_defer = llist_del_all(&gpuvm->bo_defer);
|
| 1786 | if (!bo_defer)
|
| 1787 | return;
|
| 1788 |
|
| 1789 | if (drm_gpuvm_resv_protected(gpuvm)) {
|
| 1790 | dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL);
|
| 1791 | llist_for_each_entry(vm_bo, bo_defer, list.entry.bo_defer) {
|
| 1792 | drm_gpuvm_bo_list_del(vm_bo, extobj, false);
|
| 1793 | drm_gpuvm_bo_list_del(vm_bo, evict, false);
|
| 1794 | }
|
| 1795 | dma_resv_unlock(drm_gpuvm_resv(gpuvm));
|
| 1796 | }
|
| 1797 |
|
| 1798 | while (bo_defer) {
|
| 1799 | vm_bo = llist_entry(bo_defer, struct drm_gpuvm_bo, list.entry.bo_defer);
|
| 1800 | bo_defer = bo_defer->next;
|
| 1801 | drm_gpuvm_bo_destroy_not_in_lists(vm_bo);
|
| 1802 | }
|
| 1803 | }
|
| 1804 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_deferred_cleanup);
|
| 1805 |
|
| 1806 | static struct drm_gpuvm_bo *
|
| 1807 | __drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
|
| 1808 | struct drm_gem_object *obj)
|
| 1809 | {
|
| 1810 | struct drm_gpuvm_bo *vm_bo;
|
| 1811 |
|
| 1812 | drm_gem_gpuva_assert_lock_held(gpuvm, obj);
|
| 1813 | drm_gem_for_each_gpuvm_bo(vm_bo, obj)
|
| 1814 | if (vm_bo->vm == gpuvm)
|
| 1815 | return vm_bo;
|
| 1816 |
|
| 1817 | return NULL;
|
| 1818 | }
|
| 1819 |
|
| 1820 | |
| 1821 | * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given
|
| 1822 | * &drm_gpuvm and &drm_gem_object
|
| 1823 | * @gpuvm: The &drm_gpuvm the @obj is mapped in.
|
| 1824 | * @obj: The &drm_gem_object being mapped in the @gpuvm.
|
| 1825 | *
|
| 1826 | * Find the &drm_gpuvm_bo representing the combination of the given
|
| 1827 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference
|
| 1828 | * count of the &drm_gpuvm_bo accordingly.
|
| 1829 | *
|
| 1830 | * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
|
| 1831 |
|
| 1832 | struct drm_gpuvm_bo *
|
| 1833 | drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
|
| 1834 | struct drm_gem_object *obj)
|
| 1835 | {
|
| 1836 | struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
|
| 1837 |
|
| 1838 | return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;
|
| 1839 | }
|
| 1840 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
|
| 1841 |
|
| 1842 | |
| 1843 | * drm_gpuvm_bo_obtain_locked() - obtains an instance of the &drm_gpuvm_bo for
|
| 1844 | * the given &drm_gpuvm and &drm_gem_object
|
| 1845 | * @gpuvm: The &drm_gpuvm the @obj is mapped in.
|
| 1846 | * @obj: The &drm_gem_object being mapped in the @gpuvm.
|
| 1847 | *
|
| 1848 | * Find the &drm_gpuvm_bo representing the combination of the given
|
| 1849 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference
|
| 1850 | * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new
|
| 1851 | * &drm_gpuvm_bo.
|
| 1852 | *
|
| 1853 | * Requires the lock for the GEMs gpuva list.
|
| 1854 | *
|
| 1855 | * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
|
| 1856 | *
|
| 1857 | * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure
|
| 1858 |
|
| 1859 | struct drm_gpuvm_bo *
|
| 1860 | drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm,
|
| 1861 | struct drm_gem_object *obj)
|
| 1862 | {
|
| 1863 | struct drm_gpuvm_bo *vm_bo;
|
| 1864 |
|
| 1865 | |
| 1866 | * In immediate mode this would require the caller to hold the GEMs
|
| 1867 | * gpuva mutex, but it's not okay to allocate while holding that lock,
|
| 1868 | * and this method allocates. Immediate mode drivers should use
|
| 1869 | * drm_gpuvm_bo_obtain_prealloc() instead.
|
| 1870 |
|
| 1871 | drm_WARN_ON(gpuvm->drm, drm_gpuvm_immediate_mode(gpuvm));
|
| 1872 |
|
| 1873 | vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
|
| 1874 | if (vm_bo)
|
| 1875 | return vm_bo;
|
| 1876 |
|
| 1877 | vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
|
| 1878 | if (!vm_bo)
|
| 1879 | return ERR_PTR(-ENOMEM);
|
| 1880 |
|
| 1881 | drm_gem_gpuva_assert_lock_held(gpuvm, obj);
|
| 1882 | list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list);
|
| 1883 |
|
| 1884 | return vm_bo;
|
| 1885 | }
|
| 1886 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_locked);
|
| 1887 |
|
| 1888 | |
| 1889 | * drm_gpuvm_bo_obtain_prealloc() - obtains an instance of the &drm_gpuvm_bo
|
| 1890 | * for the given &drm_gpuvm and &drm_gem_object
|
| 1891 | * @__vm_bo: A pre-allocated struct drm_gpuvm_bo.
|
| 1892 | *
|
| 1893 | * Find the &drm_gpuvm_bo representing the combination of the given
|
| 1894 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference
|
| 1895 | * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference
|
| 1896 | * count is decreased. If not found @__vm_bo is returned without further
|
| 1897 | * increase of the reference count.
|
| 1898 | *
|
| 1899 | * The provided @__vm_bo must not already be in the gpuva, evict, or extobj
|
| 1900 | * lists prior to calling this method.
|
| 1901 | *
|
| 1902 | * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
|
| 1903 | *
|
| 1904 | * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing
|
| 1905 | * &drm_gpuvm_bo was found
|
| 1906 |
|
| 1907 | struct drm_gpuvm_bo *
|
| 1908 | drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
|
| 1909 | {
|
| 1910 | struct drm_gpuvm *gpuvm = __vm_bo->vm;
|
| 1911 | struct drm_gem_object *obj = __vm_bo->obj;
|
| 1912 | struct drm_gpuvm_bo *vm_bo;
|
| 1913 |
|
| 1914 | drm_WARN_ON(gpuvm->drm, !drm_gpuvm_immediate_mode(gpuvm));
|
| 1915 |
|
| 1916 | mutex_lock(&obj->gpuva.lock);
|
| 1917 | vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
|
| 1918 | if (vm_bo) {
|
| 1919 | mutex_unlock(&obj->gpuva.lock);
|
| 1920 | kref_put(&__vm_bo->kref, drm_gpuvm_bo_destroy_not_in_lists_kref);
|
| 1921 | return vm_bo;
|
| 1922 | }
|
| 1923 |
|
| 1924 | drm_gem_gpuva_assert_lock_held(gpuvm, obj);
|
| 1925 | list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list);
|
| 1926 | mutex_unlock(&obj->gpuva.lock);
|
| 1927 |
|
| 1928 | return __vm_bo;
|
| 1929 | }
|
| 1930 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
|
| 1931 |
|
| 1932 | |
| 1933 | * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's
|
| 1934 | * extobj list
|
| 1935 | * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.
|
| 1936 | *
|
| 1937 | * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list
|
| 1938 | * already and if the corresponding &drm_gem_object is an external object,
|
| 1939 | * actually.
|
| 1940 |
|
| 1941 | void
|
| 1942 | drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)
|
| 1943 | {
|
| 1944 | struct drm_gpuvm *gpuvm = vm_bo->vm;
|
| 1945 | bool lock = !drm_gpuvm_resv_protected(gpuvm);
|
| 1946 |
|
| 1947 | if (!lock)
|
| 1948 | drm_gpuvm_resv_assert_held(gpuvm);
|
| 1949 |
|
| 1950 | if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj))
|
| 1951 | drm_gpuvm_bo_list_add(vm_bo, extobj, lock);
|
| 1952 | }
|
| 1953 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
|
| 1954 |
|
| 1955 | |
| 1956 | * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms
|
| 1957 | * evicted list
|
| 1958 | * @vm_bo: the &drm_gpuvm_bo to add or remove
|
| 1959 | * @evict: indicates whether the object is evicted
|
| 1960 | *
|
| 1961 | * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvm's evicted list.
|
| 1962 |
|
| 1963 | void
|
| 1964 | drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)
|
| 1965 | {
|
| 1966 | struct drm_gpuvm *gpuvm = vm_bo->vm;
|
| 1967 | struct drm_gem_object *obj = vm_bo->obj;
|
| 1968 | bool lock = !drm_gpuvm_resv_protected(gpuvm);
|
| 1969 |
|
| 1970 | dma_resv_assert_held(obj->resv);
|
| 1971 | vm_bo->evicted = evict;
|
| 1972 |
|
| 1973 | |
| 1974 | * internal spinlocks, since in this case the evicted list is protected
|
| 1975 | * with the VM's common dma-resv lock.
|
| 1976 |
|
| 1977 | if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)
|
| 1978 | return;
|
| 1979 |
|
| 1980 | if (evict)
|
| 1981 | drm_gpuvm_bo_list_add(vm_bo, evict, lock);
|
| 1982 | else
|
| 1983 | drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);
|
| 1984 | }
|
| 1985 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
|
| 1986 |
|
| 1987 | static int
|
| 1988 | __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
|
| 1989 | struct drm_gpuva *va)
|
| 1990 | {
|
| 1991 | struct rb_node *node;
|
| 1992 | struct list_head *head;
|
| 1993 |
|
| 1994 | if (drm_gpuva_it_iter_first(&gpuvm->rb.tree,
|
| 1995 | GPUVA_START(va),
|
| 1996 | GPUVA_LAST(va)))
|
| 1997 | return -EEXIST;
|
| 1998 |
|
| 1999 | va->vm = gpuvm;
|
| 2000 |
|
| 2001 | drm_gpuva_it_insert(va, &gpuvm->rb.tree);
|
| 2002 |
|
| 2003 | node = rb_prev(&va->rb.node);
|
| 2004 | if (node)
|
| 2005 | head = &(to_drm_gpuva(node))->rb.entry;
|
| 2006 | else
|
| 2007 | head = &gpuvm->rb.list;
|
| 2008 |
|
| 2009 | list_add(&va->rb.entry, head);
|
| 2010 |
|
| 2011 | return 0;
|
| 2012 | }
|
| 2013 |
|
| 2014 | |
| 2015 | * drm_gpuva_insert() - insert a &drm_gpuva
|
| 2016 | * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in
|
| 2017 | * @va: the &drm_gpuva to insert
|
| 2018 | *
|
| 2019 | * Insert a &drm_gpuva with a given address and range into a
|
| 2020 | * &drm_gpuvm.
|
| 2021 | *
|
| 2022 | * It is safe to use this function using the safe versions of iterating the GPU
|
| 2023 | * VA space, such as drm_gpuvm_for_each_va_safe() and
|
| 2024 | * drm_gpuvm_for_each_va_range_safe().
|
| 2025 | *
|
| 2026 | * Returns: 0 on success, negative error code on failure.
|
| 2027 |
|
| 2028 | int
|
| 2029 | drm_gpuva_insert(struct drm_gpuvm *gpuvm,
|
| 2030 | struct drm_gpuva *va)
|
| 2031 | {
|
| 2032 | u64 addr = va->va.addr;
|
| 2033 | u64 range = va->va.range;
|
| 2034 | int ret;
|
| 2035 |
|
| 2036 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))
|
| 2037 | return -EINVAL;
|
| 2038 |
|
| 2039 | ret = __drm_gpuva_insert(gpuvm, va);
|
| 2040 | if (likely(!ret))
|
| 2041 | |
| 2042 | * drm_gpuva. We can't take the reference in
|
| 2043 | * __drm_gpuva_insert() itself, since we don't want to increse
|
| 2044 | * the reference count for the GPUVM's kernel_alloc_node.
|
| 2045 |
|
| 2046 | drm_gpuvm_get(gpuvm);
|
| 2047 |
|
| 2048 | return ret;
|
| 2049 | }
|
| 2050 | EXPORT_SYMBOL_GPL(drm_gpuva_insert);
|
| 2051 |
|
| 2052 | static void
|
| 2053 | __drm_gpuva_remove(struct drm_gpuva *va)
|
| 2054 | {
|
| 2055 | drm_gpuva_it_remove(va, &va->vm->rb.tree);
|
| 2056 | list_del_init(&va->rb.entry);
|
| 2057 | }
|
| 2058 |
|
| 2059 | |
| 2060 | * drm_gpuva_remove() - remove a &drm_gpuva
|
| 2061 | * @va: the &drm_gpuva to remove
|
| 2062 | *
|
| 2063 | * This removes the given &va from the underlying tree.
|
| 2064 | *
|
| 2065 | * It is safe to use this function using the safe versions of iterating the GPU
|
| 2066 | * VA space, such as drm_gpuvm_for_each_va_safe() and
|
| 2067 | * drm_gpuvm_for_each_va_range_safe().
|
| 2068 |
|
| 2069 | void
|
| 2070 | drm_gpuva_remove(struct drm_gpuva *va)
|
| 2071 | {
|
| 2072 | struct drm_gpuvm *gpuvm = va->vm;
|
| 2073 |
|
| 2074 | if (unlikely(va == &gpuvm->kernel_alloc_node)) {
|
| 2075 | drm_WARN(gpuvm->drm, 1,
|
| 2076 | "Can't destroy kernel reserved node.\n");
|
| 2077 | return;
|
| 2078 | }
|
| 2079 |
|
| 2080 | __drm_gpuva_remove(va);
|
| 2081 | drm_gpuvm_put(va->vm);
|
| 2082 | }
|
| 2083 | EXPORT_SYMBOL_GPL(drm_gpuva_remove);
|
| 2084 |
|
| 2085 | |
| 2086 | * drm_gpuva_link() - link a &drm_gpuva
|
| 2087 | * @va: the &drm_gpuva to link
|
| 2088 | * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to
|
| 2089 | *
|
| 2090 | * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the
|
| 2091 | * &drm_gpuvm_bo to the &drm_gem_object it is associated with.
|
| 2092 | *
|
| 2093 | * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional
|
| 2094 | * reference of the latter is taken.
|
| 2095 | *
|
| 2096 | * This function expects the caller to protect the GEM's GPUVA list against
|
| 2097 | * concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
|
| 2098 |
|
| 2099 | void
|
| 2100 | drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)
|
| 2101 | {
|
| 2102 | struct drm_gem_object *obj = va->gem.obj;
|
| 2103 | struct drm_gpuvm *gpuvm = va->vm;
|
| 2104 |
|
| 2105 | if (unlikely(!obj))
|
| 2106 | return;
|
| 2107 |
|
| 2108 | drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
|
| 2109 |
|
| 2110 | va->vm_bo = drm_gpuvm_bo_get(vm_bo);
|
| 2111 |
|
| 2112 | drm_gem_gpuva_assert_lock_held(gpuvm, obj);
|
| 2113 | list_add_tail(&va->gem.entry, &vm_bo->list.gpuva);
|
| 2114 | }
|
| 2115 | EXPORT_SYMBOL_GPL(drm_gpuva_link);
|
| 2116 |
|
| 2117 | |
| 2118 | * drm_gpuva_unlink() - unlink a &drm_gpuva
|
| 2119 | * @va: the &drm_gpuva to unlink
|
| 2120 | *
|
| 2121 | * This removes the given &va from the GPU VA list of the &drm_gem_object it is
|
| 2122 | * associated with.
|
| 2123 | *
|
| 2124 | * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and
|
| 2125 | * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case
|
| 2126 | * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.
|
| 2127 | *
|
| 2128 | * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of
|
| 2129 | * the latter is dropped.
|
| 2130 | *
|
| 2131 | * This function expects the caller to protect the GEM's GPUVA list against
|
| 2132 | * concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
|
| 2133 |
|
| 2134 | void
|
| 2135 | drm_gpuva_unlink(struct drm_gpuva *va)
|
| 2136 | {
|
| 2137 | struct drm_gem_object *obj = va->gem.obj;
|
| 2138 | struct drm_gpuvm_bo *vm_bo = va->vm_bo;
|
| 2139 |
|
| 2140 | if (unlikely(!obj))
|
| 2141 | return;
|
| 2142 |
|
| 2143 | drm_gem_gpuva_assert_lock_held(va->vm, obj);
|
| 2144 | list_del_init(&va->gem.entry);
|
| 2145 |
|
| 2146 | va->vm_bo = NULL;
|
| 2147 | drm_gpuvm_bo_put(vm_bo);
|
| 2148 | }
|
| 2149 | EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
|
| 2150 |
|
| 2151 | |
| 2152 | * drm_gpuva_unlink_defer() - unlink a &drm_gpuva with deferred vm_bo cleanup
|
| 2153 | * @va: the &drm_gpuva to unlink
|
| 2154 | *
|
| 2155 | * Similar to drm_gpuva_unlink(), but uses drm_gpuvm_bo_put_deferred() and takes
|
| 2156 | * the lock for the caller.
|
| 2157 |
|
| 2158 | void
|
| 2159 | drm_gpuva_unlink_defer(struct drm_gpuva *va)
|
| 2160 | {
|
| 2161 | struct drm_gem_object *obj = va->gem.obj;
|
| 2162 | struct drm_gpuvm_bo *vm_bo = va->vm_bo;
|
| 2163 | bool should_defer_bo;
|
| 2164 |
|
| 2165 | if (unlikely(!obj))
|
| 2166 | return;
|
| 2167 |
|
| 2168 | drm_WARN_ON(vm_bo->vm->drm, !drm_gpuvm_immediate_mode(vm_bo->vm));
|
| 2169 |
|
| 2170 | mutex_lock(&obj->gpuva.lock);
|
| 2171 | list_del_init(&va->gem.entry);
|
| 2172 |
|
| 2173 | |
| 2174 | * This is drm_gpuvm_bo_put_deferred() except we already hold the mutex.
|
| 2175 |
|
| 2176 | should_defer_bo = kref_put(&vm_bo->kref, drm_gpuvm_bo_into_zombie);
|
| 2177 | mutex_unlock(&obj->gpuva.lock);
|
| 2178 | if (should_defer_bo)
|
| 2179 | drm_gpuvm_bo_defer_zombie_cleanup(vm_bo);
|
| 2180 |
|
| 2181 | va->vm_bo = NULL;
|
| 2182 | }
|
| 2183 | EXPORT_SYMBOL_GPL(drm_gpuva_unlink_defer);
|
| 2184 |
|
| 2185 | |
| 2186 | * drm_gpuva_find_first() - find the first &drm_gpuva in the given range
|
| 2187 | * @gpuvm: the &drm_gpuvm to search in
|
| 2188 | * @addr: the &drm_gpuvas address
|
| 2189 | * @range: the &drm_gpuvas range
|
| 2190 | *
|
| 2191 | * Returns: the first &drm_gpuva within the given range
|
| 2192 |
|
| 2193 | struct drm_gpuva *
|
| 2194 | drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
|
| 2195 | u64 addr, u64 range)
|
| 2196 | {
|
| 2197 | u64 last = addr + range - 1;
|
| 2198 |
|
| 2199 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last);
|
| 2200 | }
|
| 2201 | EXPORT_SYMBOL_GPL(drm_gpuva_find_first);
|
| 2202 |
|
| 2203 | |
| 2204 | * drm_gpuva_find() - find a &drm_gpuva
|
| 2205 | * @gpuvm: the &drm_gpuvm to search in
|
| 2206 | * @addr: the &drm_gpuvas address
|
| 2207 | * @range: the &drm_gpuvas range
|
| 2208 | *
|
| 2209 | * Returns: the &drm_gpuva at a given &addr and with a given &range
|
| 2210 |
|
| 2211 | struct drm_gpuva *
|
| 2212 | drm_gpuva_find(struct drm_gpuvm *gpuvm,
|
| 2213 | u64 addr, u64 range)
|
| 2214 | {
|
| 2215 | struct drm_gpuva *va;
|
| 2216 |
|
| 2217 | va = drm_gpuva_find_first(gpuvm, addr, range);
|
| 2218 | if (!va)
|
| 2219 | goto out;
|
| 2220 |
|
| 2221 | if (va->va.addr != addr ||
|
| 2222 | va->va.range != range)
|
| 2223 | goto out;
|
| 2224 |
|
| 2225 | return va;
|
| 2226 |
|
| 2227 | out:
|
| 2228 | return NULL;
|
| 2229 | }
|
| 2230 | EXPORT_SYMBOL_GPL(drm_gpuva_find);
|
| 2231 |
|
| 2232 | |
| 2233 | * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space
|
| 2234 | * is empty
|
| 2235 | * @gpuvm: the &drm_gpuvm to check the range for
|
| 2236 | * @addr: the start address of the range
|
| 2237 | * @range: the range of the interval
|
| 2238 | *
|
| 2239 | * Returns: true if the interval is empty, false otherwise
|
| 2240 |
|
| 2241 | bool
|
| 2242 | drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
|
| 2243 | {
|
| 2244 | return !drm_gpuva_find_first(gpuvm, addr, range);
|
| 2245 | }
|
| 2246 | EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty);
|
| 2247 |
|
| 2248 | |
| 2249 | * drm_gpuva_map() - helper to insert a &drm_gpuva according to a
|
| 2250 | * &drm_gpuva_op_map
|
| 2251 | * @gpuvm: the &drm_gpuvm
|
| 2252 | * @va: the &drm_gpuva to insert
|
| 2253 | * @op: the &drm_gpuva_op_map to initialize @va with
|
| 2254 | *
|
| 2255 | * Initializes the @va from the @op and inserts it into the given @gpuvm.
|
| 2256 |
|
| 2257 | void
|
| 2258 | drm_gpuva_map(struct drm_gpuvm *gpuvm,
|
| 2259 | struct drm_gpuva *va,
|
| 2260 | const struct drm_gpuva_op_map *op)
|
| 2261 | {
|
| 2262 | drm_gpuva_init_from_op(va, op);
|
| 2263 | drm_gpuva_insert(gpuvm, va);
|
| 2264 | }
|
| 2265 | EXPORT_SYMBOL_GPL(drm_gpuva_map);
|
| 2266 |
|
| 2267 | |
| 2268 | * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a
|
| 2269 | * &drm_gpuva_op_remap
|
| 2270 | * @prev: the &drm_gpuva to remap when keeping the start of a mapping
|
| 2271 | * @next: the &drm_gpuva to remap when keeping the end of a mapping
|
| 2272 | * @op: the &drm_gpuva_op_remap to initialize @prev and @next with
|
| 2273 | *
|
| 2274 | * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or
|
| 2275 | * @next.
|
| 2276 |
|
| 2277 | void
|
| 2278 | drm_gpuva_remap(struct drm_gpuva *prev,
|
| 2279 | struct drm_gpuva *next,
|
| 2280 | const struct drm_gpuva_op_remap *op)
|
| 2281 | {
|
| 2282 | struct drm_gpuva *va = op->unmap->va;
|
| 2283 | struct drm_gpuvm *gpuvm = va->vm;
|
| 2284 |
|
| 2285 | drm_gpuva_remove(va);
|
| 2286 |
|
| 2287 | if (op->prev) {
|
| 2288 | drm_gpuva_init_from_op(prev, op->prev);
|
| 2289 | drm_gpuva_insert(gpuvm, prev);
|
| 2290 | }
|
| 2291 |
|
| 2292 | if (op->next) {
|
| 2293 | drm_gpuva_init_from_op(next, op->next);
|
| 2294 | drm_gpuva_insert(gpuvm, next);
|
| 2295 | }
|
| 2296 | }
|
| 2297 | EXPORT_SYMBOL_GPL(drm_gpuva_remap);
|
| 2298 |
|
| 2299 | |
| 2300 | * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a
|
| 2301 | * &drm_gpuva_op_unmap
|
| 2302 | * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove
|
| 2303 | *
|
| 2304 | * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap.
|
| 2305 |
|
| 2306 | void
|
| 2307 | drm_gpuva_unmap(const struct drm_gpuva_op_unmap *op)
|
| 2308 | {
|
| 2309 | drm_gpuva_remove(op->va);
|
| 2310 | }
|
| 2311 | EXPORT_SYMBOL_GPL(drm_gpuva_unmap);
|
| 2312 |
|
| 2313 | static int
|
| 2314 | op_map_cb(const struct drm_gpuvm_ops *fn, void *priv,
|
| 2315 | const struct drm_gpuvm_map_req *req)
|
| 2316 | {
|
| 2317 | struct drm_gpuva_op op = {};
|
| 2318 |
|
| 2319 | if (!req)
|
| 2320 | return 0;
|
| 2321 |
|
| 2322 | op.op = DRM_GPUVA_OP_MAP;
|
| 2323 | op.map.va.addr = req->map.va.addr;
|
| 2324 | op.map.va.range = req->map.va.range;
|
| 2325 | op.map.gem.obj = req->map.gem.obj;
|
| 2326 | op.map.gem.offset = req->map.gem.offset;
|
| 2327 |
|
| 2328 | return fn->sm_step_map(&op, priv);
|
| 2329 | }
|
| 2330 |
|
| 2331 | static int
|
| 2332 | op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv,
|
| 2333 | struct drm_gpuva_op_map *prev,
|
| 2334 | struct drm_gpuva_op_map *next,
|
| 2335 | struct drm_gpuva_op_unmap *unmap)
|
| 2336 | {
|
| 2337 | struct drm_gpuva_op op = {};
|
| 2338 | struct drm_gpuva_op_remap *r;
|
| 2339 |
|
| 2340 | op.op = DRM_GPUVA_OP_REMAP;
|
| 2341 | r = &op.remap;
|
| 2342 | r->prev = prev;
|
| 2343 | r->next = next;
|
| 2344 | r->unmap = unmap;
|
| 2345 |
|
| 2346 | return fn->sm_step_remap(&op, priv);
|
| 2347 | }
|
| 2348 |
|
| 2349 | static int
|
| 2350 | op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv,
|
| 2351 | struct drm_gpuva *va, bool merge, bool madvise)
|
| 2352 | {
|
| 2353 | struct drm_gpuva_op op = {};
|
| 2354 |
|
| 2355 | if (madvise)
|
| 2356 | return 0;
|
| 2357 |
|
| 2358 | op.op = DRM_GPUVA_OP_UNMAP;
|
| 2359 | op.unmap.va = va;
|
| 2360 | op.unmap.keep = merge;
|
| 2361 |
|
| 2362 | return fn->sm_step_unmap(&op, priv);
|
| 2363 | }
|
| 2364 |
|
| 2365 | static int
|
| 2366 | __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm,
|
| 2367 | const struct drm_gpuvm_ops *ops, void *priv,
|
| 2368 | const struct drm_gpuvm_map_req *req,
|
| 2369 | bool madvise)
|
| 2370 | {
|
| 2371 | struct drm_gem_object *req_obj = req->map.gem.obj;
|
| 2372 | const struct drm_gpuvm_map_req *op_map = madvise ? NULL : req;
|
| 2373 | struct drm_gpuva *va, *next;
|
| 2374 | u64 req_offset = req->map.gem.offset;
|
| 2375 | u64 req_range = req->map.va.range;
|
| 2376 | u64 req_addr = req->map.va.addr;
|
| 2377 | u64 req_end = req_addr + req_range;
|
| 2378 | int ret;
|
| 2379 |
|
| 2380 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
|
| 2381 | return -EINVAL;
|
| 2382 |
|
| 2383 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
|
| 2384 | struct drm_gem_object *obj = va->gem.obj;
|
| 2385 | u64 offset = va->gem.offset;
|
| 2386 | u64 addr = va->va.addr;
|
| 2387 | u64 range = va->va.range;
|
| 2388 | u64 end = addr + range;
|
| 2389 | bool merge = !!va->gem.obj;
|
| 2390 |
|
| 2391 | if (madvise && obj)
|
| 2392 | continue;
|
| 2393 |
|
| 2394 | if (addr == req_addr) {
|
| 2395 | merge &= obj == req_obj &&
|
| 2396 | offset == req_offset;
|
| 2397 |
|
| 2398 | if (end == req_end) {
|
| 2399 | ret = op_unmap_cb(ops, priv, va, merge, madvise);
|
| 2400 | if (ret)
|
| 2401 | return ret;
|
| 2402 | break;
|
| 2403 | }
|
| 2404 |
|
| 2405 | if (end < req_end) {
|
| 2406 | ret = op_unmap_cb(ops, priv, va, merge, madvise);
|
| 2407 | if (ret)
|
| 2408 | return ret;
|
| 2409 | continue;
|
| 2410 | }
|
| 2411 |
|
| 2412 | if (end > req_end) {
|
| 2413 | struct drm_gpuva_op_map n = {
|
| 2414 | .va.addr = req_end,
|
| 2415 | .va.range = range - req_range,
|
| 2416 | .gem.obj = obj,
|
| 2417 | .gem.offset = offset + req_range,
|
| 2418 | };
|
| 2419 | struct drm_gpuva_op_unmap u = {
|
| 2420 | .va = va,
|
| 2421 | .keep = merge,
|
| 2422 | };
|
| 2423 |
|
| 2424 | ret = op_remap_cb(ops, priv, NULL, &n, &u);
|
| 2425 | if (ret)
|
| 2426 | return ret;
|
| 2427 |
|
| 2428 | if (madvise)
|
| 2429 | op_map = req;
|
| 2430 | break;
|
| 2431 | }
|
| 2432 | } else if (addr < req_addr) {
|
| 2433 | u64 ls_range = req_addr - addr;
|
| 2434 | struct drm_gpuva_op_map p = {
|
| 2435 | .va.addr = addr,
|
| 2436 | .va.range = ls_range,
|
| 2437 | .gem.obj = obj,
|
| 2438 | .gem.offset = offset,
|
| 2439 | };
|
| 2440 | struct drm_gpuva_op_unmap u = { .va = va };
|
| 2441 |
|
| 2442 | merge &= obj == req_obj &&
|
| 2443 | offset + ls_range == req_offset;
|
| 2444 | u.keep = merge;
|
| 2445 |
|
| 2446 | if (end == req_end) {
|
| 2447 | ret = op_remap_cb(ops, priv, &p, NULL, &u);
|
| 2448 | if (ret)
|
| 2449 | return ret;
|
| 2450 |
|
| 2451 | if (madvise)
|
| 2452 | op_map = req;
|
| 2453 | break;
|
| 2454 | }
|
| 2455 |
|
| 2456 | if (end < req_end) {
|
| 2457 | ret = op_remap_cb(ops, priv, &p, NULL, &u);
|
| 2458 | if (ret)
|
| 2459 | return ret;
|
| 2460 |
|
| 2461 | if (madvise) {
|
| 2462 | struct drm_gpuvm_map_req map_req = {
|
| 2463 | .map.va.addr = req_addr,
|
| 2464 | .map.va.range = end - req_addr,
|
| 2465 | };
|
| 2466 |
|
| 2467 | ret = op_map_cb(ops, priv, &map_req);
|
| 2468 | if (ret)
|
| 2469 | return ret;
|
| 2470 | }
|
| 2471 |
|
| 2472 | continue;
|
| 2473 | }
|
| 2474 |
|
| 2475 | if (end > req_end) {
|
| 2476 | struct drm_gpuva_op_map n = {
|
| 2477 | .va.addr = req_end,
|
| 2478 | .va.range = end - req_end,
|
| 2479 | .gem.obj = obj,
|
| 2480 | .gem.offset = offset + ls_range +
|
| 2481 | req_range,
|
| 2482 | };
|
| 2483 |
|
| 2484 | ret = op_remap_cb(ops, priv, &p, &n, &u);
|
| 2485 | if (ret)
|
| 2486 | return ret;
|
| 2487 |
|
| 2488 | if (madvise)
|
| 2489 | op_map = req;
|
| 2490 | break;
|
| 2491 | }
|
| 2492 | } else if (addr > req_addr) {
|
| 2493 | merge &= obj == req_obj &&
|
| 2494 | offset == req_offset +
|
| 2495 | (addr - req_addr);
|
| 2496 |
|
| 2497 | if (end == req_end) {
|
| 2498 | ret = op_unmap_cb(ops, priv, va, merge, madvise);
|
| 2499 | if (ret)
|
| 2500 | return ret;
|
| 2501 |
|
| 2502 | break;
|
| 2503 | }
|
| 2504 |
|
| 2505 | if (end < req_end) {
|
| 2506 | ret = op_unmap_cb(ops, priv, va, merge, madvise);
|
| 2507 | if (ret)
|
| 2508 | return ret;
|
| 2509 |
|
| 2510 | continue;
|
| 2511 | }
|
| 2512 |
|
| 2513 | if (end > req_end) {
|
| 2514 | struct drm_gpuva_op_map n = {
|
| 2515 | .va.addr = req_end,
|
| 2516 | .va.range = end - req_end,
|
| 2517 | .gem.obj = obj,
|
| 2518 | .gem.offset = offset + req_end - addr,
|
| 2519 | };
|
| 2520 | struct drm_gpuva_op_unmap u = {
|
| 2521 | .va = va,
|
| 2522 | .keep = merge,
|
| 2523 | };
|
| 2524 |
|
| 2525 | ret = op_remap_cb(ops, priv, NULL, &n, &u);
|
| 2526 | if (ret)
|
| 2527 | return ret;
|
| 2528 |
|
| 2529 | if (madvise) {
|
| 2530 | struct drm_gpuvm_map_req map_req = {
|
| 2531 | .map.va.addr = addr,
|
| 2532 | .map.va.range = req_end - addr,
|
| 2533 | };
|
| 2534 |
|
| 2535 | return op_map_cb(ops, priv, &map_req);
|
| 2536 | }
|
| 2537 | break;
|
| 2538 | }
|
| 2539 | }
|
| 2540 | }
|
| 2541 | return op_map_cb(ops, priv, op_map);
|
| 2542 | }
|
| 2543 |
|
| 2544 | static int
|
| 2545 | __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm,
|
| 2546 | const struct drm_gpuvm_ops *ops, void *priv,
|
| 2547 | u64 req_addr, u64 req_range)
|
| 2548 | {
|
| 2549 | struct drm_gpuva *va, *next;
|
| 2550 | u64 req_end = req_addr + req_range;
|
| 2551 | int ret;
|
| 2552 |
|
| 2553 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range)))
|
| 2554 | return -EINVAL;
|
| 2555 |
|
| 2556 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) {
|
| 2557 | struct drm_gpuva_op_map prev = {}, next = {};
|
| 2558 | bool prev_split = false, next_split = false;
|
| 2559 | struct drm_gem_object *obj = va->gem.obj;
|
| 2560 | u64 offset = va->gem.offset;
|
| 2561 | u64 addr = va->va.addr;
|
| 2562 | u64 range = va->va.range;
|
| 2563 | u64 end = addr + range;
|
| 2564 |
|
| 2565 | if (addr < req_addr) {
|
| 2566 | prev.va.addr = addr;
|
| 2567 | prev.va.range = req_addr - addr;
|
| 2568 | prev.gem.obj = obj;
|
| 2569 | prev.gem.offset = offset;
|
| 2570 |
|
| 2571 | prev_split = true;
|
| 2572 | }
|
| 2573 |
|
| 2574 | if (end > req_end) {
|
| 2575 | next.va.addr = req_end;
|
| 2576 | next.va.range = end - req_end;
|
| 2577 | next.gem.obj = obj;
|
| 2578 | next.gem.offset = offset + (req_end - addr);
|
| 2579 |
|
| 2580 | next_split = true;
|
| 2581 | }
|
| 2582 |
|
| 2583 | if (prev_split || next_split) {
|
| 2584 | struct drm_gpuva_op_unmap unmap = { .va = va };
|
| 2585 |
|
| 2586 | ret = op_remap_cb(ops, priv,
|
| 2587 | prev_split ? &prev : NULL,
|
| 2588 | next_split ? &next : NULL,
|
| 2589 | &unmap);
|
| 2590 | if (ret)
|
| 2591 | return ret;
|
| 2592 | } else {
|
| 2593 | ret = op_unmap_cb(ops, priv, va, false, false);
|
| 2594 | if (ret)
|
| 2595 | return ret;
|
| 2596 | }
|
| 2597 | }
|
| 2598 |
|
| 2599 | return 0;
|
| 2600 | }
|
| 2601 |
|
| 2602 | |
| 2603 | * drm_gpuvm_sm_map() - calls the &drm_gpuva_op split/merge steps
|
| 2604 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2605 | * @priv: pointer to a driver private data structure
|
| 2606 | * @req: ptr to struct drm_gpuvm_map_req
|
| 2607 | *
|
| 2608 | * This function iterates the given range of the GPU VA space. It utilizes the
|
| 2609 | * &drm_gpuvm_ops to call back into the driver providing the split and merge
|
| 2610 | * steps.
|
| 2611 | *
|
| 2612 | * Drivers may use these callbacks to update the GPU VA space right away within
|
| 2613 | * the callback. In case the driver decides to copy and store the operations for
|
| 2614 | * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to
|
| 2615 | * be called before the &drm_gpuvm's view of the GPU VA space was
|
| 2616 | * updated with the previous set of operations. To update the
|
| 2617 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
|
| 2618 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
|
| 2619 | * used.
|
| 2620 | *
|
| 2621 | * A sequence of callbacks can contain map, unmap and remap operations, but
|
| 2622 | * the sequence of callbacks might also be empty if no operation is required,
|
| 2623 | * e.g. if the requested mapping already exists in the exact same way.
|
| 2624 | *
|
| 2625 | * There can be an arbitrary amount of unmap operations, a maximum of two remap
|
| 2626 | * operations and a single map operation. The latter one represents the original
|
| 2627 | * map operation requested by the caller.
|
| 2628 | *
|
| 2629 | * Returns: 0 on success or a negative error code
|
| 2630 |
|
| 2631 | int
|
| 2632 | drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,
|
| 2633 | const struct drm_gpuvm_map_req *req)
|
| 2634 | {
|
| 2635 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 2636 |
|
| 2637 | if (unlikely(!(ops && ops->sm_step_map &&
|
| 2638 | ops->sm_step_remap &&
|
| 2639 | ops->sm_step_unmap)))
|
| 2640 | return -EINVAL;
|
| 2641 |
|
| 2642 | return __drm_gpuvm_sm_map(gpuvm, ops, priv, req, false);
|
| 2643 | }
|
| 2644 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map);
|
| 2645 |
|
| 2646 | |
| 2647 | * drm_gpuvm_sm_unmap() - calls the &drm_gpuva_ops to split on unmap
|
| 2648 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2649 | * @priv: pointer to a driver private data structure
|
| 2650 | * @req_addr: the start address of the range to unmap
|
| 2651 | * @req_range: the range of the mappings to unmap
|
| 2652 | *
|
| 2653 | * This function iterates the given range of the GPU VA space. It utilizes the
|
| 2654 | * &drm_gpuvm_ops to call back into the driver providing the operations to
|
| 2655 | * unmap and, if required, split existing mappings.
|
| 2656 | *
|
| 2657 | * Drivers may use these callbacks to update the GPU VA space right away within
|
| 2658 | * the callback. In case the driver decides to copy and store the operations for
|
| 2659 | * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be
|
| 2660 | * called before the &drm_gpuvm's view of the GPU VA space was updated
|
| 2661 | * with the previous set of operations. To update the &drm_gpuvm's view
|
| 2662 | * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or
|
| 2663 | * drm_gpuva_destroy_unlocked() should be used.
|
| 2664 | *
|
| 2665 | * A sequence of callbacks can contain unmap and remap operations, depending on
|
| 2666 | * whether there are actual overlapping mappings to split.
|
| 2667 | *
|
| 2668 | * There can be an arbitrary amount of unmap operations and a maximum of two
|
| 2669 | * remap operations.
|
| 2670 | *
|
| 2671 | * Returns: 0 on success or a negative error code
|
| 2672 |
|
| 2673 | int
|
| 2674 | drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,
|
| 2675 | u64 req_addr, u64 req_range)
|
| 2676 | {
|
| 2677 | const struct drm_gpuvm_ops *ops = gpuvm->ops;
|
| 2678 |
|
| 2679 | if (unlikely(!(ops && ops->sm_step_remap &&
|
| 2680 | ops->sm_step_unmap)))
|
| 2681 | return -EINVAL;
|
| 2682 |
|
| 2683 | return __drm_gpuvm_sm_unmap(gpuvm, ops, priv,
|
| 2684 | req_addr, req_range);
|
| 2685 | }
|
| 2686 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap);
|
| 2687 |
|
| 2688 | static int
|
| 2689 | drm_gpuva_sm_step_lock(struct drm_gpuva_op *op, void *priv)
|
| 2690 | {
|
| 2691 | struct drm_exec *exec = priv;
|
| 2692 |
|
| 2693 | switch (op->op) {
|
| 2694 | case DRM_GPUVA_OP_REMAP:
|
| 2695 | if (op->remap.unmap->va->gem.obj)
|
| 2696 | return drm_exec_lock_obj(exec, op->remap.unmap->va->gem.obj);
|
| 2697 | return 0;
|
| 2698 | case DRM_GPUVA_OP_UNMAP:
|
| 2699 | if (op->unmap.va->gem.obj)
|
| 2700 | return drm_exec_lock_obj(exec, op->unmap.va->gem.obj);
|
| 2701 | return 0;
|
| 2702 | default:
|
| 2703 | return 0;
|
| 2704 | }
|
| 2705 | }
|
| 2706 |
|
| 2707 | static const struct drm_gpuvm_ops lock_ops = {
|
| 2708 | .sm_step_map = drm_gpuva_sm_step_lock,
|
| 2709 | .sm_step_remap = drm_gpuva_sm_step_lock,
|
| 2710 | .sm_step_unmap = drm_gpuva_sm_step_lock,
|
| 2711 | };
|
| 2712 |
|
| 2713 | |
| 2714 | * drm_gpuvm_sm_map_exec_lock() - locks the objects touched by a drm_gpuvm_sm_map()
|
| 2715 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2716 | * @exec: the &drm_exec locking context
|
| 2717 | * @num_fences: for newly mapped objects, the # of fences to reserve
|
| 2718 | * @req: ptr to drm_gpuvm_map_req struct
|
| 2719 | *
|
| 2720 | * This function locks (drm_exec_lock_obj()) objects that will be unmapped/
|
| 2721 | * remapped, and locks+prepares (drm_exec_prepare_object()) objects that
|
| 2722 | * will be newly mapped.
|
| 2723 | *
|
| 2724 | * The expected usage is::
|
| 2725 | *
|
| 2726 | * vm_bind {
|
| 2727 | * struct drm_exec exec;
|
| 2728 | *
|
| 2729 | * // IGNORE_DUPLICATES is required, INTERRUPTIBLE_WAIT is recommended:
|
| 2730 | * drm_exec_init(&exec, IGNORE_DUPLICATES | INTERRUPTIBLE_WAIT, 0);
|
| 2731 | *
|
| 2732 | * drm_exec_until_all_locked (&exec) {
|
| 2733 | * for_each_vm_bind_operation {
|
| 2734 | * switch (op->op) {
|
| 2735 | * case DRIVER_OP_UNMAP:
|
| 2736 | * ret = drm_gpuvm_sm_unmap_exec_lock(gpuvm, &exec, op->addr, op->range);
|
| 2737 | * break;
|
| 2738 | * case DRIVER_OP_MAP:
|
| 2739 | * ret = drm_gpuvm_sm_map_exec_lock(gpuvm, &exec, num_fences, &req);
|
| 2740 | * break;
|
| 2741 | * }
|
| 2742 | *
|
| 2743 | * drm_exec_retry_on_contention(&exec);
|
| 2744 | * if (ret)
|
| 2745 | * return ret;
|
| 2746 | * }
|
| 2747 | * }
|
| 2748 | * }
|
| 2749 | *
|
| 2750 | * This enables all locking to be performed before the driver begins modifying
|
| 2751 | * the VM. This is safe to do in the case of overlapping DRIVER_VM_BIND_OPs,
|
| 2752 | * where an earlier op can alter the sequence of steps generated for a later
|
| 2753 | * op, because the later altered step will involve the same GEM object(s)
|
| 2754 | * already seen in the earlier locking step. For example:
|
| 2755 | *
|
| 2756 | * 1) An earlier driver DRIVER_OP_UNMAP op removes the need for a
|
| 2757 | * DRM_GPUVA_OP_REMAP/UNMAP step. This is safe because we've already
|
| 2758 | * locked the GEM object in the earlier DRIVER_OP_UNMAP op.
|
| 2759 | *
|
| 2760 | * 2) An earlier DRIVER_OP_MAP op overlaps with a later DRIVER_OP_MAP/UNMAP
|
| 2761 | * op, introducing a DRM_GPUVA_OP_REMAP/UNMAP that wouldn't have been
|
| 2762 | * required without the earlier DRIVER_OP_MAP. This is safe because we've
|
| 2763 | * already locked the GEM object in the earlier DRIVER_OP_MAP step.
|
| 2764 | *
|
| 2765 | * Returns: 0 on success or a negative error code
|
| 2766 |
|
| 2767 | int
|
| 2768 | drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm,
|
| 2769 | struct drm_exec *exec, unsigned int num_fences,
|
| 2770 | struct drm_gpuvm_map_req *req)
|
| 2771 | {
|
| 2772 | struct drm_gem_object *req_obj = req->map.gem.obj;
|
| 2773 |
|
| 2774 | if (req_obj) {
|
| 2775 | int ret = drm_exec_prepare_obj(exec, req_obj, num_fences);
|
| 2776 | if (ret)
|
| 2777 | return ret;
|
| 2778 | }
|
| 2779 |
|
| 2780 | return __drm_gpuvm_sm_map(gpuvm, &lock_ops, exec, req, false);
|
| 2781 |
|
| 2782 | }
|
| 2783 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_exec_lock);
|
| 2784 |
|
| 2785 | |
| 2786 | * drm_gpuvm_sm_unmap_exec_lock() - locks the objects touched by drm_gpuvm_sm_unmap()
|
| 2787 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2788 | * @exec: the &drm_exec locking context
|
| 2789 | * @req_addr: the start address of the range to unmap
|
| 2790 | * @req_range: the range of the mappings to unmap
|
| 2791 | *
|
| 2792 | * This function locks (drm_exec_lock_obj()) objects that will be unmapped/
|
| 2793 | * remapped by drm_gpuvm_sm_unmap().
|
| 2794 | *
|
| 2795 | * See drm_gpuvm_sm_map_exec_lock() for expected usage.
|
| 2796 | *
|
| 2797 | * Returns: 0 on success or a negative error code
|
| 2798 |
|
| 2799 | int
|
| 2800 | drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
|
| 2801 | u64 req_addr, u64 req_range)
|
| 2802 | {
|
| 2803 | return __drm_gpuvm_sm_unmap(gpuvm, &lock_ops, exec,
|
| 2804 | req_addr, req_range);
|
| 2805 | }
|
| 2806 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_exec_lock);
|
| 2807 |
|
| 2808 | static struct drm_gpuva_op *
|
| 2809 | gpuva_op_alloc(struct drm_gpuvm *gpuvm)
|
| 2810 | {
|
| 2811 | const struct drm_gpuvm_ops *fn = gpuvm->ops;
|
| 2812 | struct drm_gpuva_op *op;
|
| 2813 |
|
| 2814 | if (fn && fn->op_alloc)
|
| 2815 | op = fn->op_alloc();
|
| 2816 | else
|
| 2817 | op = kzalloc_obj(*op);
|
| 2818 |
|
| 2819 | if (unlikely(!op))
|
| 2820 | return NULL;
|
| 2821 |
|
| 2822 | return op;
|
| 2823 | }
|
| 2824 |
|
| 2825 | static void
|
| 2826 | gpuva_op_free(struct drm_gpuvm *gpuvm,
|
| 2827 | struct drm_gpuva_op *op)
|
| 2828 | {
|
| 2829 | const struct drm_gpuvm_ops *fn = gpuvm->ops;
|
| 2830 |
|
| 2831 | if (fn && fn->op_free)
|
| 2832 | fn->op_free(op);
|
| 2833 | else
|
| 2834 | kfree(op);
|
| 2835 | }
|
| 2836 |
|
| 2837 | static int
|
| 2838 | drm_gpuva_sm_step(struct drm_gpuva_op *__op,
|
| 2839 | void *priv)
|
| 2840 | {
|
| 2841 | struct {
|
| 2842 | struct drm_gpuvm *vm;
|
| 2843 | struct drm_gpuva_ops *ops;
|
| 2844 | } *args = priv;
|
| 2845 | struct drm_gpuvm *gpuvm = args->vm;
|
| 2846 | struct drm_gpuva_ops *ops = args->ops;
|
| 2847 | struct drm_gpuva_op *op;
|
| 2848 |
|
| 2849 | op = gpuva_op_alloc(gpuvm);
|
| 2850 | if (unlikely(!op))
|
| 2851 | goto err;
|
| 2852 |
|
| 2853 | memcpy(op, __op, sizeof(*op));
|
| 2854 |
|
| 2855 | if (op->op == DRM_GPUVA_OP_REMAP) {
|
| 2856 | struct drm_gpuva_op_remap *__r = &__op->remap;
|
| 2857 | struct drm_gpuva_op_remap *r = &op->remap;
|
| 2858 |
|
| 2859 | r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap),
|
| 2860 | GFP_KERNEL);
|
| 2861 | if (unlikely(!r->unmap))
|
| 2862 | goto err_free_op;
|
| 2863 |
|
| 2864 | if (__r->prev) {
|
| 2865 | r->prev = kmemdup(__r->prev, sizeof(*r->prev),
|
| 2866 | GFP_KERNEL);
|
| 2867 | if (unlikely(!r->prev))
|
| 2868 | goto err_free_unmap;
|
| 2869 | }
|
| 2870 |
|
| 2871 | if (__r->next) {
|
| 2872 | r->next = kmemdup(__r->next, sizeof(*r->next),
|
| 2873 | GFP_KERNEL);
|
| 2874 | if (unlikely(!r->next))
|
| 2875 | goto err_free_prev;
|
| 2876 | }
|
| 2877 | }
|
| 2878 |
|
| 2879 | list_add_tail(&op->entry, &ops->list);
|
| 2880 |
|
| 2881 | return 0;
|
| 2882 |
|
| 2883 | err_free_unmap:
|
| 2884 | kfree(op->remap.unmap);
|
| 2885 | err_free_prev:
|
| 2886 | kfree(op->remap.prev);
|
| 2887 | err_free_op:
|
| 2888 | gpuva_op_free(gpuvm, op);
|
| 2889 | err:
|
| 2890 | return -ENOMEM;
|
| 2891 | }
|
| 2892 |
|
| 2893 | static const struct drm_gpuvm_ops gpuvm_list_ops = {
|
| 2894 | .sm_step_map = drm_gpuva_sm_step,
|
| 2895 | .sm_step_remap = drm_gpuva_sm_step,
|
| 2896 | .sm_step_unmap = drm_gpuva_sm_step,
|
| 2897 | };
|
| 2898 |
|
| 2899 | static struct drm_gpuva_ops *
|
| 2900 | __drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
|
| 2901 | const struct drm_gpuvm_map_req *req,
|
| 2902 | bool madvise)
|
| 2903 | {
|
| 2904 | struct drm_gpuva_ops *ops;
|
| 2905 | struct {
|
| 2906 | struct drm_gpuvm *vm;
|
| 2907 | struct drm_gpuva_ops *ops;
|
| 2908 | } args;
|
| 2909 | int ret;
|
| 2910 |
|
| 2911 | ops = kzalloc_obj(*ops);
|
| 2912 | if (unlikely(!ops))
|
| 2913 | return ERR_PTR(-ENOMEM);
|
| 2914 |
|
| 2915 | INIT_LIST_HEAD(&ops->list);
|
| 2916 |
|
| 2917 | args.vm = gpuvm;
|
| 2918 | args.ops = ops;
|
| 2919 |
|
| 2920 | ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, req, madvise);
|
| 2921 | if (ret)
|
| 2922 | goto err_free_ops;
|
| 2923 |
|
| 2924 | return ops;
|
| 2925 |
|
| 2926 | err_free_ops:
|
| 2927 | drm_gpuva_ops_free(gpuvm, ops);
|
| 2928 | return ERR_PTR(ret);
|
| 2929 | }
|
| 2930 |
|
| 2931 | |
| 2932 | * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge
|
| 2933 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2934 | * @req: map request arguments
|
| 2935 | *
|
| 2936 | * This function creates a list of operations to perform splitting and merging
|
| 2937 | * of existing mapping(s) with the newly requested one.
|
| 2938 | *
|
| 2939 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed
|
| 2940 | * in the given order. It can contain map, unmap and remap operations, but it
|
| 2941 | * also can be empty if no operation is required, e.g. if the requested mapping
|
| 2942 | * already exists in the exact same way.
|
| 2943 | *
|
| 2944 | * There can be an arbitrary amount of unmap operations, a maximum of two remap
|
| 2945 | * operations and a single map operation. The latter one represents the original
|
| 2946 | * map operation requested by the caller.
|
| 2947 | *
|
| 2948 | * Note that before calling this function again with another mapping request it
|
| 2949 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The
|
| 2950 | * previously obtained operations must be either processed or abandoned. To
|
| 2951 | * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
|
| 2952 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
|
| 2953 | * used.
|
| 2954 | *
|
| 2955 | * After the caller finished processing the returned &drm_gpuva_ops, they must
|
| 2956 | * be freed with &drm_gpuva_ops_free.
|
| 2957 | *
|
| 2958 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
|
| 2959 |
|
| 2960 | struct drm_gpuva_ops *
|
| 2961 | drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
|
| 2962 | const struct drm_gpuvm_map_req *req)
|
| 2963 | {
|
| 2964 | return __drm_gpuvm_sm_map_ops_create(gpuvm, req, false);
|
| 2965 | }
|
| 2966 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create);
|
| 2967 |
|
| 2968 | |
| 2969 | * drm_gpuvm_madvise_ops_create() - creates the &drm_gpuva_ops to split
|
| 2970 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 2971 | * @req: map request arguments
|
| 2972 | *
|
| 2973 | * This function creates a list of operations to perform splitting
|
| 2974 | * of existent mapping(s) at start or end, based on the request map.
|
| 2975 | *
|
| 2976 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed
|
| 2977 | * in the given order. It can contain map and remap operations, but it
|
| 2978 | * also can be empty if no operation is required, e.g. if the requested mapping
|
| 2979 | * already exists is the exact same way.
|
| 2980 | *
|
| 2981 | * There will be no unmap operations, a maximum of two remap operations and two
|
| 2982 | * map operations. The two map operations correspond to: one from start to the
|
| 2983 | * end of drm_gpuvaX, and another from the start of drm_gpuvaY to end.
|
| 2984 | *
|
| 2985 | * Note that before calling this function again with another mapping request it
|
| 2986 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The
|
| 2987 | * previously obtained operations must be either processed or abandoned. To
|
| 2988 | * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
|
| 2989 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
|
| 2990 | * used.
|
| 2991 | *
|
| 2992 | * After the caller finished processing the returned &drm_gpuva_ops, they must
|
| 2993 | * be freed with &drm_gpuva_ops_free.
|
| 2994 | *
|
| 2995 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
|
| 2996 |
|
| 2997 | struct drm_gpuva_ops *
|
| 2998 | drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm,
|
| 2999 | const struct drm_gpuvm_map_req *req)
|
| 3000 | {
|
| 3001 | return __drm_gpuvm_sm_map_ops_create(gpuvm, req, true);
|
| 3002 | }
|
| 3003 | EXPORT_SYMBOL_GPL(drm_gpuvm_madvise_ops_create);
|
| 3004 |
|
| 3005 | |
| 3006 | * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on
|
| 3007 | * unmap
|
| 3008 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 3009 | * @req_addr: the start address of the range to unmap
|
| 3010 | * @req_range: the range of the mappings to unmap
|
| 3011 | *
|
| 3012 | * This function creates a list of operations to perform unmapping and, if
|
| 3013 | * required, splitting of the mappings overlapping the unmap range.
|
| 3014 | *
|
| 3015 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed
|
| 3016 | * in the given order. It can contain unmap and remap operations, depending on
|
| 3017 | * whether there are actual overlapping mappings to split.
|
| 3018 | *
|
| 3019 | * There can be an arbitrary amount of unmap operations and a maximum of two
|
| 3020 | * remap operations.
|
| 3021 | *
|
| 3022 | * Note that before calling this function again with another range to unmap it
|
| 3023 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The
|
| 3024 | * previously obtained operations must be processed or abandoned. To update the
|
| 3025 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(),
|
| 3026 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be
|
| 3027 | * used.
|
| 3028 | *
|
| 3029 | * After the caller finished processing the returned &drm_gpuva_ops, they must
|
| 3030 | * be freed with &drm_gpuva_ops_free.
|
| 3031 | *
|
| 3032 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
|
| 3033 |
|
| 3034 | struct drm_gpuva_ops *
|
| 3035 | drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,
|
| 3036 | u64 req_addr, u64 req_range)
|
| 3037 | {
|
| 3038 | struct drm_gpuva_ops *ops;
|
| 3039 | struct {
|
| 3040 | struct drm_gpuvm *vm;
|
| 3041 | struct drm_gpuva_ops *ops;
|
| 3042 | } args;
|
| 3043 | int ret;
|
| 3044 |
|
| 3045 | ops = kzalloc_obj(*ops);
|
| 3046 | if (unlikely(!ops))
|
| 3047 | return ERR_PTR(-ENOMEM);
|
| 3048 |
|
| 3049 | INIT_LIST_HEAD(&ops->list);
|
| 3050 |
|
| 3051 | args.vm = gpuvm;
|
| 3052 | args.ops = ops;
|
| 3053 |
|
| 3054 | ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args,
|
| 3055 | req_addr, req_range);
|
| 3056 | if (ret)
|
| 3057 | goto err_free_ops;
|
| 3058 |
|
| 3059 | return ops;
|
| 3060 |
|
| 3061 | err_free_ops:
|
| 3062 | drm_gpuva_ops_free(gpuvm, ops);
|
| 3063 | return ERR_PTR(ret);
|
| 3064 | }
|
| 3065 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create);
|
| 3066 |
|
| 3067 | |
| 3068 | * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch
|
| 3069 | * @gpuvm: the &drm_gpuvm representing the GPU VA space
|
| 3070 | * @addr: the start address of the range to prefetch
|
| 3071 | * @range: the range of the mappings to prefetch
|
| 3072 | *
|
| 3073 | * This function creates a list of operations to perform prefetching.
|
| 3074 | *
|
| 3075 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed
|
| 3076 | * in the given order. It can contain prefetch operations.
|
| 3077 | *
|
| 3078 | * There can be an arbitrary amount of prefetch operations.
|
| 3079 | *
|
| 3080 | * After the caller finished processing the returned &drm_gpuva_ops, they must
|
| 3081 | * be freed with &drm_gpuva_ops_free.
|
| 3082 | *
|
| 3083 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
|
| 3084 |
|
| 3085 | struct drm_gpuva_ops *
|
| 3086 | drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
|
| 3087 | u64 addr, u64 range)
|
| 3088 | {
|
| 3089 | struct drm_gpuva_ops *ops;
|
| 3090 | struct drm_gpuva_op *op;
|
| 3091 | struct drm_gpuva *va;
|
| 3092 | u64 end = addr + range;
|
| 3093 | int ret;
|
| 3094 |
|
| 3095 | ops = kzalloc_obj(*ops);
|
| 3096 | if (!ops)
|
| 3097 | return ERR_PTR(-ENOMEM);
|
| 3098 |
|
| 3099 | INIT_LIST_HEAD(&ops->list);
|
| 3100 |
|
| 3101 | drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
|
| 3102 | op = gpuva_op_alloc(gpuvm);
|
| 3103 | if (!op) {
|
| 3104 | ret = -ENOMEM;
|
| 3105 | goto err_free_ops;
|
| 3106 | }
|
| 3107 |
|
| 3108 | op->op = DRM_GPUVA_OP_PREFETCH;
|
| 3109 | op->prefetch.va = va;
|
| 3110 | list_add_tail(&op->entry, &ops->list);
|
| 3111 | }
|
| 3112 |
|
| 3113 | return ops;
|
| 3114 |
|
| 3115 | err_free_ops:
|
| 3116 | drm_gpuva_ops_free(gpuvm, ops);
|
| 3117 | return ERR_PTR(ret);
|
| 3118 | }
|
| 3119 | EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
|
| 3120 |
|
| 3121 | |
| 3122 | * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
|
| 3123 | * @vm_bo: the &drm_gpuvm_bo abstraction
|
| 3124 | *
|
| 3125 | * This function creates a list of operations to perform unmapping for every
|
| 3126 | * GPUVA attached to a GEM.
|
| 3127 | *
|
| 3128 | * The list can be iterated with &drm_gpuva_for_each_op and consists out of an
|
| 3129 | * arbitrary amount of unmap operations.
|
| 3130 | *
|
| 3131 | * After the caller finished processing the returned &drm_gpuva_ops, they must
|
| 3132 | * be freed with &drm_gpuva_ops_free.
|
| 3133 | *
|
| 3134 | * This function expects the caller to protect the GEM's GPUVA list against
|
| 3135 | * concurrent access using either the GEM's dma-resv or gpuva.lock mutex.
|
| 3136 | *
|
| 3137 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
|
| 3138 |
|
| 3139 | struct drm_gpuva_ops *
|
| 3140 | drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)
|
| 3141 | {
|
| 3142 | struct drm_gpuva_ops *ops;
|
| 3143 | struct drm_gpuva_op *op;
|
| 3144 | struct drm_gpuva *va;
|
| 3145 | int ret;
|
| 3146 |
|
| 3147 | drm_gem_gpuva_assert_lock_held(vm_bo->vm, vm_bo->obj);
|
| 3148 |
|
| 3149 | ops = kzalloc_obj(*ops);
|
| 3150 | if (!ops)
|
| 3151 | return ERR_PTR(-ENOMEM);
|
| 3152 |
|
| 3153 | INIT_LIST_HEAD(&ops->list);
|
| 3154 |
|
| 3155 | drm_gpuvm_bo_for_each_va(va, vm_bo) {
|
| 3156 | op = gpuva_op_alloc(vm_bo->vm);
|
| 3157 | if (!op) {
|
| 3158 | ret = -ENOMEM;
|
| 3159 | goto err_free_ops;
|
| 3160 | }
|
| 3161 |
|
| 3162 | op->op = DRM_GPUVA_OP_UNMAP;
|
| 3163 | op->unmap.va = va;
|
| 3164 | list_add_tail(&op->entry, &ops->list);
|
| 3165 | }
|
| 3166 |
|
| 3167 | return ops;
|
| 3168 |
|
| 3169 | err_free_ops:
|
| 3170 | drm_gpuva_ops_free(vm_bo->vm, ops);
|
| 3171 | return ERR_PTR(ret);
|
| 3172 | }
|
| 3173 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
|
| 3174 |
|
| 3175 | |
| 3176 | * drm_gpuva_ops_free() - free the given &drm_gpuva_ops
|
| 3177 | * @gpuvm: the &drm_gpuvm the ops were created for
|
| 3178 | * @ops: the &drm_gpuva_ops to free
|
| 3179 | *
|
| 3180 | * Frees the given &drm_gpuva_ops structure including all the ops associated
|
| 3181 | * with it.
|
| 3182 |
|
| 3183 | void
|
| 3184 | drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
|
| 3185 | struct drm_gpuva_ops *ops)
|
| 3186 | {
|
| 3187 | struct drm_gpuva_op *op, *next;
|
| 3188 |
|
| 3189 | drm_gpuva_for_each_op_safe(op, next, ops) {
|
| 3190 | list_del(&op->entry);
|
| 3191 |
|
| 3192 | if (op->op == DRM_GPUVA_OP_REMAP) {
|
| 3193 | kfree(op->remap.prev);
|
| 3194 | kfree(op->remap.next);
|
| 3195 | kfree(op->remap.unmap);
|
| 3196 | }
|
| 3197 |
|
| 3198 | gpuva_op_free(gpuvm, op);
|
| 3199 | }
|
| 3200 |
|
| 3201 | kfree(ops);
|
| 3202 | }
|
| 3203 | EXPORT_SYMBOL_GPL(drm_gpuva_ops_free);
|
| 3204 |
|
| 3205 | MODULE_DESCRIPTION("DRM GPUVM");
|
| 3206 | MODULE_LICENSE("GPL");
|