projects/linux·blob master · refs

1041 lines · 30.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * arch-independent dma-mapping routines
4 *
5 * Copyright (c) 2006 SUSE Linux Products GmbH
6 * Copyright (c) 2006 Tejun Heo <[email protected]>
7 */
8#include <linux/memblock.h> /* for max_pfn */
9#include <linux/acpi.h>
10#include <linux/dma-map-ops.h>
11#include <linux/export.h>
12#include <linux/gfp.h>
13#include <linux/iommu-dma.h>
14#include <linux/kmsan.h>
15#include <linux/of_device.h>
16#include <linux/slab.h>
17#include <linux/vmalloc.h>
18#include "debug.h"
19#include "direct.h"
20
21#define CREATE_TRACE_POINTS
22#include <trace/events/dma.h>
23
24#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
25 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
26 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
27bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT);
28#endif
29
30/*
31 * Managed DMA API
32 */
33struct dma_devres {
34 size_t size;
35 void *vaddr;
36 dma_addr_t dma_handle;
37 unsigned long attrs;
38};
39
40static void dmam_release(struct device *dev, void *res)
41{
42 struct dma_devres *this = res;
43
44 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,
45 this->attrs);
46}
47
48static int dmam_match(struct device *dev, void *res, void *match_data)
49{
50 struct dma_devres *this = res, *match = match_data;
51
52 if (this->vaddr == match->vaddr) {
53 WARN_ON(this->size != match->size ||
54 this->dma_handle != match->dma_handle);
55 return 1;
56 }
57 return 0;
58}
59
60/**
61 * dmam_free_coherent - Managed dma_free_coherent()
62 * @dev: Device to free coherent memory for
63 * @size: Size of allocation
64 * @vaddr: Virtual address of the memory to free
65 * @dma_handle: DMA handle of the memory to free
66 *
67 * Managed dma_free_coherent().
68 */
69void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
70 dma_addr_t dma_handle)
71{
72 struct dma_devres match_data = { size, vaddr, dma_handle };
73
74 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data));
75 dma_free_coherent(dev, size, vaddr, dma_handle);
76}
77EXPORT_SYMBOL(dmam_free_coherent);
78
79/**
80 * dmam_alloc_attrs - Managed dma_alloc_attrs()
81 * @dev: Device to allocate non_coherent memory for
82 * @size: Size of allocation
83 * @dma_handle: Out argument for allocated DMA handle
84 * @gfp: Allocation flags
85 * @attrs: Flags in the DMA_ATTR_* namespace.
86 *
87 * Managed dma_alloc_attrs(). Memory allocated using this function will be
88 * automatically released on driver detach.
89 *
90 * RETURNS:
91 * Pointer to allocated memory on success, NULL on failure.
92 */
93void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
94 gfp_t gfp, unsigned long attrs)
95{
96 struct dma_devres *dr;
97 void *vaddr;
98
99 dr = devres_alloc(dmam_release, sizeof(*dr), gfp);
100 if (!dr)
101 return NULL;
102
103 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);
104 if (!vaddr) {
105 devres_free(dr);
106 return NULL;
107 }
108
109 dr->vaddr = vaddr;
110 dr->dma_handle = *dma_handle;
111 dr->size = size;
112 dr->attrs = attrs;
113
114 devres_add(dev, dr);
115
116 return vaddr;
117}
118EXPORT_SYMBOL(dmam_alloc_attrs);
119
120static bool dma_go_direct(struct device *dev, dma_addr_t mask,
121 const struct dma_map_ops *ops)
122{
123 if (use_dma_iommu(dev))
124 return false;
125
126 if (likely(!ops))
127 return true;
128
129 if (IS_ENABLED(CONFIG_DMA_OPS_BYPASS) && dev_dma_ops_bypass(dev))
130 return min_not_zero(mask, dev->bus_dma_limit) >=
131 dma_direct_get_required_mask(dev);
132 return false;
133}
134
135
136/*
137 * Check if the devices uses a direct mapping for streaming DMA operations.
138 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
139 * enough.
140 */
141static inline bool dma_alloc_direct(struct device *dev,
142 const struct dma_map_ops *ops)
143{
144 return dma_go_direct(dev, dev->coherent_dma_mask, ops);
145}
146
147static inline bool dma_map_direct(struct device *dev,
148 const struct dma_map_ops *ops)
149{
150 return dma_go_direct(dev, *dev->dma_mask, ops);
151}
152
153dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
154 enum dma_data_direction dir, unsigned long attrs)
155{
156 const struct dma_map_ops *ops = get_dma_ops(dev);
157 bool is_mmio = attrs & DMA_ATTR_MMIO;
158 bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED;
159 dma_addr_t addr = DMA_MAPPING_ERROR;
160
161 BUG_ON(!valid_dma_direction(dir));
162
163 if (WARN_ON_ONCE(!dev->dma_mask))
164 return DMA_MAPPING_ERROR;
165
166 if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
167 return DMA_MAPPING_ERROR;
168
169 if (dma_map_direct(dev, ops) ||
170 (!is_mmio && !is_cc_shared &&
171 arch_dma_map_phys_direct(dev, phys + size)))
172 addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true);
173 else if (is_cc_shared)
174 return DMA_MAPPING_ERROR;
175 else if (use_dma_iommu(dev))
176 addr = iommu_dma_map_phys(dev, phys, size, dir, attrs);
177 else if (ops->map_phys)
178 addr = ops->map_phys(dev, phys, size, dir, attrs);
179
180 if (!is_mmio)
181 kmsan_handle_dma(phys, size, dir);
182 trace_dma_map_phys(dev, phys, addr, size, dir, attrs);
183 debug_dma_map_phys(dev, phys, size, dir, addr, attrs);
184
185 return addr;
186}
187EXPORT_SYMBOL_GPL(dma_map_phys);
188
189dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
190 size_t offset, size_t size, enum dma_data_direction dir,
191 unsigned long attrs)
192{
193 phys_addr_t phys = page_to_phys(page) + offset;
194
195 if (unlikely(attrs & DMA_ATTR_MMIO))
196 return DMA_MAPPING_ERROR;
197
198 if (IS_ENABLED(CONFIG_DMA_API_DEBUG) &&
199 WARN_ON_ONCE(is_zone_device_page(page)))
200 return DMA_MAPPING_ERROR;
201
202 return dma_map_phys(dev, phys, size, dir, attrs);
203}
204EXPORT_SYMBOL(dma_map_page_attrs);
205
206void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,
207 enum dma_data_direction dir, unsigned long attrs)
208{
209 const struct dma_map_ops *ops = get_dma_ops(dev);
210 bool is_mmio = attrs & DMA_ATTR_MMIO;
211 bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED;
212
213 BUG_ON(!valid_dma_direction(dir));
214
215 if (dma_map_direct(dev, ops) ||
216 (!is_mmio && !is_cc_shared &&
217 arch_dma_unmap_phys_direct(dev, addr + size)))
218 dma_direct_unmap_phys(dev, addr, size, dir, attrs, true);
219 else if (is_cc_shared)
220 return;
221 else if (use_dma_iommu(dev))
222 iommu_dma_unmap_phys(dev, addr, size, dir, attrs);
223 else if (ops->unmap_phys)
224 ops->unmap_phys(dev, addr, size, dir, attrs);
225 trace_dma_unmap_phys(dev, addr, size, dir, attrs);
226 debug_dma_unmap_phys(dev, addr, size, dir, attrs);
227}
228EXPORT_SYMBOL_GPL(dma_unmap_phys);
229
230void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
231 enum dma_data_direction dir, unsigned long attrs)
232{
233 if (unlikely(attrs & DMA_ATTR_MMIO))
234 return;
235
236 dma_unmap_phys(dev, addr, size, dir, attrs);
237}
238EXPORT_SYMBOL(dma_unmap_page_attrs);
239
240static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
241 int nents, enum dma_data_direction dir, unsigned long attrs)
242{
243 const struct dma_map_ops *ops = get_dma_ops(dev);
244 int ents;
245
246 BUG_ON(!valid_dma_direction(dir));
247
248 if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT))
249 return -EOPNOTSUPP;
250
251 if (WARN_ON_ONCE(!dev->dma_mask))
252 return 0;
253
254 if (dma_map_direct(dev, ops) ||
255 arch_dma_map_sg_direct(dev, sg, nents))
256 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
257 else if (use_dma_iommu(dev))
258 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs);
259 else
260 ents = ops->map_sg(dev, sg, nents, dir, attrs);
261
262 if (ents > 0) {
263 kmsan_handle_dma_sg(sg, nents, dir);
264 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs);
265 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs);
266 } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&
267 ents != -EIO && ents != -EREMOTEIO)) {
268 trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs);
269 return -EIO;
270 }
271
272 return ents;
273}
274
275/**
276 * dma_map_sg_attrs - Map the given buffer for DMA
277 * @dev: The device for which to perform the DMA operation
278 * @sg: The sg_table object describing the buffer
279 * @nents: Number of entries to map
280 * @dir: DMA direction
281 * @attrs: Optional DMA attributes for the map operation
282 *
283 * Maps a buffer described by a scatterlist passed in the sg argument with
284 * nents segments for the @dir DMA operation by the @dev device.
285 *
286 * Returns the number of mapped entries (which can be less than nents)
287 * on success. Zero is returned for any error.
288 *
289 * dma_unmap_sg_attrs() should be used to unmap the buffer with the
290 * original sg and original nents (not the value returned by this funciton).
291 */
292unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
293 int nents, enum dma_data_direction dir, unsigned long attrs)
294{
295 int ret;
296
297 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs);
298 if (ret < 0)
299 return 0;
300 return ret;
301}
302EXPORT_SYMBOL(dma_map_sg_attrs);
303
304/**
305 * dma_map_sgtable - Map the given buffer for DMA
306 * @dev: The device for which to perform the DMA operation
307 * @sgt: The sg_table object describing the buffer
308 * @dir: DMA direction
309 * @attrs: Optional DMA attributes for the map operation
310 *
311 * Maps a buffer described by a scatterlist stored in the given sg_table
312 * object for the @dir DMA operation by the @dev device. After success, the
313 * ownership for the buffer is transferred to the DMA domain. One has to
314 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the
315 * ownership of the buffer back to the CPU domain before touching the
316 * buffer by the CPU.
317 *
318 * Returns 0 on success or a negative error code on error. The following
319 * error codes are supported with the given meaning:
320 *
321 * -EINVAL An invalid argument, unaligned access or other error
322 * in usage. Will not succeed if retried.
323 * -ENOMEM Insufficient resources (like memory or IOVA space) to
324 * complete the mapping. Should succeed if retried later.
325 * -EIO Legacy error code with an unknown meaning. eg. this is
326 * returned if a lower level call returned
327 * DMA_MAPPING_ERROR.
328 * -EREMOTEIO The DMA device cannot access P2PDMA memory specified
329 * in the sg_table. This will not succeed if retried.
330 */
331int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
332 enum dma_data_direction dir, unsigned long attrs)
333{
334 int nents;
335
336 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);
337 if (nents < 0)
338 return nents;
339 sgt->nents = nents;
340 return 0;
341}
342EXPORT_SYMBOL_GPL(dma_map_sgtable);
343
344void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
345 int nents, enum dma_data_direction dir,
346 unsigned long attrs)
347{
348 const struct dma_map_ops *ops = get_dma_ops(dev);
349
350 BUG_ON(!valid_dma_direction(dir));
351 trace_dma_unmap_sg(dev, sg, nents, dir, attrs);
352 debug_dma_unmap_sg(dev, sg, nents, dir, attrs);
353 if (dma_map_direct(dev, ops) ||
354 arch_dma_unmap_sg_direct(dev, sg, nents))
355 dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
356 else if (use_dma_iommu(dev))
357 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs);
358 else if (ops->unmap_sg)
359 ops->unmap_sg(dev, sg, nents, dir, attrs);
360}
361EXPORT_SYMBOL(dma_unmap_sg_attrs);
362
363dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
364 size_t size, enum dma_data_direction dir, unsigned long attrs)
365{
366 return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);
367}
368EXPORT_SYMBOL(dma_map_resource);
369
370void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
371 enum dma_data_direction dir, unsigned long attrs)
372{
373 dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO);
374}
375EXPORT_SYMBOL(dma_unmap_resource);
376
377#ifdef CONFIG_DMA_NEED_SYNC
378void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
379 enum dma_data_direction dir)
380{
381 const struct dma_map_ops *ops = get_dma_ops(dev);
382
383 BUG_ON(!valid_dma_direction(dir));
384 if (dma_map_direct(dev, ops))
385 dma_direct_sync_single_for_cpu(dev, addr, size, dir, true);
386 else if (use_dma_iommu(dev))
387 iommu_dma_sync_single_for_cpu(dev, addr, size, dir);
388 else if (ops->sync_single_for_cpu)
389 ops->sync_single_for_cpu(dev, addr, size, dir);
390 trace_dma_sync_single_for_cpu(dev, addr, size, dir);
391 debug_dma_sync_single_for_cpu(dev, addr, size, dir);
392}
393EXPORT_SYMBOL(__dma_sync_single_for_cpu);
394
395void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
396 size_t size, enum dma_data_direction dir)
397{
398 const struct dma_map_ops *ops = get_dma_ops(dev);
399
400 BUG_ON(!valid_dma_direction(dir));
401 if (dma_map_direct(dev, ops))
402 dma_direct_sync_single_for_device(dev, addr, size, dir);
403 else if (use_dma_iommu(dev))
404 iommu_dma_sync_single_for_device(dev, addr, size, dir);
405 else if (ops->sync_single_for_device)
406 ops->sync_single_for_device(dev, addr, size, dir);
407 trace_dma_sync_single_for_device(dev, addr, size, dir);
408 debug_dma_sync_single_for_device(dev, addr, size, dir);
409}
410EXPORT_SYMBOL(__dma_sync_single_for_device);
411
412void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
413 int nelems, enum dma_data_direction dir)
414{
415 const struct dma_map_ops *ops = get_dma_ops(dev);
416
417 BUG_ON(!valid_dma_direction(dir));
418 if (dma_map_direct(dev, ops))
419 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
420 else if (use_dma_iommu(dev))
421 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
422 else if (ops->sync_sg_for_cpu)
423 ops->sync_sg_for_cpu(dev, sg, nelems, dir);
424 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
425 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
426}
427EXPORT_SYMBOL(__dma_sync_sg_for_cpu);
428
429void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
430 int nelems, enum dma_data_direction dir)
431{
432 const struct dma_map_ops *ops = get_dma_ops(dev);
433
434 BUG_ON(!valid_dma_direction(dir));
435 if (dma_map_direct(dev, ops))
436 dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
437 else if (use_dma_iommu(dev))
438 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir);
439 else if (ops->sync_sg_for_device)
440 ops->sync_sg_for_device(dev, sg, nelems, dir);
441 trace_dma_sync_sg_for_device(dev, sg, nelems, dir);
442 debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
443}
444EXPORT_SYMBOL(__dma_sync_sg_for_device);
445
446bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr)
447{
448 const struct dma_map_ops *ops = get_dma_ops(dev);
449
450 if (dma_map_direct(dev, ops))
451 /*
452 * dma_skip_sync could've been reset on first SWIOTLB buffer
453 * mapping, but @dma_addr is not necessary an SWIOTLB buffer.
454 * In this case, fall back to more granular check.
455 */
456 return dma_direct_need_sync(dev, dma_addr);
457 return true;
458}
459EXPORT_SYMBOL_GPL(__dma_need_sync);
460
461/**
462 * dma_need_unmap - does this device need dma_unmap_* operations
463 * @dev: device to check
464 *
465 * If this function returns %false, drivers can skip calling dma_unmap_* after
466 * finishing an I/O. This function must be called after all mappings that might
467 * need to be unmapped have been performed.
468 */
469bool dma_need_unmap(struct device *dev)
470{
471 if (!dma_map_direct(dev, get_dma_ops(dev)))
472 return true;
473 if (!dev_dma_skip_sync(dev))
474 return true;
475 return IS_ENABLED(CONFIG_DMA_API_DEBUG);
476}
477EXPORT_SYMBOL_GPL(dma_need_unmap);
478
479static void dma_setup_need_sync(struct device *dev)
480{
481 const struct dma_map_ops *ops = get_dma_ops(dev);
482
483 if (dma_map_direct(dev, ops) || use_dma_iommu(dev))
484 /*
485 * dma_skip_sync will be reset to %false on first SWIOTLB buffer
486 * mapping, if any. During the device initialization, it's
487 * enough to check only for the DMA coherence.
488 */
489 dev_assign_dma_skip_sync(dev, dev_is_dma_coherent(dev));
490 else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu &&
491 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu)
492 /*
493 * Synchronization is not possible when none of DMA sync ops
494 * is set.
495 */
496 dev_set_dma_skip_sync(dev);
497 else
498 dev_clear_dma_skip_sync(dev);
499}
500#else /* !CONFIG_DMA_NEED_SYNC */
501static inline void dma_setup_need_sync(struct device *dev) { }
502#endif /* !CONFIG_DMA_NEED_SYNC */
503
504/*
505 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems
506 * that the intention is to allow exporting memory allocated via the
507 * coherent DMA APIs through the dma_buf API, which only accepts a
508 * scattertable. This presents a couple of problems:
509 * 1. Not all memory allocated via the coherent DMA APIs is backed by
510 * a struct page
511 * 2. Passing coherent DMA memory into the streaming APIs is not allowed
512 * as we will try to flush the memory through a different alias to that
513 * actually being used (and the flushes are redundant.)
514 */
515int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
516 void *cpu_addr, dma_addr_t dma_addr, size_t size,
517 unsigned long attrs)
518{
519 const struct dma_map_ops *ops = get_dma_ops(dev);
520
521 if (dma_alloc_direct(dev, ops))
522 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
523 size, attrs);
524 if (use_dma_iommu(dev))
525 return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr,
526 size, attrs);
527 if (!ops->get_sgtable)
528 return -ENXIO;
529 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);
530}
531EXPORT_SYMBOL(dma_get_sgtable_attrs);
532
533#ifdef CONFIG_MMU
534/*
535 * Return the page attributes used for mapping dma_alloc_* memory, either in
536 * kernel space if remapping is needed, or to userspace through dma_mmap_*.
537 */
538pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
539{
540 pgprot_t dma_prot;
541
542 if (dev_is_dma_coherent(dev))
543 dma_prot = prot;
544#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
545 else if (attrs & DMA_ATTR_WRITE_COMBINE)
546 dma_prot = pgprot_writecombine(prot);
547#endif
548 else
549 dma_prot = pgprot_dmacoherent(prot);
550
551 if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED))
552 return pgprot_decrypted(dma_prot);
553 else
554 return pgprot_encrypted(dma_prot);
555}
556#endif /* CONFIG_MMU */
557
558/**
559 * dma_can_mmap - check if a given device supports dma_mmap_*
560 * @dev: device to check
561 *
562 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to
563 * map DMA allocations to userspace.
564 */
565bool dma_can_mmap(struct device *dev)
566{
567 const struct dma_map_ops *ops = get_dma_ops(dev);
568
569 if (dma_alloc_direct(dev, ops))
570 return dma_direct_can_mmap(dev);
571 if (use_dma_iommu(dev))
572 return true;
573 return ops->mmap != NULL;
574}
575EXPORT_SYMBOL_GPL(dma_can_mmap);
576
577/**
578 * dma_mmap_attrs - map a coherent DMA allocation into user space
579 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
580 * @vma: vm_area_struct describing requested user mapping
581 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
582 * @dma_addr: device-view address returned from dma_alloc_attrs
583 * @size: size of memory originally requested in dma_alloc_attrs
584 * @attrs: attributes of mapping properties requested in dma_alloc_attrs
585 *
586 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user
587 * space. The coherent DMA buffer must not be freed by the driver until the
588 * user space mapping has been released.
589 */
590int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
591 void *cpu_addr, dma_addr_t dma_addr, size_t size,
592 unsigned long attrs)
593{
594 const struct dma_map_ops *ops = get_dma_ops(dev);
595
596 if (dma_alloc_direct(dev, ops))
597 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
598 attrs);
599 if (use_dma_iommu(dev))
600 return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size,
601 attrs);
602 if (!ops->mmap)
603 return -ENXIO;
604 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
605}
606EXPORT_SYMBOL(dma_mmap_attrs);
607
608u64 dma_get_required_mask(struct device *dev)
609{
610 const struct dma_map_ops *ops = get_dma_ops(dev);
611
612 if (dma_alloc_direct(dev, ops))
613 return dma_direct_get_required_mask(dev);
614
615 if (use_dma_iommu(dev))
616 return DMA_BIT_MASK(32);
617
618 if (ops->get_required_mask)
619 return ops->get_required_mask(dev);
620
621 /*
622 * We require every DMA ops implementation to at least support a 32-bit
623 * DMA mask (and use bounce buffering if that isn't supported in
624 * hardware). As the direct mapping code has its own routine to
625 * actually report an optimal mask we default to 32-bit here as that
626 * is the right thing for most IOMMUs, and at least not actively
627 * harmful in general.
628 */
629 return DMA_BIT_MASK(32);
630}
631EXPORT_SYMBOL_GPL(dma_get_required_mask);
632
633void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
634 gfp_t flag, unsigned long attrs)
635{
636 const struct dma_map_ops *ops = get_dma_ops(dev);
637 void *cpu_addr;
638
639 WARN_ON_ONCE(!dev->coherent_dma_mask);
640
641 /*
642 * DMA allocations can never be turned back into a page pointer, so
643 * requesting compound pages doesn't make sense (and can't even be
644 * supported at all by various backends).
645 */
646 if (WARN_ON_ONCE(flag & __GFP_COMP))
647 return NULL;
648
649 if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) {
650 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
651 attrs);
652 return NULL;
653 }
654
655 if (force_dma_unencrypted(dev))
656 attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
657
658 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
659 trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
660 DMA_BIDIRECTIONAL, flag, attrs);
661 return cpu_addr;
662 }
663
664 /* let the implementation decide on the zone to allocate from: */
665 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
666
667 if (dma_alloc_direct(dev, ops) || arch_dma_alloc_direct(dev)) {
668 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
669 } else if (use_dma_iommu(dev)) {
670 cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);
671 } else if (ops->alloc) {
672 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
673 } else {
674 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
675 attrs);
676 return NULL;
677 }
678
679 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL,
680 flag, attrs);
681 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);
682 return cpu_addr;
683}
684EXPORT_SYMBOL(dma_alloc_attrs);
685
686void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
687 dma_addr_t dma_handle, unsigned long attrs)
688{
689 const struct dma_map_ops *ops = get_dma_ops(dev);
690
691 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
692 return;
693 /*
694 * On non-coherent platforms which implement DMA-coherent buffers via
695 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting
696 * this far in IRQ context is a) at risk of a BUG_ON() or trying to
697 * sleep on some machines, and b) an indication that the driver is
698 * probably misusing the coherent API anyway.
699 */
700 WARN_ON(irqs_disabled());
701
702 trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL,
703 attrs);
704 if (!cpu_addr)
705 return;
706
707 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs);
708 if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle))
709 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
710 else if (use_dma_iommu(dev))
711 iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs);
712 else if (ops->free)
713 ops->free(dev, size, cpu_addr, dma_handle, attrs);
714}
715EXPORT_SYMBOL(dma_free_attrs);
716
717static struct page *__dma_alloc_pages(struct device *dev, size_t size,
718 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
719{
720 const struct dma_map_ops *ops = get_dma_ops(dev);
721
722 if (WARN_ON_ONCE(!dev->coherent_dma_mask))
723 return NULL;
724 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)))
725 return NULL;
726 if (WARN_ON_ONCE(gfp & __GFP_COMP))
727 return NULL;
728
729 size = PAGE_ALIGN(size);
730 if (dma_alloc_direct(dev, ops))
731 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);
732 if (use_dma_iommu(dev))
733 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp);
734 if (!ops->alloc_pages_op)
735 return NULL;
736 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp);
737}
738
739struct page *dma_alloc_pages(struct device *dev, size_t size,
740 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
741{
742 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp);
743
744 if (page) {
745 trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle,
746 size, dir, gfp, 0);
747 debug_dma_alloc_pages(dev, page, size, dir, *dma_handle);
748 } else {
749 trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0);
750 }
751 return page;
752}
753EXPORT_SYMBOL_GPL(dma_alloc_pages);
754
755static void __dma_free_pages(struct device *dev, size_t size, struct page *page,
756 dma_addr_t dma_handle, enum dma_data_direction dir)
757{
758 const struct dma_map_ops *ops = get_dma_ops(dev);
759
760 size = PAGE_ALIGN(size);
761 if (dma_alloc_direct(dev, ops))
762 dma_direct_free_pages(dev, size, page, dma_handle, dir);
763 else if (use_dma_iommu(dev))
764 dma_common_free_pages(dev, size, page, dma_handle, dir);
765 else if (ops->free_pages)
766 ops->free_pages(dev, size, page, dma_handle, dir);
767}
768
769void dma_free_pages(struct device *dev, size_t size, struct page *page,
770 dma_addr_t dma_handle, enum dma_data_direction dir)
771{
772 trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0);
773 debug_dma_free_pages(dev, page, size, dir, dma_handle);
774 __dma_free_pages(dev, size, page, dma_handle, dir);
775}
776EXPORT_SYMBOL_GPL(dma_free_pages);
777
778int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma,
779 size_t size, struct page *page)
780{
781 const pgoff_t pgoff_start = vma_start_pgoff(vma);
782 const pgoff_t pgoff_end = vma_end_pgoff(vma);
783 const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
784
785 if (pgoff_start >= count || pgoff_end > count)
786 return -ENXIO;
787 return remap_pfn_range(vma, vma->vm_start,
788 page_to_pfn(page) + pgoff_start,
789 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot);
790}
791EXPORT_SYMBOL_GPL(dma_mmap_pages);
792
793static struct sg_table *alloc_single_sgt(struct device *dev, size_t size,
794 enum dma_data_direction dir, gfp_t gfp)
795{
796 struct sg_table *sgt;
797 struct page *page;
798
799 sgt = kmalloc_obj(*sgt, gfp);
800 if (!sgt)
801 return NULL;
802 if (sg_alloc_table(sgt, 1, gfp))
803 goto out_free_sgt;
804 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp);
805 if (!page)
806 goto out_free_table;
807 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
808 sg_dma_len(sgt->sgl) = sgt->sgl->length;
809 return sgt;
810out_free_table:
811 sg_free_table(sgt);
812out_free_sgt:
813 kfree(sgt);
814 return NULL;
815}
816
817struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
818 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs)
819{
820 struct sg_table *sgt;
821
822 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES))
823 return NULL;
824 if (WARN_ON_ONCE(gfp & __GFP_COMP))
825 return NULL;
826
827 if (use_dma_iommu(dev))
828 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs);
829 else
830 sgt = alloc_single_sgt(dev, size, dir, gfp);
831
832 if (sgt) {
833 sgt->nents = 1;
834 trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs);
835 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs);
836 } else {
837 trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs);
838 }
839 return sgt;
840}
841EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous);
842
843static void free_single_sgt(struct device *dev, size_t size,
844 struct sg_table *sgt, enum dma_data_direction dir)
845{
846 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address,
847 dir);
848 sg_free_table(sgt);
849 kfree(sgt);
850}
851
852void dma_free_noncontiguous(struct device *dev, size_t size,
853 struct sg_table *sgt, enum dma_data_direction dir)
854{
855 trace_dma_free_sgt(dev, sgt, size, dir);
856 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0);
857
858 if (use_dma_iommu(dev))
859 iommu_dma_free_noncontiguous(dev, size, sgt, dir);
860 else
861 free_single_sgt(dev, size, sgt, dir);
862}
863EXPORT_SYMBOL_GPL(dma_free_noncontiguous);
864
865void *dma_vmap_noncontiguous(struct device *dev, size_t size,
866 struct sg_table *sgt)
867{
868
869 if (use_dma_iommu(dev))
870 return iommu_dma_vmap_noncontiguous(dev, size, sgt);
871
872 return page_address(sg_page(sgt->sgl));
873}
874EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous);
875
876void dma_vunmap_noncontiguous(struct device *dev, void *vaddr)
877{
878 if (use_dma_iommu(dev))
879 iommu_dma_vunmap_noncontiguous(dev, vaddr);
880}
881EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous);
882
883int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma,
884 size_t size, struct sg_table *sgt)
885{
886 if (use_dma_iommu(dev))
887 return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt);
888 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl));
889}
890EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous);
891
892static int dma_supported(struct device *dev, u64 mask)
893{
894 const struct dma_map_ops *ops = get_dma_ops(dev);
895
896 if (use_dma_iommu(dev)) {
897 if (WARN_ON(ops))
898 return false;
899 return true;
900 }
901
902 /*
903 * ->dma_supported sets and clears the bypass flag, so ignore it here
904 * and always call into the method if there is one.
905 */
906 if (ops) {
907 if (!ops->dma_supported)
908 return true;
909 return ops->dma_supported(dev, mask);
910 }
911
912 return dma_direct_supported(dev, mask);
913}
914
915bool dma_pci_p2pdma_supported(struct device *dev)
916{
917 const struct dma_map_ops *ops = get_dma_ops(dev);
918
919 /*
920 * Note: dma_ops_bypass is not checked here because P2PDMA should
921 * not be used with dma mapping ops that do not have support even
922 * if the specific device is bypassing them.
923 */
924
925 /* if ops is not set, dma direct and default IOMMU support P2PDMA */
926 return !ops;
927}
928EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported);
929
930int dma_set_mask(struct device *dev, u64 mask)
931{
932 /*
933 * Truncate the mask to the actually supported dma_addr_t width to
934 * avoid generating unsupportable addresses.
935 */
936 mask = (dma_addr_t)mask;
937
938 if (!dev->dma_mask || !dma_supported(dev, mask))
939 return -EIO;
940
941 arch_dma_set_mask(dev, mask);
942 *dev->dma_mask = mask;
943 dma_setup_need_sync(dev);
944
945 return 0;
946}
947EXPORT_SYMBOL(dma_set_mask);
948
949int dma_set_coherent_mask(struct device *dev, u64 mask)
950{
951 /*
952 * Truncate the mask to the actually supported dma_addr_t width to
953 * avoid generating unsupportable addresses.
954 */
955 mask = (dma_addr_t)mask;
956
957 if (!dma_supported(dev, mask))
958 return -EIO;
959
960 dev->coherent_dma_mask = mask;
961 return 0;
962}
963EXPORT_SYMBOL(dma_set_coherent_mask);
964
965static bool __dma_addressing_limited(struct device *dev)
966{
967 const struct dma_map_ops *ops = get_dma_ops(dev);
968
969 if (min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) <
970 dma_get_required_mask(dev))
971 return true;
972
973 if (unlikely(ops) || use_dma_iommu(dev))
974 return false;
975 return !dma_direct_all_ram_mapped(dev);
976}
977
978/**
979 * dma_addressing_limited - return if the device is addressing limited
980 * @dev: device to check
981 *
982 * Return %true if the devices DMA mask is too small to address all memory in
983 * the system, else %false. Lack of addressing bits is the prime reason for
984 * bounce buffering, but might not be the only one.
985 */
986bool dma_addressing_limited(struct device *dev)
987{
988 if (!__dma_addressing_limited(dev))
989 return false;
990
991 dev_dbg(dev, "device is DMA addressing limited\n");
992 return true;
993}
994EXPORT_SYMBOL_GPL(dma_addressing_limited);
995
996size_t dma_max_mapping_size(struct device *dev)
997{
998 const struct dma_map_ops *ops = get_dma_ops(dev);
999 size_t size = SIZE_MAX;
1000
1001 if (!dev->dma_mask)
1002 return 0;
1003
1004 if (dma_map_direct(dev, ops))
1005 size = dma_direct_max_mapping_size(dev);
1006 else if (use_dma_iommu(dev))
1007 size = iommu_dma_max_mapping_size(dev);
1008 else if (ops && ops->max_mapping_size)
1009 size = ops->max_mapping_size(dev);
1010
1011 return size;
1012}
1013EXPORT_SYMBOL_GPL(dma_max_mapping_size);
1014
1015size_t dma_opt_mapping_size(struct device *dev)
1016{
1017 const struct dma_map_ops *ops = get_dma_ops(dev);
1018 size_t size = SIZE_MAX;
1019
1020 if (use_dma_iommu(dev))
1021 size = iommu_dma_opt_mapping_size();
1022 else if (ops && ops->opt_mapping_size)
1023 size = ops->opt_mapping_size();
1024
1025 return min(dma_max_mapping_size(dev), size);
1026}
1027EXPORT_SYMBOL_GPL(dma_opt_mapping_size);
1028
1029unsigned long dma_get_merge_boundary(struct device *dev)
1030{
1031 const struct dma_map_ops *ops = get_dma_ops(dev);
1032
1033 if (use_dma_iommu(dev))
1034 return iommu_dma_get_merge_boundary(dev);
1035
1036 if (!ops || !ops->get_merge_boundary)
1037 return 0; /* can't merge */
1038
1039 return ops->get_merge_boundary(dev);
1040}
1041EXPORT_SYMBOL_GPL(dma_get_merge_boundary);