projects/linux·blob master · refs

2098 lines · 59.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Dynamic DMA mapping support.
4 *
5 * This implementation is a fallback for platforms that do not support
6 * I/O TLBs (aka DMA address translation hardware).
7 * Copyright (C) 2000 Asit Mallick <[email protected]>
8 * Copyright (C) 2000 Goutham Rao <[email protected]>
9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
10 * David Mosberger-Tang <[email protected]>
11 *
12 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14 * unnecessary i-cache flushing.
15 * 04/07/.. ak Better overflow handling. Assorted fixes.
16 * 05/09/10 linville Add support for syncing ranges, support syncing for
17 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18 * 08/12/11 beckyb Add highmem support
19 */
20
21#define pr_fmt(fmt) "software IO TLB: " fmt
22
23#include <linux/cache.h>
24#include <linux/cc_platform.h>
25#include <linux/ctype.h>
26#include <linux/debugfs.h>
27#include <linux/dma-direct.h>
28#include <linux/dma-map-ops.h>
29#include <linux/export.h>
30#include <linux/gfp.h>
31#include <linux/highmem.h>
32#include <linux/io.h>
33#include <linux/kmsan-checks.h>
34#include <linux/iommu-helper.h>
35#include <linux/init.h>
36#include <linux/memblock.h>
37#include <linux/mm.h>
38#include <linux/pfn.h>
39#include <linux/rculist.h>
40#include <linux/scatterlist.h>
41#include <linux/set_memory.h>
42#include <linux/spinlock.h>
43#include <linux/string.h>
44#include <linux/swiotlb.h>
45#include <linux/types.h>
46#ifdef CONFIG_DMA_RESTRICTED_POOL
47#include <linux/of.h>
48#include <linux/of_fdt.h>
49#include <linux/of_reserved_mem.h>
50#include <linux/slab.h>
51#endif
52
53#define CREATE_TRACE_POINTS
54#include <trace/events/swiotlb.h>
55
56#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
57
58/*
59 * Minimum IO TLB size to bother booting with. Systems with mainly
60 * 64bit capable cards will only lightly use the swiotlb. If we can't
61 * allocate a contiguous 1MB, we're probably in trouble anyway.
62 */
63#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
64
65/**
66 * struct io_tlb_slot - IO TLB slot descriptor
67 * @orig_addr: The original address corresponding to a mapped entry.
68 * @alloc_size: Size of the allocated buffer.
69 * @list: The free list describing the number of free entries available
70 * from each index.
71 * @pad_slots: Number of preceding padding slots. Valid only in the first
72 * allocated non-padding slot.
73 */
74struct io_tlb_slot {
75 phys_addr_t orig_addr;
76 size_t alloc_size;
77 unsigned short list;
78 unsigned short pad_slots;
79};
80
81static bool swiotlb_force_bounce;
82static bool swiotlb_force_disable;
83
84#ifdef CONFIG_SWIOTLB_DYNAMIC
85
86static void swiotlb_dyn_alloc(struct work_struct *work);
87
88static struct io_tlb_mem io_tlb_default_mem = {
89 .lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
90 .pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
91 .dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
92 swiotlb_dyn_alloc),
93};
94
95#else /* !CONFIG_SWIOTLB_DYNAMIC */
96
97static struct io_tlb_mem io_tlb_default_mem;
98
99#endif /* CONFIG_SWIOTLB_DYNAMIC */
100
101static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
102static unsigned long default_nareas;
103
104/**
105 * struct io_tlb_area - IO TLB memory area descriptor
106 *
107 * This is a single area with a single lock.
108 *
109 * @used: The number of used IO TLB block.
110 * @index: The slot index to start searching in this area for next round.
111 * @lock: The lock to protect the above data structures in the map and
112 * unmap calls.
113 */
114struct io_tlb_area {
115 unsigned long used;
116 unsigned int index;
117 spinlock_t lock;
118};
119
120/*
121 * Round up number of slabs to the next power of 2. The last area is going
122 * be smaller than the rest if default_nslabs is not power of two.
123 * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
124 * otherwise a segment may span two or more areas. It conflicts with free
125 * contiguous slots tracking: free slots are treated contiguous no matter
126 * whether they cross an area boundary.
127 *
128 * Return true if default_nslabs is rounded up.
129 */
130static bool round_up_default_nslabs(void)
131{
132 if (!default_nareas)
133 return false;
134
135 if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
136 default_nslabs = IO_TLB_SEGSIZE * default_nareas;
137 else if (is_power_of_2(default_nslabs))
138 return false;
139 default_nslabs = roundup_pow_of_two(default_nslabs);
140 return true;
141}
142
143/**
144 * swiotlb_adjust_nareas() - adjust the number of areas and slots
145 * @nareas: Desired number of areas. Zero is treated as 1.
146 *
147 * Adjust the default number of areas in a memory pool.
148 * The default size of the memory pool may also change to meet minimum area
149 * size requirements.
150 */
151static void swiotlb_adjust_nareas(unsigned int nareas)
152{
153 if (!nareas)
154 nareas = 1;
155 else if (!is_power_of_2(nareas))
156 nareas = roundup_pow_of_two(nareas);
157
158 default_nareas = nareas;
159
160 pr_info("area num %d.\n", nareas);
161 if (round_up_default_nslabs())
162 pr_info("SWIOTLB bounce buffer size roundup to %luMB",
163 (default_nslabs << IO_TLB_SHIFT) >> 20);
164}
165
166/**
167 * limit_nareas() - get the maximum number of areas for a given memory pool size
168 * @nareas: Desired number of areas.
169 * @nslots: Total number of slots in the memory pool.
170 *
171 * Limit the number of areas to the maximum possible number of areas in
172 * a memory pool of the given size.
173 *
174 * Return: Maximum possible number of areas.
175 */
176static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
177{
178 if (nslots < nareas * IO_TLB_SEGSIZE)
179 return nslots / IO_TLB_SEGSIZE;
180 return nareas;
181}
182
183#ifdef CONFIG_DEBUG_FS
184/*
185 * Track the total used slots with a global atomic value in order to have
186 * correct information to determine the high water mark.
187 */
188static void inc_used_and_hiwater_real(struct io_tlb_mem *mem,
189 unsigned int nslots)
190{
191 unsigned long old_hiwater, new_used;
192
193 new_used = atomic_long_add_return(nslots, &mem->total_used);
194 old_hiwater = atomic_long_read(&mem->used_hiwater);
195 do {
196 if (new_used <= old_hiwater)
197 break;
198 } while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
199 &old_hiwater, new_used));
200}
201
202static void dec_used_real(struct io_tlb_mem *mem, unsigned int nslots)
203{
204 atomic_long_sub(nslots, &mem->total_used);
205}
206
207static void inc_used_and_hiwater_nop(struct io_tlb_mem *mem,
208 unsigned int nslots)
209{
210}
211static void dec_used_nop(struct io_tlb_mem *mem, unsigned int nslots)
212{
213}
214
215DEFINE_STATIC_CALL(swiotlb_inc_used, inc_used_and_hiwater_nop);
216DEFINE_STATIC_CALL(swiotlb_dec_used, dec_used_nop);
217
218static __always_inline void inc_used_and_hiwater(struct io_tlb_mem *mem,
219 unsigned int nslots)
220{
221 static_call(swiotlb_inc_used)(mem, nslots);
222}
223
224static __always_inline void dec_used(struct io_tlb_mem *mem,
225 unsigned int nslots)
226{
227 static_call(swiotlb_dec_used)(mem, nslots);
228}
229
230static bool track_hiwater_enabled __read_mostly;
231
232#else
233
234static __always_inline void inc_used_and_hiwater(struct io_tlb_mem *mem,
235 unsigned int nslots)
236{
237}
238
239static __always_inline void dec_used(struct io_tlb_mem *mem,
240 unsigned int nslots)
241{
242}
243#endif
244
245/*
246 * The tracking of used slots high watermark can be enabled
247 * by appending "track_hiwater" to the swiotlb= boot parameter.
248 * When disabled the tracking functions are no-ops with near-zero
249 * overhead via static_call.
250 */
251static int __init
252setup_io_tlb_npages(char *str)
253{
254 if (isdigit(*str)) {
255 /* avoid tail segment of size < IO_TLB_SEGSIZE */
256 default_nslabs =
257 ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
258 }
259 if (*str == ',')
260 ++str;
261 if (isdigit(*str))
262 swiotlb_adjust_nareas(simple_strtoul(str, &str, 0));
263 if (*str == ',')
264 ++str;
265 if (!strncmp(str, "force", 5)) {
266 swiotlb_force_bounce = true;
267 str += 5;
268 } else if (!strncmp(str, "noforce", 7)) {
269 swiotlb_force_disable = true;
270 str += 7;
271 }
272
273#ifdef CONFIG_DEBUG_FS
274 if (*str == ',')
275 ++str;
276 if (!strncmp(str, "track_hiwater", 13)) {
277 track_hiwater_enabled = true;
278 static_call_update(swiotlb_inc_used,
279 inc_used_and_hiwater_real);
280 static_call_update(swiotlb_dec_used, dec_used_real);
281 }
282#endif
283
284 return 0;
285}
286early_param("swiotlb", setup_io_tlb_npages);
287
288unsigned long swiotlb_size_or_default(void)
289{
290 return default_nslabs << IO_TLB_SHIFT;
291}
292
293void __init swiotlb_adjust_size(unsigned long size)
294{
295 /*
296 * If swiotlb parameter has not been specified, give a chance to
297 * architectures such as those supporting memory encryption to
298 * adjust/expand SWIOTLB size for their use.
299 */
300 if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
301 return;
302
303 size = ALIGN(size, IO_TLB_SIZE);
304 default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
305 if (round_up_default_nslabs())
306 size = default_nslabs << IO_TLB_SHIFT;
307 pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
308}
309
310void swiotlb_print_info(void)
311{
312 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
313
314 if (!mem->nslabs) {
315 pr_warn("No low mem\n");
316 return;
317 }
318
319 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
320 (mem->nslabs << IO_TLB_SHIFT) >> 20);
321}
322
323static inline unsigned long io_tlb_offset(unsigned long val)
324{
325 return val & (IO_TLB_SEGSIZE - 1);
326}
327
328static inline unsigned long nr_slots(u64 val)
329{
330 return DIV_ROUND_UP(val, IO_TLB_SIZE);
331}
332
333static void swiotlb_mark_pool_used(struct io_tlb_pool *pool)
334{
335 unsigned long i;
336
337 for (i = 0; i < pool->nareas; i++) {
338 pool->areas[i].index = 0;
339 pool->areas[i].used = pool->area_nslabs;
340 }
341
342 for (i = 0; i < pool->nslabs; i++) {
343 pool->slots[i].list = 0;
344 pool->slots[i].orig_addr = INVALID_PHYS_ADDR;
345 pool->slots[i].alloc_size = 0;
346 pool->slots[i].pad_slots = 0;
347 }
348}
349
350/*
351 * Early SWIOTLB allocation may be too early to allow an architecture to
352 * perform the desired operations. This function allows the architecture to
353 * call SWIOTLB when the operations are possible. It needs to be called
354 * before the SWIOTLB memory is used.
355 */
356void __init swiotlb_update_mem_attributes(void)
357{
358 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
359 unsigned long bytes;
360
361 /*
362 * if platform support memory encryption, swiotlb buffers are
363 * shared by default.
364 */
365 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
366 io_tlb_default_mem.cc_shared = true;
367 else
368 io_tlb_default_mem.cc_shared = false;
369
370 if (!mem->nslabs || mem->late_alloc)
371 return;
372 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
373
374 if (io_tlb_default_mem.cc_shared) {
375 int ret;
376
377 ret = set_memory_decrypted((unsigned long)mem->vaddr,
378 bytes >> PAGE_SHIFT);
379 if (ret) {
380 pr_warn("Failed to decrypt default memory pool, disabling it\n");
381 swiotlb_mark_pool_used(mem);
382 }
383 }
384}
385
386static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
387 void *vaddr, unsigned long nslabs, bool late_alloc,
388 unsigned int nareas)
389{
390 unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
391
392 mem->nslabs = nslabs;
393 mem->start = start;
394 mem->end = mem->start + bytes;
395 mem->late_alloc = late_alloc;
396 mem->nareas = nareas;
397 mem->area_nslabs = nslabs / mem->nareas;
398
399 for (i = 0; i < mem->nareas; i++) {
400 spin_lock_init(&mem->areas[i].lock);
401 mem->areas[i].index = 0;
402 mem->areas[i].used = 0;
403 }
404
405 for (i = 0; i < mem->nslabs; i++) {
406 mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
407 mem->nslabs - i);
408 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
409 mem->slots[i].alloc_size = 0;
410 mem->slots[i].pad_slots = 0;
411 }
412
413 memset(vaddr, 0, bytes);
414 mem->vaddr = vaddr;
415 return;
416}
417
418/**
419 * add_mem_pool() - add a memory pool to the allocator
420 * @mem: Software IO TLB allocator.
421 * @pool: Memory pool to be added.
422 */
423static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
424{
425#ifdef CONFIG_SWIOTLB_DYNAMIC
426 spin_lock(&mem->lock);
427 list_add_rcu(&pool->node, &mem->pools);
428 mem->nslabs += pool->nslabs;
429 spin_unlock(&mem->lock);
430#else
431 mem->nslabs = pool->nslabs;
432#endif
433}
434
435static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
436 unsigned int flags,
437 int (*remap)(void *tlb, unsigned long nslabs))
438{
439 size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
440 void *tlb;
441
442 /*
443 * By default allocate the bounce buffer memory from low memory, but
444 * allow to pick a location everywhere for hypervisors with guest
445 * memory encryption.
446 */
447 if (flags & SWIOTLB_ANY)
448 tlb = memblock_alloc(bytes, PAGE_SIZE);
449 else
450 tlb = memblock_alloc_low(bytes, PAGE_SIZE);
451
452 if (!tlb) {
453 pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
454 __func__, bytes);
455 return NULL;
456 }
457
458 if (remap && remap(tlb, nslabs) < 0) {
459 memblock_free(tlb, PAGE_ALIGN(bytes));
460 pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
461 return NULL;
462 }
463
464 return tlb;
465}
466
467/*
468 * Statically reserve bounce buffer space and initialize bounce buffer data
469 * structures for the software IO TLB used to implement the DMA API.
470 */
471void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
472 int (*remap)(void *tlb, unsigned long nslabs))
473{
474 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
475 unsigned long nslabs;
476 unsigned int nareas;
477 size_t alloc_size;
478 void *tlb;
479
480 if (!addressing_limit && !swiotlb_force_bounce)
481 return;
482 if (swiotlb_force_disable)
483 return;
484
485 io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
486
487#ifdef CONFIG_SWIOTLB_DYNAMIC
488 if (!remap)
489 io_tlb_default_mem.can_grow = true;
490 if (flags & SWIOTLB_ANY)
491 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
492 else
493 io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
494#endif
495
496 if (!default_nareas)
497 swiotlb_adjust_nareas(num_possible_cpus());
498
499 nslabs = default_nslabs;
500 nareas = limit_nareas(default_nareas, nslabs);
501 while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
502 if (nslabs <= IO_TLB_MIN_SLABS)
503 return;
504 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
505 nareas = limit_nareas(nareas, nslabs);
506 }
507
508 if (default_nslabs != nslabs) {
509 pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
510 default_nslabs, nslabs);
511 default_nslabs = nslabs;
512 }
513
514 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
515 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
516 if (!mem->slots) {
517 pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
518 __func__, alloc_size, PAGE_SIZE);
519 return;
520 }
521
522 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
523 nareas), SMP_CACHE_BYTES);
524 if (!mem->areas) {
525 pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
526 return;
527 }
528
529 swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas);
530 add_mem_pool(&io_tlb_default_mem, mem);
531
532 if (flags & SWIOTLB_VERBOSE)
533 swiotlb_print_info();
534}
535
536void __init swiotlb_init(bool addressing_limit, unsigned int flags)
537{
538 swiotlb_init_remap(addressing_limit, flags, NULL);
539}
540
541/*
542 * Systems with larger DMA zones (those that don't support ISA) can
543 * initialize the swiotlb later using the slab allocator if needed.
544 * This should be just like above, but with some error catching.
545 */
546int swiotlb_init_late(size_t size, gfp_t gfp_mask,
547 int (*remap)(void *tlb, unsigned long nslabs))
548{
549 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
550 unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
551 unsigned int order, area_order, slot_order;
552 bool leak_pages = false;
553 unsigned int nareas;
554 unsigned char *vstart = NULL;
555 bool retried = false;
556 int rc = 0;
557
558 if (io_tlb_default_mem.nslabs)
559 return 0;
560
561 if (swiotlb_force_disable)
562 return 0;
563
564 io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
565
566#ifdef CONFIG_SWIOTLB_DYNAMIC
567 if (!remap)
568 io_tlb_default_mem.can_grow = true;
569 if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
570 io_tlb_default_mem.phys_limit = zone_dma_limit;
571 else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
572 io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
573 else
574 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
575#endif
576
577 if (!default_nareas)
578 swiotlb_adjust_nareas(num_possible_cpus());
579
580retry:
581 order = get_order(nslabs << IO_TLB_SHIFT);
582 nslabs = SLABS_PER_PAGE << order;
583
584 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
585 vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
586 order);
587 if (vstart)
588 break;
589 order--;
590 nslabs = SLABS_PER_PAGE << order;
591 retried = true;
592 }
593
594 if (!vstart)
595 return -ENOMEM;
596
597 if (remap)
598 rc = remap(vstart, nslabs);
599 if (rc) {
600 free_pages((unsigned long)vstart, order);
601
602 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
603 if (nslabs < IO_TLB_MIN_SLABS)
604 return rc;
605 retried = true;
606 goto retry;
607 }
608
609 if (retried) {
610 pr_warn("only able to allocate %ld MB\n",
611 (PAGE_SIZE << order) >> 20);
612 }
613
614 rc = -ENOMEM;
615 nareas = limit_nareas(default_nareas, nslabs);
616 area_order = get_order(array_size(sizeof(*mem->areas), nareas));
617 mem->areas = (struct io_tlb_area *)
618 __get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
619 if (!mem->areas)
620 goto error_area;
621
622 slot_order = get_order(array_size(sizeof(*mem->slots), nslabs));
623 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
624 slot_order);
625 if (!mem->slots)
626 goto error_slots;
627
628 if (io_tlb_default_mem.cc_shared) {
629 rc = set_memory_decrypted((unsigned long)vstart,
630 (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
631 if (rc) {
632 leak_pages = true;
633 goto error_decrypt;
634 }
635 }
636
637 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true,
638 nareas);
639 add_mem_pool(&io_tlb_default_mem, mem);
640
641 swiotlb_print_info();
642 return 0;
643
644error_decrypt:
645 free_pages((unsigned long)mem->slots, slot_order);
646error_slots:
647 free_pages((unsigned long)mem->areas, area_order);
648error_area:
649 if (!leak_pages)
650 free_pages((unsigned long)vstart, order);
651 return rc;
652}
653
654void __init swiotlb_exit(void)
655{
656 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
657 bool leak_pages = false;
658 unsigned long tbl_vaddr;
659 size_t tbl_size, slots_size;
660 unsigned int area_order;
661
662 if (swiotlb_force_bounce)
663 return;
664
665 if (!mem->nslabs)
666 return;
667
668 pr_info("tearing down default memory pool\n");
669 tbl_vaddr = (unsigned long)phys_to_virt(mem->start);
670 tbl_size = PAGE_ALIGN(mem->end - mem->start);
671 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
672
673 if (io_tlb_default_mem.cc_shared) {
674 if (set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT))
675 leak_pages = true;
676 }
677
678 if (mem->late_alloc) {
679 area_order = get_order(array_size(sizeof(*mem->areas),
680 mem->nareas));
681 free_pages((unsigned long)mem->areas, area_order);
682 if (!leak_pages)
683 free_pages(tbl_vaddr, get_order(tbl_size));
684 free_pages((unsigned long)mem->slots, get_order(slots_size));
685 } else {
686 memblock_free(mem->areas,
687 array_size(sizeof(*mem->areas), mem->nareas));
688 if (!leak_pages)
689 memblock_phys_free(mem->start, tbl_size);
690 memblock_free(mem->slots, slots_size);
691 }
692
693 memset(mem, 0, sizeof(*mem));
694}
695
696#ifdef CONFIG_SWIOTLB_DYNAMIC
697
698/**
699 * alloc_dma_pages() - allocate pages to be used for DMA
700 * @gfp: GFP flags for the allocation.
701 * @bytes: Size of the buffer.
702 * @phys_limit: Maximum allowed physical address of the buffer.
703 * @attrs: DMA attributes for the allocation.
704 *
705 * Allocate pages from the buddy allocator. If successful, make the allocated
706 * pages decrypted that they can be used for DMA.
707 *
708 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
709 * if the allocated physical address was above @phys_limit.
710 */
711static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
712 u64 phys_limit, unsigned long attrs)
713{
714 unsigned int order = get_order(bytes);
715 bool cc_shared = attrs & __DMA_ATTR_ALLOC_CC_SHARED;
716 struct page *page;
717 phys_addr_t paddr;
718 void *vaddr;
719
720 page = alloc_pages(gfp, order);
721 if (!page)
722 return NULL;
723
724 paddr = page_to_phys(page);
725 if (paddr + bytes - 1 > phys_limit) {
726 __free_pages(page, order);
727 return ERR_PTR(-EAGAIN);
728 }
729
730 vaddr = phys_to_virt(paddr);
731 if (cc_shared && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
732 goto error;
733 return page;
734
735error:
736 /* Intentional leak if pages cannot be encrypted again. */
737 if (cc_shared && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
738 __free_pages(page, order);
739 return NULL;
740}
741
742/**
743 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
744 * @dev: Device for which a memory pool is allocated.
745 * @mem: SWIOTLB allocator for the pool.
746 * @bytes: Size of the buffer.
747 * @phys_limit: Maximum allowed physical address of the buffer.
748 * @gfp: GFP flags for the allocation.
749 * @vaddr: Receives the virtual address for the allocated buffer.
750 *
751 * Return: Allocated pages, or %NULL on allocation failure.
752 */
753static struct page *swiotlb_alloc_tlb(struct device *dev,
754 struct io_tlb_mem *mem, size_t bytes,
755 u64 phys_limit, gfp_t gfp, void **vaddr)
756{
757 struct page *page;
758 unsigned long attrs = mem->cc_shared ? __DMA_ATTR_ALLOC_CC_SHARED : 0;
759
760 *vaddr = NULL;
761 /*
762 * Allocate from the atomic pools if memory is encrypted and
763 * the allocation is atomic, because decrypting may block.
764 */
765 if (!gfpflags_allow_blocking(gfp) && dev && mem->cc_shared) {
766
767 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
768 return NULL;
769
770 return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
771 attrs, dma_coherent_ok);
772 }
773
774 gfp &= ~GFP_ZONEMASK;
775 if (phys_limit <= zone_dma_limit)
776 gfp |= __GFP_DMA;
777 else if (phys_limit <= DMA_BIT_MASK(32))
778 gfp |= __GFP_DMA32;
779
780 while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit, attrs))) {
781 if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
782 phys_limit < DMA_BIT_MASK(64) &&
783 !(gfp & (__GFP_DMA32 | __GFP_DMA)))
784 gfp |= __GFP_DMA32;
785 else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
786 !(gfp & __GFP_DMA))
787 gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
788 else
789 return NULL;
790 }
791
792 if (page)
793 *vaddr = phys_to_virt(page_to_phys(page));
794 return page;
795}
796
797/**
798 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
799 * @vaddr: Virtual address of the buffer.
800 * @bytes: Size of the buffer.
801 * @cc_shared: true if @vaddr was allocated decrypted and must be
802 * re-encrypted before being freed
803 */
804static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool cc_shared)
805{
806 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
807 dma_free_from_pool(NULL, vaddr, bytes))
808 return;
809
810 /* Intentional leak if pages cannot be encrypted again. */
811 if (!cc_shared ||
812 !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
813 __free_pages(virt_to_page(vaddr), get_order(bytes));
814}
815
816/**
817 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
818 * @dev: Device for which a memory pool is allocated.
819 * @mem: SWIOTLB allocator for the pool.
820 * @minslabs: Minimum number of slabs.
821 * @nslabs: Desired (maximum) number of slabs.
822 * @nareas: Number of areas.
823 * @phys_limit: Maximum DMA buffer physical address.
824 * @gfp: GFP flags for the allocations.
825 *
826 * Allocate and initialize a new IO TLB memory pool. The actual number of
827 * slabs may be reduced if allocation of @nslabs fails. If even
828 * @minslabs cannot be allocated, this function fails.
829 *
830 * Return: New memory pool, or %NULL on allocation failure.
831 */
832static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
833 struct io_tlb_mem *mem, unsigned long minslabs,
834 unsigned long nslabs, unsigned int nareas, u64 phys_limit,
835 gfp_t gfp)
836{
837 struct io_tlb_pool *pool;
838 unsigned int slot_order;
839 void *tlb_vaddr;
840 struct page *tlb;
841 size_t pool_size;
842 size_t tlb_size;
843
844 if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
845 nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
846 nareas = limit_nareas(nareas, nslabs);
847 }
848
849 pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
850 pool = kzalloc(pool_size, gfp);
851 if (!pool)
852 goto error;
853 pool->areas = (void *)pool + sizeof(*pool);
854 pool->cc_shared = mem->cc_shared;
855
856 tlb_size = nslabs << IO_TLB_SHIFT;
857 while (!(tlb = swiotlb_alloc_tlb(dev, mem, tlb_size,
858 phys_limit, gfp, &tlb_vaddr))) {
859 if (nslabs <= minslabs)
860 goto error_tlb;
861 nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
862 nareas = limit_nareas(nareas, nslabs);
863 tlb_size = nslabs << IO_TLB_SHIFT;
864 }
865
866 slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
867 pool->slots = (struct io_tlb_slot *)
868 __get_free_pages(gfp, slot_order);
869 if (!pool->slots)
870 goto error_slots;
871
872 swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs,
873 true, nareas);
874 return pool;
875
876error_slots:
877 swiotlb_free_tlb(tlb_vaddr, tlb_size, mem->cc_shared);
878error_tlb:
879 kfree(pool);
880error:
881 return NULL;
882}
883
884/**
885 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
886 * @work: Pointer to dyn_alloc in struct io_tlb_mem.
887 */
888static void swiotlb_dyn_alloc(struct work_struct *work)
889{
890 struct io_tlb_mem *mem =
891 container_of(work, struct io_tlb_mem, dyn_alloc);
892 struct io_tlb_pool *pool;
893
894 pool = swiotlb_alloc_pool(NULL, mem, IO_TLB_MIN_SLABS, default_nslabs,
895 default_nareas, mem->phys_limit, GFP_KERNEL);
896 if (!pool) {
897 pr_warn_ratelimited("Failed to allocate new pool");
898 return;
899 }
900
901 add_mem_pool(mem, pool);
902}
903
904static void swiotlb_dyn_free_work(struct work_struct *work)
905{
906 struct io_tlb_pool *pool =
907 container_of(to_rcu_work(work), struct io_tlb_pool, dyn_free);
908 size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
909 size_t tlb_size = pool->end - pool->start;
910
911 free_pages((unsigned long)pool->slots, get_order(slots_size));
912 swiotlb_free_tlb(pool->vaddr, tlb_size, pool->cc_shared);
913 kfree(pool);
914}
915
916static void swiotlb_schedule_dyn_free(struct io_tlb_pool *pool)
917{
918 INIT_RCU_WORK(&pool->dyn_free, swiotlb_dyn_free_work);
919 queue_rcu_work(system_wq, &pool->dyn_free);
920}
921
922/**
923 * __swiotlb_find_pool() - find the IO TLB pool for a physical address
924 * @dev: Device which has mapped the DMA buffer.
925 * @paddr: Physical address within the DMA buffer.
926 *
927 * Find the IO TLB memory pool descriptor which contains the given physical
928 * address, if any. This function is for use only when the dev is known to
929 * be using swiotlb. Use swiotlb_find_pool() for the more general case
930 * when this condition is not met.
931 *
932 * Return: Memory pool which contains @paddr, or %NULL if none.
933 */
934struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
935{
936 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
937 struct io_tlb_pool *pool;
938
939 rcu_read_lock();
940 list_for_each_entry_rcu(pool, &mem->pools, node) {
941 if (paddr >= pool->start && paddr < pool->end)
942 goto out;
943 }
944
945 list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
946 if (paddr >= pool->start && paddr < pool->end)
947 goto out;
948 }
949 pool = NULL;
950out:
951 rcu_read_unlock();
952 return pool;
953}
954
955/**
956 * swiotlb_del_pool() - remove an IO TLB pool from a device
957 * @dev: Owning device.
958 * @pool: Memory pool to be removed.
959 */
960static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
961{
962 unsigned long flags;
963
964 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
965 list_del_rcu(&pool->node);
966 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
967
968 swiotlb_schedule_dyn_free(pool);
969}
970
971#endif /* CONFIG_SWIOTLB_DYNAMIC */
972
973/**
974 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
975 * @dev: Device to be initialized.
976 */
977void swiotlb_dev_init(struct device *dev)
978{
979 dev->dma_io_tlb_mem = &io_tlb_default_mem;
980#ifdef CONFIG_SWIOTLB_DYNAMIC
981 INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
982 spin_lock_init(&dev->dma_io_tlb_lock);
983 dev->dma_uses_io_tlb = false;
984#endif
985}
986
987/**
988 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
989 * @dev: Owning device.
990 * @align_mask: Allocation alignment mask.
991 * @addr: DMA address.
992 *
993 * Return the minimum offset from the start of an IO TLB allocation which is
994 * required for a given buffer address and allocation alignment to keep the
995 * device happy.
996 *
997 * First, the address bits covered by min_align_mask must be identical in the
998 * original address and the bounce buffer address. High bits are preserved by
999 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
1000 * padding bytes before the bounce buffer.
1001 *
1002 * Second, @align_mask specifies which bits of the first allocated slot must
1003 * be zero. This may require allocating additional padding slots, and then the
1004 * offset (in bytes) from the first such padding slot is returned.
1005 */
1006static unsigned int swiotlb_align_offset(struct device *dev,
1007 unsigned int align_mask, u64 addr)
1008{
1009 return addr & dma_get_min_align_mask(dev) &
1010 (align_mask | (IO_TLB_SIZE - 1));
1011}
1012
1013/*
1014 * Bounce: copy the swiotlb buffer from or back to the original dma location
1015 */
1016static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
1017 enum dma_data_direction dir, struct io_tlb_pool *mem)
1018{
1019 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
1020 phys_addr_t orig_addr = mem->slots[index].orig_addr;
1021 size_t alloc_size = mem->slots[index].alloc_size;
1022 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
1023 int tlb_offset;
1024
1025 if (orig_addr == INVALID_PHYS_ADDR)
1026 return;
1027
1028 if (dir == DMA_FROM_DEVICE && !dev_is_dma_coherent(dev))
1029 arch_sync_dma_flush();
1030
1031 /*
1032 * It's valid for tlb_offset to be negative. This can happen when the
1033 * "offset" returned by swiotlb_align_offset() is non-zero, and the
1034 * tlb_addr is pointing within the first "offset" bytes of the second
1035 * or subsequent slots of the allocated swiotlb area. While it's not
1036 * valid for tlb_addr to be pointing within the first "offset" bytes
1037 * of the first slot, there's no way to check for such an error since
1038 * this function can't distinguish the first slot from the second and
1039 * subsequent slots.
1040 */
1041 tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
1042 swiotlb_align_offset(dev, 0, orig_addr);
1043
1044 orig_addr += tlb_offset;
1045 alloc_size -= tlb_offset;
1046
1047 if (size > alloc_size) {
1048 dev_WARN_ONCE(dev, 1,
1049 "Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
1050 alloc_size, size);
1051 size = alloc_size;
1052 }
1053
1054 if (PhysHighMem(orig_addr)) {
1055 unsigned long pfn = PFN_DOWN(orig_addr);
1056 unsigned int offset = orig_addr & ~PAGE_MASK;
1057 struct page *page;
1058 unsigned int sz = 0;
1059 unsigned long flags;
1060
1061 while (size) {
1062 sz = min_t(size_t, PAGE_SIZE - offset, size);
1063
1064 local_irq_save(flags);
1065 page = pfn_to_page(pfn);
1066 if (dir == DMA_TO_DEVICE) {
1067 /*
1068 * Ideally, kmsan_check_highmem_page()
1069 * could be used here to detect infoleaks,
1070 * but callers may map uninitialized buffers
1071 * that will be written by the device,
1072 * causing false positives.
1073 */
1074 memcpy_from_page(vaddr, page, offset, sz);
1075 } else {
1076 kmsan_unpoison_memory(vaddr, sz);
1077 memcpy_to_page(page, offset, vaddr, sz);
1078 }
1079 local_irq_restore(flags);
1080
1081 size -= sz;
1082 pfn++;
1083 vaddr += sz;
1084 offset = 0;
1085 }
1086 } else if (dir == DMA_TO_DEVICE) {
1087 /*
1088 * Ideally, kmsan_check_memory() could be used here to detect
1089 * infoleaks (uninitialized data being sent to device), but
1090 * callers may map uninitialized buffers that will be written
1091 * by the device, causing false positives.
1092 */
1093 memcpy(vaddr, phys_to_virt(orig_addr), size);
1094 } else {
1095 kmsan_unpoison_memory(vaddr, size);
1096 memcpy(phys_to_virt(orig_addr), vaddr, size);
1097 }
1098}
1099
1100static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
1101{
1102 return start + (idx << IO_TLB_SHIFT);
1103}
1104
1105/*
1106 * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
1107 */
1108static inline unsigned long get_max_slots(unsigned long boundary_mask)
1109{
1110 return (boundary_mask >> IO_TLB_SHIFT) + 1;
1111}
1112
1113static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
1114{
1115 if (index >= mem->area_nslabs)
1116 return 0;
1117 return index;
1118}
1119
1120#ifdef CONFIG_SWIOTLB_DYNAMIC
1121#ifdef CONFIG_DEBUG_FS
1122static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1123{
1124 atomic_long_add(nslots, &mem->transient_nslabs);
1125}
1126
1127static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1128{
1129 atomic_long_sub(nslots, &mem->transient_nslabs);
1130}
1131
1132#else /* !CONFIG_DEBUG_FS */
1133static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1134{
1135}
1136static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
1137{
1138}
1139#endif /* CONFIG_DEBUG_FS */
1140#endif /* CONFIG_SWIOTLB_DYNAMIC */
1141
1142/**
1143 * swiotlb_search_pool_area() - search one memory area in one pool
1144 * @dev: Device which maps the buffer.
1145 * @pool: Memory pool to be searched.
1146 * @area_index: Index of the IO TLB memory area to be searched.
1147 * @orig_addr: Original (non-bounced) IO buffer address.
1148 * @tbl_dma_addr: DMA address of the bounce buffer.
1149 * @alloc_size: Total requested size of the bounce buffer,
1150 * including initial alignment padding.
1151 * @alloc_align_mask: Required alignment of the allocated buffer.
1152 *
1153 * Find a suitable sequence of IO TLB entries for the request and allocate
1154 * a buffer from the given IO TLB memory area.
1155 * This function takes care of locking.
1156 *
1157 * Return: Index of the first allocated slot, or -1 on error.
1158 */
1159static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1160 int area_index, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr,
1161 size_t alloc_size, unsigned int alloc_align_mask)
1162{
1163 struct io_tlb_area *area = pool->areas + area_index;
1164 unsigned long boundary_mask = dma_get_seg_boundary(dev);
1165 unsigned long max_slots = get_max_slots(boundary_mask);
1166 unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1167 unsigned int nslots = nr_slots(alloc_size), stride;
1168 unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
1169 unsigned int index, slots_checked, count = 0, i;
1170 unsigned long flags;
1171 unsigned int slot_base;
1172 unsigned int slot_index;
1173
1174 BUG_ON(!nslots);
1175 BUG_ON(area_index >= pool->nareas);
1176
1177 tbl_dma_addr &= boundary_mask;
1178
1179 /*
1180 * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1181 * page-aligned in the absence of any other alignment requirements.
1182 * 'alloc_align_mask' was later introduced to specify the alignment
1183 * explicitly, however this is passed as zero for streaming mappings
1184 * and so we preserve the old behaviour there in case any drivers are
1185 * relying on it.
1186 */
1187 if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1188 alloc_align_mask = PAGE_SIZE - 1;
1189
1190 /*
1191 * Ensure that the allocation is at least slot-aligned and update
1192 * 'iotlb_align_mask' to ignore bits that will be preserved when
1193 * offsetting into the allocation.
1194 */
1195 alloc_align_mask |= (IO_TLB_SIZE - 1);
1196 iotlb_align_mask &= ~alloc_align_mask;
1197
1198 /*
1199 * For mappings with an alignment requirement don't bother looping to
1200 * unaligned slots once we found an aligned one.
1201 */
1202 stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1203
1204 spin_lock_irqsave(&area->lock, flags);
1205 if (unlikely(nslots > pool->area_nslabs - area->used))
1206 goto not_found;
1207
1208 slot_base = area_index * pool->area_nslabs;
1209 index = area->index;
1210
1211 for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1212 phys_addr_t tlb_addr;
1213
1214 slot_index = slot_base + index;
1215 tlb_addr = slot_addr(tbl_dma_addr, slot_index);
1216
1217 if ((tlb_addr & alloc_align_mask) ||
1218 (orig_addr && (tlb_addr & iotlb_align_mask) !=
1219 (orig_addr & iotlb_align_mask))) {
1220 index = wrap_area_index(pool, index + 1);
1221 slots_checked++;
1222 continue;
1223 }
1224
1225 if (!iommu_is_span_boundary(slot_index, nslots,
1226 nr_slots(tbl_dma_addr),
1227 max_slots)) {
1228 if (pool->slots[slot_index].list >= nslots)
1229 goto found;
1230 }
1231 index = wrap_area_index(pool, index + stride);
1232 slots_checked += stride;
1233 }
1234
1235not_found:
1236 spin_unlock_irqrestore(&area->lock, flags);
1237 return -1;
1238
1239found:
1240 /*
1241 * If we find a slot that indicates we have 'nslots' number of
1242 * contiguous buffers, we allocate the buffers from that slot onwards
1243 * and set the list of free entries to '0' indicating unavailable.
1244 */
1245 for (i = slot_index; i < slot_index + nslots; i++) {
1246 pool->slots[i].list = 0;
1247 pool->slots[i].alloc_size = alloc_size - (offset +
1248 ((i - slot_index) << IO_TLB_SHIFT));
1249 }
1250 for (i = slot_index - 1;
1251 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
1252 pool->slots[i].list; i--)
1253 pool->slots[i].list = ++count;
1254
1255 /*
1256 * Update the indices to avoid searching in the next round.
1257 */
1258 area->index = wrap_area_index(pool, index + nslots);
1259 area->used += nslots;
1260 spin_unlock_irqrestore(&area->lock, flags);
1261
1262 inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
1263 return slot_index;
1264}
1265
1266#ifdef CONFIG_SWIOTLB_DYNAMIC
1267
1268/**
1269 * swiotlb_search_area() - search one memory area in all pools
1270 * @dev: Device which maps the buffer.
1271 * @start_cpu: Start CPU number.
1272 * @cpu_offset: Offset from @start_cpu.
1273 * @orig_addr: Original (non-bounced) IO buffer address.
1274 * @alloc_size: Total requested size of the bounce buffer,
1275 * including initial alignment padding.
1276 * @alloc_align_mask: Required alignment of the allocated buffer.
1277 * @retpool: Used memory pool, updated on return.
1278 *
1279 * Search one memory area in all pools for a sequence of slots that match the
1280 * allocation constraints.
1281 *
1282 * Return: Index of the first allocated slot, or -1 on error.
1283 */
1284static int swiotlb_search_area(struct device *dev, int start_cpu,
1285 int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1286 unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1287{
1288 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1289 struct io_tlb_pool *pool;
1290 dma_addr_t tbl_dma_addr;
1291 int area_index;
1292 int index = -1;
1293
1294 rcu_read_lock();
1295 list_for_each_entry_rcu(pool, &mem->pools, node) {
1296 if (cpu_offset >= pool->nareas)
1297 continue;
1298 area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1299
1300 if (mem->cc_shared)
1301 tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
1302 else
1303 tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
1304
1305 index = swiotlb_search_pool_area(dev, pool, area_index,
1306 orig_addr, tbl_dma_addr,
1307 alloc_size, alloc_align_mask);
1308 if (index >= 0) {
1309 *retpool = pool;
1310 break;
1311 }
1312 }
1313 rcu_read_unlock();
1314 return index;
1315}
1316
1317/**
1318 * swiotlb_find_slots() - search for slots in the whole swiotlb
1319 * @dev: Device which maps the buffer.
1320 * @orig_addr: Original (non-bounced) IO buffer address.
1321 * @alloc_size: Total requested size of the bounce buffer,
1322 * including initial alignment padding.
1323 * @alloc_align_mask: Required alignment of the allocated buffer.
1324 * @retpool: Used memory pool, updated on return.
1325 *
1326 * Search through the whole software IO TLB to find a sequence of slots that
1327 * match the allocation constraints.
1328 *
1329 * Return: Index of the first allocated slot, or -1 on error.
1330 */
1331static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1332 size_t alloc_size, unsigned int alloc_align_mask,
1333 struct io_tlb_pool **retpool)
1334{
1335 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1336 struct io_tlb_pool *pool;
1337 dma_addr_t tbl_dma_addr;
1338 unsigned long nslabs;
1339 unsigned long flags;
1340 u64 phys_limit;
1341 int cpu, i;
1342 int index;
1343
1344 if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1345 return -1;
1346
1347 cpu = raw_smp_processor_id();
1348 for (i = 0; i < default_nareas; ++i) {
1349 index = swiotlb_search_area(dev, cpu, i, orig_addr, alloc_size,
1350 alloc_align_mask, &pool);
1351 if (index >= 0)
1352 goto found;
1353 }
1354
1355 if (!mem->can_grow)
1356 return -1;
1357
1358 schedule_work(&mem->dyn_alloc);
1359
1360 nslabs = nr_slots(alloc_size);
1361 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1362 pool = swiotlb_alloc_pool(dev, mem, nslabs, nslabs, 1, phys_limit,
1363 GFP_NOWAIT);
1364 if (!pool)
1365 return -1;
1366
1367 if (mem->cc_shared)
1368 tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
1369 else
1370 tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
1371
1372 index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr,
1373 alloc_size, alloc_align_mask);
1374 if (index < 0) {
1375 swiotlb_schedule_dyn_free(pool);
1376 return -1;
1377 }
1378
1379 pool->transient = true;
1380 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1381 list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
1382 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
1383 inc_transient_used(mem, pool->nslabs);
1384
1385found:
1386 WRITE_ONCE(dev->dma_uses_io_tlb, true);
1387
1388 /*
1389 * The general barrier orders reads and writes against a presumed store
1390 * of the SWIOTLB buffer address by a device driver (to a driver private
1391 * data structure). It serves two purposes.
1392 *
1393 * First, the store to dev->dma_uses_io_tlb must be ordered before the
1394 * presumed store. This guarantees that the returned buffer address
1395 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1396 *
1397 * Second, the load from mem->pools must be ordered before the same
1398 * presumed store. This guarantees that the returned buffer address
1399 * cannot be observed by another CPU before an update of the RCU list
1400 * that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1401 * atomicity).
1402 *
1403 * See also the comment in swiotlb_find_pool().
1404 */
1405 smp_mb();
1406
1407 *retpool = pool;
1408 return index;
1409}
1410
1411#else /* !CONFIG_SWIOTLB_DYNAMIC */
1412
1413static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1414 size_t alloc_size, unsigned int alloc_align_mask,
1415 struct io_tlb_pool **retpool)
1416{
1417 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1418 struct io_tlb_pool *pool;
1419 dma_addr_t tbl_dma_addr;
1420 int start, i;
1421 int index;
1422
1423 *retpool = pool = &mem->defpool;
1424 if (mem->cc_shared)
1425 tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
1426 else
1427 tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
1428
1429 i = start = raw_smp_processor_id() & (pool->nareas - 1);
1430 do {
1431 index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1432 tbl_dma_addr, alloc_size,
1433 alloc_align_mask);
1434 if (index >= 0)
1435 return index;
1436 if (++i >= pool->nareas)
1437 i = 0;
1438 } while (i != start);
1439 return -1;
1440}
1441
1442#endif /* CONFIG_SWIOTLB_DYNAMIC */
1443
1444/**
1445 * mem_pool_used() - get number of used slots in a memory pool
1446 * @pool: Software IO TLB memory pool.
1447 *
1448 * The result is not accurate, see mem_used().
1449 *
1450 * Return: Approximate number of used slots.
1451 */
1452static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1453{
1454 int i;
1455 unsigned long used = 0;
1456
1457 for (i = 0; i < pool->nareas; i++)
1458 used += pool->areas[i].used;
1459 return used;
1460}
1461
1462/**
1463 * mem_used() - get number of used slots in an allocator
1464 * @mem: Software IO TLB allocator.
1465 *
1466 * When trace_hiwater and CONFIG_DEBUG_FS is enabled, the result is accurate
1467 * because the total number of used slots is tracked in mem->total_used.
1468 * Otherwise, the result is an approximation, because there is no locking of
1469 * individual areas.
1470 *
1471 * Return: Number of used slots.
1472 */
1473static unsigned long mem_used(struct io_tlb_mem *mem)
1474{
1475#ifdef CONFIG_DEBUG_FS
1476 if (track_hiwater_enabled)
1477 return atomic_long_read(&mem->total_used);
1478#endif
1479
1480#ifdef CONFIG_SWIOTLB_DYNAMIC
1481 struct io_tlb_pool *pool;
1482 unsigned long used = 0;
1483
1484 rcu_read_lock();
1485 list_for_each_entry_rcu(pool, &mem->pools, node)
1486 used += mem_pool_used(pool);
1487 rcu_read_unlock();
1488
1489 return used;
1490#else
1491 return mem_pool_used(&mem->defpool);
1492#endif
1493}
1494
1495/**
1496 * swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area
1497 * @dev: Device which maps the buffer.
1498 * @orig_addr: Original (non-bounced) physical IO buffer address
1499 * @mapping_size: Requested size of the actual bounce buffer, excluding
1500 * any pre- or post-padding for alignment
1501 * @alloc_align_mask: Required start and end alignment of the allocated buffer
1502 * @dir: DMA direction
1503 * @attrs: Optional DMA attributes for the map operation, updated
1504 * to match the selected SWIOTLB pool
1505 *
1506 * Find and allocate a suitable sequence of IO TLB slots for the request.
1507 * The device's SWIOTLB pool must match the device's current DMA encryption
1508 * requirements. If the device requires decrypted DMA, bouncing is done through
1509 * an unencrypted pool and the mapping is marked shared. If the device can DMA
1510 * to encrypted memory, bouncing is done through an encrypted pool even when the
1511 * original DMA address was unencrypted. Enabling encrypted DMA for a device is
1512 * therefore expected to update its default io_tlb_mem to an encrypted pool, so
1513 * later bounce mappings for both encrypted and decrypted original memory use
1514 * that encrypted pool.
1515 *
1516 * The allocated space starts at an alignment specified by alloc_align_mask,
1517 * and the size of the allocated space is rounded up so that the total amount
1518 * of allocated space is a multiple of (alloc_align_mask + 1). If
1519 * alloc_align_mask is zero, the allocated space may be at any alignment and
1520 * the size is not rounded up.
1521 *
1522 * The returned address is within the allocated space and matches the bits
1523 * of orig_addr that are specified in the DMA min_align_mask for the device. As
1524 * such, this returned address may be offset from the beginning of the allocated
1525 * space. The bounce buffer space starting at the returned address for
1526 * mapping_size bytes is initialized to the contents of the original IO buffer
1527 * area. Any pre-padding (due to an offset) and any post-padding (due to
1528 * rounding-up the size) is not initialized.
1529 */
1530phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1531 size_t mapping_size, unsigned int alloc_align_mask,
1532 enum dma_data_direction dir, unsigned long *attrs)
1533{
1534 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1535 unsigned int offset;
1536 struct io_tlb_pool *pool;
1537 unsigned int i;
1538 size_t size;
1539 int index;
1540 phys_addr_t tlb_addr;
1541 unsigned short pad_slots;
1542
1543 if (!mem || !mem->nslabs) {
1544 dev_warn_ratelimited(dev,
1545 "Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1546 return (phys_addr_t)DMA_MAPPING_ERROR;
1547 }
1548
1549 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
1550 pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1551
1552 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
1553
1554 /* swiotlb pool is incorrect for this device */
1555 if (unlikely(mem->cc_shared != force_dma_unencrypted(dev)))
1556 return (phys_addr_t)DMA_MAPPING_ERROR;
1557
1558 } else if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
1559 /*
1560 * On hosts with memory encryption, SWIOTLB-backed memory is
1561 * unencrypted. DMA addresses returned for bounce buffers must
1562 * therefore be marked unencrypted, even for devices that can
1563 * address encrypted memory. This also preserves swiotlb=force
1564 * behavior for those devices.
1565 */
1566 if (unlikely(!mem->cc_shared))
1567 return (phys_addr_t)DMA_MAPPING_ERROR;
1568 }
1569
1570 /* Force attrs to match the kind of memory in the pool */
1571 if (mem->cc_shared)
1572 *attrs |= DMA_ATTR_CC_SHARED;
1573 else
1574 *attrs &= ~DMA_ATTR_CC_SHARED;
1575
1576 /*
1577 * The default swiotlb memory pool is allocated with PAGE_SIZE
1578 * alignment. If a mapping is requested with larger alignment,
1579 * the mapping may be unable to use the initial slot(s) in all
1580 * sets of IO_TLB_SEGSIZE slots. In such case, a mapping request
1581 * of or near the maximum mapping size would always fail.
1582 */
1583 dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,
1584 "Alloc alignment may prevent fulfilling requests with max mapping_size\n");
1585
1586 offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
1587 size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
1588 index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
1589 if (index == -1) {
1590 if (!(*attrs & DMA_ATTR_NO_WARN))
1591 dev_warn_ratelimited(dev,
1592 "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1593 size, mem->nslabs, mem_used(mem));
1594 return (phys_addr_t)DMA_MAPPING_ERROR;
1595 }
1596
1597 /*
1598 * If dma_skip_sync was set, reset it on first SWIOTLB buffer
1599 * mapping to always sync SWIOTLB buffers.
1600 */
1601 dma_reset_need_sync(dev);
1602
1603 /*
1604 * Save away the mapping from the original address to the DMA address.
1605 * This is needed when we sync the memory. Then we sync the buffer if
1606 * needed.
1607 */
1608 pad_slots = offset >> IO_TLB_SHIFT;
1609 offset &= (IO_TLB_SIZE - 1);
1610 index += pad_slots;
1611 pool->slots[index].pad_slots = pad_slots;
1612 for (i = 0; i < (nr_slots(size) - pad_slots); i++)
1613 pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
1614 tlb_addr = slot_addr(pool->start, index) + offset;
1615 /*
1616 * When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1617 * the original buffer to the TLB buffer before initiating DMA in order
1618 * to preserve the original's data if the device does a partial write,
1619 * i.e. if the device doesn't overwrite the entire buffer. Preserving
1620 * the original data, even if it's garbage, is necessary to match
1621 * hardware behavior. Use of swiotlb is supposed to be transparent,
1622 * i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1623 */
1624 swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);
1625 return tlb_addr;
1626}
1627
1628static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1629 struct io_tlb_pool *mem)
1630{
1631 unsigned long flags;
1632 unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
1633 int index, nslots, aindex;
1634 struct io_tlb_area *area;
1635 int count, i;
1636
1637 index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1638 index -= mem->slots[index].pad_slots;
1639 nslots = nr_slots(mem->slots[index].alloc_size + offset);
1640 aindex = index / mem->area_nslabs;
1641 area = &mem->areas[aindex];
1642
1643 /*
1644 * Return the buffer to the free list by setting the corresponding
1645 * entries to indicate the number of contiguous entries available.
1646 * While returning the entries to the free list, we merge the entries
1647 * with slots below and above the pool being returned.
1648 */
1649 BUG_ON(aindex >= mem->nareas);
1650
1651 spin_lock_irqsave(&area->lock, flags);
1652 if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1653 count = mem->slots[index + nslots].list;
1654 else
1655 count = 0;
1656
1657 /*
1658 * Step 1: return the slots to the free list, merging the slots with
1659 * superceeding slots
1660 */
1661 for (i = index + nslots - 1; i >= index; i--) {
1662 mem->slots[i].list = ++count;
1663 mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1664 mem->slots[i].alloc_size = 0;
1665 mem->slots[i].pad_slots = 0;
1666 }
1667
1668 /*
1669 * Step 2: merge the returned slots with the preceding slots, if
1670 * available (non zero)
1671 */
1672 for (i = index - 1;
1673 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1674 i--)
1675 mem->slots[i].list = ++count;
1676 area->used -= nslots;
1677 spin_unlock_irqrestore(&area->lock, flags);
1678
1679 dec_used(dev->dma_io_tlb_mem, nslots);
1680}
1681
1682#ifdef CONFIG_SWIOTLB_DYNAMIC
1683
1684/**
1685 * swiotlb_del_transient() - delete a transient memory pool
1686 * @dev: Device which mapped the buffer.
1687 * @tlb_addr: Physical address within a bounce buffer.
1688 * @pool: Pointer to the transient memory pool to be checked and deleted.
1689 *
1690 * Check whether the address belongs to a transient SWIOTLB memory pool.
1691 * If yes, then delete the pool.
1692 *
1693 * Return: %true if @tlb_addr belonged to a transient pool that was released.
1694 */
1695static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1696 struct io_tlb_pool *pool)
1697{
1698 if (!pool->transient)
1699 return false;
1700
1701 dec_used(dev->dma_io_tlb_mem, pool->nslabs);
1702 swiotlb_del_pool(dev, pool);
1703 dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);
1704 return true;
1705}
1706
1707#else /* !CONFIG_SWIOTLB_DYNAMIC */
1708
1709static inline bool swiotlb_del_transient(struct device *dev,
1710 phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1711{
1712 return false;
1713}
1714
1715#endif /* CONFIG_SWIOTLB_DYNAMIC */
1716
1717/*
1718 * tlb_addr is the physical address of the bounce buffer to unmap.
1719 */
1720void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1721 size_t mapping_size, enum dma_data_direction dir,
1722 unsigned long attrs, struct io_tlb_pool *pool)
1723{
1724 /*
1725 * First, sync the memory before unmapping the entry
1726 */
1727 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1728 (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1729 swiotlb_bounce(dev, tlb_addr, mapping_size,
1730 DMA_FROM_DEVICE, pool);
1731
1732 if (swiotlb_del_transient(dev, tlb_addr, pool))
1733 return;
1734 swiotlb_release_slots(dev, tlb_addr, pool);
1735}
1736
1737void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1738 size_t size, enum dma_data_direction dir,
1739 struct io_tlb_pool *pool)
1740{
1741 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1742 swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);
1743 else
1744 BUG_ON(dir != DMA_FROM_DEVICE);
1745}
1746
1747void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1748 size_t size, enum dma_data_direction dir,
1749 struct io_tlb_pool *pool)
1750{
1751 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1752 swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);
1753 else
1754 BUG_ON(dir != DMA_TO_DEVICE);
1755}
1756
1757/*
1758 * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1759 * to the device copy the data into it as well.
1760 */
1761dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1762 enum dma_data_direction dir, unsigned long attrs)
1763{
1764 phys_addr_t swiotlb_addr;
1765 dma_addr_t dma_addr;
1766
1767 trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
1768
1769 swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs);
1770 if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1771 return DMA_MAPPING_ERROR;
1772
1773 if (attrs & DMA_ATTR_CC_SHARED)
1774 dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
1775 else
1776 dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
1777
1778 if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs))) {
1779 __swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
1780 attrs | DMA_ATTR_SKIP_CPU_SYNC,
1781 swiotlb_find_pool(dev, swiotlb_addr));
1782 dev_WARN_ONCE(dev, 1,
1783 "swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1784 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1785 return DMA_MAPPING_ERROR;
1786 }
1787
1788 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
1789 arch_sync_dma_for_device(swiotlb_addr, size, dir);
1790 arch_sync_dma_flush();
1791 }
1792 return dma_addr;
1793}
1794
1795size_t swiotlb_max_mapping_size(struct device *dev)
1796{
1797 int min_align_mask = dma_get_min_align_mask(dev);
1798 int min_align = 0;
1799
1800 /*
1801 * swiotlb_find_slots() skips slots according to
1802 * min align mask. This affects max mapping size.
1803 * Take it into acount here.
1804 */
1805 if (min_align_mask)
1806 min_align = roundup(min_align_mask, IO_TLB_SIZE);
1807
1808 return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1809}
1810
1811/**
1812 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1813 */
1814bool is_swiotlb_allocated(void)
1815{
1816 return io_tlb_default_mem.nslabs;
1817}
1818
1819bool is_swiotlb_active(struct device *dev)
1820{
1821 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1822
1823 return mem && mem->nslabs;
1824}
1825
1826/**
1827 * default_swiotlb_base() - get the base address of the default SWIOTLB
1828 *
1829 * Get the lowest physical address used by the default software IO TLB pool.
1830 */
1831phys_addr_t default_swiotlb_base(void)
1832{
1833#ifdef CONFIG_SWIOTLB_DYNAMIC
1834 io_tlb_default_mem.can_grow = false;
1835#endif
1836 return io_tlb_default_mem.defpool.start;
1837}
1838
1839/**
1840 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1841 *
1842 * Get the highest physical address used by the default software IO TLB pool.
1843 */
1844phys_addr_t default_swiotlb_limit(void)
1845{
1846#ifdef CONFIG_SWIOTLB_DYNAMIC
1847 if (io_tlb_default_mem.can_grow)
1848 return io_tlb_default_mem.phys_limit;
1849#endif
1850 return io_tlb_default_mem.defpool.end - 1;
1851}
1852
1853#ifdef CONFIG_DEBUG_FS
1854#ifdef CONFIG_SWIOTLB_DYNAMIC
1855static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1856{
1857 return atomic_long_read(&mem->transient_nslabs);
1858}
1859
1860static int io_tlb_transient_used_get(void *data, u64 *val)
1861{
1862 struct io_tlb_mem *mem = data;
1863
1864 *val = mem_transient_used(mem);
1865 return 0;
1866}
1867
1868DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1869 NULL, "%llu\n");
1870#endif /* CONFIG_SWIOTLB_DYNAMIC */
1871
1872static int io_tlb_used_get(void *data, u64 *val)
1873{
1874 struct io_tlb_mem *mem = data;
1875
1876 *val = mem_used(mem);
1877 return 0;
1878}
1879
1880static int io_tlb_hiwater_get(void *data, u64 *val)
1881{
1882 struct io_tlb_mem *mem = data;
1883
1884 *val = atomic_long_read(&mem->used_hiwater);
1885 return 0;
1886}
1887
1888static int io_tlb_hiwater_set(void *data, u64 val)
1889{
1890 struct io_tlb_mem *mem = data;
1891
1892 /* Only allow setting to zero */
1893 if (val != 0)
1894 return -EINVAL;
1895
1896 atomic_long_set(&mem->used_hiwater, val);
1897 return 0;
1898}
1899
1900DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1901DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1902 io_tlb_hiwater_set, "%llu\n");
1903
1904static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1905 const char *dirname)
1906{
1907 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
1908 if (!mem->nslabs)
1909 return;
1910
1911 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
1912 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1913 &fops_io_tlb_used);
1914 debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1915 &fops_io_tlb_hiwater);
1916#ifdef CONFIG_SWIOTLB_DYNAMIC
1917 debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1918 mem, &fops_io_tlb_transient_used);
1919#endif
1920}
1921
1922static int __init swiotlb_create_default_debugfs(void)
1923{
1924 swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
1925 return 0;
1926}
1927
1928late_initcall(swiotlb_create_default_debugfs);
1929
1930#else /* !CONFIG_DEBUG_FS */
1931
1932static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1933 const char *dirname)
1934{
1935}
1936
1937#endif /* CONFIG_DEBUG_FS */
1938
1939#ifdef CONFIG_DMA_RESTRICTED_POOL
1940
1941struct page *swiotlb_alloc(struct device *dev, size_t size, unsigned long attrs)
1942{
1943 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1944 struct io_tlb_pool *pool;
1945 phys_addr_t tlb_addr;
1946 unsigned int align;
1947 int index;
1948
1949 if (!mem)
1950 return NULL;
1951
1952 if (mem->cc_shared != !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED))
1953 return NULL;
1954
1955 align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1956 index = swiotlb_find_slots(dev, 0, size, align, &pool);
1957 if (index == -1)
1958 return NULL;
1959
1960 tlb_addr = slot_addr(pool->start, index);
1961 if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1962 dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1963 &tlb_addr);
1964 swiotlb_release_slots(dev, tlb_addr, pool);
1965 return NULL;
1966 }
1967
1968 return pfn_to_page(PFN_DOWN(tlb_addr));
1969}
1970
1971bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1972{
1973 phys_addr_t tlb_addr = page_to_phys(page);
1974 struct io_tlb_pool *pool;
1975
1976 pool = swiotlb_find_pool(dev, tlb_addr);
1977 if (!pool)
1978 return false;
1979
1980 swiotlb_release_slots(dev, tlb_addr, pool);
1981
1982 return true;
1983}
1984
1985void swiotlb_free_from_pool(struct device *dev,
1986 phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1987{
1988 swiotlb_release_slots(dev, tlb_addr, pool);
1989}
1990
1991static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1992 struct device *dev)
1993{
1994 struct io_tlb_mem *mem = rmem->priv;
1995 unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1996
1997 /* Set Per-device io tlb area to one */
1998 unsigned int nareas = 1;
1999
2000 if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
2001 dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
2002 return -EINVAL;
2003 }
2004
2005 /*
2006 * Since multiple devices can share the same pool, the private data,
2007 * io_tlb_mem struct, will be initialized by the first device attached
2008 * to it.
2009 */
2010 if (!mem) {
2011 struct io_tlb_pool *pool;
2012
2013 mem = kzalloc_obj(*mem);
2014 if (!mem)
2015 return -ENOMEM;
2016 pool = &mem->defpool;
2017
2018 pool->slots = kzalloc_objs(*pool->slots, nslabs);
2019 if (!pool->slots) {
2020 kfree(mem);
2021 return -ENOMEM;
2022 }
2023
2024 pool->areas = kzalloc_objs(*pool->areas, nareas);
2025 if (!pool->areas) {
2026 kfree(pool->slots);
2027 kfree(mem);
2028 return -ENOMEM;
2029 }
2030 /*
2031 * if platform supports memory encryption,
2032 * restricted mem pool is shared by default
2033 */
2034 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
2035 int ret;
2036
2037 mem->cc_shared = true;
2038 ret = set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
2039 rmem->size >> PAGE_SHIFT);
2040 if (ret) {
2041 dev_err(dev, "Failed to decrypt restricted DMA pool\n");
2042 kfree(pool->areas);
2043 kfree(pool->slots);
2044 kfree(mem);
2045 return ret;
2046 }
2047 } else {
2048 mem->cc_shared = false;
2049 }
2050
2051 swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base),
2052 nslabs, false, nareas);
2053 mem->force_bounce = true;
2054 mem->for_alloc = true;
2055#ifdef CONFIG_SWIOTLB_DYNAMIC
2056 spin_lock_init(&mem->lock);
2057 INIT_LIST_HEAD_RCU(&mem->pools);
2058#endif
2059 add_mem_pool(mem, pool);
2060
2061 rmem->priv = mem;
2062
2063 swiotlb_create_debugfs_files(mem, rmem->name);
2064 }
2065
2066 dev->dma_io_tlb_mem = mem;
2067
2068 return 0;
2069}
2070
2071static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
2072 struct device *dev)
2073{
2074 dev->dma_io_tlb_mem = &io_tlb_default_mem;
2075}
2076
2077static int __init rmem_swiotlb_setup(unsigned long node,
2078 struct reserved_mem *rmem)
2079{
2080 if (of_get_flat_dt_prop(node, "reusable", NULL) ||
2081 of_get_flat_dt_prop(node, "linux,cma-default", NULL) ||
2082 of_get_flat_dt_prop(node, "linux,dma-default", NULL) ||
2083 of_get_flat_dt_prop(node, "no-map", NULL))
2084 return -EINVAL;
2085
2086 pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
2087 &rmem->base, (unsigned long)rmem->size / SZ_1M);
2088 return 0;
2089}
2090
2091static const struct reserved_mem_ops rmem_swiotlb_ops = {
2092 .node_init = rmem_swiotlb_setup,
2093 .device_init = rmem_swiotlb_device_init,
2094 .device_release = rmem_swiotlb_device_release,
2095};
2096
2097RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", &rmem_swiotlb_ops);
2098#endif /* CONFIG_DMA_RESTRICTED_POOL */