projects/linux·blob master · refs

1670 lines · 44.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2008 Advanced Micro Devices, Inc.
4 *
5 * Author: Joerg Roedel <[email protected]>
6 */
7
8#define pr_fmt(fmt) "DMA-API: " fmt
9
10#include <linux/sched/task_stack.h>
11#include <linux/scatterlist.h>
12#include <linux/dma-map-ops.h>
13#include <linux/sched/task.h>
14#include <linux/stacktrace.h>
15#include <linux/spinlock.h>
16#include <linux/vmalloc.h>
17#include <linux/debugfs.h>
18#include <linux/uaccess.h>
19#include <linux/export.h>
20#include <linux/device.h>
21#include <linux/types.h>
22#include <linux/sched.h>
23#include <linux/ctype.h>
24#include <linux/list.h>
25#include <linux/slab.h>
26#include <linux/swiotlb.h>
27#include <asm/sections.h>
28#include "debug.h"
29
30#define HASH_SIZE 16384ULL
31#define HASH_FN_SHIFT 13
32#define HASH_FN_MASK (HASH_SIZE - 1)
33
34#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
35/* If the pool runs out, add this many new entries at once */
36#define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry))
37
38enum {
39 dma_debug_single,
40 dma_debug_sg,
41 dma_debug_coherent,
42 dma_debug_noncoherent,
43 dma_debug_phy,
44};
45
46enum map_err_types {
47 MAP_ERR_CHECK_NOT_APPLICABLE,
48 MAP_ERR_NOT_CHECKED,
49 MAP_ERR_CHECKED,
50};
51
52#define DMA_DEBUG_STACKTRACE_ENTRIES 5
53
54/**
55 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
56 * @list: node on pre-allocated free_entries list
57 * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
58 * @dev_addr: dma address
59 * @size: length of the mapping
60 * @type: single, page, sg, coherent
61 * @direction: enum dma_data_direction
62 * @sg_call_ents: 'nents' from dma_map_sg
63 * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
64 * @paddr: physical start address of the mapping
65 * @map_err_type: track whether dma_mapping_error() was checked
66 * @attrs: dma attributes
67 * @stack_len: number of backtrace entries in @stack_entries
68 * @stack_entries: stack of backtrace history
69 */
70struct dma_debug_entry {
71 struct list_head list;
72 struct device *dev;
73 u64 dev_addr;
74 u64 size;
75 int type;
76 int direction;
77 int sg_call_ents;
78 int sg_mapped_ents;
79 phys_addr_t paddr;
80 enum map_err_types map_err_type;
81 unsigned long attrs;
82#ifdef CONFIG_STACKTRACE
83 unsigned int stack_len;
84 unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
85#endif
86} ____cacheline_aligned_in_smp;
87
88typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
89
90struct hash_bucket {
91 struct list_head list;
92 spinlock_t lock;
93};
94
95/* Hash list to save the allocated dma addresses */
96static struct hash_bucket dma_entry_hash[HASH_SIZE];
97/* List of pre-allocated dma_debug_entry's */
98static LIST_HEAD(free_entries);
99/* Lock for the list above */
100static DEFINE_SPINLOCK(free_entries_lock);
101
102/* Global disable flag - will be set in case of an error */
103static bool global_disable __read_mostly;
104
105/* Early initialization disable flag, set at the end of dma_debug_init */
106static bool dma_debug_initialized __read_mostly;
107
108static inline bool dma_debug_disabled(void)
109{
110 return global_disable || !dma_debug_initialized;
111}
112
113/* Global error count */
114static u32 error_count;
115
116/* Global error show enable*/
117static u32 show_all_errors __read_mostly;
118/* Number of errors to show */
119static u32 show_num_errors = 1;
120
121static u32 num_free_entries;
122static u32 min_free_entries;
123static u32 nr_total_entries;
124
125/* number of preallocated entries requested by kernel cmdline */
126static u32 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
127
128/* per-driver filter related state */
129
130#define NAME_MAX_LEN 64
131
132static char current_driver_name[NAME_MAX_LEN] __read_mostly;
133static struct device_driver *current_driver __read_mostly;
134
135static DEFINE_RWLOCK(driver_name_lock);
136
137static const char *const maperr2str[] = {
138 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
139 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
140 [MAP_ERR_CHECKED] = "dma map error checked",
141};
142
143static const char *type2name[] = {
144 [dma_debug_single] = "single",
145 [dma_debug_sg] = "scatter-gather",
146 [dma_debug_coherent] = "coherent",
147 [dma_debug_noncoherent] = "noncoherent",
148 [dma_debug_phy] = "phy",
149};
150
151static const char *dir2name[] = {
152 [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL",
153 [DMA_TO_DEVICE] = "DMA_TO_DEVICE",
154 [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE",
155 [DMA_NONE] = "DMA_NONE",
156};
157
158/*
159 * The access to some variables in this macro is racy. We can't use atomic_t
160 * here because all these variables are exported to debugfs. Some of them even
161 * writeable. This is also the reason why a lock won't help much. But anyway,
162 * the races are no big deal. Here is why:
163 *
164 * error_count: the addition is racy, but the worst thing that can happen is
165 * that we don't count some errors
166 * show_num_errors: the subtraction is racy. Also no big deal because in
167 * worst case this will result in one warning more in the
168 * system log than the user configured. This variable is
169 * writeable via debugfs.
170 */
171static inline void dump_entry_trace(struct dma_debug_entry *entry)
172{
173#ifdef CONFIG_STACKTRACE
174 if (entry) {
175 pr_warn("Mapped at:\n");
176 stack_trace_print(entry->stack_entries, entry->stack_len, 0);
177 }
178#endif
179}
180
181static bool driver_filter(struct device *dev)
182{
183 struct device_driver *drv;
184 unsigned long flags;
185 bool ret;
186
187 /* driver filter off */
188 if (likely(!current_driver_name[0]))
189 return true;
190
191 /* driver filter on and initialized */
192 if (current_driver && dev && dev->driver == current_driver)
193 return true;
194
195 /* driver filter on, but we can't filter on a NULL device... */
196 if (!dev)
197 return false;
198
199 if (current_driver || !current_driver_name[0])
200 return false;
201
202 /* driver filter on but not yet initialized */
203 drv = dev->driver;
204 if (!drv)
205 return false;
206
207 /* lock to protect against change of current_driver_name */
208 read_lock_irqsave(&driver_name_lock, flags);
209
210 ret = false;
211 if (drv->name &&
212 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
213 current_driver = drv;
214 ret = true;
215 }
216
217 read_unlock_irqrestore(&driver_name_lock, flags);
218
219 return ret;
220}
221
222#define err_printk(dev, entry, format, arg...) do { \
223 error_count += 1; \
224 if (driver_filter(dev) && \
225 (show_all_errors || show_num_errors > 0)) { \
226 WARN(1, pr_fmt("%s %s: ") format, \
227 dev ? dev_driver_string(dev) : "NULL", \
228 dev ? dev_name(dev) : "NULL", ## arg); \
229 dump_entry_trace(entry); \
230 } \
231 if (!show_all_errors && show_num_errors > 0) \
232 show_num_errors -= 1; \
233 } while (0);
234
235/*
236 * Hash related functions
237 *
238 * Every DMA-API request is saved into a struct dma_debug_entry. To
239 * have quick access to these structs they are stored into a hash.
240 */
241static int hash_fn(struct dma_debug_entry *entry)
242{
243 /*
244 * Hash function is based on the dma address.
245 * We use bits 20-27 here as the index into the hash
246 */
247 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
248}
249
250/*
251 * Request exclusive access to a hash bucket for a given dma_debug_entry.
252 */
253static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
254 unsigned long *flags)
255 __acquires(&dma_entry_hash[idx].lock)
256{
257 int idx = hash_fn(entry);
258 unsigned long __flags;
259
260 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
261 *flags = __flags;
262 return &dma_entry_hash[idx];
263}
264
265/*
266 * Give up exclusive access to the hash bucket
267 */
268static void put_hash_bucket(struct hash_bucket *bucket,
269 unsigned long flags)
270 __releases(&bucket->lock)
271{
272 spin_unlock_irqrestore(&bucket->lock, flags);
273}
274
275static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
276{
277 return ((a->dev_addr == b->dev_addr) &&
278 (a->dev == b->dev)) ? true : false;
279}
280
281static bool containing_match(struct dma_debug_entry *a,
282 struct dma_debug_entry *b)
283{
284 if (a->dev != b->dev)
285 return false;
286
287 if ((b->dev_addr <= a->dev_addr) &&
288 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
289 return true;
290
291 return false;
292}
293
294/*
295 * Search a given entry in the hash bucket list
296 */
297static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
298 struct dma_debug_entry *ref,
299 match_fn match)
300{
301 struct dma_debug_entry *entry, *ret = NULL;
302 int matches = 0, match_lvl, last_lvl = -1;
303
304 list_for_each_entry(entry, &bucket->list, list) {
305 if (!match(ref, entry))
306 continue;
307
308 /*
309 * Some drivers map the same physical address multiple
310 * times. Without a hardware IOMMU this results in the
311 * same device addresses being put into the dma-debug
312 * hash multiple times too. This can result in false
313 * positives being reported. Therefore we implement a
314 * best-fit algorithm here which returns the entry from
315 * the hash which fits best to the reference value
316 * instead of the first-fit.
317 */
318 matches += 1;
319 match_lvl = 0;
320 entry->size == ref->size ? ++match_lvl : 0;
321 entry->type == ref->type ? ++match_lvl : 0;
322 entry->direction == ref->direction ? ++match_lvl : 0;
323 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
324
325 if (match_lvl == 4) {
326 /* perfect-fit - return the result */
327 return entry;
328 } else if (match_lvl > last_lvl) {
329 /*
330 * We found an entry that fits better then the
331 * previous one or it is the 1st match.
332 */
333 last_lvl = match_lvl;
334 ret = entry;
335 }
336 }
337
338 /*
339 * If we have multiple matches but no perfect-fit, just return
340 * NULL.
341 */
342 ret = (matches == 1) ? ret : NULL;
343
344 return ret;
345}
346
347static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
348 struct dma_debug_entry *ref)
349{
350 return __hash_bucket_find(bucket, ref, exact_match);
351}
352
353static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
354 struct dma_debug_entry *ref,
355 unsigned long *flags)
356{
357
358 struct dma_debug_entry *entry, index = *ref;
359 int limit = min(HASH_SIZE, (index.dev_addr >> HASH_FN_SHIFT) + 1);
360
361 for (int i = 0; i < limit; i++) {
362 entry = __hash_bucket_find(*bucket, ref, containing_match);
363
364 if (entry)
365 return entry;
366
367 /*
368 * Nothing found, go back a hash bucket
369 */
370 put_hash_bucket(*bucket, *flags);
371 index.dev_addr -= (1 << HASH_FN_SHIFT);
372 *bucket = get_hash_bucket(&index, flags);
373 }
374
375 return NULL;
376}
377
378/*
379 * Add an entry to a hash bucket
380 */
381static void hash_bucket_add(struct hash_bucket *bucket,
382 struct dma_debug_entry *entry)
383{
384 list_add_tail(&entry->list, &bucket->list);
385}
386
387/*
388 * Remove entry from a hash bucket list
389 */
390static void hash_bucket_del(struct dma_debug_entry *entry)
391{
392 list_del(&entry->list);
393}
394
395/*
396 * For each mapping (initial cacheline in the case of
397 * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
398 * scatterlist, or the cacheline specified in dma_map_single) insert
399 * into this tree using the cacheline as the key. At
400 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
401 * the entry already exists at insertion time add a tag as a reference
402 * count for the overlapping mappings. For now, the overlap tracking
403 * just ensures that 'unmaps' balance 'maps' before marking the
404 * cacheline idle, but we should also be flagging overlaps as an API
405 * violation.
406 *
407 * Memory usage is mostly constrained by the maximum number of available
408 * dma-debug entries in that we need a free dma_debug_entry before
409 * inserting into the tree. In the case of dma_map_page and
410 * dma_alloc_coherent there is only one dma_debug_entry and one
411 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
412 * other hand, consumes a single dma_debug_entry, but inserts 'nents'
413 * entries into the tree.
414 *
415 * Use __GFP_NOWARN because the printk from an OOM, to netconsole, could end
416 * up right back in the DMA debugging code, leading to a deadlock.
417 */
418static RADIX_TREE(dma_active_cacheline, GFP_ATOMIC | __GFP_NOWARN);
419static DEFINE_SPINLOCK(radix_lock);
420#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
421#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
422#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
423
424static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
425{
426 return ((entry->paddr >> PAGE_SHIFT) << CACHELINE_PER_PAGE_SHIFT) +
427 (offset_in_page(entry->paddr) >> L1_CACHE_SHIFT);
428}
429
430static int active_cacheline_read_overlap(phys_addr_t cln)
431{
432 int overlap = 0, i;
433
434 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
435 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
436 overlap |= 1 << i;
437 return overlap;
438}
439
440static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
441{
442 int i;
443
444 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
445 return overlap;
446
447 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
448 if (overlap & 1 << i)
449 radix_tree_tag_set(&dma_active_cacheline, cln, i);
450 else
451 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
452
453 return overlap;
454}
455
456static void active_cacheline_inc_overlap(phys_addr_t cln, bool is_cache_clean)
457{
458 int overlap = active_cacheline_read_overlap(cln);
459
460 overlap = active_cacheline_set_overlap(cln, ++overlap);
461
462 /* If we overflowed the overlap counter then we're potentially
463 * leaking dma-mappings.
464 */
465 WARN_ONCE(!is_cache_clean && overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
466 pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
467 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
468}
469
470static int active_cacheline_dec_overlap(phys_addr_t cln)
471{
472 int overlap = active_cacheline_read_overlap(cln);
473
474 return active_cacheline_set_overlap(cln, --overlap);
475}
476
477static int active_cacheline_insert(struct dma_debug_entry *entry,
478 bool *overlap_cache_clean)
479{
480 phys_addr_t cln = to_cacheline_number(entry);
481 bool is_cache_clean = entry->attrs &
482 (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
483 DMA_ATTR_REQUIRE_COHERENT);
484 unsigned long flags;
485 int rc;
486
487 *overlap_cache_clean = false;
488
489 /* If the device is not writing memory then we don't have any
490 * concerns about the cpu consuming stale data. This mitigates
491 * legitimate usages of overlapping mappings.
492 */
493 if (entry->direction == DMA_TO_DEVICE)
494 return 0;
495
496 spin_lock_irqsave(&radix_lock, flags);
497 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
498 if (rc == -EEXIST) {
499 struct dma_debug_entry *existing;
500
501 active_cacheline_inc_overlap(cln, is_cache_clean);
502 existing = radix_tree_lookup(&dma_active_cacheline, cln);
503 /* A lookup failure here after we got -EEXIST is unexpected. */
504 WARN_ON(!existing);
505 if (existing)
506 *overlap_cache_clean =
507 existing->attrs &
508 (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
509 DMA_ATTR_REQUIRE_COHERENT);
510 }
511 spin_unlock_irqrestore(&radix_lock, flags);
512
513 return rc;
514}
515
516static void active_cacheline_remove(struct dma_debug_entry *entry)
517{
518 phys_addr_t cln = to_cacheline_number(entry);
519 unsigned long flags;
520
521 /* ...mirror the insert case */
522 if (entry->direction == DMA_TO_DEVICE)
523 return;
524
525 spin_lock_irqsave(&radix_lock, flags);
526 /* since we are counting overlaps the final put of the
527 * cacheline will occur when the overlap count is 0.
528 * active_cacheline_dec_overlap() returns -1 in that case
529 */
530 if (active_cacheline_dec_overlap(cln) < 0)
531 radix_tree_delete(&dma_active_cacheline, cln);
532 spin_unlock_irqrestore(&radix_lock, flags);
533}
534
535/*
536 * Dump mappings entries on kernel space for debugging purposes
537 */
538void debug_dma_dump_mappings(struct device *dev)
539{
540 int idx;
541 phys_addr_t cln;
542
543 for (idx = 0; idx < HASH_SIZE; idx++) {
544 struct hash_bucket *bucket = &dma_entry_hash[idx];
545 struct dma_debug_entry *entry;
546 unsigned long flags;
547
548 spin_lock_irqsave(&bucket->lock, flags);
549 list_for_each_entry(entry, &bucket->list, list) {
550 if (!dev || dev == entry->dev) {
551 cln = to_cacheline_number(entry);
552 dev_info(entry->dev,
553 "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",
554 type2name[entry->type], idx,
555 &entry->paddr, entry->dev_addr,
556 entry->size, &cln,
557 dir2name[entry->direction],
558 maperr2str[entry->map_err_type],
559 entry->attrs);
560 }
561 }
562 spin_unlock_irqrestore(&bucket->lock, flags);
563
564 cond_resched();
565 }
566}
567
568/*
569 * Dump mappings entries on user space via debugfs
570 */
571static int dump_show(struct seq_file *seq, void *v)
572{
573 int idx;
574 phys_addr_t cln;
575
576 for (idx = 0; idx < HASH_SIZE; idx++) {
577 struct hash_bucket *bucket = &dma_entry_hash[idx];
578 struct dma_debug_entry *entry;
579 unsigned long flags;
580
581 spin_lock_irqsave(&bucket->lock, flags);
582 list_for_each_entry(entry, &bucket->list, list) {
583 cln = to_cacheline_number(entry);
584 seq_printf(seq,
585 "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",
586 dev_driver_string(entry->dev),
587 dev_name(entry->dev),
588 type2name[entry->type], idx,
589 &entry->paddr, entry->dev_addr,
590 entry->size, &cln,
591 dir2name[entry->direction],
592 maperr2str[entry->map_err_type],
593 entry->attrs);
594 }
595 spin_unlock_irqrestore(&bucket->lock, flags);
596 }
597 return 0;
598}
599DEFINE_SHOW_ATTRIBUTE(dump);
600
601/*
602 * Wrapper function for adding an entry to the hash.
603 * This function takes care of locking itself.
604 */
605static void add_dma_entry(struct dma_debug_entry *entry)
606{
607 unsigned long attrs = entry->attrs;
608 bool overlap_cache_clean;
609 struct hash_bucket *bucket;
610 unsigned long flags;
611 int rc;
612
613 bucket = get_hash_bucket(entry, &flags);
614 hash_bucket_add(bucket, entry);
615 put_hash_bucket(bucket, flags);
616
617 rc = active_cacheline_insert(entry, &overlap_cache_clean);
618 if (rc == -ENOMEM) {
619 pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n");
620 global_disable = true;
621 } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
622 !(attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
623 DMA_ATTR_REQUIRE_COHERENT) &&
624 overlap_cache_clean) &&
625 dma_get_cache_alignment() >= L1_CACHE_BYTES &&
626 !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&
627 is_swiotlb_active(entry->dev))) {
628 err_printk(entry->dev, entry,
629 "cacheline tracking EEXIST, overlapping mappings aren't supported\n");
630 }
631}
632
633static int dma_debug_create_entries(gfp_t gfp)
634{
635 struct dma_debug_entry *entry;
636 int i;
637
638 entry = (void *)get_zeroed_page(gfp);
639 if (!entry)
640 return -ENOMEM;
641
642 for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)
643 list_add_tail(&entry[i].list, &free_entries);
644
645 num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
646 nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
647
648 return 0;
649}
650
651static struct dma_debug_entry *__dma_entry_alloc(void)
652{
653 struct dma_debug_entry *entry;
654
655 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
656 list_del(&entry->list);
657 memset(entry, 0, sizeof(*entry));
658
659 num_free_entries -= 1;
660 if (num_free_entries < min_free_entries)
661 min_free_entries = num_free_entries;
662
663 return entry;
664}
665
666/*
667 * This should be called outside of free_entries_lock scope to avoid potential
668 * deadlocks with serial consoles that use DMA.
669 */
670static void __dma_entry_alloc_check_leak(u32 nr_entries)
671{
672 u32 tmp = nr_entries % nr_prealloc_entries;
673
674 /* Shout each time we tick over some multiple of the initial pool */
675 if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {
676 pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",
677 nr_entries,
678 (nr_entries / nr_prealloc_entries));
679 }
680}
681
682/* struct dma_entry allocator
683 *
684 * The next two functions implement the allocator for
685 * struct dma_debug_entries.
686 */
687static struct dma_debug_entry *dma_entry_alloc(void)
688{
689 bool alloc_check_leak = false;
690 struct dma_debug_entry *entry;
691 unsigned long flags;
692 u32 nr_entries;
693
694 spin_lock_irqsave(&free_entries_lock, flags);
695 if (num_free_entries == 0) {
696 if (dma_debug_create_entries(GFP_ATOMIC)) {
697 global_disable = true;
698 spin_unlock_irqrestore(&free_entries_lock, flags);
699 pr_err("debugging out of memory - disabling\n");
700 return NULL;
701 }
702 alloc_check_leak = true;
703 nr_entries = nr_total_entries;
704 }
705
706 entry = __dma_entry_alloc();
707
708 spin_unlock_irqrestore(&free_entries_lock, flags);
709
710 if (alloc_check_leak)
711 __dma_entry_alloc_check_leak(nr_entries);
712
713#ifdef CONFIG_STACKTRACE
714 entry->stack_len = stack_trace_save(entry->stack_entries,
715 ARRAY_SIZE(entry->stack_entries),
716 1);
717#endif
718 return entry;
719}
720
721static void dma_entry_free(struct dma_debug_entry *entry)
722{
723 unsigned long flags;
724
725 active_cacheline_remove(entry);
726
727 /*
728 * add to beginning of the list - this way the entries are
729 * more likely cache hot when they are reallocated.
730 */
731 spin_lock_irqsave(&free_entries_lock, flags);
732 list_add(&entry->list, &free_entries);
733 num_free_entries += 1;
734 spin_unlock_irqrestore(&free_entries_lock, flags);
735}
736
737/*
738 * DMA-API debugging init code
739 *
740 * The init code does two things:
741 * 1. Initialize core data structures
742 * 2. Preallocate a given number of dma_debug_entry structs
743 */
744
745static ssize_t filter_read(struct file *file, char __user *user_buf,
746 size_t count, loff_t *ppos)
747{
748 char buf[NAME_MAX_LEN + 1];
749 unsigned long flags;
750 int len;
751
752 if (!current_driver_name[0])
753 return 0;
754
755 /*
756 * We can't copy to userspace directly because current_driver_name can
757 * only be read under the driver_name_lock with irqs disabled. So
758 * create a temporary copy first.
759 */
760 read_lock_irqsave(&driver_name_lock, flags);
761 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
762 read_unlock_irqrestore(&driver_name_lock, flags);
763
764 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
765}
766
767static ssize_t filter_write(struct file *file, const char __user *userbuf,
768 size_t count, loff_t *ppos)
769{
770 char buf[NAME_MAX_LEN];
771 unsigned long flags;
772 size_t len;
773 int i;
774
775 /*
776 * We can't copy from userspace directly. Access to
777 * current_driver_name is protected with a write_lock with irqs
778 * disabled. Since copy_from_user can fault and may sleep we
779 * need to copy to temporary buffer first
780 */
781 len = min(count, (size_t)(NAME_MAX_LEN - 1));
782 if (copy_from_user(buf, userbuf, len))
783 return -EFAULT;
784
785 buf[len] = 0;
786
787 write_lock_irqsave(&driver_name_lock, flags);
788
789 /*
790 * Now handle the string we got from userspace very carefully.
791 * The rules are:
792 * - only use the first token we got
793 * - token delimiter is everything looking like a space
794 * character (' ', '\n', '\t' ...)
795 *
796 */
797 if (!isalnum(buf[0])) {
798 /*
799 * If the first character userspace gave us is not
800 * alphanumerical then assume the filter should be
801 * switched off.
802 */
803 if (current_driver_name[0])
804 pr_info("switching off dma-debug driver filter\n");
805 current_driver_name[0] = 0;
806 current_driver = NULL;
807 goto out_unlock;
808 }
809
810 /*
811 * Now parse out the first token and use it as the name for the
812 * driver to filter for.
813 */
814 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
815 current_driver_name[i] = buf[i];
816 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
817 break;
818 }
819 current_driver_name[i] = 0;
820 current_driver = NULL;
821
822 pr_info("enable driver filter for driver [%s]\n",
823 current_driver_name);
824
825out_unlock:
826 write_unlock_irqrestore(&driver_name_lock, flags);
827
828 return count;
829}
830
831static const struct file_operations filter_fops = {
832 .read = filter_read,
833 .write = filter_write,
834 .llseek = default_llseek,
835};
836
837static int __init dma_debug_fs_init(void)
838{
839 struct dentry *dentry = debugfs_create_dir("dma-api", NULL);
840
841 debugfs_create_bool("disabled", 0444, dentry, &global_disable);
842 debugfs_create_u32("error_count", 0444, dentry, &error_count);
843 debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors);
844 debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors);
845 debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries);
846 debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries);
847 debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);
848 debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);
849 debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops);
850
851 return 0;
852}
853core_initcall_sync(dma_debug_fs_init);
854
855static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
856{
857 struct dma_debug_entry *entry;
858 unsigned long flags;
859 int count = 0, i;
860
861 for (i = 0; i < HASH_SIZE; ++i) {
862 spin_lock_irqsave(&dma_entry_hash[i].lock, flags);
863 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
864 if (entry->dev == dev) {
865 count += 1;
866 *out_entry = entry;
867 }
868 }
869 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);
870 }
871
872 return count;
873}
874
875static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
876{
877 struct device *dev = data;
878 struct dma_debug_entry *entry;
879 int count;
880
881 if (dma_debug_disabled())
882 return 0;
883
884 switch (action) {
885 case BUS_NOTIFY_UNBOUND_DRIVER:
886 count = device_dma_allocations(dev, &entry);
887 if (count == 0)
888 break;
889 err_printk(dev, entry, "device driver has pending "
890 "DMA allocations while released from device "
891 "[count=%d]\n"
892 "One of leaked entries details: "
893 "[device address=0x%016llx] [size=%llu bytes] "
894 "[mapped with %s] [mapped as %s]\n",
895 count, entry->dev_addr, entry->size,
896 dir2name[entry->direction], type2name[entry->type]);
897 break;
898 default:
899 break;
900 }
901
902 return 0;
903}
904
905void dma_debug_add_bus(const struct bus_type *bus)
906{
907 struct notifier_block *nb;
908
909 if (dma_debug_disabled())
910 return;
911
912 nb = kzalloc_obj(struct notifier_block);
913 if (nb == NULL) {
914 pr_err("dma_debug_add_bus: out of memory\n");
915 return;
916 }
917
918 nb->notifier_call = dma_debug_device_change;
919
920 bus_register_notifier(bus, nb);
921}
922
923static int dma_debug_init(void)
924{
925 int i, nr_pages;
926
927 /* Do not use dma_debug_initialized here, since we really want to be
928 * called to set dma_debug_initialized
929 */
930 if (global_disable)
931 return 0;
932
933 for (i = 0; i < HASH_SIZE; ++i) {
934 INIT_LIST_HEAD(&dma_entry_hash[i].list);
935 spin_lock_init(&dma_entry_hash[i].lock);
936 }
937
938 nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);
939 for (i = 0; i < nr_pages; ++i)
940 dma_debug_create_entries(GFP_KERNEL);
941 if (num_free_entries >= nr_prealloc_entries) {
942 pr_info("preallocated %d debug entries\n", nr_total_entries);
943 } else if (num_free_entries > 0) {
944 pr_warn("%d debug entries requested but only %d allocated\n",
945 nr_prealloc_entries, nr_total_entries);
946 } else {
947 pr_err("debugging out of memory error - disabled\n");
948 global_disable = true;
949
950 return 0;
951 }
952 min_free_entries = num_free_entries;
953
954 dma_debug_initialized = true;
955
956 pr_info("debugging enabled by kernel config\n");
957 return 0;
958}
959core_initcall(dma_debug_init);
960
961static __init int dma_debug_cmdline(char *str)
962{
963 if (!str)
964 return -EINVAL;
965
966 if (strncmp(str, "off", 3) == 0) {
967 pr_info("debugging disabled on kernel command line\n");
968 global_disable = true;
969 }
970
971 return 1;
972}
973
974static __init int dma_debug_entries_cmdline(char *str)
975{
976 if (!str)
977 return -EINVAL;
978 if (!get_option(&str, &nr_prealloc_entries))
979 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
980 return 1;
981}
982
983__setup("dma_debug=", dma_debug_cmdline);
984__setup("dma_debug_entries=", dma_debug_entries_cmdline);
985
986static void check_unmap(struct dma_debug_entry *ref)
987{
988 struct dma_debug_entry *entry;
989 struct hash_bucket *bucket;
990 unsigned long flags;
991
992 bucket = get_hash_bucket(ref, &flags);
993 entry = bucket_find_exact(bucket, ref);
994
995 if (!entry) {
996 /* must drop lock before calling dma_mapping_error */
997 put_hash_bucket(bucket, flags);
998
999 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1000 err_printk(ref->dev, NULL,
1001 "device driver tries to free an "
1002 "invalid DMA memory address\n");
1003 } else {
1004 err_printk(ref->dev, NULL,
1005 "device driver tries to free DMA "
1006 "memory it has not allocated [device "
1007 "address=0x%016llx] [size=%llu bytes]\n",
1008 ref->dev_addr, ref->size);
1009 }
1010 return;
1011 }
1012
1013 if (ref->size != entry->size) {
1014 err_printk(ref->dev, entry, "device driver frees "
1015 "DMA memory with different size "
1016 "[device address=0x%016llx] [map size=%llu bytes] "
1017 "[unmap size=%llu bytes]\n",
1018 ref->dev_addr, entry->size, ref->size);
1019 }
1020
1021 if (ref->type != entry->type) {
1022 err_printk(ref->dev, entry, "device driver frees "
1023 "DMA memory with wrong function "
1024 "[device address=0x%016llx] [size=%llu bytes] "
1025 "[mapped as %s] [unmapped as %s]\n",
1026 ref->dev_addr, ref->size,
1027 type2name[entry->type], type2name[ref->type]);
1028 } else if ((entry->type == dma_debug_coherent ||
1029 entry->type == dma_debug_noncoherent) &&
1030 ref->paddr != entry->paddr) {
1031 err_printk(ref->dev, entry, "device driver frees "
1032 "DMA memory with different CPU address "
1033 "[device address=0x%016llx] [size=%llu bytes] "
1034 "[cpu alloc address=0x%pa] "
1035 "[cpu free address=0x%pa]",
1036 ref->dev_addr, ref->size,
1037 &entry->paddr,
1038 &ref->paddr);
1039 }
1040
1041 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1042 ref->sg_call_ents != entry->sg_call_ents) {
1043 err_printk(ref->dev, entry, "device driver frees "
1044 "DMA sg list with different entry count "
1045 "[map count=%d] [unmap count=%d]\n",
1046 entry->sg_call_ents, ref->sg_call_ents);
1047 }
1048
1049 /*
1050 * This may be no bug in reality - but most implementations of the
1051 * DMA API don't handle this properly, so check for it here
1052 */
1053 if (ref->direction != entry->direction) {
1054 err_printk(ref->dev, entry, "device driver frees "
1055 "DMA memory with different direction "
1056 "[device address=0x%016llx] [size=%llu bytes] "
1057 "[mapped with %s] [unmapped with %s]\n",
1058 ref->dev_addr, ref->size,
1059 dir2name[entry->direction],
1060 dir2name[ref->direction]);
1061 }
1062
1063 /*
1064 * Drivers should use dma_mapping_error() to check the returned
1065 * addresses of dma_map_single() and dma_map_page().
1066 * If not, print this warning message. See Documentation/core-api/dma-api.rst.
1067 */
1068 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1069 err_printk(ref->dev, entry,
1070 "device driver failed to check map error"
1071 "[device address=0x%016llx] [size=%llu bytes] "
1072 "[mapped as %s]",
1073 ref->dev_addr, ref->size,
1074 type2name[entry->type]);
1075 }
1076
1077 /*
1078 * This may be no bug in reality - but DMA API still expects
1079 * that entry is unmapped with same attributes as it was mapped.
1080 *
1081 * DMA_ATTR_UNMAP_VALID lists the attributes that must be identical
1082 * between map and unmap. Any attribute outside this set (e.g.
1083 * DMA_ATTR_NO_WARN, DMA_ATTR_SKIP_CPU_SYNC) is allowed to differ.
1084 */
1085#define DMA_ATTR_UNMAP_VALID \
1086 (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_FORCE_CONTIGUOUS | \
1087 DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_PRIVILEGED | \
1088 DMA_ATTR_CC_SHARED)
1089 if ((ref->attrs & DMA_ATTR_UNMAP_VALID) !=
1090 (entry->attrs & DMA_ATTR_UNMAP_VALID)) {
1091 err_printk(ref->dev, entry,
1092 "device driver frees "
1093 "DMA memory with different attributes "
1094 "[device address=0x%016llx] [size=%llu bytes] "
1095 "[mapped with 0x%lx] [unmapped with 0x%lx]\n",
1096 ref->dev_addr, ref->size, entry->attrs, ref->attrs);
1097 }
1098#undef DMA_ATTR_UNMAP_VALID
1099
1100 hash_bucket_del(entry);
1101 put_hash_bucket(bucket, flags);
1102
1103 /*
1104 * Free the entry outside of bucket_lock to avoid ABBA deadlocks
1105 * between that and radix_lock.
1106 */
1107 dma_entry_free(entry);
1108}
1109
1110static void check_for_stack(struct device *dev, phys_addr_t phys)
1111{
1112 void *addr;
1113 struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1114
1115 if (!stack_vm_area) {
1116 /* Stack is direct-mapped. */
1117 if (PhysHighMem(phys))
1118 return;
1119 addr = phys_to_virt(phys);
1120 if (object_is_on_stack(addr))
1121 err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr);
1122 } else {
1123 /* Stack is vmalloced. */
1124 int i;
1125
1126 for (i = 0; i < stack_vm_area->nr_pages; i++) {
1127 if (__phys_to_pfn(phys) !=
1128 page_to_pfn(stack_vm_area->pages[i]))
1129 continue;
1130
1131 addr = (u8 *)current->stack + i * PAGE_SIZE +
1132 (phys % PAGE_SIZE);
1133 err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr);
1134 break;
1135 }
1136 }
1137}
1138
1139static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1140{
1141 if (memory_intersects(_stext, _etext, addr, len) ||
1142 memory_intersects(__start_rodata, __end_rodata, addr, len))
1143 err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1144}
1145
1146static void check_sync(struct device *dev,
1147 struct dma_debug_entry *ref,
1148 bool to_cpu)
1149{
1150 struct dma_debug_entry *entry;
1151 struct hash_bucket *bucket;
1152 unsigned long flags;
1153
1154 bucket = get_hash_bucket(ref, &flags);
1155
1156 entry = bucket_find_contain(&bucket, ref, &flags);
1157
1158 if (!entry) {
1159 err_printk(dev, NULL, "device driver tries "
1160 "to sync DMA memory it has not allocated "
1161 "[device address=0x%016llx] [size=%llu bytes]\n",
1162 (unsigned long long)ref->dev_addr, ref->size);
1163 goto out;
1164 }
1165
1166 if (ref->size > entry->size) {
1167 err_printk(dev, entry, "device driver syncs"
1168 " DMA memory outside allocated range "
1169 "[device address=0x%016llx] "
1170 "[allocation size=%llu bytes] "
1171 "[sync offset+size=%llu]\n",
1172 entry->dev_addr, entry->size,
1173 ref->size);
1174 }
1175
1176 if (entry->direction == DMA_BIDIRECTIONAL)
1177 goto out;
1178
1179 if (ref->direction != entry->direction) {
1180 err_printk(dev, entry, "device driver syncs "
1181 "DMA memory with different direction "
1182 "[device address=0x%016llx] [size=%llu bytes] "
1183 "[mapped with %s] [synced with %s]\n",
1184 (unsigned long long)ref->dev_addr, entry->size,
1185 dir2name[entry->direction],
1186 dir2name[ref->direction]);
1187 }
1188
1189 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1190 !(ref->direction == DMA_TO_DEVICE))
1191 err_printk(dev, entry, "device driver syncs "
1192 "device read-only DMA memory for cpu "
1193 "[device address=0x%016llx] [size=%llu bytes] "
1194 "[mapped with %s] [synced with %s]\n",
1195 (unsigned long long)ref->dev_addr, entry->size,
1196 dir2name[entry->direction],
1197 dir2name[ref->direction]);
1198
1199 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1200 !(ref->direction == DMA_FROM_DEVICE))
1201 err_printk(dev, entry, "device driver syncs "
1202 "device write-only DMA memory to device "
1203 "[device address=0x%016llx] [size=%llu bytes] "
1204 "[mapped with %s] [synced with %s]\n",
1205 (unsigned long long)ref->dev_addr, entry->size,
1206 dir2name[entry->direction],
1207 dir2name[ref->direction]);
1208
1209 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1210 ref->sg_call_ents != entry->sg_call_ents) {
1211 err_printk(ref->dev, entry, "device driver syncs "
1212 "DMA sg list with different entry count "
1213 "[map count=%d] [sync count=%d]\n",
1214 entry->sg_call_ents, ref->sg_call_ents);
1215 }
1216
1217out:
1218 put_hash_bucket(bucket, flags);
1219}
1220
1221static void check_sg_segment(struct device *dev, struct scatterlist *sg)
1222{
1223 unsigned int max_seg = dma_get_max_seg_size(dev);
1224 u64 start, end, boundary = dma_get_seg_boundary(dev);
1225
1226 /*
1227 * Either the driver forgot to set dma_parms appropriately, or
1228 * whoever generated the list forgot to check them.
1229 */
1230 if (sg->length > max_seg)
1231 err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n",
1232 sg->length, max_seg);
1233 /*
1234 * In some cases this could potentially be the DMA API
1235 * implementation's fault, but it would usually imply that
1236 * the scatterlist was built inappropriately to begin with.
1237 */
1238 start = sg_dma_address(sg);
1239 end = start + sg_dma_len(sg) - 1;
1240 if ((start ^ end) & ~boundary)
1241 err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
1242 start, end, boundary);
1243}
1244
1245void debug_dma_map_single(struct device *dev, const void *addr,
1246 unsigned long len)
1247{
1248 if (unlikely(dma_debug_disabled()))
1249 return;
1250
1251 if (!virt_addr_valid(addr))
1252 err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
1253 addr, len);
1254
1255 if (is_vmalloc_addr(addr))
1256 err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
1257 addr, len);
1258}
1259EXPORT_SYMBOL(debug_dma_map_single);
1260
1261void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
1262 int direction, dma_addr_t dma_addr, unsigned long attrs)
1263{
1264 struct dma_debug_entry *entry;
1265
1266 if (unlikely(dma_debug_disabled()))
1267 return;
1268
1269 if (dma_mapping_error(dev, dma_addr))
1270 return;
1271
1272 entry = dma_entry_alloc();
1273 if (!entry)
1274 return;
1275
1276 entry->dev = dev;
1277 entry->type = dma_debug_phy;
1278 entry->paddr = phys;
1279 entry->dev_addr = dma_addr;
1280 entry->size = size;
1281 entry->direction = direction;
1282 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1283 entry->attrs = attrs;
1284
1285 if (attrs & DMA_ATTR_MMIO) {
1286 unsigned long pfn = PHYS_PFN(phys);
1287
1288 if (pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
1289 err_printk(dev, entry,
1290 "dma_map_resource called for RAM address %pa\n",
1291 &phys);
1292 } else {
1293 check_for_stack(dev, phys);
1294
1295 if (!PhysHighMem(phys))
1296 check_for_illegal_area(dev, phys_to_virt(phys), size);
1297 }
1298
1299 add_dma_entry(entry);
1300}
1301
1302void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1303{
1304 struct dma_debug_entry ref;
1305 struct dma_debug_entry *entry;
1306 struct hash_bucket *bucket;
1307 unsigned long flags;
1308
1309 if (unlikely(dma_debug_disabled()))
1310 return;
1311
1312 ref.dev = dev;
1313 ref.dev_addr = dma_addr;
1314 bucket = get_hash_bucket(&ref, &flags);
1315
1316 list_for_each_entry(entry, &bucket->list, list) {
1317 if (!exact_match(&ref, entry))
1318 continue;
1319
1320 /*
1321 * The same physical address can be mapped multiple
1322 * times. Without a hardware IOMMU this results in the
1323 * same device addresses being put into the dma-debug
1324 * hash multiple times too. This can result in false
1325 * positives being reported. Therefore we implement a
1326 * best-fit algorithm here which updates the first entry
1327 * from the hash which fits the reference value and is
1328 * not currently listed as being checked.
1329 */
1330 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1331 entry->map_err_type = MAP_ERR_CHECKED;
1332 break;
1333 }
1334 }
1335
1336 put_hash_bucket(bucket, flags);
1337}
1338EXPORT_SYMBOL(debug_dma_mapping_error);
1339
1340void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size,
1341 int direction, unsigned long attrs)
1342{
1343 struct dma_debug_entry ref = {
1344 .type = dma_debug_phy,
1345 .dev = dev,
1346 .dev_addr = dma_addr,
1347 .size = size,
1348 .direction = direction,
1349 .attrs = attrs,
1350 };
1351
1352 if (unlikely(dma_debug_disabled()))
1353 return;
1354 check_unmap(&ref);
1355}
1356
1357void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1358 int nents, int mapped_ents, int direction,
1359 unsigned long attrs)
1360{
1361 struct dma_debug_entry *entry;
1362 struct scatterlist *s;
1363 int i;
1364
1365 if (unlikely(dma_debug_disabled()))
1366 return;
1367
1368 for_each_sg(sg, s, nents, i) {
1369 check_for_stack(dev, sg_phys(s));
1370 if (!PageHighMem(sg_page(s)))
1371 check_for_illegal_area(dev, sg_virt(s), s->length);
1372 }
1373
1374 for_each_sg(sg, s, mapped_ents, i) {
1375 entry = dma_entry_alloc();
1376 if (!entry)
1377 return;
1378
1379 entry->type = dma_debug_sg;
1380 entry->dev = dev;
1381 entry->paddr = sg_phys(s);
1382 entry->size = sg_dma_len(s);
1383 entry->dev_addr = sg_dma_address(s);
1384 entry->direction = direction;
1385 entry->sg_call_ents = nents;
1386 entry->sg_mapped_ents = mapped_ents;
1387 entry->attrs = attrs;
1388
1389 check_sg_segment(dev, s);
1390
1391 add_dma_entry(entry);
1392 }
1393}
1394
1395static int get_nr_mapped_entries(struct device *dev,
1396 struct dma_debug_entry *ref)
1397{
1398 struct dma_debug_entry *entry;
1399 struct hash_bucket *bucket;
1400 unsigned long flags;
1401 int mapped_ents;
1402
1403 bucket = get_hash_bucket(ref, &flags);
1404 entry = bucket_find_exact(bucket, ref);
1405 mapped_ents = 0;
1406
1407 if (entry)
1408 mapped_ents = entry->sg_mapped_ents;
1409 put_hash_bucket(bucket, flags);
1410
1411 return mapped_ents;
1412}
1413
1414void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1415 int nelems, int dir, unsigned long attrs)
1416{
1417 struct scatterlist *s;
1418 int mapped_ents = 0, i;
1419
1420 if (unlikely(dma_debug_disabled()))
1421 return;
1422
1423 for_each_sg(sglist, s, nelems, i) {
1424
1425 struct dma_debug_entry ref = {
1426 .type = dma_debug_sg,
1427 .dev = dev,
1428 .paddr = sg_phys(s),
1429 .dev_addr = sg_dma_address(s),
1430 .size = sg_dma_len(s),
1431 .direction = dir,
1432 .sg_call_ents = nelems,
1433 .attrs = attrs,
1434 };
1435
1436 if (mapped_ents && i >= mapped_ents)
1437 break;
1438
1439 if (!i)
1440 mapped_ents = get_nr_mapped_entries(dev, &ref);
1441
1442 check_unmap(&ref);
1443 }
1444}
1445
1446static phys_addr_t virt_to_paddr(void *virt)
1447{
1448 struct page *page;
1449
1450 if (is_vmalloc_addr(virt))
1451 page = vmalloc_to_page(virt);
1452 else
1453 page = virt_to_page(virt);
1454
1455 return page_to_phys(page) + offset_in_page(virt);
1456}
1457
1458void debug_dma_alloc_coherent(struct device *dev, size_t size,
1459 dma_addr_t dma_addr, void *virt,
1460 unsigned long attrs)
1461{
1462 struct dma_debug_entry *entry;
1463
1464 if (unlikely(dma_debug_disabled()))
1465 return;
1466
1467 if (unlikely(virt == NULL))
1468 return;
1469
1470 /* handle vmalloc and linear addresses */
1471 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1472 return;
1473
1474 entry = dma_entry_alloc();
1475 if (!entry)
1476 return;
1477
1478 entry->type = dma_debug_coherent;
1479 entry->dev = dev;
1480 entry->paddr = virt_to_paddr(virt);
1481 entry->size = size;
1482 entry->dev_addr = dma_addr;
1483 entry->direction = DMA_BIDIRECTIONAL;
1484 entry->attrs = attrs;
1485
1486 add_dma_entry(entry);
1487}
1488
1489void debug_dma_free_coherent(struct device *dev, size_t size, void *virt,
1490 dma_addr_t dma_addr, unsigned long attrs)
1491{
1492 struct dma_debug_entry ref = {
1493 .type = dma_debug_coherent,
1494 .dev = dev,
1495 .dev_addr = dma_addr,
1496 .size = size,
1497 .direction = DMA_BIDIRECTIONAL,
1498 .attrs = attrs,
1499 };
1500
1501 /* handle vmalloc and linear addresses */
1502 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1503 return;
1504
1505 ref.paddr = virt_to_paddr(virt);
1506
1507 if (unlikely(dma_debug_disabled()))
1508 return;
1509
1510 check_unmap(&ref);
1511}
1512
1513void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1514 size_t size, int direction)
1515{
1516 struct dma_debug_entry ref;
1517
1518 if (unlikely(dma_debug_disabled()))
1519 return;
1520
1521 ref.type = dma_debug_single;
1522 ref.dev = dev;
1523 ref.dev_addr = dma_handle;
1524 ref.size = size;
1525 ref.direction = direction;
1526 ref.sg_call_ents = 0;
1527
1528 check_sync(dev, &ref, true);
1529}
1530
1531void debug_dma_sync_single_for_device(struct device *dev,
1532 dma_addr_t dma_handle, size_t size,
1533 int direction)
1534{
1535 struct dma_debug_entry ref;
1536
1537 if (unlikely(dma_debug_disabled()))
1538 return;
1539
1540 ref.type = dma_debug_single;
1541 ref.dev = dev;
1542 ref.dev_addr = dma_handle;
1543 ref.size = size;
1544 ref.direction = direction;
1545 ref.sg_call_ents = 0;
1546
1547 check_sync(dev, &ref, false);
1548}
1549
1550void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1551 int nelems, int direction)
1552{
1553 struct scatterlist *s;
1554 int mapped_ents = 0, i;
1555
1556 if (unlikely(dma_debug_disabled()))
1557 return;
1558
1559 for_each_sg(sg, s, nelems, i) {
1560
1561 struct dma_debug_entry ref = {
1562 .type = dma_debug_sg,
1563 .dev = dev,
1564 .paddr = sg_phys(s),
1565 .dev_addr = sg_dma_address(s),
1566 .size = sg_dma_len(s),
1567 .direction = direction,
1568 .sg_call_ents = nelems,
1569 };
1570
1571 if (!i)
1572 mapped_ents = get_nr_mapped_entries(dev, &ref);
1573
1574 if (i >= mapped_ents)
1575 break;
1576
1577 check_sync(dev, &ref, true);
1578 }
1579}
1580
1581void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1582 int nelems, int direction)
1583{
1584 struct scatterlist *s;
1585 int mapped_ents = 0, i;
1586
1587 if (unlikely(dma_debug_disabled()))
1588 return;
1589
1590 for_each_sg(sg, s, nelems, i) {
1591
1592 struct dma_debug_entry ref = {
1593 .type = dma_debug_sg,
1594 .dev = dev,
1595 .paddr = sg_phys(s),
1596 .dev_addr = sg_dma_address(s),
1597 .size = sg_dma_len(s),
1598 .direction = direction,
1599 .sg_call_ents = nelems,
1600 };
1601 if (!i)
1602 mapped_ents = get_nr_mapped_entries(dev, &ref);
1603
1604 if (i >= mapped_ents)
1605 break;
1606
1607 check_sync(dev, &ref, false);
1608 }
1609}
1610
1611void debug_dma_alloc_pages(struct device *dev, struct page *page,
1612 size_t size, int direction,
1613 dma_addr_t dma_addr)
1614{
1615 struct dma_debug_entry *entry;
1616
1617 if (unlikely(dma_debug_disabled()))
1618 return;
1619
1620 entry = dma_entry_alloc();
1621 if (!entry)
1622 return;
1623
1624 entry->type = dma_debug_noncoherent;
1625 entry->dev = dev;
1626 entry->paddr = page_to_phys(page);
1627 entry->size = size;
1628 entry->dev_addr = dma_addr;
1629 entry->direction = direction;
1630
1631 add_dma_entry(entry);
1632}
1633
1634void debug_dma_free_pages(struct device *dev, struct page *page,
1635 size_t size, int direction,
1636 dma_addr_t dma_addr)
1637{
1638 struct dma_debug_entry ref = {
1639 .type = dma_debug_noncoherent,
1640 .dev = dev,
1641 .paddr = page_to_phys(page),
1642 .dev_addr = dma_addr,
1643 .size = size,
1644 .direction = direction,
1645 };
1646
1647 if (unlikely(dma_debug_disabled()))
1648 return;
1649
1650 check_unmap(&ref);
1651}
1652
1653static int __init dma_debug_driver_setup(char *str)
1654{
1655 int i;
1656
1657 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1658 current_driver_name[i] = *str;
1659 if (*str == 0)
1660 break;
1661 }
1662
1663 if (current_driver_name[0])
1664 pr_info("enable driver filter for driver [%s]\n",
1665 current_driver_name);
1666
1667
1668 return 1;
1669}
1670__setup("dma_debug_driver=", dma_debug_driver_setup);