projects/linux·blob master · refs

705 lines · 18.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3/* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2019-2024 Linaro Ltd.
5 */
6
7#include <linux/dma-mapping.h>
8#include <linux/io.h>
9#include <linux/iommu.h>
10#include <linux/of_address.h>
11#include <linux/platform_device.h>
12#include <linux/types.h>
13
14#include <linux/soc/qcom/smem.h>
15
16#include "gsi_trans.h"
17#include "ipa.h"
18#include "ipa_cmd.h"
19#include "ipa_data.h"
20#include "ipa_mem.h"
21#include "ipa_reg.h"
22#include "ipa_table.h"
23
24/* "Canary" value placed between memory regions to detect overflow */
25#define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef)
26
27/* SMEM host id representing the modem. */
28#define QCOM_SMEM_HOST_MODEM 1
29
30#define SMEM_IPA_FILTER_TABLE 497
31
32const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id)
33{
34 u32 i;
35
36 for (i = 0; i < ipa->mem_count; i++) {
37 const struct ipa_mem *mem = &ipa->mem[i];
38
39 if (mem->id == mem_id)
40 return mem;
41 }
42
43 return NULL;
44}
45
46/* Add an immediate command to a transaction that zeroes a memory region */
47static void
48ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
49{
50 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi);
51 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id);
52 dma_addr_t addr = ipa->zero_addr;
53
54 if (!mem->size)
55 return;
56
57 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true);
58}
59
60/**
61 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas
62 * @ipa: IPA pointer
63 *
64 * Set up the shared memory regions in IPA local memory. This involves
65 * zero-filling memory regions, and in the case of header memory, telling
66 * the IPA where it's located.
67 *
68 * This function performs the initial setup of this memory. If the modem
69 * crashes, its regions are re-zeroed in ipa_mem_zero_modem().
70 *
71 * The AP informs the modem where its portions of memory are located
72 * in a QMI exchange that occurs at modem startup.
73 *
74 * There is no need for a matching ipa_mem_teardown() function.
75 *
76 * Return: 0 if successful, or a negative error code
77 */
78int ipa_mem_setup(struct ipa *ipa)
79{
80 dma_addr_t addr = ipa->zero_addr;
81 const struct ipa_mem *mem;
82 struct gsi_trans *trans;
83 const struct reg *reg;
84 u32 offset;
85 u16 size;
86 u32 val;
87
88 /* Get a transaction to define the header memory region and to zero
89 * the processing context and modem memory regions.
90 */
91 trans = ipa_cmd_trans_alloc(ipa, 4);
92 if (!trans) {
93 dev_err(ipa->dev, "no transaction for memory setup\n");
94 return -EBUSY;
95 }
96
97 /* Initialize IPA-local header memory. The AP header region, if
98 * present, is contiguous with and follows the modem header region,
99 * and they are initialized together.
100 */
101 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER);
102 offset = mem->offset;
103 size = mem->size;
104 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER);
105 if (mem)
106 size += mem->size;
107
108 ipa_cmd_hdr_init_local_add(trans, offset, size, addr);
109
110 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
111 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX);
112 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
113
114 gsi_trans_commit_wait(trans);
115
116 /* Tell the hardware where the processing context area is located */
117 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
118 offset = ipa->mem_offset + mem->offset;
119
120 reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT);
121 val = reg_encode(reg, IPA_BASE_ADDR, offset);
122 iowrite32(val, ipa->reg_virt + reg_offset(reg));
123
124 return 0;
125}
126
127/* Is the given memory region ID is valid for the current IPA version? */
128static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id)
129{
130 enum ipa_version version = ipa->version;
131
132 switch (mem_id) {
133 case IPA_MEM_UC_SHARED:
134 case IPA_MEM_UC_INFO:
135 case IPA_MEM_V4_FILTER_HASHED:
136 case IPA_MEM_V4_FILTER:
137 case IPA_MEM_V6_FILTER_HASHED:
138 case IPA_MEM_V6_FILTER:
139 case IPA_MEM_V4_ROUTE_HASHED:
140 case IPA_MEM_V4_ROUTE:
141 case IPA_MEM_V6_ROUTE_HASHED:
142 case IPA_MEM_V6_ROUTE:
143 case IPA_MEM_MODEM_HEADER:
144 case IPA_MEM_AP_HEADER:
145 case IPA_MEM_MODEM_PROC_CTX:
146 case IPA_MEM_AP_PROC_CTX:
147 case IPA_MEM_MODEM:
148 case IPA_MEM_UC_EVENT_RING:
149 case IPA_MEM_PDN_CONFIG:
150 case IPA_MEM_STATS_QUOTA_MODEM:
151 case IPA_MEM_STATS_QUOTA_AP:
152 case IPA_MEM_END_MARKER: /* pseudo region */
153 break;
154
155 case IPA_MEM_STATS_TETHERING:
156 case IPA_MEM_STATS_DROP:
157 if (version < IPA_VERSION_4_0)
158 return false;
159 break;
160
161 case IPA_MEM_STATS_V4_FILTER:
162 case IPA_MEM_STATS_V6_FILTER:
163 case IPA_MEM_STATS_V4_ROUTE:
164 case IPA_MEM_STATS_V6_ROUTE:
165 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2)
166 return false;
167 break;
168
169 case IPA_MEM_AP_V4_FILTER:
170 case IPA_MEM_AP_V6_FILTER:
171 if (version < IPA_VERSION_5_0)
172 return false;
173 break;
174
175 case IPA_MEM_NAT_TABLE:
176 case IPA_MEM_STATS_FILTER_ROUTE:
177 if (version < IPA_VERSION_4_5)
178 return false;
179 break;
180
181 default:
182 return false;
183 }
184
185 return true;
186}
187
188/* Must the given memory region be present in the configuration? */
189static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id)
190{
191 switch (mem_id) {
192 case IPA_MEM_UC_SHARED:
193 case IPA_MEM_UC_INFO:
194 case IPA_MEM_V4_FILTER_HASHED:
195 case IPA_MEM_V4_FILTER:
196 case IPA_MEM_V6_FILTER_HASHED:
197 case IPA_MEM_V6_FILTER:
198 case IPA_MEM_V4_ROUTE_HASHED:
199 case IPA_MEM_V4_ROUTE:
200 case IPA_MEM_V6_ROUTE_HASHED:
201 case IPA_MEM_V6_ROUTE:
202 case IPA_MEM_MODEM_HEADER:
203 case IPA_MEM_MODEM_PROC_CTX:
204 case IPA_MEM_AP_PROC_CTX:
205 case IPA_MEM_MODEM:
206 return true;
207
208 case IPA_MEM_PDN_CONFIG:
209 case IPA_MEM_STATS_QUOTA_MODEM:
210 return ipa->version >= IPA_VERSION_4_0;
211
212 case IPA_MEM_STATS_TETHERING:
213 return ipa->version >= IPA_VERSION_4_0 &&
214 ipa->version != IPA_VERSION_5_0;
215
216 default:
217 return false; /* Anything else is optional */
218 }
219}
220
221static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem)
222{
223 enum ipa_mem_id mem_id = mem->id;
224 struct device *dev = ipa->dev;
225 u16 size_multiple;
226
227 /* Make sure the memory region is valid for this version of IPA */
228 if (!ipa_mem_id_valid(ipa, mem_id)) {
229 dev_err(dev, "region id %u not valid\n", mem_id);
230 return false;
231 }
232
233 if (!mem->size && !mem->canary_count) {
234 dev_err(dev, "empty memory region %u\n", mem_id);
235 return false;
236 }
237
238 /* Other than modem memory, sizes must be a multiple of 8 */
239 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8;
240 if (mem->size % size_multiple)
241 dev_err(dev, "region %u size not a multiple of %u bytes\n",
242 mem_id, size_multiple);
243 else if (mem->offset % 8)
244 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id);
245 else if (mem->offset < mem->canary_count * sizeof(__le32))
246 dev_err(dev, "region %u offset too small for %hu canaries\n",
247 mem_id, mem->canary_count);
248 else if (mem_id == IPA_MEM_END_MARKER && mem->size)
249 dev_err(dev, "non-zero end marker region size\n");
250 else
251 return true;
252
253 return false;
254}
255
256/* Verify each defined memory region is valid. */
257static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data)
258{
259 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { };
260 struct device *dev = ipa->dev;
261 enum ipa_mem_id mem_id;
262 u32 i;
263
264 if (mem_data->local_count > IPA_MEM_COUNT) {
265 dev_err(dev, "too many memory regions (%u > %u)\n",
266 mem_data->local_count, IPA_MEM_COUNT);
267 return false;
268 }
269
270 for (i = 0; i < mem_data->local_count; i++) {
271 const struct ipa_mem *mem = &mem_data->local[i];
272
273 if (__test_and_set_bit(mem->id, regions)) {
274 dev_err(dev, "duplicate memory region %u\n", mem->id);
275 return false;
276 }
277
278 /* Defined regions have non-zero size and/or canary count */
279 if (!ipa_mem_valid_one(ipa, mem))
280 return false;
281 }
282
283 /* Now see if any required regions are not defined */
284 for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) {
285 if (ipa_mem_id_required(ipa, mem_id))
286 dev_err(dev, "required memory region %u missing\n",
287 mem_id);
288 }
289
290 return true;
291}
292
293/* Do all memory regions fit within the IPA local memory? */
294static bool ipa_mem_size_valid(struct ipa *ipa)
295{
296 struct device *dev = ipa->dev;
297 u32 limit = ipa->mem_size;
298 u32 i;
299
300 for (i = 0; i < ipa->mem_count; i++) {
301 const struct ipa_mem *mem = &ipa->mem[i];
302
303 if (mem->offset + mem->size <= limit)
304 continue;
305
306 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n",
307 mem->id, limit);
308
309 return false;
310 }
311
312 return true;
313}
314
315/**
316 * ipa_mem_config() - Configure IPA shared memory
317 * @ipa: IPA pointer
318 *
319 * Return: 0 if successful, or a negative error code
320 */
321int ipa_mem_config(struct ipa *ipa)
322{
323 struct device *dev = ipa->dev;
324 const struct ipa_mem *mem;
325 const struct reg *reg;
326 dma_addr_t addr;
327 u32 mem_size;
328 void *virt;
329 u32 val;
330 u32 i;
331
332 /* Check the advertised location and size of the shared memory area */
333 reg = ipa_reg(ipa, SHARED_MEM_SIZE);
334 val = ioread32(ipa->reg_virt + reg_offset(reg));
335
336 /* The fields in the register are in 8 byte units */
337 ipa->mem_offset = 8 * reg_decode(reg, MEM_BADDR, val);
338
339 /* Make sure the end is within the region's mapped space */
340 mem_size = 8 * reg_decode(reg, MEM_SIZE, val);
341
342 /* If the sizes don't match, issue a warning */
343 if (ipa->mem_offset + mem_size < ipa->mem_size) {
344 dev_warn(dev, "limiting IPA memory size to 0x%08x\n",
345 mem_size);
346 ipa->mem_size = mem_size;
347 } else if (ipa->mem_offset + mem_size > ipa->mem_size) {
348 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n",
349 mem_size);
350 }
351
352 /* We know our memory size; make sure regions are all in range */
353 if (!ipa_mem_size_valid(ipa))
354 return -EINVAL;
355
356 /* Prealloc DMA memory for zeroing regions */
357 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL);
358 if (!virt)
359 return -ENOMEM;
360 ipa->zero_addr = addr;
361 ipa->zero_virt = virt;
362 ipa->zero_size = IPA_MEM_MAX;
363
364 /* For each defined region, write "canary" values in the
365 * space prior to the region's base address if indicated.
366 */
367 for (i = 0; i < ipa->mem_count; i++) {
368 u16 canary_count = ipa->mem[i].canary_count;
369 __le32 *canary;
370
371 if (!canary_count)
372 continue;
373
374 /* Write canary values in the space before the region */
375 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset;
376 do
377 *--canary = IPA_MEM_CANARY_VAL;
378 while (--canary_count);
379 }
380
381 /* Verify the microcontroller ring alignment (if defined) */
382 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING);
383 if (mem && mem->offset % 1024) {
384 dev_err(dev, "microcontroller ring not 1024-byte aligned\n");
385 goto err_dma_free;
386 }
387
388 return 0;
389
390err_dma_free:
391 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr);
392
393 return -EINVAL;
394}
395
396/* Inverse of ipa_mem_config() */
397void ipa_mem_deconfig(struct ipa *ipa)
398{
399 struct device *dev = ipa->dev;
400
401 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr);
402 ipa->zero_size = 0;
403 ipa->zero_virt = NULL;
404 ipa->zero_addr = 0;
405}
406
407/**
408 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem
409 * @ipa: IPA pointer
410 *
411 * Zero regions of IPA-local memory used by the modem. These are configured
412 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and
413 * restarts via SSR we need to re-initialize them. A QMI message tells the
414 * modem where to find regions of IPA local memory it needs to know about
415 * (these included).
416 */
417int ipa_mem_zero_modem(struct ipa *ipa)
418{
419 struct gsi_trans *trans;
420
421 /* Get a transaction to zero the modem memory, modem header,
422 * and modem processing context regions.
423 */
424 trans = ipa_cmd_trans_alloc(ipa, 3);
425 if (!trans) {
426 dev_err(ipa->dev, "no transaction to zero modem memory\n");
427 return -EBUSY;
428 }
429
430 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER);
431 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX);
432 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM);
433
434 gsi_trans_commit_wait(trans);
435
436 return 0;
437}
438
439/**
440 * ipa_imem_init() - Initialize IMEM memory used by the IPA
441 * @ipa: IPA pointer
442 * @addr: Physical address of the IPA region in IMEM
443 * @size: Size (bytes) of the IPA region in IMEM
444 *
445 * IMEM is a block of shared memory separate from system DRAM, and
446 * a portion of this memory is available for the IPA to use. The
447 * modem accesses this memory directly, but the IPA accesses it
448 * via the IOMMU, using the AP's credentials.
449 *
450 * If this region exists (size > 0) we map it for read/write access
451 * through the IOMMU using the IPA device.
452 *
453 * Note: @addr and @size are not guaranteed to be page-aligned.
454 */
455static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size)
456{
457 struct device *dev = ipa->dev;
458 struct iommu_domain *domain;
459 unsigned long iova;
460 phys_addr_t phys;
461 int ret;
462
463 if (!size)
464 return 0; /* IMEM memory not used */
465
466 domain = iommu_get_domain_for_dev(dev);
467 if (!domain) {
468 dev_err(dev, "no IOMMU domain found for IMEM\n");
469 return -EINVAL;
470 }
471
472 /* Align the address down and the size up to page boundaries */
473 phys = addr & PAGE_MASK;
474 size = PAGE_ALIGN(size + addr - phys);
475 iova = phys; /* We just want a direct mapping */
476
477 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,
478 GFP_KERNEL);
479 if (ret)
480 return ret;
481
482 ipa->imem_iova = iova;
483 ipa->imem_size = size;
484
485 return 0;
486}
487
488static void ipa_imem_exit(struct ipa *ipa)
489{
490 struct device *dev = ipa->dev;
491 struct iommu_domain *domain;
492
493 if (!ipa->imem_size)
494 return;
495
496 domain = iommu_get_domain_for_dev(dev);
497 if (domain) {
498 size_t size;
499
500 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size);
501 if (size != ipa->imem_size)
502 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n",
503 size, ipa->imem_size);
504 } else {
505 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n");
506 }
507
508 ipa->imem_size = 0;
509 ipa->imem_iova = 0;
510}
511
512/**
513 * ipa_smem_init() - Initialize SMEM memory used by the IPA
514 * @ipa: IPA pointer
515 * @size: Size (bytes) of SMEM memory region
516 *
517 * SMEM is a managed block of shared DRAM, from which numbered "items"
518 * can be allocated. One item is designated for use by the IPA.
519 *
520 * The modem accesses SMEM memory directly, but the IPA accesses it
521 * via the IOMMU, using the AP's credentials.
522 *
523 * If size provided is non-zero, we allocate it and map it for
524 * access through the IOMMU.
525 *
526 * Note: @size and the item address are is not guaranteed to be page-aligned.
527 */
528static int ipa_smem_init(struct ipa *ipa, size_t size)
529{
530 struct device *dev = ipa->dev;
531 struct iommu_domain *domain;
532 unsigned long iova;
533 phys_addr_t phys;
534 phys_addr_t addr;
535 size_t actual;
536 void *virt;
537 int ret;
538
539 if (!size)
540 return 0; /* SMEM memory not used */
541
542 /* SMEM is memory shared between the AP and another system entity
543 * (in this case, the modem). An allocation from SMEM is persistent
544 * until the AP reboots; there is no way to free an allocated SMEM
545 * region. Allocation only reserves the space; to use it you need
546 * to "get" a pointer it (this does not imply reference counting).
547 * The item might have already been allocated, in which case we
548 * use it unless the size isn't what we expect.
549 */
550 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, SMEM_IPA_FILTER_TABLE, size);
551 if (ret && ret != -EEXIST) {
552 dev_err(dev, "error %d allocating size %zu SMEM item\n",
553 ret, size);
554 return ret;
555 }
556
557 /* Now get the address of the SMEM memory region */
558 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, SMEM_IPA_FILTER_TABLE, &actual);
559 if (IS_ERR(virt)) {
560 ret = PTR_ERR(virt);
561 dev_err(dev, "error %d getting SMEM item\n", ret);
562 return ret;
563 }
564
565 /* In case the region was already allocated, verify the size */
566 if (ret && actual != size) {
567 dev_err(dev, "SMEM item has size %zu, expected %zu\n",
568 actual, size);
569 return -EINVAL;
570 }
571
572 domain = iommu_get_domain_for_dev(dev);
573 if (!domain) {
574 dev_err(dev, "no IOMMU domain found for SMEM\n");
575 return -EINVAL;
576 }
577
578 /* Align the address down and the size up to a page boundary */
579 addr = qcom_smem_virt_to_phys(virt);
580 phys = addr & PAGE_MASK;
581 size = PAGE_ALIGN(size + addr - phys);
582 iova = phys; /* We just want a direct mapping */
583
584 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE,
585 GFP_KERNEL);
586 if (ret)
587 return ret;
588
589 ipa->smem_iova = iova;
590 ipa->smem_size = size;
591
592 return 0;
593}
594
595static void ipa_smem_exit(struct ipa *ipa)
596{
597 struct device *dev = ipa->dev;
598 struct iommu_domain *domain;
599
600 domain = iommu_get_domain_for_dev(dev);
601 if (domain) {
602 size_t size;
603
604 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size);
605 if (size != ipa->smem_size)
606 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n",
607 size, ipa->smem_size);
608
609 } else {
610 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n");
611 }
612
613 ipa->smem_size = 0;
614 ipa->smem_iova = 0;
615}
616
617/* Perform memory region-related initialization */
618int ipa_mem_init(struct ipa *ipa, struct platform_device *pdev,
619 const struct ipa_mem_data *mem_data)
620{
621 struct device_node *ipa_slice_np;
622 struct device *dev = &pdev->dev;
623 u32 imem_base, imem_size;
624 struct resource *res;
625 int ret;
626
627 /* Make sure the set of defined memory regions is valid */
628 if (!ipa_mem_valid(ipa, mem_data))
629 return -EINVAL;
630
631 ipa->mem_count = mem_data->local_count;
632 ipa->mem = mem_data->local;
633
634 /* Check the route and filter table memory regions */
635 if (!ipa_table_mem_valid(ipa, false))
636 return -EINVAL;
637 if (!ipa_table_mem_valid(ipa, true))
638 return -EINVAL;
639
640 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
641 if (ret) {
642 dev_err(dev, "error %d setting DMA mask\n", ret);
643 return ret;
644 }
645
646 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ipa-shared");
647 if (!res) {
648 dev_err(dev,
649 "DT error getting \"ipa-shared\" memory property\n");
650 return -ENODEV;
651 }
652
653 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC);
654 if (!ipa->mem_virt) {
655 dev_err(dev, "unable to remap \"ipa-shared\" memory\n");
656 return -ENOMEM;
657 }
658
659 ipa->mem_addr = res->start;
660 ipa->mem_size = resource_size(res);
661
662 ipa_slice_np = of_parse_phandle(dev->of_node, "sram", 0);
663 if (ipa_slice_np) {
664 struct resource sram_res;
665
666 ret = of_address_to_resource(ipa_slice_np, 0, &sram_res);
667 of_node_put(ipa_slice_np);
668 if (ret)
669 goto err_unmap;
670
671 imem_base = sram_res.start;
672 imem_size = resource_size(&sram_res);
673 } else {
674 /* Backwards compatibility for DTs lacking
675 * an explicit reference
676 */
677 imem_base = mem_data->imem_addr;
678 imem_size = mem_data->imem_size;
679 }
680
681 ret = ipa_imem_init(ipa, imem_base, imem_size);
682 if (ret)
683 goto err_unmap;
684
685 ret = ipa_smem_init(ipa, mem_data->smem_size);
686 if (ret)
687 goto err_imem_exit;
688
689 return 0;
690
691err_imem_exit:
692 ipa_imem_exit(ipa);
693err_unmap:
694 memunmap(ipa->mem_virt);
695
696 return ret;
697}
698
699/* Inverse of ipa_mem_init() */
700void ipa_mem_exit(struct ipa *ipa)
701{
702 ipa_smem_exit(ipa);
703 ipa_imem_exit(ipa);
704 memunmap(ipa->mem_virt);
705}