linux - drivers/misc/fastrpc.c

2817 lines · 67.7 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3	// Copyright (c) 2018, Linaro Limited
4
5	#include <linux/completion.h>
6	#include <linux/device.h>
7	#include <linux/dma-buf.h>
8	#include <linux/dma-mapping.h>
9	#include <linux/dma-resv.h>
10	#include <linux/idr.h>
11	#include <linux/list.h>
12	#include <linux/miscdevice.h>
13	#include <linux/module.h>
14	#include <linux/of_address.h>
15	#include <linux/of.h>
16	#include <linux/platform_device.h>
17	#include <linux/sort.h>
18	#include <linux/of_platform.h>
19	#include <linux/rpmsg.h>
20	#include <linux/scatterlist.h>
21	#include <linux/slab.h>
22	#include <linux/firmware/qcom/qcom_scm.h>
23	#include <uapi/misc/fastrpc.h>
24	#include <linux/of_reserved_mem.h>
25	#include <linux/bitfield.h>
26	#include <linux/bits.h>
27	#include <linux/bitops.h>
28	#include <linux/compiler.h>
29	#include <linux/iopoll.h>
30
31	#define ADSP_DOMAIN_ID (0)
32	#define MDSP_DOMAIN_ID (1)
33	#define SDSP_DOMAIN_ID (2)
34	#define CDSP_DOMAIN_ID (3)
35	#define GDSP_DOMAIN_ID (4)
36	#define FASTRPC_MAX_SESSIONS 14
37	#define FASTRPC_MAX_VMIDS 16
38	#define FASTRPC_ALIGN 128
39	#define FASTRPC_MAX_FDLIST 16
40	#define FASTRPC_MAX_CRCLIST 64
41	#define FASTRPC_CTX_MAX (256)
42	#define FASTRPC_INIT_HANDLE 1
43	#define FASTRPC_DSP_UTILITIES_HANDLE 2
44	/*
45	* Maximum handle value for static handles.
46	* Static handles are pre-defined, fixed numeric values statically assigned
47	* in the IDL file or FastRPC framework.
48	*/
49	#define FASTRPC_MAX_STATIC_HANDLE (20)
50	#define FASTRPC_CTXID_MASK GENMASK(15, 8)
51	/* Sequence number occupies bits 63:16 of the ctxid / message context */
52	#define FASTRPC_CTXID_SEQ_SHIFT 16
53	#define FASTRPC_CTXID_SEQ_MASK GENMASK_ULL(63, 16)
54
55	/*
56	* Newer DSP firmware implements a PD (Protection Domain) notification
57	* framework that sends PD state notifications upon request. The PD exit
58	* notification is unconditionally sent by the DSP with this fixed sentinel
59	* in the context field rather than the context of an outstanding invocation.
60	* Since the fastrpc driver does not support the DSP PD notification framework,
61	* this message must be dropped rather than matched against the context idr.
62	*/
63	#define FASTRPC_DSP_PD_NOTIFY_CTX 0xABCDABCD
64
65	#define INIT_FILELEN_MAX (2 * 1024 * 1024)
66	#define INIT_FILE_NAMELEN_MAX (128)
67	#define FASTRPC_DEVICE_NAME "fastrpc"
68
69	/* Add memory to static PD pool, protection thru XPU */
70	#define ADSP_MMAP_HEAP_ADDR 4
71	/* MAP static DMA buffer on DSP User PD */
72	#define ADSP_MMAP_DMA_BUFFER 6
73	/* Add memory to static PD pool protection thru hypervisor */
74	#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
75	/* Add memory to userPD pool, for user heap */
76	#define ADSP_MMAP_ADD_PAGES 0x1000
77	/* Add memory to userPD pool, for LLC heap */
78	#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
79
80	#define DSP_UNSUPPORTED_API (0x80000414)
81	/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
82	#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
83	#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
84
85	/* Retrives number of input buffers from the scalars parameter */
86	#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
87
88	/* Retrives number of output buffers from the scalars parameter */
89	#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
90
91	/* Retrives number of input handles from the scalars parameter */
92	#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
93
94	/* Retrives number of output handles from the scalars parameter */
95	#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
96
97	#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
98	REMOTE_SCALARS_OUTBUFS(sc) + \
99	REMOTE_SCALARS_INHANDLES(sc)+ \
100	REMOTE_SCALARS_OUTHANDLES(sc))
101	#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
102	(((attr & 0x07) << 29) \| \
103	((method & 0x1f) << 24) \| \
104	((in & 0xff) << 16) \| \
105	((out & 0xff) << 8) \| \
106	((oin & 0x0f) << 4) \| \
107	(oout & 0x0f))
108
109	#define FASTRPC_SCALARS(method, in, out) \
110	FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
111
112	#define FASTRPC_CREATE_PROCESS_NARGS 6
113	#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
114	/* Remote Method id table */
115	#define FASTRPC_RMID_INIT_ATTACH 0
116	#define FASTRPC_RMID_INIT_RELEASE 1
117	#define FASTRPC_RMID_INIT_MMAP 4
118	#define FASTRPC_RMID_INIT_MUNMAP 5
119	#define FASTRPC_RMID_INIT_CREATE 6
120	#define FASTRPC_RMID_INIT_CREATE_ATTR 7
121	#define FASTRPC_RMID_INIT_CREATE_STATIC 8
122	#define FASTRPC_RMID_INIT_MEM_MAP 10
123	#define FASTRPC_RMID_INIT_MEM_UNMAP 11
124
125	/* Protection Domain(PD) ids */
126	#define ROOT_PD (0)
127	#define USER_PD (1)
128	#define SENSORS_PD (2)
129
130	#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
131
132	/* Poll response number from remote processor for call completion */
133	#define FASTRPC_POLL_RESPONSE (0xdecaf)
134
135	/* Polling mode timeout limit */
136	#define FASTRPC_POLL_MAX_TIMEOUT_US (10000)
137
138	struct fastrpc_phy_page {
139	dma_addr_t addr; /* dma address */
140	u64 size; /* size of contiguous region */
141	};
142
143	struct fastrpc_invoke_buf {
144	u32 num; /* number of contiguous regions */
145	u32 pgidx; /* index to start of contiguous region */
146	};
147
148	struct fastrpc_remote_dmahandle {
149	s32 fd; /* dma handle fd */
150	u32 offset; /* dma handle offset */
151	u32 len; /* dma handle length */
152	};
153
154	struct fastrpc_remote_buf {
155	u64 pv; /* buffer pointer */
156	u64 len; /* length of buffer */
157	};
158
159	union fastrpc_remote_arg {
160	struct fastrpc_remote_buf buf;
161	struct fastrpc_remote_dmahandle dma;
162	};
163
164	struct fastrpc_mmap_rsp_msg {
165	u64 vaddr;
166	};
167
168	struct fastrpc_mmap_req_msg {
169	s32 client_id;
170	u32 flags;
171	u64 vaddr;
172	s32 num;
173	};
174
175	struct fastrpc_mem_map_req_msg {
176	s32 client_id;
177	s32 fd;
178	s32 offset;
179	u32 flags;
180	u64 vaddrin;
181	s32 num;
182	s32 data_len;
183	};
184
185	struct fastrpc_munmap_req_msg {
186	s32 client_id;
187	u64 vaddr;
188	u64 size;
189	};
190
191	struct fastrpc_mem_unmap_req_msg {
192	s32 client_id;
193	s32 fd;
194	u64 vaddrin;
195	u64 len;
196	};
197
198	struct fastrpc_msg {
199	int client_id; /* process client id */
200	int tid; /* thread id */
201	u64 ctx; /* invoke caller context */
202	u32 handle; /* handle to invoke */
203	u32 sc; /* scalars structure describing the data */
204	dma_addr_t addr; /* dma address */
205	u64 size; /* size of contiguous region */
206	};
207
208	struct fastrpc_invoke_rsp {
209	u64 ctx; /* invoke caller context */
210	int retval; /* invoke return value */
211	};
212
213	struct fastrpc_buf_overlap {
214	u64 start;
215	u64 end;
216	int raix;
217	u64 mstart;
218	u64 mend;
219	u64 offset;
220	};
221
222	struct fastrpc_buf {
223	struct fastrpc_user *fl;
224	struct dma_buf *dmabuf;
225	struct device *dev;
226	void *virt;
227	dma_addr_t dma_addr;
228	u64 size;
229	/* Lock for dma buf attachments */
230	struct mutex lock;
231	struct list_head attachments;
232	/* mmap support */
233	struct list_head node; /* list of user requested mmaps */
234	uintptr_t raddr;
235	};
236
237	struct fastrpc_dma_buf_attachment {
238	struct device *dev;
239	struct sg_table sgt;
240	struct list_head node;
241	};
242
243	struct fastrpc_map {
244	struct list_head node;
245	struct fastrpc_user *fl;
246	int fd;
247	struct dma_buf *buf;
248	struct sg_table *table;
249	struct dma_buf_attachment *attach;
250	dma_addr_t dma_addr;
251	u64 size;
252	void *va;
253	u64 len;
254	u64 raddr;
255	u32 attr;
256	struct kref refcount;
257	};
258
259	struct fastrpc_invoke_ctx {
260	int nscalars;
261	int nbufs;
262	int retval;
263	int pid;
264	int client_id;
265	u32 sc;
266	u64 *fdlist;
267	u32 *crc;
268	/* Poll memory that DSP updates */
269	u32 *poll_addr;
270	u64 ctxid;
271	u64 msg_sz;
272	/* work done status flag */
273	bool is_work_done;
274	/* process updates poll memory instead of glink response */
275	bool is_polled;
276	struct kref refcount;
277	struct list_head node; /* list of ctxs */
278	struct completion work;
279	struct work_struct put_work;
280	struct fastrpc_msg msg;
281	struct fastrpc_user *fl;
282	union fastrpc_remote_arg *rpra;
283	struct fastrpc_map **maps;
284	struct fastrpc_buf *buf;
285	struct fastrpc_invoke_args *args;
286	struct fastrpc_buf_overlap *olaps;
287	struct fastrpc_channel_ctx *cctx;
288	};
289
290	struct fastrpc_session_ctx {
291	struct device *dev;
292	int sid;
293	bool used;
294	bool valid;
295	};
296
297	struct fastrpc_soc_data {
298	u32 sid_pos;
299	u32 dma_addr_bits_cdsp;
300	u32 dma_addr_bits_default;
301	bool poll_mode_supported;
302	};
303
304	struct fastrpc_channel_ctx {
305	int domain_id;
306	int sesscount;
307	int vmcount;
308	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
309	struct rpmsg_device *rpdev;
310	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
311	spinlock_t lock;
312	struct idr ctx_idr;
313	struct list_head users;
314	struct kref refcount;
315	/* Flag if dsp attributes are cached */
316	bool valid_attributes;
317	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
318	struct fastrpc_device *secure_fdevice;
319	struct fastrpc_device *fdevice;
320	struct fastrpc_buf *remote_heap;
321	struct list_head invoke_interrupted_mmaps;
322	bool secure;
323	bool unsigned_support;
324	bool poll_mode_supported;
325	/* Per-channel sequence counter; incremented on every context allocation */
326	atomic_t ctx_seq;
327	u64 dma_mask;
328	const struct fastrpc_soc_data *soc_data;
329	};
330
331	struct fastrpc_device {
332	struct fastrpc_channel_ctx *cctx;
333	struct miscdevice miscdev;
334	bool secure;
335	};
336
337	struct fastrpc_user {
338	struct list_head user;
339	struct list_head maps;
340	struct list_head pending;
341	struct list_head mmaps;
342
343	struct fastrpc_channel_ctx *cctx;
344	struct fastrpc_session_ctx *sctx;
345	struct fastrpc_buf *init_mem;
346
347	int client_id;
348	int pd;
349	bool is_secure_dev;
350	/* Flags poll mode state */
351	bool poll_mode;
352	/* Lock for lists */
353	spinlock_t lock;
354	/* lock for allocations */
355	struct mutex mutex;
356	/* Reference count */
357	struct kref refcount;
358	};
359
360	/* Extract SMMU PA from consolidated IOVA */
361	static inline dma_addr_t fastrpc_ipa_to_dma_addr(struct fastrpc_channel_ctx *cctx, dma_addr_t iova)
362	{
363	if (!cctx->soc_data->sid_pos)
364	return 0;
365	return iova & GENMASK_ULL(cctx->soc_data->sid_pos - 1, 0);
366	}
367
368	/*
369	* Prepare the consolidated iova to send to DSP by prepending the SID
370	* to smmu PA at the appropriate position
371	*/
372	static inline u64 fastrpc_sid_offset(struct fastrpc_channel_ctx *cctx,
373	struct fastrpc_session_ctx *sctx)
374	{
375	return (u64)sctx->sid << cctx->soc_data->sid_pos;
376	}
377
378	static void fastrpc_free_map(struct kref *ref)
379	{
380	struct fastrpc_map *map;
381
382	map = container_of(ref, struct fastrpc_map, refcount);
383
384	if (map->table) {
385	if (map->attr & FASTRPC_ATTR_SECUREMAP) {
386	struct qcom_scm_vmperm perm;
387	int vmid = map->fl->cctx->vmperms[0].vmid;
388	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) \| BIT(vmid);
389	int err = 0;
390
391	perm.vmid = QCOM_SCM_VMID_HLOS;
392	perm.perm = QCOM_SCM_PERM_RWX;
393	err = qcom_scm_assign_mem(map->dma_addr, map->len,
394	&src_perms, &perm, 1);
395	if (err) {
396	dev_err(map->fl->sctx->dev,
397	"Failed to assign memory dma_addr %pad size 0x%llx err %d\n",
398	&map->dma_addr, map->len, err);
399	return;
400	}
401	}
402	dma_buf_unmap_attachment_unlocked(map->attach, map->table,
403	DMA_BIDIRECTIONAL);
404	dma_buf_detach(map->buf, map->attach);
405	dma_buf_put(map->buf);
406	}
407
408	if (map->fl) {
409	spin_lock(&map->fl->lock);
410	list_del(&map->node);
411	spin_unlock(&map->fl->lock);
412	map->fl = NULL;
413	}
414
415	kfree(map);
416	}
417
418	static void fastrpc_map_put(struct fastrpc_map *map)
419	{
420	if (map)
421	kref_put(&map->refcount, fastrpc_free_map);
422	}
423
424	static int fastrpc_map_get(struct fastrpc_map *map)
425	{
426	if (!map)
427	return -ENOENT;
428
429	return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;
430	}
431
432
433	static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
434	struct fastrpc_map **ppmap, bool take_ref)
435	{
436	struct fastrpc_map *map = NULL;
437	struct dma_buf *buf;
438	int ret = -ENOENT;
439
440	buf = dma_buf_get(fd);
441	if (IS_ERR(buf))
442	return PTR_ERR(buf);
443
444	spin_lock(&fl->lock);
445	list_for_each_entry(map, &fl->maps, node) {
446	if (map->fd != fd \|\| map->buf != buf)
447	continue;
448
449	if (take_ref) {
450	ret = fastrpc_map_get(map);
451	if (ret)
452	break;
453	}
454
455	*ppmap = map;
456	ret = 0;
457	break;
458	}
459	spin_unlock(&fl->lock);
460
461	dma_buf_put(buf);
462
463	return ret;
464	}
465
466	static void fastrpc_buf_free(struct fastrpc_buf *buf)
467	{
468	if (!buf)
469	return;
470
471	dma_free_coherent(buf->dev, buf->size, buf->virt,
472	fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr));
473	kfree(buf);
474	}
475
476	static int __fastrpc_buf_alloc(struct fastrpc_user fl, struct device dev,
477	u64 size, struct fastrpc_buf **obuf)
478	{
479	struct fastrpc_buf *buf;
480
481	buf = kzalloc_obj(*buf);
482	if (!buf)
483	return -ENOMEM;
484
485	INIT_LIST_HEAD(&buf->attachments);
486	INIT_LIST_HEAD(&buf->node);
487	mutex_init(&buf->lock);
488
489	buf->fl = fl;
490	buf->virt = NULL;
491	buf->dma_addr = 0;
492	buf->size = size;
493	buf->dev = dev;
494	buf->raddr = 0;
495
496	buf->virt = dma_alloc_coherent(dev, buf->size, &buf->dma_addr,
497	GFP_KERNEL);
498	if (!buf->virt) {
499	mutex_destroy(&buf->lock);
500	kfree(buf);
501	return -ENOMEM;
502	}
503
504	*obuf = buf;
505
506	return 0;
507	}
508
509	static int fastrpc_buf_alloc(struct fastrpc_user fl, struct device dev,
510	u64 size, struct fastrpc_buf **obuf)
511	{
512	int ret;
513	struct fastrpc_buf *buf;
514
515	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
516	if (ret)
517	return ret;
518
519	buf = *obuf;
520
521	if (fl->sctx && fl->sctx->sid)
522	buf->dma_addr += fastrpc_sid_offset(fl->cctx, fl->sctx);
523
524	return 0;
525	}
526
527	static int fastrpc_remote_heap_alloc(struct fastrpc_user fl, struct device dev,
528	u64 size, struct fastrpc_buf **obuf)
529	{
530	struct device *rdev = &fl->cctx->rpdev->dev;
531
532	return __fastrpc_buf_alloc(fl, rdev, size, obuf);
533	}
534
535	static void fastrpc_channel_ctx_free(struct kref *ref)
536	{
537	struct fastrpc_channel_ctx *cctx;
538
539	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
540
541	idr_destroy(&cctx->ctx_idr);
542	kfree(cctx);
543	}
544
545	static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
546	{
547	kref_get(&cctx->refcount);
548	}
549
550	static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
551	{
552	kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
553	}
554
555	static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx);
556
557	static void fastrpc_user_free(struct kref *ref)
558	{
559	struct fastrpc_user *fl = container_of(ref, struct fastrpc_user, refcount);
560	struct fastrpc_invoke_ctx ctx, n;
561	struct fastrpc_map map, m;
562	struct fastrpc_buf buf, b;
563
564	fastrpc_buf_free(fl->init_mem);
565
566	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
567	list_del(&ctx->node);
568	fastrpc_context_put(ctx);
569	}
570
571	list_for_each_entry_safe(map, m, &fl->maps, node)
572	fastrpc_map_put(map);
573
574	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
575	list_del(&buf->node);
576	fastrpc_buf_free(buf);
577	}
578
579	fastrpc_channel_ctx_put(fl->cctx);
580	mutex_destroy(&fl->mutex);
581	kfree(fl);
582	}
583
584	static void fastrpc_user_get(struct fastrpc_user *fl)
585	{
586	kref_get(&fl->refcount);
587	}
588
589	static void fastrpc_user_put(struct fastrpc_user *fl)
590	{
591	kref_put(&fl->refcount, fastrpc_user_free);
592	}
593
594	static void fastrpc_context_free(struct kref *ref)
595	{
596	struct fastrpc_invoke_ctx *ctx;
597	struct fastrpc_channel_ctx *cctx;
598	struct fastrpc_user *fl;
599	unsigned long flags;
600	int i;
601
602	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
603	cctx = ctx->cctx;
604	fl = ctx->fl;
605
606	for (i = 0; i < ctx->nbufs; i++)
607	fastrpc_map_put(ctx->maps[i]);
608
609	fastrpc_buf_free(ctx->buf);
610
611	spin_lock_irqsave(&cctx->lock, flags);
612	idr_remove(&cctx->ctx_idr, FIELD_GET(FASTRPC_CTXID_MASK, ctx->ctxid));
613	spin_unlock_irqrestore(&cctx->lock, flags);
614
615	kfree(ctx->maps);
616	kfree(ctx->olaps);
617	kfree(ctx);
618
619	/* Release the reference taken in fastrpc_context_alloc() */
620	fastrpc_user_put(fl);
621	fastrpc_channel_ctx_put(cctx);
622	}
623
624	static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
625	{
626	kref_get(&ctx->refcount);
627	}
628
629	static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
630	{
631	kref_put(&ctx->refcount, fastrpc_context_free);
632	}
633
634	static void fastrpc_context_put_wq(struct work_struct *work)
635	{
636	struct fastrpc_invoke_ctx *ctx =
637	container_of(work, struct fastrpc_invoke_ctx, put_work);
638
639	fastrpc_context_put(ctx);
640	}
641
642	#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
643	static int olaps_cmp(const void a, const void b)
644	{
645	struct fastrpc_buf_overlap pa = (struct fastrpc_buf_overlap )a;
646	struct fastrpc_buf_overlap pb = (struct fastrpc_buf_overlap )b;
647	/* sort with lowest starting buffer first */
648	int st = CMP(pa->start, pb->start);
649	/* sort with highest ending buffer first */
650	int ed = CMP(pb->end, pa->end);
651
652	return st == 0 ? ed : st;
653	}
654
655	static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
656	{
657	u64 max_end = 0;
658	int i;
659
660	for (i = 0; i < ctx->nbufs; ++i) {
661	ctx->olaps[i].start = ctx->args[i].ptr;
662	ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
663	ctx->olaps[i].raix = i;
664	}
665
666	sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);
667
668	for (i = 0; i < ctx->nbufs; ++i) {
669	/* Falling inside previous range */
670	if (ctx->olaps[i].start < max_end) {
671	ctx->olaps[i].mstart = max_end;
672	ctx->olaps[i].mend = ctx->olaps[i].end;
673	ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
674
675	if (ctx->olaps[i].end > max_end) {
676	max_end = ctx->olaps[i].end;
677	} else {
678	ctx->olaps[i].mend = 0;
679	ctx->olaps[i].mstart = 0;
680	}
681
682	} else {
683	ctx->olaps[i].mend = ctx->olaps[i].end;
684	ctx->olaps[i].mstart = ctx->olaps[i].start;
685	ctx->olaps[i].offset = 0;
686	max_end = ctx->olaps[i].end;
687	}
688	}
689	}
690
691	static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
692	struct fastrpc_user *user, u32 kernel, u32 sc,
693	struct fastrpc_invoke_args *args)
694	{
695	struct fastrpc_channel_ctx *cctx = user->cctx;
696	struct fastrpc_invoke_ctx *ctx = NULL;
697	unsigned long flags;
698	int ret;
699
700	ctx = kzalloc_obj(*ctx);
701	if (!ctx)
702	return ERR_PTR(-ENOMEM);
703
704	INIT_LIST_HEAD(&ctx->node);
705	ctx->fl = user;
706	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
707	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
708	REMOTE_SCALARS_OUTBUFS(sc);
709
710	if (ctx->nscalars) {
711	ctx->maps = kzalloc_objs(*ctx->maps, ctx->nscalars);
712	if (!ctx->maps) {
713	kfree(ctx);
714	return ERR_PTR(-ENOMEM);
715	}
716	ctx->olaps = kzalloc_objs(*ctx->olaps, ctx->nscalars);
717	if (!ctx->olaps) {
718	kfree(ctx->maps);
719	kfree(ctx);
720	return ERR_PTR(-ENOMEM);
721	}
722	ctx->args = args;
723	fastrpc_get_buff_overlaps(ctx);
724	}
725
726	/* Released in fastrpc_context_put() */
727	fastrpc_channel_ctx_get(cctx);
728	/* Take a reference to user, released in fastrpc_context_free() */
729	fastrpc_user_get(user);
730
731	ctx->sc = sc;
732	ctx->retval = -1;
733	ctx->pid = current->pid;
734	ctx->client_id = user->client_id;
735	ctx->cctx = cctx;
736	init_completion(&ctx->work);
737	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
738
739	spin_lock(&user->lock);
740	list_add_tail(&ctx->node, &user->pending);
741	spin_unlock(&user->lock);
742
743	spin_lock_irqsave(&cctx->lock, flags);
744	ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,
745	FASTRPC_CTX_MAX, GFP_ATOMIC);
746	if (ret < 0) {
747	spin_unlock_irqrestore(&cctx->lock, flags);
748	goto err_idr;
749	}
750	ctx->ctxid = FIELD_PREP(FASTRPC_CTXID_MASK, ret) \|
751	FIELD_PREP(FASTRPC_CTXID_SEQ_MASK,
752	(u64)atomic_inc_return(&cctx->ctx_seq));
753	spin_unlock_irqrestore(&cctx->lock, flags);
754
755	kref_init(&ctx->refcount);
756
757	return ctx;
758	err_idr:
759	spin_lock(&user->lock);
760	list_del(&ctx->node);
761	spin_unlock(&user->lock);
762	fastrpc_user_put(user);
763	fastrpc_channel_ctx_put(cctx);
764	kfree(ctx->maps);
765	kfree(ctx->olaps);
766	kfree(ctx);
767
768	return ERR_PTR(ret);
769	}
770
771	static struct sg_table *
772	fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
773	enum dma_data_direction dir)
774	{
775	struct fastrpc_dma_buf_attachment *a = attachment->priv;
776	struct sg_table *table;
777	int ret;
778
779	table = &a->sgt;
780
781	ret = dma_map_sgtable(attachment->dev, table, dir, 0);
782	if (ret)
783	table = ERR_PTR(ret);
784	return table;
785	}
786
787	static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
788	struct sg_table *table,
789	enum dma_data_direction dir)
790	{
791	dma_unmap_sgtable(attach->dev, table, dir, 0);
792	}
793
794	static void fastrpc_release(struct dma_buf *dmabuf)
795	{
796	struct fastrpc_buf *buffer = dmabuf->priv;
797
798	fastrpc_buf_free(buffer);
799	}
800
801	static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
802	struct dma_buf_attachment *attachment)
803	{
804	struct fastrpc_dma_buf_attachment *a;
805	struct fastrpc_buf *buffer = dmabuf->priv;
806	int ret;
807
808	a = kzalloc_obj(*a);
809	if (!a)
810	return -ENOMEM;
811
812	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
813	fastrpc_ipa_to_dma_addr(buffer->fl->cctx, buffer->dma_addr),
814	buffer->size);
815	if (ret < 0) {
816	dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
817	kfree(a);
818	return -EINVAL;
819	}
820
821	a->dev = attachment->dev;
822	INIT_LIST_HEAD(&a->node);
823	attachment->priv = a;
824
825	mutex_lock(&buffer->lock);
826	list_add(&a->node, &buffer->attachments);
827	mutex_unlock(&buffer->lock);
828
829	return 0;
830	}
831
832	static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
833	struct dma_buf_attachment *attachment)
834	{
835	struct fastrpc_dma_buf_attachment *a = attachment->priv;
836	struct fastrpc_buf *buffer = dmabuf->priv;
837
838	mutex_lock(&buffer->lock);
839	list_del(&a->node);
840	mutex_unlock(&buffer->lock);
841	sg_free_table(&a->sgt);
842	kfree(a);
843	}
844
845	static int fastrpc_vmap(struct dma_buf dmabuf, struct iosys_map map)
846	{
847	struct fastrpc_buf *buf = dmabuf->priv;
848
849	iosys_map_set_vaddr(map, buf->virt);
850
851	return 0;
852	}
853
854	static int fastrpc_mmap(struct dma_buf *dmabuf,
855	struct vm_area_struct *vma)
856	{
857	struct fastrpc_buf *buf = dmabuf->priv;
858	size_t size = vma->vm_end - vma->vm_start;
859
860	dma_resv_assert_held(dmabuf->resv);
861
862	return dma_mmap_coherent(buf->dev, vma, buf->virt,
863	fastrpc_ipa_to_dma_addr(buf->fl->cctx, buf->dma_addr), size);
864	}
865
866	static const struct dma_buf_ops fastrpc_dma_buf_ops = {
867	.attach = fastrpc_dma_buf_attach,
868	.detach = fastrpc_dma_buf_detatch,
869	.map_dma_buf = fastrpc_map_dma_buf,
870	.unmap_dma_buf = fastrpc_unmap_dma_buf,
871	.mmap = fastrpc_mmap,
872	.vmap = fastrpc_vmap,
873	.release = fastrpc_release,
874	};
875
876	static dma_addr_t fastrpc_compute_dma_addr(struct fastrpc_user *fl, dma_addr_t sg_dma_addr)
877	{
878	return sg_dma_addr + fastrpc_sid_offset(fl->cctx, fl->sctx);
879	}
880
881	static int fastrpc_map_attach(struct fastrpc_user *fl, int fd,
882	u64 len, u32 attr, struct fastrpc_map **ppmap)
883	{
884	struct fastrpc_session_ctx *sess = fl->sctx;
885	struct fastrpc_map *map = NULL;
886	struct sg_table *table;
887	struct scatterlist *sgl = NULL;
888	int err = 0, sgl_index = 0;
889
890	map = kzalloc_obj(*map);
891	if (!map)
892	return -ENOMEM;
893
894	INIT_LIST_HEAD(&map->node);
895	kref_init(&map->refcount);
896
897	map->fl = fl;
898	map->fd = fd;
899	map->buf = dma_buf_get(fd);
900	if (IS_ERR(map->buf)) {
901	err = PTR_ERR(map->buf);
902	goto get_err;
903	}
904
905	map->attach = dma_buf_attach(map->buf, sess->dev);
906	if (IS_ERR(map->attach)) {
907	dev_err(sess->dev, "Failed to attach dmabuf\n");
908	err = PTR_ERR(map->attach);
909	goto attach_err;
910	}
911
912	table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);
913	if (IS_ERR(table)) {
914	err = PTR_ERR(table);
915	goto map_err;
916	}
917	map->table = table;
918
919	if (attr & FASTRPC_ATTR_SECUREMAP)
920	map->dma_addr = sg_phys(map->table->sgl);
921	else
922	map->dma_addr = fastrpc_compute_dma_addr(fl, sg_dma_address(map->table->sgl));
923	for_each_sg(map->table->sgl, sgl, map->table->nents,
924	sgl_index)
925	map->size += sg_dma_len(sgl);
926	if (len > map->size) {
927	dev_dbg(sess->dev, "Bad size passed len 0x%llx map size 0x%llx\n",
928	len, map->size);
929	err = -EINVAL;
930	goto map_err;
931	}
932	map->va = sg_virt(map->table->sgl);
933	map->len = len;
934
935	if (attr & FASTRPC_ATTR_SECUREMAP) {
936	/*
937	* If subsystem VMIDs are defined in DTSI, then do
938	* hyp_assign from HLOS to those VM(s)
939	*/
940	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
941	struct qcom_scm_vmperm dst_perms[2] = {0};
942
943	dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
944	dst_perms[0].perm = QCOM_SCM_PERM_RW;
945	dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
946	dst_perms[1].perm = QCOM_SCM_PERM_RWX;
947	map->attr = attr;
948	err = qcom_scm_assign_mem(map->dma_addr, (u64)map->len, &src_perms, dst_perms, 2);
949	if (err) {
950	dev_err(sess->dev,
951	"Failed to assign memory with dma_addr %pad size 0x%llx err %d\n",
952	&map->dma_addr, map->len, err);
953	goto map_err;
954	}
955	}
956	spin_lock(&fl->lock);
957	list_add_tail(&map->node, &fl->maps);
958	spin_unlock(&fl->lock);
959	*ppmap = map;
960
961	return 0;
962
963	map_err:
964	dma_buf_detach(map->buf, map->attach);
965	attach_err:
966	dma_buf_put(map->buf);
967	get_err:
968	fastrpc_map_put(map);
969
970	return err;
971	}
972
973	static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
974	u64 len, u32 attr, struct fastrpc_map **ppmap)
975	{
976	if (!fastrpc_map_lookup(fl, fd, ppmap, true))
977	return 0;
978
979	return fastrpc_map_attach(fl, fd, len, attr, ppmap);
980	}
981
982	/*
983	* Fastrpc payload buffer with metadata looks like:
984	*
985	* >>>>>> START of METADATA <<<<<<<<<
986	* +---------------------------------+
987	* \| Arguments \|
988	* \| type:(union fastrpc_remote_arg)\|
989	* \| (0 - N) \|
990	* +---------------------------------+
991	* \| Invoke Buffer list \|
992	* \| type:(struct fastrpc_invoke_buf)\|
993	* \| (0 - N) \|
994	* +---------------------------------+
995	* \| Page info list \|
996	* \| type:(struct fastrpc_phy_page) \|
997	* \| (0 - N) \|
998	* +---------------------------------+
999	* \| Optional info \|
1000	* \|(can be specific to SoC/Firmware)\|
1001	* +---------------------------------+
1002	* >>>>>>>> END of METADATA <<<<<<<<<
1003	* +---------------------------------+
1004	* \| Inline ARGS \|
1005	* \| (0-N) \|
1006	* +---------------------------------+
1007	*/
1008
1009	static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
1010	{
1011	int size = 0;
1012
1013	size = (sizeof(struct fastrpc_remote_buf) +
1014	sizeof(struct fastrpc_invoke_buf) +
1015	sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
1016	sizeof(u64) * FASTRPC_MAX_FDLIST +
1017	sizeof(u32) * FASTRPC_MAX_CRCLIST +
1018	sizeof(u32);
1019
1020	return size;
1021	}
1022
1023	static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
1024	{
1025	u64 size = 0;
1026	int oix;
1027
1028	size = ALIGN(metalen, FASTRPC_ALIGN);
1029	for (oix = 0; oix < ctx->nbufs; oix++) {
1030	int i = ctx->olaps[oix].raix;
1031
1032	if (ctx->args[i].fd == 0 \|\| ctx->args[i].fd == -1) {
1033
1034	if (ctx->olaps[oix].offset == 0)
1035	size = ALIGN(size, FASTRPC_ALIGN);
1036
1037	size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
1038	}
1039	}
1040
1041	return size;
1042	}
1043
1044	static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
1045	{
1046	struct device *dev = ctx->fl->sctx->dev;
1047	int i, err;
1048
1049	for (i = 0; i < ctx->nscalars; ++i) {
1050
1051	if (ctx->args[i].fd == 0 \|\| ctx->args[i].fd == -1 \|\|
1052	ctx->args[i].length == 0)
1053	continue;
1054
1055	if (i < ctx->nbufs)
1056	err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,
1057	ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
1058	else
1059	err = fastrpc_map_attach(ctx->fl, ctx->args[i].fd,
1060	ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);
1061	if (err) {
1062	dev_err(dev, "Error Creating map %d\n", err);
1063	return -EINVAL;
1064	}
1065
1066	}
1067	return 0;
1068	}
1069
1070	static struct fastrpc_invoke_buf fastrpc_invoke_buf_start(union fastrpc_remote_arg pra, int len)
1071	{
1072	return (struct fastrpc_invoke_buf *)(&pra[len]);
1073	}
1074
1075	static struct fastrpc_phy_page fastrpc_phy_page_start(struct fastrpc_invoke_buf buf, int len)
1076	{
1077	return (struct fastrpc_phy_page *)(&buf[len]);
1078	}
1079
1080	static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
1081	{
1082	struct device *dev = ctx->fl->sctx->dev;
1083	union fastrpc_remote_arg *rpra;
1084	struct fastrpc_invoke_buf *list;
1085	struct fastrpc_phy_page *pages;
1086	int inbufs, i, oix, err = 0;
1087	u64 len, rlen, pkt_size;
1088	u64 pg_start, pg_end;
1089	uintptr_t args;
1090	int metalen;
1091
1092	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1093	metalen = fastrpc_get_meta_size(ctx);
1094	pkt_size = fastrpc_get_payload_size(ctx, metalen);
1095
1096	err = fastrpc_create_maps(ctx);
1097	if (err)
1098	return err;
1099
1100	ctx->msg_sz = pkt_size;
1101
1102	if (ctx->fl->sctx->sid)
1103	err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
1104	else
1105	err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf);
1106	if (err)
1107	return err;
1108
1109	memset(ctx->buf->virt, 0, pkt_size);
1110	rpra = ctx->buf->virt;
1111	list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);
1112	pages = fastrpc_phy_page_start(list, ctx->nscalars);
1113	ctx->fdlist = (u64 *)(pages + ctx->nscalars);
1114	ctx->poll_addr = (u32 )((uintptr_t)ctx->fdlist + sizeof(u64) FASTRPC_MAX_FDLIST +
1115	sizeof(u32) * FASTRPC_MAX_CRCLIST);
1116
1117	args = (uintptr_t)ctx->buf->virt + metalen;
1118	rlen = pkt_size - metalen;
1119	ctx->rpra = rpra;
1120
1121	for (oix = 0; oix < ctx->nbufs; ++oix) {
1122	int mlen;
1123
1124	i = ctx->olaps[oix].raix;
1125	len = ctx->args[i].length;
1126
1127	rpra[i].buf.pv = 0;
1128	rpra[i].buf.len = len;
1129	list[i].num = len ? 1 : 0;
1130	list[i].pgidx = i;
1131
1132	if (!len)
1133	continue;
1134
1135	if (ctx->maps[i]) {
1136	struct vm_area_struct *vma = NULL;
1137
1138	rpra[i].buf.pv = (u64) ctx->args[i].ptr;
1139	pages[i].addr = ctx->maps[i]->dma_addr;
1140
1141	mmap_read_lock(current->mm);
1142	vma = vma_lookup(current->mm, ctx->args[i].ptr);
1143	if (vma)
1144	pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) -
1145	vma->vm_start;
1146	mmap_read_unlock(current->mm);
1147
1148	pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
1149	pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
1150	PAGE_SHIFT;
1151	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1152
1153	} else {
1154
1155	if (ctx->olaps[oix].offset == 0) {
1156	rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1157	args = ALIGN(args, FASTRPC_ALIGN);
1158	}
1159
1160	mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1161
1162	if (rlen < mlen)
1163	goto bail;
1164
1165	rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1166	pages[i].addr = ctx->buf->dma_addr -
1167	ctx->olaps[oix].offset +
1168	(pkt_size - rlen);
1169	pages[i].addr = pages[i].addr & PAGE_MASK;
1170
1171	pg_start = (rpra[i].buf.pv & PAGE_MASK) >> PAGE_SHIFT;
1172	pg_end = ((rpra[i].buf.pv + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1173	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1174	args = args + mlen;
1175	rlen -= mlen;
1176	}
1177
1178	if (i < inbufs && !ctx->maps[i]) {
1179	void dst = (void )(uintptr_t)rpra[i].buf.pv;
1180	void src = (void )(uintptr_t)ctx->args[i].ptr;
1181
1182	if (!kernel) {
1183	if (copy_from_user(dst, (void __user *)src,
1184	len)) {
1185	err = -EFAULT;
1186	goto bail;
1187	}
1188	} else {
1189	memcpy(dst, src, len);
1190	}
1191	}
1192	}
1193
1194	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1195	list[i].num = ctx->args[i].length ? 1 : 0;
1196	list[i].pgidx = i;
1197	if (ctx->maps[i]) {
1198	pages[i].addr = ctx->maps[i]->dma_addr;
1199	pages[i].size = ctx->maps[i]->size;
1200	}
1201	rpra[i].dma.fd = ctx->args[i].fd;
1202	rpra[i].dma.len = ctx->args[i].length;
1203	rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1204	}
1205
1206	bail:
1207	if (err)
1208	dev_err(dev, "Error: get invoke args failed:%d\n", err);
1209
1210	return err;
1211	}
1212
1213	static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1214	u32 kernel)
1215	{
1216	union fastrpc_remote_arg *rpra = ctx->rpra;
1217	struct fastrpc_user *fl = ctx->fl;
1218	struct fastrpc_map *mmap = NULL;
1219	u64 *fdlist = ctx->fdlist;
1220	int i, inbufs;
1221	int ret = 0;
1222
1223	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1224
1225	for (i = inbufs; i < ctx->nbufs; ++i) {
1226	if (!ctx->maps[i]) {
1227	void src = (void )(uintptr_t)rpra[i].buf.pv;
1228	void dst = (void )(uintptr_t)ctx->args[i].ptr;
1229	u64 len = rpra[i].buf.len;
1230
1231	if (!kernel) {
1232	if (copy_to_user((void __user *)dst, src, len)) {
1233	ret = -EFAULT;
1234	goto cleanup_fdlist;
1235	}
1236	} else {
1237	memcpy(dst, src, len);
1238	}
1239	}
1240	}
1241
1242	cleanup_fdlist:
1243	/* Clean up fdlist which is updated by DSP */
1244	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1245	if (!fdlist[i])
1246	break;
1247	if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))
1248	fastrpc_map_put(mmap);
1249	}
1250
1251	return ret;
1252	}
1253
1254	static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1255	struct fastrpc_invoke_ctx *ctx,
1256	u32 kernel, uint32_t handle)
1257	{
1258	struct fastrpc_channel_ctx *cctx;
1259	struct fastrpc_user *fl = ctx->fl;
1260	struct fastrpc_msg *msg = &ctx->msg;
1261	int ret;
1262
1263	cctx = fl->cctx;
1264	msg->client_id = fl->client_id;
1265	msg->tid = current->pid;
1266
1267	if (kernel)
1268	msg->client_id = 0;
1269
1270	msg->ctx = ctx->ctxid \| fl->pd;
1271	msg->handle = handle;
1272	msg->sc = ctx->sc;
1273	msg->addr = ctx->buf ? ctx->buf->dma_addr : 0;
1274	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1275	fastrpc_context_get(ctx);
1276
1277	ret = rpmsg_send(cctx->rpdev->ept, (void )msg, sizeof(msg));
1278
1279	if (ret)
1280	fastrpc_context_put(ctx);
1281
1282	return ret;
1283
1284	}
1285
1286	static u32 fastrpc_read_poll_addr(struct fastrpc_invoke_ctx *ctx)
1287	{
1288	dma_rmb();
1289	return READ_ONCE(*ctx->poll_addr);
1290	}
1291
1292	static int poll_for_remote_response(struct fastrpc_invoke_ctx *ctx)
1293	{
1294	u32 val;
1295	int ret;
1296
1297	/*
1298	* Poll until DSP writes FASTRPC_POLL_RESPONSE into *ctx->poll_addr
1299	* or until another path marks the work done.
1300	*/
1301	ret = read_poll_timeout_atomic(fastrpc_read_poll_addr, val,
1302	(val == FASTRPC_POLL_RESPONSE) \|\| ctx->is_work_done, 1,
1303	FASTRPC_POLL_MAX_TIMEOUT_US, false, ctx);
1304
1305	if (!ret && val == FASTRPC_POLL_RESPONSE) {
1306	/*
1307	* DSP writes FASTRPC_POLL_RESPONSE to signal successful
1308	* completion via the poll path.
1309	*/
1310	ctx->is_work_done = true;
1311	ctx->retval = 0;
1312	}
1313
1314	if (ret == -ETIMEDOUT)
1315	ret = -EIO;
1316
1317	return ret;
1318	}
1319
1320	static inline int fastrpc_wait_for_response(struct fastrpc_invoke_ctx *ctx,
1321	u32 kernel)
1322	{
1323	int err = 0;
1324
1325	if (kernel) {
1326	if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))
1327	err = -ETIMEDOUT;
1328	} else {
1329	err = wait_for_completion_interruptible(&ctx->work);
1330	}
1331
1332	return err;
1333	}
1334
1335	static int fastrpc_wait_for_completion(struct fastrpc_invoke_ctx *ctx,
1336	u32 kernel)
1337	{
1338	int err;
1339
1340	if (ctx->is_polled) {
1341	err = poll_for_remote_response(ctx);
1342	if (!err)
1343	return 0;
1344	/* If polling timed out or failed, move to normal response mode */
1345	ctx->is_polled = false;
1346	}
1347
1348	return fastrpc_wait_for_response(ctx, kernel);
1349	}
1350
1351	static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1352	u32 handle, u32 sc,
1353	struct fastrpc_invoke_args *args)
1354	{
1355	struct fastrpc_invoke_ctx *ctx = NULL;
1356	struct fastrpc_buf buf, b;
1357
1358	int err = 0;
1359
1360	if (!fl->sctx)
1361	return -EINVAL;
1362
1363	if (!fl->cctx->rpdev)
1364	return -EPIPE;
1365
1366	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1367	dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
1368	return -EPERM;
1369	}
1370
1371	ctx = fastrpc_context_alloc(fl, kernel, sc, args);
1372	if (IS_ERR(ctx))
1373	return PTR_ERR(ctx);
1374
1375	err = fastrpc_get_args(kernel, ctx);
1376	if (err)
1377	goto bail;
1378
1379	/* make sure that all CPU memory writes are seen by DSP */
1380	dma_wmb();
1381	/* Send invoke buffer to remote dsp */
1382	err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);
1383	if (err)
1384	goto bail;
1385
1386	/*
1387	* Set message context as polled if the call is for a user PD
1388	* dynamic module and user has enabled poll mode.
1389	*/
1390	if (handle > FASTRPC_MAX_STATIC_HANDLE && fl->pd == USER_PD && fl->poll_mode)
1391	ctx->is_polled = true;
1392
1393	err = fastrpc_wait_for_completion(ctx, kernel);
1394	if (err)
1395	goto bail;
1396
1397	/* make sure that all memory writes by DSP are seen by CPU */
1398	dma_rmb();
1399	/* populate all the output buffers with results */
1400	err = fastrpc_put_args(ctx, kernel);
1401	if (err)
1402	goto bail;
1403
1404	/* Check the response from remote dsp */
1405	err = ctx->retval;
1406	if (err)
1407	goto bail;
1408
1409	bail:
1410	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1411	/* We are done with this compute context */
1412	spin_lock(&fl->lock);
1413	list_del(&ctx->node);
1414	spin_unlock(&fl->lock);
1415	fastrpc_context_put(ctx);
1416	}
1417
1418	if (err == -ERESTARTSYS) {
1419	spin_lock(&fl->lock);
1420	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1421	list_del(&buf->node);
1422	list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);
1423	}
1424	spin_unlock(&fl->lock);
1425	}
1426
1427	if (err)
1428	dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1429
1430	return err;
1431	}
1432
1433	static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1434	{
1435	/* Check if the device node is non-secure and channel is secure*/
1436	if (!fl->is_secure_dev && fl->cctx->secure) {
1437	/*
1438	* Allow untrusted applications to offload only to Unsigned PD when
1439	* channel is configured as secure and block untrusted apps on channel
1440	* that does not support unsigned PD offload
1441	*/
1442	if (!fl->cctx->unsigned_support \|\| !unsigned_pd_request) {
1443	dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");
1444	return true;
1445	}
1446	}
1447
1448	return false;
1449	}
1450
1451	static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1452	char __user *argp)
1453	{
1454	struct fastrpc_init_create_static init;
1455	struct fastrpc_invoke_args *args;
1456	struct fastrpc_phy_page pages[1];
1457	char *name;
1458	int err;
1459	bool scm_done = false;
1460	struct {
1461	int client_id;
1462	u32 namelen;
1463	u32 pageslen;
1464	} inbuf;
1465	u32 sc;
1466
1467	args = kzalloc_objs(*args, FASTRPC_CREATE_STATIC_PROCESS_NARGS);
1468	if (!args)
1469	return -ENOMEM;
1470
1471	if (copy_from_user(&init, argp, sizeof(init))) {
1472	err = -EFAULT;
1473	goto err;
1474	}
1475
1476	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1477	err = -EINVAL;
1478	goto err;
1479	}
1480
1481	name = memdup_user(u64_to_user_ptr(init.name), init.namelen);
1482	if (IS_ERR(name)) {
1483	err = PTR_ERR(name);
1484	goto err;
1485	}
1486	inbuf.client_id = fl->client_id;
1487	inbuf.namelen = init.namelen;
1488	inbuf.pageslen = 0;
1489	if (!fl->cctx->remote_heap) {
1490	err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,
1491	&fl->cctx->remote_heap);
1492	if (err)
1493	goto err_name;
1494
1495	/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1496	if (fl->cctx->vmcount) {
1497	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1498
1499	err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr,
1500	(u64)fl->cctx->remote_heap->size,
1501	&src_perms,
1502	fl->cctx->vmperms, fl->cctx->vmcount);
1503	if (err) {
1504	dev_err(fl->sctx->dev,
1505	"Failed to assign memory with dma_addr %pad size 0x%llx err %d\n",
1506	&fl->cctx->remote_heap->dma_addr,
1507	fl->cctx->remote_heap->size, err);
1508	goto err_map;
1509	}
1510	scm_done = true;
1511	inbuf.pageslen = 1;
1512	}
1513	}
1514
1515	fl->pd = USER_PD;
1516
1517	args[0].ptr = (u64)(uintptr_t)&inbuf;
1518	args[0].length = sizeof(inbuf);
1519	args[0].fd = -1;
1520
1521	args[1].ptr = (u64)(uintptr_t)name;
1522	args[1].length = inbuf.namelen;
1523	args[1].fd = -1;
1524
1525	pages[0].addr = fl->cctx->remote_heap->dma_addr;
1526	pages[0].size = fl->cctx->remote_heap->size;
1527
1528	args[2].ptr = (u64)(uintptr_t) pages;
1529	args[2].length = sizeof(*pages);
1530	args[2].fd = -1;
1531
1532	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1533
1534	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1535	sc, args);
1536	if (err)
1537	goto err_invoke;
1538
1539	kfree(args);
1540	kfree(name);
1541
1542	return 0;
1543	err_invoke:
1544	if (fl->cctx->vmcount && scm_done) {
1545	u64 src_perms = 0;
1546	struct qcom_scm_vmperm dst_perms;
1547	u32 i;
1548
1549	for (i = 0; i < fl->cctx->vmcount; i++)
1550	src_perms \|= BIT(fl->cctx->vmperms[i].vmid);
1551
1552	dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1553	dst_perms.perm = QCOM_SCM_PERM_RWX;
1554	err = qcom_scm_assign_mem(fl->cctx->remote_heap->dma_addr,
1555	(u64)fl->cctx->remote_heap->size,
1556	&src_perms, &dst_perms, 1);
1557	if (err)
1558	dev_err(fl->sctx->dev, "Failed to assign memory dma_addr %pad size 0x%llx err %d\n",
1559	&fl->cctx->remote_heap->dma_addr, fl->cctx->remote_heap->size, err);
1560	}
1561	err_map:
1562	fastrpc_buf_free(fl->cctx->remote_heap);
1563	fl->cctx->remote_heap = NULL;
1564	err_name:
1565	kfree(name);
1566	err:
1567	kfree(args);
1568
1569	return err;
1570	}
1571
1572	static int fastrpc_init_create_process(struct fastrpc_user *fl,
1573	char __user *argp)
1574	{
1575	struct fastrpc_init_create init;
1576	struct fastrpc_invoke_args *args;
1577	struct fastrpc_phy_page pages[1];
1578	struct fastrpc_map *map = NULL;
1579	struct fastrpc_buf *imem = NULL;
1580	int memlen;
1581	int err;
1582	struct {
1583	int client_id;
1584	u32 namelen;
1585	u32 filelen;
1586	u32 pageslen;
1587	u32 attrs;
1588	u32 siglen;
1589	} inbuf;
1590	u32 sc;
1591	bool unsigned_module = false;
1592
1593	args = kzalloc_objs(*args, FASTRPC_CREATE_PROCESS_NARGS);
1594	if (!args)
1595	return -ENOMEM;
1596
1597	if (copy_from_user(&init, argp, sizeof(init))) {
1598	err = -EFAULT;
1599	goto err;
1600	}
1601
1602	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1603	unsigned_module = true;
1604
1605	if (is_session_rejected(fl, unsigned_module)) {
1606	err = -ECONNREFUSED;
1607	goto err;
1608	}
1609
1610	if (init.filelen > INIT_FILELEN_MAX) {
1611	err = -EINVAL;
1612	goto err;
1613	}
1614
1615	inbuf.client_id = fl->client_id;
1616	inbuf.namelen = strlen(current->comm) + 1;
1617	inbuf.filelen = init.filelen;
1618	inbuf.pageslen = 1;
1619	inbuf.attrs = init.attrs;
1620	inbuf.siglen = init.siglen;
1621	fl->pd = USER_PD;
1622
1623	if (init.filelen && init.filefd) {
1624	err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);
1625	if (err)
1626	goto err;
1627	}
1628
1629	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1630	1024 * 1024);
1631	err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,
1632	&imem);
1633	if (err)
1634	goto err_alloc;
1635
1636	fl->init_mem = imem;
1637	args[0].ptr = (u64)(uintptr_t)&inbuf;
1638	args[0].length = sizeof(inbuf);
1639	args[0].fd = -1;
1640
1641	args[1].ptr = (u64)(uintptr_t)current->comm;
1642	args[1].length = inbuf.namelen;
1643	args[1].fd = -1;
1644
1645	args[2].ptr = (u64) init.file;
1646	args[2].length = inbuf.filelen;
1647	args[2].fd = init.filefd;
1648
1649	pages[0].addr = imem->dma_addr;
1650	pages[0].size = imem->size;
1651
1652	args[3].ptr = (u64)(uintptr_t) pages;
1653	args[3].length = 1 * sizeof(*pages);
1654	args[3].fd = -1;
1655
1656	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1657	args[4].length = sizeof(inbuf.attrs);
1658	args[4].fd = -1;
1659
1660	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1661	args[5].length = sizeof(inbuf.siglen);
1662	args[5].fd = -1;
1663
1664	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1665	if (init.attrs)
1666	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1667
1668	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1669	sc, args);
1670	if (err)
1671	goto err_invoke;
1672
1673	kfree(args);
1674
1675	return 0;
1676
1677	err_invoke:
1678	fl->init_mem = NULL;
1679	fastrpc_buf_free(imem);
1680	err_alloc:
1681	fastrpc_map_put(map);
1682	err:
1683	kfree(args);
1684
1685	return err;
1686	}
1687
1688	static struct fastrpc_session_ctx *fastrpc_session_alloc(
1689	struct fastrpc_user *fl)
1690	{
1691	struct fastrpc_channel_ctx *cctx = fl->cctx;
1692	struct fastrpc_session_ctx *session = NULL;
1693	unsigned long flags;
1694	int i;
1695
1696	spin_lock_irqsave(&cctx->lock, flags);
1697	for (i = 0; i < cctx->sesscount; i++) {
1698	if (!cctx->session[i].used && cctx->session[i].valid) {
1699	cctx->session[i].used = true;
1700	session = &cctx->session[i];
1701	/* any non-zero ID will work, session_idx + 1 is the simplest one */
1702	fl->client_id = i + 1;
1703	break;
1704	}
1705	}
1706	spin_unlock_irqrestore(&cctx->lock, flags);
1707
1708	return session;
1709	}
1710
1711	static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1712	struct fastrpc_session_ctx *session)
1713	{
1714	unsigned long flags;
1715
1716	spin_lock_irqsave(&cctx->lock, flags);
1717	session->used = false;
1718	spin_unlock_irqrestore(&cctx->lock, flags);
1719	}
1720
1721	static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1722	{
1723	struct fastrpc_invoke_args args[1];
1724	int client_id = 0;
1725	u32 sc;
1726
1727	client_id = fl->client_id;
1728	args[0].ptr = (u64)(uintptr_t) &client_id;
1729	args[0].length = sizeof(client_id);
1730	args[0].fd = -1;
1731	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1732
1733	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1734	sc, &args[0]);
1735	}
1736
1737	static int fastrpc_device_release(struct inode inode, struct file file)
1738	{
1739	struct fastrpc_user fl = (struct fastrpc_user )file->private_data;
1740	struct fastrpc_channel_ctx *cctx = fl->cctx;
1741	unsigned long flags;
1742
1743	fastrpc_release_current_dsp_process(fl);
1744
1745	spin_lock_irqsave(&cctx->lock, flags);
1746	list_del(&fl->user);
1747	spin_unlock_irqrestore(&cctx->lock, flags);
1748
1749	fastrpc_session_free(cctx, fl->sctx);
1750	file->private_data = NULL;
1751	/* Release the reference taken in fastrpc_device_open */
1752	fastrpc_user_put(fl);
1753
1754	return 0;
1755	}
1756
1757	static int fastrpc_device_open(struct inode inode, struct file filp)
1758	{
1759	struct fastrpc_channel_ctx *cctx;
1760	struct fastrpc_device *fdevice;
1761	struct fastrpc_user *fl = NULL;
1762	unsigned long flags;
1763
1764	fdevice = miscdev_to_fdevice(filp->private_data);
1765	cctx = fdevice->cctx;
1766
1767	fl = kzalloc_obj(*fl);
1768	if (!fl)
1769	return -ENOMEM;
1770
1771	/* Released in fastrpc_device_release() */
1772	fastrpc_channel_ctx_get(cctx);
1773
1774	filp->private_data = fl;
1775	spin_lock_init(&fl->lock);
1776	mutex_init(&fl->mutex);
1777	INIT_LIST_HEAD(&fl->pending);
1778	INIT_LIST_HEAD(&fl->maps);
1779	INIT_LIST_HEAD(&fl->mmaps);
1780	INIT_LIST_HEAD(&fl->user);
1781	fl->cctx = cctx;
1782	fl->is_secure_dev = fdevice->secure;
1783
1784	fl->sctx = fastrpc_session_alloc(fl);
1785	if (!fl->sctx) {
1786	dev_err(&cctx->rpdev->dev, "No session available\n");
1787	mutex_destroy(&fl->mutex);
1788	kfree(fl);
1789	fastrpc_channel_ctx_put(cctx);
1790	return -EBUSY;
1791	}
1792
1793	spin_lock_irqsave(&cctx->lock, flags);
1794	list_add_tail(&fl->user, &cctx->users);
1795	spin_unlock_irqrestore(&cctx->lock, flags);
1796	kref_init(&fl->refcount);
1797
1798	return 0;
1799	}
1800
1801	static int fastrpc_dmabuf_alloc(struct fastrpc_user fl, char __user argp)
1802	{
1803	struct fastrpc_alloc_dma_buf bp;
1804	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1805	struct fastrpc_buf *buf = NULL;
1806	int err;
1807
1808	if (copy_from_user(&bp, argp, sizeof(bp)))
1809	return -EFAULT;
1810
1811	err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);
1812	if (err)
1813	return err;
1814	exp_info.ops = &fastrpc_dma_buf_ops;
1815	exp_info.size = bp.size;
1816	exp_info.flags = O_RDWR;
1817	exp_info.priv = buf;
1818	buf->dmabuf = dma_buf_export(&exp_info);
1819	if (IS_ERR(buf->dmabuf)) {
1820	err = PTR_ERR(buf->dmabuf);
1821	fastrpc_buf_free(buf);
1822	return err;
1823	}
1824
1825	bp.fd = get_unused_fd_flags(O_ACCMODE);
1826	if (bp.fd < 0) {
1827	dma_buf_put(buf->dmabuf);
1828	return bp.fd;
1829	}
1830
1831	if (copy_to_user(argp, &bp, sizeof(bp))) {
1832	put_unused_fd(bp.fd);
1833	dma_buf_put(buf->dmabuf);
1834	return -EFAULT;
1835	}
1836
1837	dma_buf_fd_install(buf->dmabuf, bp.fd);
1838
1839	return 0;
1840	}
1841
1842	static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1843	{
1844	struct fastrpc_invoke_args args[1];
1845	int client_id = fl->client_id;
1846	u32 sc;
1847
1848	args[0].ptr = (u64)(uintptr_t) &client_id;
1849	args[0].length = sizeof(client_id);
1850	args[0].fd = -1;
1851	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1852	fl->pd = pd;
1853
1854	return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,
1855	sc, &args[0]);
1856	}
1857
1858	static int fastrpc_invoke(struct fastrpc_user fl, char __user argp)
1859	{
1860	struct fastrpc_invoke_args *args = NULL;
1861	struct fastrpc_invoke inv;
1862	u32 nscalars;
1863	int err;
1864
1865	if (copy_from_user(&inv, argp, sizeof(inv)))
1866	return -EFAULT;
1867
1868	/* nscalars is truncated here to max supported value */
1869	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1870	if (nscalars) {
1871	args = kzalloc_objs(*args, nscalars);
1872	if (!args)
1873	return -ENOMEM;
1874
1875	if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,
1876	nscalars * sizeof(*args))) {
1877	kfree(args);
1878	return -EFAULT;
1879	}
1880	}
1881
1882	err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);
1883	kfree(args);
1884
1885	return err;
1886	}
1887
1888	static int fastrpc_get_info_from_dsp(struct fastrpc_user fl, uint32_t dsp_attr_buf,
1889	uint32_t dsp_attr_buf_len)
1890	{
1891	struct fastrpc_invoke_args args[2] = { 0 };
1892
1893	/*
1894	* Capability filled in userspace. This carries the information
1895	* about the remoteproc support which is fetched from the remoteproc
1896	* sysfs node by userspace.
1897	*/
1898	dsp_attr_buf[0] = 0;
1899	dsp_attr_buf_len -= 1;
1900
1901	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1902	args[0].length = sizeof(dsp_attr_buf_len);
1903	args[0].fd = -1;
1904	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1905	args[1].length = dsp_attr_buf_len * sizeof(u32);
1906	args[1].fd = -1;
1907
1908	return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,
1909	FASTRPC_SCALARS(0, 1, 1), args);
1910	}
1911
1912	static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1913	struct fastrpc_user *fl)
1914	{
1915	struct fastrpc_channel_ctx *cctx = fl->cctx;
1916	uint32_t attribute_id = cap->attribute_id;
1917	uint32_t *dsp_attributes;
1918	unsigned long flags;
1919	int err;
1920
1921	spin_lock_irqsave(&cctx->lock, flags);
1922	/* check if we already have queried dsp for attributes */
1923	if (cctx->valid_attributes) {
1924	spin_unlock_irqrestore(&cctx->lock, flags);
1925	goto done;
1926	}
1927	spin_unlock_irqrestore(&cctx->lock, flags);
1928
1929	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1930	if (!dsp_attributes)
1931	return -ENOMEM;
1932
1933	err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);
1934	if (err == DSP_UNSUPPORTED_API) {
1935	dev_info(&cctx->rpdev->dev,
1936	"Warning: DSP capabilities not supported\n");
1937	kfree(dsp_attributes);
1938	return -EOPNOTSUPP;
1939	} else if (err) {
1940	dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1941	kfree(dsp_attributes);
1942	return err;
1943	}
1944
1945	spin_lock_irqsave(&cctx->lock, flags);
1946	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1947	cctx->valid_attributes = true;
1948	spin_unlock_irqrestore(&cctx->lock, flags);
1949	kfree(dsp_attributes);
1950	done:
1951	cap->capability = cctx->dsp_attributes[attribute_id];
1952	return 0;
1953	}
1954
1955	static int fastrpc_set_option(struct fastrpc_user fl, char __user argp)
1956	{
1957	struct fastrpc_ioctl_set_option opt = {0};
1958	int i;
1959
1960	if (copy_from_user(&opt, argp, sizeof(opt)))
1961	return -EFAULT;
1962
1963	for (i = 0; i < ARRAY_SIZE(opt.reserved); i++) {
1964	if (opt.reserved[i] != 0)
1965	return -EINVAL;
1966	}
1967
1968	if (opt.request_id != FASTRPC_POLL_MODE)
1969	return -EINVAL;
1970
1971	if (!fl->cctx->poll_mode_supported)
1972	return -EOPNOTSUPP;
1973
1974	if (opt.value == FASTRPC_POLL_MODE_ENABLE)
1975	fl->poll_mode = true;
1976	else if (opt.value == FASTRPC_POLL_MODE_DISABLE)
1977	fl->poll_mode = false;
1978	else
1979	return -EINVAL;
1980
1981	return 0;
1982	}
1983
1984	static int fastrpc_get_dsp_info(struct fastrpc_user fl, char __user argp)
1985	{
1986	struct fastrpc_ioctl_capability cap = {0};
1987	int err = 0;
1988
1989	if (copy_from_user(&cap, argp, sizeof(cap)))
1990	return -EFAULT;
1991
1992	cap.capability = 0;
1993
1994	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1995	dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1996	cap.attribute_id, err);
1997	return -EOVERFLOW;
1998	}
1999
2000	err = fastrpc_get_info_from_kernel(&cap, fl);
2001	if (err)
2002	return err;
2003
2004	if (copy_to_user(argp, &cap, sizeof(cap)))
2005	return -EFAULT;
2006
2007	return 0;
2008	}
2009
2010	static int fastrpc_req_munmap_impl(struct fastrpc_user fl, struct fastrpc_buf buf)
2011	{
2012	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
2013	struct fastrpc_munmap_req_msg req_msg;
2014	struct device *dev = fl->sctx->dev;
2015	int err;
2016	u32 sc;
2017
2018	req_msg.client_id = fl->client_id;
2019	req_msg.size = buf->size;
2020	req_msg.vaddr = buf->raddr;
2021
2022	args[0].ptr = (u64) (uintptr_t) &req_msg;
2023	args[0].length = sizeof(req_msg);
2024
2025	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
2026	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
2027	&args[0]);
2028	if (!err) {
2029	dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
2030	fastrpc_buf_free(buf);
2031	} else {
2032	dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
2033	}
2034
2035	return err;
2036	}
2037
2038	static int fastrpc_req_munmap(struct fastrpc_user fl, char __user argp)
2039	{
2040	struct fastrpc_buf buf = NULL, iter, *b;
2041	struct fastrpc_req_munmap req;
2042	struct device *dev = fl->sctx->dev;
2043	int err;
2044
2045	if (copy_from_user(&req, argp, sizeof(req)))
2046	return -EFAULT;
2047
2048	spin_lock(&fl->lock);
2049	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
2050	if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
2051	list_del(&iter->node);
2052	buf = iter;
2053	break;
2054	}
2055	}
2056	spin_unlock(&fl->lock);
2057
2058	if (!buf) {
2059	dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
2060	req.vaddrout, req.size);
2061	return -EINVAL;
2062	}
2063
2064	err = fastrpc_req_munmap_impl(fl, buf);
2065	if (err) {
2066	spin_lock(&fl->lock);
2067	list_add_tail(&buf->node, &fl->mmaps);
2068	spin_unlock(&fl->lock);
2069	}
2070
2071	return err;
2072	}
2073
2074	static int fastrpc_req_mmap(struct fastrpc_user fl, char __user argp)
2075	{
2076	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
2077	struct fastrpc_buf *buf = NULL;
2078	struct fastrpc_mmap_req_msg req_msg;
2079	struct fastrpc_mmap_rsp_msg rsp_msg;
2080	struct fastrpc_phy_page pages;
2081	struct fastrpc_req_mmap req;
2082	struct device *dev = fl->sctx->dev;
2083	int err;
2084	u32 sc;
2085
2086	if (copy_from_user(&req, argp, sizeof(req)))
2087	return -EFAULT;
2088
2089	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
2090	dev_err(dev, "flag not supported 0x%x\n", req.flags);
2091
2092	return -EINVAL;
2093	}
2094
2095	if (req.vaddrin) {
2096	dev_err(dev, "adding user allocated pages is not supported\n");
2097	return -EINVAL;
2098	}
2099
2100	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
2101	err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);
2102	else
2103	err = fastrpc_buf_alloc(fl, dev, req.size, &buf);
2104
2105	if (err) {
2106	dev_err(dev, "failed to allocate buffer\n");
2107	return err;
2108	}
2109
2110	req_msg.client_id = fl->client_id;
2111	req_msg.flags = req.flags;
2112	req_msg.vaddr = req.vaddrin;
2113	req_msg.num = sizeof(pages);
2114
2115	args[0].ptr = (u64) (uintptr_t) &req_msg;
2116	args[0].length = sizeof(req_msg);
2117
2118	pages.addr = buf->dma_addr;
2119	pages.size = buf->size;
2120
2121	args[1].ptr = (u64) (uintptr_t) &pages;
2122	args[1].length = sizeof(pages);
2123
2124	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
2125	args[2].length = sizeof(rsp_msg);
2126
2127	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
2128	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
2129	&args[0]);
2130	if (err) {
2131	dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
2132	fastrpc_buf_free(buf);
2133	return err;
2134	}
2135
2136	/* update the buffer to be able to deallocate the memory on the DSP */
2137	buf->raddr = (uintptr_t) rsp_msg.vaddr;
2138
2139	/* let the client know the address to use */
2140	req.vaddrout = rsp_msg.vaddr;
2141
2142	/* Add memory to static PD pool, protection thru hypervisor */
2143	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
2144	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
2145
2146	err = qcom_scm_assign_mem(buf->dma_addr, (u64)buf->size,
2147	&src_perms, fl->cctx->vmperms, fl->cctx->vmcount);
2148	if (err) {
2149	dev_err(fl->sctx->dev,
2150	"Failed to assign memory dma_addr %pad size 0x%llx err %d",
2151	&buf->dma_addr, buf->size, err);
2152	goto err_assign;
2153	}
2154	}
2155
2156	spin_lock(&fl->lock);
2157	list_add_tail(&buf->node, &fl->mmaps);
2158	spin_unlock(&fl->lock);
2159
2160	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2161	err = -EFAULT;
2162	goto err_assign;
2163	}
2164
2165	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
2166	buf->raddr, buf->size);
2167
2168	return 0;
2169
2170	err_assign:
2171	fastrpc_req_munmap_impl(fl, buf);
2172
2173	return err;
2174	}
2175
2176	static int fastrpc_req_mem_unmap_impl(struct fastrpc_user fl, struct fastrpc_mem_unmap req)
2177	{
2178	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
2179	struct fastrpc_map map = NULL, iter, *m;
2180	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
2181	int err = 0;
2182	u32 sc;
2183	struct device *dev = fl->sctx->dev;
2184
2185	spin_lock(&fl->lock);
2186	list_for_each_entry_safe(iter, m, &fl->maps, node) {
2187	if ((req->fd < 0 \|\| iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
2188	map = iter;
2189	break;
2190	}
2191	}
2192
2193	spin_unlock(&fl->lock);
2194
2195	if (!map) {
2196	dev_err(dev, "map not in list\n");
2197	return -EINVAL;
2198	}
2199
2200	req_msg.client_id = fl->client_id;
2201	req_msg.len = map->len;
2202	req_msg.vaddrin = map->raddr;
2203	req_msg.fd = map->fd;
2204
2205	args[0].ptr = (u64) (uintptr_t) &req_msg;
2206	args[0].length = sizeof(req_msg);
2207
2208	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
2209	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,
2210	&args[0]);
2211	if (err) {
2212	dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);
2213	return err;
2214	}
2215	fastrpc_map_put(map);
2216
2217	return 0;
2218	}
2219
2220	static int fastrpc_req_mem_unmap(struct fastrpc_user fl, char __user argp)
2221	{
2222	struct fastrpc_mem_unmap req;
2223
2224	if (copy_from_user(&req, argp, sizeof(req)))
2225	return -EFAULT;
2226
2227	return fastrpc_req_mem_unmap_impl(fl, &req);
2228	}
2229
2230	static int fastrpc_req_mem_map(struct fastrpc_user fl, char __user argp)
2231	{
2232	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2233	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2234	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2235	struct fastrpc_mem_unmap req_unmap = { 0 };
2236	struct fastrpc_phy_page pages = { 0 };
2237	struct fastrpc_mem_map req;
2238	struct device *dev = fl->sctx->dev;
2239	struct fastrpc_map *map = NULL;
2240	int err;
2241	u32 sc;
2242
2243	if (copy_from_user(&req, argp, sizeof(req)))
2244	return -EFAULT;
2245
2246	/* create SMMU mapping */
2247	err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);
2248	if (err) {
2249	dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2250	return err;
2251	}
2252
2253	req_msg.client_id = fl->client_id;
2254	req_msg.fd = req.fd;
2255	req_msg.offset = req.offset;
2256	req_msg.vaddrin = req.vaddrin;
2257	map->va = (void *) (uintptr_t) req.vaddrin;
2258	req_msg.flags = req.flags;
2259	req_msg.num = sizeof(pages);
2260	req_msg.data_len = 0;
2261
2262	args[0].ptr = (u64) (uintptr_t) &req_msg;
2263	args[0].length = sizeof(req_msg);
2264
2265	pages.addr = map->dma_addr;
2266	pages.size = map->len;
2267
2268	args[1].ptr = (u64) (uintptr_t) &pages;
2269	args[1].length = sizeof(pages);
2270
2271	args[2].ptr = (u64) (uintptr_t) &pages;
2272	args[2].length = 0;
2273
2274	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2275	args[3].length = sizeof(rsp_msg);
2276
2277	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2278	err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);
2279	if (err) {
2280	dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2281	req.fd, req.vaddrin, map->len);
2282	goto err_invoke;
2283	}
2284
2285	/* update the buffer to be able to deallocate the memory on the DSP */
2286	map->raddr = rsp_msg.vaddr;
2287
2288	/* let the client know the address to use */
2289	req.vaddrout = rsp_msg.vaddr;
2290
2291	if (copy_to_user((void __user *)argp, &req, sizeof(req))) {
2292	/* unmap the memory and release the buffer */
2293	req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2294	req_unmap.length = map->len;
2295	fastrpc_req_mem_unmap_impl(fl, &req_unmap);
2296	return -EFAULT;
2297	}
2298
2299	return 0;
2300
2301	err_invoke:
2302	fastrpc_map_put(map);
2303
2304	return err;
2305	}
2306
2307	static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2308	unsigned long arg)
2309	{
2310	struct fastrpc_user fl = (struct fastrpc_user )file->private_data;
2311	char __user argp = (char __user )arg;
2312	int err;
2313
2314	switch (cmd) {
2315	case FASTRPC_IOCTL_INVOKE:
2316	err = fastrpc_invoke(fl, argp);
2317	break;
2318	case FASTRPC_IOCTL_INIT_ATTACH:
2319	err = fastrpc_init_attach(fl, ROOT_PD);
2320	break;
2321	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2322	err = fastrpc_init_attach(fl, SENSORS_PD);
2323	break;
2324	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2325	err = fastrpc_init_create_static_process(fl, argp);
2326	break;
2327	case FASTRPC_IOCTL_INIT_CREATE:
2328	err = fastrpc_init_create_process(fl, argp);
2329	break;
2330	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2331	err = fastrpc_dmabuf_alloc(fl, argp);
2332	break;
2333	case FASTRPC_IOCTL_MMAP:
2334	err = fastrpc_req_mmap(fl, argp);
2335	break;
2336	case FASTRPC_IOCTL_MUNMAP:
2337	err = fastrpc_req_munmap(fl, argp);
2338	break;
2339	case FASTRPC_IOCTL_MEM_MAP:
2340	err = fastrpc_req_mem_map(fl, argp);
2341	break;
2342	case FASTRPC_IOCTL_MEM_UNMAP:
2343	err = fastrpc_req_mem_unmap(fl, argp);
2344	break;
2345	case FASTRPC_IOCTL_SET_OPTION:
2346	err = fastrpc_set_option(fl, argp);
2347	break;
2348	case FASTRPC_IOCTL_GET_DSP_INFO:
2349	err = fastrpc_get_dsp_info(fl, argp);
2350	break;
2351	default:
2352	err = -ENOTTY;
2353	break;
2354	}
2355
2356	return err;
2357	}
2358
2359	static const struct file_operations fastrpc_fops = {
2360	.open = fastrpc_device_open,
2361	.release = fastrpc_device_release,
2362	.unlocked_ioctl = fastrpc_device_ioctl,
2363	.compat_ioctl = fastrpc_device_ioctl,
2364	};
2365
2366	static int fastrpc_cb_probe(struct platform_device *pdev)
2367	{
2368	struct fastrpc_channel_ctx *cctx;
2369	struct fastrpc_session_ctx *sess;
2370	struct device *dev = &pdev->dev;
2371	int i, sessions = 0;
2372	unsigned long flags;
2373	u32 dma_bits;
2374	u32 sid = 0;
2375	int rc;
2376
2377	cctx = dev_get_drvdata(dev->parent);
2378	if (!cctx)
2379	return -EINVAL;
2380
2381	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
2382	if (of_property_read_u32(dev->of_node, "reg", &sid))
2383	dev_info(dev, "FastRPC Session ID not specified in DT\n");
2384
2385	spin_lock_irqsave(&cctx->lock, flags);
2386	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2387	spin_unlock_irqrestore(&cctx->lock, flags);
2388	dev_err(&pdev->dev, "too many sessions\n");
2389	return -ENOSPC;
2390	}
2391	dma_bits = cctx->soc_data->dma_addr_bits_default;
2392	sess = &cctx->session[cctx->sesscount++];
2393	sess->used = false;
2394	sess->valid = true;
2395	sess->dev = dev;
2396	dev_set_drvdata(dev, sess);
2397	sess->sid = sid;
2398
2399	if (cctx->domain_id == CDSP_DOMAIN_ID)
2400	dma_bits = cctx->soc_data->dma_addr_bits_cdsp;
2401
2402	if (sessions > 0) {
2403	struct fastrpc_session_ctx *dup_sess;
2404
2405	for (i = 1; i < sessions; i++) {
2406	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2407	break;
2408	dup_sess = &cctx->session[cctx->sesscount++];
2409	memcpy(dup_sess, sess, sizeof(*dup_sess));
2410	}
2411	}
2412	spin_unlock_irqrestore(&cctx->lock, flags);
2413	rc = dma_set_mask(dev, DMA_BIT_MASK(dma_bits));
2414	if (rc) {
2415	dev_err(dev, "%u-bit DMA enable failed\n", dma_bits);
2416	return rc;
2417	}
2418
2419	return 0;
2420	}
2421
2422	static void fastrpc_cb_remove(struct platform_device *pdev)
2423	{
2424	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);
2425	struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);
2426	unsigned long flags;
2427	int i;
2428
2429	spin_lock_irqsave(&cctx->lock, flags);
2430	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2431	if (cctx->session[i].sid == sess->sid) {
2432	cctx->session[i].valid = false;
2433	cctx->sesscount--;
2434	}
2435	}
2436	spin_unlock_irqrestore(&cctx->lock, flags);
2437	}
2438
2439	static const struct of_device_id fastrpc_match_table[] = {
2440	{ .compatible = "qcom,fastrpc-compute-cb" },
2441	{ }
2442	};
2443
2444	static struct platform_driver fastrpc_cb_driver = {
2445	.probe = fastrpc_cb_probe,
2446	.remove = fastrpc_cb_remove,
2447	.driver = {
2448	.name = "qcom,fastrpc-cb",
2449	.of_match_table = fastrpc_match_table,
2450	.suppress_bind_attrs = true,
2451	},
2452	};
2453
2454	static int fastrpc_device_register(struct device dev, struct fastrpc_channel_ctx cctx,
2455	bool is_secured, const char *domain)
2456	{
2457	struct fastrpc_device *fdev;
2458	int err;
2459
2460	fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);
2461	if (!fdev)
2462	return -ENOMEM;
2463
2464	fdev->secure = is_secured;
2465	fdev->cctx = cctx;
2466	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2467	fdev->miscdev.fops = &fastrpc_fops;
2468	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",
2469	domain, is_secured ? "-secure" : "");
2470	if (!fdev->miscdev.name)
2471	return -ENOMEM;
2472
2473	err = misc_register(&fdev->miscdev);
2474	if (!err) {
2475	if (is_secured)
2476	cctx->secure_fdevice = fdev;
2477	else
2478	cctx->fdevice = fdev;
2479	}
2480
2481	return err;
2482	}
2483
2484	static int fastrpc_get_domain_id(const char *domain)
2485	{
2486	if (!strncmp(domain, "adsp", 4))
2487	return ADSP_DOMAIN_ID;
2488	else if (!strncmp(domain, "cdsp", 4))
2489	return CDSP_DOMAIN_ID;
2490	else if (!strncmp(domain, "mdsp", 4))
2491	return MDSP_DOMAIN_ID;
2492	else if (!strncmp(domain, "sdsp", 4))
2493	return SDSP_DOMAIN_ID;
2494	else if (!strncmp(domain, "gdsp", 4))
2495	return GDSP_DOMAIN_ID;
2496
2497	return -EINVAL;
2498	}
2499
2500	static const struct fastrpc_soc_data kaanapali_soc_data = {
2501	.sid_pos = 56,
2502	.dma_addr_bits_cdsp = 34,
2503	.dma_addr_bits_default = 32,
2504	.poll_mode_supported = true,
2505	};
2506
2507	static const struct fastrpc_soc_data default_soc_data = {
2508	.sid_pos = 32,
2509	.dma_addr_bits_cdsp = 32,
2510	.dma_addr_bits_default = 32,
2511	};
2512
2513	/*
2514	* Exception list for older platforms that use default_soc_data but whose
2515	* DSP firmware supports FastRPC polling mode.
2516	*
2517	* NOTE: This list is intentionally closed.
2518	* Do NOT add new platforms here. New SoCs must advertise polling mode
2519	* support via their soc_data.
2520	*/
2521
2522	static const struct of_device_id fastrpc_poll_supported_machines[] __maybe_unused = {
2523	{ .compatible = "qcom,milos" },
2524	{ .compatible = "qcom,qcs8300" },
2525	{ .compatible = "qcom,sa8775p" },
2526	{ .compatible = "qcom,sar2130p" },
2527	{ .compatible = "qcom,sm8450" },
2528	{ .compatible = "qcom,sm8550" },
2529	{ .compatible = "qcom,sm8650" },
2530	{ .compatible = "qcom,sm8750" },
2531	{ .compatible = "qcom,x1e80100" },
2532	{ .compatible = "qcom,x1p42100" },
2533	{},
2534	};
2535
2536	static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2537	{
2538	struct device *rdev = &rpdev->dev;
2539	struct fastrpc_channel_ctx *data;
2540	int i, err, domain_id = -1, vmcount;
2541	const char *domain;
2542	bool secure_dsp;
2543	unsigned int vmids[FASTRPC_MAX_VMIDS];
2544	const struct fastrpc_soc_data *soc_data;
2545
2546	soc_data = device_get_match_data(rdev);
2547
2548	err = of_property_read_string(rdev->of_node, "label", &domain);
2549	if (err) {
2550	dev_info(rdev, "FastRPC Domain not specified in DT\n");
2551	return err;
2552	}
2553
2554	domain_id = fastrpc_get_domain_id(domain);
2555
2556	if (domain_id < 0) {
2557	dev_info(rdev, "FastRPC Domain %s not supported\n", domain);
2558	return -EINVAL;
2559	}
2560
2561	if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))
2562	dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2563
2564	vmcount = of_property_read_variable_u32_array(rdev->of_node,
2565	"qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);
2566	if (vmcount < 0)
2567	vmcount = 0;
2568	else if (!qcom_scm_is_available())
2569	return -EPROBE_DEFER;
2570
2571	data = kzalloc_obj(*data);
2572	if (!data)
2573	return -ENOMEM;
2574
2575	if (vmcount) {
2576	data->vmcount = vmcount;
2577	for (i = 0; i < data->vmcount; i++) {
2578	data->vmperms[i].vmid = vmids[i];
2579	data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2580	}
2581	}
2582
2583	if (domain_id == SDSP_DOMAIN_ID) {
2584	struct resource res;
2585	u64 src_perms;
2586
2587	err = of_reserved_mem_region_to_resource(rdev->of_node, 0, &res);
2588	if (!err) {
2589	src_perms = BIT(QCOM_SCM_VMID_HLOS);
2590
2591	err = qcom_scm_assign_mem(res.start, resource_size(&res), &src_perms,
2592	data->vmperms, data->vmcount);
2593	if (err)
2594	goto err_free_data;
2595	}
2596
2597	}
2598
2599	secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));
2600	data->secure = secure_dsp;
2601	data->soc_data = soc_data;
2602	data->poll_mode_supported = soc_data->poll_mode_supported \|\|
2603	of_machine_get_match(fastrpc_poll_supported_machines);
2604
2605	switch (domain_id) {
2606	case ADSP_DOMAIN_ID:
2607	case MDSP_DOMAIN_ID:
2608	case SDSP_DOMAIN_ID:
2609	/* Unsigned PD offloading is only supported on CDSP and GDSP */
2610	data->unsigned_support = false;
2611	err = fastrpc_device_register(rdev, data, secure_dsp, domain);
2612	if (err)
2613	goto err_free_data;
2614	break;
2615	case CDSP_DOMAIN_ID:
2616	case GDSP_DOMAIN_ID:
2617	data->unsigned_support = true;
2618	/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2619	err = fastrpc_device_register(rdev, data, true, domain);
2620	if (err)
2621	goto err_free_data;
2622
2623	err = fastrpc_device_register(rdev, data, false, domain);
2624	if (err)
2625	goto err_deregister_fdev;
2626	break;
2627	default:
2628	err = -EINVAL;
2629	goto err_free_data;
2630	}
2631
2632	kref_init(&data->refcount);
2633	atomic_set(&data->ctx_seq, 0);
2634
2635	rdev->dma_mask = &data->dma_mask;
2636	dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
2637	INIT_LIST_HEAD(&data->users);
2638	INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);
2639	spin_lock_init(&data->lock);
2640	idr_init(&data->ctx_idr);
2641	data->domain_id = domain_id;
2642	data->rpdev = rpdev;
2643	dev_set_drvdata(&rpdev->dev, data);
2644
2645	err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
2646	if (err)
2647	goto err_deregister_fdev;
2648
2649	return 0;
2650
2651	err_deregister_fdev:
2652	if (data->fdevice)
2653	misc_deregister(&data->fdevice->miscdev);
2654	if (data->secure_fdevice)
2655	misc_deregister(&data->secure_fdevice->miscdev);
2656
2657	err_free_data:
2658	kfree(data);
2659	return err;
2660	}
2661
2662	static void fastrpc_notify_users(struct fastrpc_user *user)
2663	{
2664	struct fastrpc_invoke_ctx *ctx;
2665
2666	spin_lock(&user->lock);
2667	list_for_each_entry(ctx, &user->pending, node) {
2668	ctx->retval = -EPIPE;
2669	complete(&ctx->work);
2670	}
2671	spin_unlock(&user->lock);
2672	}
2673
2674	static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2675	{
2676	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2677	struct fastrpc_buf buf, b;
2678	struct fastrpc_user *user;
2679	unsigned long flags;
2680
2681	/* No invocations past this point */
2682	spin_lock_irqsave(&cctx->lock, flags);
2683	cctx->rpdev = NULL;
2684	list_for_each_entry(user, &cctx->users, user)
2685	fastrpc_notify_users(user);
2686	spin_unlock_irqrestore(&cctx->lock, flags);
2687
2688	if (cctx->fdevice)
2689	misc_deregister(&cctx->fdevice->miscdev);
2690
2691	if (cctx->secure_fdevice)
2692	misc_deregister(&cctx->secure_fdevice->miscdev);
2693
2694	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2695	list_del(&buf->node);
2696
2697	if (cctx->remote_heap)
2698	fastrpc_buf_free(cctx->remote_heap);
2699
2700	of_platform_depopulate(&rpdev->dev);
2701
2702	fastrpc_channel_ctx_put(cctx);
2703	}
2704
2705	static int fastrpc_rpmsg_callback(struct rpmsg_device rpdev, void data,
2706	int len, void *priv, u32 addr)
2707	{
2708	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);
2709	struct fastrpc_invoke_rsp *rsp = data;
2710	struct fastrpc_invoke_ctx *ctx;
2711	unsigned long flags;
2712	unsigned long ctxid;
2713
2714	if (len < sizeof(*rsp))
2715	return -EINVAL;
2716
2717	if (!cctx)
2718	return -ENODEV;
2719
2720	/*
2721	* A PD exit notification from the DSP PD notification framework carries
2722	* this sentinel rather than a real context. Drop it: a real context is
2723	* (idr_index << 4) \| pd and can never collide with this value.
2724	*/
2725	if (rsp->ctx == FASTRPC_DSP_PD_NOTIFY_CTX)
2726	return 0;
2727
2728	ctxid = FIELD_GET(FASTRPC_CTXID_MASK, rsp->ctx);
2729
2730	spin_lock_irqsave(&cctx->lock, flags);
2731	ctx = idr_find(&cctx->ctx_idr, ctxid);
2732	spin_unlock_irqrestore(&cctx->lock, flags);
2733
2734	if (!ctx) {
2735	dev_err(&rpdev->dev, "No context ID matches response\n");
2736	return -ENOENT;
2737	}
2738
2739	/*
2740	* Validate the sequence number embedded in the upper bits of the
2741	* context ID. Under high concurrency the IDR slot can be recycled
2742	* for a new request before a late (or duplicate) glink COMPLETE
2743	* response for the previous request arrives. Without this check the
2744	* driver would call complete() on the wrong context, waking a thread
2745	* whose buffers are still being accessed by the DSP.
2746	*/
2747	if (FIELD_GET(FASTRPC_CTXID_SEQ_MASK, rsp->ctx) !=
2748	FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid)) {
2749	dev_dbg(&rpdev->dev,
2750	"Stale glink response ctx 0x%llx (expected seq 0x%llx), dropping\n",
2751	rsp->ctx,
2752	FIELD_GET(FASTRPC_CTXID_SEQ_MASK, ctx->ctxid));
2753	return 0;
2754	}
2755
2756	ctx->retval = rsp->retval;
2757	ctx->is_work_done = true;
2758	complete(&ctx->work);
2759
2760	/*
2761	* The DMA buffer associated with the context cannot be freed in
2762	* interrupt context so schedule it through a worker thread to
2763	* avoid a kernel BUG.
2764	*/
2765	schedule_work(&ctx->put_work);
2766
2767	return 0;
2768	}
2769
2770	static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2771	{ .compatible = "qcom,kaanapali-fastrpc", .data = &kaanapali_soc_data },
2772	{ .compatible = "qcom,fastrpc", .data = &default_soc_data },
2773	{ }
2774	};
2775	MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2776
2777	static struct rpmsg_driver fastrpc_driver = {
2778	.probe = fastrpc_rpmsg_probe,
2779	.remove = fastrpc_rpmsg_remove,
2780	.callback = fastrpc_rpmsg_callback,
2781	.drv = {
2782	.name = "qcom,fastrpc",
2783	.of_match_table = fastrpc_rpmsg_of_match,
2784	},
2785	};
2786
2787	static int fastrpc_init(void)
2788	{
2789	int ret;
2790
2791	ret = platform_driver_register(&fastrpc_cb_driver);
2792	if (ret < 0) {
2793	pr_err("fastrpc: failed to register cb driver\n");
2794	return ret;
2795	}
2796
2797	ret = register_rpmsg_driver(&fastrpc_driver);
2798	if (ret < 0) {
2799	pr_err("fastrpc: failed to register rpmsg driver\n");
2800	platform_driver_unregister(&fastrpc_cb_driver);
2801	return ret;
2802	}
2803
2804	return 0;
2805	}
2806	module_init(fastrpc_init);
2807
2808	static void fastrpc_exit(void)
2809	{
2810	platform_driver_unregister(&fastrpc_cb_driver);
2811	unregister_rpmsg_driver(&fastrpc_driver);
2812	}
2813	module_exit(fastrpc_exit);
2814
2815	MODULE_DESCRIPTION("Qualcomm FastRPC");
2816	MODULE_LICENSE("GPL v2");
2817	MODULE_IMPORT_NS("DMA_BUF");