projects/linux·blob master · refs

1986 lines · 49.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2016-2017, Linaro Ltd
4 */
5
6#include <linux/idr.h>
7#include <linux/interrupt.h>
8#include <linux/io.h>
9#include <linux/list.h>
10#include <linux/mfd/syscon.h>
11#include <linux/module.h>
12#include <linux/of.h>
13#include <linux/of_address.h>
14#include <linux/platform_device.h>
15#include <linux/regmap.h>
16#include <linux/rpmsg.h>
17#include <linux/sizes.h>
18#include <linux/slab.h>
19#include <linux/wait.h>
20#include <linux/workqueue.h>
21#include <linux/mailbox_client.h>
22
23#include "rpmsg_internal.h"
24#include "qcom_glink_native.h"
25
26#define CREATE_TRACE_POINTS
27#include "qcom_glink_trace.h"
28
29#define GLINK_NAME_SIZE 32
30#define GLINK_VERSION_1 1
31
32#define RPM_GLINK_CID_MIN 1
33#define RPM_GLINK_CID_MAX 65536
34
35struct glink_msg {
36 /* New members MUST be added within the __struct_group() macro below. */
37 __struct_group(glink_msg_hdr, hdr, __packed,
38 __le16 cmd;
39 __le16 param1;
40 __le32 param2;
41 );
42 u8 data[];
43} __packed;
44static_assert(offsetof(struct glink_msg, data) == sizeof(struct glink_msg_hdr),
45 "struct member likely outside of __struct_group()");
46
47/**
48 * struct glink_defer_cmd - deferred incoming control message
49 * @node: list node
50 * @msg: message header
51 * @data: payload of the message
52 *
53 * Copy of a received control message, to be added to @rx_queue and processed
54 * by @rx_work of @qcom_glink.
55 */
56struct glink_defer_cmd {
57 struct list_head node;
58
59 struct glink_msg_hdr msg;
60 u8 data[];
61};
62
63/**
64 * struct glink_core_rx_intent - RX intent
65 * RX intent
66 *
67 * @data: pointer to the data (may be NULL for zero-copy)
68 * @id: remote or local intent ID
69 * @size: size of the original intent (do not modify)
70 * @reuse: To mark if the intent can be reused after first use
71 * @in_use: To mark if intent is already in use for the channel
72 * @offset: next write offset (initially 0)
73 * @node: list node
74 */
75struct glink_core_rx_intent {
76 void *data;
77 u32 id;
78 size_t size;
79 bool reuse;
80 bool in_use;
81 u32 offset;
82
83 struct list_head node;
84};
85
86/**
87 * struct qcom_glink - driver context, relates to one remote subsystem
88 * @dev: reference to the associated struct device
89 * @label: identifier of the glink edge
90 * @rx_pipe: pipe object for receive FIFO
91 * @tx_pipe: pipe object for transmit FIFO
92 * @rx_work: worker for handling received control messages
93 * @rx_lock: protects the @rx_queue
94 * @rx_queue: queue of received control messages to be processed in @rx_work
95 * @tx_lock: synchronizes operations on the tx fifo
96 * @idr_lock: synchronizes @lcids and @rcids modifications
97 * @lcids: idr of all channels with a known local channel id
98 * @rcids: idr of all channels with a known remote channel id
99 * @features: remote features
100 * @intentless: flag to indicate that there is no intent
101 * @tx_avail_notify: Waitqueue for pending tx tasks
102 * @sent_read_notify: flag to check cmd sent or not
103 * @abort_tx: flag indicating that all tx attempts should fail
104 */
105struct qcom_glink {
106 struct device *dev;
107
108 const char *label;
109
110 struct qcom_glink_pipe *rx_pipe;
111 struct qcom_glink_pipe *tx_pipe;
112
113 struct work_struct rx_work;
114 spinlock_t rx_lock;
115 struct list_head rx_queue;
116
117 spinlock_t tx_lock;
118
119 spinlock_t idr_lock;
120 struct idr lcids;
121 struct idr rcids;
122 unsigned long features;
123
124 bool intentless;
125 wait_queue_head_t tx_avail_notify;
126 bool sent_read_notify;
127
128 bool abort_tx;
129};
130
131enum {
132 GLINK_STATE_CLOSED,
133 GLINK_STATE_OPENING,
134 GLINK_STATE_OPEN,
135 GLINK_STATE_CLOSING,
136};
137
138/**
139 * struct glink_channel - internal representation of a channel
140 * @rpdev: rpdev reference, only used for primary endpoints
141 * @ept: rpmsg endpoint this channel is associated with
142 * @glink: qcom_glink context handle
143 * @refcount: refcount for the channel object
144 * @recv_lock: guard for @ept.cb
145 * @name: unique channel name/identifier
146 * @lcid: channel id, in local space
147 * @rcid: channel id, in remote space
148 * @intent_lock: lock for protection of @liids, @riids
149 * @liids: idr of all local intents
150 * @riids: idr of all remote intents
151 * @intent_work: worker responsible for transmitting rx_done packets
152 * @done_intents: list of intents that needs to be announced rx_done
153 * @buf: receive buffer, for gathering fragments
154 * @buf_offset: write offset in @buf
155 * @buf_size: size of current @buf
156 * @open_ack: completed once remote has acked the open-request
157 * @open_req: completed once open-request has been received
158 * @intent_req_lock: Synchronises multiple intent requests
159 * @intent_req_result: Result of intent request
160 * @intent_received: flag indicating that an intent has been received
161 * @intent_req_wq: wait queue for intent_req signalling
162 */
163struct glink_channel {
164 struct rpmsg_endpoint ept;
165
166 struct rpmsg_device *rpdev;
167 struct qcom_glink *glink;
168
169 struct kref refcount;
170
171 spinlock_t recv_lock;
172
173 char *name;
174 unsigned int lcid;
175 unsigned int rcid;
176
177 spinlock_t intent_lock;
178 struct idr liids;
179 struct idr riids;
180 struct work_struct intent_work;
181 struct list_head done_intents;
182
183 struct glink_core_rx_intent *buf;
184 int buf_offset;
185 int buf_size;
186
187 struct completion open_ack;
188 struct completion open_req;
189
190 struct mutex intent_req_lock;
191 int intent_req_result;
192 bool intent_received;
193 wait_queue_head_t intent_req_wq;
194};
195
196#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
197
198static const struct rpmsg_endpoint_ops glink_endpoint_ops;
199
200#define GLINK_CMD_VERSION 0
201#define GLINK_CMD_VERSION_ACK 1
202#define GLINK_CMD_OPEN 2
203#define GLINK_CMD_CLOSE 3
204#define GLINK_CMD_OPEN_ACK 4
205#define GLINK_CMD_INTENT 5
206#define GLINK_CMD_RX_DONE 6
207#define GLINK_CMD_RX_INTENT_REQ 7
208#define GLINK_CMD_RX_INTENT_REQ_ACK 8
209#define GLINK_CMD_TX_DATA 9
210#define GLINK_CMD_CLOSE_ACK 11
211#define GLINK_CMD_TX_DATA_CONT 12
212#define GLINK_CMD_READ_NOTIF 13
213#define GLINK_CMD_RX_DONE_W_REUSE 14
214#define GLINK_CMD_SIGNALS 15
215
216#define GLINK_FEATURE_INTENTLESS BIT(1)
217
218#define NATIVE_DTR_SIG NATIVE_DSR_SIG
219#define NATIVE_DSR_SIG BIT(31)
220#define NATIVE_RTS_SIG NATIVE_CTS_SIG
221#define NATIVE_CTS_SIG BIT(30)
222
223static void qcom_glink_rx_done_work(struct work_struct *work);
224
225static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
226 const char *name)
227{
228 struct glink_channel *channel;
229
230 channel = kzalloc_obj(*channel);
231 if (!channel)
232 return ERR_PTR(-ENOMEM);
233
234 /* Setup glink internal glink_channel data */
235 spin_lock_init(&channel->recv_lock);
236 spin_lock_init(&channel->intent_lock);
237 mutex_init(&channel->intent_req_lock);
238
239 channel->glink = glink;
240 channel->name = kstrdup(name, GFP_KERNEL);
241 if (!channel->name) {
242 kfree(channel);
243 return ERR_PTR(-ENOMEM);
244 }
245
246 init_completion(&channel->open_req);
247 init_completion(&channel->open_ack);
248 init_waitqueue_head(&channel->intent_req_wq);
249
250 INIT_LIST_HEAD(&channel->done_intents);
251 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
252
253 idr_init(&channel->liids);
254 idr_init(&channel->riids);
255 kref_init(&channel->refcount);
256
257 return channel;
258}
259
260static void qcom_glink_channel_release(struct kref *ref)
261{
262 struct glink_channel *channel = container_of(ref, struct glink_channel,
263 refcount);
264 struct glink_core_rx_intent *intent;
265 struct glink_core_rx_intent *tmp;
266 unsigned long flags;
267 int iid;
268
269 /* cancel pending rx_done work */
270 cancel_work_sync(&channel->intent_work);
271
272 spin_lock_irqsave(&channel->intent_lock, flags);
273 /* Free all non-reuse intents pending rx_done work */
274 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
275 if (!intent->reuse) {
276 kfree(intent->data);
277 kfree(intent);
278 }
279 }
280
281 idr_for_each_entry(&channel->liids, tmp, iid) {
282 kfree(tmp->data);
283 kfree(tmp);
284 }
285 idr_destroy(&channel->liids);
286
287 idr_for_each_entry(&channel->riids, tmp, iid)
288 kfree(tmp);
289 idr_destroy(&channel->riids);
290 spin_unlock_irqrestore(&channel->intent_lock, flags);
291
292 kfree(channel->name);
293 kfree(channel);
294}
295
296static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
297{
298 return glink->rx_pipe->avail(glink->rx_pipe);
299}
300
301static void qcom_glink_rx_peek(struct qcom_glink *glink,
302 void *data, unsigned int offset, size_t count)
303{
304 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
305}
306
307static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
308{
309 glink->rx_pipe->advance(glink->rx_pipe, count);
310}
311
312static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
313{
314 return glink->tx_pipe->avail(glink->tx_pipe);
315}
316
317static void qcom_glink_tx_write(struct qcom_glink *glink,
318 const void *hdr, size_t hlen,
319 const void *data, size_t dlen)
320{
321 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
322}
323
324static void qcom_glink_tx_kick(struct qcom_glink *glink)
325{
326 glink->tx_pipe->kick(glink->tx_pipe);
327}
328
329static void qcom_glink_send_read_notify(struct qcom_glink *glink)
330{
331 struct glink_msg msg;
332
333 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
334 msg.param1 = 0;
335 msg.param2 = 0;
336
337 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
338
339 qcom_glink_tx_kick(glink);
340}
341
342static int qcom_glink_tx(struct qcom_glink *glink,
343 const void *hdr, size_t hlen,
344 const void *data, size_t dlen, bool wait)
345{
346 unsigned int tlen = hlen + dlen;
347 unsigned long flags;
348 int ret = 0;
349
350 /* Reject packets that are too big */
351 if (tlen >= glink->tx_pipe->length)
352 return -EINVAL;
353
354 spin_lock_irqsave(&glink->tx_lock, flags);
355
356 if (glink->abort_tx) {
357 ret = -EIO;
358 goto out;
359 }
360
361 while (qcom_glink_tx_avail(glink) < tlen) {
362 if (!wait) {
363 ret = -EAGAIN;
364 goto out;
365 }
366
367 if (glink->abort_tx) {
368 ret = -EIO;
369 goto out;
370 }
371
372 if (!glink->sent_read_notify) {
373 glink->sent_read_notify = true;
374 qcom_glink_send_read_notify(glink);
375 }
376
377 /* Wait without holding the tx_lock */
378 spin_unlock_irqrestore(&glink->tx_lock, flags);
379
380 wait_event_timeout(glink->tx_avail_notify,
381 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
382
383 spin_lock_irqsave(&glink->tx_lock, flags);
384
385 if (qcom_glink_tx_avail(glink) >= tlen)
386 glink->sent_read_notify = false;
387 }
388
389 qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
390 qcom_glink_tx_kick(glink);
391
392out:
393 spin_unlock_irqrestore(&glink->tx_lock, flags);
394
395 return ret;
396}
397
398static int qcom_glink_send_version(struct qcom_glink *glink)
399{
400 struct glink_msg msg;
401
402 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
403 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
404 msg.param2 = cpu_to_le32(glink->features);
405
406 trace_qcom_glink_cmd_version_tx(glink->label, GLINK_VERSION_1, glink->features);
407
408 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
409}
410
411static void qcom_glink_send_version_ack(struct qcom_glink *glink)
412{
413 struct glink_msg msg;
414
415 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
416 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
417 msg.param2 = cpu_to_le32(glink->features);
418
419 trace_qcom_glink_cmd_version_ack_tx(glink->label, msg.param1, msg.param2);
420
421 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
422}
423
424static void qcom_glink_send_open_ack(struct qcom_glink *glink,
425 struct glink_channel *channel)
426{
427 struct glink_msg msg;
428
429 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
430 msg.param1 = cpu_to_le16(channel->rcid);
431 msg.param2 = cpu_to_le32(0);
432
433 trace_qcom_glink_cmd_open_ack_tx(glink->label, channel->name,
434 channel->lcid, channel->rcid);
435
436 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
437}
438
439static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
440 unsigned int cid, bool granted)
441{
442 struct glink_channel *channel;
443 unsigned long flags;
444
445 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
446
447 spin_lock_irqsave(&glink->idr_lock, flags);
448 channel = idr_find(&glink->rcids, cid);
449 spin_unlock_irqrestore(&glink->idr_lock, flags);
450
451 trace_qcom_glink_cmd_rx_intent_req_ack_rx(glink->label,
452 channel ? channel->name : NULL,
453 channel ? channel->lcid : 0,
454 cid, granted);
455 if (!channel) {
456 dev_err(glink->dev, "unable to find channel\n");
457 return;
458 }
459
460 WRITE_ONCE(channel->intent_req_result, granted);
461 wake_up_all(&channel->intent_req_wq);
462}
463
464static void qcom_glink_intent_req_abort(struct glink_channel *channel)
465{
466 WRITE_ONCE(channel->intent_req_result, 0);
467 wake_up_all(&channel->intent_req_wq);
468}
469
470/**
471 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
472 * @glink: Ptr to the glink edge
473 * @channel: Ptr to the channel that the open req is sent
474 *
475 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
476 * Will return with refcount held, regardless of outcome.
477 *
478 * Return: 0 on success, negative errno otherwise.
479 */
480static int qcom_glink_send_open_req(struct qcom_glink *glink,
481 struct glink_channel *channel)
482{
483 DEFINE_RAW_FLEX(struct glink_msg, req, data, GLINK_NAME_SIZE);
484 int name_len = strlen(channel->name) + 1;
485 int req_len = ALIGN(sizeof(*req) + name_len, 8);
486 int ret;
487 unsigned long flags;
488
489 kref_get(&channel->refcount);
490
491 spin_lock_irqsave(&glink->idr_lock, flags);
492 ret = idr_alloc_cyclic(&glink->lcids, channel,
493 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
494 GFP_ATOMIC);
495 spin_unlock_irqrestore(&glink->idr_lock, flags);
496 if (ret < 0)
497 return ret;
498
499 channel->lcid = ret;
500
501 req->cmd = cpu_to_le16(GLINK_CMD_OPEN);
502 req->param1 = cpu_to_le16(channel->lcid);
503 req->param2 = cpu_to_le32(name_len);
504 strscpy(req->data, channel->name, GLINK_NAME_SIZE);
505
506 trace_qcom_glink_cmd_open_tx(glink->label, channel->name,
507 channel->lcid, channel->rcid);
508
509 ret = qcom_glink_tx(glink, req, req_len, NULL, 0, true);
510 if (ret)
511 goto remove_idr;
512
513 return 0;
514
515remove_idr:
516 spin_lock_irqsave(&glink->idr_lock, flags);
517 idr_remove(&glink->lcids, channel->lcid);
518 channel->lcid = 0;
519 spin_unlock_irqrestore(&glink->idr_lock, flags);
520
521 return ret;
522}
523
524static void qcom_glink_send_close_req(struct qcom_glink *glink,
525 struct glink_channel *channel)
526{
527 struct glink_msg req;
528
529 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
530 req.param1 = cpu_to_le16(channel->lcid);
531 req.param2 = 0;
532
533 trace_qcom_glink_cmd_close_tx(glink->label, channel->name,
534 channel->lcid, channel->rcid);
535
536 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
537}
538
539static void qcom_glink_send_close_ack(struct qcom_glink *glink,
540 struct glink_channel *channel)
541{
542 struct glink_msg req;
543
544 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
545 req.param1 = cpu_to_le16(channel->rcid);
546 req.param2 = 0;
547
548 trace_qcom_glink_cmd_close_ack_tx(glink->label, channel->name,
549 channel->lcid, channel->rcid);
550
551 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
552}
553
554static void qcom_glink_rx_done_work(struct work_struct *work)
555{
556 struct glink_channel *channel = container_of(work, struct glink_channel,
557 intent_work);
558 struct qcom_glink *glink = channel->glink;
559 struct glink_core_rx_intent *intent, *tmp;
560 struct {
561 u16 id;
562 u16 lcid;
563 u32 liid;
564 } __packed cmd;
565
566 unsigned int cid = channel->lcid;
567 unsigned int iid;
568 bool reuse;
569 unsigned long flags;
570
571 spin_lock_irqsave(&channel->intent_lock, flags);
572 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
573 list_del(&intent->node);
574 spin_unlock_irqrestore(&channel->intent_lock, flags);
575 iid = intent->id;
576 reuse = intent->reuse;
577
578 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
579 cmd.lcid = cid;
580 cmd.liid = iid;
581
582 trace_qcom_glink_cmd_rx_done_tx(glink->label, channel->name,
583 channel->lcid, channel->rcid, cmd.liid, reuse);
584
585 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
586 if (!reuse) {
587 kfree(intent->data);
588 kfree(intent);
589 }
590 spin_lock_irqsave(&channel->intent_lock, flags);
591 }
592 spin_unlock_irqrestore(&channel->intent_lock, flags);
593}
594
595static void qcom_glink_rx_done(struct qcom_glink *glink,
596 struct glink_channel *channel,
597 struct glink_core_rx_intent *intent)
598{
599 /* We don't send RX_DONE to intentless systems */
600 if (glink->intentless) {
601 kfree(intent->data);
602 kfree(intent);
603 return;
604 }
605
606 /* Take it off the tree of receive intents */
607 if (!intent->reuse) {
608 spin_lock(&channel->intent_lock);
609 idr_remove(&channel->liids, intent->id);
610 spin_unlock(&channel->intent_lock);
611 }
612
613 /* Schedule the sending of a rx_done indication */
614 spin_lock(&channel->intent_lock);
615 list_add_tail(&intent->node, &channel->done_intents);
616 spin_unlock(&channel->intent_lock);
617
618 schedule_work(&channel->intent_work);
619}
620
621/**
622 * qcom_glink_receive_version() - receive version/features from remote system
623 *
624 * @glink: pointer to transport interface
625 * @version: remote version
626 * @features: remote features
627 *
628 * This function is called in response to a remote-initiated version/feature
629 * negotiation sequence.
630 */
631static void qcom_glink_receive_version(struct qcom_glink *glink,
632 u32 version,
633 u32 features)
634{
635 trace_qcom_glink_cmd_version_rx(glink->label, version, features);
636
637 switch (version) {
638 case 0:
639 break;
640 case GLINK_VERSION_1:
641 glink->features &= features;
642 fallthrough;
643 default:
644 qcom_glink_send_version_ack(glink);
645 break;
646 }
647}
648
649/**
650 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
651 *
652 * @glink: pointer to transport interface
653 * @version: remote version response
654 * @features: remote features response
655 *
656 * This function is called in response to a local-initiated version/feature
657 * negotiation sequence and is the counter-offer from the remote side based
658 * upon the initial version and feature set requested.
659 */
660static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
661 u32 version,
662 u32 features)
663{
664 trace_qcom_glink_cmd_version_ack_rx(glink->label, version, features);
665
666 switch (version) {
667 case 0:
668 /* Version negotiation failed */
669 break;
670 case GLINK_VERSION_1:
671 if (features == glink->features)
672 break;
673
674 glink->features &= features;
675 fallthrough;
676 default:
677 qcom_glink_send_version(glink);
678 break;
679 }
680}
681
682/**
683 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
684 * wire format and transmit
685 * @glink: The transport to transmit on.
686 * @channel: The glink channel
687 * @granted: The request response to encode.
688 *
689 * Return: 0 on success or standard Linux error code.
690 */
691static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
692 struct glink_channel *channel,
693 bool granted)
694{
695 struct glink_msg msg;
696
697 trace_qcom_glink_cmd_rx_intent_req_ack_tx(glink->label, channel->name,
698 channel->lcid, channel->rcid,
699 granted);
700
701 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
702 msg.param1 = cpu_to_le16(channel->lcid);
703 msg.param2 = cpu_to_le32(granted);
704
705 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
706
707 return 0;
708}
709
710/**
711 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
712 * transmit
713 * @glink: The transport to transmit on.
714 * @channel: The local channel
715 * @intent: The intent to pass on to remote.
716 *
717 * Return: 0 on success or standard Linux error code.
718 */
719static int qcom_glink_advertise_intent(struct qcom_glink *glink,
720 struct glink_channel *channel,
721 struct glink_core_rx_intent *intent)
722{
723 struct command {
724 __le16 id;
725 __le16 lcid;
726 __le32 count;
727 __le32 size;
728 __le32 liid;
729 } __packed;
730 struct command cmd;
731
732 cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
733 cmd.lcid = cpu_to_le16(channel->lcid);
734 cmd.count = cpu_to_le32(1);
735 cmd.size = cpu_to_le32(intent->size);
736 cmd.liid = cpu_to_le32(intent->id);
737
738 trace_qcom_glink_cmd_intent_tx(glink->label, channel->name,
739 channel->lcid, channel->rcid,
740 cmd.count, cmd.size, cmd.liid);
741
742 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
743
744 return 0;
745}
746
747static struct glink_core_rx_intent *
748qcom_glink_alloc_intent(struct qcom_glink *glink,
749 struct glink_channel *channel,
750 size_t size,
751 bool reuseable)
752{
753 struct glink_core_rx_intent *intent;
754 int ret;
755 unsigned long flags;
756
757 intent = kzalloc_obj(*intent);
758 if (!intent)
759 return NULL;
760
761 intent->data = kzalloc(size, GFP_KERNEL);
762 if (!intent->data)
763 goto free_intent;
764
765 spin_lock_irqsave(&channel->intent_lock, flags);
766 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
767 if (ret < 0) {
768 spin_unlock_irqrestore(&channel->intent_lock, flags);
769 goto free_data;
770 }
771 spin_unlock_irqrestore(&channel->intent_lock, flags);
772
773 intent->id = ret;
774 intent->size = size;
775 intent->reuse = reuseable;
776
777 return intent;
778
779free_data:
780 kfree(intent->data);
781free_intent:
782 kfree(intent);
783 return NULL;
784}
785
786static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
787 u32 cid, uint32_t iid,
788 bool reuse)
789{
790 struct glink_core_rx_intent *intent;
791 struct glink_channel *channel;
792 unsigned long flags;
793
794 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
795
796 spin_lock_irqsave(&glink->idr_lock, flags);
797 channel = idr_find(&glink->rcids, cid);
798 spin_unlock_irqrestore(&glink->idr_lock, flags);
799
800 trace_qcom_glink_cmd_rx_done_rx(glink->label, channel ? channel->name : NULL,
801 channel ? channel->lcid : 0, cid, iid, reuse);
802 if (!channel) {
803 dev_err(glink->dev, "invalid channel id received\n");
804 return;
805 }
806
807 spin_lock_irqsave(&channel->intent_lock, flags);
808 intent = idr_find(&channel->riids, iid);
809
810 if (!intent) {
811 spin_unlock_irqrestore(&channel->intent_lock, flags);
812 dev_err(glink->dev, "invalid intent id received\n");
813 return;
814 }
815
816 intent->in_use = false;
817
818 if (!reuse) {
819 idr_remove(&channel->riids, intent->id);
820 kfree(intent);
821 }
822 spin_unlock_irqrestore(&channel->intent_lock, flags);
823
824 if (reuse) {
825 WRITE_ONCE(channel->intent_received, true);
826 wake_up_all(&channel->intent_req_wq);
827 }
828}
829
830/**
831 * qcom_glink_handle_intent_req() - Receive a request for rx_intent
832 * from remote side
833 * @glink: Pointer to the transport interface
834 * @cid: Remote channel ID
835 * @size: size of the intent
836 *
837 * The function searches for the local channel to which the request for
838 * rx_intent has arrived and allocates and notifies the remote back
839 */
840static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
841 u32 cid, size_t size)
842{
843 struct glink_core_rx_intent *intent;
844 struct glink_channel *channel;
845 unsigned long flags;
846
847 spin_lock_irqsave(&glink->idr_lock, flags);
848 channel = idr_find(&glink->rcids, cid);
849 spin_unlock_irqrestore(&glink->idr_lock, flags);
850
851 trace_qcom_glink_cmd_rx_intent_req_rx(glink->label,
852 channel ? channel->name : NULL,
853 channel ? channel->lcid : 0,
854 cid, size);
855 if (!channel) {
856 pr_err("%s channel not found for cid %d\n", __func__, cid);
857 return;
858 }
859
860 intent = qcom_glink_alloc_intent(glink, channel, size, false);
861 if (intent)
862 qcom_glink_advertise_intent(glink, channel, intent);
863
864 qcom_glink_send_intent_req_ack(glink, channel, !!intent);
865}
866
867static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
868{
869 struct glink_defer_cmd *dcmd;
870
871 extra = ALIGN(extra, 8);
872
873 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
874 dev_dbg(glink->dev, "Insufficient data in rx fifo");
875 return -ENXIO;
876 }
877
878 dcmd = kzalloc_flex(*dcmd, data, extra, GFP_ATOMIC);
879 if (!dcmd)
880 return -ENOMEM;
881
882 INIT_LIST_HEAD(&dcmd->node);
883
884 qcom_glink_rx_peek(glink,
885 container_of(&dcmd->msg, struct glink_msg, hdr), 0,
886 sizeof(dcmd->msg) + extra);
887
888 spin_lock(&glink->rx_lock);
889 list_add_tail(&dcmd->node, &glink->rx_queue);
890 spin_unlock(&glink->rx_lock);
891
892 schedule_work(&glink->rx_work);
893 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
894
895 return 0;
896}
897
898static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
899{
900 struct glink_core_rx_intent *intent;
901 struct glink_channel *channel;
902 struct {
903 struct glink_msg_hdr msg;
904 __le32 chunk_size;
905 __le32 left_size;
906 } __packed hdr;
907 unsigned int chunk_size;
908 unsigned int left_size;
909 unsigned int rcid;
910 unsigned int liid;
911 int ret = 0;
912 unsigned long flags;
913
914 if (avail < sizeof(hdr)) {
915 dev_dbg(glink->dev, "Not enough data in fifo\n");
916 return -EAGAIN;
917 }
918
919 qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr));
920 chunk_size = le32_to_cpu(hdr.chunk_size);
921 left_size = le32_to_cpu(hdr.left_size);
922
923 if (avail < sizeof(hdr) + chunk_size) {
924 dev_dbg(glink->dev, "Payload not yet in fifo\n");
925 return -EAGAIN;
926 }
927
928 rcid = le16_to_cpu(hdr.msg.param1);
929 liid = le32_to_cpu(hdr.msg.param2);
930 spin_lock_irqsave(&glink->idr_lock, flags);
931 channel = idr_find(&glink->rcids, rcid);
932 spin_unlock_irqrestore(&glink->idr_lock, flags);
933
934 trace_qcom_glink_cmd_tx_data_rx(glink->label, channel ? channel->name : NULL,
935 channel ? channel->lcid : 0, rcid,
936 liid, chunk_size, left_size,
937 hdr.msg.cmd == GLINK_CMD_TX_DATA_CONT);
938 if (!channel) {
939 dev_dbg(glink->dev, "Data on non-existing channel\n");
940
941 /* Drop the message */
942 goto advance_rx;
943 }
944
945 if (glink->intentless) {
946 /* Might have an ongoing, fragmented, message to append */
947 if (!channel->buf) {
948 intent = kzalloc_obj(*intent, GFP_ATOMIC);
949 if (!intent)
950 return -ENOMEM;
951
952 intent->data = kmalloc(chunk_size + left_size,
953 GFP_ATOMIC);
954 if (!intent->data) {
955 kfree(intent);
956 return -ENOMEM;
957 }
958
959 intent->id = 0xbabababa;
960 intent->size = chunk_size + left_size;
961 intent->offset = 0;
962
963 channel->buf = intent;
964 } else {
965 intent = channel->buf;
966 }
967 } else {
968 spin_lock_irqsave(&channel->intent_lock, flags);
969 intent = idr_find(&channel->liids, liid);
970 spin_unlock_irqrestore(&channel->intent_lock, flags);
971
972 if (!intent) {
973 dev_err(glink->dev,
974 "no intent found for channel %s intent %d",
975 channel->name, liid);
976 ret = -ENOENT;
977 goto advance_rx;
978 }
979 }
980
981 if (intent->size - intent->offset < chunk_size) {
982 dev_err(glink->dev, "Insufficient space in intent\n");
983
984 /* The packet header lied, drop payload */
985 goto advance_rx;
986 }
987
988 qcom_glink_rx_peek(glink, intent->data + intent->offset,
989 sizeof(hdr), chunk_size);
990 intent->offset += chunk_size;
991
992 /* Handle message when no fragments remain to be received */
993 if (!left_size) {
994 spin_lock(&channel->recv_lock);
995 if (channel->ept.cb) {
996 channel->ept.cb(channel->ept.rpdev,
997 intent->data,
998 intent->offset,
999 channel->ept.priv,
1000 RPMSG_ADDR_ANY);
1001 }
1002 spin_unlock(&channel->recv_lock);
1003
1004 intent->offset = 0;
1005 channel->buf = NULL;
1006
1007 qcom_glink_rx_done(glink, channel, intent);
1008 }
1009
1010advance_rx:
1011 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
1012
1013 return ret;
1014}
1015
1016static void qcom_glink_rx_read_notif(struct qcom_glink *glink)
1017{
1018 trace_qcom_glink_cmd_read_notif_rx(glink->label);
1019
1020 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1021 qcom_glink_tx_kick(glink);
1022}
1023
1024static void qcom_glink_handle_intent(struct qcom_glink *glink,
1025 unsigned int cid,
1026 unsigned int count,
1027 size_t avail)
1028{
1029 struct glink_core_rx_intent *intent;
1030 struct glink_channel *channel;
1031 struct intent_pair {
1032 __le32 size;
1033 __le32 iid;
1034 };
1035
1036 struct {
1037 struct glink_msg_hdr msg;
1038 struct intent_pair intents[];
1039 } __packed * msg;
1040
1041 const size_t msglen = struct_size(msg, intents, count);
1042 int ret;
1043 int i;
1044 unsigned long flags;
1045
1046 if (avail < msglen) {
1047 dev_dbg(glink->dev, "Not enough data in fifo\n");
1048 return;
1049 }
1050
1051 spin_lock_irqsave(&glink->idr_lock, flags);
1052 channel = idr_find(&glink->rcids, cid);
1053 spin_unlock_irqrestore(&glink->idr_lock, flags);
1054 if (!channel) {
1055 trace_qcom_glink_cmd_intent_rx(glink->label, NULL, 0, cid, count, 0, 0);
1056 dev_err(glink->dev, "intents for non-existing channel\n");
1057 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1058 return;
1059 }
1060
1061 msg = kmalloc(msglen, GFP_ATOMIC);
1062 if (!msg)
1063 return;
1064
1065 qcom_glink_rx_peek(glink, msg, 0, msglen);
1066
1067 trace_qcom_glink_cmd_intent_rx(glink->label, channel->name,
1068 channel->lcid, cid, count,
1069 count > 0 ? msg->intents[0].size : 0,
1070 count > 0 ? msg->intents[0].iid : 0);
1071
1072 for (i = 0; i < count; ++i) {
1073 intent = kzalloc_obj(*intent, GFP_ATOMIC);
1074 if (!intent)
1075 break;
1076
1077 intent->id = le32_to_cpu(msg->intents[i].iid);
1078 intent->size = le32_to_cpu(msg->intents[i].size);
1079
1080 spin_lock_irqsave(&channel->intent_lock, flags);
1081 ret = idr_alloc(&channel->riids, intent,
1082 intent->id, intent->id + 1, GFP_ATOMIC);
1083 spin_unlock_irqrestore(&channel->intent_lock, flags);
1084
1085 if (ret < 0)
1086 dev_err(glink->dev, "failed to store remote intent\n");
1087 }
1088
1089 WRITE_ONCE(channel->intent_received, true);
1090 wake_up_all(&channel->intent_req_wq);
1091
1092 kfree(msg);
1093 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1094}
1095
1096static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1097{
1098 struct glink_channel *channel;
1099
1100 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1101
1102 spin_lock(&glink->idr_lock);
1103 channel = idr_find(&glink->lcids, lcid);
1104 spin_unlock(&glink->idr_lock);
1105
1106 trace_qcom_glink_cmd_open_ack_rx(glink->label, channel ? channel->name : NULL,
1107 lcid, channel ? channel->rcid : 0);
1108 if (!channel) {
1109 dev_err(glink->dev, "Invalid open ack packet\n");
1110 return -EINVAL;
1111 }
1112
1113 complete_all(&channel->open_ack);
1114
1115 return 0;
1116}
1117
1118/**
1119 * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1120 * @ept: Rpmsg endpoint for channel.
1121 * @pause: Pause transmission
1122 * @dst: destination address of the endpoint
1123 *
1124 * Return: 0 on success or standard Linux error code.
1125 */
1126static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1127{
1128 struct glink_channel *channel = to_glink_channel(ept);
1129 struct qcom_glink *glink = channel->glink;
1130 struct glink_msg msg;
1131 u32 sigs = 0;
1132
1133 if (pause)
1134 sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1135
1136 msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1137 msg.param1 = cpu_to_le16(channel->lcid);
1138 msg.param2 = cpu_to_le32(sigs);
1139
1140 trace_qcom_glink_cmd_signal_tx(glink->label, channel->name,
1141 channel->lcid, channel->rcid, sigs);
1142
1143 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
1144}
1145
1146static void qcom_glink_handle_signals(struct qcom_glink *glink,
1147 unsigned int rcid, unsigned int sigs)
1148{
1149 struct glink_channel *channel;
1150 unsigned long flags;
1151 bool enable;
1152
1153 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1154
1155 spin_lock_irqsave(&glink->idr_lock, flags);
1156 channel = idr_find(&glink->rcids, rcid);
1157 spin_unlock_irqrestore(&glink->idr_lock, flags);
1158
1159 trace_qcom_glink_cmd_signal_rx(glink->label, channel ? channel->name : NULL,
1160 channel ? channel->lcid : 0, rcid, sigs);
1161 if (!channel) {
1162 dev_err(glink->dev, "signal for non-existing channel\n");
1163 return;
1164 }
1165
1166 enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1167
1168 if (channel->ept.flow_cb)
1169 channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1170}
1171
1172void qcom_glink_native_rx(struct qcom_glink *glink)
1173{
1174 struct glink_msg msg;
1175 unsigned int param1;
1176 unsigned int param2;
1177 unsigned int avail;
1178 unsigned int cmd;
1179 int ret = 0;
1180
1181 /* To wakeup any blocking writers */
1182 wake_up_all(&glink->tx_avail_notify);
1183
1184 for (;;) {
1185 avail = qcom_glink_rx_avail(glink);
1186 if (avail < sizeof(msg))
1187 break;
1188
1189 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg));
1190
1191 cmd = le16_to_cpu(msg.cmd);
1192 param1 = le16_to_cpu(msg.param1);
1193 param2 = le32_to_cpu(msg.param2);
1194
1195 switch (cmd) {
1196 case GLINK_CMD_VERSION:
1197 case GLINK_CMD_VERSION_ACK:
1198 case GLINK_CMD_CLOSE:
1199 case GLINK_CMD_CLOSE_ACK:
1200 case GLINK_CMD_RX_INTENT_REQ:
1201 ret = qcom_glink_rx_defer(glink, 0);
1202 break;
1203 case GLINK_CMD_OPEN_ACK:
1204 ret = qcom_glink_rx_open_ack(glink, param1);
1205 break;
1206 case GLINK_CMD_OPEN:
1207 /* upper 16 bits of param2 are the "prio" field */
1208 ret = qcom_glink_rx_defer(glink, param2 & 0xffff);
1209 break;
1210 case GLINK_CMD_TX_DATA:
1211 case GLINK_CMD_TX_DATA_CONT:
1212 ret = qcom_glink_rx_data(glink, avail);
1213 break;
1214 case GLINK_CMD_READ_NOTIF:
1215 qcom_glink_rx_read_notif(glink);
1216 break;
1217 case GLINK_CMD_INTENT:
1218 qcom_glink_handle_intent(glink, param1, param2, avail);
1219 break;
1220 case GLINK_CMD_RX_DONE:
1221 qcom_glink_handle_rx_done(glink, param1, param2, false);
1222 break;
1223 case GLINK_CMD_RX_DONE_W_REUSE:
1224 qcom_glink_handle_rx_done(glink, param1, param2, true);
1225 break;
1226 case GLINK_CMD_RX_INTENT_REQ_ACK:
1227 qcom_glink_handle_intent_req_ack(glink, param1, param2);
1228 break;
1229 case GLINK_CMD_SIGNALS:
1230 qcom_glink_handle_signals(glink, param1, param2);
1231 break;
1232 default:
1233 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1234 ret = -EINVAL;
1235 break;
1236 }
1237
1238 if (ret)
1239 break;
1240 }
1241}
1242EXPORT_SYMBOL(qcom_glink_native_rx);
1243
1244/* Locally initiated rpmsg_create_ept */
1245static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1246 const char *name)
1247{
1248 struct glink_channel *channel;
1249 int ret;
1250 unsigned long flags;
1251
1252 channel = qcom_glink_alloc_channel(glink, name);
1253 if (IS_ERR(channel))
1254 return ERR_CAST(channel);
1255
1256 ret = qcom_glink_send_open_req(glink, channel);
1257 if (ret)
1258 goto release_channel;
1259
1260 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1261 if (!ret)
1262 goto err_timeout;
1263
1264 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1265 if (!ret)
1266 goto err_timeout;
1267
1268 qcom_glink_send_open_ack(glink, channel);
1269
1270 return channel;
1271
1272err_timeout:
1273 /* qcom_glink_send_open_req() did register the channel in lcids*/
1274 spin_lock_irqsave(&glink->idr_lock, flags);
1275 idr_remove(&glink->lcids, channel->lcid);
1276 spin_unlock_irqrestore(&glink->idr_lock, flags);
1277
1278release_channel:
1279 /* Release qcom_glink_send_open_req() reference */
1280 kref_put(&channel->refcount, qcom_glink_channel_release);
1281 /* Release qcom_glink_alloc_channel() reference */
1282 kref_put(&channel->refcount, qcom_glink_channel_release);
1283
1284 return ERR_PTR(-ETIMEDOUT);
1285}
1286
1287/* Remote initiated rpmsg_create_ept */
1288static int qcom_glink_create_remote(struct qcom_glink *glink,
1289 struct glink_channel *channel)
1290{
1291 int ret;
1292
1293 qcom_glink_send_open_ack(glink, channel);
1294
1295 ret = qcom_glink_send_open_req(glink, channel);
1296 if (ret)
1297 goto close_link;
1298
1299 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1300 if (!ret) {
1301 ret = -ETIMEDOUT;
1302 goto close_link;
1303 }
1304
1305 return 0;
1306
1307close_link:
1308 /*
1309 * Send a close request to "undo" our open-ack. The close-ack will
1310 * release qcom_glink_send_open_req() reference and the last reference
1311 * will be relesed after receiving remote_close or transport unregister
1312 * by calling qcom_glink_native_remove().
1313 */
1314 qcom_glink_send_close_req(glink, channel);
1315
1316 return ret;
1317}
1318
1319static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1320 rpmsg_rx_cb_t cb,
1321 void *priv,
1322 struct rpmsg_channel_info
1323 chinfo)
1324{
1325 struct glink_channel *parent = to_glink_channel(rpdev->ept);
1326 struct glink_channel *channel;
1327 struct qcom_glink *glink = parent->glink;
1328 struct rpmsg_endpoint *ept;
1329 const char *name = chinfo.name;
1330 int cid;
1331 int ret;
1332 unsigned long flags;
1333
1334 spin_lock_irqsave(&glink->idr_lock, flags);
1335 idr_for_each_entry(&glink->rcids, channel, cid) {
1336 if (!strcmp(channel->name, name))
1337 break;
1338 }
1339 spin_unlock_irqrestore(&glink->idr_lock, flags);
1340
1341 if (!channel) {
1342 channel = qcom_glink_create_local(glink, name);
1343 if (IS_ERR(channel))
1344 return NULL;
1345 } else {
1346 ret = qcom_glink_create_remote(glink, channel);
1347 if (ret)
1348 return NULL;
1349 }
1350
1351 ept = &channel->ept;
1352 ept->rpdev = rpdev;
1353 ept->cb = cb;
1354 ept->priv = priv;
1355 ept->ops = &glink_endpoint_ops;
1356
1357 return ept;
1358}
1359
1360static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1361{
1362 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1363 struct device_node *np = rpdev->dev.of_node;
1364 struct qcom_glink *glink = channel->glink;
1365 struct glink_core_rx_intent *intent;
1366 const struct property *prop = NULL;
1367 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1368 int num_intents;
1369 int num_groups = 1;
1370 __be32 *val = defaults;
1371 int size;
1372
1373 if (glink->intentless || !completion_done(&channel->open_ack))
1374 return 0;
1375
1376 prop = of_find_property(np, "qcom,intents", NULL);
1377 if (prop) {
1378 val = prop->value;
1379 num_groups = prop->length / sizeof(u32) / 2;
1380 }
1381
1382 /* Channel is now open, advertise base set of intents */
1383 while (num_groups--) {
1384 size = be32_to_cpup(val++);
1385 num_intents = be32_to_cpup(val++);
1386 while (num_intents--) {
1387 intent = qcom_glink_alloc_intent(glink, channel, size,
1388 true);
1389 if (!intent)
1390 break;
1391
1392 qcom_glink_advertise_intent(glink, channel, intent);
1393 }
1394 }
1395 return 0;
1396}
1397
1398static void qcom_glink_remove_rpmsg_device(struct qcom_glink *glink, struct glink_channel *channel)
1399{
1400 struct rpmsg_channel_info chinfo;
1401
1402 if (channel->rpdev) {
1403 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1404 chinfo.src = RPMSG_ADDR_ANY;
1405 chinfo.dst = RPMSG_ADDR_ANY;
1406 rpmsg_unregister_device(glink->dev, &chinfo);
1407 }
1408 channel->rpdev = NULL;
1409}
1410
1411static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1412{
1413 struct glink_channel *channel = to_glink_channel(ept);
1414 struct qcom_glink *glink = channel->glink;
1415 unsigned long flags;
1416
1417 spin_lock_irqsave(&channel->recv_lock, flags);
1418 channel->ept.cb = NULL;
1419 spin_unlock_irqrestore(&channel->recv_lock, flags);
1420
1421 qcom_glink_send_close_req(glink, channel);
1422}
1423
1424static int qcom_glink_request_intent(struct qcom_glink *glink,
1425 struct glink_channel *channel,
1426 size_t size)
1427{
1428 struct {
1429 u16 id;
1430 u16 cid;
1431 u32 size;
1432 } __packed cmd;
1433
1434 int ret;
1435
1436 mutex_lock(&channel->intent_req_lock);
1437
1438 WRITE_ONCE(channel->intent_req_result, -1);
1439 WRITE_ONCE(channel->intent_received, false);
1440
1441 cmd.id = GLINK_CMD_RX_INTENT_REQ;
1442 cmd.cid = channel->lcid;
1443 cmd.size = size;
1444
1445 trace_qcom_glink_cmd_rx_intent_req_tx(glink->label, channel->name,
1446 channel->lcid, channel->rcid,
1447 cmd.size);
1448
1449 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1450 if (ret)
1451 goto unlock;
1452
1453 ret = wait_event_timeout(channel->intent_req_wq,
1454 READ_ONCE(channel->intent_req_result) == 0 ||
1455 (READ_ONCE(channel->intent_req_result) > 0 &&
1456 READ_ONCE(channel->intent_received)) ||
1457 glink->abort_tx,
1458 10 * HZ);
1459 if (!ret) {
1460 dev_err(glink->dev, "intent request timed out\n");
1461 ret = -ETIMEDOUT;
1462 } else if (glink->abort_tx) {
1463 ret = -ECANCELED;
1464 } else {
1465 ret = READ_ONCE(channel->intent_req_result) ? 0 : -EAGAIN;
1466 }
1467
1468unlock:
1469 mutex_unlock(&channel->intent_req_lock);
1470 return ret;
1471}
1472
1473static int __qcom_glink_send(struct glink_channel *channel,
1474 const void *data, int len, bool wait)
1475{
1476 struct qcom_glink *glink = channel->glink;
1477 struct glink_core_rx_intent *intent = NULL;
1478 struct glink_core_rx_intent *tmp;
1479 int iid = 0;
1480 struct {
1481 struct glink_msg_hdr msg;
1482 __le32 chunk_size;
1483 __le32 left_size;
1484 } __packed req;
1485 int ret;
1486 unsigned long flags;
1487 int chunk_size = len;
1488 size_t offset = 0;
1489
1490 if (!glink->intentless) {
1491 while (!intent) {
1492 spin_lock_irqsave(&channel->intent_lock, flags);
1493 idr_for_each_entry(&channel->riids, tmp, iid) {
1494 if (tmp->size >= len && !tmp->in_use) {
1495 if (!intent)
1496 intent = tmp;
1497 else if (intent->size > tmp->size)
1498 intent = tmp;
1499 if (intent->size == len)
1500 break;
1501 }
1502 }
1503 if (intent)
1504 intent->in_use = true;
1505 spin_unlock_irqrestore(&channel->intent_lock, flags);
1506
1507 /* We found an available intent */
1508 if (intent)
1509 break;
1510
1511 if (!wait)
1512 return -EBUSY;
1513
1514 ret = qcom_glink_request_intent(glink, channel, len);
1515 if (ret < 0)
1516 return ret;
1517 }
1518
1519 iid = intent->id;
1520 }
1521
1522 while (offset < len) {
1523 chunk_size = len - offset;
1524 if (chunk_size > SZ_8K && wait)
1525 chunk_size = SZ_8K;
1526
1527 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1528 req.msg.param1 = cpu_to_le16(channel->lcid);
1529 req.msg.param2 = cpu_to_le32(iid);
1530 req.chunk_size = cpu_to_le32(chunk_size);
1531 req.left_size = cpu_to_le32(len - offset - chunk_size);
1532
1533 trace_qcom_glink_cmd_tx_data_tx(glink->label, channel->name,
1534 channel->lcid, channel->rcid,
1535 iid, chunk_size,
1536 len - offset - chunk_size,
1537 offset > 0);
1538
1539 ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
1540 if (ret) {
1541 /* Mark intent available if we failed */
1542 if (intent)
1543 intent->in_use = false;
1544 return ret;
1545 }
1546
1547 offset += chunk_size;
1548 }
1549
1550 return 0;
1551}
1552
1553static int qcom_glink_send(struct rpmsg_endpoint *ept, const void *data, int len)
1554{
1555 struct glink_channel *channel = to_glink_channel(ept);
1556
1557 return __qcom_glink_send(channel, data, len, true);
1558}
1559
1560static int qcom_glink_trysend(struct rpmsg_endpoint *ept, const void *data,
1561 int len)
1562{
1563 struct glink_channel *channel = to_glink_channel(ept);
1564
1565 return __qcom_glink_send(channel, data, len, false);
1566}
1567
1568static int qcom_glink_sendto(struct rpmsg_endpoint *ept, const void *data,
1569 int len, u32 dst)
1570{
1571 struct glink_channel *channel = to_glink_channel(ept);
1572
1573 return __qcom_glink_send(channel, data, len, true);
1574}
1575
1576static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, const void *data,
1577 int len, u32 dst)
1578{
1579 struct glink_channel *channel = to_glink_channel(ept);
1580
1581 return __qcom_glink_send(channel, data, len, false);
1582}
1583
1584/*
1585 * Finds the device_node for the glink child interested in this channel.
1586 */
1587static struct device_node *qcom_glink_match_channel(struct device_node *node,
1588 const char *channel)
1589{
1590 struct device_node *child;
1591 const char *name;
1592 const char *key;
1593 int ret;
1594
1595 for_each_available_child_of_node(node, child) {
1596 key = "qcom,glink-channels";
1597 ret = of_property_read_string(child, key, &name);
1598 if (ret)
1599 continue;
1600
1601 if (strcmp(name, channel) == 0)
1602 return child;
1603 }
1604
1605 return NULL;
1606}
1607
1608static const struct rpmsg_device_ops glink_device_ops = {
1609 .create_ept = qcom_glink_create_ept,
1610 .announce_create = qcom_glink_announce_create,
1611};
1612
1613static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1614 .destroy_ept = qcom_glink_destroy_ept,
1615 .send = qcom_glink_send,
1616 .sendto = qcom_glink_sendto,
1617 .trysend = qcom_glink_trysend,
1618 .trysendto = qcom_glink_trysendto,
1619 .set_flow_control = qcom_glink_set_flow_control,
1620};
1621
1622static void qcom_glink_rpdev_release(struct device *dev)
1623{
1624 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1625
1626 kfree(rpdev);
1627}
1628
1629static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1630 char *name)
1631{
1632 struct glink_channel *channel;
1633 struct rpmsg_device *rpdev;
1634 bool create_device = false;
1635 struct device_node *node;
1636 int lcid;
1637 int ret;
1638 unsigned long flags;
1639
1640 spin_lock_irqsave(&glink->idr_lock, flags);
1641 idr_for_each_entry(&glink->lcids, channel, lcid) {
1642 if (!strcmp(channel->name, name))
1643 break;
1644 }
1645 spin_unlock_irqrestore(&glink->idr_lock, flags);
1646
1647 if (!channel) {
1648 channel = qcom_glink_alloc_channel(glink, name);
1649 if (IS_ERR(channel))
1650 return PTR_ERR(channel);
1651
1652 /* The opening dance was initiated by the remote */
1653 create_device = true;
1654 }
1655
1656 trace_qcom_glink_cmd_open_rx(glink->label, name, channel->lcid, rcid);
1657
1658 spin_lock_irqsave(&glink->idr_lock, flags);
1659 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1660 if (ret < 0) {
1661 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1662 spin_unlock_irqrestore(&glink->idr_lock, flags);
1663 goto free_channel;
1664 }
1665 channel->rcid = ret;
1666 spin_unlock_irqrestore(&glink->idr_lock, flags);
1667
1668 complete_all(&channel->open_req);
1669
1670 if (create_device) {
1671 rpdev = kzalloc_obj(*rpdev);
1672 if (!rpdev) {
1673 ret = -ENOMEM;
1674 goto rcid_remove;
1675 }
1676
1677 rpdev->ept = &channel->ept;
1678 strscpy(rpdev->id.name, name);
1679 rpdev->src = RPMSG_ADDR_ANY;
1680 rpdev->dst = RPMSG_ADDR_ANY;
1681 rpdev->ops = &glink_device_ops;
1682
1683 node = qcom_glink_match_channel(glink->dev->of_node, name);
1684 rpdev->dev.of_node = node;
1685 rpdev->dev.parent = glink->dev;
1686 rpdev->dev.release = qcom_glink_rpdev_release;
1687
1688 ret = rpmsg_register_device(rpdev);
1689 if (ret)
1690 goto rcid_remove;
1691
1692 channel->rpdev = rpdev;
1693 }
1694
1695 return 0;
1696
1697rcid_remove:
1698 spin_lock_irqsave(&glink->idr_lock, flags);
1699 idr_remove(&glink->rcids, channel->rcid);
1700 channel->rcid = 0;
1701 spin_unlock_irqrestore(&glink->idr_lock, flags);
1702free_channel:
1703 /* Release the reference, iff we took it */
1704 if (create_device)
1705 kref_put(&channel->refcount, qcom_glink_channel_release);
1706
1707 return ret;
1708}
1709
1710static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1711{
1712 struct glink_channel *channel;
1713 unsigned long flags;
1714
1715 spin_lock_irqsave(&glink->idr_lock, flags);
1716 channel = idr_find(&glink->rcids, rcid);
1717 spin_unlock_irqrestore(&glink->idr_lock, flags);
1718
1719 trace_qcom_glink_cmd_close_rx(glink->label, channel ? channel->name : NULL,
1720 channel ? channel->lcid : 0, rcid);
1721 if (WARN(!channel, "close request on unknown channel\n"))
1722 return;
1723
1724 /* cancel pending rx_done work */
1725 cancel_work_sync(&channel->intent_work);
1726
1727 qcom_glink_remove_rpmsg_device(glink, channel);
1728
1729 qcom_glink_send_close_ack(glink, channel);
1730
1731 spin_lock_irqsave(&glink->idr_lock, flags);
1732 idr_remove(&glink->rcids, channel->rcid);
1733 channel->rcid = 0;
1734 spin_unlock_irqrestore(&glink->idr_lock, flags);
1735
1736 kref_put(&channel->refcount, qcom_glink_channel_release);
1737}
1738
1739static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1740{
1741 struct glink_channel *channel;
1742 unsigned long flags;
1743
1744 /* To wakeup any blocking writers */
1745 wake_up_all(&glink->tx_avail_notify);
1746
1747 spin_lock_irqsave(&glink->idr_lock, flags);
1748 channel = idr_find(&glink->lcids, lcid);
1749
1750 trace_qcom_glink_cmd_close_ack_rx(glink->label, channel ? channel->name : NULL,
1751 lcid, channel ? channel->rcid : 0);
1752 if (WARN(!channel, "close ack on unknown channel\n")) {
1753 spin_unlock_irqrestore(&glink->idr_lock, flags);
1754 return;
1755 }
1756
1757 idr_remove(&glink->lcids, channel->lcid);
1758 channel->lcid = 0;
1759 spin_unlock_irqrestore(&glink->idr_lock, flags);
1760
1761 /* Decouple the potential rpdev from the channel */
1762 qcom_glink_remove_rpmsg_device(glink, channel);
1763
1764 kref_put(&channel->refcount, qcom_glink_channel_release);
1765}
1766
1767static void qcom_glink_work(struct work_struct *work)
1768{
1769 struct qcom_glink *glink = container_of(work, struct qcom_glink,
1770 rx_work);
1771 struct glink_defer_cmd *dcmd;
1772 struct glink_msg *msg;
1773 unsigned long flags;
1774 unsigned int param1;
1775 unsigned int param2;
1776 unsigned int cmd;
1777
1778 for (;;) {
1779 spin_lock_irqsave(&glink->rx_lock, flags);
1780 if (list_empty(&glink->rx_queue)) {
1781 spin_unlock_irqrestore(&glink->rx_lock, flags);
1782 break;
1783 }
1784 dcmd = list_first_entry(&glink->rx_queue,
1785 struct glink_defer_cmd, node);
1786 list_del(&dcmd->node);
1787 spin_unlock_irqrestore(&glink->rx_lock, flags);
1788
1789 msg = container_of(&dcmd->msg, struct glink_msg, hdr);
1790 cmd = le16_to_cpu(msg->cmd);
1791 param1 = le16_to_cpu(msg->param1);
1792 param2 = le32_to_cpu(msg->param2);
1793
1794 switch (cmd) {
1795 case GLINK_CMD_VERSION:
1796 qcom_glink_receive_version(glink, param1, param2);
1797 break;
1798 case GLINK_CMD_VERSION_ACK:
1799 qcom_glink_receive_version_ack(glink, param1, param2);
1800 break;
1801 case GLINK_CMD_OPEN:
1802 qcom_glink_rx_open(glink, param1, msg->data);
1803 break;
1804 case GLINK_CMD_CLOSE:
1805 qcom_glink_rx_close(glink, param1);
1806 break;
1807 case GLINK_CMD_CLOSE_ACK:
1808 qcom_glink_rx_close_ack(glink, param1);
1809 break;
1810 case GLINK_CMD_RX_INTENT_REQ:
1811 qcom_glink_handle_intent_req(glink, param1, param2);
1812 break;
1813 default:
1814 WARN(1, "Unknown defer object %d\n", cmd);
1815 break;
1816 }
1817
1818 kfree(dcmd);
1819 }
1820}
1821
1822static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1823{
1824 struct glink_defer_cmd *dcmd;
1825 struct glink_defer_cmd *tmp;
1826
1827 /* cancel any pending deferred rx_work */
1828 cancel_work_sync(&glink->rx_work);
1829
1830 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1831 kfree(dcmd);
1832}
1833
1834static ssize_t rpmsg_name_show(struct device *dev,
1835 struct device_attribute *attr, char *buf)
1836{
1837 int ret = 0;
1838 const char *name;
1839
1840 ret = of_property_read_string(dev->of_node, "label", &name);
1841 if (ret < 0)
1842 name = dev->of_node->name;
1843
1844 return sysfs_emit(buf, "%s\n", name);
1845}
1846static DEVICE_ATTR_RO(rpmsg_name);
1847
1848static struct attribute *qcom_glink_attrs[] = {
1849 &dev_attr_rpmsg_name.attr,
1850 NULL
1851};
1852ATTRIBUTE_GROUPS(qcom_glink);
1853
1854static void qcom_glink_device_release(struct device *dev)
1855{
1856 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1857 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1858
1859 /* Release qcom_glink_alloc_channel() reference */
1860 kref_put(&channel->refcount, qcom_glink_channel_release);
1861 kfree(rpdev);
1862}
1863
1864static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1865{
1866 struct rpmsg_device *rpdev;
1867 struct glink_channel *channel;
1868
1869 rpdev = kzalloc_obj(*rpdev);
1870 if (!rpdev)
1871 return -ENOMEM;
1872
1873 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1874 if (IS_ERR(channel)) {
1875 kfree(rpdev);
1876 return PTR_ERR(channel);
1877 }
1878 channel->rpdev = rpdev;
1879
1880 rpdev->ept = &channel->ept;
1881 rpdev->ops = &glink_device_ops;
1882 rpdev->dev.parent = glink->dev;
1883 rpdev->dev.release = qcom_glink_device_release;
1884
1885 return rpmsg_ctrldev_register_device(rpdev);
1886}
1887
1888struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1889 unsigned long features,
1890 struct qcom_glink_pipe *rx,
1891 struct qcom_glink_pipe *tx,
1892 bool intentless)
1893{
1894 int ret;
1895 struct qcom_glink *glink;
1896
1897 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1898 if (!glink)
1899 return ERR_PTR(-ENOMEM);
1900
1901 glink->dev = dev;
1902 glink->tx_pipe = tx;
1903 glink->rx_pipe = rx;
1904
1905 glink->features = features;
1906 glink->intentless = intentless;
1907
1908 spin_lock_init(&glink->tx_lock);
1909 spin_lock_init(&glink->rx_lock);
1910 INIT_LIST_HEAD(&glink->rx_queue);
1911 INIT_WORK(&glink->rx_work, qcom_glink_work);
1912 init_waitqueue_head(&glink->tx_avail_notify);
1913
1914 spin_lock_init(&glink->idr_lock);
1915 idr_init(&glink->lcids);
1916 idr_init(&glink->rcids);
1917
1918 ret = of_property_read_string(dev->of_node, "label", &glink->label);
1919 if (ret < 0)
1920 glink->label = dev->of_node->name;
1921
1922 glink->dev->groups = qcom_glink_groups;
1923
1924 ret = device_add_groups(dev, qcom_glink_groups);
1925 if (ret)
1926 dev_err(dev, "failed to add groups\n");
1927
1928 ret = qcom_glink_send_version(glink);
1929 if (ret)
1930 return ERR_PTR(ret);
1931
1932 ret = qcom_glink_create_chrdev(glink);
1933 if (ret)
1934 dev_err(glink->dev, "failed to register chrdev\n");
1935
1936 return glink;
1937}
1938EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1939
1940static int qcom_glink_remove_device(struct device *dev, void *data)
1941{
1942 device_unregister(dev);
1943
1944 return 0;
1945}
1946
1947void qcom_glink_native_remove(struct qcom_glink *glink)
1948{
1949 struct glink_channel *channel;
1950 unsigned long flags;
1951 int cid;
1952 int ret;
1953
1954 qcom_glink_cancel_rx_work(glink);
1955
1956 /* Fail all attempts at sending messages */
1957 spin_lock_irqsave(&glink->tx_lock, flags);
1958 glink->abort_tx = true;
1959 wake_up_all(&glink->tx_avail_notify);
1960 spin_unlock_irqrestore(&glink->tx_lock, flags);
1961
1962 /* Abort any senders waiting for intent requests */
1963 spin_lock_irqsave(&glink->idr_lock, flags);
1964 idr_for_each_entry(&glink->lcids, channel, cid)
1965 qcom_glink_intent_req_abort(channel);
1966 spin_unlock_irqrestore(&glink->idr_lock, flags);
1967
1968 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1969 if (ret)
1970 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1971
1972 /* Release any defunct local channels, waiting for close-ack */
1973 idr_for_each_entry(&glink->lcids, channel, cid)
1974 kref_put(&channel->refcount, qcom_glink_channel_release);
1975
1976 /* Release any defunct local channels, waiting for close-req */
1977 idr_for_each_entry(&glink->rcids, channel, cid)
1978 kref_put(&channel->refcount, qcom_glink_channel_release);
1979
1980 idr_destroy(&glink->lcids);
1981 idr_destroy(&glink->rcids);
1982}
1983EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1984
1985MODULE_DESCRIPTION("Qualcomm GLINK driver");
1986MODULE_LICENSE("GPL v2");