projects/linux·blob master · refs

2262 lines · 56.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Arm Firmware Framework for ARMv8-A(FFA) interface driver
4 *
5 * The Arm FFA specification[1] describes a software architecture to
6 * leverages the virtualization extension to isolate software images
7 * provided by an ecosystem of vendors from each other and describes
8 * interfaces that standardize communication between the various software
9 * images including communication between images in the Secure world and
10 * Normal world. Any Hypervisor could use the FFA interfaces to enable
11 * communication between VMs it manages.
12 *
13 * The Hypervisor a.k.a Partition managers in FFA terminology can assign
14 * system resources(Memory regions, Devices, CPU cycles) to the partitions
15 * and manage isolation amongst them.
16 *
17 * [1] https://developer.arm.com/docs/den0077/latest
18 *
19 * Copyright (C) 2021 ARM Ltd.
20 */
21
22#define DRIVER_NAME "ARM FF-A"
23#define pr_fmt(fmt) DRIVER_NAME ": " fmt
24
25#include <linux/acpi.h>
26#include <linux/arm_ffa.h>
27#include <linux/bitfield.h>
28#include <linux/cpuhotplug.h>
29#include <linux/delay.h>
30#include <linux/device.h>
31#include <linux/hashtable.h>
32#include <linux/interrupt.h>
33#include <linux/io.h>
34#include <linux/kernel.h>
35#include <linux/minmax.h>
36#include <linux/module.h>
37#include <linux/mm.h>
38#include <linux/mutex.h>
39#include <linux/of_irq.h>
40#include <linux/platform_device.h>
41#include <linux/scatterlist.h>
42#include <linux/slab.h>
43#include <linux/smp.h>
44#include <linux/uuid.h>
45#include <linux/xarray.h>
46
47#include <asm/virt.h>
48
49#include "common.h"
50
51#define FFA_DRIVER_VERSION FFA_VERSION_1_2
52#define FFA_MIN_VERSION FFA_VERSION_1_0
53#define FFA_PLATFORM_NAME "arm-ffa"
54
55#define SENDER_ID_MASK GENMASK(31, 16)
56#define RECEIVER_ID_MASK GENMASK(15, 0)
57#define SENDER_ID(x) ((u16)(FIELD_GET(SENDER_ID_MASK, (x))))
58#define RECEIVER_ID(x) ((u16)(FIELD_GET(RECEIVER_ID_MASK, (x))))
59#define PACK_TARGET_INFO(s, r) \
60 (FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r)))
61
62#define RXTX_MAP_MIN_BUFSZ_MASK GENMASK(1, 0)
63#define RXTX_MAP_MAX_BUFSZ_MASK GENMASK(31, 16)
64#define RXTX_MAP_MIN_BUFSZ(x) (FIELD_GET(RXTX_MAP_MIN_BUFSZ_MASK, (x)))
65#define RXTX_MAP_MAX_BUFSZ(x) (FIELD_GET(RXTX_MAP_MAX_BUFSZ_MASK, (x)))
66
67#define FFA_MAX_NOTIFICATIONS 64
68
69static ffa_fn *invoke_ffa_fn;
70
71static const int ffa_linux_errmap[] = {
72 /* better than switch case as long as return value is continuous */
73 0, /* FFA_RET_SUCCESS */
74 -EOPNOTSUPP, /* FFA_RET_NOT_SUPPORTED */
75 -EINVAL, /* FFA_RET_INVALID_PARAMETERS */
76 -ENOMEM, /* FFA_RET_NO_MEMORY */
77 -EBUSY, /* FFA_RET_BUSY */
78 -EINTR, /* FFA_RET_INTERRUPTED */
79 -EACCES, /* FFA_RET_DENIED */
80 -EAGAIN, /* FFA_RET_RETRY */
81 -ECANCELED, /* FFA_RET_ABORTED */
82 -ENODATA, /* FFA_RET_NO_DATA */
83 -EAGAIN, /* FFA_RET_NOT_READY */
84};
85
86static inline int ffa_to_linux_errno(int errno)
87{
88 int err_idx = -errno;
89
90 if (err_idx >= 0 && err_idx < ARRAY_SIZE(ffa_linux_errmap))
91 return ffa_linux_errmap[err_idx];
92 return -EINVAL;
93}
94
95struct ffa_pcpu_irq {
96 struct ffa_drv_info *info;
97 struct work_struct notif_pcpu_work;
98};
99
100struct ffa_drv_info {
101 u32 version;
102 u16 vm_id;
103 struct mutex rx_lock; /* lock to protect Rx buffer */
104 struct mutex tx_lock; /* lock to protect Tx buffer */
105 void *rx_buffer;
106 void *tx_buffer;
107 size_t rxtx_bufsz;
108 bool mem_ops_native;
109 bool msg_direct_req2_supp;
110 bool bitmap_created;
111 bool bus_notifier_registered;
112 bool notif_enabled;
113 unsigned int sched_recv_irq;
114 unsigned int notif_pend_irq;
115 unsigned int cpuhp_state;
116 struct ffa_pcpu_irq __percpu *irq_pcpu;
117 struct workqueue_struct *notif_pcpu_wq;
118 struct work_struct sched_recv_irq_work;
119 struct xarray partition_info;
120 DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS));
121 rwlock_t notify_lock; /* lock to protect notifier hashtable */
122};
123
124static struct ffa_drv_info *drv_info;
125static struct platform_device *ffa_pdev;
126
127/*
128 * The driver must be able to support all the versions from the earliest
129 * supported FFA_MIN_VERSION to the latest supported FFA_DRIVER_VERSION.
130 * The specification states that if firmware supports a FFA implementation
131 * that is incompatible with and at a greater version number than specified
132 * by the caller(FFA_DRIVER_VERSION passed as parameter to FFA_VERSION),
133 * it must return the NOT_SUPPORTED error code.
134 */
135static u32 ffa_compatible_version_find(u32 version)
136{
137 u16 major = FFA_MAJOR_VERSION(version), minor = FFA_MINOR_VERSION(version);
138 u16 drv_major = FFA_MAJOR_VERSION(FFA_DRIVER_VERSION);
139 u16 drv_minor = FFA_MINOR_VERSION(FFA_DRIVER_VERSION);
140
141 if ((major < drv_major) || (major == drv_major && minor <= drv_minor))
142 return version;
143
144 pr_info("Firmware version higher than driver version, downgrading\n");
145 return FFA_DRIVER_VERSION;
146}
147
148static int ffa_version_check(u32 *version)
149{
150 ffa_value_t ver;
151
152 invoke_ffa_fn((ffa_value_t){
153 .a0 = FFA_VERSION, .a1 = FFA_DRIVER_VERSION,
154 }, &ver);
155
156 if ((s32)ver.a0 == FFA_RET_NOT_SUPPORTED) {
157 pr_info("FFA_VERSION returned not supported\n");
158 return -EOPNOTSUPP;
159 }
160
161 if (FFA_MAJOR_VERSION(ver.a0) > FFA_MAJOR_VERSION(FFA_DRIVER_VERSION)) {
162 pr_err("Incompatible v%d.%d! Latest supported v%d.%d\n",
163 FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0),
164 FFA_MAJOR_VERSION(FFA_DRIVER_VERSION),
165 FFA_MINOR_VERSION(FFA_DRIVER_VERSION));
166 return -EINVAL;
167 }
168
169 if (ver.a0 < FFA_MIN_VERSION) {
170 pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n",
171 FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0),
172 FFA_MAJOR_VERSION(FFA_MIN_VERSION),
173 FFA_MINOR_VERSION(FFA_MIN_VERSION));
174 return -EINVAL;
175 }
176
177 pr_info("Driver version %d.%d\n", FFA_MAJOR_VERSION(FFA_DRIVER_VERSION),
178 FFA_MINOR_VERSION(FFA_DRIVER_VERSION));
179 pr_info("Firmware version %d.%d found\n", FFA_MAJOR_VERSION(ver.a0),
180 FFA_MINOR_VERSION(ver.a0));
181 *version = ffa_compatible_version_find(ver.a0);
182
183 return 0;
184}
185
186static int ffa_rx_release(void)
187{
188 ffa_value_t ret;
189
190 invoke_ffa_fn((ffa_value_t){
191 .a0 = FFA_RX_RELEASE,
192 }, &ret);
193
194 if (ret.a0 == FFA_ERROR)
195 return ffa_to_linux_errno((int)ret.a2);
196
197 /* check for ret.a0 == FFA_RX_RELEASE ? */
198
199 return 0;
200}
201
202static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt)
203{
204 ffa_value_t ret;
205
206 invoke_ffa_fn((ffa_value_t){
207 .a0 = FFA_FN_NATIVE(RXTX_MAP),
208 .a1 = tx_buf, .a2 = rx_buf, .a3 = pg_cnt,
209 }, &ret);
210
211 if (ret.a0 == FFA_ERROR)
212 return ffa_to_linux_errno((int)ret.a2);
213
214 return 0;
215}
216
217static int ffa_rxtx_unmap(void)
218{
219 ffa_value_t ret;
220
221 invoke_ffa_fn((ffa_value_t){
222 .a0 = FFA_RXTX_UNMAP,
223 }, &ret);
224
225 if (ret.a0 == FFA_ERROR)
226 return ffa_to_linux_errno((int)ret.a2);
227
228 return 0;
229}
230
231static int ffa_features(u32 func_feat_id, u32 input_props,
232 u32 *if_props_1, u32 *if_props_2)
233{
234 ffa_value_t id;
235
236 if (!ARM_SMCCC_IS_FAST_CALL(func_feat_id) && input_props) {
237 pr_err("%s: Invalid Parameters: %x, %x", __func__,
238 func_feat_id, input_props);
239 return ffa_to_linux_errno(FFA_RET_INVALID_PARAMETERS);
240 }
241
242 invoke_ffa_fn((ffa_value_t){
243 .a0 = FFA_FEATURES, .a1 = func_feat_id, .a2 = input_props,
244 }, &id);
245
246 if (id.a0 == FFA_ERROR)
247 return ffa_to_linux_errno((int)id.a2);
248
249 if (if_props_1)
250 *if_props_1 = id.a2;
251 if (if_props_2)
252 *if_props_2 = id.a3;
253
254 return 0;
255}
256
257#define PARTITION_INFO_GET_RETURN_COUNT_ONLY BIT(0)
258#define FFA_SUPPORTS_GET_COUNT_ONLY(version) ((version) > FFA_VERSION_1_0)
259#define FFA_PART_INFO_HAS_SIZE_IN_RESP(version) ((version) > FFA_VERSION_1_0)
260#define FFA_PART_INFO_HAS_UUID_IN_RESP(version) ((version) > FFA_VERSION_1_0)
261#define FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(version) \
262 ((version) > FFA_VERSION_1_0)
263
264/* buffer must be sizeof(struct ffa_partition_info) * num_partitions */
265static int
266__ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
267 struct ffa_partition_info *buffer, int num_partitions)
268{
269 int idx, count, flags = 0, sz, buf_sz;
270 ffa_value_t partition_info;
271
272 if (FFA_SUPPORTS_GET_COUNT_ONLY(drv_info->version) &&
273 (!buffer || !num_partitions)) /* Just get the count for now */
274 flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY;
275
276 mutex_lock(&drv_info->rx_lock);
277 invoke_ffa_fn((ffa_value_t){
278 .a0 = FFA_PARTITION_INFO_GET,
279 .a1 = uuid0, .a2 = uuid1, .a3 = uuid2, .a4 = uuid3,
280 .a5 = flags,
281 }, &partition_info);
282
283 if (partition_info.a0 == FFA_ERROR) {
284 mutex_unlock(&drv_info->rx_lock);
285 return ffa_to_linux_errno((int)partition_info.a2);
286 }
287
288 count = partition_info.a2;
289
290 if (FFA_PART_INFO_HAS_SIZE_IN_RESP(drv_info->version)) {
291 buf_sz = sz = partition_info.a3;
292 if (sz > sizeof(*buffer))
293 buf_sz = sizeof(*buffer);
294 } else {
295 buf_sz = sz = 8;
296 }
297
298 if (buffer && count <= num_partitions)
299 for (idx = 0; idx < count; idx++) {
300 struct ffa_partition_info_le {
301 __le16 id;
302 __le16 exec_ctxt;
303 __le32 properties;
304 uuid_t uuid;
305 } *rx_buf = drv_info->rx_buffer + idx * sz;
306 struct ffa_partition_info *buf = buffer + idx;
307
308 buf->id = le16_to_cpu(rx_buf->id);
309 buf->exec_ctxt = le16_to_cpu(rx_buf->exec_ctxt);
310 buf->properties = le32_to_cpu(rx_buf->properties);
311 if (buf_sz > 8)
312 import_uuid(&buf->uuid, (u8 *)&rx_buf->uuid);
313 }
314
315 if (!(flags & PARTITION_INFO_GET_RETURN_COUNT_ONLY))
316 ffa_rx_release();
317
318 mutex_unlock(&drv_info->rx_lock);
319
320 return count;
321}
322
323#define LAST_INDEX_MASK GENMASK(15, 0)
324#define CURRENT_INDEX_MASK GENMASK(31, 16)
325#define UUID_INFO_TAG_MASK GENMASK(47, 32)
326#define PARTITION_INFO_SZ_MASK GENMASK(63, 48)
327#define PARTITION_COUNT(x) ((u16)(FIELD_GET(LAST_INDEX_MASK, (x))) + 1)
328#define CURRENT_INDEX(x) ((u16)(FIELD_GET(CURRENT_INDEX_MASK, (x))))
329#define UUID_INFO_TAG(x) ((u16)(FIELD_GET(UUID_INFO_TAG_MASK, (x))))
330#define PARTITION_INFO_SZ(x) ((u16)(FIELD_GET(PARTITION_INFO_SZ_MASK, (x))))
331#define PART_INFO_ID_MASK GENMASK(15, 0)
332#define PART_INFO_EXEC_CXT_MASK GENMASK(31, 16)
333#define PART_INFO_PROPS_MASK GENMASK(63, 32)
334#define FFA_PART_INFO_GET_REGS_FIRST_REG 3
335#define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC 3
336#define FFA_PART_INFO_GET_REGS_NUM_REGS \
337 (sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0))
338#define PART_INFO_ID(x) ((u16)(FIELD_GET(PART_INFO_ID_MASK, (x))))
339#define PART_INFO_EXEC_CXT(x) ((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x))))
340#define PART_INFO_PROPERTIES(x) ((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x))))
341static int
342__ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
343 struct ffa_partition_info *buffer, int num_parts)
344{
345 u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0;
346 struct ffa_partition_info *buf = buffer;
347 ffa_value_t partition_info;
348
349 do {
350 __le64 *regs;
351 int idx, nr_desc, buf_idx, regs_per_desc, max_desc;
352
353 invoke_ffa_fn((ffa_value_t){
354 .a0 = FFA_PARTITION_INFO_GET_REGS,
355 .a1 = (u64)uuid1 << 32 | uuid0,
356 .a2 = (u64)uuid3 << 32 | uuid2,
357 .a3 = start_idx | tag << 16,
358 }, &partition_info);
359
360 if (partition_info.a0 == FFA_ERROR)
361 return ffa_to_linux_errno((int)partition_info.a2);
362
363 if (!count)
364 count = PARTITION_COUNT(partition_info.a2);
365 if (!buffer || !num_parts) /* count only */
366 return count;
367 if (count > num_parts)
368 return -EINVAL;
369
370 cur_idx = CURRENT_INDEX(partition_info.a2);
371 if (cur_idx < start_idx || cur_idx >= count)
372 return -EINVAL;
373
374 buf_sz = PARTITION_INFO_SZ(partition_info.a2);
375 if (buf_sz % sizeof(*regs))
376 return -EINVAL;
377
378 regs_per_desc = buf_sz / sizeof(*regs);
379 if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC)
380 return -EINVAL;
381
382 nr_desc = cur_idx - start_idx + 1;
383 max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS -
384 FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc;
385 if (nr_desc > max_desc)
386 return -EINVAL;
387
388 buf_idx = buf - buffer;
389 if (buf_idx + nr_desc > num_parts)
390 return -EINVAL;
391
392 tag = UUID_INFO_TAG(partition_info.a2);
393
394 regs = (void *)&partition_info.a3;
395 for (idx = 0; idx < nr_desc; idx++, buf++) {
396 union {
397 uuid_t uuid;
398 u64 regs[2];
399 } uuid_regs = {
400 .regs = {
401 le64_to_cpu(*(regs + 1)),
402 le64_to_cpu(*(regs + 2)),
403 }
404 };
405 u64 val = *(u64 *)regs;
406
407 buf->id = PART_INFO_ID(val);
408 buf->exec_ctxt = PART_INFO_EXEC_CXT(val);
409 buf->properties = PART_INFO_PROPERTIES(val);
410 uuid_copy(&buf->uuid, &uuid_regs.uuid);
411 regs += regs_per_desc;
412 }
413 start_idx = cur_idx + 1;
414
415 } while (cur_idx < (count - 1));
416
417 return count;
418}
419
420/* buffer is allocated and caller must free the same if returned count > 0 */
421static int
422ffa_partition_probe(const uuid_t *uuid, struct ffa_partition_info **buffer)
423{
424 int count;
425 u32 uuid0_4[4];
426 bool reg_mode = false;
427 struct ffa_partition_info *pbuf;
428
429 if (!ffa_features(FFA_PARTITION_INFO_GET_REGS, 0, NULL, NULL))
430 reg_mode = true;
431
432 export_uuid((u8 *)uuid0_4, uuid);
433 if (reg_mode)
434 count = __ffa_partition_info_get_regs(uuid0_4[0], uuid0_4[1],
435 uuid0_4[2], uuid0_4[3],
436 NULL, 0);
437 else
438 count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1],
439 uuid0_4[2], uuid0_4[3],
440 NULL, 0);
441 if (count <= 0)
442 return count;
443
444 pbuf = kzalloc_objs(*pbuf, count);
445 if (!pbuf)
446 return -ENOMEM;
447
448 if (reg_mode)
449 count = __ffa_partition_info_get_regs(uuid0_4[0], uuid0_4[1],
450 uuid0_4[2], uuid0_4[3],
451 pbuf, count);
452 else
453 count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1],
454 uuid0_4[2], uuid0_4[3],
455 pbuf, count);
456 if (count <= 0)
457 kfree(pbuf);
458 else
459 *buffer = pbuf;
460
461 return count;
462}
463
464#define VM_ID_MASK GENMASK(15, 0)
465static int ffa_id_get(u16 *vm_id)
466{
467 ffa_value_t id;
468
469 invoke_ffa_fn((ffa_value_t){
470 .a0 = FFA_ID_GET,
471 }, &id);
472
473 if (id.a0 == FFA_ERROR)
474 return ffa_to_linux_errno((int)id.a2);
475
476 *vm_id = FIELD_GET(VM_ID_MASK, (id.a2));
477
478 return 0;
479}
480
481static inline void ffa_msg_send_wait_for_completion(ffa_value_t *ret)
482{
483 while (ret->a0 == FFA_INTERRUPT || ret->a0 == FFA_YIELD) {
484 if (ret->a0 == FFA_YIELD)
485 fsleep(1000);
486
487 invoke_ffa_fn((ffa_value_t){
488 .a0 = FFA_RUN, .a1 = ret->a1,
489 }, ret);
490 }
491}
492
493static int ffa_msg_send_direct_req(u16 src_id, u16 dst_id, bool mode_32bit,
494 struct ffa_send_direct_data *data)
495{
496 u32 req_id, resp_id, src_dst_ids = PACK_TARGET_INFO(src_id, dst_id);
497 ffa_value_t ret;
498
499 if (mode_32bit) {
500 req_id = FFA_MSG_SEND_DIRECT_REQ;
501 resp_id = FFA_MSG_SEND_DIRECT_RESP;
502 } else {
503 req_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_REQ);
504 resp_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_RESP);
505 }
506
507 invoke_ffa_fn((ffa_value_t){
508 .a0 = req_id, .a1 = src_dst_ids, .a2 = 0,
509 .a3 = data->data0, .a4 = data->data1, .a5 = data->data2,
510 .a6 = data->data3, .a7 = data->data4,
511 }, &ret);
512
513 ffa_msg_send_wait_for_completion(&ret);
514
515 if (ret.a0 == FFA_ERROR)
516 return ffa_to_linux_errno((int)ret.a2);
517
518 if (ret.a0 == resp_id) {
519 data->data0 = ret.a3;
520 data->data1 = ret.a4;
521 data->data2 = ret.a5;
522 data->data3 = ret.a6;
523 data->data4 = ret.a7;
524 return 0;
525 }
526
527 return -EINVAL;
528}
529
530static int ffa_msg_send2(struct ffa_device *dev, u16 src_id, void *buf, size_t sz)
531{
532 u32 src_dst_ids = PACK_TARGET_INFO(src_id, dev->vm_id);
533 struct ffa_indirect_msg_hdr *msg;
534 ffa_value_t ret;
535 int retval = 0;
536
537 if (sz > (drv_info->rxtx_bufsz - sizeof(*msg)))
538 return -ERANGE;
539
540 mutex_lock(&drv_info->tx_lock);
541
542 msg = drv_info->tx_buffer;
543 msg->flags = 0;
544 msg->res0 = 0;
545 msg->offset = sizeof(*msg);
546 msg->send_recv_id = src_dst_ids;
547 msg->size = sz;
548 uuid_copy(&msg->uuid, &dev->uuid);
549 memcpy((u8 *)msg + msg->offset, buf, sz);
550
551 /* flags = 0, sender VMID = 0 works for both physical/virtual NS */
552 invoke_ffa_fn((ffa_value_t){
553 .a0 = FFA_MSG_SEND2, .a1 = 0, .a2 = 0
554 }, &ret);
555
556 if (ret.a0 == FFA_ERROR)
557 retval = ffa_to_linux_errno((int)ret.a2);
558
559 mutex_unlock(&drv_info->tx_lock);
560 return retval;
561}
562
563static int ffa_msg_send_direct_req2(u16 src_id, u16 dst_id, const uuid_t *uuid,
564 struct ffa_send_direct_data2 *data)
565{
566 u32 src_dst_ids = PACK_TARGET_INFO(src_id, dst_id);
567 union {
568 uuid_t uuid;
569 __le64 regs[2];
570 } uuid_regs = { .uuid = *uuid };
571 ffa_value_t ret, args = {
572 .a0 = FFA_MSG_SEND_DIRECT_REQ2,
573 .a1 = src_dst_ids,
574 .a2 = le64_to_cpu(uuid_regs.regs[0]),
575 .a3 = le64_to_cpu(uuid_regs.regs[1]),
576 };
577 memcpy((void *)&args + offsetof(ffa_value_t, a4), data, sizeof(*data));
578
579 invoke_ffa_fn(args, &ret);
580
581 ffa_msg_send_wait_for_completion(&ret);
582
583 if (ret.a0 == FFA_ERROR)
584 return ffa_to_linux_errno((int)ret.a2);
585
586 if (ret.a0 == FFA_MSG_SEND_DIRECT_RESP2) {
587 memcpy(data, (void *)&ret + offsetof(ffa_value_t, a4), sizeof(*data));
588 return 0;
589 }
590
591 return -EINVAL;
592}
593
594static int ffa_mem_first_frag(u32 func_id, phys_addr_t buf, u32 buf_sz,
595 u32 frag_len, u32 len, u64 *handle)
596{
597 ffa_value_t ret;
598
599 invoke_ffa_fn((ffa_value_t){
600 .a0 = func_id, .a1 = len, .a2 = frag_len,
601 .a3 = buf, .a4 = buf_sz,
602 }, &ret);
603
604 while (ret.a0 == FFA_MEM_OP_PAUSE)
605 invoke_ffa_fn((ffa_value_t){
606 .a0 = FFA_MEM_OP_RESUME,
607 .a1 = ret.a1, .a2 = ret.a2,
608 }, &ret);
609
610 if (ret.a0 == FFA_ERROR)
611 return ffa_to_linux_errno((int)ret.a2);
612
613 if (ret.a0 == FFA_SUCCESS) {
614 if (handle)
615 *handle = PACK_HANDLE(ret.a2, ret.a3);
616 } else if (ret.a0 == FFA_MEM_FRAG_RX) {
617 if (handle)
618 *handle = PACK_HANDLE(ret.a1, ret.a2);
619 } else {
620 return -EOPNOTSUPP;
621 }
622
623 return frag_len;
624}
625
626static int ffa_mem_next_frag(u64 handle, u32 frag_len)
627{
628 ffa_value_t ret;
629
630 invoke_ffa_fn((ffa_value_t){
631 .a0 = FFA_MEM_FRAG_TX,
632 .a1 = HANDLE_LOW(handle), .a2 = HANDLE_HIGH(handle),
633 .a3 = frag_len,
634 }, &ret);
635
636 while (ret.a0 == FFA_MEM_OP_PAUSE)
637 invoke_ffa_fn((ffa_value_t){
638 .a0 = FFA_MEM_OP_RESUME,
639 .a1 = ret.a1, .a2 = ret.a2,
640 }, &ret);
641
642 if (ret.a0 == FFA_ERROR)
643 return ffa_to_linux_errno((int)ret.a2);
644
645 if (ret.a0 == FFA_MEM_FRAG_RX)
646 return ret.a3;
647 else if (ret.a0 == FFA_SUCCESS)
648 return 0;
649
650 return -EOPNOTSUPP;
651}
652
653static int
654ffa_transmit_fragment(u32 func_id, phys_addr_t buf, u32 buf_sz, u32 frag_len,
655 u32 len, u64 *handle, bool first)
656{
657 if (!first)
658 return ffa_mem_next_frag(*handle, frag_len);
659
660 return ffa_mem_first_frag(func_id, buf, buf_sz, frag_len, len, handle);
661}
662
663static u32 ffa_get_num_pages_sg(struct scatterlist *sg)
664{
665 u32 num_pages = 0;
666
667 do {
668 num_pages += sg->length / FFA_PAGE_SIZE;
669 } while ((sg = sg_next(sg)));
670
671 return num_pages;
672}
673
674static u16 ffa_memory_attributes_get(u32 func_id)
675{
676 /*
677 * For the memory lend or donate operation, if the receiver is a PE or
678 * a proxy endpoint, the owner/sender must not specify the attributes
679 */
680 if (func_id == FFA_FN_NATIVE(MEM_LEND) ||
681 func_id == FFA_MEM_LEND)
682 return 0;
683
684 return FFA_MEM_NORMAL | FFA_MEM_WRITE_BACK | FFA_MEM_INNER_SHAREABLE;
685}
686
687static void ffa_emad_impdef_value_init(u32 version, void *dst, void *src)
688{
689 struct ffa_mem_region_attributes *ep_mem_access;
690
691 if (FFA_EMAD_HAS_IMPDEF_FIELD(version))
692 memcpy(dst, src, sizeof(ep_mem_access->impdef_val));
693}
694
695static void
696ffa_mem_region_additional_setup(u32 version, struct ffa_mem_region *mem_region)
697{
698 if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version)) {
699 mem_region->ep_mem_size = 0;
700 } else {
701 mem_region->ep_mem_size = ffa_emad_size_get(version);
702 mem_region->ep_mem_offset = sizeof(*mem_region);
703 memset(mem_region->reserved, 0, 12);
704 }
705}
706
707static int
708ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize,
709 struct ffa_mem_ops_args *args)
710{
711 int rc = 0;
712 bool first = true;
713 u32 composite_offset;
714 phys_addr_t addr = 0;
715 struct ffa_mem_region *mem_region = buffer;
716 struct ffa_composite_mem_region *composite;
717 struct ffa_mem_region_addr_range *constituents;
718 struct ffa_mem_region_attributes *ep_mem_access;
719 u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg), ep_offset;
720 u32 emad_end, emad_size = ffa_emad_size_get(drv_info->version);
721
722 mem_region->tag = args->tag;
723 mem_region->flags = args->flags;
724 mem_region->sender_id = drv_info->vm_id;
725 mem_region->attributes = ffa_memory_attributes_get(func_id);
726
727 ffa_mem_region_additional_setup(drv_info->version, mem_region);
728 composite_offset = ffa_mem_desc_offset(buffer, args->nattrs,
729 drv_info->version);
730 if (composite_offset + sizeof(*composite) > max_fragsize)
731 return -ENXIO;
732
733 for (idx = 0; idx < args->nattrs; idx++) {
734 ep_offset = ffa_mem_desc_offset(buffer, idx, drv_info->version);
735 if (check_add_overflow(ep_offset, emad_size, &emad_end))
736 return -ENXIO;
737
738 if (emad_end > max_fragsize)
739 return -ENXIO;
740
741 ep_mem_access = buffer + ep_offset;
742 memset(ep_mem_access, 0, emad_size);
743 ep_mem_access->receiver = args->attrs[idx].receiver;
744 ep_mem_access->attrs = args->attrs[idx].attrs;
745 ep_mem_access->composite_off = composite_offset;
746 ffa_emad_impdef_value_init(drv_info->version,
747 ep_mem_access->impdef_val,
748 args->attrs[idx].impdef_val);
749 }
750 mem_region->handle = 0;
751 mem_region->ep_count = args->nattrs;
752
753 composite = buffer + composite_offset;
754 composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg);
755 composite->addr_range_cnt = num_entries;
756 composite->reserved = 0;
757
758 length = composite_offset + CONSTITUENTS_OFFSET(num_entries);
759 frag_len = composite_offset + CONSTITUENTS_OFFSET(0);
760 if (frag_len > max_fragsize)
761 return -ENXIO;
762
763 if (!args->use_txbuf) {
764 addr = virt_to_phys(buffer);
765 buf_sz = max_fragsize / FFA_PAGE_SIZE;
766 }
767
768 constituents = buffer + frag_len;
769 idx = 0;
770 do {
771 if (frag_len == max_fragsize) {
772 rc = ffa_transmit_fragment(func_id, addr, buf_sz,
773 frag_len, length,
774 &args->g_handle, first);
775 if (rc < 0)
776 return -ENXIO;
777
778 first = false;
779 idx = 0;
780 frag_len = 0;
781 constituents = buffer;
782 }
783
784 if ((void *)constituents + sizeof(*constituents) - buffer > max_fragsize) {
785 pr_err("Memory Region Fragment > Tx Buffer size\n");
786 return -EFAULT;
787 }
788
789 constituents->address = sg_phys(args->sg);
790 constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE;
791 constituents->reserved = 0;
792 constituents++;
793 frag_len += sizeof(*constituents);
794 } while ((args->sg = sg_next(args->sg)));
795
796 return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len,
797 length, &args->g_handle, first);
798}
799
800static int ffa_memory_ops(u32 func_id, struct ffa_mem_ops_args *args)
801{
802 int ret;
803 void *buffer;
804 size_t rxtx_bufsz = drv_info->rxtx_bufsz;
805
806 if (!args->use_txbuf) {
807 buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
808 if (!buffer)
809 return -ENOMEM;
810 } else {
811 buffer = drv_info->tx_buffer;
812 mutex_lock(&drv_info->tx_lock);
813 }
814
815 ret = ffa_setup_and_transmit(func_id, buffer, rxtx_bufsz, args);
816
817 if (args->use_txbuf)
818 mutex_unlock(&drv_info->tx_lock);
819 else
820 free_pages_exact(buffer, rxtx_bufsz);
821
822 return ret < 0 ? ret : 0;
823}
824
825static int ffa_memory_reclaim(u64 g_handle, u32 flags)
826{
827 ffa_value_t ret;
828
829 invoke_ffa_fn((ffa_value_t){
830 .a0 = FFA_MEM_RECLAIM,
831 .a1 = HANDLE_LOW(g_handle), .a2 = HANDLE_HIGH(g_handle),
832 .a3 = flags,
833 }, &ret);
834
835 if (ret.a0 == FFA_ERROR)
836 return ffa_to_linux_errno((int)ret.a2);
837
838 return 0;
839}
840
841static int ffa_notification_bitmap_create(void)
842{
843 ffa_value_t ret;
844 u16 vcpu_count = nr_cpu_ids;
845
846 invoke_ffa_fn((ffa_value_t){
847 .a0 = FFA_NOTIFICATION_BITMAP_CREATE,
848 .a1 = drv_info->vm_id, .a2 = vcpu_count,
849 }, &ret);
850
851 if (ret.a0 == FFA_ERROR)
852 return ffa_to_linux_errno((int)ret.a2);
853
854 return 0;
855}
856
857static int ffa_notification_bitmap_destroy(void)
858{
859 ffa_value_t ret;
860
861 invoke_ffa_fn((ffa_value_t){
862 .a0 = FFA_NOTIFICATION_BITMAP_DESTROY,
863 .a1 = drv_info->vm_id,
864 }, &ret);
865
866 if (ret.a0 == FFA_ERROR)
867 return ffa_to_linux_errno((int)ret.a2);
868
869 return 0;
870}
871
872enum notify_type {
873 SECURE_PARTITION,
874 NON_SECURE_VM,
875 SPM_FRAMEWORK,
876 NS_HYP_FRAMEWORK,
877};
878
879#define NOTIFICATION_LOW_MASK GENMASK(31, 0)
880#define NOTIFICATION_HIGH_MASK GENMASK(63, 32)
881#define NOTIFICATION_BITMAP_HIGH(x) \
882 ((u32)(FIELD_GET(NOTIFICATION_HIGH_MASK, (x))))
883#define NOTIFICATION_BITMAP_LOW(x) \
884 ((u32)(FIELD_GET(NOTIFICATION_LOW_MASK, (x))))
885#define PACK_NOTIFICATION_BITMAP(low, high) \
886 (FIELD_PREP(NOTIFICATION_LOW_MASK, (low)) | \
887 FIELD_PREP(NOTIFICATION_HIGH_MASK, (high)))
888
889#define RECEIVER_VCPU_MASK GENMASK(31, 16)
890#define PACK_NOTIFICATION_GET_RECEIVER_INFO(vcpu_r, r) \
891 (FIELD_PREP(RECEIVER_VCPU_MASK, (vcpu_r)) | \
892 FIELD_PREP(RECEIVER_ID_MASK, (r)))
893
894#define NOTIFICATION_INFO_GET_MORE_PEND_MASK BIT(0)
895#define NOTIFICATION_INFO_GET_ID_COUNT GENMASK(11, 7)
896#define ID_LIST_MASK_64 GENMASK(51, 12)
897#define ID_LIST_MASK_32 GENMASK(31, 12)
898#define MAX_IDS_64 20
899#define MAX_IDS_32 10
900
901#define PER_VCPU_NOTIFICATION_FLAG BIT(0)
902#define SECURE_PARTITION_BITMAP_ENABLE BIT(SECURE_PARTITION)
903#define NON_SECURE_VM_BITMAP_ENABLE BIT(NON_SECURE_VM)
904#define SPM_FRAMEWORK_BITMAP_ENABLE BIT(SPM_FRAMEWORK)
905#define NS_HYP_FRAMEWORK_BITMAP_ENABLE BIT(NS_HYP_FRAMEWORK)
906#define FFA_BITMAP_SECURE_ENABLE_MASK \
907 (SECURE_PARTITION_BITMAP_ENABLE | SPM_FRAMEWORK_BITMAP_ENABLE)
908#define FFA_BITMAP_NS_ENABLE_MASK \
909 (NON_SECURE_VM_BITMAP_ENABLE | NS_HYP_FRAMEWORK_BITMAP_ENABLE)
910#define FFA_BITMAP_ALL_ENABLE_MASK \
911 (FFA_BITMAP_SECURE_ENABLE_MASK | FFA_BITMAP_NS_ENABLE_MASK)
912
913#define FFA_SECURE_PARTITION_ID_FLAG BIT(15)
914
915#define SPM_FRAMEWORK_BITMAP(x) NOTIFICATION_BITMAP_LOW(x)
916#define NS_HYP_FRAMEWORK_BITMAP(x) NOTIFICATION_BITMAP_HIGH(x)
917#define FRAMEWORK_NOTIFY_RX_BUFFER_FULL BIT(0)
918
919static int ffa_notification_bind_common(u16 dst_id, u64 bitmap,
920 u32 flags, bool is_bind)
921{
922 ffa_value_t ret;
923 u32 func, src_dst_ids = PACK_TARGET_INFO(dst_id, drv_info->vm_id);
924
925 func = is_bind ? FFA_NOTIFICATION_BIND : FFA_NOTIFICATION_UNBIND;
926
927 invoke_ffa_fn((ffa_value_t){
928 .a0 = func, .a1 = src_dst_ids, .a2 = flags,
929 .a3 = NOTIFICATION_BITMAP_LOW(bitmap),
930 .a4 = NOTIFICATION_BITMAP_HIGH(bitmap),
931 }, &ret);
932
933 if (ret.a0 == FFA_ERROR)
934 return ffa_to_linux_errno((int)ret.a2);
935 else if (ret.a0 != FFA_SUCCESS)
936 return -EINVAL;
937
938 return 0;
939}
940
941static
942int ffa_notification_set(u16 src_id, u16 dst_id, u32 flags, u64 bitmap)
943{
944 ffa_value_t ret;
945 u32 src_dst_ids = PACK_TARGET_INFO(dst_id, src_id);
946
947 invoke_ffa_fn((ffa_value_t) {
948 .a0 = FFA_NOTIFICATION_SET, .a1 = src_dst_ids, .a2 = flags,
949 .a3 = NOTIFICATION_BITMAP_LOW(bitmap),
950 .a4 = NOTIFICATION_BITMAP_HIGH(bitmap),
951 }, &ret);
952
953 if (ret.a0 == FFA_ERROR)
954 return ffa_to_linux_errno((int)ret.a2);
955 else if (ret.a0 != FFA_SUCCESS)
956 return -EINVAL;
957
958 return 0;
959}
960
961struct ffa_notify_bitmaps {
962 u64 sp_map;
963 u64 vm_map;
964 u64 arch_map;
965};
966
967static int ffa_notification_get(u32 flags, struct ffa_notify_bitmaps *notify)
968{
969 ffa_value_t ret;
970 u16 src_id = drv_info->vm_id;
971 u16 cpu_id = smp_processor_id();
972 u32 rec_vcpu_ids = PACK_NOTIFICATION_GET_RECEIVER_INFO(cpu_id, src_id);
973
974 invoke_ffa_fn((ffa_value_t){
975 .a0 = FFA_NOTIFICATION_GET, .a1 = rec_vcpu_ids, .a2 = flags,
976 }, &ret);
977
978 if (ret.a0 == FFA_ERROR)
979 return ffa_to_linux_errno((int)ret.a2);
980 else if (ret.a0 != FFA_SUCCESS)
981 return -EINVAL; /* Something else went wrong. */
982
983 if (flags & SECURE_PARTITION_BITMAP_ENABLE)
984 notify->sp_map = PACK_NOTIFICATION_BITMAP(ret.a2, ret.a3);
985 if (flags & NON_SECURE_VM_BITMAP_ENABLE)
986 notify->vm_map = PACK_NOTIFICATION_BITMAP(ret.a4, ret.a5);
987 if (flags & SPM_FRAMEWORK_BITMAP_ENABLE)
988 notify->arch_map = SPM_FRAMEWORK_BITMAP(ret.a6);
989 if (flags & NS_HYP_FRAMEWORK_BITMAP_ENABLE)
990 notify->arch_map = PACK_NOTIFICATION_BITMAP(notify->arch_map,
991 ret.a7);
992
993 return 0;
994}
995
996struct ffa_dev_part_info {
997 ffa_sched_recv_cb callback;
998 void *cb_data;
999 rwlock_t rw_lock;
1000 struct ffa_device *dev;
1001 struct list_head node;
1002};
1003
1004static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu)
1005{
1006 struct ffa_dev_part_info *partition = NULL, *tmp;
1007 ffa_sched_recv_cb callback;
1008 struct list_head *phead;
1009 void *cb_data;
1010
1011 phead = xa_load(&drv_info->partition_info, part_id);
1012 if (!phead) {
1013 pr_err("%s: Invalid partition ID 0x%x\n", __func__, part_id);
1014 return;
1015 }
1016
1017 list_for_each_entry_safe(partition, tmp, phead, node) {
1018 read_lock(&partition->rw_lock);
1019 callback = partition->callback;
1020 cb_data = partition->cb_data;
1021 read_unlock(&partition->rw_lock);
1022
1023 if (callback)
1024 callback(vcpu, is_per_vcpu, cb_data);
1025 }
1026}
1027
1028/*
1029 * Map logical ID index to the u16 index within the packed ID list.
1030 *
1031 * For native responses (FF-A width == kernel word size), IDs are
1032 * tightly packed: idx -> idx.
1033 *
1034 * For 32-bit responses on a 64-bit kernel, each 64-bit register
1035 * contributes 4 x u16 values but only the lower 2 are defined; the
1036 * upper 2 are garbage. This mapping skips those upper halves:
1037 * 0,1,2,3,4,5,... -> 0,1,4,5,8,9,...
1038 */
1039static int list_idx_to_u16_idx(int idx, bool is_native_resp)
1040{
1041 return is_native_resp ? idx : idx + 2 * (idx >> 1);
1042}
1043
1044static void ffa_notification_info_get(void)
1045{
1046 int ids_processed, ids_count[MAX_IDS_64];
1047 int idx, list, max_ids, lists_cnt;
1048 bool is_64b_resp, is_native_resp;
1049 ffa_value_t ret;
1050 u64 id_list;
1051
1052 do {
1053 invoke_ffa_fn((ffa_value_t){
1054 .a0 = FFA_FN_NATIVE(NOTIFICATION_INFO_GET),
1055 }, &ret);
1056
1057 if (ret.a0 != FFA_FN_NATIVE(SUCCESS) && ret.a0 != FFA_SUCCESS) {
1058 if ((s32)ret.a2 != FFA_RET_NO_DATA)
1059 pr_err("Notification Info fetch failed: 0x%lx (0x%lx)",
1060 ret.a0, ret.a2);
1061 return;
1062 }
1063
1064 is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS);
1065 is_native_resp = (ret.a0 == FFA_FN_NATIVE(SUCCESS));
1066
1067 ids_processed = 0;
1068 lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2);
1069 if (is_64b_resp) {
1070 max_ids = MAX_IDS_64;
1071 id_list = FIELD_GET(ID_LIST_MASK_64, ret.a2);
1072 } else {
1073 max_ids = MAX_IDS_32;
1074 id_list = FIELD_GET(ID_LIST_MASK_32, ret.a2);
1075 }
1076
1077 for (idx = 0; idx < lists_cnt; idx++, id_list >>= 2)
1078 ids_count[idx] = (id_list & 0x3) + 1;
1079
1080 /* Process IDs */
1081 for (list = 0; list < lists_cnt; list++) {
1082 int u16_idx;
1083 u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3;
1084
1085 if (ids_processed >= max_ids - 1)
1086 break;
1087
1088 u16_idx = list_idx_to_u16_idx(ids_processed,
1089 is_native_resp);
1090 part_id = packed_id_list[u16_idx];
1091 ids_processed++;
1092
1093 if (ids_count[list] == 1) { /* Global Notification */
1094 __do_sched_recv_cb(part_id, 0, false);
1095 continue;
1096 }
1097
1098 /* Per vCPU Notification */
1099 for (idx = 1; idx < ids_count[list]; idx++) {
1100 if (ids_processed >= max_ids - 1)
1101 break;
1102
1103 u16_idx = list_idx_to_u16_idx(ids_processed,
1104 is_native_resp);
1105 vcpu_id = packed_id_list[u16_idx];
1106 ids_processed++;
1107
1108 __do_sched_recv_cb(part_id, vcpu_id, true);
1109 }
1110 }
1111 } while (ret.a2 & NOTIFICATION_INFO_GET_MORE_PEND_MASK);
1112}
1113
1114static int ffa_run(struct ffa_device *dev, u16 vcpu)
1115{
1116 ffa_value_t ret;
1117 u32 target = dev->vm_id << 16 | vcpu;
1118
1119 invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = target, }, &ret);
1120
1121 while (ret.a0 == FFA_INTERRUPT)
1122 invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = ret.a1, },
1123 &ret);
1124
1125 if (ret.a0 == FFA_ERROR)
1126 return ffa_to_linux_errno((int)ret.a2);
1127
1128 return 0;
1129}
1130
1131static void ffa_drvinfo_flags_init(void)
1132{
1133 if (!ffa_features(FFA_FN_NATIVE(MEM_LEND), 0, NULL, NULL) ||
1134 !ffa_features(FFA_FN_NATIVE(MEM_SHARE), 0, NULL, NULL))
1135 drv_info->mem_ops_native = true;
1136
1137 if (!ffa_features(FFA_MSG_SEND_DIRECT_REQ2, 0, NULL, NULL) ||
1138 !ffa_features(FFA_MSG_SEND_DIRECT_RESP2, 0, NULL, NULL))
1139 drv_info->msg_direct_req2_supp = true;
1140}
1141
1142static u32 ffa_api_version_get(void)
1143{
1144 return drv_info->version;
1145}
1146
1147static int ffa_partition_info_get(const char *uuid_str,
1148 struct ffa_partition_info *buffer)
1149{
1150 int count;
1151 uuid_t uuid;
1152 struct ffa_partition_info *pbuf;
1153
1154 if (!uuid_str || uuid_parse(uuid_str, &uuid)) {
1155 pr_err("invalid uuid (%s)\n", uuid_str);
1156 return -ENODEV;
1157 }
1158
1159 count = ffa_partition_probe(&uuid, &pbuf);
1160 if (count <= 0)
1161 return -ENOENT;
1162
1163 memcpy(buffer, pbuf, sizeof(*pbuf) * count);
1164 kfree(pbuf);
1165 return 0;
1166}
1167
1168static void ffa_mode_32bit_set(struct ffa_device *dev)
1169{
1170 dev->mode_32bit = true;
1171}
1172
1173static int ffa_sync_send_receive(struct ffa_device *dev,
1174 struct ffa_send_direct_data *data)
1175{
1176 return ffa_msg_send_direct_req(drv_info->vm_id, dev->vm_id,
1177 dev->mode_32bit, data);
1178}
1179
1180static int ffa_indirect_msg_send(struct ffa_device *dev, void *buf, size_t sz)
1181{
1182 return ffa_msg_send2(dev, drv_info->vm_id, buf, sz);
1183}
1184
1185static int ffa_sync_send_receive2(struct ffa_device *dev,
1186 struct ffa_send_direct_data2 *data)
1187{
1188 if (!drv_info->msg_direct_req2_supp)
1189 return -EOPNOTSUPP;
1190
1191 return ffa_msg_send_direct_req2(drv_info->vm_id, dev->vm_id,
1192 &dev->uuid, data);
1193}
1194
1195static int ffa_memory_share(struct ffa_mem_ops_args *args)
1196{
1197 if (drv_info->mem_ops_native)
1198 return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args);
1199
1200 return ffa_memory_ops(FFA_MEM_SHARE, args);
1201}
1202
1203static int ffa_memory_lend(struct ffa_mem_ops_args *args)
1204{
1205 /* Note that upon a successful MEM_LEND request the caller
1206 * must ensure that the memory region specified is not accessed
1207 * until a successful MEM_RECALIM call has been made.
1208 * On systems with a hypervisor present this will been enforced,
1209 * however on systems without a hypervisor the responsibility
1210 * falls to the calling kernel driver to prevent access.
1211 */
1212 if (drv_info->mem_ops_native)
1213 return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args);
1214
1215 return ffa_memory_ops(FFA_MEM_LEND, args);
1216}
1217
1218#define ffa_notifications_disabled() (!drv_info->notif_enabled)
1219
1220struct notifier_cb_info {
1221 struct hlist_node hnode;
1222 struct ffa_device *dev;
1223 ffa_fwk_notifier_cb fwk_cb;
1224 ffa_notifier_cb cb;
1225 void *cb_data;
1226};
1227
1228static int
1229ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback,
1230 void *cb_data, bool is_registration)
1231{
1232 struct ffa_dev_part_info *partition = NULL;
1233 struct list_head *phead;
1234 bool cb_valid;
1235
1236 if (ffa_notifications_disabled())
1237 return -EOPNOTSUPP;
1238
1239 phead = xa_load(&drv_info->partition_info, dev->vm_id);
1240 if (!phead) {
1241 pr_err("%s: Invalid partition ID 0x%x\n", __func__, dev->vm_id);
1242 return -EINVAL;
1243 }
1244
1245 list_for_each_entry(partition, phead, node)
1246 if (partition->dev == dev)
1247 break;
1248
1249 if (&partition->node == phead) {
1250 pr_err("%s: No such partition ID 0x%x\n", __func__, dev->vm_id);
1251 return -EINVAL;
1252 }
1253
1254 write_lock(&partition->rw_lock);
1255
1256 cb_valid = !!partition->callback;
1257 if (!(is_registration ^ cb_valid)) {
1258 write_unlock(&partition->rw_lock);
1259 return -EINVAL;
1260 }
1261
1262 partition->callback = callback;
1263 partition->cb_data = cb_data;
1264
1265 write_unlock(&partition->rw_lock);
1266 return 0;
1267}
1268
1269static int ffa_sched_recv_cb_register(struct ffa_device *dev,
1270 ffa_sched_recv_cb cb, void *cb_data)
1271{
1272 return ffa_sched_recv_cb_update(dev, cb, cb_data, true);
1273}
1274
1275static int ffa_sched_recv_cb_unregister(struct ffa_device *dev)
1276{
1277 return ffa_sched_recv_cb_update(dev, NULL, NULL, false);
1278}
1279
1280static int ffa_notification_bind(u16 dst_id, u64 bitmap, u32 flags)
1281{
1282 return ffa_notification_bind_common(dst_id, bitmap, flags, true);
1283}
1284
1285static int ffa_notification_unbind(u16 dst_id, u64 bitmap)
1286{
1287 return ffa_notification_bind_common(dst_id, bitmap, 0, false);
1288}
1289
1290static enum notify_type ffa_notify_type_get(u16 vm_id)
1291{
1292 if (vm_id & FFA_SECURE_PARTITION_ID_FLAG)
1293 return SECURE_PARTITION;
1294 else
1295 return NON_SECURE_VM;
1296}
1297
1298/* notifier_hnode_get* should be called with notify_lock held */
1299static struct notifier_cb_info *
1300notifier_hnode_get_by_vmid(u16 notify_id, int vmid)
1301{
1302 struct notifier_cb_info *node;
1303
1304 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id)
1305 if (node->fwk_cb && vmid == node->dev->vm_id)
1306 return node;
1307
1308 return NULL;
1309}
1310
1311static struct notifier_cb_info *
1312notifier_hnode_get_by_vmid_uuid(u16 notify_id, int vmid, const uuid_t *uuid)
1313{
1314 struct notifier_cb_info *node;
1315
1316 if (uuid_is_null(uuid))
1317 return notifier_hnode_get_by_vmid(notify_id, vmid);
1318
1319 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id)
1320 if (node->fwk_cb && vmid == node->dev->vm_id &&
1321 uuid_equal(&node->dev->uuid, uuid))
1322 return node;
1323
1324 return NULL;
1325}
1326
1327static struct notifier_cb_info *
1328notifier_hnode_get_by_type(u16 notify_id, enum notify_type type)
1329{
1330 struct notifier_cb_info *node;
1331
1332 hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id)
1333 if (node->cb && type == ffa_notify_type_get(node->dev->vm_id))
1334 return node;
1335
1336 return NULL;
1337}
1338
1339static int update_notifier_cb(struct ffa_device *dev, int notify_id,
1340 struct notifier_cb_info *cb, bool is_framework)
1341{
1342 struct notifier_cb_info *cb_info = NULL;
1343 enum notify_type type = ffa_notify_type_get(dev->vm_id);
1344 bool cb_found, is_registration = !!cb;
1345
1346 if (is_framework)
1347 cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, dev->vm_id,
1348 &dev->uuid);
1349 else
1350 cb_info = notifier_hnode_get_by_type(notify_id, type);
1351
1352 cb_found = !!cb_info;
1353
1354 if (!(is_registration ^ cb_found))
1355 return -EINVAL;
1356
1357 if (is_registration) {
1358 hash_add(drv_info->notifier_hash, &cb->hnode, notify_id);
1359 } else {
1360 hash_del(&cb_info->hnode);
1361 kfree(cb_info);
1362 }
1363
1364 return 0;
1365}
1366
1367static int __ffa_notify_relinquish(struct ffa_device *dev, int notify_id,
1368 bool is_framework)
1369{
1370 int rc;
1371
1372 if (ffa_notifications_disabled())
1373 return -EOPNOTSUPP;
1374
1375 if (notify_id >= FFA_MAX_NOTIFICATIONS)
1376 return -EINVAL;
1377
1378 write_lock(&drv_info->notify_lock);
1379
1380 rc = update_notifier_cb(dev, notify_id, NULL, is_framework);
1381 if (rc) {
1382 pr_err("Could not unregister notification callback\n");
1383 write_unlock(&drv_info->notify_lock);
1384 return rc;
1385 }
1386
1387 if (!is_framework)
1388 rc = ffa_notification_unbind(dev->vm_id, BIT(notify_id));
1389
1390 write_unlock(&drv_info->notify_lock);
1391
1392 return rc;
1393}
1394
1395static int ffa_notify_relinquish(struct ffa_device *dev, int notify_id)
1396{
1397 return __ffa_notify_relinquish(dev, notify_id, false);
1398}
1399
1400static int ffa_fwk_notify_relinquish(struct ffa_device *dev, int notify_id)
1401{
1402 return __ffa_notify_relinquish(dev, notify_id, true);
1403}
1404
1405static int __ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu,
1406 void *cb, void *cb_data,
1407 int notify_id, bool is_framework)
1408{
1409 int rc;
1410 u32 flags = 0;
1411 struct notifier_cb_info *cb_info = NULL;
1412
1413 if (ffa_notifications_disabled())
1414 return -EOPNOTSUPP;
1415
1416 if (notify_id >= FFA_MAX_NOTIFICATIONS)
1417 return -EINVAL;
1418
1419 cb_info = kzalloc_obj(*cb_info);
1420 if (!cb_info)
1421 return -ENOMEM;
1422
1423 cb_info->dev = dev;
1424 cb_info->cb_data = cb_data;
1425 if (is_framework)
1426 cb_info->fwk_cb = cb;
1427 else
1428 cb_info->cb = cb;
1429
1430 write_lock(&drv_info->notify_lock);
1431
1432 if (!is_framework) {
1433 if (is_per_vcpu)
1434 flags = PER_VCPU_NOTIFICATION_FLAG;
1435
1436 rc = ffa_notification_bind(dev->vm_id, BIT(notify_id), flags);
1437 if (rc)
1438 goto out_unlock_free;
1439 }
1440
1441 rc = update_notifier_cb(dev, notify_id, cb_info, is_framework);
1442 if (rc) {
1443 pr_err("Failed to register callback for %d - %d\n",
1444 notify_id, rc);
1445 if (!is_framework)
1446 ffa_notification_unbind(dev->vm_id, BIT(notify_id));
1447 }
1448
1449out_unlock_free:
1450 write_unlock(&drv_info->notify_lock);
1451 if (rc)
1452 kfree(cb_info);
1453
1454 return rc;
1455}
1456
1457static int ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu,
1458 ffa_notifier_cb cb, void *cb_data, int notify_id)
1459{
1460 return __ffa_notify_request(dev, is_per_vcpu, cb, cb_data, notify_id,
1461 false);
1462}
1463
1464static int
1465ffa_fwk_notify_request(struct ffa_device *dev, ffa_fwk_notifier_cb cb,
1466 void *cb_data, int notify_id)
1467{
1468 return __ffa_notify_request(dev, false, cb, cb_data, notify_id, true);
1469}
1470
1471static int ffa_notify_send(struct ffa_device *dev, int notify_id,
1472 bool is_per_vcpu, u16 vcpu)
1473{
1474 u32 flags = 0;
1475
1476 if (ffa_notifications_disabled())
1477 return -EOPNOTSUPP;
1478
1479 if (is_per_vcpu)
1480 flags |= (PER_VCPU_NOTIFICATION_FLAG | vcpu << 16);
1481
1482 return ffa_notification_set(dev->vm_id, drv_info->vm_id, flags,
1483 BIT(notify_id));
1484}
1485
1486static void handle_notif_callbacks(u64 bitmap, enum notify_type type)
1487{
1488 ffa_notifier_cb cb;
1489 void *cb_data;
1490 int notify_id;
1491
1492 for (notify_id = 0; notify_id <= FFA_MAX_NOTIFICATIONS && bitmap;
1493 notify_id++, bitmap >>= 1) {
1494 if (!(bitmap & 1))
1495 continue;
1496
1497 scoped_guard(read_lock, &drv_info->notify_lock) {
1498 struct notifier_cb_info *cb_info;
1499
1500 cb_info = notifier_hnode_get_by_type(notify_id, type);
1501 cb = cb_info ? cb_info->cb : NULL;
1502 cb_data = cb_info ? cb_info->cb_data : NULL;
1503 }
1504
1505 if (cb)
1506 cb(notify_id, cb_data);
1507 }
1508}
1509
1510static void handle_fwk_notif_callbacks(u32 bitmap)
1511{
1512 void *buf;
1513 uuid_t uuid;
1514 void *fwk_cb_data;
1515 int notify_id = 0, target;
1516 ffa_fwk_notifier_cb fwk_cb;
1517 struct ffa_indirect_msg_hdr *msg;
1518 size_t min_offset = offsetof(struct ffa_indirect_msg_hdr, uuid);
1519
1520 /* Only one framework notification defined and supported for now */
1521 if (!(bitmap & FRAMEWORK_NOTIFY_RX_BUFFER_FULL))
1522 return;
1523
1524 scoped_guard(mutex, &drv_info->rx_lock) {
1525 u32 offset, size;
1526
1527 msg = drv_info->rx_buffer;
1528 offset = msg->offset;
1529 size = msg->size;
1530
1531 if (!size || (offset != min_offset && offset < sizeof(*msg)) ||
1532 offset > drv_info->rxtx_bufsz ||
1533 size > drv_info->rxtx_bufsz - offset) {
1534 pr_err("invalid framework notification message\n");
1535 ffa_rx_release();
1536 return;
1537 }
1538
1539 buf = kmemdup((void *)msg + offset, size, GFP_KERNEL);
1540 if (!buf) {
1541 ffa_rx_release();
1542 return;
1543 }
1544
1545 target = SENDER_ID(msg->send_recv_id);
1546 if (offset >= sizeof(*msg))
1547 uuid_copy(&uuid, &msg->uuid);
1548 else
1549 uuid_copy(&uuid, &uuid_null);
1550 ffa_rx_release();
1551 }
1552
1553 scoped_guard(read_lock, &drv_info->notify_lock) {
1554 struct notifier_cb_info *cb_info;
1555
1556 cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target,
1557 &uuid);
1558 fwk_cb = cb_info ? cb_info->fwk_cb : NULL;
1559 fwk_cb_data = cb_info ? cb_info->cb_data : NULL;
1560 }
1561
1562 if (fwk_cb)
1563 fwk_cb(notify_id, fwk_cb_data, buf);
1564 kfree(buf);
1565}
1566
1567static void notif_get_and_handle(void *cb_data)
1568{
1569 int rc;
1570 u32 flags;
1571 struct ffa_drv_info *info = cb_data;
1572 struct ffa_notify_bitmaps bitmaps = { 0 };
1573
1574 if (info->vm_id == 0) /* Non secure physical instance */
1575 flags = FFA_BITMAP_SECURE_ENABLE_MASK;
1576 else
1577 flags = FFA_BITMAP_ALL_ENABLE_MASK;
1578
1579 rc = ffa_notification_get(flags, &bitmaps);
1580 if (rc) {
1581 pr_err("Failed to retrieve notifications with %d!\n", rc);
1582 return;
1583 }
1584
1585 handle_fwk_notif_callbacks(SPM_FRAMEWORK_BITMAP(bitmaps.arch_map));
1586 handle_fwk_notif_callbacks(NS_HYP_FRAMEWORK_BITMAP(bitmaps.arch_map));
1587 handle_notif_callbacks(bitmaps.vm_map, NON_SECURE_VM);
1588 handle_notif_callbacks(bitmaps.sp_map, SECURE_PARTITION);
1589}
1590
1591static void
1592ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data)
1593{
1594 struct ffa_drv_info *info = cb_data;
1595
1596 if (!is_per_vcpu)
1597 notif_get_and_handle(info);
1598 else
1599 smp_call_function_single(vcpu, notif_get_and_handle, info, 0);
1600}
1601
1602static void notif_pcpu_irq_work_fn(struct work_struct *work)
1603{
1604 struct ffa_pcpu_irq *pcpu = container_of(work, struct ffa_pcpu_irq,
1605 notif_pcpu_work);
1606 struct ffa_drv_info *info = pcpu->info;
1607
1608 notif_get_and_handle(info);
1609}
1610
1611static const struct ffa_info_ops ffa_drv_info_ops = {
1612 .api_version_get = ffa_api_version_get,
1613 .partition_info_get = ffa_partition_info_get,
1614};
1615
1616static const struct ffa_msg_ops ffa_drv_msg_ops = {
1617 .mode_32bit_set = ffa_mode_32bit_set,
1618 .sync_send_receive = ffa_sync_send_receive,
1619 .indirect_send = ffa_indirect_msg_send,
1620 .sync_send_receive2 = ffa_sync_send_receive2,
1621};
1622
1623static const struct ffa_mem_ops ffa_drv_mem_ops = {
1624 .memory_reclaim = ffa_memory_reclaim,
1625 .memory_share = ffa_memory_share,
1626 .memory_lend = ffa_memory_lend,
1627};
1628
1629static const struct ffa_cpu_ops ffa_drv_cpu_ops = {
1630 .run = ffa_run,
1631};
1632
1633static const struct ffa_notifier_ops ffa_drv_notifier_ops = {
1634 .sched_recv_cb_register = ffa_sched_recv_cb_register,
1635 .sched_recv_cb_unregister = ffa_sched_recv_cb_unregister,
1636 .notify_request = ffa_notify_request,
1637 .notify_relinquish = ffa_notify_relinquish,
1638 .fwk_notify_request = ffa_fwk_notify_request,
1639 .fwk_notify_relinquish = ffa_fwk_notify_relinquish,
1640 .notify_send = ffa_notify_send,
1641};
1642
1643static const struct ffa_ops ffa_drv_ops = {
1644 .info_ops = &ffa_drv_info_ops,
1645 .msg_ops = &ffa_drv_msg_ops,
1646 .mem_ops = &ffa_drv_mem_ops,
1647 .cpu_ops = &ffa_drv_cpu_ops,
1648 .notifier_ops = &ffa_drv_notifier_ops,
1649};
1650
1651void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid)
1652{
1653 int count, idx;
1654 struct ffa_partition_info *pbuf, *tpbuf;
1655
1656 count = ffa_partition_probe(uuid, &pbuf);
1657 if (count <= 0)
1658 return;
1659
1660 for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++)
1661 if (tpbuf->id == ffa_dev->vm_id)
1662 uuid_copy(&ffa_dev->uuid, uuid);
1663 kfree(pbuf);
1664}
1665
1666static int
1667ffa_bus_notifier(struct notifier_block *nb, unsigned long action, void *data)
1668{
1669 struct device *dev = data;
1670 struct ffa_device *fdev = to_ffa_dev(dev);
1671
1672 if (action == BUS_NOTIFY_BIND_DRIVER) {
1673 struct ffa_driver *ffa_drv = to_ffa_driver(dev->driver);
1674 const struct ffa_device_id *id_table = ffa_drv->id_table;
1675
1676 /*
1677 * FF-A v1.1 provides UUID for each partition as part of the
1678 * discovery API, the discovered UUID must be populated in the
1679 * device's UUID and there is no need to workaround by copying
1680 * the same from the driver table.
1681 */
1682 if (uuid_is_null(&fdev->uuid))
1683 ffa_device_match_uuid(fdev, &id_table->uuid);
1684
1685 return NOTIFY_OK;
1686 }
1687
1688 return NOTIFY_DONE;
1689}
1690
1691static struct notifier_block ffa_bus_nb = {
1692 .notifier_call = ffa_bus_notifier,
1693};
1694
1695static void ffa_bus_notifier_unregister(void)
1696{
1697 if (!drv_info->bus_notifier_registered)
1698 return;
1699
1700 bus_unregister_notifier(&ffa_bus_type, &ffa_bus_nb);
1701 drv_info->bus_notifier_registered = false;
1702}
1703
1704static int ffa_xa_add_partition_info(struct ffa_device *dev)
1705{
1706 struct ffa_dev_part_info *info;
1707 struct list_head *head, *phead;
1708 int ret = -ENOMEM;
1709
1710 phead = xa_load(&drv_info->partition_info, dev->vm_id);
1711 if (phead) {
1712 head = phead;
1713 list_for_each_entry(info, head, node) {
1714 if (info->dev == dev) {
1715 pr_err("%s: duplicate dev %p part ID 0x%x\n",
1716 __func__, dev, dev->vm_id);
1717 return -EEXIST;
1718 }
1719 }
1720 }
1721
1722 info = kzalloc_obj(*info);
1723 if (!info)
1724 return ret;
1725
1726 rwlock_init(&info->rw_lock);
1727 info->dev = dev;
1728
1729 if (!phead) {
1730 phead = kzalloc_obj(*phead);
1731 if (!phead)
1732 goto free_out;
1733
1734 INIT_LIST_HEAD(phead);
1735
1736 ret = xa_insert(&drv_info->partition_info, dev->vm_id, phead,
1737 GFP_KERNEL);
1738 if (ret) {
1739 pr_err("%s: failed to save part ID 0x%x Ret:%d\n",
1740 __func__, dev->vm_id, ret);
1741 goto free_out;
1742 }
1743 }
1744 list_add(&info->node, phead);
1745 return 0;
1746
1747free_out:
1748 kfree(phead);
1749 kfree(info);
1750 return ret;
1751}
1752
1753static int ffa_setup_host_partition(int vm_id)
1754{
1755 struct ffa_partition_info buf = { 0 };
1756 struct ffa_device *ffa_dev;
1757 int ret;
1758
1759 buf.id = vm_id;
1760 ffa_dev = ffa_device_register(&buf, &ffa_drv_ops, &ffa_pdev->dev);
1761 if (!ffa_dev) {
1762 pr_err("%s: failed to register host partition ID 0x%x\n",
1763 __func__, vm_id);
1764 return -EINVAL;
1765 }
1766
1767 ret = ffa_xa_add_partition_info(ffa_dev);
1768 if (ret)
1769 return ret;
1770
1771 if (ffa_notifications_disabled())
1772 return 0;
1773
1774 ret = ffa_sched_recv_cb_update(ffa_dev, ffa_self_notif_handle,
1775 drv_info, true);
1776 if (ret)
1777 pr_info("Failed to register driver sched callback %d\n", ret);
1778
1779 return ret;
1780}
1781
1782static void ffa_partitions_cleanup(void)
1783{
1784 struct list_head *phead;
1785 unsigned long idx;
1786
1787 ffa_bus_notifier_unregister();
1788
1789 /* Clean up/free all registered devices */
1790 ffa_devices_unregister();
1791
1792 xa_for_each(&drv_info->partition_info, idx, phead) {
1793 struct ffa_dev_part_info *info, *tmp;
1794
1795 xa_erase(&drv_info->partition_info, idx);
1796 list_for_each_entry_safe(info, tmp, phead, node) {
1797 list_del(&info->node);
1798 kfree(info);
1799 }
1800 kfree(phead);
1801 }
1802
1803 xa_destroy(&drv_info->partition_info);
1804}
1805
1806static int ffa_setup_partitions(void)
1807{
1808 int count, idx, ret;
1809 struct ffa_device *ffa_dev;
1810 struct ffa_partition_info *pbuf, *tpbuf;
1811
1812 if (!FFA_PART_INFO_HAS_UUID_IN_RESP(drv_info->version)) {
1813 ret = bus_register_notifier(&ffa_bus_type, &ffa_bus_nb);
1814 if (ret)
1815 pr_err("Failed to register FF-A bus notifiers\n");
1816 else
1817 drv_info->bus_notifier_registered = true;
1818 }
1819
1820 count = ffa_partition_probe(&uuid_null, &pbuf);
1821 if (count <= 0) {
1822 pr_info("%s: No partitions found, error %d\n", __func__, count);
1823 ffa_bus_notifier_unregister();
1824 return -EINVAL;
1825 }
1826
1827 xa_init(&drv_info->partition_info);
1828 for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) {
1829 /* Note that if the UUID will be uuid_null, that will require
1830 * ffa_bus_notifier() to find the UUID of this partition id
1831 * with help of ffa_device_match_uuid(). FF-A v1.1 and above
1832 * provides UUID here for each partition as part of the
1833 * discovery API and the same is passed.
1834 */
1835 ffa_dev = ffa_device_register(tpbuf, &ffa_drv_ops,
1836 &ffa_pdev->dev);
1837 if (!ffa_dev) {
1838 pr_err("%s: failed to register partition ID 0x%x\n",
1839 __func__, tpbuf->id);
1840 continue;
1841 }
1842
1843 if (FFA_PART_INFO_HAS_EXEC_STATE_IN_RESP(drv_info->version) &&
1844 !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC))
1845 ffa_mode_32bit_set(ffa_dev);
1846
1847 if (ffa_xa_add_partition_info(ffa_dev)) {
1848 ffa_device_unregister(ffa_dev);
1849 continue;
1850 }
1851 }
1852
1853 kfree(pbuf);
1854
1855 /*
1856 * Check if the host is already added as part of partition info
1857 * No multiple UUID possible for the host, so just checking if
1858 * there is an entry will suffice
1859 */
1860 if (xa_load(&drv_info->partition_info, drv_info->vm_id))
1861 return 0;
1862
1863 /* Allocate for the host */
1864 ret = ffa_setup_host_partition(drv_info->vm_id);
1865 if (ret)
1866 ffa_partitions_cleanup();
1867
1868 return ret;
1869}
1870
1871/* FFA FEATURE IDs */
1872#define FFA_FEAT_NOTIFICATION_PENDING_INT (1)
1873#define FFA_FEAT_SCHEDULE_RECEIVER_INT (2)
1874#define FFA_FEAT_MANAGED_EXIT_INT (3)
1875
1876static irqreturn_t ffa_sched_recv_irq_handler(int irq, void *irq_data)
1877{
1878 struct ffa_pcpu_irq *pcpu = irq_data;
1879 struct ffa_drv_info *info = pcpu->info;
1880
1881 queue_work(info->notif_pcpu_wq, &info->sched_recv_irq_work);
1882
1883 return IRQ_HANDLED;
1884}
1885
1886static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data)
1887{
1888 struct ffa_pcpu_irq *pcpu = irq_data;
1889 struct ffa_drv_info *info = pcpu->info;
1890
1891 queue_work_on(smp_processor_id(), info->notif_pcpu_wq,
1892 &pcpu->notif_pcpu_work);
1893
1894 return IRQ_HANDLED;
1895}
1896
1897static void ffa_sched_recv_irq_work_fn(struct work_struct *work)
1898{
1899 ffa_notification_info_get();
1900}
1901
1902static int ffa_irq_map(u32 id)
1903{
1904 char *err_str;
1905 int ret, irq, intid;
1906
1907 if (id == FFA_FEAT_NOTIFICATION_PENDING_INT)
1908 err_str = "Notification Pending Interrupt";
1909 else if (id == FFA_FEAT_SCHEDULE_RECEIVER_INT)
1910 err_str = "Schedule Receiver Interrupt";
1911 else
1912 err_str = "Unknown ID";
1913
1914 /* The returned intid is assumed to be SGI donated to NS world */
1915 ret = ffa_features(id, 0, &intid, NULL);
1916 if (ret < 0) {
1917 if (ret != -EOPNOTSUPP)
1918 pr_err("Failed to retrieve FF-A %s %u\n", err_str, id);
1919 return ret;
1920 }
1921
1922 if (acpi_disabled) {
1923 struct of_phandle_args oirq = {};
1924 struct device_node *gic;
1925
1926 /* Only GICv3 supported currently with the device tree */
1927 gic = of_find_compatible_node(NULL, NULL, "arm,gic-v3");
1928 if (!gic)
1929 return -ENXIO;
1930
1931 oirq.np = gic;
1932 oirq.args_count = 1;
1933 oirq.args[0] = intid;
1934 irq = irq_create_of_mapping(&oirq);
1935 of_node_put(gic);
1936#ifdef CONFIG_ACPI
1937 } else {
1938 irq = acpi_register_gsi(NULL, intid, ACPI_EDGE_SENSITIVE,
1939 ACPI_ACTIVE_HIGH);
1940#endif
1941 }
1942
1943 if (irq <= 0) {
1944 pr_err("Failed to create IRQ mapping!\n");
1945 return -ENODATA;
1946 }
1947
1948 return irq;
1949}
1950
1951static void ffa_irq_unmap(unsigned int irq)
1952{
1953 if (!irq)
1954 return;
1955 irq_dispose_mapping(irq);
1956}
1957
1958static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu)
1959{
1960 if (drv_info->sched_recv_irq)
1961 enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE);
1962 if (drv_info->notif_pend_irq)
1963 enable_percpu_irq(drv_info->notif_pend_irq, IRQ_TYPE_NONE);
1964 return 0;
1965}
1966
1967static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu)
1968{
1969 if (drv_info->sched_recv_irq)
1970 disable_percpu_irq(drv_info->sched_recv_irq);
1971 if (drv_info->notif_pend_irq)
1972 disable_percpu_irq(drv_info->notif_pend_irq);
1973 return 0;
1974}
1975
1976static void ffa_uninit_pcpu_irq(void)
1977{
1978 if (drv_info->cpuhp_state) {
1979 cpuhp_remove_state(drv_info->cpuhp_state);
1980 drv_info->cpuhp_state = 0;
1981 }
1982
1983 if (drv_info->notif_pcpu_wq) {
1984 destroy_workqueue(drv_info->notif_pcpu_wq);
1985 drv_info->notif_pcpu_wq = NULL;
1986 }
1987
1988 if (drv_info->sched_recv_irq)
1989 free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu);
1990
1991 if (drv_info->notif_pend_irq)
1992 free_percpu_irq(drv_info->notif_pend_irq, drv_info->irq_pcpu);
1993
1994 if (drv_info->irq_pcpu) {
1995 free_percpu(drv_info->irq_pcpu);
1996 drv_info->irq_pcpu = NULL;
1997 }
1998}
1999
2000static int ffa_init_pcpu_irq(void)
2001{
2002 struct ffa_pcpu_irq __percpu *irq_pcpu;
2003 int ret, cpu;
2004
2005 irq_pcpu = alloc_percpu(struct ffa_pcpu_irq);
2006 if (!irq_pcpu)
2007 return -ENOMEM;
2008
2009 for_each_present_cpu(cpu) {
2010 per_cpu_ptr(irq_pcpu, cpu)->info = drv_info;
2011 INIT_WORK(&per_cpu_ptr(irq_pcpu, cpu)->notif_pcpu_work,
2012 notif_pcpu_irq_work_fn);
2013 }
2014
2015 drv_info->irq_pcpu = irq_pcpu;
2016
2017 if (drv_info->sched_recv_irq) {
2018 ret = request_percpu_irq(drv_info->sched_recv_irq,
2019 ffa_sched_recv_irq_handler,
2020 "ARM-FFA-SRI", irq_pcpu);
2021 if (ret) {
2022 pr_err("Error registering percpu SRI nIRQ %d : %d\n",
2023 drv_info->sched_recv_irq, ret);
2024 drv_info->sched_recv_irq = 0;
2025 return ret;
2026 }
2027 }
2028
2029 if (drv_info->notif_pend_irq) {
2030 ret = request_percpu_irq(drv_info->notif_pend_irq,
2031 notif_pend_irq_handler,
2032 "ARM-FFA-NPI", irq_pcpu);
2033 if (ret) {
2034 pr_err("Error registering percpu NPI nIRQ %d : %d\n",
2035 drv_info->notif_pend_irq, ret);
2036 drv_info->notif_pend_irq = 0;
2037 return ret;
2038 }
2039 }
2040
2041 INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn);
2042 drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification");
2043 if (!drv_info->notif_pcpu_wq)
2044 return -EINVAL;
2045
2046 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ffa/pcpu-irq:starting",
2047 ffa_cpuhp_pcpu_irq_enable,
2048 ffa_cpuhp_pcpu_irq_disable);
2049
2050 if (ret < 0)
2051 return ret;
2052
2053 drv_info->cpuhp_state = ret;
2054 return 0;
2055}
2056
2057static void ffa_notifications_cleanup(void)
2058{
2059 ffa_uninit_pcpu_irq();
2060 ffa_irq_unmap(drv_info->sched_recv_irq);
2061 drv_info->sched_recv_irq = 0;
2062 ffa_irq_unmap(drv_info->notif_pend_irq);
2063 drv_info->notif_pend_irq = 0;
2064
2065 if (drv_info->bitmap_created) {
2066 ffa_notification_bitmap_destroy();
2067 drv_info->bitmap_created = false;
2068 }
2069 drv_info->notif_enabled = false;
2070}
2071
2072static void ffa_notifications_setup(void)
2073{
2074 int ret;
2075
2076 ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL);
2077 if (!ret) {
2078 ret = ffa_notification_bitmap_create();
2079 if (ret) {
2080 pr_err("Notification bitmap create error %d\n", ret);
2081 return;
2082 }
2083
2084 drv_info->bitmap_created = true;
2085 }
2086
2087 ret = ffa_irq_map(FFA_FEAT_SCHEDULE_RECEIVER_INT);
2088 if (ret > 0)
2089 drv_info->sched_recv_irq = ret;
2090
2091 ret = ffa_irq_map(FFA_FEAT_NOTIFICATION_PENDING_INT);
2092 if (ret > 0)
2093 drv_info->notif_pend_irq = ret;
2094
2095 if (!drv_info->sched_recv_irq && !drv_info->notif_pend_irq)
2096 goto cleanup;
2097
2098 ret = ffa_init_pcpu_irq();
2099 if (ret)
2100 goto cleanup;
2101
2102 hash_init(drv_info->notifier_hash);
2103 rwlock_init(&drv_info->notify_lock);
2104
2105 drv_info->notif_enabled = true;
2106 return;
2107cleanup:
2108 pr_info("Notification setup failed %d, not enabled\n", ret);
2109 ffa_notifications_cleanup();
2110}
2111
2112static int ffa_probe(struct platform_device *pdev)
2113{
2114 int ret;
2115 u32 buf_sz;
2116 size_t rxtx_min_bufsz = SZ_4K, rxtx_max_bufsz = 0, rxtx_bufsz;
2117
2118 if (IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) &&
2119 is_protected_kvm_enabled() && !is_pkvm_initialized())
2120 return -EPROBE_DEFER;
2121
2122 ret = ffa_transport_init(&invoke_ffa_fn);
2123 if (ret)
2124 return ret == -EOPNOTSUPP ? -ENODEV : ret;
2125
2126 drv_info = kzalloc_obj(*drv_info);
2127 if (!drv_info)
2128 return -ENOMEM;
2129 platform_set_drvdata(pdev, drv_info);
2130
2131 ret = ffa_version_check(&drv_info->version);
2132 if (ret) {
2133 if (ret == -EOPNOTSUPP)
2134 ret = -ENODEV;
2135 goto free_drv_info;
2136 }
2137
2138 if (ffa_id_get(&drv_info->vm_id)) {
2139 pr_err("failed to obtain VM id for self\n");
2140 ret = -ENODEV;
2141 goto free_drv_info;
2142 }
2143
2144 ret = ffa_features(FFA_FN_NATIVE(RXTX_MAP), 0, &buf_sz, NULL);
2145 if (!ret) {
2146 if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 1)
2147 rxtx_min_bufsz = SZ_64K;
2148 else if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 2)
2149 rxtx_min_bufsz = SZ_16K;
2150 else
2151 rxtx_min_bufsz = SZ_4K;
2152
2153 rxtx_max_bufsz = RXTX_MAP_MAX_BUFSZ(buf_sz) * SZ_4K;
2154 if (rxtx_max_bufsz != 0 && rxtx_max_bufsz < rxtx_min_bufsz)
2155 rxtx_max_bufsz = rxtx_min_bufsz;
2156 }
2157
2158 rxtx_bufsz = min_not_zero(PAGE_ALIGN(rxtx_min_bufsz), rxtx_max_bufsz);
2159 drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
2160 if (!drv_info->rx_buffer) {
2161 ret = -ENOMEM;
2162 goto free_drv_info;
2163 }
2164
2165 drv_info->tx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
2166 if (!drv_info->tx_buffer) {
2167 ret = -ENOMEM;
2168 goto free_pages;
2169 }
2170
2171 ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
2172 virt_to_phys(drv_info->rx_buffer),
2173 rxtx_bufsz / FFA_PAGE_SIZE);
2174 if (ret == -EINVAL && !rxtx_max_bufsz && rxtx_min_bufsz < rxtx_bufsz) {
2175 rxtx_bufsz = rxtx_min_bufsz;
2176 ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
2177 virt_to_phys(drv_info->rx_buffer),
2178 rxtx_bufsz / FFA_PAGE_SIZE);
2179 }
2180 if (ret) {
2181 pr_err("failed to register FFA RxTx buffers\n");
2182 goto free_pages;
2183 }
2184 drv_info->rxtx_bufsz = rxtx_bufsz;
2185
2186 mutex_init(&drv_info->rx_lock);
2187 mutex_init(&drv_info->tx_lock);
2188
2189 ffa_drvinfo_flags_init();
2190
2191 ffa_notifications_setup();
2192
2193 ret = ffa_setup_partitions();
2194 if (!ret)
2195 return ret;
2196
2197 pr_err("failed to setup partitions\n");
2198 ffa_notifications_cleanup();
2199 ffa_rxtx_unmap();
2200free_pages:
2201 if (drv_info->tx_buffer)
2202 free_pages_exact(drv_info->tx_buffer, rxtx_bufsz);
2203 free_pages_exact(drv_info->rx_buffer, rxtx_bufsz);
2204free_drv_info:
2205 platform_set_drvdata(pdev, NULL);
2206 kfree(drv_info);
2207 drv_info = NULL;
2208 return ret;
2209}
2210
2211static void ffa_remove(struct platform_device *pdev)
2212{
2213 struct ffa_drv_info *info = platform_get_drvdata(pdev);
2214
2215 ffa_notifications_cleanup();
2216 ffa_partitions_cleanup();
2217 ffa_rxtx_unmap();
2218 free_pages_exact(info->tx_buffer, info->rxtx_bufsz);
2219 free_pages_exact(info->rx_buffer, info->rxtx_bufsz);
2220 kfree(info);
2221 platform_set_drvdata(pdev, NULL);
2222 drv_info = NULL;
2223}
2224
2225static struct platform_driver ffa_driver = {
2226 .probe = ffa_probe,
2227 .remove = ffa_remove,
2228 .shutdown = ffa_remove,
2229 .driver = {
2230 .name = FFA_PLATFORM_NAME,
2231 },
2232};
2233
2234static int __init ffa_init(void)
2235{
2236 int ret;
2237
2238 ffa_pdev = platform_device_register_simple(FFA_PLATFORM_NAME,
2239 PLATFORM_DEVID_NONE,
2240 NULL, 0);
2241 if (IS_ERR(ffa_pdev))
2242 return PTR_ERR(ffa_pdev);
2243
2244 ret = platform_driver_register(&ffa_driver);
2245 if (ret)
2246 platform_device_unregister(ffa_pdev);
2247
2248 return ret;
2249}
2250module_init(ffa_init);
2251
2252static void __exit ffa_exit(void)
2253{
2254 platform_device_unregister(ffa_pdev);
2255 platform_driver_unregister(&ffa_driver);
2256}
2257module_exit(ffa_exit);
2258
2259MODULE_ALIAS("arm-ffa");
2260MODULE_AUTHOR("Sudeep Holla <[email protected]>");
2261MODULE_DESCRIPTION("Arm FF-A interface driver");
2262MODULE_LICENSE("GPL v2");