projects/linux·blob uaccess-cleanups · refs

1990 lines · 51.4 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Char device for device raw access
4 *
5 * Copyright (C) 2005-2007 Kristian Hoegsberg <[email protected]>
6 */
7
8#include <linux/bug.h>
9#include <linux/compat.h>
10#include <linux/delay.h>
11#include <linux/device.h>
12#include <linux/dma-mapping.h>
13#include <linux/err.h>
14#include <linux/errno.h>
15#include <linux/firewire.h>
16#include <linux/firewire-cdev.h>
17#include <linux/irqflags.h>
18#include <linux/jiffies.h>
19#include <linux/kernel.h>
20#include <linux/kref.h>
21#include <linux/mm.h>
22#include <linux/module.h>
23#include <linux/mutex.h>
24#include <linux/poll.h>
25#include <linux/sched.h> /* required for linux/wait.h */
26#include <linux/slab.h>
27#include <linux/spinlock.h>
28#include <linux/string.h>
29#include <linux/time.h>
30#include <linux/uaccess.h>
31#include <linux/vmalloc.h>
32#include <linux/wait.h>
33#include <linux/workqueue.h>
34
35
36#include "core.h"
37#include <trace/events/firewire.h>
38
39#include "packet-header-definitions.h"
40
41/*
42 * ABI version history is documented in linux/firewire-cdev.h.
43 */
44#define FW_CDEV_KERNEL_VERSION 6
45#define FW_CDEV_VERSION_EVENT_REQUEST2 4
46#define FW_CDEV_VERSION_ALLOCATE_REGION_END 4
47#define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 5
48#define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 6
49
50static DEFINE_SPINLOCK(phy_receiver_list_lock);
51static LIST_HEAD(phy_receiver_list);
52
53struct client {
54 u32 version;
55 struct fw_device *device;
56
57 spinlock_t lock;
58 bool in_shutdown;
59 struct xarray resource_xa;
60 struct list_head event_list;
61 wait_queue_head_t wait;
62 wait_queue_head_t tx_flush_wait;
63 u64 bus_reset_closure;
64
65 struct fw_iso_context *iso_context;
66 struct mutex iso_context_mutex;
67 u64 iso_closure;
68 struct fw_iso_buffer buffer;
69 unsigned long vm_start;
70
71 struct list_head phy_receiver_link;
72 u64 phy_receiver_closure;
73
74 struct list_head link;
75 struct kref kref;
76};
77
78static inline void client_get(struct client *client)
79{
80 kref_get(&client->kref);
81}
82
83static void client_release(struct kref *kref)
84{
85 struct client *client = container_of(kref, struct client, kref);
86
87 fw_device_put(client->device);
88 kfree(client);
89}
90
91static void client_put(struct client *client)
92{
93 kref_put(&client->kref, client_release);
94}
95
96struct client_resource;
97typedef void (*client_resource_release_fn_t)(struct client *,
98 struct client_resource *);
99struct client_resource {
100 client_resource_release_fn_t release;
101 int handle;
102};
103
104struct address_handler_resource {
105 struct client_resource resource;
106 struct fw_address_handler handler;
107 __u64 closure;
108 struct client *client;
109};
110
111struct outbound_transaction_resource {
112 struct client_resource resource;
113 struct fw_transaction transaction;
114};
115
116struct inbound_transaction_resource {
117 struct client_resource resource;
118 struct fw_card *card;
119 struct fw_request *request;
120 bool is_fcp;
121 void *data;
122 size_t length;
123};
124
125struct descriptor_resource {
126 struct client_resource resource;
127 struct fw_descriptor descriptor;
128 u32 data[];
129};
130
131struct iso_resource_params {
132 u64 channels_mask;
133 s32 bandwidth;
134};
135
136struct iso_resource_auto {
137 struct client_resource resource;
138 struct client *client;
139 /* Schedule work and access todo only with client->lock held. */
140 struct delayed_work work;
141 enum {
142 ISO_RES_AUTO_ALLOC,
143 ISO_RES_AUTO_REALLOC,
144 ISO_RES_AUTO_DEALLOC,
145 } todo;
146 int generation;
147 struct iso_resource_params params;
148 struct iso_resource_event *e_alloc, *e_dealloc;
149};
150
151struct iso_resource_once {
152 struct client *client;
153 struct work_struct work;
154 enum {
155 ISO_RES_ONCE_ALLOC,
156 ISO_RES_ONCE_DEALLOC,
157 } todo;
158 struct iso_resource_params params;
159 struct iso_resource_event *event;
160};
161
162static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)
163{
164 return container_of(resource, struct address_handler_resource, resource);
165}
166
167static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource)
168{
169 return container_of(resource, struct inbound_transaction_resource, resource);
170}
171
172static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource)
173{
174 return container_of(resource, struct descriptor_resource, resource);
175}
176
177static struct iso_resource_auto *to_iso_resource_auto(struct client_resource *resource)
178{
179 return container_of(resource, struct iso_resource_auto, resource);
180}
181
182static void release_iso_resource_auto(struct client *, struct client_resource *);
183
184static int is_iso_resource_auto(const struct client_resource *resource)
185{
186 return resource->release == release_iso_resource_auto;
187}
188
189static void release_transaction(struct client *client,
190 struct client_resource *resource);
191
192static int is_outbound_transaction_resource(const struct client_resource *resource)
193{
194 return resource->release == release_transaction;
195}
196
197static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay)
198{
199 client_get(r->client);
200 if (!queue_delayed_work(fw_workqueue, &r->work, delay))
201 client_put(r->client);
202}
203
204/*
205 * dequeue_event() just kfree()'s the event, so the event has to be
206 * the first field in a struct XYZ_event.
207 */
208struct event {
209 struct { void *data; size_t size; } v[2];
210 struct list_head link;
211};
212
213struct bus_reset_event {
214 struct event event;
215 struct fw_cdev_event_bus_reset reset;
216};
217
218struct outbound_transaction_event {
219 struct event event;
220 struct client *client;
221 struct outbound_transaction_resource r;
222 union {
223 struct fw_cdev_event_response without_tstamp;
224 struct fw_cdev_event_response2 with_tstamp;
225 } rsp;
226};
227
228struct inbound_transaction_event {
229 struct event event;
230 union {
231 struct fw_cdev_event_request request;
232 struct fw_cdev_event_request2 request2;
233 struct fw_cdev_event_request3 with_tstamp;
234 } req;
235};
236
237struct iso_interrupt_event {
238 struct event event;
239 struct fw_cdev_event_iso_interrupt interrupt;
240};
241
242struct iso_interrupt_mc_event {
243 struct event event;
244 struct fw_cdev_event_iso_interrupt_mc interrupt;
245};
246
247struct iso_resource_event {
248 struct event event;
249 struct fw_cdev_event_iso_resource iso_resource;
250};
251
252struct outbound_phy_packet_event {
253 struct event event;
254 struct client *client;
255 struct fw_packet p;
256 union {
257 struct fw_cdev_event_phy_packet without_tstamp;
258 struct fw_cdev_event_phy_packet2 with_tstamp;
259 } phy_packet;
260};
261
262struct inbound_phy_packet_event {
263 struct event event;
264 union {
265 struct fw_cdev_event_phy_packet without_tstamp;
266 struct fw_cdev_event_phy_packet2 with_tstamp;
267 } phy_packet;
268};
269
270#ifdef CONFIG_COMPAT
271static void __user *u64_to_uptr(u64 value)
272{
273 if (in_compat_syscall())
274 return compat_ptr(value);
275 else
276 return (void __user *)(unsigned long)value;
277}
278
279static u64 uptr_to_u64(void __user *ptr)
280{
281 if (in_compat_syscall())
282 return ptr_to_compat(ptr);
283 else
284 return (u64)(unsigned long)ptr;
285}
286#else
287static inline void __user *u64_to_uptr(u64 value)
288{
289 return (void __user *)(unsigned long)value;
290}
291
292static inline u64 uptr_to_u64(void __user *ptr)
293{
294 return (u64)(unsigned long)ptr;
295}
296#endif /* CONFIG_COMPAT */
297
298static int fw_device_op_open(struct inode *inode, struct file *file)
299{
300 struct fw_device *device;
301 struct client *client;
302
303 device = fw_device_get_by_devt(inode->i_rdev);
304 if (device == NULL)
305 return -ENODEV;
306
307 if (fw_device_is_shutdown(device)) {
308 fw_device_put(device);
309 return -ENODEV;
310 }
311
312 client = kzalloc_obj(*client);
313 if (client == NULL) {
314 fw_device_put(device);
315 return -ENOMEM;
316 }
317
318 client->device = device;
319 spin_lock_init(&client->lock);
320 xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);
321 INIT_LIST_HEAD(&client->event_list);
322 init_waitqueue_head(&client->wait);
323 init_waitqueue_head(&client->tx_flush_wait);
324 INIT_LIST_HEAD(&client->phy_receiver_link);
325 INIT_LIST_HEAD(&client->link);
326 kref_init(&client->kref);
327 mutex_init(&client->iso_context_mutex);
328
329 file->private_data = client;
330
331 return nonseekable_open(inode, file);
332}
333
334static void queue_event(struct client *client, struct event *event,
335 void *data0, size_t size0, void *data1, size_t size1)
336{
337 event->v[0].data = data0;
338 event->v[0].size = size0;
339 event->v[1].data = data1;
340 event->v[1].size = size1;
341
342 scoped_guard(spinlock_irqsave, &client->lock) {
343 if (client->in_shutdown)
344 kfree(event);
345 else
346 list_add_tail(&event->link, &client->event_list);
347 }
348
349 wake_up_interruptible(&client->wait);
350}
351
352static int dequeue_event(struct client *client,
353 char __user *buffer, size_t count)
354{
355 struct event *event;
356 size_t size, total;
357 int i, ret;
358
359 ret = wait_event_interruptible(client->wait,
360 !list_empty(&client->event_list) ||
361 fw_device_is_shutdown(client->device));
362 if (ret < 0)
363 return ret;
364
365 if (list_empty(&client->event_list) &&
366 fw_device_is_shutdown(client->device))
367 return -ENODEV;
368
369 scoped_guard(spinlock_irq, &client->lock) {
370 event = list_first_entry(&client->event_list, struct event, link);
371 list_del(&event->link);
372 }
373
374 total = 0;
375 for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) {
376 size = min(event->v[i].size, count - total);
377 if (copy_to_user(buffer + total, event->v[i].data, size)) {
378 ret = -EFAULT;
379 goto out;
380 }
381 total += size;
382 }
383 ret = total;
384
385 out:
386 kfree(event);
387
388 return ret;
389}
390
391static ssize_t fw_device_op_read(struct file *file, char __user *buffer,
392 size_t count, loff_t *offset)
393{
394 struct client *client = file->private_data;
395
396 return dequeue_event(client, buffer, count);
397}
398
399static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,
400 struct client *client)
401{
402 struct fw_card *card = client->device->card;
403
404 guard(spinlock_irq)(&card->lock);
405
406 event->closure = client->bus_reset_closure;
407 event->type = FW_CDEV_EVENT_BUS_RESET;
408 event->generation = client->device->generation;
409 event->node_id = client->device->node_id;
410 event->local_node_id = card->local_node->node_id;
411 event->bm_node_id = card->bm_node_id;
412 event->irm_node_id = card->irm_node->node_id;
413 event->root_node_id = card->root_node->node_id;
414}
415
416static void for_each_client(struct fw_device *device,
417 void (*callback)(struct client *client))
418{
419 struct client *c;
420
421 guard(mutex)(&device->client_list_mutex);
422
423 list_for_each_entry(c, &device->client_list, link)
424 callback(c);
425}
426
427static void queue_bus_reset_event(struct client *client)
428{
429 struct bus_reset_event *e;
430 struct client_resource *resource;
431 unsigned long index;
432
433 e = kzalloc_obj(*e);
434 if (e == NULL)
435 return;
436
437 fill_bus_reset_event(&e->reset, client);
438
439 queue_event(client, &e->event,
440 &e->reset, sizeof(e->reset), NULL, 0);
441
442 guard(spinlock_irq)(&client->lock);
443
444 xa_for_each(&client->resource_xa, index, resource) {
445 if (is_iso_resource_auto(resource))
446 schedule_iso_resource_auto(to_iso_resource_auto(resource), 0);
447 }
448}
449
450void fw_device_cdev_update(struct fw_device *device)
451{
452 for_each_client(device, queue_bus_reset_event);
453}
454
455static void wake_up_client(struct client *client)
456{
457 wake_up_interruptible(&client->wait);
458}
459
460void fw_device_cdev_remove(struct fw_device *device)
461{
462 for_each_client(device, wake_up_client);
463}
464
465union ioctl_arg {
466 struct fw_cdev_get_info get_info;
467 struct fw_cdev_send_request send_request;
468 struct fw_cdev_allocate allocate;
469 struct fw_cdev_deallocate deallocate;
470 struct fw_cdev_send_response send_response;
471 struct fw_cdev_initiate_bus_reset initiate_bus_reset;
472 struct fw_cdev_add_descriptor add_descriptor;
473 struct fw_cdev_remove_descriptor remove_descriptor;
474 struct fw_cdev_create_iso_context create_iso_context;
475 struct fw_cdev_queue_iso queue_iso;
476 struct fw_cdev_start_iso start_iso;
477 struct fw_cdev_stop_iso stop_iso;
478 struct fw_cdev_get_cycle_timer get_cycle_timer;
479 struct fw_cdev_allocate_iso_resource allocate_iso_resource;
480 struct fw_cdev_send_stream_packet send_stream_packet;
481 struct fw_cdev_get_cycle_timer2 get_cycle_timer2;
482 struct fw_cdev_send_phy_packet send_phy_packet;
483 struct fw_cdev_receive_phy_packets receive_phy_packets;
484 struct fw_cdev_set_iso_channels set_iso_channels;
485 struct fw_cdev_flush_iso flush_iso;
486};
487
488static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
489{
490 struct fw_cdev_get_info *a = &arg->get_info;
491 struct fw_cdev_event_bus_reset bus_reset;
492 unsigned long ret = 0;
493
494 client->version = a->version;
495 a->version = FW_CDEV_KERNEL_VERSION;
496 a->card = client->device->card->index;
497
498 scoped_guard(rwsem_read, &fw_device_rwsem) {
499 if (a->rom != 0) {
500 size_t want = a->rom_length;
501 size_t have = client->device->config_rom_length * 4;
502
503 ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom,
504 min(want, have));
505 if (ret != 0)
506 return -EFAULT;
507 }
508 a->rom_length = client->device->config_rom_length * 4;
509 }
510
511 guard(mutex)(&client->device->client_list_mutex);
512
513 client->bus_reset_closure = a->bus_reset_closure;
514 if (a->bus_reset != 0) {
515 fill_bus_reset_event(&bus_reset, client);
516 /* unaligned size of bus_reset is 36 bytes */
517 ret = copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36);
518 }
519 if (ret == 0 && list_empty(&client->link))
520 list_add_tail(&client->link, &client->device->client_list);
521
522 return ret ? -EFAULT : 0;
523}
524
525static int add_client_resource(struct client *client, struct client_resource *resource,
526 gfp_t gfp_mask)
527{
528 scoped_guard(spinlock_irqsave, &client->lock) {
529 u32 index;
530 int ret;
531
532 if (client->in_shutdown)
533 return -ECANCELED;
534
535 if (gfpflags_allow_blocking(gfp_mask)) {
536 ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
537 GFP_NOWAIT);
538 } else {
539 ret = xa_alloc_bh(&client->resource_xa, &index, resource,
540 xa_limit_32b, GFP_NOWAIT);
541 }
542 if (ret < 0)
543 return ret;
544
545 resource->handle = index;
546 client_get(client);
547 }
548
549 return 0;
550}
551
552static int release_client_resource(struct client *client, u32 handle,
553 client_resource_release_fn_t release,
554 struct client_resource **return_resource)
555{
556 unsigned long index = handle;
557 struct client_resource *resource;
558
559 scoped_guard(spinlock_irq, &client->lock) {
560 if (client->in_shutdown)
561 return -EINVAL;
562
563 resource = xa_load(&client->resource_xa, index);
564 if (!resource || resource->release != release)
565 return -EINVAL;
566
567 xa_erase(&client->resource_xa, handle);
568 }
569
570 if (return_resource)
571 *return_resource = resource;
572 else
573 resource->release(client, resource);
574
575 client_put(client);
576
577 return 0;
578}
579
580static void release_transaction(struct client *client,
581 struct client_resource *resource)
582{
583}
584
585static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp,
586 u32 response_tstamp, void *payload, size_t length, void *data)
587{
588 struct outbound_transaction_event *e = data;
589 struct client *client = e->client;
590 unsigned long index = e->r.resource.handle;
591
592 scoped_guard(spinlock_irqsave, &client->lock) {
593 xa_erase(&client->resource_xa, index);
594 if (client->in_shutdown)
595 wake_up(&client->tx_flush_wait);
596 }
597
598 switch (e->rsp.without_tstamp.type) {
599 case FW_CDEV_EVENT_RESPONSE:
600 {
601 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
602
603 if (length < rsp->length)
604 rsp->length = length;
605 if (rcode == RCODE_COMPLETE)
606 memcpy(rsp->data, payload, rsp->length);
607
608 rsp->rcode = rcode;
609
610 // In the case that sizeof(*rsp) doesn't align with the position of the
611 // data, and the read is short, preserve an extra copy of the data
612 // to stay compatible with a pre-2.6.27 bug. Since the bug is harmless
613 // for short reads and some apps depended on it, this is both safe
614 // and prudent for compatibility.
615 if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))
616 queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length);
617 else
618 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
619
620 break;
621 }
622 case FW_CDEV_EVENT_RESPONSE2:
623 {
624 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
625
626 if (length < rsp->length)
627 rsp->length = length;
628 if (rcode == RCODE_COMPLETE)
629 memcpy(rsp->data, payload, rsp->length);
630
631 rsp->rcode = rcode;
632 rsp->request_tstamp = request_tstamp;
633 rsp->response_tstamp = response_tstamp;
634
635 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);
636
637 break;
638 }
639 default:
640 WARN_ON(1);
641 break;
642 }
643
644 // Drop the xarray's reference.
645 client_put(client);
646}
647
648static int init_request(struct client *client,
649 struct fw_cdev_send_request *request,
650 int destination_id, int speed)
651{
652 struct outbound_transaction_event *e;
653 void *payload;
654 int ret;
655
656 if (request->tcode != TCODE_STREAM_DATA &&
657 (request->length > 4096 || request->length > 512 << speed))
658 return -EIO;
659
660 if (request->tcode == TCODE_WRITE_QUADLET_REQUEST &&
661 request->length < 4)
662 return -EINVAL;
663
664 e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL);
665 if (e == NULL)
666 return -ENOMEM;
667 e->client = client;
668
669 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
670 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;
671
672 rsp->type = FW_CDEV_EVENT_RESPONSE;
673 rsp->length = request->length;
674 rsp->closure = request->closure;
675 payload = rsp->data;
676 } else {
677 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;
678
679 rsp->type = FW_CDEV_EVENT_RESPONSE2;
680 rsp->length = request->length;
681 rsp->closure = request->closure;
682 payload = rsp->data;
683 }
684
685 if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) {
686 ret = -EFAULT;
687 goto failed;
688 }
689
690 e->r.resource.release = release_transaction;
691 ret = add_client_resource(client, &e->r.resource, GFP_KERNEL);
692 if (ret < 0)
693 goto failed;
694
695 fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode,
696 destination_id, request->generation, speed, request->offset,
697 payload, request->length, complete_transaction, e);
698 return 0;
699
700 failed:
701 kfree(e);
702
703 return ret;
704}
705
706static int ioctl_send_request(struct client *client, union ioctl_arg *arg)
707{
708 switch (arg->send_request.tcode) {
709 case TCODE_WRITE_QUADLET_REQUEST:
710 case TCODE_WRITE_BLOCK_REQUEST:
711 case TCODE_READ_QUADLET_REQUEST:
712 case TCODE_READ_BLOCK_REQUEST:
713 case TCODE_LOCK_MASK_SWAP:
714 case TCODE_LOCK_COMPARE_SWAP:
715 case TCODE_LOCK_FETCH_ADD:
716 case TCODE_LOCK_LITTLE_ADD:
717 case TCODE_LOCK_BOUNDED_ADD:
718 case TCODE_LOCK_WRAP_ADD:
719 case TCODE_LOCK_VENDOR_DEPENDENT:
720 break;
721 default:
722 return -EINVAL;
723 }
724
725 return init_request(client, &arg->send_request, client->device->node_id,
726 client->device->max_speed);
727}
728
729static void release_request(struct client *client,
730 struct client_resource *resource)
731{
732 struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource);
733
734 if (r->is_fcp)
735 fw_request_put(r->request);
736 else
737 fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR);
738
739 fw_card_put(r->card);
740 kfree(r);
741}
742
743static void handle_request(struct fw_card *card, struct fw_request *request,
744 int tcode, int destination, int source,
745 int generation, unsigned long long offset,
746 void *payload, size_t length, void *callback_data)
747{
748 struct address_handler_resource *handler = callback_data;
749 bool is_fcp = is_in_fcp_region(offset, length);
750 struct inbound_transaction_resource *r;
751 struct inbound_transaction_event *e;
752 size_t event_size0;
753 int ret;
754
755 /* card may be different from handler->client->device->card */
756 fw_card_get(card);
757
758 // Extend the lifetime of data for request so that its payload is safely accessible in
759 // the process context for the client.
760 if (is_fcp)
761 fw_request_get(request);
762
763 r = kmalloc_obj(*r, GFP_ATOMIC);
764 e = kmalloc_obj(*e, GFP_ATOMIC);
765 if (r == NULL || e == NULL)
766 goto failed;
767
768 r->card = card;
769 r->request = request;
770 r->is_fcp = is_fcp;
771 r->data = payload;
772 r->length = length;
773
774 r->resource.release = release_request;
775 ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC);
776 if (ret < 0)
777 goto failed;
778
779 if (handler->client->version < FW_CDEV_VERSION_EVENT_REQUEST2) {
780 struct fw_cdev_event_request *req = &e->req.request;
781
782 if (tcode & 0x10)
783 tcode = TCODE_LOCK_REQUEST;
784
785 req->type = FW_CDEV_EVENT_REQUEST;
786 req->tcode = tcode;
787 req->offset = offset;
788 req->length = length;
789 req->handle = r->resource.handle;
790 req->closure = handler->closure;
791 event_size0 = sizeof(*req);
792 } else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
793 struct fw_cdev_event_request2 *req = &e->req.request2;
794
795 req->type = FW_CDEV_EVENT_REQUEST2;
796 req->tcode = tcode;
797 req->offset = offset;
798 req->source_node_id = source;
799 req->destination_node_id = destination;
800 req->card = card->index;
801 req->generation = generation;
802 req->length = length;
803 req->handle = r->resource.handle;
804 req->closure = handler->closure;
805 event_size0 = sizeof(*req);
806 } else {
807 struct fw_cdev_event_request3 *req = &e->req.with_tstamp;
808
809 req->type = FW_CDEV_EVENT_REQUEST3;
810 req->tcode = tcode;
811 req->offset = offset;
812 req->source_node_id = source;
813 req->destination_node_id = destination;
814 req->card = card->index;
815 req->generation = generation;
816 req->length = length;
817 req->handle = r->resource.handle;
818 req->closure = handler->closure;
819 req->tstamp = fw_request_get_timestamp(request);
820 event_size0 = sizeof(*req);
821 }
822
823 queue_event(handler->client, &e->event,
824 &e->req, event_size0, r->data, length);
825 return;
826
827 failed:
828 kfree(r);
829 kfree(e);
830
831 if (!is_fcp)
832 fw_send_response(card, request, RCODE_CONFLICT_ERROR);
833 else
834 fw_request_put(request);
835
836 fw_card_put(card);
837}
838
839static void release_address_handler(struct client *client,
840 struct client_resource *resource)
841{
842 struct address_handler_resource *r = to_address_handler_resource(resource);
843
844 fw_core_remove_address_handler(&r->handler);
845 kfree(r);
846}
847
848static int ioctl_allocate(struct client *client, union ioctl_arg *arg)
849{
850 struct fw_cdev_allocate *a = &arg->allocate;
851 struct address_handler_resource *r;
852 struct fw_address_region region;
853 int ret;
854
855 r = kmalloc_obj(*r);
856 if (r == NULL)
857 return -ENOMEM;
858
859 region.start = a->offset;
860 if (client->version < FW_CDEV_VERSION_ALLOCATE_REGION_END)
861 region.end = a->offset + a->length;
862 else
863 region.end = a->region_end;
864
865 r->handler.length = a->length;
866 r->handler.address_callback = handle_request;
867 r->handler.callback_data = r;
868 r->closure = a->closure;
869 r->client = client;
870
871 ret = fw_core_add_address_handler(&r->handler, &region);
872 if (ret < 0) {
873 kfree(r);
874 return ret;
875 }
876 a->offset = r->handler.offset;
877
878 r->resource.release = release_address_handler;
879 ret = add_client_resource(client, &r->resource, GFP_KERNEL);
880 if (ret < 0) {
881 release_address_handler(client, &r->resource);
882 return ret;
883 }
884 a->handle = r->resource.handle;
885
886 return 0;
887}
888
889static int ioctl_deallocate(struct client *client, union ioctl_arg *arg)
890{
891 return release_client_resource(client, arg->deallocate.handle,
892 release_address_handler, NULL);
893}
894
895static int ioctl_send_response(struct client *client, union ioctl_arg *arg)
896{
897 struct fw_cdev_send_response *a = &arg->send_response;
898 struct client_resource *resource;
899 struct inbound_transaction_resource *r;
900 int ret = 0;
901
902 if (release_client_resource(client, a->handle,
903 release_request, &resource) < 0)
904 return -EINVAL;
905
906 r = to_inbound_transaction_resource(resource);
907 if (r->is_fcp) {
908 fw_request_put(r->request);
909 goto out;
910 }
911
912 if (a->length != fw_get_response_length(r->request)) {
913 ret = -EINVAL;
914 fw_request_put(r->request);
915 goto out;
916 }
917 if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) {
918 ret = -EFAULT;
919 fw_request_put(r->request);
920 goto out;
921 }
922 fw_send_response(r->card, r->request, a->rcode);
923 out:
924 fw_card_put(r->card);
925 kfree(r);
926
927 return ret;
928}
929
930static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg)
931{
932 fw_schedule_bus_reset(client->device->card, true,
933 arg->initiate_bus_reset.type == FW_CDEV_SHORT_RESET);
934 return 0;
935}
936
937static void release_descriptor(struct client *client,
938 struct client_resource *resource)
939{
940 struct descriptor_resource *r = to_descriptor_resource(resource);
941
942 fw_core_remove_descriptor(&r->descriptor);
943 kfree(r);
944}
945
946static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)
947{
948 struct fw_cdev_add_descriptor *a = &arg->add_descriptor;
949 struct descriptor_resource *r;
950 int ret;
951
952 /* Access policy: Allow this ioctl only on local nodes' device files. */
953 if (!client->device->is_local)
954 return -ENOSYS;
955
956 if (a->length > 256)
957 return -EINVAL;
958
959 r = kmalloc_flex(*r, data, a->length);
960 if (r == NULL)
961 return -ENOMEM;
962
963 if (copy_from_user(r->data, u64_to_uptr(a->data),
964 flex_array_size(r, data, a->length))) {
965 ret = -EFAULT;
966 goto failed;
967 }
968
969 r->descriptor.length = a->length;
970 r->descriptor.immediate = a->immediate;
971 r->descriptor.key = a->key;
972 r->descriptor.data = r->data;
973
974 ret = fw_core_add_descriptor(&r->descriptor);
975 if (ret < 0)
976 goto failed;
977
978 r->resource.release = release_descriptor;
979 ret = add_client_resource(client, &r->resource, GFP_KERNEL);
980 if (ret < 0) {
981 fw_core_remove_descriptor(&r->descriptor);
982 goto failed;
983 }
984 a->handle = r->resource.handle;
985
986 return 0;
987 failed:
988 kfree(r);
989
990 return ret;
991}
992
993static int ioctl_remove_descriptor(struct client *client, union ioctl_arg *arg)
994{
995 return release_client_resource(client, arg->remove_descriptor.handle,
996 release_descriptor, NULL);
997}
998
999static void iso_callback(struct fw_iso_context *context, u32 cycle,
1000 size_t header_length, void *header, void *data)
1001{
1002 struct client *client = data;
1003 struct iso_interrupt_event *e;
1004
1005 e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL);
1006 if (e == NULL)
1007 return;
1008
1009 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT;
1010 e->interrupt.closure = client->iso_closure;
1011 e->interrupt.cycle = cycle;
1012 e->interrupt.header_length = header_length;
1013 memcpy(e->interrupt.header, header, header_length);
1014 queue_event(client, &e->event, &e->interrupt,
1015 sizeof(e->interrupt) + header_length, NULL, 0);
1016}
1017
1018static void iso_mc_callback(struct fw_iso_context *context,
1019 dma_addr_t completed, void *data)
1020{
1021 struct client *client = data;
1022 struct iso_interrupt_mc_event *e;
1023
1024 e = kmalloc_obj(*e);
1025 if (e == NULL)
1026 return;
1027
1028 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL;
1029 e->interrupt.closure = client->iso_closure;
1030 e->interrupt.completed = fw_iso_buffer_lookup(&client->buffer,
1031 completed);
1032 queue_event(client, &e->event, &e->interrupt,
1033 sizeof(e->interrupt), NULL, 0);
1034}
1035
1036static enum dma_data_direction iso_dma_direction(struct fw_iso_context *context)
1037{
1038 if (context->type == FW_ISO_CONTEXT_TRANSMIT)
1039 return DMA_TO_DEVICE;
1040 else
1041 return DMA_FROM_DEVICE;
1042}
1043
1044static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)
1045{
1046 struct fw_cdev_create_iso_context *a = &arg->create_iso_context;
1047 struct fw_iso_context *context;
1048 int ret;
1049
1050 BUILD_BUG_ON(FW_CDEV_ISO_CONTEXT_TRANSMIT != FW_ISO_CONTEXT_TRANSMIT ||
1051 FW_CDEV_ISO_CONTEXT_RECEIVE != FW_ISO_CONTEXT_RECEIVE ||
1052 FW_CDEV_ISO_CONTEXT_RECEIVE_MULTICHANNEL !=
1053 FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL);
1054
1055 switch (a->type) {
1056 case FW_ISO_CONTEXT_TRANSMIT:
1057 if (a->speed > SCODE_3200 || a->channel > 63)
1058 return -EINVAL;
1059 break;
1060
1061 case FW_ISO_CONTEXT_RECEIVE:
1062 if (a->header_size < 4 || (a->header_size & 3) ||
1063 a->channel > 63)
1064 return -EINVAL;
1065 break;
1066
1067 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1068 break;
1069
1070 default:
1071 return -EINVAL;
1072 }
1073
1074 if (a->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL)
1075 context = fw_iso_mc_context_create(client->device->card, iso_mc_callback, client);
1076 else
1077 context = fw_iso_context_create(client->device->card, a->type, a->channel, a->speed,
1078 a->header_size, iso_callback, client);
1079 if (IS_ERR(context))
1080 return PTR_ERR(context);
1081 if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW)
1082 context->flags |= FW_ISO_CONTEXT_FLAG_DROP_OVERFLOW_HEADERS;
1083
1084 // We only support one context at this time.
1085 scoped_guard(mutex, &client->iso_context_mutex) {
1086 if (client->iso_context != NULL) {
1087 fw_iso_context_destroy(context);
1088
1089 return -EBUSY;
1090 }
1091 // The DMA mapping operation is available if the buffer is already allocated by
1092 // mmap(2) system call. If not, it is delegated to the system call.
1093 if (client->buffer.pages && !client->buffer.dma_addrs) {
1094 ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
1095 iso_dma_direction(context));
1096 if (ret < 0) {
1097 fw_iso_context_destroy(context);
1098
1099 return ret;
1100 }
1101 }
1102 client->iso_closure = a->closure;
1103 client->iso_context = context;
1104 }
1105
1106 a->handle = 0;
1107
1108 return 0;
1109}
1110
1111static int ioctl_set_iso_channels(struct client *client, union ioctl_arg *arg)
1112{
1113 struct fw_cdev_set_iso_channels *a = &arg->set_iso_channels;
1114 struct fw_iso_context *ctx = client->iso_context;
1115
1116 if (ctx == NULL || a->handle != 0)
1117 return -EINVAL;
1118
1119 return fw_iso_context_set_channels(ctx, &a->channels);
1120}
1121
1122/* Macros for decoding the iso packet control header. */
1123#define GET_PAYLOAD_LENGTH(v) ((v) & 0xffff)
1124#define GET_INTERRUPT(v) (((v) >> 16) & 0x01)
1125#define GET_SKIP(v) (((v) >> 17) & 0x01)
1126#define GET_TAG(v) (((v) >> 18) & 0x03)
1127#define GET_SY(v) (((v) >> 20) & 0x0f)
1128#define GET_HEADER_LENGTH(v) (((v) >> 24) & 0xff)
1129
1130static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)
1131{
1132 struct fw_cdev_queue_iso *a = &arg->queue_iso;
1133 struct fw_cdev_iso_packet __user *p, *end, *next;
1134 struct fw_iso_context *ctx = client->iso_context;
1135 unsigned long payload, buffer_end, transmit_header_bytes = 0;
1136 u32 control;
1137 int count;
1138 DEFINE_RAW_FLEX(struct fw_iso_packet, u, header, 64);
1139
1140 if (ctx == NULL || a->handle != 0)
1141 return -EINVAL;
1142
1143 /*
1144 * If the user passes a non-NULL data pointer, has mmap()'ed
1145 * the iso buffer, and the pointer points inside the buffer,
1146 * we setup the payload pointers accordingly. Otherwise we
1147 * set them both to 0, which will still let packets with
1148 * payload_length == 0 through. In other words, if no packets
1149 * use the indirect payload, the iso buffer need not be mapped
1150 * and the a->data pointer is ignored.
1151 */
1152 payload = (unsigned long)a->data - client->vm_start;
1153 buffer_end = client->buffer.page_count << PAGE_SHIFT;
1154 if (a->data == 0 || client->buffer.pages == NULL ||
1155 payload >= buffer_end) {
1156 payload = 0;
1157 buffer_end = 0;
1158 }
1159
1160 if (ctx->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL && payload & 3)
1161 return -EINVAL;
1162
1163 p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets);
1164
1165 end = (void __user *)p + a->size;
1166 count = 0;
1167 while (p < end) {
1168 if (get_user(control, &p->control))
1169 return -EFAULT;
1170 u->payload_length = GET_PAYLOAD_LENGTH(control);
1171 u->interrupt = GET_INTERRUPT(control);
1172 u->skip = GET_SKIP(control);
1173 u->tag = GET_TAG(control);
1174 u->sy = GET_SY(control);
1175 u->header_length = GET_HEADER_LENGTH(control);
1176
1177 switch (ctx->type) {
1178 case FW_ISO_CONTEXT_TRANSMIT:
1179 if (u->header_length & 3)
1180 return -EINVAL;
1181 transmit_header_bytes = u->header_length;
1182 break;
1183
1184 case FW_ISO_CONTEXT_RECEIVE:
1185 if (u->header_length == 0 ||
1186 u->header_length % ctx->header_size != 0)
1187 return -EINVAL;
1188 break;
1189
1190 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:
1191 if (u->payload_length == 0 ||
1192 u->payload_length & 3)
1193 return -EINVAL;
1194 break;
1195 }
1196
1197 next = (struct fw_cdev_iso_packet __user *)
1198 &p->header[transmit_header_bytes / 4];
1199 if (next > end)
1200 return -EINVAL;
1201 if (copy_from_user
1202 (u->header, p->header, transmit_header_bytes))
1203 return -EFAULT;
1204 if (u->skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&
1205 u->header_length + u->payload_length > 0)
1206 return -EINVAL;
1207 if (payload + u->payload_length > buffer_end)
1208 return -EINVAL;
1209
1210 if (fw_iso_context_queue(ctx, u, &client->buffer, payload))
1211 break;
1212
1213 p = next;
1214 payload += u->payload_length;
1215 count++;
1216 }
1217 fw_iso_context_queue_flush(ctx);
1218
1219 a->size -= uptr_to_u64(p) - a->packets;
1220 a->packets = uptr_to_u64(p);
1221 a->data = client->vm_start + payload;
1222
1223 return count;
1224}
1225
1226static int ioctl_start_iso(struct client *client, union ioctl_arg *arg)
1227{
1228 struct fw_cdev_start_iso *a = &arg->start_iso;
1229
1230 BUILD_BUG_ON(
1231 FW_CDEV_ISO_CONTEXT_MATCH_TAG0 != FW_ISO_CONTEXT_MATCH_TAG0 ||
1232 FW_CDEV_ISO_CONTEXT_MATCH_TAG1 != FW_ISO_CONTEXT_MATCH_TAG1 ||
1233 FW_CDEV_ISO_CONTEXT_MATCH_TAG2 != FW_ISO_CONTEXT_MATCH_TAG2 ||
1234 FW_CDEV_ISO_CONTEXT_MATCH_TAG3 != FW_ISO_CONTEXT_MATCH_TAG3 ||
1235 FW_CDEV_ISO_CONTEXT_MATCH_ALL_TAGS != FW_ISO_CONTEXT_MATCH_ALL_TAGS);
1236
1237 if (client->iso_context == NULL || a->handle != 0)
1238 return -EINVAL;
1239
1240 if (client->iso_context->type == FW_ISO_CONTEXT_RECEIVE &&
1241 (a->tags == 0 || a->tags > 15 || a->sync > 15))
1242 return -EINVAL;
1243
1244 return fw_iso_context_start(client->iso_context,
1245 a->cycle, a->sync, a->tags);
1246}
1247
1248static int ioctl_stop_iso(struct client *client, union ioctl_arg *arg)
1249{
1250 struct fw_cdev_stop_iso *a = &arg->stop_iso;
1251
1252 if (client->iso_context == NULL || a->handle != 0)
1253 return -EINVAL;
1254
1255 return fw_iso_context_stop(client->iso_context);
1256}
1257
1258static int ioctl_flush_iso(struct client *client, union ioctl_arg *arg)
1259{
1260 struct fw_cdev_flush_iso *a = &arg->flush_iso;
1261
1262 if (client->iso_context == NULL || a->handle != 0)
1263 return -EINVAL;
1264
1265 return fw_iso_context_flush_completions(client->iso_context);
1266}
1267
1268static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg)
1269{
1270 struct fw_cdev_get_cycle_timer2 *a = &arg->get_cycle_timer2;
1271 struct fw_card *card = client->device->card;
1272 struct timespec64 ts = {0, 0};
1273 u32 cycle_time = 0;
1274 int ret;
1275
1276 guard(irq)();
1277
1278 ret = fw_card_read_cycle_time(card, &cycle_time);
1279 if (ret < 0)
1280 return ret;
1281
1282 switch (a->clk_id) {
1283 case CLOCK_REALTIME: ktime_get_real_ts64(&ts); break;
1284 case CLOCK_MONOTONIC: ktime_get_ts64(&ts); break;
1285 case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts); break;
1286 default:
1287 return -EINVAL;
1288 }
1289
1290 a->tv_sec = ts.tv_sec;
1291 a->tv_nsec = ts.tv_nsec;
1292 a->cycle_timer = cycle_time;
1293
1294 return 0;
1295}
1296
1297static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
1298{
1299 struct fw_cdev_get_cycle_timer *a = &arg->get_cycle_timer;
1300 struct fw_cdev_get_cycle_timer2 ct2;
1301
1302 ct2.clk_id = CLOCK_REALTIME;
1303 ioctl_get_cycle_timer2(client, (union ioctl_arg *)&ct2);
1304
1305 a->local_time = ct2.tv_sec * USEC_PER_SEC + ct2.tv_nsec / NSEC_PER_USEC;
1306 a->cycle_timer = ct2.cycle_timer;
1307
1308 return 0;
1309}
1310
1311static int fill_iso_resource_params(struct iso_resource_params *params,
1312 struct fw_cdev_allocate_iso_resource *request)
1313{
1314 if ((request->channels == 0 && request->bandwidth == 0) ||
1315 request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)
1316 return -EINVAL;
1317
1318 params->channels_mask = request->channels;
1319 params->bandwidth = request->bandwidth;
1320
1321 return 0;
1322}
1323
1324static void iso_resource_auto_work(struct work_struct *work)
1325{
1326 struct iso_resource_event *e;
1327 struct iso_resource_auto *r = from_work(r, work, work.work);
1328 struct client *client = r->client;
1329 unsigned long index = r->resource.handle;
1330 int current_generation, resource_generation, channel, bandwidth, todo;
1331 u64 reset_jiffies;
1332 bool free;
1333
1334 scoped_guard(spinlock_irq, &client->lock) {
1335 reset_jiffies = client->device->card->reset_jiffies;
1336 current_generation = client->device->generation;
1337 resource_generation = r->generation;
1338 r->generation = current_generation;
1339 todo = r->todo;
1340 }
1341
1342 switch (todo) {
1343 case ISO_RES_AUTO_ALLOC:
1344 // Allow 1000ms grace period for other reallocations.
1345 if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) {
1346 schedule_iso_resource_auto(r, msecs_to_jiffies(333));
1347 goto out;
1348 }
1349 break;
1350 case ISO_RES_AUTO_REALLOC:
1351 // We could be called twice within the same generation.
1352 if (resource_generation == current_generation)
1353 goto out;
1354 break;
1355 case ISO_RES_AUTO_DEALLOC:
1356 default:
1357 break;
1358 }
1359
1360 bandwidth = r->params.bandwidth;
1361
1362 fw_iso_resource_manage(client->device->card, current_generation, r->params.channels_mask,
1363 &channel, &bandwidth, todo != ISO_RES_AUTO_DEALLOC);
1364
1365 if (todo == ISO_RES_AUTO_DEALLOC) {
1366 free = true;
1367 e = r->e_dealloc;
1368 r->e_dealloc = NULL;
1369 } else {
1370 free = false;
1371
1372 // Is this generation outdated already? As long as this resource sticks in the
1373 // xarray, it will be scheduled again for a newer generation or at shutdown.
1374 if (channel == -EAGAIN)
1375 goto out;
1376
1377 bool success = channel >= 0 || bandwidth > 0;
1378
1379 if (!success) {
1380 // Allocation or reallocation failure? Pull this resource out of the
1381 // xarray and prepare for deletion, unless the client is shutting down.
1382 scoped_guard(spinlock_irq, &client->lock) {
1383 if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) {
1384 client_put(client);
1385 free = true;
1386 }
1387 }
1388 }
1389
1390 if (todo == ISO_RES_AUTO_REALLOC) {
1391 if (success)
1392 goto out;
1393
1394 // Notify the userspace client of the failure through a deallocation event.
1395 e = r->e_dealloc;
1396 r->e_dealloc = NULL;
1397 } else {
1398 // Transit from allocation to reallocation, except if the client requested
1399 // deallocation in the meantime.
1400 scoped_guard(spinlock_irq, &client->lock) {
1401 if (r->todo == ISO_RES_AUTO_ALLOC)
1402 r->todo = ISO_RES_AUTO_REALLOC;
1403 }
1404
1405 if (channel >= 0)
1406 r->params.channels_mask = BIT_ULL(channel);
1407
1408 e = r->e_alloc;
1409 r->e_alloc = NULL;
1410 }
1411 }
1412
1413 e->iso_resource.handle = r->resource.handle;
1414 e->iso_resource.channel = channel;
1415 e->iso_resource.bandwidth = bandwidth;
1416
1417 queue_event(client, &e->event,
1418 &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
1419
1420 if (free) {
1421 cancel_delayed_work(&r->work);
1422 kfree(r->e_alloc);
1423 kfree(r->e_dealloc);
1424 kfree(r);
1425 }
1426 out:
1427 client_put(client);
1428}
1429
1430static void release_iso_resource_auto(struct client *client, struct client_resource *resource)
1431{
1432 struct iso_resource_auto *r = to_iso_resource_auto(resource);
1433
1434 guard(spinlock_irq)(&client->lock);
1435
1436 r->todo = ISO_RES_AUTO_DEALLOC;
1437 schedule_iso_resource_auto(r, 0);
1438}
1439
1440static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *arg)
1441{
1442 struct fw_cdev_allocate_iso_resource *request = &arg->allocate_iso_resource;
1443 struct iso_resource_event *e1 __free(kfree) = kmalloc_obj(*e1);
1444 struct iso_resource_event *e2 __free(kfree) = kmalloc_obj(*e2);
1445 struct iso_resource_auto *r __free(kfree) = kmalloc_obj(*r);
1446 int err;
1447
1448 if (!r || !e1 || !e2)
1449 return -ENOMEM;
1450
1451 err = fill_iso_resource_params(&r->params, request);
1452 if (err < 0)
1453 return err;
1454
1455 INIT_DELAYED_WORK(&r->work, iso_resource_auto_work);
1456 r->client = client;
1457 r->todo = ISO_RES_AUTO_ALLOC;
1458 r->e_alloc = e1;
1459 r->e_dealloc = e2;
1460
1461 e1->iso_resource.closure = request->closure;
1462 e1->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
1463 e2->iso_resource.closure = request->closure;
1464 e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
1465
1466 r->resource.release = release_iso_resource_auto;
1467 err = add_client_resource(client, &r->resource, GFP_KERNEL);
1468 if (err < 0)
1469 return err;
1470 request->handle = r->resource.handle;
1471
1472 retain_and_null_ptr(e1);
1473 retain_and_null_ptr(e2);
1474 schedule_iso_resource_auto(no_free_ptr(r), 0);
1475
1476 return 0;
1477}
1478
1479static int ioctl_deallocate_iso_resource(struct client *client,
1480 union ioctl_arg *arg)
1481{
1482 return release_client_resource(client,
1483 arg->deallocate.handle, release_iso_resource_auto, NULL);
1484}
1485
1486#define UNAVAILABLE_HANDLE -1
1487
1488static void iso_resource_once_work(struct work_struct *work)
1489{
1490 struct iso_resource_once *r = from_work(r, work, work);
1491 struct client *client = r->client;
1492 struct iso_resource_event *e = r->event;
1493 int generation, channel, bandwidth;
1494
1495 scoped_guard(spinlock_irq, &client->lock)
1496 generation = client->device->generation;
1497
1498 bandwidth = r->params.bandwidth;
1499
1500 fw_iso_resource_manage(client->device->card, generation, r->params.channels_mask, &channel,
1501 &bandwidth, r->todo == ISO_RES_ONCE_ALLOC);
1502
1503 e->iso_resource.handle = UNAVAILABLE_HANDLE;
1504 e->iso_resource.channel = channel;
1505 e->iso_resource.bandwidth = bandwidth;
1506
1507 queue_event(client, &e->event, &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
1508
1509 cancel_work(&r->work);
1510 kfree(r);
1511
1512 client_put(client);
1513}
1514
1515static int init_iso_resource_once(struct client *client,
1516 struct fw_cdev_allocate_iso_resource *request, int todo)
1517{
1518 struct iso_resource_event *e __free(kfree) = kmalloc_obj(*e);
1519 struct iso_resource_once *r __free(kfree) = kmalloc_obj(*r);
1520 int err;
1521
1522 if (!r || !e)
1523 return -ENOMEM;
1524
1525 err = fill_iso_resource_params(&r->params, request);
1526 if (err < 0)
1527 return err;
1528
1529 INIT_WORK(&r->work, iso_resource_once_work);
1530 r->client = client;
1531 r->todo = todo;
1532
1533 if (todo == ISO_RES_ONCE_ALLOC)
1534 e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
1535 else
1536 e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
1537 e->iso_resource.closure = request->closure;
1538 r->event = no_free_ptr(e);
1539
1540 // Keep the client until work item finishing.
1541 client_get(r->client);
1542
1543 queue_work(fw_workqueue, &no_free_ptr(r)->work);
1544
1545 request->handle = UNAVAILABLE_HANDLE;
1546
1547 return 0;
1548}
1549
1550static int ioctl_allocate_iso_resource_once(struct client *client,
1551 union ioctl_arg *arg)
1552{
1553 return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_ALLOC);
1554}
1555
1556static int ioctl_deallocate_iso_resource_once(struct client *client,
1557 union ioctl_arg *arg)
1558{
1559 return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_DEALLOC);
1560}
1561
1562/*
1563 * Returns a speed code: Maximum speed to or from this device,
1564 * limited by the device's link speed, the local node's link speed,
1565 * and all PHY port speeds between the two links.
1566 */
1567static int ioctl_get_speed(struct client *client, union ioctl_arg *arg)
1568{
1569 return client->device->max_speed;
1570}
1571
1572static int ioctl_send_broadcast_request(struct client *client,
1573 union ioctl_arg *arg)
1574{
1575 struct fw_cdev_send_request *a = &arg->send_request;
1576
1577 switch (a->tcode) {
1578 case TCODE_WRITE_QUADLET_REQUEST:
1579 case TCODE_WRITE_BLOCK_REQUEST:
1580 break;
1581 default:
1582 return -EINVAL;
1583 }
1584
1585 /* Security policy: Only allow accesses to Units Space. */
1586 if (a->offset < CSR_REGISTER_BASE + CSR_CONFIG_ROM_END)
1587 return -EACCES;
1588
1589 return init_request(client, a, LOCAL_BUS | 0x3f, SCODE_100);
1590}
1591
1592static int ioctl_send_stream_packet(struct client *client, union ioctl_arg *arg)
1593{
1594 struct fw_cdev_send_stream_packet *a = &arg->send_stream_packet;
1595 struct fw_cdev_send_request request;
1596 int dest;
1597
1598 if (a->speed > client->device->card->link_speed ||
1599 a->length > 1024 << a->speed)
1600 return -EIO;
1601
1602 if (a->tag > 3 || a->channel > 63 || a->sy > 15)
1603 return -EINVAL;
1604
1605 dest = fw_stream_packet_destination_id(a->tag, a->channel, a->sy);
1606 request.tcode = TCODE_STREAM_DATA;
1607 request.length = a->length;
1608 request.closure = a->closure;
1609 request.data = a->data;
1610 request.generation = a->generation;
1611
1612 return init_request(client, &request, dest, a->speed);
1613}
1614
1615static void outbound_phy_packet_callback(struct fw_packet *packet,
1616 struct fw_card *card, int status)
1617{
1618 struct outbound_phy_packet_event *e =
1619 container_of(packet, struct outbound_phy_packet_event, p);
1620 struct client *e_client = e->client;
1621 u32 rcode;
1622
1623 trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation,
1624 packet->timestamp);
1625
1626 switch (status) {
1627 // expected:
1628 case ACK_COMPLETE:
1629 rcode = RCODE_COMPLETE;
1630 break;
1631 // should never happen with PHY packets:
1632 case ACK_PENDING:
1633 rcode = RCODE_COMPLETE;
1634 break;
1635 case ACK_BUSY_X:
1636 case ACK_BUSY_A:
1637 case ACK_BUSY_B:
1638 rcode = RCODE_BUSY;
1639 break;
1640 case ACK_DATA_ERROR:
1641 rcode = RCODE_DATA_ERROR;
1642 break;
1643 case ACK_TYPE_ERROR:
1644 rcode = RCODE_TYPE_ERROR;
1645 break;
1646 // stale generation; cancelled; on certain controllers: no ack
1647 default:
1648 rcode = status;
1649 break;
1650 }
1651
1652 switch (e->phy_packet.without_tstamp.type) {
1653 case FW_CDEV_EVENT_PHY_PACKET_SENT:
1654 {
1655 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1656
1657 pp->rcode = rcode;
1658 pp->data[0] = packet->timestamp;
1659 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1660 NULL, 0);
1661 break;
1662 }
1663 case FW_CDEV_EVENT_PHY_PACKET_SENT2:
1664 {
1665 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1666
1667 pp->rcode = rcode;
1668 pp->tstamp = packet->timestamp;
1669 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,
1670 NULL, 0);
1671 break;
1672 }
1673 default:
1674 WARN_ON(1);
1675 break;
1676 }
1677
1678 client_put(e_client);
1679}
1680
1681static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)
1682{
1683 struct fw_cdev_send_phy_packet *a = &arg->send_phy_packet;
1684 struct fw_card *card = client->device->card;
1685 struct outbound_phy_packet_event *e;
1686
1687 /* Access policy: Allow this ioctl only on local nodes' device files. */
1688 if (!client->device->is_local)
1689 return -ENOSYS;
1690
1691 e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL);
1692 if (e == NULL)
1693 return -ENOMEM;
1694
1695 client_get(client);
1696 e->client = client;
1697 e->p.speed = SCODE_100;
1698 e->p.generation = a->generation;
1699 async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL);
1700 e->p.header[1] = a->data[0];
1701 e->p.header[2] = a->data[1];
1702 e->p.header_length = 12;
1703 e->p.callback = outbound_phy_packet_callback;
1704
1705 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1706 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1707
1708 pp->closure = a->closure;
1709 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT;
1710 if (is_ping_packet(a->data))
1711 pp->length = 4;
1712 } else {
1713 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1714
1715 pp->closure = a->closure;
1716 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2;
1717 // Keep the data field so that application can match the response event to the
1718 // request.
1719 pp->length = sizeof(a->data);
1720 memcpy(pp->data, a->data, sizeof(a->data));
1721 }
1722
1723 trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation,
1724 e->p.header[1], e->p.header[2]);
1725
1726 card->driver->send_request(card, &e->p);
1727
1728 return 0;
1729}
1730
1731static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg)
1732{
1733 struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets;
1734
1735 /* Access policy: Allow this ioctl only on local nodes' device files. */
1736 if (!client->device->is_local)
1737 return -ENOSYS;
1738
1739 // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
1740 // destination never runs in any type of IRQ context.
1741 scoped_guard(spinlock_irq, &phy_receiver_list_lock)
1742 list_move_tail(&client->phy_receiver_link, &phy_receiver_list);
1743
1744 client->phy_receiver_closure = a->closure;
1745
1746 return 0;
1747}
1748
1749void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)
1750{
1751 struct client *client;
1752
1753 // NOTE: This can be without irqsave when we can guarantee that __fw_send_request() for local
1754 // destination never runs in any type of IRQ context.
1755 guard(spinlock_irqsave)(&phy_receiver_list_lock);
1756
1757 list_for_each_entry(client, &phy_receiver_list, phy_receiver_link) {
1758 struct inbound_phy_packet_event *e;
1759
1760 if (client->device->card != card)
1761 continue;
1762
1763 e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC);
1764 if (e == NULL)
1765 break;
1766
1767 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {
1768 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;
1769
1770 pp->closure = client->phy_receiver_closure;
1771 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;
1772 pp->rcode = RCODE_COMPLETE;
1773 pp->length = 8;
1774 pp->data[0] = p->header[1];
1775 pp->data[1] = p->header[2];
1776 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1777 } else {
1778 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;
1779
1780 pp = &e->phy_packet.with_tstamp;
1781 pp->closure = client->phy_receiver_closure;
1782 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2;
1783 pp->rcode = RCODE_COMPLETE;
1784 pp->length = 8;
1785 pp->tstamp = p->timestamp;
1786 pp->data[0] = p->header[1];
1787 pp->data[1] = p->header[2];
1788 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);
1789 }
1790 }
1791}
1792
1793static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = {
1794 [0x00] = ioctl_get_info,
1795 [0x01] = ioctl_send_request,
1796 [0x02] = ioctl_allocate,
1797 [0x03] = ioctl_deallocate,
1798 [0x04] = ioctl_send_response,
1799 [0x05] = ioctl_initiate_bus_reset,
1800 [0x06] = ioctl_add_descriptor,
1801 [0x07] = ioctl_remove_descriptor,
1802 [0x08] = ioctl_create_iso_context,
1803 [0x09] = ioctl_queue_iso,
1804 [0x0a] = ioctl_start_iso,
1805 [0x0b] = ioctl_stop_iso,
1806 [0x0c] = ioctl_get_cycle_timer,
1807 [0x0d] = ioctl_allocate_iso_resource,
1808 [0x0e] = ioctl_deallocate_iso_resource,
1809 [0x0f] = ioctl_allocate_iso_resource_once,
1810 [0x10] = ioctl_deallocate_iso_resource_once,
1811 [0x11] = ioctl_get_speed,
1812 [0x12] = ioctl_send_broadcast_request,
1813 [0x13] = ioctl_send_stream_packet,
1814 [0x14] = ioctl_get_cycle_timer2,
1815 [0x15] = ioctl_send_phy_packet,
1816 [0x16] = ioctl_receive_phy_packets,
1817 [0x17] = ioctl_set_iso_channels,
1818 [0x18] = ioctl_flush_iso,
1819};
1820
1821static int dispatch_ioctl(struct client *client,
1822 unsigned int cmd, void __user *arg)
1823{
1824 union ioctl_arg buffer;
1825 int ret;
1826
1827 if (fw_device_is_shutdown(client->device))
1828 return -ENODEV;
1829
1830 if (_IOC_TYPE(cmd) != '#' ||
1831 _IOC_NR(cmd) >= ARRAY_SIZE(ioctl_handlers) ||
1832 _IOC_SIZE(cmd) > sizeof(buffer))
1833 return -ENOTTY;
1834
1835 memset(&buffer, 0, sizeof(buffer));
1836
1837 if (_IOC_DIR(cmd) & _IOC_WRITE)
1838 if (copy_from_user(&buffer, arg, _IOC_SIZE(cmd)))
1839 return -EFAULT;
1840
1841 ret = ioctl_handlers[_IOC_NR(cmd)](client, &buffer);
1842 if (ret < 0)
1843 return ret;
1844
1845 if (_IOC_DIR(cmd) & _IOC_READ)
1846 if (copy_to_user(arg, &buffer, _IOC_SIZE(cmd)))
1847 return -EFAULT;
1848
1849 return ret;
1850}
1851
1852static long fw_device_op_ioctl(struct file *file,
1853 unsigned int cmd, unsigned long arg)
1854{
1855 return dispatch_ioctl(file->private_data, cmd, (void __user *)arg);
1856}
1857
1858static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)
1859{
1860 struct client *client = file->private_data;
1861 unsigned long size;
1862 int page_count, ret;
1863
1864 if (fw_device_is_shutdown(client->device))
1865 return -ENODEV;
1866
1867 /* FIXME: We could support multiple buffers, but we don't. */
1868 if (client->buffer.pages != NULL)
1869 return -EBUSY;
1870
1871 if (!(vma->vm_flags & VM_SHARED))
1872 return -EINVAL;
1873
1874 if (vma->vm_start & ~PAGE_MASK)
1875 return -EINVAL;
1876
1877 client->vm_start = vma->vm_start;
1878 size = vma->vm_end - vma->vm_start;
1879 page_count = size >> PAGE_SHIFT;
1880 if (size & ~PAGE_MASK)
1881 return -EINVAL;
1882
1883 ret = fw_iso_buffer_alloc(&client->buffer, page_count);
1884 if (ret < 0)
1885 return ret;
1886
1887 scoped_guard(mutex, &client->iso_context_mutex) {
1888 // The direction of DMA can be determined if the isochronous context is already
1889 // allocated. If not, the DMA mapping operation is postponed after the allocation.
1890 if (client->iso_context) {
1891 ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,
1892 iso_dma_direction(client->iso_context));
1893 if (ret < 0)
1894 goto fail;
1895 }
1896 }
1897
1898 ret = vm_map_pages_zero(vma, client->buffer.pages,
1899 client->buffer.page_count);
1900 if (ret < 0)
1901 goto fail;
1902
1903 return 0;
1904 fail:
1905 fw_iso_buffer_destroy(&client->buffer, client->device->card);
1906 return ret;
1907}
1908
1909static bool has_outbound_transactions(struct client *client)
1910{
1911 struct client_resource *resource;
1912 unsigned long index;
1913
1914 guard(spinlock_irq)(&client->lock);
1915
1916 xa_for_each(&client->resource_xa, index, resource) {
1917 if (is_outbound_transaction_resource(resource))
1918 return true;
1919 }
1920
1921 return false;
1922}
1923
1924static int fw_device_op_release(struct inode *inode, struct file *file)
1925{
1926 struct client *client = file->private_data;
1927 struct event *event, *next_event;
1928 struct client_resource *resource;
1929 unsigned long index;
1930
1931 // NOTE: This can be without irq when we can guarantee that __fw_send_request() for local
1932 // destination never runs in any type of IRQ context.
1933 scoped_guard(spinlock_irq, &phy_receiver_list_lock)
1934 list_del(&client->phy_receiver_link);
1935
1936 scoped_guard(mutex, &client->device->client_list_mutex)
1937 list_del(&client->link);
1938
1939 if (client->iso_context)
1940 fw_iso_context_destroy(client->iso_context);
1941 mutex_destroy(&client->iso_context_mutex);
1942
1943 if (client->buffer.pages)
1944 fw_iso_buffer_destroy(&client->buffer, client->device->card);
1945
1946 // Freeze client->resource_xa and client->event_list.
1947 scoped_guard(spinlock_irq, &client->lock)
1948 client->in_shutdown = true;
1949
1950 wait_event(client->tx_flush_wait, !has_outbound_transactions(client));
1951
1952 xa_for_each(&client->resource_xa, index, resource) {
1953 resource->release(client, resource);
1954 client_put(client);
1955 }
1956 xa_destroy(&client->resource_xa);
1957
1958 list_for_each_entry_safe(event, next_event, &client->event_list, link)
1959 kfree(event);
1960
1961 client_put(client);
1962
1963 return 0;
1964}
1965
1966static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)
1967{
1968 struct client *client = file->private_data;
1969 __poll_t mask = 0;
1970
1971 poll_wait(file, &client->wait, pt);
1972
1973 if (fw_device_is_shutdown(client->device))
1974 mask |= EPOLLHUP | EPOLLERR;
1975 if (!list_empty(&client->event_list))
1976 mask |= EPOLLIN | EPOLLRDNORM;
1977
1978 return mask;
1979}
1980
1981const struct file_operations fw_device_ops = {
1982 .owner = THIS_MODULE,
1983 .open = fw_device_op_open,
1984 .read = fw_device_op_read,
1985 .unlocked_ioctl = fw_device_op_ioctl,
1986 .mmap = fw_device_op_mmap,
1987 .release = fw_device_op_release,
1988 .poll = fw_device_op_poll,
1989 .compat_ioctl = compat_ptr_ioctl,
1990};