linux - drivers/gpio/gpiolib-cdev.c

2784 lines · 71.9 KiB

1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/anon_inodes.h>
4	#include <linux/atomic.h>
5	#include <linux/bitmap.h>
6	#include <linux/build_bug.h>
7	#include <linux/cdev.h>
8	#include <linux/cleanup.h>
9	#include <linux/compat.h>
10	#include <linux/compiler.h>
11	#include <linux/device.h>
12	#include <linux/err.h>
13	#include <linux/file.h>
14	#include <linux/gpio.h>
15	#include <linux/gpio/driver.h>
16	#include <linux/hte.h>
17	#include <linux/interrupt.h>
18	#include <linux/irqreturn.h>
19	#include <linux/kfifo.h>
20	#include <linux/module.h>
21	#include <linux/mutex.h>
22	#include <linux/overflow.h>
23	#include <linux/pinctrl/consumer.h>
24	#include <linux/poll.h>
25	#include <linux/seq_file.h>
26	#include <linux/spinlock.h>
27	#include <linux/string.h>
28	#include <linux/timekeeping.h>
29	#include <linux/uaccess.h>
30	#include <linux/workqueue.h>
31
32	#include <uapi/linux/gpio.h>
33
34	#include "gpiolib.h"
35	#include "gpiolib-cdev.h"
36
37	/*
38	* Array sizes must ensure 64-bit alignment and not create holes in the
39	* struct packing.
40	*/
41	static_assert(IS_ALIGNED(GPIO_V2_LINES_MAX, 2));
42	static_assert(IS_ALIGNED(GPIO_MAX_NAME_SIZE, 8));
43
44	/*
45	* Check that uAPI structs are 64-bit aligned for 32/64-bit compatibility
46	*/
47	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_attribute), 8));
48	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config_attribute), 8));
49	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config), 8));
50	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_request), 8));
51	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info), 8));
52	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info_changed), 8));
53	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_event), 8));
54	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_values), 8));
55
56	/* Character device interface to GPIO.
57	*
58	* The GPIO character device, /dev/gpiochipN, provides userspace an
59	* interface to gpiolib GPIOs via ioctl()s.
60	*/
61
62	/*
63	* GPIO line handle management
64	*/
65
66	#ifdef CONFIG_GPIO_CDEV_V1
67	/**
68	* struct linehandle_state - contains the state of a userspace handle
69	* @gdev: the GPIO device the handle pertains to
70	* @label: consumer label used to tag descriptors
71	* @descs: the GPIO descriptors held by this handle
72	* @num_descs: the number of descriptors held in the descs array
73	*/
74	struct linehandle_state {
75	struct gpio_device *gdev;
76	const char *label;
77	struct gpio_desc *descs[GPIOHANDLES_MAX];
78	u32 num_descs;
79	};
80
81	#define GPIOHANDLE_REQUEST_VALID_FLAGS \
82	(GPIOHANDLE_REQUEST_INPUT \| \
83	GPIOHANDLE_REQUEST_OUTPUT \| \
84	GPIOHANDLE_REQUEST_ACTIVE_LOW \| \
85	GPIOHANDLE_REQUEST_BIAS_PULL_UP \| \
86	GPIOHANDLE_REQUEST_BIAS_PULL_DOWN \| \
87	GPIOHANDLE_REQUEST_BIAS_DISABLE \| \
88	GPIOHANDLE_REQUEST_OPEN_DRAIN \| \
89	GPIOHANDLE_REQUEST_OPEN_SOURCE)
90
91	#define GPIOHANDLE_REQUEST_DIRECTION_FLAGS \
92	(GPIOHANDLE_REQUEST_INPUT \| \
93	GPIOHANDLE_REQUEST_OUTPUT)
94
95	static int linehandle_validate_flags(u32 flags)
96	{
97	/* Return an error if an unknown flag is set */
98	if (flags & ~GPIOHANDLE_REQUEST_VALID_FLAGS)
99	return -EINVAL;
100
101	/*
102	* Do not allow both INPUT & OUTPUT flags to be set as they are
103	* contradictory.
104	*/
105	if ((flags & GPIOHANDLE_REQUEST_INPUT) &&
106	(flags & GPIOHANDLE_REQUEST_OUTPUT))
107	return -EINVAL;
108
109	/*
110	* Do not allow OPEN_SOURCE & OPEN_DRAIN flags in a single request. If
111	* the hardware actually supports enabling both at the same time the
112	* electrical result would be disastrous.
113	*/
114	if ((flags & GPIOHANDLE_REQUEST_OPEN_DRAIN) &&
115	(flags & GPIOHANDLE_REQUEST_OPEN_SOURCE))
116	return -EINVAL;
117
118	/* OPEN_DRAIN and OPEN_SOURCE flags only make sense for output mode. */
119	if (!(flags & GPIOHANDLE_REQUEST_OUTPUT) &&
120	((flags & GPIOHANDLE_REQUEST_OPEN_DRAIN) \|\|
121	(flags & GPIOHANDLE_REQUEST_OPEN_SOURCE)))
122	return -EINVAL;
123
124	/* Bias flags only allowed for input or output mode. */
125	if (!((flags & GPIOHANDLE_REQUEST_INPUT) \|\|
126	(flags & GPIOHANDLE_REQUEST_OUTPUT)) &&
127	((flags & GPIOHANDLE_REQUEST_BIAS_DISABLE) \|\|
128	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_UP) \|\|
129	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN)))
130	return -EINVAL;
131
132	/* Only one bias flag can be set. */
133	if (((flags & GPIOHANDLE_REQUEST_BIAS_DISABLE) &&
134	(flags & (GPIOHANDLE_REQUEST_BIAS_PULL_DOWN \|
135	GPIOHANDLE_REQUEST_BIAS_PULL_UP))) \|\|
136	((flags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN) &&
137	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_UP)))
138	return -EINVAL;
139
140	return 0;
141	}
142
143	static void linehandle_flags_to_desc_flags(u32 lflags, unsigned long *flagsp)
144	{
145	unsigned long flags = READ_ONCE(*flagsp);
146
147	assign_bit(GPIOD_FLAG_ACTIVE_LOW, &flags,
148	lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW);
149	assign_bit(GPIOD_FLAG_OPEN_DRAIN, &flags,
150	lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN);
151	assign_bit(GPIOD_FLAG_OPEN_SOURCE, &flags,
152	lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE);
153	assign_bit(GPIOD_FLAG_PULL_UP, &flags,
154	lflags & GPIOHANDLE_REQUEST_BIAS_PULL_UP);
155	assign_bit(GPIOD_FLAG_PULL_DOWN, &flags,
156	lflags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN);
157	assign_bit(GPIOD_FLAG_BIAS_DISABLE, &flags,
158	lflags & GPIOHANDLE_REQUEST_BIAS_DISABLE);
159
160	WRITE_ONCE(*flagsp, flags);
161	}
162
163	static long linehandle_set_config(struct linehandle_state *lh,
164	void __user *ip)
165	{
166	struct gpiohandle_config gcnf;
167	struct gpio_desc *desc;
168	int i, ret;
169	u32 lflags;
170
171	if (copy_from_user(&gcnf, ip, sizeof(gcnf)))
172	return -EFAULT;
173
174	lflags = gcnf.flags;
175	ret = linehandle_validate_flags(lflags);
176	if (ret)
177	return ret;
178
179	/* Lines must be reconfigured explicitly as input or output. */
180	if (!(lflags & GPIOHANDLE_REQUEST_DIRECTION_FLAGS))
181	return -EINVAL;
182
183	for (i = 0; i < lh->num_descs; i++) {
184	desc = lh->descs[i];
185	linehandle_flags_to_desc_flags(lflags, &desc->flags);
186
187	if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
188	int val = !!gcnf.default_values[i];
189
190	ret = gpiod_direction_output_nonotify(desc, val);
191	if (ret)
192	return ret;
193	} else {
194	ret = gpiod_direction_input_nonotify(desc);
195	if (ret)
196	return ret;
197	}
198
199	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_CONFIG);
200	}
201	return 0;
202	}
203
204	static long linehandle_ioctl(struct file *file, unsigned int cmd,
205	unsigned long arg)
206	{
207	struct linehandle_state *lh = file->private_data;
208	void __user ip = (void __user )arg;
209	struct gpiohandle_data ghd;
210	DECLARE_BITMAP(vals, GPIOHANDLES_MAX);
211	unsigned int i;
212	int ret;
213
214	guard(srcu)(&lh->gdev->srcu);
215
216	if (!rcu_access_pointer(lh->gdev->chip))
217	return -ENODEV;
218
219	switch (cmd) {
220	case GPIOHANDLE_GET_LINE_VALUES_IOCTL:
221	/* NOTE: It's okay to read values of output lines */
222	ret = gpiod_get_array_value_complex(false, true,
223	lh->num_descs, lh->descs,
224	NULL, vals);
225	if (ret)
226	return ret;
227
228	memset(&ghd, 0, sizeof(ghd));
229	for (i = 0; i < lh->num_descs; i++)
230	ghd.values[i] = test_bit(i, vals);
231
232	if (copy_to_user(ip, &ghd, sizeof(ghd)))
233	return -EFAULT;
234
235	return 0;
236	case GPIOHANDLE_SET_LINE_VALUES_IOCTL:
237	/*
238	* All line descriptors were created at once with the same
239	* flags so just check if the first one is really output.
240	*/
241	if (!test_bit(GPIOD_FLAG_IS_OUT, &lh->descs[0]->flags))
242	return -EPERM;
243
244	if (copy_from_user(&ghd, ip, sizeof(ghd)))
245	return -EFAULT;
246
247	/* Clamp all values to [0,1] */
248	for (i = 0; i < lh->num_descs; i++)
249	__assign_bit(i, vals, ghd.values[i]);
250
251	/* Reuse the array setting function */
252	return gpiod_set_array_value_complex(false,
253	true,
254	lh->num_descs,
255	lh->descs,
256	NULL,
257	vals);
258	case GPIOHANDLE_SET_CONFIG_IOCTL:
259	return linehandle_set_config(lh, ip);
260	default:
261	return -EINVAL;
262	}
263	}
264
265	#ifdef CONFIG_COMPAT
266	static long linehandle_ioctl_compat(struct file *file, unsigned int cmd,
267	unsigned long arg)
268	{
269	return linehandle_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
270	}
271	#endif
272
273	static void linehandle_free(struct linehandle_state *lh)
274	{
275	int i;
276
277	for (i = 0; i < lh->num_descs; i++)
278	if (lh->descs[i])
279	gpiod_free(lh->descs[i]);
280	kfree(lh->label);
281	gpio_device_put(lh->gdev);
282	kfree(lh);
283	}
284
285	static int linehandle_release(struct inode inode, struct file file)
286	{
287	linehandle_free(file->private_data);
288	return 0;
289	}
290
291	static const struct file_operations linehandle_fileops = {
292	.release = linehandle_release,
293	.owner = THIS_MODULE,
294	.llseek = noop_llseek,
295	.unlocked_ioctl = linehandle_ioctl,
296	#ifdef CONFIG_COMPAT
297	.compat_ioctl = linehandle_ioctl_compat,
298	#endif
299	};
300
301	DEFINE_FREE(linehandle_free, struct linehandle_state *, if (!IS_ERR_OR_NULL(_T)) linehandle_free(_T))
302
303	static int linehandle_create(struct gpio_device gdev, void __user ip)
304	{
305	struct gpiohandle_request handlereq;
306	struct linehandle_state *lh __free(linehandle_free) = NULL;
307	int i, ret;
308	u32 lflags;
309
310	if (copy_from_user(&handlereq, ip, sizeof(handlereq)))
311	return -EFAULT;
312	if ((handlereq.lines == 0) \|\| (handlereq.lines > GPIOHANDLES_MAX))
313	return -EINVAL;
314
315	lflags = handlereq.flags;
316
317	ret = linehandle_validate_flags(lflags);
318	if (ret)
319	return ret;
320
321	lh = kzalloc_obj(*lh);
322	if (!lh)
323	return -ENOMEM;
324	lh->gdev = gpio_device_get(gdev);
325
326	if (handlereq.consumer_label[0] != '\0') {
327	/* label is only initialized if consumer_label is set */
328	lh->label = kstrndup(handlereq.consumer_label,
329	sizeof(handlereq.consumer_label) - 1,
330	GFP_KERNEL);
331	if (!lh->label)
332	return -ENOMEM;
333	}
334
335	lh->num_descs = handlereq.lines;
336
337	/* Request each GPIO */
338	for (i = 0; i < handlereq.lines; i++) {
339	u32 offset = handlereq.lineoffsets[i];
340	struct gpio_desc *desc = gpio_device_get_desc(gdev, offset);
341
342	if (IS_ERR(desc))
343	return PTR_ERR(desc);
344
345	ret = gpiod_request_user(desc, lh->label);
346	if (ret)
347	return ret;
348	lh->descs[i] = desc;
349	linehandle_flags_to_desc_flags(handlereq.flags, &desc->flags);
350
351	ret = gpiod_set_transitory(desc, false);
352	if (ret < 0)
353	return ret;
354
355	/*
356	* Lines have to be requested explicitly for input
357	* or output, else the line will be treated "as is".
358	*/
359	if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
360	int val = !!handlereq.default_values[i];
361
362	ret = gpiod_direction_output_nonotify(desc, val);
363	if (ret)
364	return ret;
365	} else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
366	ret = gpiod_direction_input_nonotify(desc);
367	if (ret)
368	return ret;
369	}
370
371	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED);
372
373	dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
374	offset);
375	}
376
377	FD_PREPARE(fdf, O_RDONLY \| O_CLOEXEC,
378	anon_inode_getfile("gpio-linehandle", &linehandle_fileops,
379	lh, O_RDONLY \| O_CLOEXEC));
380	if (fdf->fd < 0)
381	return fdf->fd;
382	retain_and_null_ptr(lh);
383
384	handlereq.fd = fdf->fd;
385	if (copy_to_user(ip, &handlereq, sizeof(handlereq)))
386	return -EFAULT;
387
388	fd_publish(fdf);
389
390	dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
391	handlereq.lines);
392
393	return 0;
394	}
395	#endif /* CONFIG_GPIO_CDEV_V1 */
396
397	/**
398	* struct line - contains the state of a requested line
399	* @desc: the GPIO descriptor for this line.
400	* @req: the corresponding line request
401	* @irq: the interrupt triggered in response to events on this GPIO
402	* @edflags: the edge flags, GPIO_V2_LINE_FLAG_EDGE_RISING and/or
403	* GPIO_V2_LINE_FLAG_EDGE_FALLING, indicating the edge detection applied
404	* @timestamp_ns: cache for the timestamp storing it between hardirq and
405	* IRQ thread, used to bring the timestamp close to the actual event
406	* @req_seqno: the seqno for the current edge event in the sequence of
407	* events for the corresponding line request. This is drawn from the @req.
408	* @line_seqno: the seqno for the current edge event in the sequence of
409	* events for this line.
410	* @work: the worker that implements software debouncing
411	* @sw_debounced: flag indicating if the software debouncer is active
412	* @level: the current debounced physical level of the line
413	* @hdesc: the Hardware Timestamp Engine (HTE) descriptor
414	* @raw_level: the line level at the time of event
415	* @total_discard_seq: the running counter of the discarded events
416	* @last_seqno: the last sequence number before debounce period expires
417	*/
418	struct line {
419	struct gpio_desc *desc;
420	/*
421	* -- edge detector specific fields --
422	*/
423	struct linereq *req;
424	unsigned int irq;
425	/*
426	* The flags for the active edge detector configuration.
427	*
428	* edflags is set by linereq_create(), linereq_free(), and
429	* linereq_set_config(), which are themselves mutually
430	* exclusive, and is accessed by edge_irq_thread(),
431	* process_hw_ts_thread() and debounce_work_func(),
432	* which can all live with a slightly stale value.
433	*/
434	u64 edflags;
435	/*
436	* timestamp_ns and req_seqno are accessed only by
437	* edge_irq_handler() and edge_irq_thread(), which are themselves
438	* mutually exclusive, so no additional protection is necessary.
439	*/
440	u64 timestamp_ns;
441	u32 req_seqno;
442	/*
443	* line_seqno is accessed by either edge_irq_thread() or
444	* debounce_work_func(), which are themselves mutually exclusive,
445	* so no additional protection is necessary.
446	*/
447	u32 line_seqno;
448	/*
449	* -- debouncer specific fields --
450	*/
451	struct delayed_work work;
452	/*
453	* sw_debounce is accessed by linereq_set_config(), which is the
454	* only setter, and linereq_get_values(), which can live with a
455	* slightly stale value.
456	*/
457	unsigned int sw_debounced;
458	/*
459	* level is accessed by debounce_work_func(), which is the only
460	* setter, and linereq_get_values() which can live with a slightly
461	* stale value.
462	*/
463	unsigned int level;
464	#ifdef CONFIG_HTE
465	struct hte_ts_desc hdesc;
466	/*
467	* HTE provider sets line level at the time of event. The valid
468	* value is 0 or 1 and negative value for an error.
469	*/
470	int raw_level;
471	/*
472	* when sw_debounce is set on HTE enabled line, this is running
473	* counter of the discarded events.
474	*/
475	u32 total_discard_seq;
476	/*
477	* when sw_debounce is set on HTE enabled line, this variable records
478	* last sequence number before debounce period expires.
479	*/
480	u32 last_seqno;
481	#endif /* CONFIG_HTE */
482	};
483
484	/**
485	* struct linereq - contains the state of a userspace line request
486	* @gdev: the GPIO device the line request pertains to
487	* @label: consumer label used to tag GPIO descriptors
488	* @num_lines: the number of lines in the lines array
489	* @wait: wait queue that handles blocking reads of events
490	* @device_unregistered_nb: notifier block for receiving gdev unregister events
491	* @event_buffer_size: the number of elements allocated in @events
492	* @events: KFIFO for the GPIO events
493	* @seqno: the sequence number for edge events generated on all lines in
494	* this line request. Note that this is not used when @num_lines is 1, as
495	* the line_seqno is then the same and is cheaper to calculate.
496	* @config_mutex: mutex for serializing ioctl() calls to ensure consistency
497	* of configuration, particularly multi-step accesses to desc flags.
498	* @lines: the lines held by this line request, with @num_lines elements.
499	*/
500	struct linereq {
501	struct gpio_device *gdev;
502	const char *label;
503	u32 num_lines;
504	wait_queue_head_t wait;
505	struct notifier_block device_unregistered_nb;
506	u32 event_buffer_size;
507	DECLARE_KFIFO_PTR(events, struct gpio_v2_line_event);
508	atomic_t seqno;
509	struct mutex config_mutex;
510	struct line lines[] __counted_by(num_lines);
511	};
512
513	#define GPIO_V2_LINE_BIAS_FLAGS \
514	(GPIO_V2_LINE_FLAG_BIAS_PULL_UP \| \
515	GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN \| \
516	GPIO_V2_LINE_FLAG_BIAS_DISABLED)
517
518	#define GPIO_V2_LINE_DIRECTION_FLAGS \
519	(GPIO_V2_LINE_FLAG_INPUT \| \
520	GPIO_V2_LINE_FLAG_OUTPUT)
521
522	#define GPIO_V2_LINE_DRIVE_FLAGS \
523	(GPIO_V2_LINE_FLAG_OPEN_DRAIN \| \
524	GPIO_V2_LINE_FLAG_OPEN_SOURCE)
525
526	#define GPIO_V2_LINE_EDGE_FLAGS \
527	(GPIO_V2_LINE_FLAG_EDGE_RISING \| \
528	GPIO_V2_LINE_FLAG_EDGE_FALLING)
529
530	#define GPIO_V2_LINE_FLAG_EDGE_BOTH GPIO_V2_LINE_EDGE_FLAGS
531
532	#define GPIO_V2_LINE_VALID_FLAGS \
533	(GPIO_V2_LINE_FLAG_ACTIVE_LOW \| \
534	GPIO_V2_LINE_DIRECTION_FLAGS \| \
535	GPIO_V2_LINE_DRIVE_FLAGS \| \
536	GPIO_V2_LINE_EDGE_FLAGS \| \
537	GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME \| \
538	GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE \| \
539	GPIO_V2_LINE_BIAS_FLAGS)
540
541	/* subset of flags relevant for edge detector configuration */
542	#define GPIO_V2_LINE_EDGE_DETECTOR_FLAGS \
543	(GPIO_V2_LINE_FLAG_ACTIVE_LOW \| \
544	GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE \| \
545	GPIO_V2_LINE_EDGE_FLAGS)
546
547	static int linereq_unregistered_notify(struct notifier_block *nb,
548	unsigned long action, void *data)
549	{
550	struct linereq *lr = container_of(nb, struct linereq,
551	device_unregistered_nb);
552
553	wake_up_poll(&lr->wait, EPOLLIN \| EPOLLERR);
554
555	return NOTIFY_OK;
556	}
557
558	static void linereq_put_event(struct linereq *lr,
559	struct gpio_v2_line_event *le)
560	{
561	bool overflow = false;
562
563	scoped_guard(spinlock, &lr->wait.lock) {
564	if (kfifo_is_full(&lr->events)) {
565	overflow = true;
566	kfifo_skip(&lr->events);
567	}
568	kfifo_in(&lr->events, le, 1);
569	}
570	if (!overflow)
571	wake_up_poll(&lr->wait, EPOLLIN);
572	else
573	pr_debug_ratelimited("event FIFO is full - event dropped\n");
574	}
575
576	static u64 line_event_timestamp(struct line *line)
577	{
578	if (test_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &line->desc->flags))
579	return ktime_get_real_ns();
580	else if (IS_ENABLED(CONFIG_HTE) &&
581	test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &line->desc->flags))
582	return line->timestamp_ns;
583
584	return ktime_get_ns();
585	}
586
587	static u32 line_event_id(int level)
588	{
589	return level ? GPIO_V2_LINE_EVENT_RISING_EDGE :
590	GPIO_V2_LINE_EVENT_FALLING_EDGE;
591	}
592
593	static inline char make_irq_label(const char orig)
594	{
595	char *new;
596
597	if (!orig)
598	return NULL;
599
600	new = kstrdup_and_replace(orig, '/', ':', GFP_KERNEL);
601	if (!new)
602	return ERR_PTR(-ENOMEM);
603
604	return new;
605	}
606
607	static inline void free_irq_label(const char *label)
608	{
609	kfree(label);
610	}
611
612	#ifdef CONFIG_HTE
613
614	static enum hte_return process_hw_ts_thread(void *p)
615	{
616	struct line *line;
617	struct linereq *lr;
618	struct gpio_v2_line_event le;
619	u64 edflags;
620	int level;
621
622	if (!p)
623	return HTE_CB_HANDLED;
624
625	line = p;
626	lr = line->req;
627
628	memset(&le, 0, sizeof(le));
629
630	le.timestamp_ns = line->timestamp_ns;
631	edflags = READ_ONCE(line->edflags);
632
633	switch (edflags & GPIO_V2_LINE_EDGE_FLAGS) {
634	case GPIO_V2_LINE_FLAG_EDGE_BOTH:
635	level = (line->raw_level >= 0) ?
636	line->raw_level :
637	gpiod_get_raw_value_cansleep(line->desc);
638
639	if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
640	level = !level;
641
642	le.id = line_event_id(level);
643	break;
644	case GPIO_V2_LINE_FLAG_EDGE_RISING:
645	le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
646	break;
647	case GPIO_V2_LINE_FLAG_EDGE_FALLING:
648	le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
649	break;
650	default:
651	return HTE_CB_HANDLED;
652	}
653	le.line_seqno = line->line_seqno;
654	le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno;
655	le.offset = gpiod_hwgpio(line->desc);
656
657	linereq_put_event(lr, &le);
658
659	return HTE_CB_HANDLED;
660	}
661
662	static enum hte_return process_hw_ts(struct hte_ts_data ts, void p)
663	{
664	struct line *line;
665	struct linereq *lr;
666	int diff_seqno = 0;
667
668	if (!ts \|\| !p)
669	return HTE_CB_HANDLED;
670
671	line = p;
672	line->timestamp_ns = ts->tsc;
673	line->raw_level = ts->raw_level;
674	lr = line->req;
675
676	if (READ_ONCE(line->sw_debounced)) {
677	line->total_discard_seq++;
678	line->last_seqno = ts->seq;
679	mod_delayed_work(system_percpu_wq, &line->work,
680	usecs_to_jiffies(READ_ONCE(line->desc->debounce_period_us)));
681	} else {
682	if (unlikely(ts->seq < line->line_seqno))
683	return HTE_CB_HANDLED;
684
685	diff_seqno = ts->seq - line->line_seqno;
686	line->line_seqno = ts->seq;
687	if (lr->num_lines != 1)
688	line->req_seqno = atomic_add_return(diff_seqno,
689	&lr->seqno);
690
691	return HTE_RUN_SECOND_CB;
692	}
693
694	return HTE_CB_HANDLED;
695	}
696
697	static int hte_edge_setup(struct line *line, u64 eflags)
698	{
699	int ret;
700	unsigned long flags = 0;
701	struct hte_ts_desc *hdesc = &line->hdesc;
702
703	if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING)
704	flags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ?
705	HTE_FALLING_EDGE_TS :
706	HTE_RISING_EDGE_TS;
707	if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
708	flags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ?
709	HTE_RISING_EDGE_TS :
710	HTE_FALLING_EDGE_TS;
711
712	line->total_discard_seq = 0;
713
714	hte_init_line_attr(hdesc, desc_to_gpio(line->desc), flags, NULL,
715	line->desc);
716
717	ret = hte_ts_get(NULL, hdesc, 0);
718	if (ret)
719	return ret;
720
721	return hte_request_ts_ns(hdesc, process_hw_ts, process_hw_ts_thread,
722	line);
723	}
724
725	#else
726
727	static int hte_edge_setup(struct line *line, u64 eflags)
728	{
729	return 0;
730	}
731	#endif /* CONFIG_HTE */
732
733	static irqreturn_t edge_irq_thread(int irq, void *p)
734	{
735	struct line *line = p;
736	struct linereq *lr = line->req;
737	struct gpio_v2_line_event le;
738
739	/* Do not leak kernel stack to userspace */
740	memset(&le, 0, sizeof(le));
741
742	if (line->timestamp_ns) {
743	le.timestamp_ns = line->timestamp_ns;
744	} else {
745	/*
746	* We may be running from a nested threaded interrupt in
747	* which case we didn't get the timestamp from
748	* edge_irq_handler().
749	*/
750	le.timestamp_ns = line_event_timestamp(line);
751	if (lr->num_lines != 1)
752	line->req_seqno = atomic_inc_return(&lr->seqno);
753	}
754	line->timestamp_ns = 0;
755
756	switch (READ_ONCE(line->edflags) & GPIO_V2_LINE_EDGE_FLAGS) {
757	case GPIO_V2_LINE_FLAG_EDGE_BOTH:
758	le.id = line_event_id(gpiod_get_value_cansleep(line->desc));
759	break;
760	case GPIO_V2_LINE_FLAG_EDGE_RISING:
761	le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
762	break;
763	case GPIO_V2_LINE_FLAG_EDGE_FALLING:
764	le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
765	break;
766	default:
767	return IRQ_NONE;
768	}
769	line->line_seqno++;
770	le.line_seqno = line->line_seqno;
771	le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno;
772	le.offset = gpiod_hwgpio(line->desc);
773
774	linereq_put_event(lr, &le);
775
776	return IRQ_HANDLED;
777	}
778
779	static irqreturn_t edge_irq_handler(int irq, void *p)
780	{
781	struct line *line = p;
782	struct linereq *lr = line->req;
783
784	/*
785	* Just store the timestamp in hardirq context so we get it as
786	* close in time as possible to the actual event.
787	*/
788	line->timestamp_ns = line_event_timestamp(line);
789
790	if (lr->num_lines != 1)
791	line->req_seqno = atomic_inc_return(&lr->seqno);
792
793	return IRQ_WAKE_THREAD;
794	}
795
796	/*
797	* returns the current debounced logical value.
798	*/
799	static bool debounced_value(struct line *line)
800	{
801	bool value;
802
803	/*
804	* minor race - debouncer may be stopped here, so edge_detector_stop()
805	* must leave the value unchanged so the following will read the level
806	* from when the debouncer was last running.
807	*/
808	value = READ_ONCE(line->level);
809
810	if (test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags))
811	value = !value;
812
813	return value;
814	}
815
816	static irqreturn_t debounce_irq_handler(int irq, void *p)
817	{
818	struct line *line = p;
819
820	mod_delayed_work(system_percpu_wq, &line->work,
821	usecs_to_jiffies(READ_ONCE(line->desc->debounce_period_us)));
822
823	return IRQ_HANDLED;
824	}
825
826	static void debounce_work_func(struct work_struct *work)
827	{
828	struct gpio_v2_line_event le;
829	struct line *line = container_of(work, struct line, work.work);
830	struct linereq *lr;
831	u64 eflags, edflags = READ_ONCE(line->edflags);
832	int level = -1;
833	#ifdef CONFIG_HTE
834	int diff_seqno;
835
836	if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)
837	level = line->raw_level;
838	#endif
839	if (level < 0)
840	level = gpiod_get_raw_value_cansleep(line->desc);
841	if (level < 0) {
842	pr_debug_ratelimited("debouncer failed to read line value\n");
843	return;
844	}
845
846	if (READ_ONCE(line->level) == level)
847	return;
848
849	WRITE_ONCE(line->level, level);
850
851	/* -- edge detection -- */
852	eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS;
853	if (!eflags)
854	return;
855
856	/* switch from physical level to logical - if they differ */
857	if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
858	level = !level;
859
860	/* ignore edges that are not being monitored */
861	if (((eflags == GPIO_V2_LINE_FLAG_EDGE_RISING) && !level) \|\|
862	((eflags == GPIO_V2_LINE_FLAG_EDGE_FALLING) && level))
863	return;
864
865	/* Do not leak kernel stack to userspace */
866	memset(&le, 0, sizeof(le));
867
868	lr = line->req;
869	le.timestamp_ns = line_event_timestamp(line);
870	le.offset = gpiod_hwgpio(line->desc);
871	#ifdef CONFIG_HTE
872	if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) {
873	/* discard events except the last one */
874	line->total_discard_seq -= 1;
875	diff_seqno = line->last_seqno - line->total_discard_seq -
876	line->line_seqno;
877	line->line_seqno = line->last_seqno - line->total_discard_seq;
878	le.line_seqno = line->line_seqno;
879	le.seqno = (lr->num_lines == 1) ?
880	le.line_seqno : atomic_add_return(diff_seqno, &lr->seqno);
881	} else
882	#endif /* CONFIG_HTE */
883	{
884	line->line_seqno++;
885	le.line_seqno = line->line_seqno;
886	le.seqno = (lr->num_lines == 1) ?
887	le.line_seqno : atomic_inc_return(&lr->seqno);
888	}
889
890	le.id = line_event_id(level);
891
892	linereq_put_event(lr, &le);
893	}
894
895	static int debounce_setup(struct line *line, unsigned int debounce_period_us)
896	{
897	unsigned long irqflags;
898	int ret, level, irq;
899	char *label;
900
901	/*
902	* Try hardware. Skip gpiod_set_config() to avoid emitting two
903	* CHANGED_CONFIG line state events.
904	*/
905	ret = gpio_do_set_config(line->desc,
906	pinconf_to_config_packed(PIN_CONFIG_INPUT_DEBOUNCE,
907	debounce_period_us));
908	if (ret != -ENOTSUPP)
909	return ret;
910
911	if (debounce_period_us) {
912	/* setup software debounce */
913	level = gpiod_get_raw_value_cansleep(line->desc);
914	if (level < 0)
915	return level;
916
917	if (!(IS_ENABLED(CONFIG_HTE) &&
918	test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &line->desc->flags))) {
919	irq = gpiod_to_irq(line->desc);
920	if (irq < 0)
921	return -ENXIO;
922
923	label = make_irq_label(line->req->label);
924	if (IS_ERR(label))
925	return -ENOMEM;
926
927	irqflags = IRQF_TRIGGER_FALLING \| IRQF_TRIGGER_RISING;
928	ret = request_irq(irq, debounce_irq_handler, irqflags,
929	label, line);
930	if (ret) {
931	free_irq_label(label);
932	return ret;
933	}
934	line->irq = irq;
935	} else {
936	ret = hte_edge_setup(line, GPIO_V2_LINE_FLAG_EDGE_BOTH);
937	if (ret)
938	return ret;
939	}
940
941	WRITE_ONCE(line->level, level);
942	WRITE_ONCE(line->sw_debounced, 1);
943	}
944	return 0;
945	}
946
947	static bool gpio_v2_line_config_debounced(struct gpio_v2_line_config *lc,
948	unsigned int line_idx)
949	{
950	unsigned int i;
951	u64 mask = BIT_ULL(line_idx);
952
953	for (i = 0; i < lc->num_attrs; i++) {
954	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
955	(lc->attrs[i].mask & mask))
956	return true;
957	}
958	return false;
959	}
960
961	static u32 gpio_v2_line_config_debounce_period(struct gpio_v2_line_config *lc,
962	unsigned int line_idx)
963	{
964	unsigned int i;
965	u64 mask = BIT_ULL(line_idx);
966
967	for (i = 0; i < lc->num_attrs; i++) {
968	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
969	(lc->attrs[i].mask & mask))
970	return lc->attrs[i].attr.debounce_period_us;
971	}
972	return 0;
973	}
974
975	static void edge_detector_stop(struct line *line)
976	{
977	if (line->irq) {
978	free_irq_label(free_irq(line->irq, line));
979	line->irq = 0;
980	}
981
982	#ifdef CONFIG_HTE
983	if (READ_ONCE(line->edflags) & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)
984	hte_ts_put(&line->hdesc);
985	#endif
986
987	cancel_delayed_work_sync(&line->work);
988	WRITE_ONCE(line->sw_debounced, 0);
989	WRITE_ONCE(line->edflags, 0);
990	if (line->desc)
991	WRITE_ONCE(line->desc->debounce_period_us, 0);
992	/* do not change line->level - see comment in debounced_value() */
993	}
994
995	static int edge_detector_fifo_init(struct linereq *req)
996	{
997	if (kfifo_initialized(&req->events))
998	return 0;
999
1000	return kfifo_alloc(&req->events, req->event_buffer_size, GFP_KERNEL);
1001	}
1002
1003	static int edge_detector_setup(struct line *line,
1004	struct gpio_v2_line_config *lc,
1005	unsigned int line_idx, u64 edflags)
1006	{
1007	u32 debounce_period_us;
1008	unsigned long irqflags = 0;
1009	u64 eflags;
1010	int irq, ret;
1011	char *label;
1012
1013	eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS;
1014	if (eflags) {
1015	ret = edge_detector_fifo_init(line->req);
1016	if (ret)
1017	return ret;
1018	}
1019	if (gpio_v2_line_config_debounced(lc, line_idx)) {
1020	debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx);
1021	ret = debounce_setup(line, debounce_period_us);
1022	if (ret)
1023	return ret;
1024	WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us);
1025	}
1026
1027	/* detection disabled or sw debouncer will provide edge detection */
1028	if (!eflags \|\| READ_ONCE(line->sw_debounced))
1029	return 0;
1030
1031	if (IS_ENABLED(CONFIG_HTE) &&
1032	(edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1033	return hte_edge_setup(line, edflags);
1034
1035	irq = gpiod_to_irq(line->desc);
1036	if (irq < 0)
1037	return -ENXIO;
1038
1039	if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING)
1040	irqflags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ?
1041	IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
1042	if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
1043	irqflags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &line->desc->flags) ?
1044	IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
1045	irqflags \|= IRQF_ONESHOT;
1046
1047	label = make_irq_label(line->req->label);
1048	if (IS_ERR(label))
1049	return PTR_ERR(label);
1050
1051	/* Request a thread to read the events */
1052	ret = request_threaded_irq(irq, edge_irq_handler, edge_irq_thread,
1053	irqflags, label, line);
1054	if (ret) {
1055	free_irq_label(label);
1056	return ret;
1057	}
1058
1059	line->irq = irq;
1060	return 0;
1061	}
1062
1063	static int edge_detector_update(struct line *line,
1064	struct gpio_v2_line_config *lc,
1065	unsigned int line_idx, u64 edflags)
1066	{
1067	u64 active_edflags = READ_ONCE(line->edflags);
1068	unsigned int debounce_period_us =
1069	gpio_v2_line_config_debounce_period(lc, line_idx);
1070
1071	if ((active_edflags == edflags) &&
1072	(READ_ONCE(line->desc->debounce_period_us) == debounce_period_us))
1073	return 0;
1074
1075	/* sw debounced and still will be...*/
1076	if (debounce_period_us && READ_ONCE(line->sw_debounced)) {
1077	WRITE_ONCE(line->desc->debounce_period_us, debounce_period_us);
1078	/*
1079	* ensure event fifo is initialised if edge detection
1080	* is now enabled.
1081	*/
1082	if (edflags & GPIO_V2_LINE_EDGE_FLAGS)
1083	return edge_detector_fifo_init(line->req);
1084
1085	return 0;
1086	}
1087
1088	/* reconfiguring edge detection or sw debounce being disabled */
1089	if ((line->irq && !READ_ONCE(line->sw_debounced)) \|\|
1090	(active_edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) \|\|
1091	(!debounce_period_us && READ_ONCE(line->sw_debounced)))
1092	edge_detector_stop(line);
1093
1094	return edge_detector_setup(line, lc, line_idx, edflags);
1095	}
1096
1097	static u64 gpio_v2_line_config_flags(struct gpio_v2_line_config *lc,
1098	unsigned int line_idx)
1099	{
1100	unsigned int i;
1101	u64 mask = BIT_ULL(line_idx);
1102
1103	for (i = 0; i < lc->num_attrs; i++) {
1104	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_FLAGS) &&
1105	(lc->attrs[i].mask & mask))
1106	return lc->attrs[i].attr.flags;
1107	}
1108	return lc->flags;
1109	}
1110
1111	static int gpio_v2_line_config_output_value(struct gpio_v2_line_config *lc,
1112	unsigned int line_idx)
1113	{
1114	unsigned int i;
1115	u64 mask = BIT_ULL(line_idx);
1116
1117	for (i = 0; i < lc->num_attrs; i++) {
1118	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES) &&
1119	(lc->attrs[i].mask & mask))
1120	return !!(lc->attrs[i].attr.values & mask);
1121	}
1122	return 0;
1123	}
1124
1125	static int gpio_v2_line_flags_validate(u64 flags)
1126	{
1127	/* Return an error if an unknown flag is set */
1128	if (flags & ~GPIO_V2_LINE_VALID_FLAGS)
1129	return -EINVAL;
1130
1131	if (!IS_ENABLED(CONFIG_HTE) &&
1132	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1133	return -EOPNOTSUPP;
1134
1135	/*
1136	* Do not allow both INPUT and OUTPUT flags to be set as they are
1137	* contradictory.
1138	*/
1139	if ((flags & GPIO_V2_LINE_FLAG_INPUT) &&
1140	(flags & GPIO_V2_LINE_FLAG_OUTPUT))
1141	return -EINVAL;
1142
1143	/* Only allow one event clock source */
1144	if (IS_ENABLED(CONFIG_HTE) &&
1145	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME) &&
1146	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1147	return -EINVAL;
1148
1149	/* Edge detection requires explicit input. */
1150	if ((flags & GPIO_V2_LINE_EDGE_FLAGS) &&
1151	!(flags & GPIO_V2_LINE_FLAG_INPUT))
1152	return -EINVAL;
1153
1154	/*
1155	* Do not allow OPEN_SOURCE and OPEN_DRAIN flags in a single
1156	* request. If the hardware actually supports enabling both at the
1157	* same time the electrical result would be disastrous.
1158	*/
1159	if ((flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN) &&
1160	(flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE))
1161	return -EINVAL;
1162
1163	/* Drive requires explicit output direction. */
1164	if ((flags & GPIO_V2_LINE_DRIVE_FLAGS) &&
1165	!(flags & GPIO_V2_LINE_FLAG_OUTPUT))
1166	return -EINVAL;
1167
1168	/* Bias requires explicit direction. */
1169	if ((flags & GPIO_V2_LINE_BIAS_FLAGS) &&
1170	!(flags & GPIO_V2_LINE_DIRECTION_FLAGS))
1171	return -EINVAL;
1172
1173	/* Only one bias flag can be set. */
1174	if (((flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED) &&
1175	(flags & (GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN \|
1176	GPIO_V2_LINE_FLAG_BIAS_PULL_UP))) \|\|
1177	((flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN) &&
1178	(flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)))
1179	return -EINVAL;
1180
1181	return 0;
1182	}
1183
1184	static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc,
1185	unsigned int num_lines)
1186	{
1187	size_t unused_attrs;
1188	unsigned int i;
1189	u64 flags;
1190	int ret;
1191
1192	if (lc->num_attrs > GPIO_V2_LINE_NUM_ATTRS_MAX)
1193	return -EINVAL;
1194
1195	unused_attrs = GPIO_V2_LINE_NUM_ATTRS_MAX - lc->num_attrs;
1196
1197	if (!mem_is_zero(lc->padding, sizeof(lc->padding)))
1198	return -EINVAL;
1199
1200	for (i = 0; i < lc->num_attrs; i++) {
1201	if (lc->attrs[i].attr.padding != 0)
1202	return -EINVAL;
1203	}
1204
1205	if (unused_attrs) {
1206	if (!mem_is_zero(&lc->attrs[lc->num_attrs], unused_attrs * sizeof(*lc->attrs)))
1207	return -EINVAL;
1208	}
1209
1210	for (i = 0; i < num_lines; i++) {
1211	flags = gpio_v2_line_config_flags(lc, i);
1212	ret = gpio_v2_line_flags_validate(flags);
1213	if (ret)
1214	return ret;
1215
1216	/* debounce requires explicit input */
1217	if (gpio_v2_line_config_debounced(lc, i) &&
1218	!(flags & GPIO_V2_LINE_FLAG_INPUT))
1219	return -EINVAL;
1220	}
1221	return 0;
1222	}
1223
1224	static void gpio_v2_line_config_flags_to_desc_flags(u64 lflags,
1225	unsigned long *flagsp)
1226	{
1227	unsigned long flags = READ_ONCE(*flagsp);
1228
1229	assign_bit(GPIOD_FLAG_ACTIVE_LOW, &flags,
1230	lflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW);
1231
1232	if (lflags & GPIO_V2_LINE_FLAG_OUTPUT)
1233	set_bit(GPIOD_FLAG_IS_OUT, &flags);
1234	else if (lflags & GPIO_V2_LINE_FLAG_INPUT)
1235	clear_bit(GPIOD_FLAG_IS_OUT, &flags);
1236
1237	assign_bit(GPIOD_FLAG_EDGE_RISING, &flags,
1238	lflags & GPIO_V2_LINE_FLAG_EDGE_RISING);
1239	assign_bit(GPIOD_FLAG_EDGE_FALLING, &flags,
1240	lflags & GPIO_V2_LINE_FLAG_EDGE_FALLING);
1241
1242	assign_bit(GPIOD_FLAG_OPEN_DRAIN, &flags,
1243	lflags & GPIO_V2_LINE_FLAG_OPEN_DRAIN);
1244	assign_bit(GPIOD_FLAG_OPEN_SOURCE, &flags,
1245	lflags & GPIO_V2_LINE_FLAG_OPEN_SOURCE);
1246
1247	assign_bit(GPIOD_FLAG_PULL_UP, &flags,
1248	lflags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP);
1249	assign_bit(GPIOD_FLAG_PULL_DOWN, &flags,
1250	lflags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN);
1251	assign_bit(GPIOD_FLAG_BIAS_DISABLE, &flags,
1252	lflags & GPIO_V2_LINE_FLAG_BIAS_DISABLED);
1253
1254	assign_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &flags,
1255	lflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME);
1256	assign_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &flags,
1257	lflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE);
1258
1259	WRITE_ONCE(*flagsp, flags);
1260	}
1261
1262	static long linereq_get_values(struct linereq lr, void __user ip)
1263	{
1264	struct gpio_v2_line_values lv;
1265	DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
1266	struct gpio_desc **descs;
1267	unsigned int i, didx, num_get;
1268	bool val;
1269	int ret;
1270
1271	/* NOTE: It's ok to read values of output lines. */
1272	if (copy_from_user(&lv, ip, sizeof(lv)))
1273	return -EFAULT;
1274
1275	/*
1276	* gpiod_get_array_value_complex() requires compacted desc and val
1277	* arrays, rather than the sparse ones in lv.
1278	* Calculation of num_get and construction of the desc array is
1279	* optimized to avoid allocation for the desc array for the common
1280	* num_get == 1 case.
1281	*/
1282	/* scan requested lines to calculate the subset to get */
1283	for (num_get = 0, i = 0; i < lr->num_lines; i++) {
1284	if (lv.mask & BIT_ULL(i)) {
1285	num_get++;
1286	/* capture desc for the num_get == 1 case */
1287	descs = &lr->lines[i].desc;
1288	}
1289	}
1290
1291	if (num_get == 0)
1292	return -EINVAL;
1293
1294	if (num_get != 1) {
1295	/* build compacted desc array */
1296	descs = kmalloc_objs(*descs, num_get);
1297	if (!descs)
1298	return -ENOMEM;
1299	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1300	if (lv.mask & BIT_ULL(i)) {
1301	descs[didx] = lr->lines[i].desc;
1302	didx++;
1303	}
1304	}
1305	}
1306	ret = gpiod_get_array_value_complex(false, true, num_get,
1307	descs, NULL, vals);
1308
1309	if (num_get != 1)
1310	kfree(descs);
1311	if (ret)
1312	return ret;
1313
1314	lv.bits = 0;
1315	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1316	/* unpack compacted vals for the response */
1317	if (lv.mask & BIT_ULL(i)) {
1318	if (lr->lines[i].sw_debounced)
1319	val = debounced_value(&lr->lines[i]);
1320	else
1321	val = test_bit(didx, vals);
1322	if (val)
1323	lv.bits \|= BIT_ULL(i);
1324	didx++;
1325	}
1326	}
1327
1328	if (copy_to_user(ip, &lv, sizeof(lv)))
1329	return -EFAULT;
1330
1331	return 0;
1332	}
1333
1334	static long linereq_set_values(struct linereq lr, void __user ip)
1335	{
1336	DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
1337	struct gpio_v2_line_values lv;
1338	struct gpio_desc **descs;
1339	unsigned int i, didx, num_set;
1340	int ret;
1341
1342	if (copy_from_user(&lv, ip, sizeof(lv)))
1343	return -EFAULT;
1344
1345	guard(mutex)(&lr->config_mutex);
1346
1347	/*
1348	* gpiod_set_array_value_complex() requires compacted desc and val
1349	* arrays, rather than the sparse ones in lv.
1350	* Calculation of num_set and construction of the descs and vals arrays
1351	* is optimized to minimize scanning the lv->mask, and to avoid
1352	* allocation for the desc array for the common num_set == 1 case.
1353	*/
1354	bitmap_zero(vals, GPIO_V2_LINES_MAX);
1355	/* scan requested lines to determine the subset to be set */
1356	for (num_set = 0, i = 0; i < lr->num_lines; i++) {
1357	if (lv.mask & BIT_ULL(i)) {
1358	/* add to compacted values */
1359	if (lv.bits & BIT_ULL(i))
1360	__set_bit(num_set, vals);
1361	num_set++;
1362	/* capture desc for the num_set == 1 case */
1363	descs = &lr->lines[i].desc;
1364	}
1365	}
1366	if (num_set == 0)
1367	return -EINVAL;
1368
1369	if (num_set != 1) {
1370	/* build compacted desc array */
1371	descs = kmalloc_objs(*descs, num_set);
1372	if (!descs)
1373	return -ENOMEM;
1374	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1375	if (lv.mask & BIT_ULL(i)) {
1376	descs[didx] = lr->lines[i].desc;
1377	didx++;
1378	}
1379	}
1380	}
1381	ret = gpiod_set_array_value_complex(false, true, num_set,
1382	descs, NULL, vals);
1383
1384	if (num_set != 1)
1385	kfree(descs);
1386	return ret;
1387	}
1388
1389	static long linereq_set_config(struct linereq lr, void __user ip)
1390	{
1391	struct gpio_v2_line_config lc;
1392	struct gpio_desc *desc;
1393	struct line *line;
1394	unsigned int i;
1395	u64 flags, edflags;
1396	int ret;
1397
1398	if (copy_from_user(&lc, ip, sizeof(lc)))
1399	return -EFAULT;
1400
1401	ret = gpio_v2_line_config_validate(&lc, lr->num_lines);
1402	if (ret)
1403	return ret;
1404
1405	guard(mutex)(&lr->config_mutex);
1406
1407	for (i = 0; i < lr->num_lines; i++) {
1408	line = &lr->lines[i];
1409	desc = lr->lines[i].desc;
1410	flags = gpio_v2_line_config_flags(&lc, i);
1411	/*
1412	* Lines not explicitly reconfigured as input or output
1413	* are left unchanged.
1414	*/
1415	if (!(flags & GPIO_V2_LINE_DIRECTION_FLAGS))
1416	continue;
1417	gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags);
1418	edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS;
1419	if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
1420	int val = gpio_v2_line_config_output_value(&lc, i);
1421
1422	edge_detector_stop(line);
1423	ret = gpiod_direction_output_nonotify(desc, val);
1424	if (ret)
1425	return ret;
1426	} else {
1427	ret = gpiod_direction_input_nonotify(desc);
1428	if (ret)
1429	return ret;
1430
1431	ret = edge_detector_update(line, &lc, i, edflags);
1432	if (ret)
1433	return ret;
1434	}
1435
1436	WRITE_ONCE(line->edflags, edflags);
1437
1438	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_CONFIG);
1439	}
1440	return 0;
1441	}
1442
1443	static long linereq_ioctl(struct file *file, unsigned int cmd,
1444	unsigned long arg)
1445	{
1446	struct linereq *lr = file->private_data;
1447	void __user ip = (void __user )arg;
1448
1449	guard(srcu)(&lr->gdev->srcu);
1450
1451	if (!rcu_access_pointer(lr->gdev->chip))
1452	return -ENODEV;
1453
1454	switch (cmd) {
1455	case GPIO_V2_LINE_GET_VALUES_IOCTL:
1456	return linereq_get_values(lr, ip);
1457	case GPIO_V2_LINE_SET_VALUES_IOCTL:
1458	return linereq_set_values(lr, ip);
1459	case GPIO_V2_LINE_SET_CONFIG_IOCTL:
1460	return linereq_set_config(lr, ip);
1461	default:
1462	return -EINVAL;
1463	}
1464	}
1465
1466	#ifdef CONFIG_COMPAT
1467	static long linereq_ioctl_compat(struct file *file, unsigned int cmd,
1468	unsigned long arg)
1469	{
1470	return linereq_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
1471	}
1472	#endif
1473
1474	static __poll_t linereq_poll(struct file *file,
1475	struct poll_table_struct *wait)
1476	{
1477	struct linereq *lr = file->private_data;
1478	__poll_t events = 0;
1479
1480	guard(srcu)(&lr->gdev->srcu);
1481
1482	if (!rcu_access_pointer(lr->gdev->chip))
1483	return EPOLLHUP \| EPOLLERR;
1484
1485	poll_wait(file, &lr->wait, wait);
1486
1487	if (!kfifo_is_empty_spinlocked_noirqsave(&lr->events,
1488	&lr->wait.lock))
1489	events = EPOLLIN \| EPOLLRDNORM;
1490
1491	return events;
1492	}
1493
1494	static ssize_t linereq_read(struct file file, char __user buf,
1495	size_t count, loff_t *f_ps)
1496	{
1497	struct linereq *lr = file->private_data;
1498	struct gpio_v2_line_event le;
1499	ssize_t bytes_read = 0;
1500	int ret;
1501
1502	guard(srcu)(&lr->gdev->srcu);
1503
1504	if (!rcu_access_pointer(lr->gdev->chip))
1505	return -ENODEV;
1506
1507	if (count < sizeof(le))
1508	return -EINVAL;
1509
1510	do {
1511	scoped_guard(spinlock, &lr->wait.lock) {
1512	if (kfifo_is_empty(&lr->events)) {
1513	if (bytes_read)
1514	return bytes_read;
1515
1516	if (file->f_flags & O_NONBLOCK)
1517	return -EAGAIN;
1518
1519	ret = wait_event_interruptible_locked(lr->wait,
1520	!kfifo_is_empty(&lr->events));
1521	if (ret)
1522	return ret;
1523	}
1524
1525	if (kfifo_out(&lr->events, &le, 1) != 1) {
1526	/*
1527	* This should never happen - we hold the
1528	* lock from the moment we learned the fifo
1529	* is no longer empty until now.
1530	*/
1531	WARN(1, "failed to read from non-empty kfifo");
1532	return -EIO;
1533	}
1534	}
1535
1536	if (copy_to_user(buf + bytes_read, &le, sizeof(le)))
1537	return -EFAULT;
1538	bytes_read += sizeof(le);
1539	} while (count >= bytes_read + sizeof(le));
1540
1541	return bytes_read;
1542	}
1543
1544	static void linereq_free(struct linereq *lr)
1545	{
1546	unsigned int i;
1547
1548	if (lr->device_unregistered_nb.notifier_call)
1549	blocking_notifier_chain_unregister(&lr->gdev->device_notifier,
1550	&lr->device_unregistered_nb);
1551
1552	for (i = 0; i < lr->num_lines; i++) {
1553	if (lr->lines[i].desc) {
1554	edge_detector_stop(&lr->lines[i]);
1555	gpiod_free(lr->lines[i].desc);
1556	}
1557	}
1558	kfifo_free(&lr->events);
1559	kfree(lr->label);
1560	gpio_device_put(lr->gdev);
1561	kvfree(lr);
1562	}
1563
1564	static int linereq_release(struct inode inode, struct file file)
1565	{
1566	struct linereq *lr = file->private_data;
1567
1568	linereq_free(lr);
1569	return 0;
1570	}
1571
1572	#ifdef CONFIG_PROC_FS
1573	static void linereq_show_fdinfo(struct seq_file out, struct file file)
1574	{
1575	struct linereq *lr = file->private_data;
1576	struct device *dev = &lr->gdev->dev;
1577	u16 i;
1578
1579	seq_printf(out, "gpio-chip:\t%s\n", dev_name(dev));
1580
1581	for (i = 0; i < lr->num_lines; i++)
1582	seq_printf(out, "gpio-line:\t%d\n",
1583	gpiod_hwgpio(lr->lines[i].desc));
1584	}
1585	#endif
1586
1587	static const struct file_operations line_fileops = {
1588	.release = linereq_release,
1589	.read = linereq_read,
1590	.poll = linereq_poll,
1591	.owner = THIS_MODULE,
1592	.llseek = noop_llseek,
1593	.unlocked_ioctl = linereq_ioctl,
1594	#ifdef CONFIG_COMPAT
1595	.compat_ioctl = linereq_ioctl_compat,
1596	#endif
1597	#ifdef CONFIG_PROC_FS
1598	.show_fdinfo = linereq_show_fdinfo,
1599	#endif
1600	};
1601
1602	DEFINE_FREE(linereq_free, struct linereq *, if (!IS_ERR_OR_NULL(_T)) linereq_free(_T))
1603
1604	static int linereq_create(struct gpio_device gdev, void __user ip)
1605	{
1606	struct gpio_v2_line_request ulr;
1607	struct gpio_v2_line_config *lc;
1608	struct linereq *lr __free(linereq_free) = NULL;
1609	u64 flags, edflags;
1610	unsigned int i;
1611	int ret;
1612
1613	if (copy_from_user(&ulr, ip, sizeof(ulr)))
1614	return -EFAULT;
1615
1616	if ((ulr.num_lines == 0) \|\| (ulr.num_lines > GPIO_V2_LINES_MAX))
1617	return -EINVAL;
1618
1619	if (!mem_is_zero(ulr.padding, sizeof(ulr.padding)))
1620	return -EINVAL;
1621
1622	lc = &ulr.config;
1623	ret = gpio_v2_line_config_validate(lc, ulr.num_lines);
1624	if (ret)
1625	return ret;
1626
1627	lr = kvzalloc_flex(*lr, lines, ulr.num_lines);
1628	if (!lr)
1629	return -ENOMEM;
1630	lr->num_lines = ulr.num_lines;
1631
1632	lr->gdev = gpio_device_get(gdev);
1633
1634	for (i = 0; i < ulr.num_lines; i++) {
1635	lr->lines[i].req = lr;
1636	WRITE_ONCE(lr->lines[i].sw_debounced, 0);
1637	INIT_DELAYED_WORK(&lr->lines[i].work, debounce_work_func);
1638	}
1639
1640	if (ulr.consumer[0] != '\0') {
1641	/* label is only initialized if consumer is set */
1642	lr->label = kstrndup(ulr.consumer, sizeof(ulr.consumer) - 1,
1643	GFP_KERNEL);
1644	if (!lr->label)
1645	return -ENOMEM;
1646	}
1647
1648	mutex_init(&lr->config_mutex);
1649	init_waitqueue_head(&lr->wait);
1650	INIT_KFIFO(lr->events);
1651	lr->event_buffer_size = ulr.event_buffer_size;
1652	if (lr->event_buffer_size == 0)
1653	lr->event_buffer_size = ulr.num_lines * 16;
1654	else if (lr->event_buffer_size > GPIO_V2_LINES_MAX * 16)
1655	lr->event_buffer_size = GPIO_V2_LINES_MAX * 16;
1656
1657	atomic_set(&lr->seqno, 0);
1658
1659	/* Request each GPIO */
1660	for (i = 0; i < ulr.num_lines; i++) {
1661	u32 offset = ulr.offsets[i];
1662	struct gpio_desc *desc = gpio_device_get_desc(gdev, offset);
1663
1664	if (IS_ERR(desc))
1665	return PTR_ERR(desc);
1666
1667	ret = gpiod_request_user(desc, lr->label);
1668	if (ret)
1669	return ret;
1670
1671	lr->lines[i].desc = desc;
1672	flags = gpio_v2_line_config_flags(lc, i);
1673	gpio_v2_line_config_flags_to_desc_flags(flags, &desc->flags);
1674
1675	ret = gpiod_set_transitory(desc, false);
1676	if (ret < 0)
1677	return ret;
1678
1679	edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS;
1680	/*
1681	* Lines have to be requested explicitly for input
1682	* or output, else the line will be treated "as is".
1683	*/
1684	if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
1685	int val = gpio_v2_line_config_output_value(lc, i);
1686
1687	ret = gpiod_direction_output_nonotify(desc, val);
1688	if (ret)
1689	return ret;
1690	} else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
1691	ret = gpiod_direction_input_nonotify(desc);
1692	if (ret)
1693	return ret;
1694
1695	ret = edge_detector_setup(&lr->lines[i], lc, i,
1696	edflags);
1697	if (ret)
1698	return ret;
1699	}
1700
1701	lr->lines[i].edflags = edflags;
1702
1703	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED);
1704
1705	dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
1706	offset);
1707	}
1708
1709	lr->device_unregistered_nb.notifier_call = linereq_unregistered_notify;
1710	ret = blocking_notifier_chain_register(&gdev->device_notifier,
1711	&lr->device_unregistered_nb);
1712	if (ret)
1713	return ret;
1714
1715	FD_PREPARE(fdf, O_RDONLY \| O_CLOEXEC,
1716	anon_inode_getfile("gpio-line", &line_fileops, lr,
1717	O_RDONLY \| O_CLOEXEC));
1718	if (fdf->fd < 0)
1719	return fdf->fd;
1720	retain_and_null_ptr(lr);
1721
1722	ulr.fd = fdf->fd;
1723	if (copy_to_user(ip, &ulr, sizeof(ulr)))
1724	return -EFAULT;
1725
1726	fd_publish(fdf);
1727
1728	dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
1729	ulr.num_lines);
1730
1731	return 0;
1732	}
1733
1734	#ifdef CONFIG_GPIO_CDEV_V1
1735
1736	/*
1737	* GPIO line event management
1738	*/
1739
1740	/**
1741	* struct lineevent_state - contains the state of a userspace event
1742	* @gdev: the GPIO device the event pertains to
1743	* @label: consumer label used to tag descriptors
1744	* @desc: the GPIO descriptor held by this event
1745	* @eflags: the event flags this line was requested with
1746	* @irq: the interrupt that trigger in response to events on this GPIO
1747	* @wait: wait queue that handles blocking reads of events
1748	* @device_unregistered_nb: notifier block for receiving gdev unregister events
1749	* @events: KFIFO for the GPIO events
1750	* @timestamp: cache for the timestamp storing it between hardirq
1751	* and IRQ thread, used to bring the timestamp close to the actual
1752	* event
1753	*/
1754	struct lineevent_state {
1755	struct gpio_device *gdev;
1756	const char *label;
1757	struct gpio_desc *desc;
1758	u32 eflags;
1759	int irq;
1760	wait_queue_head_t wait;
1761	struct notifier_block device_unregistered_nb;
1762	DECLARE_KFIFO(events, struct gpioevent_data, 16);
1763	u64 timestamp;
1764	};
1765
1766	#define GPIOEVENT_REQUEST_VALID_FLAGS \
1767	(GPIOEVENT_REQUEST_RISING_EDGE \| \
1768	GPIOEVENT_REQUEST_FALLING_EDGE)
1769
1770	static __poll_t lineevent_poll(struct file *file,
1771	struct poll_table_struct *wait)
1772	{
1773	struct lineevent_state *le = file->private_data;
1774	__poll_t events = 0;
1775
1776	guard(srcu)(&le->gdev->srcu);
1777
1778	if (!rcu_access_pointer(le->gdev->chip))
1779	return EPOLLHUP \| EPOLLERR;
1780
1781	poll_wait(file, &le->wait, wait);
1782
1783	if (!kfifo_is_empty_spinlocked_noirqsave(&le->events, &le->wait.lock))
1784	events = EPOLLIN \| EPOLLRDNORM;
1785
1786	return events;
1787	}
1788
1789	static int lineevent_unregistered_notify(struct notifier_block *nb,
1790	unsigned long action, void *data)
1791	{
1792	struct lineevent_state *le = container_of(nb, struct lineevent_state,
1793	device_unregistered_nb);
1794
1795	wake_up_poll(&le->wait, EPOLLIN \| EPOLLERR);
1796
1797	return NOTIFY_OK;
1798	}
1799
1800	struct compat_gpioeevent_data {
1801	compat_u64 timestamp;
1802	u32 id;
1803	};
1804
1805	static ssize_t lineevent_read(struct file file, char __user buf,
1806	size_t count, loff_t *f_ps)
1807	{
1808	struct lineevent_state *le = file->private_data;
1809	struct gpioevent_data ge;
1810	ssize_t bytes_read = 0;
1811	ssize_t ge_size;
1812	int ret;
1813
1814	guard(srcu)(&le->gdev->srcu);
1815
1816	if (!rcu_access_pointer(le->gdev->chip))
1817	return -ENODEV;
1818
1819	/*
1820	* When compatible system call is being used the struct gpioevent_data,
1821	* in case of at least ia32, has different size due to the alignment
1822	* differences. Because we have first member 64 bits followed by one of
1823	* 32 bits there is no gap between them. The only difference is the
1824	* padding at the end of the data structure. Hence, we calculate the
1825	* actual sizeof() and pass this as an argument to copy_to_user() to
1826	* drop unneeded bytes from the output.
1827	*/
1828	if (compat_need_64bit_alignment_fixup())
1829	ge_size = sizeof(struct compat_gpioeevent_data);
1830	else
1831	ge_size = sizeof(struct gpioevent_data);
1832	if (count < ge_size)
1833	return -EINVAL;
1834
1835	do {
1836	scoped_guard(spinlock, &le->wait.lock) {
1837	if (kfifo_is_empty(&le->events)) {
1838	if (bytes_read)
1839	return bytes_read;
1840
1841	if (file->f_flags & O_NONBLOCK)
1842	return -EAGAIN;
1843
1844	ret = wait_event_interruptible_locked(le->wait,
1845	!kfifo_is_empty(&le->events));
1846	if (ret)
1847	return ret;
1848	}
1849
1850	if (kfifo_out(&le->events, &ge, 1) != 1) {
1851	/*
1852	* This should never happen - we hold the
1853	* lock from the moment we learned the fifo
1854	* is no longer empty until now.
1855	*/
1856	WARN(1, "failed to read from non-empty kfifo");
1857	return -EIO;
1858	}
1859	}
1860
1861	if (copy_to_user(buf + bytes_read, &ge, ge_size))
1862	return -EFAULT;
1863	bytes_read += ge_size;
1864	} while (count >= bytes_read + ge_size);
1865
1866	return bytes_read;
1867	}
1868
1869	static void lineevent_free(struct lineevent_state *le)
1870	{
1871	if (le->device_unregistered_nb.notifier_call)
1872	blocking_notifier_chain_unregister(&le->gdev->device_notifier,
1873	&le->device_unregistered_nb);
1874	if (le->irq)
1875	free_irq_label(free_irq(le->irq, le));
1876	if (le->desc)
1877	gpiod_free(le->desc);
1878	kfree(le->label);
1879	gpio_device_put(le->gdev);
1880	kfree(le);
1881	}
1882
1883	static int lineevent_release(struct inode inode, struct file file)
1884	{
1885	lineevent_free(file->private_data);
1886	return 0;
1887	}
1888
1889	static long lineevent_ioctl(struct file *file, unsigned int cmd,
1890	unsigned long arg)
1891	{
1892	struct lineevent_state *le = file->private_data;
1893	void __user ip = (void __user )arg;
1894	struct gpiohandle_data ghd;
1895
1896	guard(srcu)(&le->gdev->srcu);
1897
1898	if (!rcu_access_pointer(le->gdev->chip))
1899	return -ENODEV;
1900
1901	/*
1902	* We can get the value for an event line but not set it,
1903	* because it is input by definition.
1904	*/
1905	if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) {
1906	int val;
1907
1908	memset(&ghd, 0, sizeof(ghd));
1909
1910	val = gpiod_get_value_cansleep(le->desc);
1911	if (val < 0)
1912	return val;
1913	ghd.values[0] = val;
1914
1915	if (copy_to_user(ip, &ghd, sizeof(ghd)))
1916	return -EFAULT;
1917
1918	return 0;
1919	}
1920	return -EINVAL;
1921	}
1922
1923	#ifdef CONFIG_COMPAT
1924	static long lineevent_ioctl_compat(struct file *file, unsigned int cmd,
1925	unsigned long arg)
1926	{
1927	return lineevent_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
1928	}
1929	#endif
1930
1931	static const struct file_operations lineevent_fileops = {
1932	.release = lineevent_release,
1933	.read = lineevent_read,
1934	.poll = lineevent_poll,
1935	.owner = THIS_MODULE,
1936	.llseek = noop_llseek,
1937	.unlocked_ioctl = lineevent_ioctl,
1938	#ifdef CONFIG_COMPAT
1939	.compat_ioctl = lineevent_ioctl_compat,
1940	#endif
1941	};
1942
1943	static irqreturn_t lineevent_irq_thread(int irq, void *p)
1944	{
1945	struct lineevent_state *le = p;
1946	struct gpioevent_data ge;
1947	int ret;
1948
1949	/* Do not leak kernel stack to userspace */
1950	memset(&ge, 0, sizeof(ge));
1951
1952	/*
1953	* We may be running from a nested threaded interrupt in which case
1954	* we didn't get the timestamp from lineevent_irq_handler().
1955	*/
1956	if (!le->timestamp)
1957	ge.timestamp = ktime_get_ns();
1958	else
1959	ge.timestamp = le->timestamp;
1960
1961	if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE
1962	&& le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) {
1963	int level = gpiod_get_value_cansleep(le->desc);
1964
1965	if (level)
1966	/* Emit low-to-high event */
1967	ge.id = GPIOEVENT_EVENT_RISING_EDGE;
1968	else
1969	/* Emit high-to-low event */
1970	ge.id = GPIOEVENT_EVENT_FALLING_EDGE;
1971	} else if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE) {
1972	/* Emit low-to-high event */
1973	ge.id = GPIOEVENT_EVENT_RISING_EDGE;
1974	} else if (le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) {
1975	/* Emit high-to-low event */
1976	ge.id = GPIOEVENT_EVENT_FALLING_EDGE;
1977	} else {
1978	return IRQ_NONE;
1979	}
1980
1981	ret = kfifo_in_spinlocked_noirqsave(&le->events, &ge,
1982	1, &le->wait.lock);
1983	if (ret)
1984	wake_up_poll(&le->wait, EPOLLIN);
1985	else
1986	pr_debug_ratelimited("event FIFO is full - event dropped\n");
1987
1988	return IRQ_HANDLED;
1989	}
1990
1991	static irqreturn_t lineevent_irq_handler(int irq, void *p)
1992	{
1993	struct lineevent_state *le = p;
1994
1995	/*
1996	* Just store the timestamp in hardirq context so we get it as
1997	* close in time as possible to the actual event.
1998	*/
1999	le->timestamp = ktime_get_ns();
2000
2001	return IRQ_WAKE_THREAD;
2002	}
2003
2004	DEFINE_FREE(lineevent_free, struct lineevent_state *, if (!IS_ERR_OR_NULL(_T)) lineevent_free(_T))
2005
2006	static int lineevent_create(struct gpio_device gdev, void __user ip)
2007	{
2008	struct gpioevent_request eventreq;
2009	struct lineevent_state *le __free(lineevent_free) = NULL;
2010	struct gpio_desc *desc;
2011	u32 offset;
2012	u32 lflags;
2013	u32 eflags;
2014	int ret;
2015	int irq, irqflags = 0;
2016	char *label;
2017
2018	if (copy_from_user(&eventreq, ip, sizeof(eventreq)))
2019	return -EFAULT;
2020
2021	offset = eventreq.lineoffset;
2022	lflags = eventreq.handleflags;
2023	eflags = eventreq.eventflags;
2024
2025	desc = gpio_device_get_desc(gdev, offset);
2026	if (IS_ERR(desc))
2027	return PTR_ERR(desc);
2028
2029	/* Return an error if a unknown flag is set */
2030	if ((lflags & ~GPIOHANDLE_REQUEST_VALID_FLAGS) \|\|
2031	(eflags & ~GPIOEVENT_REQUEST_VALID_FLAGS))
2032	return -EINVAL;
2033
2034	/* This is just wrong: we don't look for events on output lines */
2035	if ((lflags & GPIOHANDLE_REQUEST_OUTPUT) \|\|
2036	(lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) \|\|
2037	(lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE))
2038	return -EINVAL;
2039
2040	/* Only one bias flag can be set. */
2041	if (((lflags & GPIOHANDLE_REQUEST_BIAS_DISABLE) &&
2042	(lflags & (GPIOHANDLE_REQUEST_BIAS_PULL_DOWN \|
2043	GPIOHANDLE_REQUEST_BIAS_PULL_UP))) \|\|
2044	((lflags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN) &&
2045	(lflags & GPIOHANDLE_REQUEST_BIAS_PULL_UP)))
2046	return -EINVAL;
2047
2048	le = kzalloc_obj(*le);
2049	if (!le)
2050	return -ENOMEM;
2051	le->gdev = gpio_device_get(gdev);
2052
2053	if (eventreq.consumer_label[0] != '\0') {
2054	/* label is only initialized if consumer_label is set */
2055	le->label = kstrndup(eventreq.consumer_label,
2056	sizeof(eventreq.consumer_label) - 1,
2057	GFP_KERNEL);
2058	if (!le->label)
2059	return -ENOMEM;
2060	}
2061
2062	ret = gpiod_request_user(desc, le->label);
2063	if (ret)
2064	return ret;
2065	le->desc = desc;
2066	le->eflags = eflags;
2067
2068	linehandle_flags_to_desc_flags(lflags, &desc->flags);
2069
2070	ret = gpiod_direction_input(desc);
2071	if (ret)
2072	return ret;
2073
2074	gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED);
2075
2076	irq = gpiod_to_irq(desc);
2077	if (irq <= 0)
2078	return -ENODEV;
2079
2080	if (eflags & GPIOEVENT_REQUEST_RISING_EDGE)
2081	irqflags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags) ?
2082	IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
2083	if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE)
2084	irqflags \|= test_bit(GPIOD_FLAG_ACTIVE_LOW, &desc->flags) ?
2085	IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
2086	irqflags \|= IRQF_ONESHOT;
2087
2088	INIT_KFIFO(le->events);
2089	init_waitqueue_head(&le->wait);
2090
2091	le->device_unregistered_nb.notifier_call = lineevent_unregistered_notify;
2092	ret = blocking_notifier_chain_register(&gdev->device_notifier,
2093	&le->device_unregistered_nb);
2094	if (ret)
2095	return ret;
2096
2097	label = make_irq_label(le->label);
2098	if (IS_ERR(label))
2099	return PTR_ERR(label);
2100
2101	/* Request a thread to read the events */
2102	ret = request_threaded_irq(irq,
2103	lineevent_irq_handler,
2104	lineevent_irq_thread,
2105	irqflags,
2106	label,
2107	le);
2108	if (ret) {
2109	free_irq_label(label);
2110	return ret;
2111	}
2112
2113	le->irq = irq;
2114
2115	FD_PREPARE(fdf, O_RDONLY \| O_CLOEXEC,
2116	anon_inode_getfile("gpio-event", &lineevent_fileops, le,
2117	O_RDONLY \| O_CLOEXEC));
2118	if (fdf->fd < 0)
2119	return fdf->fd;
2120	retain_and_null_ptr(le);
2121
2122	eventreq.fd = fdf->fd;
2123	if (copy_to_user(ip, &eventreq, sizeof(eventreq)))
2124	return -EFAULT;
2125
2126	fd_publish(fdf);
2127
2128	return 0;
2129	}
2130
2131	static void gpio_v2_line_info_to_v1(struct gpio_v2_line_info *info_v2,
2132	struct gpioline_info *info_v1)
2133	{
2134	u64 flagsv2 = info_v2->flags;
2135
2136	memcpy(info_v1->name, info_v2->name, sizeof(info_v1->name));
2137	memcpy(info_v1->consumer, info_v2->consumer, sizeof(info_v1->consumer));
2138	info_v1->line_offset = info_v2->offset;
2139	info_v1->flags = 0;
2140
2141	if (flagsv2 & GPIO_V2_LINE_FLAG_USED)
2142	info_v1->flags \|= GPIOLINE_FLAG_KERNEL;
2143
2144	if (flagsv2 & GPIO_V2_LINE_FLAG_OUTPUT)
2145	info_v1->flags \|= GPIOLINE_FLAG_IS_OUT;
2146
2147	if (flagsv2 & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
2148	info_v1->flags \|= GPIOLINE_FLAG_ACTIVE_LOW;
2149
2150	if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_DRAIN)
2151	info_v1->flags \|= GPIOLINE_FLAG_OPEN_DRAIN;
2152	if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_SOURCE)
2153	info_v1->flags \|= GPIOLINE_FLAG_OPEN_SOURCE;
2154
2155	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)
2156	info_v1->flags \|= GPIOLINE_FLAG_BIAS_PULL_UP;
2157	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN)
2158	info_v1->flags \|= GPIOLINE_FLAG_BIAS_PULL_DOWN;
2159	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_DISABLED)
2160	info_v1->flags \|= GPIOLINE_FLAG_BIAS_DISABLE;
2161	}
2162
2163	static void gpio_v2_line_info_changed_to_v1(
2164	struct gpio_v2_line_info_changed *lic_v2,
2165	struct gpioline_info_changed *lic_v1)
2166	{
2167	memset(lic_v1, 0, sizeof(*lic_v1));
2168	gpio_v2_line_info_to_v1(&lic_v2->info, &lic_v1->info);
2169	lic_v1->timestamp = lic_v2->timestamp_ns;
2170	lic_v1->event_type = lic_v2->event_type;
2171	}
2172
2173	#endif /* CONFIG_GPIO_CDEV_V1 */
2174
2175	static int gpio_desc_to_lineinfo(struct gpio_desc *desc,
2176	struct gpio_v2_line_info *info, bool atomic)
2177	{
2178	u32 debounce_period_us;
2179	unsigned long dflags;
2180	const char *label;
2181
2182	memset(info, 0, sizeof(*info));
2183
2184	CLASS(gpio_chip_guard, guard)(desc);
2185	if (!guard.gc)
2186	return -ENODEV;
2187
2188	info->offset = gpiod_hwgpio(desc);
2189
2190	if (desc->name)
2191	strscpy(info->name, desc->name, sizeof(info->name));
2192
2193	dflags = READ_ONCE(desc->flags);
2194
2195	scoped_guard(srcu, &desc->gdev->desc_srcu) {
2196	label = gpiod_get_label(desc);
2197	if (label && test_bit(GPIOD_FLAG_REQUESTED, &dflags))
2198	strscpy(info->consumer, label,
2199	sizeof(info->consumer));
2200	}
2201
2202	/*
2203	* Userspace only need know that the kernel is using this GPIO so it
2204	* can't use it.
2205	* The calculation of the used flag is slightly racy, as it may read
2206	* desc, gc and pinctrl state without a lock covering all three at
2207	* once. Worst case if the line is in transition and the calculation
2208	* is inconsistent then it looks to the user like they performed the
2209	* read on the other side of the transition - but that can always
2210	* happen.
2211	* The definitive test that a line is available to userspace is to
2212	* request it.
2213	*/
2214	if (test_bit(GPIOD_FLAG_REQUESTED, &dflags) \|\|
2215	test_bit(GPIOD_FLAG_IS_HOGGED, &dflags) \|\|
2216	test_bit(GPIOD_FLAG_EXPORT, &dflags) \|\|
2217	test_bit(GPIOD_FLAG_SYSFS, &dflags) \|\|
2218	!gpiochip_line_is_valid(guard.gc, info->offset)) {
2219	info->flags \|= GPIO_V2_LINE_FLAG_USED;
2220	} else if (!atomic) {
2221	if (!pinctrl_gpio_can_use_line(guard.gc, info->offset))
2222	info->flags \|= GPIO_V2_LINE_FLAG_USED;
2223	}
2224
2225	if (test_bit(GPIOD_FLAG_IS_OUT, &dflags))
2226	info->flags \|= GPIO_V2_LINE_FLAG_OUTPUT;
2227	else
2228	info->flags \|= GPIO_V2_LINE_FLAG_INPUT;
2229
2230	if (test_bit(GPIOD_FLAG_ACTIVE_LOW, &dflags))
2231	info->flags \|= GPIO_V2_LINE_FLAG_ACTIVE_LOW;
2232
2233	if (test_bit(GPIOD_FLAG_OPEN_DRAIN, &dflags))
2234	info->flags \|= GPIO_V2_LINE_FLAG_OPEN_DRAIN;
2235	if (test_bit(GPIOD_FLAG_OPEN_SOURCE, &dflags))
2236	info->flags \|= GPIO_V2_LINE_FLAG_OPEN_SOURCE;
2237
2238	if (test_bit(GPIOD_FLAG_BIAS_DISABLE, &dflags))
2239	info->flags \|= GPIO_V2_LINE_FLAG_BIAS_DISABLED;
2240	if (test_bit(GPIOD_FLAG_PULL_DOWN, &dflags))
2241	info->flags \|= GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN;
2242	if (test_bit(GPIOD_FLAG_PULL_UP, &dflags))
2243	info->flags \|= GPIO_V2_LINE_FLAG_BIAS_PULL_UP;
2244
2245	if (test_bit(GPIOD_FLAG_EDGE_RISING, &dflags))
2246	info->flags \|= GPIO_V2_LINE_FLAG_EDGE_RISING;
2247	if (test_bit(GPIOD_FLAG_EDGE_FALLING, &dflags))
2248	info->flags \|= GPIO_V2_LINE_FLAG_EDGE_FALLING;
2249
2250	if (test_bit(GPIOD_FLAG_EVENT_CLOCK_REALTIME, &dflags))
2251	info->flags \|= GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME;
2252	else if (test_bit(GPIOD_FLAG_EVENT_CLOCK_HTE, &dflags))
2253	info->flags \|= GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE;
2254
2255	debounce_period_us = READ_ONCE(desc->debounce_period_us);
2256	if (debounce_period_us) {
2257	info->attrs[info->num_attrs].id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE;
2258	info->attrs[info->num_attrs].debounce_period_us =
2259	debounce_period_us;
2260	info->num_attrs++;
2261	}
2262
2263	return 0;
2264	}
2265
2266	struct gpio_chardev_data {
2267	struct gpio_device *gdev;
2268	wait_queue_head_t wait;
2269	DECLARE_KFIFO(events, struct gpio_v2_line_info_changed, 32);
2270	struct notifier_block lineinfo_changed_nb;
2271	struct notifier_block device_unregistered_nb;
2272	unsigned long *watched_lines;
2273	#ifdef CONFIG_GPIO_CDEV_V1
2274	atomic_t watch_abi_version;
2275	#endif
2276	struct file *fp;
2277	};
2278
2279	static int chipinfo_get(struct gpio_chardev_data cdev, void __user ip)
2280	{
2281	struct gpio_device *gdev = cdev->gdev;
2282	struct gpiochip_info chipinfo;
2283
2284	memset(&chipinfo, 0, sizeof(chipinfo));
2285
2286	strscpy(chipinfo.name, dev_name(&gdev->dev), sizeof(chipinfo.name));
2287	strscpy(chipinfo.label, gdev->label, sizeof(chipinfo.label));
2288	chipinfo.lines = gdev->ngpio;
2289	if (copy_to_user(ip, &chipinfo, sizeof(chipinfo)))
2290	return -EFAULT;
2291	return 0;
2292	}
2293
2294	#ifdef CONFIG_GPIO_CDEV_V1
2295	/*
2296	* returns 0 if the versions match, else the previously selected ABI version
2297	*/
2298	static int lineinfo_ensure_abi_version(struct gpio_chardev_data *cdata,
2299	unsigned int version)
2300	{
2301	int abiv = atomic_cmpxchg(&cdata->watch_abi_version, 0, version);
2302
2303	if (abiv == version)
2304	return 0;
2305
2306	return abiv;
2307	}
2308
2309	static int lineinfo_get_v1(struct gpio_chardev_data cdev, void __user ip,
2310	bool watch)
2311	{
2312	struct gpio_desc *desc;
2313	struct gpioline_info lineinfo;
2314	struct gpio_v2_line_info lineinfo_v2;
2315	int ret;
2316
2317	if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
2318	return -EFAULT;
2319
2320	/* this doubles as a range check on line_offset */
2321	desc = gpio_device_get_desc(cdev->gdev, lineinfo.line_offset);
2322	if (IS_ERR(desc))
2323	return PTR_ERR(desc);
2324
2325	if (watch) {
2326	if (lineinfo_ensure_abi_version(cdev, 1))
2327	return -EPERM;
2328
2329	if (test_and_set_bit(lineinfo.line_offset, cdev->watched_lines))
2330	return -EBUSY;
2331	}
2332
2333	ret = gpio_desc_to_lineinfo(desc, &lineinfo_v2, false);
2334	if (ret)
2335	return ret;
2336
2337	gpio_v2_line_info_to_v1(&lineinfo_v2, &lineinfo);
2338
2339	if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
2340	if (watch)
2341	clear_bit(lineinfo.line_offset, cdev->watched_lines);
2342	return -EFAULT;
2343	}
2344
2345	return 0;
2346	}
2347	#endif
2348
2349	static int lineinfo_get(struct gpio_chardev_data cdev, void __user ip,
2350	bool watch)
2351	{
2352	struct gpio_desc *desc;
2353	struct gpio_v2_line_info lineinfo;
2354	int ret;
2355
2356	if (copy_from_user(&lineinfo, ip, sizeof(lineinfo)))
2357	return -EFAULT;
2358
2359	if (!mem_is_zero(lineinfo.padding, sizeof(lineinfo.padding)))
2360	return -EINVAL;
2361
2362	desc = gpio_device_get_desc(cdev->gdev, lineinfo.offset);
2363	if (IS_ERR(desc))
2364	return PTR_ERR(desc);
2365
2366	if (watch) {
2367	#ifdef CONFIG_GPIO_CDEV_V1
2368	if (lineinfo_ensure_abi_version(cdev, 2))
2369	return -EPERM;
2370	#endif
2371	if (test_and_set_bit(lineinfo.offset, cdev->watched_lines))
2372	return -EBUSY;
2373	}
2374
2375	ret = gpio_desc_to_lineinfo(desc, &lineinfo, false);
2376	if (ret)
2377	return ret;
2378
2379	if (copy_to_user(ip, &lineinfo, sizeof(lineinfo))) {
2380	if (watch)
2381	clear_bit(lineinfo.offset, cdev->watched_lines);
2382	return -EFAULT;
2383	}
2384
2385	return 0;
2386	}
2387
2388	static int lineinfo_unwatch(struct gpio_chardev_data cdev, void __user ip)
2389	{
2390	__u32 offset;
2391
2392	if (copy_from_user(&offset, ip, sizeof(offset)))
2393	return -EFAULT;
2394
2395	if (offset >= cdev->gdev->ngpio)
2396	return -EINVAL;
2397
2398	if (!test_and_clear_bit(offset, cdev->watched_lines))
2399	return -EBUSY;
2400
2401	return 0;
2402	}
2403
2404	/*
2405	* gpio_ioctl() - ioctl handler for the GPIO chardev
2406	*/
2407	static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2408	{
2409	struct gpio_chardev_data *cdev = file->private_data;
2410	struct gpio_device *gdev = cdev->gdev;
2411	void __user ip = (void __user )arg;
2412
2413	guard(srcu)(&gdev->srcu);
2414
2415	/* We fail any subsequent ioctl():s when the chip is gone */
2416	if (!rcu_access_pointer(gdev->chip))
2417	return -ENODEV;
2418
2419	/* Fill in the struct and pass to userspace */
2420	switch (cmd) {
2421	case GPIO_GET_CHIPINFO_IOCTL:
2422	return chipinfo_get(cdev, ip);
2423	#ifdef CONFIG_GPIO_CDEV_V1
2424	case GPIO_GET_LINEHANDLE_IOCTL:
2425	return linehandle_create(gdev, ip);
2426	case GPIO_GET_LINEEVENT_IOCTL:
2427	return lineevent_create(gdev, ip);
2428	case GPIO_GET_LINEINFO_IOCTL:
2429	return lineinfo_get_v1(cdev, ip, false);
2430	case GPIO_GET_LINEINFO_WATCH_IOCTL:
2431	return lineinfo_get_v1(cdev, ip, true);
2432	#endif /* CONFIG_GPIO_CDEV_V1 */
2433	case GPIO_V2_GET_LINEINFO_IOCTL:
2434	return lineinfo_get(cdev, ip, false);
2435	case GPIO_V2_GET_LINEINFO_WATCH_IOCTL:
2436	return lineinfo_get(cdev, ip, true);
2437	case GPIO_V2_GET_LINE_IOCTL:
2438	return linereq_create(gdev, ip);
2439	case GPIO_GET_LINEINFO_UNWATCH_IOCTL:
2440	return lineinfo_unwatch(cdev, ip);
2441	default:
2442	return -EINVAL;
2443	}
2444	}
2445
2446	#ifdef CONFIG_COMPAT
2447	static long gpio_ioctl_compat(struct file *file, unsigned int cmd,
2448	unsigned long arg)
2449	{
2450	return gpio_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
2451	}
2452	#endif
2453
2454	struct lineinfo_changed_ctx {
2455	struct work_struct work;
2456	struct gpio_v2_line_info_changed chg;
2457	struct gpio_device *gdev;
2458	struct gpio_chardev_data *cdev;
2459	};
2460
2461	static void lineinfo_changed_func(struct work_struct *work)
2462	{
2463	struct lineinfo_changed_ctx *ctx =
2464	container_of(work, struct lineinfo_changed_ctx, work);
2465	struct gpio_chip *gc;
2466	int ret;
2467
2468	if (!(ctx->chg.info.flags & GPIO_V2_LINE_FLAG_USED)) {
2469	/*
2470	* If nobody set the USED flag earlier, let's see with pinctrl
2471	* now. We're doing this late because it's a sleeping function.
2472	* Pin functions are in general much more static and while it's
2473	* not 100% bullet-proof, it's good enough for most cases.
2474	*/
2475	scoped_guard(srcu, &ctx->gdev->srcu) {
2476	gc = srcu_dereference(ctx->gdev->chip, &ctx->gdev->srcu);
2477	if (gc &&
2478	!pinctrl_gpio_can_use_line(gc, ctx->chg.info.offset))
2479	ctx->chg.info.flags \|= GPIO_V2_LINE_FLAG_USED;
2480	}
2481	}
2482
2483	ret = kfifo_in_spinlocked(&ctx->cdev->events, &ctx->chg, 1,
2484	&ctx->cdev->wait.lock);
2485	if (ret)
2486	wake_up_poll(&ctx->cdev->wait, EPOLLIN);
2487	else
2488	pr_debug_ratelimited("lineinfo event FIFO is full - event dropped\n");
2489
2490	gpio_device_put(ctx->gdev);
2491	fput(ctx->cdev->fp);
2492	kfree(ctx);
2493	}
2494
2495	static int lineinfo_changed_notify(struct notifier_block *nb,
2496	unsigned long action, void *data)
2497	{
2498	struct gpio_chardev_data *cdev =
2499	container_of(nb, struct gpio_chardev_data, lineinfo_changed_nb);
2500	struct lineinfo_changed_ctx *ctx;
2501	struct gpio_desc *desc = data;
2502	struct file *fp;
2503	int ret;
2504
2505	if (!test_bit(gpiod_hwgpio(desc), cdev->watched_lines))
2506	return NOTIFY_DONE;
2507
2508	/* Keep the file descriptor alive for the duration of the notification. */
2509	fp = get_file_active(&cdev->fp);
2510	if (!fp)
2511	/* Chardev file descriptor was or is being released. */
2512	return NOTIFY_DONE;
2513
2514	/*
2515	* If this is called from atomic context (for instance: with a spinlock
2516	* taken by the atomic notifier chain), any sleeping calls must be done
2517	* outside of this function in process context of the dedicated
2518	* workqueue.
2519	*
2520	* Let's gather as much info as possible from the descriptor and
2521	* postpone just the call to pinctrl_gpio_can_use_line() until the work
2522	* is executed.
2523	*/
2524
2525	ctx = kzalloc_obj(*ctx, GFP_ATOMIC);
2526	if (!ctx) {
2527	pr_err("Failed to allocate memory for line info notification\n");
2528	fput(fp);
2529	return NOTIFY_DONE;
2530	}
2531
2532	ctx->chg.event_type = action;
2533	ctx->chg.timestamp_ns = ktime_get_ns();
2534
2535	ret = gpio_desc_to_lineinfo(desc, &ctx->chg.info, true);
2536	if (ret) {
2537	fput(fp);
2538	return NOTIFY_DONE;
2539	}
2540
2541	/* Keep the GPIO device alive until we emit the event. */
2542	ctx->gdev = gpio_device_get(desc->gdev);
2543	ctx->cdev = cdev;
2544
2545	INIT_WORK(&ctx->work, lineinfo_changed_func);
2546	queue_work(ctx->gdev->line_state_wq, &ctx->work);
2547
2548	return NOTIFY_OK;
2549	}
2550
2551	static int gpio_device_unregistered_notify(struct notifier_block *nb,
2552	unsigned long action, void *data)
2553	{
2554	struct gpio_chardev_data *cdev = container_of(nb,
2555	struct gpio_chardev_data,
2556	device_unregistered_nb);
2557
2558	wake_up_poll(&cdev->wait, EPOLLIN \| EPOLLERR);
2559
2560	return NOTIFY_OK;
2561	}
2562
2563	static __poll_t lineinfo_watch_poll(struct file *file,
2564	struct poll_table_struct *pollt)
2565	{
2566	struct gpio_chardev_data *cdev = file->private_data;
2567	__poll_t events = 0;
2568
2569	guard(srcu)(&cdev->gdev->srcu);
2570
2571	if (!rcu_access_pointer(cdev->gdev->chip))
2572	return EPOLLHUP \| EPOLLERR;
2573
2574	poll_wait(file, &cdev->wait, pollt);
2575
2576	if (!kfifo_is_empty_spinlocked_noirqsave(&cdev->events,
2577	&cdev->wait.lock))
2578	events = EPOLLIN \| EPOLLRDNORM;
2579
2580	return events;
2581	}
2582
2583	static ssize_t lineinfo_watch_read(struct file file, char __user buf,
2584	size_t count, loff_t *off)
2585	{
2586	struct gpio_chardev_data *cdev = file->private_data;
2587	struct gpio_v2_line_info_changed event;
2588	ssize_t bytes_read = 0;
2589	int ret;
2590	size_t event_size;
2591
2592	guard(srcu)(&cdev->gdev->srcu);
2593
2594	if (!rcu_access_pointer(cdev->gdev->chip))
2595	return -ENODEV;
2596
2597	#ifndef CONFIG_GPIO_CDEV_V1
2598	event_size = sizeof(struct gpio_v2_line_info_changed);
2599	if (count < event_size)
2600	return -EINVAL;
2601	#endif
2602
2603	do {
2604	scoped_guard(spinlock, &cdev->wait.lock) {
2605	if (kfifo_is_empty(&cdev->events)) {
2606	if (bytes_read)
2607	return bytes_read;
2608
2609	if (file->f_flags & O_NONBLOCK)
2610	return -EAGAIN;
2611
2612	ret = wait_event_interruptible_locked(cdev->wait,
2613	!kfifo_is_empty(&cdev->events));
2614	if (ret)
2615	return ret;
2616	}
2617	#ifdef CONFIG_GPIO_CDEV_V1
2618	/* must be after kfifo check so watch_abi_version is set */
2619	if (atomic_read(&cdev->watch_abi_version) == 2)
2620	event_size = sizeof(struct gpio_v2_line_info_changed);
2621	else
2622	event_size = sizeof(struct gpioline_info_changed);
2623	if (count < event_size)
2624	return -EINVAL;
2625	#endif
2626	if (kfifo_out(&cdev->events, &event, 1) != 1) {
2627	/*
2628	* This should never happen - we hold the
2629	* lock from the moment we learned the fifo
2630	* is no longer empty until now.
2631	*/
2632	WARN(1, "failed to read from non-empty kfifo");
2633	return -EIO;
2634	}
2635	}
2636
2637	#ifdef CONFIG_GPIO_CDEV_V1
2638	if (event_size == sizeof(struct gpio_v2_line_info_changed)) {
2639	if (copy_to_user(buf + bytes_read, &event, event_size))
2640	return -EFAULT;
2641	} else {
2642	struct gpioline_info_changed event_v1;
2643
2644	gpio_v2_line_info_changed_to_v1(&event, &event_v1);
2645	if (copy_to_user(buf + bytes_read, &event_v1,
2646	event_size))
2647	return -EFAULT;
2648	}
2649	#else
2650	if (copy_to_user(buf + bytes_read, &event, event_size))
2651	return -EFAULT;
2652	#endif
2653	bytes_read += event_size;
2654	} while (count >= bytes_read + sizeof(event));
2655
2656	return bytes_read;
2657	}
2658
2659	/**
2660	* gpio_chrdev_open() - open the chardev for ioctl operations
2661	* @inode: inode for this chardev
2662	* @file: file struct for storing private data
2663	*
2664	* Returns:
2665	* 0 on success, or negative errno on failure.
2666	*/
2667	static int gpio_chrdev_open(struct inode inode, struct file file)
2668	{
2669	struct gpio_device *gdev = container_of(inode->i_cdev,
2670	struct gpio_device, chrdev);
2671	struct gpio_chardev_data *cdev;
2672	int ret = -ENOMEM;
2673
2674	cdev = kzalloc_obj(*cdev);
2675	if (!cdev)
2676	return -ENOMEM;
2677
2678	cdev->watched_lines = bitmap_zalloc(gdev->ngpio, GFP_KERNEL);
2679	if (!cdev->watched_lines)
2680	goto out_free_cdev;
2681
2682	init_waitqueue_head(&cdev->wait);
2683	INIT_KFIFO(cdev->events);
2684	cdev->gdev = gpio_device_get(gdev);
2685
2686	cdev->lineinfo_changed_nb.notifier_call = lineinfo_changed_notify;
2687	scoped_guard(write_lock_irqsave, &gdev->line_state_lock)
2688	ret = raw_notifier_chain_register(&gdev->line_state_notifier,
2689	&cdev->lineinfo_changed_nb);
2690	if (ret)
2691	goto out_free_bitmap;
2692
2693	cdev->device_unregistered_nb.notifier_call =
2694	gpio_device_unregistered_notify;
2695	ret = blocking_notifier_chain_register(&gdev->device_notifier,
2696	&cdev->device_unregistered_nb);
2697	if (ret)
2698	goto out_unregister_line_notifier;
2699
2700	file->private_data = cdev;
2701	cdev->fp = file;
2702
2703	return nonseekable_open(inode, file);
2704
2705	out_unregister_line_notifier:
2706	scoped_guard(write_lock_irqsave, &gdev->line_state_lock)
2707	raw_notifier_chain_unregister(&gdev->line_state_notifier,
2708	&cdev->lineinfo_changed_nb);
2709	out_free_bitmap:
2710	gpio_device_put(gdev);
2711	bitmap_free(cdev->watched_lines);
2712	out_free_cdev:
2713	kfree(cdev);
2714	return ret;
2715	}
2716
2717	/**
2718	* gpio_chrdev_release() - close chardev after ioctl operations
2719	* @inode: inode for this chardev
2720	* @file: file struct for storing private data
2721	*
2722	* Returns:
2723	* 0 on success, or negative errno on failure.
2724	*/
2725	static int gpio_chrdev_release(struct inode inode, struct file file)
2726	{
2727	struct gpio_chardev_data *cdev = file->private_data;
2728	struct gpio_device *gdev = cdev->gdev;
2729
2730	blocking_notifier_chain_unregister(&gdev->device_notifier,
2731	&cdev->device_unregistered_nb);
2732	scoped_guard(write_lock_irqsave, &gdev->line_state_lock)
2733	raw_notifier_chain_unregister(&gdev->line_state_notifier,
2734	&cdev->lineinfo_changed_nb);
2735	bitmap_free(cdev->watched_lines);
2736	gpio_device_put(gdev);
2737	kfree(cdev);
2738
2739	return 0;
2740	}
2741
2742	static const struct file_operations gpio_fileops = {
2743	.release = gpio_chrdev_release,
2744	.open = gpio_chrdev_open,
2745	.poll = lineinfo_watch_poll,
2746	.read = lineinfo_watch_read,
2747	.owner = THIS_MODULE,
2748	.unlocked_ioctl = gpio_ioctl,
2749	#ifdef CONFIG_COMPAT
2750	.compat_ioctl = gpio_ioctl_compat,
2751	#endif
2752	};
2753
2754	int gpiolib_cdev_register(struct gpio_chip *gc, dev_t devt)
2755	{
2756	struct gpio_device *gdev = gc->gpiodev;
2757	int ret;
2758
2759	cdev_init(&gdev->chrdev, &gpio_fileops);
2760	gdev->chrdev.owner = THIS_MODULE;
2761	gdev->dev.devt = MKDEV(MAJOR(devt), gdev->id);
2762
2763	gdev->line_state_wq = alloc_ordered_workqueue("%s", WQ_HIGHPRI,
2764	dev_name(&gdev->dev));
2765	if (!gdev->line_state_wq)
2766	return -ENOMEM;
2767
2768	ret = cdev_device_add(&gdev->chrdev, &gdev->dev);
2769	if (ret) {
2770	destroy_workqueue(gdev->line_state_wq);
2771	return ret;
2772	}
2773
2774	gpiochip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id);
2775
2776	return 0;
2777	}
2778
2779	void gpiolib_cdev_unregister(struct gpio_device *gdev)
2780	{
2781	destroy_workqueue(gdev->line_state_wq);
2782	cdev_device_del(&gdev->chrdev, &gdev->dev);
2783	blocking_notifier_call_chain(&gdev->device_notifier, 0, NULL);
2784	}