projects/linux·blob master · refs

1153 lines · 29.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Loopback bridge driver for the Greybus loopback module.
4 *
5 * Copyright 2014 Google Inc.
6 * Copyright 2014 Linaro Ltd.
7 */
8
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/mutex.h>
14#include <linux/slab.h>
15#include <linux/kthread.h>
16#include <linux/delay.h>
17#include <linux/random.h>
18#include <linux/sizes.h>
19#include <linux/cdev.h>
20#include <linux/fs.h>
21#include <linux/kfifo.h>
22#include <linux/debugfs.h>
23#include <linux/list_sort.h>
24#include <linux/spinlock.h>
25#include <linux/workqueue.h>
26#include <linux/atomic.h>
27#include <linux/pm_runtime.h>
28#include <linux/greybus.h>
29#include <asm/div64.h>
30
31#define NSEC_PER_DAY 86400000000000ULL
32
33struct gb_loopback_stats {
34 u32 min;
35 u32 max;
36 u64 sum;
37 u32 count;
38};
39
40struct gb_loopback_device {
41 struct dentry *root;
42 u32 count;
43 size_t size_max;
44
45 /* We need to take a lock in atomic context */
46 spinlock_t lock;
47 wait_queue_head_t wq;
48};
49
50static struct gb_loopback_device gb_dev;
51
52struct gb_loopback_async_operation {
53 struct gb_loopback *gb;
54 struct gb_operation *operation;
55 ktime_t ts;
56 int (*completion)(struct gb_loopback_async_operation *op_async);
57};
58
59struct gb_loopback {
60 struct gb_connection *connection;
61
62 struct dentry *file;
63 struct kfifo kfifo_lat;
64 struct mutex mutex;
65 struct task_struct *task;
66 struct device *dev;
67 wait_queue_head_t wq;
68 wait_queue_head_t wq_completion;
69 atomic_t outstanding_operations;
70
71 /* Per connection stats */
72 ktime_t ts;
73 struct gb_loopback_stats latency;
74 struct gb_loopback_stats throughput;
75 struct gb_loopback_stats requests_per_second;
76 struct gb_loopback_stats apbridge_unipro_latency;
77 struct gb_loopback_stats gbphy_firmware_latency;
78
79 int type;
80 int async;
81 int id;
82 u32 size;
83 u32 iteration_max;
84 u32 iteration_count;
85 int us_wait;
86 u32 error;
87 u32 requests_completed;
88 u32 requests_timedout;
89 u32 timeout;
90 u32 jiffy_timeout;
91 u32 timeout_min;
92 u32 timeout_max;
93 u32 outstanding_operations_max;
94 u64 elapsed_nsecs;
95 u32 apbridge_latency_ts;
96 u32 gbphy_latency_ts;
97
98 u32 send_count;
99};
100
101static struct class loopback_class = {
102 .name = "gb_loopback",
103};
104
105static DEFINE_IDA(loopback_ida);
106
107/* Min/max values in jiffies */
108#define GB_LOOPBACK_TIMEOUT_MIN 1
109#define GB_LOOPBACK_TIMEOUT_MAX 10000
110
111#define GB_LOOPBACK_FIFO_DEFAULT 8192
112
113static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT;
114module_param(kfifo_depth, uint, 0444);
115
116/* Maximum size of any one send data buffer we support */
117#define MAX_PACKET_SIZE (PAGE_SIZE * 2)
118
119#define GB_LOOPBACK_US_WAIT_MAX 1000000
120
121/* interface sysfs attributes */
122#define gb_loopback_ro_attr(field) \
123static ssize_t field##_show(struct device *dev, \
124 struct device_attribute *attr, \
125 char *buf) \
126{ \
127 struct gb_loopback *gb = dev_get_drvdata(dev); \
128 return sysfs_emit(buf, "%u\n", gb->field); \
129} \
130static DEVICE_ATTR_RO(field)
131
132#define gb_loopback_ro_stats_attr(name, field, type) \
133static ssize_t name##_##field##_show(struct device *dev, \
134 struct device_attribute *attr, \
135 char *buf) \
136{ \
137 struct gb_loopback *gb = dev_get_drvdata(dev); \
138 /* Report 0 for min and max if no transfer succeeded */ \
139 if (!gb->requests_completed) \
140 return sysfs_emit(buf, "0\n"); \
141 return sysfs_emit(buf, "%" #type "\n", gb->name.field); \
142} \
143static DEVICE_ATTR_RO(name##_##field)
144
145#define gb_loopback_ro_avg_attr(name) \
146static ssize_t name##_avg_show(struct device *dev, \
147 struct device_attribute *attr, \
148 char *buf) \
149{ \
150 struct gb_loopback_stats *stats; \
151 struct gb_loopback *gb; \
152 u64 avg, rem; \
153 u32 count; \
154 gb = dev_get_drvdata(dev); \
155 stats = &gb->name; \
156 count = stats->count ? stats->count : 1; \
157 avg = stats->sum + count / 2000000; /* round closest */ \
158 rem = do_div(avg, count); \
159 rem *= 1000000; \
160 do_div(rem, count); \
161 return sysfs_emit(buf, "%llu.%06u\n", avg, (u32)rem); \
162} \
163static DEVICE_ATTR_RO(name##_avg)
164
165#define gb_loopback_stats_attrs(field) \
166 gb_loopback_ro_stats_attr(field, min, u); \
167 gb_loopback_ro_stats_attr(field, max, u); \
168 gb_loopback_ro_avg_attr(field)
169
170#define gb_dev_loopback_ro_attr(field) \
171static ssize_t field##_show(struct device *dev, \
172 struct device_attribute *attr, \
173 char *buf) \
174{ \
175 struct gb_loopback *gb = dev_get_drvdata(dev); \
176 return sysfs_emit(buf, "%u\n", gb->field); \
177} \
178static DEVICE_ATTR_RO(field)
179
180#define gb_dev_loopback_rw_attr(field, type) \
181static ssize_t field##_show(struct device *dev, \
182 struct device_attribute *attr, \
183 char *buf) \
184{ \
185 struct gb_loopback *gb = dev_get_drvdata(dev); \
186 return sysfs_emit(buf, "%" #type "\n", gb->field); \
187} \
188static ssize_t field##_store(struct device *dev, \
189 struct device_attribute *attr, \
190 const char *buf, \
191 size_t len) \
192{ \
193 int ret; \
194 struct gb_loopback *gb = dev_get_drvdata(dev); \
195 mutex_lock(&gb->mutex); \
196 ret = sscanf(buf, "%"#type, &gb->field); \
197 if (ret != 1) \
198 len = -EINVAL; \
199 else \
200 gb_loopback_check_attr(gb); \
201 mutex_unlock(&gb->mutex); \
202 return len; \
203} \
204static DEVICE_ATTR_RW(field)
205
206static void gb_loopback_reset_stats(struct gb_loopback *gb);
207static void gb_loopback_check_attr(struct gb_loopback *gb)
208{
209 if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX)
210 gb->us_wait = GB_LOOPBACK_US_WAIT_MAX;
211 if (gb->size > gb_dev.size_max)
212 gb->size = gb_dev.size_max;
213 gb->requests_timedout = 0;
214 gb->requests_completed = 0;
215 gb->iteration_count = 0;
216 gb->send_count = 0;
217 gb->error = 0;
218
219 if (kfifo_depth < gb->iteration_max) {
220 dev_warn(gb->dev,
221 "cannot log bytes %u kfifo_depth %u\n",
222 gb->iteration_max, kfifo_depth);
223 }
224 kfifo_reset_out(&gb->kfifo_lat);
225
226 switch (gb->type) {
227 case GB_LOOPBACK_TYPE_PING:
228 case GB_LOOPBACK_TYPE_TRANSFER:
229 case GB_LOOPBACK_TYPE_SINK:
230 gb->jiffy_timeout = usecs_to_jiffies(gb->timeout);
231 if (!gb->jiffy_timeout)
232 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN;
233 else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX)
234 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX;
235 gb_loopback_reset_stats(gb);
236 wake_up(&gb->wq);
237 break;
238 default:
239 gb->type = 0;
240 break;
241 }
242}
243
244/* Time to send and receive one message */
245gb_loopback_stats_attrs(latency);
246/* Number of requests sent per second on this cport */
247gb_loopback_stats_attrs(requests_per_second);
248/* Quantity of data sent and received on this cport */
249gb_loopback_stats_attrs(throughput);
250/* Latency across the UniPro link from APBridge's perspective */
251gb_loopback_stats_attrs(apbridge_unipro_latency);
252/* Firmware induced overhead in the GPBridge */
253gb_loopback_stats_attrs(gbphy_firmware_latency);
254
255/* Number of errors encountered during loop */
256gb_loopback_ro_attr(error);
257/* Number of requests successfully completed async */
258gb_loopback_ro_attr(requests_completed);
259/* Number of requests timed out async */
260gb_loopback_ro_attr(requests_timedout);
261/* Timeout minimum in useconds */
262gb_loopback_ro_attr(timeout_min);
263/* Timeout minimum in useconds */
264gb_loopback_ro_attr(timeout_max);
265
266/*
267 * Type of loopback message to send based on protocol type definitions
268 * 0 => Don't send message
269 * 2 => Send ping message continuously (message without payload)
270 * 3 => Send transfer message continuously (message with payload,
271 * payload returned in response)
272 * 4 => Send a sink message (message with payload, no payload in response)
273 */
274gb_dev_loopback_rw_attr(type, d);
275/* Size of transfer message payload: 0-4096 bytes */
276gb_dev_loopback_rw_attr(size, u);
277/* Time to wait between two messages: 0-1000 ms */
278gb_dev_loopback_rw_attr(us_wait, d);
279/* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */
280gb_dev_loopback_rw_attr(iteration_max, u);
281/* The current index of the for (i = 0; i < iteration_max; i++) loop */
282gb_dev_loopback_ro_attr(iteration_count);
283/* A flag to indicate synchronous or asynchronous operations */
284gb_dev_loopback_rw_attr(async, u);
285/* Timeout of an individual asynchronous request */
286gb_dev_loopback_rw_attr(timeout, u);
287/* Maximum number of in-flight operations before back-off */
288gb_dev_loopback_rw_attr(outstanding_operations_max, u);
289
290static struct attribute *loopback_attrs[] = {
291 &dev_attr_latency_min.attr,
292 &dev_attr_latency_max.attr,
293 &dev_attr_latency_avg.attr,
294 &dev_attr_requests_per_second_min.attr,
295 &dev_attr_requests_per_second_max.attr,
296 &dev_attr_requests_per_second_avg.attr,
297 &dev_attr_throughput_min.attr,
298 &dev_attr_throughput_max.attr,
299 &dev_attr_throughput_avg.attr,
300 &dev_attr_apbridge_unipro_latency_min.attr,
301 &dev_attr_apbridge_unipro_latency_max.attr,
302 &dev_attr_apbridge_unipro_latency_avg.attr,
303 &dev_attr_gbphy_firmware_latency_min.attr,
304 &dev_attr_gbphy_firmware_latency_max.attr,
305 &dev_attr_gbphy_firmware_latency_avg.attr,
306 &dev_attr_type.attr,
307 &dev_attr_size.attr,
308 &dev_attr_us_wait.attr,
309 &dev_attr_iteration_count.attr,
310 &dev_attr_iteration_max.attr,
311 &dev_attr_async.attr,
312 &dev_attr_error.attr,
313 &dev_attr_requests_completed.attr,
314 &dev_attr_requests_timedout.attr,
315 &dev_attr_timeout.attr,
316 &dev_attr_outstanding_operations_max.attr,
317 &dev_attr_timeout_min.attr,
318 &dev_attr_timeout_max.attr,
319 NULL,
320};
321ATTRIBUTE_GROUPS(loopback);
322
323static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error);
324
325static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs)
326{
327 do_div(elapsed_nsecs, NSEC_PER_USEC);
328 return elapsed_nsecs;
329}
330
331static u64 __gb_loopback_calc_latency(u64 t1, u64 t2)
332{
333 if (t2 > t1)
334 return t2 - t1;
335 else
336 return NSEC_PER_DAY - t2 + t1;
337}
338
339static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te)
340{
341 return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te));
342}
343
344static int gb_loopback_operation_sync(struct gb_loopback *gb, int type,
345 void *request, int request_size,
346 void *response, int response_size)
347{
348 struct gb_operation *operation;
349 ktime_t ts, te;
350 int ret;
351
352 ts = ktime_get();
353 operation = gb_operation_create(gb->connection, type, request_size,
354 response_size, GFP_KERNEL);
355 if (!operation)
356 return -ENOMEM;
357
358 if (request_size)
359 memcpy(operation->request->payload, request, request_size);
360
361 ret = gb_operation_request_send_sync(operation);
362 if (ret) {
363 dev_err(&gb->connection->bundle->dev,
364 "synchronous operation failed: %d\n", ret);
365 goto out_put_operation;
366 } else {
367 if (response_size == operation->response->payload_size) {
368 memcpy(response, operation->response->payload,
369 response_size);
370 } else {
371 dev_err(&gb->connection->bundle->dev,
372 "response size %zu expected %d\n",
373 operation->response->payload_size,
374 response_size);
375 ret = -EINVAL;
376 goto out_put_operation;
377 }
378 }
379
380 te = ktime_get();
381
382 /* Calculate the total time the message took */
383 gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te);
384
385out_put_operation:
386 gb_operation_put(operation);
387
388 return ret;
389}
390
391static void gb_loopback_async_wait_all(struct gb_loopback *gb)
392{
393 wait_event(gb->wq_completion,
394 !atomic_read(&gb->outstanding_operations));
395}
396
397static void gb_loopback_async_operation_callback(struct gb_operation *operation)
398{
399 struct gb_loopback_async_operation *op_async;
400 struct gb_loopback *gb;
401 ktime_t te;
402 int result;
403
404 te = ktime_get();
405 result = gb_operation_result(operation);
406 op_async = gb_operation_get_data(operation);
407 gb = op_async->gb;
408
409 mutex_lock(&gb->mutex);
410
411 if (!result && op_async->completion)
412 result = op_async->completion(op_async);
413
414 if (!result) {
415 gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te);
416 } else {
417 gb->error++;
418 if (result == -ETIMEDOUT)
419 gb->requests_timedout++;
420 }
421
422 gb->iteration_count++;
423 gb_loopback_calculate_stats(gb, result);
424
425 mutex_unlock(&gb->mutex);
426
427 dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n",
428 operation->id);
429
430 /* Wake up waiters */
431 atomic_dec(&op_async->gb->outstanding_operations);
432 wake_up(&gb->wq_completion);
433
434 /* Release resources */
435 gb_operation_put(operation);
436 kfree(op_async);
437}
438
439static int gb_loopback_async_operation(struct gb_loopback *gb, int type,
440 void *request, int request_size,
441 int response_size,
442 void *completion)
443{
444 struct gb_loopback_async_operation *op_async;
445 struct gb_operation *operation;
446 int ret;
447
448 op_async = kzalloc_obj(*op_async);
449 if (!op_async)
450 return -ENOMEM;
451
452 operation = gb_operation_create(gb->connection, type, request_size,
453 response_size, GFP_KERNEL);
454 if (!operation) {
455 kfree(op_async);
456 return -ENOMEM;
457 }
458
459 if (request_size)
460 memcpy(operation->request->payload, request, request_size);
461
462 gb_operation_set_data(operation, op_async);
463
464 op_async->gb = gb;
465 op_async->operation = operation;
466 op_async->completion = completion;
467
468 op_async->ts = ktime_get();
469
470 atomic_inc(&gb->outstanding_operations);
471 ret = gb_operation_request_send(operation,
472 gb_loopback_async_operation_callback,
473 jiffies_to_msecs(gb->jiffy_timeout),
474 GFP_KERNEL);
475 if (ret) {
476 atomic_dec(&gb->outstanding_operations);
477 gb_operation_put(operation);
478 kfree(op_async);
479 }
480 return ret;
481}
482
483static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len)
484{
485 struct gb_loopback_transfer_request *request;
486 int retval;
487
488 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
489 if (!request)
490 return -ENOMEM;
491
492 request->len = cpu_to_le32(len);
493 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK,
494 request, len + sizeof(*request),
495 NULL, 0);
496 kfree(request);
497 return retval;
498}
499
500static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len)
501{
502 struct gb_loopback_transfer_request *request;
503 struct gb_loopback_transfer_response *response;
504 int retval;
505
506 gb->apbridge_latency_ts = 0;
507 gb->gbphy_latency_ts = 0;
508
509 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
510 if (!request)
511 return -ENOMEM;
512 response = kmalloc(len + sizeof(*response), GFP_KERNEL);
513 if (!response) {
514 kfree(request);
515 return -ENOMEM;
516 }
517
518 memset(request->data, 0x5A, len);
519
520 request->len = cpu_to_le32(len);
521 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER,
522 request, len + sizeof(*request),
523 response, len + sizeof(*response));
524 if (retval)
525 goto gb_error;
526
527 if (memcmp(request->data, response->data, len)) {
528 dev_err(&gb->connection->bundle->dev,
529 "Loopback Data doesn't match\n");
530 retval = -EREMOTEIO;
531 }
532 gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0);
533 gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1);
534
535gb_error:
536 kfree(request);
537 kfree(response);
538
539 return retval;
540}
541
542static int gb_loopback_sync_ping(struct gb_loopback *gb)
543{
544 return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING,
545 NULL, 0, NULL, 0);
546}
547
548static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len)
549{
550 struct gb_loopback_transfer_request *request;
551 int retval;
552
553 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
554 if (!request)
555 return -ENOMEM;
556
557 request->len = cpu_to_le32(len);
558 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK,
559 request, len + sizeof(*request),
560 0, NULL);
561 kfree(request);
562 return retval;
563}
564
565static int gb_loopback_async_transfer_complete(
566 struct gb_loopback_async_operation *op_async)
567{
568 struct gb_loopback *gb;
569 struct gb_operation *operation;
570 struct gb_loopback_transfer_request *request;
571 struct gb_loopback_transfer_response *response;
572 size_t len;
573 int retval = 0;
574
575 gb = op_async->gb;
576 operation = op_async->operation;
577 request = operation->request->payload;
578 response = operation->response->payload;
579 len = le32_to_cpu(request->len);
580
581 if (memcmp(request->data, response->data, len)) {
582 dev_err(&gb->connection->bundle->dev,
583 "Loopback Data doesn't match operation id %d\n",
584 operation->id);
585 retval = -EREMOTEIO;
586 } else {
587 gb->apbridge_latency_ts =
588 (u32)__le32_to_cpu(response->reserved0);
589 gb->gbphy_latency_ts =
590 (u32)__le32_to_cpu(response->reserved1);
591 }
592
593 return retval;
594}
595
596static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len)
597{
598 struct gb_loopback_transfer_request *request;
599 int retval, response_len;
600
601 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
602 if (!request)
603 return -ENOMEM;
604
605 memset(request->data, 0x5A, len);
606
607 request->len = cpu_to_le32(len);
608 response_len = sizeof(struct gb_loopback_transfer_response);
609 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER,
610 request, len + sizeof(*request),
611 len + response_len,
612 gb_loopback_async_transfer_complete);
613 if (retval)
614 goto gb_error;
615
616gb_error:
617 kfree(request);
618 return retval;
619}
620
621static int gb_loopback_async_ping(struct gb_loopback *gb)
622{
623 return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING,
624 NULL, 0, 0, NULL);
625}
626
627static int gb_loopback_request_handler(struct gb_operation *operation)
628{
629 struct gb_connection *connection = operation->connection;
630 struct gb_loopback_transfer_request *request;
631 struct gb_loopback_transfer_response *response;
632 struct device *dev = &connection->bundle->dev;
633 size_t len;
634
635 /* By convention, the AP initiates the version operation */
636 switch (operation->type) {
637 case GB_LOOPBACK_TYPE_PING:
638 case GB_LOOPBACK_TYPE_SINK:
639 return 0;
640 case GB_LOOPBACK_TYPE_TRANSFER:
641 if (operation->request->payload_size < sizeof(*request)) {
642 dev_err(dev, "transfer request too small (%zu < %zu)\n",
643 operation->request->payload_size,
644 sizeof(*request));
645 return -EINVAL; /* -EMSGSIZE */
646 }
647 request = operation->request->payload;
648 len = le32_to_cpu(request->len);
649 if (len > gb_dev.size_max) {
650 dev_err(dev, "transfer request too large (%zu > %zu)\n",
651 len, gb_dev.size_max);
652 return -EINVAL;
653 }
654
655 if (!gb_operation_response_alloc(operation,
656 len + sizeof(*response), GFP_KERNEL)) {
657 dev_err(dev, "error allocating response\n");
658 return -ENOMEM;
659 }
660 response = operation->response->payload;
661 response->len = cpu_to_le32(len);
662 if (len)
663 memcpy(response->data, request->data, len);
664
665 return 0;
666 default:
667 dev_err(dev, "unsupported request: %u\n", operation->type);
668 return -EINVAL;
669 }
670}
671
672static void gb_loopback_reset_stats(struct gb_loopback *gb)
673{
674 struct gb_loopback_stats reset = {
675 .min = U32_MAX,
676 };
677
678 /* Reset per-connection stats */
679 memcpy(&gb->latency, &reset,
680 sizeof(struct gb_loopback_stats));
681 memcpy(&gb->throughput, &reset,
682 sizeof(struct gb_loopback_stats));
683 memcpy(&gb->requests_per_second, &reset,
684 sizeof(struct gb_loopback_stats));
685 memcpy(&gb->apbridge_unipro_latency, &reset,
686 sizeof(struct gb_loopback_stats));
687 memcpy(&gb->gbphy_firmware_latency, &reset,
688 sizeof(struct gb_loopback_stats));
689
690 /* Should be initialized at least once per transaction set */
691 gb->apbridge_latency_ts = 0;
692 gb->gbphy_latency_ts = 0;
693 gb->ts = ktime_set(0, 0);
694}
695
696static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val)
697{
698 if (stats->min > val)
699 stats->min = val;
700 if (stats->max < val)
701 stats->max = val;
702 stats->sum += val;
703 stats->count++;
704}
705
706static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats,
707 u64 val, u32 count)
708{
709 stats->sum += val;
710 stats->count += count;
711
712 do_div(val, count);
713 if (stats->min > val)
714 stats->min = val;
715 if (stats->max < val)
716 stats->max = val;
717}
718
719static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency)
720{
721 u64 req = gb->requests_completed * USEC_PER_SEC;
722
723 gb_loopback_update_stats_window(&gb->requests_per_second, req, latency);
724}
725
726static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency)
727{
728 u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2;
729
730 switch (gb->type) {
731 case GB_LOOPBACK_TYPE_PING:
732 break;
733 case GB_LOOPBACK_TYPE_SINK:
734 aggregate_size += sizeof(struct gb_loopback_transfer_request) +
735 gb->size;
736 break;
737 case GB_LOOPBACK_TYPE_TRANSFER:
738 aggregate_size += sizeof(struct gb_loopback_transfer_request) +
739 sizeof(struct gb_loopback_transfer_response) +
740 gb->size * 2;
741 break;
742 default:
743 return;
744 }
745
746 aggregate_size *= gb->requests_completed;
747 aggregate_size *= USEC_PER_SEC;
748 gb_loopback_update_stats_window(&gb->throughput, aggregate_size,
749 latency);
750}
751
752static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb)
753{
754 u32 lat;
755
756 /* Express latency in terms of microseconds */
757 lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs);
758
759 /* Log latency stastic */
760 gb_loopback_update_stats(&gb->latency, lat);
761
762 /* Raw latency log on a per thread basis */
763 kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat));
764
765 /* Log the firmware supplied latency values */
766 gb_loopback_update_stats(&gb->apbridge_unipro_latency,
767 gb->apbridge_latency_ts);
768 gb_loopback_update_stats(&gb->gbphy_firmware_latency,
769 gb->gbphy_latency_ts);
770}
771
772static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error)
773{
774 u64 nlat;
775 u32 lat;
776 ktime_t te;
777
778 if (!error) {
779 gb->requests_completed++;
780 gb_loopback_calculate_latency_stats(gb);
781 }
782
783 te = ktime_get();
784 nlat = gb_loopback_calc_latency(gb->ts, te);
785 if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) {
786 lat = gb_loopback_nsec_to_usec_latency(nlat);
787
788 gb_loopback_throughput_update(gb, lat);
789 gb_loopback_requests_update(gb, lat);
790
791 if (gb->iteration_count != gb->iteration_max) {
792 gb->ts = te;
793 gb->requests_completed = 0;
794 }
795 }
796}
797
798static void gb_loopback_async_wait_to_send(struct gb_loopback *gb)
799{
800 if (!(gb->async && gb->outstanding_operations_max))
801 return;
802 wait_event_interruptible(gb->wq_completion,
803 (atomic_read(&gb->outstanding_operations) <
804 gb->outstanding_operations_max) ||
805 kthread_should_stop());
806}
807
808static int gb_loopback_fn(void *data)
809{
810 int error = 0;
811 int us_wait = 0;
812 int type;
813 int ret;
814 u32 size;
815
816 struct gb_loopback *gb = data;
817 struct gb_bundle *bundle = gb->connection->bundle;
818
819 ret = gb_pm_runtime_get_sync(bundle);
820 if (ret)
821 return ret;
822
823 while (1) {
824 if (!gb->type) {
825 gb_pm_runtime_put_autosuspend(bundle);
826 wait_event_interruptible(gb->wq, gb->type ||
827 kthread_should_stop());
828 ret = gb_pm_runtime_get_sync(bundle);
829 if (ret)
830 return ret;
831 }
832
833 if (kthread_should_stop())
834 break;
835
836 /* Limit the maximum number of in-flight async operations */
837 gb_loopback_async_wait_to_send(gb);
838 if (kthread_should_stop())
839 break;
840
841 mutex_lock(&gb->mutex);
842
843 /* Optionally terminate */
844 if (gb->send_count == gb->iteration_max) {
845 mutex_unlock(&gb->mutex);
846
847 /* Wait for synchronous and asynchronous completion */
848 gb_loopback_async_wait_all(gb);
849
850 /* Mark complete unless user-space has poked us */
851 mutex_lock(&gb->mutex);
852 if (gb->iteration_count == gb->iteration_max) {
853 gb->type = 0;
854 gb->send_count = 0;
855 sysfs_notify(&gb->dev->kobj, NULL,
856 "iteration_count");
857 dev_dbg(&bundle->dev, "load test complete\n");
858 } else {
859 dev_dbg(&bundle->dev,
860 "continuing on with new test set\n");
861 }
862 mutex_unlock(&gb->mutex);
863 continue;
864 }
865 size = gb->size;
866 us_wait = gb->us_wait;
867 type = gb->type;
868 if (ktime_to_ns(gb->ts) == 0)
869 gb->ts = ktime_get();
870
871 /* Else operations to perform */
872 if (gb->async) {
873 if (type == GB_LOOPBACK_TYPE_PING)
874 error = gb_loopback_async_ping(gb);
875 else if (type == GB_LOOPBACK_TYPE_TRANSFER)
876 error = gb_loopback_async_transfer(gb, size);
877 else if (type == GB_LOOPBACK_TYPE_SINK)
878 error = gb_loopback_async_sink(gb, size);
879
880 if (error) {
881 gb->error++;
882 gb->iteration_count++;
883 }
884 } else {
885 /* We are effectively single threaded here */
886 if (type == GB_LOOPBACK_TYPE_PING)
887 error = gb_loopback_sync_ping(gb);
888 else if (type == GB_LOOPBACK_TYPE_TRANSFER)
889 error = gb_loopback_sync_transfer(gb, size);
890 else if (type == GB_LOOPBACK_TYPE_SINK)
891 error = gb_loopback_sync_sink(gb, size);
892
893 if (error)
894 gb->error++;
895 gb->iteration_count++;
896 gb_loopback_calculate_stats(gb, !!error);
897 }
898 gb->send_count++;
899 mutex_unlock(&gb->mutex);
900
901 if (us_wait) {
902 if (us_wait < 20000)
903 usleep_range(us_wait, us_wait + 100);
904 else
905 msleep(us_wait / 1000);
906 }
907 }
908
909 gb_pm_runtime_put_autosuspend(bundle);
910
911 return 0;
912}
913
914static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s,
915 struct kfifo *kfifo,
916 struct mutex *mutex)
917{
918 u32 latency;
919 int retval;
920
921 if (kfifo_len(kfifo) == 0) {
922 retval = -EAGAIN;
923 goto done;
924 }
925
926 mutex_lock(mutex);
927 retval = kfifo_out(kfifo, &latency, sizeof(latency));
928 if (retval > 0) {
929 seq_printf(s, "%u", latency);
930 retval = 0;
931 }
932 mutex_unlock(mutex);
933done:
934 return retval;
935}
936
937static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused)
938{
939 struct gb_loopback *gb = s->private;
940
941 return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat,
942 &gb->mutex);
943}
944DEFINE_SHOW_ATTRIBUTE(gb_loopback_dbgfs_latency);
945
946#define DEBUGFS_NAMELEN 32
947
948static int gb_loopback_probe(struct gb_bundle *bundle,
949 const struct greybus_bundle_id *id)
950{
951 struct greybus_descriptor_cport *cport_desc;
952 struct gb_connection *connection;
953 struct gb_loopback *gb;
954 struct device *dev;
955 int retval;
956 char name[DEBUGFS_NAMELEN];
957 unsigned long flags;
958
959 if (bundle->num_cports != 1)
960 return -ENODEV;
961
962 cport_desc = &bundle->cport_desc[0];
963 if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK)
964 return -ENODEV;
965
966 gb = kzalloc_obj(*gb);
967 if (!gb)
968 return -ENOMEM;
969
970 connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
971 gb_loopback_request_handler);
972 if (IS_ERR(connection)) {
973 retval = PTR_ERR(connection);
974 goto out_kzalloc;
975 }
976
977 gb->connection = connection;
978 greybus_set_drvdata(bundle, gb);
979
980 init_waitqueue_head(&gb->wq);
981 init_waitqueue_head(&gb->wq_completion);
982 atomic_set(&gb->outstanding_operations, 0);
983 gb_loopback_reset_stats(gb);
984
985 /* Reported values to user-space for min/max timeouts */
986 gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN);
987 gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX);
988
989 if (!gb_dev.count) {
990 /* Calculate maximum payload */
991 gb_dev.size_max = gb_operation_get_payload_size_max(connection);
992 if (gb_dev.size_max <=
993 sizeof(struct gb_loopback_transfer_request)) {
994 retval = -EINVAL;
995 goto out_connection_destroy;
996 }
997 gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request);
998 }
999
1000 /* Create per-connection sysfs and debugfs data-points */
1001 snprintf(name, sizeof(name), "raw_latency_%s",
1002 dev_name(&connection->bundle->dev));
1003 gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb,
1004 &gb_loopback_dbgfs_latency_fops);
1005
1006 gb->id = ida_alloc(&loopback_ida, GFP_KERNEL);
1007 if (gb->id < 0) {
1008 retval = gb->id;
1009 goto out_debugfs_remove;
1010 }
1011
1012 retval = gb_connection_enable(connection);
1013 if (retval)
1014 goto out_ida_remove;
1015
1016 dev = device_create_with_groups(&loopback_class,
1017 &connection->bundle->dev,
1018 MKDEV(0, 0), gb, loopback_groups,
1019 "gb_loopback%d", gb->id);
1020 if (IS_ERR(dev)) {
1021 retval = PTR_ERR(dev);
1022 goto out_connection_disable;
1023 }
1024 gb->dev = dev;
1025
1026 /* Allocate kfifo */
1027 if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32),
1028 GFP_KERNEL)) {
1029 retval = -ENOMEM;
1030 goto out_conn;
1031 }
1032 /* Fork worker thread */
1033 mutex_init(&gb->mutex);
1034 gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback");
1035 if (IS_ERR(gb->task)) {
1036 retval = PTR_ERR(gb->task);
1037 goto out_kfifo;
1038 }
1039
1040 spin_lock_irqsave(&gb_dev.lock, flags);
1041 gb_dev.count++;
1042 spin_unlock_irqrestore(&gb_dev.lock, flags);
1043
1044 gb_connection_latency_tag_enable(connection);
1045
1046 gb_pm_runtime_put_autosuspend(bundle);
1047
1048 return 0;
1049
1050out_kfifo:
1051 kfifo_free(&gb->kfifo_lat);
1052out_conn:
1053 device_unregister(dev);
1054out_connection_disable:
1055 gb_connection_disable(connection);
1056out_ida_remove:
1057 ida_free(&loopback_ida, gb->id);
1058out_debugfs_remove:
1059 debugfs_remove(gb->file);
1060out_connection_destroy:
1061 gb_connection_destroy(connection);
1062out_kzalloc:
1063 kfree(gb);
1064
1065 return retval;
1066}
1067
1068static void gb_loopback_disconnect(struct gb_bundle *bundle)
1069{
1070 struct gb_loopback *gb = greybus_get_drvdata(bundle);
1071 unsigned long flags;
1072 int ret;
1073
1074 ret = gb_pm_runtime_get_sync(bundle);
1075 if (ret)
1076 gb_pm_runtime_get_noresume(bundle);
1077
1078 gb_connection_disable(gb->connection);
1079
1080 if (!IS_ERR_OR_NULL(gb->task))
1081 kthread_stop(gb->task);
1082
1083 kfifo_free(&gb->kfifo_lat);
1084 gb_connection_latency_tag_disable(gb->connection);
1085 debugfs_remove(gb->file);
1086
1087 /*
1088 * FIXME: gb_loopback_async_wait_all() is redundant now, as connection
1089 * is disabled at the beginning and so we can't have any more
1090 * incoming/outgoing requests.
1091 */
1092 gb_loopback_async_wait_all(gb);
1093
1094 spin_lock_irqsave(&gb_dev.lock, flags);
1095 gb_dev.count--;
1096 spin_unlock_irqrestore(&gb_dev.lock, flags);
1097
1098 device_unregister(gb->dev);
1099 ida_free(&loopback_ida, gb->id);
1100
1101 gb_connection_destroy(gb->connection);
1102 kfree(gb);
1103}
1104
1105static const struct greybus_bundle_id gb_loopback_id_table[] = {
1106 { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) },
1107 { }
1108};
1109MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table);
1110
1111static struct greybus_driver gb_loopback_driver = {
1112 .name = "loopback",
1113 .probe = gb_loopback_probe,
1114 .disconnect = gb_loopback_disconnect,
1115 .id_table = gb_loopback_id_table,
1116};
1117
1118static int loopback_init(void)
1119{
1120 int retval;
1121
1122 spin_lock_init(&gb_dev.lock);
1123 gb_dev.root = debugfs_create_dir("gb_loopback", NULL);
1124
1125 retval = class_register(&loopback_class);
1126 if (retval)
1127 goto err;
1128
1129 retval = greybus_register(&gb_loopback_driver);
1130 if (retval)
1131 goto err_unregister;
1132
1133 return 0;
1134
1135err_unregister:
1136 class_unregister(&loopback_class);
1137err:
1138 debugfs_remove_recursive(gb_dev.root);
1139 return retval;
1140}
1141module_init(loopback_init);
1142
1143static void __exit loopback_exit(void)
1144{
1145 debugfs_remove_recursive(gb_dev.root);
1146 greybus_deregister(&gb_loopback_driver);
1147 class_unregister(&loopback_class);
1148 ida_destroy(&loopback_ida);
1149}
1150module_exit(loopback_exit);
1151
1152MODULE_DESCRIPTION("Loopback bridge driver for the Greybus loopback module");
1153MODULE_LICENSE("GPL v2");