| 1 | |
| 2 | |
| 3 | * SSH packet transport layer.
|
| 4 | *
|
| 5 | * Copyright (C) 2019-2022 Maximilian Luz <[email protected]>
|
| 6 |
|
| 7 |
|
| 8 | #include <linux/unaligned.h>
|
| 9 | #include <linux/atomic.h>
|
| 10 | #include <linux/error-injection.h>
|
| 11 | #include <linux/jiffies.h>
|
| 12 | #include <linux/kfifo.h>
|
| 13 | #include <linux/kref.h>
|
| 14 | #include <linux/kthread.h>
|
| 15 | #include <linux/ktime.h>
|
| 16 | #include <linux/limits.h>
|
| 17 | #include <linux/list.h>
|
| 18 | #include <linux/lockdep.h>
|
| 19 | #include <linux/serdev.h>
|
| 20 | #include <linux/slab.h>
|
| 21 | #include <linux/spinlock.h>
|
| 22 | #include <linux/workqueue.h>
|
| 23 |
|
| 24 | #include <linux/surface_aggregator/serial_hub.h>
|
| 25 |
|
| 26 | #include "ssh_msgb.h"
|
| 27 | #include "ssh_packet_layer.h"
|
| 28 | #include "ssh_parser.h"
|
| 29 |
|
| 30 | #include "trace.h"
|
| 31 |
|
| 32 | |
| 33 | * To simplify reasoning about the code below, we define a few concepts. The
|
| 34 | * system below is similar to a state-machine for packets, however, there are
|
| 35 | * too many states to explicitly write them down. To (somewhat) manage the
|
| 36 | * states and packets we rely on flags, reference counting, and some simple
|
| 37 | * concepts. State transitions are triggered by actions.
|
| 38 | *
|
| 39 | * >> Actions <<
|
| 40 | *
|
| 41 | * - submit
|
| 42 | * - transmission start (process next item in queue)
|
| 43 | * - transmission finished (guaranteed to never be parallel to transmission
|
| 44 | * start)
|
| 45 | * - ACK received
|
| 46 | * - NAK received (this is equivalent to issuing re-submit for all pending
|
| 47 | * packets)
|
| 48 | * - timeout (this is equivalent to re-issuing a submit or canceling)
|
| 49 | * - cancel (non-pending and pending)
|
| 50 | *
|
| 51 | * >> Data Structures, Packet Ownership, General Overview <<
|
| 52 | *
|
| 53 | * The code below employs two main data structures: The packet queue,
|
| 54 | * containing all packets scheduled for transmission, and the set of pending
|
| 55 | * packets, containing all packets awaiting an ACK.
|
| 56 | *
|
| 57 | * Shared ownership of a packet is controlled via reference counting. Inside
|
| 58 | * the transport system are a total of five packet owners:
|
| 59 | *
|
| 60 | * - the packet queue,
|
| 61 | * - the pending set,
|
| 62 | * - the transmitter thread,
|
| 63 | * - the receiver thread (via ACKing), and
|
| 64 | * - the timeout work item.
|
| 65 | *
|
| 66 | * Normal operation is as follows: The initial reference of the packet is
|
| 67 | * obtained by submitting the packet and queuing it. The receiver thread takes
|
| 68 | * packets from the queue. By doing this, it does not increment the refcount
|
| 69 | * but takes over the reference (removing it from the queue). If the packet is
|
| 70 | * sequenced (i.e. needs to be ACKed by the client), the transmitter thread
|
| 71 | * sets-up the timeout and adds the packet to the pending set before starting
|
| 72 | * to transmit it. As the timeout is handled by a reaper task, no additional
|
| 73 | * reference for it is needed. After the transmit is done, the reference held
|
| 74 | * by the transmitter thread is dropped. If the packet is unsequenced (i.e.
|
| 75 | * does not need an ACK), the packet is completed by the transmitter thread
|
| 76 | * before dropping that reference.
|
| 77 | *
|
| 78 | * On receival of an ACK, the receiver thread removes and obtains the
|
| 79 | * reference to the packet from the pending set. The receiver thread will then
|
| 80 | * complete the packet and drop its reference.
|
| 81 | *
|
| 82 | * On receival of a NAK, the receiver thread re-submits all currently pending
|
| 83 | * packets.
|
| 84 | *
|
| 85 | * Packet timeouts are detected by the timeout reaper. This is a task,
|
| 86 | * scheduled depending on the earliest packet timeout expiration date,
|
| 87 | * checking all currently pending packets if their timeout has expired. If the
|
| 88 | * timeout of a packet has expired, it is re-submitted and the number of tries
|
| 89 | * of this packet is incremented. If this number reaches its limit, the packet
|
| 90 | * will be completed with a failure.
|
| 91 | *
|
| 92 | * On transmission failure (such as repeated packet timeouts), the completion
|
| 93 | * callback is immediately run by on thread on which the error was detected.
|
| 94 | *
|
| 95 | * To ensure that a packet eventually leaves the system it is marked as
|
| 96 | * "locked" directly before it is going to be completed or when it is
|
| 97 | * canceled. Marking a packet as "locked" has the effect that passing and
|
| 98 | * creating new references of the packet is disallowed. This means that the
|
| 99 | * packet cannot be added to the queue, the pending set, and the timeout, or
|
| 100 | * be picked up by the transmitter thread or receiver thread. To remove a
|
| 101 | * packet from the system it has to be marked as locked and subsequently all
|
| 102 | * references from the data structures (queue, pending) have to be removed.
|
| 103 | * References held by threads will eventually be dropped automatically as
|
| 104 | * their execution progresses.
|
| 105 | *
|
| 106 | * Note that the packet completion callback is, in case of success and for a
|
| 107 | * sequenced packet, guaranteed to run on the receiver thread, thus providing
|
| 108 | * a way to reliably identify responses to the packet. The packet completion
|
| 109 | * callback is only run once and it does not indicate that the packet has
|
| 110 | * fully left the system (for this, one should rely on the release method,
|
| 111 | * triggered when the reference count of the packet reaches zero). In case of
|
| 112 | * re-submission (and with somewhat unlikely timing), it may be possible that
|
| 113 | * the packet is being re-transmitted while the completion callback runs.
|
| 114 | * Completion will occur both on success and internal error, as well as when
|
| 115 | * the packet is canceled.
|
| 116 | *
|
| 117 | * >> Flags <<
|
| 118 | *
|
| 119 | * Flags are used to indicate the state and progression of a packet. Some flags
|
| 120 | * have stricter guarantees than other:
|
| 121 | *
|
| 122 | * - locked
|
| 123 | * Indicates if the packet is locked. If the packet is locked, passing and/or
|
| 124 | * creating additional references to the packet is forbidden. The packet thus
|
| 125 | * may not be queued, dequeued, or removed or added to the pending set. Note
|
| 126 | * that the packet state flags may still change (e.g. it may be marked as
|
| 127 | * ACKed, transmitted, ...).
|
| 128 | *
|
| 129 | * - completed
|
| 130 | * Indicates if the packet completion callback has been executed or is about
|
| 131 | * to be executed. This flag is used to ensure that the packet completion
|
| 132 | * callback is only run once.
|
| 133 | *
|
| 134 | * - queued
|
| 135 | * Indicates if a packet is present in the submission queue or not. This flag
|
| 136 | * must only be modified with the queue lock held, and must be coherent to the
|
| 137 | * presence of the packet in the queue.
|
| 138 | *
|
| 139 | * - pending
|
| 140 | * Indicates if a packet is present in the set of pending packets or not.
|
| 141 | * This flag must only be modified with the pending lock held, and must be
|
| 142 | * coherent to the presence of the packet in the pending set.
|
| 143 | *
|
| 144 | * - transmitting
|
| 145 | * Indicates if the packet is currently transmitting. In case of
|
| 146 | * re-transmissions, it is only safe to wait on the "transmitted" completion
|
| 147 | * after this flag has been set. The completion will be set both in success
|
| 148 | * and error case.
|
| 149 | *
|
| 150 | * - transmitted
|
| 151 | * Indicates if the packet has been transmitted. This flag is not cleared by
|
| 152 | * the system, thus it indicates the first transmission only.
|
| 153 | *
|
| 154 | * - acked
|
| 155 | * Indicates if the packet has been acknowledged by the client. There are no
|
| 156 | * other guarantees given. For example, the packet may still be canceled
|
| 157 | * and/or the completion may be triggered an error even though this bit is
|
| 158 | * set. Rely on the status provided to the completion callback instead.
|
| 159 | *
|
| 160 | * - canceled
|
| 161 | * Indicates if the packet has been canceled from the outside. There are no
|
| 162 | * other guarantees given. Specifically, the packet may be completed by
|
| 163 | * another part of the system before the cancellation attempts to complete it.
|
| 164 | *
|
| 165 | * >> General Notes <<
|
| 166 | *
|
| 167 | * - To avoid deadlocks, if both queue and pending locks are required, the
|
| 168 | * pending lock must be acquired before the queue lock.
|
| 169 | *
|
| 170 | * - The packet priority must be accessed only while holding the queue lock.
|
| 171 | *
|
| 172 | * - The packet timestamp must be accessed only while holding the pending
|
| 173 | * lock.
|
| 174 |
|
| 175 |
|
| 176 | |
| 177 | * SSH_PTL_MAX_PACKET_TRIES - Maximum transmission attempts for packet.
|
| 178 | *
|
| 179 | * Maximum number of transmission attempts per sequenced packet in case of
|
| 180 | * time-outs. Must be smaller than 16. If the packet times out after this
|
| 181 | * amount of tries, the packet will be completed with %-ETIMEDOUT as status
|
| 182 | * code.
|
| 183 |
|
| 184 | #define SSH_PTL_MAX_PACKET_TRIES 3
|
| 185 |
|
| 186 | |
| 187 | * SSH_PTL_TX_TIMEOUT - Packet transmission timeout.
|
| 188 | *
|
| 189 | * Timeout in jiffies for packet transmission via the underlying serial
|
| 190 | * device. If transmitting the packet takes longer than this timeout, the
|
| 191 | * packet will be completed with -ETIMEDOUT. It will not be re-submitted.
|
| 192 |
|
| 193 | #define SSH_PTL_TX_TIMEOUT HZ
|
| 194 |
|
| 195 | |
| 196 | * SSH_PTL_PACKET_TIMEOUT - Packet response timeout.
|
| 197 | *
|
| 198 | * Timeout as ktime_t delta for ACKs. If we have not received an ACK in this
|
| 199 | * time-frame after starting transmission, the packet will be re-submitted.
|
| 200 |
|
| 201 | #define SSH_PTL_PACKET_TIMEOUT ms_to_ktime(1000)
|
| 202 |
|
| 203 | |
| 204 | * SSH_PTL_PACKET_TIMEOUT_RESOLUTION - Packet timeout granularity.
|
| 205 | *
|
| 206 | * Time-resolution for timeouts. Should be larger than one jiffy to avoid
|
| 207 | * direct re-scheduling of reaper work_struct.
|
| 208 |
|
| 209 | #define SSH_PTL_PACKET_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50))
|
| 210 |
|
| 211 | |
| 212 | * SSH_PTL_MAX_PENDING - Maximum number of pending packets.
|
| 213 | *
|
| 214 | * Maximum number of sequenced packets concurrently waiting for an ACK.
|
| 215 | * Packets marked as blocking will not be transmitted while this limit is
|
| 216 | * reached.
|
| 217 |
|
| 218 | #define SSH_PTL_MAX_PENDING 1
|
| 219 |
|
| 220 | |
| 221 | * SSH_PTL_RX_BUF_LEN - Evaluation-buffer size in bytes.
|
| 222 |
|
| 223 | #define SSH_PTL_RX_BUF_LEN 4096
|
| 224 |
|
| 225 | |
| 226 | * SSH_PTL_RX_FIFO_LEN - Fifo input-buffer size in bytes.
|
| 227 |
|
| 228 | #define SSH_PTL_RX_FIFO_LEN 4096
|
| 229 |
|
| 230 | #ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION
|
| 231 |
|
| 232 | |
| 233 | * ssh_ptl_should_drop_ack_packet() - Error injection hook to drop ACK packets.
|
| 234 | *
|
| 235 | * Useful to test detection and handling of automated re-transmits by the EC.
|
| 236 | * Specifically of packets that the EC considers not-ACKed but the driver
|
| 237 | * already considers ACKed (due to dropped ACK). In this case, the EC
|
| 238 | * re-transmits the packet-to-be-ACKed and the driver should detect it as
|
| 239 | * duplicate/already handled. Note that the driver should still send an ACK
|
| 240 | * for the re-transmitted packet.
|
| 241 |
|
| 242 | static noinline bool ssh_ptl_should_drop_ack_packet(void)
|
| 243 | {
|
| 244 | return false;
|
| 245 | }
|
| 246 | ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_ack_packet, TRUE);
|
| 247 |
|
| 248 | |
| 249 | * ssh_ptl_should_drop_nak_packet() - Error injection hook to drop NAK packets.
|
| 250 | *
|
| 251 | * Useful to test/force automated (timeout-based) re-transmit by the EC.
|
| 252 | * Specifically, packets that have not reached the driver completely/with valid
|
| 253 | * checksums. Only useful in combination with receival of (injected) bad data.
|
| 254 |
|
| 255 | static noinline bool ssh_ptl_should_drop_nak_packet(void)
|
| 256 | {
|
| 257 | return false;
|
| 258 | }
|
| 259 | ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_nak_packet, TRUE);
|
| 260 |
|
| 261 | |
| 262 | * ssh_ptl_should_drop_dsq_packet() - Error injection hook to drop sequenced
|
| 263 | * data packet.
|
| 264 | *
|
| 265 | * Useful to test re-transmit timeout of the driver. If the data packet has not
|
| 266 | * been ACKed after a certain time, the driver should re-transmit the packet up
|
| 267 | * to limited number of times defined in SSH_PTL_MAX_PACKET_TRIES.
|
| 268 |
|
| 269 | static noinline bool ssh_ptl_should_drop_dsq_packet(void)
|
| 270 | {
|
| 271 | return false;
|
| 272 | }
|
| 273 | ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_dsq_packet, TRUE);
|
| 274 |
|
| 275 | |
| 276 | * ssh_ptl_should_fail_write() - Error injection hook to make
|
| 277 | * serdev_device_write() fail.
|
| 278 | *
|
| 279 | * Hook to simulate errors in serdev_device_write when transmitting packets.
|
| 280 |
|
| 281 | static noinline int ssh_ptl_should_fail_write(void)
|
| 282 | {
|
| 283 | return 0;
|
| 284 | }
|
| 285 | ALLOW_ERROR_INJECTION(ssh_ptl_should_fail_write, ERRNO);
|
| 286 |
|
| 287 | |
| 288 | * ssh_ptl_should_corrupt_tx_data() - Error injection hook to simulate invalid
|
| 289 | * data being sent to the EC.
|
| 290 | *
|
| 291 | * Hook to simulate corrupt/invalid data being sent from host (driver) to EC.
|
| 292 | * Causes the packet data to be actively corrupted by overwriting it with
|
| 293 | * pre-defined values, such that it becomes invalid, causing the EC to respond
|
| 294 | * with a NAK packet. Useful to test handling of NAK packets received by the
|
| 295 | * driver.
|
| 296 |
|
| 297 | static noinline bool ssh_ptl_should_corrupt_tx_data(void)
|
| 298 | {
|
| 299 | return false;
|
| 300 | }
|
| 301 | ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_tx_data, TRUE);
|
| 302 |
|
| 303 | |
| 304 | * ssh_ptl_should_corrupt_rx_syn() - Error injection hook to simulate invalid
|
| 305 | * data being sent by the EC.
|
| 306 | *
|
| 307 | * Hook to simulate invalid SYN bytes, i.e. an invalid start of messages and
|
| 308 | * test handling thereof in the driver.
|
| 309 |
|
| 310 | static noinline bool ssh_ptl_should_corrupt_rx_syn(void)
|
| 311 | {
|
| 312 | return false;
|
| 313 | }
|
| 314 | ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_syn, TRUE);
|
| 315 |
|
| 316 | |
| 317 | * ssh_ptl_should_corrupt_rx_data() - Error injection hook to simulate invalid
|
| 318 | * data being sent by the EC.
|
| 319 | *
|
| 320 | * Hook to simulate invalid data/checksum of the message frame and test handling
|
| 321 | * thereof in the driver.
|
| 322 |
|
| 323 | static noinline bool ssh_ptl_should_corrupt_rx_data(void)
|
| 324 | {
|
| 325 | return false;
|
| 326 | }
|
| 327 | ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_data, TRUE);
|
| 328 |
|
| 329 | static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet)
|
| 330 | {
|
| 331 | if (likely(!ssh_ptl_should_drop_ack_packet()))
|
| 332 | return false;
|
| 333 |
|
| 334 | trace_ssam_ei_tx_drop_ack_packet(packet);
|
| 335 | ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n",
|
| 336 | packet);
|
| 337 |
|
| 338 | return true;
|
| 339 | }
|
| 340 |
|
| 341 | static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet)
|
| 342 | {
|
| 343 | if (likely(!ssh_ptl_should_drop_nak_packet()))
|
| 344 | return false;
|
| 345 |
|
| 346 | trace_ssam_ei_tx_drop_nak_packet(packet);
|
| 347 | ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n",
|
| 348 | packet);
|
| 349 |
|
| 350 | return true;
|
| 351 | }
|
| 352 |
|
| 353 | static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet)
|
| 354 | {
|
| 355 | if (likely(!ssh_ptl_should_drop_dsq_packet()))
|
| 356 | return false;
|
| 357 |
|
| 358 | trace_ssam_ei_tx_drop_dsq_packet(packet);
|
| 359 | ptl_info(packet->ptl,
|
| 360 | "packet error injection: dropping sequenced data packet %p\n",
|
| 361 | packet);
|
| 362 |
|
| 363 | return true;
|
| 364 | }
|
| 365 |
|
| 366 | static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)
|
| 367 | {
|
| 368 |
|
| 369 | if (!packet->data.ptr || !packet->data.len)
|
| 370 | return false;
|
| 371 |
|
| 372 | switch (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)]) {
|
| 373 | case SSH_FRAME_TYPE_ACK:
|
| 374 | return __ssh_ptl_should_drop_ack_packet(packet);
|
| 375 |
|
| 376 | case SSH_FRAME_TYPE_NAK:
|
| 377 | return __ssh_ptl_should_drop_nak_packet(packet);
|
| 378 |
|
| 379 | case SSH_FRAME_TYPE_DATA_SEQ:
|
| 380 | return __ssh_ptl_should_drop_dsq_packet(packet);
|
| 381 |
|
| 382 | default:
|
| 383 | return false;
|
| 384 | }
|
| 385 | }
|
| 386 |
|
| 387 | static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet,
|
| 388 | const unsigned char *buf, size_t count)
|
| 389 | {
|
| 390 | int status;
|
| 391 |
|
| 392 | status = ssh_ptl_should_fail_write();
|
| 393 | if (unlikely(status)) {
|
| 394 | trace_ssam_ei_tx_fail_write(packet, status);
|
| 395 | ptl_info(packet->ptl,
|
| 396 | "packet error injection: simulating transmit error %d, packet %p\n",
|
| 397 | status, packet);
|
| 398 |
|
| 399 | return status;
|
| 400 | }
|
| 401 |
|
| 402 | return serdev_device_write_buf(ptl->serdev, buf, count);
|
| 403 | }
|
| 404 |
|
| 405 | static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)
|
| 406 | {
|
| 407 |
|
| 408 | if (!packet->data.ptr || !packet->data.len)
|
| 409 | return;
|
| 410 |
|
| 411 |
|
| 412 | if (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)] != SSH_FRAME_TYPE_DATA_SEQ)
|
| 413 | return;
|
| 414 |
|
| 415 | if (likely(!ssh_ptl_should_corrupt_tx_data()))
|
| 416 | return;
|
| 417 |
|
| 418 | trace_ssam_ei_tx_corrupt_data(packet);
|
| 419 | ptl_info(packet->ptl,
|
| 420 | "packet error injection: simulating invalid transmit data on packet %p\n",
|
| 421 | packet);
|
| 422 |
|
| 423 | |
| 424 | * NB: The value 0xb3 has been chosen more or less randomly so that it
|
| 425 | * doesn't have any (major) overlap with the SYN bytes (aa 55) and is
|
| 426 | * non-trivial (i.e. non-zero, non-0xff).
|
| 427 |
|
| 428 | memset(packet->data.ptr, 0xb3, packet->data.len);
|
| 429 | }
|
| 430 |
|
| 431 | static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
|
| 432 | struct ssam_span *data)
|
| 433 | {
|
| 434 | struct ssam_span frame;
|
| 435 |
|
| 436 |
|
| 437 | if (!sshp_find_syn(data, &frame))
|
| 438 | return;
|
| 439 |
|
| 440 | if (likely(!ssh_ptl_should_corrupt_rx_syn()))
|
| 441 | return;
|
| 442 |
|
| 443 | trace_ssam_ei_rx_corrupt_syn(data->len);
|
| 444 |
|
| 445 | data->ptr[1] = 0xb3;
|
| 446 | }
|
| 447 |
|
| 448 | static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
|
| 449 | struct ssam_span *frame)
|
| 450 | {
|
| 451 | size_t payload_len, message_len;
|
| 452 | struct ssh_frame *sshf;
|
| 453 |
|
| 454 |
|
| 455 | if (frame->len < SSH_MESSAGE_LENGTH(0))
|
| 456 | return;
|
| 457 |
|
| 458 |
|
| 459 | payload_len = get_unaligned_le16(&frame->ptr[SSH_MSGOFFSET_FRAME(len)]);
|
| 460 | message_len = SSH_MESSAGE_LENGTH(payload_len);
|
| 461 | if (frame->len < message_len)
|
| 462 | return;
|
| 463 |
|
| 464 | if (likely(!ssh_ptl_should_corrupt_rx_data()))
|
| 465 | return;
|
| 466 |
|
| 467 | sshf = (struct ssh_frame *)&frame->ptr[SSH_MSGOFFSET_FRAME(type)];
|
| 468 | trace_ssam_ei_rx_corrupt_data(sshf);
|
| 469 |
|
| 470 | |
| 471 | * Flip bits in first byte of payload checksum. This is basically
|
| 472 | * equivalent to a payload/frame data error without us having to worry
|
| 473 | * about (the, arguably pretty small, probability of) accidental
|
| 474 | * checksum collisions.
|
| 475 |
|
| 476 | frame->ptr[frame->len - 2] = ~frame->ptr[frame->len - 2];
|
| 477 | }
|
| 478 |
|
| 479 | #else
|
| 480 |
|
| 481 | static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)
|
| 482 | {
|
| 483 | return false;
|
| 484 | }
|
| 485 |
|
| 486 | static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl,
|
| 487 | struct ssh_packet *packet,
|
| 488 | const unsigned char *buf,
|
| 489 | size_t count)
|
| 490 | {
|
| 491 | return serdev_device_write_buf(ptl->serdev, buf, count);
|
| 492 | }
|
| 493 |
|
| 494 | static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)
|
| 495 | {
|
| 496 | }
|
| 497 |
|
| 498 | static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
|
| 499 | struct ssam_span *data)
|
| 500 | {
|
| 501 | }
|
| 502 |
|
| 503 | static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
|
| 504 | struct ssam_span *frame)
|
| 505 | {
|
| 506 | }
|
| 507 |
|
| 508 | #endif
|
| 509 |
|
| 510 | static void __ssh_ptl_packet_release(struct kref *kref)
|
| 511 | {
|
| 512 | struct ssh_packet *p = container_of(kref, struct ssh_packet, refcnt);
|
| 513 |
|
| 514 | trace_ssam_packet_release(p);
|
| 515 |
|
| 516 | ptl_dbg_cond(p->ptl, "ptl: releasing packet %p\n", p);
|
| 517 | p->ops->release(p);
|
| 518 | }
|
| 519 |
|
| 520 | |
| 521 | * ssh_packet_get() - Increment reference count of packet.
|
| 522 | * @packet: The packet to increment the reference count of.
|
| 523 | *
|
| 524 | * Increments the reference count of the given packet. See ssh_packet_put()
|
| 525 | * for the counter-part of this function.
|
| 526 | *
|
| 527 | * Return: Returns the packet provided as input.
|
| 528 |
|
| 529 | struct ssh_packet *ssh_packet_get(struct ssh_packet *packet)
|
| 530 | {
|
| 531 | if (packet)
|
| 532 | kref_get(&packet->refcnt);
|
| 533 | return packet;
|
| 534 | }
|
| 535 | EXPORT_SYMBOL_GPL(ssh_packet_get);
|
| 536 |
|
| 537 | |
| 538 | * ssh_packet_put() - Decrement reference count of packet.
|
| 539 | * @packet: The packet to decrement the reference count of.
|
| 540 | *
|
| 541 | * If the reference count reaches zero, the ``release`` callback specified in
|
| 542 | * the packet's &struct ssh_packet_ops, i.e. ``packet->ops->release``, will be
|
| 543 | * called.
|
| 544 | *
|
| 545 | * See ssh_packet_get() for the counter-part of this function.
|
| 546 |
|
| 547 | void ssh_packet_put(struct ssh_packet *packet)
|
| 548 | {
|
| 549 | if (packet)
|
| 550 | kref_put(&packet->refcnt, __ssh_ptl_packet_release);
|
| 551 | }
|
| 552 | EXPORT_SYMBOL_GPL(ssh_packet_put);
|
| 553 |
|
| 554 | static u8 ssh_packet_get_seq(struct ssh_packet *packet)
|
| 555 | {
|
| 556 | return packet->data.ptr[SSH_MSGOFFSET_FRAME(seq)];
|
| 557 | }
|
| 558 |
|
| 559 | |
| 560 | * ssh_packet_init() - Initialize SSH packet.
|
| 561 | * @packet: The packet to initialize.
|
| 562 | * @type: Type-flags of the packet.
|
| 563 | * @priority: Priority of the packet. See SSH_PACKET_PRIORITY() for details.
|
| 564 | * @ops: Packet operations.
|
| 565 | *
|
| 566 | * Initializes the given SSH packet. Sets the transmission buffer pointer to
|
| 567 | * %NULL and the transmission buffer length to zero. For data-type packets,
|
| 568 | * this buffer has to be set separately via ssh_packet_set_data() before
|
| 569 | * submission, and must contain a valid SSH message, i.e. frame with optional
|
| 570 | * payload of any type.
|
| 571 |
|
| 572 | void ssh_packet_init(struct ssh_packet *packet, unsigned long type,
|
| 573 | u8 priority, const struct ssh_packet_ops *ops)
|
| 574 | {
|
| 575 | kref_init(&packet->refcnt);
|
| 576 |
|
| 577 | packet->ptl = NULL;
|
| 578 | INIT_LIST_HEAD(&packet->queue_node);
|
| 579 | INIT_LIST_HEAD(&packet->pending_node);
|
| 580 |
|
| 581 | packet->state = type & SSH_PACKET_FLAGS_TY_MASK;
|
| 582 | packet->priority = priority;
|
| 583 | packet->timestamp = KTIME_MAX;
|
| 584 |
|
| 585 | packet->data.ptr = NULL;
|
| 586 | packet->data.len = 0;
|
| 587 |
|
| 588 | packet->ops = ops;
|
| 589 | }
|
| 590 |
|
| 591 | static struct kmem_cache *ssh_ctrl_packet_cache;
|
| 592 |
|
| 593 | |
| 594 | * ssh_ctrl_packet_cache_init() - Initialize the control packet cache.
|
| 595 |
|
| 596 | int ssh_ctrl_packet_cache_init(void)
|
| 597 | {
|
| 598 | const unsigned int size = sizeof(struct ssh_packet) + SSH_MSG_LEN_CTRL;
|
| 599 | const unsigned int align = __alignof__(struct ssh_packet);
|
| 600 | struct kmem_cache *cache;
|
| 601 |
|
| 602 | cache = kmem_cache_create("ssam_ctrl_packet", size, align, 0, NULL);
|
| 603 | if (!cache)
|
| 604 | return -ENOMEM;
|
| 605 |
|
| 606 | ssh_ctrl_packet_cache = cache;
|
| 607 | return 0;
|
| 608 | }
|
| 609 |
|
| 610 | |
| 611 | * ssh_ctrl_packet_cache_destroy() - Deinitialize the control packet cache.
|
| 612 |
|
| 613 | void ssh_ctrl_packet_cache_destroy(void)
|
| 614 | {
|
| 615 | kmem_cache_destroy(ssh_ctrl_packet_cache);
|
| 616 | ssh_ctrl_packet_cache = NULL;
|
| 617 | }
|
| 618 |
|
| 619 | |
| 620 | * ssh_ctrl_packet_alloc() - Allocate packet from control packet cache.
|
| 621 | * @packet: Where the pointer to the newly allocated packet should be stored.
|
| 622 | * @buffer: The buffer corresponding to this packet.
|
| 623 | * @flags: Flags used for allocation.
|
| 624 | *
|
| 625 | * Allocates a packet and corresponding transport buffer from the control
|
| 626 | * packet cache. Sets the packet's buffer reference to the allocated buffer.
|
| 627 | * The packet must be freed via ssh_ctrl_packet_free(), which will also free
|
| 628 | * the corresponding buffer. The corresponding buffer must not be freed
|
| 629 | * separately. Intended to be used with %ssh_ptl_ctrl_packet_ops as packet
|
| 630 | * operations.
|
| 631 | *
|
| 632 | * Return: Returns zero on success, %-ENOMEM if the allocation failed.
|
| 633 |
|
| 634 | static int ssh_ctrl_packet_alloc(struct ssh_packet **packet,
|
| 635 | struct ssam_span *buffer, gfp_t flags)
|
| 636 | {
|
| 637 | *packet = kmem_cache_alloc(ssh_ctrl_packet_cache, flags);
|
| 638 | if (!*packet)
|
| 639 | return -ENOMEM;
|
| 640 |
|
| 641 | buffer->ptr = (u8 *)(*packet + 1);
|
| 642 | buffer->len = SSH_MSG_LEN_CTRL;
|
| 643 |
|
| 644 | trace_ssam_ctrl_packet_alloc(*packet, buffer->len);
|
| 645 | return 0;
|
| 646 | }
|
| 647 |
|
| 648 | |
| 649 | * ssh_ctrl_packet_free() - Free packet allocated from control packet cache.
|
| 650 | * @p: The packet to free.
|
| 651 |
|
| 652 | static void ssh_ctrl_packet_free(struct ssh_packet *p)
|
| 653 | {
|
| 654 | trace_ssam_ctrl_packet_free(p);
|
| 655 | kmem_cache_free(ssh_ctrl_packet_cache, p);
|
| 656 | }
|
| 657 |
|
| 658 | static const struct ssh_packet_ops ssh_ptl_ctrl_packet_ops = {
|
| 659 | .complete = NULL,
|
| 660 | .release = ssh_ctrl_packet_free,
|
| 661 | };
|
| 662 |
|
| 663 | static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now,
|
| 664 | ktime_t expires)
|
| 665 | {
|
| 666 | unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now));
|
| 667 | ktime_t aexp = ktime_add(expires, SSH_PTL_PACKET_TIMEOUT_RESOLUTION);
|
| 668 |
|
| 669 | spin_lock(&ptl->rtx_timeout.lock);
|
| 670 |
|
| 671 |
|
| 672 | if (ktime_before(aexp, ptl->rtx_timeout.expires)) {
|
| 673 | ptl->rtx_timeout.expires = expires;
|
| 674 | mod_delayed_work(system_percpu_wq, &ptl->rtx_timeout.reaper, delta);
|
| 675 | }
|
| 676 |
|
| 677 | spin_unlock(&ptl->rtx_timeout.lock);
|
| 678 | }
|
| 679 |
|
| 680 |
|
| 681 | static void ssh_packet_next_try(struct ssh_packet *p)
|
| 682 | {
|
| 683 | u8 base = ssh_packet_priority_get_base(p->priority);
|
| 684 | u8 try = ssh_packet_priority_get_try(p->priority);
|
| 685 |
|
| 686 | lockdep_assert_held(&p->ptl->queue.lock);
|
| 687 |
|
| 688 | |
| 689 | * Ensure that we write the priority in one go via WRITE_ONCE() so we
|
| 690 | * can access it via READ_ONCE() for tracing. Note that other access
|
| 691 | * is guarded by the queue lock, so no need to use READ_ONCE() there.
|
| 692 |
|
| 693 | WRITE_ONCE(p->priority, __SSH_PACKET_PRIORITY(base, try + 1));
|
| 694 | }
|
| 695 |
|
| 696 |
|
| 697 | static struct list_head *__ssh_ptl_queue_find_entrypoint(struct ssh_packet *p)
|
| 698 | {
|
| 699 | struct list_head *head;
|
| 700 | struct ssh_packet *q;
|
| 701 |
|
| 702 | lockdep_assert_held(&p->ptl->queue.lock);
|
| 703 |
|
| 704 | |
| 705 | * We generally assume that there are less control (ACK/NAK) packets
|
| 706 | * and re-submitted data packets as there are normal data packets (at
|
| 707 | * least in situations in which many packets are queued; if there
|
| 708 | * aren't many packets queued the decision on how to iterate should be
|
| 709 | * basically irrelevant; the number of control/data packets is more or
|
| 710 | * less limited via the maximum number of pending packets). Thus, when
|
| 711 | * inserting a control or re-submitted data packet, (determined by
|
| 712 | * their priority), we search from front to back. Normal data packets
|
| 713 | * are, usually queued directly at the tail of the queue, so for those
|
| 714 | * search from back to front.
|
| 715 |
|
| 716 |
|
| 717 | if (p->priority > SSH_PACKET_PRIORITY(DATA, 0)) {
|
| 718 | list_for_each(head, &p->ptl->queue.head) {
|
| 719 | q = list_entry(head, struct ssh_packet, queue_node);
|
| 720 |
|
| 721 | if (q->priority < p->priority)
|
| 722 | break;
|
| 723 | }
|
| 724 | } else {
|
| 725 | list_for_each_prev(head, &p->ptl->queue.head) {
|
| 726 | q = list_entry(head, struct ssh_packet, queue_node);
|
| 727 |
|
| 728 | if (q->priority >= p->priority) {
|
| 729 | head = head->next;
|
| 730 | break;
|
| 731 | }
|
| 732 | }
|
| 733 | }
|
| 734 |
|
| 735 | return head;
|
| 736 | }
|
| 737 |
|
| 738 |
|
| 739 | static int __ssh_ptl_queue_push(struct ssh_packet *packet)
|
| 740 | {
|
| 741 | struct ssh_ptl *ptl = packet->ptl;
|
| 742 | struct list_head *head;
|
| 743 |
|
| 744 | lockdep_assert_held(&ptl->queue.lock);
|
| 745 |
|
| 746 | if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
|
| 747 | return -ESHUTDOWN;
|
| 748 |
|
| 749 |
|
| 750 | if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state))
|
| 751 | return -EINVAL;
|
| 752 |
|
| 753 |
|
| 754 | if (test_and_set_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state))
|
| 755 | return -EALREADY;
|
| 756 |
|
| 757 | head = __ssh_ptl_queue_find_entrypoint(packet);
|
| 758 |
|
| 759 | list_add_tail(&ssh_packet_get(packet)->queue_node, head);
|
| 760 | return 0;
|
| 761 | }
|
| 762 |
|
| 763 | static int ssh_ptl_queue_push(struct ssh_packet *packet)
|
| 764 | {
|
| 765 | int status;
|
| 766 |
|
| 767 | spin_lock(&packet->ptl->queue.lock);
|
| 768 | status = __ssh_ptl_queue_push(packet);
|
| 769 | spin_unlock(&packet->ptl->queue.lock);
|
| 770 |
|
| 771 | return status;
|
| 772 | }
|
| 773 |
|
| 774 | static void ssh_ptl_queue_remove(struct ssh_packet *packet)
|
| 775 | {
|
| 776 | struct ssh_ptl *ptl = packet->ptl;
|
| 777 |
|
| 778 | spin_lock(&ptl->queue.lock);
|
| 779 |
|
| 780 | if (!test_and_clear_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) {
|
| 781 | spin_unlock(&ptl->queue.lock);
|
| 782 | return;
|
| 783 | }
|
| 784 |
|
| 785 | list_del(&packet->queue_node);
|
| 786 |
|
| 787 | spin_unlock(&ptl->queue.lock);
|
| 788 | ssh_packet_put(packet);
|
| 789 | }
|
| 790 |
|
| 791 | static void ssh_ptl_pending_push(struct ssh_packet *p)
|
| 792 | {
|
| 793 | struct ssh_ptl *ptl = p->ptl;
|
| 794 | const ktime_t timestamp = ktime_get_coarse_boottime();
|
| 795 | const ktime_t timeout = ptl->rtx_timeout.timeout;
|
| 796 |
|
| 797 | |
| 798 | * Note: We can get the time for the timestamp before acquiring the
|
| 799 | * lock as this is the only place we're setting it and this function
|
| 800 | * is called only from the transmitter thread. Thus it is not possible
|
| 801 | * to overwrite the timestamp with an outdated value below.
|
| 802 |
|
| 803 |
|
| 804 | spin_lock(&ptl->pending.lock);
|
| 805 |
|
| 806 |
|
| 807 | if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state)) {
|
| 808 | spin_unlock(&ptl->pending.lock);
|
| 809 | return;
|
| 810 | }
|
| 811 |
|
| 812 | |
| 813 | * On re-submission, the packet has already been added the pending
|
| 814 | * set. We still need to update the timestamp as the packet timeout is
|
| 815 | * reset for each (re-)submission.
|
| 816 |
|
| 817 | p->timestamp = timestamp;
|
| 818 |
|
| 819 |
|
| 820 | if (!test_and_set_bit(SSH_PACKET_SF_PENDING_BIT, &p->state)) {
|
| 821 | atomic_inc(&ptl->pending.count);
|
| 822 | list_add_tail(&ssh_packet_get(p)->pending_node, &ptl->pending.head);
|
| 823 | }
|
| 824 |
|
| 825 | spin_unlock(&ptl->pending.lock);
|
| 826 |
|
| 827 |
|
| 828 | ssh_ptl_timeout_reaper_mod(ptl, timestamp, timestamp + timeout);
|
| 829 | }
|
| 830 |
|
| 831 | static void ssh_ptl_pending_remove(struct ssh_packet *packet)
|
| 832 | {
|
| 833 | struct ssh_ptl *ptl = packet->ptl;
|
| 834 |
|
| 835 | spin_lock(&ptl->pending.lock);
|
| 836 |
|
| 837 | if (!test_and_clear_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) {
|
| 838 | spin_unlock(&ptl->pending.lock);
|
| 839 | return;
|
| 840 | }
|
| 841 |
|
| 842 | list_del(&packet->pending_node);
|
| 843 | atomic_dec(&ptl->pending.count);
|
| 844 |
|
| 845 | spin_unlock(&ptl->pending.lock);
|
| 846 |
|
| 847 | ssh_packet_put(packet);
|
| 848 | }
|
| 849 |
|
| 850 |
|
| 851 | static void __ssh_ptl_complete(struct ssh_packet *p, int status)
|
| 852 | {
|
| 853 | struct ssh_ptl *ptl = READ_ONCE(p->ptl);
|
| 854 |
|
| 855 | trace_ssam_packet_complete(p, status);
|
| 856 | ptl_dbg_cond(ptl, "ptl: completing packet %p (status: %d)\n", p, status);
|
| 857 |
|
| 858 | if (p->ops->complete)
|
| 859 | p->ops->complete(p, status);
|
| 860 | }
|
| 861 |
|
| 862 | static void ssh_ptl_remove_and_complete(struct ssh_packet *p, int status)
|
| 863 | {
|
| 864 | |
| 865 | * A call to this function should in general be preceded by
|
| 866 | * set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->flags) to avoid re-adding the
|
| 867 | * packet to the structures it's going to be removed from.
|
| 868 | *
|
| 869 | * The set_bit call does not need explicit memory barriers as the
|
| 870 | * implicit barrier of the test_and_set_bit() call below ensure that the
|
| 871 | * flag is visible before we actually attempt to remove the packet.
|
| 872 |
|
| 873 |
|
| 874 | if (test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
|
| 875 | return;
|
| 876 |
|
| 877 | ssh_ptl_queue_remove(p);
|
| 878 | ssh_ptl_pending_remove(p);
|
| 879 |
|
| 880 | __ssh_ptl_complete(p, status);
|
| 881 | }
|
| 882 |
|
| 883 | static bool ssh_ptl_tx_can_process(struct ssh_packet *packet)
|
| 884 | {
|
| 885 | struct ssh_ptl *ptl = packet->ptl;
|
| 886 |
|
| 887 | if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &packet->state))
|
| 888 | return !atomic_read(&ptl->pending.count);
|
| 889 |
|
| 890 |
|
| 891 | if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &packet->state))
|
| 892 | return true;
|
| 893 |
|
| 894 |
|
| 895 | if (test_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state))
|
| 896 | return true;
|
| 897 |
|
| 898 |
|
| 899 | return atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING;
|
| 900 | }
|
| 901 |
|
| 902 | static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl)
|
| 903 | {
|
| 904 | struct ssh_packet *packet = ERR_PTR(-ENOENT);
|
| 905 | struct ssh_packet *p, *n;
|
| 906 |
|
| 907 | spin_lock(&ptl->queue.lock);
|
| 908 | list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
|
| 909 | |
| 910 | * If we are canceling or completing this packet, ignore it.
|
| 911 | * It's going to be removed from this queue shortly.
|
| 912 |
|
| 913 | if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
|
| 914 | continue;
|
| 915 |
|
| 916 | |
| 917 | * Packets should be ordered non-blocking/to-be-resent first.
|
| 918 | * If we cannot process this packet, assume that we can't
|
| 919 | * process any following packet either and abort.
|
| 920 |
|
| 921 | if (!ssh_ptl_tx_can_process(p)) {
|
| 922 | packet = ERR_PTR(-EBUSY);
|
| 923 | break;
|
| 924 | }
|
| 925 |
|
| 926 | |
| 927 | * We are allowed to change the state now. Remove it from the
|
| 928 | * queue and mark it as being transmitted.
|
| 929 |
|
| 930 |
|
| 931 | list_del(&p->queue_node);
|
| 932 |
|
| 933 | set_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &p->state);
|
| 934 |
|
| 935 | smp_mb__before_atomic();
|
| 936 | clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);
|
| 937 |
|
| 938 | |
| 939 | * Update number of tries. This directly influences the
|
| 940 | * priority in case the packet is re-submitted (e.g. via
|
| 941 | * timeout/NAK). Note that all reads and writes to the
|
| 942 | * priority after the first submission are guarded by the
|
| 943 | * queue lock.
|
| 944 |
|
| 945 | ssh_packet_next_try(p);
|
| 946 |
|
| 947 | packet = p;
|
| 948 | break;
|
| 949 | }
|
| 950 | spin_unlock(&ptl->queue.lock);
|
| 951 |
|
| 952 | return packet;
|
| 953 | }
|
| 954 |
|
| 955 | static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl)
|
| 956 | {
|
| 957 | struct ssh_packet *p;
|
| 958 |
|
| 959 | p = ssh_ptl_tx_pop(ptl);
|
| 960 | if (IS_ERR(p))
|
| 961 | return p;
|
| 962 |
|
| 963 | if (test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state)) {
|
| 964 | ptl_dbg(ptl, "ptl: transmitting sequenced packet %p\n", p);
|
| 965 | ssh_ptl_pending_push(p);
|
| 966 | } else {
|
| 967 | ptl_dbg(ptl, "ptl: transmitting non-sequenced packet %p\n", p);
|
| 968 | }
|
| 969 |
|
| 970 | return p;
|
| 971 | }
|
| 972 |
|
| 973 | static void ssh_ptl_tx_compl_success(struct ssh_packet *packet)
|
| 974 | {
|
| 975 | struct ssh_ptl *ptl = packet->ptl;
|
| 976 |
|
| 977 | ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet);
|
| 978 |
|
| 979 |
|
| 980 | set_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state);
|
| 981 |
|
| 982 | smp_mb__before_atomic();
|
| 983 | clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);
|
| 984 |
|
| 985 |
|
| 986 | if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &packet->state)) {
|
| 987 | set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);
|
| 988 | ssh_ptl_remove_and_complete(packet, 0);
|
| 989 | }
|
| 990 |
|
| 991 | |
| 992 | * Notify that a packet transmission has finished. In general we're only
|
| 993 | * waiting for one packet (if any), so wake_up_all should be fine.
|
| 994 |
|
| 995 | wake_up_all(&ptl->tx.packet_wq);
|
| 996 | }
|
| 997 |
|
| 998 | static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status)
|
| 999 | {
|
| 1000 |
|
| 1001 | set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);
|
| 1002 |
|
| 1003 | smp_mb__before_atomic();
|
| 1004 | clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);
|
| 1005 |
|
| 1006 | ptl_err(packet->ptl, "ptl: transmission error: %d\n", status);
|
| 1007 | ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet);
|
| 1008 |
|
| 1009 | ssh_ptl_remove_and_complete(packet, status);
|
| 1010 |
|
| 1011 | |
| 1012 | * Notify that a packet transmission has finished. In general we're only
|
| 1013 | * waiting for one packet (if any), so wake_up_all should be fine.
|
| 1014 |
|
| 1015 | wake_up_all(&packet->ptl->tx.packet_wq);
|
| 1016 | }
|
| 1017 |
|
| 1018 | static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl)
|
| 1019 | {
|
| 1020 | int status;
|
| 1021 |
|
| 1022 | status = wait_for_completion_interruptible(&ptl->tx.thread_cplt_pkt);
|
| 1023 | reinit_completion(&ptl->tx.thread_cplt_pkt);
|
| 1024 |
|
| 1025 | |
| 1026 | * Ensure completion is cleared before continuing to avoid lost update
|
| 1027 | * problems.
|
| 1028 |
|
| 1029 | smp_mb__after_atomic();
|
| 1030 |
|
| 1031 | return status;
|
| 1032 | }
|
| 1033 |
|
| 1034 | static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout)
|
| 1035 | {
|
| 1036 | long status;
|
| 1037 |
|
| 1038 | status = wait_for_completion_interruptible_timeout(&ptl->tx.thread_cplt_tx,
|
| 1039 | timeout);
|
| 1040 | reinit_completion(&ptl->tx.thread_cplt_tx);
|
| 1041 |
|
| 1042 | |
| 1043 | * Ensure completion is cleared before continuing to avoid lost update
|
| 1044 | * problems.
|
| 1045 |
|
| 1046 | smp_mb__after_atomic();
|
| 1047 |
|
| 1048 | return status;
|
| 1049 | }
|
| 1050 |
|
| 1051 | static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet)
|
| 1052 | {
|
| 1053 | long timeout = SSH_PTL_TX_TIMEOUT;
|
| 1054 | size_t offset = 0;
|
| 1055 |
|
| 1056 |
|
| 1057 | if (unlikely(!packet->data.ptr))
|
| 1058 | return 0;
|
| 1059 |
|
| 1060 |
|
| 1061 | if (ssh_ptl_should_drop_packet(packet))
|
| 1062 | return 0;
|
| 1063 |
|
| 1064 |
|
| 1065 | ssh_ptl_tx_inject_invalid_data(packet);
|
| 1066 |
|
| 1067 | ptl_dbg(ptl, "tx: sending data (length: %zu)\n", packet->data.len);
|
| 1068 | print_hex_dump_debug("tx: ", DUMP_PREFIX_OFFSET, 16, 1,
|
| 1069 | packet->data.ptr, packet->data.len, false);
|
| 1070 |
|
| 1071 | do {
|
| 1072 | ssize_t status, len;
|
| 1073 | u8 *buf;
|
| 1074 |
|
| 1075 | buf = packet->data.ptr + offset;
|
| 1076 | len = packet->data.len - offset;
|
| 1077 |
|
| 1078 | status = ssh_ptl_write_buf(ptl, packet, buf, len);
|
| 1079 | if (status < 0)
|
| 1080 | return status;
|
| 1081 |
|
| 1082 | if (status == len)
|
| 1083 | return 0;
|
| 1084 |
|
| 1085 | offset += status;
|
| 1086 |
|
| 1087 | timeout = ssh_ptl_tx_wait_transfer(ptl, timeout);
|
| 1088 | if (kthread_should_stop() || !atomic_read(&ptl->tx.running))
|
| 1089 | return -ESHUTDOWN;
|
| 1090 |
|
| 1091 | if (timeout < 0)
|
| 1092 | return -EINTR;
|
| 1093 |
|
| 1094 | if (timeout == 0)
|
| 1095 | return -ETIMEDOUT;
|
| 1096 | } while (true);
|
| 1097 | }
|
| 1098 |
|
| 1099 | static int ssh_ptl_tx_threadfn(void *data)
|
| 1100 | {
|
| 1101 | struct ssh_ptl *ptl = data;
|
| 1102 |
|
| 1103 | while (!kthread_should_stop() && atomic_read(&ptl->tx.running)) {
|
| 1104 | struct ssh_packet *packet;
|
| 1105 | int status;
|
| 1106 |
|
| 1107 |
|
| 1108 | packet = ssh_ptl_tx_next(ptl);
|
| 1109 |
|
| 1110 |
|
| 1111 | if (IS_ERR(packet)) {
|
| 1112 | ssh_ptl_tx_wait_packet(ptl);
|
| 1113 | continue;
|
| 1114 | }
|
| 1115 |
|
| 1116 |
|
| 1117 | status = ssh_ptl_tx_packet(ptl, packet);
|
| 1118 | if (status)
|
| 1119 | ssh_ptl_tx_compl_error(packet, status);
|
| 1120 | else
|
| 1121 | ssh_ptl_tx_compl_success(packet);
|
| 1122 |
|
| 1123 | ssh_packet_put(packet);
|
| 1124 | }
|
| 1125 |
|
| 1126 | return 0;
|
| 1127 | }
|
| 1128 |
|
| 1129 | |
| 1130 | * ssh_ptl_tx_wakeup_packet() - Wake up packet transmitter thread for new
|
| 1131 | * packet.
|
| 1132 | * @ptl: The packet transport layer.
|
| 1133 | *
|
| 1134 | * Wakes up the packet transmitter thread, notifying it that a new packet has
|
| 1135 | * arrived and is ready for transfer. If the packet transport layer has been
|
| 1136 | * shut down, calls to this function will be ignored.
|
| 1137 |
|
| 1138 | static void ssh_ptl_tx_wakeup_packet(struct ssh_ptl *ptl)
|
| 1139 | {
|
| 1140 | if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
|
| 1141 | return;
|
| 1142 |
|
| 1143 | complete(&ptl->tx.thread_cplt_pkt);
|
| 1144 | }
|
| 1145 |
|
| 1146 | |
| 1147 | * ssh_ptl_tx_start() - Start packet transmitter thread.
|
| 1148 | * @ptl: The packet transport layer.
|
| 1149 | *
|
| 1150 | * Return: Returns zero on success, a negative error code on failure.
|
| 1151 |
|
| 1152 | int ssh_ptl_tx_start(struct ssh_ptl *ptl)
|
| 1153 | {
|
| 1154 | atomic_set_release(&ptl->tx.running, 1);
|
| 1155 |
|
| 1156 | ptl->tx.thread = kthread_run(ssh_ptl_tx_threadfn, ptl, "ssam_serial_hub-tx");
|
| 1157 | if (IS_ERR(ptl->tx.thread))
|
| 1158 | return PTR_ERR(ptl->tx.thread);
|
| 1159 |
|
| 1160 | return 0;
|
| 1161 | }
|
| 1162 |
|
| 1163 | |
| 1164 | * ssh_ptl_tx_stop() - Stop packet transmitter thread.
|
| 1165 | * @ptl: The packet transport layer.
|
| 1166 | *
|
| 1167 | * Return: Returns zero on success, a negative error code on failure.
|
| 1168 |
|
| 1169 | int ssh_ptl_tx_stop(struct ssh_ptl *ptl)
|
| 1170 | {
|
| 1171 | int status = 0;
|
| 1172 |
|
| 1173 | if (!IS_ERR_OR_NULL(ptl->tx.thread)) {
|
| 1174 |
|
| 1175 | atomic_set_release(&ptl->tx.running, 0);
|
| 1176 |
|
| 1177 | |
| 1178 | * Wake up thread in case it is paused. Do not use wakeup
|
| 1179 | * helpers as this may be called when the shutdown bit has
|
| 1180 | * already been set.
|
| 1181 |
|
| 1182 | complete(&ptl->tx.thread_cplt_pkt);
|
| 1183 | complete(&ptl->tx.thread_cplt_tx);
|
| 1184 |
|
| 1185 |
|
| 1186 | status = kthread_stop(ptl->tx.thread);
|
| 1187 | ptl->tx.thread = NULL;
|
| 1188 | }
|
| 1189 |
|
| 1190 | return status;
|
| 1191 | }
|
| 1192 |
|
| 1193 | static struct ssh_packet *ssh_ptl_ack_pop(struct ssh_ptl *ptl, u8 seq_id)
|
| 1194 | {
|
| 1195 | struct ssh_packet *packet = ERR_PTR(-ENOENT);
|
| 1196 | struct ssh_packet *p, *n;
|
| 1197 |
|
| 1198 | spin_lock(&ptl->pending.lock);
|
| 1199 | list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
|
| 1200 | |
| 1201 | * We generally expect packets to be in order, so first packet
|
| 1202 | * to be added to pending is first to be sent, is first to be
|
| 1203 | * ACKed.
|
| 1204 |
|
| 1205 | if (unlikely(ssh_packet_get_seq(p) != seq_id))
|
| 1206 | continue;
|
| 1207 |
|
| 1208 | |
| 1209 | * In case we receive an ACK while handling a transmission
|
| 1210 | * error completion. The packet will be removed shortly.
|
| 1211 |
|
| 1212 | if (unlikely(test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {
|
| 1213 | packet = ERR_PTR(-EPERM);
|
| 1214 | break;
|
| 1215 | }
|
| 1216 |
|
| 1217 | |
| 1218 | * Mark the packet as ACKed and remove it from pending by
|
| 1219 | * removing its node and decrementing the pending counter.
|
| 1220 |
|
| 1221 | set_bit(SSH_PACKET_SF_ACKED_BIT, &p->state);
|
| 1222 |
|
| 1223 | smp_mb__before_atomic();
|
| 1224 | clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
|
| 1225 |
|
| 1226 | atomic_dec(&ptl->pending.count);
|
| 1227 | list_del(&p->pending_node);
|
| 1228 | packet = p;
|
| 1229 |
|
| 1230 | break;
|
| 1231 | }
|
| 1232 | spin_unlock(&ptl->pending.lock);
|
| 1233 |
|
| 1234 | return packet;
|
| 1235 | }
|
| 1236 |
|
| 1237 | static void ssh_ptl_wait_until_transmitted(struct ssh_packet *packet)
|
| 1238 | {
|
| 1239 | wait_event(packet->ptl->tx.packet_wq,
|
| 1240 | test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state) ||
|
| 1241 | test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state));
|
| 1242 | }
|
| 1243 |
|
| 1244 | static void ssh_ptl_acknowledge(struct ssh_ptl *ptl, u8 seq)
|
| 1245 | {
|
| 1246 | struct ssh_packet *p;
|
| 1247 |
|
| 1248 | p = ssh_ptl_ack_pop(ptl, seq);
|
| 1249 | if (IS_ERR(p)) {
|
| 1250 | if (PTR_ERR(p) == -ENOENT) {
|
| 1251 | |
| 1252 | * The packet has not been found in the set of pending
|
| 1253 | * packets.
|
| 1254 |
|
| 1255 | ptl_warn(ptl, "ptl: received ACK for non-pending packet\n");
|
| 1256 | } else {
|
| 1257 | |
| 1258 | * The packet is pending, but we are not allowed to take
|
| 1259 | * it because it has been locked.
|
| 1260 |
|
| 1261 | WARN_ON(PTR_ERR(p) != -EPERM);
|
| 1262 | }
|
| 1263 | return;
|
| 1264 | }
|
| 1265 |
|
| 1266 | ptl_dbg(ptl, "ptl: received ACK for packet %p\n", p);
|
| 1267 |
|
| 1268 | |
| 1269 | * It is possible that the packet has been transmitted, but the state
|
| 1270 | * has not been updated from "transmitting" to "transmitted" yet.
|
| 1271 | * In that case, we need to wait for this transition to occur in order
|
| 1272 | * to determine between success or failure.
|
| 1273 | *
|
| 1274 | * On transmission failure, the packet will be locked after this call.
|
| 1275 | * On success, the transmitted bit will be set.
|
| 1276 |
|
| 1277 | ssh_ptl_wait_until_transmitted(p);
|
| 1278 |
|
| 1279 | |
| 1280 | * The packet will already be locked in case of a transmission error or
|
| 1281 | * cancellation. Let the transmitter or cancellation issuer complete the
|
| 1282 | * packet.
|
| 1283 |
|
| 1284 | if (unlikely(test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {
|
| 1285 | if (unlikely(!test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &p->state)))
|
| 1286 | ptl_err(ptl, "ptl: received ACK before packet had been fully transmitted\n");
|
| 1287 |
|
| 1288 | ssh_packet_put(p);
|
| 1289 | return;
|
| 1290 | }
|
| 1291 |
|
| 1292 | ssh_ptl_remove_and_complete(p, 0);
|
| 1293 | ssh_packet_put(p);
|
| 1294 |
|
| 1295 | if (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING)
|
| 1296 | ssh_ptl_tx_wakeup_packet(ptl);
|
| 1297 | }
|
| 1298 |
|
| 1299 | |
| 1300 | * ssh_ptl_submit() - Submit a packet to the transport layer.
|
| 1301 | * @ptl: The packet transport layer to submit the packet to.
|
| 1302 | * @p: The packet to submit.
|
| 1303 | *
|
| 1304 | * Submits a new packet to the transport layer, queuing it to be sent. This
|
| 1305 | * function should not be used for re-submission.
|
| 1306 | *
|
| 1307 | * Return: Returns zero on success, %-EINVAL if a packet field is invalid or
|
| 1308 | * the packet has been canceled prior to submission, %-EALREADY if the packet
|
| 1309 | * has already been submitted, or %-ESHUTDOWN if the packet transport layer
|
| 1310 | * has been shut down.
|
| 1311 |
|
| 1312 | int ssh_ptl_submit(struct ssh_ptl *ptl, struct ssh_packet *p)
|
| 1313 | {
|
| 1314 | struct ssh_ptl *ptl_old;
|
| 1315 | int status;
|
| 1316 |
|
| 1317 | trace_ssam_packet_submit(p);
|
| 1318 |
|
| 1319 |
|
| 1320 | if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &p->state)) {
|
| 1321 | if (p->data.ptr || test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state))
|
| 1322 | return -EINVAL;
|
| 1323 | } else if (!p->data.ptr) {
|
| 1324 | return -EINVAL;
|
| 1325 | }
|
| 1326 |
|
| 1327 | |
| 1328 | * The ptl reference only gets set on or before the first submission.
|
| 1329 | * After the first submission, it has to be read-only.
|
| 1330 | *
|
| 1331 | * Note that ptl may already be set from upper-layer request
|
| 1332 | * submission, thus we cannot expect it to be NULL.
|
| 1333 |
|
| 1334 | ptl_old = READ_ONCE(p->ptl);
|
| 1335 | if (!ptl_old)
|
| 1336 | WRITE_ONCE(p->ptl, ptl);
|
| 1337 | else if (WARN_ON(ptl_old != ptl))
|
| 1338 | return -EALREADY;
|
| 1339 |
|
| 1340 | status = ssh_ptl_queue_push(p);
|
| 1341 | if (status)
|
| 1342 | return status;
|
| 1343 |
|
| 1344 | if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &p->state) ||
|
| 1345 | (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING))
|
| 1346 | ssh_ptl_tx_wakeup_packet(ptl);
|
| 1347 |
|
| 1348 | return 0;
|
| 1349 | }
|
| 1350 |
|
| 1351 | |
| 1352 | * __ssh_ptl_resubmit() - Re-submit a packet to the transport layer.
|
| 1353 | * @packet: The packet to re-submit.
|
| 1354 | *
|
| 1355 | * Re-submits the given packet: Checks if it can be re-submitted and queues it
|
| 1356 | * if it can, resetting the packet timestamp in the process. Must be called
|
| 1357 | * with the pending lock held.
|
| 1358 | *
|
| 1359 | * Return: Returns %-ECANCELED if the packet has exceeded its number of tries,
|
| 1360 | * %-EINVAL if the packet has been locked, %-EALREADY if the packet is already
|
| 1361 | * on the queue, and %-ESHUTDOWN if the transmission layer has been shut down.
|
| 1362 |
|
| 1363 | static int __ssh_ptl_resubmit(struct ssh_packet *packet)
|
| 1364 | {
|
| 1365 | int status;
|
| 1366 | u8 try;
|
| 1367 |
|
| 1368 | lockdep_assert_held(&packet->ptl->pending.lock);
|
| 1369 |
|
| 1370 | trace_ssam_packet_resubmit(packet);
|
| 1371 |
|
| 1372 | spin_lock(&packet->ptl->queue.lock);
|
| 1373 |
|
| 1374 |
|
| 1375 | try = ssh_packet_priority_get_try(packet->priority);
|
| 1376 | if (try >= SSH_PTL_MAX_PACKET_TRIES) {
|
| 1377 | spin_unlock(&packet->ptl->queue.lock);
|
| 1378 | return -ECANCELED;
|
| 1379 | }
|
| 1380 |
|
| 1381 | status = __ssh_ptl_queue_push(packet);
|
| 1382 | if (status) {
|
| 1383 | |
| 1384 | * An error here indicates that the packet has either already
|
| 1385 | * been queued, been locked, or the transport layer is being
|
| 1386 | * shut down. In all cases: Ignore the error.
|
| 1387 |
|
| 1388 | spin_unlock(&packet->ptl->queue.lock);
|
| 1389 | return status;
|
| 1390 | }
|
| 1391 |
|
| 1392 | packet->timestamp = KTIME_MAX;
|
| 1393 |
|
| 1394 | spin_unlock(&packet->ptl->queue.lock);
|
| 1395 | return 0;
|
| 1396 | }
|
| 1397 |
|
| 1398 | static void ssh_ptl_resubmit_pending(struct ssh_ptl *ptl)
|
| 1399 | {
|
| 1400 | struct ssh_packet *p;
|
| 1401 | bool resub = false;
|
| 1402 |
|
| 1403 | |
| 1404 | * Note: We deliberately do not remove/attempt to cancel and complete
|
| 1405 | * packets that are out of tires in this function. The packet will be
|
| 1406 | * eventually canceled and completed by the timeout. Removing the packet
|
| 1407 | * here could lead to overly eager cancellation if the packet has not
|
| 1408 | * been re-transmitted yet but the tries-counter already updated (i.e
|
| 1409 | * ssh_ptl_tx_next() removed the packet from the queue and updated the
|
| 1410 | * counter, but re-transmission for the last try has not actually
|
| 1411 | * started yet).
|
| 1412 |
|
| 1413 |
|
| 1414 | spin_lock(&ptl->pending.lock);
|
| 1415 |
|
| 1416 |
|
| 1417 | list_for_each_entry(p, &ptl->pending.head, pending_node) {
|
| 1418 | |
| 1419 | * Re-submission fails if the packet is out of tries, has been
|
| 1420 | * locked, is already queued, or the layer is being shut down.
|
| 1421 | * No need to re-schedule tx-thread in those cases.
|
| 1422 |
|
| 1423 | if (!__ssh_ptl_resubmit(p))
|
| 1424 | resub = true;
|
| 1425 | }
|
| 1426 |
|
| 1427 | spin_unlock(&ptl->pending.lock);
|
| 1428 |
|
| 1429 | if (resub)
|
| 1430 | ssh_ptl_tx_wakeup_packet(ptl);
|
| 1431 | }
|
| 1432 |
|
| 1433 | |
| 1434 | * ssh_ptl_cancel() - Cancel a packet.
|
| 1435 | * @p: The packet to cancel.
|
| 1436 | *
|
| 1437 | * Cancels a packet. There are no guarantees on when completion and release
|
| 1438 | * callbacks will be called. This may occur during execution of this function
|
| 1439 | * or may occur at any point later.
|
| 1440 | *
|
| 1441 | * Note that it is not guaranteed that the packet will actually be canceled if
|
| 1442 | * the packet is concurrently completed by another process. The only guarantee
|
| 1443 | * of this function is that the packet will be completed (with success,
|
| 1444 | * failure, or cancellation) and released from the transport layer in a
|
| 1445 | * reasonable time-frame.
|
| 1446 | *
|
| 1447 | * May be called before the packet has been submitted, in which case any later
|
| 1448 | * packet submission fails.
|
| 1449 |
|
| 1450 | void ssh_ptl_cancel(struct ssh_packet *p)
|
| 1451 | {
|
| 1452 | if (test_and_set_bit(SSH_PACKET_SF_CANCELED_BIT, &p->state))
|
| 1453 | return;
|
| 1454 |
|
| 1455 | trace_ssam_packet_cancel(p);
|
| 1456 |
|
| 1457 | |
| 1458 | * Lock packet and commit with memory barrier. If this packet has
|
| 1459 | * already been locked, it's going to be removed and completed by
|
| 1460 | * another party, which should have precedence.
|
| 1461 |
|
| 1462 | if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
|
| 1463 | return;
|
| 1464 |
|
| 1465 | |
| 1466 | * By marking the packet as locked and employing the implicit memory
|
| 1467 | * barrier of test_and_set_bit, we have guaranteed that, at this point,
|
| 1468 | * the packet cannot be added to the queue any more.
|
| 1469 | *
|
| 1470 | * In case the packet has never been submitted, packet->ptl is NULL. If
|
| 1471 | * the packet is currently being submitted, packet->ptl may be NULL or
|
| 1472 | * non-NULL. Due marking the packet as locked above and committing with
|
| 1473 | * the memory barrier, we have guaranteed that, if packet->ptl is NULL,
|
| 1474 | * the packet will never be added to the queue. If packet->ptl is
|
| 1475 | * non-NULL, we don't have any guarantees.
|
| 1476 |
|
| 1477 |
|
| 1478 | if (READ_ONCE(p->ptl)) {
|
| 1479 | ssh_ptl_remove_and_complete(p, -ECANCELED);
|
| 1480 |
|
| 1481 | if (atomic_read(&p->ptl->pending.count) < SSH_PTL_MAX_PENDING)
|
| 1482 | ssh_ptl_tx_wakeup_packet(p->ptl);
|
| 1483 |
|
| 1484 | } else if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {
|
| 1485 | __ssh_ptl_complete(p, -ECANCELED);
|
| 1486 | }
|
| 1487 | }
|
| 1488 |
|
| 1489 |
|
| 1490 | static ktime_t ssh_packet_get_expiration(struct ssh_packet *p, ktime_t timeout)
|
| 1491 | {
|
| 1492 | lockdep_assert_held(&p->ptl->pending.lock);
|
| 1493 |
|
| 1494 | if (p->timestamp != KTIME_MAX)
|
| 1495 | return ktime_add(p->timestamp, timeout);
|
| 1496 | else
|
| 1497 | return KTIME_MAX;
|
| 1498 | }
|
| 1499 |
|
| 1500 | static void ssh_ptl_timeout_reap(struct work_struct *work)
|
| 1501 | {
|
| 1502 | struct ssh_ptl *ptl = to_ssh_ptl(work, rtx_timeout.reaper.work);
|
| 1503 | struct ssh_packet *p, *n;
|
| 1504 | LIST_HEAD(claimed);
|
| 1505 | ktime_t now = ktime_get_coarse_boottime();
|
| 1506 | ktime_t timeout = ptl->rtx_timeout.timeout;
|
| 1507 | ktime_t next = KTIME_MAX;
|
| 1508 | bool resub = false;
|
| 1509 | int status;
|
| 1510 |
|
| 1511 | trace_ssam_ptl_timeout_reap(atomic_read(&ptl->pending.count));
|
| 1512 |
|
| 1513 | |
| 1514 | * Mark reaper as "not pending". This is done before checking any
|
| 1515 | * packets to avoid lost-update type problems.
|
| 1516 |
|
| 1517 | spin_lock(&ptl->rtx_timeout.lock);
|
| 1518 | ptl->rtx_timeout.expires = KTIME_MAX;
|
| 1519 | spin_unlock(&ptl->rtx_timeout.lock);
|
| 1520 |
|
| 1521 | spin_lock(&ptl->pending.lock);
|
| 1522 |
|
| 1523 | list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
|
| 1524 | ktime_t expires = ssh_packet_get_expiration(p, timeout);
|
| 1525 |
|
| 1526 | |
| 1527 | * Check if the timeout hasn't expired yet. Find out next
|
| 1528 | * expiration date to be handled after this run.
|
| 1529 |
|
| 1530 | if (ktime_after(expires, now)) {
|
| 1531 | next = ktime_before(expires, next) ? expires : next;
|
| 1532 | continue;
|
| 1533 | }
|
| 1534 |
|
| 1535 | trace_ssam_packet_timeout(p);
|
| 1536 |
|
| 1537 | status = __ssh_ptl_resubmit(p);
|
| 1538 |
|
| 1539 | |
| 1540 | * Re-submission fails if the packet is out of tries, has been
|
| 1541 | * locked, is already queued, or the layer is being shut down.
|
| 1542 | * No need to re-schedule tx-thread in those cases.
|
| 1543 |
|
| 1544 | if (!status)
|
| 1545 | resub = true;
|
| 1546 |
|
| 1547 |
|
| 1548 | if (status != -ECANCELED)
|
| 1549 | continue;
|
| 1550 |
|
| 1551 |
|
| 1552 |
|
| 1553 | |
| 1554 | * If someone else has locked the packet already, don't use it
|
| 1555 | * and let the other party complete it.
|
| 1556 |
|
| 1557 | if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))
|
| 1558 | continue;
|
| 1559 |
|
| 1560 | |
| 1561 | * We have now marked the packet as locked. Thus it cannot be
|
| 1562 | * added to the pending list again after we've removed it here.
|
| 1563 | * We can therefore re-use the pending_node of this packet
|
| 1564 | * temporarily.
|
| 1565 |
|
| 1566 |
|
| 1567 | clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
|
| 1568 |
|
| 1569 | atomic_dec(&ptl->pending.count);
|
| 1570 | list_move_tail(&p->pending_node, &claimed);
|
| 1571 | }
|
| 1572 |
|
| 1573 | spin_unlock(&ptl->pending.lock);
|
| 1574 |
|
| 1575 |
|
| 1576 | list_for_each_entry_safe(p, n, &claimed, pending_node) {
|
| 1577 | if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {
|
| 1578 | ssh_ptl_queue_remove(p);
|
| 1579 | __ssh_ptl_complete(p, -ETIMEDOUT);
|
| 1580 | }
|
| 1581 |
|
| 1582 | |
| 1583 | * Drop the reference we've obtained by removing it from
|
| 1584 | * the pending set.
|
| 1585 |
|
| 1586 | list_del(&p->pending_node);
|
| 1587 | ssh_packet_put(p);
|
| 1588 | }
|
| 1589 |
|
| 1590 |
|
| 1591 | next = max(next, ktime_add(now, SSH_PTL_PACKET_TIMEOUT_RESOLUTION));
|
| 1592 | if (next != KTIME_MAX)
|
| 1593 | ssh_ptl_timeout_reaper_mod(ptl, now, next);
|
| 1594 |
|
| 1595 | if (resub)
|
| 1596 | ssh_ptl_tx_wakeup_packet(ptl);
|
| 1597 | }
|
| 1598 |
|
| 1599 | static bool ssh_ptl_rx_retransmit_check(struct ssh_ptl *ptl, const struct ssh_frame *frame)
|
| 1600 | {
|
| 1601 | int i;
|
| 1602 |
|
| 1603 | |
| 1604 | * Ignore unsequenced packets. On some devices (notably Surface Pro 9),
|
| 1605 | * unsequenced events will always be sent with SEQ=0x00. Attempting to
|
| 1606 | * detect retransmission would thus just block all events.
|
| 1607 | *
|
| 1608 | * While sequence numbers would also allow detection of retransmitted
|
| 1609 | * packets in unsequenced communication, they have only ever been used
|
| 1610 | * to cover edge-cases in sequenced transmission. In particular, the
|
| 1611 | * only instance of packets being retransmitted (that we are aware of)
|
| 1612 | * is due to an ACK timeout. As this does not happen in unsequenced
|
| 1613 | * communication, skip the retransmission check for those packets
|
| 1614 | * entirely.
|
| 1615 |
|
| 1616 | if (frame->type == SSH_FRAME_TYPE_DATA_NSQ)
|
| 1617 | return false;
|
| 1618 |
|
| 1619 | |
| 1620 | * Check if SEQ has been seen recently (i.e. packet was
|
| 1621 | * re-transmitted and we should ignore it).
|
| 1622 |
|
| 1623 | for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++) {
|
| 1624 | if (likely(ptl->rx.blocked.seqs[i] != frame->seq))
|
| 1625 | continue;
|
| 1626 |
|
| 1627 | ptl_dbg(ptl, "ptl: ignoring repeated data packet\n");
|
| 1628 | return true;
|
| 1629 | }
|
| 1630 |
|
| 1631 |
|
| 1632 | ptl->rx.blocked.seqs[ptl->rx.blocked.offset] = frame->seq;
|
| 1633 | ptl->rx.blocked.offset = (ptl->rx.blocked.offset + 1)
|
| 1634 | % ARRAY_SIZE(ptl->rx.blocked.seqs);
|
| 1635 |
|
| 1636 | return false;
|
| 1637 | }
|
| 1638 |
|
| 1639 | static void ssh_ptl_rx_dataframe(struct ssh_ptl *ptl,
|
| 1640 | const struct ssh_frame *frame,
|
| 1641 | const struct ssam_span *payload)
|
| 1642 | {
|
| 1643 | if (ssh_ptl_rx_retransmit_check(ptl, frame))
|
| 1644 | return;
|
| 1645 |
|
| 1646 | ptl->ops.data_received(ptl, payload);
|
| 1647 | }
|
| 1648 |
|
| 1649 | static void ssh_ptl_send_ack(struct ssh_ptl *ptl, u8 seq)
|
| 1650 | {
|
| 1651 | struct ssh_packet *packet;
|
| 1652 | struct ssam_span buf;
|
| 1653 | struct msgbuf msgb;
|
| 1654 | int status;
|
| 1655 |
|
| 1656 | status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);
|
| 1657 | if (status) {
|
| 1658 | ptl_err(ptl, "ptl: failed to allocate ACK packet\n");
|
| 1659 | return;
|
| 1660 | }
|
| 1661 |
|
| 1662 | ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(ACK, 0),
|
| 1663 | &ssh_ptl_ctrl_packet_ops);
|
| 1664 |
|
| 1665 | msgb_init(&msgb, buf.ptr, buf.len);
|
| 1666 | msgb_push_ack(&msgb, seq);
|
| 1667 | ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));
|
| 1668 |
|
| 1669 | ssh_ptl_submit(ptl, packet);
|
| 1670 | ssh_packet_put(packet);
|
| 1671 | }
|
| 1672 |
|
| 1673 | static void ssh_ptl_send_nak(struct ssh_ptl *ptl)
|
| 1674 | {
|
| 1675 | struct ssh_packet *packet;
|
| 1676 | struct ssam_span buf;
|
| 1677 | struct msgbuf msgb;
|
| 1678 | int status;
|
| 1679 |
|
| 1680 | status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);
|
| 1681 | if (status) {
|
| 1682 | ptl_err(ptl, "ptl: failed to allocate NAK packet\n");
|
| 1683 | return;
|
| 1684 | }
|
| 1685 |
|
| 1686 | ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(NAK, 0),
|
| 1687 | &ssh_ptl_ctrl_packet_ops);
|
| 1688 |
|
| 1689 | msgb_init(&msgb, buf.ptr, buf.len);
|
| 1690 | msgb_push_nak(&msgb);
|
| 1691 | ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));
|
| 1692 |
|
| 1693 | ssh_ptl_submit(ptl, packet);
|
| 1694 | ssh_packet_put(packet);
|
| 1695 | }
|
| 1696 |
|
| 1697 | static size_t ssh_ptl_rx_eval(struct ssh_ptl *ptl, struct ssam_span *source)
|
| 1698 | {
|
| 1699 | struct ssh_frame *frame;
|
| 1700 | struct ssam_span payload;
|
| 1701 | struct ssam_span aligned;
|
| 1702 | bool syn_found;
|
| 1703 | int status;
|
| 1704 |
|
| 1705 |
|
| 1706 | ssh_ptl_rx_inject_invalid_syn(ptl, source);
|
| 1707 |
|
| 1708 |
|
| 1709 | syn_found = sshp_find_syn(source, &aligned);
|
| 1710 |
|
| 1711 | if (unlikely(aligned.ptr != source->ptr)) {
|
| 1712 | |
| 1713 | * We expect aligned.ptr == source->ptr. If this is not the
|
| 1714 | * case, then aligned.ptr > source->ptr and we've encountered
|
| 1715 | * some unexpected data where we'd expect the start of a new
|
| 1716 | * message (i.e. the SYN sequence).
|
| 1717 | *
|
| 1718 | * This can happen when a CRC check for the previous message
|
| 1719 | * failed and we start actively searching for the next one
|
| 1720 | * (via the call to sshp_find_syn() above), or the first bytes
|
| 1721 | * of a message got dropped or corrupted.
|
| 1722 | *
|
| 1723 | * In any case, we issue a warning, send a NAK to the EC to
|
| 1724 | * request re-transmission of any data we haven't acknowledged
|
| 1725 | * yet, and finally, skip everything up to the next SYN
|
| 1726 | * sequence.
|
| 1727 |
|
| 1728 |
|
| 1729 | ptl_warn(ptl, "rx: parser: invalid start of frame, skipping\n");
|
| 1730 |
|
| 1731 | |
| 1732 | * Notes:
|
| 1733 | * - This might send multiple NAKs in case the communication
|
| 1734 | * starts with an invalid SYN and is broken down into multiple
|
| 1735 | * pieces. This should generally be handled fine, we just
|
| 1736 | * might receive duplicate data in this case, which is
|
| 1737 | * detected when handling data frames.
|
| 1738 | * - This path will also be executed on invalid CRCs: When an
|
| 1739 | * invalid CRC is encountered, the code below will skip data
|
| 1740 | * until directly after the SYN. This causes the search for
|
| 1741 | * the next SYN, which is generally not placed directly after
|
| 1742 | * the last one.
|
| 1743 | *
|
| 1744 | * Open question: Should we send this in case of invalid
|
| 1745 | * payload CRCs if the frame-type is non-sequential (current
|
| 1746 | * implementation) or should we drop that frame without
|
| 1747 | * telling the EC?
|
| 1748 |
|
| 1749 | ssh_ptl_send_nak(ptl);
|
| 1750 | }
|
| 1751 |
|
| 1752 | if (unlikely(!syn_found))
|
| 1753 | return aligned.ptr - source->ptr;
|
| 1754 |
|
| 1755 |
|
| 1756 | ssh_ptl_rx_inject_invalid_data(ptl, &aligned);
|
| 1757 |
|
| 1758 |
|
| 1759 | status = sshp_parse_frame(&ptl->serdev->dev, &aligned, &frame, &payload,
|
| 1760 | SSH_PTL_RX_BUF_LEN);
|
| 1761 | if (status)
|
| 1762 | return aligned.ptr - source->ptr + sizeof(u16);
|
| 1763 | if (!frame)
|
| 1764 | return aligned.ptr - source->ptr;
|
| 1765 |
|
| 1766 | trace_ssam_rx_frame_received(frame);
|
| 1767 |
|
| 1768 | switch (frame->type) {
|
| 1769 | case SSH_FRAME_TYPE_ACK:
|
| 1770 | ssh_ptl_acknowledge(ptl, frame->seq);
|
| 1771 | break;
|
| 1772 |
|
| 1773 | case SSH_FRAME_TYPE_NAK:
|
| 1774 | ssh_ptl_resubmit_pending(ptl);
|
| 1775 | break;
|
| 1776 |
|
| 1777 | case SSH_FRAME_TYPE_DATA_SEQ:
|
| 1778 | ssh_ptl_send_ack(ptl, frame->seq);
|
| 1779 | fallthrough;
|
| 1780 |
|
| 1781 | case SSH_FRAME_TYPE_DATA_NSQ:
|
| 1782 | ssh_ptl_rx_dataframe(ptl, frame, &payload);
|
| 1783 | break;
|
| 1784 |
|
| 1785 | default:
|
| 1786 | ptl_warn(ptl, "ptl: received frame with unknown type %#04x\n",
|
| 1787 | frame->type);
|
| 1788 | break;
|
| 1789 | }
|
| 1790 |
|
| 1791 | return aligned.ptr - source->ptr + SSH_MESSAGE_LENGTH(payload.len);
|
| 1792 | }
|
| 1793 |
|
| 1794 | static int ssh_ptl_rx_threadfn(void *data)
|
| 1795 | {
|
| 1796 | struct ssh_ptl *ptl = data;
|
| 1797 |
|
| 1798 | while (true) {
|
| 1799 | struct ssam_span span;
|
| 1800 | size_t offs = 0;
|
| 1801 | size_t n;
|
| 1802 |
|
| 1803 | wait_event_interruptible(ptl->rx.wq,
|
| 1804 | !kfifo_is_empty(&ptl->rx.fifo) ||
|
| 1805 | kthread_should_stop());
|
| 1806 | if (kthread_should_stop())
|
| 1807 | break;
|
| 1808 |
|
| 1809 |
|
| 1810 | n = sshp_buf_read_from_fifo(&ptl->rx.buf, &ptl->rx.fifo);
|
| 1811 |
|
| 1812 | ptl_dbg(ptl, "rx: received data (size: %zu)\n", n);
|
| 1813 | print_hex_dump_debug("rx: ", DUMP_PREFIX_OFFSET, 16, 1,
|
| 1814 | ptl->rx.buf.ptr + ptl->rx.buf.len - n,
|
| 1815 | n, false);
|
| 1816 |
|
| 1817 |
|
| 1818 | while (offs < ptl->rx.buf.len) {
|
| 1819 | sshp_buf_span_from(&ptl->rx.buf, offs, &span);
|
| 1820 | n = ssh_ptl_rx_eval(ptl, &span);
|
| 1821 | if (n == 0)
|
| 1822 | break;
|
| 1823 |
|
| 1824 | offs += n;
|
| 1825 | }
|
| 1826 |
|
| 1827 |
|
| 1828 | sshp_buf_drop(&ptl->rx.buf, offs);
|
| 1829 | }
|
| 1830 |
|
| 1831 | return 0;
|
| 1832 | }
|
| 1833 |
|
| 1834 | static void ssh_ptl_rx_wakeup(struct ssh_ptl *ptl)
|
| 1835 | {
|
| 1836 | wake_up(&ptl->rx.wq);
|
| 1837 | }
|
| 1838 |
|
| 1839 | |
| 1840 | * ssh_ptl_rx_start() - Start packet transport layer receiver thread.
|
| 1841 | * @ptl: The packet transport layer.
|
| 1842 | *
|
| 1843 | * Return: Returns zero on success, a negative error code on failure.
|
| 1844 |
|
| 1845 | int ssh_ptl_rx_start(struct ssh_ptl *ptl)
|
| 1846 | {
|
| 1847 | if (ptl->rx.thread)
|
| 1848 | return 0;
|
| 1849 |
|
| 1850 | ptl->rx.thread = kthread_run(ssh_ptl_rx_threadfn, ptl,
|
| 1851 | "ssam_serial_hub-rx");
|
| 1852 | if (IS_ERR(ptl->rx.thread))
|
| 1853 | return PTR_ERR(ptl->rx.thread);
|
| 1854 |
|
| 1855 | return 0;
|
| 1856 | }
|
| 1857 |
|
| 1858 | |
| 1859 | * ssh_ptl_rx_stop() - Stop packet transport layer receiver thread.
|
| 1860 | * @ptl: The packet transport layer.
|
| 1861 | *
|
| 1862 | * Return: Returns zero on success, a negative error code on failure.
|
| 1863 |
|
| 1864 | int ssh_ptl_rx_stop(struct ssh_ptl *ptl)
|
| 1865 | {
|
| 1866 | int status = 0;
|
| 1867 |
|
| 1868 | if (ptl->rx.thread) {
|
| 1869 | status = kthread_stop(ptl->rx.thread);
|
| 1870 | ptl->rx.thread = NULL;
|
| 1871 | }
|
| 1872 |
|
| 1873 | return status;
|
| 1874 | }
|
| 1875 |
|
| 1876 | |
| 1877 | * ssh_ptl_rx_rcvbuf() - Push data from lower-layer transport to the packet
|
| 1878 | * layer.
|
| 1879 | * @ptl: The packet transport layer.
|
| 1880 | * @buf: Pointer to the data to push to the layer.
|
| 1881 | * @n: Size of the data to push to the layer, in bytes.
|
| 1882 | *
|
| 1883 | * Pushes data from a lower-layer transport to the receiver fifo buffer of the
|
| 1884 | * packet layer and notifies the receiver thread. Calls to this function are
|
| 1885 | * ignored once the packet layer has been shut down.
|
| 1886 | *
|
| 1887 | * Return: Returns the number of bytes transferred (positive or zero) on
|
| 1888 | * success. Returns %-ESHUTDOWN if the packet layer has been shut down.
|
| 1889 |
|
| 1890 | ssize_t ssh_ptl_rx_rcvbuf(struct ssh_ptl *ptl, const u8 *buf, size_t n)
|
| 1891 | {
|
| 1892 | size_t used;
|
| 1893 |
|
| 1894 | if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
|
| 1895 | return -ESHUTDOWN;
|
| 1896 |
|
| 1897 | used = kfifo_in(&ptl->rx.fifo, buf, n);
|
| 1898 | if (used)
|
| 1899 | ssh_ptl_rx_wakeup(ptl);
|
| 1900 |
|
| 1901 | return used;
|
| 1902 | }
|
| 1903 |
|
| 1904 | |
| 1905 | * ssh_ptl_shutdown() - Shut down the packet transport layer.
|
| 1906 | * @ptl: The packet transport layer.
|
| 1907 | *
|
| 1908 | * Shuts down the packet transport layer, removing and canceling all queued
|
| 1909 | * and pending packets. Packets canceled by this operation will be completed
|
| 1910 | * with %-ESHUTDOWN as status. Receiver and transmitter threads will be
|
| 1911 | * stopped.
|
| 1912 | *
|
| 1913 | * As a result of this function, the transport layer will be marked as shut
|
| 1914 | * down. Submission of packets after the transport layer has been shut down
|
| 1915 | * will fail with %-ESHUTDOWN.
|
| 1916 |
|
| 1917 | void ssh_ptl_shutdown(struct ssh_ptl *ptl)
|
| 1918 | {
|
| 1919 | LIST_HEAD(complete_q);
|
| 1920 | LIST_HEAD(complete_p);
|
| 1921 | struct ssh_packet *p, *n;
|
| 1922 | int status;
|
| 1923 |
|
| 1924 |
|
| 1925 | set_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state);
|
| 1926 | |
| 1927 | * Ensure that the layer gets marked as shut-down before actually
|
| 1928 | * stopping it. In combination with the check in ssh_ptl_queue_push(),
|
| 1929 | * this guarantees that no new packets can be added and all already
|
| 1930 | * queued packets are properly canceled. In combination with the check
|
| 1931 | * in ssh_ptl_rx_rcvbuf(), this guarantees that received data is
|
| 1932 | * properly cut off.
|
| 1933 |
|
| 1934 | smp_mb__after_atomic();
|
| 1935 |
|
| 1936 | status = ssh_ptl_rx_stop(ptl);
|
| 1937 | if (status)
|
| 1938 | ptl_err(ptl, "ptl: failed to stop receiver thread\n");
|
| 1939 |
|
| 1940 | status = ssh_ptl_tx_stop(ptl);
|
| 1941 | if (status)
|
| 1942 | ptl_err(ptl, "ptl: failed to stop transmitter thread\n");
|
| 1943 |
|
| 1944 | cancel_delayed_work_sync(&ptl->rtx_timeout.reaper);
|
| 1945 |
|
| 1946 | |
| 1947 | * At this point, all threads have been stopped. This means that the
|
| 1948 | * only references to packets from inside the system are in the queue
|
| 1949 | * and pending set.
|
| 1950 | *
|
| 1951 | * Note: We still need locks here because someone could still be
|
| 1952 | * canceling packets.
|
| 1953 | *
|
| 1954 | * Note 2: We can re-use queue_node (or pending_node) if we mark the
|
| 1955 | * packet as locked an then remove it from the queue (or pending set
|
| 1956 | * respectively). Marking the packet as locked avoids re-queuing
|
| 1957 | * (which should already be prevented by having stopped the treads...)
|
| 1958 | * and not setting QUEUED_BIT (or PENDING_BIT) prevents removal from a
|
| 1959 | * new list via other threads (e.g. cancellation).
|
| 1960 | *
|
| 1961 | * Note 3: There may be overlap between complete_p and complete_q.
|
| 1962 | * This is handled via test_and_set_bit() on the "completed" flag
|
| 1963 | * (also handles cancellation).
|
| 1964 |
|
| 1965 |
|
| 1966 |
|
| 1967 | spin_lock(&ptl->queue.lock);
|
| 1968 | list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
|
| 1969 | set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);
|
| 1970 |
|
| 1971 | smp_mb__before_atomic();
|
| 1972 | clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);
|
| 1973 |
|
| 1974 | list_move_tail(&p->queue_node, &complete_q);
|
| 1975 | }
|
| 1976 | spin_unlock(&ptl->queue.lock);
|
| 1977 |
|
| 1978 |
|
| 1979 | spin_lock(&ptl->pending.lock);
|
| 1980 | list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
|
| 1981 | set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);
|
| 1982 |
|
| 1983 | smp_mb__before_atomic();
|
| 1984 | clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);
|
| 1985 |
|
| 1986 | list_move_tail(&p->pending_node, &complete_q);
|
| 1987 | }
|
| 1988 | atomic_set(&ptl->pending.count, 0);
|
| 1989 | spin_unlock(&ptl->pending.lock);
|
| 1990 |
|
| 1991 |
|
| 1992 | list_for_each_entry(p, &complete_q, queue_node) {
|
| 1993 | if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
|
| 1994 | __ssh_ptl_complete(p, -ESHUTDOWN);
|
| 1995 |
|
| 1996 | ssh_packet_put(p);
|
| 1997 | }
|
| 1998 |
|
| 1999 |
|
| 2000 | list_for_each_entry(p, &complete_p, pending_node) {
|
| 2001 | if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))
|
| 2002 | __ssh_ptl_complete(p, -ESHUTDOWN);
|
| 2003 |
|
| 2004 | ssh_packet_put(p);
|
| 2005 | }
|
| 2006 |
|
| 2007 | |
| 2008 | * At this point we have guaranteed that the system doesn't reference
|
| 2009 | * any packets any more.
|
| 2010 |
|
| 2011 | }
|
| 2012 |
|
| 2013 | |
| 2014 | * ssh_ptl_init() - Initialize packet transport layer.
|
| 2015 | * @ptl: The packet transport layer to initialize.
|
| 2016 | * @serdev: The underlying serial device, i.e. the lower-level transport.
|
| 2017 | * @ops: Packet layer operations.
|
| 2018 | *
|
| 2019 | * Initializes the given packet transport layer. Transmitter and receiver
|
| 2020 | * threads must be started separately via ssh_ptl_tx_start() and
|
| 2021 | * ssh_ptl_rx_start(), after the packet-layer has been initialized and the
|
| 2022 | * lower-level transport layer has been set up.
|
| 2023 | *
|
| 2024 | * Return: Returns zero on success and a nonzero error code on failure.
|
| 2025 |
|
| 2026 | int ssh_ptl_init(struct ssh_ptl *ptl, struct serdev_device *serdev,
|
| 2027 | struct ssh_ptl_ops *ops)
|
| 2028 | {
|
| 2029 | int i, status;
|
| 2030 |
|
| 2031 | ptl->serdev = serdev;
|
| 2032 | ptl->state = 0;
|
| 2033 |
|
| 2034 | spin_lock_init(&ptl->queue.lock);
|
| 2035 | INIT_LIST_HEAD(&ptl->queue.head);
|
| 2036 |
|
| 2037 | spin_lock_init(&ptl->pending.lock);
|
| 2038 | INIT_LIST_HEAD(&ptl->pending.head);
|
| 2039 | atomic_set_release(&ptl->pending.count, 0);
|
| 2040 |
|
| 2041 | ptl->tx.thread = NULL;
|
| 2042 | atomic_set(&ptl->tx.running, 0);
|
| 2043 | init_completion(&ptl->tx.thread_cplt_pkt);
|
| 2044 | init_completion(&ptl->tx.thread_cplt_tx);
|
| 2045 | init_waitqueue_head(&ptl->tx.packet_wq);
|
| 2046 |
|
| 2047 | ptl->rx.thread = NULL;
|
| 2048 | init_waitqueue_head(&ptl->rx.wq);
|
| 2049 |
|
| 2050 | spin_lock_init(&ptl->rtx_timeout.lock);
|
| 2051 | ptl->rtx_timeout.timeout = SSH_PTL_PACKET_TIMEOUT;
|
| 2052 | ptl->rtx_timeout.expires = KTIME_MAX;
|
| 2053 | INIT_DELAYED_WORK(&ptl->rtx_timeout.reaper, ssh_ptl_timeout_reap);
|
| 2054 |
|
| 2055 | ptl->ops = *ops;
|
| 2056 |
|
| 2057 |
|
| 2058 | for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++)
|
| 2059 | ptl->rx.blocked.seqs[i] = U16_MAX;
|
| 2060 | ptl->rx.blocked.offset = 0;
|
| 2061 |
|
| 2062 | status = kfifo_alloc(&ptl->rx.fifo, SSH_PTL_RX_FIFO_LEN, GFP_KERNEL);
|
| 2063 | if (status)
|
| 2064 | return status;
|
| 2065 |
|
| 2066 | status = sshp_buf_alloc(&ptl->rx.buf, SSH_PTL_RX_BUF_LEN, GFP_KERNEL);
|
| 2067 | if (status)
|
| 2068 | kfifo_free(&ptl->rx.fifo);
|
| 2069 |
|
| 2070 | return status;
|
| 2071 | }
|
| 2072 |
|
| 2073 | |
| 2074 | * ssh_ptl_destroy() - Deinitialize packet transport layer.
|
| 2075 | * @ptl: The packet transport layer to deinitialize.
|
| 2076 | *
|
| 2077 | * Deinitializes the given packet transport layer and frees resources
|
| 2078 | * associated with it. If receiver and/or transmitter threads have been
|
| 2079 | * started, the layer must first be shut down via ssh_ptl_shutdown() before
|
| 2080 | * this function can be called.
|
| 2081 |
|
| 2082 | void ssh_ptl_destroy(struct ssh_ptl *ptl)
|
| 2083 | {
|
| 2084 | kfifo_free(&ptl->rx.fifo);
|
| 2085 | sshp_buf_free(&ptl->rx.buf);
|
| 2086 | }
|