projects/linux·blob master · refs

1332 lines · 37.8 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Processing of received RxRPC packets
3 *
4 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include "ar-internal.h"
11
12/* Override priority when generating ACKs for received DATA */
13static const u8 rxrpc_ack_priority[RXRPC_ACK__INVALID] = {
14 [RXRPC_ACK_IDLE] = 1,
15 [RXRPC_ACK_DELAY] = 2,
16 [RXRPC_ACK_REQUESTED] = 3,
17 [RXRPC_ACK_DUPLICATE] = 4,
18 [RXRPC_ACK_EXCEEDS_WINDOW] = 5,
19 [RXRPC_ACK_NOSPACE] = 6,
20 [RXRPC_ACK_OUT_OF_SEQUENCE] = 7,
21};
22
23static void rxrpc_proto_abort(struct rxrpc_call *call, rxrpc_seq_t seq,
24 enum rxrpc_abort_reason why)
25{
26 rxrpc_abort_call(call, seq, RX_PROTOCOL_ERROR, -EBADMSG, why);
27}
28
29/*
30 * Do TCP-style congestion management [RFC5681].
31 */
32static void rxrpc_congestion_management(struct rxrpc_call *call,
33 struct rxrpc_ack_summary *summary)
34{
35 summary->change = rxrpc_cong_no_change;
36 summary->in_flight = rxrpc_tx_in_flight(call);
37
38 if (test_and_clear_bit(RXRPC_CALL_RETRANS_TIMEOUT, &call->flags)) {
39 summary->retrans_timeo = true;
40 call->cong_ssthresh = umax(summary->in_flight / 2, 2);
41 call->cong_cwnd = 1;
42 if (call->cong_cwnd >= call->cong_ssthresh &&
43 call->cong_ca_state == RXRPC_CA_SLOW_START) {
44 call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;
45 call->cong_tstamp = call->acks_latest_ts;
46 call->cong_cumul_acks = 0;
47 }
48 }
49
50 call->cong_cumul_acks += summary->nr_new_sacks;
51 call->cong_cumul_acks += summary->nr_new_hacks;
52 if (call->cong_cumul_acks > 255)
53 call->cong_cumul_acks = 255;
54
55 switch (call->cong_ca_state) {
56 case RXRPC_CA_SLOW_START:
57 if (call->acks_nr_snacks > 0)
58 goto packet_loss_detected;
59 if (call->cong_cumul_acks > 0)
60 call->cong_cwnd += 1;
61 if (call->cong_cwnd >= call->cong_ssthresh) {
62 call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;
63 call->cong_tstamp = call->acks_latest_ts;
64 }
65 goto out;
66
67 case RXRPC_CA_CONGEST_AVOIDANCE:
68 if (call->acks_nr_snacks > 0)
69 goto packet_loss_detected;
70
71 /* We analyse the number of packets that get ACK'd per RTT
72 * period and increase the window if we managed to fill it.
73 */
74 if (call->rtt_count == 0)
75 goto out;
76 if (ktime_before(call->acks_latest_ts,
77 ktime_add_us(call->cong_tstamp,
78 call->srtt_us >> 3)))
79 goto out_no_clear_ca;
80 summary->change = rxrpc_cong_rtt_window_end;
81 call->cong_tstamp = call->acks_latest_ts;
82 if (call->cong_cumul_acks >= call->cong_cwnd)
83 call->cong_cwnd++;
84 goto out;
85
86 case RXRPC_CA_PACKET_LOSS:
87 if (call->acks_nr_snacks == 0)
88 goto resume_normality;
89
90 if (summary->new_low_snack) {
91 summary->change = rxrpc_cong_new_low_nack;
92 call->cong_dup_acks = 1;
93 if (call->cong_extra > 1)
94 call->cong_extra = 1;
95 goto send_extra_data;
96 }
97
98 call->cong_dup_acks++;
99 if (call->cong_dup_acks < 3)
100 goto send_extra_data;
101
102 summary->change = rxrpc_cong_begin_retransmission;
103 call->cong_ca_state = RXRPC_CA_FAST_RETRANSMIT;
104 call->cong_ssthresh = umax(summary->in_flight / 2, 2);
105 call->cong_cwnd = call->cong_ssthresh + 3;
106 call->cong_extra = 0;
107 call->cong_dup_acks = 0;
108 summary->need_retransmit = true;
109 summary->in_fast_or_rto_recovery = true;
110 goto out;
111
112 case RXRPC_CA_FAST_RETRANSMIT:
113 rxrpc_tlp_init(call);
114 summary->in_fast_or_rto_recovery = true;
115 if (!summary->new_low_snack) {
116 if (summary->nr_new_sacks == 0)
117 call->cong_cwnd += 1;
118 call->cong_dup_acks++;
119 if (call->cong_dup_acks == 2) {
120 summary->change = rxrpc_cong_retransmit_again;
121 call->cong_dup_acks = 0;
122 summary->need_retransmit = true;
123 }
124 } else {
125 summary->change = rxrpc_cong_progress;
126 call->cong_cwnd = call->cong_ssthresh;
127 if (call->acks_nr_snacks == 0) {
128 summary->exiting_fast_or_rto_recovery = true;
129 goto resume_normality;
130 }
131 }
132 goto out;
133
134 default:
135 BUG();
136 goto out;
137 }
138
139resume_normality:
140 summary->change = rxrpc_cong_cleared_nacks;
141 call->cong_dup_acks = 0;
142 call->cong_extra = 0;
143 call->cong_tstamp = call->acks_latest_ts;
144 if (call->cong_cwnd < call->cong_ssthresh)
145 call->cong_ca_state = RXRPC_CA_SLOW_START;
146 else
147 call->cong_ca_state = RXRPC_CA_CONGEST_AVOIDANCE;
148out:
149 call->cong_cumul_acks = 0;
150out_no_clear_ca:
151 if (call->cong_cwnd >= RXRPC_TX_MAX_WINDOW)
152 call->cong_cwnd = RXRPC_TX_MAX_WINDOW;
153 trace_rxrpc_congest(call, summary);
154 return;
155
156packet_loss_detected:
157 summary->change = rxrpc_cong_saw_nack;
158 call->cong_ca_state = RXRPC_CA_PACKET_LOSS;
159 call->cong_dup_acks = 0;
160 goto send_extra_data;
161
162send_extra_data:
163 /* Send some previously unsent DATA if we have some to advance the ACK
164 * state.
165 */
166 if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) ||
167 call->acks_nr_sacks != call->tx_top - call->tx_bottom) {
168 call->cong_extra++;
169 wake_up(&call->waitq);
170 }
171 goto out_no_clear_ca;
172}
173
174/*
175 * Degrade the congestion window if we haven't transmitted a packet for >1RTT.
176 */
177void rxrpc_congestion_degrade(struct rxrpc_call *call)
178{
179 ktime_t rtt, now, time_since;
180
181 if (call->cong_ca_state != RXRPC_CA_SLOW_START &&
182 call->cong_ca_state != RXRPC_CA_CONGEST_AVOIDANCE)
183 return;
184 if (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_ACK ||
185 __rxrpc_call_state(call) == RXRPC_CALL_CLIENT_AWAIT_REPLY)
186 return;
187
188 rtt = ns_to_ktime(call->srtt_us * (NSEC_PER_USEC / 8));
189 now = ktime_get_real();
190 time_since = ktime_sub(now, call->tx_last_sent);
191 if (ktime_before(time_since, rtt))
192 return;
193
194 trace_rxrpc_reset_cwnd(call, time_since, rtt);
195 rxrpc_inc_stat(call->rxnet, stat_tx_data_cwnd_reset);
196 call->tx_last_sent = now;
197 call->cong_ca_state = RXRPC_CA_SLOW_START;
198 call->cong_ssthresh = umax(call->cong_ssthresh, call->cong_cwnd * 3 / 4);
199 call->cong_cwnd = umax(call->cong_cwnd / 2, RXRPC_MIN_CWND);
200}
201
202/*
203 * Add an RTT sample derived from an ACK'd DATA packet.
204 */
205static void rxrpc_add_data_rtt_sample(struct rxrpc_call *call,
206 struct rxrpc_ack_summary *summary,
207 struct rxrpc_txqueue *tq,
208 int ix)
209{
210 ktime_t xmit_ts = ktime_add_us(tq->xmit_ts_base, tq->segment_xmit_ts[ix]);
211
212 rxrpc_call_add_rtt(call, rxrpc_rtt_rx_data_ack, -1,
213 summary->acked_serial, summary->ack_serial,
214 xmit_ts, call->acks_latest_ts);
215 __clear_bit(ix, &tq->rtt_samples); /* Prevent repeat RTT sample */
216}
217
218/*
219 * Apply a hard ACK by advancing the Tx window.
220 */
221static bool rxrpc_rotate_tx_window(struct rxrpc_call *call, rxrpc_seq_t to,
222 struct rxrpc_ack_summary *summary)
223{
224 struct rxrpc_txqueue *tq = call->tx_queue;
225 rxrpc_seq_t seq = call->tx_bottom + 1;
226 bool rot_last = false, trace = false;
227
228 _enter("%x,%x", call->tx_bottom, to);
229
230 trace_rxrpc_tx_rotate(call, seq, to);
231 trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate);
232
233 if (call->acks_lowest_nak == call->tx_bottom) {
234 call->acks_lowest_nak = to;
235 } else if (after(to, call->acks_lowest_nak)) {
236 summary->new_low_snack = true;
237 call->acks_lowest_nak = to;
238 }
239
240 if (after(seq, to))
241 return false;
242
243 /* We may have a left over fully-consumed buffer at the front that we
244 * couldn't drop before (rotate_and_keep below).
245 */
246 if (seq == call->tx_qbase + RXRPC_NR_TXQUEUE) {
247 call->tx_qbase += RXRPC_NR_TXQUEUE;
248 call->tx_queue = tq->next;
249 trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);
250 kfree(tq);
251 tq = call->tx_queue;
252 }
253
254 while (before_eq(seq, to)) {
255 unsigned int ix = seq - call->tx_qbase;
256
257 _debug("tq=%x seq=%x i=%d f=%x", tq->qbase, seq, ix, tq->bufs[ix]->flags);
258 if (tq->bufs[ix]->flags & RXRPC_LAST_PACKET) {
259 set_bit(RXRPC_CALL_TX_LAST, &call->flags);
260 rot_last = true;
261 }
262
263 if (summary->acked_serial == tq->segment_serial[ix] &&
264 test_bit(ix, &tq->rtt_samples))
265 rxrpc_add_data_rtt_sample(call, summary, tq, ix);
266
267 if (ix == tq->nr_reported_acks) {
268 /* Packet directly hard ACK'd. */
269 tq->nr_reported_acks++;
270 rxrpc_input_rack_one(call, summary, tq, ix);
271 if (seq == call->tlp_seq)
272 summary->tlp_probe_acked = true;
273 summary->nr_new_hacks++;
274 __set_bit(ix, &tq->segment_acked);
275 trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_hack);
276 } else if (test_bit(ix, &tq->segment_acked)) {
277 /* Soft ACK -> hard ACK. */
278 call->acks_nr_sacks--;
279 trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_sack);
280 } else {
281 /* Soft NAK -> hard ACK. */
282 call->acks_nr_snacks--;
283 rxrpc_input_rack_one(call, summary, tq, ix);
284 if (seq == call->tlp_seq)
285 summary->tlp_probe_acked = true;
286 summary->nr_new_hacks++;
287 __set_bit(ix, &tq->segment_acked);
288 trace_rxrpc_rotate(call, tq, summary, seq, rxrpc_rotate_trace_snak);
289 }
290
291 call->tx_nr_sent--;
292 if (__test_and_clear_bit(ix, &tq->segment_lost))
293 call->tx_nr_lost--;
294 if (__test_and_clear_bit(ix, &tq->segment_retransmitted))
295 call->tx_nr_resent--;
296 __clear_bit(ix, &tq->ever_retransmitted);
297
298 rxrpc_put_txbuf(tq->bufs[ix], rxrpc_txbuf_put_rotated);
299 tq->bufs[ix] = NULL;
300
301 WRITE_ONCE(call->tx_bottom, seq);
302 trace_rxrpc_txqueue(call, (rot_last ?
303 rxrpc_txqueue_rotate_last :
304 rxrpc_txqueue_rotate));
305
306 seq++;
307 trace = true;
308 if (!(seq & RXRPC_TXQ_MASK)) {
309 trace_rxrpc_rack_update(call, summary);
310 trace = false;
311 prefetch(tq->next);
312 if (tq != call->tx_qtail) {
313 call->tx_qbase += RXRPC_NR_TXQUEUE;
314 call->tx_queue = tq->next;
315 trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);
316 kfree(tq);
317 tq = call->tx_queue;
318 } else {
319 trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_keep);
320 tq = NULL;
321 break;
322 }
323 }
324 }
325
326 if (trace)
327 trace_rxrpc_rack_update(call, summary);
328
329 if (rot_last) {
330 set_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags);
331 if (tq) {
332 trace_rxrpc_tq(call, tq, seq, rxrpc_tq_rotate_and_free);
333 kfree(tq);
334 call->tx_queue = NULL;
335 }
336 }
337
338 _debug("%x,%x,%x,%d", to, call->tx_bottom, call->tx_top, rot_last);
339
340 wake_up(&call->waitq);
341 return rot_last;
342}
343
344/*
345 * End the transmission phase of a call.
346 *
347 * This occurs when we get an ACKALL packet, the first DATA packet of a reply,
348 * or a final ACK packet.
349 */
350static void rxrpc_end_tx_phase(struct rxrpc_call *call, bool reply_begun,
351 enum rxrpc_abort_reason abort_why)
352{
353 ASSERT(test_bit(RXRPC_CALL_TX_LAST, &call->flags));
354
355 call->rack_timer_mode = RXRPC_CALL_RACKTIMER_OFF;
356 call->rack_timo_at = KTIME_MAX;
357 trace_rxrpc_rack_timer(call, 0, false);
358 trace_rxrpc_timer_can(call, rxrpc_timer_trace_rack_off + call->rack_timer_mode);
359
360 switch (__rxrpc_call_state(call)) {
361 case RXRPC_CALL_CLIENT_SEND_REQUEST:
362 case RXRPC_CALL_CLIENT_AWAIT_ACK:
363 case RXRPC_CALL_CLIENT_AWAIT_REPLY:
364 if (reply_begun) {
365 rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_RECV_REPLY);
366 trace_rxrpc_txqueue(call, rxrpc_txqueue_end);
367 break;
368 }
369
370 rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY);
371 trace_rxrpc_txqueue(call, rxrpc_txqueue_await_reply);
372 break;
373
374 case RXRPC_CALL_SERVER_AWAIT_ACK:
375 rxrpc_call_completed(call);
376 trace_rxrpc_txqueue(call, rxrpc_txqueue_end);
377 break;
378
379 default:
380 kdebug("end_tx %s", rxrpc_call_states[__rxrpc_call_state(call)]);
381 rxrpc_proto_abort(call, call->tx_top, abort_why);
382 break;
383 }
384}
385
386/*
387 * Begin the reply reception phase of a call.
388 */
389static bool rxrpc_receiving_reply(struct rxrpc_call *call)
390{
391 struct rxrpc_ack_summary summary = { 0 };
392 rxrpc_seq_t top = READ_ONCE(call->tx_top);
393
394 if (call->ackr_reason) {
395 call->delay_ack_at = KTIME_MAX;
396 trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack);
397 }
398
399 /* Deal with an apparent reply coming in before we've got the request
400 * queued or transmitted.
401 */
402 if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
403 rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply);
404 return false;
405 }
406
407 if (!test_bit(RXRPC_CALL_TX_LAST, &call->flags)) {
408 if (!rxrpc_rotate_tx_window(call, top, &summary)) {
409 rxrpc_proto_abort(call, top, rxrpc_eproto_early_reply);
410 return false;
411 }
412 }
413
414 rxrpc_end_tx_phase(call, true, rxrpc_eproto_unexpected_reply);
415 return true;
416}
417
418/*
419 * End the packet reception phase.
420 */
421static void rxrpc_end_rx_phase(struct rxrpc_call *call, rxrpc_serial_t serial)
422{
423 rxrpc_seq_t whigh = READ_ONCE(call->rx_highest_seq);
424
425 _enter("%d,%s", call->debug_id, rxrpc_call_states[__rxrpc_call_state(call)]);
426
427 trace_rxrpc_receive(call, rxrpc_receive_end, 0, whigh);
428
429 switch (__rxrpc_call_state(call)) {
430 case RXRPC_CALL_CLIENT_RECV_REPLY:
431 rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_terminal_ack);
432 rxrpc_call_completed(call);
433 break;
434
435 case RXRPC_CALL_SERVER_RECV_REQUEST:
436 rxrpc_set_call_state(call, RXRPC_CALL_SERVER_ACK_REQUEST);
437 call->expect_req_by = KTIME_MAX;
438 rxrpc_propose_delay_ACK(call, serial, rxrpc_propose_ack_processing_op);
439 break;
440
441 default:
442 break;
443 }
444}
445
446static void rxrpc_input_update_ack_window(struct rxrpc_call *call,
447 rxrpc_seq_t window, rxrpc_seq_t wtop)
448{
449 call->ackr_window = window;
450 call->ackr_wtop = wtop;
451}
452
453/*
454 * Push a DATA packet onto the Rx queue.
455 */
456static void rxrpc_input_queue_data(struct rxrpc_call *call, struct sk_buff *skb,
457 rxrpc_seq_t window, rxrpc_seq_t wtop,
458 enum rxrpc_receive_trace why)
459{
460 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
461 bool last = sp->hdr.flags & RXRPC_LAST_PACKET;
462
463 spin_lock_irq(&call->recvmsg_queue.lock);
464
465 __skb_queue_tail(&call->recvmsg_queue, skb);
466 rxrpc_input_update_ack_window(call, window, wtop);
467 trace_rxrpc_receive(call, last ? why + 1 : why, sp->hdr.serial, sp->hdr.seq);
468 if (last)
469 /* Change the state inside the lock so that recvmsg syncs
470 * correctly with it and using sendmsg() to send a reply
471 * doesn't race.
472 */
473 rxrpc_end_rx_phase(call, sp->hdr.serial);
474
475 spin_unlock_irq(&call->recvmsg_queue.lock);
476}
477
478/*
479 * Process a DATA packet.
480 */
481static void rxrpc_input_data_one(struct rxrpc_call *call, struct sk_buff *skb,
482 bool *_notify, rxrpc_serial_t *_ack_serial, int *_ack_reason)
483{
484 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
485 struct sk_buff *oos;
486 rxrpc_serial_t serial = sp->hdr.serial;
487 unsigned int sack = call->ackr_sack_base;
488 rxrpc_seq_t window = call->ackr_window;
489 rxrpc_seq_t wtop = call->ackr_wtop;
490 rxrpc_seq_t wlimit = window + call->rx_winsize - 1;
491 rxrpc_seq_t seq = sp->hdr.seq;
492 bool last = sp->hdr.flags & RXRPC_LAST_PACKET;
493 int ack_reason = -1;
494
495 rxrpc_inc_stat(call->rxnet, stat_rx_data);
496 if (sp->hdr.flags & RXRPC_REQUEST_ACK)
497 rxrpc_inc_stat(call->rxnet, stat_rx_data_reqack);
498 if (sp->hdr.flags & RXRPC_JUMBO_PACKET)
499 rxrpc_inc_stat(call->rxnet, stat_rx_data_jumbo);
500
501 if (last) {
502 if (test_and_set_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
503 seq + 1 != wtop)
504 return rxrpc_proto_abort(call, seq, rxrpc_eproto_different_last);
505 } else {
506 if (test_bit(RXRPC_CALL_RX_LAST, &call->flags) &&
507 after_eq(seq, wtop)) {
508 pr_warn("Packet beyond last: c=%x q=%x window=%x-%x wlimit=%x\n",
509 call->debug_id, seq, window, wtop, wlimit);
510 return rxrpc_proto_abort(call, seq, rxrpc_eproto_data_after_last);
511 }
512 }
513
514 if (after(seq, call->rx_highest_seq))
515 call->rx_highest_seq = seq;
516
517 trace_rxrpc_rx_data(call->debug_id, seq, serial, sp->hdr.flags);
518
519 if (before(seq, window)) {
520 ack_reason = RXRPC_ACK_DUPLICATE;
521 goto send_ack;
522 }
523 if (after(seq, wlimit)) {
524 ack_reason = RXRPC_ACK_EXCEEDS_WINDOW;
525 goto send_ack;
526 }
527
528 /* Queue the packet. */
529 if (seq == window) {
530 if (sp->hdr.flags & RXRPC_REQUEST_ACK)
531 ack_reason = RXRPC_ACK_REQUESTED;
532 /* Send an immediate ACK if we fill in a hole */
533 else if (!skb_queue_empty(&call->rx_oos_queue))
534 ack_reason = RXRPC_ACK_DELAY;
535
536 window++;
537 if (after(window, wtop)) {
538 trace_rxrpc_sack(call, seq, sack, rxrpc_sack_none);
539 wtop = window;
540 } else {
541 trace_rxrpc_sack(call, seq, sack, rxrpc_sack_advance);
542 sack = (sack + 1) % RXRPC_SACK_SIZE;
543 }
544
545
546 rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg);
547
548 rxrpc_input_queue_data(call, skb, window, wtop, rxrpc_receive_queue);
549 *_notify = true;
550
551 while ((oos = skb_peek(&call->rx_oos_queue))) {
552 struct rxrpc_skb_priv *osp = rxrpc_skb(oos);
553
554 if (after(osp->hdr.seq, window))
555 break;
556
557 __skb_unlink(oos, &call->rx_oos_queue);
558 last = osp->hdr.flags & RXRPC_LAST_PACKET;
559 seq = osp->hdr.seq;
560 call->ackr_sack_table[sack] = 0;
561 trace_rxrpc_sack(call, seq, sack, rxrpc_sack_fill);
562 sack = (sack + 1) % RXRPC_SACK_SIZE;
563
564 window++;
565 rxrpc_input_queue_data(call, oos, window, wtop,
566 rxrpc_receive_queue_oos);
567 }
568
569 call->ackr_sack_base = sack;
570 } else {
571 unsigned int slot;
572
573 ack_reason = RXRPC_ACK_OUT_OF_SEQUENCE;
574
575 slot = seq - window;
576 sack = (sack + slot) % RXRPC_SACK_SIZE;
577
578 if (call->ackr_sack_table[sack % RXRPC_SACK_SIZE]) {
579 ack_reason = RXRPC_ACK_DUPLICATE;
580 goto send_ack;
581 }
582
583 call->ackr_sack_table[sack % RXRPC_SACK_SIZE] |= 1;
584 trace_rxrpc_sack(call, seq, sack, rxrpc_sack_oos);
585
586 if (after(seq + 1, wtop)) {
587 wtop = seq + 1;
588 rxrpc_input_update_ack_window(call, window, wtop);
589 }
590
591 skb_queue_walk(&call->rx_oos_queue, oos) {
592 struct rxrpc_skb_priv *osp = rxrpc_skb(oos);
593
594 if (after(osp->hdr.seq, seq)) {
595 rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg_oos);
596 __skb_queue_before(&call->rx_oos_queue, oos, skb);
597 goto oos_queued;
598 }
599 }
600
601 rxrpc_get_skb(skb, rxrpc_skb_get_to_recvmsg_oos);
602 __skb_queue_tail(&call->rx_oos_queue, skb);
603 oos_queued:
604 trace_rxrpc_receive(call, last ? rxrpc_receive_oos_last : rxrpc_receive_oos,
605 sp->hdr.serial, sp->hdr.seq);
606 }
607
608send_ack:
609 if (ack_reason >= 0) {
610 if (rxrpc_ack_priority[ack_reason] > rxrpc_ack_priority[*_ack_reason]) {
611 *_ack_serial = serial;
612 *_ack_reason = ack_reason;
613 } else if (rxrpc_ack_priority[ack_reason] == rxrpc_ack_priority[*_ack_reason] &&
614 ack_reason == RXRPC_ACK_REQUESTED) {
615 *_ack_serial = serial;
616 *_ack_reason = ack_reason;
617 }
618 }
619}
620
621/*
622 * Split a jumbo packet and file the bits separately.
623 */
624static bool rxrpc_input_split_jumbo(struct rxrpc_call *call, struct sk_buff *skb)
625{
626 struct rxrpc_jumbo_header jhdr;
627 struct rxrpc_skb_priv *sp = rxrpc_skb(skb), *jsp;
628 struct sk_buff *jskb;
629 rxrpc_serial_t ack_serial = 0;
630 unsigned int offset = sizeof(struct rxrpc_wire_header);
631 unsigned int len = skb->len - offset;
632 bool notify = false;
633 int ack_reason = 0, count = 1, stat_ix;
634
635 while (sp->hdr.flags & RXRPC_JUMBO_PACKET) {
636 if (len < RXRPC_JUMBO_SUBPKTLEN)
637 goto protocol_error;
638 if (sp->hdr.flags & RXRPC_LAST_PACKET)
639 goto protocol_error;
640 if (skb_copy_bits(skb, offset + RXRPC_JUMBO_DATALEN,
641 &jhdr, sizeof(jhdr)) < 0)
642 goto protocol_error;
643
644 jskb = skb_clone(skb, GFP_NOFS);
645 if (!jskb) {
646 kdebug("couldn't clone");
647 return false;
648 }
649 rxrpc_new_skb(jskb, rxrpc_skb_new_jumbo_subpacket);
650 jsp = rxrpc_skb(jskb);
651 jsp->offset = offset;
652 jsp->len = RXRPC_JUMBO_DATALEN;
653 rxrpc_input_data_one(call, jskb, &notify, &ack_serial, &ack_reason);
654 rxrpc_free_skb(jskb, rxrpc_skb_put_jumbo_subpacket);
655
656 sp->hdr.flags = jhdr.flags;
657 sp->hdr._rsvd = ntohs(jhdr._rsvd);
658 sp->hdr.seq++;
659 sp->hdr.serial++;
660 offset += RXRPC_JUMBO_SUBPKTLEN;
661 len -= RXRPC_JUMBO_SUBPKTLEN;
662 count++;
663 }
664
665 sp->offset = offset;
666 sp->len = len;
667 rxrpc_input_data_one(call, skb, &notify, &ack_serial, &ack_reason);
668
669 stat_ix = umin(count, ARRAY_SIZE(call->rxnet->stat_rx_jumbo)) - 1;
670 atomic_inc(&call->rxnet->stat_rx_jumbo[stat_ix]);
671
672 if (ack_reason > 0) {
673 rxrpc_send_ACK(call, ack_reason, ack_serial,
674 rxrpc_propose_ack_input_data);
675 } else {
676 call->ackr_nr_unacked++;
677 rxrpc_propose_delay_ACK(call, sp->hdr.serial,
678 rxrpc_propose_ack_input_data);
679 }
680 if (notify && !test_bit(RXRPC_CALL_CONN_CHALLENGING, &call->flags)) {
681 trace_rxrpc_notify_socket(call->debug_id, sp->hdr.serial);
682 rxrpc_notify_socket(call);
683 }
684 return true;
685
686protocol_error:
687 return false;
688}
689
690/*
691 * Process a DATA packet, adding the packet to the Rx ring. The caller's
692 * packet ref must be passed on or discarded.
693 */
694static void rxrpc_input_data(struct rxrpc_call *call, struct sk_buff *skb)
695{
696 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
697 rxrpc_serial_t serial = sp->hdr.serial;
698 rxrpc_seq_t seq0 = sp->hdr.seq;
699
700 _enter("{%x,%x,%x},{%u,%x}",
701 call->ackr_window, call->ackr_wtop, call->rx_highest_seq,
702 skb->len, seq0);
703
704 if (__rxrpc_call_is_complete(call))
705 return;
706
707 switch (__rxrpc_call_state(call)) {
708 case RXRPC_CALL_CLIENT_SEND_REQUEST:
709 case RXRPC_CALL_CLIENT_AWAIT_ACK:
710 case RXRPC_CALL_CLIENT_AWAIT_REPLY:
711 /* Received data implicitly ACKs all of the request
712 * packets we sent when we're acting as a client.
713 */
714 if (!rxrpc_receiving_reply(call))
715 goto out_notify;
716 break;
717
718 case RXRPC_CALL_SERVER_RECV_REQUEST: {
719 unsigned long timo = READ_ONCE(call->next_req_timo);
720
721 if (timo) {
722 ktime_t delay = ms_to_ktime(timo);
723
724 call->expect_req_by = ktime_add(ktime_get_real(), delay);
725 trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_idle);
726 }
727 break;
728 }
729
730 default:
731 break;
732 }
733
734 if (!rxrpc_input_split_jumbo(call, skb)) {
735 rxrpc_proto_abort(call, sp->hdr.seq, rxrpc_badmsg_bad_jumbo);
736 goto out_notify;
737 }
738 return;
739
740out_notify:
741 trace_rxrpc_notify_socket(call->debug_id, serial);
742 rxrpc_notify_socket(call);
743 _leave(" [queued]");
744}
745
746/*
747 * See if there's a cached RTT probe to complete.
748 */
749static void rxrpc_complete_rtt_probe(struct rxrpc_call *call,
750 ktime_t resp_time,
751 rxrpc_serial_t acked_serial,
752 rxrpc_serial_t ack_serial,
753 enum rxrpc_rtt_rx_trace type)
754{
755 rxrpc_serial_t orig_serial;
756 unsigned long avail;
757 ktime_t sent_at;
758 bool matched = false;
759 int i;
760
761 avail = READ_ONCE(call->rtt_avail);
762 smp_rmb(); /* Read avail bits before accessing data. */
763
764 for (i = 0; i < ARRAY_SIZE(call->rtt_serial); i++) {
765 if (!test_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &avail))
766 continue;
767
768 sent_at = call->rtt_sent_at[i];
769 orig_serial = call->rtt_serial[i];
770
771 if (orig_serial == acked_serial) {
772 clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
773 smp_mb(); /* Read data before setting avail bit */
774 set_bit(i, &call->rtt_avail);
775 rxrpc_call_add_rtt(call, type, i, acked_serial, ack_serial,
776 sent_at, resp_time);
777 matched = true;
778 }
779
780 /* If a later serial is being acked, then mark this slot as
781 * being available.
782 */
783 if (after(acked_serial, orig_serial)) {
784 trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_obsolete, i,
785 orig_serial, acked_serial, 0, 0, 0);
786 clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
787 smp_wmb();
788 set_bit(i, &call->rtt_avail);
789 }
790 }
791
792 if (!matched)
793 trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_lost, 9, 0, acked_serial, 0, 0, 0);
794}
795
796/*
797 * Process the extra information that may be appended to an ACK packet
798 */
799static void rxrpc_input_ack_trailer(struct rxrpc_call *call, struct sk_buff *skb,
800 struct rxrpc_acktrailer *trailer)
801{
802 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
803 struct rxrpc_peer *peer = call->peer;
804 unsigned int max_data, capacity;
805 bool wake = false;
806 u32 max_mtu = ntohl(trailer->maxMTU);
807 //u32 if_mtu = ntohl(trailer->ifMTU);
808 u32 rwind = ntohl(trailer->rwind);
809 u32 jumbo_max = ntohl(trailer->jumbo_max);
810
811 if (rwind > RXRPC_TX_MAX_WINDOW)
812 rwind = RXRPC_TX_MAX_WINDOW;
813 if (call->tx_winsize != rwind) {
814 if (rwind > call->tx_winsize)
815 wake = true;
816 trace_rxrpc_rx_rwind_change(call, sp->hdr.serial, rwind, wake);
817 call->tx_winsize = rwind;
818 }
819
820 max_mtu = clamp(max_mtu, 500, 65535);
821 peer->ackr_max_data = max_mtu;
822
823 if (max_mtu < peer->max_data) {
824 trace_rxrpc_pmtud_reduce(peer, sp->hdr.serial, max_mtu,
825 rxrpc_pmtud_reduce_ack);
826 peer->max_data = max_mtu;
827 }
828
829 max_data = umin(max_mtu, peer->max_data);
830 capacity = max_data;
831 capacity += sizeof(struct rxrpc_jumbo_header); /* First subpacket has main hdr, not jumbo */
832 capacity /= sizeof(struct rxrpc_jumbo_header) + RXRPC_JUMBO_DATALEN;
833
834 if (jumbo_max == 0) {
835 /* The peer says it supports pmtu discovery */
836 peer->ackr_adv_pmtud = true;
837 } else {
838 peer->ackr_adv_pmtud = false;
839 capacity = clamp(capacity, 1, jumbo_max);
840 }
841
842 call->tx_jumbo_max = capacity;
843
844 if (wake)
845 wake_up(&call->waitq);
846}
847
848#if defined(CONFIG_X86) && __GNUC__ && !defined(__clang__)
849/* Clang doesn't support the %z constraint modifier */
850#define shiftr_adv_rotr(shift_from, rotate_into) ({ \
851 asm(" shr%z1 %1\n" \
852 " inc %0\n" \
853 " rcr%z2 %2\n" \
854 : "+d"(shift_from), "+m"(*(shift_from)), "+rm"(rotate_into) \
855 ); \
856 })
857#else
858#define shiftr_adv_rotr(shift_from, rotate_into) ({ \
859 typeof(rotate_into) __bit0 = *(shift_from) & 1; \
860 *(shift_from) >>= 1; \
861 shift_from++; \
862 rotate_into >>= 1; \
863 rotate_into |= __bit0 << (sizeof(rotate_into) * 8 - 1); \
864 })
865#endif
866
867/*
868 * Deal with RTT samples from soft ACKs.
869 */
870static void rxrpc_input_soft_rtt(struct rxrpc_call *call,
871 struct rxrpc_ack_summary *summary,
872 struct rxrpc_txqueue *tq)
873{
874 for (int ix = 0; ix < RXRPC_NR_TXQUEUE; ix++)
875 if (summary->acked_serial == tq->segment_serial[ix])
876 return rxrpc_add_data_rtt_sample(call, summary, tq, ix);
877}
878
879/*
880 * Process a batch of soft ACKs specific to a transmission queue segment.
881 */
882static void rxrpc_input_soft_ack_tq(struct rxrpc_call *call,
883 struct rxrpc_ack_summary *summary,
884 struct rxrpc_txqueue *tq,
885 unsigned long extracted_acks,
886 int nr_reported,
887 rxrpc_seq_t seq,
888 rxrpc_seq_t *lowest_nak)
889{
890 unsigned long old_reported = 0, flipped, new_acks = 0;
891 unsigned long a_to_n, n_to_a = 0;
892 int new, a, n;
893
894 if (tq->nr_reported_acks > 0)
895 old_reported = ~0UL >> (RXRPC_NR_TXQUEUE - tq->nr_reported_acks);
896
897 _enter("{%x,%lx,%d},%lx,%d,%x",
898 tq->qbase, tq->segment_acked, tq->nr_reported_acks,
899 extracted_acks, nr_reported, seq);
900
901 _debug("[%x]", tq->qbase);
902 _debug("tq %16lx %u", tq->segment_acked, tq->nr_reported_acks);
903 _debug("sack %16lx %u", extracted_acks, nr_reported);
904
905 /* See how many previously logged ACKs/NAKs have flipped. */
906 flipped = (tq->segment_acked ^ extracted_acks) & old_reported;
907 if (flipped) {
908 n_to_a = ~tq->segment_acked & flipped; /* Old NAK -> ACK */
909 a_to_n = tq->segment_acked & flipped; /* Old ACK -> NAK */
910 a = hweight_long(n_to_a);
911 n = hweight_long(a_to_n);
912 _debug("flip %16lx", flipped);
913 _debug("ntoa %16lx %d", n_to_a, a);
914 _debug("aton %16lx %d", a_to_n, n);
915 call->acks_nr_sacks += a - n;
916 call->acks_nr_snacks += n - a;
917 summary->nr_new_sacks += a;
918 summary->nr_new_snacks += n;
919 }
920
921 /* See how many new ACKs/NAKs have been acquired. */
922 new = nr_reported - tq->nr_reported_acks;
923 if (new > 0) {
924 new_acks = extracted_acks & ~old_reported;
925 if (new_acks) {
926 a = hweight_long(new_acks);
927 n = new - a;
928 _debug("new_a %16lx new=%d a=%d n=%d", new_acks, new, a, n);
929 call->acks_nr_sacks += a;
930 call->acks_nr_snacks += n;
931 summary->nr_new_sacks += a;
932 summary->nr_new_snacks += n;
933 } else {
934 call->acks_nr_snacks += new;
935 summary->nr_new_snacks += new;
936 }
937 }
938
939 tq->nr_reported_acks = nr_reported;
940 tq->segment_acked = extracted_acks;
941 trace_rxrpc_apply_acks(call, tq);
942
943 if (extracted_acks != ~0UL) {
944 rxrpc_seq_t lowest = seq + ffz(extracted_acks);
945
946 if (before(lowest, *lowest_nak))
947 *lowest_nak = lowest;
948 }
949
950 if (summary->acked_serial)
951 rxrpc_input_soft_rtt(call, summary, tq);
952
953 new_acks |= n_to_a;
954 if (new_acks)
955 rxrpc_input_rack(call, summary, tq, new_acks);
956
957 if (call->tlp_serial &&
958 rxrpc_seq_in_txq(tq, call->tlp_seq) &&
959 test_bit(call->tlp_seq - tq->qbase, &new_acks))
960 summary->tlp_probe_acked = true;
961}
962
963/*
964 * Process individual soft ACKs.
965 *
966 * Each ACK in the array corresponds to one packet and can be either an ACK or
967 * a NAK. If we get find an explicitly NAK'd packet we resend immediately;
968 * packets that lie beyond the end of the ACK list are scheduled for resend by
969 * the timer on the basis that the peer might just not have processed them at
970 * the time the ACK was sent.
971 */
972static void rxrpc_input_soft_acks(struct rxrpc_call *call,
973 struct rxrpc_ack_summary *summary,
974 struct sk_buff *skb)
975{
976 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
977 struct rxrpc_txqueue *tq = call->tx_queue;
978 unsigned long extracted = ~0UL;
979 unsigned int nr = 0, nsack;
980 rxrpc_seq_t seq = call->acks_hard_ack + 1;
981 rxrpc_seq_t lowest_nak = seq + sp->ack.nr_acks;
982 u8 sack[256] __aligned(sizeof(unsigned long));
983 u8 *acks = sack;
984
985 _enter("%x,%x,%u", tq->qbase, seq, sp->ack.nr_acks);
986
987 while (after(seq, tq->qbase + RXRPC_NR_TXQUEUE - 1))
988 tq = tq->next;
989
990 /* Extract an individual SACK table. A normal SACK table is up to 255
991 * bytes with 1 ACK flag per byte, but an extended SACK table can be up
992 * to 256 bytes with up to 8 ACK/NACK flags per byte. The ACK flags go
993 * across all bit 0's then all bit 1's, then all bit 2's, ...
994 */
995 memset(sack, 0, sizeof(sack));
996 nsack = umin(sp->ack.nr_acks, 256);
997 if (skb_copy_bits(skb,
998 sizeof(struct rxrpc_wire_header) + sizeof(struct rxrpc_ackpacket),
999 sack, nsack) < 0)
1000 return;
1001
1002 for (unsigned int i = 0; i < sp->ack.nr_acks; i++) {
1003 /* Decant ACKs until we hit a txqueue boundary. */
1004 if ((i & 255) == 0)
1005 acks = sack;
1006 shiftr_adv_rotr(acks, extracted);
1007 seq++;
1008 nr++;
1009 if ((seq & RXRPC_TXQ_MASK) != 0)
1010 continue;
1011
1012 _debug("bound %16lx %u", extracted, nr);
1013
1014 rxrpc_input_soft_ack_tq(call, summary, tq, extracted, RXRPC_NR_TXQUEUE,
1015 seq - RXRPC_NR_TXQUEUE, &lowest_nak);
1016 extracted = ~0UL;
1017 nr = 0;
1018 tq = tq->next;
1019 prefetch(tq);
1020 }
1021
1022 if (nr) {
1023 unsigned int nr_reported = seq & RXRPC_TXQ_MASK;
1024
1025 extracted >>= RXRPC_NR_TXQUEUE - nr_reported;
1026 _debug("tail %16lx %u", extracted, nr_reported);
1027 rxrpc_input_soft_ack_tq(call, summary, tq, extracted, nr_reported,
1028 seq & ~RXRPC_TXQ_MASK, &lowest_nak);
1029 }
1030
1031 /* We *can* have more nacks than we did - the peer is permitted to drop
1032 * packets it has soft-acked and re-request them. Further, it is
1033 * possible for the nack distribution to change whilst the number of
1034 * nacks stays the same or goes down.
1035 */
1036 if (lowest_nak != call->acks_lowest_nak) {
1037 call->acks_lowest_nak = lowest_nak;
1038 summary->new_low_snack = true;
1039 }
1040
1041 _debug("summary A=%d+%d N=%d+%d",
1042 call->acks_nr_sacks, summary->nr_new_sacks,
1043 call->acks_nr_snacks, summary->nr_new_snacks);
1044}
1045
1046/*
1047 * Return true if the ACK is valid - ie. it doesn't appear to have regressed
1048 * with respect to the ack state conveyed by preceding ACKs.
1049 */
1050static bool rxrpc_is_ack_valid(struct rxrpc_call *call,
1051 rxrpc_seq_t hard_ack, rxrpc_seq_t prev_pkt)
1052{
1053 rxrpc_seq_t base = READ_ONCE(call->acks_hard_ack);
1054
1055 if (after(hard_ack, base))
1056 return true; /* The window advanced */
1057
1058 if (before(hard_ack, base))
1059 return false; /* firstPacket regressed */
1060
1061 if (after_eq(prev_pkt, call->acks_prev_seq))
1062 return true; /* previousPacket hasn't regressed. */
1063
1064 /* Some rx implementations put a serial number in previousPacket. */
1065 if (after(prev_pkt, base + call->tx_winsize))
1066 return false;
1067 return true;
1068}
1069
1070/*
1071 * Process an ACK packet.
1072 *
1073 * ack.firstPacket is the sequence number of the first soft-ACK'd/NAK'd packet
1074 * in the ACK array. Anything before that is hard-ACK'd and may be discarded.
1075 *
1076 * A hard-ACK means that a packet has been processed and may be discarded; a
1077 * soft-ACK means that the packet may be discarded and retransmission
1078 * requested. A phase is complete when all packets are hard-ACK'd.
1079 */
1080static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
1081{
1082 struct rxrpc_ack_summary summary = { 0 };
1083 struct rxrpc_acktrailer trailer;
1084 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1085 rxrpc_seq_t first_soft_ack, hard_ack, prev_pkt;
1086 int nr_acks, offset, ioffset;
1087
1088 _enter("");
1089
1090 offset = sizeof(struct rxrpc_wire_header) + sizeof(struct rxrpc_ackpacket);
1091
1092 summary.ack_serial = sp->hdr.serial;
1093 first_soft_ack = sp->ack.first_ack;
1094 prev_pkt = sp->ack.prev_ack;
1095 nr_acks = sp->ack.nr_acks;
1096 hard_ack = first_soft_ack - 1;
1097 summary.acked_serial = sp->ack.acked_serial;
1098 summary.ack_reason = (sp->ack.reason < RXRPC_ACK__INVALID ?
1099 sp->ack.reason : RXRPC_ACK__INVALID);
1100
1101 trace_rxrpc_rx_ack(call, sp);
1102 rxrpc_inc_stat(call->rxnet, stat_rx_acks[summary.ack_reason]);
1103 prefetch(call->tx_queue);
1104
1105 /* If we get an EXCEEDS_WINDOW ACK from the server, it probably
1106 * indicates that the client address changed due to NAT. The server
1107 * lost the call because it switched to a different peer.
1108 */
1109 if (unlikely(summary.ack_reason == RXRPC_ACK_EXCEEDS_WINDOW) &&
1110 hard_ack == 0 &&
1111 prev_pkt == 0 &&
1112 rxrpc_is_client_call(call)) {
1113 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
1114 0, -ENETRESET);
1115 goto send_response;
1116 }
1117
1118 /* If we get an OUT_OF_SEQUENCE ACK from the server, that can also
1119 * indicate a change of address. However, we can retransmit the call
1120 * if we still have it buffered to the beginning.
1121 */
1122 if (unlikely(summary.ack_reason == RXRPC_ACK_OUT_OF_SEQUENCE) &&
1123 hard_ack == 0 &&
1124 prev_pkt == 0 &&
1125 call->tx_bottom == 0 &&
1126 rxrpc_is_client_call(call)) {
1127 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
1128 0, -ENETRESET);
1129 goto send_response;
1130 }
1131
1132 /* Discard any out-of-order or duplicate ACKs (outside lock). */
1133 if (!rxrpc_is_ack_valid(call, hard_ack, prev_pkt)) {
1134 trace_rxrpc_rx_discard_ack(call, summary.ack_serial, hard_ack, prev_pkt);
1135 goto send_response; /* Still respond if requested. */
1136 }
1137
1138 trailer.maxMTU = 0;
1139 ioffset = offset + nr_acks + 3;
1140 if (skb->len >= ioffset + sizeof(trailer) &&
1141 skb_copy_bits(skb, ioffset, &trailer, sizeof(trailer)) < 0)
1142 return rxrpc_proto_abort(call, 0, rxrpc_badmsg_short_ack_trailer);
1143
1144 call->acks_latest_ts = ktime_get_real();
1145 call->acks_hard_ack = hard_ack;
1146 call->acks_prev_seq = prev_pkt;
1147
1148 if (summary.acked_serial) {
1149 switch (summary.ack_reason) {
1150 case RXRPC_ACK_PING_RESPONSE:
1151 rxrpc_complete_rtt_probe(call, call->acks_latest_ts,
1152 summary.acked_serial, summary.ack_serial,
1153 rxrpc_rtt_rx_ping_response);
1154 break;
1155 default:
1156 if (after(summary.acked_serial, call->acks_highest_serial))
1157 call->acks_highest_serial = summary.acked_serial;
1158 summary.rtt_sample_avail = true;
1159 break;
1160 }
1161 }
1162
1163 /* Parse rwind and mtu sizes if provided. */
1164 if (trailer.maxMTU)
1165 rxrpc_input_ack_trailer(call, skb, &trailer);
1166
1167 if (hard_ack + 1 == 0)
1168 return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_zero);
1169
1170 /* Ignore ACKs unless we are transmitting or are waiting for
1171 * acknowledgement of the packets we've just been transmitting.
1172 */
1173 switch (__rxrpc_call_state(call)) {
1174 case RXRPC_CALL_CLIENT_SEND_REQUEST:
1175 case RXRPC_CALL_CLIENT_AWAIT_ACK:
1176 case RXRPC_CALL_SERVER_SEND_REPLY:
1177 case RXRPC_CALL_SERVER_AWAIT_ACK:
1178 break;
1179 default:
1180 goto send_response;
1181 }
1182
1183 if (before(hard_ack, call->tx_bottom) ||
1184 after(hard_ack, call->tx_top))
1185 return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_outside_window);
1186 if (nr_acks > call->tx_top - hard_ack)
1187 return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_sack_overflow);
1188
1189 if (after(hard_ack, call->tx_bottom)) {
1190 if (rxrpc_rotate_tx_window(call, hard_ack, &summary)) {
1191 rxrpc_end_tx_phase(call, false, rxrpc_eproto_unexpected_ack);
1192 goto send_response;
1193 }
1194 }
1195
1196 if (nr_acks > 0) {
1197 if (offset > (int)skb->len - nr_acks)
1198 return rxrpc_proto_abort(call, 0, rxrpc_eproto_ackr_short_sack);
1199 rxrpc_input_soft_acks(call, &summary, skb);
1200 }
1201
1202 if (test_bit(RXRPC_CALL_TX_LAST, &call->flags) &&
1203 call->acks_nr_sacks == call->tx_top - hard_ack &&
1204 rxrpc_is_client_call(call))
1205 rxrpc_propose_ping(call, summary.ack_serial,
1206 rxrpc_propose_ack_ping_for_lost_reply);
1207
1208 /* Drive the congestion management algorithm first and then RACK-TLP as
1209 * the latter depends on the state/change in state in the former.
1210 */
1211 rxrpc_congestion_management(call, &summary);
1212 rxrpc_rack_detect_loss_and_arm_timer(call, &summary);
1213 rxrpc_tlp_process_ack(call, &summary);
1214 if (call->tlp_serial && after_eq(summary.acked_serial, call->tlp_serial))
1215 call->tlp_serial = 0;
1216
1217send_response:
1218 if (summary.ack_reason == RXRPC_ACK_PING)
1219 rxrpc_send_ACK(call, RXRPC_ACK_PING_RESPONSE, summary.ack_serial,
1220 rxrpc_propose_ack_respond_to_ping);
1221 else if (sp->hdr.flags & RXRPC_REQUEST_ACK)
1222 rxrpc_send_ACK(call, RXRPC_ACK_REQUESTED, summary.ack_serial,
1223 rxrpc_propose_ack_respond_to_ack);
1224}
1225
1226/*
1227 * Process an ACKALL packet.
1228 */
1229static void rxrpc_input_ackall(struct rxrpc_call *call, struct sk_buff *skb)
1230{
1231 struct rxrpc_ack_summary summary = { 0 };
1232
1233 switch (__rxrpc_call_state(call)) {
1234 case RXRPC_CALL_CLIENT_SEND_REQUEST:
1235 case RXRPC_CALL_CLIENT_AWAIT_ACK:
1236 case RXRPC_CALL_SERVER_SEND_REPLY:
1237 case RXRPC_CALL_SERVER_AWAIT_ACK:
1238 break;
1239 default:
1240 return;
1241 }
1242
1243 if (rxrpc_rotate_tx_window(call, call->tx_transmitted, &summary))
1244 rxrpc_end_tx_phase(call, false, rxrpc_eproto_unexpected_ackall);
1245}
1246
1247/*
1248 * Process an ABORT packet directed at a call.
1249 */
1250static void rxrpc_input_abort(struct rxrpc_call *call, struct sk_buff *skb)
1251{
1252 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1253
1254 trace_rxrpc_rx_abort(call, sp->hdr.serial, skb->priority);
1255
1256 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
1257 skb->priority, -ECONNABORTED);
1258}
1259
1260/*
1261 * Process an incoming call packet.
1262 */
1263void rxrpc_input_call_packet(struct rxrpc_call *call, struct sk_buff *skb)
1264{
1265 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
1266 unsigned long timo;
1267
1268 _enter("%p,%p", call, skb);
1269
1270 if (sp->hdr.serviceId != call->dest_srx.srx_service)
1271 call->dest_srx.srx_service = sp->hdr.serviceId;
1272 if ((int)sp->hdr.serial - (int)call->rx_serial > 0)
1273 call->rx_serial = sp->hdr.serial;
1274 if (!test_bit(RXRPC_CALL_RX_HEARD, &call->flags))
1275 set_bit(RXRPC_CALL_RX_HEARD, &call->flags);
1276
1277 timo = READ_ONCE(call->next_rx_timo);
1278 if (timo) {
1279 ktime_t delay = ms_to_ktime(timo);
1280
1281 call->expect_rx_by = ktime_add(ktime_get_real(), delay);
1282 trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_expect_rx);
1283 }
1284
1285 switch (sp->hdr.type) {
1286 case RXRPC_PACKET_TYPE_DATA:
1287 return rxrpc_input_data(call, skb);
1288
1289 case RXRPC_PACKET_TYPE_ACK:
1290 return rxrpc_input_ack(call, skb);
1291
1292 case RXRPC_PACKET_TYPE_BUSY:
1293 /* Just ignore BUSY packets from the server; the retry and
1294 * lifespan timers will take care of business. BUSY packets
1295 * from the client don't make sense.
1296 */
1297 return;
1298
1299 case RXRPC_PACKET_TYPE_ABORT:
1300 return rxrpc_input_abort(call, skb);
1301
1302 case RXRPC_PACKET_TYPE_ACKALL:
1303 return rxrpc_input_ackall(call, skb);
1304
1305 default:
1306 break;
1307 }
1308}
1309
1310/*
1311 * Handle a new service call on a channel implicitly completing the preceding
1312 * call on that channel. This does not apply to client conns.
1313 *
1314 * TODO: If callNumber > call_id + 1, renegotiate security.
1315 */
1316void rxrpc_implicit_end_call(struct rxrpc_call *call, struct sk_buff *skb)
1317{
1318 switch (__rxrpc_call_state(call)) {
1319 case RXRPC_CALL_SERVER_AWAIT_ACK:
1320 rxrpc_call_completed(call);
1321 fallthrough;
1322 case RXRPC_CALL_COMPLETE:
1323 break;
1324 default:
1325 rxrpc_abort_call(call, 0, RX_CALL_DEAD, -ESHUTDOWN,
1326 rxrpc_eproto_improper_term);
1327 trace_rxrpc_improper_term(call);
1328 break;
1329 }
1330
1331 rxrpc_input_call_event(call);
1332}