linux - net/ceph/messenger

3819 lines · 97.5 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Ceph msgr2 protocol implementation
4	*
5	* Copyright (C) 2020 Ilya Dryomov <[email protected]>
6	*/
7
8	#include <linux/ceph/ceph_debug.h>
9
10	#include <crypto/aead.h>
11	#include <crypto/sha2.h>
12	#include <crypto/utils.h>
13	#include <linux/bvec.h>
14	#include <linux/crc32c.h>
15	#include <linux/net.h>
16	#include <linux/scatterlist.h>
17	#include <linux/socket.h>
18	#include <linux/sched/mm.h>
19	#include <net/sock.h>
20	#include <net/tcp.h>
21
22	#include <linux/ceph/ceph_features.h>
23	#include <linux/ceph/decode.h>
24	#include <linux/ceph/libceph.h>
25	#include <linux/ceph/messenger.h>
26
27	#include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
28
29	#define FRAME_TAG_HELLO 1
30	#define FRAME_TAG_AUTH_REQUEST 2
31	#define FRAME_TAG_AUTH_BAD_METHOD 3
32	#define FRAME_TAG_AUTH_REPLY_MORE 4
33	#define FRAME_TAG_AUTH_REQUEST_MORE 5
34	#define FRAME_TAG_AUTH_DONE 6
35	#define FRAME_TAG_AUTH_SIGNATURE 7
36	#define FRAME_TAG_CLIENT_IDENT 8
37	#define FRAME_TAG_SERVER_IDENT 9
38	#define FRAME_TAG_IDENT_MISSING_FEATURES 10
39	#define FRAME_TAG_SESSION_RECONNECT 11
40	#define FRAME_TAG_SESSION_RESET 12
41	#define FRAME_TAG_SESSION_RETRY 13
42	#define FRAME_TAG_SESSION_RETRY_GLOBAL 14
43	#define FRAME_TAG_SESSION_RECONNECT_OK 15
44	#define FRAME_TAG_WAIT 16
45	#define FRAME_TAG_MESSAGE 17
46	#define FRAME_TAG_KEEPALIVE2 18
47	#define FRAME_TAG_KEEPALIVE2_ACK 19
48	#define FRAME_TAG_ACK 20
49
50	#define FRAME_LATE_STATUS_ABORTED 0x1
51	#define FRAME_LATE_STATUS_COMPLETE 0xe
52	#define FRAME_LATE_STATUS_ABORTED_MASK 0xf
53
54	#define IN_S_HANDLE_PREAMBLE 1
55	#define IN_S_HANDLE_CONTROL 2
56	#define IN_S_HANDLE_CONTROL_REMAINDER 3
57	#define IN_S_PREPARE_READ_DATA 4
58	#define IN_S_PREPARE_READ_DATA_CONT 5
59	#define IN_S_PREPARE_READ_ENC_PAGE 6
60	#define IN_S_PREPARE_SPARSE_DATA 7
61	#define IN_S_PREPARE_SPARSE_DATA_CONT 8
62	#define IN_S_HANDLE_EPILOGUE 9
63	#define IN_S_FINISH_SKIP 10
64
65	#define OUT_S_QUEUE_DATA 1
66	#define OUT_S_QUEUE_DATA_CONT 2
67	#define OUT_S_QUEUE_ENC_PAGE 3
68	#define OUT_S_QUEUE_ZEROS 4
69	#define OUT_S_FINISH_MESSAGE 5
70	#define OUT_S_GET_NEXT 6
71
72	#define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
73	#define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
74	#define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
75	#define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
76
77	#define CEPH_MSG_FLAGS (MSG_DONTWAIT \| MSG_NOSIGNAL)
78
79	static int do_recvmsg(struct socket sock, struct iov_iter it)
80	{
81	struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
82	int ret;
83
84	msg.msg_iter = *it;
85	while (iov_iter_count(it)) {
86	ret = sock_recvmsg(sock, &msg, msg.msg_flags);
87	if (ret <= 0) {
88	if (ret == -EAGAIN)
89	ret = 0;
90	return ret;
91	}
92
93	iov_iter_advance(it, ret);
94	}
95
96	WARN_ON(msg_data_left(&msg));
97	return 1;
98	}
99
100	/*
101	* Read as much as possible.
102	*
103	* Return:
104	* 1 - done, nothing (else) to read
105	* 0 - socket is empty, need to wait
106	* <0 - error
107	*/
108	static int ceph_tcp_recv(struct ceph_connection *con)
109	{
110	int ret;
111
112	dout("%s con %p %s %zu\n", __func__, con,
113	iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
114	iov_iter_count(&con->v2.in_iter));
115	ret = do_recvmsg(con->sock, &con->v2.in_iter);
116	dout("%s con %p ret %d left %zu\n", __func__, con, ret,
117	iov_iter_count(&con->v2.in_iter));
118	return ret;
119	}
120
121	static int do_sendmsg(struct socket sock, struct iov_iter it)
122	{
123	struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
124	int ret;
125
126	msg.msg_iter = *it;
127	while (iov_iter_count(it)) {
128	ret = sock_sendmsg(sock, &msg);
129	if (ret <= 0) {
130	if (ret == -EAGAIN)
131	ret = 0;
132	return ret;
133	}
134
135	iov_iter_advance(it, ret);
136	}
137
138	WARN_ON(msg_data_left(&msg));
139	return 1;
140	}
141
142	static int do_try_sendpage(struct socket sock, struct iov_iter it)
143	{
144	struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
145	struct bio_vec bv;
146	int ret;
147
148	if (WARN_ON(!iov_iter_is_bvec(it)))
149	return -EINVAL;
150
151	while (iov_iter_count(it)) {
152	/* iov_iter_iovec() for ITER_BVEC */
153	bvec_set_page(&bv, it->bvec->bv_page,
154	min(iov_iter_count(it),
155	it->bvec->bv_len - it->iov_offset),
156	it->bvec->bv_offset + it->iov_offset);
157
158	/*
159	* MSG_SPLICE_PAGES cannot properly handle pages with
160	* page_count == 0, we need to fall back to sendmsg if
161	* that's the case.
162	*
163	* Same goes for slab pages: skb_can_coalesce() allows
164	* coalescing neighboring slab objects into a single frag
165	* which triggers one of hardened usercopy checks.
166	*/
167	if (sendpage_ok(bv.bv_page))
168	msg.msg_flags \|= MSG_SPLICE_PAGES;
169	else
170	msg.msg_flags &= ~MSG_SPLICE_PAGES;
171
172	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len);
173	ret = sock_sendmsg(sock, &msg);
174	if (ret <= 0) {
175	if (ret == -EAGAIN)
176	ret = 0;
177	return ret;
178	}
179
180	iov_iter_advance(it, ret);
181	}
182
183	return 1;
184	}
185
186	/*
187	* Write as much as possible. The socket is expected to be corked,
188	* so we don't bother with MSG_MORE here.
189	*
190	* Return:
191	* 1 - done, nothing (else) to write
192	* 0 - socket is full, need to wait
193	* <0 - error
194	*/
195	static int ceph_tcp_send(struct ceph_connection *con)
196	{
197	int ret;
198
199	dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
200	iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
201	if (con->v2.out_iter_sendpage)
202	ret = do_try_sendpage(con->sock, &con->v2.out_iter);
203	else
204	ret = do_sendmsg(con->sock, &con->v2.out_iter);
205	dout("%s con %p ret %d left %zu\n", __func__, con, ret,
206	iov_iter_count(&con->v2.out_iter));
207	return ret;
208	}
209
210	static void add_in_kvec(struct ceph_connection con, void buf, int len)
211	{
212	BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
213	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
214
215	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
216	con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
217	con->v2.in_kvec_cnt++;
218
219	con->v2.in_iter.nr_segs++;
220	con->v2.in_iter.count += len;
221	}
222
223	static void reset_in_kvecs(struct ceph_connection *con)
224	{
225	WARN_ON(iov_iter_count(&con->v2.in_iter));
226
227	con->v2.in_kvec_cnt = 0;
228	iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
229	}
230
231	static void set_in_bvec(struct ceph_connection con, const struct bio_vec bv)
232	{
233	WARN_ON(iov_iter_count(&con->v2.in_iter));
234
235	con->v2.in_bvec = *bv;
236	iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
237	}
238
239	static void set_in_skip(struct ceph_connection *con, int len)
240	{
241	WARN_ON(iov_iter_count(&con->v2.in_iter));
242
243	dout("%s con %p len %d\n", __func__, con, len);
244	iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
245	}
246
247	static void add_out_kvec(struct ceph_connection con, void buf, int len)
248	{
249	BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
250	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
251	WARN_ON(con->v2.out_zero);
252
253	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
254	con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
255	con->v2.out_kvec_cnt++;
256
257	con->v2.out_iter.nr_segs++;
258	con->v2.out_iter.count += len;
259	}
260
261	static void reset_out_kvecs(struct ceph_connection *con)
262	{
263	WARN_ON(iov_iter_count(&con->v2.out_iter));
264	WARN_ON(con->v2.out_zero);
265
266	con->v2.out_kvec_cnt = 0;
267
268	iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
269	con->v2.out_iter_sendpage = false;
270	}
271
272	static void set_out_bvec(struct ceph_connection con, const struct bio_vec bv,
273	bool zerocopy)
274	{
275	WARN_ON(iov_iter_count(&con->v2.out_iter));
276	WARN_ON(con->v2.out_zero);
277
278	con->v2.out_bvec = *bv;
279	con->v2.out_iter_sendpage = zerocopy;
280	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
281	con->v2.out_bvec.bv_len);
282	}
283
284	static void set_out_bvec_zero(struct ceph_connection *con)
285	{
286	WARN_ON(iov_iter_count(&con->v2.out_iter));
287	WARN_ON(!con->v2.out_zero);
288
289	bvec_set_page(&con->v2.out_bvec, ceph_zero_page,
290	min(con->v2.out_zero, (int)PAGE_SIZE), 0);
291	con->v2.out_iter_sendpage = true;
292	iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
293	con->v2.out_bvec.bv_len);
294	}
295
296	static void out_zero_add(struct ceph_connection *con, int len)
297	{
298	dout("%s con %p len %d\n", __func__, con, len);
299	con->v2.out_zero += len;
300	}
301
302	static void alloc_conn_buf(struct ceph_connection con, int len)
303	{
304	void *buf;
305
306	dout("%s con %p len %d\n", __func__, con, len);
307
308	if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
309	return NULL;
310
311	buf = kvmalloc(len, GFP_NOIO);
312	if (!buf)
313	return NULL;
314
315	con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
316	return buf;
317	}
318
319	static void free_conn_bufs(struct ceph_connection *con)
320	{
321	while (con->v2.conn_buf_cnt)
322	kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
323	}
324
325	static void add_in_sign_kvec(struct ceph_connection con, void buf, int len)
326	{
327	BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
328
329	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
330	con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
331	con->v2.in_sign_kvec_cnt++;
332	}
333
334	static void clear_in_sign_kvecs(struct ceph_connection *con)
335	{
336	con->v2.in_sign_kvec_cnt = 0;
337	}
338
339	static void add_out_sign_kvec(struct ceph_connection con, void buf, int len)
340	{
341	BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
342
343	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
344	con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
345	con->v2.out_sign_kvec_cnt++;
346	}
347
348	static void clear_out_sign_kvecs(struct ceph_connection *con)
349	{
350	con->v2.out_sign_kvec_cnt = 0;
351	}
352
353	static bool con_secure(struct ceph_connection *con)
354	{
355	return con->v2.con_mode == CEPH_CON_MODE_SECURE;
356	}
357
358	static int front_len(const struct ceph_msg *msg)
359	{
360	return le32_to_cpu(msg->hdr.front_len);
361	}
362
363	static int middle_len(const struct ceph_msg *msg)
364	{
365	return le32_to_cpu(msg->hdr.middle_len);
366	}
367
368	static int data_len(const struct ceph_msg *msg)
369	{
370	return le32_to_cpu(msg->hdr.data_len);
371	}
372
373	static bool need_padding(int len)
374	{
375	return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
376	}
377
378	static int padded_len(int len)
379	{
380	return ALIGN(len, CEPH_GCM_BLOCK_LEN);
381	}
382
383	static int padding_len(int len)
384	{
385	return padded_len(len) - len;
386	}
387
388	/* preamble + control segment */
389	static int head_onwire_len(int ctrl_len, bool secure)
390	{
391	int head_len;
392	int rem_len;
393
394	BUG_ON(ctrl_len < 1 \|\| ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
395
396	if (secure) {
397	head_len = CEPH_PREAMBLE_SECURE_LEN;
398	if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
399	rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
400	head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
401	}
402	} else {
403	head_len = CEPH_PREAMBLE_PLAIN_LEN + ctrl_len + CEPH_CRC_LEN;
404	}
405	return head_len;
406	}
407
408	/* front, middle and data segments + epilogue */
409	static int __tail_onwire_len(int front_len, int middle_len, int data_len,
410	bool secure)
411	{
412	BUG_ON(front_len < 0 \|\| front_len > CEPH_MSG_MAX_FRONT_LEN \|\|
413	middle_len < 0 \|\| middle_len > CEPH_MSG_MAX_MIDDLE_LEN \|\|
414	data_len < 0 \|\| data_len > CEPH_MSG_MAX_DATA_LEN);
415
416	if (!front_len && !middle_len && !data_len)
417	return 0;
418
419	if (!secure)
420	return front_len + middle_len + data_len +
421	CEPH_EPILOGUE_PLAIN_LEN;
422
423	return padded_len(front_len) + padded_len(middle_len) +
424	padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
425	}
426
427	static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
428	{
429	return __tail_onwire_len(front_len(msg), middle_len(msg),
430	data_len(msg), secure);
431	}
432
433	/* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
434	#define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
435	sizeof(struct ceph_msg_header2) + \
436	CEPH_CRC_LEN)
437
438	static const int frame_aligns[] = {
439	sizeof(void *),
440	sizeof(void *),
441	sizeof(void *),
442	PAGE_SIZE
443	};
444
445	/*
446	* Discards trailing empty segments, unless there is just one segment.
447	* A frame always has at least one (possibly empty) segment.
448	*/
449	static int calc_segment_count(const int *lens, int len_cnt)
450	{
451	int i;
452
453	for (i = len_cnt - 1; i >= 0; i--) {
454	if (lens[i])
455	return i + 1;
456	}
457
458	return 1;
459	}
460
461	static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
462	const int *lens, int len_cnt)
463	{
464	int i;
465
466	memset(desc, 0, sizeof(*desc));
467
468	desc->fd_tag = tag;
469	desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
470	BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
471	for (i = 0; i < desc->fd_seg_cnt; i++) {
472	desc->fd_lens[i] = lens[i];
473	desc->fd_aligns[i] = frame_aligns[i];
474	}
475	}
476
477	/*
478	* Preamble crc covers everything up to itself (28 bytes) and
479	* is calculated and verified irrespective of the connection mode
480	* (i.e. even if the frame is encrypted).
481	*/
482	static void encode_preamble(const struct ceph_frame_desc desc, void p)
483	{
484	void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
485	void *start = p;
486	int i;
487
488	memset(p, 0, CEPH_PREAMBLE_LEN);
489
490	ceph_encode_8(&p, desc->fd_tag);
491	ceph_encode_8(&p, desc->fd_seg_cnt);
492	for (i = 0; i < desc->fd_seg_cnt; i++) {
493	ceph_encode_32(&p, desc->fd_lens[i]);
494	ceph_encode_16(&p, desc->fd_aligns[i]);
495	}
496
497	put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
498	}
499
500	static int decode_preamble(void p, struct ceph_frame_desc desc)
501	{
502	void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
503	u32 crc, expected_crc;
504	int i;
505
506	crc = crc32c(0, p, crcp - p);
507	expected_crc = get_unaligned_le32(crcp);
508	if (crc != expected_crc) {
509	pr_err("bad preamble crc, calculated %u, expected %u\n",
510	crc, expected_crc);
511	return -EBADMSG;
512	}
513
514	memset(desc, 0, sizeof(*desc));
515
516	desc->fd_tag = ceph_decode_8(&p);
517	desc->fd_seg_cnt = ceph_decode_8(&p);
518	if (desc->fd_seg_cnt < 1 \|\|
519	desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
520	pr_err("bad segment count %d\n", desc->fd_seg_cnt);
521	return -EINVAL;
522	}
523	for (i = 0; i < desc->fd_seg_cnt; i++) {
524	desc->fd_lens[i] = ceph_decode_32(&p);
525	desc->fd_aligns[i] = ceph_decode_16(&p);
526	}
527
528	/*
529	* This would fire for FRAME_TAG_WAIT (it has one empty
530	* segment), but we should never get it as client.
531	*/
532	if (desc->fd_lens[0] < 1 \|\|
533	desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
534	pr_err("bad control segment length %d\n", desc->fd_lens[0]);
535	return -EINVAL;
536	}
537
538	if (desc->fd_lens[1] < 0 \|\|
539	desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
540	pr_err("bad front segment length %d\n", desc->fd_lens[1]);
541	return -EINVAL;
542	}
543	if (desc->fd_lens[2] < 0 \|\|
544	desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
545	pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
546	return -EINVAL;
547	}
548	if (desc->fd_lens[3] < 0 \|\|
549	desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
550	pr_err("bad data segment length %d\n", desc->fd_lens[3]);
551	return -EINVAL;
552	}
553
554	if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
555	pr_err("last segment empty, segment count %d\n",
556	desc->fd_seg_cnt);
557	return -EINVAL;
558	}
559
560	return 0;
561	}
562
563	static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
564	{
565	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
566	FRAME_LATE_STATUS_COMPLETE;
567	cpu_to_le32s(&con->v2.out_epil.front_crc);
568	cpu_to_le32s(&con->v2.out_epil.middle_crc);
569	cpu_to_le32s(&con->v2.out_epil.data_crc);
570	}
571
572	static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
573	{
574	memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
575	con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
576	FRAME_LATE_STATUS_COMPLETE;
577	}
578
579	static int decode_epilogue(void p, u32 front_crc, u32 *middle_crc,
580	u32 *data_crc)
581	{
582	u8 late_status;
583
584	late_status = ceph_decode_8(&p);
585	if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
586	FRAME_LATE_STATUS_COMPLETE) {
587	/* we should never get an aborted message as client */
588	pr_err("bad late_status 0x%x\n", late_status);
589	return -EINVAL;
590	}
591
592	if (front_crc && middle_crc && data_crc) {
593	*front_crc = ceph_decode_32(&p);
594	*middle_crc = ceph_decode_32(&p);
595	*data_crc = ceph_decode_32(&p);
596	}
597
598	return 0;
599	}
600
601	static void fill_header(struct ceph_msg_header *hdr,
602	const struct ceph_msg_header2 *hdr2,
603	int front_len, int middle_len, int data_len,
604	const struct ceph_entity_name *peer_name)
605	{
606	hdr->seq = hdr2->seq;
607	hdr->tid = hdr2->tid;
608	hdr->type = hdr2->type;
609	hdr->priority = hdr2->priority;
610	hdr->version = hdr2->version;
611	hdr->front_len = cpu_to_le32(front_len);
612	hdr->middle_len = cpu_to_le32(middle_len);
613	hdr->data_len = cpu_to_le32(data_len);
614	hdr->data_off = hdr2->data_off;
615	hdr->src = *peer_name;
616	hdr->compat_version = hdr2->compat_version;
617	hdr->reserved = 0;
618	hdr->crc = 0;
619	}
620
621	static void fill_header2(struct ceph_msg_header2 *hdr2,
622	const struct ceph_msg_header *hdr, u64 ack_seq)
623	{
624	hdr2->seq = hdr->seq;
625	hdr2->tid = hdr->tid;
626	hdr2->type = hdr->type;
627	hdr2->priority = hdr->priority;
628	hdr2->version = hdr->version;
629	hdr2->data_pre_padding_len = 0;
630	hdr2->data_off = hdr->data_off;
631	hdr2->ack_seq = cpu_to_le64(ack_seq);
632	hdr2->flags = 0;
633	hdr2->compat_version = hdr->compat_version;
634	hdr2->reserved = 0;
635	}
636
637	static int verify_control_crc(struct ceph_connection *con)
638	{
639	int ctrl_len = con->v2.in_desc.fd_lens[0];
640	u32 crc, expected_crc;
641
642	WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
643	WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
644
645	crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
646	expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
647	if (crc != expected_crc) {
648	pr_err("bad control crc, calculated %u, expected %u\n",
649	crc, expected_crc);
650	return -EBADMSG;
651	}
652
653	return 0;
654	}
655
656	static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
657	u32 middle_crc, u32 data_crc)
658	{
659	if (front_len(con->in_msg)) {
660	con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
661	front_len(con->in_msg));
662	} else {
663	WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
664	con->in_front_crc = -1;
665	}
666
667	if (middle_len(con->in_msg))
668	con->in_middle_crc = crc32c(-1,
669	con->in_msg->middle->vec.iov_base,
670	middle_len(con->in_msg));
671	else if (data_len(con->in_msg))
672	con->in_middle_crc = -1;
673	else
674	con->in_middle_crc = 0;
675
676	if (!data_len(con->in_msg))
677	con->in_data_crc = 0;
678
679	dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
680	con->in_front_crc, con->in_middle_crc, con->in_data_crc);
681
682	if (con->in_front_crc != front_crc) {
683	pr_err("bad front crc, calculated %u, expected %u\n",
684	con->in_front_crc, front_crc);
685	return -EBADMSG;
686	}
687	if (con->in_middle_crc != middle_crc) {
688	pr_err("bad middle crc, calculated %u, expected %u\n",
689	con->in_middle_crc, middle_crc);
690	return -EBADMSG;
691	}
692	if (con->in_data_crc != data_crc) {
693	pr_err("bad data crc, calculated %u, expected %u\n",
694	con->in_data_crc, data_crc);
695	return -EBADMSG;
696	}
697
698	return 0;
699	}
700
701	static int setup_crypto(struct ceph_connection *con,
702	const u8 *session_key, int session_key_len,
703	const u8 *con_secret, int con_secret_len)
704	{
705	unsigned int noio_flag;
706	int ret;
707
708	dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
709	__func__, con, con->v2.con_mode, session_key_len, con_secret_len);
710	WARN_ON(con->v2.hmac_key_set \|\| con->v2.gcm_tfm \|\| con->v2.gcm_req);
711
712	if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
713	con->v2.con_mode != CEPH_CON_MODE_SECURE) {
714	pr_err("bad con_mode %d\n", con->v2.con_mode);
715	return -EINVAL;
716	}
717
718	if (!session_key_len) {
719	WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
720	WARN_ON(con_secret_len);
721	return 0; /* auth_none */
722	}
723
724	hmac_sha256_preparekey(&con->v2.hmac_key, session_key, session_key_len);
725	con->v2.hmac_key_set = true;
726
727	if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
728	WARN_ON(con_secret_len);
729	return 0; /* auth_x, plain mode */
730	}
731
732	if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
733	pr_err("con_secret too small %d\n", con_secret_len);
734	return -EINVAL;
735	}
736
737	noio_flag = memalloc_noio_save();
738	con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
739	memalloc_noio_restore(noio_flag);
740	if (IS_ERR(con->v2.gcm_tfm)) {
741	ret = PTR_ERR(con->v2.gcm_tfm);
742	con->v2.gcm_tfm = NULL;
743	pr_err("failed to allocate gcm tfm context: %d\n", ret);
744	return ret;
745	}
746
747	WARN_ON((unsigned long)con_secret &
748	crypto_aead_alignmask(con->v2.gcm_tfm));
749	ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
750	if (ret) {
751	pr_err("failed to set gcm key: %d\n", ret);
752	return ret;
753	}
754
755	WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
756	ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
757	if (ret) {
758	pr_err("failed to set gcm tag size: %d\n", ret);
759	return ret;
760	}
761
762	con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
763	if (!con->v2.gcm_req) {
764	pr_err("failed to allocate gcm request\n");
765	return -ENOMEM;
766	}
767
768	crypto_init_wait(&con->v2.gcm_wait);
769	aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
770	crypto_req_done, &con->v2.gcm_wait);
771
772	memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
773	CEPH_GCM_IV_LEN);
774	memcpy(&con->v2.out_gcm_nonce,
775	con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
776	CEPH_GCM_IV_LEN);
777	return 0; /* auth_x, secure mode */
778	}
779
780	static void con_hmac_sha256(struct ceph_connection *con,
781	const struct kvec *kvecs, int kvec_cnt,
782	u8 hmac[SHA256_DIGEST_SIZE])
783	{
784	struct hmac_sha256_ctx ctx;
785	int i;
786
787	dout("%s con %p hmac_key_set %d kvec_cnt %d\n", __func__, con,
788	con->v2.hmac_key_set, kvec_cnt);
789
790	if (!con->v2.hmac_key_set) {
791	memset(hmac, 0, SHA256_DIGEST_SIZE);
792	return; /* auth_none */
793	}
794
795	/* auth_x, both plain and secure modes */
796	hmac_sha256_init(&ctx, &con->v2.hmac_key);
797	for (i = 0; i < kvec_cnt; i++)
798	hmac_sha256_update(&ctx, kvecs[i].iov_base, kvecs[i].iov_len);
799	hmac_sha256_final(&ctx, hmac);
800	}
801
802	static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
803	{
804	u64 counter;
805
806	counter = le64_to_cpu(nonce->counter);
807	nonce->counter = cpu_to_le64(counter + 1);
808	}
809
810	static int gcm_crypt(struct ceph_connection *con, bool encrypt,
811	struct scatterlist src, struct scatterlist dst,
812	int src_len)
813	{
814	struct ceph_gcm_nonce *nonce;
815	int ret;
816
817	nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
818
819	aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */
820	aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
821	ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
822	crypto_aead_decrypt(con->v2.gcm_req),
823	&con->v2.gcm_wait);
824	if (ret)
825	return ret;
826
827	gcm_inc_nonce(nonce);
828	return 0;
829	}
830
831	static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
832	struct bio_vec *bv)
833	{
834	struct page *page;
835	size_t off, len;
836
837	WARN_ON(!cursor->total_resid);
838
839	/* skip zero-length data items */
840	while (!cursor->resid)
841	ceph_msg_data_advance(cursor, 0);
842
843	/* get a piece of data, cursor isn't advanced */
844	page = ceph_msg_data_next(cursor, &off, &len);
845	bvec_set_page(bv, page, len, off);
846	}
847
848	static int calc_sg_cnt(void *buf, int buf_len)
849	{
850	int sg_cnt;
851
852	if (!buf_len)
853	return 0;
854
855	sg_cnt = need_padding(buf_len) ? 1 : 0;
856	if (is_vmalloc_addr(buf)) {
857	WARN_ON(offset_in_page(buf));
858	sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
859	} else {
860	sg_cnt++;
861	}
862
863	return sg_cnt;
864	}
865
866	static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
867	{
868	int data_len = cursor->total_resid;
869	struct bio_vec bv;
870	int sg_cnt;
871
872	if (!data_len)
873	return 0;
874
875	sg_cnt = need_padding(data_len) ? 1 : 0;
876	do {
877	get_bvec_at(cursor, &bv);
878	sg_cnt++;
879
880	ceph_msg_data_advance(cursor, bv.bv_len);
881	} while (cursor->total_resid);
882
883	return sg_cnt;
884	}
885
886	static void init_sgs(struct scatterlist *sg, void buf, int buf_len, u8 *pad)
887	{
888	void *end = buf + buf_len;
889	struct page *page;
890	int len;
891	void *p;
892
893	if (!buf_len)
894	return;
895
896	if (is_vmalloc_addr(buf)) {
897	p = buf;
898	do {
899	page = vmalloc_to_page(p);
900	len = min_t(int, end - p, PAGE_SIZE);
901	WARN_ON(!page \|\| !len \|\| offset_in_page(p));
902	sg_set_page(*sg, page, len, 0);
903	sg = sg_next(sg);
904	p += len;
905	} while (p != end);
906	} else {
907	sg_set_buf(*sg, buf, buf_len);
908	sg = sg_next(sg);
909	}
910
911	if (need_padding(buf_len)) {
912	sg_set_buf(*sg, pad, padding_len(buf_len));
913	sg = sg_next(sg);
914	}
915	}
916
917	static void init_sgs_cursor(struct scatterlist **sg,
918	struct ceph_msg_data_cursor cursor, u8 pad)
919	{
920	int data_len = cursor->total_resid;
921	struct bio_vec bv;
922
923	if (!data_len)
924	return;
925
926	do {
927	get_bvec_at(cursor, &bv);
928	sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
929	sg = sg_next(sg);
930
931	ceph_msg_data_advance(cursor, bv.bv_len);
932	} while (cursor->total_resid);
933
934	if (need_padding(data_len)) {
935	sg_set_buf(*sg, pad, padding_len(data_len));
936	sg = sg_next(sg);
937	}
938	}
939
940	/**
941	* init_sgs_pages: set up scatterlist on an array of page pointers
942	* @sg: scatterlist to populate
943	* @pages: pointer to page array
944	* @dpos: position in the array to start (bytes)
945	* @dlen: len to add to sg (bytes)
946	* @pad: pointer to pad destination (if any)
947	*
948	* Populate the scatterlist from the page array, starting at an arbitrary
949	* byte in the array and running for a specified length.
950	*/
951	static void init_sgs_pages(struct scatterlist sg, struct page pages,
952	int dpos, int dlen, u8 *pad)
953	{
954	int idx = dpos >> PAGE_SHIFT;
955	int off = offset_in_page(dpos);
956	int resid = dlen;
957
958	do {
959	int len = min(resid, (int)PAGE_SIZE - off);
960
961	sg_set_page(*sg, pages[idx], len, off);
962	sg = sg_next(sg);
963	off = 0;
964	++idx;
965	resid -= len;
966	} while (resid);
967
968	if (need_padding(dlen)) {
969	sg_set_buf(*sg, pad, padding_len(dlen));
970	sg = sg_next(sg);
971	}
972	}
973
974	static int setup_message_sgs(struct sg_table sgt, struct ceph_msg msg,
975	u8 front_pad, u8 middle_pad, u8 *data_pad,
976	void epilogue, struct page *pages, int dpos,
977	bool add_tag)
978	{
979	struct ceph_msg_data_cursor cursor;
980	struct scatterlist *cur_sg;
981	int dlen = data_len(msg);
982	int sg_cnt;
983	int ret;
984
985	if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
986	return 0;
987
988	sg_cnt = 1; /* epilogue + [auth tag] */
989	if (front_len(msg))
990	sg_cnt += calc_sg_cnt(msg->front.iov_base,
991	front_len(msg));
992	if (middle_len(msg))
993	sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
994	middle_len(msg));
995	if (dlen) {
996	if (pages) {
997	sg_cnt += calc_pages_for(dpos, dlen);
998	if (need_padding(dlen))
999	sg_cnt++;
1000	} else {
1001	ceph_msg_data_cursor_init(&cursor, msg, dlen);
1002	sg_cnt += calc_sg_cnt_cursor(&cursor);
1003	}
1004	}
1005
1006	ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1007	if (ret)
1008	return ret;
1009
1010	cur_sg = sgt->sgl;
1011	if (front_len(msg))
1012	init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1013	front_pad);
1014	if (middle_len(msg))
1015	init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1016	middle_pad);
1017	if (dlen) {
1018	if (pages) {
1019	init_sgs_pages(&cur_sg, pages, dpos, dlen, data_pad);
1020	} else {
1021	ceph_msg_data_cursor_init(&cursor, msg, dlen);
1022	init_sgs_cursor(&cur_sg, &cursor, data_pad);
1023	}
1024	}
1025
1026	WARN_ON(!sg_is_last(cur_sg));
1027	sg_set_buf(cur_sg, epilogue,
1028	CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1029	return 0;
1030	}
1031
1032	static int decrypt_preamble(struct ceph_connection *con)
1033	{
1034	struct scatterlist sg;
1035
1036	sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1037	return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1038	}
1039
1040	static int decrypt_control_remainder(struct ceph_connection *con)
1041	{
1042	int ctrl_len = con->v2.in_desc.fd_lens[0];
1043	int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1044	int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1045	struct scatterlist sgs[2];
1046
1047	WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1048	WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1049
1050	sg_init_table(sgs, 2);
1051	sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1052	sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1053
1054	return gcm_crypt(con, false, sgs, sgs,
1055	padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1056	}
1057
1058	/* Process sparse read data that lives in a buffer */
1059	static int process_v2_sparse_read(struct ceph_connection *con,
1060	struct page **pages, int spos)
1061	{
1062	struct ceph_msg_data_cursor cursor;
1063	int ret;
1064
1065	ceph_msg_data_cursor_init(&cursor, con->in_msg,
1066	con->in_msg->sparse_read_total);
1067
1068	for (;;) {
1069	char *buf = NULL;
1070
1071	ret = con->ops->sparse_read(con, &cursor, &buf);
1072	if (ret <= 0)
1073	return ret;
1074
1075	dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf);
1076
1077	do {
1078	int idx = spos >> PAGE_SHIFT;
1079	int soff = offset_in_page(spos);
1080	struct page *spage = con->v2.in_enc_pages[idx];
1081	int len = min_t(int, ret, PAGE_SIZE - soff);
1082
1083	if (buf) {
1084	memcpy_from_page(buf, spage, soff, len);
1085	buf += len;
1086	} else {
1087	struct bio_vec bv;
1088
1089	get_bvec_at(&cursor, &bv);
1090	len = min_t(int, len, bv.bv_len);
1091	memcpy_page(bv.bv_page, bv.bv_offset,
1092	spage, soff, len);
1093	ceph_msg_data_advance(&cursor, len);
1094	}
1095	spos += len;
1096	ret -= len;
1097	} while (ret);
1098	}
1099	}
1100
1101	static int decrypt_tail(struct ceph_connection *con)
1102	{
1103	struct sg_table enc_sgt = {};
1104	struct sg_table sgt = {};
1105	struct page **pages = NULL;
1106	bool sparse = !!con->in_msg->sparse_read_total;
1107	int dpos = 0;
1108	int tail_len;
1109	int ret;
1110
1111	tail_len = tail_onwire_len(con->in_msg, true);
1112	ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
1113	con->v2.in_enc_page_cnt, 0, tail_len,
1114	GFP_NOIO);
1115	if (ret)
1116	goto out;
1117
1118	if (sparse) {
1119	dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg)));
1120	pages = con->v2.in_enc_pages;
1121	}
1122
1123	ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1124	MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1125	con->v2.in_buf, pages, dpos, true);
1126	if (ret)
1127	goto out;
1128
1129	dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1130	con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1131	ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1132	if (ret)
1133	goto out;
1134
1135	if (sparse && data_len(con->in_msg)) {
1136	ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos);
1137	if (ret)
1138	goto out;
1139	}
1140
1141	WARN_ON(!con->v2.in_enc_page_cnt);
1142	ceph_release_page_vector(con->v2.in_enc_pages,
1143	con->v2.in_enc_page_cnt);
1144	con->v2.in_enc_pages = NULL;
1145	con->v2.in_enc_page_cnt = 0;
1146
1147	out:
1148	sg_free_table(&sgt);
1149	sg_free_table(&enc_sgt);
1150	return ret;
1151	}
1152
1153	static int prepare_banner(struct ceph_connection *con)
1154	{
1155	int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1156	void buf, p;
1157
1158	buf = alloc_conn_buf(con, buf_len);
1159	if (!buf)
1160	return -ENOMEM;
1161
1162	p = buf;
1163	ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1164	ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1165	ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1166	ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1167	WARN_ON(p != buf + buf_len);
1168
1169	add_out_kvec(con, buf, buf_len);
1170	add_out_sign_kvec(con, buf, buf_len);
1171	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1172	return 0;
1173	}
1174
1175	/*
1176	* base:
1177	* preamble
1178	* control body (ctrl_len bytes)
1179	* space for control crc
1180	*
1181	* extdata (optional):
1182	* control body (extdata_len bytes)
1183	*
1184	* Compute control crc and gather base and extdata into:
1185	*
1186	* preamble
1187	* control body (ctrl_len + extdata_len bytes)
1188	* control crc
1189	*
1190	* Preamble should already be encoded at the start of base.
1191	*/
1192	static void prepare_head_plain(struct ceph_connection con, void base,
1193	int ctrl_len, void *extdata, int extdata_len,
1194	bool to_be_signed)
1195	{
1196	int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1197	void *crcp = base + base_len - CEPH_CRC_LEN;
1198	u32 crc;
1199
1200	crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1201	if (extdata_len)
1202	crc = crc32c(crc, extdata, extdata_len);
1203	put_unaligned_le32(crc, crcp);
1204
1205	if (!extdata_len) {
1206	add_out_kvec(con, base, base_len);
1207	if (to_be_signed)
1208	add_out_sign_kvec(con, base, base_len);
1209	return;
1210	}
1211
1212	add_out_kvec(con, base, crcp - base);
1213	add_out_kvec(con, extdata, extdata_len);
1214	add_out_kvec(con, crcp, CEPH_CRC_LEN);
1215	if (to_be_signed) {
1216	add_out_sign_kvec(con, base, crcp - base);
1217	add_out_sign_kvec(con, extdata, extdata_len);
1218	add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1219	}
1220	}
1221
1222	static int prepare_head_secure_small(struct ceph_connection *con,
1223	void *base, int ctrl_len)
1224	{
1225	struct scatterlist sg;
1226	int ret;
1227
1228	/* inline buffer padding? */
1229	if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1230	memset(CTRL_BODY(base) + ctrl_len, 0,
1231	CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1232
1233	sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1234	ret = gcm_crypt(con, true, &sg, &sg,
1235	CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1236	if (ret)
1237	return ret;
1238
1239	add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1240	return 0;
1241	}
1242
1243	/*
1244	* base:
1245	* preamble
1246	* control body (ctrl_len bytes)
1247	* space for padding, if needed
1248	* space for control remainder auth tag
1249	* space for preamble auth tag
1250	*
1251	* Encrypt preamble and the inline portion, then encrypt the remainder
1252	* and gather into:
1253	*
1254	* preamble
1255	* control body (48 bytes)
1256	* preamble auth tag
1257	* control body (ctrl_len - 48 bytes)
1258	* zero padding, if needed
1259	* control remainder auth tag
1260	*
1261	* Preamble should already be encoded at the start of base.
1262	*/
1263	static int prepare_head_secure_big(struct ceph_connection *con,
1264	void *base, int ctrl_len)
1265	{
1266	int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1267	void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1268	void *rem_tag = rem + padded_len(rem_len);
1269	void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1270	struct scatterlist sgs[2];
1271	int ret;
1272
1273	sg_init_table(sgs, 2);
1274	sg_set_buf(&sgs[0], base, rem - base);
1275	sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1276	ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1277	if (ret)
1278	return ret;
1279
1280	/* control remainder padding? */
1281	if (need_padding(rem_len))
1282	memset(rem + rem_len, 0, padding_len(rem_len));
1283
1284	sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1285	ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1286	if (ret)
1287	return ret;
1288
1289	add_out_kvec(con, base, rem - base);
1290	add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1291	add_out_kvec(con, rem, pmbl_tag - rem);
1292	return 0;
1293	}
1294
1295	static int __prepare_control(struct ceph_connection *con, int tag,
1296	void base, int ctrl_len, void extdata,
1297	int extdata_len, bool to_be_signed)
1298	{
1299	int total_len = ctrl_len + extdata_len;
1300	struct ceph_frame_desc desc;
1301	int ret;
1302
1303	dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1304	total_len, ctrl_len, extdata_len);
1305
1306	/* extdata may be vmalloc'ed but not base */
1307	if (WARN_ON(is_vmalloc_addr(base) \|\| !ctrl_len))
1308	return -EINVAL;
1309
1310	init_frame_desc(&desc, tag, &total_len, 1);
1311	encode_preamble(&desc, base);
1312
1313	if (con_secure(con)) {
1314	if (WARN_ON(extdata_len \|\| to_be_signed))
1315	return -EINVAL;
1316
1317	if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1318	/* fully inlined, inline buffer may need padding */
1319	ret = prepare_head_secure_small(con, base, ctrl_len);
1320	else
1321	/* partially inlined, inline buffer is full */
1322	ret = prepare_head_secure_big(con, base, ctrl_len);
1323	if (ret)
1324	return ret;
1325	} else {
1326	prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1327	to_be_signed);
1328	}
1329
1330	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1331	return 0;
1332	}
1333
1334	static int prepare_control(struct ceph_connection *con, int tag,
1335	void *base, int ctrl_len)
1336	{
1337	return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1338	}
1339
1340	static int prepare_hello(struct ceph_connection *con)
1341	{
1342	void buf, p;
1343	int ctrl_len;
1344
1345	ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1346	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1347	if (!buf)
1348	return -ENOMEM;
1349
1350	p = CTRL_BODY(buf);
1351	ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1352	ceph_encode_entity_addr(&p, &con->peer_addr);
1353	WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1354
1355	return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1356	NULL, 0, true);
1357	}
1358
1359	/* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1360	#define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1361
1362	static int prepare_auth_request(struct ceph_connection *con)
1363	{
1364	void authorizer, authorizer_copy;
1365	int ctrl_len, authorizer_len;
1366	void *buf;
1367	int ret;
1368
1369	ctrl_len = AUTH_BUF_LEN;
1370	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1371	if (!buf)
1372	return -ENOMEM;
1373
1374	mutex_unlock(&con->mutex);
1375	ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1376	&authorizer, &authorizer_len);
1377	mutex_lock(&con->mutex);
1378	if (con->state != CEPH_CON_S_V2_HELLO) {
1379	dout("%s con %p state changed to %d\n", __func__, con,
1380	con->state);
1381	return -EAGAIN;
1382	}
1383
1384	dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1385	if (ret)
1386	return ret;
1387
1388	authorizer_copy = alloc_conn_buf(con, authorizer_len);
1389	if (!authorizer_copy)
1390	return -ENOMEM;
1391
1392	memcpy(authorizer_copy, authorizer, authorizer_len);
1393
1394	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1395	authorizer_copy, authorizer_len, true);
1396	}
1397
1398	static int prepare_auth_request_more(struct ceph_connection *con,
1399	void *reply, int reply_len)
1400	{
1401	int ctrl_len, authorizer_len;
1402	void *authorizer;
1403	void *buf;
1404	int ret;
1405
1406	ctrl_len = AUTH_BUF_LEN;
1407	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1408	if (!buf)
1409	return -ENOMEM;
1410
1411	mutex_unlock(&con->mutex);
1412	ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1413	CTRL_BODY(buf), &ctrl_len,
1414	&authorizer, &authorizer_len);
1415	mutex_lock(&con->mutex);
1416	if (con->state != CEPH_CON_S_V2_AUTH) {
1417	dout("%s con %p state changed to %d\n", __func__, con,
1418	con->state);
1419	return -EAGAIN;
1420	}
1421
1422	dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1423	if (ret)
1424	return ret;
1425
1426	return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1427	ctrl_len, authorizer, authorizer_len, true);
1428	}
1429
1430	static int prepare_auth_signature(struct ceph_connection *con)
1431	{
1432	void *buf;
1433
1434	buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1435	con_secure(con)));
1436	if (!buf)
1437	return -ENOMEM;
1438
1439	con_hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1440	CTRL_BODY(buf));
1441
1442	return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1443	SHA256_DIGEST_SIZE);
1444	}
1445
1446	static int prepare_client_ident(struct ceph_connection *con)
1447	{
1448	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1449	struct ceph_client *client = from_msgr(con->msgr);
1450	u64 global_id = ceph_client_gid(client);
1451	void buf, p;
1452	int ctrl_len;
1453
1454	WARN_ON(con->v2.server_cookie);
1455	WARN_ON(con->v2.connect_seq);
1456	WARN_ON(con->v2.peer_global_seq);
1457
1458	if (!con->v2.client_cookie) {
1459	do {
1460	get_random_bytes(&con->v2.client_cookie,
1461	sizeof(con->v2.client_cookie));
1462	} while (!con->v2.client_cookie);
1463	dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1464	con->v2.client_cookie);
1465	} else {
1466	dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1467	con->v2.client_cookie);
1468	}
1469
1470	dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1471	__func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1472	ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1473	global_id, con->v2.global_seq, client->supported_features,
1474	client->required_features, con->v2.client_cookie);
1475
1476	ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1477	ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1478	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1479	if (!buf)
1480	return -ENOMEM;
1481
1482	p = CTRL_BODY(buf);
1483	ceph_encode_8(&p, 2); /* addrvec marker */
1484	ceph_encode_32(&p, 1); /* addr_cnt */
1485	ceph_encode_entity_addr(&p, my_addr);
1486	ceph_encode_entity_addr(&p, &con->peer_addr);
1487	ceph_encode_64(&p, global_id);
1488	ceph_encode_64(&p, con->v2.global_seq);
1489	ceph_encode_64(&p, client->supported_features);
1490	ceph_encode_64(&p, client->required_features);
1491	ceph_encode_64(&p, 0); /* flags */
1492	ceph_encode_64(&p, con->v2.client_cookie);
1493	WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1494
1495	return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1496	}
1497
1498	static int prepare_session_reconnect(struct ceph_connection *con)
1499	{
1500	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1501	void buf, p;
1502	int ctrl_len;
1503
1504	WARN_ON(!con->v2.client_cookie);
1505	WARN_ON(!con->v2.server_cookie);
1506	WARN_ON(!con->v2.connect_seq);
1507	WARN_ON(!con->v2.peer_global_seq);
1508
1509	dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1510	__func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1511	con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1512	con->v2.connect_seq, con->in_seq);
1513
1514	ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1515	buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1516	if (!buf)
1517	return -ENOMEM;
1518
1519	p = CTRL_BODY(buf);
1520	ceph_encode_8(&p, 2); /* entity_addrvec_t marker */
1521	ceph_encode_32(&p, 1); /* my_addrs len */
1522	ceph_encode_entity_addr(&p, my_addr);
1523	ceph_encode_64(&p, con->v2.client_cookie);
1524	ceph_encode_64(&p, con->v2.server_cookie);
1525	ceph_encode_64(&p, con->v2.global_seq);
1526	ceph_encode_64(&p, con->v2.connect_seq);
1527	ceph_encode_64(&p, con->in_seq);
1528	WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1529
1530	return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1531	}
1532
1533	static int prepare_keepalive2(struct ceph_connection *con)
1534	{
1535	struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1536	struct timespec64 now;
1537
1538	ktime_get_real_ts64(&now);
1539	dout("%s con %p timestamp %ptSp\n", __func__, con, &now);
1540
1541	ceph_encode_timespec64(ts, &now);
1542
1543	reset_out_kvecs(con);
1544	return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1545	sizeof(struct ceph_timespec));
1546	}
1547
1548	static int prepare_ack(struct ceph_connection *con)
1549	{
1550	void *p;
1551
1552	dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1553	con->in_seq_acked, con->in_seq);
1554	con->in_seq_acked = con->in_seq;
1555
1556	p = CTRL_BODY(con->v2.out_buf);
1557	ceph_encode_64(&p, con->in_seq_acked);
1558
1559	reset_out_kvecs(con);
1560	return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1561	}
1562
1563	static void prepare_epilogue_plain(struct ceph_connection *con,
1564	struct ceph_msg *msg, bool aborted)
1565	{
1566	dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1567	msg, aborted, con->v2.out_epil.front_crc,
1568	con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1569
1570	encode_epilogue_plain(con, aborted);
1571	add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1572	}
1573
1574	/*
1575	* For "used" empty segments, crc is -1. For unused (trailing)
1576	* segments, crc is 0.
1577	*/
1578	static void prepare_message_plain(struct ceph_connection *con,
1579	struct ceph_msg *msg)
1580	{
1581	prepare_head_plain(con, con->v2.out_buf,
1582	sizeof(struct ceph_msg_header2), NULL, 0, false);
1583
1584	if (!front_len(msg) && !middle_len(msg)) {
1585	if (!data_len(msg)) {
1586	/*
1587	* Empty message: once the head is written,
1588	* we are done -- there is no epilogue.
1589	*/
1590	con->v2.out_state = OUT_S_FINISH_MESSAGE;
1591	return;
1592	}
1593
1594	con->v2.out_epil.front_crc = -1;
1595	con->v2.out_epil.middle_crc = -1;
1596	con->v2.out_state = OUT_S_QUEUE_DATA;
1597	return;
1598	}
1599
1600	if (front_len(msg)) {
1601	con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1602	front_len(msg));
1603	add_out_kvec(con, msg->front.iov_base, front_len(msg));
1604	} else {
1605	/* middle (at least) is there, checked above */
1606	con->v2.out_epil.front_crc = -1;
1607	}
1608
1609	if (middle_len(msg)) {
1610	con->v2.out_epil.middle_crc =
1611	crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1612	add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1613	} else {
1614	con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1615	}
1616
1617	if (data_len(msg)) {
1618	con->v2.out_state = OUT_S_QUEUE_DATA;
1619	} else {
1620	con->v2.out_epil.data_crc = 0;
1621	prepare_epilogue_plain(con, msg, false);
1622	con->v2.out_state = OUT_S_FINISH_MESSAGE;
1623	}
1624	}
1625
1626	/*
1627	* Unfortunately the kernel crypto API doesn't support streaming
1628	* (piecewise) operation for AEAD algorithms, so we can't get away
1629	* with a fixed size buffer and a couple sgs. Instead, we have to
1630	* allocate pages for the entire tail of the message (currently up
1631	* to ~32M) and two sgs arrays (up to ~256K each)...
1632	*/
1633	static int prepare_message_secure(struct ceph_connection *con,
1634	struct ceph_msg *msg)
1635	{
1636	void *zerop = page_address(ceph_zero_page);
1637	struct sg_table enc_sgt = {};
1638	struct sg_table sgt = {};
1639	struct page **enc_pages;
1640	int enc_page_cnt;
1641	int tail_len;
1642	int ret;
1643
1644	ret = prepare_head_secure_small(con, con->v2.out_buf,
1645	sizeof(struct ceph_msg_header2));
1646	if (ret)
1647	return ret;
1648
1649	tail_len = tail_onwire_len(msg, true);
1650	if (!tail_len) {
1651	/*
1652	* Empty message: once the head is written,
1653	* we are done -- there is no epilogue.
1654	*/
1655	con->v2.out_state = OUT_S_FINISH_MESSAGE;
1656	return 0;
1657	}
1658
1659	encode_epilogue_secure(con, false);
1660	ret = setup_message_sgs(&sgt, msg, zerop, zerop, zerop,
1661	&con->v2.out_epil, NULL, 0, false);
1662	if (ret)
1663	goto out;
1664
1665	enc_page_cnt = calc_pages_for(0, tail_len);
1666	enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1667	if (IS_ERR(enc_pages)) {
1668	ret = PTR_ERR(enc_pages);
1669	goto out;
1670	}
1671
1672	WARN_ON(con->v2.out_enc_pages \|\| con->v2.out_enc_page_cnt);
1673	con->v2.out_enc_pages = enc_pages;
1674	con->v2.out_enc_page_cnt = enc_page_cnt;
1675	con->v2.out_enc_resid = tail_len;
1676	con->v2.out_enc_i = 0;
1677
1678	ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1679	0, tail_len, GFP_NOIO);
1680	if (ret)
1681	goto out;
1682
1683	ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1684	tail_len - CEPH_GCM_TAG_LEN);
1685	if (ret)
1686	goto out;
1687
1688	dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1689	msg, sgt.orig_nents, enc_page_cnt);
1690	con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1691
1692	out:
1693	sg_free_table(&sgt);
1694	sg_free_table(&enc_sgt);
1695	return ret;
1696	}
1697
1698	static int prepare_message(struct ceph_connection con, struct ceph_msg msg)
1699	{
1700	int lens[] = {
1701	sizeof(struct ceph_msg_header2),
1702	front_len(msg),
1703	middle_len(msg),
1704	data_len(msg)
1705	};
1706	struct ceph_frame_desc desc;
1707	int ret;
1708
1709	dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1710	msg, lens[0], lens[1], lens[2], lens[3]);
1711
1712	if (con->in_seq > con->in_seq_acked) {
1713	dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1714	con->in_seq_acked, con->in_seq);
1715	con->in_seq_acked = con->in_seq;
1716	}
1717
1718	reset_out_kvecs(con);
1719	init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1720	encode_preamble(&desc, con->v2.out_buf);
1721	fill_header2(CTRL_BODY(con->v2.out_buf), &msg->hdr,
1722	con->in_seq_acked);
1723
1724	if (con_secure(con)) {
1725	ret = prepare_message_secure(con, msg);
1726	if (ret)
1727	return ret;
1728	} else {
1729	prepare_message_plain(con, msg);
1730	}
1731
1732	ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1733	return 0;
1734	}
1735
1736	static int prepare_read_banner_prefix(struct ceph_connection *con)
1737	{
1738	void *buf;
1739
1740	buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1741	if (!buf)
1742	return -ENOMEM;
1743
1744	reset_in_kvecs(con);
1745	add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1746	add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1747	con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1748	return 0;
1749	}
1750
1751	static int prepare_read_banner_payload(struct ceph_connection *con,
1752	int payload_len)
1753	{
1754	void *buf;
1755
1756	buf = alloc_conn_buf(con, payload_len);
1757	if (!buf)
1758	return -ENOMEM;
1759
1760	reset_in_kvecs(con);
1761	add_in_kvec(con, buf, payload_len);
1762	add_in_sign_kvec(con, buf, payload_len);
1763	con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1764	return 0;
1765	}
1766
1767	static void prepare_read_preamble(struct ceph_connection *con)
1768	{
1769	reset_in_kvecs(con);
1770	add_in_kvec(con, con->v2.in_buf,
1771	con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1772	CEPH_PREAMBLE_PLAIN_LEN);
1773	con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1774	}
1775
1776	static int prepare_read_control(struct ceph_connection *con)
1777	{
1778	int ctrl_len = con->v2.in_desc.fd_lens[0];
1779	int head_len;
1780	void *buf;
1781
1782	reset_in_kvecs(con);
1783	if (con->state == CEPH_CON_S_V2_HELLO \|\|
1784	con->state == CEPH_CON_S_V2_AUTH) {
1785	head_len = head_onwire_len(ctrl_len, false);
1786	buf = alloc_conn_buf(con, head_len);
1787	if (!buf)
1788	return -ENOMEM;
1789
1790	/* preserve preamble */
1791	memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1792
1793	add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1794	add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1795	add_in_sign_kvec(con, buf, head_len);
1796	} else {
1797	if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1798	buf = alloc_conn_buf(con, ctrl_len);
1799	if (!buf)
1800	return -ENOMEM;
1801
1802	add_in_kvec(con, buf, ctrl_len);
1803	} else {
1804	add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1805	}
1806	add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1807	}
1808	con->v2.in_state = IN_S_HANDLE_CONTROL;
1809	return 0;
1810	}
1811
1812	static int prepare_read_control_remainder(struct ceph_connection *con)
1813	{
1814	int ctrl_len = con->v2.in_desc.fd_lens[0];
1815	int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1816	void *buf;
1817
1818	buf = alloc_conn_buf(con, ctrl_len);
1819	if (!buf)
1820	return -ENOMEM;
1821
1822	memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1823
1824	reset_in_kvecs(con);
1825	add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1826	add_in_kvec(con, con->v2.in_buf,
1827	padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1828	con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1829	return 0;
1830	}
1831
1832	static int prepare_read_data(struct ceph_connection *con)
1833	{
1834	struct bio_vec bv;
1835
1836	con->in_data_crc = -1;
1837	ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1838	data_len(con->in_msg));
1839
1840	get_bvec_at(&con->v2.in_cursor, &bv);
1841	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1842	if (unlikely(!con->bounce_page)) {
1843	con->bounce_page = alloc_page(GFP_NOIO);
1844	if (!con->bounce_page) {
1845	pr_err("failed to allocate bounce page\n");
1846	return -ENOMEM;
1847	}
1848	}
1849
1850	bv.bv_page = con->bounce_page;
1851	bv.bv_offset = 0;
1852	}
1853	set_in_bvec(con, &bv);
1854	con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1855	return 0;
1856	}
1857
1858	static void prepare_read_data_cont(struct ceph_connection *con)
1859	{
1860	struct bio_vec bv;
1861
1862	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1863	con->in_data_crc = crc32c(con->in_data_crc,
1864	page_address(con->bounce_page),
1865	con->v2.in_bvec.bv_len);
1866
1867	get_bvec_at(&con->v2.in_cursor, &bv);
1868	memcpy_to_page(bv.bv_page, bv.bv_offset,
1869	page_address(con->bounce_page),
1870	con->v2.in_bvec.bv_len);
1871	} else {
1872	con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1873	con->v2.in_bvec.bv_page,
1874	con->v2.in_bvec.bv_offset,
1875	con->v2.in_bvec.bv_len);
1876	}
1877
1878	ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1879	if (con->v2.in_cursor.total_resid) {
1880	get_bvec_at(&con->v2.in_cursor, &bv);
1881	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1882	bv.bv_page = con->bounce_page;
1883	bv.bv_offset = 0;
1884	}
1885	set_in_bvec(con, &bv);
1886	WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1887	return;
1888	}
1889
1890	/*
1891	* We've read all data. Prepare to read epilogue.
1892	*/
1893	reset_in_kvecs(con);
1894	add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1895	con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1896	}
1897
1898	static int prepare_sparse_read_cont(struct ceph_connection *con)
1899	{
1900	int ret;
1901	struct bio_vec bv;
1902	char *buf = NULL;
1903	struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1904
1905	WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);
1906
1907	if (iov_iter_is_bvec(&con->v2.in_iter)) {
1908	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1909	con->in_data_crc = crc32c(con->in_data_crc,
1910	page_address(con->bounce_page),
1911	con->v2.in_bvec.bv_len);
1912	get_bvec_at(cursor, &bv);
1913	memcpy_to_page(bv.bv_page, bv.bv_offset,
1914	page_address(con->bounce_page),
1915	con->v2.in_bvec.bv_len);
1916	} else {
1917	con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1918	con->v2.in_bvec.bv_page,
1919	con->v2.in_bvec.bv_offset,
1920	con->v2.in_bvec.bv_len);
1921	}
1922
1923	ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);
1924	cursor->sr_resid -= con->v2.in_bvec.bv_len;
1925	dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__,
1926	con->v2.in_bvec.bv_len, cursor->sr_resid);
1927	WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid);
1928	if (cursor->sr_resid) {
1929	get_bvec_at(cursor, &bv);
1930	if (bv.bv_len > cursor->sr_resid)
1931	bv.bv_len = cursor->sr_resid;
1932	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1933	bv.bv_page = con->bounce_page;
1934	bv.bv_offset = 0;
1935	}
1936	set_in_bvec(con, &bv);
1937	con->v2.data_len_remain -= bv.bv_len;
1938	return 0;
1939	}
1940	} else if (iov_iter_is_kvec(&con->v2.in_iter)) {
1941	/* On first call, we have no kvec so don't compute crc */
1942	if (con->v2.in_kvec_cnt) {
1943	WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);
1944	con->in_data_crc = crc32c(con->in_data_crc,
1945	con->v2.in_kvecs[0].iov_base,
1946	con->v2.in_kvecs[0].iov_len);
1947	}
1948	} else {
1949	return -EIO;
1950	}
1951
1952	/* get next extent */
1953	ret = con->ops->sparse_read(con, cursor, &buf);
1954	if (ret <= 0) {
1955	if (ret < 0)
1956	return ret;
1957
1958	reset_in_kvecs(con);
1959	add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1960	con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1961	return 0;
1962	}
1963
1964	if (buf) {
1965	/* receive into buffer */
1966	reset_in_kvecs(con);
1967	add_in_kvec(con, buf, ret);
1968	con->v2.data_len_remain -= ret;
1969	return 0;
1970	}
1971
1972	if (ret > cursor->total_resid) {
1973	pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n",
1974	__func__, ret, cursor->total_resid, cursor->resid);
1975	return -EIO;
1976	}
1977	get_bvec_at(cursor, &bv);
1978	if (bv.bv_len > cursor->sr_resid)
1979	bv.bv_len = cursor->sr_resid;
1980	if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1981	if (unlikely(!con->bounce_page)) {
1982	con->bounce_page = alloc_page(GFP_NOIO);
1983	if (!con->bounce_page) {
1984	pr_err("failed to allocate bounce page\n");
1985	return -ENOMEM;
1986	}
1987	}
1988
1989	bv.bv_page = con->bounce_page;
1990	bv.bv_offset = 0;
1991	}
1992	set_in_bvec(con, &bv);
1993	con->v2.data_len_remain -= ret;
1994	return ret;
1995	}
1996
1997	static int prepare_sparse_read_data(struct ceph_connection *con)
1998	{
1999	struct ceph_msg *msg = con->in_msg;
2000
2001	dout("%s: starting sparse read\n", __func__);
2002
2003	if (WARN_ON_ONCE(!con->ops->sparse_read))
2004	return -EOPNOTSUPP;
2005
2006	if (!con_secure(con))
2007	con->in_data_crc = -1;
2008
2009	ceph_msg_data_cursor_init(&con->v2.in_cursor, msg,
2010	msg->sparse_read_total);
2011
2012	reset_in_kvecs(con);
2013	con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;
2014	con->v2.data_len_remain = data_len(msg);
2015	return prepare_sparse_read_cont(con);
2016	}
2017
2018	static int prepare_read_tail_plain(struct ceph_connection *con)
2019	{
2020	struct ceph_msg *msg = con->in_msg;
2021
2022	if (!front_len(msg) && !middle_len(msg)) {
2023	WARN_ON(!data_len(msg));
2024	return prepare_read_data(con);
2025	}
2026
2027	reset_in_kvecs(con);
2028	if (front_len(msg)) {
2029	add_in_kvec(con, msg->front.iov_base, front_len(msg));
2030	WARN_ON(msg->front.iov_len != front_len(msg));
2031	}
2032	if (middle_len(msg)) {
2033	add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
2034	WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
2035	}
2036
2037	if (data_len(msg)) {
2038	if (msg->sparse_read_total)
2039	con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;
2040	else
2041	con->v2.in_state = IN_S_PREPARE_READ_DATA;
2042	} else {
2043	add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
2044	con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2045	}
2046	return 0;
2047	}
2048
2049	static void prepare_read_enc_page(struct ceph_connection *con)
2050	{
2051	struct bio_vec bv;
2052
2053	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
2054	con->v2.in_enc_resid);
2055	WARN_ON(!con->v2.in_enc_resid);
2056
2057	bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],
2058	min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);
2059
2060	set_in_bvec(con, &bv);
2061	con->v2.in_enc_i++;
2062	con->v2.in_enc_resid -= bv.bv_len;
2063
2064	if (con->v2.in_enc_resid) {
2065	con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
2066	return;
2067	}
2068
2069	/*
2070	* We are set to read the last piece of ciphertext (ending
2071	* with epilogue) + auth tag.
2072	*/
2073	WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
2074	con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2075	}
2076
2077	static int prepare_read_tail_secure(struct ceph_connection *con)
2078	{
2079	struct page **enc_pages;
2080	int enc_page_cnt;
2081	int tail_len;
2082
2083	tail_len = tail_onwire_len(con->in_msg, true);
2084	WARN_ON(!tail_len);
2085
2086	enc_page_cnt = calc_pages_for(0, tail_len);
2087	enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
2088	if (IS_ERR(enc_pages))
2089	return PTR_ERR(enc_pages);
2090
2091	WARN_ON(con->v2.in_enc_pages \|\| con->v2.in_enc_page_cnt);
2092	con->v2.in_enc_pages = enc_pages;
2093	con->v2.in_enc_page_cnt = enc_page_cnt;
2094	con->v2.in_enc_resid = tail_len;
2095	con->v2.in_enc_i = 0;
2096
2097	prepare_read_enc_page(con);
2098	return 0;
2099	}
2100
2101	static void __finish_skip(struct ceph_connection *con)
2102	{
2103	con->in_seq++;
2104	prepare_read_preamble(con);
2105	}
2106
2107	static void prepare_skip_message(struct ceph_connection *con)
2108	{
2109	struct ceph_frame_desc *desc = &con->v2.in_desc;
2110	int tail_len;
2111
2112	dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
2113	desc->fd_lens[2], desc->fd_lens[3]);
2114
2115	tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
2116	desc->fd_lens[3], con_secure(con));
2117	if (!tail_len) {
2118	__finish_skip(con);
2119	} else {
2120	set_in_skip(con, tail_len);
2121	con->v2.in_state = IN_S_FINISH_SKIP;
2122	}
2123	}
2124
2125	static int process_banner_prefix(struct ceph_connection *con)
2126	{
2127	int payload_len;
2128	void *p;
2129
2130	WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
2131
2132	p = con->v2.in_kvecs[0].iov_base;
2133	if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
2134	if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
2135	con->error_msg = "server is speaking msgr1 protocol";
2136	else
2137	con->error_msg = "protocol error, bad banner";
2138	return -EINVAL;
2139	}
2140
2141	p += CEPH_BANNER_V2_LEN;
2142	payload_len = ceph_decode_16(&p);
2143	dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2144
2145	if (payload_len < sizeof(u64) + sizeof(u64)) {
2146	con->error_msg = "protocol error, bad banner payload len";
2147	return -EINVAL;
2148	}
2149
2150	return prepare_read_banner_payload(con, payload_len);
2151	}
2152
2153	static int process_banner_payload(struct ceph_connection *con)
2154	{
2155	void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
2156	u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
2157	u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
2158	u64 server_feat, server_req_feat;
2159	void *p;
2160	int ret;
2161
2162	p = con->v2.in_kvecs[0].iov_base;
2163	ceph_decode_64_safe(&p, end, server_feat, bad);
2164	ceph_decode_64_safe(&p, end, server_req_feat, bad);
2165
2166	dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
2167	__func__, con, server_feat, server_req_feat);
2168
2169	if (req_feat & ~server_feat) {
2170	pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2171	server_feat, req_feat & ~server_feat);
2172	con->error_msg = "missing required protocol features";
2173	return -EINVAL;
2174	}
2175	if (server_req_feat & ~feat) {
2176	pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2177	feat, server_req_feat & ~feat);
2178	con->error_msg = "missing required protocol features";
2179	return -EINVAL;
2180	}
2181
2182	/* no reset_out_kvecs() as our banner may still be pending */
2183	ret = prepare_hello(con);
2184	if (ret) {
2185	pr_err("prepare_hello failed: %d\n", ret);
2186	return ret;
2187	}
2188
2189	con->state = CEPH_CON_S_V2_HELLO;
2190	prepare_read_preamble(con);
2191	return 0;
2192
2193	bad:
2194	pr_err("failed to decode banner payload\n");
2195	return -EINVAL;
2196	}
2197
2198	static int process_hello(struct ceph_connection con, void p, void *end)
2199	{
2200	struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2201	struct ceph_entity_addr addr_for_me;
2202	u8 entity_type;
2203	int ret;
2204
2205	if (con->state != CEPH_CON_S_V2_HELLO) {
2206	con->error_msg = "protocol error, unexpected hello";
2207	return -EINVAL;
2208	}
2209
2210	ceph_decode_8_safe(&p, end, entity_type, bad);
2211	ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
2212	if (ret) {
2213	pr_err("failed to decode addr_for_me: %d\n", ret);
2214	return ret;
2215	}
2216
2217	dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2218	entity_type, ceph_pr_addr(&addr_for_me));
2219
2220	if (entity_type != con->peer_name.type) {
2221	pr_err("bad peer type, want %d, got %d\n",
2222	con->peer_name.type, entity_type);
2223	con->error_msg = "wrong peer at address";
2224	return -EINVAL;
2225	}
2226
2227	/*
2228	* Set our address to the address our first peer (i.e. monitor)
2229	* sees that we are connecting from. If we are behind some sort
2230	* of NAT and want to be identified by some private (not NATed)
2231	* address, ip option should be used.
2232	*/
2233	if (ceph_addr_is_blank(my_addr)) {
2234	memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2235	sizeof(my_addr->in_addr));
2236	ceph_addr_set_port(my_addr, 0);
2237	dout("%s con %p set my addr %s, as seen by peer %s\n",
2238	__func__, con, ceph_pr_addr(my_addr),
2239	ceph_pr_addr(&con->peer_addr));
2240	} else {
2241	dout("%s con %p my addr already set %s\n",
2242	__func__, con, ceph_pr_addr(my_addr));
2243	}
2244
2245	WARN_ON(ceph_addr_is_blank(my_addr) \|\| ceph_addr_port(my_addr));
2246	WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2247	WARN_ON(!my_addr->nonce);
2248
2249	/* no reset_out_kvecs() as our hello may still be pending */
2250	ret = prepare_auth_request(con);
2251	if (ret) {
2252	if (ret != -EAGAIN)
2253	pr_err("prepare_auth_request failed: %d\n", ret);
2254	return ret;
2255	}
2256
2257	con->state = CEPH_CON_S_V2_AUTH;
2258	return 0;
2259
2260	bad:
2261	pr_err("failed to decode hello\n");
2262	return -EINVAL;
2263	}
2264
2265	static int process_auth_bad_method(struct ceph_connection *con,
2266	void p, void end)
2267	{
2268	int allowed_protos[8], allowed_modes[8];
2269	int allowed_proto_cnt, allowed_mode_cnt;
2270	int used_proto, result;
2271	int ret;
2272	int i;
2273
2274	if (con->state != CEPH_CON_S_V2_AUTH) {
2275	con->error_msg = "protocol error, unexpected auth_bad_method";
2276	return -EINVAL;
2277	}
2278
2279	ceph_decode_32_safe(&p, end, used_proto, bad);
2280	ceph_decode_32_safe(&p, end, result, bad);
2281	dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2282	result);
2283
2284	ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2285	if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2286	pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2287	return -EINVAL;
2288	}
2289	for (i = 0; i < allowed_proto_cnt; i++) {
2290	ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2291	dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2292	i, allowed_protos[i]);
2293	}
2294
2295	ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2296	if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2297	pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2298	return -EINVAL;
2299	}
2300	for (i = 0; i < allowed_mode_cnt; i++) {
2301	ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2302	dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2303	i, allowed_modes[i]);
2304	}
2305
2306	mutex_unlock(&con->mutex);
2307	ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2308	allowed_protos,
2309	allowed_proto_cnt,
2310	allowed_modes,
2311	allowed_mode_cnt);
2312	mutex_lock(&con->mutex);
2313	if (con->state != CEPH_CON_S_V2_AUTH) {
2314	dout("%s con %p state changed to %d\n", __func__, con,
2315	con->state);
2316	return -EAGAIN;
2317	}
2318
2319	dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2320	return ret;
2321
2322	bad:
2323	pr_err("failed to decode auth_bad_method\n");
2324	return -EINVAL;
2325	}
2326
2327	static int process_auth_reply_more(struct ceph_connection *con,
2328	void p, void end)
2329	{
2330	int payload_len;
2331	int ret;
2332
2333	if (con->state != CEPH_CON_S_V2_AUTH) {
2334	con->error_msg = "protocol error, unexpected auth_reply_more";
2335	return -EINVAL;
2336	}
2337
2338	ceph_decode_32_safe(&p, end, payload_len, bad);
2339	ceph_decode_need(&p, end, payload_len, bad);
2340
2341	dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2342
2343	reset_out_kvecs(con);
2344	ret = prepare_auth_request_more(con, p, payload_len);
2345	if (ret) {
2346	if (ret != -EAGAIN)
2347	pr_err("prepare_auth_request_more failed: %d\n", ret);
2348	return ret;
2349	}
2350
2351	return 0;
2352
2353	bad:
2354	pr_err("failed to decode auth_reply_more\n");
2355	return -EINVAL;
2356	}
2357
2358	/*
2359	* Align con_secret to avoid GFP_ATOMIC allocation inside
2360	* crypto_aead_setkey() called from setup_crypto(). __aligned(16)
2361	* isn't guaranteed to work for stack objects, so do it by hand.
2362	*/
2363	static int process_auth_done(struct ceph_connection con, void p, void *end)
2364	{
2365	u8 session_key[CEPH_MAX_KEY_LEN];
2366	u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2367	u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2368	int session_key_len, con_secret_len;
2369	int payload_len;
2370	u64 global_id;
2371	int ret;
2372
2373	if (con->state != CEPH_CON_S_V2_AUTH) {
2374	con->error_msg = "protocol error, unexpected auth_done";
2375	return -EINVAL;
2376	}
2377
2378	ceph_decode_64_safe(&p, end, global_id, bad);
2379	ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2380
2381	ceph_decode_32_safe(&p, end, payload_len, bad);
2382	ceph_decode_need(&p, end, payload_len, bad);
2383
2384	dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2385	__func__, con, global_id, con->v2.con_mode, payload_len);
2386
2387	mutex_unlock(&con->mutex);
2388	session_key_len = 0;
2389	con_secret_len = 0;
2390	ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2391	session_key, &session_key_len,
2392	con_secret, &con_secret_len);
2393	mutex_lock(&con->mutex);
2394	if (con->state != CEPH_CON_S_V2_AUTH) {
2395	dout("%s con %p state changed to %d\n", __func__, con,
2396	con->state);
2397	ret = -EAGAIN;
2398	goto out;
2399	}
2400
2401	dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2402	if (ret)
2403	goto out;
2404
2405	ret = setup_crypto(con, session_key, session_key_len, con_secret,
2406	con_secret_len);
2407	if (ret)
2408	goto out;
2409
2410	reset_out_kvecs(con);
2411	ret = prepare_auth_signature(con);
2412	if (ret) {
2413	pr_err("prepare_auth_signature failed: %d\n", ret);
2414	goto out;
2415	}
2416
2417	con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2418
2419	out:
2420	memzero_explicit(session_key, sizeof(session_key));
2421	memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2422	return ret;
2423
2424	bad:
2425	pr_err("failed to decode auth_done\n");
2426	return -EINVAL;
2427	}
2428
2429	static int process_auth_signature(struct ceph_connection *con,
2430	void p, void end)
2431	{
2432	u8 hmac[SHA256_DIGEST_SIZE];
2433	int ret;
2434
2435	if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2436	con->error_msg = "protocol error, unexpected auth_signature";
2437	return -EINVAL;
2438	}
2439
2440	con_hmac_sha256(con, con->v2.out_sign_kvecs, con->v2.out_sign_kvec_cnt,
2441	hmac);
2442
2443	ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2444	if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2445	con->error_msg = "integrity error, bad auth signature";
2446	return -EBADMSG;
2447	}
2448
2449	dout("%s con %p auth signature ok\n", __func__, con);
2450
2451	/* no reset_out_kvecs() as our auth_signature may still be pending */
2452	if (!con->v2.server_cookie) {
2453	ret = prepare_client_ident(con);
2454	if (ret) {
2455	pr_err("prepare_client_ident failed: %d\n", ret);
2456	return ret;
2457	}
2458
2459	con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2460	} else {
2461	ret = prepare_session_reconnect(con);
2462	if (ret) {
2463	pr_err("prepare_session_reconnect failed: %d\n", ret);
2464	return ret;
2465	}
2466
2467	con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2468	}
2469
2470	return 0;
2471
2472	bad:
2473	pr_err("failed to decode auth_signature\n");
2474	return -EINVAL;
2475	}
2476
2477	static int process_server_ident(struct ceph_connection *con,
2478	void p, void end)
2479	{
2480	struct ceph_client *client = from_msgr(con->msgr);
2481	u64 features, required_features;
2482	struct ceph_entity_addr addr;
2483	u64 global_seq;
2484	u64 global_id;
2485	u64 cookie;
2486	u64 flags;
2487	int ret;
2488
2489	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2490	con->error_msg = "protocol error, unexpected server_ident";
2491	return -EINVAL;
2492	}
2493
2494	ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2495	if (ret) {
2496	pr_err("failed to decode server addrs: %d\n", ret);
2497	return ret;
2498	}
2499
2500	ceph_decode_64_safe(&p, end, global_id, bad);
2501	ceph_decode_64_safe(&p, end, global_seq, bad);
2502	ceph_decode_64_safe(&p, end, features, bad);
2503	ceph_decode_64_safe(&p, end, required_features, bad);
2504	ceph_decode_64_safe(&p, end, flags, bad);
2505	ceph_decode_64_safe(&p, end, cookie, bad);
2506
2507	dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2508	__func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2509	global_id, global_seq, features, required_features, flags, cookie);
2510
2511	/* is this who we intended to talk to? */
2512	if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2513	pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2514	ceph_pr_addr(&con->peer_addr),
2515	le32_to_cpu(con->peer_addr.nonce),
2516	ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2517	con->error_msg = "wrong peer at address";
2518	return -EINVAL;
2519	}
2520
2521	if (client->required_features & ~features) {
2522	pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2523	features, client->required_features & ~features);
2524	con->error_msg = "missing required protocol features";
2525	return -EINVAL;
2526	}
2527
2528	/*
2529	* Both name->type and name->num are set in ceph_con_open() but
2530	* name->num may be bogus in the initial monmap. name->type is
2531	* verified in handle_hello().
2532	*/
2533	WARN_ON(!con->peer_name.type);
2534	con->peer_name.num = cpu_to_le64(global_id);
2535	con->v2.peer_global_seq = global_seq;
2536	con->peer_features = features;
2537	WARN_ON(required_features & ~client->supported_features);
2538	con->v2.server_cookie = cookie;
2539
2540	if (flags & CEPH_MSG_CONNECT_LOSSY) {
2541	ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2542	WARN_ON(con->v2.server_cookie);
2543	} else {
2544	WARN_ON(!con->v2.server_cookie);
2545	}
2546
2547	clear_in_sign_kvecs(con);
2548	clear_out_sign_kvecs(con);
2549	free_conn_bufs(con);
2550	con->delay = 0; /* reset backoff memory */
2551
2552	con->state = CEPH_CON_S_OPEN;
2553	con->v2.out_state = OUT_S_GET_NEXT;
2554	return 0;
2555
2556	bad:
2557	pr_err("failed to decode server_ident\n");
2558	return -EINVAL;
2559	}
2560
2561	static int process_ident_missing_features(struct ceph_connection *con,
2562	void p, void end)
2563	{
2564	struct ceph_client *client = from_msgr(con->msgr);
2565	u64 missing_features;
2566
2567	if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2568	con->error_msg = "protocol error, unexpected ident_missing_features";
2569	return -EINVAL;
2570	}
2571
2572	ceph_decode_64_safe(&p, end, missing_features, bad);
2573	pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2574	client->supported_features, missing_features);
2575	con->error_msg = "missing required protocol features";
2576	return -EINVAL;
2577
2578	bad:
2579	pr_err("failed to decode ident_missing_features\n");
2580	return -EINVAL;
2581	}
2582
2583	static int process_session_reconnect_ok(struct ceph_connection *con,
2584	void p, void end)
2585	{
2586	u64 seq;
2587
2588	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2589	con->error_msg = "protocol error, unexpected session_reconnect_ok";
2590	return -EINVAL;
2591	}
2592
2593	ceph_decode_64_safe(&p, end, seq, bad);
2594
2595	dout("%s con %p seq %llu\n", __func__, con, seq);
2596	ceph_con_discard_requeued(con, seq);
2597
2598	clear_in_sign_kvecs(con);
2599	clear_out_sign_kvecs(con);
2600	free_conn_bufs(con);
2601	con->delay = 0; /* reset backoff memory */
2602
2603	con->state = CEPH_CON_S_OPEN;
2604	con->v2.out_state = OUT_S_GET_NEXT;
2605	return 0;
2606
2607	bad:
2608	pr_err("failed to decode session_reconnect_ok\n");
2609	return -EINVAL;
2610	}
2611
2612	static int process_session_retry(struct ceph_connection *con,
2613	void p, void end)
2614	{
2615	u64 connect_seq;
2616	int ret;
2617
2618	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2619	con->error_msg = "protocol error, unexpected session_retry";
2620	return -EINVAL;
2621	}
2622
2623	ceph_decode_64_safe(&p, end, connect_seq, bad);
2624
2625	dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2626	WARN_ON(connect_seq <= con->v2.connect_seq);
2627	con->v2.connect_seq = connect_seq + 1;
2628
2629	free_conn_bufs(con);
2630
2631	reset_out_kvecs(con);
2632	ret = prepare_session_reconnect(con);
2633	if (ret) {
2634	pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2635	return ret;
2636	}
2637
2638	return 0;
2639
2640	bad:
2641	pr_err("failed to decode session_retry\n");
2642	return -EINVAL;
2643	}
2644
2645	static int process_session_retry_global(struct ceph_connection *con,
2646	void p, void end)
2647	{
2648	u64 global_seq;
2649	int ret;
2650
2651	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2652	con->error_msg = "protocol error, unexpected session_retry_global";
2653	return -EINVAL;
2654	}
2655
2656	ceph_decode_64_safe(&p, end, global_seq, bad);
2657
2658	dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2659	WARN_ON(global_seq <= con->v2.global_seq);
2660	con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2661
2662	free_conn_bufs(con);
2663
2664	reset_out_kvecs(con);
2665	ret = prepare_session_reconnect(con);
2666	if (ret) {
2667	pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2668	return ret;
2669	}
2670
2671	return 0;
2672
2673	bad:
2674	pr_err("failed to decode session_retry_global\n");
2675	return -EINVAL;
2676	}
2677
2678	static int process_session_reset(struct ceph_connection *con,
2679	void p, void end)
2680	{
2681	bool full;
2682	int ret;
2683
2684	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2685	con->error_msg = "protocol error, unexpected session_reset";
2686	return -EINVAL;
2687	}
2688
2689	ceph_decode_8_safe(&p, end, full, bad);
2690	if (!full) {
2691	con->error_msg = "protocol error, bad session_reset";
2692	return -EINVAL;
2693	}
2694
2695	pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2696	ceph_pr_addr(&con->peer_addr));
2697	ceph_con_reset_session(con);
2698
2699	mutex_unlock(&con->mutex);
2700	if (con->ops->peer_reset)
2701	con->ops->peer_reset(con);
2702	mutex_lock(&con->mutex);
2703	if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2704	dout("%s con %p state changed to %d\n", __func__, con,
2705	con->state);
2706	return -EAGAIN;
2707	}
2708
2709	free_conn_bufs(con);
2710
2711	reset_out_kvecs(con);
2712	ret = prepare_client_ident(con);
2713	if (ret) {
2714	pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2715	return ret;
2716	}
2717
2718	con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2719	return 0;
2720
2721	bad:
2722	pr_err("failed to decode session_reset\n");
2723	return -EINVAL;
2724	}
2725
2726	static int process_keepalive2_ack(struct ceph_connection *con,
2727	void p, void end)
2728	{
2729	if (con->state != CEPH_CON_S_OPEN) {
2730	con->error_msg = "protocol error, unexpected keepalive2_ack";
2731	return -EINVAL;
2732	}
2733
2734	ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2735	ceph_decode_timespec64(&con->last_keepalive_ack, p);
2736
2737	dout("%s con %p timestamp %ptSp\n", __func__, con, &con->last_keepalive_ack);
2738
2739	return 0;
2740
2741	bad:
2742	pr_err("failed to decode keepalive2_ack\n");
2743	return -EINVAL;
2744	}
2745
2746	static int process_ack(struct ceph_connection con, void p, void *end)
2747	{
2748	u64 seq;
2749
2750	if (con->state != CEPH_CON_S_OPEN) {
2751	con->error_msg = "protocol error, unexpected ack";
2752	return -EINVAL;
2753	}
2754
2755	ceph_decode_64_safe(&p, end, seq, bad);
2756
2757	dout("%s con %p seq %llu\n", __func__, con, seq);
2758	ceph_con_discard_sent(con, seq);
2759	return 0;
2760
2761	bad:
2762	pr_err("failed to decode ack\n");
2763	return -EINVAL;
2764	}
2765
2766	static int process_control(struct ceph_connection con, void p, void *end)
2767	{
2768	int tag = con->v2.in_desc.fd_tag;
2769	int ret;
2770
2771	dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2772
2773	switch (tag) {
2774	case FRAME_TAG_HELLO:
2775	ret = process_hello(con, p, end);
2776	break;
2777	case FRAME_TAG_AUTH_BAD_METHOD:
2778	ret = process_auth_bad_method(con, p, end);
2779	break;
2780	case FRAME_TAG_AUTH_REPLY_MORE:
2781	ret = process_auth_reply_more(con, p, end);
2782	break;
2783	case FRAME_TAG_AUTH_DONE:
2784	ret = process_auth_done(con, p, end);
2785	break;
2786	case FRAME_TAG_AUTH_SIGNATURE:
2787	ret = process_auth_signature(con, p, end);
2788	break;
2789	case FRAME_TAG_SERVER_IDENT:
2790	ret = process_server_ident(con, p, end);
2791	break;
2792	case FRAME_TAG_IDENT_MISSING_FEATURES:
2793	ret = process_ident_missing_features(con, p, end);
2794	break;
2795	case FRAME_TAG_SESSION_RECONNECT_OK:
2796	ret = process_session_reconnect_ok(con, p, end);
2797	break;
2798	case FRAME_TAG_SESSION_RETRY:
2799	ret = process_session_retry(con, p, end);
2800	break;
2801	case FRAME_TAG_SESSION_RETRY_GLOBAL:
2802	ret = process_session_retry_global(con, p, end);
2803	break;
2804	case FRAME_TAG_SESSION_RESET:
2805	ret = process_session_reset(con, p, end);
2806	break;
2807	case FRAME_TAG_KEEPALIVE2_ACK:
2808	ret = process_keepalive2_ack(con, p, end);
2809	break;
2810	case FRAME_TAG_ACK:
2811	ret = process_ack(con, p, end);
2812	break;
2813	default:
2814	pr_err("bad tag %d\n", tag);
2815	con->error_msg = "protocol error, bad tag";
2816	return -EINVAL;
2817	}
2818	if (ret) {
2819	dout("%s con %p error %d\n", __func__, con, ret);
2820	return ret;
2821	}
2822
2823	prepare_read_preamble(con);
2824	return 0;
2825	}
2826
2827	/*
2828	* Return:
2829	* 1 - con->in_msg set, read message
2830	* 0 - skip message
2831	* <0 - error
2832	*/
2833	static int process_message_header(struct ceph_connection *con,
2834	void p, void end)
2835	{
2836	struct ceph_frame_desc *desc = &con->v2.in_desc;
2837	struct ceph_msg_header2 *hdr2;
2838	struct ceph_msg_header hdr;
2839	int skip;
2840	int ret;
2841	u64 seq;
2842
2843	ceph_decode_need(&p, end, sizeof(*hdr2), bad);
2844	hdr2 = p;
2845
2846	/* verify seq# */
2847	seq = le64_to_cpu(hdr2->seq);
2848	if ((s64)seq - (s64)con->in_seq < 1) {
2849	pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2850	ENTITY_NAME(con->peer_name),
2851	ceph_pr_addr(&con->peer_addr),
2852	seq, con->in_seq + 1);
2853	return 0;
2854	}
2855	if ((s64)seq - (s64)con->in_seq > 1) {
2856	pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2857	con->error_msg = "bad message sequence # for incoming message";
2858	return -EBADE;
2859	}
2860
2861	ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2862
2863	fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2864	desc->fd_lens[3], &con->peer_name);
2865	ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2866	if (ret)
2867	return ret;
2868
2869	WARN_ON(!con->in_msg ^ skip);
2870	if (skip)
2871	return 0;
2872
2873	WARN_ON(!con->in_msg);
2874	WARN_ON(con->in_msg->con != con);
2875	return 1;
2876
2877	bad:
2878	pr_err("failed to decode message header\n");
2879	return -EINVAL;
2880	}
2881
2882	static int process_message(struct ceph_connection *con)
2883	{
2884	ceph_con_process_message(con);
2885
2886	/*
2887	* We could have been closed by ceph_con_close() because
2888	* ceph_con_process_message() temporarily drops con->mutex.
2889	*/
2890	if (con->state != CEPH_CON_S_OPEN) {
2891	dout("%s con %p state changed to %d\n", __func__, con,
2892	con->state);
2893	return -EAGAIN;
2894	}
2895
2896	prepare_read_preamble(con);
2897	return 0;
2898	}
2899
2900	static int __handle_control(struct ceph_connection con, void p)
2901	{
2902	void *end = p + con->v2.in_desc.fd_lens[0];
2903	struct ceph_msg *msg;
2904	int ret;
2905
2906	if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2907	return process_control(con, p, end);
2908
2909	if (con->state != CEPH_CON_S_OPEN) {
2910	con->error_msg = "protocol error, unexpected message";
2911	return -EINVAL;
2912	}
2913
2914	ret = process_message_header(con, p, end);
2915	if (ret < 0)
2916	return ret;
2917	if (ret == 0) {
2918	prepare_skip_message(con);
2919	return 0;
2920	}
2921
2922	msg = con->in_msg; /* set in process_message_header() */
2923	if (front_len(msg)) {
2924	WARN_ON(front_len(msg) > msg->front_alloc_len);
2925	msg->front.iov_len = front_len(msg);
2926	} else {
2927	msg->front.iov_len = 0;
2928	}
2929	if (middle_len(msg)) {
2930	WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2931	msg->middle->vec.iov_len = middle_len(msg);
2932	} else if (msg->middle) {
2933	msg->middle->vec.iov_len = 0;
2934	}
2935
2936	if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2937	return process_message(con);
2938
2939	if (con_secure(con))
2940	return prepare_read_tail_secure(con);
2941
2942	return prepare_read_tail_plain(con);
2943	}
2944
2945	static int handle_preamble(struct ceph_connection *con)
2946	{
2947	struct ceph_frame_desc *desc = &con->v2.in_desc;
2948	int ret;
2949
2950	if (con_secure(con)) {
2951	ret = decrypt_preamble(con);
2952	if (ret) {
2953	if (ret == -EBADMSG)
2954	con->error_msg = "integrity error, bad preamble auth tag";
2955	return ret;
2956	}
2957	}
2958
2959	ret = decode_preamble(con->v2.in_buf, desc);
2960	if (ret) {
2961	if (ret == -EBADMSG)
2962	con->error_msg = "integrity error, bad crc";
2963	else
2964	con->error_msg = "protocol error, bad preamble";
2965	return ret;
2966	}
2967
2968	dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2969	con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2970	desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2971
2972	if (!con_secure(con))
2973	return prepare_read_control(con);
2974
2975	if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2976	return prepare_read_control_remainder(con);
2977
2978	return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2979	}
2980
2981	static int handle_control(struct ceph_connection *con)
2982	{
2983	int ctrl_len = con->v2.in_desc.fd_lens[0];
2984	void *buf;
2985	int ret;
2986
2987	WARN_ON(con_secure(con));
2988
2989	ret = verify_control_crc(con);
2990	if (ret) {
2991	con->error_msg = "integrity error, bad crc";
2992	return ret;
2993	}
2994
2995	if (con->state == CEPH_CON_S_V2_AUTH) {
2996	buf = alloc_conn_buf(con, ctrl_len);
2997	if (!buf)
2998	return -ENOMEM;
2999
3000	memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
3001	return __handle_control(con, buf);
3002	}
3003
3004	return __handle_control(con, con->v2.in_kvecs[0].iov_base);
3005	}
3006
3007	static int handle_control_remainder(struct ceph_connection *con)
3008	{
3009	int ret;
3010
3011	WARN_ON(!con_secure(con));
3012
3013	ret = decrypt_control_remainder(con);
3014	if (ret) {
3015	if (ret == -EBADMSG)
3016	con->error_msg = "integrity error, bad control remainder auth tag";
3017	return ret;
3018	}
3019
3020	return __handle_control(con, con->v2.in_kvecs[0].iov_base -
3021	CEPH_PREAMBLE_INLINE_LEN);
3022	}
3023
3024	static int handle_epilogue(struct ceph_connection *con)
3025	{
3026	u32 front_crc, middle_crc, data_crc;
3027	int ret;
3028
3029	if (con_secure(con)) {
3030	ret = decrypt_tail(con);
3031	if (ret) {
3032	if (ret == -EBADMSG)
3033	con->error_msg = "integrity error, bad epilogue auth tag";
3034	return ret;
3035	}
3036
3037	/* just late_status */
3038	ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
3039	if (ret) {
3040	con->error_msg = "protocol error, bad epilogue";
3041	return ret;
3042	}
3043	} else {
3044	ret = decode_epilogue(con->v2.in_buf, &front_crc,
3045	&middle_crc, &data_crc);
3046	if (ret) {
3047	con->error_msg = "protocol error, bad epilogue";
3048	return ret;
3049	}
3050
3051	ret = verify_epilogue_crcs(con, front_crc, middle_crc,
3052	data_crc);
3053	if (ret) {
3054	con->error_msg = "integrity error, bad crc";
3055	return ret;
3056	}
3057	}
3058
3059	return process_message(con);
3060	}
3061
3062	static void finish_skip(struct ceph_connection *con)
3063	{
3064	dout("%s con %p\n", __func__, con);
3065
3066	if (con_secure(con))
3067	gcm_inc_nonce(&con->v2.in_gcm_nonce);
3068
3069	__finish_skip(con);
3070	}
3071
3072	static int populate_in_iter(struct ceph_connection *con)
3073	{
3074	int ret;
3075
3076	dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
3077	con->v2.in_state);
3078	WARN_ON(iov_iter_count(&con->v2.in_iter));
3079
3080	if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
3081	ret = process_banner_prefix(con);
3082	} else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
3083	ret = process_banner_payload(con);
3084	} else if ((con->state >= CEPH_CON_S_V2_HELLO &&
3085	con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) \|\|
3086	con->state == CEPH_CON_S_OPEN) {
3087	switch (con->v2.in_state) {
3088	case IN_S_HANDLE_PREAMBLE:
3089	ret = handle_preamble(con);
3090	break;
3091	case IN_S_HANDLE_CONTROL:
3092	ret = handle_control(con);
3093	break;
3094	case IN_S_HANDLE_CONTROL_REMAINDER:
3095	ret = handle_control_remainder(con);
3096	break;
3097	case IN_S_PREPARE_READ_DATA:
3098	ret = prepare_read_data(con);
3099	break;
3100	case IN_S_PREPARE_READ_DATA_CONT:
3101	prepare_read_data_cont(con);
3102	ret = 0;
3103	break;
3104	case IN_S_PREPARE_READ_ENC_PAGE:
3105	prepare_read_enc_page(con);
3106	ret = 0;
3107	break;
3108	case IN_S_PREPARE_SPARSE_DATA:
3109	ret = prepare_sparse_read_data(con);
3110	break;
3111	case IN_S_PREPARE_SPARSE_DATA_CONT:
3112	ret = prepare_sparse_read_cont(con);
3113	break;
3114	case IN_S_HANDLE_EPILOGUE:
3115	ret = handle_epilogue(con);
3116	break;
3117	case IN_S_FINISH_SKIP:
3118	finish_skip(con);
3119	ret = 0;
3120	break;
3121	default:
3122	WARN(1, "bad in_state %d", con->v2.in_state);
3123	return -EINVAL;
3124	}
3125	} else {
3126	WARN(1, "bad state %d", con->state);
3127	return -EINVAL;
3128	}
3129	if (ret) {
3130	dout("%s con %p error %d\n", __func__, con, ret);
3131	return ret;
3132	}
3133
3134	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3135	return -ENODATA;
3136	dout("%s con %p populated %zu\n", __func__, con,
3137	iov_iter_count(&con->v2.in_iter));
3138	return 1;
3139	}
3140
3141	int ceph_con_v2_try_read(struct ceph_connection *con)
3142	{
3143	int ret;
3144
3145	dout("%s con %p state %d need %zu\n", __func__, con, con->state,
3146	iov_iter_count(&con->v2.in_iter));
3147
3148	if (con->state == CEPH_CON_S_PREOPEN)
3149	return 0;
3150
3151	/*
3152	* We should always have something pending here. If not,
3153	* avoid calling populate_in_iter() as if we read something
3154	* (ceph_tcp_recv() would immediately return 1).
3155	*/
3156	if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3157	return -ENODATA;
3158
3159	for (;;) {
3160	ret = ceph_tcp_recv(con);
3161	if (ret <= 0)
3162	return ret;
3163
3164	ret = populate_in_iter(con);
3165	if (ret <= 0) {
3166	if (ret && ret != -EAGAIN && !con->error_msg)
3167	con->error_msg = "read processing error";
3168	return ret;
3169	}
3170	}
3171	}
3172
3173	static void queue_data(struct ceph_connection con, struct ceph_msg msg)
3174	{
3175	struct bio_vec bv;
3176
3177	con->v2.out_epil.data_crc = -1;
3178	ceph_msg_data_cursor_init(&con->v2.out_cursor, msg,
3179	data_len(msg));
3180
3181	get_bvec_at(&con->v2.out_cursor, &bv);
3182	set_out_bvec(con, &bv, true);
3183	con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
3184	}
3185
3186	static void queue_data_cont(struct ceph_connection con, struct ceph_msg msg)
3187	{
3188	struct bio_vec bv;
3189
3190	con->v2.out_epil.data_crc = ceph_crc32c_page(
3191	con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3192	con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
3193
3194	ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
3195	if (con->v2.out_cursor.total_resid) {
3196	get_bvec_at(&con->v2.out_cursor, &bv);
3197	set_out_bvec(con, &bv, true);
3198	WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
3199	return;
3200	}
3201
3202	/*
3203	* We've written all data. Queue epilogue. Once it's written,
3204	* we are done.
3205	*/
3206	reset_out_kvecs(con);
3207	prepare_epilogue_plain(con, msg, false);
3208	con->v2.out_state = OUT_S_FINISH_MESSAGE;
3209	}
3210
3211	static void queue_enc_page(struct ceph_connection *con)
3212	{
3213	struct bio_vec bv;
3214
3215	dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3216	con->v2.out_enc_resid);
3217	WARN_ON(!con->v2.out_enc_resid);
3218
3219	bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],
3220	min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);
3221
3222	set_out_bvec(con, &bv, false);
3223	con->v2.out_enc_i++;
3224	con->v2.out_enc_resid -= bv.bv_len;
3225
3226	if (con->v2.out_enc_resid) {
3227	WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3228	return;
3229	}
3230
3231	/*
3232	* We've queued the last piece of ciphertext (ending with
3233	* epilogue) + auth tag. Once it's written, we are done.
3234	*/
3235	WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3236	con->v2.out_state = OUT_S_FINISH_MESSAGE;
3237	}
3238
3239	static void queue_zeros(struct ceph_connection con, struct ceph_msg msg)
3240	{
3241	dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3242
3243	if (con->v2.out_zero) {
3244	set_out_bvec_zero(con);
3245	con->v2.out_zero -= con->v2.out_bvec.bv_len;
3246	con->v2.out_state = OUT_S_QUEUE_ZEROS;
3247	return;
3248	}
3249
3250	/*
3251	* We've zero-filled everything up to epilogue. Queue epilogue
3252	* with late_status set to ABORTED and crcs adjusted for zeros.
3253	* Once it's written, we are done patching up for the revoke.
3254	*/
3255	reset_out_kvecs(con);
3256	prepare_epilogue_plain(con, msg, true);
3257	con->v2.out_state = OUT_S_FINISH_MESSAGE;
3258	}
3259
3260	static void finish_message(struct ceph_connection *con)
3261	{
3262	dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3263
3264	/* we end up here both plain and secure modes */
3265	if (con->v2.out_enc_pages) {
3266	WARN_ON(!con->v2.out_enc_page_cnt);
3267	ceph_release_page_vector(con->v2.out_enc_pages,
3268	con->v2.out_enc_page_cnt);
3269	con->v2.out_enc_pages = NULL;
3270	con->v2.out_enc_page_cnt = 0;
3271	}
3272	/* message may have been revoked */
3273	if (con->out_msg) {
3274	ceph_msg_put(con->out_msg);
3275	con->out_msg = NULL;
3276	}
3277
3278	con->v2.out_state = OUT_S_GET_NEXT;
3279	}
3280
3281	static int populate_out_iter(struct ceph_connection *con)
3282	{
3283	struct ceph_msg *msg;
3284	int ret;
3285
3286	dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3287	con->v2.out_state);
3288	WARN_ON(iov_iter_count(&con->v2.out_iter));
3289
3290	if (con->state != CEPH_CON_S_OPEN) {
3291	WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX \|\|
3292	con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3293	goto nothing_pending;
3294	}
3295
3296	switch (con->v2.out_state) {
3297	case OUT_S_QUEUE_DATA:
3298	WARN_ON(!con->out_msg);
3299	queue_data(con, con->out_msg);
3300	goto populated;
3301	case OUT_S_QUEUE_DATA_CONT:
3302	WARN_ON(!con->out_msg);
3303	queue_data_cont(con, con->out_msg);
3304	goto populated;
3305	case OUT_S_QUEUE_ENC_PAGE:
3306	queue_enc_page(con);
3307	goto populated;
3308	case OUT_S_QUEUE_ZEROS:
3309	WARN_ON(con->out_msg); /* revoked */
3310	queue_zeros(con, con->out_msg);
3311	goto populated;
3312	case OUT_S_FINISH_MESSAGE:
3313	finish_message(con);
3314	break;
3315	case OUT_S_GET_NEXT:
3316	break;
3317	default:
3318	WARN(1, "bad out_state %d", con->v2.out_state);
3319	return -EINVAL;
3320	}
3321
3322	WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3323	if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3324	ret = prepare_keepalive2(con);
3325	if (ret) {
3326	pr_err("prepare_keepalive2 failed: %d\n", ret);
3327	return ret;
3328	}
3329	} else if ((msg = ceph_con_get_out_msg(con)) != NULL) {
3330	ret = prepare_message(con, msg);
3331	if (ret) {
3332	pr_err("prepare_message failed: %d\n", ret);
3333	return ret;
3334	}
3335	} else if (con->in_seq > con->in_seq_acked) {
3336	ret = prepare_ack(con);
3337	if (ret) {
3338	pr_err("prepare_ack failed: %d\n", ret);
3339	return ret;
3340	}
3341	} else {
3342	goto nothing_pending;
3343	}
3344
3345	populated:
3346	if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3347	return -ENODATA;
3348	dout("%s con %p populated %zu\n", __func__, con,
3349	iov_iter_count(&con->v2.out_iter));
3350	return 1;
3351
3352	nothing_pending:
3353	WARN_ON(iov_iter_count(&con->v2.out_iter));
3354	dout("%s con %p nothing pending\n", __func__, con);
3355	ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3356	return 0;
3357	}
3358
3359	int ceph_con_v2_try_write(struct ceph_connection *con)
3360	{
3361	int ret;
3362
3363	dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3364	iov_iter_count(&con->v2.out_iter));
3365
3366	/* open the socket first? */
3367	if (con->state == CEPH_CON_S_PREOPEN) {
3368	WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3369
3370	/*
3371	* Always bump global_seq. Bump connect_seq only if
3372	* there is a session (i.e. we are reconnecting and will
3373	* send session_reconnect instead of client_ident).
3374	*/
3375	con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3376	if (con->v2.server_cookie)
3377	con->v2.connect_seq++;
3378
3379	ret = prepare_read_banner_prefix(con);
3380	if (ret) {
3381	pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3382	con->error_msg = "connect error";
3383	return ret;
3384	}
3385
3386	reset_out_kvecs(con);
3387	ret = prepare_banner(con);
3388	if (ret) {
3389	pr_err("prepare_banner failed: %d\n", ret);
3390	con->error_msg = "connect error";
3391	return ret;
3392	}
3393
3394	ret = ceph_tcp_connect(con);
3395	if (ret) {
3396	pr_err("ceph_tcp_connect failed: %d\n", ret);
3397	con->error_msg = "connect error";
3398	return ret;
3399	}
3400	}
3401
3402	if (!iov_iter_count(&con->v2.out_iter)) {
3403	ret = populate_out_iter(con);
3404	if (ret <= 0) {
3405	if (ret && ret != -EAGAIN && !con->error_msg)
3406	con->error_msg = "write processing error";
3407	return ret;
3408	}
3409	}
3410
3411	tcp_sock_set_cork(con->sock->sk, true);
3412	for (;;) {
3413	ret = ceph_tcp_send(con);
3414	if (ret <= 0)
3415	break;
3416
3417	ret = populate_out_iter(con);
3418	if (ret <= 0) {
3419	if (ret && ret != -EAGAIN && !con->error_msg)
3420	con->error_msg = "write processing error";
3421	break;
3422	}
3423	}
3424
3425	tcp_sock_set_cork(con->sock->sk, false);
3426	return ret;
3427	}
3428
3429	static u32 crc32c_zeros(u32 crc, int zero_len)
3430	{
3431	int len;
3432
3433	while (zero_len) {
3434	len = min(zero_len, (int)PAGE_SIZE);
3435	crc = crc32c(crc, page_address(ceph_zero_page), len);
3436	zero_len -= len;
3437	}
3438
3439	return crc;
3440	}
3441
3442	static void prepare_zero_front(struct ceph_connection *con,
3443	struct ceph_msg *msg, int resid)
3444	{
3445	int sent;
3446
3447	WARN_ON(!resid \|\| resid > front_len(msg));
3448	sent = front_len(msg) - resid;
3449	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3450
3451	if (sent) {
3452	con->v2.out_epil.front_crc =
3453	crc32c(-1, msg->front.iov_base, sent);
3454	con->v2.out_epil.front_crc =
3455	crc32c_zeros(con->v2.out_epil.front_crc, resid);
3456	} else {
3457	con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3458	}
3459
3460	con->v2.out_iter.count -= resid;
3461	out_zero_add(con, resid);
3462	}
3463
3464	static void prepare_zero_middle(struct ceph_connection *con,
3465	struct ceph_msg *msg, int resid)
3466	{
3467	int sent;
3468
3469	WARN_ON(!resid \|\| resid > middle_len(msg));
3470	sent = middle_len(msg) - resid;
3471	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3472
3473	if (sent) {
3474	con->v2.out_epil.middle_crc =
3475	crc32c(-1, msg->middle->vec.iov_base, sent);
3476	con->v2.out_epil.middle_crc =
3477	crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3478	} else {
3479	con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3480	}
3481
3482	con->v2.out_iter.count -= resid;
3483	out_zero_add(con, resid);
3484	}
3485
3486	static void prepare_zero_data(struct ceph_connection *con,
3487	struct ceph_msg *msg)
3488	{
3489	dout("%s con %p\n", __func__, con);
3490	con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(msg));
3491	out_zero_add(con, data_len(msg));
3492	}
3493
3494	static void revoke_at_queue_data(struct ceph_connection *con,
3495	struct ceph_msg *msg)
3496	{
3497	int boundary;
3498	int resid;
3499
3500	WARN_ON(!data_len(msg));
3501	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3502	resid = iov_iter_count(&con->v2.out_iter);
3503
3504	boundary = front_len(msg) + middle_len(msg);
3505	if (resid > boundary) {
3506	resid -= boundary;
3507	WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3508	dout("%s con %p was sending head\n", __func__, con);
3509	if (front_len(msg))
3510	prepare_zero_front(con, msg, front_len(msg));
3511	if (middle_len(msg))
3512	prepare_zero_middle(con, msg, middle_len(msg));
3513	prepare_zero_data(con, msg);
3514	WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3515	con->v2.out_state = OUT_S_QUEUE_ZEROS;
3516	return;
3517	}
3518
3519	boundary = middle_len(msg);
3520	if (resid > boundary) {
3521	resid -= boundary;
3522	dout("%s con %p was sending front\n", __func__, con);
3523	prepare_zero_front(con, msg, resid);
3524	if (middle_len(msg))
3525	prepare_zero_middle(con, msg, middle_len(msg));
3526	prepare_zero_data(con, msg);
3527	queue_zeros(con, msg);
3528	return;
3529	}
3530
3531	WARN_ON(!resid);
3532	dout("%s con %p was sending middle\n", __func__, con);
3533	prepare_zero_middle(con, msg, resid);
3534	prepare_zero_data(con, msg);
3535	queue_zeros(con, msg);
3536	}
3537
3538	static void revoke_at_queue_data_cont(struct ceph_connection *con,
3539	struct ceph_msg *msg)
3540	{
3541	int sent, resid; /* current piece of data */
3542
3543	WARN_ON(!data_len(msg));
3544	WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3545	resid = iov_iter_count(&con->v2.out_iter);
3546	WARN_ON(!resid \|\| resid > con->v2.out_bvec.bv_len);
3547	sent = con->v2.out_bvec.bv_len - resid;
3548	dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3549
3550	if (sent) {
3551	con->v2.out_epil.data_crc = ceph_crc32c_page(
3552	con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3553	con->v2.out_bvec.bv_offset, sent);
3554	ceph_msg_data_advance(&con->v2.out_cursor, sent);
3555	}
3556	WARN_ON(resid > con->v2.out_cursor.total_resid);
3557	con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3558	con->v2.out_cursor.total_resid);
3559
3560	con->v2.out_iter.count -= resid;
3561	out_zero_add(con, con->v2.out_cursor.total_resid);
3562	queue_zeros(con, msg);
3563	}
3564
3565	static void revoke_at_finish_message(struct ceph_connection *con,
3566	struct ceph_msg *msg)
3567	{
3568	int boundary;
3569	int resid;
3570
3571	WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3572	resid = iov_iter_count(&con->v2.out_iter);
3573
3574	if (!front_len(msg) && !middle_len(msg) &&
3575	!data_len(msg)) {
3576	WARN_ON(!resid \|\| resid > MESSAGE_HEAD_PLAIN_LEN);
3577	dout("%s con %p was sending head (empty message) - noop\n",
3578	__func__, con);
3579	return;
3580	}
3581
3582	boundary = front_len(msg) + middle_len(msg) +
3583	CEPH_EPILOGUE_PLAIN_LEN;
3584	if (resid > boundary) {
3585	resid -= boundary;
3586	WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3587	dout("%s con %p was sending head\n", __func__, con);
3588	if (front_len(msg))
3589	prepare_zero_front(con, msg, front_len(msg));
3590	if (middle_len(msg))
3591	prepare_zero_middle(con, msg, middle_len(msg));
3592	con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3593	WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3594	con->v2.out_state = OUT_S_QUEUE_ZEROS;
3595	return;
3596	}
3597
3598	boundary = middle_len(msg) + CEPH_EPILOGUE_PLAIN_LEN;
3599	if (resid > boundary) {
3600	resid -= boundary;
3601	dout("%s con %p was sending front\n", __func__, con);
3602	prepare_zero_front(con, msg, resid);
3603	if (middle_len(msg))
3604	prepare_zero_middle(con, msg, middle_len(msg));
3605	con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3606	queue_zeros(con, msg);
3607	return;
3608	}
3609
3610	boundary = CEPH_EPILOGUE_PLAIN_LEN;
3611	if (resid > boundary) {
3612	resid -= boundary;
3613	dout("%s con %p was sending middle\n", __func__, con);
3614	prepare_zero_middle(con, msg, resid);
3615	con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3616	queue_zeros(con, msg);
3617	return;
3618	}
3619
3620	WARN_ON(!resid);
3621	dout("%s con %p was sending epilogue - noop\n", __func__, con);
3622	}
3623
3624	void ceph_con_v2_revoke(struct ceph_connection con, struct ceph_msg msg)
3625	{
3626	WARN_ON(con->v2.out_zero);
3627
3628	if (con_secure(con)) {
3629	WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3630	con->v2.out_state != OUT_S_FINISH_MESSAGE);
3631	dout("%s con %p secure - noop\n", __func__, con);
3632	return;
3633	}
3634
3635	switch (con->v2.out_state) {
3636	case OUT_S_QUEUE_DATA:
3637	revoke_at_queue_data(con, msg);
3638	break;
3639	case OUT_S_QUEUE_DATA_CONT:
3640	revoke_at_queue_data_cont(con, msg);
3641	break;
3642	case OUT_S_FINISH_MESSAGE:
3643	revoke_at_finish_message(con, msg);
3644	break;
3645	default:
3646	WARN(1, "bad out_state %d", con->v2.out_state);
3647	break;
3648	}
3649	}
3650
3651	static void revoke_at_prepare_read_data(struct ceph_connection *con)
3652	{
3653	int remaining;
3654	int resid;
3655
3656	WARN_ON(con_secure(con));
3657	WARN_ON(!data_len(con->in_msg));
3658	WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3659	resid = iov_iter_count(&con->v2.in_iter);
3660	WARN_ON(!resid);
3661
3662	remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3663	dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3664	remaining);
3665	con->v2.in_iter.count -= resid;
3666	set_in_skip(con, resid + remaining);
3667	con->v2.in_state = IN_S_FINISH_SKIP;
3668	}
3669
3670	static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3671	{
3672	int recved, resid; /* current piece of data */
3673	int remaining;
3674
3675	WARN_ON(con_secure(con));
3676	WARN_ON(!data_len(con->in_msg));
3677	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3678	resid = iov_iter_count(&con->v2.in_iter);
3679	WARN_ON(!resid \|\| resid > con->v2.in_bvec.bv_len);
3680	recved = con->v2.in_bvec.bv_len - resid;
3681	dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3682
3683	if (recved)
3684	ceph_msg_data_advance(&con->v2.in_cursor, recved);
3685	WARN_ON(resid > con->v2.in_cursor.total_resid);
3686
3687	remaining = CEPH_EPILOGUE_PLAIN_LEN;
3688	dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3689	con->v2.in_cursor.total_resid, remaining);
3690	con->v2.in_iter.count -= resid;
3691	set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3692	con->v2.in_state = IN_S_FINISH_SKIP;
3693	}
3694
3695	static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3696	{
3697	int resid; /* current enc page (not necessarily data) */
3698
3699	WARN_ON(!con_secure(con));
3700	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3701	resid = iov_iter_count(&con->v2.in_iter);
3702	WARN_ON(!resid \|\| resid > con->v2.in_bvec.bv_len);
3703
3704	dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3705	con->v2.in_enc_resid);
3706	con->v2.in_iter.count -= resid;
3707	set_in_skip(con, resid + con->v2.in_enc_resid);
3708	con->v2.in_state = IN_S_FINISH_SKIP;
3709	}
3710
3711	static void revoke_at_prepare_sparse_data(struct ceph_connection *con)
3712	{
3713	int resid; /* current piece of data */
3714	int remaining;
3715
3716	WARN_ON(con_secure(con));
3717	WARN_ON(!data_len(con->in_msg));
3718	WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3719	resid = iov_iter_count(&con->v2.in_iter);
3720	dout("%s con %p resid %d\n", __func__, con, resid);
3721
3722	remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;
3723	con->v2.in_iter.count -= resid;
3724	set_in_skip(con, resid + remaining);
3725	con->v2.in_state = IN_S_FINISH_SKIP;
3726	}
3727
3728	static void revoke_at_handle_epilogue(struct ceph_connection *con)
3729	{
3730	int resid;
3731
3732	resid = iov_iter_count(&con->v2.in_iter);
3733	WARN_ON(!resid);
3734
3735	dout("%s con %p resid %d\n", __func__, con, resid);
3736	con->v2.in_iter.count -= resid;
3737	set_in_skip(con, resid);
3738	con->v2.in_state = IN_S_FINISH_SKIP;
3739	}
3740
3741	void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3742	{
3743	switch (con->v2.in_state) {
3744	case IN_S_PREPARE_SPARSE_DATA:
3745	case IN_S_PREPARE_READ_DATA:
3746	revoke_at_prepare_read_data(con);
3747	break;
3748	case IN_S_PREPARE_READ_DATA_CONT:
3749	revoke_at_prepare_read_data_cont(con);
3750	break;
3751	case IN_S_PREPARE_READ_ENC_PAGE:
3752	revoke_at_prepare_read_enc_page(con);
3753	break;
3754	case IN_S_PREPARE_SPARSE_DATA_CONT:
3755	revoke_at_prepare_sparse_data(con);
3756	break;
3757	case IN_S_HANDLE_EPILOGUE:
3758	revoke_at_handle_epilogue(con);
3759	break;
3760	default:
3761	WARN(1, "bad in_state %d", con->v2.in_state);
3762	break;
3763	}
3764	}
3765
3766	bool ceph_con_v2_opened(struct ceph_connection *con)
3767	{
3768	return con->v2.peer_global_seq;
3769	}
3770
3771	void ceph_con_v2_reset_session(struct ceph_connection *con)
3772	{
3773	con->v2.client_cookie = 0;
3774	con->v2.server_cookie = 0;
3775	con->v2.global_seq = 0;
3776	con->v2.connect_seq = 0;
3777	con->v2.peer_global_seq = 0;
3778	}
3779
3780	void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3781	{
3782	iov_iter_truncate(&con->v2.in_iter, 0);
3783	iov_iter_truncate(&con->v2.out_iter, 0);
3784	con->v2.out_zero = 0;
3785
3786	clear_in_sign_kvecs(con);
3787	clear_out_sign_kvecs(con);
3788	free_conn_bufs(con);
3789
3790	if (con->v2.in_enc_pages) {
3791	WARN_ON(!con->v2.in_enc_page_cnt);
3792	ceph_release_page_vector(con->v2.in_enc_pages,
3793	con->v2.in_enc_page_cnt);
3794	con->v2.in_enc_pages = NULL;
3795	con->v2.in_enc_page_cnt = 0;
3796	}
3797	if (con->v2.out_enc_pages) {
3798	WARN_ON(!con->v2.out_enc_page_cnt);
3799	ceph_release_page_vector(con->v2.out_enc_pages,
3800	con->v2.out_enc_page_cnt);
3801	con->v2.out_enc_pages = NULL;
3802	con->v2.out_enc_page_cnt = 0;
3803	}
3804
3805	con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3806	memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3807	memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3808
3809	memzero_explicit(&con->v2.hmac_key, sizeof(con->v2.hmac_key));
3810	con->v2.hmac_key_set = false;
3811	if (con->v2.gcm_req) {
3812	aead_request_free(con->v2.gcm_req);
3813	con->v2.gcm_req = NULL;
3814	}
3815	if (con->v2.gcm_tfm) {
3816	crypto_free_aead(con->v2.gcm_tfm);
3817	con->v2.gcm_tfm = NULL;
3818	}
3819	}