projects/linux·blob master · refs

1117 lines · 29.5 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
4 */
5
6#include <linux/crc32.h>
7#include <linux/base64.h>
8#include <linux/prandom.h>
9#include <linux/unaligned.h>
10#include <crypto/dh.h>
11#include <crypto/utils.h>
12#include "nvme.h"
13#include "fabrics.h"
14#include <linux/nvme-auth.h>
15#include <linux/nvme-keyring.h>
16
17#define CHAP_BUF_SIZE 4096
18static struct kmem_cache *nvme_chap_buf_cache;
19static mempool_t *nvme_chap_buf_pool;
20
21struct nvme_dhchap_queue_context {
22 struct list_head entry;
23 struct work_struct auth_work;
24 struct nvme_ctrl *ctrl;
25 struct crypto_kpp *dh_tfm;
26 struct nvme_dhchap_key *transformed_key;
27 void *buf;
28 int qid;
29 int error;
30 u32 s1;
31 u32 s2;
32 bool bi_directional;
33 bool authenticated;
34 u16 transaction;
35 u8 status;
36 u8 dhgroup_id;
37 u8 hash_id;
38 u8 sc_c;
39 size_t hash_len;
40 u8 c1[NVME_AUTH_MAX_DIGEST_SIZE];
41 u8 c2[NVME_AUTH_MAX_DIGEST_SIZE];
42 u8 response[NVME_AUTH_MAX_DIGEST_SIZE];
43 u8 *ctrl_key;
44 u8 *host_key;
45 u8 *sess_key;
46 int ctrl_key_len;
47 int host_key_len;
48 int sess_key_len;
49};
50
51static struct workqueue_struct *nvme_auth_wq;
52
53static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)
54{
55 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
56 ctrl->opts->nr_poll_queues + 1;
57}
58
59static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
60 void *data, size_t data_len, bool auth_send)
61{
62 struct nvme_command cmd = {};
63 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY;
64 struct request_queue *q = ctrl->fabrics_q;
65 int ret;
66
67 if (qid != 0) {
68 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED;
69 q = ctrl->connect_q;
70 }
71
72 cmd.auth_common.opcode = nvme_fabrics_command;
73 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
74 cmd.auth_common.spsp0 = 0x01;
75 cmd.auth_common.spsp1 = 0x01;
76 if (auth_send) {
77 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
78 cmd.auth_send.tl = cpu_to_le32(data_len);
79 } else {
80 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
81 cmd.auth_receive.al = cpu_to_le32(data_len);
82 }
83
84 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
85 qid == 0 ? NVME_QID_ANY : qid, flags);
86 if (ret > 0)
87 dev_warn(ctrl->device,
88 "qid %d auth_send failed with status %d\n", qid, ret);
89 else if (ret < 0)
90 dev_err(ctrl->device,
91 "qid %d auth_send failed with error %d\n", qid, ret);
92 return ret;
93}
94
95static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
96 struct nvmf_auth_dhchap_failure_data *data,
97 u16 transaction, u8 expected_msg)
98{
99 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
100 __func__, qid, data->auth_type, data->auth_id);
101
102 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
103 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
104 return data->rescode_exp;
105 }
106 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
107 data->auth_id != expected_msg) {
108 dev_warn(ctrl->device,
109 "qid %d invalid message %02x/%02x\n",
110 qid, data->auth_type, data->auth_id);
111 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
112 }
113 if (le16_to_cpu(data->t_id) != transaction) {
114 dev_warn(ctrl->device,
115 "qid %d invalid transaction ID %d\n",
116 qid, le16_to_cpu(data->t_id));
117 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
118 }
119 return 0;
120}
121
122static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
123 struct nvme_dhchap_queue_context *chap)
124{
125 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
126 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
127 u8 dh_list_offset = NVME_AUTH_DHCHAP_MAX_DH_IDS;
128 u8 *idlist = data->auth_protocol[0].dhchap.idlist;
129
130 if (size > CHAP_BUF_SIZE) {
131 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
132 return -EINVAL;
133 }
134 memset((u8 *)chap->buf, 0, size);
135 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
136 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
137 data->t_id = cpu_to_le16(chap->transaction);
138 if (ctrl->opts->concat && chap->qid == 0) {
139 if (ctrl->opts->tls_key)
140 data->sc_c = NVME_AUTH_SECP_REPLACETLSPSK;
141 else
142 data->sc_c = NVME_AUTH_SECP_NEWTLSPSK;
143 } else
144 data->sc_c = NVME_AUTH_SECP_NOSC;
145 chap->sc_c = data->sc_c;
146 data->napd = 1;
147 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
148 data->auth_protocol[0].dhchap.halen = 3;
149 idlist[0] = NVME_AUTH_HASH_SHA256;
150 idlist[1] = NVME_AUTH_HASH_SHA384;
151 idlist[2] = NVME_AUTH_HASH_SHA512;
152 if (chap->sc_c == NVME_AUTH_SECP_NOSC)
153 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_NULL;
154 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_2048;
155 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_3072;
156 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_4096;
157 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_6144;
158 idlist[dh_list_offset++] = NVME_AUTH_DHGROUP_8192;
159 data->auth_protocol[0].dhchap.dhlen =
160 dh_list_offset - NVME_AUTH_DHCHAP_MAX_DH_IDS;
161
162 return size;
163}
164
165static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
166 struct nvme_dhchap_queue_context *chap)
167{
168 struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
169 u16 dhvlen = le16_to_cpu(data->dhvlen);
170 size_t size = sizeof(*data) + data->hl + dhvlen;
171 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
172 const char *hmac_name, *kpp_name;
173
174 if (size > CHAP_BUF_SIZE) {
175 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
176 return -EINVAL;
177 }
178
179 hmac_name = nvme_auth_hmac_name(data->hashid);
180 if (!hmac_name) {
181 dev_warn(ctrl->device,
182 "qid %d: invalid HASH ID %d\n",
183 chap->qid, data->hashid);
184 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
185 return -EPROTO;
186 }
187
188 if (chap->hash_id == data->hashid && chap->hash_len == data->hl) {
189 dev_dbg(ctrl->device,
190 "qid %d: reuse existing hash %s\n",
191 chap->qid, hmac_name);
192 goto select_kpp;
193 }
194
195 if (nvme_auth_hmac_hash_len(data->hashid) != data->hl) {
196 dev_warn(ctrl->device,
197 "qid %d: invalid hash length %d\n",
198 chap->qid, data->hl);
199 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
200 return -EPROTO;
201 }
202
203 chap->hash_id = data->hashid;
204 chap->hash_len = data->hl;
205 dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
206 chap->qid, hmac_name);
207
208select_kpp:
209 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
210 if (!kpp_name) {
211 dev_warn(ctrl->device,
212 "qid %d: invalid DH group id %d\n",
213 chap->qid, data->dhgid);
214 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
215 /* Leave previous dh_tfm intact */
216 return -EPROTO;
217 }
218
219 if (chap->dhgroup_id == data->dhgid &&
220 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
221 dev_dbg(ctrl->device,
222 "qid %d: reuse existing DH group %s\n",
223 chap->qid, gid_name);
224 goto skip_kpp;
225 }
226
227 /* Reset dh_tfm if it can't be reused */
228 if (chap->dh_tfm) {
229 crypto_free_kpp(chap->dh_tfm);
230 chap->dh_tfm = NULL;
231 }
232
233 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
234 if (dhvlen == 0) {
235 dev_warn(ctrl->device,
236 "qid %d: empty DH value\n",
237 chap->qid);
238 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
239 return -EPROTO;
240 }
241
242 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
243 if (IS_ERR(chap->dh_tfm)) {
244 int ret = PTR_ERR(chap->dh_tfm);
245
246 dev_warn(ctrl->device,
247 "qid %d: error %d initializing DH group %s\n",
248 chap->qid, ret, gid_name);
249 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
250 chap->dh_tfm = NULL;
251 return ret;
252 }
253 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
254 chap->qid, gid_name);
255 } else if (dhvlen != 0) {
256 dev_warn(ctrl->device,
257 "qid %d: invalid DH value for NULL DH\n",
258 chap->qid);
259 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
260 return -EPROTO;
261 }
262 chap->dhgroup_id = data->dhgid;
263
264skip_kpp:
265 chap->s1 = le32_to_cpu(data->seqnum);
266 memcpy(chap->c1, data->cval, chap->hash_len);
267 if (dhvlen) {
268 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
269 if (!chap->ctrl_key) {
270 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
271 return -ENOMEM;
272 }
273 chap->ctrl_key_len = dhvlen;
274 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
275 dhvlen);
276 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
277 (int)chap->ctrl_key_len, chap->ctrl_key);
278 }
279
280 return 0;
281}
282
283static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
284 struct nvme_dhchap_queue_context *chap)
285{
286 struct nvmf_auth_dhchap_reply_data *data = chap->buf;
287 size_t size = sizeof(*data);
288
289 size += 2 * chap->hash_len;
290
291 if (chap->host_key_len)
292 size += chap->host_key_len;
293
294 if (size > CHAP_BUF_SIZE) {
295 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
296 return -EINVAL;
297 }
298
299 memset(chap->buf, 0, size);
300 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
301 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
302 data->t_id = cpu_to_le16(chap->transaction);
303 data->hl = chap->hash_len;
304 data->dhvlen = cpu_to_le16(chap->host_key_len);
305 memcpy(data->rval, chap->response, chap->hash_len);
306 if (ctrl->ctrl_key)
307 chap->bi_directional = true;
308 if (ctrl->ctrl_key || ctrl->opts->concat) {
309 get_random_bytes(chap->c2, chap->hash_len);
310 data->cvalid = 1;
311 memcpy(data->rval + chap->hash_len, chap->c2,
312 chap->hash_len);
313 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
314 __func__, chap->qid, (int)chap->hash_len, chap->c2);
315 } else {
316 memset(chap->c2, 0, chap->hash_len);
317 }
318 if (ctrl->opts->concat) {
319 chap->s2 = 0;
320 chap->bi_directional = false;
321 } else
322 chap->s2 = nvme_auth_get_seqnum();
323 data->seqnum = cpu_to_le32(chap->s2);
324 if (chap->host_key_len) {
325 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
326 __func__, chap->qid,
327 chap->host_key_len, chap->host_key);
328 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
329 chap->host_key_len);
330 }
331
332 return size;
333}
334
335static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
336 struct nvme_dhchap_queue_context *chap)
337{
338 struct nvmf_auth_dhchap_success1_data *data = chap->buf;
339 size_t size = sizeof(*data) + chap->hash_len;
340
341 if (size > CHAP_BUF_SIZE) {
342 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
343 return -EINVAL;
344 }
345
346 if (data->hl != chap->hash_len) {
347 dev_warn(ctrl->device,
348 "qid %d: invalid hash length %u\n",
349 chap->qid, data->hl);
350 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
351 return -EPROTO;
352 }
353
354 /* Just print out information for the admin queue */
355 if (chap->qid == 0)
356 dev_info(ctrl->device,
357 "qid 0: authenticated with hash %s dhgroup %s\n",
358 nvme_auth_hmac_name(chap->hash_id),
359 nvme_auth_dhgroup_name(chap->dhgroup_id));
360
361 if (!data->rvalid)
362 return 0;
363
364 /* Validate controller response */
365 if (crypto_memneq(chap->response, data->rval, data->hl)) {
366 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
367 __func__, chap->qid, (int)chap->hash_len, data->rval);
368 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
369 __func__, chap->qid, (int)chap->hash_len,
370 chap->response);
371 dev_warn(ctrl->device,
372 "qid %d: controller authentication failed\n",
373 chap->qid);
374 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
375 return -ECONNREFUSED;
376 }
377
378 /* Just print out information for the admin queue */
379 if (chap->qid == 0)
380 dev_info(ctrl->device,
381 "qid 0: controller authenticated\n");
382 return 0;
383}
384
385static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
386 struct nvme_dhchap_queue_context *chap)
387{
388 struct nvmf_auth_dhchap_success2_data *data = chap->buf;
389 size_t size = sizeof(*data);
390
391 memset(chap->buf, 0, size);
392 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
393 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
394 data->t_id = cpu_to_le16(chap->transaction);
395
396 return size;
397}
398
399static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
400 struct nvme_dhchap_queue_context *chap)
401{
402 struct nvmf_auth_dhchap_failure_data *data = chap->buf;
403 size_t size = sizeof(*data);
404
405 memset(chap->buf, 0, size);
406 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
407 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
408 data->t_id = cpu_to_le16(chap->transaction);
409 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
410 data->rescode_exp = chap->status;
411
412 return size;
413}
414
415static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
416 struct nvme_dhchap_queue_context *chap)
417{
418 struct nvme_auth_hmac_ctx hmac;
419 u8 buf[4], *challenge = chap->c1;
420 int ret;
421
422 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
423 __func__, chap->qid, chap->s1, chap->transaction);
424
425 if (!chap->transformed_key) {
426 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,
427 ctrl->opts->host->nqn);
428 if (IS_ERR(chap->transformed_key)) {
429 ret = PTR_ERR(chap->transformed_key);
430 chap->transformed_key = NULL;
431 return ret;
432 }
433 } else {
434 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
435 __func__, chap->qid);
436 }
437
438 ret = nvme_auth_hmac_init(&hmac, chap->hash_id,
439 chap->transformed_key->key,
440 chap->transformed_key->len);
441 if (ret)
442 goto out;
443
444 if (chap->dh_tfm) {
445 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
446 if (!challenge) {
447 ret = -ENOMEM;
448 goto out;
449 }
450 ret = nvme_auth_augmented_challenge(chap->hash_id,
451 chap->sess_key,
452 chap->sess_key_len,
453 chap->c1, challenge,
454 chap->hash_len);
455 if (ret)
456 goto out;
457 }
458
459 nvme_auth_hmac_update(&hmac, challenge, chap->hash_len);
460
461 put_unaligned_le32(chap->s1, buf);
462 nvme_auth_hmac_update(&hmac, buf, 4);
463
464 put_unaligned_le16(chap->transaction, buf);
465 nvme_auth_hmac_update(&hmac, buf, 2);
466
467 *buf = chap->sc_c;
468 nvme_auth_hmac_update(&hmac, buf, 1);
469 nvme_auth_hmac_update(&hmac, "HostHost", 8);
470 nvme_auth_hmac_update(&hmac, ctrl->opts->host->nqn,
471 strlen(ctrl->opts->host->nqn));
472 memset(buf, 0, sizeof(buf));
473 nvme_auth_hmac_update(&hmac, buf, 1);
474 nvme_auth_hmac_update(&hmac, ctrl->opts->subsysnqn,
475 strlen(ctrl->opts->subsysnqn));
476 nvme_auth_hmac_final(&hmac, chap->response);
477 ret = 0;
478out:
479 if (challenge != chap->c1)
480 kfree(challenge);
481 memzero_explicit(&hmac, sizeof(hmac));
482 return ret;
483}
484
485static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
486 struct nvme_dhchap_queue_context *chap)
487{
488 struct nvme_auth_hmac_ctx hmac;
489 struct nvme_dhchap_key *transformed_key;
490 u8 buf[4], *challenge = chap->c2;
491 int ret;
492
493 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,
494 ctrl->opts->subsysnqn);
495 if (IS_ERR(transformed_key)) {
496 ret = PTR_ERR(transformed_key);
497 return ret;
498 }
499
500 ret = nvme_auth_hmac_init(&hmac, chap->hash_id, transformed_key->key,
501 transformed_key->len);
502 if (ret) {
503 dev_warn(ctrl->device, "qid %d: failed to init hmac, error %d\n",
504 chap->qid, ret);
505 goto out;
506 }
507
508 if (chap->dh_tfm) {
509 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
510 if (!challenge) {
511 ret = -ENOMEM;
512 goto out;
513 }
514 ret = nvme_auth_augmented_challenge(chap->hash_id,
515 chap->sess_key,
516 chap->sess_key_len,
517 chap->c2, challenge,
518 chap->hash_len);
519 if (ret)
520 goto out;
521 }
522 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
523 __func__, chap->qid, chap->s2, chap->transaction);
524 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
525 __func__, chap->qid, (int)chap->hash_len, challenge);
526 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
527 __func__, chap->qid, ctrl->opts->subsysnqn);
528 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
529 __func__, chap->qid, ctrl->opts->host->nqn);
530
531 nvme_auth_hmac_update(&hmac, challenge, chap->hash_len);
532
533 put_unaligned_le32(chap->s2, buf);
534 nvme_auth_hmac_update(&hmac, buf, 4);
535
536 put_unaligned_le16(chap->transaction, buf);
537 nvme_auth_hmac_update(&hmac, buf, 2);
538
539 *buf = chap->sc_c;
540 nvme_auth_hmac_update(&hmac, buf, 1);
541 nvme_auth_hmac_update(&hmac, "Controller", 10);
542 nvme_auth_hmac_update(&hmac, ctrl->opts->subsysnqn,
543 strlen(ctrl->opts->subsysnqn));
544 memset(buf, 0, 4);
545 nvme_auth_hmac_update(&hmac, buf, 1);
546 nvme_auth_hmac_update(&hmac, ctrl->opts->host->nqn,
547 strlen(ctrl->opts->host->nqn));
548 nvme_auth_hmac_final(&hmac, chap->response);
549 ret = 0;
550out:
551 if (challenge != chap->c2)
552 kfree(challenge);
553 memzero_explicit(&hmac, sizeof(hmac));
554 nvme_auth_free_key(transformed_key);
555 return ret;
556}
557
558static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
559 struct nvme_dhchap_queue_context *chap)
560{
561 int ret;
562
563 if (chap->host_key && chap->host_key_len) {
564 dev_dbg(ctrl->device,
565 "qid %d: reusing host key\n", chap->qid);
566 goto gen_sesskey;
567 }
568 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
569 if (ret < 0) {
570 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
571 return ret;
572 }
573
574 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
575
576 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
577 if (!chap->host_key) {
578 chap->host_key_len = 0;
579 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
580 return -ENOMEM;
581 }
582 ret = nvme_auth_gen_pubkey(chap->dh_tfm,
583 chap->host_key, chap->host_key_len);
584 if (ret) {
585 dev_dbg(ctrl->device,
586 "failed to generate public key, error %d\n", ret);
587 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
588 return ret;
589 }
590
591gen_sesskey:
592 chap->sess_key_len = chap->hash_len;
593 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
594 if (!chap->sess_key) {
595 chap->sess_key_len = 0;
596 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
597 return -ENOMEM;
598 }
599
600 ret = nvme_auth_gen_session_key(chap->dh_tfm,
601 chap->ctrl_key, chap->ctrl_key_len,
602 chap->sess_key, chap->sess_key_len,
603 chap->hash_id);
604 if (ret) {
605 dev_dbg(ctrl->device,
606 "failed to generate session key, error %d\n", ret);
607 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
608 return ret;
609 }
610 dev_dbg(ctrl->device, "session key %*ph\n",
611 (int)chap->sess_key_len, chap->sess_key);
612 return 0;
613}
614
615static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)
616{
617 nvme_auth_free_key(chap->transformed_key);
618 chap->transformed_key = NULL;
619 kfree_sensitive(chap->host_key);
620 chap->host_key = NULL;
621 chap->host_key_len = 0;
622 kfree_sensitive(chap->ctrl_key);
623 chap->ctrl_key = NULL;
624 chap->ctrl_key_len = 0;
625 kfree_sensitive(chap->sess_key);
626 chap->sess_key = NULL;
627 chap->sess_key_len = 0;
628 chap->status = 0;
629 chap->error = 0;
630 chap->s1 = 0;
631 chap->s2 = 0;
632 chap->bi_directional = false;
633 chap->transaction = 0;
634 memset(chap->c1, 0, sizeof(chap->c1));
635 memset(chap->c2, 0, sizeof(chap->c2));
636 mempool_free(chap->buf, nvme_chap_buf_pool);
637 chap->buf = NULL;
638}
639
640static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)
641{
642 nvme_auth_reset_dhchap(chap);
643 chap->authenticated = false;
644 if (chap->dh_tfm)
645 crypto_free_kpp(chap->dh_tfm);
646}
647
648void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl)
649{
650 dev_dbg(ctrl->device, "Wipe generated TLS PSK %08x\n",
651 key_serial(ctrl->opts->tls_key));
652 key_revoke(ctrl->opts->tls_key);
653 key_put(ctrl->opts->tls_key);
654 ctrl->opts->tls_key = NULL;
655}
656EXPORT_SYMBOL_GPL(nvme_auth_revoke_tls_key);
657
658static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl,
659 struct nvme_dhchap_queue_context *chap)
660{
661 u8 *psk, *tls_psk;
662 char *digest;
663 struct key *tls_key;
664 size_t psk_len;
665 int ret = 0;
666
667 if (!chap->sess_key) {
668 dev_warn(ctrl->device,
669 "%s: qid %d no session key negotiated\n",
670 __func__, chap->qid);
671 return -ENOKEY;
672 }
673
674 if (chap->qid) {
675 dev_warn(ctrl->device,
676 "qid %d: secure concatenation not supported on I/O queues\n",
677 chap->qid);
678 return -EINVAL;
679 }
680 ret = nvme_auth_generate_psk(chap->hash_id, chap->sess_key,
681 chap->sess_key_len,
682 chap->c1, chap->c2,
683 chap->hash_len, &psk, &psk_len);
684 if (ret) {
685 dev_warn(ctrl->device,
686 "%s: qid %d failed to generate PSK, error %d\n",
687 __func__, chap->qid, ret);
688 return ret;
689 }
690 dev_dbg(ctrl->device,
691 "%s: generated psk %*ph\n", __func__, (int)psk_len, psk);
692
693 ret = nvme_auth_generate_digest(chap->hash_id, psk, psk_len,
694 ctrl->opts->subsysnqn,
695 ctrl->opts->host->nqn, &digest);
696 if (ret) {
697 dev_warn(ctrl->device,
698 "%s: qid %d failed to generate digest, error %d\n",
699 __func__, chap->qid, ret);
700 goto out_free_psk;
701 }
702 dev_dbg(ctrl->device, "%s: generated digest %s\n",
703 __func__, digest);
704 ret = nvme_auth_derive_tls_psk(chap->hash_id, psk, psk_len,
705 digest, &tls_psk);
706 if (ret) {
707 dev_warn(ctrl->device,
708 "%s: qid %d failed to derive TLS psk, error %d\n",
709 __func__, chap->qid, ret);
710 goto out_free_digest;
711 }
712
713 tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring,
714 ctrl->opts->host->nqn,
715 ctrl->opts->subsysnqn, chap->hash_id,
716 tls_psk, psk_len, digest);
717 if (IS_ERR(tls_key)) {
718 ret = PTR_ERR(tls_key);
719 dev_warn(ctrl->device,
720 "%s: qid %d failed to insert generated key, error %d\n",
721 __func__, chap->qid, ret);
722 tls_key = NULL;
723 }
724 kfree_sensitive(tls_psk);
725 if (ctrl->opts->tls_key)
726 nvme_auth_revoke_tls_key(ctrl);
727 ctrl->opts->tls_key = tls_key;
728out_free_digest:
729 kfree_sensitive(digest);
730out_free_psk:
731 kfree_sensitive(psk);
732 return ret;
733}
734
735static void nvme_queue_auth_work(struct work_struct *work)
736{
737 struct nvme_dhchap_queue_context *chap =
738 container_of(work, struct nvme_dhchap_queue_context, auth_work);
739 struct nvme_ctrl *ctrl = chap->ctrl;
740 size_t tl;
741 int ret = 0;
742
743 /*
744 * Allocate a large enough buffer for the entire negotiation:
745 * 4k is enough to ffdhe8192.
746 */
747 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);
748 if (!chap->buf) {
749 chap->error = -ENOMEM;
750 return;
751 }
752
753 chap->transaction = ctrl->transaction++;
754
755 /* DH-HMAC-CHAP Step 1: send negotiate */
756 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
757 __func__, chap->qid);
758 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
759 if (ret < 0) {
760 chap->error = ret;
761 return;
762 }
763 tl = ret;
764 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
765 if (ret) {
766 chap->error = ret;
767 return;
768 }
769
770 /* DH-HMAC-CHAP Step 2: receive challenge */
771 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
772 __func__, chap->qid);
773
774 memset(chap->buf, 0, CHAP_BUF_SIZE);
775 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
776 false);
777 if (ret) {
778 dev_warn(ctrl->device,
779 "qid %d failed to receive challenge, %s %d\n",
780 chap->qid, ret < 0 ? "error" : "nvme status", ret);
781 chap->error = ret;
782 return;
783 }
784 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
785 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
786 if (ret) {
787 chap->status = ret;
788 chap->error = -EKEYREJECTED;
789 return;
790 }
791
792 ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
793 if (ret) {
794 /* Invalid challenge parameters */
795 chap->error = ret;
796 goto fail2;
797 }
798
799 if (chap->ctrl_key_len) {
800 dev_dbg(ctrl->device,
801 "%s: qid %d DH exponential\n",
802 __func__, chap->qid);
803 ret = nvme_auth_dhchap_exponential(ctrl, chap);
804 if (ret) {
805 chap->error = ret;
806 goto fail2;
807 }
808 }
809
810 dev_dbg(ctrl->device, "%s: qid %d host response\n",
811 __func__, chap->qid);
812 mutex_lock(&ctrl->dhchap_auth_mutex);
813 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
814 mutex_unlock(&ctrl->dhchap_auth_mutex);
815 if (ret) {
816 chap->error = ret;
817 goto fail2;
818 }
819
820 /* DH-HMAC-CHAP Step 3: send reply */
821 dev_dbg(ctrl->device, "%s: qid %d send reply\n",
822 __func__, chap->qid);
823 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
824 if (ret < 0) {
825 chap->error = ret;
826 goto fail2;
827 }
828
829 tl = ret;
830 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
831 if (ret) {
832 chap->error = ret;
833 goto fail2;
834 }
835
836 /* DH-HMAC-CHAP Step 4: receive success1 */
837 dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
838 __func__, chap->qid);
839
840 memset(chap->buf, 0, CHAP_BUF_SIZE);
841 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
842 false);
843 if (ret) {
844 dev_warn(ctrl->device,
845 "qid %d failed to receive success1, %s %d\n",
846 chap->qid, ret < 0 ? "error" : "nvme status", ret);
847 chap->error = ret;
848 return;
849 }
850 ret = nvme_auth_receive_validate(ctrl, chap->qid,
851 chap->buf, chap->transaction,
852 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
853 if (ret) {
854 chap->status = ret;
855 chap->error = -EKEYREJECTED;
856 return;
857 }
858
859 mutex_lock(&ctrl->dhchap_auth_mutex);
860 if (ctrl->ctrl_key) {
861 dev_dbg(ctrl->device,
862 "%s: qid %d controller response\n",
863 __func__, chap->qid);
864 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
865 if (ret) {
866 mutex_unlock(&ctrl->dhchap_auth_mutex);
867 chap->error = ret;
868 goto fail2;
869 }
870 }
871 mutex_unlock(&ctrl->dhchap_auth_mutex);
872
873 ret = nvme_auth_process_dhchap_success1(ctrl, chap);
874 if (ret) {
875 /* Controller authentication failed */
876 chap->error = -EKEYREJECTED;
877 goto fail2;
878 }
879
880 if (chap->bi_directional) {
881 /* DH-HMAC-CHAP Step 5: send success2 */
882 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
883 __func__, chap->qid);
884 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
885 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
886 if (ret)
887 chap->error = ret;
888 }
889 if (!ret) {
890 chap->error = 0;
891 chap->authenticated = true;
892 if (ctrl->opts->concat &&
893 (ret = nvme_auth_secure_concat(ctrl, chap))) {
894 dev_warn(ctrl->device,
895 "%s: qid %d failed to enable secure concatenation\n",
896 __func__, chap->qid);
897 chap->error = ret;
898 chap->authenticated = false;
899 }
900 return;
901 }
902
903fail2:
904 if (chap->status == 0)
905 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
906 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
907 __func__, chap->qid, chap->status);
908 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
909 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
910 /*
911 * only update error if send failure2 failed and no other
912 * error had been set during authentication.
913 */
914 if (ret && !chap->error)
915 chap->error = ret;
916}
917
918int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
919{
920 struct nvme_dhchap_queue_context *chap;
921
922 if (!ctrl->host_key) {
923 dev_warn(ctrl->device, "qid %d: no key\n", qid);
924 return -ENOKEY;
925 }
926
927 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
928 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
929 return -ENOKEY;
930 }
931
932 chap = &ctrl->dhchap_ctxs[qid];
933 cancel_work_sync(&chap->auth_work);
934 queue_work(nvme_auth_wq, &chap->auth_work);
935 return 0;
936}
937EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
938
939int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
940{
941 struct nvme_dhchap_queue_context *chap;
942 int ret;
943
944 chap = &ctrl->dhchap_ctxs[qid];
945 flush_work(&chap->auth_work);
946 ret = chap->error;
947 /* clear sensitive info */
948 nvme_auth_reset_dhchap(chap);
949 return ret;
950}
951EXPORT_SYMBOL_GPL(nvme_auth_wait);
952
953static void nvme_ctrl_auth_work(struct work_struct *work)
954{
955 struct nvme_ctrl *ctrl =
956 container_of(work, struct nvme_ctrl, dhchap_auth_work);
957 int ret, q;
958
959 /*
960 * If the ctrl is no connected, bail as reconnect will handle
961 * authentication.
962 */
963 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
964 return;
965
966 /* Authenticate admin queue first */
967 ret = nvme_auth_negotiate(ctrl, 0);
968 if (ret) {
969 dev_warn(ctrl->device,
970 "qid 0: error %d setting up authentication\n", ret);
971 return;
972 }
973 ret = nvme_auth_wait(ctrl, 0);
974 if (ret) {
975 dev_warn(ctrl->device,
976 "qid 0: authentication failed\n");
977 return;
978 }
979 /*
980 * Only run authentication on the admin queue for secure concatenation.
981 */
982 if (ctrl->opts->concat)
983 return;
984
985 for (q = 1; q < ctrl->queue_count; q++) {
986 struct nvme_dhchap_queue_context *chap =
987 &ctrl->dhchap_ctxs[q];
988 /*
989 * Skip re-authentication if the queue had
990 * not been authenticated initially.
991 */
992 if (!chap->authenticated)
993 continue;
994 cancel_work_sync(&chap->auth_work);
995 queue_work(nvme_auth_wq, &chap->auth_work);
996 }
997
998 /*
999 * Failure is a soft-state; credentials remain valid until
1000 * the controller terminates the connection.
1001 */
1002 for (q = 1; q < ctrl->queue_count; q++) {
1003 struct nvme_dhchap_queue_context *chap =
1004 &ctrl->dhchap_ctxs[q];
1005 if (!chap->authenticated)
1006 continue;
1007 flush_work(&chap->auth_work);
1008 ret = chap->error;
1009 nvme_auth_reset_dhchap(chap);
1010 if (ret)
1011 dev_warn(ctrl->device,
1012 "qid %d: authentication failed\n", q);
1013 }
1014}
1015
1016int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
1017{
1018 struct nvme_dhchap_queue_context *chap;
1019 int i, ret;
1020
1021 mutex_init(&ctrl->dhchap_auth_mutex);
1022 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);
1023 if (!ctrl->opts)
1024 return 0;
1025 ret = nvme_auth_parse_key(ctrl->opts->dhchap_secret, &ctrl->host_key);
1026 if (ret)
1027 return ret;
1028 ret = nvme_auth_parse_key(ctrl->opts->dhchap_ctrl_secret,
1029 &ctrl->ctrl_key);
1030 if (ret)
1031 goto err_free_dhchap_secret;
1032
1033 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)
1034 return 0;
1035
1036 ctrl->dhchap_ctxs = kvzalloc_objs(*chap, ctrl_max_dhchaps(ctrl));
1037 if (!ctrl->dhchap_ctxs) {
1038 ret = -ENOMEM;
1039 goto err_free_dhchap_ctrl_secret;
1040 }
1041
1042 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {
1043 chap = &ctrl->dhchap_ctxs[i];
1044 chap->qid = i;
1045 chap->ctrl = ctrl;
1046 chap->authenticated = false;
1047 INIT_WORK(&chap->auth_work, nvme_queue_auth_work);
1048 }
1049
1050 return 0;
1051err_free_dhchap_ctrl_secret:
1052 nvme_auth_free_key(ctrl->ctrl_key);
1053 ctrl->ctrl_key = NULL;
1054err_free_dhchap_secret:
1055 nvme_auth_free_key(ctrl->host_key);
1056 ctrl->host_key = NULL;
1057 return ret;
1058}
1059EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
1060
1061void nvme_auth_stop(struct nvme_ctrl *ctrl)
1062{
1063 cancel_work_sync(&ctrl->dhchap_auth_work);
1064}
1065EXPORT_SYMBOL_GPL(nvme_auth_stop);
1066
1067void nvme_auth_free(struct nvme_ctrl *ctrl)
1068{
1069 int i;
1070
1071 if (ctrl->dhchap_ctxs) {
1072 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)
1073 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);
1074 kvfree(ctrl->dhchap_ctxs);
1075 }
1076 if (ctrl->host_key) {
1077 nvme_auth_free_key(ctrl->host_key);
1078 ctrl->host_key = NULL;
1079 }
1080 if (ctrl->ctrl_key) {
1081 nvme_auth_free_key(ctrl->ctrl_key);
1082 ctrl->ctrl_key = NULL;
1083 }
1084}
1085EXPORT_SYMBOL_GPL(nvme_auth_free);
1086
1087int __init nvme_init_auth(void)
1088{
1089 nvme_auth_wq = alloc_workqueue("nvme-auth-wq",
1090 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1091 if (!nvme_auth_wq)
1092 return -ENOMEM;
1093
1094 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",
1095 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
1096 if (!nvme_chap_buf_cache)
1097 goto err_destroy_workqueue;
1098
1099 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,
1100 mempool_free_slab, nvme_chap_buf_cache);
1101 if (!nvme_chap_buf_pool)
1102 goto err_destroy_chap_buf_cache;
1103
1104 return 0;
1105err_destroy_chap_buf_cache:
1106 kmem_cache_destroy(nvme_chap_buf_cache);
1107err_destroy_workqueue:
1108 destroy_workqueue(nvme_auth_wq);
1109 return -ENOMEM;
1110}
1111
1112void __exit nvme_exit_auth(void)
1113{
1114 mempool_destroy(nvme_chap_buf_pool);
1115 kmem_cache_destroy(nvme_chap_buf_cache);
1116 destroy_workqueue(nvme_auth_wq);
1117}