linux - drivers/crypto/ccp/sev-dev.c

3170 lines · 79.6 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* AMD Secure Encrypted Virtualization (SEV) interface
4	*
5	* Copyright (C) 2016,2019 Advanced Micro Devices, Inc.
6	*
7	* Author: Brijesh Singh <[email protected]>
8	*/
9
10	#include <linux/bitfield.h>
11	#include <linux/module.h>
12	#include <linux/kernel.h>
13	#include <linux/kthread.h>
14	#include <linux/sched.h>
15	#include <linux/interrupt.h>
16	#include <linux/spinlock.h>
17	#include <linux/spinlock_types.h>
18	#include <linux/types.h>
19	#include <linux/mutex.h>
20	#include <linux/delay.h>
21	#include <linux/hw_random.h>
22	#include <linux/ccp.h>
23	#include <linux/firmware.h>
24	#include <linux/panic_notifier.h>
25	#include <linux/gfp.h>
26	#include <linux/cpufeature.h>
27	#include <linux/fs.h>
28	#include <linux/fs_struct.h>
29	#include <linux/psp.h>
30	#include <linux/amd-iommu.h>
31	#include <linux/crash_dump.h>
32
33	#include <asm/smp.h>
34	#include <asm/cacheflush.h>
35	#include <asm/e820/types.h>
36	#include <asm/sev.h>
37	#include <asm/msr.h>
38
39	#include "psp-dev.h"
40	#include "sev-dev.h"
41
42	#define DEVICE_NAME "sev"
43	#define SEV_FW_FILE "amd/sev.fw"
44	#define SEV_FW_NAME_SIZE 64
45
46	/* Minimum firmware version required for the SEV-SNP support */
47	#define SNP_MIN_API_MAJOR 1
48	#define SNP_MIN_API_MINOR 51
49
50	/*
51	* Maximum number of firmware-writable buffers that might be specified
52	* in the parameters of a legacy SEV command buffer.
53	*/
54	#define CMD_BUF_FW_WRITABLE_MAX 2
55
56	/* Leave room in the descriptor array for an end-of-list indicator. */
57	#define CMD_BUF_DESC_MAX (CMD_BUF_FW_WRITABLE_MAX + 1)
58
59	static DEFINE_MUTEX(sev_cmd_mutex);
60	static struct sev_misc_dev *misc_dev;
61
62	static int psp_cmd_timeout = 100;
63	module_param(psp_cmd_timeout, int, 0644);
64	MODULE_PARM_DESC(psp_cmd_timeout, " default timeout value, in seconds, for PSP commands");
65
66	static int psp_probe_timeout = 5;
67	module_param(psp_probe_timeout, int, 0644);
68	MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
69
70	static char *init_ex_path;
71	module_param(init_ex_path, charp, 0444);
72	MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
73
74	static bool psp_init_on_probe = true;
75	module_param(psp_init_on_probe, bool, 0444);
76	MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
77
78	#if IS_ENABLED(CONFIG_PCI_TSM)
79	static bool sev_tio_enabled = true;
80	module_param_named(tio, sev_tio_enabled, bool, 0444);
81	MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX");
82	#else
83	static const bool sev_tio_enabled = false;
84	#endif
85
86	MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
87	MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
88	MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
89	MODULE_FIRMWARE("amd/amd_sev_fam19h_model1xh.sbin"); /* 4th gen EPYC */
90
91	static bool psp_dead;
92	static int psp_timeout;
93
94	enum snp_hv_fixed_pages_state {
95	ALLOCATED,
96	HV_FIXED,
97	};
98
99	struct snp_hv_fixed_pages_entry {
100	struct list_head list;
101	struct page *page;
102	unsigned int order;
103	bool free;
104	enum snp_hv_fixed_pages_state page_state;
105	};
106
107	static LIST_HEAD(snp_hv_fixed_pages);
108
109	/* Trusted Memory Region (TMR):
110	* The TMR is a 1MB area that must be 1MB aligned. Use the page allocator
111	* to allocate the memory, which will return aligned memory for the specified
112	* allocation order.
113	*
114	* When SEV-SNP is enabled the TMR needs to be 2MB aligned and 2MB sized.
115	*/
116	#define SEV_TMR_SIZE (1024 * 1024)
117	#define SNP_TMR_SIZE (2 * 1024 * 1024)
118
119	static void *sev_es_tmr;
120	static size_t sev_es_tmr_size = SEV_TMR_SIZE;
121
122	/* INIT_EX NV Storage:
123	* The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
124	* allocator to allocate the memory, which will return aligned memory for the
125	* specified allocation order.
126	*/
127	#define NV_LENGTH (32 * 1024)
128	static void *sev_init_ex_buffer;
129
130	static void __sev_firmware_shutdown(struct sev_device *sev, bool panic);
131
132	static int snp_shutdown_on_panic(struct notifier_block *nb,
133	unsigned long reason, void *arg);
134
135	static struct notifier_block snp_panic_notifier = {
136	.notifier_call = snp_shutdown_on_panic,
137	};
138
139	static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
140	{
141	struct sev_device *sev = psp_master->sev_data;
142
143	if (sev->api_major > maj)
144	return true;
145
146	if (sev->api_major == maj && sev->api_minor >= min)
147	return true;
148
149	return false;
150	}
151
152	static void sev_irq_handler(int irq, void *data, unsigned int status)
153	{
154	struct sev_device *sev = data;
155	int reg;
156
157	/* Check if it is command completion: */
158	if (!(status & SEV_CMD_COMPLETE))
159	return;
160
161	/* Check if it is SEV command completion: */
162	reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
163	if (FIELD_GET(PSP_CMDRESP_RESP, reg)) {
164	sev->int_rcvd = 1;
165	wake_up(&sev->int_queue);
166	}
167	}
168
169	static int sev_wait_cmd_ioc(struct sev_device *sev,
170	unsigned int *reg, unsigned int timeout)
171	{
172	int ret;
173
174	/*
175	* If invoked during panic handling, local interrupts are disabled,
176	* so the PSP command completion interrupt can't be used. Poll for
177	* PSP command completion instead.
178	*/
179	if (irqs_disabled()) {
180	unsigned long timeout_usecs = (timeout * USEC_PER_SEC) / 10;
181
182	/* Poll for SEV command completion: */
183	while (timeout_usecs--) {
184	*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
185	if (*reg & PSP_CMDRESP_RESP)
186	return 0;
187
188	udelay(10);
189	}
190	return -ETIMEDOUT;
191	}
192
193	ret = wait_event_timeout(sev->int_queue,
194	sev->int_rcvd, timeout * HZ);
195	if (!ret)
196	return -ETIMEDOUT;
197
198	*reg = ioread32(sev->io_regs + sev->vdata->cmdresp_reg);
199
200	return 0;
201	}
202
203	static int sev_cmd_buffer_len(int cmd)
204	{
205	switch (cmd) {
206	case SEV_CMD_INIT: return sizeof(struct sev_data_init);
207	case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
208	case SEV_CMD_SNP_SHUTDOWN_EX: return sizeof(struct sev_data_snp_shutdown_ex);
209	case SEV_CMD_SNP_INIT_EX: return sizeof(struct sev_data_snp_init_ex);
210	case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
211	case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
212	case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
213	case SEV_CMD_PDH_CERT_EXPORT: return sizeof(struct sev_data_pdh_cert_export);
214	case SEV_CMD_LAUNCH_START: return sizeof(struct sev_data_launch_start);
215	case SEV_CMD_LAUNCH_UPDATE_DATA: return sizeof(struct sev_data_launch_update_data);
216	case SEV_CMD_LAUNCH_UPDATE_VMSA: return sizeof(struct sev_data_launch_update_vmsa);
217	case SEV_CMD_LAUNCH_FINISH: return sizeof(struct sev_data_launch_finish);
218	case SEV_CMD_LAUNCH_MEASURE: return sizeof(struct sev_data_launch_measure);
219	case SEV_CMD_ACTIVATE: return sizeof(struct sev_data_activate);
220	case SEV_CMD_DEACTIVATE: return sizeof(struct sev_data_deactivate);
221	case SEV_CMD_DECOMMISSION: return sizeof(struct sev_data_decommission);
222	case SEV_CMD_GUEST_STATUS: return sizeof(struct sev_data_guest_status);
223	case SEV_CMD_DBG_DECRYPT: return sizeof(struct sev_data_dbg);
224	case SEV_CMD_DBG_ENCRYPT: return sizeof(struct sev_data_dbg);
225	case SEV_CMD_SEND_START: return sizeof(struct sev_data_send_start);
226	case SEV_CMD_SEND_UPDATE_DATA: return sizeof(struct sev_data_send_update_data);
227	case SEV_CMD_SEND_UPDATE_VMSA: return sizeof(struct sev_data_send_update_vmsa);
228	case SEV_CMD_SEND_FINISH: return sizeof(struct sev_data_send_finish);
229	case SEV_CMD_RECEIVE_START: return sizeof(struct sev_data_receive_start);
230	case SEV_CMD_RECEIVE_FINISH: return sizeof(struct sev_data_receive_finish);
231	case SEV_CMD_RECEIVE_UPDATE_DATA: return sizeof(struct sev_data_receive_update_data);
232	case SEV_CMD_RECEIVE_UPDATE_VMSA: return sizeof(struct sev_data_receive_update_vmsa);
233	case SEV_CMD_LAUNCH_UPDATE_SECRET: return sizeof(struct sev_data_launch_secret);
234	case SEV_CMD_DOWNLOAD_FIRMWARE: return sizeof(struct sev_data_download_firmware);
235	case SEV_CMD_GET_ID: return sizeof(struct sev_data_get_id);
236	case SEV_CMD_ATTESTATION_REPORT: return sizeof(struct sev_data_attestation_report);
237	case SEV_CMD_SEND_CANCEL: return sizeof(struct sev_data_send_cancel);
238	case SEV_CMD_SNP_GCTX_CREATE: return sizeof(struct sev_data_snp_addr);
239	case SEV_CMD_SNP_LAUNCH_START: return sizeof(struct sev_data_snp_launch_start);
240	case SEV_CMD_SNP_LAUNCH_UPDATE: return sizeof(struct sev_data_snp_launch_update);
241	case SEV_CMD_SNP_ACTIVATE: return sizeof(struct sev_data_snp_activate);
242	case SEV_CMD_SNP_DECOMMISSION: return sizeof(struct sev_data_snp_addr);
243	case SEV_CMD_SNP_PAGE_RECLAIM: return sizeof(struct sev_data_snp_page_reclaim);
244	case SEV_CMD_SNP_GUEST_STATUS: return sizeof(struct sev_data_snp_guest_status);
245	case SEV_CMD_SNP_LAUNCH_FINISH: return sizeof(struct sev_data_snp_launch_finish);
246	case SEV_CMD_SNP_DBG_DECRYPT: return sizeof(struct sev_data_snp_dbg);
247	case SEV_CMD_SNP_DBG_ENCRYPT: return sizeof(struct sev_data_snp_dbg);
248	case SEV_CMD_SNP_VERIFY_MITIGATION: return sizeof(struct sev_data_snp_verify_mitigation);
249	case SEV_CMD_SNP_PAGE_UNSMASH: return sizeof(struct sev_data_snp_page_unsmash);
250	case SEV_CMD_SNP_PLATFORM_STATUS: return sizeof(struct sev_data_snp_addr);
251	case SEV_CMD_SNP_GUEST_REQUEST: return sizeof(struct sev_data_snp_guest_request);
252	case SEV_CMD_SNP_CONFIG: return sizeof(struct sev_user_data_snp_config);
253	case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit);
254	case SEV_CMD_SNP_FEATURE_INFO: return sizeof(struct sev_data_snp_feature_info);
255	case SEV_CMD_SNP_VLEK_LOAD: return sizeof(struct sev_user_data_snp_vlek_load);
256	default: return sev_tio_cmd_buffer_len(cmd);
257	}
258
259	return 0;
260	}
261
262	static struct file open_file_as_root(const char filename, int flags, umode_t mode)
263	{
264	CLASS(prepare_creds, cred)();
265	if (!cred)
266	return ERR_PTR(-ENOMEM);
267
268	cred->fsuid = GLOBAL_ROOT_UID;
269
270	scoped_with_init_fs() {
271	scoped_with_creds(cred)
272	return filp_open(filename, flags, mode);
273	}
274	}
275
276	static int sev_read_init_ex_file(void)
277	{
278	struct sev_device *sev = psp_master->sev_data;
279	struct file *fp;
280	ssize_t nread;
281
282	lockdep_assert_held(&sev_cmd_mutex);
283
284	if (!sev_init_ex_buffer)
285	return -EOPNOTSUPP;
286
287	fp = open_file_as_root(init_ex_path, O_RDONLY, 0);
288	if (IS_ERR(fp)) {
289	int ret = PTR_ERR(fp);
290
291	if (ret == -ENOENT) {
292	dev_info(sev->dev,
293	"SEV: %s does not exist and will be created later.\n",
294	init_ex_path);
295	ret = 0;
296	} else {
297	dev_err(sev->dev,
298	"SEV: could not open %s for read, error %d\n",
299	init_ex_path, ret);
300	}
301	return ret;
302	}
303
304	nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
305	if (nread != NV_LENGTH) {
306	dev_info(sev->dev,
307	"SEV: could not read %u bytes to non volatile memory area, ret %ld\n",
308	NV_LENGTH, nread);
309	}
310
311	dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
312	filp_close(fp, NULL);
313
314	return 0;
315	}
316
317	static int sev_write_init_ex_file(void)
318	{
319	struct sev_device *sev = psp_master->sev_data;
320	struct file *fp;
321	loff_t offset = 0;
322	ssize_t nwrite;
323
324	lockdep_assert_held(&sev_cmd_mutex);
325
326	if (!sev_init_ex_buffer)
327	return 0;
328
329	fp = open_file_as_root(init_ex_path, O_CREAT \| O_WRONLY, 0600);
330	if (IS_ERR(fp)) {
331	int ret = PTR_ERR(fp);
332
333	dev_err(sev->dev,
334	"SEV: could not open file for write, error %d\n",
335	ret);
336	return ret;
337	}
338
339	nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
340	vfs_fsync(fp, 0);
341	filp_close(fp, NULL);
342
343	if (nwrite != NV_LENGTH) {
344	dev_err(sev->dev,
345	"SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
346	NV_LENGTH, nwrite);
347	return -EIO;
348	}
349
350	dev_dbg(sev->dev, "SEV: write successful to NV file\n");
351
352	return 0;
353	}
354
355	static int sev_write_init_ex_file_if_required(int cmd_id)
356	{
357	lockdep_assert_held(&sev_cmd_mutex);
358
359	if (!sev_init_ex_buffer)
360	return 0;
361
362	/*
363	* Only a few platform commands modify the SPI/NV area, but none of the
364	* non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
365	* PEK_CERT_IMPORT, and PDH_GEN do.
366	*/
367	switch (cmd_id) {
368	case SEV_CMD_FACTORY_RESET:
369	case SEV_CMD_INIT_EX:
370	case SEV_CMD_PDH_GEN:
371	case SEV_CMD_PEK_CERT_IMPORT:
372	case SEV_CMD_PEK_GEN:
373	break;
374	default:
375	return 0;
376	}
377
378	return sev_write_init_ex_file();
379	}
380
381	int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked)
382	{
383	int ret, err, i;
384
385	paddr = __sme_clr(ALIGN_DOWN(paddr, PAGE_SIZE));
386
387	for (i = 0; i < npages; i++, paddr += PAGE_SIZE) {
388	struct sev_data_snp_page_reclaim data = {0};
389
390	data.paddr = paddr;
391
392	if (locked)
393	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
394	else
395	ret = sev_do_cmd(SEV_CMD_SNP_PAGE_RECLAIM, &data, &err);
396
397	if (ret)
398	goto cleanup;
399
400	ret = rmp_make_shared(__phys_to_pfn(paddr), PG_LEVEL_4K);
401	if (ret)
402	goto cleanup;
403	}
404
405	return 0;
406
407	cleanup:
408	/*
409	* If there was a failure reclaiming the page then it is no longer safe
410	* to release it back to the system; leak it instead.
411	*/
412	snp_leak_pages(__phys_to_pfn(paddr), npages - i);
413	return ret;
414	}
415	EXPORT_SYMBOL_GPL(snp_reclaim_pages);
416
417	static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked)
418	{
419	unsigned long pfn = __sme_clr(paddr) >> PAGE_SHIFT;
420	int rc, i;
421
422	for (i = 0; i < npages; i++, pfn++) {
423	rc = rmp_make_private(pfn, 0, PG_LEVEL_4K, 0, true);
424	if (rc)
425	goto cleanup;
426	}
427
428	return 0;
429
430	cleanup:
431	/*
432	* Try unrolling the firmware state changes by
433	* reclaiming the pages which were already changed to the
434	* firmware state.
435	*/
436	snp_reclaim_pages(paddr, i, locked);
437
438	return rc;
439	}
440
441	static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked)
442	{
443	unsigned long npages = 1ul << order, paddr;
444	struct sev_device *sev;
445	struct page *page;
446
447	if (!psp_master \|\| !psp_master->sev_data)
448	return NULL;
449
450	page = alloc_pages(gfp_mask, order);
451	if (!page)
452	return NULL;
453
454	/* If SEV-SNP is initialized then add the page in RMP table. */
455	sev = psp_master->sev_data;
456	if (!sev->snp_initialized)
457	return page;
458
459	paddr = __pa((unsigned long)page_address(page));
460	if (rmp_mark_pages_firmware(paddr, npages, locked))
461	return NULL;
462
463	return page;
464	}
465
466	void *snp_alloc_firmware_page(gfp_t gfp_mask)
467	{
468	struct page *page;
469
470	page = __snp_alloc_firmware_pages(gfp_mask, 0, false);
471
472	return page ? page_address(page) : NULL;
473	}
474	EXPORT_SYMBOL_GPL(snp_alloc_firmware_page);
475
476	static void __snp_free_firmware_pages(struct page *page, int order, bool locked)
477	{
478	struct sev_device *sev = psp_master->sev_data;
479	unsigned long paddr, npages = 1ul << order;
480
481	if (!page)
482	return;
483
484	paddr = __pa((unsigned long)page_address(page));
485	if (sev->snp_initialized &&
486	snp_reclaim_pages(paddr, npages, locked))
487	return;
488
489	__free_pages(page, order);
490	}
491
492	void snp_free_firmware_page(void *addr)
493	{
494	if (!addr)
495	return;
496
497	__snp_free_firmware_pages(virt_to_page(addr), 0, false);
498	}
499	EXPORT_SYMBOL_GPL(snp_free_firmware_page);
500
501	static void *sev_fw_alloc(unsigned long len)
502	{
503	struct page *page;
504
505	page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true);
506	if (!page)
507	return NULL;
508
509	return page_address(page);
510	}
511
512	/**
513	* struct cmd_buf_desc - descriptors for managing legacy SEV command address
514	* parameters corresponding to buffers that may be written to by firmware.
515	*
516	* @paddr_ptr: pointer to the address parameter in the command buffer which may
517	* need to be saved/restored depending on whether a bounce buffer
518	* is used. In the case of a bounce buffer, the command buffer
519	* needs to be updated with the address of the new bounce buffer
520	* snp_map_cmd_buf_desc() has allocated specifically for it. Must
521	* be NULL if this descriptor is only an end-of-list indicator.
522	*
523	* @paddr_orig: storage for the original address parameter, which can be used to
524	* restore the original value in @paddr_ptr in cases where it is
525	* replaced with the address of a bounce buffer.
526	*
527	* @len: length of buffer located at the address originally stored at @paddr_ptr
528	*
529	* @guest_owned: true if the address corresponds to guest-owned pages, in which
530	* case bounce buffers are not needed.
531	*/
532	struct cmd_buf_desc {
533	u64 *paddr_ptr;
534	u64 paddr_orig;
535	u32 len;
536	bool guest_owned;
537	};
538
539	/*
540	* If a legacy SEV command parameter is a memory address, those pages in
541	* turn need to be transitioned to/from firmware-owned before/after
542	* executing the firmware command.
543	*
544	* Additionally, in cases where those pages are not guest-owned, a bounce
545	* buffer is needed in place of the original memory address parameter.
546	*
547	* A set of descriptors are used to keep track of this handling, and
548	* initialized here based on the specific commands being executed.
549	*/
550	static void snp_populate_cmd_buf_desc_list(int cmd, void *cmd_buf,
551	struct cmd_buf_desc *desc_list)
552	{
553	switch (cmd) {
554	case SEV_CMD_PDH_CERT_EXPORT: {
555	struct sev_data_pdh_cert_export *data = cmd_buf;
556
557	desc_list[0].paddr_ptr = &data->pdh_cert_address;
558	desc_list[0].len = data->pdh_cert_len;
559	desc_list[1].paddr_ptr = &data->cert_chain_address;
560	desc_list[1].len = data->cert_chain_len;
561	break;
562	}
563	case SEV_CMD_GET_ID: {
564	struct sev_data_get_id *data = cmd_buf;
565
566	desc_list[0].paddr_ptr = &data->address;
567	desc_list[0].len = data->len;
568	break;
569	}
570	case SEV_CMD_PEK_CSR: {
571	struct sev_data_pek_csr *data = cmd_buf;
572
573	desc_list[0].paddr_ptr = &data->address;
574	desc_list[0].len = data->len;
575	break;
576	}
577	case SEV_CMD_LAUNCH_UPDATE_DATA: {
578	struct sev_data_launch_update_data *data = cmd_buf;
579
580	desc_list[0].paddr_ptr = &data->address;
581	desc_list[0].len = data->len;
582	desc_list[0].guest_owned = true;
583	break;
584	}
585	case SEV_CMD_LAUNCH_UPDATE_VMSA: {
586	struct sev_data_launch_update_vmsa *data = cmd_buf;
587
588	desc_list[0].paddr_ptr = &data->address;
589	desc_list[0].len = data->len;
590	desc_list[0].guest_owned = true;
591	break;
592	}
593	case SEV_CMD_LAUNCH_MEASURE: {
594	struct sev_data_launch_measure *data = cmd_buf;
595
596	desc_list[0].paddr_ptr = &data->address;
597	desc_list[0].len = data->len;
598	break;
599	}
600	case SEV_CMD_LAUNCH_UPDATE_SECRET: {
601	struct sev_data_launch_secret *data = cmd_buf;
602
603	desc_list[0].paddr_ptr = &data->guest_address;
604	desc_list[0].len = data->guest_len;
605	desc_list[0].guest_owned = true;
606	break;
607	}
608	case SEV_CMD_DBG_DECRYPT: {
609	struct sev_data_dbg *data = cmd_buf;
610
611	desc_list[0].paddr_ptr = &data->dst_addr;
612	desc_list[0].len = data->len;
613	desc_list[0].guest_owned = true;
614	break;
615	}
616	case SEV_CMD_DBG_ENCRYPT: {
617	struct sev_data_dbg *data = cmd_buf;
618
619	desc_list[0].paddr_ptr = &data->dst_addr;
620	desc_list[0].len = data->len;
621	desc_list[0].guest_owned = true;
622	break;
623	}
624	case SEV_CMD_ATTESTATION_REPORT: {
625	struct sev_data_attestation_report *data = cmd_buf;
626
627	desc_list[0].paddr_ptr = &data->address;
628	desc_list[0].len = data->len;
629	break;
630	}
631	case SEV_CMD_SEND_START: {
632	struct sev_data_send_start *data = cmd_buf;
633
634	desc_list[0].paddr_ptr = &data->session_address;
635	desc_list[0].len = data->session_len;
636	break;
637	}
638	case SEV_CMD_SEND_UPDATE_DATA: {
639	struct sev_data_send_update_data *data = cmd_buf;
640
641	desc_list[0].paddr_ptr = &data->hdr_address;
642	desc_list[0].len = data->hdr_len;
643	desc_list[1].paddr_ptr = &data->trans_address;
644	desc_list[1].len = data->trans_len;
645	break;
646	}
647	case SEV_CMD_SEND_UPDATE_VMSA: {
648	struct sev_data_send_update_vmsa *data = cmd_buf;
649
650	desc_list[0].paddr_ptr = &data->hdr_address;
651	desc_list[0].len = data->hdr_len;
652	desc_list[1].paddr_ptr = &data->trans_address;
653	desc_list[1].len = data->trans_len;
654	break;
655	}
656	case SEV_CMD_RECEIVE_UPDATE_DATA: {
657	struct sev_data_receive_update_data *data = cmd_buf;
658
659	desc_list[0].paddr_ptr = &data->guest_address;
660	desc_list[0].len = data->guest_len;
661	desc_list[0].guest_owned = true;
662	break;
663	}
664	case SEV_CMD_RECEIVE_UPDATE_VMSA: {
665	struct sev_data_receive_update_vmsa *data = cmd_buf;
666
667	desc_list[0].paddr_ptr = &data->guest_address;
668	desc_list[0].len = data->guest_len;
669	desc_list[0].guest_owned = true;
670	break;
671	}
672	default:
673	break;
674	}
675	}
676
677	static int snp_map_cmd_buf_desc(struct cmd_buf_desc *desc)
678	{
679	unsigned int npages;
680
681	if (!desc->len)
682	return 0;
683
684	/* Allocate a bounce buffer if this isn't a guest owned page. */
685	if (!desc->guest_owned) {
686	struct page *page;
687
688	page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(desc->len));
689	if (!page) {
690	pr_warn("Failed to allocate bounce buffer for SEV legacy command.\n");
691	return -ENOMEM;
692	}
693
694	desc->paddr_orig = *desc->paddr_ptr;
695	*desc->paddr_ptr = __psp_pa(page_to_virt(page));
696	}
697
698	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
699
700	/* Transition the buffer to firmware-owned. */
701	if (rmp_mark_pages_firmware(*desc->paddr_ptr, npages, true)) {
702	pr_warn("Error moving pages to firmware-owned state for SEV legacy command.\n");
703	return -EFAULT;
704	}
705
706	return 0;
707	}
708
709	static int snp_unmap_cmd_buf_desc(struct cmd_buf_desc *desc)
710	{
711	unsigned int npages;
712
713	if (!desc->len)
714	return 0;
715
716	npages = PAGE_ALIGN(desc->len) >> PAGE_SHIFT;
717
718	/* Transition the buffers back to hypervisor-owned. */
719	if (snp_reclaim_pages(*desc->paddr_ptr, npages, true)) {
720	pr_warn("Failed to reclaim firmware-owned pages while issuing SEV legacy command.\n");
721	return -EFAULT;
722	}
723
724	/* Copy data from bounce buffer and then free it. */
725	if (!desc->guest_owned) {
726	void bounce_buf = __va(__sme_clr(desc->paddr_ptr));
727	void *dst_buf = __va(__sme_clr(desc->paddr_orig));
728
729	memcpy(dst_buf, bounce_buf, desc->len);
730	__free_pages(virt_to_page(bounce_buf), get_order(desc->len));
731
732	/* Restore the original address in the command buffer. */
733	*desc->paddr_ptr = desc->paddr_orig;
734	}
735
736	return 0;
737	}
738
739	static int snp_map_cmd_buf_desc_list(int cmd, void cmd_buf, struct cmd_buf_desc desc_list)
740	{
741	int i;
742
743	snp_populate_cmd_buf_desc_list(cmd, cmd_buf, desc_list);
744
745	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
746	struct cmd_buf_desc *desc = &desc_list[i];
747
748	if (!desc->paddr_ptr)
749	break;
750
751	if (snp_map_cmd_buf_desc(desc))
752	goto err_unmap;
753	}
754
755	return 0;
756
757	err_unmap:
758	for (i--; i >= 0; i--)
759	snp_unmap_cmd_buf_desc(&desc_list[i]);
760
761	return -EFAULT;
762	}
763
764	static int snp_unmap_cmd_buf_desc_list(struct cmd_buf_desc *desc_list)
765	{
766	int i, ret = 0;
767
768	for (i = 0; i < CMD_BUF_DESC_MAX; i++) {
769	struct cmd_buf_desc *desc = &desc_list[i];
770
771	if (!desc->paddr_ptr)
772	break;
773
774	if (snp_unmap_cmd_buf_desc(&desc_list[i]))
775	ret = -EFAULT;
776	}
777
778	return ret;
779	}
780
781	static bool sev_cmd_buf_writable(int cmd)
782	{
783	switch (cmd) {
784	case SEV_CMD_PLATFORM_STATUS:
785	case SEV_CMD_GUEST_STATUS:
786	case SEV_CMD_LAUNCH_START:
787	case SEV_CMD_RECEIVE_START:
788	case SEV_CMD_LAUNCH_MEASURE:
789	case SEV_CMD_SEND_START:
790	case SEV_CMD_SEND_UPDATE_DATA:
791	case SEV_CMD_SEND_UPDATE_VMSA:
792	case SEV_CMD_PEK_CSR:
793	case SEV_CMD_PDH_CERT_EXPORT:
794	case SEV_CMD_GET_ID:
795	case SEV_CMD_ATTESTATION_REPORT:
796	return true;
797	default:
798	return false;
799	}
800	}
801
802	/* After SNP is INIT'ed, the behavior of legacy SEV commands is changed. */
803	static bool snp_legacy_handling_needed(int cmd)
804	{
805	struct sev_device *sev = psp_master->sev_data;
806
807	return cmd < SEV_CMD_SNP_INIT && sev->snp_initialized;
808	}
809
810	static int snp_prep_cmd_buf(int cmd, void cmd_buf, struct cmd_buf_desc desc_list)
811	{
812	if (!snp_legacy_handling_needed(cmd))
813	return 0;
814
815	if (snp_map_cmd_buf_desc_list(cmd, cmd_buf, desc_list))
816	return -EFAULT;
817
818	/*
819	* Before command execution, the command buffer needs to be put into
820	* the firmware-owned state.
821	*/
822	if (sev_cmd_buf_writable(cmd)) {
823	if (rmp_mark_pages_firmware(__pa(cmd_buf), 1, true))
824	return -EFAULT;
825	}
826
827	return 0;
828	}
829
830	static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf)
831	{
832	if (!snp_legacy_handling_needed(cmd))
833	return 0;
834
835	/*
836	* After command completion, the command buffer needs to be put back
837	* into the hypervisor-owned state.
838	*/
839	if (sev_cmd_buf_writable(cmd))
840	if (snp_reclaim_pages(__pa(cmd_buf), 1, true))
841	return -EFAULT;
842
843	return 0;
844	}
845
846	int __sev_do_cmd_locked(int cmd, void data, int psp_ret)
847	{
848	struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0};
849	struct psp_device *psp = psp_master;
850	struct sev_device *sev;
851	unsigned int cmdbuff_hi, cmdbuff_lo;
852	unsigned int phys_lsb, phys_msb;
853	unsigned int reg;
854	void *cmd_buf;
855	int buf_len;
856	int ret = 0;
857
858	if (!psp \|\| !psp->sev_data)
859	return -ENODEV;
860
861	if (psp_dead)
862	return -EBUSY;
863
864	sev = psp->sev_data;
865
866	buf_len = sev_cmd_buffer_len(cmd);
867	if (WARN_ON_ONCE(!data != !buf_len))
868	return -EINVAL;
869
870	/*
871	* Copy the incoming data to driver's scratch buffer as __pa() will not
872	* work for some memory, e.g. vmalloc'd addresses, and @data may not be
873	* physically contiguous.
874	*/
875	if (data) {
876	/*
877	* Commands are generally issued one at a time and require the
878	* sev_cmd_mutex, but there could be recursive firmware requests
879	* due to SEV_CMD_SNP_PAGE_RECLAIM needing to be issued while
880	* preparing buffers for another command. This is the only known
881	* case of nesting in the current code, so exactly one
882	* additional command buffer is available for that purpose.
883	*/
884	if (!sev->cmd_buf_active) {
885	cmd_buf = sev->cmd_buf;
886	sev->cmd_buf_active = true;
887	} else if (!sev->cmd_buf_backup_active) {
888	cmd_buf = sev->cmd_buf_backup;
889	sev->cmd_buf_backup_active = true;
890	} else {
891	dev_err(sev->dev,
892	"SEV: too many firmware commands in progress, no command buffers available.\n");
893	return -EBUSY;
894	}
895
896	memcpy(cmd_buf, data, buf_len);
897
898	/*
899	* The behavior of the SEV-legacy commands is altered when the
900	* SNP firmware is in the INIT state.
901	*/
902	ret = snp_prep_cmd_buf(cmd, cmd_buf, desc_list);
903	if (ret) {
904	dev_err(sev->dev,
905	"SEV: failed to prepare buffer for legacy command 0x%x. Error: %d\n",
906	cmd, ret);
907	return ret;
908	}
909	} else {
910	cmd_buf = sev->cmd_buf;
911	}
912
913	/* Get the physical address of the command buffer */
914	phys_lsb = data ? lower_32_bits(__psp_pa(cmd_buf)) : 0;
915	phys_msb = data ? upper_32_bits(__psp_pa(cmd_buf)) : 0;
916
917	dev_dbg(sev->dev, "sev command id %#x buffer 0x%08x%08x timeout %us\n",
918	cmd, phys_msb, phys_lsb, psp_timeout);
919
920	print_hex_dump_debug("(in): ", DUMP_PREFIX_OFFSET, 16, 2, data,
921	buf_len, false);
922
923	iowrite32(phys_lsb, sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
924	iowrite32(phys_msb, sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
925
926	sev->int_rcvd = 0;
927
928	reg = FIELD_PREP(SEV_CMDRESP_CMD, cmd);
929
930	/*
931	* If invoked during panic handling, local interrupts are disabled so
932	* the PSP command completion interrupt can't be used.
933	* sev_wait_cmd_ioc() already checks for interrupts disabled and
934	* polls for PSP command completion. Ensure we do not request an
935	* interrupt from the PSP if irqs disabled.
936	*/
937	if (!irqs_disabled())
938	reg \|= SEV_CMDRESP_IOC;
939
940	iowrite32(reg, sev->io_regs + sev->vdata->cmdresp_reg);
941
942	/* wait for command completion */
943	ret = sev_wait_cmd_ioc(sev, &reg, psp_timeout);
944	if (ret) {
945	if (psp_ret)
946	*psp_ret = 0;
947
948	dev_err(sev->dev, "sev command %#x timed out, disabling PSP\n", cmd);
949	psp_dead = true;
950
951	return ret;
952	}
953
954	psp_timeout = psp_cmd_timeout;
955
956	if (psp_ret)
957	*psp_ret = FIELD_GET(PSP_CMDRESP_STS, reg);
958
959	if (FIELD_GET(PSP_CMDRESP_STS, reg)) {
960	dev_dbg(sev->dev, "sev command %#x failed (%#010lx)\n",
961	cmd, FIELD_GET(PSP_CMDRESP_STS, reg));
962
963	/*
964	* PSP firmware may report additional error information in the
965	* command buffer registers on error. Print contents of command
966	* buffer registers if they changed.
967	*/
968	cmdbuff_hi = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_hi_reg);
969	cmdbuff_lo = ioread32(sev->io_regs + sev->vdata->cmdbuff_addr_lo_reg);
970	if (cmdbuff_hi != phys_msb \|\| cmdbuff_lo != phys_lsb) {
971	dev_dbg(sev->dev, "Additional error information reported in cmdbuff:");
972	dev_dbg(sev->dev, " cmdbuff hi: %#010x\n", cmdbuff_hi);
973	dev_dbg(sev->dev, " cmdbuff lo: %#010x\n", cmdbuff_lo);
974	}
975	ret = -EIO;
976	} else {
977	ret = sev_write_init_ex_file_if_required(cmd);
978	}
979
980	/*
981	* Copy potential output from the PSP back to data. Do this even on
982	* failure in case the caller wants to glean something from the error.
983	*/
984	if (data) {
985	int ret_reclaim;
986	/*
987	* Restore the page state after the command completes.
988	*/
989	ret_reclaim = snp_reclaim_cmd_buf(cmd, cmd_buf);
990	if (ret_reclaim) {
991	dev_err(sev->dev,
992	"SEV: failed to reclaim buffer for legacy command %#x. Error: %d\n",
993	cmd, ret_reclaim);
994	return ret_reclaim;
995	}
996
997	memcpy(data, cmd_buf, buf_len);
998
999	if (sev->cmd_buf_backup_active)
1000	sev->cmd_buf_backup_active = false;
1001	else
1002	sev->cmd_buf_active = false;
1003
1004	if (snp_unmap_cmd_buf_desc_list(desc_list))
1005	return -EFAULT;
1006	}
1007
1008	print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
1009	buf_len, false);
1010
1011	return ret;
1012	}
1013
1014	int sev_do_cmd(int cmd, void data, int psp_ret)
1015	{
1016	int rc;
1017
1018	mutex_lock(&sev_cmd_mutex);
1019	rc = __sev_do_cmd_locked(cmd, data, psp_ret);
1020	mutex_unlock(&sev_cmd_mutex);
1021
1022	return rc;
1023	}
1024	EXPORT_SYMBOL_GPL(sev_do_cmd);
1025
1026	static int __sev_init_locked(int *error)
1027	{
1028	struct sev_data_init data;
1029
1030	memset(&data, 0, sizeof(data));
1031	if (sev_es_tmr) {
1032	/*
1033	* Do not include the encryption mask on the physical
1034	* address of the TMR (firmware should clear it anyway).
1035	*/
1036	data.tmr_address = __pa(sev_es_tmr);
1037
1038	data.flags \|= SEV_INIT_FLAGS_SEV_ES;
1039	data.tmr_len = sev_es_tmr_size;
1040	}
1041
1042	return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
1043	}
1044
1045	static int __sev_init_ex_locked(int *error)
1046	{
1047	struct sev_data_init_ex data;
1048
1049	memset(&data, 0, sizeof(data));
1050	data.length = sizeof(data);
1051	data.nv_address = __psp_pa(sev_init_ex_buffer);
1052	data.nv_len = NV_LENGTH;
1053
1054	if (sev_es_tmr) {
1055	/*
1056	* Do not include the encryption mask on the physical
1057	* address of the TMR (firmware should clear it anyway).
1058	*/
1059	data.tmr_address = __pa(sev_es_tmr);
1060
1061	data.flags \|= SEV_INIT_FLAGS_SEV_ES;
1062	data.tmr_len = sev_es_tmr_size;
1063	}
1064
1065	return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
1066	}
1067
1068	static inline int __sev_do_init_locked(int *psp_ret)
1069	{
1070	if (sev_init_ex_buffer)
1071	return __sev_init_ex_locked(psp_ret);
1072	else
1073	return __sev_init_locked(psp_ret);
1074	}
1075
1076	/* Hypervisor Fixed pages API interface */
1077	static void snp_hv_fixed_pages_state_update(struct sev_device *sev,
1078	enum snp_hv_fixed_pages_state page_state)
1079	{
1080	struct snp_hv_fixed_pages_entry *entry;
1081
1082	/* List is protected by sev_cmd_mutex */
1083	lockdep_assert_held(&sev_cmd_mutex);
1084
1085	if (list_empty(&snp_hv_fixed_pages))
1086	return;
1087
1088	list_for_each_entry(entry, &snp_hv_fixed_pages, list)
1089	entry->page_state = page_state;
1090	}
1091
1092	/*
1093	* Allocate HV_FIXED pages in 2MB aligned sizes to ensure the whole
1094	* 2MB pages are marked as HV_FIXED.
1095	*/
1096	struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages)
1097	{
1098	struct psp_device *psp_master = psp_get_master_device();
1099	struct snp_hv_fixed_pages_entry *entry;
1100	unsigned int order;
1101	struct page *page;
1102
1103	if (!psp_master)
1104	return NULL;
1105
1106	order = get_order(PMD_SIZE * num_2mb_pages);
1107
1108	/*
1109	* SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1110	* also needs to be protected using the same mutex.
1111	*/
1112	guard(mutex)(&sev_cmd_mutex);
1113
1114	/*
1115	* This API uses SNP_INIT_EX to transition allocated pages to HV_Fixed
1116	* page state, fail if SNP is already initialized.
1117	*/
1118	if (psp_master->sev_data &&
1119	((struct sev_device *)psp_master->sev_data)->snp_initialized)
1120	return NULL;
1121
1122	/* Re-use freed pages that match the request */
1123	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1124	/* Hypervisor fixed page allocator implements exact fit policy */
1125	if (entry->order == order && entry->free) {
1126	entry->free = false;
1127	memset(page_address(entry->page), 0,
1128	(1 << entry->order) * PAGE_SIZE);
1129	return entry->page;
1130	}
1131	}
1132
1133	page = alloc_pages(GFP_KERNEL \| __GFP_ZERO, order);
1134	if (!page)
1135	return NULL;
1136
1137	entry = kzalloc_obj(*entry);
1138	if (!entry) {
1139	__free_pages(page, order);
1140	return NULL;
1141	}
1142
1143	entry->page = page;
1144	entry->order = order;
1145	list_add_tail(&entry->list, &snp_hv_fixed_pages);
1146
1147	return page;
1148	}
1149
1150	void snp_free_hv_fixed_pages(struct page *page)
1151	{
1152	struct psp_device *psp_master = psp_get_master_device();
1153	struct snp_hv_fixed_pages_entry entry, nentry;
1154
1155	if (!psp_master)
1156	return;
1157
1158	/*
1159	* SNP_INIT_EX is protected by sev_cmd_mutex, therefore this list
1160	* also needs to be protected using the same mutex.
1161	*/
1162	guard(mutex)(&sev_cmd_mutex);
1163
1164	list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1165	if (entry->page != page)
1166	continue;
1167
1168	/*
1169	* HV_FIXED page state cannot be changed until reboot
1170	* and they cannot be used by an SNP guest, so they cannot
1171	* be returned back to the page allocator.
1172	* Mark the pages as free internally to allow possible re-use.
1173	*/
1174	if (entry->page_state == HV_FIXED) {
1175	entry->free = true;
1176	} else {
1177	__free_pages(page, entry->order);
1178	list_del(&entry->list);
1179	kfree(entry);
1180	}
1181	return;
1182	}
1183	}
1184
1185	static void snp_add_hv_fixed_pages(struct sev_device sev, struct sev_data_range_list range_list)
1186	{
1187	struct snp_hv_fixed_pages_entry *entry;
1188	struct sev_data_range *range;
1189	int num_elements;
1190
1191	lockdep_assert_held(&sev_cmd_mutex);
1192
1193	if (list_empty(&snp_hv_fixed_pages))
1194	return;
1195
1196	num_elements = list_count_nodes(&snp_hv_fixed_pages) +
1197	range_list->num_elements;
1198
1199	/*
1200	* Ensure the list of HV_FIXED pages that will be passed to firmware
1201	* do not exceed the page-sized argument buffer.
1202	*/
1203	if (num_elements * sizeof(range) + sizeof(range_list) > PAGE_SIZE) {
1204	dev_warn(sev->dev, "Additional HV_Fixed pages cannot be accommodated, omitting\n");
1205	return;
1206	}
1207
1208	range = &range_list->ranges[range_list->num_elements];
1209	list_for_each_entry(entry, &snp_hv_fixed_pages, list) {
1210	range->base = page_to_pfn(entry->page) << PAGE_SHIFT;
1211	range->page_count = 1 << entry->order;
1212	range++;
1213	}
1214	range_list->num_elements = num_elements;
1215	}
1216
1217	static void snp_leak_hv_fixed_pages(void)
1218	{
1219	struct snp_hv_fixed_pages_entry entry, nentry;
1220
1221	/* List is protected by sev_cmd_mutex */
1222	lockdep_assert_held(&sev_cmd_mutex);
1223
1224	if (list_empty(&snp_hv_fixed_pages))
1225	return;
1226
1227	list_for_each_entry_safe(entry, nentry, &snp_hv_fixed_pages, list) {
1228	if (entry->free && entry->page_state != HV_FIXED)
1229	__free_pages(entry->page, entry->order);
1230	else
1231	__snp_leak_pages(page_to_pfn(entry->page),
1232	1 << entry->order, false);
1233
1234	list_del(&entry->list);
1235	kfree(entry);
1236	}
1237	}
1238
1239	bool sev_is_snp_ciphertext_hiding_supported(void)
1240	{
1241	struct psp_device *psp = psp_master;
1242	struct sev_device *sev;
1243
1244	if (!psp \|\| !psp->sev_data)
1245	return false;
1246
1247	sev = psp->sev_data;
1248
1249	/*
1250	* Feature information indicates if CipherTextHiding feature is
1251	* supported by the SEV firmware and additionally platform status
1252	* indicates if CipherTextHiding feature is enabled in the
1253	* Platform BIOS.
1254	*/
1255	return ((sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED) &&
1256	sev->snp_plat_status.ciphertext_hiding_cap);
1257	}
1258	EXPORT_SYMBOL_GPL(sev_is_snp_ciphertext_hiding_supported);
1259
1260	static int snp_get_platform_data(struct sev_device sev, int error)
1261	{
1262	struct sev_data_snp_feature_info snp_feat_info;
1263	struct snp_feature_info *feat_info;
1264	struct sev_data_snp_addr buf;
1265	struct page *page;
1266	int rc;
1267
1268	/*
1269	* This function is expected to be called before SNP is
1270	* initialized.
1271	*/
1272	if (sev->snp_initialized)
1273	return -EINVAL;
1274
1275	buf.address = __psp_pa(&sev->snp_plat_status);
1276	rc = sev_do_cmd(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
1277	if (rc) {
1278	dev_err(sev->dev, "SNP PLATFORM_STATUS command failed, ret = %d, error = %#x\n",
1279	rc, *error);
1280	return rc;
1281	}
1282
1283	sev->api_major = sev->snp_plat_status.api_major;
1284	sev->api_minor = sev->snp_plat_status.api_minor;
1285	sev->build = sev->snp_plat_status.build_id;
1286
1287	/*
1288	* Do feature discovery of the currently loaded firmware,
1289	* and cache feature information from CPUID 0x8000_0024,
1290	* sub-function 0.
1291	*/
1292	if (!sev->snp_plat_status.feature_info)
1293	return 0;
1294
1295	/*
1296	* Use dynamically allocated structure for the SNP_FEATURE_INFO
1297	* command to ensure structure is 8-byte aligned, and does not
1298	* cross a page boundary.
1299	*/
1300	page = alloc_page(GFP_KERNEL);
1301	if (!page)
1302	return -ENOMEM;
1303
1304	feat_info = page_address(page);
1305	snp_feat_info.length = sizeof(snp_feat_info);
1306	snp_feat_info.ecx_in = 0;
1307	snp_feat_info.feature_info_paddr = __psp_pa(feat_info);
1308
1309	rc = sev_do_cmd(SEV_CMD_SNP_FEATURE_INFO, &snp_feat_info, error);
1310	if (!rc)
1311	sev->snp_feat_info_0 = *feat_info;
1312	else
1313	dev_err(sev->dev, "SNP FEATURE_INFO command failed, ret = %d, error = %#x\n",
1314	rc, *error);
1315
1316	__free_page(page);
1317
1318	return rc;
1319	}
1320
1321	static int snp_filter_reserved_mem_regions(struct resource rs, void arg)
1322	{
1323	struct sev_data_range_list *range_list = arg;
1324	struct sev_data_range *range = &range_list->ranges[range_list->num_elements];
1325	size_t size;
1326
1327	/*
1328	* Ensure the list of HV_FIXED pages passed to the firmware including
1329	* the one about to be written to do not exceed the page-sized argument
1330	* buffer.
1331	*/
1332	if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) +
1333	sizeof(struct sev_data_range_list)) > PAGE_SIZE)
1334	return -E2BIG;
1335
1336	switch (rs->desc) {
1337	case E820_TYPE_RESERVED:
1338	case E820_TYPE_PMEM:
1339	case E820_TYPE_ACPI:
1340	range->base = rs->start & PAGE_MASK;
1341	size = PAGE_ALIGN((rs->end + 1) - rs->start);
1342	range->page_count = size >> PAGE_SHIFT;
1343	range_list->num_elements++;
1344	break;
1345	default:
1346	break;
1347	}
1348
1349	return 0;
1350	}
1351
1352	#ifdef CONFIG_SYSFS
1353	static int snp_verify_mitigation(u16 command, u64 vector,
1354	struct sev_data_snp_verify_mitigation_dst *dst)
1355	{
1356	struct sev_data_snp_verify_mitigation_dst *mit_dst = NULL;
1357	struct sev_data_snp_verify_mitigation data = {0};
1358	struct sev_device *sev = psp_master->sev_data;
1359	int ret, error = 0;
1360
1361	mit_dst = snp_alloc_firmware_page(GFP_KERNEL \| __GFP_ZERO);
1362	if (!mit_dst)
1363	return -ENOMEM;
1364
1365	data.length = sizeof(data);
1366	data.subcommand = command;
1367	data.vector = vector;
1368	data.dst_paddr = __psp_pa(mit_dst);
1369	data.dst_paddr_en = true;
1370
1371	ret = sev_do_cmd(SEV_CMD_SNP_VERIFY_MITIGATION, &data, &error);
1372	if (!ret)
1373	memcpy(dst, mit_dst, sizeof(*mit_dst));
1374	else
1375	dev_err(sev->dev, "SNP_VERIFY_MITIGATION command failed, ret = %d, error = %#x\n",
1376	ret, error);
1377
1378	snp_free_firmware_page(mit_dst);
1379
1380	return ret;
1381	}
1382
1383	static ssize_t supported_mitigations_show(struct kobject *kobj,
1384	struct kobj_attribute attr, char buf)
1385	{
1386	struct sev_data_snp_verify_mitigation_dst dst;
1387	int ret;
1388
1389	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
1390	if (ret)
1391	return ret;
1392
1393	return sysfs_emit(buf, "0x%llx\n", dst.mit_supported_vector);
1394	}
1395
1396	static struct kobj_attribute supported_attr =
1397	__ATTR_RO_MODE(supported_mitigations, 0400);
1398
1399	static ssize_t verified_mitigations_show(struct kobject *kobj,
1400	struct kobj_attribute attr, char buf)
1401	{
1402	struct sev_data_snp_verify_mitigation_dst dst;
1403	int ret;
1404
1405	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_STATUS, 0, &dst);
1406	if (ret)
1407	return ret;
1408
1409	return sysfs_emit(buf, "0x%llx\n", dst.mit_verified_vector);
1410	}
1411
1412	static ssize_t verified_mitigations_store(struct kobject *kobj,
1413	struct kobj_attribute *attr,
1414	const char *buf, size_t count)
1415	{
1416	struct sev_data_snp_verify_mitigation_dst dst;
1417	struct sev_device *sev = psp_master->sev_data;
1418	u64 vector;
1419	int ret;
1420
1421	ret = kstrtoull(buf, 0, &vector);
1422	if (ret)
1423	return ret;
1424
1425	/*
1426	* The firmware verifies a single mitigation per call. Reject vectors
1427	* with more than one bit set early to avoid a guaranteed-to-fail call
1428	*/
1429	if (hweight64(vector) != 1)
1430	return -EINVAL;
1431
1432	ret = snp_verify_mitigation(SNP_MIT_SUBCMD_REQ_VERIFY, vector, &dst);
1433	if (ret)
1434	return ret;
1435
1436	if (dst.mit_failure_status) {
1437	dev_err(sev->dev, "Verify Mitigation - failure status: 0x%x\n",
1438	dst.mit_failure_status);
1439	return -EINVAL;
1440	}
1441
1442	return count;
1443	}
1444
1445	static struct kobj_attribute verified_attr =
1446	__ATTR_RW_MODE(verified_mitigations, 0600);
1447
1448	static struct attribute *mitigation_attrs[] = {
1449	&supported_attr.attr,
1450	&verified_attr.attr,
1451	NULL
1452	};
1453
1454	static const struct attribute_group mit_attr_group = {
1455	.attrs = mitigation_attrs,
1456	};
1457
1458	static void sev_snp_register_verify_mitigation(struct sev_device *sev)
1459	{
1460	int rc;
1461
1462	if (!(sev->snp_feat_info_0.ecx & SNP_VERIFY_MITIGATION_SUPPORTED) \|\|
1463	sev->verify_mit)
1464	return;
1465
1466	if (!sev->sev_kobj) {
1467	sev->sev_kobj = kobject_create_and_add("sev", firmware_kobj);
1468	if (!sev->sev_kobj)
1469	return;
1470	}
1471
1472	sev->verify_mit = kobject_create_and_add("vulnerabilities", sev->sev_kobj);
1473	if (!sev->verify_mit)
1474	goto err_sev_kobj;
1475
1476	rc = sysfs_create_group(sev->verify_mit, &mit_attr_group);
1477	if (rc)
1478	goto err_verify_mit;
1479
1480	return;
1481
1482	err_verify_mit:
1483	kobject_put(sev->verify_mit);
1484	sev->verify_mit = NULL;
1485	err_sev_kobj:
1486	kobject_put(sev->sev_kobj);
1487	sev->sev_kobj = NULL;
1488	}
1489
1490	static void sev_snp_unregister_verify_mitigation(struct sev_device *sev)
1491	{
1492	if (sev->verify_mit) {
1493	sysfs_remove_group(sev->verify_mit, &mit_attr_group);
1494	kobject_put(sev->verify_mit);
1495	sev->verify_mit = NULL;
1496	}
1497
1498	if (sev->sev_kobj) {
1499	kobject_put(sev->sev_kobj);
1500	sev->sev_kobj = NULL;
1501	}
1502	}
1503	#else // CONFIG_SYSFS
1504	static void sev_snp_register_verify_mitigation(struct sev_device *sev) { }
1505	static void sev_snp_unregister_verify_mitigation(struct sev_device *sev) { }
1506	#endif // CONFIG_SYSFS
1507
1508	static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
1509	{
1510	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
1511	struct psp_device *psp = psp_master;
1512	struct sev_data_snp_init_ex data = {};
1513	struct sev_device *sev;
1514	void *arg = &data;
1515	int cmd, rc = 0;
1516
1517	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
1518	return -ENODEV;
1519
1520	sev = psp->sev_data;
1521
1522	if (sev->snp_initialized)
1523	return 0;
1524
1525	if (!sev_version_greater_or_equal(SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR)) {
1526	dev_dbg(sev->dev, "SEV-SNP support requires firmware version >= %d:%d\n",
1527	SNP_MIN_API_MAJOR, SNP_MIN_API_MINOR);
1528	return -EOPNOTSUPP;
1529	}
1530
1531	rc = snp_prepare();
1532	if (rc)
1533	return rc;
1534
1535	/*
1536	* Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
1537	* of system physical address ranges to convert into HV-fixed page
1538	* states during the RMP initialization. For instance, the memory that
1539	* UEFI reserves should be included in the that list. This allows system
1540	* components that occasionally write to memory (e.g. logging to UEFI
1541	* reserved regions) to not fail due to RMP initialization and SNP
1542	* enablement.
1543	*
1544	*/
1545	if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) {
1546	bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED);
1547
1548	/*
1549	* Firmware checks that the pages containing the ranges enumerated
1550	* in the RANGES structure are either in the default page state or in the
1551	* firmware page state.
1552	*/
1553	snp_range_list = kzalloc(PAGE_SIZE, GFP_KERNEL);
1554	if (!snp_range_list) {
1555	dev_err(sev->dev,
1556	"SEV: SNP_INIT_EX range list memory allocation failed\n");
1557	return -ENOMEM;
1558	}
1559
1560	/*
1561	* Retrieve all reserved memory regions from the e820 memory map
1562	* to be setup as HV-fixed pages.
1563	*/
1564	rc = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM, 0, ~0,
1565	snp_range_list, snp_filter_reserved_mem_regions);
1566	if (rc) {
1567	dev_err(sev->dev,
1568	"SEV: SNP_INIT_EX walk_iomem_res_desc failed rc = %d\n", rc);
1569	return rc;
1570	}
1571
1572	/*
1573	* Add HV_Fixed pages from other PSP sub-devices, such as SFS to the
1574	* HV_Fixed page list.
1575	*/
1576	snp_add_hv_fixed_pages(sev, snp_range_list);
1577
1578	if (max_snp_asid) {
1579	data.ciphertext_hiding_en = 1;
1580	data.max_snp_asid = max_snp_asid;
1581	}
1582
1583	data.init_rmp = 1;
1584	data.list_paddr_en = 1;
1585	data.list_paddr = __psp_pa(snp_range_list);
1586
1587	data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported();
1588
1589	/*
1590	* When psp_init_on_probe is disabled, the userspace calling
1591	* SEV ioctl can inadvertently shut down SNP and SEV-TIO causing
1592	* unexpected state loss.
1593	*/
1594	if (data.tio_en && !psp_init_on_probe)
1595	dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n");
1596
1597	cmd = SEV_CMD_SNP_INIT_EX;
1598	} else {
1599	cmd = SEV_CMD_SNP_INIT;
1600	arg = NULL;
1601	}
1602
1603	/*
1604	* The following sequence must be issued before launching the first SNP
1605	* guest to ensure all dirty cache lines are flushed, including from
1606	* updates to the RMP table itself via the RMPUPDATE instruction:
1607	*
1608	* - WBINVD on all running CPUs
1609	* - SEV_CMD_SNP_INIT[_EX] firmware command
1610	* - WBINVD on all running CPUs
1611	* - SEV_CMD_SNP_DF_FLUSH firmware command
1612	*/
1613	wbinvd_on_all_cpus();
1614
1615	rc = __sev_do_cmd_locked(cmd, arg, error);
1616	if (rc) {
1617	dev_err(sev->dev, "SEV-SNP: %s failed rc %d, error %#x\n",
1618	cmd == SEV_CMD_SNP_INIT_EX ? "SNP_INIT_EX" : "SNP_INIT",
1619	rc, *error);
1620	return rc;
1621	}
1622
1623	/* Prepare for first SNP guest launch after INIT. */
1624	wbinvd_on_all_cpus();
1625	rc = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, error);
1626	if (rc) {
1627	dev_err(sev->dev, "SEV-SNP: SNP_DF_FLUSH failed rc %d, error %#x\n",
1628	rc, *error);
1629	return rc;
1630	}
1631
1632	snp_hv_fixed_pages_state_update(sev, HV_FIXED);
1633	sev->snp_initialized = true;
1634	dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n",
1635	data.tio_en ? "enabled" : "disabled");
1636
1637	dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major,
1638	sev->api_minor, sev->build);
1639
1640	atomic_notifier_chain_register(&panic_notifier_list,
1641	&snp_panic_notifier);
1642
1643	if (data.tio_en) {
1644	struct page *page;
1645
1646	/*
1647	* This executes with the sev_cmd_mutex held so down the stack
1648	* snp_reclaim_pages(locked=false) might be needed (which is extremely
1649	* unlikely) but will cause a deadlock.
1650	* Instead of exporting __snp_alloc_firmware_pages(), allocate a page
1651	* for this one call here.
1652	*/
1653	page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT \| __GFP_ZERO,
1654	0, true);
1655	if (page) {
1656	void *tio_status = page_address(page);
1657
1658	sev_tsm_init_locked(sev, tio_status);
1659
1660	__snp_free_firmware_pages(page, 0, true);
1661	}
1662	}
1663
1664	sev_es_tmr_size = SNP_TMR_SIZE;
1665
1666	snp_enable_rmpopt();
1667
1668	return 0;
1669	}
1670
1671	static void __sev_platform_init_handle_tmr(struct sev_device *sev)
1672	{
1673	if (sev_es_tmr)
1674	return;
1675
1676	/* Obtain the TMR memory area for SEV-ES use */
1677	sev_es_tmr = sev_fw_alloc(sev_es_tmr_size);
1678	if (sev_es_tmr) {
1679	/* Must flush the cache before giving it to the firmware */
1680	if (!sev->snp_initialized)
1681	clflush_cache_range(sev_es_tmr, sev_es_tmr_size);
1682	} else {
1683	dev_warn(sev->dev, "SEV: TMR allocation failed, SEV-ES support unavailable\n");
1684	}
1685	}
1686
1687	/*
1688	* If an init_ex_path is provided allocate a buffer for the file and
1689	* read in the contents. Additionally, if SNP is initialized, convert
1690	* the buffer pages to firmware pages.
1691	*/
1692	static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev)
1693	{
1694	struct page *page;
1695	int rc;
1696
1697	if (!init_ex_path)
1698	return 0;
1699
1700	if (sev_init_ex_buffer)
1701	return 0;
1702
1703	page = alloc_pages(GFP_KERNEL \| __GFP_ZERO, get_order(NV_LENGTH));
1704	if (!page) {
1705	dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n");
1706	return -ENOMEM;
1707	}
1708
1709	sev_init_ex_buffer = page_address(page);
1710
1711	rc = sev_read_init_ex_file();
1712	if (rc)
1713	goto err_free;
1714
1715	/* If SEV-SNP is initialized, transition to firmware page. */
1716	if (sev->snp_initialized) {
1717	unsigned long npages;
1718
1719	npages = 1UL << get_order(NV_LENGTH);
1720	if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) {
1721	dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n");
1722	rc = -ENOMEM;
1723	/*
1724	* Pages can be in an inconsistent state, don't release them back to the
1725	* system.
1726	*/
1727	goto err_reset;
1728	}
1729	}
1730
1731	return 0;
1732
1733	err_free:
1734	__free_pages(page, get_order(NV_LENGTH));
1735	err_reset:
1736	sev_init_ex_buffer = NULL;
1737	return rc;
1738	}
1739
1740	static int __sev_platform_init_locked(int *error)
1741	{
1742	int rc, psp_ret, dfflush_error;
1743	struct sev_device *sev;
1744
1745	psp_ret = dfflush_error = SEV_RET_NO_FW_CALL;
1746
1747	if (!psp_master \|\| !psp_master->sev_data)
1748	return -ENODEV;
1749
1750	sev = psp_master->sev_data;
1751
1752	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1753	return 0;
1754
1755	__sev_platform_init_handle_tmr(sev);
1756
1757	rc = __sev_platform_init_handle_init_ex_path(sev);
1758	if (rc)
1759	return rc;
1760
1761	rc = __sev_do_init_locked(&psp_ret);
1762	if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
1763	/*
1764	* Initialization command returned an integrity check failure
1765	* status code, meaning that firmware load and validation of SEV
1766	* related persistent data has failed. Retrying the
1767	* initialization function should succeed by replacing the state
1768	* with a reset state.
1769	*/
1770	dev_err(sev->dev,
1771	"SEV: retrying INIT command because of SECURE_DATA_INVALID error. Retrying once to reset PSP SEV state.");
1772	rc = __sev_do_init_locked(&psp_ret);
1773	}
1774
1775	if (error)
1776	*error = psp_ret;
1777
1778	if (rc) {
1779	dev_err(sev->dev, "SEV: %s failed %#x, rc %d\n",
1780	sev_init_ex_buffer ? "INIT_EX" : "INIT", psp_ret, rc);
1781	return rc;
1782	}
1783
1784	sev->sev_plat_status.state = SEV_STATE_INIT;
1785
1786	/* Prepare for first SEV guest launch after INIT */
1787	wbinvd_on_all_cpus();
1788	rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error);
1789	if (rc) {
1790	dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n",
1791	dfflush_error, rc);
1792	return rc;
1793	}
1794
1795	dev_dbg(sev->dev, "SEV firmware initialized\n");
1796
1797	dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
1798	sev->api_minor, sev->build);
1799
1800	return 0;
1801	}
1802
1803	static int _sev_platform_init_locked(struct sev_platform_init_args *args)
1804	{
1805	struct sev_device *sev;
1806	int rc;
1807
1808	if (!psp_master \|\| !psp_master->sev_data)
1809	return -ENODEV;
1810
1811	/*
1812	* Skip SNP/SEV initialization under a kdump kernel as SEV/SNP
1813	* may already be initialized in the previous kernel. Since no
1814	* SNP/SEV guests are run under a kdump kernel, there is no
1815	* need to initialize SNP or SEV during kdump boot.
1816	*/
1817	if (is_kdump_kernel())
1818	return 0;
1819
1820	sev = psp_master->sev_data;
1821
1822	if (sev->sev_plat_status.state == SEV_STATE_INIT)
1823	return 0;
1824
1825	rc = __sev_snp_init_locked(&args->error, args->max_snp_asid);
1826	if (rc && rc != -ENODEV)
1827	return rc;
1828
1829	/* Defer legacy SEV/SEV-ES support if allowed by caller/module. */
1830	if (args->probe && !psp_init_on_probe)
1831	return 0;
1832
1833	return __sev_platform_init_locked(&args->error);
1834	}
1835
1836	int sev_platform_init(struct sev_platform_init_args *args)
1837	{
1838	int rc;
1839
1840	mutex_lock(&sev_cmd_mutex);
1841	rc = _sev_platform_init_locked(args);
1842	mutex_unlock(&sev_cmd_mutex);
1843
1844	/*
1845	* Register the sysfs interface outside the sev_cmd_mutex. The
1846	* _show()/_store() handlers issue SEV commands that acquire the
1847	* sev_cmd_mutex, so creating (and on the shutdown path, removing) the
1848	* sysfs group must stay outside that lock. sysfs provides its own
1849	* synchronization between group creation/removal and concurrent
1850	* attribute access.
1851	*/
1852	if (!rc)
1853	sev_snp_register_verify_mitigation(psp_master->sev_data);
1854
1855	return rc;
1856	}
1857	EXPORT_SYMBOL_GPL(sev_platform_init);
1858
1859	static int __sev_platform_shutdown_locked(int *error)
1860	{
1861	struct psp_device *psp = psp_master;
1862	struct sev_device *sev;
1863	int ret;
1864
1865	if (!psp \|\| !psp->sev_data)
1866	return 0;
1867
1868	sev = psp->sev_data;
1869
1870	if (sev->sev_plat_status.state == SEV_STATE_UNINIT)
1871	return 0;
1872
1873	ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
1874	if (ret) {
1875	dev_err(sev->dev, "SEV: failed to SHUTDOWN error %#x, rc %d\n",
1876	*error, ret);
1877	return ret;
1878	}
1879
1880	sev->sev_plat_status.state = SEV_STATE_UNINIT;
1881	dev_dbg(sev->dev, "SEV firmware shutdown\n");
1882
1883	return ret;
1884	}
1885
1886	static int sev_get_platform_state(int state, int error)
1887	{
1888	struct sev_user_data_status data;
1889	int rc;
1890
1891	rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, error);
1892	if (rc)
1893	return rc;
1894
1895	*state = data.state;
1896	return rc;
1897	}
1898
1899	static int sev_move_to_init_state(struct sev_issue_cmd argp, bool shutdown_required)
1900	{
1901	int rc;
1902
1903	rc = __sev_platform_init_locked(&argp->error);
1904	if (rc)
1905	return rc;
1906
1907	*shutdown_required = true;
1908
1909	return 0;
1910	}
1911
1912	static int sev_ioctl_do_reset(struct sev_issue_cmd *argp, bool writable)
1913	{
1914	int state, rc;
1915
1916	if (!writable)
1917	return -EPERM;
1918
1919	/*
1920	* The SEV spec requires that FACTORY_RESET must be issued in
1921	* UNINIT state. Before we go further lets check if any guest is
1922	* active.
1923	*
1924	* If FW is in WORKING state then deny the request otherwise issue
1925	* SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
1926	*
1927	*/
1928	rc = sev_get_platform_state(&state, &argp->error);
1929	if (rc)
1930	return rc;
1931
1932	if (state == SEV_STATE_WORKING)
1933	return -EBUSY;
1934
1935	if (state == SEV_STATE_INIT) {
1936	rc = __sev_platform_shutdown_locked(&argp->error);
1937	if (rc)
1938	return rc;
1939	}
1940
1941	return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
1942	}
1943
1944	static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
1945	{
1946	struct sev_user_data_status data;
1947	int ret;
1948
1949	memset(&data, 0, sizeof(data));
1950
1951	ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, &data, &argp->error);
1952	if (ret)
1953	return ret;
1954
1955	if (copy_to_user((void __user *)argp->data, &data, sizeof(data)))
1956	ret = -EFAULT;
1957
1958	return ret;
1959	}
1960
1961	static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp, bool writable)
1962	{
1963	struct sev_device *sev = psp_master->sev_data;
1964	bool shutdown_required = false;
1965	int rc;
1966
1967	if (!writable)
1968	return -EPERM;
1969
1970	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
1971	rc = sev_move_to_init_state(argp, &shutdown_required);
1972	if (rc)
1973	return rc;
1974	}
1975
1976	rc = __sev_do_cmd_locked(cmd, NULL, &argp->error);
1977
1978	if (shutdown_required)
1979	__sev_firmware_shutdown(sev, false);
1980
1981	return rc;
1982	}
1983
1984	static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable)
1985	{
1986	struct sev_device *sev = psp_master->sev_data;
1987	struct sev_user_data_pek_csr input;
1988	bool shutdown_required = false;
1989	struct sev_data_pek_csr data;
1990	void __user *input_address;
1991	void *blob = NULL;
1992	int ret;
1993
1994	if (!writable)
1995	return -EPERM;
1996
1997	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
1998	return -EFAULT;
1999
2000	memset(&data, 0, sizeof(data));
2001
2002	/* userspace wants to query CSR length */
2003	if (!input.address \|\| !input.length)
2004	goto cmd;
2005
2006	/* allocate a physically contiguous buffer to store the CSR blob */
2007	input_address = (void __user *)input.address;
2008	if (input.length > SEV_FW_BLOB_MAX_SIZE)
2009	return -EFAULT;
2010
2011	blob = kzalloc(input.length, GFP_KERNEL);
2012	if (!blob)
2013	return -ENOMEM;
2014
2015	data.address = __psp_pa(blob);
2016	data.len = input.length;
2017
2018	cmd:
2019	if (sev->sev_plat_status.state == SEV_STATE_UNINIT) {
2020	ret = sev_move_to_init_state(argp, &shutdown_required);
2021	if (ret)
2022	goto e_free_blob;
2023	}
2024
2025	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error);
2026
2027	/*
2028	* Firmware will returns the length of the CSR blob (either the minimum
2029	* required length or the actual length written), return it to the user.
2030	*/
2031	input.length = data.len;
2032
2033	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2034	ret = -EFAULT;
2035	goto e_free_blob;
2036	}
2037
2038	if (ret \|\| WARN_ON_ONCE(argp->error))
2039	goto e_free_blob;
2040
2041	if (blob) {
2042	if (copy_to_user(input_address, blob, input.length))
2043	ret = -EFAULT;
2044	}
2045
2046	e_free_blob:
2047	if (shutdown_required)
2048	__sev_firmware_shutdown(sev, false);
2049
2050	kfree(blob);
2051	return ret;
2052	}
2053
2054	void *psp_copy_user_blob(u64 uaddr, u32 len)
2055	{
2056	if (!uaddr \|\| !len)
2057	return ERR_PTR(-EINVAL);
2058
2059	/* verify that blob length does not exceed our limit */
2060	if (len > SEV_FW_BLOB_MAX_SIZE)
2061	return ERR_PTR(-EINVAL);
2062
2063	return memdup_user((void __user *)uaddr, len);
2064	}
2065	EXPORT_SYMBOL_GPL(psp_copy_user_blob);
2066
2067	static int sev_get_api_version(void)
2068	{
2069	struct sev_device *sev = psp_master->sev_data;
2070	struct sev_user_data_status status;
2071	int error = 0, ret;
2072
2073	/*
2074	* Cache SNP platform status and SNP feature information
2075	* if SNP is available.
2076	*/
2077	if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) {
2078	ret = snp_get_platform_data(sev, &error);
2079	if (ret)
2080	return 1;
2081	}
2082
2083	ret = sev_platform_status(&status, &error);
2084	if (ret) {
2085	dev_err(sev->dev,
2086	"SEV: failed to get status. Error: %#x\n", error);
2087	return 1;
2088	}
2089
2090	/* Cache SEV platform status */
2091	sev->sev_plat_status = status;
2092
2093	sev->api_major = status.api_major;
2094	sev->api_minor = status.api_minor;
2095	sev->build = status.build;
2096
2097	return 0;
2098	}
2099
2100	static int sev_get_firmware(struct device *dev,
2101	const struct firmware **firmware)
2102	{
2103	char fw_name_specific[SEV_FW_NAME_SIZE];
2104	char fw_name_subset[SEV_FW_NAME_SIZE];
2105
2106	snprintf(fw_name_specific, sizeof(fw_name_specific),
2107	"amd/amd_sev_fam%.2xh_model%.2xh.sbin",
2108	boot_cpu_data.x86, boot_cpu_data.x86_model);
2109
2110	snprintf(fw_name_subset, sizeof(fw_name_subset),
2111	"amd/amd_sev_fam%.2xh_model%.1xxh.sbin",
2112	boot_cpu_data.x86, (boot_cpu_data.x86_model & 0xf0) >> 4);
2113
2114	/* Check for SEV FW for a particular model.
2115	* Ex. amd_sev_fam17h_model00h.sbin for Family 17h Model 00h
2116	*
2117	* or
2118	*
2119	* Check for SEV FW common to a subset of models.
2120	* Ex. amd_sev_fam17h_model0xh.sbin for
2121	* Family 17h Model 00h -- Family 17h Model 0Fh
2122	*
2123	* or
2124	*
2125	* Fall-back to using generic name: sev.fw
2126	*/
2127	if ((firmware_request_nowarn(firmware, fw_name_specific, dev) >= 0) \|\|
2128	(firmware_request_nowarn(firmware, fw_name_subset, dev) >= 0) \|\|
2129	(firmware_request_nowarn(firmware, SEV_FW_FILE, dev) >= 0))
2130	return 0;
2131
2132	return -ENOENT;
2133	}
2134
2135	/* Don't fail if SEV FW couldn't be updated. Continue with existing SEV FW */
2136	static int sev_update_firmware(struct device *dev)
2137	{
2138	struct sev_data_download_firmware data;
2139	const struct firmware *firmware;
2140	int ret, error, order;
2141	struct page *p;
2142	void *fw_blob;
2143
2144	if (!sev_version_greater_or_equal(0, 15)) {
2145	dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n");
2146	return -1;
2147	}
2148
2149	if (sev_get_firmware(dev, &firmware) == -ENOENT) {
2150	dev_dbg(dev, "No SEV firmware file present\n");
2151	return -1;
2152	}
2153
2154	order = get_order(firmware->size);
2155	p = alloc_pages(GFP_KERNEL, order);
2156	if (!p) {
2157	ret = -1;
2158	goto fw_err;
2159	}
2160
2161	/*
2162	* Copy firmware data to a kernel allocated contiguous
2163	* memory region.
2164	*/
2165	fw_blob = page_address(p);
2166	memcpy(fw_blob, firmware->data, firmware->size);
2167
2168	data.address = __psp_pa(fw_blob);
2169	data.len = firmware->size;
2170
2171	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
2172
2173	/*
2174	* A quirk for fixing the committed TCB version, when upgrading from
2175	* earlier firmware version than 1.50.
2176	*/
2177	if (!ret && !sev_version_greater_or_equal(1, 50))
2178	ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, &data, &error);
2179
2180	if (ret)
2181	dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error);
2182
2183	__free_pages(p, order);
2184
2185	fw_err:
2186	release_firmware(firmware);
2187
2188	return ret;
2189	}
2190
2191	static int __sev_snp_shutdown_locked(int *error, bool panic)
2192	{
2193	struct psp_device *psp = psp_master;
2194	struct sev_device *sev;
2195	struct sev_data_snp_shutdown_ex data;
2196	int ret;
2197
2198	if (!psp \|\| !psp->sev_data)
2199	return 0;
2200
2201	sev = psp->sev_data;
2202
2203	if (!sev->snp_initialized)
2204	return 0;
2205
2206	memset(&data, 0, sizeof(data));
2207	data.len = sizeof(data);
2208	data.iommu_snp_shutdown = 1;
2209	if (sev->snp_feat_info_0.ecx & SNP_X86_SHUTDOWN_SUPPORTED)
2210	data.x86_snp_shutdown = 1;
2211
2212	/*
2213	* If invoked during panic handling, local interrupts are disabled
2214	* and all CPUs are stopped, so wbinvd_on_all_cpus() can't be called.
2215	* In that case, a wbinvd() is done on remote CPUs via the NMI
2216	* callback, so only a local wbinvd() is needed here.
2217	*/
2218	if (!panic)
2219	wbinvd_on_all_cpus();
2220	else
2221	wbinvd();
2222
2223	ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data, error);
2224	/* SHUTDOWN may require DF_FLUSH */
2225	if (*error == SEV_RET_DFFLUSH_REQUIRED) {
2226	int dfflush_error = SEV_RET_NO_FW_CALL;
2227
2228	ret = __sev_do_cmd_locked(SEV_CMD_SNP_DF_FLUSH, NULL, &dfflush_error);
2229	if (ret) {
2230	dev_err(sev->dev, "SEV-SNP DF_FLUSH failed, ret = %d, error = %#x\n",
2231	ret, dfflush_error);
2232	return ret;
2233	}
2234	/* reissue the shutdown command */
2235	ret = __sev_do_cmd_locked(SEV_CMD_SNP_SHUTDOWN_EX, &data,
2236	error);
2237	}
2238	if (ret) {
2239	dev_err(sev->dev, "SEV-SNP firmware shutdown failed, rc %d, error %#x\n",
2240	ret, *error);
2241	return ret;
2242	}
2243
2244	if (data.x86_snp_shutdown) {
2245	if (!panic)
2246	snp_shutdown();
2247	snp_hv_fixed_pages_state_update(sev, ALLOCATED);
2248	} else {
2249	/*
2250	* SNP_SHUTDOWN_EX with IOMMU_SNP_SHUTDOWN set to 1 disables SNP
2251	* enforcement by the IOMMU and also transitions all pages
2252	* associated with the IOMMU to the Reclaim state.
2253	* Firmware was transitioning the IOMMU pages to Hypervisor state
2254	* before version 1.53. But, accounting for the number of assigned
2255	* 4kB pages in a 2M page was done incorrectly by not transitioning
2256	* to the Reclaim state. This resulted in RMP #PF when later accessing
2257	* the 2M page containing those pages during kexec boot. Hence, the
2258	* firmware now transitions these pages to Reclaim state and hypervisor
2259	* needs to transition these pages to shared state. SNP Firmware
2260	* version 1.53 and above are needed for kexec boot.
2261	*/
2262	ret = amd_iommu_snp_disable();
2263	if (ret) {
2264	dev_err(sev->dev, "SNP IOMMU shutdown failed\n");
2265	return ret;
2266	}
2267	}
2268
2269	snp_leak_hv_fixed_pages();
2270	sev->snp_initialized = false;
2271	dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
2272
2273	/*
2274	* __sev_snp_shutdown_locked() deadlocks when it tries to unregister
2275	* itself during panic as the panic notifier is called with RCU read
2276	* lock held and notifier unregistration does RCU synchronization.
2277	*/
2278	if (!panic)
2279	atomic_notifier_chain_unregister(&panic_notifier_list,
2280	&snp_panic_notifier);
2281
2282	/* Reset TMR size back to default */
2283	sev_es_tmr_size = SEV_TMR_SIZE;
2284
2285	return ret;
2286	}
2287
2288	static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp, bool writable)
2289	{
2290	struct sev_device *sev = psp_master->sev_data;
2291	struct sev_user_data_pek_cert_import input;
2292	struct sev_data_pek_cert_import data;
2293	bool shutdown_required = false;
2294	void pek_blob, oca_blob;
2295	int ret;
2296
2297	if (!writable)
2298	return -EPERM;
2299
2300	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2301	return -EFAULT;
2302
2303	/* copy PEK certificate blobs from userspace */
2304	pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
2305	if (IS_ERR(pek_blob))
2306	return PTR_ERR(pek_blob);
2307
2308	data.reserved = 0;
2309	data.pek_cert_address = __psp_pa(pek_blob);
2310	data.pek_cert_len = input.pek_cert_len;
2311
2312	/* copy PEK certificate blobs from userspace */
2313	oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
2314	if (IS_ERR(oca_blob)) {
2315	ret = PTR_ERR(oca_blob);
2316	goto e_free_pek;
2317	}
2318
2319	data.oca_cert_address = __psp_pa(oca_blob);
2320	data.oca_cert_len = input.oca_cert_len;
2321
2322	/* If platform is not in INIT state then transition it to INIT */
2323	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2324	ret = sev_move_to_init_state(argp, &shutdown_required);
2325	if (ret)
2326	goto e_free_oca;
2327	}
2328
2329	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, &data, &argp->error);
2330
2331	e_free_oca:
2332	if (shutdown_required)
2333	__sev_firmware_shutdown(sev, false);
2334
2335	kfree(oca_blob);
2336	e_free_pek:
2337	kfree(pek_blob);
2338	return ret;
2339	}
2340
2341	static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp)
2342	{
2343	struct sev_user_data_get_id2 input;
2344	struct sev_data_get_id data;
2345	void __user *input_address;
2346	void *id_blob = NULL;
2347	int ret;
2348
2349	/* SEV GET_ID is available from SEV API v0.16 and up */
2350	if (!sev_version_greater_or_equal(0, 16))
2351	return -ENOTSUPP;
2352
2353	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2354	return -EFAULT;
2355
2356	input_address = (void __user *)input.address;
2357
2358	if (input.address && input.length) {
2359	/*
2360	* The length of the ID shouldn't be assumed by software since
2361	* it may change in the future. The allocation size is limited
2362	* to 1 << (PAGE_SHIFT + MAX_PAGE_ORDER) by the page allocator.
2363	* If the allocation fails, simply return ENOMEM rather than
2364	* warning in the kernel log.
2365	*/
2366	id_blob = kzalloc(input.length, GFP_KERNEL \| __GFP_NOWARN);
2367	if (!id_blob)
2368	return -ENOMEM;
2369
2370	data.address = __psp_pa(id_blob);
2371	data.len = input.length;
2372	} else {
2373	data.address = 0;
2374	data.len = 0;
2375	}
2376
2377	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, &data, &argp->error);
2378
2379	/*
2380	* Firmware will return the length of the ID value (either the minimum
2381	* required length or the actual length written), return it to the user.
2382	*/
2383	input.length = data.len;
2384
2385	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2386	ret = -EFAULT;
2387	goto e_free;
2388	}
2389
2390	if (ret \|\| WARN_ON_ONCE(argp->error))
2391	goto e_free;
2392
2393	if (id_blob) {
2394	if (copy_to_user(input_address, id_blob, data.len)) {
2395	ret = -EFAULT;
2396	goto e_free;
2397	}
2398	}
2399
2400	e_free:
2401	kfree(id_blob);
2402
2403	return ret;
2404	}
2405
2406	static int sev_ioctl_do_get_id(struct sev_issue_cmd *argp)
2407	{
2408	struct sev_data_get_id *data;
2409	u64 data_size, user_size;
2410	void id_blob, mem;
2411	int ret;
2412
2413	/* SEV GET_ID available from SEV API v0.16 and up */
2414	if (!sev_version_greater_or_equal(0, 16))
2415	return -ENOTSUPP;
2416
2417	/* SEV FW expects the buffer it fills with the ID to be
2418	* 8-byte aligned. Memory allocated should be enough to
2419	* hold data structure + alignment padding + memory
2420	* where SEV FW writes the ID.
2421	*/
2422	data_size = ALIGN(sizeof(struct sev_data_get_id), 8);
2423	user_size = sizeof(struct sev_user_data_get_id);
2424
2425	mem = kzalloc(data_size + user_size, GFP_KERNEL);
2426	if (!mem)
2427	return -ENOMEM;
2428
2429	data = mem;
2430	id_blob = mem + data_size;
2431
2432	data->address = __psp_pa(id_blob);
2433	data->len = user_size;
2434
2435	ret = __sev_do_cmd_locked(SEV_CMD_GET_ID, data, &argp->error);
2436	if (!ret) {
2437	if (copy_to_user((void __user *)argp->data, id_blob, data->len))
2438	ret = -EFAULT;
2439	}
2440
2441	kfree(mem);
2442
2443	return ret;
2444	}
2445
2446	static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
2447	{
2448	struct sev_device *sev = psp_master->sev_data;
2449	struct sev_user_data_pdh_cert_export input;
2450	void pdh_blob = NULL, cert_blob = NULL;
2451	struct sev_data_pdh_cert_export data;
2452	void __user *input_cert_chain_address;
2453	void __user *input_pdh_cert_address;
2454	bool shutdown_required = false;
2455	int ret;
2456
2457	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
2458	return -EFAULT;
2459
2460	memset(&data, 0, sizeof(data));
2461
2462	input_pdh_cert_address = (void __user *)input.pdh_cert_address;
2463	input_cert_chain_address = (void __user *)input.cert_chain_address;
2464
2465	/* Userspace wants to query the certificate length. */
2466	if (!input.pdh_cert_address \|\|
2467	!input.pdh_cert_len \|\|
2468	!input.cert_chain_address \|\|
2469	!input.cert_chain_len)
2470	goto cmd;
2471
2472	/* Allocate a physically contiguous buffer to store the PDH blob. */
2473	if (input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE)
2474	return -EFAULT;
2475
2476	/* Allocate a physically contiguous buffer to store the cert chain blob. */
2477	if (input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE)
2478	return -EFAULT;
2479
2480	pdh_blob = kzalloc(input.pdh_cert_len, GFP_KERNEL);
2481	if (!pdh_blob)
2482	return -ENOMEM;
2483
2484	data.pdh_cert_address = __psp_pa(pdh_blob);
2485	data.pdh_cert_len = input.pdh_cert_len;
2486
2487	cert_blob = kzalloc(input.cert_chain_len, GFP_KERNEL);
2488	if (!cert_blob) {
2489	ret = -ENOMEM;
2490	goto e_free_pdh;
2491	}
2492
2493	data.cert_chain_address = __psp_pa(cert_blob);
2494	data.cert_chain_len = input.cert_chain_len;
2495
2496	cmd:
2497	/* If platform is not in INIT state then transition it to INIT. */
2498	if (sev->sev_plat_status.state != SEV_STATE_INIT) {
2499	if (!writable) {
2500	ret = -EPERM;
2501	goto e_free_cert;
2502	}
2503	ret = sev_move_to_init_state(argp, &shutdown_required);
2504	if (ret)
2505	goto e_free_cert;
2506	}
2507
2508	ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error);
2509
2510	/*
2511	* Firmware will return the length of the blobs (either the minimum
2512	* required length or the actual length written), return 'em to the user.
2513	*/
2514	input.cert_chain_len = data.cert_chain_len;
2515	input.pdh_cert_len = data.pdh_cert_len;
2516
2517	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
2518	ret = -EFAULT;
2519	goto e_free_cert;
2520	}
2521
2522	if (ret \|\| WARN_ON_ONCE(argp->error))
2523	goto e_free_cert;
2524
2525	if (pdh_blob) {
2526	if (copy_to_user(input_pdh_cert_address,
2527	pdh_blob, input.pdh_cert_len)) {
2528	ret = -EFAULT;
2529	goto e_free_cert;
2530	}
2531	}
2532
2533	if (cert_blob) {
2534	if (copy_to_user(input_cert_chain_address,
2535	cert_blob, input.cert_chain_len))
2536	ret = -EFAULT;
2537	}
2538
2539	e_free_cert:
2540	if (shutdown_required)
2541	__sev_firmware_shutdown(sev, false);
2542
2543	kfree(cert_blob);
2544	e_free_pdh:
2545	kfree(pdh_blob);
2546	return ret;
2547	}
2548
2549	static int __sev_do_snp_platform_status(struct sev_user_data_snp_status *status,
2550	int *error)
2551	{
2552	struct sev_device *sev = psp_master->sev_data;
2553	struct sev_data_snp_addr buf;
2554	struct page *status_page;
2555	void *data;
2556	int ret;
2557
2558	status_page = alloc_page(GFP_KERNEL_ACCOUNT);
2559	if (!status_page)
2560	return -ENOMEM;
2561
2562	data = page_address(status_page);
2563
2564	/*
2565	* SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed
2566	* when SNP has been initialized, the status page must be firmware-owned.
2567	*/
2568	if (sev->snp_initialized) {
2569	/*
2570	* Firmware expects the status page to be in Firmware state,
2571	* otherwise it will report an error INVALID_PAGE_STATE.
2572	*/
2573	if (rmp_mark_pages_firmware(__pa(data), 1, true)) {
2574	ret = -EFAULT;
2575	goto cleanup;
2576	}
2577	}
2578
2579	buf.address = __psp_pa(data);
2580	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
2581
2582	if (sev->snp_initialized) {
2583	/*
2584	* The status page will be in Reclaim state on success, or left
2585	* in Firmware state on failure. Use snp_reclaim_pages() to
2586	* transition either case back to Hypervisor-owned state.
2587	*/
2588	if (snp_reclaim_pages(__pa(data), 1, true))
2589	return -EFAULT;
2590	}
2591
2592	if (ret)
2593	goto cleanup;
2594
2595	memcpy(status, data, sizeof(*status));
2596
2597	cleanup:
2598	__free_pages(status_page, 0);
2599	return ret;
2600	}
2601
2602	static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
2603	{
2604	struct sev_user_data_snp_status status;
2605	int ret;
2606
2607	if (!argp->data)
2608	return -EINVAL;
2609
2610	ret = __sev_do_snp_platform_status(&status, &argp->error);
2611	if (ret < 0)
2612	return ret;
2613
2614	if (copy_to_user((void __user *)argp->data, &status,
2615	sizeof(struct sev_user_data_snp_status)))
2616	ret = -EFAULT;
2617
2618	return ret;
2619	}
2620
2621	static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
2622	{
2623	struct sev_data_snp_commit buf;
2624	int ret;
2625
2626	buf.len = sizeof(buf);
2627
2628	ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
2629
2630	return ret;
2631	}
2632
2633	static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable)
2634	{
2635	struct sev_device *sev = psp_master->sev_data;
2636	struct sev_user_data_snp_config config;
2637
2638	if (!argp->data)
2639	return -EINVAL;
2640
2641	if (!writable)
2642	return -EPERM;
2643
2644	if (!sev->snp_initialized)
2645	return -ENODEV;
2646
2647	if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
2648	return -EFAULT;
2649
2650	return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
2651	}
2652
2653	static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
2654	{
2655	struct sev_device *sev = psp_master->sev_data;
2656	struct sev_user_data_snp_vlek_load input;
2657	void *blob;
2658	int ret;
2659
2660	if (!argp->data)
2661	return -EINVAL;
2662
2663	if (!writable)
2664	return -EPERM;
2665
2666	if (!sev->snp_initialized)
2667	return -ENODEV;
2668
2669	if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
2670	return -EFAULT;
2671
2672	if (input.len != sizeof(input) \|\| input.vlek_wrapped_version != 0)
2673	return -EINVAL;
2674
2675	blob = psp_copy_user_blob(input.vlek_wrapped_address,
2676	sizeof(struct sev_user_data_snp_wrapped_vlek_hashstick));
2677	if (IS_ERR(blob))
2678	return PTR_ERR(blob);
2679
2680	input.vlek_wrapped_address = __psp_pa(blob);
2681
2682	ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
2683	kfree(blob);
2684
2685	return ret;
2686	}
2687
2688	static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
2689	{
2690	void __user argp = (void __user )arg;
2691	struct sev_issue_cmd input;
2692	int ret = -EFAULT;
2693	bool writable = file->f_mode & FMODE_WRITE;
2694
2695	if (!psp_master \|\| !psp_master->sev_data)
2696	return -ENODEV;
2697
2698	if (ioctl != SEV_ISSUE_CMD)
2699	return -EINVAL;
2700
2701	if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
2702	return -EFAULT;
2703
2704	if (input.cmd > SEV_MAX)
2705	return -EINVAL;
2706
2707	mutex_lock(&sev_cmd_mutex);
2708
2709	switch (input.cmd) {
2710
2711	case SEV_FACTORY_RESET:
2712	ret = sev_ioctl_do_reset(&input, writable);
2713	break;
2714	case SEV_PLATFORM_STATUS:
2715	ret = sev_ioctl_do_platform_status(&input);
2716	break;
2717	case SEV_PEK_GEN:
2718	ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input, writable);
2719	break;
2720	case SEV_PDH_GEN:
2721	ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input, writable);
2722	break;
2723	case SEV_PEK_CSR:
2724	ret = sev_ioctl_do_pek_csr(&input, writable);
2725	break;
2726	case SEV_PEK_CERT_IMPORT:
2727	ret = sev_ioctl_do_pek_import(&input, writable);
2728	break;
2729	case SEV_PDH_CERT_EXPORT:
2730	ret = sev_ioctl_do_pdh_export(&input, writable);
2731	break;
2732	case SEV_GET_ID:
2733	pr_warn_once("SEV_GET_ID command is deprecated, use SEV_GET_ID2\n");
2734	ret = sev_ioctl_do_get_id(&input);
2735	break;
2736	case SEV_GET_ID2:
2737	ret = sev_ioctl_do_get_id2(&input);
2738	break;
2739	case SNP_PLATFORM_STATUS:
2740	ret = sev_ioctl_do_snp_platform_status(&input);
2741	break;
2742	case SNP_COMMIT:
2743	ret = sev_ioctl_do_snp_commit(&input);
2744	break;
2745	case SNP_SET_CONFIG:
2746	ret = sev_ioctl_do_snp_set_config(&input, writable);
2747	break;
2748	case SNP_VLEK_LOAD:
2749	ret = sev_ioctl_do_snp_vlek_load(&input, writable);
2750	break;
2751	default:
2752	ret = -EINVAL;
2753	goto out;
2754	}
2755
2756	if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
2757	ret = -EFAULT;
2758	out:
2759	mutex_unlock(&sev_cmd_mutex);
2760
2761	return ret;
2762	}
2763
2764	static const struct file_operations sev_fops = {
2765	.owner = THIS_MODULE,
2766	.unlocked_ioctl = sev_ioctl,
2767	};
2768
2769	int sev_platform_status(struct sev_user_data_status data, int error)
2770	{
2771	return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
2772	}
2773	EXPORT_SYMBOL_GPL(sev_platform_status);
2774
2775	int sev_guest_deactivate(struct sev_data_deactivate data, int error)
2776	{
2777	return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
2778	}
2779	EXPORT_SYMBOL_GPL(sev_guest_deactivate);
2780
2781	int sev_guest_activate(struct sev_data_activate data, int error)
2782	{
2783	return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
2784	}
2785	EXPORT_SYMBOL_GPL(sev_guest_activate);
2786
2787	int sev_guest_decommission(struct sev_data_decommission data, int error)
2788	{
2789	return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
2790	}
2791	EXPORT_SYMBOL_GPL(sev_guest_decommission);
2792
2793	int sev_guest_df_flush(int *error)
2794	{
2795	return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
2796	}
2797	EXPORT_SYMBOL_GPL(sev_guest_df_flush);
2798
2799	static void sev_exit(struct kref *ref)
2800	{
2801	misc_deregister(&misc_dev->misc);
2802	kfree(misc_dev);
2803	misc_dev = NULL;
2804	}
2805
2806	static int sev_misc_init(struct sev_device *sev)
2807	{
2808	struct device *dev = sev->dev;
2809	int ret;
2810
2811	/*
2812	* SEV feature support can be detected on multiple devices but the SEV
2813	* FW commands must be issued on the master. During probe, we do not
2814	* know the master hence we create /dev/sev on the first device probe.
2815	* sev_do_cmd() finds the right master device to which to issue the
2816	* command to the firmware.
2817	*/
2818	if (!misc_dev) {
2819	struct miscdevice *misc;
2820
2821	misc_dev = kzalloc_obj(*misc_dev);
2822	if (!misc_dev)
2823	return -ENOMEM;
2824
2825	misc = &misc_dev->misc;
2826	misc->minor = MISC_DYNAMIC_MINOR;
2827	misc->name = DEVICE_NAME;
2828	misc->fops = &sev_fops;
2829
2830	ret = misc_register(misc);
2831	if (ret)
2832	return ret;
2833
2834	kref_init(&misc_dev->refcount);
2835	} else {
2836	kref_get(&misc_dev->refcount);
2837	}
2838
2839	init_waitqueue_head(&sev->int_queue);
2840	sev->misc = misc_dev;
2841	dev_dbg(dev, "registered SEV device\n");
2842
2843	return 0;
2844	}
2845
2846	int sev_dev_init(struct psp_device *psp)
2847	{
2848	struct device *dev = psp->dev;
2849	struct sev_device *sev;
2850	int ret = -ENOMEM;
2851
2852	if (!boot_cpu_has(X86_FEATURE_SEV)) {
2853	dev_info_once(dev, "SEV: memory encryption not enabled by BIOS\n");
2854	return 0;
2855	}
2856
2857	sev = devm_kzalloc(dev, sizeof(*sev), GFP_KERNEL);
2858	if (!sev)
2859	goto e_err;
2860
2861	sev->cmd_buf = (void *)devm_get_free_pages(dev, GFP_KERNEL, 1);
2862	if (!sev->cmd_buf)
2863	goto e_sev;
2864
2865	sev->cmd_buf_backup = (uint8_t *)sev->cmd_buf + PAGE_SIZE;
2866
2867	psp->sev_data = sev;
2868
2869	sev->dev = dev;
2870	sev->psp = psp;
2871
2872	sev->io_regs = psp->io_regs;
2873
2874	sev->vdata = (struct sev_vdata *)psp->vdata->sev;
2875	if (!sev->vdata) {
2876	ret = -ENODEV;
2877	dev_err(dev, "sev: missing driver data\n");
2878	goto e_buf;
2879	}
2880
2881	psp_set_sev_irq_handler(psp, sev_irq_handler, sev);
2882
2883	ret = sev_misc_init(sev);
2884	if (ret)
2885	goto e_irq;
2886
2887	dev_notice(dev, "sev enabled\n");
2888
2889	return 0;
2890
2891	e_irq:
2892	psp_clear_sev_irq_handler(psp);
2893	e_buf:
2894	devm_free_pages(dev, (unsigned long)sev->cmd_buf);
2895	e_sev:
2896	devm_kfree(dev, sev);
2897	e_err:
2898	psp->sev_data = NULL;
2899
2900	dev_notice(dev, "sev initialization failed\n");
2901
2902	return ret;
2903	}
2904
2905	static void __sev_firmware_shutdown(struct sev_device *sev, bool panic)
2906	{
2907	int error;
2908
2909	__sev_platform_shutdown_locked(&error);
2910
2911	if (sev_es_tmr) {
2912	/*
2913	* The TMR area was encrypted, flush it from the cache.
2914	*
2915	* If invoked during panic handling, local interrupts are
2916	* disabled and all CPUs are stopped, so wbinvd_on_all_cpus()
2917	* can't be used. In that case, wbinvd() is done on remote CPUs
2918	* via the NMI callback, and done for this CPU later during
2919	* SNP shutdown, so wbinvd_on_all_cpus() can be skipped.
2920	*/
2921	if (!panic)
2922	wbinvd_on_all_cpus();
2923
2924	__snp_free_firmware_pages(virt_to_page(sev_es_tmr),
2925	get_order(sev_es_tmr_size),
2926	true);
2927	sev_es_tmr = NULL;
2928	}
2929
2930	if (sev_init_ex_buffer) {
2931	__snp_free_firmware_pages(virt_to_page(sev_init_ex_buffer),
2932	get_order(NV_LENGTH),
2933	true);
2934	sev_init_ex_buffer = NULL;
2935	}
2936
2937	__sev_snp_shutdown_locked(&error, panic);
2938	}
2939
2940	static void sev_firmware_shutdown(struct sev_device *sev)
2941	{
2942	/*
2943	* Calling without sev_cmd_mutex held as TSM will likely try disconnecting
2944	* IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex.
2945	*/
2946	if (sev->tio_status)
2947	sev_tsm_uninit(sev);
2948
2949	/*
2950	* Remove the sysfs interface before taking the sev_cmd_mutex.
2951	* sysfs_remove_group() waits for in-flight _show()/_store() handlers
2952	* to drain, and those handlers issue SNP_VERIFY_MITIGATION via
2953	* sev_do_cmd() which acquires the sev_cmd_mutex. Removing the group
2954	* while holding the mutex could therefore deadlock.
2955	*/
2956	sev_snp_unregister_verify_mitigation(sev);
2957
2958	mutex_lock(&sev_cmd_mutex);
2959
2960	__sev_firmware_shutdown(sev, false);
2961
2962	kfree(sev->tio_status);
2963	sev->tio_status = NULL;
2964
2965	mutex_unlock(&sev_cmd_mutex);
2966	}
2967
2968	void sev_platform_shutdown(void)
2969	{
2970	if (!psp_master \|\| !psp_master->sev_data)
2971	return;
2972
2973	sev_firmware_shutdown(psp_master->sev_data);
2974	}
2975	EXPORT_SYMBOL_GPL(sev_platform_shutdown);
2976
2977	u64 sev_get_snp_policy_bits(void)
2978	{
2979	struct psp_device *psp = psp_master;
2980	struct sev_device *sev;
2981	u64 policy_bits;
2982
2983	if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
2984	return 0;
2985
2986	if (!psp \|\| !psp->sev_data)
2987	return 0;
2988
2989	sev = psp->sev_data;
2990
2991	policy_bits = SNP_POLICY_MASK_BASE;
2992
2993	if (sev->snp_plat_status.feature_info) {
2994	if (sev->snp_feat_info_0.ecx & SNP_RAPL_DISABLE_SUPPORTED)
2995	policy_bits \|= SNP_POLICY_MASK_RAPL_DIS;
2996
2997	if (sev->snp_feat_info_0.ecx & SNP_CIPHER_TEXT_HIDING_SUPPORTED)
2998	policy_bits \|= SNP_POLICY_MASK_CIPHERTEXT_HIDING_DRAM;
2999
3000	if (sev->snp_feat_info_0.ecx & SNP_AES_256_XTS_POLICY_SUPPORTED)
3001	policy_bits \|= SNP_POLICY_MASK_MEM_AES_256_XTS;
3002
3003	if (sev->snp_feat_info_0.ecx & SNP_CXL_ALLOW_POLICY_SUPPORTED)
3004	policy_bits \|= SNP_POLICY_MASK_CXL_ALLOW;
3005
3006	if (sev_version_greater_or_equal(1, 58))
3007	policy_bits \|= SNP_POLICY_MASK_PAGE_SWAP_DISABLE;
3008	}
3009
3010	return policy_bits;
3011	}
3012	EXPORT_SYMBOL_GPL(sev_get_snp_policy_bits);
3013
3014	void sev_dev_destroy(struct psp_device *psp)
3015	{
3016	struct sev_device *sev = psp->sev_data;
3017
3018	if (!sev)
3019	return;
3020
3021	sev_firmware_shutdown(sev);
3022
3023	if (sev->misc)
3024	kref_put(&misc_dev->refcount, sev_exit);
3025
3026	psp_clear_sev_irq_handler(psp);
3027	}
3028
3029	static int snp_shutdown_on_panic(struct notifier_block *nb,
3030	unsigned long reason, void *arg)
3031	{
3032	struct sev_device *sev = psp_master->sev_data;
3033
3034	/*
3035	* If sev_cmd_mutex is already acquired, then it's likely
3036	* another PSP command is in flight and issuing a shutdown
3037	* would fail in unexpected ways. Rather than create even
3038	* more confusion during a panic, just bail out here.
3039	*/
3040	if (mutex_is_locked(&sev_cmd_mutex))
3041	return NOTIFY_DONE;
3042
3043	__sev_firmware_shutdown(sev, true);
3044
3045	return NOTIFY_DONE;
3046	}
3047
3048	int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
3049	void data, int error)
3050	{
3051	if (!filep \|\| filep->f_op != &sev_fops)
3052	return -EBADF;
3053
3054	return sev_do_cmd(cmd, data, error);
3055	}
3056	EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
3057
3058	void sev_pci_init(void)
3059	{
3060	struct sev_device *sev = psp_master->sev_data;
3061	u8 api_major, api_minor, build;
3062
3063	if (!sev)
3064	return;
3065
3066	psp_timeout = psp_probe_timeout;
3067
3068	if (sev_get_api_version())
3069	goto err;
3070
3071	api_major = sev->api_major;
3072	api_minor = sev->api_minor;
3073	build = sev->build;
3074
3075	if (sev_update_firmware(sev->dev) == 0)
3076	sev_get_api_version();
3077
3078	if (api_major != sev->api_major \|\| api_minor != sev->api_minor \|\|
3079	build != sev->build)
3080	dev_info(sev->dev, "SEV firmware updated from %d.%d.%d to %d.%d.%d\n",
3081	api_major, api_minor, build,
3082	sev->api_major, sev->api_minor, sev->build);
3083
3084	return;
3085
3086	err:
3087	sev_dev_destroy(psp_master);
3088
3089	psp_master->sev_data = NULL;
3090	}
3091
3092	void sev_pci_exit(void)
3093	{
3094	struct sev_device *sev = psp_master->sev_data;
3095
3096	if (!sev)
3097	return;
3098
3099	sev_firmware_shutdown(sev);
3100	}
3101
3102	static int get_v1_svn(struct sev_device *sev)
3103	{
3104	struct sev_snp_tcb_version_genoa_milan *tcb;
3105	struct sev_user_data_snp_status status;
3106	int ret, error = 0;
3107
3108	mutex_lock(&sev_cmd_mutex);
3109	ret = __sev_do_snp_platform_status(&status, &error);
3110	mutex_unlock(&sev_cmd_mutex);
3111	if (ret < 0)
3112	return ret;
3113
3114	tcb = (struct sev_snp_tcb_version_genoa_milan *)&status
3115	.current_tcb_version;
3116	return tcb->snp;
3117	}
3118
3119	static int get_v2_svn(struct sev_device *sev)
3120	{
3121	struct sev_user_data_snp_status status;
3122	struct sev_snp_tcb_version_turin *tcb;
3123	int ret, error = 0;
3124
3125	mutex_lock(&sev_cmd_mutex);
3126	ret = __sev_do_snp_platform_status(&status, &error);
3127	mutex_unlock(&sev_cmd_mutex);
3128	if (ret < 0)
3129	return ret;
3130
3131	tcb = (struct sev_snp_tcb_version_turin *)&status
3132	.current_tcb_version;
3133	return tcb->snp;
3134	}
3135
3136	static bool sev_firmware_allows_es(struct sev_device *sev)
3137	{
3138	/* Documented in AMD-SB-3023 */
3139	if (boot_cpu_has(X86_FEATURE_ZEN4) \|\| boot_cpu_has(X86_FEATURE_ZEN3))
3140	return get_v1_svn(sev) < 0x1b;
3141	else if (boot_cpu_has(X86_FEATURE_ZEN5))
3142	return get_v2_svn(sev) < 0x4;
3143	else
3144	return true;
3145	}
3146
3147	int sev_firmware_supported_vm_types(void)
3148	{
3149	int supported_vm_types = 0;
3150	struct sev_device *sev;
3151
3152	if (!psp_master \|\| !psp_master->sev_data)
3153	return supported_vm_types;
3154	sev = psp_master->sev_data;
3155
3156	supported_vm_types \|= BIT(KVM_X86_SEV_VM);
3157	supported_vm_types \|= BIT(KVM_X86_SEV_ES_VM);
3158
3159	if (!sev->snp_initialized)
3160	return supported_vm_types;
3161
3162	supported_vm_types \|= BIT(KVM_X86_SNP_VM);
3163
3164	if (!sev_firmware_allows_es(sev))
3165	supported_vm_types &= ~BIT(KVM_X86_SEV_ES_VM);
3166
3167	return supported_vm_types;
3168
3169	}
3170	EXPORT_SYMBOL_FOR_MODULES(sev_firmware_supported_vm_types, "kvm-amd");