linux - drivers/mmc/core/block.c

3392 lines · 84.5 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Block driver for media (i.e., flash cards)
4	*
5	* Copyright 2002 Hewlett-Packard Company
6	* Copyright 2005-2008 Pierre Ossman
7	*
8	* Use consistent with the GNU GPL is permitted,
9	* provided that this copyright notice is
10	* preserved in its entirety in all copies and derived works.
11	*
12	* HEWLETT-PACKARD COMPANY MAKES NO WARRANTIES, EXPRESSED OR IMPLIED,
13	* AS TO THE USEFULNESS OR CORRECTNESS OF THIS CODE OR ITS
14	* FITNESS FOR ANY PARTICULAR PURPOSE.
15	*
16	* Many thanks to Alessandro Rubini and Jonathan Corbet!
17	*
18	* Author: Andrew Christian
19	* 28 May 2002
20	*/
21	#include <linux/moduleparam.h>
22	#include <linux/module.h>
23	#include <linux/init.h>
24
25	#include <linux/kernel.h>
26	#include <linux/fs.h>
27	#include <linux/slab.h>
28	#include <linux/errno.h>
29	#include <linux/hdreg.h>
30	#include <linux/kdev_t.h>
31	#include <linux/kref.h>
32	#include <linux/blkdev.h>
33	#include <linux/cdev.h>
34	#include <linux/mutex.h>
35	#include <linux/scatterlist.h>
36	#include <linux/string.h>
37	#include <linux/string_helpers.h>
38	#include <linux/delay.h>
39	#include <linux/capability.h>
40	#include <linux/compat.h>
41	#include <linux/pm_runtime.h>
42	#include <linux/idr.h>
43	#include <linux/debugfs.h>
44	#include <linux/rpmb.h>
45
46	#include <linux/mmc/ioctl.h>
47	#include <linux/mmc/card.h>
48	#include <linux/mmc/host.h>
49	#include <linux/mmc/mmc.h>
50	#include <linux/mmc/sd.h>
51
52	#include <linux/uaccess.h>
53	#include <linux/unaligned.h>
54
55	#include "queue.h"
56	#include "block.h"
57	#include "core.h"
58	#include "card.h"
59	#include "crypto.h"
60	#include "host.h"
61	#include "bus.h"
62	#include "mmc_ops.h"
63	#include "quirks.h"
64	#include "sd_ops.h"
65
66	MODULE_ALIAS("mmc:block");
67	#ifdef MODULE_PARAM_PREFIX
68	#undef MODULE_PARAM_PREFIX
69	#endif
70	#define MODULE_PARAM_PREFIX "mmcblk."
71
72	/*
73	* Set a 10 second timeout for polling write request busy state. Note, mmc core
74	* is setting a 3 second timeout for SD cards, and SDHCI has long had a 10
75	* second software timer to timeout the whole request, so 10 seconds should be
76	* ample.
77	*/
78	#define MMC_BLK_TIMEOUT_MS (10 * 1000)
79	#define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
80	#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
81
82	#define RPMB_FRAME_SIZE sizeof(struct rpmb_frame)
83	#define CHECK_SIZE_NEQ(val) ((val) != sizeof(struct rpmb_frame))
84	#define CHECK_SIZE_ALIGNED(val) IS_ALIGNED((val), sizeof(struct rpmb_frame))
85
86	static DEFINE_MUTEX(block_mutex);
87
88	/*
89	* The defaults come from config options but can be overriden by module
90	* or bootarg options.
91	*/
92	static int perdev_minors = CONFIG_MMC_BLOCK_MINORS;
93
94	/*
95	* We've only got one major, so number of mmcblk devices is
96	* limited to (1 << 20) / number of minors per device. It is also
97	* limited by the MAX_DEVICES below.
98	*/
99	static int max_devices;
100
101	#define MAX_DEVICES 256
102
103	static DEFINE_IDA(mmc_blk_ida);
104	static DEFINE_IDA(mmc_rpmb_ida);
105
106	struct mmc_blk_busy_data {
107	struct mmc_card *card;
108	u32 status;
109	};
110
111	/*
112	* There is one mmc_blk_data per slot.
113	*/
114	struct mmc_blk_data {
115	struct device *parent;
116	struct gendisk *disk;
117	struct mmc_queue queue;
118	struct list_head part;
119	struct list_head rpmbs;
120
121	unsigned int flags;
122	#define MMC_BLK_CMD23 (1 << 0) /* Can do SET_BLOCK_COUNT for multiblock */
123	#define MMC_BLK_REL_WR (1 << 1) /* MMC Reliable write support */
124
125	struct kref kref;
126	unsigned int read_only;
127	unsigned int part_type;
128	unsigned int reset_done;
129	#define MMC_BLK_READ BIT(0)
130	#define MMC_BLK_WRITE BIT(1)
131	#define MMC_BLK_DISCARD BIT(2)
132	#define MMC_BLK_SECDISCARD BIT(3)
133	#define MMC_BLK_CQE_RECOVERY BIT(4)
134	#define MMC_BLK_TRIM BIT(5)
135
136	/*
137	* Only set in main mmc_blk_data associated
138	* with mmc_card with dev_set_drvdata, and keeps
139	* track of the current selected device partition.
140	*/
141	unsigned int part_curr;
142	#define MMC_BLK_PART_INVALID UINT_MAX /* Unknown partition active */
143	int area_type;
144
145	/* debugfs files (only in main mmc_blk_data) */
146	struct dentry *status_dentry;
147	struct dentry *ext_csd_dentry;
148	};
149
150	/* Device type for RPMB character devices */
151	static dev_t mmc_rpmb_devt;
152
153	/* Bus type for RPMB character devices */
154	static const struct bus_type mmc_rpmb_bus_type = {
155	.name = "mmc_rpmb",
156	};
157
158	/**
159	* struct mmc_rpmb_data - special RPMB device type for these areas
160	* @dev: the device for the RPMB area
161	* @chrdev: character device for the RPMB area
162	* @id: unique device ID number
163	* @part_index: partition index (0 on first)
164	* @md: parent MMC block device
165	* @rdev: registered RPMB device
166	* @node: list item, so we can put this device on a list
167	*/
168	struct mmc_rpmb_data {
169	struct device dev;
170	struct cdev chrdev;
171	int id;
172	unsigned int part_index;
173	struct mmc_blk_data *md;
174	struct rpmb_dev *rdev;
175	struct list_head node;
176	};
177
178	static DEFINE_MUTEX(open_lock);
179
180	module_param(perdev_minors, int, 0444);
181	MODULE_PARM_DESC(perdev_minors, "Minors numbers to allocate per device");
182
183	static inline int mmc_blk_part_switch(struct mmc_card *card,
184	unsigned int part_type);
185	static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
186	struct mmc_card *card,
187	int recovery_mode,
188	struct mmc_queue *mq);
189	static void mmc_blk_hsq_req_done(struct mmc_request *mrq);
190	static int mmc_spi_err_check(struct mmc_card *card);
191	static int mmc_blk_busy_cb(void cb_data, bool busy);
192
193	static struct mmc_blk_data mmc_blk_get(struct gendisk disk)
194	{
195	struct mmc_blk_data *md;
196
197	mutex_lock(&open_lock);
198	md = disk->private_data;
199	if (md && !kref_get_unless_zero(&md->kref))
200	md = NULL;
201	mutex_unlock(&open_lock);
202
203	return md;
204	}
205
206	static inline int mmc_get_devidx(struct gendisk *disk)
207	{
208	int devidx = disk->first_minor / perdev_minors;
209	return devidx;
210	}
211
212	static void mmc_blk_kref_release(struct kref *ref)
213	{
214	struct mmc_blk_data *md = container_of(ref, struct mmc_blk_data, kref);
215	int devidx;
216
217	devidx = mmc_get_devidx(md->disk);
218	ida_free(&mmc_blk_ida, devidx);
219
220	mutex_lock(&open_lock);
221	md->disk->private_data = NULL;
222	mutex_unlock(&open_lock);
223
224	put_disk(md->disk);
225	kfree(md);
226	}
227
228	static void mmc_blk_put(struct mmc_blk_data *md)
229	{
230	kref_put(&md->kref, mmc_blk_kref_release);
231	}
232
233	static ssize_t power_ro_lock_show(struct device *dev,
234	struct device_attribute attr, char buf)
235	{
236	int ret;
237	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
238	struct mmc_card *card = md->queue.card;
239	int locked = 0;
240
241	if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PERM_WP_EN)
242	locked = 2;
243	else if (card->ext_csd.boot_ro_lock & EXT_CSD_BOOT_WP_B_PWR_WP_EN)
244	locked = 1;
245
246	ret = sysfs_emit(buf, "%d\n", locked);
247
248	mmc_blk_put(md);
249
250	return ret;
251	}
252
253	static ssize_t power_ro_lock_store(struct device *dev,
254	struct device_attribute attr, const char buf, size_t count)
255	{
256	int ret;
257	struct mmc_blk_data md, part_md;
258	struct mmc_queue *mq;
259	struct request *req;
260	unsigned long set;
261
262	if (kstrtoul(buf, 0, &set))
263	return -EINVAL;
264
265	if (set != 1)
266	return count;
267
268	md = mmc_blk_get(dev_to_disk(dev));
269	mq = &md->queue;
270
271	/* Dispatch locking to the block layer */
272	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_OUT, 0);
273	if (IS_ERR(req)) {
274	count = PTR_ERR(req);
275	goto out_put;
276	}
277	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_BOOT_WP;
278	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
279	blk_execute_rq(req, false);
280	ret = req_to_mmc_queue_req(req)->drv_op_result;
281	blk_mq_free_request(req);
282
283	if (!ret) {
284	pr_info("%s: Locking boot partition ro until next power on\n",
285	md->disk->disk_name);
286	set_disk_ro(md->disk, 1);
287
288	list_for_each_entry(part_md, &md->part, part)
289	if (part_md->area_type == MMC_BLK_DATA_AREA_BOOT) {
290	pr_info("%s: Locking boot partition ro until next power on\n", part_md->disk->disk_name);
291	set_disk_ro(part_md->disk, 1);
292	}
293	}
294	out_put:
295	mmc_blk_put(md);
296	return count;
297	}
298
299	static DEVICE_ATTR(ro_lock_until_next_power_on, 0,
300	power_ro_lock_show, power_ro_lock_store);
301
302	static ssize_t force_ro_show(struct device dev, struct device_attribute attr,
303	char *buf)
304	{
305	int ret;
306	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
307
308	ret = sysfs_emit(buf, "%d\n",
309	get_disk_ro(dev_to_disk(dev)) ^
310	md->read_only);
311	mmc_blk_put(md);
312	return ret;
313	}
314
315	static ssize_t force_ro_store(struct device dev, struct device_attribute attr,
316	const char *buf, size_t count)
317	{
318	int ret;
319	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
320	unsigned long set;
321
322	if (kstrtoul(buf, 0, &set)) {
323	ret = -EINVAL;
324	goto out;
325	}
326
327	set_disk_ro(dev_to_disk(dev), set \|\| md->read_only);
328	ret = count;
329	out:
330	mmc_blk_put(md);
331	return ret;
332	}
333
334	static DEVICE_ATTR(force_ro, 0644, force_ro_show, force_ro_store);
335
336	static struct attribute *mmc_disk_attrs[] = {
337	&dev_attr_force_ro.attr,
338	&dev_attr_ro_lock_until_next_power_on.attr,
339	NULL,
340	};
341
342	static umode_t mmc_disk_attrs_is_visible(struct kobject *kobj,
343	struct attribute *a, int n)
344	{
345	struct device *dev = kobj_to_dev(kobj);
346	struct mmc_blk_data *md = mmc_blk_get(dev_to_disk(dev));
347	umode_t mode = a->mode;
348
349	if (a == &dev_attr_ro_lock_until_next_power_on.attr &&
350	(md->area_type & MMC_BLK_DATA_AREA_BOOT) &&
351	md->queue.card->ext_csd.boot_ro_lockable) {
352	mode = 0444;
353	if (!(md->queue.card->ext_csd.boot_ro_lock &
354	EXT_CSD_BOOT_WP_B_PWR_WP_DIS))
355	mode \|= 0200;
356	}
357
358	mmc_blk_put(md);
359	return mode;
360	}
361
362	static const struct attribute_group mmc_disk_attr_group = {
363	.is_visible = mmc_disk_attrs_is_visible,
364	.attrs = mmc_disk_attrs,
365	};
366
367	static const struct attribute_group *mmc_disk_attr_groups[] = {
368	&mmc_disk_attr_group,
369	NULL,
370	};
371
372	static int mmc_blk_open(struct gendisk *disk, blk_mode_t mode)
373	{
374	struct mmc_blk_data *md = mmc_blk_get(disk);
375	int ret = -ENXIO;
376
377	mutex_lock(&block_mutex);
378	if (md) {
379	ret = 0;
380	if ((mode & BLK_OPEN_WRITE) && md->read_only) {
381	mmc_blk_put(md);
382	ret = -EROFS;
383	}
384	}
385	mutex_unlock(&block_mutex);
386
387	return ret;
388	}
389
390	static void mmc_blk_release(struct gendisk *disk)
391	{
392	struct mmc_blk_data *md = disk->private_data;
393
394	mutex_lock(&block_mutex);
395	mmc_blk_put(md);
396	mutex_unlock(&block_mutex);
397	}
398
399	static int
400	mmc_blk_getgeo(struct gendisk disk, struct hd_geometry geo)
401	{
402	geo->cylinders = get_capacity(disk) / (4 * 16);
403	geo->heads = 4;
404	geo->sectors = 16;
405	return 0;
406	}
407
408	struct mmc_blk_ioc_data {
409	struct mmc_ioc_cmd ic;
410	unsigned char *buf;
411	u64 buf_bytes;
412	unsigned int flags;
413	#define MMC_BLK_IOC_DROP BIT(0) /* drop this mrq */
414	#define MMC_BLK_IOC_SBC BIT(1) /* use mrq.sbc */
415
416	struct mmc_rpmb_data *rpmb;
417	};
418
419	static struct mmc_blk_ioc_data *mmc_blk_ioctl_copy_from_user(
420	struct mmc_ioc_cmd __user *user)
421	{
422	struct mmc_blk_ioc_data *idata;
423	int err;
424
425	idata = kzalloc_obj(*idata);
426	if (!idata) {
427	err = -ENOMEM;
428	goto out;
429	}
430
431	if (copy_from_user(&idata->ic, user, sizeof(idata->ic))) {
432	err = -EFAULT;
433	goto idata_err;
434	}
435
436	idata->buf_bytes = (u64) idata->ic.blksz * idata->ic.blocks;
437	if (idata->buf_bytes > MMC_IOC_MAX_BYTES) {
438	err = -EOVERFLOW;
439	goto idata_err;
440	}
441
442	if (!idata->buf_bytes) {
443	idata->buf = NULL;
444	return idata;
445	}
446
447	idata->buf = memdup_user((void __user *)(unsigned long)
448	idata->ic.data_ptr, idata->buf_bytes);
449	if (IS_ERR(idata->buf)) {
450	err = PTR_ERR(idata->buf);
451	goto idata_err;
452	}
453
454	return idata;
455
456	idata_err:
457	kfree(idata);
458	out:
459	return ERR_PTR(err);
460	}
461
462	static int mmc_blk_ioctl_copy_to_user(struct mmc_ioc_cmd __user *ic_ptr,
463	struct mmc_blk_ioc_data *idata)
464	{
465	struct mmc_ioc_cmd *ic = &idata->ic;
466
467	if (copy_to_user(&(ic_ptr->response), ic->response,
468	sizeof(ic->response)))
469	return -EFAULT;
470
471	if (!idata->ic.write_flag) {
472	if (copy_to_user((void __user *)(unsigned long)ic->data_ptr,
473	idata->buf, idata->buf_bytes))
474	return -EFAULT;
475	}
476
477	return 0;
478	}
479
480	static int __mmc_blk_ioctl_cmd(struct mmc_card card, struct mmc_blk_data md,
481	struct mmc_blk_ioc_data **idatas, int i)
482	{
483	struct mmc_command cmd = {}, sbc = {};
484	struct mmc_data data = {};
485	struct mmc_request mrq = {};
486	struct scatterlist sg;
487	bool r1b_resp;
488	unsigned int busy_timeout_ms;
489	int err;
490	unsigned int target_part;
491	struct mmc_blk_ioc_data *idata = idatas[i];
492	struct mmc_blk_ioc_data *prev_idata = NULL;
493
494	if (!card \|\| !md \|\| !idata)
495	return -EINVAL;
496
497	if (idata->flags & MMC_BLK_IOC_DROP)
498	return 0;
499
500	if (idata->flags & MMC_BLK_IOC_SBC && i > 0)
501	prev_idata = idatas[i - 1];
502
503	/*
504	* The RPMB accesses comes in from the character device, so we
505	* need to target these explicitly. Else we just target the
506	* partition type for the block device the ioctl() was issued
507	* on.
508	*/
509	if (idata->rpmb) {
510	/* Support multiple RPMB partitions */
511	target_part = idata->rpmb->part_index;
512	target_part \|= EXT_CSD_PART_CONFIG_ACC_RPMB;
513	} else {
514	target_part = md->part_type;
515	}
516
517	cmd.opcode = idata->ic.opcode;
518	cmd.arg = idata->ic.arg;
519	cmd.flags = idata->ic.flags;
520
521	if (idata->buf_bytes) {
522	data.sg = &sg;
523	data.sg_len = 1;
524	data.blksz = idata->ic.blksz;
525	data.blocks = idata->ic.blocks;
526
527	sg_init_one(data.sg, idata->buf, idata->buf_bytes);
528
529	if (idata->ic.write_flag)
530	data.flags = MMC_DATA_WRITE;
531	else
532	data.flags = MMC_DATA_READ;
533
534	/* data.flags must already be set before doing this. */
535	mmc_set_data_timeout(&data, card);
536
537	/* Allow overriding the timeout_ns for empirical tuning. */
538	if (idata->ic.data_timeout_ns)
539	data.timeout_ns = idata->ic.data_timeout_ns;
540
541	mrq.data = &data;
542	}
543
544	mrq.cmd = &cmd;
545
546	err = mmc_blk_part_switch(card, target_part);
547	if (err)
548	return err;
549
550	if (idata->ic.is_acmd) {
551	err = mmc_app_cmd(card->host, card);
552	if (err)
553	return err;
554	}
555
556	if (idata->rpmb \|\| prev_idata) {
557	sbc.opcode = MMC_SET_BLOCK_COUNT;
558	/*
559	* We don't do any blockcount validation because the max size
560	* may be increased by a future standard. We just copy the
561	* 'Reliable Write' bit here.
562	*/
563	sbc.arg = data.blocks \| (idata->ic.write_flag & BIT(31));
564	if (prev_idata)
565	sbc.arg = prev_idata->ic.arg;
566	sbc.flags = MMC_RSP_R1 \| MMC_CMD_AC;
567	mrq.sbc = &sbc;
568	}
569
570	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_SANITIZE_START) &&
571	(cmd.opcode == MMC_SWITCH))
572	return mmc_sanitize(card, idata->ic.cmd_timeout_ms);
573
574	/* If it's an R1B response we need some more preparations. */
575	busy_timeout_ms = idata->ic.cmd_timeout_ms ? : MMC_BLK_TIMEOUT_MS;
576	r1b_resp = (cmd.flags & MMC_RSP_R1B) == MMC_RSP_R1B;
577	if (r1b_resp)
578	mmc_prepare_busy_cmd(card->host, &cmd, busy_timeout_ms);
579
580	mmc_wait_for_req(card->host, &mrq);
581	memcpy(&idata->ic.response, cmd.resp, sizeof(cmd.resp));
582
583	if (prev_idata) {
584	memcpy(&prev_idata->ic.response, sbc.resp, sizeof(sbc.resp));
585	if (sbc.error) {
586	dev_err(mmc_dev(card->host), "%s: sbc error %d\n",
587	__func__, sbc.error);
588	return sbc.error;
589	}
590	}
591
592	if (cmd.error) {
593	dev_err(mmc_dev(card->host), "%s: cmd error %d\n",
594	__func__, cmd.error);
595	return cmd.error;
596	}
597	if (data.error) {
598	dev_err(mmc_dev(card->host), "%s: data error %d\n",
599	__func__, data.error);
600	return data.error;
601	}
602
603	/*
604	* Make sure the cache of the PARTITION_CONFIG register and
605	* PARTITION_ACCESS bits is updated in case the ioctl ext_csd write
606	* changed it successfully.
607	*/
608	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) &&
609	(cmd.opcode == MMC_SWITCH)) {
610	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
611	u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg);
612
613	/*
614	* Update cache so the next mmc_blk_part_switch call operates
615	* on up-to-date data.
616	*/
617	card->ext_csd.part_config = value;
618	main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK;
619	}
620
621	/*
622	* Make sure to update CACHE_CTRL in case it was changed. The cache
623	* will get turned back on if the card is re-initialized, e.g.
624	* suspend/resume or hw reset in recovery.
625	*/
626	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_CACHE_CTRL) &&
627	(cmd.opcode == MMC_SWITCH)) {
628	u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg) & 1;
629
630	card->ext_csd.cache_ctrl = value;
631	}
632
633	/*
634	* According to the SD specs, some commands require a delay after
635	* issuing the command.
636	*/
637	if (idata->ic.postsleep_min_us)
638	usleep_range(idata->ic.postsleep_min_us, idata->ic.postsleep_max_us);
639
640	if (mmc_host_is_spi(card->host)) {
641	if (idata->ic.write_flag \|\| r1b_resp \|\| cmd.flags & MMC_RSP_SPI_BUSY)
642	return mmc_spi_err_check(card);
643	return err;
644	}
645
646	/*
647	* Ensure RPMB, writes and R1B responses are completed by polling with
648	* CMD13. Note that, usually we don't need to poll when using HW busy
649	* detection, but here it's needed since some commands may indicate the
650	* error through the R1 status bits.
651	*/
652	if (idata->rpmb \|\| idata->ic.write_flag \|\| r1b_resp) {
653	struct mmc_blk_busy_data cb_data = {
654	.card = card,
655	};
656
657	err = __mmc_poll_for_busy(card->host, 0, busy_timeout_ms,
658	&mmc_blk_busy_cb, &cb_data);
659
660	idata->ic.response[0] = cb_data.status;
661	}
662
663	return err;
664	}
665
666	static int mmc_blk_ioctl_cmd(struct mmc_blk_data *md,
667	struct mmc_ioc_cmd __user *ic_ptr,
668	struct mmc_rpmb_data *rpmb)
669	{
670	struct mmc_blk_ioc_data *idata;
671	struct mmc_blk_ioc_data *idatas[1];
672	struct mmc_queue *mq;
673	struct mmc_card *card;
674	int err = 0, ioc_err = 0;
675	struct request *req;
676
677	idata = mmc_blk_ioctl_copy_from_user(ic_ptr);
678	if (IS_ERR(idata))
679	return PTR_ERR(idata);
680	/* This will be NULL on non-RPMB ioctl():s */
681	idata->rpmb = rpmb;
682
683	card = md->queue.card;
684	if (IS_ERR(card)) {
685	err = PTR_ERR(card);
686	goto cmd_done;
687	}
688
689	/*
690	* Dispatch the ioctl() into the block request queue.
691	*/
692	mq = &md->queue;
693	req = blk_mq_alloc_request(mq->queue,
694	idata->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
695	if (IS_ERR(req)) {
696	err = PTR_ERR(req);
697	goto cmd_done;
698	}
699	idatas[0] = idata;
700	req_to_mmc_queue_req(req)->drv_op =
701	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
702	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
703	req_to_mmc_queue_req(req)->drv_op_data = idatas;
704	req_to_mmc_queue_req(req)->ioc_count = 1;
705	blk_execute_rq(req, false);
706	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
707	err = mmc_blk_ioctl_copy_to_user(ic_ptr, idata);
708	blk_mq_free_request(req);
709
710	cmd_done:
711	kfree(idata->buf);
712	kfree(idata);
713	return ioc_err ? ioc_err : err;
714	}
715
716	static int mmc_blk_ioctl_multi_cmd(struct mmc_blk_data *md,
717	struct mmc_ioc_multi_cmd __user *user,
718	struct mmc_rpmb_data *rpmb)
719	{
720	struct mmc_blk_ioc_data **idata = NULL;
721	struct mmc_ioc_cmd __user *cmds = user->cmds;
722	struct mmc_card *card;
723	struct mmc_queue *mq;
724	int err = 0, ioc_err = 0;
725	__u64 num_of_cmds;
726	unsigned int i, n;
727	struct request *req;
728
729	if (copy_from_user(&num_of_cmds, &user->num_of_cmds,
730	sizeof(num_of_cmds)))
731	return -EFAULT;
732
733	if (!num_of_cmds)
734	return 0;
735
736	if (num_of_cmds > MMC_IOC_MAX_CMDS)
737	return -EINVAL;
738
739	n = num_of_cmds;
740	idata = kzalloc_objs(*idata, n);
741	if (!idata)
742	return -ENOMEM;
743
744	for (i = 0; i < n; i++) {
745	idata[i] = mmc_blk_ioctl_copy_from_user(&cmds[i]);
746	if (IS_ERR(idata[i])) {
747	err = PTR_ERR(idata[i]);
748	n = i;
749	goto cmd_err;
750	}
751	/* This will be NULL on non-RPMB ioctl():s */
752	idata[i]->rpmb = rpmb;
753	}
754
755	card = md->queue.card;
756	if (IS_ERR(card)) {
757	err = PTR_ERR(card);
758	goto cmd_err;
759	}
760
761
762	/*
763	* Dispatch the ioctl()s into the block request queue.
764	*/
765	mq = &md->queue;
766	req = blk_mq_alloc_request(mq->queue,
767	idata[0]->ic.write_flag ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
768	if (IS_ERR(req)) {
769	err = PTR_ERR(req);
770	goto cmd_err;
771	}
772	req_to_mmc_queue_req(req)->drv_op =
773	rpmb ? MMC_DRV_OP_IOCTL_RPMB : MMC_DRV_OP_IOCTL;
774	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
775	req_to_mmc_queue_req(req)->drv_op_data = idata;
776	req_to_mmc_queue_req(req)->ioc_count = n;
777	blk_execute_rq(req, false);
778	ioc_err = req_to_mmc_queue_req(req)->drv_op_result;
779
780	/* copy to user if data and response */
781	for (i = 0; i < n && !err; i++)
782	err = mmc_blk_ioctl_copy_to_user(&cmds[i], idata[i]);
783
784	blk_mq_free_request(req);
785
786	cmd_err:
787	for (i = 0; i < n; i++) {
788	kfree(idata[i]->buf);
789	kfree(idata[i]);
790	}
791	kfree(idata);
792	return ioc_err ? ioc_err : err;
793	}
794
795	static int mmc_blk_check_blkdev(struct block_device *bdev)
796	{
797	/*
798	* The caller must have CAP_SYS_RAWIO, and must be calling this on the
799	* whole block device, not on a partition. This prevents overspray
800	* between sibling partitions.
801	*/
802	if (!capable(CAP_SYS_RAWIO) \|\| bdev_is_partition(bdev))
803	return -EPERM;
804	return 0;
805	}
806
807	static int mmc_blk_ioctl(struct block_device *bdev, blk_mode_t mode,
808	unsigned int cmd, unsigned long arg)
809	{
810	struct mmc_blk_data *md;
811	int ret;
812
813	switch (cmd) {
814	case MMC_IOC_CMD:
815	ret = mmc_blk_check_blkdev(bdev);
816	if (ret)
817	return ret;
818	md = mmc_blk_get(bdev->bd_disk);
819	if (!md)
820	return -EINVAL;
821	ret = mmc_blk_ioctl_cmd(md,
822	(struct mmc_ioc_cmd __user *)arg,
823	NULL);
824	mmc_blk_put(md);
825	return ret;
826	case MMC_IOC_MULTI_CMD:
827	ret = mmc_blk_check_blkdev(bdev);
828	if (ret)
829	return ret;
830	md = mmc_blk_get(bdev->bd_disk);
831	if (!md)
832	return -EINVAL;
833	ret = mmc_blk_ioctl_multi_cmd(md,
834	(struct mmc_ioc_multi_cmd __user *)arg,
835	NULL);
836	mmc_blk_put(md);
837	return ret;
838	default:
839	return -EINVAL;
840	}
841	}
842
843	#ifdef CONFIG_COMPAT
844	static int mmc_blk_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
845	unsigned int cmd, unsigned long arg)
846	{
847	return mmc_blk_ioctl(bdev, mode, cmd, (unsigned long) compat_ptr(arg));
848	}
849	#endif
850
851	static int mmc_blk_alternative_gpt_sector(struct gendisk *disk,
852	sector_t *sector)
853	{
854	struct mmc_blk_data *md;
855	int ret;
856
857	md = mmc_blk_get(disk);
858	if (!md)
859	return -EINVAL;
860
861	if (md->queue.card)
862	ret = mmc_card_alternative_gpt_sector(md->queue.card, sector);
863	else
864	ret = -ENODEV;
865
866	mmc_blk_put(md);
867
868	return ret;
869	}
870
871	static const struct block_device_operations mmc_bdops = {
872	.open = mmc_blk_open,
873	.release = mmc_blk_release,
874	.getgeo = mmc_blk_getgeo,
875	.owner = THIS_MODULE,
876	.ioctl = mmc_blk_ioctl,
877	#ifdef CONFIG_COMPAT
878	.compat_ioctl = mmc_blk_compat_ioctl,
879	#endif
880	.alternative_gpt_sector = mmc_blk_alternative_gpt_sector,
881	};
882
883	static int mmc_blk_part_switch_pre(struct mmc_card *card,
884	unsigned int part_type)
885	{
886	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK;
887	const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB;
888	int ret = 0;
889
890	if ((part_type & mask) == rpmb) {
891	if (card->ext_csd.cmdq_en) {
892	ret = mmc_cmdq_disable(card);
893	if (ret)
894	return ret;
895	}
896	mmc_retune_pause(card->host);
897	}
898
899	return ret;
900	}
901
902	static int mmc_blk_part_switch_post(struct mmc_card *card,
903	unsigned int part_type)
904	{
905	const unsigned int mask = EXT_CSD_PART_CONFIG_ACC_MASK;
906	const unsigned int rpmb = EXT_CSD_PART_CONFIG_ACC_RPMB;
907	int ret = 0;
908
909	if ((part_type & mask) == rpmb) {
910	mmc_retune_unpause(card->host);
911	if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
912	ret = mmc_cmdq_enable(card);
913	}
914
915	return ret;
916	}
917
918	static inline int mmc_blk_part_switch(struct mmc_card *card,
919	unsigned int part_type)
920	{
921	int ret = 0;
922	struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
923
924	if (main_md->part_curr == part_type)
925	return 0;
926
927	if (mmc_card_mmc(card)) {
928	u8 part_config = card->ext_csd.part_config;
929
930	ret = mmc_blk_part_switch_pre(card, part_type);
931	if (ret)
932	return ret;
933
934	part_config &= ~EXT_CSD_PART_CONFIG_ACC_MASK;
935	part_config \|= part_type;
936
937	ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
938	EXT_CSD_PART_CONFIG, part_config,
939	card->ext_csd.part_time);
940	if (ret) {
941	mmc_blk_part_switch_post(card, part_type);
942	return ret;
943	}
944
945	card->ext_csd.part_config = part_config;
946
947	ret = mmc_blk_part_switch_post(card, main_md->part_curr);
948	}
949
950	main_md->part_curr = part_type;
951	return ret;
952	}
953
954	static int mmc_sd_num_wr_blocks(struct mmc_card card, u32 written_blocks)
955	{
956	int err;
957	u32 result;
958	__be32 *blocks;
959	u8 resp_sz = mmc_card_ult_capacity(card) ? 8 : 4;
960
961	struct mmc_request mrq = {};
962	struct mmc_command cmd = {};
963	struct mmc_data data = {};
964	struct scatterlist sg;
965
966	err = mmc_app_cmd(card->host, card);
967	if (err)
968	return err;
969
970	cmd.opcode = SD_APP_SEND_NUM_WR_BLKS;
971	cmd.arg = 0;
972	cmd.flags = MMC_RSP_SPI_R1 \| MMC_RSP_R1 \| MMC_CMD_ADTC;
973
974	data.blksz = resp_sz;
975	data.blocks = 1;
976	data.flags = MMC_DATA_READ;
977	data.sg = &sg;
978	data.sg_len = 1;
979	mmc_set_data_timeout(&data, card);
980
981	mrq.cmd = &cmd;
982	mrq.data = &data;
983
984	blocks = kmalloc(resp_sz, GFP_NOIO);
985	if (!blocks)
986	return -ENOMEM;
987
988	sg_init_one(&sg, blocks, resp_sz);
989
990	mmc_wait_for_req(card->host, &mrq);
991
992	if (mmc_card_ult_capacity(card)) {
993	/*
994	* Normally, ACMD22 returns the number of written sectors as
995	* u32. SDUC, however, returns it as u64. This is not a
996	* superfluous requirement, because SDUC writes may exceed 2TB.
997	* For Linux mmc however, the previously write operation could
998	* not be more than the block layer limits, thus just make room
999	* for a u64 and cast the response back to u32.
1000	*/
1001	result = clamp_val(get_unaligned_be64(blocks), 0, UINT_MAX);
1002	} else {
1003	result = ntohl(*blocks);
1004	}
1005	kfree(blocks);
1006
1007	if (cmd.error \|\| data.error)
1008	return -EIO;
1009
1010	*written_blocks = result;
1011
1012	return 0;
1013	}
1014
1015	static unsigned int mmc_blk_clock_khz(struct mmc_host *host)
1016	{
1017	if (host->actual_clock)
1018	return host->actual_clock / 1000;
1019
1020	/* Clock may be subject to a divisor, fudge it by a factor of 2. */
1021	if (host->ios.clock)
1022	return host->ios.clock / 2000;
1023
1024	/* How can there be no clock */
1025	WARN_ON_ONCE(1);
1026	return 100; /* 100 kHz is minimum possible value */
1027	}
1028
1029	static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host,
1030	struct mmc_data *data)
1031	{
1032	unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000);
1033	unsigned int khz;
1034
1035	if (data->timeout_clks) {
1036	khz = mmc_blk_clock_khz(host);
1037	ms += DIV_ROUND_UP(data->timeout_clks, khz);
1038	}
1039
1040	return ms;
1041	}
1042
1043	/*
1044	* Attempts to reset the card and get back to the requested partition.
1045	* Therefore any error here must result in cancelling the block layer
1046	* request, it must not be reattempted without going through the mmc_blk
1047	* partition sanity checks.
1048	*/
1049	static int mmc_blk_reset(struct mmc_blk_data md, struct mmc_host host,
1050	int type)
1051	{
1052	int err;
1053	struct mmc_blk_data *main_md = dev_get_drvdata(&host->card->dev);
1054
1055	if (md->reset_done & type)
1056	return -EEXIST;
1057
1058	md->reset_done \|= type;
1059	err = mmc_hw_reset(host->card);
1060	/*
1061	* A successful reset will leave the card in the main partition, but
1062	* upon failure it might not be, so set it to MMC_BLK_PART_INVALID
1063	* in that case.
1064	*/
1065	main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type;
1066	if (err)
1067	return err;
1068	/* Ensure we switch back to the correct partition */
1069	if (mmc_blk_part_switch(host->card, md->part_type))
1070	/*
1071	* We have failed to get back into the correct
1072	* partition, so we need to abort the whole request.
1073	*/
1074	return -ENODEV;
1075	return 0;
1076	}
1077
1078	static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type)
1079	{
1080	md->reset_done &= ~type;
1081	}
1082
1083	static void mmc_blk_check_sbc(struct mmc_queue_req *mq_rq)
1084	{
1085	struct mmc_blk_ioc_data **idata = mq_rq->drv_op_data;
1086	int i;
1087
1088	for (i = 1; i < mq_rq->ioc_count; i++) {
1089	if (idata[i - 1]->ic.opcode == MMC_SET_BLOCK_COUNT &&
1090	mmc_op_multi(idata[i]->ic.opcode)) {
1091	idata[i - 1]->flags \|= MMC_BLK_IOC_DROP;
1092	idata[i]->flags \|= MMC_BLK_IOC_SBC;
1093	}
1094	}
1095	}
1096
1097	/*
1098	* The non-block commands come back from the block layer after it queued it and
1099	* processed it with all other requests and then they get issued in this
1100	* function.
1101	*/
1102	static void mmc_blk_issue_drv_op(struct mmc_queue mq, struct request req)
1103	{
1104	struct mmc_queue_req *mq_rq;
1105	struct mmc_card *card = mq->card;
1106	struct mmc_blk_data *md = mq->blkdata;
1107	struct mmc_blk_ioc_data **idata;
1108	bool rpmb_ioctl;
1109	u8 **ext_csd;
1110	u32 status;
1111	int ret;
1112	int i;
1113
1114	mq_rq = req_to_mmc_queue_req(req);
1115	rpmb_ioctl = (mq_rq->drv_op == MMC_DRV_OP_IOCTL_RPMB);
1116
1117	switch (mq_rq->drv_op) {
1118	case MMC_DRV_OP_IOCTL:
1119	if (card->ext_csd.cmdq_en) {
1120	ret = mmc_cmdq_disable(card);
1121	if (ret)
1122	break;
1123	}
1124
1125	mmc_blk_check_sbc(mq_rq);
1126
1127	fallthrough;
1128	case MMC_DRV_OP_IOCTL_RPMB:
1129	idata = mq_rq->drv_op_data;
1130	for (i = 0, ret = 0; i < mq_rq->ioc_count; i++) {
1131	ret = __mmc_blk_ioctl_cmd(card, md, idata, i);
1132	if (ret)
1133	break;
1134	}
1135	/* Always switch back to main area after RPMB access */
1136	if (rpmb_ioctl)
1137	mmc_blk_part_switch(card, 0);
1138	else if (card->reenable_cmdq && !card->ext_csd.cmdq_en)
1139	mmc_cmdq_enable(card);
1140	break;
1141	case MMC_DRV_OP_BOOT_WP:
1142	ret = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BOOT_WP,
1143	card->ext_csd.boot_ro_lock \|
1144	EXT_CSD_BOOT_WP_B_PWR_WP_EN,
1145	card->ext_csd.part_time);
1146	if (ret)
1147	pr_err("%s: Locking boot partition ro until next power on failed: %d\n",
1148	md->disk->disk_name, ret);
1149	else
1150	card->ext_csd.boot_ro_lock \|=
1151	EXT_CSD_BOOT_WP_B_PWR_WP_EN;
1152	break;
1153	case MMC_DRV_OP_GET_CARD_STATUS:
1154	ret = mmc_send_status(card, &status);
1155	if (!ret)
1156	ret = status;
1157	break;
1158	case MMC_DRV_OP_GET_EXT_CSD:
1159	ext_csd = mq_rq->drv_op_data;
1160	ret = mmc_get_ext_csd(card, ext_csd);
1161	break;
1162	default:
1163	pr_err("%s: unknown driver specific operation\n",
1164	md->disk->disk_name);
1165	ret = -EINVAL;
1166	break;
1167	}
1168	mq_rq->drv_op_result = ret;
1169	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
1170	}
1171
1172	static void mmc_blk_issue_erase_rq(struct mmc_queue mq, struct request req,
1173	int type, unsigned int erase_arg)
1174	{
1175	struct mmc_blk_data *md = mq->blkdata;
1176	struct mmc_card *card = md->queue.card;
1177	unsigned int nr;
1178	sector_t from;
1179	int err = 0;
1180	blk_status_t status = BLK_STS_OK;
1181
1182	if (!mmc_card_can_erase(card)) {
1183	status = BLK_STS_NOTSUPP;
1184	goto fail;
1185	}
1186
1187	from = blk_rq_pos(req);
1188	nr = blk_rq_sectors(req);
1189
1190	do {
1191	err = 0;
1192	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1193	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1194	INAND_CMD38_ARG_EXT_CSD,
1195	erase_arg == MMC_TRIM_ARG ?
1196	INAND_CMD38_ARG_TRIM :
1197	INAND_CMD38_ARG_ERASE,
1198	card->ext_csd.generic_cmd6_time);
1199	}
1200	if (!err)
1201	err = mmc_erase(card, from, nr, erase_arg);
1202	} while (err == -EIO && !mmc_blk_reset(md, card->host, type));
1203	if (err)
1204	status = BLK_STS_IOERR;
1205	else
1206	mmc_blk_reset_success(md, type);
1207	fail:
1208	blk_mq_end_request(req, status);
1209	}
1210
1211	static void mmc_blk_issue_trim_rq(struct mmc_queue mq, struct request req)
1212	{
1213	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_TRIM, MMC_TRIM_ARG);
1214	}
1215
1216	static void mmc_blk_issue_discard_rq(struct mmc_queue mq, struct request req)
1217	{
1218	struct mmc_blk_data *md = mq->blkdata;
1219	struct mmc_card *card = md->queue.card;
1220	unsigned int arg = card->erase_arg;
1221
1222	if (mmc_card_broken_sd_discard(card))
1223	arg = SD_ERASE_ARG;
1224
1225	mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg);
1226	}
1227
1228	static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
1229	struct request *req)
1230	{
1231	struct mmc_blk_data *md = mq->blkdata;
1232	struct mmc_card *card = md->queue.card;
1233	unsigned int nr, arg;
1234	sector_t from;
1235	int err = 0, type = MMC_BLK_SECDISCARD;
1236	blk_status_t status = BLK_STS_OK;
1237
1238	if (!(mmc_card_can_secure_erase_trim(card))) {
1239	status = BLK_STS_NOTSUPP;
1240	goto out;
1241	}
1242
1243	from = blk_rq_pos(req);
1244	nr = blk_rq_sectors(req);
1245
1246	if (mmc_card_can_trim(card) && !mmc_erase_group_aligned(card, from, nr))
1247	arg = MMC_SECURE_TRIM1_ARG;
1248	else
1249	arg = MMC_SECURE_ERASE_ARG;
1250
1251	retry:
1252	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1253	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1254	INAND_CMD38_ARG_EXT_CSD,
1255	arg == MMC_SECURE_TRIM1_ARG ?
1256	INAND_CMD38_ARG_SECTRIM1 :
1257	INAND_CMD38_ARG_SECERASE,
1258	card->ext_csd.generic_cmd6_time);
1259	if (err)
1260	goto out_retry;
1261	}
1262
1263	err = mmc_erase(card, from, nr, arg);
1264	if (err == -EIO)
1265	goto out_retry;
1266	if (err) {
1267	status = BLK_STS_IOERR;
1268	goto out;
1269	}
1270
1271	if (arg == MMC_SECURE_TRIM1_ARG) {
1272	if (card->quirks & MMC_QUIRK_INAND_CMD38) {
1273	err = mmc_switch(card, EXT_CSD_CMD_SET_NORMAL,
1274	INAND_CMD38_ARG_EXT_CSD,
1275	INAND_CMD38_ARG_SECTRIM2,
1276	card->ext_csd.generic_cmd6_time);
1277	if (err)
1278	goto out_retry;
1279	}
1280
1281	err = mmc_erase(card, from, nr, MMC_SECURE_TRIM2_ARG);
1282	if (err == -EIO)
1283	goto out_retry;
1284	if (err) {
1285	status = BLK_STS_IOERR;
1286	goto out;
1287	}
1288	}
1289
1290	out_retry:
1291	if (err && !mmc_blk_reset(md, card->host, type))
1292	goto retry;
1293	if (!err)
1294	mmc_blk_reset_success(md, type);
1295	out:
1296	blk_mq_end_request(req, status);
1297	}
1298
1299	static void mmc_blk_issue_flush(struct mmc_queue mq, struct request req)
1300	{
1301	struct mmc_blk_data *md = mq->blkdata;
1302	struct mmc_card *card = md->queue.card;
1303	int ret = 0;
1304
1305	ret = mmc_flush_cache(card->host);
1306	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
1307	}
1308
1309	/*
1310	* Reformat current write as a reliable write, supporting
1311	* both legacy and the enhanced reliable write MMC cards.
1312	* In each transfer we'll handle only as much as a single
1313	* reliable write can handle, thus finish the request in
1314	* partial completions.
1315	*/
1316	static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
1317	struct mmc_card *card,
1318	struct request *req)
1319	{
1320	if (!(card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN)) {
1321	/* Legacy mode imposes restrictions on transfers. */
1322	if (!IS_ALIGNED(blk_rq_pos(req), card->ext_csd.rel_sectors))
1323	brq->data.blocks = 1;
1324
1325	if (brq->data.blocks > card->ext_csd.rel_sectors)
1326	brq->data.blocks = card->ext_csd.rel_sectors;
1327	else if (brq->data.blocks < card->ext_csd.rel_sectors)
1328	brq->data.blocks = 1;
1329	}
1330	}
1331
1332	#define CMD_ERRORS_EXCL_OOR \
1333	(R1_ADDRESS_ERROR \| /* Misaligned address */ \
1334	R1_BLOCK_LEN_ERROR \| /* Transferred block length incorrect */\
1335	R1_WP_VIOLATION \| /* Tried to write to protected block */ \
1336	R1_CARD_ECC_FAILED \| /* Card ECC failed */ \
1337	R1_CC_ERROR \| /* Card controller error */ \
1338	R1_ERROR) /* General/unknown error */
1339
1340	#define CMD_ERRORS \
1341	(CMD_ERRORS_EXCL_OOR \| \
1342	R1_OUT_OF_RANGE) /* Command argument out of range */ \
1343
1344	static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
1345	{
1346	u32 val;
1347
1348	/*
1349	* Per the SD specification(physical layer version 4.10)[1],
1350	* section 4.3.3, it explicitly states that "When the last
1351	* block of user area is read using CMD18, the host should
1352	* ignore OUT_OF_RANGE error that may occur even the sequence
1353	* is correct". And JESD84-B51 for eMMC also has a similar
1354	* statement on section 6.8.3.
1355	*
1356	* Multiple block read/write could be done by either predefined
1357	* method, namely CMD23, or open-ending mode. For open-ending mode,
1358	* we should ignore the OUT_OF_RANGE error as it's normal behaviour.
1359	*
1360	* However the spec[1] doesn't tell us whether we should also
1361	* ignore that for predefined method. But per the spec[1], section
1362	* 4.15 Set Block Count Command, it says"If illegal block count
1363	* is set, out of range error will be indicated during read/write
1364	* operation (For example, data transfer is stopped at user area
1365	* boundary)." In another word, we could expect a out of range error
1366	* in the response for the following CMD18/25. And if argument of
1367	* CMD23 + the argument of CMD18/25 exceed the max number of blocks,
1368	* we could also expect to get a -ETIMEDOUT or any error number from
1369	* the host drivers due to missing data response(for write)/data(for
1370	* read), as the cards will stop the data transfer by itself per the
1371	* spec. So we only need to check R1_OUT_OF_RANGE for open-ending mode.
1372	*/
1373
1374	if (!brq->stop.error) {
1375	bool oor_with_open_end;
1376	/* If there is no error yet, check R1 response */
1377
1378	val = brq->stop.resp[0] & CMD_ERRORS;
1379	oor_with_open_end = val & R1_OUT_OF_RANGE && !brq->mrq.sbc;
1380
1381	if (val && !oor_with_open_end)
1382	brq->stop.error = -EIO;
1383	}
1384	}
1385
1386	static void mmc_blk_data_prep(struct mmc_queue mq, struct mmc_queue_req mqrq,
1387	int recovery_mode, bool *do_rel_wr_p,
1388	bool *do_data_tag_p)
1389	{
1390	struct mmc_blk_data *md = mq->blkdata;
1391	struct mmc_card *card = md->queue.card;
1392	struct mmc_blk_request *brq = &mqrq->brq;
1393	struct request *req = mmc_queue_req_to_req(mqrq);
1394	bool do_rel_wr, do_data_tag;
1395
1396	/*
1397	* Reliable writes are used to implement Forced Unit Access and
1398	* are supported only on MMCs.
1399	*/
1400	do_rel_wr = (req->cmd_flags & REQ_FUA) &&
1401	rq_data_dir(req) == WRITE &&
1402	(md->flags & MMC_BLK_REL_WR);
1403
1404	if (mqrq->flags & MQRQ_XFER_SINGLE_BLOCK)
1405	recovery_mode = 1;
1406
1407	memset(brq, 0, sizeof(struct mmc_blk_request));
1408
1409	mmc_crypto_prepare_req(mqrq);
1410
1411	brq->mrq.data = &brq->data;
1412	brq->mrq.tag = req->tag;
1413
1414	brq->stop.opcode = MMC_STOP_TRANSMISSION;
1415	brq->stop.arg = 0;
1416
1417	if (rq_data_dir(req) == READ) {
1418	brq->data.flags = MMC_DATA_READ;
1419	brq->stop.flags = MMC_RSP_SPI_R1 \| MMC_RSP_R1 \| MMC_CMD_AC;
1420	} else {
1421	brq->data.flags = MMC_DATA_WRITE;
1422	brq->stop.flags = MMC_RSP_SPI_R1B \| MMC_RSP_R1B \| MMC_CMD_AC;
1423	}
1424
1425	brq->data.blksz = 512;
1426	brq->data.blocks = blk_rq_sectors(req);
1427	brq->data.blk_addr = blk_rq_pos(req);
1428
1429	/*
1430	* The command queue supports 2 priorities: "high" (1) and "simple" (0).
1431	* The eMMC will give "high" priority tasks priority over "simple"
1432	* priority tasks. Here we always set "simple" priority by not setting
1433	* MMC_DATA_PRIO.
1434	*/
1435
1436	/*
1437	* The block layer doesn't support all sector count
1438	* restrictions, so we need to be prepared for too big
1439	* requests.
1440	*/
1441	if (brq->data.blocks > card->host->max_blk_count)
1442	brq->data.blocks = card->host->max_blk_count;
1443
1444	if (brq->data.blocks > 1) {
1445	/*
1446	* Some SD cards in SPI mode return a CRC error or even lock up
1447	* completely when trying to read the last block using a
1448	* multiblock read command.
1449	*/
1450	if (mmc_host_is_spi(card->host) && (rq_data_dir(req) == READ) &&
1451	(blk_rq_pos(req) + blk_rq_sectors(req) ==
1452	get_capacity(md->disk)))
1453	brq->data.blocks--;
1454
1455	/*
1456	* After a read error, we redo the request one (native) sector
1457	* at a time in order to accurately determine which
1458	* sectors can be read successfully.
1459	*/
1460	if (recovery_mode)
1461	brq->data.blocks = queue_physical_block_size(mq->queue) >> SECTOR_SHIFT;
1462
1463	/*
1464	* Some controllers have HW issues while operating
1465	* in multiple I/O mode
1466	*/
1467	if (card->host->ops->multi_io_quirk)
1468	brq->data.blocks = card->host->ops->multi_io_quirk(card,
1469	(rq_data_dir(req) == READ) ?
1470	MMC_DATA_READ : MMC_DATA_WRITE,
1471	brq->data.blocks);
1472	}
1473
1474	if (do_rel_wr) {
1475	mmc_apply_rel_rw(brq, card, req);
1476	brq->data.flags \|= MMC_DATA_REL_WR;
1477	}
1478
1479	/*
1480	* Data tag is used only during writing meta data to speed
1481	* up write and any subsequent read of this meta data
1482	*/
1483	do_data_tag = card->ext_csd.data_tag_unit_size &&
1484	(req->cmd_flags & REQ_META) &&
1485	(rq_data_dir(req) == WRITE) &&
1486	((brq->data.blocks * brq->data.blksz) >=
1487	card->ext_csd.data_tag_unit_size);
1488
1489	if (do_data_tag)
1490	brq->data.flags \|= MMC_DATA_DAT_TAG;
1491
1492	mmc_set_data_timeout(&brq->data, card);
1493
1494	brq->data.sg = mqrq->sg;
1495	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);
1496
1497	/*
1498	* Adjust the sg list so it is the same size as the
1499	* request.
1500	*/
1501	if (brq->data.blocks != blk_rq_sectors(req)) {
1502	int i, data_size = brq->data.blocks << 9;
1503	struct scatterlist *sg;
1504
1505	for_each_sg(brq->data.sg, sg, brq->data.sg_len, i) {
1506	data_size -= sg->length;
1507	if (data_size <= 0) {
1508	sg->length += data_size;
1509	i++;
1510	break;
1511	}
1512	}
1513	brq->data.sg_len = i;
1514	}
1515
1516	if (do_rel_wr_p)
1517	*do_rel_wr_p = do_rel_wr;
1518
1519	if (do_data_tag_p)
1520	*do_data_tag_p = do_data_tag;
1521	}
1522
1523	#define MMC_CQE_RETRIES 2
1524
1525	static void mmc_blk_cqe_complete_rq(struct mmc_queue mq, struct request req)
1526	{
1527	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1528	struct mmc_request *mrq = &mqrq->brq.mrq;
1529	struct request_queue *q = req->q;
1530	struct mmc_host *host = mq->card->host;
1531	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
1532	unsigned long flags;
1533	bool put_card;
1534	int err;
1535
1536	mmc_cqe_post_req(host, mrq);
1537
1538	if (mrq->cmd && mrq->cmd->error)
1539	err = mrq->cmd->error;
1540	else if (mrq->data && mrq->data->error)
1541	err = mrq->data->error;
1542	else
1543	err = 0;
1544
1545	if (err) {
1546	if (mqrq->retries++ < MMC_CQE_RETRIES) {
1547	mqrq->flags \|= MQRQ_XFER_SINGLE_BLOCK;
1548	blk_mq_requeue_request(req, true);
1549	} else {
1550	blk_mq_end_request(req, BLK_STS_IOERR);
1551	}
1552	} else if (mrq->data) {
1553	if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered))
1554	blk_mq_requeue_request(req, true);
1555	else
1556	__blk_mq_end_request(req, BLK_STS_OK);
1557	} else if (mq->in_recovery) {
1558	blk_mq_requeue_request(req, true);
1559	} else {
1560	blk_mq_end_request(req, BLK_STS_OK);
1561	}
1562
1563	spin_lock_irqsave(&mq->lock, flags);
1564
1565	mq->in_flight[issue_type] -= 1;
1566
1567	put_card = (mmc_tot_in_flight(mq) == 0);
1568
1569	mmc_cqe_check_busy(mq);
1570
1571	spin_unlock_irqrestore(&mq->lock, flags);
1572
1573	if (!mq->cqe_busy)
1574	blk_mq_run_hw_queues(q, true);
1575
1576	if (put_card)
1577	mmc_put_card(mq->card, &mq->ctx);
1578	}
1579
1580	void mmc_blk_cqe_recovery(struct mmc_queue *mq)
1581	{
1582	struct mmc_card *card = mq->card;
1583	struct mmc_host *host = card->host;
1584	int err;
1585
1586	pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
1587
1588	err = mmc_cqe_recovery(host);
1589	if (err)
1590	mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
1591	mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);
1592
1593	pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
1594	}
1595
1596	static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
1597	{
1598	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
1599	brq.mrq);
1600	struct request *req = mmc_queue_req_to_req(mqrq);
1601	struct request_queue *q = req->q;
1602	struct mmc_queue *mq = q->queuedata;
1603
1604	/*
1605	* Block layer timeouts race with completions which means the normal
1606	* completion path cannot be used during recovery.
1607	*/
1608	if (mq->in_recovery)
1609	mmc_blk_cqe_complete_rq(mq, req);
1610	else if (likely(!blk_should_fake_timeout(req->q)))
1611	blk_mq_complete_request(req);
1612	}
1613
1614	static int mmc_blk_cqe_start_req(struct mmc_host host, struct mmc_request mrq)
1615	{
1616	mrq->done = mmc_blk_cqe_req_done;
1617	mrq->recovery_notifier = mmc_cqe_recovery_notifier;
1618
1619	return mmc_cqe_start_req(host, mrq);
1620	}
1621
1622	static struct mmc_request mmc_blk_cqe_prep_dcmd(struct mmc_queue_req mqrq,
1623	struct request *req)
1624	{
1625	struct mmc_blk_request *brq = &mqrq->brq;
1626
1627	memset(brq, 0, sizeof(*brq));
1628
1629	brq->mrq.cmd = &brq->cmd;
1630	brq->mrq.tag = req->tag;
1631
1632	return &brq->mrq;
1633	}
1634
1635	static int mmc_blk_cqe_issue_flush(struct mmc_queue mq, struct request req)
1636	{
1637	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1638	struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
1639
1640	mrq->cmd->opcode = MMC_SWITCH;
1641	mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) \|
1642	(EXT_CSD_FLUSH_CACHE << 16) \|
1643	(1 << 8) \|
1644	EXT_CSD_CMD_SET_NORMAL;
1645	mrq->cmd->flags = MMC_CMD_AC \| MMC_RSP_R1B;
1646
1647	return mmc_blk_cqe_start_req(mq->card->host, mrq);
1648	}
1649
1650	static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue mq, struct request req)
1651	{
1652	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1653	struct mmc_host *host = mq->card->host;
1654	int err;
1655
1656	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
1657	mqrq->brq.mrq.done = mmc_blk_hsq_req_done;
1658	mmc_pre_req(host, &mqrq->brq.mrq);
1659
1660	err = mmc_cqe_start_req(host, &mqrq->brq.mrq);
1661	if (err)
1662	mmc_post_req(host, &mqrq->brq.mrq, err);
1663
1664	return err;
1665	}
1666
1667	static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue mq, struct request req)
1668	{
1669	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1670	struct mmc_host *host = mq->card->host;
1671
1672	if (host->hsq_enabled)
1673	return mmc_blk_hsq_issue_rw_rq(mq, req);
1674
1675	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);
1676
1677	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);
1678	}
1679
1680	static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
1681	struct mmc_card *card,
1682	int recovery_mode,
1683	struct mmc_queue *mq)
1684	{
1685	u32 readcmd, writecmd;
1686	struct mmc_blk_request *brq = &mqrq->brq;
1687	struct request *req = mmc_queue_req_to_req(mqrq);
1688	struct mmc_blk_data *md = mq->blkdata;
1689	bool do_rel_wr, do_data_tag;
1690
1691	mmc_blk_data_prep(mq, mqrq, recovery_mode, &do_rel_wr, &do_data_tag);
1692
1693	brq->mrq.cmd = &brq->cmd;
1694
1695	brq->cmd.arg = blk_rq_pos(req);
1696	if (!mmc_card_blockaddr(card))
1697	brq->cmd.arg <<= 9;
1698	brq->cmd.flags = MMC_RSP_SPI_R1 \| MMC_RSP_R1 \| MMC_CMD_ADTC;
1699
1700	if (brq->data.blocks > 1 \|\| do_rel_wr) {
1701	/* SPI multiblock writes terminate using a special
1702	* token, not a STOP_TRANSMISSION request.
1703	*/
1704	if (!mmc_host_is_spi(card->host) \|\|
1705	rq_data_dir(req) == READ)
1706	brq->mrq.stop = &brq->stop;
1707	readcmd = MMC_READ_MULTIPLE_BLOCK;
1708	writecmd = MMC_WRITE_MULTIPLE_BLOCK;
1709	} else {
1710	brq->mrq.stop = NULL;
1711	readcmd = MMC_READ_SINGLE_BLOCK;
1712	writecmd = MMC_WRITE_BLOCK;
1713	}
1714	brq->cmd.opcode = rq_data_dir(req) == READ ? readcmd : writecmd;
1715
1716	/*
1717	* Pre-defined multi-block transfers are preferable to
1718	* open ended-ones (and necessary for reliable writes).
1719	* However, it is not sufficient to just send CMD23,
1720	* and avoid the final CMD12, as on an error condition
1721	* CMD12 (stop) needs to be sent anyway. This, coupled
1722	* with Auto-CMD23 enhancements provided by some
1723	* hosts, means that the complexity of dealing
1724	* with this is best left to the host. If CMD23 is
1725	* supported by card and host, we'll fill sbc in and let
1726	* the host deal with handling it correctly. This means
1727	* that for hosts that don't expose MMC_CAP_CMD23, no
1728	* change of behavior will be observed.
1729	*
1730	* N.B: Some MMC cards experience perf degradation.
1731	* We'll avoid using CMD23-bounded multiblock writes for
1732	* these, while retaining features like reliable writes.
1733	*/
1734	if ((md->flags & MMC_BLK_CMD23) && mmc_op_multi(brq->cmd.opcode) &&
1735	(do_rel_wr \|\| !mmc_card_blk_no_cmd23(card) \|\| do_data_tag)) {
1736	brq->sbc.opcode = MMC_SET_BLOCK_COUNT;
1737	brq->sbc.arg = brq->data.blocks \|
1738	(do_rel_wr ? (1 << 31) : 0) \|
1739	(do_data_tag ? (1 << 29) : 0);
1740	brq->sbc.flags = MMC_RSP_R1 \| MMC_CMD_AC;
1741	brq->mrq.sbc = &brq->sbc;
1742	}
1743
1744	if (mmc_card_ult_capacity(card)) {
1745	brq->cmd.ext_addr = blk_rq_pos(req) >> 32;
1746	brq->cmd.has_ext_addr = true;
1747	}
1748	}
1749
1750	#define MMC_MAX_RETRIES 5
1751	#define MMC_DATA_RETRIES 2
1752	#define MMC_NO_RETRIES (MMC_MAX_RETRIES + 1)
1753
1754	static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout)
1755	{
1756	struct mmc_command cmd = {
1757	.opcode = MMC_STOP_TRANSMISSION,
1758	.flags = MMC_RSP_SPI_R1 \| MMC_RSP_R1 \| MMC_CMD_AC,
1759	/* Some hosts wait for busy anyway, so provide a busy timeout */
1760	.busy_timeout = timeout,
1761	};
1762
1763	return mmc_wait_for_cmd(card->host, &cmd, 5);
1764	}
1765
1766	static int mmc_blk_fix_state(struct mmc_card card, struct request req)
1767	{
1768	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1769	struct mmc_blk_request *brq = &mqrq->brq;
1770	unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data);
1771	int err;
1772
1773	mmc_retune_hold_now(card->host);
1774
1775	mmc_blk_send_stop(card, timeout);
1776
1777	err = mmc_poll_for_busy(card, timeout, false, MMC_BUSY_IO);
1778
1779	mmc_retune_release(card->host);
1780
1781	return err;
1782	}
1783
1784	static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq)
1785	{
1786	return !!brq->mrq.sbc;
1787	}
1788
1789	static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq)
1790	{
1791	return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR;
1792	}
1793
1794	/*
1795	* Check for errors the host controller driver might not have seen such as
1796	* response mode errors or invalid card state.
1797	*/
1798	static bool mmc_blk_status_error(struct request *req, u32 status)
1799	{
1800	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1801	struct mmc_blk_request *brq = &mqrq->brq;
1802	struct mmc_queue *mq = req->q->queuedata;
1803	u32 stop_err_bits;
1804
1805	if (mmc_host_is_spi(mq->card->host))
1806	return false;
1807
1808	stop_err_bits = mmc_blk_stop_err_bits(brq);
1809
1810	return brq->cmd.resp[0] & CMD_ERRORS \|\|
1811	brq->stop.resp[0] & stop_err_bits \|\|
1812	status & stop_err_bits \|\|
1813	(rq_data_dir(req) == WRITE && !mmc_ready_for_data(status));
1814	}
1815
1816	static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq)
1817	{
1818	return !brq->sbc.error && !brq->cmd.error &&
1819	!(brq->cmd.resp[0] & CMD_ERRORS);
1820	}
1821
1822	/*
1823	* Requests are completed by mmc_blk_mq_complete_rq() which sets simple
1824	* policy:
1825	* 1. A request that has transferred at least some data is considered
1826	* successful and will be requeued if there is remaining data to
1827	* transfer.
1828	* 2. Otherwise the number of retries is incremented and the request
1829	* will be requeued if there are remaining retries.
1830	* 3. Otherwise the request will be errored out.
1831	* That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and
1832	* mqrq->retries. So there are only 4 possible actions here:
1833	* 1. do not accept the bytes_xfered value i.e. set it to zero
1834	* 2. change mqrq->retries to determine the number of retries
1835	* 3. try to reset the card
1836	* 4. read one sector at a time
1837	*/
1838	static void mmc_blk_mq_rw_recovery(struct mmc_queue mq, struct request req)
1839	{
1840	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
1841	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1842	struct mmc_blk_request *brq = &mqrq->brq;
1843	struct mmc_blk_data *md = mq->blkdata;
1844	struct mmc_card *card = mq->card;
1845	u32 status;
1846	u32 blocks;
1847	int err;
1848
1849	/*
1850	* Some errors the host driver might not have seen. Set the number of
1851	* bytes transferred to zero in that case.
1852	*/
1853	err = __mmc_send_status(card, &status, 0);
1854	if (err \|\| mmc_blk_status_error(req, status))
1855	brq->data.bytes_xfered = 0;
1856
1857	mmc_retune_release(card->host);
1858
1859	/*
1860	* Try again to get the status. This also provides an opportunity for
1861	* re-tuning.
1862	*/
1863	if (err)
1864	err = __mmc_send_status(card, &status, 0);
1865
1866	/*
1867	* Nothing more to do after the number of bytes transferred has been
1868	* updated and there is no card.
1869	*/
1870	if (err && mmc_detect_card_removed(card->host))
1871	return;
1872
1873	/* Try to get back to "tran" state */
1874	if (!mmc_host_is_spi(mq->card->host) &&
1875	(err \|\| !mmc_ready_for_data(status)))
1876	err = mmc_blk_fix_state(mq->card, req);
1877
1878	/*
1879	* Special case for SD cards where the card might record the number of
1880	* blocks written.
1881	*/
1882	if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) &&
1883	rq_data_dir(req) == WRITE) {
1884	if (mmc_sd_num_wr_blocks(card, &blocks))
1885	brq->data.bytes_xfered = 0;
1886	else
1887	brq->data.bytes_xfered = blocks << 9;
1888	}
1889
1890	/* Reset if the card is in a bad state */
1891	if (!mmc_host_is_spi(mq->card->host) &&
1892	err && mmc_blk_reset(md, card->host, type)) {
1893	pr_err("%s: recovery failed!\n", req->q->disk->disk_name);
1894	mqrq->retries = MMC_NO_RETRIES;
1895	return;
1896	}
1897
1898	/*
1899	* If anything was done, just return and if there is anything remaining
1900	* on the request it will get requeued.
1901	*/
1902	if (brq->data.bytes_xfered)
1903	return;
1904
1905	/* Reset before last retry */
1906	if (mqrq->retries + 1 == MMC_MAX_RETRIES &&
1907	mmc_blk_reset(md, card->host, type))
1908	return;
1909
1910	/* Command errors fail fast, so use all MMC_MAX_RETRIES */
1911	if (brq->sbc.error \|\| brq->cmd.error)
1912	return;
1913
1914	/* Reduce the remaining retries for data errors */
1915	if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) {
1916	mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES;
1917	return;
1918	}
1919	}
1920
1921	static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
1922	{
1923	mmc_blk_eval_resp_error(brq);
1924
1925	return brq->sbc.error \|\| brq->cmd.error \|\| brq->stop.error \|\|
1926	brq->data.error \|\| brq->cmd.resp[0] & CMD_ERRORS;
1927	}
1928
1929	static int mmc_spi_err_check(struct mmc_card *card)
1930	{
1931	u32 status = 0;
1932	int err;
1933
1934	/*
1935	* SPI does not have a TRAN state we have to wait on, instead the
1936	* card is ready again when it no longer holds the line LOW.
1937	* We still have to ensure two things here before we know the write
1938	* was successful:
1939	* 1. The card has not disconnected during busy and we actually read our
1940	* own pull-up, thinking it was still connected, so ensure it
1941	* still responds.
1942	* 2. Check for any error bits, in particular R1_SPI_IDLE to catch a
1943	* just reconnected card after being disconnected during busy.
1944	*/
1945	err = __mmc_send_status(card, &status, 0);
1946	if (err)
1947	return err;
1948	/* All R1 and R2 bits of SPI are errors in our case */
1949	if (status)
1950	return -EIO;
1951	return 0;
1952	}
1953
1954	static int mmc_blk_busy_cb(void cb_data, bool busy)
1955	{
1956	struct mmc_blk_busy_data *data = cb_data;
1957	u32 status = 0;
1958	int err;
1959
1960	err = mmc_send_status(data->card, &status);
1961	if (err)
1962	return err;
1963
1964	/* Accumulate response error bits. */
1965	data->status \|= status;
1966
1967	*busy = !mmc_ready_for_data(status);
1968	return 0;
1969	}
1970
1971	static int mmc_blk_card_busy(struct mmc_card card, struct request req)
1972	{
1973	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
1974	struct mmc_blk_busy_data cb_data;
1975	int err;
1976
1977	if (rq_data_dir(req) == READ)
1978	return 0;
1979
1980	if (mmc_host_is_spi(card->host)) {
1981	err = mmc_spi_err_check(card);
1982	if (err)
1983	mqrq->brq.data.bytes_xfered = 0;
1984	return err;
1985	}
1986
1987	cb_data.card = card;
1988	cb_data.status = 0;
1989	err = __mmc_poll_for_busy(card->host, 0, MMC_BLK_TIMEOUT_MS,
1990	&mmc_blk_busy_cb, &cb_data);
1991
1992	/*
1993	* Do not assume data transferred correctly if there are any error bits
1994	* set.
1995	*/
1996	if (cb_data.status & mmc_blk_stop_err_bits(&mqrq->brq)) {
1997	mqrq->brq.data.bytes_xfered = 0;
1998	err = err ? err : -EIO;
1999	}
2000
2001	/* Copy the exception bit so it will be seen later on */
2002	if (mmc_card_mmc(card) && cb_data.status & R1_EXCEPTION_EVENT)
2003	mqrq->brq.cmd.resp[0] \|= R1_EXCEPTION_EVENT;
2004
2005	return err;
2006	}
2007
2008	static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,
2009	struct request *req)
2010	{
2011	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
2012
2013	mmc_blk_reset_success(mq->blkdata, type);
2014	}
2015
2016	static void mmc_blk_mq_complete_rq(struct mmc_queue mq, struct request req)
2017	{
2018	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2019	unsigned int nr_bytes = mqrq->brq.data.bytes_xfered;
2020
2021	if (nr_bytes) {
2022	if (blk_update_request(req, BLK_STS_OK, nr_bytes))
2023	blk_mq_requeue_request(req, true);
2024	else
2025	__blk_mq_end_request(req, BLK_STS_OK);
2026	} else if (!blk_rq_bytes(req)) {
2027	__blk_mq_end_request(req, BLK_STS_IOERR);
2028	} else if (mqrq->retries++ < MMC_MAX_RETRIES) {
2029	mqrq->flags \|= MQRQ_XFER_SINGLE_BLOCK;
2030	blk_mq_requeue_request(req, true);
2031	} else {
2032	if (mmc_card_removed(mq->card))
2033	req->rq_flags \|= RQF_QUIET;
2034	blk_mq_end_request(req, BLK_STS_IOERR);
2035	}
2036	}
2037
2038	static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,
2039	struct mmc_queue_req *mqrq)
2040	{
2041	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&
2042	(mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT \|\|
2043	mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT);
2044	}
2045
2046	static void mmc_blk_urgent_bkops(struct mmc_queue *mq,
2047	struct mmc_queue_req *mqrq)
2048	{
2049	if (mmc_blk_urgent_bkops_needed(mq, mqrq))
2050	mmc_run_bkops(mq->card);
2051	}
2052
2053	static void mmc_blk_hsq_req_done(struct mmc_request *mrq)
2054	{
2055	struct mmc_queue_req *mqrq =
2056	container_of(mrq, struct mmc_queue_req, brq.mrq);
2057	struct request *req = mmc_queue_req_to_req(mqrq);
2058	struct request_queue *q = req->q;
2059	struct mmc_queue *mq = q->queuedata;
2060	struct mmc_host *host = mq->card->host;
2061	unsigned long flags;
2062
2063	if (mmc_blk_rq_error(&mqrq->brq) \|\|
2064	mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2065	spin_lock_irqsave(&mq->lock, flags);
2066	mq->recovery_needed = true;
2067	mq->recovery_req = req;
2068	spin_unlock_irqrestore(&mq->lock, flags);
2069
2070	host->cqe_ops->cqe_recovery_start(host);
2071
2072	schedule_work(&mq->recovery_work);
2073	return;
2074	}
2075
2076	mmc_blk_rw_reset_success(mq, req);
2077
2078	/*
2079	* Block layer timeouts race with completions which means the normal
2080	* completion path cannot be used during recovery.
2081	*/
2082	if (mq->in_recovery)
2083	mmc_blk_cqe_complete_rq(mq, req);
2084	else if (likely(!blk_should_fake_timeout(req->q)))
2085	blk_mq_complete_request(req);
2086	}
2087
2088	void mmc_blk_mq_complete(struct request *req)
2089	{
2090	struct mmc_queue *mq = req->q->queuedata;
2091	struct mmc_host *host = mq->card->host;
2092
2093	if (host->cqe_enabled)
2094	mmc_blk_cqe_complete_rq(mq, req);
2095	else if (likely(!blk_should_fake_timeout(req->q)))
2096	mmc_blk_mq_complete_rq(mq, req);
2097	}
2098
2099	static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,
2100	struct request *req)
2101	{
2102	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2103	struct mmc_host *host = mq->card->host;
2104
2105	if (mmc_blk_rq_error(&mqrq->brq) \|\|
2106	mmc_blk_card_busy(mq->card, req)) {
2107	mmc_blk_mq_rw_recovery(mq, req);
2108	} else {
2109	mmc_blk_rw_reset_success(mq, req);
2110	mmc_retune_release(host);
2111	}
2112
2113	mmc_blk_urgent_bkops(mq, mqrq);
2114	}
2115
2116	static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, enum mmc_issue_type issue_type)
2117	{
2118	unsigned long flags;
2119	bool put_card;
2120
2121	spin_lock_irqsave(&mq->lock, flags);
2122
2123	mq->in_flight[issue_type] -= 1;
2124
2125	put_card = (mmc_tot_in_flight(mq) == 0);
2126
2127	spin_unlock_irqrestore(&mq->lock, flags);
2128
2129	if (put_card)
2130	mmc_put_card(mq->card, &mq->ctx);
2131	}
2132
2133	static void mmc_blk_mq_post_req(struct mmc_queue mq, struct request req,
2134	bool can_sleep)
2135	{
2136	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
2137	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2138	struct mmc_request *mrq = &mqrq->brq.mrq;
2139	struct mmc_host *host = mq->card->host;
2140
2141	mmc_post_req(host, mrq, 0);
2142
2143	/*
2144	* Block layer timeouts race with completions which means the normal
2145	* completion path cannot be used during recovery.
2146	*/
2147	if (mq->in_recovery) {
2148	mmc_blk_mq_complete_rq(mq, req);
2149	} else if (likely(!blk_should_fake_timeout(req->q))) {
2150	if (can_sleep)
2151	blk_mq_complete_request_direct(req, mmc_blk_mq_complete);
2152	else
2153	blk_mq_complete_request(req);
2154	}
2155
2156	mmc_blk_mq_dec_in_flight(mq, issue_type);
2157	}
2158
2159	void mmc_blk_mq_recovery(struct mmc_queue *mq)
2160	{
2161	struct request *req = mq->recovery_req;
2162	struct mmc_host *host = mq->card->host;
2163	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2164
2165	mq->recovery_req = NULL;
2166	mq->rw_wait = false;
2167
2168	if (mmc_blk_rq_error(&mqrq->brq)) {
2169	mmc_retune_hold_now(host);
2170	mmc_blk_mq_rw_recovery(mq, req);
2171	}
2172
2173	mmc_blk_urgent_bkops(mq, mqrq);
2174
2175	mmc_blk_mq_post_req(mq, req, true);
2176	}
2177
2178	static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,
2179	struct request **prev_req)
2180	{
2181	if (mmc_host_can_done_complete(mq->card->host))
2182	return;
2183
2184	mutex_lock(&mq->complete_lock);
2185
2186	if (!mq->complete_req)
2187	goto out_unlock;
2188
2189	mmc_blk_mq_poll_completion(mq, mq->complete_req);
2190
2191	if (prev_req)
2192	*prev_req = mq->complete_req;
2193	else
2194	mmc_blk_mq_post_req(mq, mq->complete_req, true);
2195
2196	mq->complete_req = NULL;
2197
2198	out_unlock:
2199	mutex_unlock(&mq->complete_lock);
2200	}
2201
2202	void mmc_blk_mq_complete_work(struct work_struct *work)
2203	{
2204	struct mmc_queue *mq = container_of(work, struct mmc_queue,
2205	complete_work);
2206
2207	mmc_blk_mq_complete_prev_req(mq, NULL);
2208	}
2209
2210	static void mmc_blk_mq_req_done(struct mmc_request *mrq)
2211	{
2212	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
2213	brq.mrq);
2214	struct request *req = mmc_queue_req_to_req(mqrq);
2215	struct request_queue *q = req->q;
2216	struct mmc_queue *mq = q->queuedata;
2217	struct mmc_host *host = mq->card->host;
2218	unsigned long flags;
2219
2220	if (!mmc_host_can_done_complete(host)) {
2221	bool waiting;
2222
2223	/*
2224	* We cannot complete the request in this context, so record
2225	* that there is a request to complete, and that a following
2226	* request does not need to wait (although it does need to
2227	* complete complete_req first).
2228	*/
2229	spin_lock_irqsave(&mq->lock, flags);
2230	mq->complete_req = req;
2231	mq->rw_wait = false;
2232	waiting = mq->waiting;
2233	spin_unlock_irqrestore(&mq->lock, flags);
2234
2235	/*
2236	* If 'waiting' then the waiting task will complete this
2237	* request, otherwise queue a work to do it. Note that
2238	* complete_work may still race with the dispatch of a following
2239	* request.
2240	*/
2241	if (waiting)
2242	wake_up(&mq->wait);
2243	else
2244	queue_work(mq->card->complete_wq, &mq->complete_work);
2245
2246	return;
2247	}
2248
2249	/* Take the recovery path for errors or urgent background operations */
2250	if (mmc_blk_rq_error(&mqrq->brq) \|\|
2251	mmc_blk_urgent_bkops_needed(mq, mqrq)) {
2252	spin_lock_irqsave(&mq->lock, flags);
2253	mq->recovery_needed = true;
2254	mq->recovery_req = req;
2255	spin_unlock_irqrestore(&mq->lock, flags);
2256	wake_up(&mq->wait);
2257	schedule_work(&mq->recovery_work);
2258	return;
2259	}
2260
2261	mmc_blk_rw_reset_success(mq, req);
2262
2263	mq->rw_wait = false;
2264	wake_up(&mq->wait);
2265
2266	/* context unknown */
2267	mmc_blk_mq_post_req(mq, req, false);
2268	}
2269
2270	static bool mmc_blk_rw_wait_cond(struct mmc_queue mq, int err)
2271	{
2272	unsigned long flags;
2273	bool done;
2274
2275	/*
2276	* Wait while there is another request in progress, but not if recovery
2277	* is needed. Also indicate whether there is a request waiting to start.
2278	*/
2279	spin_lock_irqsave(&mq->lock, flags);
2280	if (mq->recovery_needed) {
2281	*err = -EBUSY;
2282	done = true;
2283	} else {
2284	done = !mq->rw_wait;
2285	}
2286	mq->waiting = !done;
2287	spin_unlock_irqrestore(&mq->lock, flags);
2288
2289	return done;
2290	}
2291
2292	static int mmc_blk_rw_wait(struct mmc_queue mq, struct request *prev_req)
2293	{
2294	int err = 0;
2295
2296	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));
2297
2298	/* Always complete the previous request if there is one */
2299	mmc_blk_mq_complete_prev_req(mq, prev_req);
2300
2301	return err;
2302	}
2303
2304	static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,
2305	struct request *req)
2306	{
2307	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
2308	struct mmc_host *host = mq->card->host;
2309	struct request *prev_req = NULL;
2310	int err = 0;
2311
2312	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
2313
2314	mqrq->brq.mrq.done = mmc_blk_mq_req_done;
2315
2316	mmc_pre_req(host, &mqrq->brq.mrq);
2317
2318	err = mmc_blk_rw_wait(mq, &prev_req);
2319	if (err)
2320	goto out_post_req;
2321
2322	mq->rw_wait = true;
2323
2324	err = mmc_start_request(host, &mqrq->brq.mrq);
2325
2326	if (prev_req)
2327	mmc_blk_mq_post_req(mq, prev_req, true);
2328
2329	if (err)
2330	mq->rw_wait = false;
2331
2332	/* Release re-tuning here where there is no synchronization required */
2333	if (err \|\| mmc_host_can_done_complete(host))
2334	mmc_retune_release(host);
2335
2336	out_post_req:
2337	if (err)
2338	mmc_post_req(host, &mqrq->brq.mrq, err);
2339
2340	return err;
2341	}
2342
2343	static int mmc_blk_wait_for_idle(struct mmc_queue mq, struct mmc_host host)
2344	{
2345	if (host->cqe_enabled)
2346	return host->cqe_ops->cqe_wait_for_idle(host);
2347
2348	return mmc_blk_rw_wait(mq, NULL);
2349	}
2350
2351	enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue mq, struct request req)
2352	{
2353	struct mmc_blk_data *md = mq->blkdata;
2354	struct mmc_card *card = md->queue.card;
2355	struct mmc_host *host = card->host;
2356	int ret;
2357
2358	ret = mmc_blk_part_switch(card, md->part_type);
2359	if (ret)
2360	return MMC_REQ_FAILED_TO_START;
2361
2362	switch (mmc_issue_type(mq, req)) {
2363	case MMC_ISSUE_SYNC:
2364	ret = mmc_blk_wait_for_idle(mq, host);
2365	if (ret)
2366	return MMC_REQ_BUSY;
2367	switch (req_op(req)) {
2368	case REQ_OP_DRV_IN:
2369	case REQ_OP_DRV_OUT:
2370	mmc_blk_issue_drv_op(mq, req);
2371	break;
2372	case REQ_OP_DISCARD:
2373	mmc_blk_issue_discard_rq(mq, req);
2374	break;
2375	case REQ_OP_SECURE_ERASE:
2376	mmc_blk_issue_secdiscard_rq(mq, req);
2377	break;
2378	case REQ_OP_WRITE_ZEROES:
2379	mmc_blk_issue_trim_rq(mq, req);
2380	break;
2381	case REQ_OP_FLUSH:
2382	mmc_blk_issue_flush(mq, req);
2383	break;
2384	default:
2385	WARN_ON_ONCE(1);
2386	return MMC_REQ_FAILED_TO_START;
2387	}
2388	return MMC_REQ_FINISHED;
2389	case MMC_ISSUE_DCMD:
2390	case MMC_ISSUE_ASYNC:
2391	switch (req_op(req)) {
2392	case REQ_OP_FLUSH:
2393	if (!mmc_cache_enabled(host)) {
2394	blk_mq_end_request(req, BLK_STS_OK);
2395	return MMC_REQ_FINISHED;
2396	}
2397	ret = mmc_blk_cqe_issue_flush(mq, req);
2398	break;
2399	case REQ_OP_WRITE:
2400	card->written_flag = true;
2401	fallthrough;
2402	case REQ_OP_READ:
2403	if (host->cqe_enabled)
2404	ret = mmc_blk_cqe_issue_rw_rq(mq, req);
2405	else
2406	ret = mmc_blk_mq_issue_rw_rq(mq, req);
2407	break;
2408	default:
2409	WARN_ON_ONCE(1);
2410	ret = -EINVAL;
2411	}
2412	if (!ret)
2413	return MMC_REQ_STARTED;
2414	return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
2415	default:
2416	WARN_ON_ONCE(1);
2417	return MMC_REQ_FAILED_TO_START;
2418	}
2419	}
2420
2421	static inline int mmc_blk_readonly(struct mmc_card *card)
2422	{
2423	return mmc_card_readonly(card) \|\|
2424	!(card->csd.cmdclass & CCC_BLOCK_WRITE);
2425	}
2426
2427	/*
2428	* Search for a declared partitions node for the disk in mmc-card related node.
2429	*
2430	* This is to permit support for partition table defined in DT in special case
2431	* where a partition table is not written in the disk and is expected to be
2432	* passed from the running system.
2433	*
2434	* For the user disk, "partitions" node is searched.
2435	* For the special HW disk, "partitions-" node with the appended name is used
2436	* following this conversion table (to adhere to JEDEC naming)
2437	* - boot0 -> partitions-boot1
2438	* - boot1 -> partitions-boot2
2439	* - gp0 -> partitions-gp1
2440	* - gp1 -> partitions-gp2
2441	* - gp2 -> partitions-gp3
2442	* - gp3 -> partitions-gp4
2443	*/
2444	static struct fwnode_handle mmc_blk_get_partitions_node(struct device mmc_dev,
2445	const char *subname)
2446	{
2447	const char *node_name = "partitions";
2448
2449	if (subname) {
2450	mmc_dev = mmc_dev->parent;
2451
2452	/*
2453	* Check if we are allocating a BOOT disk boot0/1 disk.
2454	* In DT we use the JEDEC naming boot1/2.
2455	*/
2456	if (!strcmp(subname, "boot0"))
2457	node_name = "partitions-boot1";
2458	if (!strcmp(subname, "boot1"))
2459	node_name = "partitions-boot2";
2460	/*
2461	* Check if we are allocating a GP disk gp0/1/2/3 disk.
2462	* In DT we use the JEDEC naming gp1/2/3/4.
2463	*/
2464	if (!strcmp(subname, "gp0"))
2465	node_name = "partitions-gp1";
2466	if (!strcmp(subname, "gp1"))
2467	node_name = "partitions-gp2";
2468	if (!strcmp(subname, "gp2"))
2469	node_name = "partitions-gp3";
2470	if (!strcmp(subname, "gp3"))
2471	node_name = "partitions-gp4";
2472	}
2473
2474	return device_get_named_child_node(mmc_dev, node_name);
2475	}
2476
2477	static struct mmc_blk_data mmc_blk_alloc_req(struct mmc_card card,
2478	struct device *parent,
2479	sector_t size,
2480	bool default_ro,
2481	const char *subname,
2482	int area_type,
2483	unsigned int part_type)
2484	{
2485	struct fwnode_handle *disk_fwnode;
2486	struct mmc_blk_data *md;
2487	int devidx, ret;
2488	char cap_str[10];
2489	unsigned int features = 0;
2490
2491	devidx = ida_alloc_max(&mmc_blk_ida, max_devices - 1, GFP_KERNEL);
2492	if (devidx < 0) {
2493	/*
2494	* We get -ENOSPC because there are no more any available
2495	* devidx. The reason may be that, either userspace haven't yet
2496	* unmounted the partitions, which postpones mmc_blk_release()
2497	* from being called, or the device has more partitions than
2498	* what we support.
2499	*/
2500	if (devidx == -ENOSPC)
2501	dev_err(mmc_dev(card->host),
2502	"no more device IDs available\n");
2503
2504	return ERR_PTR(devidx);
2505	}
2506
2507	md = kzalloc_obj(*md);
2508	if (!md) {
2509	ret = -ENOMEM;
2510	goto out;
2511	}
2512
2513	md->area_type = area_type;
2514
2515	/*
2516	* Set the read-only status based on the supported commands
2517	* and the write protect switch.
2518	*/
2519	md->read_only = mmc_blk_readonly(card);
2520
2521	if (mmc_host_can_cmd23(card->host) && mmc_card_can_cmd23(card))
2522	md->flags \|= MMC_BLK_CMD23;
2523
2524	if (md->flags & MMC_BLK_CMD23 &&
2525	((card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN) \|\|
2526	card->ext_csd.rel_sectors)) {
2527	md->flags \|= MMC_BLK_REL_WR;
2528	features \|= (BLK_FEAT_WRITE_CACHE \| BLK_FEAT_FUA);
2529	} else if (mmc_cache_enabled(card->host)) {
2530	features \|= BLK_FEAT_WRITE_CACHE;
2531	}
2532
2533	md->disk = mmc_init_queue(&md->queue, card, features);
2534	if (IS_ERR(md->disk)) {
2535	ret = PTR_ERR(md->disk);
2536	goto err_kfree;
2537	}
2538
2539	INIT_LIST_HEAD(&md->part);
2540	INIT_LIST_HEAD(&md->rpmbs);
2541	kref_init(&md->kref);
2542
2543	md->queue.blkdata = md;
2544	md->part_type = part_type;
2545
2546	md->disk->major = MMC_BLOCK_MAJOR;
2547	md->disk->minors = perdev_minors;
2548	md->disk->first_minor = devidx * perdev_minors;
2549	md->disk->fops = &mmc_bdops;
2550	md->disk->private_data = md;
2551	md->parent = parent;
2552	set_disk_ro(md->disk, md->read_only \|\| default_ro);
2553	if (area_type & MMC_BLK_DATA_AREA_RPMB)
2554	md->disk->flags \|= GENHD_FL_NO_PART;
2555
2556	/*
2557	* As discussed on lkml, GENHD_FL_REMOVABLE should:
2558	*
2559	* - be set for removable media with permanent block devices
2560	* - be unset for removable block devices with permanent media
2561	*
2562	* Since MMC block devices clearly fall under the second
2563	* case, we do not set GENHD_FL_REMOVABLE. Userspace
2564	* should use the block device creation/destruction hotplug
2565	* messages to tell when the card is present.
2566	*/
2567
2568	snprintf(md->disk->disk_name, sizeof(md->disk->disk_name),
2569	"mmcblk%u%s", card->host->index, subname ? subname : "");
2570
2571	set_capacity(md->disk, size);
2572
2573	string_get_size((u64)size, 512, STRING_UNITS_2,
2574	cap_str, sizeof(cap_str));
2575	pr_info("%s: %s %s %s%s\n",
2576	md->disk->disk_name, mmc_card_id(card), mmc_card_name(card),
2577	cap_str, md->read_only ? " (ro)" : "");
2578
2579	/* used in ->open, must be set before add_disk: */
2580	if (area_type == MMC_BLK_DATA_AREA_MAIN)
2581	dev_set_drvdata(&card->dev, md);
2582	disk_fwnode = mmc_blk_get_partitions_node(parent, subname);
2583	ret = add_disk_fwnode(md->parent, md->disk, mmc_disk_attr_groups,
2584	disk_fwnode);
2585	if (ret)
2586	goto err_put_disk;
2587	return md;
2588
2589	err_put_disk:
2590	put_disk(md->disk);
2591	blk_mq_free_tag_set(&md->queue.tag_set);
2592	err_kfree:
2593	kfree(md);
2594	out:
2595	ida_free(&mmc_blk_ida, devidx);
2596	return ERR_PTR(ret);
2597	}
2598
2599	static struct mmc_blk_data mmc_blk_alloc(struct mmc_card card)
2600	{
2601	sector_t size;
2602
2603	if (!mmc_card_sd(card) && mmc_card_blockaddr(card)) {
2604	/*
2605	* The EXT_CSD sector count is in number or 512 byte
2606	* sectors.
2607	*/
2608	size = card->ext_csd.sectors;
2609	} else {
2610	/*
2611	* The CSD capacity field is in units of read_blkbits.
2612	* set_capacity takes units of 512 bytes.
2613	*/
2614	size = (typeof(sector_t))card->csd.capacity
2615	<< (card->csd.read_blkbits - 9);
2616	}
2617
2618	return mmc_blk_alloc_req(card, &card->dev, size, false, NULL,
2619	MMC_BLK_DATA_AREA_MAIN, 0);
2620	}
2621
2622	static int mmc_blk_alloc_part(struct mmc_card *card,
2623	struct mmc_blk_data *md,
2624	unsigned int part_type,
2625	sector_t size,
2626	bool default_ro,
2627	const char *subname,
2628	int area_type)
2629	{
2630	struct mmc_blk_data *part_md;
2631
2632	part_md = mmc_blk_alloc_req(card, disk_to_dev(md->disk), size, default_ro,
2633	subname, area_type, part_type);
2634	if (IS_ERR(part_md))
2635	return PTR_ERR(part_md);
2636	list_add(&part_md->part, &md->part);
2637
2638	return 0;
2639	}
2640
2641	/**
2642	* mmc_rpmb_ioctl() - ioctl handler for the RPMB chardev
2643	* @filp: the character device file
2644	* @cmd: the ioctl() command
2645	* @arg: the argument from userspace
2646	*
2647	* This will essentially just redirect the ioctl()s coming in over to
2648	* the main block device spawning the RPMB character device.
2649	*/
2650	static long mmc_rpmb_ioctl(struct file *filp, unsigned int cmd,
2651	unsigned long arg)
2652	{
2653	struct mmc_rpmb_data *rpmb = filp->private_data;
2654	int ret;
2655
2656	switch (cmd) {
2657	case MMC_IOC_CMD:
2658	ret = mmc_blk_ioctl_cmd(rpmb->md,
2659	(struct mmc_ioc_cmd __user *)arg,
2660	rpmb);
2661	break;
2662	case MMC_IOC_MULTI_CMD:
2663	ret = mmc_blk_ioctl_multi_cmd(rpmb->md,
2664	(struct mmc_ioc_multi_cmd __user *)arg,
2665	rpmb);
2666	break;
2667	default:
2668	ret = -EINVAL;
2669	break;
2670	}
2671
2672	return ret;
2673	}
2674
2675	#ifdef CONFIG_COMPAT
2676	static long mmc_rpmb_ioctl_compat(struct file *filp, unsigned int cmd,
2677	unsigned long arg)
2678	{
2679	return mmc_rpmb_ioctl(filp, cmd, (unsigned long)compat_ptr(arg));
2680	}
2681	#endif
2682
2683	static int mmc_rpmb_chrdev_open(struct inode inode, struct file filp)
2684	{
2685	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2686	struct mmc_rpmb_data, chrdev);
2687
2688	get_device(&rpmb->dev);
2689	filp->private_data = rpmb;
2690
2691	return nonseekable_open(inode, filp);
2692	}
2693
2694	static int mmc_rpmb_chrdev_release(struct inode inode, struct file filp)
2695	{
2696	struct mmc_rpmb_data *rpmb = container_of(inode->i_cdev,
2697	struct mmc_rpmb_data, chrdev);
2698
2699	put_device(&rpmb->dev);
2700
2701	return 0;
2702	}
2703
2704	static const struct file_operations mmc_rpmb_fileops = {
2705	.release = mmc_rpmb_chrdev_release,
2706	.open = mmc_rpmb_chrdev_open,
2707	.owner = THIS_MODULE,
2708	.unlocked_ioctl = mmc_rpmb_ioctl,
2709	#ifdef CONFIG_COMPAT
2710	.compat_ioctl = mmc_rpmb_ioctl_compat,
2711	#endif
2712	};
2713
2714	static void mmc_blk_rpmb_device_release(struct device *dev)
2715	{
2716	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2717
2718	mmc_blk_put(rpmb->md);
2719	ida_free(&mmc_rpmb_ida, rpmb->id);
2720	kfree(rpmb);
2721	}
2722
2723	static void free_idata(struct mmc_blk_ioc_data **idata, unsigned int cmd_count)
2724	{
2725	unsigned int n;
2726
2727	for (n = 0; n < cmd_count; n++)
2728	kfree(idata[n]);
2729	kfree(idata);
2730	}
2731
2732	static struct mmc_blk_ioc_data *alloc_idata(struct mmc_rpmb_data rpmb,
2733	unsigned int cmd_count)
2734	{
2735	struct mmc_blk_ioc_data **idata;
2736	unsigned int n;
2737
2738	idata = kzalloc_objs(*idata, cmd_count);
2739	if (!idata)
2740	return NULL;
2741
2742	for (n = 0; n < cmd_count; n++) {
2743	idata[n] = kzalloc_objs(**idata, 1);
2744	if (!idata[n]) {
2745	free_idata(idata, n);
2746	return NULL;
2747	}
2748	idata[n]->rpmb = rpmb;
2749	}
2750
2751	return idata;
2752	}
2753
2754	static void set_idata(struct mmc_blk_ioc_data *idata, u32 opcode,
2755	int write_flag, u8 *buf, unsigned int buf_bytes)
2756	{
2757	/*
2758	* The size of an RPMB frame must match what's expected by the
2759	* hardware.
2760	*/
2761	static_assert(!CHECK_SIZE_NEQ(512), "RPMB frame size must be 512 bytes");
2762
2763	idata->ic.opcode = opcode;
2764	idata->ic.flags = MMC_RSP_R1 \| MMC_CMD_ADTC;
2765	idata->ic.write_flag = write_flag;
2766	idata->ic.blksz = RPMB_FRAME_SIZE;
2767	idata->ic.blocks = buf_bytes / idata->ic.blksz;
2768	idata->buf = buf;
2769	idata->buf_bytes = buf_bytes;
2770	}
2771
2772	static int mmc_route_rpmb_frames(struct device dev, u8 req,
2773	unsigned int req_len, u8 *resp,
2774	unsigned int resp_len)
2775	{
2776	struct rpmb_frame frm = (struct rpmb_frame )req;
2777	struct mmc_rpmb_data *rpmb = dev_get_drvdata(dev);
2778	struct mmc_blk_data *md = rpmb->md;
2779	struct mmc_blk_ioc_data **idata;
2780	struct mmc_queue_req *mq_rq;
2781	unsigned int cmd_count;
2782	struct request *rq;
2783	u16 req_type;
2784	bool write;
2785	int ret;
2786
2787	if (IS_ERR(md->queue.card))
2788	return PTR_ERR(md->queue.card);
2789
2790	if (req_len < RPMB_FRAME_SIZE)
2791	return -EINVAL;
2792
2793	req_type = be16_to_cpu(frm->req_resp);
2794	switch (req_type) {
2795	case RPMB_PROGRAM_KEY:
2796	if (CHECK_SIZE_NEQ(req_len) \|\| CHECK_SIZE_NEQ(resp_len))
2797	return -EINVAL;
2798	write = true;
2799	break;
2800	case RPMB_GET_WRITE_COUNTER:
2801	if (CHECK_SIZE_NEQ(req_len) \|\| CHECK_SIZE_NEQ(resp_len))
2802	return -EINVAL;
2803	write = false;
2804	break;
2805	case RPMB_WRITE_DATA:
2806	if (!CHECK_SIZE_ALIGNED(req_len) \|\| CHECK_SIZE_NEQ(resp_len))
2807	return -EINVAL;
2808	write = true;
2809	break;
2810	case RPMB_READ_DATA:
2811	if (CHECK_SIZE_NEQ(req_len) \|\| !CHECK_SIZE_ALIGNED(resp_len))
2812	return -EINVAL;
2813	write = false;
2814	break;
2815	default:
2816	return -EINVAL;
2817	}
2818
2819	/* Write operations require 3 commands, read operations require 2 */
2820	cmd_count = write ? 3 : 2;
2821
2822	idata = alloc_idata(rpmb, cmd_count);
2823	if (!idata)
2824	return -ENOMEM;
2825
2826	if (write) {
2827	struct rpmb_frame resp_frm = (struct rpmb_frame )resp;
2828
2829	/* Send write request frame(s) */
2830	set_idata(idata[0], MMC_WRITE_MULTIPLE_BLOCK,
2831	1 \| MMC_CMD23_ARG_REL_WR, req, req_len);
2832
2833	/* Send result request frame */
2834	memset(resp_frm, 0, RPMB_FRAME_SIZE);
2835	resp_frm->req_resp = cpu_to_be16(RPMB_RESULT_READ);
2836	set_idata(idata[1], MMC_WRITE_MULTIPLE_BLOCK, 1, resp,
2837	resp_len);
2838
2839	/* Read response frame */
2840	set_idata(idata[2], MMC_READ_MULTIPLE_BLOCK, 0, resp, resp_len);
2841	} else {
2842	/* Send write request frame(s) */
2843	set_idata(idata[0], MMC_WRITE_MULTIPLE_BLOCK, 1, req, req_len);
2844
2845	/* Read response frame */
2846	set_idata(idata[1], MMC_READ_MULTIPLE_BLOCK, 0, resp, resp_len);
2847	}
2848
2849	rq = blk_mq_alloc_request(md->queue.queue, REQ_OP_DRV_OUT, 0);
2850	if (IS_ERR(rq)) {
2851	ret = PTR_ERR(rq);
2852	goto out;
2853	}
2854
2855	mq_rq = req_to_mmc_queue_req(rq);
2856	mq_rq->drv_op = MMC_DRV_OP_IOCTL_RPMB;
2857	mq_rq->drv_op_result = -EIO;
2858	mq_rq->drv_op_data = idata;
2859	mq_rq->ioc_count = cmd_count;
2860	blk_execute_rq(rq, false);
2861	ret = req_to_mmc_queue_req(rq)->drv_op_result;
2862
2863	blk_mq_free_request(rq);
2864
2865	out:
2866	free_idata(idata, cmd_count);
2867	return ret;
2868	}
2869
2870	static int mmc_blk_alloc_rpmb_part(struct mmc_card *card,
2871	struct mmc_blk_data *md,
2872	unsigned int part_index,
2873	sector_t size,
2874	const char *subname)
2875	{
2876	int devidx, ret;
2877	char rpmb_name[DISK_NAME_LEN];
2878	char cap_str[10];
2879	struct mmc_rpmb_data *rpmb;
2880
2881	/* This creates the minor number for the RPMB char device */
2882	devidx = ida_alloc_max(&mmc_rpmb_ida, max_devices - 1, GFP_KERNEL);
2883	if (devidx < 0)
2884	return devidx;
2885
2886	rpmb = kzalloc_obj(*rpmb);
2887	if (!rpmb) {
2888	ida_free(&mmc_rpmb_ida, devidx);
2889	return -ENOMEM;
2890	}
2891
2892	snprintf(rpmb_name, sizeof(rpmb_name),
2893	"mmcblk%u%s", card->host->index, subname ? subname : "");
2894
2895	rpmb->id = devidx;
2896	rpmb->part_index = part_index;
2897	rpmb->dev.init_name = rpmb_name;
2898	rpmb->dev.bus = &mmc_rpmb_bus_type;
2899	rpmb->dev.devt = MKDEV(MAJOR(mmc_rpmb_devt), rpmb->id);
2900	rpmb->dev.parent = &card->dev;
2901	rpmb->dev.release = mmc_blk_rpmb_device_release;
2902	device_initialize(&rpmb->dev);
2903	dev_set_drvdata(&rpmb->dev, rpmb);
2904	mmc_blk_get(md->disk);
2905	rpmb->md = md;
2906
2907	cdev_init(&rpmb->chrdev, &mmc_rpmb_fileops);
2908	rpmb->chrdev.owner = THIS_MODULE;
2909	ret = cdev_device_add(&rpmb->chrdev, &rpmb->dev);
2910	if (ret) {
2911	pr_err("%s: could not add character device\n", rpmb_name);
2912	goto out_put_device;
2913	}
2914
2915	list_add(&rpmb->node, &md->rpmbs);
2916
2917	string_get_size((u64)size, 512, STRING_UNITS_2,
2918	cap_str, sizeof(cap_str));
2919
2920	pr_info("%s: %s %s %s, chardev (%d:%d)\n",
2921	rpmb_name, mmc_card_id(card), mmc_card_name(card), cap_str,
2922	MAJOR(mmc_rpmb_devt), rpmb->id);
2923
2924	return 0;
2925
2926	out_put_device:
2927	put_device(&rpmb->dev);
2928	return ret;
2929	}
2930
2931	static void mmc_blk_remove_rpmb_part(struct mmc_rpmb_data *rpmb)
2932	{
2933	rpmb_dev_unregister(rpmb->rdev);
2934	cdev_device_del(&rpmb->chrdev, &rpmb->dev);
2935	put_device(&rpmb->dev);
2936	}
2937
2938	/* MMC Physical partitions consist of two boot partitions and
2939	* up to four general purpose partitions.
2940	* For each partition enabled in EXT_CSD a block device will be allocatedi
2941	* to provide access to the partition.
2942	*/
2943
2944	static int mmc_blk_alloc_parts(struct mmc_card card, struct mmc_blk_data md)
2945	{
2946	int idx, ret;
2947
2948	if (!mmc_card_mmc(card))
2949	return 0;
2950
2951	for (idx = 0; idx < card->nr_parts; idx++) {
2952	if (card->part[idx].area_type & MMC_BLK_DATA_AREA_RPMB) {
2953	/*
2954	* RPMB partitions does not provide block access, they
2955	* are only accessed using ioctl():s. Thus create
2956	* special RPMB block devices that do not have a
2957	* backing block queue for these.
2958	*/
2959	ret = mmc_blk_alloc_rpmb_part(card, md,
2960	card->part[idx].part_cfg,
2961	card->part[idx].size >> SECTOR_SHIFT,
2962	card->part[idx].name);
2963	if (ret)
2964	return ret;
2965	} else if (card->part[idx].size) {
2966	ret = mmc_blk_alloc_part(card, md,
2967	card->part[idx].part_cfg,
2968	card->part[idx].size >> SECTOR_SHIFT,
2969	card->part[idx].force_ro,
2970	card->part[idx].name,
2971	card->part[idx].area_type);
2972	if (ret)
2973	return ret;
2974	}
2975	}
2976
2977	return 0;
2978	}
2979
2980	static void mmc_blk_remove_req(struct mmc_blk_data *md)
2981	{
2982	/*
2983	* Flush remaining requests and free queues. It is freeing the queue
2984	* that stops new requests from being accepted.
2985	*/
2986	del_gendisk(md->disk);
2987	mmc_cleanup_queue(&md->queue);
2988	mmc_blk_put(md);
2989	}
2990
2991	static void mmc_blk_remove_parts(struct mmc_card *card,
2992	struct mmc_blk_data *md)
2993	{
2994	struct list_head pos, q;
2995	struct mmc_blk_data *part_md;
2996	struct mmc_rpmb_data *rpmb;
2997
2998	/* Remove RPMB partitions */
2999	list_for_each_safe(pos, q, &md->rpmbs) {
3000	rpmb = list_entry(pos, struct mmc_rpmb_data, node);
3001	list_del(pos);
3002	mmc_blk_remove_rpmb_part(rpmb);
3003	}
3004	/* Remove block partitions */
3005	list_for_each_safe(pos, q, &md->part) {
3006	part_md = list_entry(pos, struct mmc_blk_data, part);
3007	list_del(pos);
3008	mmc_blk_remove_req(part_md);
3009	}
3010	}
3011
3012	#ifdef CONFIG_DEBUG_FS
3013
3014	static int mmc_dbg_card_status_get(void data, u64 val)
3015	{
3016	struct mmc_card *card = data;
3017	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3018	struct mmc_queue *mq = &md->queue;
3019	struct request *req;
3020	int ret;
3021
3022	/* Ask the block layer about the card status */
3023	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
3024	if (IS_ERR(req))
3025	return PTR_ERR(req);
3026	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_CARD_STATUS;
3027	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
3028	blk_execute_rq(req, false);
3029	ret = req_to_mmc_queue_req(req)->drv_op_result;
3030	if (ret >= 0) {
3031	*val = ret;
3032	ret = 0;
3033	}
3034	blk_mq_free_request(req);
3035
3036	return ret;
3037	}
3038	DEFINE_DEBUGFS_ATTRIBUTE(mmc_dbg_card_status_fops, mmc_dbg_card_status_get,
3039	NULL, "%08llx\n");
3040
3041	/* That is two digits * 512 + 1 for newline */
3042	#define EXT_CSD_STR_LEN 1025
3043
3044	static int mmc_ext_csd_open(struct inode inode, struct file filp)
3045	{
3046	struct mmc_card *card = inode->i_private;
3047	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3048	struct mmc_queue *mq = &md->queue;
3049	struct request *req;
3050	char *buf;
3051	ssize_t n = 0;
3052	u8 *ext_csd;
3053	int err, i;
3054
3055	buf = kmalloc(EXT_CSD_STR_LEN + 1, GFP_KERNEL);
3056	if (!buf)
3057	return -ENOMEM;
3058
3059	/* Ask the block layer for the EXT CSD */
3060	req = blk_mq_alloc_request(mq->queue, REQ_OP_DRV_IN, 0);
3061	if (IS_ERR(req)) {
3062	err = PTR_ERR(req);
3063	goto out_free;
3064	}
3065	req_to_mmc_queue_req(req)->drv_op = MMC_DRV_OP_GET_EXT_CSD;
3066	req_to_mmc_queue_req(req)->drv_op_result = -EIO;
3067	req_to_mmc_queue_req(req)->drv_op_data = &ext_csd;
3068	blk_execute_rq(req, false);
3069	err = req_to_mmc_queue_req(req)->drv_op_result;
3070	blk_mq_free_request(req);
3071	if (err) {
3072	pr_err("FAILED %d\n", err);
3073	goto out_free;
3074	}
3075
3076	for (i = 0; i < 512; i++)
3077	n += sprintf(buf + n, "%02x", ext_csd[i]);
3078	n += sprintf(buf + n, "\n");
3079
3080	if (n != EXT_CSD_STR_LEN) {
3081	err = -EINVAL;
3082	kfree(ext_csd);
3083	goto out_free;
3084	}
3085
3086	filp->private_data = buf;
3087	kfree(ext_csd);
3088	return 0;
3089
3090	out_free:
3091	kfree(buf);
3092	return err;
3093	}
3094
3095	static ssize_t mmc_ext_csd_read(struct file filp, char __user ubuf,
3096	size_t cnt, loff_t *ppos)
3097	{
3098	char *buf = filp->private_data;
3099
3100	return simple_read_from_buffer(ubuf, cnt, ppos,
3101	buf, EXT_CSD_STR_LEN);
3102	}
3103
3104	static int mmc_ext_csd_release(struct inode inode, struct file file)
3105	{
3106	kfree(file->private_data);
3107	return 0;
3108	}
3109
3110	static const struct file_operations mmc_dbg_ext_csd_fops = {
3111	.open = mmc_ext_csd_open,
3112	.read = mmc_ext_csd_read,
3113	.release = mmc_ext_csd_release,
3114	.llseek = default_llseek,
3115	};
3116
3117	static void mmc_blk_add_debugfs(struct mmc_card card, struct mmc_blk_data md)
3118	{
3119	struct dentry *root;
3120
3121	if (!card->debugfs_root)
3122	return;
3123
3124	root = card->debugfs_root;
3125
3126	if (mmc_card_mmc(card) \|\| mmc_card_sd(card)) {
3127	md->status_dentry =
3128	debugfs_create_file_unsafe("status", 0400, root,
3129	card,
3130	&mmc_dbg_card_status_fops);
3131	}
3132
3133	if (mmc_card_mmc(card)) {
3134	md->ext_csd_dentry =
3135	debugfs_create_file("ext_csd", 0400, root, card,
3136	&mmc_dbg_ext_csd_fops);
3137	}
3138	}
3139
3140	static void mmc_blk_remove_debugfs(struct mmc_card *card,
3141	struct mmc_blk_data *md)
3142	{
3143	if (!card->debugfs_root)
3144	return;
3145
3146	debugfs_remove(md->status_dentry);
3147	md->status_dentry = NULL;
3148
3149	debugfs_remove(md->ext_csd_dentry);
3150	md->ext_csd_dentry = NULL;
3151	}
3152
3153	#else
3154
3155	static void mmc_blk_add_debugfs(struct mmc_card card, struct mmc_blk_data md)
3156	{
3157	}
3158
3159	static void mmc_blk_remove_debugfs(struct mmc_card *card,
3160	struct mmc_blk_data *md)
3161	{
3162	}
3163
3164	#endif /* CONFIG_DEBUG_FS */
3165
3166	static void mmc_blk_rpmb_add(struct mmc_card *card)
3167	{
3168	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3169	struct mmc_rpmb_data *rpmb;
3170	struct rpmb_dev *rdev;
3171	unsigned int n;
3172	u32 cid[4];
3173	struct rpmb_descr descr = {
3174	.type = RPMB_TYPE_EMMC,
3175	.route_frames = mmc_route_rpmb_frames,
3176	.reliable_wr_count = card->ext_csd.enhanced_rpmb_supported ?
3177	2 : 32,
3178	.capacity = card->ext_csd.raw_rpmb_size_mult,
3179	.dev_id = (void *)cid,
3180	.dev_id_len = sizeof(cid),
3181	};
3182
3183	/*
3184	* Provice CID as an octet array. The CID needs to be interpreted
3185	* when used as input to derive the RPMB key since some fields
3186	* will change due to firmware updates.
3187	*/
3188	for (n = 0; n < 4; n++)
3189	cid[n] = be32_to_cpu((__force __be32)card->raw_cid[n]);
3190
3191	list_for_each_entry(rpmb, &md->rpmbs, node) {
3192	rdev = rpmb_dev_register(&rpmb->dev, &descr);
3193	if (IS_ERR(rdev)) {
3194	pr_warn("%s: could not register RPMB device\n",
3195	dev_name(&rpmb->dev));
3196	continue;
3197	}
3198	rpmb->rdev = rdev;
3199	}
3200	}
3201
3202	static int mmc_blk_probe(struct mmc_card *card)
3203	{
3204	struct mmc_blk_data *md;
3205	int ret = 0;
3206
3207	/*
3208	* Check that the card supports the command class(es) we need.
3209	*/
3210	if (!(card->csd.cmdclass & CCC_BLOCK_READ))
3211	return -ENODEV;
3212
3213	mmc_fixup_device(card, mmc_blk_fixups);
3214
3215	card->complete_wq = alloc_workqueue("mmc_complete",
3216	WQ_MEM_RECLAIM \| WQ_HIGHPRI \| WQ_PERCPU,
3217	0);
3218	if (!card->complete_wq) {
3219	pr_err("Failed to create mmc completion workqueue");
3220	return -ENOMEM;
3221	}
3222
3223	md = mmc_blk_alloc(card);
3224	if (IS_ERR(md)) {
3225	ret = PTR_ERR(md);
3226	goto out_free;
3227	}
3228
3229	ret = mmc_blk_alloc_parts(card, md);
3230	if (ret)
3231	goto out;
3232
3233	/* Add two debugfs entries */
3234	mmc_blk_add_debugfs(card, md);
3235
3236	pm_runtime_set_autosuspend_delay(&card->dev, 3000);
3237	pm_runtime_use_autosuspend(&card->dev);
3238
3239	/*
3240	* Don't enable runtime PM for SD-combo cards here. Leave that
3241	* decision to be taken during the SDIO init sequence instead.
3242	*/
3243	if (!mmc_card_sd_combo(card)) {
3244	pm_runtime_set_active(&card->dev);
3245	pm_runtime_enable(&card->dev);
3246	}
3247
3248	mmc_blk_rpmb_add(card);
3249
3250	return 0;
3251
3252	out:
3253	mmc_blk_remove_parts(card, md);
3254	mmc_blk_remove_req(md);
3255	out_free:
3256	destroy_workqueue(card->complete_wq);
3257	return ret;
3258	}
3259
3260	static void mmc_blk_remove(struct mmc_card *card)
3261	{
3262	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3263
3264	mmc_blk_remove_debugfs(card, md);
3265	mmc_blk_remove_parts(card, md);
3266	pm_runtime_get_sync(&card->dev);
3267	if (md->part_curr != md->part_type) {
3268	mmc_claim_host(card->host);
3269	mmc_blk_part_switch(card, md->part_type);
3270	mmc_release_host(card->host);
3271	}
3272	if (!mmc_card_sd_combo(card))
3273	pm_runtime_disable(&card->dev);
3274	pm_runtime_put_noidle(&card->dev);
3275	mmc_blk_remove_req(md);
3276	destroy_workqueue(card->complete_wq);
3277	}
3278
3279	static int _mmc_blk_suspend(struct mmc_card *card)
3280	{
3281	struct mmc_blk_data *part_md;
3282	struct mmc_blk_data *md = dev_get_drvdata(&card->dev);
3283
3284	if (md) {
3285	mmc_queue_suspend(&md->queue);
3286	list_for_each_entry(part_md, &md->part, part) {
3287	mmc_queue_suspend(&part_md->queue);
3288	}
3289	}
3290	return 0;
3291	}
3292
3293	static void mmc_blk_shutdown(struct mmc_card *card)
3294	{
3295	_mmc_blk_suspend(card);
3296	}
3297
3298	static int mmc_blk_suspend(struct device *dev)
3299	{
3300	struct mmc_card *card = mmc_dev_to_card(dev);
3301
3302	return _mmc_blk_suspend(card);
3303	}
3304
3305	static int mmc_blk_resume(struct device *dev)
3306	{
3307	struct mmc_blk_data *part_md;
3308	struct mmc_blk_data *md = dev_get_drvdata(dev);
3309
3310	if (md) {
3311	/*
3312	* Resume involves the card going into idle state,
3313	* so current partition is always the main one.
3314	*/
3315	md->part_curr = md->part_type;
3316	mmc_queue_resume(&md->queue);
3317	list_for_each_entry(part_md, &md->part, part) {
3318	mmc_queue_resume(&part_md->queue);
3319	}
3320	}
3321	return 0;
3322	}
3323
3324	static DEFINE_SIMPLE_DEV_PM_OPS(mmc_blk_pm_ops, mmc_blk_suspend, mmc_blk_resume);
3325
3326	static struct mmc_driver mmc_driver = {
3327	.drv = {
3328	.name = "mmcblk",
3329	.pm = pm_sleep_ptr(&mmc_blk_pm_ops),
3330	},
3331	.probe = mmc_blk_probe,
3332	.remove = mmc_blk_remove,
3333	.shutdown = mmc_blk_shutdown,
3334	};
3335
3336	static int __init mmc_blk_init(void)
3337	{
3338	int res;
3339
3340	if (perdev_minors <= 0) {
3341	pr_err("mmcblk: invalid minors per device: %d\n", perdev_minors);
3342	return -EINVAL;
3343	}
3344
3345	res = bus_register(&mmc_rpmb_bus_type);
3346	if (res < 0) {
3347	pr_err("mmcblk: could not register RPMB bus type\n");
3348	return res;
3349	}
3350	res = alloc_chrdev_region(&mmc_rpmb_devt, 0, MAX_DEVICES, "rpmb");
3351	if (res < 0) {
3352	pr_err("mmcblk: failed to allocate rpmb chrdev region\n");
3353	goto out_bus_unreg;
3354	}
3355
3356	if (perdev_minors != CONFIG_MMC_BLOCK_MINORS)
3357	pr_info("mmcblk: using %d minors per device\n", perdev_minors);
3358
3359	max_devices = min(MAX_DEVICES, (1 << MINORBITS) / perdev_minors);
3360
3361	res = register_blkdev(MMC_BLOCK_MAJOR, "mmc");
3362	if (res)
3363	goto out_chrdev_unreg;
3364
3365	res = mmc_register_driver(&mmc_driver);
3366	if (res)
3367	goto out_blkdev_unreg;
3368
3369	return 0;
3370
3371	out_blkdev_unreg:
3372	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
3373	out_chrdev_unreg:
3374	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3375	out_bus_unreg:
3376	bus_unregister(&mmc_rpmb_bus_type);
3377	return res;
3378	}
3379
3380	static void __exit mmc_blk_exit(void)
3381	{
3382	mmc_unregister_driver(&mmc_driver);
3383	unregister_blkdev(MMC_BLOCK_MAJOR, "mmc");
3384	unregister_chrdev_region(mmc_rpmb_devt, MAX_DEVICES);
3385	bus_unregister(&mmc_rpmb_bus_type);
3386	}
3387
3388	module_init(mmc_blk_init);
3389	module_exit(mmc_blk_exit);
3390
3391	MODULE_LICENSE("GPL");
3392	MODULE_DESCRIPTION("Multimedia Card (MMC) block device driver");