linux - drivers/ata/libata-core.c

7037 lines · 179.4 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* libata-core.c - helper library for ATA
4	*
5	* Copyright 2003-2004 Red Hat, Inc. All rights reserved.
6	* Copyright 2003-2004 Jeff Garzik
7	*
8	* libata documentation is available via 'make {ps\|pdf}docs',
9	* as Documentation/driver-api/libata.rst
10	*
11	* Hardware documentation available from http://www.t13.org/ and
12	* http://www.sata-io.org/
13	*
14	* Standards documents from:
15	* http://www.t13.org (ATA standards, PCI DMA IDE spec)
16	* http://www.t10.org (SCSI MMC - for ATAPI MMC)
17	* http://www.sata-io.org (SATA)
18	* http://www.compactflash.org (CF)
19	* http://www.qic.org (QIC157 - Tape and DSC)
20	* http://www.ce-ata.org (CE-ATA: not supported)
21	*
22	* libata is essentially a library of internal helper functions for
23	* low-level ATA host controller drivers. As such, the API/ABI is
24	* likely to change as new drivers are added and updated.
25	* Do not depend on ABI/API stability.
26	*/
27
28	#include <linux/kernel.h>
29	#include <linux/module.h>
30	#include <linux/pci.h>
31	#include <linux/init.h>
32	#include <linux/list.h>
33	#include <linux/mm.h>
34	#include <linux/spinlock.h>
35	#include <linux/blkdev.h>
36	#include <linux/delay.h>
37	#include <linux/timer.h>
38	#include <linux/time.h>
39	#include <linux/interrupt.h>
40	#include <linux/completion.h>
41	#include <linux/suspend.h>
42	#include <linux/workqueue.h>
43	#include <linux/scatterlist.h>
44	#include <linux/io.h>
45	#include <linux/log2.h>
46	#include <linux/slab.h>
47	#include <linux/glob.h>
48	#include <scsi/scsi.h>
49	#include <scsi/scsi_cmnd.h>
50	#include <scsi/scsi_host.h>
51	#include <linux/libata.h>
52	#include <asm/byteorder.h>
53	#include <linux/unaligned.h>
54	#include <linux/cdrom.h>
55	#include <linux/ratelimit.h>
56	#include <linux/leds.h>
57	#include <linux/pm_runtime.h>
58	#include <linux/platform_device.h>
59	#include <asm/setup.h>
60
61	#define CREATE_TRACE_POINTS
62	#include <trace/events/libata.h>
63
64	#include "libata.h"
65	#include "libata-transport.h"
66
67	const struct ata_port_operations ata_base_port_ops = {
68	.reset.prereset = ata_std_prereset,
69	.reset.postreset = ata_std_postreset,
70	.error_handler = ata_std_error_handler,
71	.sched_eh = ata_std_sched_eh,
72	.end_eh = ata_std_end_eh,
73	};
74
75	static unsigned int ata_dev_init_params(struct ata_device *dev,
76	u16 heads, u16 sectors);
77	static unsigned int ata_dev_set_xfermode(struct ata_device *dev);
78	static void ata_dev_xfermask(struct ata_device *dev);
79	static u64 ata_dev_quirks(const struct ata_device *dev);
80	static u64 ata_dev_get_quirk_value(struct ata_device *dev, u64 quirk);
81
82	static DEFINE_IDA(ata_ida);
83
84	#ifdef CONFIG_ATA_FORCE
85	struct ata_force_param {
86	const char *name;
87	u64 value;
88	u8 cbl;
89	u8 spd_limit;
90	unsigned int xfer_mask;
91	u64 quirk_on;
92	u64 quirk_off;
93	unsigned int pflags_on;
94	u16 lflags_on;
95	u16 lflags_off;
96	};
97
98	struct ata_force_ent {
99	int port;
100	int device;
101	struct ata_force_param param;
102	};
103
104	static struct ata_force_ent *ata_force_tbl;
105	static int ata_force_tbl_size;
106
107	static char ata_force_param_buf[COMMAND_LINE_SIZE] __initdata;
108	/* param_buf is thrown away after initialization, disallow read */
109	module_param_string(force, ata_force_param_buf, sizeof(ata_force_param_buf), 0);
110	MODULE_PARM_DESC(force, "Force ATA configurations including cable type, link speed and transfer mode (see Documentation/admin-guide/kernel-parameters.rst for details)");
111	#endif
112
113	static int atapi_enabled = 1;
114	module_param(atapi_enabled, int, 0444);
115	MODULE_PARM_DESC(atapi_enabled, "Enable discovery of ATAPI devices (0=off, 1=on [default])");
116
117	static int atapi_dmadir = 0;
118	module_param(atapi_dmadir, int, 0444);
119	MODULE_PARM_DESC(atapi_dmadir, "Enable ATAPI DMADIR bridge support (0=off [default], 1=on)");
120
121	int atapi_passthru16 = 1;
122	module_param(atapi_passthru16, int, 0444);
123	MODULE_PARM_DESC(atapi_passthru16, "Enable ATA_16 passthru for ATAPI devices (0=off, 1=on [default])");
124
125	int libata_fua = 0;
126	module_param_named(fua, libata_fua, int, 0444);
127	MODULE_PARM_DESC(fua, "FUA support (0=off [default], 1=on)");
128
129	static int ata_ignore_hpa;
130	module_param_named(ignore_hpa, ata_ignore_hpa, int, 0644);
131	MODULE_PARM_DESC(ignore_hpa, "Ignore HPA limit (0=keep BIOS limits, 1=ignore limits, using full disk)");
132
133	static int libata_dma_mask = ATA_DMA_MASK_ATA\|ATA_DMA_MASK_ATAPI\|ATA_DMA_MASK_CFA;
134	module_param_named(dma, libata_dma_mask, int, 0444);
135	MODULE_PARM_DESC(dma, "DMA enable/disable (0x1==ATA, 0x2==ATAPI, 0x4==CF)");
136
137	static int ata_probe_timeout;
138	module_param(ata_probe_timeout, int, 0444);
139	MODULE_PARM_DESC(ata_probe_timeout, "Set ATA probing timeout (seconds)");
140
141	int libata_noacpi = 0;
142	module_param_named(noacpi, libata_noacpi, int, 0444);
143	MODULE_PARM_DESC(noacpi, "Disable the use of ACPI in probe/suspend/resume (0=off [default], 1=on)");
144
145	int libata_allow_tpm = 0;
146	module_param_named(allow_tpm, libata_allow_tpm, int, 0444);
147	MODULE_PARM_DESC(allow_tpm, "Permit the use of TPM commands (0=off [default], 1=on)");
148
149	static int atapi_an;
150	module_param(atapi_an, int, 0444);
151	MODULE_PARM_DESC(atapi_an, "Enable ATAPI AN media presence notification (0=0ff [default], 1=on)");
152
153	MODULE_AUTHOR("Jeff Garzik");
154	MODULE_DESCRIPTION("Library module for ATA devices");
155	MODULE_LICENSE("GPL");
156	MODULE_VERSION(DRV_VERSION);
157
158	static inline bool ata_dev_print_info(const struct ata_device *dev)
159	{
160	struct ata_eh_context *ehc = &dev->link->eh_context;
161
162	return ehc->i.flags & ATA_EHI_PRINTINFO;
163	}
164
165	/**
166	* ata_link_next - link iteration helper
167	* @link: the previous link, NULL to start
168	* @ap: ATA port containing links to iterate
169	* @mode: iteration mode, one of ATA_LITER_*
170	*
171	* LOCKING:
172	* Host lock or EH context.
173	*
174	* RETURNS:
175	* Pointer to the next link.
176	*/
177	struct ata_link ata_link_next(struct ata_link link, struct ata_port *ap,
178	enum ata_link_iter_mode mode)
179	{
180	BUG_ON(mode != ATA_LITER_EDGE &&
181	mode != ATA_LITER_PMP_FIRST && mode != ATA_LITER_HOST_FIRST);
182
183	/* NULL link indicates start of iteration */
184	if (!link)
185	switch (mode) {
186	case ATA_LITER_EDGE:
187	case ATA_LITER_PMP_FIRST:
188	if (sata_pmp_attached(ap))
189	return ap->pmp_link;
190	fallthrough;
191	case ATA_LITER_HOST_FIRST:
192	return &ap->link;
193	}
194
195	/* we just iterated over the host link, what's next? */
196	if (link == &ap->link)
197	switch (mode) {
198	case ATA_LITER_HOST_FIRST:
199	if (sata_pmp_attached(ap))
200	return ap->pmp_link;
201	fallthrough;
202	case ATA_LITER_PMP_FIRST:
203	if (unlikely(ap->slave_link))
204	return ap->slave_link;
205	fallthrough;
206	case ATA_LITER_EDGE:
207	return NULL;
208	}
209
210	/* slave_link excludes PMP */
211	if (unlikely(link == ap->slave_link))
212	return NULL;
213
214	/* we were over a PMP link */
215	if (++link < ap->pmp_link + ap->nr_pmp_links)
216	return link;
217
218	if (mode == ATA_LITER_PMP_FIRST)
219	return &ap->link;
220
221	return NULL;
222	}
223	EXPORT_SYMBOL_GPL(ata_link_next);
224
225	/**
226	* ata_dev_next - device iteration helper
227	* @dev: the previous device, NULL to start
228	* @link: ATA link containing devices to iterate
229	* @mode: iteration mode, one of ATA_DITER_*
230	*
231	* LOCKING:
232	* Host lock or EH context.
233	*
234	* RETURNS:
235	* Pointer to the next device.
236	*/
237	struct ata_device ata_dev_next(struct ata_device dev, struct ata_link *link,
238	enum ata_dev_iter_mode mode)
239	{
240	BUG_ON(mode != ATA_DITER_ENABLED && mode != ATA_DITER_ENABLED_REVERSE &&
241	mode != ATA_DITER_ALL && mode != ATA_DITER_ALL_REVERSE);
242
243	/* NULL dev indicates start of iteration */
244	if (!dev)
245	switch (mode) {
246	case ATA_DITER_ENABLED:
247	case ATA_DITER_ALL:
248	dev = link->device;
249	goto check;
250	case ATA_DITER_ENABLED_REVERSE:
251	case ATA_DITER_ALL_REVERSE:
252	dev = link->device + ata_link_max_devices(link) - 1;
253	goto check;
254	}
255
256	next:
257	/* move to the next one */
258	switch (mode) {
259	case ATA_DITER_ENABLED:
260	case ATA_DITER_ALL:
261	if (++dev < link->device + ata_link_max_devices(link))
262	goto check;
263	return NULL;
264	case ATA_DITER_ENABLED_REVERSE:
265	case ATA_DITER_ALL_REVERSE:
266	if (--dev >= link->device)
267	goto check;
268	return NULL;
269	}
270
271	check:
272	if ((mode == ATA_DITER_ENABLED \|\| mode == ATA_DITER_ENABLED_REVERSE) &&
273	!ata_dev_enabled(dev))
274	goto next;
275	return dev;
276	}
277	EXPORT_SYMBOL_GPL(ata_dev_next);
278
279	/**
280	* ata_dev_phys_link - find physical link for a device
281	* @dev: ATA device to look up physical link for
282	*
283	* Look up physical link which @dev is attached to. Note that
284	* this is different from @dev->link only when @dev is on slave
285	* link. For all other cases, it's the same as @dev->link.
286	*
287	* LOCKING:
288	* Don't care.
289	*
290	* RETURNS:
291	* Pointer to the found physical link.
292	*/
293	struct ata_link ata_dev_phys_link(struct ata_device dev)
294	{
295	struct ata_port *ap = dev->link->ap;
296
297	if (!ap->slave_link)
298	return dev->link;
299	if (!dev->devno)
300	return &ap->link;
301	return ap->slave_link;
302	}
303
304	#ifdef CONFIG_ATA_FORCE
305	/**
306	* ata_force_cbl - force cable type according to libata.force
307	* @ap: ATA port of interest
308	*
309	* Force cable type according to libata.force and whine about it.
310	* The last entry which has matching port number is used, so it
311	* can be specified as part of device force parameters. For
312	* example, both "a:40c,1.00:udma4" and "1.00:40c,udma4" have the
313	* same effect.
314	*
315	* LOCKING:
316	* EH context.
317	*/
318	void ata_force_cbl(struct ata_port *ap)
319	{
320	int i;
321
322	for (i = ata_force_tbl_size - 1; i >= 0; i--) {
323	const struct ata_force_ent *fe = &ata_force_tbl[i];
324
325	if (fe->port != -1 && fe->port != ap->print_id)
326	continue;
327
328	if (fe->param.cbl == ATA_CBL_NONE)
329	continue;
330
331	ap->cbl = fe->param.cbl;
332	ata_port_notice(ap, "FORCE: cable set to %s\n", fe->param.name);
333	return;
334	}
335	}
336
337	/**
338	* ata_force_pflags - force port flags according to libata.force
339	* @ap: ATA port of interest
340	*
341	* Force port flags according to libata.force and whine about it.
342	*
343	* LOCKING:
344	* EH context.
345	*/
346	static void ata_force_pflags(struct ata_port *ap)
347	{
348	int i;
349
350	for (i = ata_force_tbl_size - 1; i >= 0; i--) {
351	const struct ata_force_ent *fe = &ata_force_tbl[i];
352
353	if (fe->port != -1 && fe->port != ap->print_id)
354	continue;
355
356	/* let pflags stack */
357	if (fe->param.pflags_on) {
358	ap->pflags \|= fe->param.pflags_on;
359	ata_port_notice(ap,
360	"FORCE: port flag 0x%x forced -> 0x%x\n",
361	fe->param.pflags_on, ap->pflags);
362	}
363	}
364	}
365
366	/**
367	* ata_force_link_limits - force link limits according to libata.force
368	* @link: ATA link of interest
369	*
370	* Force link flags and SATA spd limit according to libata.force
371	* and whine about it. When only the port part is specified
372	* (e.g. 1:), the limit applies to all links connected to both
373	* the host link and all fan-out ports connected via PMP. If the
374	* device part is specified as 0 (e.g. 1.00:), it specifies the
375	* first fan-out link not the host link. Device number 15 always
376	* points to the host link whether PMP is attached or not. If the
377	* controller has slave link, device number 16 points to it.
378	*
379	* LOCKING:
380	* EH context.
381	*/
382	static void ata_force_link_limits(struct ata_link *link)
383	{
384	bool did_spd = false;
385	int linkno = link->pmp;
386	int i;
387
388	if (ata_is_host_link(link))
389	linkno += 15;
390
391	for (i = ata_force_tbl_size - 1; i >= 0; i--) {
392	const struct ata_force_ent *fe = &ata_force_tbl[i];
393
394	if (fe->port != -1 && fe->port != link->ap->print_id)
395	continue;
396
397	if (fe->device != -1 && fe->device != linkno)
398	continue;
399
400	/* only honor the first spd limit */
401	if (!did_spd && fe->param.spd_limit) {
402	link->hw_sata_spd_limit = (1 << fe->param.spd_limit) - 1;
403	ata_link_notice(link, "FORCE: PHY spd limit set to %s\n",
404	fe->param.name);
405	did_spd = true;
406	}
407
408	/* let lflags stack */
409	if (fe->param.lflags_on) {
410	link->flags \|= fe->param.lflags_on;
411	ata_link_notice(link,
412	"FORCE: link flag 0x%x forced -> 0x%x\n",
413	fe->param.lflags_on, link->flags);
414	}
415	if (fe->param.lflags_off) {
416	link->flags &= ~fe->param.lflags_off;
417	ata_link_notice(link,
418	"FORCE: link flag 0x%x cleared -> 0x%x\n",
419	fe->param.lflags_off, link->flags);
420	}
421	}
422	}
423
424	/**
425	* ata_force_xfermask - force xfermask according to libata.force
426	* @dev: ATA device of interest
427	*
428	* Force xfer_mask according to libata.force and whine about it.
429	* For consistency with link selection, device number 15 selects
430	* the first device connected to the host link.
431	*
432	* LOCKING:
433	* EH context.
434	*/
435	static void ata_force_xfermask(struct ata_device *dev)
436	{
437	int devno = dev->link->pmp + dev->devno;
438	int alt_devno = devno;
439	int i;
440
441	/* allow n.15/16 for devices attached to host port */
442	if (ata_is_host_link(dev->link))
443	alt_devno += 15;
444
445	for (i = ata_force_tbl_size - 1; i >= 0; i--) {
446	const struct ata_force_ent *fe = &ata_force_tbl[i];
447	unsigned int pio_mask, mwdma_mask, udma_mask;
448
449	if (fe->port != -1 && fe->port != dev->link->ap->print_id)
450	continue;
451
452	if (fe->device != -1 && fe->device != devno &&
453	fe->device != alt_devno)
454	continue;
455
456	if (!fe->param.xfer_mask)
457	continue;
458
459	ata_unpack_xfermask(fe->param.xfer_mask,
460	&pio_mask, &mwdma_mask, &udma_mask);
461	if (udma_mask)
462	dev->udma_mask = udma_mask;
463	else if (mwdma_mask) {
464	dev->udma_mask = 0;
465	dev->mwdma_mask = mwdma_mask;
466	} else {
467	dev->udma_mask = 0;
468	dev->mwdma_mask = 0;
469	dev->pio_mask = pio_mask;
470	}
471
472	ata_dev_notice(dev, "FORCE: xfer_mask set to %s\n",
473	fe->param.name);
474	return;
475	}
476	}
477
478	static const struct ata_force_ent *
479	ata_force_get_fe_for_dev(struct ata_device *dev)
480	{
481	const struct ata_force_ent *fe;
482	int devno = dev->link->pmp + dev->devno;
483	int alt_devno = devno;
484	int i;
485
486	/* allow n.15/16 for devices attached to host port */
487	if (ata_is_host_link(dev->link))
488	alt_devno += 15;
489
490	for (i = 0; i < ata_force_tbl_size; i++) {
491	fe = &ata_force_tbl[i];
492	if (fe->port != -1 && fe->port != dev->link->ap->print_id)
493	continue;
494
495	if (fe->device != -1 && fe->device != devno &&
496	fe->device != alt_devno)
497	continue;
498
499	return fe;
500	}
501
502	return NULL;
503	}
504
505	/**
506	* ata_force_quirks - force quirks according to libata.force
507	* @dev: ATA device of interest
508	*
509	* Force quirks according to libata.force and whine about it.
510	* For consistency with link selection, device number 15 selects
511	* the first device connected to the host link.
512	*
513	* LOCKING:
514	* EH context.
515	*/
516	static void ata_force_quirks(struct ata_device *dev)
517	{
518	const struct ata_force_ent *fe = ata_force_get_fe_for_dev(dev);
519
520	if (!fe)
521	return;
522
523	if (!(~dev->quirks & fe->param.quirk_on) &&
524	!(dev->quirks & fe->param.quirk_off))
525	return;
526
527	dev->quirks \|= fe->param.quirk_on;
528	dev->quirks &= ~fe->param.quirk_off;
529
530	ata_dev_notice(dev, "FORCE: modified (%s)\n", fe->param.name);
531	}
532	#else
533	static inline void ata_force_pflags(struct ata_port *ap) { }
534	static inline void ata_force_link_limits(struct ata_link *link) { }
535	static inline void ata_force_xfermask(struct ata_device *dev) { }
536	static inline void ata_force_quirks(struct ata_device *dev) { }
537	#endif
538
539	/**
540	* atapi_cmd_type - Determine ATAPI command type from SCSI opcode
541	* @opcode: SCSI opcode
542	*
543	* Determine ATAPI command type from @opcode.
544	*
545	* LOCKING:
546	* None.
547	*
548	* RETURNS:
549	* ATAPI_{READ\|WRITE\|READ_CD\|PASS_THRU\|MISC}
550	*/
551	int atapi_cmd_type(u8 opcode)
552	{
553	switch (opcode) {
554	case GPCMD_READ_10:
555	case GPCMD_READ_12:
556	return ATAPI_READ;
557
558	case GPCMD_WRITE_10:
559	case GPCMD_WRITE_12:
560	case GPCMD_WRITE_AND_VERIFY_10:
561	return ATAPI_WRITE;
562
563	case GPCMD_READ_CD:
564	case GPCMD_READ_CD_MSF:
565	return ATAPI_READ_CD;
566
567	case ATA_16:
568	case ATA_12:
569	if (atapi_passthru16)
570	return ATAPI_PASS_THRU;
571	fallthrough;
572	default:
573	return ATAPI_MISC;
574	}
575	}
576	EXPORT_SYMBOL_GPL(atapi_cmd_type);
577
578	static const u8 ata_rw_cmds[] = {
579	/* pio multi */
580	ATA_CMD_READ_MULTI,
581	ATA_CMD_WRITE_MULTI,
582	ATA_CMD_READ_MULTI_EXT,
583	ATA_CMD_WRITE_MULTI_EXT,
584	0,
585	0,
586	0,
587	0,
588	/* pio */
589	ATA_CMD_PIO_READ,
590	ATA_CMD_PIO_WRITE,
591	ATA_CMD_PIO_READ_EXT,
592	ATA_CMD_PIO_WRITE_EXT,
593	0,
594	0,
595	0,
596	0,
597	/* dma */
598	ATA_CMD_READ,
599	ATA_CMD_WRITE,
600	ATA_CMD_READ_EXT,
601	ATA_CMD_WRITE_EXT,
602	0,
603	0,
604	0,
605	ATA_CMD_WRITE_FUA_EXT
606	};
607
608	/**
609	* ata_set_rwcmd_protocol - set taskfile r/w command and protocol
610	* @dev: target device for the taskfile
611	* @tf: taskfile to examine and configure
612	*
613	* Examine the device configuration and tf->flags to determine
614	* the proper read/write command and protocol to use for @tf.
615	*
616	* LOCKING:
617	* caller.
618	*/
619	static bool ata_set_rwcmd_protocol(struct ata_device *dev,
620	struct ata_taskfile *tf)
621	{
622	u8 cmd;
623
624	int index, fua, lba48, write;
625
626	fua = (tf->flags & ATA_TFLAG_FUA) ? 4 : 0;
627	lba48 = (tf->flags & ATA_TFLAG_LBA48) ? 2 : 0;
628	write = (tf->flags & ATA_TFLAG_WRITE) ? 1 : 0;
629
630	if (dev->flags & ATA_DFLAG_PIO) {
631	tf->protocol = ATA_PROT_PIO;
632	index = dev->multi_count ? 0 : 8;
633	} else if (lba48 && (dev->link->ap->flags & ATA_FLAG_PIO_LBA48)) {
634	/* Unable to use DMA due to host limitation */
635	tf->protocol = ATA_PROT_PIO;
636	index = dev->multi_count ? 0 : 8;
637	} else {
638	tf->protocol = ATA_PROT_DMA;
639	index = 16;
640	}
641
642	cmd = ata_rw_cmds[index + fua + lba48 + write];
643	if (!cmd)
644	return false;
645
646	tf->command = cmd;
647
648	return true;
649	}
650
651	/**
652	* ata_tf_read_block - Read block address from ATA taskfile
653	* @tf: ATA taskfile of interest
654	* @dev: ATA device @tf belongs to
655	*
656	* LOCKING:
657	* None.
658	*
659	* Read block address from @tf. This function can handle all
660	* three address formats - LBA, LBA48 and CHS. tf->protocol and
661	* flags select the address format to use.
662	*
663	* RETURNS:
664	* Block address read from @tf.
665	*/
666	u64 ata_tf_read_block(const struct ata_taskfile tf, struct ata_device dev)
667	{
668	u64 block = 0;
669
670	if (tf->flags & ATA_TFLAG_LBA) {
671	if (tf->flags & ATA_TFLAG_LBA48) {
672	block \|= (u64)tf->hob_lbah << 40;
673	block \|= (u64)tf->hob_lbam << 32;
674	block \|= (u64)tf->hob_lbal << 24;
675	} else
676	block \|= (tf->device & 0xf) << 24;
677
678	block \|= tf->lbah << 16;
679	block \|= tf->lbam << 8;
680	block \|= tf->lbal;
681	} else {
682	u32 cyl, head, sect;
683
684	cyl = tf->lbam \| (tf->lbah << 8);
685	head = tf->device & 0xf;
686	sect = tf->lbal;
687
688	if (!sect) {
689	ata_dev_warn(dev,
690	"device reported invalid CHS sector 0\n");
691	return U64_MAX;
692	}
693
694	block = (cyl * dev->heads + head) * dev->sectors + sect - 1;
695	}
696
697	return block;
698	}
699
700	/*
701	* Set a taskfile command duration limit index.
702	*/
703	static inline void ata_set_tf_cdl(struct ata_queued_cmd *qc, int cdl)
704	{
705	struct ata_taskfile *tf = &qc->tf;
706
707	if (tf->protocol == ATA_PROT_NCQ)
708	tf->auxiliary \|= cdl;
709	else
710	tf->feature \|= cdl;
711
712	/*
713	* Mark this command as having a CDL and request the result
714	* task file so that we can inspect the sense data available
715	* bit on completion.
716	*/
717	qc->flags \|= ATA_QCFLAG_HAS_CDL \| ATA_QCFLAG_RESULT_TF;
718	}
719
720	/**
721	* ata_build_rw_tf - Build ATA taskfile for given read/write request
722	* @qc: Metadata associated with the taskfile to build
723	* @block: Block address
724	* @n_block: Number of blocks
725	* @tf_flags: RW/FUA etc...
726	* @cdl: Command duration limit index
727	* @class: IO priority class
728	*
729	* LOCKING:
730	* None.
731	*
732	* Build ATA taskfile for the command @qc for read/write request described
733	* by @block, @n_block, @tf_flags and @class.
734	*
735	* RETURNS:
736	*
737	* 0 on success, -ERANGE if the request is too large for @dev,
738	* -EINVAL if the request is invalid.
739	*/
740	int ata_build_rw_tf(struct ata_queued_cmd *qc, u64 block, u32 n_block,
741	unsigned int tf_flags, int cdl, int class)
742	{
743	struct ata_taskfile *tf = &qc->tf;
744	struct ata_device *dev = qc->dev;
745
746	tf->flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_DEVICE;
747	tf->flags \|= tf_flags;
748
749	if (ata_ncq_enabled(dev)) {
750	/* yay, NCQ */
751	if (!lba_48_ok(block, n_block))
752	return -ERANGE;
753
754	tf->protocol = ATA_PROT_NCQ;
755	tf->flags \|= ATA_TFLAG_LBA \| ATA_TFLAG_LBA48;
756
757	if (tf->flags & ATA_TFLAG_WRITE)
758	tf->command = ATA_CMD_FPDMA_WRITE;
759	else
760	tf->command = ATA_CMD_FPDMA_READ;
761
762	tf->nsect = qc->hw_tag << 3;
763	tf->hob_feature = (n_block >> 8) & 0xff;
764	tf->feature = n_block & 0xff;
765
766	tf->hob_lbah = (block >> 40) & 0xff;
767	tf->hob_lbam = (block >> 32) & 0xff;
768	tf->hob_lbal = (block >> 24) & 0xff;
769	tf->lbah = (block >> 16) & 0xff;
770	tf->lbam = (block >> 8) & 0xff;
771	tf->lbal = block & 0xff;
772
773	tf->device = ATA_LBA;
774	if (tf->flags & ATA_TFLAG_FUA)
775	tf->device \|= 1 << 7;
776
777	if (dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLED &&
778	class == IOPRIO_CLASS_RT)
779	tf->hob_nsect \|= ATA_PRIO_HIGH << ATA_SHIFT_PRIO;
780
781	if ((dev->flags & ATA_DFLAG_CDL_ENABLED) && cdl)
782	ata_set_tf_cdl(qc, cdl);
783
784	} else if (dev->flags & ATA_DFLAG_LBA) {
785	tf->flags \|= ATA_TFLAG_LBA;
786
787	if ((dev->flags & ATA_DFLAG_CDL_ENABLED) && cdl)
788	ata_set_tf_cdl(qc, cdl);
789
790	/* Both FUA writes and a CDL index require 48-bit commands */
791	if (!(tf->flags & ATA_TFLAG_FUA) &&
792	!(qc->flags & ATA_QCFLAG_HAS_CDL) &&
793	lba_28_ok(block, n_block)) {
794	/* use LBA28 */
795	tf->device \|= (block >> 24) & 0xf;
796	} else if (lba_48_ok(block, n_block)) {
797	if (!(dev->flags & ATA_DFLAG_LBA48))
798	return -ERANGE;
799
800	/* use LBA48 */
801	tf->flags \|= ATA_TFLAG_LBA48;
802
803	tf->hob_nsect = (n_block >> 8) & 0xff;
804
805	tf->hob_lbah = (block >> 40) & 0xff;
806	tf->hob_lbam = (block >> 32) & 0xff;
807	tf->hob_lbal = (block >> 24) & 0xff;
808	} else {
809	/* request too large even for LBA48 */
810	return -ERANGE;
811	}
812
813	if (unlikely(!ata_set_rwcmd_protocol(dev, tf)))
814	return -EINVAL;
815
816	tf->nsect = n_block & 0xff;
817
818	tf->lbah = (block >> 16) & 0xff;
819	tf->lbam = (block >> 8) & 0xff;
820	tf->lbal = block & 0xff;
821
822	tf->device \|= ATA_LBA;
823	} else {
824	/* CHS */
825	u32 sect, head, cyl, track;
826
827	/* The request -may- be too large for CHS addressing. */
828	if (!lba_28_ok(block, n_block))
829	return -ERANGE;
830
831	if (unlikely(!ata_set_rwcmd_protocol(dev, tf)))
832	return -EINVAL;
833
834	/* Convert LBA to CHS */
835	track = (u32)block / dev->sectors;
836	cyl = track / dev->heads;
837	head = track % dev->heads;
838	sect = (u32)block % dev->sectors + 1;
839
840	/* Check whether the converted CHS can fit.
841	Cylinder: 0-65535
842	Head: 0-15
843	Sector: 1-255*/
844	if ((cyl >> 16) \|\| (head >> 4) \|\| (sect >> 8) \|\| (!sect))
845	return -ERANGE;
846
847	tf->nsect = n_block & 0xff; /* Sector count 0 means 256 sectors */
848	tf->lbal = sect;
849	tf->lbam = cyl;
850	tf->lbah = cyl >> 8;
851	tf->device \|= head;
852	}
853
854	return 0;
855	}
856
857	/**
858	* ata_pack_xfermask - Pack pio, mwdma and udma masks into xfer_mask
859	* @pio_mask: pio_mask
860	* @mwdma_mask: mwdma_mask
861	* @udma_mask: udma_mask
862	*
863	* Pack @pio_mask, @mwdma_mask and @udma_mask into a single
864	* unsigned int xfer_mask.
865	*
866	* LOCKING:
867	* None.
868	*
869	* RETURNS:
870	* Packed xfer_mask.
871	*/
872	unsigned int ata_pack_xfermask(unsigned int pio_mask,
873	unsigned int mwdma_mask,
874	unsigned int udma_mask)
875	{
876	return ((pio_mask << ATA_SHIFT_PIO) & ATA_MASK_PIO) \|
877	((mwdma_mask << ATA_SHIFT_MWDMA) & ATA_MASK_MWDMA) \|
878	((udma_mask << ATA_SHIFT_UDMA) & ATA_MASK_UDMA);
879	}
880	EXPORT_SYMBOL_GPL(ata_pack_xfermask);
881
882	/**
883	* ata_unpack_xfermask - Unpack xfer_mask into pio, mwdma and udma masks
884	* @xfer_mask: xfer_mask to unpack
885	* @pio_mask: resulting pio_mask
886	* @mwdma_mask: resulting mwdma_mask
887	* @udma_mask: resulting udma_mask
888	*
889	* Unpack @xfer_mask into @pio_mask, @mwdma_mask and @udma_mask.
890	* Any NULL destination masks will be ignored.
891	*/
892	void ata_unpack_xfermask(unsigned int xfer_mask, unsigned int *pio_mask,
893	unsigned int mwdma_mask, unsigned int udma_mask)
894	{
895	if (pio_mask)
896	*pio_mask = (xfer_mask & ATA_MASK_PIO) >> ATA_SHIFT_PIO;
897	if (mwdma_mask)
898	*mwdma_mask = (xfer_mask & ATA_MASK_MWDMA) >> ATA_SHIFT_MWDMA;
899	if (udma_mask)
900	*udma_mask = (xfer_mask & ATA_MASK_UDMA) >> ATA_SHIFT_UDMA;
901	}
902
903	static const struct ata_xfer_ent {
904	int shift, bits;
905	u8 base;
906	} ata_xfer_tbl[] = {
907	{ ATA_SHIFT_PIO, ATA_NR_PIO_MODES, XFER_PIO_0 },
908	{ ATA_SHIFT_MWDMA, ATA_NR_MWDMA_MODES, XFER_MW_DMA_0 },
909	{ ATA_SHIFT_UDMA, ATA_NR_UDMA_MODES, XFER_UDMA_0 },
910	{ -1, },
911	};
912
913	/**
914	* ata_xfer_mask2mode - Find matching XFER_* for the given xfer_mask
915	* @xfer_mask: xfer_mask of interest
916	*
917	* Return matching XFER_* value for @xfer_mask. Only the highest
918	* bit of @xfer_mask is considered.
919	*
920	* LOCKING:
921	* None.
922	*
923	* RETURNS:
924	* Matching XFER_* value, 0xff if no match found.
925	*/
926	u8 ata_xfer_mask2mode(unsigned int xfer_mask)
927	{
928	int highbit = fls(xfer_mask) - 1;
929	const struct ata_xfer_ent *ent;
930
931	for (ent = ata_xfer_tbl; ent->shift >= 0; ent++)
932	if (highbit >= ent->shift && highbit < ent->shift + ent->bits)
933	return ent->base + highbit - ent->shift;
934	return 0xff;
935	}
936	EXPORT_SYMBOL_GPL(ata_xfer_mask2mode);
937
938	/**
939	* ata_xfer_mode2mask - Find matching xfer_mask for XFER_*
940	* @xfer_mode: XFER_* of interest
941	*
942	* Return matching xfer_mask for @xfer_mode.
943	*
944	* LOCKING:
945	* None.
946	*
947	* RETURNS:
948	* Matching xfer_mask, 0 if no match found.
949	*/
950	unsigned int ata_xfer_mode2mask(u8 xfer_mode)
951	{
952	const struct ata_xfer_ent *ent;
953
954	for (ent = ata_xfer_tbl; ent->shift >= 0; ent++)
955	if (xfer_mode >= ent->base && xfer_mode < ent->base + ent->bits)
956	return ((2 << (ent->shift + xfer_mode - ent->base)) - 1)
957	& ~((1 << ent->shift) - 1);
958	return 0;
959	}
960	EXPORT_SYMBOL_GPL(ata_xfer_mode2mask);
961
962	/**
963	* ata_xfer_mode2shift - Find matching xfer_shift for XFER_*
964	* @xfer_mode: XFER_* of interest
965	*
966	* Return matching xfer_shift for @xfer_mode.
967	*
968	* LOCKING:
969	* None.
970	*
971	* RETURNS:
972	* Matching xfer_shift, -1 if no match found.
973	*/
974	int ata_xfer_mode2shift(u8 xfer_mode)
975	{
976	const struct ata_xfer_ent *ent;
977
978	for (ent = ata_xfer_tbl; ent->shift >= 0; ent++)
979	if (xfer_mode >= ent->base && xfer_mode < ent->base + ent->bits)
980	return ent->shift;
981	return -1;
982	}
983	EXPORT_SYMBOL_GPL(ata_xfer_mode2shift);
984
985	/**
986	* ata_mode_string - convert xfer_mask to string
987	* @xfer_mask: mask of bits supported; only highest bit counts.
988	*
989	* Determine string which represents the highest speed
990	* (highest bit in @modemask).
991	*
992	* LOCKING:
993	* None.
994	*
995	* RETURNS:
996	* Constant C string representing highest speed listed in
997	* @mode_mask, or the constant C string "<n/a>".
998	*/
999	const char *ata_mode_string(unsigned int xfer_mask)
1000	{
1001	static const char * const xfer_mode_str[] = {
1002	"PIO0",
1003	"PIO1",
1004	"PIO2",
1005	"PIO3",
1006	"PIO4",
1007	"PIO5",
1008	"PIO6",
1009	"MWDMA0",
1010	"MWDMA1",
1011	"MWDMA2",
1012	"MWDMA3",
1013	"MWDMA4",
1014	"UDMA/16",
1015	"UDMA/25",
1016	"UDMA/33",
1017	"UDMA/44",
1018	"UDMA/66",
1019	"UDMA/100",
1020	"UDMA/133",
1021	"UDMA7",
1022	};
1023	int highbit;
1024
1025	highbit = fls(xfer_mask) - 1;
1026	if (highbit >= 0 && highbit < ARRAY_SIZE(xfer_mode_str))
1027	return xfer_mode_str[highbit];
1028	return "<n/a>";
1029	}
1030	EXPORT_SYMBOL_GPL(ata_mode_string);
1031
1032	const char *sata_spd_string(unsigned int spd)
1033	{
1034	static const char * const spd_str[] = {
1035	"1.5 Gbps",
1036	"3.0 Gbps",
1037	"6.0 Gbps",
1038	};
1039
1040	if (spd == 0 \|\| (spd - 1) >= ARRAY_SIZE(spd_str))
1041	return "<unknown>";
1042	return spd_str[spd - 1];
1043	}
1044
1045	/**
1046	* ata_dev_classify - determine device type based on ATA-spec signature
1047	* @tf: ATA taskfile register set for device to be identified
1048	*
1049	* Determine from taskfile register contents whether a device is
1050	* ATA or ATAPI, as per "Signature and persistence" section
1051	* of ATA/PI spec (volume 1, sect 5.14).
1052	*
1053	* LOCKING:
1054	* None.
1055	*
1056	* RETURNS:
1057	* Device type, %ATA_DEV_ATA, %ATA_DEV_ATAPI, %ATA_DEV_PMP,
1058	* %ATA_DEV_ZAC, or %ATA_DEV_UNKNOWN the event of failure.
1059	*/
1060	unsigned int ata_dev_classify(const struct ata_taskfile *tf)
1061	{
1062	/* Apple's open source Darwin code hints that some devices only
1063	* put a proper signature into the LBA mid/high registers,
1064	* So, we only check those. It's sufficient for uniqueness.
1065	*
1066	* ATA/ATAPI-7 (d1532v1r1: Feb. 19, 2003) specified separate
1067	* signatures for ATA and ATAPI devices attached on SerialATA,
1068	* 0x3c/0xc3 and 0x69/0x96 respectively. However, SerialATA
1069	* spec has never mentioned about using different signatures
1070	* for ATA/ATAPI devices. Then, Serial ATA II: Port
1071	* Multiplier specification began to use 0x69/0x96 to identify
1072	* port multpliers and 0x3c/0xc3 to identify SEMB device.
1073	* ATA/ATAPI-7 dropped descriptions about 0x3c/0xc3 and
1074	* 0x69/0x96 shortly and described them as reserved for
1075	* SerialATA.
1076	*
1077	* We follow the current spec and consider that 0x69/0x96
1078	* identifies a port multiplier and 0x3c/0xc3 a SEMB device.
1079	* Unfortunately, WDC WD1600JS-62MHB5 (a hard drive) reports
1080	* SEMB signature. This is worked around in
1081	* ata_dev_read_id().
1082	*/
1083	if (tf->lbam == 0 && tf->lbah == 0)
1084	return ATA_DEV_ATA;
1085
1086	if (tf->lbam == 0x14 && tf->lbah == 0xeb)
1087	return ATA_DEV_ATAPI;
1088
1089	if (tf->lbam == 0x69 && tf->lbah == 0x96)
1090	return ATA_DEV_PMP;
1091
1092	if (tf->lbam == 0x3c && tf->lbah == 0xc3)
1093	return ATA_DEV_SEMB;
1094
1095	if (tf->lbam == 0xcd && tf->lbah == 0xab)
1096	return ATA_DEV_ZAC;
1097
1098	return ATA_DEV_UNKNOWN;
1099	}
1100	EXPORT_SYMBOL_GPL(ata_dev_classify);
1101
1102	/**
1103	* ata_id_string - Convert IDENTIFY DEVICE page into string
1104	* @id: IDENTIFY DEVICE results we will examine
1105	* @s: string into which data is output
1106	* @ofs: offset into identify device page
1107	* @len: length of string to return. must be an even number.
1108	*
1109	* The strings in the IDENTIFY DEVICE page are broken up into
1110	* 16-bit chunks. Run through the string, and output each
1111	* 8-bit chunk linearly, regardless of platform.
1112	*
1113	* LOCKING:
1114	* caller.
1115	*/
1116
1117	void ata_id_string(const u16 id, unsigned char s,
1118	unsigned int ofs, unsigned int len)
1119	{
1120	unsigned int c;
1121
1122	BUG_ON(len & 1);
1123
1124	while (len > 0) {
1125	c = id[ofs] >> 8;
1126	*s = c;
1127	s++;
1128
1129	c = id[ofs] & 0xff;
1130	*s = c;
1131	s++;
1132
1133	ofs++;
1134	len -= 2;
1135	}
1136	}
1137	EXPORT_SYMBOL_GPL(ata_id_string);
1138
1139	/**
1140	* ata_id_c_string - Convert IDENTIFY DEVICE page into C string
1141	* @id: IDENTIFY DEVICE results we will examine
1142	* @s: string into which data is output
1143	* @ofs: offset into identify device page
1144	* @len: length of string to return. must be an odd number.
1145	*
1146	* This function is identical to ata_id_string except that it
1147	* trims trailing spaces and terminates the resulting string with
1148	* null. @len must be actual maximum length (even number) + 1.
1149	*
1150	* LOCKING:
1151	* caller.
1152	*/
1153	void ata_id_c_string(const u16 id, unsigned char s,
1154	unsigned int ofs, unsigned int len)
1155	{
1156	unsigned char *p;
1157
1158	ata_id_string(id, s, ofs, len - 1);
1159
1160	p = s + strnlen(s, len - 1);
1161	while (p > s && p[-1] == ' ')
1162	p--;
1163	*p = '\0';
1164	}
1165	EXPORT_SYMBOL_GPL(ata_id_c_string);
1166
1167	static u64 ata_id_n_sectors(const u16 *id)
1168	{
1169	if (ata_id_has_lba(id)) {
1170	if (ata_id_has_lba48(id))
1171	return ata_id_u64(id, ATA_ID_LBA_CAPACITY_2);
1172
1173	return ata_id_u32(id, ATA_ID_LBA_CAPACITY);
1174	}
1175
1176	if (ata_id_current_chs_valid(id))
1177	return (u32)id[ATA_ID_CUR_CYLS] * (u32)id[ATA_ID_CUR_HEADS] *
1178	(u32)id[ATA_ID_CUR_SECTORS];
1179
1180	return (u32)id[ATA_ID_CYLS] * (u32)id[ATA_ID_HEADS] *
1181	(u32)id[ATA_ID_SECTORS];
1182	}
1183
1184	u64 ata_tf_to_lba48(const struct ata_taskfile *tf)
1185	{
1186	u64 sectors = 0;
1187
1188	sectors \|= ((u64)(tf->hob_lbah & 0xff)) << 40;
1189	sectors \|= ((u64)(tf->hob_lbam & 0xff)) << 32;
1190	sectors \|= ((u64)(tf->hob_lbal & 0xff)) << 24;
1191	sectors \|= (tf->lbah & 0xff) << 16;
1192	sectors \|= (tf->lbam & 0xff) << 8;
1193	sectors \|= (tf->lbal & 0xff);
1194
1195	return sectors;
1196	}
1197
1198	u64 ata_tf_to_lba(const struct ata_taskfile *tf)
1199	{
1200	u64 sectors = 0;
1201
1202	sectors \|= (tf->device & 0x0f) << 24;
1203	sectors \|= (tf->lbah & 0xff) << 16;
1204	sectors \|= (tf->lbam & 0xff) << 8;
1205	sectors \|= (tf->lbal & 0xff);
1206
1207	return sectors;
1208	}
1209
1210	/**
1211	* ata_read_native_max_address - Read native max address
1212	* @dev: target device
1213	* @max_sectors: out parameter for the result native max address
1214	*
1215	* Perform an LBA48 or LBA28 native size query upon the device in
1216	* question.
1217	*
1218	* RETURNS:
1219	* 0 on success, -EACCES if command is aborted by the drive.
1220	* -EIO on other errors.
1221	*/
1222	static int ata_read_native_max_address(struct ata_device dev, u64 max_sectors)
1223	{
1224	unsigned int err_mask;
1225	struct ata_taskfile tf;
1226	int lba48 = ata_id_has_lba48(dev->id);
1227
1228	ata_tf_init(dev, &tf);
1229
1230	/* always clear all address registers */
1231	tf.flags \|= ATA_TFLAG_DEVICE \| ATA_TFLAG_ISADDR;
1232
1233	if (lba48) {
1234	tf.command = ATA_CMD_READ_NATIVE_MAX_EXT;
1235	tf.flags \|= ATA_TFLAG_LBA48;
1236	} else
1237	tf.command = ATA_CMD_READ_NATIVE_MAX;
1238
1239	tf.protocol = ATA_PROT_NODATA;
1240	tf.device \|= ATA_LBA;
1241
1242	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
1243	if (err_mask) {
1244	ata_dev_warn(dev,
1245	"failed to read native max address (err_mask=0x%x)\n",
1246	err_mask);
1247	if (err_mask == AC_ERR_DEV && (tf.error & ATA_ABORTED))
1248	return -EACCES;
1249	return -EIO;
1250	}
1251
1252	if (lba48)
1253	*max_sectors = ata_tf_to_lba48(&tf) + 1;
1254	else
1255	*max_sectors = ata_tf_to_lba(&tf) + 1;
1256	if (dev->quirks & ATA_QUIRK_HPA_SIZE)
1257	(*max_sectors)--;
1258	return 0;
1259	}
1260
1261	/**
1262	* ata_set_max_sectors - Set max sectors
1263	* @dev: target device
1264	* @new_sectors: new max sectors value to set for the device
1265	*
1266	* Set max sectors of @dev to @new_sectors.
1267	*
1268	* RETURNS:
1269	* 0 on success, -EACCES if command is aborted or denied (due to
1270	* previous non-volatile SET_MAX) by the drive. -EIO on other
1271	* errors.
1272	*/
1273	static int ata_set_max_sectors(struct ata_device *dev, u64 new_sectors)
1274	{
1275	unsigned int err_mask;
1276	struct ata_taskfile tf;
1277	int lba48 = ata_id_has_lba48(dev->id);
1278
1279	new_sectors--;
1280
1281	ata_tf_init(dev, &tf);
1282
1283	tf.flags \|= ATA_TFLAG_DEVICE \| ATA_TFLAG_ISADDR;
1284
1285	if (lba48) {
1286	tf.command = ATA_CMD_SET_MAX_EXT;
1287	tf.flags \|= ATA_TFLAG_LBA48;
1288
1289	tf.hob_lbal = (new_sectors >> 24) & 0xff;
1290	tf.hob_lbam = (new_sectors >> 32) & 0xff;
1291	tf.hob_lbah = (new_sectors >> 40) & 0xff;
1292	} else {
1293	tf.command = ATA_CMD_SET_MAX;
1294
1295	tf.device \|= (new_sectors >> 24) & 0xf;
1296	}
1297
1298	tf.protocol = ATA_PROT_NODATA;
1299	tf.device \|= ATA_LBA;
1300
1301	tf.lbal = (new_sectors >> 0) & 0xff;
1302	tf.lbam = (new_sectors >> 8) & 0xff;
1303	tf.lbah = (new_sectors >> 16) & 0xff;
1304
1305	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
1306	if (err_mask) {
1307	ata_dev_warn(dev,
1308	"failed to set max address (err_mask=0x%x)\n",
1309	err_mask);
1310	if (err_mask == AC_ERR_DEV &&
1311	(tf.error & (ATA_ABORTED \| ATA_IDNF)))
1312	return -EACCES;
1313	return -EIO;
1314	}
1315
1316	return 0;
1317	}
1318
1319	/**
1320	* ata_hpa_resize - Resize a device with an HPA set
1321	* @dev: Device to resize
1322	*
1323	* Read the size of an LBA28 or LBA48 disk with HPA features and resize
1324	* it if required to the full size of the media. The caller must check
1325	* the drive has the HPA feature set enabled.
1326	*
1327	* RETURNS:
1328	* 0 on success, -errno on failure.
1329	*/
1330	static int ata_hpa_resize(struct ata_device *dev)
1331	{
1332	bool print_info = ata_dev_print_info(dev);
1333	bool unlock_hpa = ata_ignore_hpa \|\| dev->flags & ATA_DFLAG_UNLOCK_HPA;
1334	u64 sectors = ata_id_n_sectors(dev->id);
1335	u64 native_sectors;
1336	int rc;
1337
1338	/* do we need to do it? */
1339	if ((dev->class != ATA_DEV_ATA && dev->class != ATA_DEV_ZAC) \|\|
1340	!ata_id_has_lba(dev->id) \|\| !ata_id_hpa_enabled(dev->id) \|\|
1341	(dev->quirks & ATA_QUIRK_BROKEN_HPA) \|\| ata_id_is_locked(dev->id))
1342	return 0;
1343
1344	/* read native max address */
1345	rc = ata_read_native_max_address(dev, &native_sectors);
1346	if (rc) {
1347	/* If device aborted the command or HPA isn't going to
1348	* be unlocked, skip HPA resizing.
1349	*/
1350	if (rc == -EACCES \|\| !unlock_hpa) {
1351	ata_dev_warn(dev,
1352	"HPA support seems broken, skipping HPA handling\n");
1353	dev->quirks \|= ATA_QUIRK_BROKEN_HPA;
1354
1355	/* we can continue if device aborted the command */
1356	if (rc == -EACCES)
1357	rc = 0;
1358	}
1359
1360	return rc;
1361	}
1362	dev->n_native_sectors = native_sectors;
1363
1364	/* nothing to do? */
1365	if (native_sectors <= sectors \|\| !unlock_hpa) {
1366	if (!print_info \|\| native_sectors == sectors)
1367	return 0;
1368
1369	if (native_sectors > sectors)
1370	ata_dev_info(dev,
1371	"HPA detected: current %llu, native %llu\n",
1372	(unsigned long long)sectors,
1373	(unsigned long long)native_sectors);
1374	else if (native_sectors < sectors)
1375	ata_dev_warn(dev,
1376	"native sectors (%llu) is smaller than sectors (%llu)\n",
1377	(unsigned long long)native_sectors,
1378	(unsigned long long)sectors);
1379	return 0;
1380	}
1381
1382	/* let's unlock HPA */
1383	rc = ata_set_max_sectors(dev, native_sectors);
1384	if (rc == -EACCES) {
1385	/* if device aborted the command, skip HPA resizing */
1386	ata_dev_warn(dev,
1387	"device aborted resize (%llu -> %llu), skipping HPA handling\n",
1388	(unsigned long long)sectors,
1389	(unsigned long long)native_sectors);
1390	dev->quirks \|= ATA_QUIRK_BROKEN_HPA;
1391	return 0;
1392	} else if (rc)
1393	return rc;
1394
1395	/* re-read IDENTIFY data */
1396	rc = ata_dev_reread_id(dev, 0);
1397	if (rc) {
1398	ata_dev_err(dev,
1399	"failed to re-read IDENTIFY data after HPA resizing\n");
1400	return rc;
1401	}
1402
1403	if (print_info) {
1404	u64 new_sectors = ata_id_n_sectors(dev->id);
1405	ata_dev_info(dev,
1406	"HPA unlocked: %llu -> %llu, native %llu\n",
1407	(unsigned long long)sectors,
1408	(unsigned long long)new_sectors,
1409	(unsigned long long)native_sectors);
1410	}
1411
1412	return 0;
1413	}
1414
1415	/**
1416	* ata_dump_id - IDENTIFY DEVICE info debugging output
1417	* @dev: device from which the information is fetched
1418	* @id: IDENTIFY DEVICE page to dump
1419	*
1420	* Dump selected 16-bit words from the given IDENTIFY DEVICE
1421	* page.
1422	*
1423	* LOCKING:
1424	* caller.
1425	*/
1426
1427	static inline void ata_dump_id(struct ata_device dev, const u16 id)
1428	{
1429	ata_dev_dbg(dev,
1430	"49==0x%04x 53==0x%04x 63==0x%04x 64==0x%04x 75==0x%04x\n"
1431	"80==0x%04x 81==0x%04x 82==0x%04x 83==0x%04x 84==0x%04x\n"
1432	"88==0x%04x 93==0x%04x\n",
1433	id[49], id[53], id[63], id[64], id[75], id[80],
1434	id[81], id[82], id[83], id[84], id[88], id[93]);
1435	}
1436
1437	/**
1438	* ata_id_xfermask - Compute xfermask from the given IDENTIFY data
1439	* @id: IDENTIFY data to compute xfer mask from
1440	*
1441	* Compute the xfermask for this device. This is not as trivial
1442	* as it seems if we must consider early devices correctly.
1443	*
1444	* FIXME: pre IDE drive timing (do we care ?).
1445	*
1446	* LOCKING:
1447	* None.
1448	*
1449	* RETURNS:
1450	* Computed xfermask
1451	*/
1452	unsigned int ata_id_xfermask(const u16 *id)
1453	{
1454	unsigned int pio_mask, mwdma_mask, udma_mask;
1455
1456	/* Usual case. Word 53 indicates word 64 is valid */
1457	if (id[ATA_ID_FIELD_VALID] & (1 << 1)) {
1458	pio_mask = id[ATA_ID_PIO_MODES] & 0x03;
1459	pio_mask <<= 3;
1460	pio_mask \|= 0x7;
1461	} else {
1462	/* If word 64 isn't valid then Word 51 high byte holds
1463	* the PIO timing number for the maximum. Turn it into
1464	* a mask.
1465	*/
1466	u8 mode = (id[ATA_ID_OLD_PIO_MODES] >> 8) & 0xFF;
1467	if (mode < 5) /* Valid PIO range */
1468	pio_mask = (2 << mode) - 1;
1469	else
1470	pio_mask = 1;
1471
1472	/* But wait.. there's more. Design your standards by
1473	* committee and you too can get a free iordy field to
1474	* process. However it is the speeds not the modes that
1475	* are supported... Note drivers using the timing API
1476	* will get this right anyway
1477	*/
1478	}
1479
1480	mwdma_mask = id[ATA_ID_MWDMA_MODES] & 0x07;
1481
1482	if (ata_id_is_cfa(id)) {
1483	/*
1484	* Process compact flash extended modes
1485	*/
1486	int pio = (id[ATA_ID_CFA_MODES] >> 0) & 0x7;
1487	int dma = (id[ATA_ID_CFA_MODES] >> 3) & 0x7;
1488
1489	if (pio)
1490	pio_mask \|= (1 << 5);
1491	if (pio > 1)
1492	pio_mask \|= (1 << 6);
1493	if (dma)
1494	mwdma_mask \|= (1 << 3);
1495	if (dma > 1)
1496	mwdma_mask \|= (1 << 4);
1497	}
1498
1499	udma_mask = 0;
1500	if (id[ATA_ID_FIELD_VALID] & (1 << 2))
1501	udma_mask = id[ATA_ID_UDMA_MODES] & 0xff;
1502
1503	return ata_pack_xfermask(pio_mask, mwdma_mask, udma_mask);
1504	}
1505	EXPORT_SYMBOL_GPL(ata_id_xfermask);
1506
1507	static void ata_qc_complete_internal(struct ata_queued_cmd *qc)
1508	{
1509	struct completion *waiting = qc->private_data;
1510
1511	complete(waiting);
1512	}
1513
1514	/**
1515	* ata_exec_internal - execute libata internal command
1516	* @dev: Device to which the command is sent
1517	* @tf: Taskfile registers for the command and the result
1518	* @cdb: CDB for packet command
1519	* @dma_dir: Data transfer direction of the command
1520	* @buf: Data buffer of the command
1521	* @buflen: Length of data buffer
1522	* @timeout: Timeout in msecs (0 for default)
1523	*
1524	* Executes libata internal command with timeout. @tf contains
1525	* the command on entry and the result on return. Timeout and error
1526	* conditions are reported via the return value. No recovery action
1527	* is taken after a command times out. It is the caller's duty to
1528	* clean up after timeout.
1529	*
1530	* LOCKING:
1531	* None. Should be called with kernel context, might sleep.
1532	*
1533	* RETURNS:
1534	* Zero on success, AC_ERR_* mask on failure
1535	*/
1536	unsigned int ata_exec_internal(struct ata_device dev, struct ata_taskfile tf,
1537	const u8 *cdb, enum dma_data_direction dma_dir,
1538	void *buf, unsigned int buflen,
1539	unsigned int timeout)
1540	{
1541	struct ata_link *link = dev->link;
1542	struct ata_port *ap = link->ap;
1543	const bool owns_eh_mutex = ap->host->eh_owner == current;
1544	u8 command = tf->command;
1545	struct ata_queued_cmd *qc;
1546	struct scatterlist sgl;
1547	unsigned int preempted_tag;
1548	u32 preempted_sactive;
1549	u64 preempted_qc_active;
1550	int preempted_nr_active_links;
1551	bool auto_timeout = false;
1552	DECLARE_COMPLETION_ONSTACK(wait);
1553	unsigned long flags;
1554	unsigned int err_mask;
1555	int rc;
1556
1557	if (WARN_ON(dma_dir != DMA_NONE && !buf))
1558	return AC_ERR_INVALID;
1559
1560	spin_lock_irqsave(ap->lock, flags);
1561
1562	/* No internal command while frozen */
1563	if (ata_port_is_frozen(ap)) {
1564	spin_unlock_irqrestore(ap->lock, flags);
1565	return AC_ERR_SYSTEM;
1566	}
1567
1568	/* Initialize internal qc */
1569	qc = __ata_qc_from_tag(ap, ATA_TAG_INTERNAL);
1570
1571	qc->tag = ATA_TAG_INTERNAL;
1572	qc->hw_tag = 0;
1573	qc->scsicmd = NULL;
1574	qc->ap = ap;
1575	qc->dev = dev;
1576	ata_qc_reinit(qc);
1577
1578	preempted_tag = link->active_tag;
1579	preempted_sactive = link->sactive;
1580	preempted_qc_active = ap->qc_active;
1581	preempted_nr_active_links = ap->nr_active_links;
1582	link->active_tag = ATA_TAG_POISON;
1583	link->sactive = 0;
1584	ap->qc_active = 0;
1585	ap->nr_active_links = 0;
1586
1587	/* Prepare and issue qc */
1588	qc->tf = *tf;
1589	if (cdb)
1590	memcpy(qc->cdb, cdb, ATAPI_CDB_LEN);
1591
1592	/* Some SATA bridges need us to indicate data xfer direction */
1593	if (tf->protocol == ATAPI_PROT_DMA && (dev->flags & ATA_DFLAG_DMADIR) &&
1594	dma_dir == DMA_FROM_DEVICE)
1595	qc->tf.feature \|= ATAPI_DMADIR;
1596
1597	qc->flags \|= ATA_QCFLAG_RESULT_TF;
1598	qc->dma_dir = dma_dir;
1599	if (dma_dir != DMA_NONE) {
1600	sg_init_one(&sgl, buf, buflen);
1601	ata_sg_init(qc, &sgl, 1);
1602	qc->nbytes = buflen;
1603	}
1604
1605	qc->private_data = &wait;
1606	qc->complete_fn = ata_qc_complete_internal;
1607
1608	ata_qc_issue(ap, qc);
1609
1610	spin_unlock_irqrestore(ap->lock, flags);
1611
1612	if (!timeout) {
1613	if (ata_probe_timeout) {
1614	timeout = ata_probe_timeout * 1000;
1615	} else {
1616	timeout = ata_internal_cmd_timeout(dev, command);
1617	auto_timeout = true;
1618	}
1619	}
1620
1621	if (owns_eh_mutex) {
1622	/*
1623	* To prevent that the compiler complains about the
1624	* ata_eh_release() call below.
1625	*/
1626	__acquire(&ap->host->eh_mutex);
1627	ata_eh_release(ap);
1628	}
1629
1630	rc = wait_for_completion_timeout(&wait, msecs_to_jiffies(timeout));
1631
1632	if (owns_eh_mutex) {
1633	ata_eh_acquire(ap);
1634	/*
1635	* To prevent that the compiler complains about the above
1636	* ata_eh_acquire() call.
1637	*/
1638	__release(&ap->host->eh_mutex);
1639	}
1640
1641	ata_sff_flush_pio_task(ap);
1642
1643	if (!rc) {
1644	/*
1645	* We are racing with irq here. If we lose, the following test
1646	* prevents us from completing the qc twice. If we win, the port
1647	* is frozen and will be cleaned up by ->post_internal_cmd().
1648	*/
1649	spin_lock_irqsave(ap->lock, flags);
1650	if (qc->flags & ATA_QCFLAG_ACTIVE) {
1651	qc->err_mask \|= AC_ERR_TIMEOUT;
1652	ata_port_freeze(ap);
1653	ata_dev_warn(dev, "qc timeout after %u msecs (cmd 0x%x)\n",
1654	timeout, command);
1655	}
1656	spin_unlock_irqrestore(ap->lock, flags);
1657	}
1658
1659	if (ap->ops->post_internal_cmd)
1660	ap->ops->post_internal_cmd(qc);
1661
1662	/* Perform minimal error analysis */
1663	if (qc->flags & ATA_QCFLAG_EH) {
1664	if (qc->result_tf.status & (ATA_ERR \| ATA_DF))
1665	qc->err_mask \|= AC_ERR_DEV;
1666
1667	if (!qc->err_mask)
1668	qc->err_mask \|= AC_ERR_OTHER;
1669
1670	if (qc->err_mask & ~AC_ERR_OTHER)
1671	qc->err_mask &= ~AC_ERR_OTHER;
1672	} else if (qc->tf.command == ATA_CMD_REQ_SENSE_DATA) {
1673	qc->result_tf.status \|= ATA_SENSE;
1674	}
1675
1676	/* Finish up */
1677	spin_lock_irqsave(ap->lock, flags);
1678
1679	*tf = qc->result_tf;
1680	err_mask = qc->err_mask;
1681
1682	ata_qc_free(qc);
1683	link->active_tag = preempted_tag;
1684	link->sactive = preempted_sactive;
1685	ap->qc_active = preempted_qc_active;
1686	ap->nr_active_links = preempted_nr_active_links;
1687
1688	spin_unlock_irqrestore(ap->lock, flags);
1689
1690	if ((err_mask & AC_ERR_TIMEOUT) && auto_timeout)
1691	ata_internal_cmd_timed_out(dev, command);
1692
1693	return err_mask;
1694	}
1695
1696	/**
1697	* ata_pio_need_iordy - check if iordy needed
1698	* @adev: ATA device
1699	*
1700	* Check if the current speed of the device requires IORDY. Used
1701	* by various controllers for chip configuration.
1702	*/
1703	unsigned int ata_pio_need_iordy(const struct ata_device *adev)
1704	{
1705	/* Don't set IORDY if we're preparing for reset. IORDY may
1706	* lead to controller lock up on certain controllers if the
1707	* port is not occupied. See bko#11703 for details.
1708	*/
1709	if (adev->link->ap->pflags & ATA_PFLAG_RESETTING)
1710	return 0;
1711	/* Controller doesn't support IORDY. Probably a pointless
1712	* check as the caller should know this.
1713	*/
1714	if (adev->link->ap->flags & ATA_FLAG_NO_IORDY)
1715	return 0;
1716	/* CF spec. r4.1 Table 22 says no iordy on PIO5 and PIO6. */
1717	if (ata_id_is_cfa(adev->id)
1718	&& (adev->pio_mode == XFER_PIO_5 \|\| adev->pio_mode == XFER_PIO_6))
1719	return 0;
1720	/* PIO3 and higher it is mandatory */
1721	if (adev->pio_mode > XFER_PIO_2)
1722	return 1;
1723	/* We turn it on when possible */
1724	if (ata_id_has_iordy(adev->id))
1725	return 1;
1726	return 0;
1727	}
1728	EXPORT_SYMBOL_GPL(ata_pio_need_iordy);
1729
1730	/**
1731	* ata_pio_mask_no_iordy - Return the non IORDY mask
1732	* @adev: ATA device
1733	*
1734	* Compute the highest mode possible if we are not using iordy. Return
1735	* -1 if no iordy mode is available.
1736	*/
1737	static u32 ata_pio_mask_no_iordy(const struct ata_device *adev)
1738	{
1739	/* If we have no drive specific rule, then PIO 2 is non IORDY */
1740	if (adev->id[ATA_ID_FIELD_VALID] & 2) { /* EIDE */
1741	u16 pio = adev->id[ATA_ID_EIDE_PIO];
1742	/* Is the speed faster than the drive allows non IORDY ? */
1743	if (pio) {
1744	/* This is cycle times not frequency - watch the logic! */
1745	if (pio > 240) /* PIO2 is 240nS per cycle */
1746	return 3 << ATA_SHIFT_PIO;
1747	return 7 << ATA_SHIFT_PIO;
1748	}
1749	}
1750	return 3 << ATA_SHIFT_PIO;
1751	}
1752
1753	/**
1754	* ata_do_dev_read_id - default ID read method
1755	* @dev: device
1756	* @tf: proposed taskfile
1757	* @id: data buffer
1758	*
1759	* Issue the identify taskfile and hand back the buffer containing
1760	* identify data. For some RAID controllers and for pre ATA devices
1761	* this function is wrapped or replaced by the driver
1762	*/
1763	unsigned int ata_do_dev_read_id(struct ata_device *dev,
1764	struct ata_taskfile tf, __le16 id)
1765	{
1766	return ata_exec_internal(dev, tf, NULL, DMA_FROM_DEVICE,
1767	id, sizeof(id[0]) * ATA_ID_WORDS, 0);
1768	}
1769	EXPORT_SYMBOL_GPL(ata_do_dev_read_id);
1770
1771	/**
1772	* ata_dev_read_id - Read ID data from the specified device
1773	* @dev: target device
1774	* @p_class: pointer to class of the target device (may be changed)
1775	* @flags: ATA_READID_* flags
1776	* @id: buffer to read IDENTIFY data into
1777	*
1778	* Read ID data from the specified device. ATA_CMD_ID_ATA is
1779	* performed on ATA devices and ATA_CMD_ID_ATAPI on ATAPI
1780	* devices. This function also issues ATA_CMD_INIT_DEV_PARAMS
1781	* for pre-ATA4 drives.
1782	*
1783	* FIXME: ATA_CMD_ID_ATA is optional for early drives and right
1784	* now we abort if we hit that case.
1785	*
1786	* LOCKING:
1787	* Kernel thread context (may sleep)
1788	*
1789	* RETURNS:
1790	* 0 on success, -errno otherwise.
1791	*/
1792	int ata_dev_read_id(struct ata_device dev, unsigned int p_class,
1793	unsigned int flags, u16 *id)
1794	{
1795	struct ata_port *ap = dev->link->ap;
1796	unsigned int class = *p_class;
1797	struct ata_taskfile tf;
1798	unsigned int err_mask = 0;
1799	const char *reason;
1800	bool is_semb = class == ATA_DEV_SEMB;
1801	int may_fallback = 1, tried_spinup = 0;
1802	int rc;
1803
1804	retry:
1805	ata_tf_init(dev, &tf);
1806
1807	switch (class) {
1808	case ATA_DEV_SEMB:
1809	class = ATA_DEV_ATA; /* some hard drives report SEMB sig */
1810	fallthrough;
1811	case ATA_DEV_ATA:
1812	case ATA_DEV_ZAC:
1813	tf.command = ATA_CMD_ID_ATA;
1814	break;
1815	case ATA_DEV_ATAPI:
1816	tf.command = ATA_CMD_ID_ATAPI;
1817	break;
1818	default:
1819	rc = -ENODEV;
1820	reason = "unsupported class";
1821	goto err_out;
1822	}
1823
1824	tf.protocol = ATA_PROT_PIO;
1825
1826	/* Some devices choke if TF registers contain garbage. Make
1827	* sure those are properly initialized.
1828	*/
1829	tf.flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_DEVICE;
1830
1831	/* Device presence detection is unreliable on some
1832	* controllers. Always poll IDENTIFY if available.
1833	*/
1834	tf.flags \|= ATA_TFLAG_POLLING;
1835
1836	if (ap->ops->read_id)
1837	err_mask = ap->ops->read_id(dev, &tf, (__le16 *)id);
1838	else
1839	err_mask = ata_do_dev_read_id(dev, &tf, (__le16 *)id);
1840
1841	if (err_mask) {
1842	if (err_mask & AC_ERR_NODEV_HINT) {
1843	ata_dev_dbg(dev, "NODEV after polling detection\n");
1844	return -ENOENT;
1845	}
1846
1847	if (is_semb) {
1848	ata_dev_info(dev,
1849	"IDENTIFY failed on device w/ SEMB sig, disabled\n");
1850	/* SEMB is not supported yet */
1851	*p_class = ATA_DEV_SEMB_UNSUP;
1852	return 0;
1853	}
1854
1855	if ((err_mask == AC_ERR_DEV) && (tf.error & ATA_ABORTED)) {
1856	/* Device or controller might have reported
1857	* the wrong device class. Give a shot at the
1858	* other IDENTIFY if the current one is
1859	* aborted by the device.
1860	*/
1861	if (may_fallback) {
1862	may_fallback = 0;
1863
1864	if (class == ATA_DEV_ATA)
1865	class = ATA_DEV_ATAPI;
1866	else
1867	class = ATA_DEV_ATA;
1868	goto retry;
1869	}
1870
1871	/* Control reaches here iff the device aborted
1872	* both flavors of IDENTIFYs which happens
1873	* sometimes with phantom devices.
1874	*/
1875	ata_dev_dbg(dev,
1876	"both IDENTIFYs aborted, assuming NODEV\n");
1877	return -ENOENT;
1878	}
1879
1880	rc = -EIO;
1881	reason = "I/O error";
1882	goto err_out;
1883	}
1884
1885	if (dev->quirks & ATA_QUIRK_DUMP_ID) {
1886	ata_dev_info(dev, "dumping IDENTIFY data, "
1887	"class=%d may_fallback=%d tried_spinup=%d\n",
1888	class, may_fallback, tried_spinup);
1889	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET,
1890	16, 2, id, ATA_ID_WORDS * sizeof(*id), true);
1891	}
1892
1893	/* Falling back doesn't make sense if ID data was read
1894	* successfully at least once.
1895	*/
1896	may_fallback = 0;
1897
1898	swap_buf_le16(id, ATA_ID_WORDS);
1899
1900	/* sanity check */
1901	rc = -EINVAL;
1902	reason = "device reports invalid type";
1903
1904	if (class == ATA_DEV_ATA \|\| class == ATA_DEV_ZAC) {
1905	if (!ata_id_is_ata(id) && !ata_id_is_cfa(id))
1906	goto err_out;
1907	if (ap->host->flags & ATA_HOST_IGNORE_ATA &&
1908	ata_id_is_ata(id)) {
1909	ata_dev_dbg(dev,
1910	"host indicates ignore ATA devices, ignored\n");
1911	return -ENOENT;
1912	}
1913	} else {
1914	if (ata_id_is_ata(id))
1915	goto err_out;
1916	}
1917
1918	if (!tried_spinup && (id[2] == 0x37c8 \|\| id[2] == 0x738c)) {
1919	tried_spinup = 1;
1920	/*
1921	* Drive powered-up in standby mode, and requires a specific
1922	* SET_FEATURES spin-up subcommand before it will accept
1923	* anything other than the original IDENTIFY command.
1924	*/
1925	err_mask = ata_dev_set_feature(dev, SETFEATURES_SPINUP, 0);
1926	if (err_mask && id[2] != 0x738c) {
1927	rc = -EIO;
1928	reason = "SPINUP failed";
1929	goto err_out;
1930	}
1931	/*
1932	* If the drive initially returned incomplete IDENTIFY info,
1933	* we now must reissue the IDENTIFY command.
1934	*/
1935	if (id[2] == 0x37c8)
1936	goto retry;
1937	}
1938
1939	if ((flags & ATA_READID_POSTRESET) &&
1940	(class == ATA_DEV_ATA \|\| class == ATA_DEV_ZAC)) {
1941	/*
1942	* The exact sequence expected by certain pre-ATA4 drives is:
1943	* SRST RESET
1944	* IDENTIFY (optional in early ATA)
1945	* INITIALIZE DEVICE PARAMETERS (later IDE and ATA)
1946	* anything else..
1947	* Some drives were very specific about that exact sequence.
1948	*
1949	* Note that ATA4 says lba is mandatory so the second check
1950	* should never trigger.
1951	*/
1952	if (ata_id_major_version(id) < 4 \|\| !ata_id_has_lba(id)) {
1953	err_mask = ata_dev_init_params(dev, id[3], id[6]);
1954	if (err_mask) {
1955	rc = -EIO;
1956	reason = "INIT_DEV_PARAMS failed";
1957	goto err_out;
1958	}
1959
1960	/* current CHS translation info (id[53-58]) might be
1961	* changed. reread the identify device info.
1962	*/
1963	flags &= ~ATA_READID_POSTRESET;
1964	goto retry;
1965	}
1966	}
1967
1968	*p_class = class;
1969
1970	return 0;
1971
1972	err_out:
1973	ata_dev_warn(dev, "failed to IDENTIFY (%s, err_mask=0x%x)\n",
1974	reason, err_mask);
1975	return rc;
1976	}
1977
1978	bool ata_dev_power_init_tf(struct ata_device dev, struct ata_taskfile tf,
1979	bool set_active)
1980	{
1981	/* Only applies to ATA and ZAC devices */
1982	if (dev->class != ATA_DEV_ATA && dev->class != ATA_DEV_ZAC)
1983	return false;
1984
1985	ata_tf_init(dev, tf);
1986	tf->flags \|= ATA_TFLAG_DEVICE \| ATA_TFLAG_ISADDR;
1987	tf->protocol = ATA_PROT_NODATA;
1988
1989	if (set_active) {
1990	/* VERIFY for 1 sector at lba=0 */
1991	tf->command = ATA_CMD_VERIFY;
1992	tf->nsect = 1;
1993	if (dev->flags & ATA_DFLAG_LBA) {
1994	tf->flags \|= ATA_TFLAG_LBA;
1995	tf->device \|= ATA_LBA;
1996	} else {
1997	/* CHS */
1998	tf->lbal = 0x1; /* sect */
1999	}
2000	} else {
2001	tf->command = ATA_CMD_STANDBYNOW1;
2002	}
2003
2004	return true;
2005	}
2006
2007	static bool ata_dev_power_is_active(struct ata_device *dev)
2008	{
2009	struct ata_taskfile tf;
2010	unsigned int err_mask;
2011
2012	ata_tf_init(dev, &tf);
2013	tf.flags \|= ATA_TFLAG_DEVICE \| ATA_TFLAG_ISADDR;
2014	tf.protocol = ATA_PROT_NODATA;
2015	tf.command = ATA_CMD_CHK_POWER;
2016
2017	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
2018	if (err_mask) {
2019	ata_dev_err(dev, "Check power mode failed (err_mask=0x%x)\n",
2020	err_mask);
2021	/*
2022	* Assume we are in standby mode so that we always force a
2023	* spinup in ata_dev_power_set_active().
2024	*/
2025	return false;
2026	}
2027
2028	ata_dev_dbg(dev, "Power mode: 0x%02x\n", tf.nsect);
2029
2030	/* Active or idle */
2031	return tf.nsect == 0xff;
2032	}
2033
2034	/**
2035	* ata_dev_power_set_standby - Set a device power mode to standby
2036	* @dev: target device
2037	*
2038	* Issue a STANDBY IMMEDIATE command to set a device power mode to standby.
2039	* For an HDD device, this spins down the disks.
2040	*
2041	* LOCKING:
2042	* Kernel thread context (may sleep).
2043	*/
2044	void ata_dev_power_set_standby(struct ata_device *dev)
2045	{
2046	unsigned long ap_flags = dev->link->ap->flags;
2047	struct ata_taskfile tf;
2048	unsigned int err_mask;
2049
2050	/* If the device is already sleeping or in standby, do nothing. */
2051	if ((dev->flags & ATA_DFLAG_SLEEPING) \|\|
2052	!ata_dev_power_is_active(dev))
2053	return;
2054
2055	/*
2056	* Some odd clown BIOSes issue spindown on power off (ACPI S4 or S5)
2057	* causing some drives to spin up and down again. For these, do nothing
2058	* if we are being called on shutdown.
2059	*/
2060	if ((ap_flags & ATA_FLAG_NO_POWEROFF_SPINDOWN) &&
2061	system_state == SYSTEM_POWER_OFF)
2062	return;
2063
2064	if ((ap_flags & ATA_FLAG_NO_HIBERNATE_SPINDOWN) &&
2065	system_entering_hibernation())
2066	return;
2067
2068	/* Issue STANDBY IMMEDIATE command only if supported by the device */
2069	if (!ata_dev_power_init_tf(dev, &tf, false))
2070	return;
2071
2072	ata_dev_notice(dev, "Entering standby power mode\n");
2073
2074	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
2075	if (err_mask)
2076	ata_dev_err(dev, "STANDBY IMMEDIATE failed (err_mask=0x%x)\n",
2077	err_mask);
2078	}
2079
2080	/**
2081	* ata_dev_power_set_active - Set a device power mode to active
2082	* @dev: target device
2083	*
2084	* Issue a VERIFY command to enter to ensure that the device is in the
2085	* active power mode. For a spun-down HDD (standby or idle power mode),
2086	* the VERIFY command will complete after the disk spins up.
2087	*
2088	* LOCKING:
2089	* Kernel thread context (may sleep).
2090	*/
2091	void ata_dev_power_set_active(struct ata_device *dev)
2092	{
2093	struct ata_taskfile tf;
2094	unsigned int err_mask;
2095
2096	/*
2097	* Issue READ VERIFY SECTORS command for 1 sector at lba=0 only
2098	* if supported by the device.
2099	*/
2100	if (!ata_dev_power_init_tf(dev, &tf, true))
2101	return;
2102
2103	/*
2104	* Check the device power state & condition and force a spinup with
2105	* VERIFY command only if the drive is not already ACTIVE or IDLE.
2106	*/
2107	if (ata_dev_power_is_active(dev))
2108	return;
2109
2110	ata_dev_notice(dev, "Entering active power mode\n");
2111
2112	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
2113	if (err_mask)
2114	ata_dev_err(dev, "VERIFY failed (err_mask=0x%x)\n",
2115	err_mask);
2116	}
2117
2118	/**
2119	* ata_read_log_page - read a specific log page
2120	* @dev: target device
2121	* @log: log to read
2122	* @page: page to read
2123	* @buf: buffer to store read page
2124	* @sectors: number of sectors to read
2125	*
2126	* Read log page using READ_LOG_EXT command.
2127	*
2128	* LOCKING:
2129	* Kernel thread context (may sleep).
2130	*
2131	* RETURNS:
2132	* 0 on success, AC_ERR_* mask otherwise.
2133	*/
2134	unsigned int ata_read_log_page(struct ata_device *dev, u8 log,
2135	u8 page, void *buf, unsigned int sectors)
2136	{
2137	unsigned long ap_flags = dev->link->ap->flags;
2138	struct ata_taskfile tf;
2139	unsigned int err_mask;
2140	bool dma = false;
2141
2142	ata_dev_dbg(dev, "read log page - log 0x%x, page 0x%x\n", log, page);
2143
2144	/*
2145	* Return error without actually issuing the command on controllers
2146	* which e.g. lockup on a read log page.
2147	*/
2148	if (ap_flags & ATA_FLAG_NO_LOG_PAGE)
2149	return AC_ERR_DEV;
2150
2151	retry:
2152	ata_tf_init(dev, &tf);
2153	if (ata_dma_enabled(dev) && ata_id_has_read_log_dma_ext(dev->id) &&
2154	!(dev->quirks & ATA_QUIRK_NO_DMA_LOG)) {
2155	tf.command = ATA_CMD_READ_LOG_DMA_EXT;
2156	tf.protocol = ATA_PROT_DMA;
2157	dma = true;
2158	} else {
2159	tf.command = ATA_CMD_READ_LOG_EXT;
2160	tf.protocol = ATA_PROT_PIO;
2161	dma = false;
2162	}
2163	tf.lbal = log;
2164	tf.lbam = page;
2165	tf.nsect = sectors;
2166	tf.hob_nsect = sectors >> 8;
2167	tf.flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_LBA48 \| ATA_TFLAG_DEVICE;
2168
2169	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_FROM_DEVICE,
2170	buf, sectors * ATA_SECT_SIZE, 0);
2171
2172	if (err_mask) {
2173	if (dma) {
2174	dev->quirks \|= ATA_QUIRK_NO_DMA_LOG;
2175	if (!ata_port_is_frozen(dev->link->ap))
2176	goto retry;
2177	}
2178	ata_dev_err(dev,
2179	"Read log 0x%02x page 0x%02x failed, Emask 0x%x\n",
2180	(unsigned int)log, (unsigned int)page, err_mask);
2181	}
2182
2183	return err_mask;
2184	}
2185
2186	static inline void ata_clear_log_directory(struct ata_device *dev)
2187	{
2188	memset(dev->gp_log_dir, 0, ATA_SECT_SIZE);
2189	}
2190
2191	static int ata_read_log_directory(struct ata_device *dev)
2192	{
2193	u16 version;
2194
2195	/* If the log page is already cached, do nothing. */
2196	version = get_unaligned_le16(&dev->gp_log_dir[0]);
2197	if (version == 0x0001)
2198	return 0;
2199
2200	if (ata_read_log_page(dev, ATA_LOG_DIRECTORY, 0, dev->gp_log_dir, 1)) {
2201	ata_clear_log_directory(dev);
2202	return -EIO;
2203	}
2204
2205	version = get_unaligned_le16(&dev->gp_log_dir[0]);
2206	if (version != 0x0001)
2207	ata_dev_warn_once(dev,
2208	"Invalid log directory version 0x%04x\n",
2209	version);
2210
2211	return 0;
2212	}
2213
2214	static int ata_log_supported(struct ata_device *dev, u8 log)
2215	{
2216	if (dev->quirks & ATA_QUIRK_NO_LOG_DIR)
2217	return 0;
2218
2219	if (ata_read_log_directory(dev))
2220	return 0;
2221
2222	return get_unaligned_le16(&dev->gp_log_dir[log * 2]);
2223	}
2224
2225	static bool ata_identify_page_supported(struct ata_device *dev, u8 page)
2226	{
2227	unsigned int err, i;
2228
2229	if (dev->quirks & ATA_QUIRK_NO_ID_DEV_LOG)
2230	return false;
2231
2232	if (!ata_log_supported(dev, ATA_LOG_IDENTIFY_DEVICE)) {
2233	/*
2234	* IDENTIFY DEVICE data log is defined as mandatory starting
2235	* with ACS-3 (ATA version 10). Warn about the missing log
2236	* for drives which implement this ATA level or above.
2237	*/
2238	if (ata_id_major_version(dev->id) >= 10)
2239	ata_dev_warn(dev,
2240	"ATA Identify Device Log not supported\n");
2241	dev->quirks \|= ATA_QUIRK_NO_ID_DEV_LOG;
2242	return false;
2243	}
2244
2245	/*
2246	* Read IDENTIFY DEVICE data log, page 0, to figure out if the page is
2247	* supported.
2248	*/
2249	err = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE, 0,
2250	dev->sector_buf, 1);
2251	if (err)
2252	return false;
2253
2254	for (i = 0; i < dev->sector_buf[8]; i++) {
2255	if (dev->sector_buf[9 + i] == page)
2256	return true;
2257	}
2258
2259	return false;
2260	}
2261
2262	static int ata_do_link_spd_quirk(struct ata_device *dev)
2263	{
2264	struct ata_link *plink = ata_dev_phys_link(dev);
2265	u32 target, target_limit;
2266
2267	if (!sata_scr_valid(plink))
2268	return 0;
2269
2270	if (dev->quirks & ATA_QUIRK_1_5_GBPS)
2271	target = 1;
2272	else
2273	return 0;
2274
2275	target_limit = (1 << target) - 1;
2276
2277	/* if already on stricter limit, no need to push further */
2278	if (plink->sata_spd_limit <= target_limit)
2279	return 0;
2280
2281	plink->sata_spd_limit = target_limit;
2282
2283	/* Request another EH round by returning -EAGAIN if link is
2284	* going faster than the target speed. Forward progress is
2285	* guaranteed by setting sata_spd_limit to target_limit above.
2286	*/
2287	if (plink->sata_spd > target) {
2288	ata_dev_info(dev, "applying link speed limit quirk to %s\n",
2289	sata_spd_string(target));
2290	return -EAGAIN;
2291	}
2292	return 0;
2293	}
2294
2295	static inline bool ata_dev_knobble(struct ata_device *dev)
2296	{
2297	struct ata_port *ap = dev->link->ap;
2298
2299	if (ata_dev_quirks(dev) & ATA_QUIRK_BRIDGE_OK)
2300	return false;
2301
2302	return ((ap->cbl == ATA_CBL_SATA) && (!ata_id_is_sata(dev->id)));
2303	}
2304
2305	static void ata_dev_config_ncq_send_recv(struct ata_device *dev)
2306	{
2307	unsigned int err_mask;
2308
2309	if (!ata_log_supported(dev, ATA_LOG_NCQ_SEND_RECV)) {
2310	ata_dev_warn(dev, "NCQ Send/Recv Log not supported\n");
2311	return;
2312	}
2313	err_mask = ata_read_log_page(dev, ATA_LOG_NCQ_SEND_RECV,
2314	0, dev->sector_buf, 1);
2315	if (!err_mask) {
2316	u8 *cmds = dev->ncq_send_recv_cmds;
2317
2318	dev->flags \|= ATA_DFLAG_NCQ_SEND_RECV;
2319	memcpy(cmds, dev->sector_buf, ATA_LOG_NCQ_SEND_RECV_SIZE);
2320
2321	if (dev->quirks & ATA_QUIRK_NO_NCQ_TRIM) {
2322	ata_dev_dbg(dev, "disabling queued TRIM support\n");
2323	cmds[ATA_LOG_NCQ_SEND_RECV_DSM_OFFSET] &=
2324	~ATA_LOG_NCQ_SEND_RECV_DSM_TRIM;
2325	}
2326	}
2327	}
2328
2329	static void ata_dev_config_ncq_non_data(struct ata_device *dev)
2330	{
2331	unsigned int err_mask;
2332
2333	if (!ata_log_supported(dev, ATA_LOG_NCQ_NON_DATA)) {
2334	ata_dev_warn(dev,
2335	"NCQ Non-Data Log not supported\n");
2336	return;
2337	}
2338	err_mask = ata_read_log_page(dev, ATA_LOG_NCQ_NON_DATA,
2339	0, dev->sector_buf, 1);
2340	if (!err_mask)
2341	memcpy(dev->ncq_non_data_cmds, dev->sector_buf,
2342	ATA_LOG_NCQ_NON_DATA_SIZE);
2343	}
2344
2345	static void ata_dev_config_ncq_prio(struct ata_device *dev)
2346	{
2347	unsigned int err_mask;
2348
2349	if (!ata_identify_page_supported(dev, ATA_LOG_SATA_SETTINGS))
2350	return;
2351
2352	err_mask = ata_read_log_page(dev,
2353	ATA_LOG_IDENTIFY_DEVICE,
2354	ATA_LOG_SATA_SETTINGS,
2355	dev->sector_buf, 1);
2356	if (err_mask)
2357	goto not_supported;
2358
2359	if (!(dev->sector_buf[ATA_LOG_NCQ_PRIO_OFFSET] & BIT(3)))
2360	goto not_supported;
2361
2362	dev->flags \|= ATA_DFLAG_NCQ_PRIO;
2363
2364	return;
2365
2366	not_supported:
2367	dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLED;
2368	dev->flags &= ~ATA_DFLAG_NCQ_PRIO;
2369	}
2370
2371	static bool ata_dev_check_adapter(struct ata_device *dev,
2372	unsigned short vendor_id)
2373	{
2374	struct pci_dev *pcidev = NULL;
2375	struct device *parent_dev = NULL;
2376
2377	for (parent_dev = dev->tdev.parent; parent_dev != NULL;
2378	parent_dev = parent_dev->parent) {
2379	if (dev_is_pci(parent_dev)) {
2380	pcidev = to_pci_dev(parent_dev);
2381	if (pcidev->vendor == vendor_id)
2382	return true;
2383	break;
2384	}
2385	}
2386
2387	return false;
2388	}
2389
2390	bool ata_adapter_is_online(struct ata_port *ap)
2391	{
2392	struct device *dev;
2393
2394	if (!ap \|\| !ap->host)
2395	return false;
2396
2397	dev = ap->host->dev;
2398	if (!dev)
2399	return false;
2400
2401	if (dev_is_pci(dev) &&
2402	pci_channel_offline(to_pci_dev(dev)))
2403	return false;
2404
2405	return true;
2406	}
2407
2408	static int ata_dev_config_ncq(struct ata_device *dev,
2409	char *desc, size_t desc_sz)
2410	{
2411	struct ata_port *ap = dev->link->ap;
2412	int hdepth = 0, ddepth = ata_id_queue_depth(dev->id);
2413	unsigned int err_mask;
2414	char *aa_desc = "";
2415
2416	if (!ata_id_has_ncq(dev->id)) {
2417	desc[0] = '\0';
2418	return 0;
2419	}
2420	if (!IS_ENABLED(CONFIG_SATA_HOST))
2421	return 0;
2422	if (dev->quirks & ATA_QUIRK_NONCQ) {
2423	snprintf(desc, desc_sz, "NCQ (not used)");
2424	return 0;
2425	}
2426
2427	if (dev->quirks & ATA_QUIRK_NO_NCQ_ON_ATI &&
2428	ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI)) {
2429	snprintf(desc, desc_sz, "NCQ (not used)");
2430	return 0;
2431	}
2432
2433	if (ap->flags & ATA_FLAG_NCQ) {
2434	hdepth = min(ap->scsi_host->can_queue, ATA_MAX_QUEUE);
2435	dev->flags \|= ATA_DFLAG_NCQ;
2436	}
2437
2438	if (!(dev->quirks & ATA_QUIRK_BROKEN_FPDMA_AA) &&
2439	(ap->flags & ATA_FLAG_FPDMA_AA) &&
2440	ata_id_has_fpdma_aa(dev->id)) {
2441	err_mask = ata_dev_set_feature(dev, SETFEATURES_SATA_ENABLE,
2442	SATA_FPDMA_AA);
2443	if (err_mask) {
2444	ata_dev_err(dev,
2445	"failed to enable AA (error_mask=0x%x)\n",
2446	err_mask);
2447	if (err_mask != AC_ERR_DEV) {
2448	dev->quirks \|= ATA_QUIRK_BROKEN_FPDMA_AA;
2449	return -EIO;
2450	}
2451	} else
2452	aa_desc = ", AA";
2453	}
2454
2455	if (hdepth >= ddepth)
2456	snprintf(desc, desc_sz, "NCQ (depth %d)%s", ddepth, aa_desc);
2457	else
2458	snprintf(desc, desc_sz, "NCQ (depth %d/%d)%s", hdepth,
2459	ddepth, aa_desc);
2460
2461	if ((ap->flags & ATA_FLAG_FPDMA_AUX)) {
2462	if (ata_id_has_ncq_send_and_recv(dev->id))
2463	ata_dev_config_ncq_send_recv(dev);
2464	if (ata_id_has_ncq_non_data(dev->id))
2465	ata_dev_config_ncq_non_data(dev);
2466	if (ata_id_has_ncq_prio(dev->id))
2467	ata_dev_config_ncq_prio(dev);
2468	}
2469
2470	return 0;
2471	}
2472
2473	static void ata_dev_config_sense_reporting(struct ata_device *dev)
2474	{
2475	unsigned int err_mask;
2476
2477	if (!ata_id_has_sense_reporting(dev->id))
2478	return;
2479
2480	if (ata_id_sense_reporting_enabled(dev->id))
2481	return;
2482
2483	err_mask = ata_dev_set_feature(dev, SETFEATURE_SENSE_DATA, 0x1);
2484	if (err_mask) {
2485	ata_dev_dbg(dev,
2486	"failed to enable Sense Data Reporting, Emask 0x%x\n",
2487	err_mask);
2488	}
2489	}
2490
2491	static void ata_dev_config_zoned(struct ata_device *dev)
2492	{
2493	unsigned int err_mask;
2494	u8 *identify_buf = dev->sector_buf;
2495
2496	dev->zac_zones_optimal_open = U32_MAX;
2497	dev->zac_zones_optimal_nonseq = U32_MAX;
2498	dev->zac_zones_max_open = U32_MAX;
2499
2500	if (!ata_dev_is_zoned(dev))
2501	return;
2502
2503	if (!ata_identify_page_supported(dev, ATA_LOG_ZONED_INFORMATION)) {
2504	ata_dev_warn(dev,
2505	"ATA Zoned Information Log not supported\n");
2506	return;
2507	}
2508
2509	/*
2510	* Read IDENTIFY DEVICE data log, page 9 (Zoned-device information)
2511	*/
2512	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
2513	ATA_LOG_ZONED_INFORMATION,
2514	identify_buf, 1);
2515	if (!err_mask) {
2516	u64 zoned_cap, opt_open, opt_nonseq, max_open;
2517
2518	zoned_cap = get_unaligned_le64(&identify_buf[8]);
2519	if ((zoned_cap >> 63))
2520	dev->zac_zoned_cap = (zoned_cap & 1);
2521	opt_open = get_unaligned_le64(&identify_buf[24]);
2522	if ((opt_open >> 63))
2523	dev->zac_zones_optimal_open = (u32)opt_open;
2524	opt_nonseq = get_unaligned_le64(&identify_buf[32]);
2525	if ((opt_nonseq >> 63))
2526	dev->zac_zones_optimal_nonseq = (u32)opt_nonseq;
2527	max_open = get_unaligned_le64(&identify_buf[40]);
2528	if ((max_open >> 63))
2529	dev->zac_zones_max_open = (u32)max_open;
2530	}
2531	}
2532
2533	static void ata_dev_config_trusted(struct ata_device *dev)
2534	{
2535	u64 trusted_cap;
2536	unsigned int err;
2537
2538	if (!ata_id_has_trusted(dev->id))
2539	return;
2540
2541	if (!ata_identify_page_supported(dev, ATA_LOG_SECURITY)) {
2542	ata_dev_warn(dev,
2543	"Security Log not supported\n");
2544	return;
2545	}
2546
2547	err = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE, ATA_LOG_SECURITY,
2548	dev->sector_buf, 1);
2549	if (err)
2550	return;
2551
2552	trusted_cap = get_unaligned_le64(&dev->sector_buf[40]);
2553	if (!(trusted_cap & (1ULL << 63))) {
2554	ata_dev_dbg(dev,
2555	"Trusted Computing capability qword not valid!\n");
2556	return;
2557	}
2558
2559	if (trusted_cap & (1 << 0))
2560	dev->flags \|= ATA_DFLAG_TRUSTED;
2561	}
2562
2563	static void ata_dev_cleanup_cdl_resources(struct ata_device *dev)
2564	{
2565	kfree(dev->cdl);
2566	dev->cdl = NULL;
2567	}
2568
2569	static int ata_dev_init_cdl_resources(struct ata_device *dev)
2570	{
2571	struct ata_cdl *cdl = dev->cdl;
2572	unsigned int err_mask;
2573
2574	if (!cdl) {
2575	cdl = kzalloc_obj(*cdl);
2576	if (!cdl)
2577	return -ENOMEM;
2578	dev->cdl = cdl;
2579	}
2580
2581	err_mask = ata_read_log_page(dev, ATA_LOG_CDL, 0, cdl->desc_log_buf,
2582	ATA_LOG_CDL_SIZE / ATA_SECT_SIZE);
2583	if (err_mask) {
2584	ata_dev_warn(dev, "Read Command Duration Limits log failed\n");
2585	ata_dev_cleanup_cdl_resources(dev);
2586	return -EIO;
2587	}
2588
2589	return 0;
2590	}
2591
2592	static void ata_dev_config_cdl(struct ata_device *dev)
2593	{
2594	unsigned int err_mask;
2595	bool cdl_enabled;
2596	u64 val;
2597	int ret;
2598
2599	if (ata_id_major_version(dev->id) < 11)
2600	goto not_supported;
2601
2602	if (!ata_log_supported(dev, ATA_LOG_IDENTIFY_DEVICE) \|\|
2603	!ata_identify_page_supported(dev, ATA_LOG_SUPPORTED_CAPABILITIES) \|\|
2604	!ata_identify_page_supported(dev, ATA_LOG_CURRENT_SETTINGS))
2605	goto not_supported;
2606
2607	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
2608	ATA_LOG_SUPPORTED_CAPABILITIES,
2609	dev->sector_buf, 1);
2610	if (err_mask)
2611	goto not_supported;
2612
2613	/* Check Command Duration Limit Supported bits */
2614	val = get_unaligned_le64(&dev->sector_buf[168]);
2615	if (!(val & BIT_ULL(63)) \|\| !(val & BIT_ULL(0)))
2616	goto not_supported;
2617
2618	/* Warn the user if command duration guideline is not supported */
2619	if (!(val & BIT_ULL(1)))
2620	ata_dev_warn(dev,
2621	"Command duration guideline is not supported\n");
2622
2623	/*
2624	* We must have support for the sense data for successful NCQ commands
2625	* log indicated by the successful NCQ command sense data supported bit.
2626	*/
2627	val = get_unaligned_le64(&dev->sector_buf[8]);
2628	if (!(val & BIT_ULL(63)) \|\| !(val & BIT_ULL(47))) {
2629	ata_dev_warn(dev,
2630	"CDL supported but Successful NCQ Command Sense Data is not supported\n");
2631	goto not_supported;
2632	}
2633
2634	/* Without NCQ autosense, the successful NCQ commands log is useless. */
2635	if (!ata_id_has_ncq_autosense(dev->id)) {
2636	ata_dev_warn(dev,
2637	"CDL supported but NCQ autosense is not supported\n");
2638	goto not_supported;
2639	}
2640
2641	/*
2642	* If CDL is marked as enabled, make sure the feature is enabled too.
2643	* Conversely, if CDL is disabled, make sure the feature is turned off.
2644	*/
2645	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
2646	ATA_LOG_CURRENT_SETTINGS,
2647	dev->sector_buf, 1);
2648	if (err_mask)
2649	goto not_supported;
2650
2651	val = get_unaligned_le64(&dev->sector_buf[8]);
2652	cdl_enabled = val & BIT_ULL(63) && val & BIT_ULL(21);
2653	if (dev->flags & ATA_DFLAG_CDL_ENABLED) {
2654	if (!cdl_enabled) {
2655	/* Enable CDL on the device */
2656	err_mask = ata_dev_set_feature(dev, SETFEATURES_CDL, 1);
2657	if (err_mask) {
2658	ata_dev_err(dev,
2659	"Enable CDL feature failed\n");
2660	goto not_supported;
2661	}
2662	}
2663	} else {
2664	if (cdl_enabled) {
2665	/* Disable CDL on the device */
2666	err_mask = ata_dev_set_feature(dev, SETFEATURES_CDL, 0);
2667	if (err_mask) {
2668	ata_dev_err(dev,
2669	"Disable CDL feature failed\n");
2670	goto not_supported;
2671	}
2672	}
2673	}
2674
2675	/*
2676	* While CDL itself has to be enabled using sysfs, CDL requires that
2677	* sense data for successful NCQ commands is enabled to work properly.
2678	* Just like ata_dev_config_sense_reporting(), enable it unconditionally
2679	* if supported.
2680	*/
2681	if (!(val & BIT_ULL(63)) \|\| !(val & BIT_ULL(18))) {
2682	err_mask = ata_dev_set_feature(dev,
2683	SETFEATURE_SENSE_DATA_SUCC_NCQ, 0x1);
2684	if (err_mask) {
2685	ata_dev_warn(dev,
2686	"failed to enable Sense Data for successful NCQ commands, Emask 0x%x\n",
2687	err_mask);
2688	goto not_supported;
2689	}
2690	}
2691
2692	/* CDL is supported: allocate and initialize needed resources. */
2693	ret = ata_dev_init_cdl_resources(dev);
2694	if (ret) {
2695	ata_dev_warn(dev, "Initialize CDL resources failed\n");
2696	goto not_supported;
2697	}
2698
2699	dev->flags \|= ATA_DFLAG_CDL;
2700
2701	return;
2702
2703	not_supported:
2704	dev->flags &= ~(ATA_DFLAG_CDL \| ATA_DFLAG_CDL_ENABLED);
2705	ata_dev_cleanup_cdl_resources(dev);
2706	}
2707
2708	static void ata_dev_config_depop(struct ata_device *dev)
2709	{
2710	unsigned int err_mask;
2711	u64 val;
2712
2713	/* Ignore old drives. */
2714	if (ata_id_major_version(dev->id) < 11)
2715	goto not_supported;
2716
2717	/* NCQ Autosense is required. */
2718	if (!ata_identify_page_supported(dev, ATA_LOG_SUPPORTED_CAPABILITIES) \|\|
2719	!ata_id_has_ncq_autosense(dev->id))
2720	goto not_supported;
2721
2722	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
2723	ATA_LOG_SUPPORTED_CAPABILITIES,
2724	dev->sector_buf, 1);
2725	if (err_mask)
2726	goto not_supported;
2727
2728	/* Check depopulation capabilities bits. */
2729	val = get_unaligned_le64(&dev->sector_buf[152]);
2730	if (!(val & BIT_ULL(63)))
2731	goto not_supported;
2732
2733	/*
2734	* Support for at least the GET PHYSICAL ELEMENT STATUS and
2735	* REMOVE ELEMENT AND TRUNCATE commands is mandated.
2736	*/
2737	if (!(val & BIT_ULL(0)) \|\| !(val & BIT_ULL(1)))
2738	goto not_supported;
2739
2740	dev->flags \|= ATA_DFLAG_DEPOP;
2741
2742	/* Check if RESTORE ELEMENTS AND REBUILD is supported. */
2743	if (val & BIT_ULL(2))
2744	dev->flags \|= ATA_DFLAG_DEPOP_RESTORE;
2745
2746	/*
2747	* For ZAC devices, check if REMOVE ELEMENT AND MODIFY ZONES is
2748	* supported.
2749	*/
2750	if (dev->class != ATA_DEV_ZAC)
2751	return;
2752
2753	err_mask = ata_read_log_page(dev, ATA_LOG_IDENTIFY_DEVICE,
2754	ATA_LOG_ZONED_INFORMATION,
2755	dev->sector_buf, 1);
2756	if (err_mask)
2757	return;
2758
2759	val = get_unaligned_le64(&dev->sector_buf[8]);
2760	if (!(val & BIT_ULL(63)))
2761	return;
2762
2763	if (val & BIT_ULL(1))
2764	dev->flags \|= ATA_DFLAG_DEPOP_MODIFY;
2765
2766	return;
2767
2768	not_supported:
2769	dev->flags &= ~(ATA_DFLAG_DEPOP \| ATA_DFLAG_DEPOP_RESTORE \|
2770	ATA_DFLAG_DEPOP_MODIFY);
2771	}
2772
2773	static int ata_dev_config_lba(struct ata_device *dev)
2774	{
2775	const u16 *id = dev->id;
2776	const char *lba_desc;
2777	char ncq_desc[32];
2778	int ret;
2779
2780	dev->flags \|= ATA_DFLAG_LBA;
2781
2782	if (ata_id_has_lba48(id)) {
2783	lba_desc = "LBA48";
2784	dev->flags \|= ATA_DFLAG_LBA48;
2785	if (dev->n_sectors >= (1UL << 28) &&
2786	ata_id_has_flush_ext(id))
2787	dev->flags \|= ATA_DFLAG_FLUSH_EXT;
2788	} else {
2789	lba_desc = "LBA";
2790	}
2791
2792	/* config NCQ */
2793	ret = ata_dev_config_ncq(dev, ncq_desc, sizeof(ncq_desc));
2794
2795	/* print device info to dmesg */
2796	if (ata_dev_print_info(dev))
2797	ata_dev_info(dev,
2798	"%llu sectors, multi %u: %s %s\n",
2799	(unsigned long long)dev->n_sectors,
2800	dev->multi_count, lba_desc, ncq_desc);
2801
2802	return ret;
2803	}
2804
2805	static void ata_dev_config_chs(struct ata_device *dev)
2806	{
2807	const u16 *id = dev->id;
2808
2809	if (ata_id_current_chs_valid(id)) {
2810	/* Current CHS translation is valid. */
2811	dev->cylinders = id[54];
2812	dev->heads = id[55];
2813	dev->sectors = id[56];
2814	} else {
2815	/* Default translation */
2816	dev->cylinders = id[1];
2817	dev->heads = id[3];
2818	dev->sectors = id[6];
2819	}
2820
2821	/* print device info to dmesg */
2822	if (ata_dev_print_info(dev))
2823	ata_dev_info(dev,
2824	"%llu sectors, multi %u, CHS %u/%u/%u\n",
2825	(unsigned long long)dev->n_sectors,
2826	dev->multi_count, dev->cylinders,
2827	dev->heads, dev->sectors);
2828	}
2829
2830	static void ata_dev_config_fua(struct ata_device *dev)
2831	{
2832	/* Ignore FUA support if its use is disabled globally */
2833	if (!libata_fua)
2834	goto nofua;
2835
2836	/* Ignore devices without support for WRITE DMA FUA EXT */
2837	if (!(dev->flags & ATA_DFLAG_LBA48) \|\| !ata_id_has_fua(dev->id))
2838	goto nofua;
2839
2840	/* Ignore known bad devices and devices that lack NCQ support */
2841	if (!ata_ncq_supported(dev) \|\| (dev->quirks & ATA_QUIRK_NO_FUA))
2842	goto nofua;
2843
2844	dev->flags \|= ATA_DFLAG_FUA;
2845
2846	return;
2847
2848	nofua:
2849	dev->flags &= ~ATA_DFLAG_FUA;
2850	}
2851
2852	static void ata_dev_config_devslp(struct ata_device *dev)
2853	{
2854	u8 *sata_setting = dev->sector_buf;
2855	unsigned int err_mask;
2856	int i, j;
2857
2858	/*
2859	* Check device sleep capability. Get DevSlp timing variables
2860	* from SATA Settings page of Identify Device Data Log.
2861	*/
2862	if (!ata_id_has_devslp(dev->id) \|\|
2863	!ata_identify_page_supported(dev, ATA_LOG_SATA_SETTINGS))
2864	return;
2865
2866	err_mask = ata_read_log_page(dev,
2867	ATA_LOG_IDENTIFY_DEVICE,
2868	ATA_LOG_SATA_SETTINGS,
2869	sata_setting, 1);
2870	if (err_mask)
2871	return;
2872
2873	dev->flags \|= ATA_DFLAG_DEVSLP;
2874	for (i = 0; i < ATA_LOG_DEVSLP_SIZE; i++) {
2875	j = ATA_LOG_DEVSLP_OFFSET + i;
2876	dev->devslp_timing[i] = sata_setting[j];
2877	}
2878	}
2879
2880	static void ata_dev_config_cpr(struct ata_device *dev)
2881	{
2882	unsigned int err_mask;
2883	size_t buf_len;
2884	int i, nr_cpr = 0;
2885	struct ata_cpr_log *cpr_log = NULL;
2886	u8 desc, buf = NULL;
2887
2888	if (ata_id_major_version(dev->id) < 11)
2889	goto out;
2890
2891	buf_len = ata_log_supported(dev, ATA_LOG_CONCURRENT_POSITIONING_RANGES);
2892	if (buf_len == 0)
2893	goto out;
2894
2895	/*
2896	* Read the concurrent positioning ranges log (0x47). We can have at
2897	* most 255 32B range descriptors plus a 64B header. This log varies in
2898	* size, so use the size reported in the GPL directory. Reading beyond
2899	* the supported length will result in an error.
2900	*/
2901	buf_len <<= 9;
2902	buf = kzalloc(buf_len, GFP_KERNEL);
2903	if (!buf)
2904	goto out;
2905
2906	err_mask = ata_read_log_page(dev, ATA_LOG_CONCURRENT_POSITIONING_RANGES,
2907	0, buf, buf_len >> 9);
2908	if (err_mask)
2909	goto out;
2910
2911	nr_cpr = buf[0];
2912	if (!nr_cpr)
2913	goto out;
2914
2915	/*
2916	* The device reports the number of CPR descriptors independently of the
2917	* log size, and that count is also used to emit VPD page B9h into the
2918	* fixed-size rbuf. Reject a count larger than what that buffer can hold
2919	* (ATA_DEV_MAX_CPR) or larger than the log the device actually returned.
2920	*/
2921	if (nr_cpr > ATA_DEV_MAX_CPR) {
2922	ata_dev_warn(dev,
2923	"Too many concurrent positioning ranges\n");
2924	goto out;
2925	}
2926
2927	if (buf_len < 64 + (size_t)nr_cpr * 32) {
2928	ata_dev_warn(dev,
2929	"Invalid number of concurrent positioning ranges\n");
2930	goto out;
2931	}
2932
2933	cpr_log = kzalloc_flex(*cpr_log, cpr, nr_cpr);
2934	if (!cpr_log)
2935	goto out;
2936
2937	cpr_log->nr_cpr = nr_cpr;
2938	desc = &buf[64];
2939	for (i = 0; i < nr_cpr; i++, desc += 32) {
2940	cpr_log->cpr[i].num = desc[0];
2941	cpr_log->cpr[i].num_storage_elements = desc[1];
2942	cpr_log->cpr[i].start_lba = get_unaligned_le64(&desc[8]);
2943	cpr_log->cpr[i].num_lbas = get_unaligned_le64(&desc[16]);
2944	}
2945
2946	out:
2947	swap(dev->cpr_log, cpr_log);
2948	kfree(cpr_log);
2949	kfree(buf);
2950	}
2951
2952	/*
2953	* Configure features related to link power management.
2954	*/
2955	static void ata_dev_config_lpm(struct ata_device *dev)
2956	{
2957	struct ata_port *ap = dev->link->ap;
2958	unsigned int err_mask;
2959
2960	if (ap->flags & ATA_FLAG_NO_LPM) {
2961	/*
2962	* When the port does not support LPM, we cannot support it on
2963	* the device either.
2964	*/
2965	dev->quirks \|= ATA_QUIRK_NOLPM;
2966	} else {
2967	/*
2968	* Some WD SATA-1 drives have issues with LPM, turn on NOLPM for
2969	* them.
2970	*/
2971	if ((dev->quirks & ATA_QUIRK_WD_BROKEN_LPM) &&
2972	(dev->id[ATA_ID_SATA_CAPABILITY] & 0xe) == 0x2)
2973	dev->quirks \|= ATA_QUIRK_NOLPM;
2974
2975	/* ATI and AMD specific quirk */
2976	if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD) &&
2977	(ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI) \|\|
2978	ata_dev_check_adapter(dev, PCI_VENDOR_ID_AMD)))
2979	dev->quirks \|= ATA_QUIRK_NOLPM;
2980	}
2981
2982	if (dev->quirks & ATA_QUIRK_NOLPM &&
2983	ap->target_lpm_policy != ATA_LPM_MAX_POWER) {
2984	ata_dev_warn(dev, "LPM support broken, forcing max_power\n");
2985	ap->target_lpm_policy = ATA_LPM_MAX_POWER;
2986	}
2987
2988	/*
2989	* Device Initiated Power Management (DIPM) is normally disabled by
2990	* default on a device. However, DIPM may have been enabled and that
2991	* setting kept even after COMRESET because of the Software Settings
2992	* Preservation feature. So if the port does not support DIPM and the
2993	* device does, disable DIPM on the device.
2994	*/
2995	if (ap->flags & ATA_FLAG_NO_DIPM && ata_id_has_dipm(dev->id)) {
2996	err_mask = ata_dev_set_feature(dev,
2997	SETFEATURES_SATA_DISABLE, SATA_DIPM);
2998	if (err_mask && err_mask != AC_ERR_DEV)
2999	ata_dev_err(dev, "Disable DIPM failed, Emask 0x%x\n",
3000	err_mask);
3001	}
3002	}
3003
3004	static void ata_dev_print_features(struct ata_device *dev)
3005	{
3006	if (!(dev->flags & ATA_DFLAG_FEATURES_MASK) && !dev->cpr_log &&
3007	!ata_id_has_hipm(dev->id) && !ata_id_has_dipm(dev->id))
3008	return;
3009
3010	ata_dev_info(dev,
3011	"Features:%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
3012	dev->flags & ATA_DFLAG_FUA ? " FUA" : "",
3013	dev->flags & ATA_DFLAG_TRUSTED ? " Trust" : "",
3014	dev->flags & ATA_DFLAG_DA ? " Dev-Attention" : "",
3015	dev->flags & ATA_DFLAG_DEVSLP ? " Dev-Sleep" : "",
3016	ata_id_has_hipm(dev->id) ? " HIPM" : "",
3017	ata_id_has_dipm(dev->id) ? " DIPM" : "",
3018	dev->flags & ATA_DFLAG_NCQ_SEND_RECV ? " NCQ-sndrcv" : "",
3019	dev->flags & ATA_DFLAG_NCQ_PRIO ? " NCQ-prio" : "",
3020	dev->flags & ATA_DFLAG_CDL ? " CDL" : "",
3021	dev->cpr_log ? " CPR" : "",
3022	dev->flags & ATA_DFLAG_DEPOP ? " Depop" : "",
3023	dev->flags & ATA_DFLAG_DEPOP_RESTORE ? " Depop-Restore" : "",
3024	dev->flags & ATA_DFLAG_DEPOP_MODIFY ? " Depop-Modify" : "");
3025	}
3026
3027	/**
3028	* ata_dev_configure - Configure the specified ATA/ATAPI device
3029	* @dev: Target device to configure
3030	*
3031	* Configure @dev according to @dev->id. Generic and low-level
3032	* driver specific fixups are also applied.
3033	*
3034	* LOCKING:
3035	* Kernel thread context (may sleep)
3036	*
3037	* RETURNS:
3038	* 0 on success, -errno otherwise
3039	*/
3040	int ata_dev_configure(struct ata_device *dev)
3041	{
3042	struct ata_port *ap = dev->link->ap;
3043	bool print_info = ata_dev_print_info(dev);
3044	const u16 *id = dev->id;
3045	unsigned int xfer_mask;
3046	unsigned int err_mask;
3047	char revbuf[7]; /* XYZ-99\0 */
3048	char fwrevbuf[ATA_ID_FW_REV_LEN+1];
3049	char modelbuf[ATA_ID_PROD_LEN+1];
3050	int rc;
3051
3052	if (!ata_dev_enabled(dev)) {
3053	ata_dev_dbg(dev, "no device\n");
3054	return 0;
3055	}
3056
3057	/* Clear the general purpose log directory cache. */
3058	ata_clear_log_directory(dev);
3059
3060	/* Set quirks */
3061	dev->quirks \|= ata_dev_quirks(dev);
3062	ata_force_quirks(dev);
3063
3064	if (dev->quirks & ATA_QUIRK_DISABLE) {
3065	ata_dev_info(dev, "unsupported device, disabling\n");
3066	ata_dev_disable(dev);
3067	return 0;
3068	}
3069
3070	if ((!atapi_enabled \|\| (ap->flags & ATA_FLAG_NO_ATAPI)) &&
3071	dev->class == ATA_DEV_ATAPI) {
3072	ata_dev_warn(dev, "WARNING: ATAPI is %s, device ignored\n",
3073	atapi_enabled ? "not supported with this driver"
3074	: "disabled");
3075	ata_dev_disable(dev);
3076	return 0;
3077	}
3078
3079	rc = ata_do_link_spd_quirk(dev);
3080	if (rc)
3081	return rc;
3082
3083	/* let ACPI work its magic */
3084	rc = ata_acpi_on_devcfg(dev);
3085	if (rc)
3086	return rc;
3087
3088	/* massage HPA, do it early as it might change IDENTIFY data */
3089	rc = ata_hpa_resize(dev);
3090	if (rc)
3091	return rc;
3092
3093	/* print device capabilities */
3094	ata_dev_dbg(dev,
3095	"%s: cfg 49:%04x 82:%04x 83:%04x 84:%04x "
3096	"85:%04x 86:%04x 87:%04x 88:%04x\n",
3097	__func__,
3098	id[49], id[82], id[83], id[84],
3099	id[85], id[86], id[87], id[88]);
3100
3101	/* initialize to-be-configured parameters */
3102	dev->flags &= ~ATA_DFLAG_CFG_MASK;
3103	dev->max_sectors = 0;
3104	dev->cdb_len = 0;
3105	dev->n_sectors = 0;
3106	dev->cylinders = 0;
3107	dev->heads = 0;
3108	dev->sectors = 0;
3109	dev->multi_count = 0;
3110
3111	/*
3112	* common ATA, ATAPI feature tests
3113	*/
3114
3115	/* find max transfer mode; for printk only */
3116	xfer_mask = ata_id_xfermask(id);
3117
3118	ata_dump_id(dev, id);
3119
3120	/* SCSI only uses 4-char revisions, dump full 8 chars from ATA */
3121	ata_id_c_string(dev->id, fwrevbuf, ATA_ID_FW_REV,
3122	sizeof(fwrevbuf));
3123
3124	ata_id_c_string(dev->id, modelbuf, ATA_ID_PROD,
3125	sizeof(modelbuf));
3126
3127	/* ATA-specific feature tests */
3128	if (dev->class == ATA_DEV_ATA \|\| dev->class == ATA_DEV_ZAC) {
3129	if (ata_id_is_cfa(id)) {
3130	/* CPRM may make this media unusable */
3131	if (id[ATA_ID_CFA_KEY_MGMT] & 1)
3132	ata_dev_warn(dev,
3133	"supports DRM functions and may not be fully accessible\n");
3134	snprintf(revbuf, 7, "CFA");
3135	} else {
3136	snprintf(revbuf, 7, "ATA-%d", ata_id_major_version(id));
3137	/* Warn the user if the device has TPM extensions */
3138	if (ata_id_has_tpm(id))
3139	ata_dev_warn(dev,
3140	"supports DRM functions and may not be fully accessible\n");
3141	}
3142
3143	dev->n_sectors = ata_id_n_sectors(id);
3144	if (ata_id_is_locked(id)) {
3145	/*
3146	* If Security locked, set capacity to zero to prevent
3147	* any I/O, e.g. partition scanning, as any I/O to a
3148	* locked drive will result in user visible errors.
3149	*/
3150	ata_dev_info(dev,
3151	"Security locked, setting capacity to zero\n");
3152	dev->n_sectors = 0;
3153	}
3154
3155	/* get current R/W Multiple count setting */
3156	if ((dev->id[47] >> 8) == 0x80 && (dev->id[59] & 0x100)) {
3157	unsigned int max = dev->id[47] & 0xff;
3158	unsigned int cnt = dev->id[59] & 0xff;
3159	/* only recognize/allow powers of two here */
3160	if (is_power_of_2(max) && is_power_of_2(cnt))
3161	if (cnt <= max)
3162	dev->multi_count = cnt;
3163	}
3164
3165	/* print device info to dmesg */
3166	if (print_info)
3167	ata_dev_info(dev, "%s: %s, %s, max %s\n",
3168	revbuf, modelbuf, fwrevbuf,
3169	ata_mode_string(xfer_mask));
3170
3171	if (ata_id_has_lba(id)) {
3172	rc = ata_dev_config_lba(dev);
3173	if (rc)
3174	return rc;
3175	} else {
3176	ata_dev_config_chs(dev);
3177	}
3178
3179	ata_dev_config_lpm(dev);
3180	ata_dev_config_fua(dev);
3181	ata_dev_config_devslp(dev);
3182	ata_dev_config_sense_reporting(dev);
3183	ata_dev_config_zoned(dev);
3184	ata_dev_config_trusted(dev);
3185	ata_dev_config_cpr(dev);
3186	ata_dev_config_cdl(dev);
3187	ata_dev_config_depop(dev);
3188	dev->cdb_len = 32;
3189
3190	if (print_info)
3191	ata_dev_print_features(dev);
3192	}
3193
3194	/* ATAPI-specific feature tests */
3195	else if (dev->class == ATA_DEV_ATAPI) {
3196	const char *cdb_intr_string = "";
3197	const char *atapi_an_string = "";
3198	const char *dma_dir_string = "";
3199	u32 sntf;
3200
3201	rc = atapi_cdb_len(id);
3202	if ((rc < 12) \|\| (rc > ATAPI_CDB_LEN)) {
3203	ata_dev_warn(dev, "unsupported CDB len %d\n", rc);
3204	rc = -EINVAL;
3205	goto err_out_nosup;
3206	}
3207	dev->cdb_len = (unsigned int) rc;
3208
3209	/* Enable ATAPI AN if both the host and device have
3210	* the support. If PMP is attached, SNTF is required
3211	* to enable ATAPI AN to discern between PHY status
3212	* changed notifications and ATAPI ANs.
3213	*/
3214	if (atapi_an &&
3215	(ap->flags & ATA_FLAG_AN) && ata_id_has_atapi_AN(id) &&
3216	(!sata_pmp_attached(ap) \|\|
3217	sata_scr_read(&ap->link, SCR_NOTIFICATION, &sntf) == 0)) {
3218	/* issue SET feature command to turn this on */
3219	err_mask = ata_dev_set_feature(dev,
3220	SETFEATURES_SATA_ENABLE, SATA_AN);
3221	if (err_mask)
3222	ata_dev_err(dev,
3223	"failed to enable ATAPI AN (err_mask=0x%x)\n",
3224	err_mask);
3225	else {
3226	dev->flags \|= ATA_DFLAG_AN;
3227	atapi_an_string = ", ATAPI AN";
3228	}
3229	}
3230
3231	if (ata_id_cdb_intr(dev->id)) {
3232	dev->flags \|= ATA_DFLAG_CDB_INTR;
3233	cdb_intr_string = ", CDB intr";
3234	}
3235
3236	if (atapi_dmadir \|\| (dev->quirks & ATA_QUIRK_ATAPI_DMADIR) \|\|
3237	atapi_id_dmadir(dev->id)) {
3238	dev->flags \|= ATA_DFLAG_DMADIR;
3239	dma_dir_string = ", DMADIR";
3240	}
3241
3242	if (ata_id_has_da(dev->id)) {
3243	dev->flags \|= ATA_DFLAG_DA;
3244	zpodd_init(dev);
3245	}
3246
3247	/* print device info to dmesg */
3248	if (print_info)
3249	ata_dev_info(dev,
3250	"ATAPI: %s, %s, max %s%s%s%s\n",
3251	modelbuf, fwrevbuf,
3252	ata_mode_string(xfer_mask),
3253	cdb_intr_string, atapi_an_string,
3254	dma_dir_string);
3255
3256	ata_dev_config_lpm(dev);
3257
3258	if (print_info)
3259	ata_dev_print_features(dev);
3260	}
3261
3262	/* determine max_sectors */
3263	dev->max_sectors = ATA_MAX_SECTORS;
3264	if (dev->flags & ATA_DFLAG_LBA48)
3265	dev->max_sectors = ATA_MAX_SECTORS_LBA48;
3266
3267	/* Limit PATA drive on SATA cable bridge transfers to udma5,
3268	200 sectors */
3269	if (ata_dev_knobble(dev)) {
3270	if (print_info)
3271	ata_dev_info(dev, "applying bridge limits\n");
3272	dev->udma_mask &= ATA_UDMA5;
3273	dev->max_sectors = ATA_MAX_SECTORS;
3274	}
3275
3276	if ((dev->class == ATA_DEV_ATAPI) &&
3277	(atapi_command_packet_set(id) == TYPE_TAPE)) {
3278	dev->max_sectors = ATA_MAX_SECTORS_TAPE;
3279	dev->quirks \|= ATA_QUIRK_STUCK_ERR;
3280	}
3281
3282	if (dev->quirks & ATA_QUIRK_MAX_SEC)
3283	dev->max_sectors = min_t(unsigned int, dev->max_sectors,
3284	ata_dev_get_quirk_value(dev,
3285	ATA_QUIRK_MAX_SEC));
3286
3287	if (dev->quirks & ATA_QUIRK_MAX_SEC_LBA48)
3288	dev->max_sectors = ATA_MAX_SECTORS_LBA48;
3289
3290	if (ap->ops->dev_config)
3291	ap->ops->dev_config(dev);
3292
3293	if (dev->quirks & ATA_QUIRK_DIAGNOSTIC) {
3294	/* Let the user know. We don't want to disallow opens for
3295	rescue purposes, or in case the vendor is just a blithering
3296	idiot. Do this after the dev_config call as some controllers
3297	with buggy firmware may want to avoid reporting false device
3298	bugs */
3299
3300	if (print_info) {
3301	ata_dev_warn(dev,
3302	"Drive reports diagnostics failure. This may indicate a drive\n");
3303	ata_dev_warn(dev,
3304	"fault or invalid emulation. Contact drive vendor for information.\n");
3305	}
3306	}
3307
3308	if ((dev->quirks & ATA_QUIRK_FIRMWARE_WARN) && print_info) {
3309	ata_dev_warn(dev, "WARNING: device requires firmware update to be fully functional\n");
3310	ata_dev_warn(dev, " contact the vendor or visit http://ata.wiki.kernel.org\n");
3311	}
3312
3313	return 0;
3314
3315	err_out_nosup:
3316	return rc;
3317	}
3318
3319	/**
3320	* ata_cable_40wire - return 40 wire cable type
3321	* @ap: port
3322	*
3323	* Helper method for drivers which want to hardwire 40 wire cable
3324	* detection.
3325	*/
3326
3327	int ata_cable_40wire(struct ata_port *ap)
3328	{
3329	return ATA_CBL_PATA40;
3330	}
3331	EXPORT_SYMBOL_GPL(ata_cable_40wire);
3332
3333	/**
3334	* ata_cable_80wire - return 80 wire cable type
3335	* @ap: port
3336	*
3337	* Helper method for drivers which want to hardwire 80 wire cable
3338	* detection.
3339	*/
3340
3341	int ata_cable_80wire(struct ata_port *ap)
3342	{
3343	return ATA_CBL_PATA80;
3344	}
3345	EXPORT_SYMBOL_GPL(ata_cable_80wire);
3346
3347	/**
3348	* ata_cable_unknown - return unknown PATA cable.
3349	* @ap: port
3350	*
3351	* Helper method for drivers which have no PATA cable detection.
3352	*/
3353
3354	int ata_cable_unknown(struct ata_port *ap)
3355	{
3356	return ATA_CBL_PATA_UNK;
3357	}
3358	EXPORT_SYMBOL_GPL(ata_cable_unknown);
3359
3360	/**
3361	* ata_cable_ignore - return ignored PATA cable.
3362	* @ap: port
3363	*
3364	* Helper method for drivers which don't use cable type to limit
3365	* transfer mode.
3366	*/
3367	int ata_cable_ignore(struct ata_port *ap)
3368	{
3369	return ATA_CBL_PATA_IGN;
3370	}
3371	EXPORT_SYMBOL_GPL(ata_cable_ignore);
3372
3373	/**
3374	* ata_cable_sata - return SATA cable type
3375	* @ap: port
3376	*
3377	* Helper method for drivers which have SATA cables
3378	*/
3379
3380	int ata_cable_sata(struct ata_port *ap)
3381	{
3382	return ATA_CBL_SATA;
3383	}
3384	EXPORT_SYMBOL_GPL(ata_cable_sata);
3385
3386	/**
3387	* sata_print_link_status - Print SATA link status
3388	* @link: SATA link to printk link status about
3389	*
3390	* This function prints link speed and status of a SATA link.
3391	*
3392	* LOCKING:
3393	* None.
3394	*/
3395	static void sata_print_link_status(struct ata_link *link)
3396	{
3397	u32 sstatus, scontrol, tmp;
3398
3399	if (sata_scr_read(link, SCR_STATUS, &sstatus))
3400	return;
3401	if (sata_scr_read(link, SCR_CONTROL, &scontrol))
3402	return;
3403
3404	if (ata_phys_link_online(link)) {
3405	tmp = (sstatus >> 4) & 0xf;
3406	ata_link_info(link, "SATA link up %s (SStatus %X SControl %X)\n",
3407	sata_spd_string(tmp), sstatus, scontrol);
3408	} else {
3409	ata_link_info(link, "SATA link down (SStatus %X SControl %X)\n",
3410	sstatus, scontrol);
3411	}
3412	}
3413
3414	/**
3415	* ata_dev_pair - return other device on cable
3416	* @adev: device
3417	*
3418	* Obtain the other device on the same cable, or if none is
3419	* present NULL is returned
3420	*/
3421
3422	struct ata_device ata_dev_pair(struct ata_device adev)
3423	{
3424	struct ata_link *link = adev->link;
3425	struct ata_device *pair = &link->device[1 - adev->devno];
3426	if (!ata_dev_enabled(pair))
3427	return NULL;
3428	return pair;
3429	}
3430	EXPORT_SYMBOL_GPL(ata_dev_pair);
3431
3432	#ifdef CONFIG_ATA_ACPI
3433	/**
3434	* ata_timing_cycle2mode - find xfer mode for the specified cycle duration
3435	* @xfer_shift: ATA_SHIFT_* value for transfer type to examine.
3436	* @cycle: cycle duration in ns
3437	*
3438	* Return matching xfer mode for @cycle. The returned mode is of
3439	* the transfer type specified by @xfer_shift. If @cycle is too
3440	* slow for @xfer_shift, 0xff is returned. If @cycle is faster
3441	* than the fastest known mode, the fasted mode is returned.
3442	*
3443	* LOCKING:
3444	* None.
3445	*
3446	* RETURNS:
3447	* Matching xfer_mode, 0xff if no match found.
3448	*/
3449	u8 ata_timing_cycle2mode(unsigned int xfer_shift, int cycle)
3450	{
3451	u8 base_mode = 0xff, last_mode = 0xff;
3452	const struct ata_xfer_ent *ent;
3453	const struct ata_timing *t;
3454
3455	for (ent = ata_xfer_tbl; ent->shift >= 0; ent++)
3456	if (ent->shift == xfer_shift)
3457	base_mode = ent->base;
3458
3459	for (t = ata_timing_find_mode(base_mode);
3460	t && ata_xfer_mode2shift(t->mode) == xfer_shift; t++) {
3461	unsigned short this_cycle;
3462
3463	switch (xfer_shift) {
3464	case ATA_SHIFT_PIO:
3465	case ATA_SHIFT_MWDMA:
3466	this_cycle = t->cycle;
3467	break;
3468	case ATA_SHIFT_UDMA:
3469	this_cycle = t->udma;
3470	break;
3471	default:
3472	return 0xff;
3473	}
3474
3475	if (cycle > this_cycle)
3476	break;
3477
3478	last_mode = t->mode;
3479	}
3480
3481	return last_mode;
3482	}
3483	#endif
3484
3485	/**
3486	* ata_down_xfermask_limit - adjust dev xfer masks downward
3487	* @dev: Device to adjust xfer masks
3488	* @sel: ATA_DNXFER_* selector
3489	*
3490	* Adjust xfer masks of @dev downward. Note that this function
3491	* does not apply the change. Invoking ata_set_mode() afterwards
3492	* will apply the limit.
3493	*
3494	* LOCKING:
3495	* Inherited from caller.
3496	*
3497	* RETURNS:
3498	* 0 on success, negative errno on failure
3499	*/
3500	int ata_down_xfermask_limit(struct ata_device *dev, unsigned int sel)
3501	{
3502	char buf[32];
3503	unsigned int orig_mask, xfer_mask;
3504	unsigned int pio_mask, mwdma_mask, udma_mask;
3505	int quiet, highbit;
3506
3507	quiet = !!(sel & ATA_DNXFER_QUIET);
3508	sel &= ~ATA_DNXFER_QUIET;
3509
3510	xfer_mask = orig_mask = ata_pack_xfermask(dev->pio_mask,
3511	dev->mwdma_mask,
3512	dev->udma_mask);
3513	ata_unpack_xfermask(xfer_mask, &pio_mask, &mwdma_mask, &udma_mask);
3514
3515	switch (sel) {
3516	case ATA_DNXFER_PIO:
3517	highbit = fls(pio_mask) - 1;
3518	pio_mask &= ~(1 << highbit);
3519	break;
3520
3521	case ATA_DNXFER_DMA:
3522	if (udma_mask) {
3523	highbit = fls(udma_mask) - 1;
3524	udma_mask &= ~(1 << highbit);
3525	if (!udma_mask)
3526	return -ENOENT;
3527	} else if (mwdma_mask) {
3528	highbit = fls(mwdma_mask) - 1;
3529	mwdma_mask &= ~(1 << highbit);
3530	if (!mwdma_mask)
3531	return -ENOENT;
3532	}
3533	break;
3534
3535	case ATA_DNXFER_40C:
3536	udma_mask &= ATA_UDMA_MASK_40C;
3537	break;
3538
3539	case ATA_DNXFER_FORCE_PIO0:
3540	pio_mask &= 1;
3541	fallthrough;
3542	case ATA_DNXFER_FORCE_PIO:
3543	mwdma_mask = 0;
3544	udma_mask = 0;
3545	break;
3546
3547	default:
3548	BUG();
3549	}
3550
3551	xfer_mask &= ata_pack_xfermask(pio_mask, mwdma_mask, udma_mask);
3552
3553	if (!(xfer_mask & ATA_MASK_PIO) \|\| xfer_mask == orig_mask)
3554	return -ENOENT;
3555
3556	if (!quiet) {
3557	if (xfer_mask & (ATA_MASK_MWDMA \| ATA_MASK_UDMA))
3558	snprintf(buf, sizeof(buf), "%s:%s",
3559	ata_mode_string(xfer_mask),
3560	ata_mode_string(xfer_mask & ATA_MASK_PIO));
3561	else
3562	snprintf(buf, sizeof(buf), "%s",
3563	ata_mode_string(xfer_mask));
3564
3565	ata_dev_warn(dev, "limiting speed to %s\n", buf);
3566	}
3567
3568	ata_unpack_xfermask(xfer_mask, &dev->pio_mask, &dev->mwdma_mask,
3569	&dev->udma_mask);
3570
3571	return 0;
3572	}
3573
3574	static int ata_dev_set_mode(struct ata_device *dev)
3575	{
3576	struct ata_port *ap = dev->link->ap;
3577	struct ata_eh_context *ehc = &dev->link->eh_context;
3578	const bool nosetxfer = dev->quirks & ATA_QUIRK_NOSETXFER;
3579	const char *dev_err_whine = "";
3580	int ign_dev_err = 0;
3581	unsigned int err_mask = 0;
3582	int rc;
3583
3584	dev->flags &= ~ATA_DFLAG_PIO;
3585	if (dev->xfer_shift == ATA_SHIFT_PIO)
3586	dev->flags \|= ATA_DFLAG_PIO;
3587
3588	if (nosetxfer && ap->flags & ATA_FLAG_SATA && ata_id_is_sata(dev->id))
3589	dev_err_whine = " (SET_XFERMODE skipped)";
3590	else {
3591	if (nosetxfer)
3592	ata_dev_warn(dev,
3593	"NOSETXFER but PATA detected - can't "
3594	"skip SETXFER, might malfunction\n");
3595	err_mask = ata_dev_set_xfermode(dev);
3596	}
3597
3598	if (err_mask & ~AC_ERR_DEV)
3599	goto fail;
3600
3601	/* revalidate */
3602	ehc->i.flags \|= ATA_EHI_POST_SETMODE;
3603	rc = ata_dev_revalidate(dev, ATA_DEV_UNKNOWN, 0);
3604	ehc->i.flags &= ~ATA_EHI_POST_SETMODE;
3605	if (rc)
3606	return rc;
3607
3608	if (dev->xfer_shift == ATA_SHIFT_PIO) {
3609	/* Old CFA may refuse this command, which is just fine */
3610	if (ata_id_is_cfa(dev->id))
3611	ign_dev_err = 1;
3612	/* Catch several broken garbage emulations plus some pre
3613	ATA devices */
3614	if (ata_id_major_version(dev->id) == 0 &&
3615	dev->pio_mode <= XFER_PIO_2)
3616	ign_dev_err = 1;
3617	/* Some very old devices and some bad newer ones fail
3618	any kind of SET_XFERMODE request but support PIO0-2
3619	timings and no IORDY */
3620	if (!ata_id_has_iordy(dev->id) && dev->pio_mode <= XFER_PIO_2)
3621	ign_dev_err = 1;
3622	}
3623	/* Early MWDMA devices do DMA but don't allow DMA mode setting.
3624	Don't fail an MWDMA0 set IFF the device indicates it is in MWDMA0 */
3625	if (dev->xfer_shift == ATA_SHIFT_MWDMA &&
3626	dev->dma_mode == XFER_MW_DMA_0 &&
3627	(dev->id[63] >> 8) & 1)
3628	ign_dev_err = 1;
3629
3630	/* if the device is actually configured correctly, ignore dev err */
3631	if (dev->xfer_mode == ata_xfer_mask2mode(ata_id_xfermask(dev->id)))
3632	ign_dev_err = 1;
3633
3634	if (err_mask & AC_ERR_DEV) {
3635	if (!ign_dev_err)
3636	goto fail;
3637	else
3638	dev_err_whine = " (device error ignored)";
3639	}
3640
3641	ata_dev_dbg(dev, "xfer_shift=%u, xfer_mode=0x%x\n",
3642	dev->xfer_shift, (int)dev->xfer_mode);
3643
3644	if (!(ehc->i.flags & ATA_EHI_QUIET) \|\|
3645	ehc->i.flags & ATA_EHI_DID_HARDRESET)
3646	ata_dev_info(dev, "configured for %s%s\n",
3647	ata_mode_string(ata_xfer_mode2mask(dev->xfer_mode)),
3648	dev_err_whine);
3649
3650	return 0;
3651
3652	fail:
3653	ata_dev_err(dev, "failed to set xfermode (err_mask=0x%x)\n", err_mask);
3654	return -EIO;
3655	}
3656
3657	/**
3658	* ata_set_mode - Program timings and issue SET FEATURES - XFER
3659	* @link: link on which timings will be programmed
3660	* @r_failed_dev: out parameter for failed device
3661	*
3662	* Standard implementation of the function used to tune and set
3663	* ATA device disk transfer mode (PIO3, UDMA6, etc.). If
3664	* ata_dev_set_mode() fails, pointer to the failing device is
3665	* returned in @r_failed_dev.
3666	*
3667	* LOCKING:
3668	* PCI/etc. bus probe sem.
3669	*
3670	* RETURNS:
3671	* 0 on success, negative errno otherwise
3672	*/
3673
3674	int ata_set_mode(struct ata_link link, struct ata_device *r_failed_dev)
3675	{
3676	struct ata_port *ap = link->ap;
3677	struct ata_device *dev;
3678	int rc = 0, used_dma = 0, found = 0;
3679
3680	/* step 1: calculate xfer_mask */
3681	ata_for_each_dev(dev, link, ENABLED) {
3682	unsigned int pio_mask, dma_mask;
3683	unsigned int mode_mask;
3684
3685	mode_mask = ATA_DMA_MASK_ATA;
3686	if (dev->class == ATA_DEV_ATAPI)
3687	mode_mask = ATA_DMA_MASK_ATAPI;
3688	else if (ata_id_is_cfa(dev->id))
3689	mode_mask = ATA_DMA_MASK_CFA;
3690
3691	ata_dev_xfermask(dev);
3692	ata_force_xfermask(dev);
3693
3694	pio_mask = ata_pack_xfermask(dev->pio_mask, 0, 0);
3695
3696	if (libata_dma_mask & mode_mask)
3697	dma_mask = ata_pack_xfermask(0, dev->mwdma_mask,
3698	dev->udma_mask);
3699	else
3700	dma_mask = 0;
3701
3702	dev->pio_mode = ata_xfer_mask2mode(pio_mask);
3703	dev->dma_mode = ata_xfer_mask2mode(dma_mask);
3704
3705	found = 1;
3706	if (ata_dma_enabled(dev))
3707	used_dma = 1;
3708	}
3709	if (!found)
3710	goto out;
3711
3712	/* step 2: always set host PIO timings */
3713	ata_for_each_dev(dev, link, ENABLED) {
3714	if (dev->pio_mode == 0xff) {
3715	ata_dev_warn(dev, "no PIO support\n");
3716	rc = -EINVAL;
3717	goto out;
3718	}
3719
3720	dev->xfer_mode = dev->pio_mode;
3721	dev->xfer_shift = ATA_SHIFT_PIO;
3722	if (ap->ops->set_piomode)
3723	ap->ops->set_piomode(ap, dev);
3724	}
3725
3726	/* step 3: set host DMA timings */
3727	ata_for_each_dev(dev, link, ENABLED) {
3728	if (!ata_dma_enabled(dev))
3729	continue;
3730
3731	dev->xfer_mode = dev->dma_mode;
3732	dev->xfer_shift = ata_xfer_mode2shift(dev->dma_mode);
3733	if (ap->ops->set_dmamode)
3734	ap->ops->set_dmamode(ap, dev);
3735	}
3736
3737	/* step 4: update devices' xfer mode */
3738	ata_for_each_dev(dev, link, ENABLED) {
3739	rc = ata_dev_set_mode(dev);
3740	if (rc)
3741	goto out;
3742	}
3743
3744	/* Record simplex status. If we selected DMA then the other
3745	* host channels are not permitted to do so.
3746	*/
3747	if (used_dma && (ap->host->flags & ATA_HOST_SIMPLEX))
3748	ap->host->simplex_claimed = ap;
3749
3750	out:
3751	if (rc)
3752	*r_failed_dev = dev;
3753	return rc;
3754	}
3755	EXPORT_SYMBOL_GPL(ata_set_mode);
3756
3757	/**
3758	* ata_wait_ready - wait for link to become ready
3759	* @link: link to be waited on
3760	* @deadline: deadline jiffies for the operation
3761	* @check_ready: callback to check link readiness
3762	*
3763	* Wait for @link to become ready. @check_ready should return
3764	* positive number if @link is ready, 0 if it isn't, -ENODEV if
3765	* link doesn't seem to be occupied, other errno for other error
3766	* conditions.
3767	*
3768	* Transient -ENODEV conditions are allowed for
3769	* ATA_TMOUT_FF_WAIT.
3770	*
3771	* LOCKING:
3772	* EH context.
3773	*
3774	* RETURNS:
3775	* 0 if @link is ready before @deadline; otherwise, -errno.
3776	*/
3777	int ata_wait_ready(struct ata_link *link, unsigned long deadline,
3778	int (check_ready)(struct ata_link link))
3779	{
3780	unsigned long start = jiffies;
3781	unsigned long nodev_deadline;
3782	int warned = 0;
3783
3784	/* choose which 0xff timeout to use, read comment in libata.h */
3785	if (link->ap->host->flags & ATA_HOST_PARALLEL_SCAN)
3786	nodev_deadline = ata_deadline(start, ATA_TMOUT_FF_WAIT_LONG);
3787	else
3788	nodev_deadline = ata_deadline(start, ATA_TMOUT_FF_WAIT);
3789
3790	/* Slave readiness can't be tested separately from master. On
3791	* M/S emulation configuration, this function should be called
3792	* only on the master and it will handle both master and slave.
3793	*/
3794	WARN_ON(link == link->ap->slave_link);
3795
3796	if (time_after(nodev_deadline, deadline))
3797	nodev_deadline = deadline;
3798
3799	while (1) {
3800	unsigned long now = jiffies;
3801	int ready, tmp;
3802
3803	ready = tmp = check_ready(link);
3804	if (ready > 0)
3805	return 0;
3806
3807	/*
3808	* -ENODEV could be transient. Ignore -ENODEV if link
3809	* is online. Also, some SATA devices take a long
3810	* time to clear 0xff after reset. Wait for
3811	* ATA_TMOUT_FF_WAIT[_LONG] on -ENODEV if link isn't
3812	* offline.
3813	*
3814	* Note that some PATA controllers (pata_ali) explode
3815	* if status register is read more than once when
3816	* there's no device attached.
3817	*/
3818	if (ready == -ENODEV) {
3819	if (ata_link_online(link))
3820	ready = 0;
3821	else if ((link->ap->flags & ATA_FLAG_SATA) &&
3822	!ata_link_offline(link) &&
3823	time_before(now, nodev_deadline))
3824	ready = 0;
3825	}
3826
3827	if (ready)
3828	return ready;
3829	if (time_after(now, deadline))
3830	return -EBUSY;
3831
3832	if (!warned && time_after(now, start + 5 * HZ) &&
3833	(deadline - now > 3 * HZ)) {
3834	ata_link_warn(link,
3835	"link is slow to respond, please be patient "
3836	"(ready=%d)\n", tmp);
3837	warned = 1;
3838	}
3839
3840	ata_msleep(link->ap, 50);
3841	}
3842	}
3843
3844	/**
3845	* ata_wait_after_reset - wait for link to become ready after reset
3846	* @link: link to be waited on
3847	* @deadline: deadline jiffies for the operation
3848	* @check_ready: callback to check link readiness
3849	*
3850	* Wait for @link to become ready after reset.
3851	*
3852	* LOCKING:
3853	* EH context.
3854	*
3855	* RETURNS:
3856	* 0 if @link is ready before @deadline; otherwise, -errno.
3857	*/
3858	int ata_wait_after_reset(struct ata_link *link, unsigned long deadline,
3859	int (check_ready)(struct ata_link link))
3860	{
3861	ata_msleep(link->ap, ATA_WAIT_AFTER_RESET);
3862
3863	return ata_wait_ready(link, deadline, check_ready);
3864	}
3865	EXPORT_SYMBOL_GPL(ata_wait_after_reset);
3866
3867	/**
3868	* ata_std_prereset - prepare for reset
3869	* @link: ATA link to be reset
3870	* @deadline: deadline jiffies for the operation
3871	*
3872	* @link is about to be reset. Initialize it. Failure from
3873	* prereset makes libata abort whole reset sequence and give up
3874	* that port, so prereset should be best-effort. It does its
3875	* best to prepare for reset sequence but if things go wrong, it
3876	* should just whine, not fail.
3877	*
3878	* LOCKING:
3879	* Kernel thread context (may sleep)
3880	*
3881	* RETURNS:
3882	* Always 0.
3883	*/
3884	int ata_std_prereset(struct ata_link *link, unsigned long deadline)
3885	{
3886	struct ata_port *ap = link->ap;
3887	struct ata_eh_context *ehc = &link->eh_context;
3888	const unsigned int *timing = sata_ehc_deb_timing(ehc);
3889	int rc;
3890
3891	/* if we're about to do hardreset, nothing more to do */
3892	if (ehc->i.action & ATA_EH_HARDRESET)
3893	return 0;
3894
3895	/* if SATA, resume link */
3896	if (ap->flags & ATA_FLAG_SATA) {
3897	rc = sata_link_resume(link, timing, deadline);
3898	/* whine about phy resume failure but proceed */
3899	if (rc && rc != -EOPNOTSUPP)
3900	ata_link_warn(link,
3901	"failed to resume link for reset (errno=%d)\n",
3902	rc);
3903	}
3904
3905	/* no point in trying softreset on offline link */
3906	if (ata_phys_link_offline(link))
3907	ehc->i.action &= ~ATA_EH_SOFTRESET;
3908
3909	return 0;
3910	}
3911	EXPORT_SYMBOL_GPL(ata_std_prereset);
3912
3913	/**
3914	* ata_std_postreset - standard postreset callback
3915	* @link: the target ata_link
3916	* @classes: classes of attached devices
3917	*
3918	* This function is invoked after a successful reset. Note that
3919	* the device might have been reset more than once using
3920	* different reset methods before postreset is invoked.
3921	*
3922	* LOCKING:
3923	* Kernel thread context (may sleep)
3924	*/
3925	void ata_std_postreset(struct ata_link link, unsigned int classes)
3926	{
3927	u32 serror;
3928
3929	/* reset complete, clear SError */
3930	if (!sata_scr_read(link, SCR_ERROR, &serror))
3931	sata_scr_write(link, SCR_ERROR, serror);
3932
3933	/* print link status */
3934	sata_print_link_status(link);
3935	}
3936	EXPORT_SYMBOL_GPL(ata_std_postreset);
3937
3938	/**
3939	* ata_dev_same_device - Determine whether new ID matches configured device
3940	* @dev: device to compare against
3941	* @new_class: class of the new device
3942	* @new_id: IDENTIFY page of the new device
3943	*
3944	* Compare @new_class and @new_id against @dev and determine
3945	* whether @dev is the device indicated by @new_class and
3946	* @new_id.
3947	*
3948	* LOCKING:
3949	* None.
3950	*
3951	* RETURNS:
3952	* 1 if @dev matches @new_class and @new_id, 0 otherwise.
3953	*/
3954	static int ata_dev_same_device(struct ata_device *dev, unsigned int new_class,
3955	const u16 *new_id)
3956	{
3957	const u16 *old_id = dev->id;
3958	unsigned char model[2][ATA_ID_PROD_LEN + 1];
3959	unsigned char serial[2][ATA_ID_SERNO_LEN + 1];
3960
3961	if (dev->class != new_class) {
3962	ata_dev_info(dev, "class mismatch %d != %d\n",
3963	dev->class, new_class);
3964	return 0;
3965	}
3966
3967	ata_id_c_string(old_id, model[0], ATA_ID_PROD, sizeof(model[0]));
3968	ata_id_c_string(new_id, model[1], ATA_ID_PROD, sizeof(model[1]));
3969	ata_id_c_string(old_id, serial[0], ATA_ID_SERNO, sizeof(serial[0]));
3970	ata_id_c_string(new_id, serial[1], ATA_ID_SERNO, sizeof(serial[1]));
3971
3972	if (strcmp(model[0], model[1])) {
3973	ata_dev_info(dev, "model number mismatch '%s' != '%s'\n",
3974	model[0], model[1]);
3975	return 0;
3976	}
3977
3978	if (strcmp(serial[0], serial[1])) {
3979	ata_dev_info(dev, "serial number mismatch '%s' != '%s'\n",
3980	serial[0], serial[1]);
3981	return 0;
3982	}
3983
3984	return 1;
3985	}
3986
3987	/**
3988	* ata_dev_reread_id - Re-read IDENTIFY data
3989	* @dev: target ATA device
3990	* @readid_flags: read ID flags
3991	*
3992	* Re-read IDENTIFY page and make sure @dev is still attached to
3993	* the port.
3994	*
3995	* LOCKING:
3996	* Kernel thread context (may sleep)
3997	*
3998	* RETURNS:
3999	* 0 on success, negative errno otherwise
4000	*/
4001	int ata_dev_reread_id(struct ata_device *dev, unsigned int readid_flags)
4002	{
4003	unsigned int class = dev->class;
4004	u16 id = (void )dev->sector_buf;
4005	int rc;
4006
4007	/* read ID data */
4008	rc = ata_dev_read_id(dev, &class, readid_flags, id);
4009	if (rc)
4010	return rc;
4011
4012	/* is the device still there? */
4013	if (!ata_dev_same_device(dev, class, id))
4014	return -ENODEV;
4015
4016	memcpy(dev->id, id, sizeof(id[0]) * ATA_ID_WORDS);
4017	return 0;
4018	}
4019
4020	/**
4021	* ata_dev_revalidate - Revalidate ATA device
4022	* @dev: device to revalidate
4023	* @new_class: new class code
4024	* @readid_flags: read ID flags
4025	*
4026	* Re-read IDENTIFY page, make sure @dev is still attached to the
4027	* port and reconfigure it according to the new IDENTIFY page.
4028	*
4029	* LOCKING:
4030	* Kernel thread context (may sleep)
4031	*
4032	* RETURNS:
4033	* 0 on success, negative errno otherwise
4034	*/
4035	int ata_dev_revalidate(struct ata_device *dev, unsigned int new_class,
4036	unsigned int readid_flags)
4037	{
4038	u64 n_sectors = dev->n_sectors;
4039	u64 n_native_sectors = dev->n_native_sectors;
4040	int rc;
4041
4042	if (!ata_dev_enabled(dev))
4043	return -ENODEV;
4044
4045	/* fail early if !ATA && !ATAPI to avoid issuing [P]IDENTIFY to PMP */
4046	if (ata_class_enabled(new_class) && new_class == ATA_DEV_PMP) {
4047	ata_dev_info(dev, "class mismatch %u != %u\n",
4048	dev->class, new_class);
4049	rc = -ENODEV;
4050	goto fail;
4051	}
4052
4053	/* re-read ID */
4054	rc = ata_dev_reread_id(dev, readid_flags);
4055	if (rc)
4056	goto fail;
4057
4058	/* configure device according to the new ID */
4059	rc = ata_dev_configure(dev);
4060	if (rc)
4061	goto fail;
4062
4063	/* verify n_sectors hasn't changed */
4064	if (dev->class != ATA_DEV_ATA \|\| !n_sectors \|\|
4065	dev->n_sectors == n_sectors \|\| ata_id_is_locked(dev->id))
4066	return 0;
4067
4068	/* n_sectors has changed */
4069	ata_dev_warn(dev, "n_sectors mismatch %llu != %llu\n",
4070	(unsigned long long)n_sectors,
4071	(unsigned long long)dev->n_sectors);
4072
4073	/*
4074	* Something could have caused HPA to be unlocked
4075	* involuntarily. If n_native_sectors hasn't changed and the
4076	* new size matches it, keep the device.
4077	*/
4078	if (dev->n_native_sectors == n_native_sectors &&
4079	dev->n_sectors > n_sectors && dev->n_sectors == n_native_sectors) {
4080	ata_dev_warn(dev,
4081	"new n_sectors matches native, probably "
4082	"late HPA unlock, n_sectors updated\n");
4083	/* use the larger n_sectors */
4084	return 0;
4085	}
4086
4087	/*
4088	* Some BIOSes boot w/o HPA but resume w/ HPA locked. Try
4089	* unlocking HPA in those cases.
4090	*
4091	* https://bugzilla.kernel.org/show_bug.cgi?id=15396
4092	*/
4093	if (dev->n_native_sectors == n_native_sectors &&
4094	dev->n_sectors < n_sectors && n_sectors == n_native_sectors &&
4095	!(dev->quirks & ATA_QUIRK_BROKEN_HPA)) {
4096	ata_dev_warn(dev,
4097	"old n_sectors matches native, probably "
4098	"late HPA lock, will try to unlock HPA\n");
4099	/* try unlocking HPA */
4100	dev->flags \|= ATA_DFLAG_UNLOCK_HPA;
4101	rc = -EIO;
4102	} else
4103	rc = -ENODEV;
4104
4105	/* restore original n_[native_]sectors and fail */
4106	dev->n_native_sectors = n_native_sectors;
4107	dev->n_sectors = n_sectors;
4108	fail:
4109	ata_dev_err(dev, "revalidation failed (errno=%d)\n", rc);
4110	return rc;
4111	}
4112
4113	static const char * const ata_quirk_names[] = {
4114	[__ATA_QUIRK_DIAGNOSTIC] = "diagnostic",
4115	[__ATA_QUIRK_NODMA] = "nodma",
4116	[__ATA_QUIRK_NONCQ] = "noncq",
4117	[__ATA_QUIRK_BROKEN_HPA] = "brokenhpa",
4118	[__ATA_QUIRK_DISABLE] = "disable",
4119	[__ATA_QUIRK_HPA_SIZE] = "hpasize",
4120	[__ATA_QUIRK_IVB] = "ivb",
4121	[__ATA_QUIRK_STUCK_ERR] = "stuckerr",
4122	[__ATA_QUIRK_BRIDGE_OK] = "bridgeok",
4123	[__ATA_QUIRK_ATAPI_MOD16_DMA] = "atapimod16dma",
4124	[__ATA_QUIRK_FIRMWARE_WARN] = "firmwarewarn",
4125	[__ATA_QUIRK_1_5_GBPS] = "1.5gbps",
4126	[__ATA_QUIRK_NOSETXFER] = "nosetxfer",
4127	[__ATA_QUIRK_BROKEN_FPDMA_AA] = "brokenfpdmaaa",
4128	[__ATA_QUIRK_DUMP_ID] = "dumpid",
4129	[__ATA_QUIRK_MAX_SEC_LBA48] = "maxseclba48",
4130	[__ATA_QUIRK_ATAPI_DMADIR] = "atapidmadir",
4131	[__ATA_QUIRK_NO_NCQ_TRIM] = "noncqtrim",
4132	[__ATA_QUIRK_NOLPM] = "nolpm",
4133	[__ATA_QUIRK_WD_BROKEN_LPM] = "wdbrokenlpm",
4134	[__ATA_QUIRK_ZERO_AFTER_TRIM] = "zeroaftertrim",
4135	[__ATA_QUIRK_NO_DMA_LOG] = "nodmalog",
4136	[__ATA_QUIRK_NOTRIM] = "notrim",
4137	[__ATA_QUIRK_MAX_SEC] = "maxsec",
4138	[__ATA_QUIRK_MAX_TRIM_128M] = "maxtrim128m",
4139	[__ATA_QUIRK_NO_NCQ_ON_ATI] = "noncqonati",
4140	[__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD] = "nolpmonatiandamd",
4141	[__ATA_QUIRK_NO_ID_DEV_LOG] = "noiddevlog",
4142	[__ATA_QUIRK_NO_LOG_DIR] = "nologdir",
4143	[__ATA_QUIRK_NO_FUA] = "nofua",
4144	};
4145
4146	static void ata_dev_print_quirks(const struct ata_device *dev,
4147	const char model, const char rev,
4148	unsigned int quirks)
4149	{
4150	struct ata_eh_context *ehc = &dev->link->eh_context;
4151	int n = 0, i;
4152	size_t sz;
4153	char *str;
4154
4155	if (!ata_dev_print_info(dev) \|\| ehc->i.flags & ATA_EHI_DID_PRINT_QUIRKS)
4156	return;
4157
4158	ehc->i.flags \|= ATA_EHI_DID_PRINT_QUIRKS;
4159
4160	if (!quirks)
4161	return;
4162
4163	sz = 64 + ARRAY_SIZE(ata_quirk_names) * 16;
4164	str = kmalloc(sz, GFP_KERNEL);
4165	if (!str)
4166	return;
4167
4168	n = snprintf(str, sz, "Model '%s', rev '%s', applying quirks:",
4169	model, rev);
4170
4171	for (i = 0; i < ARRAY_SIZE(ata_quirk_names); i++) {
4172	if (quirks & (1U << i))
4173	n += snprintf(str + n, sz - n,
4174	" %s", ata_quirk_names[i]);
4175	}
4176
4177	ata_dev_warn(dev, "%s\n", str);
4178
4179	kfree(str);
4180	}
4181
4182	struct ata_dev_quirk_value {
4183	const char *model_num;
4184	const char *model_rev;
4185	u64 val;
4186	};
4187
4188	static const struct ata_dev_quirk_value __ata_dev_max_sec_quirks[] = {
4189	{ "TORiSAN DVD-ROM DRD-N216", NULL, 128 },
4190	{ "ST380013AS", "3.20", 1024 },
4191	{ "LITEON CX1-JB*-HP", NULL, 1024 },
4192	{ "LITEON EP1-*", NULL, 1024 },
4193	{ "DELLBOSS VD", "MV.R00-0", 8191 },
4194	{ "INTEL SSDSC2KG480G8", "XCV10120", 8191 },
4195	{ },
4196	};
4197
4198	struct ata_dev_quirks_entry {
4199	const char *model_num;
4200	const char *model_rev;
4201	u64 quirks;
4202	};
4203
4204	static const struct ata_dev_quirks_entry __ata_dev_quirks[] = {
4205	/* Devices with DMA related problems under Linux */
4206	{ "WDC AC11000H", NULL, ATA_QUIRK_NODMA },
4207	{ "WDC AC22100H", NULL, ATA_QUIRK_NODMA },
4208	{ "WDC AC32500H", NULL, ATA_QUIRK_NODMA },
4209	{ "WDC AC33100H", NULL, ATA_QUIRK_NODMA },
4210	{ "WDC AC31600H", NULL, ATA_QUIRK_NODMA },
4211	{ "WDC AC32100H", "24.09P07", ATA_QUIRK_NODMA },
4212	{ "WDC AC23200L", "21.10N21", ATA_QUIRK_NODMA },
4213	{ "Compaq CRD-8241B", NULL, ATA_QUIRK_NODMA },
4214	{ "CRD-8400B", NULL, ATA_QUIRK_NODMA },
4215	{ "CRD-848[02]B", NULL, ATA_QUIRK_NODMA },
4216	{ "CRD-84", NULL, ATA_QUIRK_NODMA },
4217	{ "SanDisk SDP3B", NULL, ATA_QUIRK_NODMA },
4218	{ "SanDisk SDP3B-64", NULL, ATA_QUIRK_NODMA },
4219	{ "SANYO CD-ROM CRD", NULL, ATA_QUIRK_NODMA },
4220	{ "HITACHI CDR-8", NULL, ATA_QUIRK_NODMA },
4221	{ "HITACHI CDR-8[34]35", NULL, ATA_QUIRK_NODMA },
4222	{ "Toshiba CD-ROM XM-6202B", NULL, ATA_QUIRK_NODMA },
4223	{ "TOSHIBA CD-ROM XM-1702BC", NULL, ATA_QUIRK_NODMA },
4224	{ "CD-532E-A", NULL, ATA_QUIRK_NODMA },
4225	{ "E-IDE CD-ROM CR-840", NULL, ATA_QUIRK_NODMA },
4226	{ "CD-ROM Drive/F5A", NULL, ATA_QUIRK_NODMA },
4227	{ "WPI CDD-820", NULL, ATA_QUIRK_NODMA },
4228	{ "SAMSUNG CD-ROM SC-148C", NULL, ATA_QUIRK_NODMA },
4229	{ "SAMSUNG CD-ROM SC", NULL, ATA_QUIRK_NODMA },
4230	{ "ATAPI CD-ROM DRIVE 40X MAXIMUM", NULL, ATA_QUIRK_NODMA },
4231	{ "_NEC DV5800A", NULL, ATA_QUIRK_NODMA },
4232	{ "SAMSUNG CD-ROM SN-124", "N001", ATA_QUIRK_NODMA },
4233	{ "Seagate STT20000A", NULL, ATA_QUIRK_NODMA },
4234	{ " 2GB ATA Flash Disk", "ADMA428M", ATA_QUIRK_NODMA },
4235	{ "VRFDFC22048UCHC-TE*", NULL, ATA_QUIRK_NODMA },
4236	/* Odd clown on sil3726/4726 PMPs */
4237	{ "Config Disk", NULL, ATA_QUIRK_DISABLE },
4238	/* Similar story with ASMedia 1092 */
4239	{ "ASMT109x- Config", NULL, ATA_QUIRK_DISABLE },
4240
4241	/* Weird ATAPI devices */
4242	{ "TORiSAN DVD-ROM DRD-N216", NULL, ATA_QUIRK_MAX_SEC },
4243	{ "QUANTUM DAT DAT72-000", NULL, ATA_QUIRK_ATAPI_MOD16_DMA },
4244	{ "Slimtype DVD A DS8A8SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 },
4245	{ "Slimtype DVD A DS8A9SH", NULL, ATA_QUIRK_MAX_SEC_LBA48 },
4246
4247	/*
4248	* Causes silent data corruption with higher max sects.
4249	* http://lkml.kernel.org/g/[email protected]
4250	*/
4251	{ "ST380013AS", "3.20", ATA_QUIRK_MAX_SEC },
4252
4253	/*
4254	* These devices time out with higher max sects.
4255	* https://bugzilla.kernel.org/show_bug.cgi?id=121671
4256	*/
4257	{ "LITEON CX1-JB*-HP", NULL, ATA_QUIRK_MAX_SEC },
4258	{ "LITEON EP1-*", NULL, ATA_QUIRK_MAX_SEC },
4259
4260	/*
4261	* These devices time out with higher max sects.
4262	* https://bugzilla.kernel.org/show_bug.cgi?id=220693
4263	*/
4264	{ "DELLBOSS VD", "MV.R00-0", ATA_QUIRK_MAX_SEC },
4265
4266	/* Devices we expect to fail diagnostics */
4267
4268	/* Devices where NCQ should be avoided */
4269	/* NCQ is slow */
4270	{ "WDC WD740ADFD-00", NULL, ATA_QUIRK_NONCQ },
4271	{ "WDC WD740ADFD-00NLR1", NULL, ATA_QUIRK_NONCQ },
4272	/* http://thread.gmane.org/gmane.linux.ide/14907 */
4273	{ "FUJITSU MHT2060BH", NULL, ATA_QUIRK_NONCQ },
4274	/* NCQ is broken */
4275	{ "Maxtor ", "BANC", ATA_QUIRK_NONCQ },
4276	{ "Maxtor 7V300F0", "VA111630", ATA_QUIRK_NONCQ },
4277	{ "ST380817AS", "3.42", ATA_QUIRK_NONCQ },
4278	{ "ST3160023AS", "3.42", ATA_QUIRK_NONCQ },
4279	{ "OCZ CORE_SSD", "02.10104", ATA_QUIRK_NONCQ },
4280
4281	/* Seagate NCQ + FLUSH CACHE firmware bug */
4282	{ "ST31500341AS", "SD1[5-9]", ATA_QUIRK_NONCQ \|
4283	ATA_QUIRK_FIRMWARE_WARN },
4284
4285	{ "ST31000333AS", "SD1[5-9]", ATA_QUIRK_NONCQ \|
4286	ATA_QUIRK_FIRMWARE_WARN },
4287
4288	{ "ST3640[36]23AS", "SD1[5-9]", ATA_QUIRK_NONCQ \|
4289	ATA_QUIRK_FIRMWARE_WARN },
4290
4291	{ "ST3320[68]13AS", "SD1[5-9]", ATA_QUIRK_NONCQ \|
4292	ATA_QUIRK_FIRMWARE_WARN },
4293
4294	/* ADATA devices with LPM issues. */
4295	{ "ADATA SU680", NULL, ATA_QUIRK_NOLPM },
4296
4297	/* Seagate disks with LPM issues */
4298	{ "ST1000DM010-2EP102", NULL, ATA_QUIRK_NOLPM },
4299	{ "ST2000DM008-2FR102", NULL, ATA_QUIRK_NOLPM },
4300
4301	/* drives which fail FPDMA_AA activation (some may freeze afterwards)
4302	the ST disks also have LPM issues */
4303	{ "ST1000LM024 HN-M101MBB", NULL, ATA_QUIRK_BROKEN_FPDMA_AA \|
4304	ATA_QUIRK_NOLPM },
4305	{ "VB0250EAVER", "HPG7", ATA_QUIRK_BROKEN_FPDMA_AA },
4306
4307	/* Blacklist entries taken from Silicon Image 3124/3132
4308	Windows driver .inf file - also several Linux problem reports */
4309	{ "HTS541060G9SA00", "MB3OC60D", ATA_QUIRK_NONCQ },
4310	{ "HTS541080G9SA00", "MB4OC60D", ATA_QUIRK_NONCQ },
4311	{ "HTS541010G9SA00", "MBZOC60D", ATA_QUIRK_NONCQ },
4312
4313	/* https://bugzilla.kernel.org/show_bug.cgi?id=15573 */
4314	{ "C300-CTFDDAC128MAG", "0001", ATA_QUIRK_NONCQ },
4315
4316	/* Sandisk SD7/8/9s lock up hard on large trims */
4317	{ "SanDisk SD[789]*", NULL, ATA_QUIRK_MAX_TRIM_128M },
4318
4319	/* devices which puke on READ_NATIVE_MAX */
4320	{ "HDS724040KLSA80", "KFAOA20N", ATA_QUIRK_BROKEN_HPA },
4321	{ "WDC WD3200JD-00KLB0", "WD-WCAMR1130137", ATA_QUIRK_BROKEN_HPA },
4322	{ "WDC WD2500JD-00HBB0", "WD-WMAL71490727", ATA_QUIRK_BROKEN_HPA },
4323	{ "MAXTOR 6L080L4", "A93.0500", ATA_QUIRK_BROKEN_HPA },
4324
4325	/* this one allows HPA unlocking but fails IOs on the area */
4326	{ "OCZ-VERTEX", "1.30", ATA_QUIRK_BROKEN_HPA },
4327
4328	/* Devices which report 1 sector over size HPA */
4329	{ "ST340823A", NULL, ATA_QUIRK_HPA_SIZE },
4330	{ "ST320413A", NULL, ATA_QUIRK_HPA_SIZE },
4331	{ "ST310211A", NULL, ATA_QUIRK_HPA_SIZE },
4332
4333	/* Devices which get the IVB wrong */
4334	{ "QUANTUM FIREBALLlct10 05", "A03.0900", ATA_QUIRK_IVB },
4335	/* Maybe we should just add all TSSTcorp devices... */
4336	{ "TSSTcorp CDDVDW SH-S202[HJN]", "SB0[01]", ATA_QUIRK_IVB },
4337
4338	/* Devices that do not need bridging limits applied */
4339	{ "MTRON MSP-SATA*", NULL, ATA_QUIRK_BRIDGE_OK },
4340	{ "BUFFALO HD-QSU2/R5", NULL, ATA_QUIRK_BRIDGE_OK },
4341	{ "QEMU HARDDISK", "2.5+", ATA_QUIRK_BRIDGE_OK },
4342
4343	/* Devices which aren't very happy with higher link speeds */
4344	{ "WD My Book", NULL, ATA_QUIRK_1_5_GBPS },
4345	{ "Seagate FreeAgent GoFlex", NULL, ATA_QUIRK_1_5_GBPS },
4346
4347	/*
4348	* Devices which choke on SETXFER. Applies only if both the
4349	* device and controller are SATA.
4350	*/
4351	{ "PIONEER DVD-RW DVRTD08", NULL, ATA_QUIRK_NOSETXFER },
4352	{ "PIONEER DVD-RW DVRTD08A", NULL, ATA_QUIRK_NOSETXFER },
4353	{ "PIONEER DVD-RW DVR-215", NULL, ATA_QUIRK_NOSETXFER },
4354	{ "PIONEER DVD-RW DVR-212D", NULL, ATA_QUIRK_NOSETXFER },
4355	{ "PIONEER DVD-RW DVR-216D", NULL, ATA_QUIRK_NOSETXFER },
4356
4357	/* These specific Pioneer models have LPM issues */
4358	{ "PIONEER BD-RW BDR-207M", NULL, ATA_QUIRK_NOLPM },
4359	{ "PIONEER BD-RW BDR-205", NULL, ATA_QUIRK_NOLPM },
4360
4361	/* Crucial devices with broken LPM support */
4362	{ "CT0BX00SSD1", NULL, ATA_QUIRK_NOLPM },
4363
4364	/* 512GB MX100 with MU01 firmware has both queued TRIM and LPM issues */
4365	{ "Crucial_CT512MX100*", "MU01", ATA_QUIRK_NO_NCQ_TRIM \|
4366	ATA_QUIRK_ZERO_AFTER_TRIM \|
4367	ATA_QUIRK_NOLPM },
4368	/* 512GB MX100 with newer firmware has only LPM issues */
4369	{ "Crucial_CT512MX100*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM \|
4370	ATA_QUIRK_NOLPM },
4371
4372	/* 480GB+ M500 SSDs have both queued TRIM and LPM issues */
4373	{ "Crucial_CT480M500*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4374	ATA_QUIRK_ZERO_AFTER_TRIM \|
4375	ATA_QUIRK_NOLPM },
4376	{ "Crucial_CT960M500*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4377	ATA_QUIRK_ZERO_AFTER_TRIM \|
4378	ATA_QUIRK_NOLPM },
4379
4380	/* AMD Radeon devices with broken LPM support */
4381	{ "R3SL240G", NULL, ATA_QUIRK_NOLPM },
4382
4383	/* Apacer models with LPM issues */
4384	{ "Apacer AS340*", NULL, ATA_QUIRK_NOLPM },
4385
4386	/* PNY CS900 (Phison PS3111-S11, DRAM-less) drops the link on DIPM */
4387	{ "PNY CS900 1TB SSD", NULL, ATA_QUIRK_NOLPM },
4388
4389	/* Silicon Motion models with LPM issues */
4390	{ "MD619HXCLDE3TC", "TCVAID", ATA_QUIRK_NOLPM },
4391	{ "MD619GXCLDE3TC", "TCV35D", ATA_QUIRK_NOLPM },
4392
4393	/* These specific Samsung models/firmware-revs do not handle LPM well */
4394	{ "SAMSUNG MZMPC128HBFU-000MV", "CXM14M1Q", ATA_QUIRK_NOLPM },
4395	{ "SAMSUNG SSD PM830 mSATA *", "CXM13D1Q", ATA_QUIRK_NOLPM },
4396	{ "SAMSUNG MZ7TD256HAFV-000L9", NULL, ATA_QUIRK_NOLPM },
4397	{ "SAMSUNG MZ7TE512HMHP-000L1", "EXT06L0Q", ATA_QUIRK_NOLPM },
4398
4399	/* devices that don't properly handle queued TRIM commands */
4400	{ "Micron_M500IT_*", "MU01", ATA_QUIRK_NO_NCQ_TRIM \|
4401	ATA_QUIRK_ZERO_AFTER_TRIM },
4402	{ "Micron_M500_*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4403	ATA_QUIRK_ZERO_AFTER_TRIM },
4404	{ "Micron_M5[15]0_*", "MU01", ATA_QUIRK_NO_NCQ_TRIM \|
4405	ATA_QUIRK_ZERO_AFTER_TRIM },
4406	{ "Micron_1100_*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4407	ATA_QUIRK_ZERO_AFTER_TRIM, },
4408	{ "Crucial_CTM500", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4409	ATA_QUIRK_ZERO_AFTER_TRIM },
4410	{ "Crucial_CTM550", "MU01", ATA_QUIRK_NO_NCQ_TRIM \|
4411	ATA_QUIRK_ZERO_AFTER_TRIM },
4412	{ "Crucial_CTMX100", "MU01", ATA_QUIRK_NO_NCQ_TRIM \|
4413	ATA_QUIRK_ZERO_AFTER_TRIM },
4414	{ "Samsung SSD 840 EVO*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4415	ATA_QUIRK_NO_DMA_LOG \|
4416	ATA_QUIRK_ZERO_AFTER_TRIM },
4417	{ "Samsung SSD 840*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4418	ATA_QUIRK_ZERO_AFTER_TRIM },
4419	{ "Samsung SSD 850*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4420	ATA_QUIRK_ZERO_AFTER_TRIM },
4421	{ "Samsung SSD 860*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4422	ATA_QUIRK_ZERO_AFTER_TRIM \|
4423	ATA_QUIRK_NO_NCQ_ON_ATI \|
4424	ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
4425	{ "Samsung SSD 870*", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4426	ATA_QUIRK_ZERO_AFTER_TRIM \|
4427	ATA_QUIRK_NO_NCQ_ON_ATI \|
4428	ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
4429	{ "SAMSUNGMZ7LH", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4430	ATA_QUIRK_ZERO_AFTER_TRIM \|
4431	ATA_QUIRK_NO_NCQ_ON_ATI \|
4432	ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD },
4433	{ "FCCTM500", NULL, ATA_QUIRK_NO_NCQ_TRIM \|
4434	ATA_QUIRK_ZERO_AFTER_TRIM },
4435
4436	/* devices that don't properly handle TRIM commands */
4437	{ "SuperSSpeed S238*", NULL, ATA_QUIRK_NOTRIM },
4438	{ "M88V29*", NULL, ATA_QUIRK_NOTRIM },
4439
4440	/*
4441	* As defined, the DRAT (Deterministic Read After Trim) and RZAT
4442	* (Return Zero After Trim) flags in the ATA Command Set are
4443	* unreliable in the sense that they only define what happens if
4444	* the device successfully executed the DSM TRIM command. TRIM
4445	* is only advisory, however, and the device is free to silently
4446	* ignore all or parts of the request.
4447	*
4448	* Whitelist drives that are known to reliably return zeroes
4449	* after TRIM.
4450	*/
4451
4452	/*
4453	* The intel 510 drive has buggy DRAT/RZAT. Explicitly exclude
4454	* that model before whitelisting all other intel SSDs.
4455	*/
4456	{ "INTELSSDSC2MH", NULL, 0 },
4457
4458	{ "Micron*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4459	{ "Crucial*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4460	{ "INTEL SSDSC2KG480G8", "XCV10120", ATA_QUIRK_ZERO_AFTER_TRIM \|
4461	ATA_QUIRK_MAX_SEC },
4462	{ "INTELSSD", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4463	{ "SSDINTEL", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4464	{ "SamsungSSD", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4465	{ "SAMSUNGSSD", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4466	{ "SAMSUNGMZ7KM", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4467	{ "ST[1248][0248]0[FH]*", NULL, ATA_QUIRK_ZERO_AFTER_TRIM },
4468
4469	/*
4470	* Some WD SATA-I drives spin up and down erratically when the link
4471	* is put into the slumber mode. We don't have full list of the
4472	* affected devices. Disable LPM if the device matches one of the
4473	* known prefixes and is SATA-1. As a side effect LPM partial is
4474	* lost too.
4475	*
4476	* https://bugzilla.kernel.org/show_bug.cgi?id=57211
4477	*/
4478	{ "WDC WD800JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4479	{ "WDC WD1200JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4480	{ "WDC WD1600JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4481	{ "WDC WD2000JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4482	{ "WDC WD2500JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4483	{ "WDC WD3000JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4484	{ "WDC WD3200JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM },
4485
4486	/*
4487	* WD drives with LPM issues (irrespective of supported SATA speeds).
4488	* (Unlike ATA_QUIRK_WD_BROKEN_LPM, which is only applied if the drive
4489	* exposes SATA Gen1 speed support, and SATA Gen1 speed support only.)
4490	*/
4491	{ "WDC WD100EFGX-68CPLN0", NULL, ATA_QUIRK_NOLPM },
4492	{ "WDC WD102KFBX-68M95N0", NULL, ATA_QUIRK_NOLPM },
4493	{ "WDC WD141KFGX-68FH9N0", NULL, ATA_QUIRK_NOLPM },
4494	{ "WD Green 2.5 480GB", NULL, ATA_QUIRK_NOLPM },
4495
4496	/*
4497	* This sata dom device goes on a walkabout when the ATA_LOG_DIRECTORY
4498	* log page is accessed. Ensure we never ask for this log page with
4499	* these devices.
4500	*/
4501	{ "SATADOM-ML 3ME", NULL, ATA_QUIRK_NO_LOG_DIR },
4502
4503	/* Buggy FUA */
4504	{ "Maxtor", "BANC1G10", ATA_QUIRK_NO_FUA },
4505	{ "WDCWD2500J", NULL, ATA_QUIRK_NO_FUA },
4506	{ "OCZ-VERTEX*", NULL, ATA_QUIRK_NO_FUA },
4507	{ "INTELSSDSC2CT", NULL, ATA_QUIRK_NO_FUA },
4508
4509	/* End Marker */
4510	{ }
4511	};
4512
4513	static u64 ata_dev_quirks(const struct ata_device *dev)
4514	{
4515	unsigned char model_num[ATA_ID_PROD_LEN + 1];
4516	unsigned char model_rev[ATA_ID_FW_REV_LEN + 1];
4517	const struct ata_dev_quirks_entry *ad = __ata_dev_quirks;
4518
4519	/* dev->quirks is an u64. */
4520	BUILD_BUG_ON(__ATA_QUIRK_MAX > 64);
4521
4522	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
4523	ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev));
4524
4525	while (ad->model_num) {
4526	if (glob_match(ad->model_num, model_num) &&
4527	(!ad->model_rev \|\| glob_match(ad->model_rev, model_rev))) {
4528	ata_dev_print_quirks(dev, model_num, model_rev,
4529	ad->quirks);
4530	return ad->quirks;
4531	}
4532	ad++;
4533	}
4534	return 0;
4535	}
4536
4537	static u64 ata_dev_get_max_sec_quirk_value(struct ata_device *dev)
4538	{
4539	unsigned char model_num[ATA_ID_PROD_LEN + 1];
4540	unsigned char model_rev[ATA_ID_FW_REV_LEN + 1];
4541	const struct ata_dev_quirk_value *ad = __ata_dev_max_sec_quirks;
4542	u64 val = 0;
4543
4544	#ifdef CONFIG_ATA_FORCE
4545	const struct ata_force_ent *fe = ata_force_get_fe_for_dev(dev);
4546	if (fe && (fe->param.quirk_on & ATA_QUIRK_MAX_SEC) && fe->param.value)
4547	val = fe->param.value;
4548	#endif
4549	if (val)
4550	goto out;
4551
4552	ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
4553	ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev));
4554
4555	while (ad->model_num) {
4556	if (glob_match(ad->model_num, model_num) &&
4557	(!ad->model_rev \|\| glob_match(ad->model_rev, model_rev))) {
4558	val = ad->val;
4559	break;
4560	}
4561	ad++;
4562	}
4563
4564	out:
4565	ata_dev_warn(dev, "%s quirk is using value: %llu\n",
4566	ata_quirk_names[__ATA_QUIRK_MAX_SEC], val);
4567
4568	return val;
4569	}
4570
4571	static u64 ata_dev_get_quirk_value(struct ata_device *dev, u64 quirk)
4572	{
4573	if (quirk == ATA_QUIRK_MAX_SEC)
4574	return ata_dev_get_max_sec_quirk_value(dev);
4575
4576	return 0;
4577	}
4578
4579	static bool ata_dev_nodma(const struct ata_device *dev)
4580	{
4581	/*
4582	* We do not support polling DMA. Deny DMA for those ATAPI devices
4583	* with CDB-intr (and use PIO) if the LLDD handles only interrupts in
4584	* the HSM_ST_LAST state.
4585	*/
4586	if ((dev->link->ap->flags & ATA_FLAG_PIO_POLLING) &&
4587	(dev->flags & ATA_DFLAG_CDB_INTR))
4588	return true;
4589	return dev->quirks & ATA_QUIRK_NODMA;
4590	}
4591
4592	/**
4593	* ata_is_40wire - check drive side detection
4594	* @dev: device
4595	*
4596	* Perform drive side detection decoding, allowing for device vendors
4597	* who can't follow the documentation.
4598	*/
4599
4600	static int ata_is_40wire(struct ata_device *dev)
4601	{
4602	if (dev->quirks & ATA_QUIRK_IVB)
4603	return ata_drive_40wire_relaxed(dev->id);
4604	return ata_drive_40wire(dev->id);
4605	}
4606
4607	/**
4608	* cable_is_40wire - 40/80/SATA decider
4609	* @ap: port to consider
4610	*
4611	* This function encapsulates the policy for speed management
4612	* in one place. At the moment we don't cache the result but
4613	* there is a good case for setting ap->cbl to the result when
4614	* we are called with unknown cables (and figuring out if it
4615	* impacts hotplug at all).
4616	*
4617	* Return 1 if the cable appears to be 40 wire.
4618	*/
4619
4620	static int cable_is_40wire(struct ata_port *ap)
4621	{
4622	struct ata_link *link;
4623	struct ata_device *dev;
4624
4625	/* If the controller thinks we are 40 wire, we are. */
4626	if (ap->cbl == ATA_CBL_PATA40)
4627	return 1;
4628
4629	/* If the controller thinks we are 80 wire, we are. */
4630	if (ap->cbl == ATA_CBL_PATA80 \|\| ap->cbl == ATA_CBL_SATA)
4631	return 0;
4632
4633	/* If the system is known to be 40 wire short cable (eg
4634	* laptop), then we allow 80 wire modes even if the drive
4635	* isn't sure.
4636	*/
4637	if (ap->cbl == ATA_CBL_PATA40_SHORT)
4638	return 0;
4639
4640	/* If the controller doesn't know, we scan.
4641	*
4642	* Note: We look for all 40 wire detects at this point. Any
4643	* 80 wire detect is taken to be 80 wire cable because
4644	* - in many setups only the one drive (slave if present) will
4645	* give a valid detect
4646	* - if you have a non detect capable drive you don't want it
4647	* to colour the choice
4648	*/
4649	ata_for_each_link(link, ap, EDGE) {
4650	ata_for_each_dev(dev, link, ENABLED) {
4651	if (!ata_is_40wire(dev))
4652	return 0;
4653	}
4654	}
4655	return 1;
4656	}
4657
4658	/**
4659	* ata_dev_xfermask - Compute supported xfermask of the given device
4660	* @dev: Device to compute xfermask for
4661	*
4662	* Compute supported xfermask of @dev and store it in
4663	* dev->*_mask. This function is responsible for applying all
4664	* known limits including host controller limits, device quirks, etc...
4665	*
4666	* LOCKING:
4667	* None.
4668	*/
4669	static void ata_dev_xfermask(struct ata_device *dev)
4670	{
4671	struct ata_link *link = dev->link;
4672	struct ata_port *ap = link->ap;
4673	struct ata_host *host = ap->host;
4674	unsigned int xfer_mask;
4675
4676	/* controller modes available */
4677	xfer_mask = ata_pack_xfermask(ap->pio_mask,
4678	ap->mwdma_mask, ap->udma_mask);
4679
4680	/* drive modes available */
4681	xfer_mask &= ata_pack_xfermask(dev->pio_mask,
4682	dev->mwdma_mask, dev->udma_mask);
4683	xfer_mask &= ata_id_xfermask(dev->id);
4684
4685	/*
4686	* CFA Advanced TrueIDE timings are not allowed on a shared
4687	* cable
4688	*/
4689	if (ata_dev_pair(dev)) {
4690	/* No PIO5 or PIO6 */
4691	xfer_mask &= ~(0x03 << (ATA_SHIFT_PIO + 5));
4692	/* No MWDMA3 or MWDMA 4 */
4693	xfer_mask &= ~(0x03 << (ATA_SHIFT_MWDMA + 3));
4694	}
4695
4696	if (ata_dev_nodma(dev)) {
4697	xfer_mask &= ~(ATA_MASK_MWDMA \| ATA_MASK_UDMA);
4698	ata_dev_warn(dev,
4699	"device does not support DMA, disabling DMA\n");
4700	}
4701
4702	if ((host->flags & ATA_HOST_SIMPLEX) &&
4703	host->simplex_claimed && host->simplex_claimed != ap) {
4704	xfer_mask &= ~(ATA_MASK_MWDMA \| ATA_MASK_UDMA);
4705	ata_dev_warn(dev,
4706	"simplex DMA is claimed by other device, disabling DMA\n");
4707	}
4708
4709	if (ap->flags & ATA_FLAG_NO_IORDY)
4710	xfer_mask &= ata_pio_mask_no_iordy(dev);
4711
4712	if (ap->ops->mode_filter)
4713	xfer_mask = ap->ops->mode_filter(dev, xfer_mask);
4714
4715	/* Apply cable rule here. Don't apply it early because when
4716	* we handle hot plug the cable type can itself change.
4717	* Check this last so that we know if the transfer rate was
4718	* solely limited by the cable.
4719	* Unknown or 80 wire cables reported host side are checked
4720	* drive side as well. Cases where we know a 40wire cable
4721	* is used safely for 80 are not checked here.
4722	*/
4723	if (xfer_mask & (0xF8 << ATA_SHIFT_UDMA))
4724	/* UDMA/44 or higher would be available */
4725	if (cable_is_40wire(ap)) {
4726	ata_dev_warn(dev,
4727	"limited to UDMA/33 due to 40-wire cable\n");
4728	xfer_mask &= ~(0xF8 << ATA_SHIFT_UDMA);
4729	}
4730
4731	ata_unpack_xfermask(xfer_mask, &dev->pio_mask,
4732	&dev->mwdma_mask, &dev->udma_mask);
4733	}
4734
4735	/**
4736	* ata_dev_set_xfermode - Issue SET FEATURES - XFER MODE command
4737	* @dev: Device to which command will be sent
4738	*
4739	* Issue SET FEATURES - XFER MODE command to device @dev
4740	* on port @ap.
4741	*
4742	* LOCKING:
4743	* PCI/etc. bus probe sem.
4744	*
4745	* RETURNS:
4746	* 0 on success, AC_ERR_* mask otherwise.
4747	*/
4748
4749	static unsigned int ata_dev_set_xfermode(struct ata_device *dev)
4750	{
4751	struct ata_taskfile tf;
4752
4753	/* set up set-features taskfile */
4754	ata_dev_dbg(dev, "set features - xfer mode\n");
4755
4756	/* Some controllers and ATAPI devices show flaky interrupt
4757	* behavior after setting xfer mode. Use polling instead.
4758	*/
4759	ata_tf_init(dev, &tf);
4760	tf.command = ATA_CMD_SET_FEATURES;
4761	tf.feature = SETFEATURES_XFER;
4762	tf.flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_DEVICE \| ATA_TFLAG_POLLING;
4763	tf.protocol = ATA_PROT_NODATA;
4764	/* If we are using IORDY we must send the mode setting command */
4765	if (ata_pio_need_iordy(dev))
4766	tf.nsect = dev->xfer_mode;
4767	/* If the device has IORDY and the controller does not - turn it off */
4768	else if (ata_id_has_iordy(dev->id))
4769	tf.nsect = 0x01;
4770	else /* In the ancient relic department - skip all of this */
4771	return 0;
4772
4773	/*
4774	* On some disks, this command causes spin-up, so we need longer
4775	* timeout.
4776	*/
4777	return ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 15000);
4778	}
4779
4780	/**
4781	* ata_dev_set_feature - Issue SET FEATURES
4782	* @dev: Device to which command will be sent
4783	* @subcmd: The SET FEATURES subcommand to be sent
4784	* @action: The sector count represents a subcommand specific action
4785	*
4786	* Issue SET FEATURES command to device @dev on port @ap with sector count
4787	*
4788	* LOCKING:
4789	* PCI/etc. bus probe sem.
4790	*
4791	* RETURNS:
4792	* 0 on success, AC_ERR_* mask otherwise.
4793	*/
4794	unsigned int ata_dev_set_feature(struct ata_device *dev, u8 subcmd, u8 action)
4795	{
4796	struct ata_taskfile tf;
4797	unsigned int timeout = 0;
4798
4799	/* set up set-features taskfile */
4800	ata_dev_dbg(dev, "set features\n");
4801
4802	ata_tf_init(dev, &tf);
4803	tf.command = ATA_CMD_SET_FEATURES;
4804	tf.feature = subcmd;
4805	tf.flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_DEVICE;
4806	tf.protocol = ATA_PROT_NODATA;
4807	tf.nsect = action;
4808
4809	if (subcmd == SETFEATURES_SPINUP)
4810	timeout = ata_probe_timeout ?
4811	ata_probe_timeout * 1000 : SETFEATURES_SPINUP_TIMEOUT;
4812
4813	return ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, timeout);
4814	}
4815	EXPORT_SYMBOL_GPL(ata_dev_set_feature);
4816
4817	/**
4818	* ata_dev_init_params - Issue INIT DEV PARAMS command
4819	* @dev: Device to which command will be sent
4820	* @heads: Number of heads (taskfile parameter)
4821	* @sectors: Number of sectors (taskfile parameter)
4822	*
4823	* LOCKING:
4824	* Kernel thread context (may sleep)
4825	*
4826	* RETURNS:
4827	* 0 on success, AC_ERR_* mask otherwise.
4828	*/
4829	static unsigned int ata_dev_init_params(struct ata_device *dev,
4830	u16 heads, u16 sectors)
4831	{
4832	struct ata_taskfile tf;
4833	unsigned int err_mask;
4834
4835	/* Number of sectors per track 1-255. Number of heads 1-16 */
4836	if (sectors < 1 \|\| sectors > 255 \|\| heads < 1 \|\| heads > 16)
4837	return AC_ERR_INVALID;
4838
4839	/* set up init dev params taskfile */
4840	ata_dev_dbg(dev, "init dev params\n");
4841
4842	ata_tf_init(dev, &tf);
4843	tf.command = ATA_CMD_INIT_DEV_PARAMS;
4844	tf.flags \|= ATA_TFLAG_ISADDR \| ATA_TFLAG_DEVICE;
4845	tf.protocol = ATA_PROT_NODATA;
4846	tf.nsect = sectors;
4847	tf.device \|= (heads - 1) & 0x0f; /* max head = num. of heads - 1 */
4848
4849	err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 0);
4850	/* A clean abort indicates an original or just out of spec drive
4851	and we should continue as we issue the setup based on the
4852	drive reported working geometry */
4853	if (err_mask == AC_ERR_DEV && (tf.error & ATA_ABORTED))
4854	err_mask = 0;
4855
4856	return err_mask;
4857	}
4858
4859	/**
4860	* atapi_check_dma - Check whether ATAPI DMA can be supported
4861	* @qc: Metadata associated with taskfile to check
4862	*
4863	* Allow low-level driver to filter ATA PACKET commands, returning
4864	* a status indicating whether or not it is OK to use DMA for the
4865	* supplied PACKET command.
4866	*
4867	* LOCKING:
4868	* spin_lock_irqsave(host lock)
4869	*
4870	* RETURNS: 0 when ATAPI DMA can be used
4871	* nonzero otherwise
4872	*/
4873	int atapi_check_dma(struct ata_queued_cmd *qc)
4874	{
4875	struct ata_port *ap = qc->ap;
4876
4877	/* Don't allow DMA if it isn't multiple of 16 bytes. Quite a
4878	* few ATAPI devices choke on such DMA requests.
4879	*/
4880	if (!(qc->dev->quirks & ATA_QUIRK_ATAPI_MOD16_DMA) &&
4881	unlikely(qc->nbytes & 15))
4882	return -EOPNOTSUPP;
4883
4884	if (ap->ops->check_atapi_dma)
4885	return ap->ops->check_atapi_dma(qc);
4886
4887	return 0;
4888	}
4889
4890	/**
4891	* ata_std_qc_defer - Check whether a qc needs to be deferred
4892	* @qc: ATA command in question
4893	*
4894	* Non-NCQ commands cannot run with any other command, NCQ or
4895	* not. As upper layer only knows the queue depth, we are
4896	* responsible for maintaining exclusion. This function checks
4897	* whether a new command @qc can be issued.
4898	*
4899	* LOCKING:
4900	* spin_lock_irqsave(host lock)
4901	*
4902	* RETURNS:
4903	* ATA_DEFER_* if deferring is needed, 0 otherwise.
4904	*/
4905	int ata_std_qc_defer(struct ata_queued_cmd *qc)
4906	{
4907	struct ata_link *link = qc->dev->link;
4908
4909	if (ata_is_ncq(qc->tf.protocol)) {
4910	if (!ata_tag_valid(link->active_tag))
4911	return 0;
4912	} else {
4913	if (!ata_tag_valid(link->active_tag) && !link->sactive)
4914	return 0;
4915	}
4916
4917	return ATA_DEFER_LINK;
4918	}
4919	EXPORT_SYMBOL_GPL(ata_std_qc_defer);
4920
4921	/**
4922	* ata_sg_init - Associate command with scatter-gather table.
4923	* @qc: Command to be associated
4924	* @sg: Scatter-gather table.
4925	* @n_elem: Number of elements in s/g table.
4926	*
4927	* Initialize the data-related elements of queued_cmd @qc
4928	* to point to a scatter-gather table @sg, containing @n_elem
4929	* elements.
4930	*
4931	* LOCKING:
4932	* spin_lock_irqsave(host lock)
4933	*/
4934	void ata_sg_init(struct ata_queued_cmd qc, struct scatterlist sg,
4935	unsigned int n_elem)
4936	{
4937	qc->sg = sg;
4938	qc->n_elem = n_elem;
4939	qc->cursg = qc->sg;
4940	}
4941
4942	#ifdef CONFIG_HAS_DMA
4943
4944	/**
4945	* ata_sg_clean - Unmap DMA memory associated with command
4946	* @qc: Command containing DMA memory to be released
4947	*
4948	* Unmap all mapped DMA memory associated with this command.
4949	*
4950	* LOCKING:
4951	* spin_lock_irqsave(host lock)
4952	*/
4953	static void ata_sg_clean(struct ata_queued_cmd *qc)
4954	{
4955	struct ata_port *ap = qc->ap;
4956	struct scatterlist *sg = qc->sg;
4957	int dir = qc->dma_dir;
4958
4959	WARN_ON_ONCE(sg == NULL);
4960
4961	if (qc->n_elem)
4962	dma_unmap_sg(ap->dev, sg, qc->orig_n_elem, dir);
4963
4964	qc->flags &= ~ATA_QCFLAG_DMAMAP;
4965	qc->sg = NULL;
4966	}
4967
4968	/**
4969	* ata_sg_setup - DMA-map the scatter-gather table associated with a command.
4970	* @qc: Command with scatter-gather table to be mapped.
4971	*
4972	* DMA-map the scatter-gather table associated with queued_cmd @qc.
4973	*
4974	* LOCKING:
4975	* spin_lock_irqsave(host lock)
4976	*
4977	* RETURNS:
4978	* Zero on success, negative on error.
4979	*
4980	*/
4981	static int ata_sg_setup(struct ata_queued_cmd *qc)
4982	{
4983	struct ata_port *ap = qc->ap;
4984	unsigned int n_elem;
4985
4986	n_elem = dma_map_sg(ap->dev, qc->sg, qc->n_elem, qc->dma_dir);
4987	if (n_elem < 1)
4988	return -1;
4989
4990	qc->orig_n_elem = qc->n_elem;
4991	qc->n_elem = n_elem;
4992	qc->flags \|= ATA_QCFLAG_DMAMAP;
4993
4994	return 0;
4995	}
4996
4997	#else /* !CONFIG_HAS_DMA */
4998
4999	static inline void ata_sg_clean(struct ata_queued_cmd *qc) {}
5000	static inline int ata_sg_setup(struct ata_queued_cmd *qc) { return -1; }
5001
5002	#endif /* !CONFIG_HAS_DMA */
5003
5004	/**
5005	* swap_buf_le16 - swap halves of 16-bit words in place
5006	* @buf: Buffer to swap
5007	* @buf_words: Number of 16-bit words in buffer.
5008	*
5009	* Swap halves of 16-bit words if needed to convert from
5010	* little-endian byte order to native cpu byte order, or
5011	* vice-versa.
5012	*
5013	* LOCKING:
5014	* Inherited from caller.
5015	*/
5016	void swap_buf_le16(u16 *buf, unsigned int buf_words)
5017	{
5018	#ifdef __BIG_ENDIAN
5019	unsigned int i;
5020
5021	for (i = 0; i < buf_words; i++)
5022	buf[i] = le16_to_cpu(buf[i]);
5023	#endif /* __BIG_ENDIAN */
5024	}
5025
5026	/**
5027	* ata_qc_free - free unused ata_queued_cmd
5028	* @qc: Command to complete
5029	*
5030	* Designed to free unused ata_queued_cmd object
5031	* in case something prevents using it.
5032	*
5033	* LOCKING:
5034	* spin_lock_irqsave(host lock)
5035	*/
5036	void ata_qc_free(struct ata_queued_cmd *qc)
5037	{
5038	qc->flags = 0;
5039	if (ata_tag_valid(qc->tag))
5040	qc->tag = ATA_TAG_POISON;
5041	}
5042
5043	void __ata_qc_complete(struct ata_queued_cmd *qc)
5044	{
5045	struct ata_port *ap;
5046	struct ata_link *link;
5047
5048	if (WARN_ON_ONCE(!(qc->flags & ATA_QCFLAG_ACTIVE)))
5049	return;
5050
5051	ap = qc->ap;
5052	link = qc->dev->link;
5053
5054	if (likely(qc->flags & ATA_QCFLAG_DMAMAP))
5055	ata_sg_clean(qc);
5056
5057	/* command should be marked inactive atomically with qc completion */
5058	if (ata_is_ncq(qc->tf.protocol)) {
5059	link->sactive &= ~(1 << qc->hw_tag);
5060	if (!link->sactive)
5061	ap->nr_active_links--;
5062	} else {
5063	link->active_tag = ATA_TAG_POISON;
5064	ap->nr_active_links--;
5065	}
5066
5067	/* clear exclusive status */
5068	if (unlikely(qc->flags & ATA_QCFLAG_CLEAR_EXCL &&
5069	ap->excl_link == link))
5070	ap->excl_link = NULL;
5071
5072	/*
5073	* Mark qc as inactive to prevent the port interrupt handler from
5074	* completing the command twice later, before the error handler is
5075	* called.
5076	*/
5077	qc->flags &= ~ATA_QCFLAG_ACTIVE;
5078	ap->qc_active &= ~(1ULL << qc->tag);
5079
5080	/* call completion callback */
5081	qc->complete_fn(qc);
5082	}
5083
5084	static void fill_result_tf(struct ata_queued_cmd *qc)
5085	{
5086	struct ata_port *ap = qc->ap;
5087
5088	/*
5089	* rtf may already be filled (e.g. for successful NCQ commands).
5090	* If that is the case, we have nothing to do.
5091	*/
5092	if (qc->flags & ATA_QCFLAG_RTF_FILLED)
5093	return;
5094
5095	qc->result_tf.flags = qc->tf.flags;
5096	ap->ops->qc_fill_rtf(qc);
5097	qc->flags \|= ATA_QCFLAG_RTF_FILLED;
5098	}
5099
5100	static void ata_verify_xfer(struct ata_queued_cmd *qc)
5101	{
5102	struct ata_device *dev = qc->dev;
5103
5104	if (!ata_is_data(qc->tf.protocol))
5105	return;
5106
5107	if ((dev->mwdma_mask \|\| dev->udma_mask) && ata_is_pio(qc->tf.protocol))
5108	return;
5109
5110	dev->flags &= ~ATA_DFLAG_DUBIOUS_XFER;
5111	}
5112
5113	/**
5114	* ata_qc_complete - Complete an active ATA command
5115	* @qc: Command to complete
5116	*
5117	* Indicate to the mid and upper layers that an ATA command has
5118	* completed, with either an ok or not-ok status.
5119	*
5120	* Refrain from calling this function multiple times when
5121	* successfully completing multiple NCQ commands.
5122	* ata_qc_complete_multiple() should be used instead, which will
5123	* properly update IRQ expect state.
5124	*
5125	* LOCKING:
5126	* spin_lock_irqsave(host lock)
5127	*/
5128	void ata_qc_complete(struct ata_queued_cmd *qc)
5129	{
5130	struct ata_port *ap = qc->ap;
5131	struct ata_device *dev = qc->dev;
5132	struct ata_eh_info *ehi = &dev->link->eh_info;
5133
5134	/* Trigger the LED (if available) */
5135	ledtrig_disk_activity(!!(qc->tf.flags & ATA_TFLAG_WRITE));
5136
5137	/*
5138	* In order to synchronize EH with the regular execution path, a qc that
5139	* is owned by EH is marked with ATA_QCFLAG_EH.
5140	*
5141	* The normal execution path is responsible for not accessing a qc owned
5142	* by EH. libata core enforces the rule by returning NULL from
5143	* ata_qc_from_tag() for qcs owned by EH.
5144	*/
5145	if (unlikely(qc->err_mask))
5146	qc->flags \|= ATA_QCFLAG_EH;
5147
5148	/*
5149	* Finish internal commands without any further processing and always
5150	* with the result TF filled.
5151	*/
5152	if (unlikely(ata_tag_internal(qc->tag))) {
5153	fill_result_tf(qc);
5154	trace_ata_qc_complete_internal(qc);
5155	__ata_qc_complete(qc);
5156	return;
5157	}
5158
5159	/* Non-internal qc has failed. Fill the result TF and summon EH. */
5160	if (unlikely(qc->flags & ATA_QCFLAG_EH)) {
5161	fill_result_tf(qc);
5162	trace_ata_qc_complete_failed(qc);
5163	ata_qc_schedule_eh(qc);
5164	return;
5165	}
5166
5167	WARN_ON_ONCE(ata_port_is_frozen(ap));
5168
5169	/* read result TF if requested */
5170	if (qc->flags & ATA_QCFLAG_RESULT_TF)
5171	fill_result_tf(qc);
5172
5173	trace_ata_qc_complete_done(qc);
5174
5175	/*
5176	* For CDL commands that completed without an error, check if we have
5177	* sense data (ATA_SENSE is set). If we do, then the command may have
5178	* been aborted by the device due to a limit timeout using the policy
5179	* 0xD. For these commands, invoke EH to get the command sense data.
5180	*/
5181	if (qc->flags & ATA_QCFLAG_HAS_CDL &&
5182	qc->result_tf.status & ATA_SENSE) {
5183	/*
5184	* Tell SCSI EH to not overwrite scmd->result even if this
5185	* command is finished with result SAM_STAT_GOOD.
5186	*/
5187	qc->scsicmd->flags \|= SCMD_FORCE_EH_SUCCESS;
5188	qc->flags \|= ATA_QCFLAG_EH_SUCCESS_CMD;
5189	ehi->dev_action[dev->devno] \|= ATA_EH_GET_SUCCESS_SENSE;
5190
5191	/*
5192	* set pending so that ata_qc_schedule_eh() does not trigger
5193	* fast drain, and freeze the port.
5194	*/
5195	ap->pflags \|= ATA_PFLAG_EH_PENDING;
5196	ata_qc_schedule_eh(qc);
5197	return;
5198	}
5199
5200	/* Some commands need post-processing after successful completion. */
5201	switch (qc->tf.command) {
5202	case ATA_CMD_SET_FEATURES:
5203	if (qc->tf.feature != SETFEATURES_WC_ON &&
5204	qc->tf.feature != SETFEATURES_WC_OFF &&
5205	qc->tf.feature != SETFEATURES_RA_ON &&
5206	qc->tf.feature != SETFEATURES_RA_OFF)
5207	break;
5208	fallthrough;
5209	case ATA_CMD_INIT_DEV_PARAMS: /* CHS translation changed */
5210	case ATA_CMD_SET_MULTI: /* multi_count changed */
5211	/* revalidate device */
5212	ehi->dev_action[dev->devno] \|= ATA_EH_REVALIDATE;
5213	ata_port_schedule_eh(ap);
5214	break;
5215
5216	case ATA_CMD_SLEEP:
5217	dev->flags \|= ATA_DFLAG_SLEEPING;
5218	break;
5219	}
5220
5221	if (unlikely(dev->flags & ATA_DFLAG_DUBIOUS_XFER))
5222	ata_verify_xfer(qc);
5223
5224	__ata_qc_complete(qc);
5225	}
5226	EXPORT_SYMBOL_GPL(ata_qc_complete);
5227
5228	/**
5229	* ata_qc_get_active - get bitmask of active qcs
5230	* @ap: port in question
5231	*
5232	* LOCKING:
5233	* spin_lock_irqsave(host lock)
5234	*
5235	* RETURNS:
5236	* Bitmask of active qcs
5237	*/
5238	u64 ata_qc_get_active(struct ata_port *ap)
5239	{
5240	u64 qc_active = ap->qc_active;
5241
5242	/* ATA_TAG_INTERNAL is sent to hw as tag 0 */
5243	if (qc_active & (1ULL << ATA_TAG_INTERNAL)) {
5244	qc_active \|= (1 << 0);
5245	qc_active &= ~(1ULL << ATA_TAG_INTERNAL);
5246	}
5247
5248	return qc_active;
5249	}
5250	EXPORT_SYMBOL_GPL(ata_qc_get_active);
5251
5252	/**
5253	* ata_qc_issue - issue taskfile to device
5254	* @ap: ATA port of interest
5255	* @qc: command to issue to device
5256	*
5257	* Prepare an ATA command to submission to device.
5258	* This includes mapping the data into a DMA-able
5259	* area, filling in the S/G table, and finally
5260	* writing the taskfile to hardware, starting the command.
5261	*
5262	* LOCKING:
5263	* spin_lock_irqsave(host lock)
5264	*/
5265	void ata_qc_issue(struct ata_port ap, struct ata_queued_cmd qc)
5266	__must_hold(ap->lock)
5267	{
5268	struct ata_link *link = qc->dev->link;
5269	u8 prot = qc->tf.protocol;
5270
5271	/*
5272	* Make sure we have a valid tag and that only one non-NCQ command is
5273	* outstanding.
5274	*/
5275	if (WARN_ON_ONCE(!ata_tag_valid(qc->tag)) \|\|
5276	WARN_ON_ONCE(ata_tag_valid(link->active_tag)))
5277	goto sys_err;
5278
5279	if (ata_is_ncq(prot)) {
5280	WARN_ON_ONCE(link->sactive & (1 << qc->hw_tag));
5281
5282	if (!link->sactive)
5283	ap->nr_active_links++;
5284	link->sactive \|= 1 << qc->hw_tag;
5285	} else {
5286	WARN_ON_ONCE(link->sactive);
5287
5288	ap->nr_active_links++;
5289	link->active_tag = qc->tag;
5290	}
5291
5292	qc->flags \|= ATA_QCFLAG_ACTIVE;
5293	ap->qc_active \|= 1ULL << qc->tag;
5294
5295	/* Make sure the device is still accessible. */
5296	if (!ata_adapter_is_online(ap)) {
5297	qc->err_mask \|= AC_ERR_HOST_BUS;
5298	goto sys_err;
5299	}
5300
5301	/*
5302	* We guarantee to LLDs that they will have at least one
5303	* non-zero sg if the command is a data command.
5304	*/
5305	if (ata_is_data(prot) && (!qc->sg \|\| !qc->n_elem \|\| !qc->nbytes))
5306	goto sys_err;
5307
5308	if (ata_is_dma(prot) \|\| (ata_is_pio(prot) &&
5309	(ap->flags & ATA_FLAG_PIO_DMA)))
5310	if (ata_sg_setup(qc))
5311	goto sys_err;
5312
5313	/* if device is sleeping, schedule reset and abort the link */
5314	if (unlikely(qc->dev->flags & ATA_DFLAG_SLEEPING)) {
5315	link->eh_info.action \|= ATA_EH_RESET;
5316	ata_ehi_push_desc(&link->eh_info, "waking up from sleep");
5317	ata_link_abort(link);
5318	return;
5319	}
5320
5321	if (ap->ops->qc_prep) {
5322	trace_ata_qc_prep(qc);
5323	qc->err_mask \|= ap->ops->qc_prep(qc);
5324	if (unlikely(qc->err_mask))
5325	goto err;
5326	}
5327
5328	trace_ata_qc_issue(qc);
5329	qc->err_mask \|= ap->ops->qc_issue(qc);
5330	if (unlikely(qc->err_mask))
5331	goto err;
5332	return;
5333
5334	sys_err:
5335	qc->err_mask \|= AC_ERR_SYSTEM;
5336	err:
5337	ata_qc_complete(qc);
5338	}
5339
5340	/**
5341	* ata_phys_link_online - test whether the given link is online
5342	* @link: ATA link to test
5343	*
5344	* Test whether @link is online. Note that this function returns
5345	* 0 if online status of @link cannot be obtained, so
5346	* ata_link_online(link) != !ata_link_offline(link).
5347	*
5348	* LOCKING:
5349	* None.
5350	*
5351	* RETURNS:
5352	* True if the port online status is available and online.
5353	*/
5354	bool ata_phys_link_online(struct ata_link *link)
5355	{
5356	u32 sstatus;
5357
5358	if (sata_scr_read(link, SCR_STATUS, &sstatus) == 0 &&
5359	ata_sstatus_online(sstatus))
5360	return true;
5361	return false;
5362	}
5363
5364	/**
5365	* ata_phys_link_offline - test whether the given link is offline
5366	* @link: ATA link to test
5367	*
5368	* Test whether @link is offline. Note that this function
5369	* returns 0 if offline status of @link cannot be obtained, so
5370	* ata_link_online(link) != !ata_link_offline(link).
5371	*
5372	* LOCKING:
5373	* None.
5374	*
5375	* RETURNS:
5376	* True if the port offline status is available and offline.
5377	*/
5378	bool ata_phys_link_offline(struct ata_link *link)
5379	{
5380	u32 sstatus;
5381
5382	if (sata_scr_read(link, SCR_STATUS, &sstatus) == 0 &&
5383	!ata_sstatus_online(sstatus))
5384	return true;
5385	return false;
5386	}
5387
5388	/**
5389	* ata_link_online - test whether the given link is online
5390	* @link: ATA link to test
5391	*
5392	* Test whether @link is online. This is identical to
5393	* ata_phys_link_online() when there's no slave link. When
5394	* there's a slave link, this function should only be called on
5395	* the master link and will return true if any of M/S links is
5396	* online.
5397	*
5398	* LOCKING:
5399	* None.
5400	*
5401	* RETURNS:
5402	* True if the port online status is available and online.
5403	*/
5404	bool ata_link_online(struct ata_link *link)
5405	{
5406	struct ata_link *slave = link->ap->slave_link;
5407
5408	WARN_ON(link == slave); /* shouldn't be called on slave link */
5409
5410	return ata_phys_link_online(link) \|\|
5411	(slave && ata_phys_link_online(slave));
5412	}
5413	EXPORT_SYMBOL_GPL(ata_link_online);
5414
5415	/**
5416	* ata_link_offline - test whether the given link is offline
5417	* @link: ATA link to test
5418	*
5419	* Test whether @link is offline. This is identical to
5420	* ata_phys_link_offline() when there's no slave link. When
5421	* there's a slave link, this function should only be called on
5422	* the master link and will return true if both M/S links are
5423	* offline.
5424	*
5425	* LOCKING:
5426	* None.
5427	*
5428	* RETURNS:
5429	* True if the port offline status is available and offline.
5430	*/
5431	bool ata_link_offline(struct ata_link *link)
5432	{
5433	struct ata_link *slave = link->ap->slave_link;
5434
5435	WARN_ON(link == slave); /* shouldn't be called on slave link */
5436
5437	return ata_phys_link_offline(link) &&
5438	(!slave \|\| ata_phys_link_offline(slave));
5439	}
5440	EXPORT_SYMBOL_GPL(ata_link_offline);
5441
5442	#ifdef CONFIG_PM
5443	static void ata_port_request_pm(struct ata_port *ap, pm_message_t mesg,
5444	unsigned int action, unsigned int ehi_flags,
5445	bool async)
5446	{
5447	struct ata_link *link;
5448	unsigned long flags;
5449
5450	spin_lock_irqsave(ap->lock, flags);
5451
5452	/*
5453	* A previous PM operation might still be in progress. Wait for
5454	* ATA_PFLAG_PM_PENDING to clear.
5455	*/
5456	if (ap->pflags & ATA_PFLAG_PM_PENDING) {
5457	spin_unlock_irqrestore(ap->lock, flags);
5458	ata_port_wait_eh(ap);
5459	spin_lock_irqsave(ap->lock, flags);
5460	}
5461
5462	/* Request PM operation to EH */
5463	ap->pm_mesg = mesg;
5464	ap->pflags \|= ATA_PFLAG_PM_PENDING;
5465	ata_for_each_link(link, ap, HOST_FIRST) {
5466	link->eh_info.action \|= action;
5467	link->eh_info.flags \|= ehi_flags;
5468	}
5469
5470	ata_port_schedule_eh(ap);
5471
5472	spin_unlock_irqrestore(ap->lock, flags);
5473
5474	if (!async)
5475	ata_port_wait_eh(ap);
5476	}
5477
5478	static void ata_port_suspend(struct ata_port *ap, pm_message_t mesg,
5479	bool async)
5480	{
5481	/*
5482	* We are about to suspend the port, so we do not care about
5483	* scsi_rescan_device() calls scheduled by previous resume operations.
5484	* The next resume will schedule the rescan again. So cancel any rescan
5485	* that is not done yet.
5486	*/
5487	cancel_delayed_work_sync(&ap->scsi_rescan_task);
5488
5489	/*
5490	* On some hardware, device fails to respond after spun down for
5491	* suspend. As the device will not be used until being resumed, we
5492	* do not need to touch the device. Ask EH to skip the usual stuff
5493	* and proceed directly to suspend.
5494	*
5495	* http://thread.gmane.org/gmane.linux.ide/46764
5496	*/
5497	ata_port_request_pm(ap, mesg, 0,
5498	ATA_EHI_QUIET \| ATA_EHI_NO_AUTOPSY \|
5499	ATA_EHI_NO_RECOVERY,
5500	async);
5501	}
5502
5503	static int ata_port_pm_suspend(struct device *dev)
5504	{
5505	struct ata_port *ap = to_ata_port(dev);
5506
5507	if (pm_runtime_suspended(dev))
5508	return 0;
5509
5510	ata_port_suspend(ap, PMSG_SUSPEND, false);
5511	return 0;
5512	}
5513
5514	static int ata_port_pm_freeze(struct device *dev)
5515	{
5516	struct ata_port *ap = to_ata_port(dev);
5517
5518	if (pm_runtime_suspended(dev))
5519	return 0;
5520
5521	ata_port_suspend(ap, PMSG_FREEZE, false);
5522	return 0;
5523	}
5524
5525	static int ata_port_pm_poweroff(struct device *dev)
5526	{
5527	if (!pm_runtime_suspended(dev))
5528	ata_port_suspend(to_ata_port(dev), PMSG_HIBERNATE, false);
5529	return 0;
5530	}
5531
5532	static void ata_port_resume(struct ata_port *ap, pm_message_t mesg,
5533	bool async)
5534	{
5535	ata_port_request_pm(ap, mesg, ATA_EH_RESET,
5536	ATA_EHI_NO_AUTOPSY \| ATA_EHI_QUIET,
5537	async);
5538	}
5539
5540	static int ata_port_pm_resume(struct device *dev)
5541	{
5542	if (!pm_runtime_suspended(dev))
5543	ata_port_resume(to_ata_port(dev), PMSG_RESUME, true);
5544	return 0;
5545	}
5546
5547	/*
5548	* For ODDs, the upper layer will poll for media change every few seconds,
5549	* which will make it enter and leave suspend state every few seconds. And
5550	* as each suspend will cause a hard/soft reset, the gain of runtime suspend
5551	* is very little and the ODD may malfunction after constantly being reset.
5552	* So the idle callback here will not proceed to suspend if a non-ZPODD capable
5553	* ODD is attached to the port.
5554	*/
5555	static int ata_port_runtime_idle(struct device *dev)
5556	{
5557	struct ata_port *ap = to_ata_port(dev);
5558	struct ata_link *link;
5559	struct ata_device *adev;
5560
5561	ata_for_each_link(link, ap, HOST_FIRST) {
5562	ata_for_each_dev(adev, link, ENABLED)
5563	if (adev->class == ATA_DEV_ATAPI &&
5564	!zpodd_dev_enabled(adev))
5565	return -EBUSY;
5566	}
5567
5568	return 0;
5569	}
5570
5571	static int ata_port_runtime_suspend(struct device *dev)
5572	{
5573	ata_port_suspend(to_ata_port(dev), PMSG_AUTO_SUSPEND, false);
5574	return 0;
5575	}
5576
5577	static int ata_port_runtime_resume(struct device *dev)
5578	{
5579	ata_port_resume(to_ata_port(dev), PMSG_AUTO_RESUME, false);
5580	return 0;
5581	}
5582
5583	static const struct dev_pm_ops ata_port_pm_ops = {
5584	.suspend = ata_port_pm_suspend,
5585	.resume = ata_port_pm_resume,
5586	.freeze = ata_port_pm_freeze,
5587	.thaw = ata_port_pm_resume,
5588	.poweroff = ata_port_pm_poweroff,
5589	.restore = ata_port_pm_resume,
5590
5591	.runtime_suspend = ata_port_runtime_suspend,
5592	.runtime_resume = ata_port_runtime_resume,
5593	.runtime_idle = ata_port_runtime_idle,
5594	};
5595
5596	/* sas ports don't participate in pm runtime management of ata_ports,
5597	* and need to resume ata devices at the domain level, not the per-port
5598	* level. sas suspend/resume is async to allow parallel port recovery
5599	* since sas has multiple ata_port instances per Scsi_Host.
5600	*/
5601	void ata_sas_port_suspend(struct ata_port *ap)
5602	{
5603	ata_port_suspend(ap, PMSG_SUSPEND, true);
5604	}
5605	EXPORT_SYMBOL_GPL(ata_sas_port_suspend);
5606
5607	void ata_sas_port_resume(struct ata_port *ap)
5608	{
5609	ata_port_resume(ap, PMSG_RESUME, true);
5610	}
5611	EXPORT_SYMBOL_GPL(ata_sas_port_resume);
5612
5613	/**
5614	* ata_host_suspend - suspend host
5615	* @host: host to suspend
5616	* @mesg: PM message
5617	*
5618	* Suspend @host. Actual operation is performed by port suspend.
5619	*/
5620	void ata_host_suspend(struct ata_host *host, pm_message_t mesg)
5621	{
5622	host->dev->power.power_state = mesg;
5623	}
5624	EXPORT_SYMBOL_GPL(ata_host_suspend);
5625
5626	/**
5627	* ata_host_resume - resume host
5628	* @host: host to resume
5629	*
5630	* Resume @host. Actual operation is performed by port resume.
5631	*/
5632	void ata_host_resume(struct ata_host *host)
5633	{
5634	host->dev->power.power_state = PMSG_ON;
5635	}
5636	EXPORT_SYMBOL_GPL(ata_host_resume);
5637	#endif
5638
5639	const struct device_type ata_port_type = {
5640	.name = ATA_PORT_TYPE_NAME,
5641	#ifdef CONFIG_PM
5642	.pm = &ata_port_pm_ops,
5643	#endif
5644	};
5645
5646	/**
5647	* ata_dev_init - Initialize an ata_device structure
5648	* @dev: Device structure to initialize
5649	*
5650	* Initialize @dev in preparation for probing.
5651	*
5652	* LOCKING:
5653	* Inherited from caller.
5654	*/
5655	void ata_dev_init(struct ata_device *dev)
5656	{
5657	struct ata_link *link = ata_dev_phys_link(dev);
5658	struct ata_port *ap = link->ap;
5659	unsigned long flags;
5660
5661	/* SATA spd limit is bound to the attached device, reset together */
5662	link->sata_spd_limit = link->hw_sata_spd_limit;
5663	link->sata_spd = 0;
5664
5665	/* High bits of dev->flags are used to record warm plug
5666	* requests which occur asynchronously. Synchronize using
5667	* host lock.
5668	*/
5669	spin_lock_irqsave(ap->lock, flags);
5670	dev->flags &= ~ATA_DFLAG_INIT_MASK;
5671	dev->quirks = 0;
5672	spin_unlock_irqrestore(ap->lock, flags);
5673
5674	memset((void *)dev + ATA_DEVICE_CLEAR_BEGIN, 0,
5675	ATA_DEVICE_CLEAR_END - ATA_DEVICE_CLEAR_BEGIN);
5676	dev->pio_mask = UINT_MAX;
5677	dev->mwdma_mask = UINT_MAX;
5678	dev->udma_mask = UINT_MAX;
5679	}
5680
5681	/**
5682	* ata_link_init - Initialize an ata_link structure
5683	* @ap: ATA port link is attached to
5684	* @link: Link structure to initialize
5685	* @pmp: Port multiplier port number
5686	*
5687	* Initialize @link.
5688	*
5689	* LOCKING:
5690	* Kernel thread context (may sleep)
5691	*/
5692	void ata_link_init(struct ata_port ap, struct ata_link link, int pmp)
5693	{
5694	int i;
5695
5696	/* clear everything except for devices */
5697	memset((void *)link + ATA_LINK_CLEAR_BEGIN, 0,
5698	ATA_LINK_CLEAR_END - ATA_LINK_CLEAR_BEGIN);
5699
5700	link->ap = ap;
5701	link->pmp = pmp;
5702	link->active_tag = ATA_TAG_POISON;
5703	link->hw_sata_spd_limit = UINT_MAX;
5704	INIT_WORK(&link->deferred_qc_work, ata_scsi_deferred_qc_work);
5705
5706	/* can't use iterator, ap isn't initialized yet */
5707	for (i = 0; i < ATA_MAX_DEVICES; i++) {
5708	struct ata_device *dev = &link->device[i];
5709
5710	dev->link = link;
5711	dev->devno = dev - link->device;
5712	#ifdef CONFIG_ATA_ACPI
5713	dev->gtf_filter = ata_acpi_gtf_filter;
5714	#endif
5715	ata_dev_init(dev);
5716	}
5717	}
5718
5719	/**
5720	* sata_link_init_spd - Initialize link->sata_spd_limit
5721	* @link: Link to configure sata_spd_limit for
5722	*
5723	* Initialize ``link->[hw_]sata_spd_limit`` to the currently
5724	* configured value.
5725	*
5726	* LOCKING:
5727	* Kernel thread context (may sleep).
5728	*
5729	* RETURNS:
5730	* 0 on success, -errno on failure.
5731	*/
5732	int sata_link_init_spd(struct ata_link *link)
5733	{
5734	u8 spd;
5735	int rc;
5736
5737	rc = sata_scr_read(link, SCR_CONTROL, &link->saved_scontrol);
5738	if (rc)
5739	return rc;
5740
5741	spd = (link->saved_scontrol >> 4) & 0xf;
5742	if (spd)
5743	link->hw_sata_spd_limit &= (1 << spd) - 1;
5744
5745	ata_force_link_limits(link);
5746
5747	link->sata_spd_limit = link->hw_sata_spd_limit;
5748
5749	return 0;
5750	}
5751
5752	/**
5753	* ata_port_alloc - allocate and initialize basic ATA port resources
5754	* @host: ATA host this allocated port belongs to
5755	*
5756	* Allocate and initialize basic ATA port resources.
5757	*
5758	* RETURNS:
5759	* Allocate ATA port on success, NULL on failure.
5760	*
5761	* LOCKING:
5762	* Inherited from calling layer (may sleep).
5763	*/
5764	struct ata_port ata_port_alloc(struct ata_host host)
5765	{
5766	struct ata_port *ap;
5767	int id;
5768
5769	ap = kzalloc_obj(*ap);
5770	if (!ap)
5771	return NULL;
5772
5773	ap->pflags \|= ATA_PFLAG_INITIALIZING \| ATA_PFLAG_FROZEN;
5774	ap->lock = &host->lock;
5775	id = ida_alloc_min(&ata_ida, 1, GFP_KERNEL);
5776	if (id < 0) {
5777	kfree(ap);
5778	return NULL;
5779	}
5780	ap->print_id = id;
5781	ap->host = host;
5782	ap->dev = host->dev;
5783
5784	mutex_init(&ap->scsi_scan_mutex);
5785	INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug);
5786	INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan);
5787	INIT_LIST_HEAD(&ap->eh_done_q);
5788	init_waitqueue_head(&ap->eh_wait_q);
5789	init_completion(&ap->park_req_pending);
5790	timer_setup(&ap->fastdrain_timer, ata_eh_fastdrain_timerfn,
5791	TIMER_DEFERRABLE);
5792
5793	ap->cbl = ATA_CBL_NONE;
5794
5795	ata_link_init(ap, &ap->link, 0);
5796
5797	#ifdef ATA_IRQ_TRAP
5798	ap->stats.unhandled_irq = 1;
5799	ap->stats.idle_irq = 1;
5800	#endif
5801	ata_sff_port_init(ap);
5802
5803	ata_force_pflags(ap);
5804
5805	return ap;
5806	}
5807	EXPORT_SYMBOL_GPL(ata_port_alloc);
5808
5809	void ata_port_free(struct ata_port *ap)
5810	{
5811	if (!ap)
5812	return;
5813
5814	kfree(ap->pmp_link);
5815	kfree(ap->slave_link);
5816	ida_free(&ata_ida, ap->print_id);
5817	kfree(ap);
5818	}
5819	EXPORT_SYMBOL_GPL(ata_port_free);
5820
5821	static void ata_devres_release(struct device gendev, void res)
5822	{
5823	struct ata_host *host = dev_get_drvdata(gendev);
5824	int i;
5825
5826	for (i = 0; i < host->n_ports; i++) {
5827	struct ata_port *ap = host->ports[i];
5828
5829	if (!ap)
5830	continue;
5831
5832	if (ap->scsi_host)
5833	scsi_host_put(ap->scsi_host);
5834
5835	}
5836
5837	dev_set_drvdata(gendev, NULL);
5838	ata_host_put(host);
5839	}
5840
5841	static void ata_host_release(struct kref *kref)
5842	{
5843	struct ata_host *host = container_of(kref, struct ata_host, kref);
5844	int i;
5845
5846	for (i = 0; i < host->n_ports; i++) {
5847	ata_port_free(host->ports[i]);
5848	host->ports[i] = NULL;
5849	}
5850	kfree(host);
5851	}
5852
5853	void ata_host_get(struct ata_host *host)
5854	{
5855	kref_get(&host->kref);
5856	}
5857
5858	void ata_host_put(struct ata_host *host)
5859	{
5860	kref_put(&host->kref, ata_host_release);
5861	}
5862	EXPORT_SYMBOL_GPL(ata_host_put);
5863
5864	/**
5865	* ata_host_alloc - allocate and init basic ATA host resources
5866	* @dev: generic device this host is associated with
5867	* @n_ports: the number of ATA ports associated with this host
5868	*
5869	* Allocate and initialize basic ATA host resources. LLD calls
5870	* this function to allocate a host, initializes it fully and
5871	* attaches it using ata_host_register().
5872	*
5873	* RETURNS:
5874	* Allocate ATA host on success, NULL on failure.
5875	*
5876	* LOCKING:
5877	* Inherited from calling layer (may sleep).
5878	*/
5879	struct ata_host ata_host_alloc(struct device dev, int n_ports)
5880	{
5881	struct ata_host *host;
5882	size_t sz;
5883	int i;
5884	void *dr;
5885
5886	/* alloc a container for our list of ATA ports (buses) */
5887	sz = sizeof(struct ata_host) + n_ports * sizeof(void *);
5888	host = kzalloc(sz, GFP_KERNEL);
5889	if (!host)
5890	return NULL;
5891
5892	if (!devres_open_group(dev, NULL, GFP_KERNEL)) {
5893	kfree(host);
5894	return NULL;
5895	}
5896
5897	dr = devres_alloc(ata_devres_release, 0, GFP_KERNEL);
5898	if (!dr) {
5899	kfree(host);
5900	goto err_out;
5901	}
5902
5903	devres_add(dev, dr);
5904	dev_set_drvdata(dev, host);
5905
5906	spin_lock_init(&host->lock);
5907	mutex_init(&host->eh_mutex);
5908	host->dev = dev;
5909	host->n_ports = n_ports;
5910	kref_init(&host->kref);
5911
5912	/* allocate ports bound to this host */
5913	for (i = 0; i < n_ports; i++) {
5914	struct ata_port *ap;
5915
5916	ap = ata_port_alloc(host);
5917	if (!ap)
5918	goto err_out;
5919
5920	ap->port_no = i;
5921	host->ports[i] = ap;
5922	}
5923
5924	devres_remove_group(dev, NULL);
5925	return host;
5926
5927	err_out:
5928	devres_release_group(dev, NULL);
5929	return NULL;
5930	}
5931	EXPORT_SYMBOL_GPL(ata_host_alloc);
5932
5933	/**
5934	* ata_host_alloc_pinfo - alloc host and init with port_info array
5935	* @dev: generic device this host is associated with
5936	* @ppi: array of ATA port_info to initialize host with
5937	* @n_ports: number of ATA ports attached to this host
5938	*
5939	* Allocate ATA host and initialize with info from @ppi. If NULL
5940	* terminated, @ppi may contain fewer entries than @n_ports. The
5941	* last entry will be used for the remaining ports.
5942	*
5943	* RETURNS:
5944	* Allocate ATA host on success, NULL on failure.
5945	*
5946	* LOCKING:
5947	* Inherited from calling layer (may sleep).
5948	*/
5949	struct ata_host ata_host_alloc_pinfo(struct device dev,
5950	const struct ata_port_info * const * ppi,
5951	int n_ports)
5952	{
5953	const struct ata_port_info *pi = &ata_dummy_port_info;
5954	struct ata_host *host;
5955	int i, j;
5956
5957	host = ata_host_alloc(dev, n_ports);
5958	if (!host)
5959	return NULL;
5960
5961	for (i = 0, j = 0; i < host->n_ports; i++) {
5962	struct ata_port *ap = host->ports[i];
5963
5964	if (ppi[j])
5965	pi = ppi[j++];
5966
5967	ap->pio_mask = pi->pio_mask;
5968	ap->mwdma_mask = pi->mwdma_mask;
5969	ap->udma_mask = pi->udma_mask;
5970	ap->flags \|= pi->flags;
5971	ap->link.flags \|= pi->link_flags;
5972	ap->ops = pi->port_ops;
5973
5974	if (!host->ops && (pi->port_ops != &ata_dummy_port_ops))
5975	host->ops = pi->port_ops;
5976	}
5977
5978	return host;
5979	}
5980	EXPORT_SYMBOL_GPL(ata_host_alloc_pinfo);
5981
5982	static void ata_host_stop(struct device gendev, void res)
5983	{
5984	struct ata_host *host = dev_get_drvdata(gendev);
5985	int i;
5986
5987	WARN_ON(!(host->flags & ATA_HOST_STARTED));
5988
5989	for (i = 0; i < host->n_ports; i++) {
5990	struct ata_port *ap = host->ports[i];
5991
5992	if (ap->ops->port_stop)
5993	ap->ops->port_stop(ap);
5994	}
5995
5996	if (host->ops->host_stop)
5997	host->ops->host_stop(host);
5998	}
5999
6000	/**
6001	* ata_finalize_port_ops - finalize ata_port_operations
6002	* @ops: ata_port_operations to finalize
6003	*
6004	* An ata_port_operations can inherit from another ops and that
6005	* ops can again inherit from another. This can go on as many
6006	* times as necessary as long as there is no loop in the
6007	* inheritance chain.
6008	*
6009	* Ops tables are finalized when the host is started. NULL or
6010	* unspecified entries are inherited from the closet ancestor
6011	* which has the method and the entry is populated with it.
6012	* After finalization, the ops table directly points to all the
6013	* methods and ->inherits is no longer necessary and cleared.
6014	*
6015	* Using ATA_OP_NULL, inheriting ops can force a method to NULL.
6016	*
6017	* LOCKING:
6018	* None.
6019	*/
6020	static void ata_finalize_port_ops(struct ata_port_operations *ops)
6021	{
6022	static DEFINE_SPINLOCK(lock);
6023	const struct ata_port_operations *cur;
6024	void begin = (void )ops;
6025	void end = (void )&ops->inherits;
6026	void **pp;
6027
6028	if (!ops \|\| !ops->inherits)
6029	return;
6030
6031	spin_lock(&lock);
6032
6033	for (cur = ops->inherits; cur; cur = cur->inherits) {
6034	void inherit = (void )cur;
6035
6036	for (pp = begin; pp < end; pp++, inherit++)
6037	if (!*pp)
6038	pp = inherit;
6039	}
6040
6041	for (pp = begin; pp < end; pp++)
6042	if (IS_ERR(*pp))
6043	*pp = NULL;
6044
6045	ops->inherits = NULL;
6046
6047	spin_unlock(&lock);
6048	}
6049
6050	/**
6051	* ata_host_start - start and freeze ports of an ATA host
6052	* @host: ATA host to start ports for
6053	*
6054	* Start and then freeze ports of @host. Started status is
6055	* recorded in host->flags, so this function can be called
6056	* multiple times. Ports are guaranteed to get started only
6057	* once. If host->ops is not initialized yet, it is set to the
6058	* first non-dummy port ops.
6059	*
6060	* LOCKING:
6061	* Inherited from calling layer (may sleep).
6062	*
6063	* RETURNS:
6064	* 0 if all ports are started successfully, -errno otherwise.
6065	*/
6066	int ata_host_start(struct ata_host *host)
6067	{
6068	int have_stop = 0;
6069	void *start_dr = NULL;
6070	int i, rc;
6071
6072	if (host->flags & ATA_HOST_STARTED)
6073	return 0;
6074
6075	ata_finalize_port_ops(host->ops);
6076
6077	for (i = 0; i < host->n_ports; i++) {
6078	struct ata_port *ap = host->ports[i];
6079
6080	ata_finalize_port_ops(ap->ops);
6081
6082	if (!host->ops && !ata_port_is_dummy(ap))
6083	host->ops = ap->ops;
6084
6085	if (ap->ops->port_stop)
6086	have_stop = 1;
6087	}
6088
6089	if (host->ops && host->ops->host_stop)
6090	have_stop = 1;
6091
6092	if (have_stop) {
6093	start_dr = devres_alloc(ata_host_stop, 0, GFP_KERNEL);
6094	if (!start_dr)
6095	return -ENOMEM;
6096	}
6097
6098	for (i = 0; i < host->n_ports; i++) {
6099	struct ata_port *ap = host->ports[i];
6100
6101	if (ap->ops->port_start) {
6102	rc = ap->ops->port_start(ap);
6103	if (rc) {
6104	if (rc != -ENODEV)
6105	dev_err(host->dev,
6106	"failed to start port %d (errno=%d)\n",
6107	i, rc);
6108	goto err_out;
6109	}
6110	}
6111	ata_eh_freeze_port(ap);
6112	}
6113
6114	if (start_dr)
6115	devres_add(host->dev, start_dr);
6116	host->flags \|= ATA_HOST_STARTED;
6117	return 0;
6118
6119	err_out:
6120	while (--i >= 0) {
6121	struct ata_port *ap = host->ports[i];
6122
6123	if (ap->ops->port_stop)
6124	ap->ops->port_stop(ap);
6125	}
6126	devres_free(start_dr);
6127	return rc;
6128	}
6129	EXPORT_SYMBOL_GPL(ata_host_start);
6130
6131	/**
6132	* ata_host_init - Initialize a host struct for sas (ipr, libsas)
6133	* @host: host to initialize
6134	* @dev: device host is attached to
6135	* @ops: port_ops
6136	*
6137	*/
6138	void ata_host_init(struct ata_host host, struct device dev,
6139	struct ata_port_operations *ops)
6140	{
6141	spin_lock_init(&host->lock);
6142	mutex_init(&host->eh_mutex);
6143	host->n_tags = ATA_MAX_QUEUE;
6144	host->dev = dev;
6145	host->ops = ops;
6146	kref_init(&host->kref);
6147	}
6148	EXPORT_SYMBOL_GPL(ata_host_init);
6149
6150	void ata_port_probe(struct ata_port *ap)
6151	{
6152	struct ata_eh_info *ehi = &ap->link.eh_info;
6153	unsigned long flags;
6154
6155	ata_acpi_port_power_on(ap);
6156
6157	/* kick EH for boot probing */
6158	spin_lock_irqsave(ap->lock, flags);
6159
6160	ehi->probe_mask \|= ATA_ALL_DEVICES;
6161	ehi->action \|= ATA_EH_RESET;
6162	ehi->flags \|= ATA_EHI_NO_AUTOPSY \| ATA_EHI_QUIET;
6163
6164	ap->pflags &= ~ATA_PFLAG_INITIALIZING;
6165	ap->pflags \|= ATA_PFLAG_LOADING;
6166	ata_port_schedule_eh(ap);
6167
6168	spin_unlock_irqrestore(ap->lock, flags);
6169	}
6170	EXPORT_SYMBOL_GPL(ata_port_probe);
6171
6172	static void async_port_probe(void *data, async_cookie_t cookie)
6173	{
6174	struct ata_port *ap = data;
6175
6176	/*
6177	* If we're not allowed to scan this host in parallel,
6178	* we need to wait until all previous scans have completed
6179	* before going further.
6180	* Jeff Garzik says this is only within a controller, so we
6181	* don't need to wait for port 0, only for later ports.
6182	*/
6183	if (!(ap->host->flags & ATA_HOST_PARALLEL_SCAN) && ap->port_no != 0)
6184	async_synchronize_cookie(cookie);
6185
6186	ata_port_probe(ap);
6187	ata_port_wait_eh(ap);
6188
6189	/* in order to keep device order, we need to synchronize at this point */
6190	async_synchronize_cookie(cookie);
6191
6192	ata_scsi_scan_host(ap, 1);
6193	}
6194
6195	/**
6196	* ata_host_register - register initialized ATA host
6197	* @host: ATA host to register
6198	* @sht: template for SCSI host
6199	*
6200	* Register initialized ATA host. @host is allocated using
6201	* ata_host_alloc() and fully initialized by LLD. This function
6202	* starts ports, registers @host with ATA and SCSI layers and
6203	* probe registered devices.
6204	*
6205	* LOCKING:
6206	* Inherited from calling layer (may sleep).
6207	*
6208	* RETURNS:
6209	* 0 on success, -errno otherwise.
6210	*/
6211	int ata_host_register(struct ata_host host, const struct scsi_host_template sht)
6212	{
6213	int i, rc;
6214
6215	host->n_tags = clamp(sht->can_queue, 1, ATA_MAX_QUEUE);
6216
6217	/* host must have been started */
6218	if (!(host->flags & ATA_HOST_STARTED)) {
6219	dev_err(host->dev, "BUG: trying to register unstarted host\n");
6220	WARN_ON(1);
6221	return -EINVAL;
6222	}
6223
6224	/* Create associated sysfs transport objects */
6225	for (i = 0; i < host->n_ports; i++) {
6226	rc = ata_tport_add(host->dev,host->ports[i]);
6227	if (rc) {
6228	goto err_tadd;
6229	}
6230	}
6231
6232	rc = ata_scsi_add_hosts(host, sht);
6233	if (rc)
6234	goto err_tadd;
6235
6236	/* set cable, sata_spd_limit and report */
6237	for (i = 0; i < host->n_ports; i++) {
6238	struct ata_port *ap = host->ports[i];
6239	unsigned int xfer_mask;
6240
6241	/* set SATA cable type if still unset */
6242	if (ap->cbl == ATA_CBL_NONE && (ap->flags & ATA_FLAG_SATA))
6243	ap->cbl = ATA_CBL_SATA;
6244
6245	/* init sata_spd_limit to the current value */
6246	sata_link_init_spd(&ap->link);
6247	if (ap->slave_link)
6248	sata_link_init_spd(ap->slave_link);
6249
6250	/* print per-port info to dmesg */
6251	xfer_mask = ata_pack_xfermask(ap->pio_mask, ap->mwdma_mask,
6252	ap->udma_mask);
6253
6254	if (!ata_port_is_dummy(ap)) {
6255	ata_port_info(ap, "%cATA max %s %s\n",
6256	(ap->flags & ATA_FLAG_SATA) ? 'S' : 'P',
6257	ata_mode_string(xfer_mask),
6258	ap->link.eh_info.desc);
6259	ata_ehi_clear_desc(&ap->link.eh_info);
6260	} else
6261	ata_port_info(ap, "DUMMY\n");
6262	}
6263
6264	/* perform each probe asynchronously */
6265	for (i = 0; i < host->n_ports; i++) {
6266	struct ata_port *ap = host->ports[i];
6267	ap->cookie = async_schedule(async_port_probe, ap);
6268	}
6269
6270	return 0;
6271
6272	err_tadd:
6273	while (--i >= 0) {
6274	ata_tport_delete(host->ports[i]);
6275	}
6276	return rc;
6277
6278	}
6279	EXPORT_SYMBOL_GPL(ata_host_register);
6280
6281	/**
6282	* ata_host_activate - start host, request IRQ and register it
6283	* @host: target ATA host
6284	* @irq: IRQ to request
6285	* @irq_handler: irq_handler used when requesting IRQ
6286	* @irq_flags: irq_flags used when requesting IRQ
6287	* @sht: scsi_host_template to use when registering the host
6288	*
6289	* After allocating an ATA host and initializing it, most libata
6290	* LLDs perform three steps to activate the host - start host,
6291	* request IRQ and register it. This helper takes necessary
6292	* arguments and performs the three steps in one go.
6293	*
6294	* An invalid IRQ skips the IRQ registration and expects the host to
6295	* have set polling mode on the port. In this case, @irq_handler
6296	* should be NULL.
6297	*
6298	* LOCKING:
6299	* Inherited from calling layer (may sleep).
6300	*
6301	* RETURNS:
6302	* 0 on success, -errno otherwise.
6303	*/
6304	int ata_host_activate(struct ata_host *host, int irq,
6305	irq_handler_t irq_handler, unsigned long irq_flags,
6306	const struct scsi_host_template *sht)
6307	{
6308	int i, rc;
6309	char *irq_desc;
6310
6311	rc = ata_host_start(host);
6312	if (rc)
6313	return rc;
6314
6315	/* Special case for polling mode */
6316	if (!irq) {
6317	WARN_ON(irq_handler);
6318	return ata_host_register(host, sht);
6319	}
6320
6321	irq_desc = devm_kasprintf(host->dev, GFP_KERNEL, "%s[%s]",
6322	dev_driver_string(host->dev),
6323	dev_name(host->dev));
6324	if (!irq_desc)
6325	return -ENOMEM;
6326
6327	rc = devm_request_irq(host->dev, irq, irq_handler, irq_flags,
6328	irq_desc, host);
6329	if (rc)
6330	return rc;
6331
6332	for (i = 0; i < host->n_ports; i++)
6333	ata_port_desc_misc(host->ports[i], irq);
6334
6335	rc = ata_host_register(host, sht);
6336	/* if failed, just free the IRQ and leave ports alone */
6337	if (rc)
6338	devm_free_irq(host->dev, irq, host);
6339
6340	return rc;
6341	}
6342	EXPORT_SYMBOL_GPL(ata_host_activate);
6343
6344	/**
6345	* ata_dev_free_resources - Free a device resources
6346	* @dev: Target ATA device
6347	*
6348	* Free resources allocated to support a device features.
6349	*
6350	* LOCKING:
6351	* Kernel thread context (may sleep).
6352	*/
6353	void ata_dev_free_resources(struct ata_device *dev)
6354	{
6355	if (zpodd_dev_enabled(dev))
6356	zpodd_exit(dev);
6357
6358	ata_dev_cleanup_cdl_resources(dev);
6359	}
6360
6361	/**
6362	* ata_port_detach - Detach ATA port in preparation of device removal
6363	* @ap: ATA port to be detached
6364	*
6365	* Detach all ATA devices and the associated SCSI devices of @ap;
6366	* then, remove the associated SCSI host. @ap is guaranteed to
6367	* be quiescent on return from this function.
6368	*
6369	* LOCKING:
6370	* Kernel thread context (may sleep).
6371	*/
6372	static void ata_port_detach(struct ata_port *ap)
6373	{
6374	unsigned long flags;
6375	struct ata_link *link;
6376	struct ata_device *dev;
6377
6378	/* Ensure ata_port probe has completed */
6379	async_synchronize_cookie(ap->cookie + 1);
6380
6381	/* Wait for any ongoing EH */
6382	ata_port_wait_eh(ap);
6383
6384	mutex_lock(&ap->scsi_scan_mutex);
6385	spin_lock_irqsave(ap->lock, flags);
6386
6387	/* Remove scsi devices */
6388	ata_for_each_link(link, ap, HOST_FIRST) {
6389	ata_for_each_dev(dev, link, ALL) {
6390	if (dev->sdev) {
6391	spin_unlock_irqrestore(ap->lock, flags);
6392	scsi_remove_device(dev->sdev);
6393	spin_lock_irqsave(ap->lock, flags);
6394	dev->sdev = NULL;
6395	}
6396	}
6397	}
6398
6399	/* Tell EH to disable all devices */
6400	ap->pflags \|= ATA_PFLAG_UNLOADING;
6401	ata_port_schedule_eh(ap);
6402
6403	spin_unlock_irqrestore(ap->lock, flags);
6404	mutex_unlock(&ap->scsi_scan_mutex);
6405
6406	/* wait till EH commits suicide */
6407	ata_port_wait_eh(ap);
6408
6409	/* It better be dead now and not have any remaining deferred qc. */
6410	WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED));
6411
6412	cancel_delayed_work_sync(&ap->hotplug_task);
6413	cancel_delayed_work_sync(&ap->scsi_rescan_task);
6414
6415	ata_for_each_link(link, ap, PMP_FIRST) {
6416	WARN_ON(link->deferred_qc);
6417	cancel_work_sync(&link->deferred_qc_work);
6418	}
6419
6420	/* Delete port multiplier link transport devices */
6421	if (ap->pmp_link) {
6422	int i;
6423
6424	for (i = 0; i < SATA_PMP_MAX_PORTS; i++)
6425	ata_tlink_delete(&ap->pmp_link[i]);
6426	}
6427
6428	/* Remove the associated SCSI host */
6429	scsi_remove_host(ap->scsi_host);
6430	ata_tport_delete(ap);
6431	}
6432
6433	/**
6434	* ata_host_detach - Detach all ports of an ATA host
6435	* @host: Host to detach
6436	*
6437	* Detach all ports of @host.
6438	*
6439	* LOCKING:
6440	* Kernel thread context (may sleep).
6441	*/
6442	void ata_host_detach(struct ata_host *host)
6443	{
6444	int i;
6445
6446	for (i = 0; i < host->n_ports; i++)
6447	ata_port_detach(host->ports[i]);
6448
6449	/* the host is dead now, dissociate ACPI */
6450	ata_acpi_dissociate(host);
6451	}
6452	EXPORT_SYMBOL_GPL(ata_host_detach);
6453
6454	#ifdef CONFIG_PCI
6455
6456	/**
6457	* ata_pci_remove_one - PCI layer callback for device removal
6458	* @pdev: PCI device that was removed
6459	*
6460	* PCI layer indicates to libata via this hook that hot-unplug or
6461	* module unload event has occurred. Detach all ports. Resource
6462	* release is handled via devres.
6463	*
6464	* LOCKING:
6465	* Inherited from PCI layer (may sleep).
6466	*/
6467	void ata_pci_remove_one(struct pci_dev *pdev)
6468	{
6469	struct ata_host *host = pci_get_drvdata(pdev);
6470
6471	ata_host_detach(host);
6472	}
6473	EXPORT_SYMBOL_GPL(ata_pci_remove_one);
6474
6475	void ata_pci_shutdown_one(struct pci_dev *pdev)
6476	{
6477	struct ata_host *host = pci_get_drvdata(pdev);
6478	int i;
6479
6480	for (i = 0; i < host->n_ports; i++) {
6481	struct ata_port *ap = host->ports[i];
6482
6483	ap->pflags \|= ATA_PFLAG_FROZEN;
6484
6485	/* Disable port interrupts */
6486	if (ap->ops->freeze)
6487	ap->ops->freeze(ap);
6488
6489	/* Stop the port DMA engines */
6490	if (ap->ops->port_stop)
6491	ap->ops->port_stop(ap);
6492	}
6493	}
6494	EXPORT_SYMBOL_GPL(ata_pci_shutdown_one);
6495
6496	/* move to PCI subsystem */
6497	int pci_test_config_bits(struct pci_dev pdev, const struct pci_bits bits)
6498	{
6499	unsigned long tmp = 0;
6500
6501	switch (bits->width) {
6502	case 1: {
6503	u8 tmp8 = 0;
6504	pci_read_config_byte(pdev, bits->reg, &tmp8);
6505	tmp = tmp8;
6506	break;
6507	}
6508	case 2: {
6509	u16 tmp16 = 0;
6510	pci_read_config_word(pdev, bits->reg, &tmp16);
6511	tmp = tmp16;
6512	break;
6513	}
6514	case 4: {
6515	u32 tmp32 = 0;
6516	pci_read_config_dword(pdev, bits->reg, &tmp32);
6517	tmp = tmp32;
6518	break;
6519	}
6520
6521	default:
6522	return -EINVAL;
6523	}
6524
6525	tmp &= bits->mask;
6526
6527	return (tmp == bits->val) ? 1 : 0;
6528	}
6529	EXPORT_SYMBOL_GPL(pci_test_config_bits);
6530
6531	#ifdef CONFIG_PM
6532	void ata_pci_device_do_suspend(struct pci_dev *pdev, pm_message_t mesg)
6533	{
6534	pci_save_state(pdev);
6535	pci_disable_device(pdev);
6536
6537	if (mesg.event & PM_EVENT_SLEEP)
6538	pci_set_power_state(pdev, PCI_D3hot);
6539	}
6540	EXPORT_SYMBOL_GPL(ata_pci_device_do_suspend);
6541
6542	int ata_pci_device_do_resume(struct pci_dev *pdev)
6543	{
6544	int rc;
6545
6546	pci_set_power_state(pdev, PCI_D0);
6547	pci_restore_state(pdev);
6548
6549	rc = pcim_enable_device(pdev);
6550	if (rc) {
6551	dev_err(&pdev->dev,
6552	"failed to enable device after resume (%d)\n", rc);
6553	return rc;
6554	}
6555
6556	pci_set_master(pdev);
6557	return 0;
6558	}
6559	EXPORT_SYMBOL_GPL(ata_pci_device_do_resume);
6560
6561	int ata_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
6562	{
6563	struct ata_host *host = pci_get_drvdata(pdev);
6564
6565	ata_host_suspend(host, mesg);
6566
6567	ata_pci_device_do_suspend(pdev, mesg);
6568
6569	return 0;
6570	}
6571	EXPORT_SYMBOL_GPL(ata_pci_device_suspend);
6572
6573	int ata_pci_device_resume(struct pci_dev *pdev)
6574	{
6575	struct ata_host *host = pci_get_drvdata(pdev);
6576	int rc;
6577
6578	rc = ata_pci_device_do_resume(pdev);
6579	if (rc == 0)
6580	ata_host_resume(host);
6581	return rc;
6582	}
6583	EXPORT_SYMBOL_GPL(ata_pci_device_resume);
6584	#endif /* CONFIG_PM */
6585	#endif /* CONFIG_PCI */
6586
6587	/**
6588	* ata_platform_remove_one - Platform layer callback for device removal
6589	* @pdev: Platform device that was removed
6590	*
6591	* Platform layer indicates to libata via this hook that hot-unplug or
6592	* module unload event has occurred. Detach all ports. Resource
6593	* release is handled via devres.
6594	*
6595	* LOCKING:
6596	* Inherited from platform layer (may sleep).
6597	*/
6598	void ata_platform_remove_one(struct platform_device *pdev)
6599	{
6600	struct ata_host *host = platform_get_drvdata(pdev);
6601
6602	ata_host_detach(host);
6603	}
6604	EXPORT_SYMBOL_GPL(ata_platform_remove_one);
6605
6606	#ifdef CONFIG_ATA_FORCE
6607
6608	#define force_cbl(name, flag) \
6609	{ #name, .cbl = (flag) }
6610
6611	#define force_spd_limit(spd, val) \
6612	{ #spd, .spd_limit = (val) }
6613
6614	#define force_xfer(mode, shift) \
6615	{ #mode, .xfer_mask = (1UL << (shift)) }
6616
6617	#define force_lflag_on(name, flags) \
6618	{ #name, .lflags_on = (flags) }
6619
6620	#define force_lflag_onoff(name, flags) \
6621	{ "no" #name, .lflags_on = (flags) }, \
6622	{ #name, .lflags_off = (flags) }
6623
6624	#define force_pflag_on(name, flags) \
6625	{ #name, .pflags_on = (flags) }
6626
6627	#define force_quirk_on(name, flag) \
6628	{ #name, .quirk_on = (flag) }
6629
6630	#define force_quirk_val(name, flag, val) \
6631	{ #name, .quirk_on = (flag), \
6632	.value = (val) }
6633
6634	#define force_quirk_onoff(name, flag) \
6635	{ "no" #name, .quirk_on = (flag) }, \
6636	{ #name, .quirk_off = (flag) }
6637
6638	/*
6639	* If the ata_force_param struct member 'name' ends with '=', then the value
6640	* after the equal sign will be parsed as an u64, and will be saved in the
6641	* ata_force_param struct member 'value'. This works because each libata.force
6642	* entry (struct ata_force_ent) is separated by commas, so each entry represents
6643	* a single quirk, and can thus only have a single value.
6644	*/
6645	static const struct ata_force_param force_tbl[] __initconst = {
6646	force_cbl(40c, ATA_CBL_PATA40),
6647	force_cbl(80c, ATA_CBL_PATA80),
6648	force_cbl(short40c, ATA_CBL_PATA40_SHORT),
6649	force_cbl(unk, ATA_CBL_PATA_UNK),
6650	force_cbl(ign, ATA_CBL_PATA_IGN),
6651	force_cbl(sata, ATA_CBL_SATA),
6652
6653	force_spd_limit(1.5Gbps, 1),
6654	force_spd_limit(3.0Gbps, 2),
6655
6656	force_xfer(pio0, ATA_SHIFT_PIO + 0),
6657	force_xfer(pio1, ATA_SHIFT_PIO + 1),
6658	force_xfer(pio2, ATA_SHIFT_PIO + 2),
6659	force_xfer(pio3, ATA_SHIFT_PIO + 3),
6660	force_xfer(pio4, ATA_SHIFT_PIO + 4),
6661	force_xfer(pio5, ATA_SHIFT_PIO + 5),
6662	force_xfer(pio6, ATA_SHIFT_PIO + 6),
6663	force_xfer(mwdma0, ATA_SHIFT_MWDMA + 0),
6664	force_xfer(mwdma1, ATA_SHIFT_MWDMA + 1),
6665	force_xfer(mwdma2, ATA_SHIFT_MWDMA + 2),
6666	force_xfer(mwdma3, ATA_SHIFT_MWDMA + 3),
6667	force_xfer(mwdma4, ATA_SHIFT_MWDMA + 4),
6668	force_xfer(udma0, ATA_SHIFT_UDMA + 0),
6669	force_xfer(udma16, ATA_SHIFT_UDMA + 0),
6670	force_xfer(udma/16, ATA_SHIFT_UDMA + 0),
6671	force_xfer(udma1, ATA_SHIFT_UDMA + 1),
6672	force_xfer(udma25, ATA_SHIFT_UDMA + 1),
6673	force_xfer(udma/25, ATA_SHIFT_UDMA + 1),
6674	force_xfer(udma2, ATA_SHIFT_UDMA + 2),
6675	force_xfer(udma33, ATA_SHIFT_UDMA + 2),
6676	force_xfer(udma/33, ATA_SHIFT_UDMA + 2),
6677	force_xfer(udma3, ATA_SHIFT_UDMA + 3),
6678	force_xfer(udma44, ATA_SHIFT_UDMA + 3),
6679	force_xfer(udma/44, ATA_SHIFT_UDMA + 3),
6680	force_xfer(udma4, ATA_SHIFT_UDMA + 4),
6681	force_xfer(udma66, ATA_SHIFT_UDMA + 4),
6682	force_xfer(udma/66, ATA_SHIFT_UDMA + 4),
6683	force_xfer(udma5, ATA_SHIFT_UDMA + 5),
6684	force_xfer(udma100, ATA_SHIFT_UDMA + 5),
6685	force_xfer(udma/100, ATA_SHIFT_UDMA + 5),
6686	force_xfer(udma6, ATA_SHIFT_UDMA + 6),
6687	force_xfer(udma133, ATA_SHIFT_UDMA + 6),
6688	force_xfer(udma/133, ATA_SHIFT_UDMA + 6),
6689	force_xfer(udma7, ATA_SHIFT_UDMA + 7),
6690
6691	force_lflag_on(nohrst, ATA_LFLAG_NO_HRST),
6692	force_lflag_on(nosrst, ATA_LFLAG_NO_SRST),
6693	force_lflag_on(norst, ATA_LFLAG_NO_HRST \| ATA_LFLAG_NO_SRST),
6694	force_lflag_on(rstonce, ATA_LFLAG_RST_ONCE),
6695	force_lflag_onoff(dbdelay, ATA_LFLAG_NO_DEBOUNCE_DELAY),
6696
6697	force_pflag_on(external, ATA_PFLAG_EXTERNAL),
6698
6699	force_quirk_onoff(ncq, ATA_QUIRK_NONCQ),
6700	force_quirk_onoff(ncqtrim, ATA_QUIRK_NO_NCQ_TRIM),
6701	force_quirk_onoff(ncqati, ATA_QUIRK_NO_NCQ_ON_ATI),
6702
6703	force_quirk_onoff(trim, ATA_QUIRK_NOTRIM),
6704	force_quirk_on(trim_zero, ATA_QUIRK_ZERO_AFTER_TRIM),
6705	force_quirk_on(max_trim_128m, ATA_QUIRK_MAX_TRIM_128M),
6706
6707	force_quirk_onoff(dma, ATA_QUIRK_NODMA),
6708	force_quirk_on(atapi_dmadir, ATA_QUIRK_ATAPI_DMADIR),
6709	force_quirk_on(atapi_mod16_dma, ATA_QUIRK_ATAPI_MOD16_DMA),
6710
6711	force_quirk_onoff(dmalog, ATA_QUIRK_NO_DMA_LOG),
6712	force_quirk_onoff(iddevlog, ATA_QUIRK_NO_ID_DEV_LOG),
6713	force_quirk_onoff(logdir, ATA_QUIRK_NO_LOG_DIR),
6714
6715	force_quirk_val(max_sec_128, ATA_QUIRK_MAX_SEC, 128),
6716	force_quirk_val(max_sec_1024, ATA_QUIRK_MAX_SEC, 1024),
6717	force_quirk_on(max_sec=, ATA_QUIRK_MAX_SEC),
6718	force_quirk_on(max_sec_lba48, ATA_QUIRK_MAX_SEC_LBA48),
6719
6720	force_quirk_onoff(lpm, ATA_QUIRK_NOLPM),
6721	force_quirk_onoff(setxfer, ATA_QUIRK_NOSETXFER),
6722	force_quirk_on(dump_id, ATA_QUIRK_DUMP_ID),
6723	force_quirk_onoff(fua, ATA_QUIRK_NO_FUA),
6724
6725	force_quirk_on(disable, ATA_QUIRK_DISABLE),
6726	};
6727
6728	static int __init ata_parse_force_one(char **cur,
6729	struct ata_force_ent *force_ent,
6730	const char **reason)
6731	{
6732	char start = cur, p = cur;
6733	char id, val, endp, equalsign, *char_after_equalsign;
6734	const struct ata_force_param *match_fp = NULL;
6735	u64 val_after_equalsign;
6736	int nr_matches = 0, i;
6737
6738	/* find where this param ends and update cur /
6739	while (p != '\0' && p != ',')
6740	p++;
6741
6742	if (*p == '\0')
6743	*cur = p;
6744	else
6745	*cur = p + 1;
6746
6747	*p = '\0';
6748
6749	/* parse */
6750	p = strchr(start, ':');
6751	if (!p) {
6752	val = strstrip(start);
6753	goto parse_val;
6754	}
6755	*p = '\0';
6756
6757	id = strstrip(start);
6758	val = strstrip(p + 1);
6759
6760	/* parse id */
6761	p = strchr(id, '.');
6762	if (p) {
6763	*p++ = '\0';
6764	force_ent->device = simple_strtoul(p, &endp, 10);
6765	if (p == endp \|\| *endp != '\0') {
6766	*reason = "invalid device";
6767	return -EINVAL;
6768	}
6769	}
6770
6771	force_ent->port = simple_strtoul(id, &endp, 10);
6772	if (id == endp \|\| *endp != '\0') {
6773	*reason = "invalid port/link";
6774	return -EINVAL;
6775	}
6776
6777	parse_val:
6778	equalsign = strchr(val, '=');
6779	if (equalsign) {
6780	char_after_equalsign = equalsign + 1;
6781	if (!strlen(char_after_equalsign) \|\|
6782	kstrtoull(char_after_equalsign, 10, &val_after_equalsign)) {
6783	*reason = "invalid value after equal sign";
6784	return -EINVAL;
6785	}
6786	}
6787
6788	/* Parse the parameter value. */
6789	for (i = 0; i < ARRAY_SIZE(force_tbl); i++) {
6790	const struct ata_force_param *fp = &force_tbl[i];
6791
6792	/*
6793	* If val contains equal sign, match has to be exact, i.e.
6794	* shortcuts are not supported.
6795	*/
6796	if (equalsign &&
6797	(strncasecmp(val, fp->name,
6798	char_after_equalsign - val) == 0)) {
6799	force_ent->param = *fp;
6800	force_ent->param.value = val_after_equalsign;
6801	return 0;
6802	}
6803
6804	/*
6805	* If val does not contain equal sign, allow shortcuts so that
6806	* both 1.5 and 1.5Gbps work.
6807	*/
6808	if (strncasecmp(val, fp->name, strlen(val)))
6809	continue;
6810
6811	nr_matches++;
6812	match_fp = fp;
6813
6814	if (strcasecmp(val, fp->name) == 0) {
6815	nr_matches = 1;
6816	break;
6817	}
6818	}
6819
6820	if (!nr_matches) {
6821	*reason = "unknown value";
6822	return -EINVAL;
6823	}
6824	if (nr_matches > 1) {
6825	*reason = "ambiguous value";
6826	return -EINVAL;
6827	}
6828
6829	force_ent->param = *match_fp;
6830
6831	return 0;
6832	}
6833
6834	static void __init ata_parse_force_param(void)
6835	{
6836	int idx = 0, size = 1;
6837	int last_port = -1, last_device = -1;
6838	char p, cur, *next;
6839
6840	/* Calculate maximum number of params and allocate ata_force_tbl */
6841	for (p = ata_force_param_buf; *p; p++)
6842	if (*p == ',')
6843	size++;
6844
6845	ata_force_tbl = kzalloc_objs(ata_force_tbl[0], size);
6846	if (!ata_force_tbl) {
6847	printk(KERN_WARNING "ata: failed to extend force table, "
6848	"libata.force ignored\n");
6849	return;
6850	}
6851
6852	/* parse and populate the table */
6853	for (cur = ata_force_param_buf; *cur != '\0'; cur = next) {
6854	const char *reason = "";
6855	struct ata_force_ent te = { .port = -1, .device = -1 };
6856
6857	next = cur;
6858	if (ata_parse_force_one(&next, &te, &reason)) {
6859	printk(KERN_WARNING "ata: failed to parse force "
6860	"parameter \"%s\" (%s)\n",
6861	cur, reason);
6862	continue;
6863	}
6864
6865	if (te.port == -1) {
6866	te.port = last_port;
6867	te.device = last_device;
6868	}
6869
6870	ata_force_tbl[idx++] = te;
6871
6872	last_port = te.port;
6873	last_device = te.device;
6874	}
6875
6876	ata_force_tbl_size = idx;
6877	}
6878
6879	static void ata_free_force_param(void)
6880	{
6881	kfree(ata_force_tbl);
6882	}
6883	#else
6884	static inline void ata_parse_force_param(void) { }
6885	static inline void ata_free_force_param(void) { }
6886	#endif
6887
6888	static int __init ata_init(void)
6889	{
6890	int rc;
6891
6892	ata_parse_force_param();
6893
6894	rc = ata_sff_init();
6895	if (rc) {
6896	ata_free_force_param();
6897	return rc;
6898	}
6899
6900	libata_transport_init();
6901
6902	printk(KERN_DEBUG "libata version " DRV_VERSION " loaded.\n");
6903
6904	return 0;
6905	}
6906
6907	static void __exit ata_exit(void)
6908	{
6909	libata_transport_exit();
6910	ata_sff_exit();
6911	ata_free_force_param();
6912	}
6913
6914	subsys_initcall(ata_init);
6915	module_exit(ata_exit);
6916
6917	static DEFINE_RATELIMIT_STATE(ratelimit, HZ / 5, 1);
6918
6919	int ata_ratelimit(void)
6920	{
6921	return __ratelimit(&ratelimit);
6922	}
6923	EXPORT_SYMBOL_GPL(ata_ratelimit);
6924
6925	/**
6926	* ata_msleep - ATA EH owner aware msleep
6927	* @ap: ATA port to attribute the sleep to
6928	* @msecs: duration to sleep in milliseconds
6929	*
6930	* Sleeps @msecs. If the current task is owner of @ap's EH, the
6931	* ownership is released before going to sleep and reacquired
6932	* after the sleep is complete. IOW, other ports sharing the
6933	* @ap->host will be allowed to own the EH while this task is
6934	* sleeping.
6935	*
6936	* LOCKING:
6937	* Might sleep.
6938	*/
6939	void ata_msleep(struct ata_port *ap, unsigned int msecs)
6940	__context_unsafe(conditional locking)
6941	{
6942	bool owns_eh = ap && ap->host->eh_owner == current;
6943
6944	if (owns_eh)
6945	ata_eh_release(ap);
6946
6947	if (msecs < 20) {
6948	unsigned long usecs = msecs * USEC_PER_MSEC;
6949	usleep_range(usecs, usecs + 50);
6950	} else {
6951	msleep(msecs);
6952	}
6953
6954	if (owns_eh)
6955	ata_eh_acquire(ap);
6956	}
6957	EXPORT_SYMBOL_GPL(ata_msleep);
6958
6959	/**
6960	* ata_wait_register - wait until register value changes
6961	* @ap: ATA port to wait register for, can be NULL
6962	* @reg: IO-mapped register
6963	* @mask: Mask to apply to read register value
6964	* @val: Wait condition
6965	* @interval: polling interval in milliseconds
6966	* @timeout: timeout in milliseconds
6967	*
6968	* Waiting for some bits of register to change is a common
6969	* operation for ATA controllers. This function reads 32bit LE
6970	* IO-mapped register @reg and tests for the following condition.
6971	*
6972	* (*@reg & mask) != val
6973	*
6974	* If the condition is met, it returns; otherwise, the process is
6975	* repeated after @interval_msec until timeout.
6976	*
6977	* LOCKING:
6978	* Kernel thread context (may sleep)
6979	*
6980	* RETURNS:
6981	* The final register value.
6982	*/
6983	u32 ata_wait_register(struct ata_port ap, void __iomem reg, u32 mask, u32 val,
6984	unsigned int interval, unsigned int timeout)
6985	{
6986	unsigned long deadline;
6987	u32 tmp;
6988
6989	tmp = ioread32(reg);
6990
6991	/* Calculate timeout _after_ the first read to make sure
6992	* preceding writes reach the controller before starting to
6993	* eat away the timeout.
6994	*/
6995	deadline = ata_deadline(jiffies, timeout);
6996
6997	while ((tmp & mask) == val && time_before(jiffies, deadline)) {
6998	ata_msleep(ap, interval);
6999	tmp = ioread32(reg);
7000	}
7001
7002	return tmp;
7003	}
7004	EXPORT_SYMBOL_GPL(ata_wait_register);
7005
7006	/*
7007	* Dummy port_ops
7008	*/
7009	static unsigned int ata_dummy_qc_issue(struct ata_queued_cmd *qc)
7010	{
7011	return AC_ERR_SYSTEM;
7012	}
7013
7014	static void ata_dummy_error_handler(struct ata_port *ap)
7015	__must_hold(&ap->host->eh_mutex)
7016	{
7017	/* truly dummy */
7018	}
7019
7020	struct ata_port_operations ata_dummy_port_ops = {
7021	.qc_issue = ata_dummy_qc_issue,
7022	.error_handler = ata_dummy_error_handler,
7023	.sched_eh = ata_std_sched_eh,
7024	.end_eh = ata_std_end_eh,
7025	};
7026	EXPORT_SYMBOL_GPL(ata_dummy_port_ops);
7027
7028	const struct ata_port_info ata_dummy_port_info = {
7029	.port_ops = &ata_dummy_port_ops,
7030	};
7031	EXPORT_SYMBOL_GPL(ata_dummy_port_info);
7032
7033	EXPORT_TRACEPOINT_SYMBOL_GPL(ata_tf_load);
7034	EXPORT_TRACEPOINT_SYMBOL_GPL(ata_exec_command);
7035	EXPORT_TRACEPOINT_SYMBOL_GPL(ata_bmdma_setup);
7036	EXPORT_TRACEPOINT_SYMBOL_GPL(ata_bmdma_start);
7037	EXPORT_TRACEPOINT_SYMBOL_GPL(ata_bmdma_status);