linux - drivers/ata/libahci.c

2787 lines · 72.5 KiB

1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* libahci.c - Common AHCI SATA low-level routines
4	*
5	* Maintained by: Tejun Heo <[email protected]>
6	* Please ALWAYS copy [email protected]
7	* on emails.
8	*
9	* Copyright 2004-2005 Red Hat, Inc.
10	*
11	* libata documentation is available via 'make {ps\|pdf}docs',
12	* as Documentation/driver-api/libata.rst
13	*
14	* AHCI hardware documentation:
15	* http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
16	* http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
17	*/
18
19	#include <linux/bitops.h>
20	#include <linux/kernel.h>
21	#include <linux/gfp.h>
22	#include <linux/module.h>
23	#include <linux/nospec.h>
24	#include <linux/blkdev.h>
25	#include <linux/delay.h>
26	#include <linux/interrupt.h>
27	#include <linux/dma-mapping.h>
28	#include <linux/device.h>
29	#include <scsi/scsi_host.h>
30	#include <scsi/scsi_cmnd.h>
31	#include <linux/libata.h>
32	#include <linux/pci.h>
33	#include "ahci.h"
34	#include "libata.h"
35
36	static int ahci_skip_host_reset;
37	int ahci_ignore_sss;
38	EXPORT_SYMBOL_GPL(ahci_ignore_sss);
39
40	module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
41	MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
42
43	module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
44	MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
45
46	static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
47	unsigned hints);
48	static ssize_t ahci_led_show(struct ata_port ap, char buf);
49	static ssize_t ahci_led_store(struct ata_port ap, const char buf,
50	size_t size);
51	static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
52	ssize_t size);
53
54
55
56	static int ahci_scr_read(struct ata_link link, unsigned int sc_reg, u32 val);
57	static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
58	static void ahci_qc_fill_rtf(struct ata_queued_cmd *qc);
59	static void ahci_qc_ncq_fill_rtf(struct ata_port *ap, u64 done_mask);
60	static int ahci_port_start(struct ata_port *ap);
61	static void ahci_port_stop(struct ata_port *ap);
62	static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc);
63	static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc);
64	static void ahci_freeze(struct ata_port *ap);
65	static void ahci_thaw(struct ata_port *ap);
66	static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep);
67	static void ahci_enable_fbs(struct ata_port *ap);
68	static void ahci_disable_fbs(struct ata_port *ap);
69	static void ahci_pmp_attach(struct ata_port *ap);
70	static void ahci_pmp_detach(struct ata_port *ap);
71	static int ahci_softreset(struct ata_link link, unsigned int class,
72	unsigned long deadline);
73	static int ahci_pmp_retry_softreset(struct ata_link link, unsigned int class,
74	unsigned long deadline);
75	static int ahci_hardreset(struct ata_link link, unsigned int class,
76	unsigned long deadline);
77	static void ahci_postreset(struct ata_link link, unsigned int class);
78	static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
79	static void ahci_dev_config(struct ata_device *dev);
80	#ifdef CONFIG_PM
81	static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
82	#endif
83	static ssize_t ahci_activity_show(struct ata_device dev, char buf);
84	static ssize_t ahci_activity_store(struct ata_device *dev,
85	enum sw_activity val);
86	static void ahci_init_sw_activity(struct ata_link *link);
87
88	static ssize_t ahci_show_host_caps(struct device *dev,
89	struct device_attribute attr, char buf);
90	static ssize_t ahci_show_host_cap2(struct device *dev,
91	struct device_attribute attr, char buf);
92	static ssize_t ahci_show_host_version(struct device *dev,
93	struct device_attribute attr, char buf);
94	static ssize_t ahci_show_port_cmd(struct device *dev,
95	struct device_attribute attr, char buf);
96	static ssize_t ahci_read_em_buffer(struct device *dev,
97	struct device_attribute attr, char buf);
98	static ssize_t ahci_store_em_buffer(struct device *dev,
99	struct device_attribute *attr,
100	const char *buf, size_t size);
101	static ssize_t ahci_show_em_supported(struct device *dev,
102	struct device_attribute attr, char buf);
103	static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance);
104
105	static DEVICE_ATTR(ahci_host_caps, S_IRUGO, ahci_show_host_caps, NULL);
106	static DEVICE_ATTR(ahci_host_cap2, S_IRUGO, ahci_show_host_cap2, NULL);
107	static DEVICE_ATTR(ahci_host_version, S_IRUGO, ahci_show_host_version, NULL);
108	static DEVICE_ATTR(ahci_port_cmd, S_IRUGO, ahci_show_port_cmd, NULL);
109	static DEVICE_ATTR(em_buffer, S_IWUSR \| S_IRUGO,
110	ahci_read_em_buffer, ahci_store_em_buffer);
111	static DEVICE_ATTR(em_message_supported, S_IRUGO, ahci_show_em_supported, NULL);
112
113	static struct attribute *ahci_shost_attrs[] = {
114	&dev_attr_link_power_management_supported.attr,
115	&dev_attr_link_power_management_policy.attr,
116	&dev_attr_em_message_type.attr,
117	&dev_attr_em_message.attr,
118	&dev_attr_ahci_host_caps.attr,
119	&dev_attr_ahci_host_cap2.attr,
120	&dev_attr_ahci_host_version.attr,
121	&dev_attr_ahci_port_cmd.attr,
122	&dev_attr_em_buffer.attr,
123	&dev_attr_em_message_supported.attr,
124	NULL
125	};
126
127	static const struct attribute_group ahci_shost_attr_group = {
128	.attrs = ahci_shost_attrs
129	};
130
131	const struct attribute_group *ahci_shost_groups[] = {
132	&ahci_shost_attr_group,
133	NULL
134	};
135	EXPORT_SYMBOL_GPL(ahci_shost_groups);
136
137	static struct attribute *ahci_sdev_attrs[] = {
138	&dev_attr_sw_activity.attr,
139	&dev_attr_unload_heads.attr,
140	&dev_attr_ncq_prio_supported.attr,
141	&dev_attr_ncq_prio_enable.attr,
142	NULL
143	};
144
145	static const struct attribute_group ahci_sdev_attr_group = {
146	.attrs = ahci_sdev_attrs
147	};
148
149	const struct attribute_group *ahci_sdev_groups[] = {
150	&ahci_sdev_attr_group,
151	NULL
152	};
153	EXPORT_SYMBOL_GPL(ahci_sdev_groups);
154
155	struct ata_port_operations ahci_ops = {
156	.inherits = &sata_pmp_port_ops,
157
158	.qc_defer = ahci_pmp_qc_defer,
159	.qc_prep = ahci_qc_prep,
160	.qc_issue = ahci_qc_issue,
161	.qc_fill_rtf = ahci_qc_fill_rtf,
162	.qc_ncq_fill_rtf = ahci_qc_ncq_fill_rtf,
163
164	.freeze = ahci_freeze,
165	.thaw = ahci_thaw,
166	.reset.softreset = ahci_softreset,
167	.reset.hardreset = ahci_hardreset,
168	.reset.postreset = ahci_postreset,
169	.pmp_reset.softreset = ahci_softreset,
170	.error_handler = ahci_error_handler,
171	.post_internal_cmd = ahci_post_internal_cmd,
172	.dev_config = ahci_dev_config,
173
174	.scr_read = ahci_scr_read,
175	.scr_write = ahci_scr_write,
176	.pmp_attach = ahci_pmp_attach,
177	.pmp_detach = ahci_pmp_detach,
178
179	.set_lpm = ahci_set_lpm,
180	.em_show = ahci_led_show,
181	.em_store = ahci_led_store,
182	.sw_activity_show = ahci_activity_show,
183	.sw_activity_store = ahci_activity_store,
184	.transmit_led_message = ahci_transmit_led_message,
185	#ifdef CONFIG_PM
186	.port_suspend = ahci_port_suspend,
187	.port_resume = ahci_port_resume,
188	#endif
189	.port_start = ahci_port_start,
190	.port_stop = ahci_port_stop,
191	};
192	EXPORT_SYMBOL_GPL(ahci_ops);
193
194	struct ata_port_operations ahci_pmp_retry_srst_ops = {
195	.inherits = &ahci_ops,
196	.reset.softreset = ahci_pmp_retry_softreset,
197	};
198	EXPORT_SYMBOL_GPL(ahci_pmp_retry_srst_ops);
199
200	static bool ahci_em_messages __read_mostly = true;
201	module_param(ahci_em_messages, bool, 0444);
202	/* add other LED protocol types when they become supported */
203	MODULE_PARM_DESC(ahci_em_messages,
204	"AHCI Enclosure Management Message control (0 = off, 1 = on)");
205
206	/* device sleep idle timeout in ms */
207	static int devslp_idle_timeout __read_mostly = 1000;
208	module_param(devslp_idle_timeout, int, 0644);
209	MODULE_PARM_DESC(devslp_idle_timeout, "device sleep idle timeout");
210
211	static void ahci_enable_ahci(void __iomem *mmio)
212	{
213	int i;
214	u32 tmp;
215
216	/* turn on AHCI_EN */
217	tmp = readl(mmio + HOST_CTL);
218	if (tmp & HOST_AHCI_EN)
219	return;
220
221	/* Some controllers need AHCI_EN to be written multiple times.
222	* Try a few times before giving up.
223	*/
224	for (i = 0; i < 5; i++) {
225	tmp \|= HOST_AHCI_EN;
226	writel(tmp, mmio + HOST_CTL);
227	tmp = readl(mmio + HOST_CTL); /* flush && sanity check */
228	if (tmp & HOST_AHCI_EN)
229	return;
230	msleep(10);
231	}
232
233	WARN_ON(1);
234	}
235
236	/**
237	* ahci_rpm_get_port - Make sure the port is powered on
238	* @ap: Port to power on
239	*
240	* Whenever there is need to access the AHCI host registers outside of
241	* normal execution paths, call this function to make sure the host is
242	* actually powered on.
243	*/
244	static int ahci_rpm_get_port(struct ata_port *ap)
245	{
246	return pm_runtime_get_sync(ap->dev);
247	}
248
249	/**
250	* ahci_rpm_put_port - Undoes ahci_rpm_get_port()
251	* @ap: Port to power down
252	*
253	* Undoes ahci_rpm_get_port() and possibly powers down the AHCI host
254	* if it has no more active users.
255	*/
256	static void ahci_rpm_put_port(struct ata_port *ap)
257	{
258	pm_runtime_put(ap->dev);
259	}
260
261	static ssize_t ahci_show_host_caps(struct device *dev,
262	struct device_attribute attr, char buf)
263	{
264	struct Scsi_Host *shost = class_to_shost(dev);
265	struct ata_port *ap = ata_shost_to_port(shost);
266	struct ahci_host_priv *hpriv = ap->host->private_data;
267
268	return sprintf(buf, "%x\n", hpriv->cap);
269	}
270
271	static ssize_t ahci_show_host_cap2(struct device *dev,
272	struct device_attribute attr, char buf)
273	{
274	struct Scsi_Host *shost = class_to_shost(dev);
275	struct ata_port *ap = ata_shost_to_port(shost);
276	struct ahci_host_priv *hpriv = ap->host->private_data;
277
278	return sprintf(buf, "%x\n", hpriv->cap2);
279	}
280
281	static ssize_t ahci_show_host_version(struct device *dev,
282	struct device_attribute attr, char buf)
283	{
284	struct Scsi_Host *shost = class_to_shost(dev);
285	struct ata_port *ap = ata_shost_to_port(shost);
286	struct ahci_host_priv *hpriv = ap->host->private_data;
287
288	return sprintf(buf, "%x\n", hpriv->version);
289	}
290
291	static ssize_t ahci_show_port_cmd(struct device *dev,
292	struct device_attribute attr, char buf)
293	{
294	struct Scsi_Host *shost = class_to_shost(dev);
295	struct ata_port *ap = ata_shost_to_port(shost);
296	void __iomem *port_mmio = ahci_port_base(ap);
297	ssize_t ret;
298
299	ahci_rpm_get_port(ap);
300	ret = sprintf(buf, "%x\n", readl(port_mmio + PORT_CMD));
301	ahci_rpm_put_port(ap);
302
303	return ret;
304	}
305
306	static ssize_t ahci_read_em_buffer(struct device *dev,
307	struct device_attribute attr, char buf)
308	{
309	struct Scsi_Host *shost = class_to_shost(dev);
310	struct ata_port *ap = ata_shost_to_port(shost);
311	struct ahci_host_priv *hpriv = ap->host->private_data;
312	void __iomem *mmio = hpriv->mmio;
313	void __iomem *em_mmio = mmio + hpriv->em_loc;
314	u32 em_ctl, msg;
315	unsigned long flags;
316	size_t count;
317	int i;
318
319	ahci_rpm_get_port(ap);
320	spin_lock_irqsave(ap->lock, flags);
321
322	em_ctl = readl(mmio + HOST_EM_CTL);
323	if (!(ap->flags & ATA_FLAG_EM) \|\| em_ctl & EM_CTL_XMT \|\|
324	!(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO)) {
325	spin_unlock_irqrestore(ap->lock, flags);
326	ahci_rpm_put_port(ap);
327	return -EINVAL;
328	}
329
330	if (!(em_ctl & EM_CTL_MR)) {
331	spin_unlock_irqrestore(ap->lock, flags);
332	ahci_rpm_put_port(ap);
333	return -EAGAIN;
334	}
335
336	if (!(em_ctl & EM_CTL_SMB))
337	em_mmio += hpriv->em_buf_sz;
338
339	count = hpriv->em_buf_sz;
340
341	/* the count should not be larger than PAGE_SIZE */
342	if (count > PAGE_SIZE) {
343	if (printk_ratelimit())
344	ata_port_warn(ap,
345	"EM read buffer size too large: "
346	"buffer size %u, page size %lu\n",
347	hpriv->em_buf_sz, PAGE_SIZE);
348	count = PAGE_SIZE;
349	}
350
351	for (i = 0; i < count; i += 4) {
352	msg = readl(em_mmio + i);
353	buf[i] = msg & 0xff;
354	buf[i + 1] = (msg >> 8) & 0xff;
355	buf[i + 2] = (msg >> 16) & 0xff;
356	buf[i + 3] = (msg >> 24) & 0xff;
357	}
358
359	spin_unlock_irqrestore(ap->lock, flags);
360	ahci_rpm_put_port(ap);
361
362	return i;
363	}
364
365	static ssize_t ahci_store_em_buffer(struct device *dev,
366	struct device_attribute *attr,
367	const char *buf, size_t size)
368	{
369	struct Scsi_Host *shost = class_to_shost(dev);
370	struct ata_port *ap = ata_shost_to_port(shost);
371	struct ahci_host_priv *hpriv = ap->host->private_data;
372	void __iomem *mmio = hpriv->mmio;
373	void __iomem *em_mmio = mmio + hpriv->em_loc;
374	const unsigned char *msg_buf = buf;
375	u32 em_ctl, msg;
376	unsigned long flags;
377	int i;
378
379	/* check size validity */
380	if (!(ap->flags & ATA_FLAG_EM) \|\|
381	!(hpriv->em_msg_type & EM_MSG_TYPE_SGPIO) \|\|
382	size % 4 \|\| size > hpriv->em_buf_sz)
383	return -EINVAL;
384
385	ahci_rpm_get_port(ap);
386	spin_lock_irqsave(ap->lock, flags);
387
388	em_ctl = readl(mmio + HOST_EM_CTL);
389	if (em_ctl & EM_CTL_TM) {
390	spin_unlock_irqrestore(ap->lock, flags);
391	ahci_rpm_put_port(ap);
392	return -EBUSY;
393	}
394
395	for (i = 0; i < size; i += 4) {
396	msg = msg_buf[i] \| msg_buf[i + 1] << 8 \|
397	msg_buf[i + 2] << 16 \| msg_buf[i + 3] << 24;
398	writel(msg, em_mmio + i);
399	}
400
401	writel(em_ctl \| EM_CTL_TM, mmio + HOST_EM_CTL);
402
403	spin_unlock_irqrestore(ap->lock, flags);
404	ahci_rpm_put_port(ap);
405
406	return size;
407	}
408
409	static ssize_t ahci_show_em_supported(struct device *dev,
410	struct device_attribute attr, char buf)
411	{
412	struct Scsi_Host *shost = class_to_shost(dev);
413	struct ata_port *ap = ata_shost_to_port(shost);
414	struct ahci_host_priv *hpriv = ap->host->private_data;
415	void __iomem *mmio = hpriv->mmio;
416	u32 em_ctl;
417
418	ahci_rpm_get_port(ap);
419	em_ctl = readl(mmio + HOST_EM_CTL);
420	ahci_rpm_put_port(ap);
421
422	return sprintf(buf, "%s%s%s%s\n",
423	em_ctl & EM_CTL_LED ? "led " : "",
424	em_ctl & EM_CTL_SAFTE ? "saf-te " : "",
425	em_ctl & EM_CTL_SES ? "ses-2 " : "",
426	em_ctl & EM_CTL_SGPIO ? "sgpio " : "");
427	}
428
429	/**
430	* ahci_save_initial_config - Save and fixup initial config values
431	* @dev: target AHCI device
432	* @hpriv: host private area to store config values
433	*
434	* Some registers containing configuration info might be setup by
435	* BIOS and might be cleared on reset. This function saves the
436	* initial values of those registers into @hpriv such that they
437	* can be restored after controller reset.
438	*
439	* If inconsistent, config values are fixed up by this function.
440	*
441	* If it is not set already this function sets hpriv->start_engine to
442	* ahci_start_engine.
443	*
444	* LOCKING:
445	* None.
446	*/
447	void ahci_save_initial_config(struct device dev, struct ahci_host_priv hpriv)
448	{
449	void __iomem *mmio = hpriv->mmio;
450	void __iomem *port_mmio;
451	unsigned long port_map;
452	u32 cap, cap2, vers;
453	int i;
454
455	/* make sure AHCI mode is enabled before accessing CAP */
456	ahci_enable_ahci(mmio);
457
458	/*
459	* Values prefixed with saved_ are written back to the HBA and ports
460	* registers after reset. Values without are used for driver operation.
461	*/
462
463	/*
464	* Override HW-init HBA capability fields with the platform-specific
465	* values. The rest of the HBA capabilities are defined as Read-only
466	* and can't be modified in CSR anyway.
467	*/
468	cap = readl(mmio + HOST_CAP);
469	if (hpriv->saved_cap)
470	cap = (cap & ~(HOST_CAP_SSS \| HOST_CAP_MPS)) \| hpriv->saved_cap;
471	hpriv->saved_cap = cap;
472
473	/* CAP2 register is only defined for AHCI 1.2 and later */
474	vers = readl(mmio + HOST_VERSION);
475	if ((vers >> 16) > 1 \|\|
476	((vers >> 16) == 1 && (vers & 0xFFFF) >= 0x200))
477	hpriv->saved_cap2 = cap2 = readl(mmio + HOST_CAP2);
478	else
479	hpriv->saved_cap2 = cap2 = 0;
480
481	/* some chips have errata preventing 64bit use */
482	if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
483	dev_info(dev, "controller can't do 64bit DMA, forcing 32bit\n");
484	cap &= ~HOST_CAP_64;
485	}
486
487	if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
488	dev_info(dev, "controller can't do NCQ, turning off CAP_NCQ\n");
489	cap &= ~HOST_CAP_NCQ;
490	}
491
492	if (!(cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_YES_NCQ)) {
493	dev_info(dev, "controller can do NCQ, turning on CAP_NCQ\n");
494	cap \|= HOST_CAP_NCQ;
495	}
496
497	if ((cap & HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
498	dev_info(dev, "controller can't do PMP, turning off CAP_PMP\n");
499	cap &= ~HOST_CAP_PMP;
500	}
501
502	if ((cap & HOST_CAP_SNTF) && (hpriv->flags & AHCI_HFLAG_NO_SNTF)) {
503	dev_info(dev,
504	"controller can't do SNTF, turning off CAP_SNTF\n");
505	cap &= ~HOST_CAP_SNTF;
506	}
507
508	if ((cap2 & HOST_CAP2_SDS) && (hpriv->flags & AHCI_HFLAG_NO_DEVSLP)) {
509	dev_info(dev,
510	"controller can't do DEVSLP, turning off\n");
511	cap2 &= ~HOST_CAP2_SDS;
512	cap2 &= ~HOST_CAP2_SADM;
513	}
514
515	if (!(cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_YES_FBS)) {
516	dev_info(dev, "controller can do FBS, turning on CAP_FBS\n");
517	cap \|= HOST_CAP_FBS;
518	}
519
520	if ((cap & HOST_CAP_FBS) && (hpriv->flags & AHCI_HFLAG_NO_FBS)) {
521	dev_info(dev, "controller can't do FBS, turning off CAP_FBS\n");
522	cap &= ~HOST_CAP_FBS;
523	}
524
525	if (!(cap & HOST_CAP_ALPM) && (hpriv->flags & AHCI_HFLAG_YES_ALPM)) {
526	dev_info(dev, "controller can do ALPM, turning on CAP_ALPM\n");
527	cap \|= HOST_CAP_ALPM;
528	}
529
530	if ((cap & HOST_CAP_SXS) && (hpriv->flags & AHCI_HFLAG_NO_SXS)) {
531	dev_info(dev, "controller does not support SXS, disabling CAP_SXS\n");
532	cap &= ~HOST_CAP_SXS;
533	}
534
535	/* Override the HBA ports mapping if the platform needs it */
536	port_map = readl(mmio + HOST_PORTS_IMPL);
537	if (hpriv->saved_port_map && port_map != hpriv->saved_port_map) {
538	dev_info(dev, "forcing port_map 0x%lx -> 0x%x\n",
539	port_map, hpriv->saved_port_map);
540	port_map = hpriv->saved_port_map;
541	} else {
542	hpriv->saved_port_map = port_map;
543	}
544
545	/* mask_port_map not set means that all ports are available */
546	if (hpriv->mask_port_map) {
547	dev_warn(dev, "masking port_map 0x%lx -> 0x%lx\n",
548	port_map,
549	port_map & hpriv->mask_port_map);
550	port_map &= hpriv->mask_port_map;
551	}
552
553	/* cross check port_map and cap.n_ports */
554	if (port_map) {
555	int map_ports = hweight_long(port_map);
556
557	/* If PI has more ports than n_ports, whine, clear
558	* port_map and let it be generated from n_ports.
559	*/
560	if (map_ports > ahci_nr_ports(cap)) {
561	dev_warn(dev,
562	"implemented port map (0x%lx) contains more ports than nr_ports (%u), using nr_ports\n",
563	port_map, ahci_nr_ports(cap));
564	port_map = 0;
565	}
566	}
567
568	/* fabricate port_map from cap.nr_ports for < AHCI 1.3 */
569	if (!port_map && vers < 0x10300) {
570	port_map = (1 << ahci_nr_ports(cap)) - 1;
571	dev_warn(dev, "forcing PORTS_IMPL to 0x%lx\n", port_map);
572
573	/* write the fixed up value to the PI register */
574	hpriv->saved_port_map = port_map;
575	}
576
577	/*
578	* Preserve the ports capabilities defined by the platform. Note there
579	* is no need in storing the rest of the P#.CMD fields since they are
580	* volatile.
581	*/
582	for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) {
583	if (hpriv->saved_port_cap[i])
584	continue;
585
586	port_mmio = __ahci_port_base(hpriv, i);
587	hpriv->saved_port_cap[i] =
588	readl(port_mmio + PORT_CMD) & PORT_CMD_CAP;
589	}
590
591	/* record values to use during operation */
592	hpriv->cap = cap;
593	hpriv->cap2 = cap2;
594	hpriv->version = vers;
595	hpriv->port_map = port_map;
596
597	if (!hpriv->start_engine)
598	hpriv->start_engine = ahci_start_engine;
599
600	if (!hpriv->stop_engine)
601	hpriv->stop_engine = ahci_stop_engine;
602
603	if (!hpriv->irq_handler)
604	hpriv->irq_handler = ahci_single_level_irq_intr;
605	}
606	EXPORT_SYMBOL_GPL(ahci_save_initial_config);
607
608	/**
609	* ahci_restore_initial_config - Restore initial config
610	* @host: target ATA host
611	*
612	* Restore initial config stored by ahci_save_initial_config().
613	*
614	* LOCKING:
615	* None.
616	*/
617	static void ahci_restore_initial_config(struct ata_host *host)
618	{
619	struct ahci_host_priv *hpriv = host->private_data;
620	unsigned long port_map = hpriv->port_map;
621	void __iomem *mmio = hpriv->mmio;
622	void __iomem *port_mmio;
623	int i;
624
625	writel(hpriv->saved_cap, mmio + HOST_CAP);
626	if (hpriv->saved_cap2)
627	writel(hpriv->saved_cap2, mmio + HOST_CAP2);
628	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
629	(void) readl(mmio + HOST_PORTS_IMPL); /* flush */
630
631	for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) {
632	port_mmio = __ahci_port_base(hpriv, i);
633	writel(hpriv->saved_port_cap[i], port_mmio + PORT_CMD);
634	}
635	}
636
637	static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
638	{
639	static const int offset[] = {
640	[SCR_STATUS] = PORT_SCR_STAT,
641	[SCR_CONTROL] = PORT_SCR_CTL,
642	[SCR_ERROR] = PORT_SCR_ERR,
643	[SCR_ACTIVE] = PORT_SCR_ACT,
644	[SCR_NOTIFICATION] = PORT_SCR_NTF,
645	};
646	struct ahci_host_priv *hpriv = ap->host->private_data;
647
648	if (sc_reg < ARRAY_SIZE(offset) &&
649	(sc_reg != SCR_NOTIFICATION \|\| (hpriv->cap & HOST_CAP_SNTF)))
650	return offset[sc_reg];
651	return 0;
652	}
653
654	static int ahci_scr_read(struct ata_link link, unsigned int sc_reg, u32 val)
655	{
656	void __iomem *port_mmio = ahci_port_base(link->ap);
657	int offset = ahci_scr_offset(link->ap, sc_reg);
658
659	if (offset) {
660	*val = readl(port_mmio + offset);
661	return 0;
662	}
663	return -EINVAL;
664	}
665
666	static int ahci_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
667	{
668	void __iomem *port_mmio = ahci_port_base(link->ap);
669	int offset = ahci_scr_offset(link->ap, sc_reg);
670
671	if (offset) {
672	writel(val, port_mmio + offset);
673	return 0;
674	}
675	return -EINVAL;
676	}
677
678	void ahci_start_engine(struct ata_port *ap)
679	{
680	void __iomem *port_mmio = ahci_port_base(ap);
681	u32 tmp;
682
683	/* start DMA */
684	tmp = readl(port_mmio + PORT_CMD);
685	tmp \|= PORT_CMD_START;
686	writel(tmp, port_mmio + PORT_CMD);
687	readl(port_mmio + PORT_CMD); /* flush */
688	}
689	EXPORT_SYMBOL_GPL(ahci_start_engine);
690
691	int ahci_stop_engine(struct ata_port *ap)
692	{
693	void __iomem *port_mmio = ahci_port_base(ap);
694	struct ahci_host_priv *hpriv = ap->host->private_data;
695	u32 tmp;
696
697	/*
698	* On some controllers, stopping a port's DMA engine while the port
699	* is in ALPM state (partial or slumber) results in failures on
700	* subsequent DMA engine starts. For those controllers, put the
701	* port back in active state before stopping its DMA engine.
702	*/
703	if ((hpriv->flags & AHCI_HFLAG_WAKE_BEFORE_STOP) &&
704	(ap->link.lpm_policy > ATA_LPM_MAX_POWER) &&
705	ahci_set_lpm(&ap->link, ATA_LPM_MAX_POWER, ATA_LPM_WAKE_ONLY)) {
706	dev_err(ap->host->dev, "Failed to wake up port before engine stop\n");
707	return -EIO;
708	}
709
710	tmp = readl(port_mmio + PORT_CMD);
711
712	/* check if the HBA is idle */
713	if ((tmp & (PORT_CMD_START \| PORT_CMD_LIST_ON)) == 0)
714	return 0;
715
716	/*
717	* Don't try to issue commands but return with ENODEV if the
718	* AHCI controller not available anymore (e.g. due to PCIe hot
719	* unplugging). Otherwise a 500ms delay for each port is added.
720	*/
721	if (tmp == 0xffffffff) {
722	dev_err(ap->host->dev, "AHCI controller unavailable!\n");
723	return -ENODEV;
724	}
725
726	/* setting HBA to idle */
727	tmp &= ~PORT_CMD_START;
728	writel(tmp, port_mmio + PORT_CMD);
729
730	/* wait for engine to stop. This could be as long as 500 msec */
731	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
732	PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
733	if (tmp & PORT_CMD_LIST_ON)
734	return -EIO;
735
736	return 0;
737	}
738	EXPORT_SYMBOL_GPL(ahci_stop_engine);
739
740	void ahci_start_fis_rx(struct ata_port *ap)
741	{
742	void __iomem *port_mmio = ahci_port_base(ap);
743	struct ahci_host_priv *hpriv = ap->host->private_data;
744	struct ahci_port_priv *pp = ap->private_data;
745	u32 tmp;
746
747	/*
748	* On HBAs that only support 32-bit addressing PxCLBU is read only '0'.
749	* When applying the AHCI_HFLAG_32BIT_ONLY quirk, PxCLBU is RW, and the
750	* reset value is Implementation Specific, so we need to clear it to 0.
751	*/
752	if (hpriv->cap & HOST_CAP_64)
753	writel((pp->cmd_slot_dma >> 16) >> 16,
754	port_mmio + PORT_LST_ADDR_HI);
755	else if (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)
756	writel(0, port_mmio + PORT_LST_ADDR_HI);
757	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
758
759	/*
760	* On HBAs that only support 32-bit addressing PxFBU is read only '0'.
761	* When applying the AHCI_HFLAG_32BIT_ONLY quirk, PxFBU is RW, and the
762	* reset value is Implementation Specific, so we need to clear it to 0.
763	*/
764	if (hpriv->cap & HOST_CAP_64)
765	writel((pp->rx_fis_dma >> 16) >> 16,
766	port_mmio + PORT_FIS_ADDR_HI);
767	else if (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)
768	writel(0, port_mmio + PORT_FIS_ADDR_HI);
769	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
770
771	/* enable FIS reception */
772	tmp = readl(port_mmio + PORT_CMD);
773	tmp \|= PORT_CMD_FIS_RX;
774	writel(tmp, port_mmio + PORT_CMD);
775
776	/* flush */
777	readl(port_mmio + PORT_CMD);
778	}
779	EXPORT_SYMBOL_GPL(ahci_start_fis_rx);
780
781	static int ahci_stop_fis_rx(struct ata_port *ap)
782	{
783	void __iomem *port_mmio = ahci_port_base(ap);
784	u32 tmp;
785
786	/* disable FIS reception */
787	tmp = readl(port_mmio + PORT_CMD);
788	tmp &= ~PORT_CMD_FIS_RX;
789	writel(tmp, port_mmio + PORT_CMD);
790
791	/* wait for completion, spec says 500ms, give it 1000 */
792	tmp = ata_wait_register(ap, port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
793	PORT_CMD_FIS_ON, 10, 1000);
794	if (tmp & PORT_CMD_FIS_ON)
795	return -EBUSY;
796
797	return 0;
798	}
799
800	static void ahci_power_up(struct ata_port *ap)
801	{
802	struct ahci_host_priv *hpriv = ap->host->private_data;
803	void __iomem *port_mmio = ahci_port_base(ap);
804	u32 cmd;
805
806	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
807
808	/* spin up device */
809	if (hpriv->cap & HOST_CAP_SSS) {
810	cmd \|= PORT_CMD_SPIN_UP;
811	writel(cmd, port_mmio + PORT_CMD);
812	}
813
814	/* wake up link */
815	writel(cmd \| PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
816	}
817
818	static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
819	unsigned int hints)
820	{
821	struct ata_port *ap = link->ap;
822	struct ahci_host_priv *hpriv = ap->host->private_data;
823	struct ahci_port_priv *pp = ap->private_data;
824	void __iomem *port_mmio = ahci_port_base(ap);
825
826	if (policy != ATA_LPM_MAX_POWER) {
827	/* wakeup flag only applies to the max power policy */
828	hints &= ~ATA_LPM_WAKE_ONLY;
829
830	/*
831	* Disable interrupts on Phy Ready. This keeps us from
832	* getting woken up due to spurious phy ready
833	* interrupts.
834	*/
835	pp->intr_mask &= ~PORT_IRQ_PHYRDY;
836	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
837
838	sata_link_scr_lpm(link, policy, false);
839	}
840
841	if (hpriv->cap & HOST_CAP_ALPM) {
842	u32 cmd = readl(port_mmio + PORT_CMD);
843
844	if (policy == ATA_LPM_MAX_POWER \|\| !(hints & ATA_LPM_HIPM)) {
845	if (!(hints & ATA_LPM_WAKE_ONLY))
846	cmd &= ~(PORT_CMD_ASP \| PORT_CMD_ALPE);
847	cmd \|= PORT_CMD_ICC_ACTIVE;
848
849	writel(cmd, port_mmio + PORT_CMD);
850	readl(port_mmio + PORT_CMD);
851
852	/* wait 10ms to be sure we've come out of LPM state */
853	ata_msleep(ap, 10);
854
855	if (hints & ATA_LPM_WAKE_ONLY)
856	return 0;
857	} else {
858	cmd \|= PORT_CMD_ALPE;
859	if (policy == ATA_LPM_MIN_POWER)
860	cmd \|= PORT_CMD_ASP;
861	else if (policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
862	cmd &= ~PORT_CMD_ASP;
863
864	/* write out new cmd value */
865	writel(cmd, port_mmio + PORT_CMD);
866	}
867	}
868
869	/* set aggressive device sleep */
870	if ((hpriv->cap2 & HOST_CAP2_SDS) &&
871	(hpriv->cap2 & HOST_CAP2_SADM) &&
872	(link->device->flags & ATA_DFLAG_DEVSLP)) {
873	if (policy == ATA_LPM_MIN_POWER \|\|
874	policy == ATA_LPM_MIN_POWER_WITH_PARTIAL)
875	ahci_set_aggressive_devslp(ap, true);
876	else
877	ahci_set_aggressive_devslp(ap, false);
878	}
879
880	if (policy == ATA_LPM_MAX_POWER) {
881	sata_link_scr_lpm(link, policy, false);
882
883	/* turn PHYRDY IRQ back on */
884	pp->intr_mask \|= PORT_IRQ_PHYRDY;
885	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
886	}
887
888	return 0;
889	}
890
891	#ifdef CONFIG_PM
892	static void ahci_power_down(struct ata_port *ap)
893	{
894	struct ahci_host_priv *hpriv = ap->host->private_data;
895	void __iomem *port_mmio = ahci_port_base(ap);
896	u32 cmd, scontrol;
897
898	if (!(hpriv->cap & HOST_CAP_SSS))
899	return;
900
901	/* put device into listen mode, first set PxSCTL.DET to 0 */
902	scontrol = readl(port_mmio + PORT_SCR_CTL);
903	scontrol &= ~0xf;
904	writel(scontrol, port_mmio + PORT_SCR_CTL);
905
906	/* then set PxCMD.SUD to 0 */
907	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
908	cmd &= ~PORT_CMD_SPIN_UP;
909	writel(cmd, port_mmio + PORT_CMD);
910	}
911	#endif
912
913	static void ahci_start_port(struct ata_port *ap)
914	{
915	struct ahci_host_priv *hpriv = ap->host->private_data;
916	struct ahci_port_priv *pp = ap->private_data;
917	struct ata_link *link;
918	struct ahci_em_priv *emp;
919	ssize_t rc;
920	int i;
921
922	/* enable FIS reception */
923	ahci_start_fis_rx(ap);
924
925	/* enable DMA */
926	if (!(hpriv->flags & AHCI_HFLAG_DELAY_ENGINE))
927	hpriv->start_engine(ap);
928
929	/* turn on LEDs */
930	if (ap->flags & ATA_FLAG_EM) {
931	ata_for_each_link(link, ap, EDGE) {
932	emp = &pp->em_priv[link->pmp];
933
934	/* EM Transmit bit maybe busy during init */
935	for (i = 0; i < EM_MAX_RETRY; i++) {
936	rc = ap->ops->transmit_led_message(ap,
937	emp->led_state,
938	4);
939	/*
940	* If busy, give a breather but do not
941	* release EH ownership by using msleep()
942	* instead of ata_msleep(). EM Transmit
943	* bit is busy for the whole host and
944	* releasing ownership will cause other
945	* ports to fail the same way.
946	*/
947	if (rc == -EBUSY)
948	msleep(1);
949	else
950	break;
951	}
952	}
953	}
954
955	if (ap->flags & ATA_FLAG_SW_ACTIVITY)
956	ata_for_each_link(link, ap, EDGE)
957	ahci_init_sw_activity(link);
958
959	}
960
961	static int ahci_deinit_port(struct ata_port ap, const char *emsg)
962	{
963	int rc;
964	struct ahci_host_priv *hpriv = ap->host->private_data;
965
966	/* disable DMA */
967	rc = hpriv->stop_engine(ap);
968	if (rc) {
969	*emsg = "failed to stop engine";
970	return rc;
971	}
972
973	/* disable FIS reception */
974	rc = ahci_stop_fis_rx(ap);
975	if (rc) {
976	*emsg = "failed stop FIS RX";
977	return rc;
978	}
979
980	return 0;
981	}
982
983	int ahci_reset_controller(struct ata_host *host)
984	{
985	struct ahci_host_priv *hpriv = host->private_data;
986	void __iomem *mmio = hpriv->mmio;
987	u32 tmp;
988
989	/*
990	* We must be in AHCI mode, before using anything AHCI-specific, such
991	* as HOST_RESET.
992	*/
993	ahci_enable_ahci(mmio);
994
995	/* Global controller reset */
996	if (ahci_skip_host_reset) {
997	dev_info(host->dev, "Skipping global host reset\n");
998	return 0;
999	}
1000
1001	tmp = readl(mmio + HOST_CTL);
1002	if (!(tmp & HOST_RESET)) {
1003	writel(tmp \| HOST_RESET, mmio + HOST_CTL);
1004	readl(mmio + HOST_CTL); /* flush */
1005	}
1006
1007	/*
1008	* To perform host reset, OS should set HOST_RESET and poll until this
1009	* bit is read to be "0". Reset must complete within 1 second, or the
1010	* hardware should be considered fried.
1011	*/
1012	tmp = ata_wait_register(NULL, mmio + HOST_CTL, HOST_RESET,
1013	HOST_RESET, 10, 1000);
1014	if (tmp & HOST_RESET) {
1015	dev_err(host->dev, "Controller reset failed (0x%x)\n",
1016	tmp);
1017	return -EIO;
1018	}
1019
1020	/* Turn on AHCI mode */
1021	ahci_enable_ahci(mmio);
1022
1023	/* Some registers might be cleared on reset. Restore initial values. */
1024	if (!(hpriv->flags & AHCI_HFLAG_NO_WRITE_TO_RO))
1025	ahci_restore_initial_config(host);
1026
1027	return 0;
1028	}
1029	EXPORT_SYMBOL_GPL(ahci_reset_controller);
1030
1031	static void ahci_sw_activity(struct ata_link *link)
1032	{
1033	struct ata_port *ap = link->ap;
1034	struct ahci_port_priv *pp = ap->private_data;
1035	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1036
1037	if (!(link->flags & ATA_LFLAG_SW_ACTIVITY))
1038	return;
1039
1040	emp->activity++;
1041	if (!timer_pending(&emp->timer))
1042	mod_timer(&emp->timer, jiffies + msecs_to_jiffies(10));
1043	}
1044
1045	static void ahci_sw_activity_blink(struct timer_list *t)
1046	{
1047	struct ahci_em_priv *emp = timer_container_of(emp, t, timer);
1048	struct ata_link *link = emp->link;
1049	struct ata_port *ap = link->ap;
1050
1051	unsigned long led_message = emp->led_state;
1052	u32 activity_led_state;
1053	unsigned long flags;
1054
1055	led_message &= EM_MSG_LED_VALUE;
1056	led_message \|= ap->port_no \| (link->pmp << 8);
1057
1058	/* check to see if we've had activity. If so,
1059	* toggle state of LED and reset timer. If not,
1060	* turn LED to desired idle state.
1061	*/
1062	spin_lock_irqsave(ap->lock, flags);
1063	if (emp->saved_activity != emp->activity) {
1064	emp->saved_activity = emp->activity;
1065	/* get the current LED state */
1066	activity_led_state = led_message & EM_MSG_LED_VALUE_ON;
1067
1068	if (activity_led_state)
1069	activity_led_state = 0;
1070	else
1071	activity_led_state = 1;
1072
1073	/* clear old state */
1074	led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
1075
1076	/* toggle state */
1077	led_message \|= (activity_led_state << 16);
1078	mod_timer(&emp->timer, jiffies + msecs_to_jiffies(100));
1079	} else {
1080	/* switch to idle */
1081	led_message &= ~EM_MSG_LED_VALUE_ACTIVITY;
1082	if (emp->blink_policy == BLINK_OFF)
1083	led_message \|= (1 << 16);
1084	}
1085	spin_unlock_irqrestore(ap->lock, flags);
1086	ap->ops->transmit_led_message(ap, led_message, 4);
1087	}
1088
1089	static void ahci_init_sw_activity(struct ata_link *link)
1090	{
1091	struct ata_port *ap = link->ap;
1092	struct ahci_port_priv *pp = ap->private_data;
1093	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1094
1095	/* init activity stats, setup timer */
1096	emp->saved_activity = emp->activity = 0;
1097	emp->link = link;
1098	timer_setup(&emp->timer, ahci_sw_activity_blink, 0);
1099
1100	/* check our blink policy and set flag for link if it's enabled */
1101	if (emp->blink_policy)
1102	link->flags \|= ATA_LFLAG_SW_ACTIVITY;
1103	}
1104
1105	int ahci_reset_em(struct ata_host *host)
1106	{
1107	struct ahci_host_priv *hpriv = host->private_data;
1108	void __iomem *mmio = hpriv->mmio;
1109	u32 em_ctl;
1110
1111	em_ctl = readl(mmio + HOST_EM_CTL);
1112	if ((em_ctl & EM_CTL_TM) \|\| (em_ctl & EM_CTL_RST))
1113	return -EINVAL;
1114
1115	writel(em_ctl \| EM_CTL_RST, mmio + HOST_EM_CTL);
1116	return 0;
1117	}
1118	EXPORT_SYMBOL_GPL(ahci_reset_em);
1119
1120	static ssize_t ahci_transmit_led_message(struct ata_port *ap, u32 state,
1121	ssize_t size)
1122	{
1123	struct ahci_host_priv *hpriv = ap->host->private_data;
1124	struct ahci_port_priv *pp = ap->private_data;
1125	void __iomem *mmio = hpriv->mmio;
1126	u32 em_ctl;
1127	u32 message[] = {0, 0};
1128	unsigned long flags;
1129	int pmp;
1130	struct ahci_em_priv *emp;
1131
1132	/* get the slot number from the message */
1133	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1134	if (pmp < EM_MAX_SLOTS)
1135	emp = &pp->em_priv[pmp];
1136	else
1137	return -EINVAL;
1138
1139	ahci_rpm_get_port(ap);
1140	spin_lock_irqsave(ap->lock, flags);
1141
1142	/*
1143	* if we are still busy transmitting a previous message,
1144	* do not allow
1145	*/
1146	em_ctl = readl(mmio + HOST_EM_CTL);
1147	if (em_ctl & EM_CTL_TM) {
1148	spin_unlock_irqrestore(ap->lock, flags);
1149	ahci_rpm_put_port(ap);
1150	return -EBUSY;
1151	}
1152
1153	if (hpriv->em_msg_type & EM_MSG_TYPE_LED) {
1154	/*
1155	* create message header - this is all zero except for
1156	* the message size, which is 4 bytes.
1157	*/
1158	message[0] \|= (4 << 8);
1159
1160	/* ignore 0:4 of byte zero, fill in port info yourself */
1161	message[1] = ((state & ~EM_MSG_LED_HBA_PORT) \| ap->port_no);
1162
1163	/* write message to EM_LOC */
1164	writel(message[0], mmio + hpriv->em_loc);
1165	writel(message[1], mmio + hpriv->em_loc+4);
1166
1167	/*
1168	* tell hardware to transmit the message
1169	*/
1170	writel(em_ctl \| EM_CTL_TM, mmio + HOST_EM_CTL);
1171	}
1172
1173	/* save off new led state for port/slot */
1174	emp->led_state = state;
1175
1176	spin_unlock_irqrestore(ap->lock, flags);
1177	ahci_rpm_put_port(ap);
1178
1179	return size;
1180	}
1181
1182	static ssize_t ahci_led_show(struct ata_port ap, char buf)
1183	{
1184	struct ahci_port_priv *pp = ap->private_data;
1185	struct ata_link *link;
1186	struct ahci_em_priv *emp;
1187	int rc = 0;
1188
1189	ata_for_each_link(link, ap, EDGE) {
1190	emp = &pp->em_priv[link->pmp];
1191	rc += sprintf(buf, "%lx\n", emp->led_state);
1192	}
1193	return rc;
1194	}
1195
1196	static ssize_t ahci_led_store(struct ata_port ap, const char buf,
1197	size_t size)
1198	{
1199	unsigned int state;
1200	int pmp;
1201	struct ahci_port_priv *pp = ap->private_data;
1202	struct ahci_em_priv *emp;
1203
1204	if (kstrtouint(buf, 0, &state) < 0)
1205	return -EINVAL;
1206
1207	/* get the slot number from the message */
1208	pmp = (state & EM_MSG_LED_PMP_SLOT) >> 8;
1209	if (pmp < EM_MAX_SLOTS) {
1210	pmp = array_index_nospec(pmp, EM_MAX_SLOTS);
1211	emp = &pp->em_priv[pmp];
1212	} else {
1213	return -EINVAL;
1214	}
1215
1216	/* mask off the activity bits if we are in sw_activity
1217	* mode, user should turn off sw_activity before setting
1218	* activity led through em_message
1219	*/
1220	if (emp->blink_policy)
1221	state &= ~EM_MSG_LED_VALUE_ACTIVITY;
1222
1223	return ap->ops->transmit_led_message(ap, state, size);
1224	}
1225
1226	static ssize_t ahci_activity_store(struct ata_device *dev, enum sw_activity val)
1227	{
1228	struct ata_link *link = dev->link;
1229	struct ata_port *ap = link->ap;
1230	struct ahci_port_priv *pp = ap->private_data;
1231	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1232	u32 port_led_state = emp->led_state;
1233
1234	/* save the desired Activity LED behavior */
1235	if (val == OFF) {
1236	/* clear LFLAG */
1237	link->flags &= ~(ATA_LFLAG_SW_ACTIVITY);
1238
1239	/* set the LED to OFF */
1240	port_led_state &= EM_MSG_LED_VALUE_OFF;
1241	port_led_state \|= (ap->port_no \| (link->pmp << 8));
1242	ap->ops->transmit_led_message(ap, port_led_state, 4);
1243	} else {
1244	link->flags \|= ATA_LFLAG_SW_ACTIVITY;
1245	if (val == BLINK_OFF) {
1246	/* set LED to ON for idle */
1247	port_led_state &= EM_MSG_LED_VALUE_OFF;
1248	port_led_state \|= (ap->port_no \| (link->pmp << 8));
1249	port_led_state \|= EM_MSG_LED_VALUE_ON; /* check this */
1250	ap->ops->transmit_led_message(ap, port_led_state, 4);
1251	}
1252	}
1253	emp->blink_policy = val;
1254	return 0;
1255	}
1256
1257	static ssize_t ahci_activity_show(struct ata_device dev, char buf)
1258	{
1259	struct ata_link *link = dev->link;
1260	struct ata_port *ap = link->ap;
1261	struct ahci_port_priv *pp = ap->private_data;
1262	struct ahci_em_priv *emp = &pp->em_priv[link->pmp];
1263
1264	/* display the saved value of activity behavior for this
1265	* disk.
1266	*/
1267	return sprintf(buf, "%d\n", emp->blink_policy);
1268	}
1269
1270	static void ahci_port_clear_pending_irq(struct ata_port *ap)
1271	{
1272	struct ahci_host_priv *hpriv = ap->host->private_data;
1273	void __iomem *port_mmio = ahci_port_base(ap);
1274	u32 tmp;
1275
1276	/* clear SError */
1277	tmp = readl(port_mmio + PORT_SCR_ERR);
1278	dev_dbg(ap->host->dev, "PORT_SCR_ERR 0x%x\n", tmp);
1279	writel(tmp, port_mmio + PORT_SCR_ERR);
1280
1281	/* clear port IRQ */
1282	tmp = readl(port_mmio + PORT_IRQ_STAT);
1283	dev_dbg(ap->host->dev, "PORT_IRQ_STAT 0x%x\n", tmp);
1284	if (tmp)
1285	writel(tmp, port_mmio + PORT_IRQ_STAT);
1286
1287	writel(1 << ap->port_no, hpriv->mmio + HOST_IRQ_STAT);
1288	}
1289
1290	static void ahci_port_init(struct device dev, struct ata_port ap,
1291	int port_no, void __iomem *mmio,
1292	void __iomem *port_mmio)
1293	{
1294	const char *emsg = NULL;
1295	int rc;
1296
1297	/* make sure port is not active */
1298	rc = ahci_deinit_port(ap, &emsg);
1299	if (rc)
1300	dev_warn(dev, "%s (%d)\n", emsg, rc);
1301
1302	ahci_port_clear_pending_irq(ap);
1303	}
1304
1305	void ahci_init_controller(struct ata_host *host)
1306	{
1307	struct ahci_host_priv *hpriv = host->private_data;
1308	void __iomem *mmio = hpriv->mmio;
1309	int i;
1310	void __iomem *port_mmio;
1311	u32 tmp;
1312
1313	for (i = 0; i < host->n_ports; i++) {
1314	struct ata_port *ap = host->ports[i];
1315
1316	port_mmio = ahci_port_base(ap);
1317	if (ata_port_is_dummy(ap))
1318	continue;
1319
1320	ahci_port_init(host->dev, ap, i, mmio, port_mmio);
1321	}
1322
1323	tmp = readl(mmio + HOST_CTL);
1324	dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);
1325	writel(tmp \| HOST_IRQ_EN, mmio + HOST_CTL);
1326	tmp = readl(mmio + HOST_CTL);
1327	dev_dbg(host->dev, "HOST_CTL 0x%x\n", tmp);
1328	}
1329	EXPORT_SYMBOL_GPL(ahci_init_controller);
1330
1331	static void ahci_dev_config(struct ata_device *dev)
1332	{
1333	struct ahci_host_priv *hpriv = dev->link->ap->host->private_data;
1334
1335	if ((dev->class == ATA_DEV_ATAPI) &&
1336	(hpriv->flags & AHCI_HFLAG_ATAPI_DMA_QUIRK))
1337	dev->quirks \|= ATA_QUIRK_ATAPI_MOD16_DMA;
1338
1339	if (hpriv->flags & AHCI_HFLAG_SECT255) {
1340	dev->max_sectors = 255;
1341	ata_dev_info(dev,
1342	"SB600 AHCI: limiting to 255 sectors per cmd\n");
1343	}
1344	}
1345
1346	unsigned int ahci_dev_classify(struct ata_port *ap)
1347	{
1348	void __iomem *port_mmio = ahci_port_base(ap);
1349	struct ata_taskfile tf;
1350	u32 tmp;
1351
1352	tmp = readl(port_mmio + PORT_SIG);
1353	tf.lbah = (tmp >> 24) & 0xff;
1354	tf.lbam = (tmp >> 16) & 0xff;
1355	tf.lbal = (tmp >> 8) & 0xff;
1356	tf.nsect = (tmp) & 0xff;
1357
1358	return ata_port_classify(ap, &tf);
1359	}
1360	EXPORT_SYMBOL_GPL(ahci_dev_classify);
1361
1362	void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
1363	u32 opts)
1364	{
1365	dma_addr_t cmd_tbl_dma;
1366
1367	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
1368
1369	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
1370	pp->cmd_slot[tag].status = 0;
1371	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
1372	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
1373	}
1374	EXPORT_SYMBOL_GPL(ahci_fill_cmd_slot);
1375
1376	int ahci_kick_engine(struct ata_port *ap)
1377	{
1378	void __iomem *port_mmio = ahci_port_base(ap);
1379	struct ahci_host_priv *hpriv = ap->host->private_data;
1380	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1381	u32 tmp;
1382	int busy, rc;
1383
1384	/* stop engine */
1385	rc = hpriv->stop_engine(ap);
1386	if (rc)
1387	goto out_restart;
1388
1389	/* need to do CLO?
1390	* always do CLO if PMP is attached (AHCI-1.3 9.2)
1391	*/
1392	busy = status & (ATA_BUSY \| ATA_DRQ);
1393	if (!busy && !sata_pmp_attached(ap)) {
1394	rc = 0;
1395	goto out_restart;
1396	}
1397
1398	if (!(hpriv->cap & HOST_CAP_CLO)) {
1399	rc = -EOPNOTSUPP;
1400	goto out_restart;
1401	}
1402
1403	/* perform CLO */
1404	tmp = readl(port_mmio + PORT_CMD);
1405	tmp \|= PORT_CMD_CLO;
1406	writel(tmp, port_mmio + PORT_CMD);
1407
1408	rc = 0;
1409	tmp = ata_wait_register(ap, port_mmio + PORT_CMD,
1410	PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1411	if (tmp & PORT_CMD_CLO)
1412	rc = -EIO;
1413
1414	/* restart engine */
1415	out_restart:
1416	hpriv->start_engine(ap);
1417	return rc;
1418	}
1419	EXPORT_SYMBOL_GPL(ahci_kick_engine);
1420
1421	static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1422	struct ata_taskfile *tf, int is_cmd, u16 flags,
1423	unsigned int timeout_msec)
1424	{
1425	const u32 cmd_fis_len = 5; /* five dwords */
1426	struct ahci_port_priv *pp = ap->private_data;
1427	void __iomem *port_mmio = ahci_port_base(ap);
1428	u8 *fis = pp->cmd_tbl;
1429	u32 tmp;
1430
1431	/* prep the command */
1432	ata_tf_to_fis(tf, pmp, is_cmd, fis);
1433	ahci_fill_cmd_slot(pp, 0, cmd_fis_len \| flags \| (pmp << 12));
1434
1435	/* set port value for softreset of Port Multiplier */
1436	if (pp->fbs_enabled && pp->fbs_last_dev != pmp) {
1437	tmp = readl(port_mmio + PORT_FBS);
1438	tmp &= ~(PORT_FBS_DEV_MASK \| PORT_FBS_DEC);
1439	tmp \|= pmp << PORT_FBS_DEV_OFFSET;
1440	writel(tmp, port_mmio + PORT_FBS);
1441	pp->fbs_last_dev = pmp;
1442	}
1443
1444	/* issue & wait */
1445	writel(1, port_mmio + PORT_CMD_ISSUE);
1446
1447	if (timeout_msec) {
1448	tmp = ata_wait_register(ap, port_mmio + PORT_CMD_ISSUE,
1449	0x1, 0x1, 1, timeout_msec);
1450	if (tmp & 0x1) {
1451	ahci_kick_engine(ap);
1452	return -EBUSY;
1453	}
1454	} else
1455	readl(port_mmio + PORT_CMD_ISSUE); /* flush */
1456
1457	return 0;
1458	}
1459
1460	int ahci_do_softreset(struct ata_link link, unsigned int class,
1461	int pmp, unsigned long deadline,
1462	int (check_ready)(struct ata_link link))
1463	{
1464	struct ata_port *ap = link->ap;
1465	struct ahci_host_priv *hpriv = ap->host->private_data;
1466	struct ahci_port_priv *pp = ap->private_data;
1467	const char *reason = NULL;
1468	unsigned long now;
1469	unsigned int msecs;
1470	struct ata_taskfile tf;
1471	bool fbs_disabled = false;
1472	int rc;
1473
1474	/* prepare for SRST (AHCI-1.1 10.4.1) */
1475	rc = ahci_kick_engine(ap);
1476	if (rc && rc != -EOPNOTSUPP)
1477	ata_link_warn(link, "failed to reset engine (errno=%d)\n", rc);
1478
1479	/*
1480	* According to AHCI-1.2 9.3.9: if FBS is enable, software shall
1481	* clear PxFBS.EN to '0' prior to issuing software reset to devices
1482	* that is attached to port multiplier.
1483	*/
1484	if (!ata_is_host_link(link) && pp->fbs_enabled) {
1485	ahci_disable_fbs(ap);
1486	fbs_disabled = true;
1487	}
1488
1489	ata_tf_init(link->device, &tf);
1490
1491	/* issue the first H2D Register FIS */
1492	msecs = 0;
1493	now = jiffies;
1494	if (time_after(deadline, now))
1495	msecs = jiffies_to_msecs(deadline - now);
1496
1497	tf.ctl \|= ATA_SRST;
1498	if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
1499	AHCI_CMD_RESET \| AHCI_CMD_CLR_BUSY, msecs)) {
1500	rc = -EIO;
1501	reason = "1st FIS failed";
1502	goto fail;
1503	}
1504
1505	/* spec says at least 5us, but be generous and sleep for 1ms */
1506	ata_msleep(ap, 1);
1507
1508	/* issue the second H2D Register FIS */
1509	tf.ctl &= ~ATA_SRST;
1510	ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
1511
1512	/* wait for link to become ready */
1513	rc = ata_wait_after_reset(link, deadline, check_ready);
1514	if (rc == -EBUSY && hpriv->flags & AHCI_HFLAG_SRST_TOUT_IS_OFFLINE) {
1515	/*
1516	* Workaround for cases where link online status can't
1517	* be trusted. Treat device readiness timeout as link
1518	* offline.
1519	*/
1520	ata_link_info(link, "device not ready, treating as offline\n");
1521	*class = ATA_DEV_NONE;
1522	} else if (rc) {
1523	/* link occupied, -ENODEV too is an error */
1524	reason = "device not ready";
1525	goto fail;
1526	} else
1527	*class = ahci_dev_classify(ap);
1528
1529	/* re-enable FBS if disabled before */
1530	if (fbs_disabled)
1531	ahci_enable_fbs(ap);
1532
1533	return 0;
1534
1535	fail:
1536	ata_link_err(link, "softreset failed (%s)\n", reason);
1537	return rc;
1538	}
1539	EXPORT_SYMBOL_GPL(ahci_do_softreset);
1540
1541	int ahci_check_ready(struct ata_link *link)
1542	{
1543	void __iomem *port_mmio = ahci_port_base(link->ap);
1544	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1545
1546	return ata_check_ready(status);
1547	}
1548	EXPORT_SYMBOL_GPL(ahci_check_ready);
1549
1550	static int ahci_softreset(struct ata_link link, unsigned int class,
1551	unsigned long deadline)
1552	{
1553	int pmp = sata_srst_pmp(link);
1554
1555	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
1556	}
1557
1558	static int ahci_bad_pmp_check_ready(struct ata_link *link)
1559	{
1560	void __iomem *port_mmio = ahci_port_base(link->ap);
1561	u8 status = readl(port_mmio + PORT_TFDATA) & 0xFF;
1562	u32 irq_status = readl(port_mmio + PORT_IRQ_STAT);
1563
1564	/*
1565	* There is no need to check TFDATA if BAD PMP is found due to HW bug,
1566	* which can save timeout delay.
1567	*/
1568	if (irq_status & PORT_IRQ_BAD_PMP)
1569	return -EIO;
1570
1571	return ata_check_ready(status);
1572	}
1573
1574	static int ahci_pmp_retry_softreset(struct ata_link link, unsigned int class,
1575	unsigned long deadline)
1576	{
1577	struct ata_port *ap = link->ap;
1578	void __iomem *port_mmio = ahci_port_base(ap);
1579	int pmp = sata_srst_pmp(link);
1580	int rc;
1581	u32 irq_sts;
1582
1583	rc = ahci_do_softreset(link, class, pmp, deadline,
1584	ahci_bad_pmp_check_ready);
1585
1586	/*
1587	* Soft reset fails with IPMS set when PMP is enabled but
1588	* SATA HDD/ODD is connected to SATA port, do soft reset
1589	* again to port 0.
1590	*/
1591	if (rc == -EIO) {
1592	irq_sts = readl(port_mmio + PORT_IRQ_STAT);
1593	if (irq_sts & PORT_IRQ_BAD_PMP) {
1594	ata_link_warn(link,
1595	"applying PMP SRST workaround "
1596	"and retrying\n");
1597	rc = ahci_do_softreset(link, class, 0, deadline,
1598	ahci_check_ready);
1599	}
1600	}
1601
1602	return rc;
1603	}
1604
1605	int ahci_do_hardreset(struct ata_link link, unsigned int class,
1606	unsigned long deadline, bool *online)
1607	{
1608	const unsigned int *timing = sata_ehc_deb_timing(&link->eh_context);
1609	struct ata_port *ap = link->ap;
1610	struct ahci_port_priv *pp = ap->private_data;
1611	struct ahci_host_priv *hpriv = ap->host->private_data;
1612	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1613	struct ata_taskfile tf;
1614	int rc;
1615
1616	hpriv->stop_engine(ap);
1617
1618	/* clear D2H reception area to properly wait for D2H FIS */
1619	ata_tf_init(link->device, &tf);
1620	tf.status = ATA_BUSY;
1621	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
1622
1623	ahci_port_clear_pending_irq(ap);
1624
1625	rc = sata_link_hardreset(link, timing, deadline, online,
1626	ahci_check_ready);
1627
1628	hpriv->start_engine(ap);
1629
1630	if (*online)
1631	*class = ahci_dev_classify(ap);
1632
1633	return rc;
1634	}
1635	EXPORT_SYMBOL_GPL(ahci_do_hardreset);
1636
1637	static int ahci_hardreset(struct ata_link link, unsigned int class,
1638	unsigned long deadline)
1639	{
1640	bool online;
1641
1642	return ahci_do_hardreset(link, class, deadline, &online);
1643	}
1644
1645	static void ahci_postreset(struct ata_link link, unsigned int class)
1646	{
1647	struct ata_port *ap = link->ap;
1648	void __iomem *port_mmio = ahci_port_base(ap);
1649	u32 new_tmp, tmp;
1650
1651	ata_std_postreset(link, class);
1652
1653	/* Make sure port's ATAPI bit is set appropriately */
1654	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1655	if (*class == ATA_DEV_ATAPI)
1656	new_tmp \|= PORT_CMD_ATAPI;
1657	else
1658	new_tmp &= ~PORT_CMD_ATAPI;
1659	if (new_tmp != tmp) {
1660	writel(new_tmp, port_mmio + PORT_CMD);
1661	readl(port_mmio + PORT_CMD); /* flush */
1662	}
1663	}
1664
1665	static unsigned int ahci_fill_sg(struct ata_queued_cmd qc, void cmd_tbl)
1666	{
1667	struct scatterlist *sg;
1668	struct ahci_sg *ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1669	unsigned int si;
1670
1671	/*
1672	* Next, the S/G list.
1673	*/
1674	for_each_sg(qc->sg, sg, qc->n_elem, si) {
1675	dma_addr_t addr = sg_dma_address(sg);
1676	u32 sg_len = sg_dma_len(sg);
1677
1678	ahci_sg[si].addr = cpu_to_le32(addr & 0xffffffff);
1679	ahci_sg[si].addr_hi = cpu_to_le32((addr >> 16) >> 16);
1680	ahci_sg[si].flags_size = cpu_to_le32(sg_len - 1);
1681	}
1682
1683	return si;
1684	}
1685
1686	static int ahci_pmp_qc_defer(struct ata_queued_cmd *qc)
1687	{
1688	struct ata_port *ap = qc->ap;
1689	struct ahci_port_priv *pp = ap->private_data;
1690
1691	if (!sata_pmp_attached(ap) \|\| pp->fbs_enabled)
1692	return ata_std_qc_defer(qc);
1693	else
1694	return sata_pmp_qc_defer_cmd_switch(qc);
1695	}
1696
1697	static enum ata_completion_errors ahci_qc_prep(struct ata_queued_cmd *qc)
1698	{
1699	struct ata_port *ap = qc->ap;
1700	struct ahci_port_priv *pp = ap->private_data;
1701	int is_atapi = ata_is_atapi(qc->tf.protocol);
1702	void *cmd_tbl;
1703	u32 opts;
1704	const u32 cmd_fis_len = 5; /* five dwords */
1705	unsigned int n_elem;
1706
1707	/*
1708	* Fill in command table information. First, the header,
1709	* a SATA Register - Host to Device command FIS.
1710	*/
1711	cmd_tbl = pp->cmd_tbl + qc->hw_tag * AHCI_CMD_TBL_SZ;
1712
1713	ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
1714	if (is_atapi) {
1715	memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1716	memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1717	}
1718
1719	n_elem = 0;
1720	if (qc->flags & ATA_QCFLAG_DMAMAP)
1721	n_elem = ahci_fill_sg(qc, cmd_tbl);
1722
1723	/*
1724	* Fill in command slot information.
1725	*/
1726	opts = cmd_fis_len \| n_elem << 16 \| (qc->dev->link->pmp << 12);
1727	if (qc->tf.flags & ATA_TFLAG_WRITE)
1728	opts \|= AHCI_CMD_WRITE;
1729	if (is_atapi)
1730	opts \|= AHCI_CMD_ATAPI \| AHCI_CMD_PREFETCH;
1731
1732	ahci_fill_cmd_slot(pp, qc->hw_tag, opts);
1733
1734	return AC_ERR_OK;
1735	}
1736
1737	static void ahci_fbs_dec_intr(struct ata_port *ap)
1738	{
1739	struct ahci_port_priv *pp = ap->private_data;
1740	void __iomem *port_mmio = ahci_port_base(ap);
1741	u32 fbs = readl(port_mmio + PORT_FBS);
1742	int retries = 3;
1743
1744	BUG_ON(!pp->fbs_enabled);
1745
1746	/* time to wait for DEC is not specified by AHCI spec,
1747	* add a retry loop for safety.
1748	*/
1749	writel(fbs \| PORT_FBS_DEC, port_mmio + PORT_FBS);
1750	fbs = readl(port_mmio + PORT_FBS);
1751	while ((fbs & PORT_FBS_DEC) && retries--) {
1752	udelay(1);
1753	fbs = readl(port_mmio + PORT_FBS);
1754	}
1755
1756	if (fbs & PORT_FBS_DEC)
1757	dev_err(ap->host->dev, "failed to clear device error\n");
1758	}
1759
1760	static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1761	{
1762	struct ahci_host_priv *hpriv = ap->host->private_data;
1763	struct ahci_port_priv *pp = ap->private_data;
1764	struct ata_eh_info *host_ehi = &ap->link.eh_info;
1765	struct ata_link *link = NULL;
1766	struct ata_queued_cmd *active_qc;
1767	struct ata_eh_info *active_ehi;
1768	bool fbs_need_dec = false;
1769	u32 serror;
1770
1771	/* determine active link with error */
1772	if (pp->fbs_enabled) {
1773	void __iomem *port_mmio = ahci_port_base(ap);
1774	u32 fbs = readl(port_mmio + PORT_FBS);
1775	int pmp = fbs >> PORT_FBS_DWE_OFFSET;
1776
1777	if ((fbs & PORT_FBS_SDE) && (pmp < ap->nr_pmp_links)) {
1778	link = &ap->pmp_link[pmp];
1779	fbs_need_dec = true;
1780	}
1781
1782	} else
1783	ata_for_each_link(link, ap, EDGE)
1784	if (ata_link_active(link))
1785	break;
1786
1787	if (!link)
1788	link = &ap->link;
1789
1790	active_qc = ata_qc_from_tag(ap, link->active_tag);
1791	active_ehi = &link->eh_info;
1792
1793	/* record irq stat */
1794	ata_ehi_clear_desc(host_ehi);
1795	ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
1796
1797	/* AHCI needs SError cleared; otherwise, it might lock up */
1798	ahci_scr_read(&ap->link, SCR_ERROR, &serror);
1799	ahci_scr_write(&ap->link, SCR_ERROR, serror);
1800	host_ehi->serror \|= serror;
1801
1802	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1803	if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
1804	irq_stat &= ~PORT_IRQ_IF_ERR;
1805
1806	if (irq_stat & PORT_IRQ_TF_ERR) {
1807	/* If qc is active, charge it; otherwise, the active
1808	* link. There's no active qc on NCQ errors. It will
1809	* be determined by EH by reading log page 10h.
1810	*/
1811	if (active_qc)
1812	active_qc->err_mask \|= AC_ERR_DEV;
1813	else
1814	active_ehi->err_mask \|= AC_ERR_DEV;
1815
1816	if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
1817	host_ehi->serror &= ~SERR_INTERNAL;
1818	}
1819
1820	if (irq_stat & PORT_IRQ_UNK_FIS) {
1821	u32 *unk = pp->rx_fis + RX_FIS_UNK;
1822
1823	active_ehi->err_mask \|= AC_ERR_HSM;
1824	active_ehi->action \|= ATA_EH_RESET;
1825	ata_ehi_push_desc(active_ehi,
1826	"unknown FIS %08x %08x %08x %08x" ,
1827	unk[0], unk[1], unk[2], unk[3]);
1828	}
1829
1830	if (sata_pmp_attached(ap) && (irq_stat & PORT_IRQ_BAD_PMP)) {
1831	active_ehi->err_mask \|= AC_ERR_HSM;
1832	active_ehi->action \|= ATA_EH_RESET;
1833	ata_ehi_push_desc(active_ehi, "incorrect PMP");
1834	}
1835
1836	if (irq_stat & (PORT_IRQ_HBUS_ERR \| PORT_IRQ_HBUS_DATA_ERR)) {
1837	host_ehi->err_mask \|= AC_ERR_HOST_BUS;
1838	host_ehi->action \|= ATA_EH_RESET;
1839	ata_ehi_push_desc(host_ehi, "host bus error");
1840	}
1841
1842	if (irq_stat & PORT_IRQ_IF_ERR) {
1843	if (fbs_need_dec)
1844	active_ehi->err_mask \|= AC_ERR_DEV;
1845	else {
1846	host_ehi->err_mask \|= AC_ERR_ATA_BUS;
1847	host_ehi->action \|= ATA_EH_RESET;
1848	}
1849
1850	ata_ehi_push_desc(host_ehi, "interface fatal error");
1851	}
1852
1853	if (irq_stat & (PORT_IRQ_CONNECT \| PORT_IRQ_PHYRDY)) {
1854	ata_ehi_hotplugged(host_ehi);
1855	ata_ehi_push_desc(host_ehi, "%s",
1856	irq_stat & PORT_IRQ_CONNECT ?
1857	"connection status changed" : "PHY RDY changed");
1858	}
1859
1860	/* okay, let's hand over to EH */
1861
1862	if (irq_stat & PORT_IRQ_FREEZE)
1863	ata_port_freeze(ap);
1864	else if (fbs_need_dec) {
1865	ata_link_abort(link);
1866	ahci_fbs_dec_intr(ap);
1867	} else
1868	ata_port_abort(ap);
1869	}
1870
1871	static void ahci_qc_complete(struct ata_port ap, void __iomem port_mmio)
1872	{
1873	struct ata_eh_info *ehi = &ap->link.eh_info;
1874	struct ahci_port_priv *pp = ap->private_data;
1875	u32 qc_active = 0;
1876	int rc;
1877
1878	/*
1879	* pp->active_link is not reliable once FBS is enabled, both
1880	* PORT_SCR_ACT and PORT_CMD_ISSUE should be checked because
1881	* NCQ and non-NCQ commands may be in flight at the same time.
1882	*/
1883	if (pp->fbs_enabled) {
1884	if (ap->qc_active) {
1885	qc_active = readl(port_mmio + PORT_SCR_ACT);
1886	qc_active \|= readl(port_mmio + PORT_CMD_ISSUE);
1887	}
1888	} else {
1889	/* pp->active_link is valid iff any command is in flight */
1890	if (ap->qc_active && pp->active_link->sactive)
1891	qc_active = readl(port_mmio + PORT_SCR_ACT);
1892	else
1893	qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1894	}
1895
1896	rc = ata_qc_complete_multiple(ap, qc_active);
1897	if (unlikely(rc < 0 && !(ap->pflags & ATA_PFLAG_RESETTING))) {
1898	ehi->err_mask \|= AC_ERR_HSM;
1899	ehi->action \|= ATA_EH_RESET;
1900	ata_port_freeze(ap);
1901	}
1902	}
1903
1904	static void ahci_handle_port_interrupt(struct ata_port *ap,
1905	void __iomem *port_mmio, u32 status)
1906	{
1907	struct ahci_port_priv *pp = ap->private_data;
1908	struct ahci_host_priv *hpriv = ap->host->private_data;
1909
1910	/* ignore BAD_PMP while resetting */
1911	if (unlikely(ap->pflags & ATA_PFLAG_RESETTING))
1912	status &= ~PORT_IRQ_BAD_PMP;
1913
1914	if (sata_lpm_ignore_phy_events(&ap->link)) {
1915	status &= ~PORT_IRQ_PHYRDY;
1916	ahci_scr_write(&ap->link, SCR_ERROR, SERR_PHYRDY_CHG);
1917	}
1918
1919	if (unlikely(status & PORT_IRQ_ERROR)) {
1920	/*
1921	* Before getting the error notification, we may have
1922	* received SDB FISes notifying successful completions.
1923	* Handle these first and then handle the error.
1924	*/
1925	ahci_qc_complete(ap, port_mmio);
1926	ahci_error_intr(ap, status);
1927	return;
1928	}
1929
1930	if (status & PORT_IRQ_SDB_FIS) {
1931	/* If SNotification is available, leave notification
1932	* handling to sata_async_notification(). If not,
1933	* emulate it by snooping SDB FIS RX area.
1934	*
1935	* Snooping FIS RX area is probably cheaper than
1936	* poking SNotification but some constrollers which
1937	* implement SNotification, ICH9 for example, don't
1938	* store AN SDB FIS into receive area.
1939	*/
1940	if (hpriv->cap & HOST_CAP_SNTF)
1941	sata_async_notification(ap);
1942	else {
1943	/* If the 'N' bit in word 0 of the FIS is set,
1944	* we just received asynchronous notification.
1945	* Tell libata about it.
1946	*
1947	* Lack of SNotification should not appear in
1948	* ahci 1.2, so the workaround is unnecessary
1949	* when FBS is enabled.
1950	*/
1951	if (pp->fbs_enabled)
1952	WARN_ON_ONCE(1);
1953	else {
1954	const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1955	u32 f0 = le32_to_cpu(f[0]);
1956	if (f0 & (1 << 15))
1957	sata_async_notification(ap);
1958	}
1959	}
1960	}
1961
1962	/* Handle completed commands */
1963	ahci_qc_complete(ap, port_mmio);
1964	}
1965
1966	static void ahci_port_intr(struct ata_port *ap)
1967	{
1968	void __iomem *port_mmio = ahci_port_base(ap);
1969	u32 status;
1970
1971	status = readl(port_mmio + PORT_IRQ_STAT);
1972	writel(status, port_mmio + PORT_IRQ_STAT);
1973
1974	ahci_handle_port_interrupt(ap, port_mmio, status);
1975	}
1976
1977	static irqreturn_t ahci_multi_irqs_intr_hard(int irq, void *dev_instance)
1978	{
1979	struct ata_port *ap = dev_instance;
1980	void __iomem *port_mmio = ahci_port_base(ap);
1981	u32 status;
1982
1983	status = readl(port_mmio + PORT_IRQ_STAT);
1984	writel(status, port_mmio + PORT_IRQ_STAT);
1985
1986	spin_lock(ap->lock);
1987	ahci_handle_port_interrupt(ap, port_mmio, status);
1988	spin_unlock(ap->lock);
1989
1990	return IRQ_HANDLED;
1991	}
1992
1993	u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked)
1994	{
1995	unsigned int i, handled = 0;
1996
1997	for (i = 0; i < host->n_ports; i++) {
1998	struct ata_port *ap;
1999
2000	if (!(irq_masked & (1 << i)))
2001	continue;
2002
2003	ap = host->ports[i];
2004	if (ap) {
2005	ahci_port_intr(ap);
2006	} else {
2007	if (ata_ratelimit())
2008	dev_warn(host->dev,
2009	"interrupt on disabled port %u\n", i);
2010	}
2011
2012	handled = 1;
2013	}
2014
2015	return handled;
2016	}
2017	EXPORT_SYMBOL_GPL(ahci_handle_port_intr);
2018
2019	static irqreturn_t ahci_single_level_irq_intr(int irq, void *dev_instance)
2020	{
2021	struct ata_host *host = dev_instance;
2022	struct ahci_host_priv *hpriv;
2023	unsigned int rc = 0;
2024	void __iomem *mmio;
2025	u32 irq_stat, irq_masked;
2026
2027	hpriv = host->private_data;
2028	mmio = hpriv->mmio;
2029
2030	/* sigh. 0xffffffff is a valid return from h/w */
2031	irq_stat = readl(mmio + HOST_IRQ_STAT);
2032	if (!irq_stat)
2033	return IRQ_NONE;
2034
2035	irq_masked = irq_stat & hpriv->port_map;
2036
2037	spin_lock(&host->lock);
2038
2039	rc = ahci_handle_port_intr(host, irq_masked);
2040
2041	/* HOST_IRQ_STAT behaves as level triggered latch meaning that
2042	* it should be cleared after all the port events are cleared;
2043	* otherwise, it will raise a spurious interrupt after each
2044	* valid one. Please read section 10.6.2 of ahci 1.1 for more
2045	* information.
2046	*
2047	* Also, use the unmasked value to clear interrupt as spurious
2048	* pending event on a dummy port might cause screaming IRQ.
2049	*/
2050	writel(irq_stat, mmio + HOST_IRQ_STAT);
2051
2052	spin_unlock(&host->lock);
2053
2054	return IRQ_RETVAL(rc);
2055	}
2056
2057	unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
2058	{
2059	struct ata_port *ap = qc->ap;
2060	void __iomem *port_mmio = ahci_port_base(ap);
2061	struct ahci_port_priv *pp = ap->private_data;
2062
2063	/* Keep track of the currently active link. It will be used
2064	* in completion path to determine whether NCQ phase is in
2065	* progress.
2066	*/
2067	pp->active_link = qc->dev->link;
2068
2069	if (ata_is_ncq(qc->tf.protocol))
2070	writel(1 << qc->hw_tag, port_mmio + PORT_SCR_ACT);
2071
2072	if (pp->fbs_enabled && pp->fbs_last_dev != qc->dev->link->pmp) {
2073	u32 fbs = readl(port_mmio + PORT_FBS);
2074	fbs &= ~(PORT_FBS_DEV_MASK \| PORT_FBS_DEC);
2075	fbs \|= qc->dev->link->pmp << PORT_FBS_DEV_OFFSET;
2076	writel(fbs, port_mmio + PORT_FBS);
2077	pp->fbs_last_dev = qc->dev->link->pmp;
2078	}
2079
2080	writel(1 << qc->hw_tag, port_mmio + PORT_CMD_ISSUE);
2081
2082	ahci_sw_activity(qc->dev->link);
2083
2084	return 0;
2085	}
2086	EXPORT_SYMBOL_GPL(ahci_qc_issue);
2087
2088	static void ahci_qc_fill_rtf(struct ata_queued_cmd *qc)
2089	{
2090	struct ahci_port_priv *pp = qc->ap->private_data;
2091	u8 *rx_fis = pp->rx_fis;
2092
2093	if (pp->fbs_enabled)
2094	rx_fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
2095
2096	/*
2097	* After a successful execution of an ATA PIO data-in command,
2098	* the device doesn't send D2H Reg FIS to update the TF and
2099	* the host should take TF and E_Status from the preceding PIO
2100	* Setup FIS.
2101	*/
2102	if (qc->tf.protocol == ATA_PROT_PIO && qc->dma_dir == DMA_FROM_DEVICE &&
2103	!(qc->flags & ATA_QCFLAG_EH)) {
2104	ata_tf_from_fis(rx_fis + RX_FIS_PIO_SETUP, &qc->result_tf);
2105	qc->result_tf.status = (rx_fis + RX_FIS_PIO_SETUP)[15];
2106	return;
2107	}
2108
2109	/*
2110	* For NCQ commands, we never get a D2H FIS, so reading the D2H Register
2111	* FIS area of the Received FIS Structure (which contains a copy of the
2112	* last D2H FIS received) will contain an outdated status code.
2113	* For NCQ commands, we instead get a SDB FIS, so read the SDB FIS area
2114	* instead. However, the SDB FIS does not contain the LBA, so we can't
2115	* use the ata_tf_from_fis() helper.
2116	*/
2117	if (ata_is_ncq(qc->tf.protocol)) {
2118	const u8 *fis = rx_fis + RX_FIS_SDB;
2119
2120	/*
2121	* Successful NCQ commands have been filled already.
2122	* A failed NCQ command will read the status here.
2123	* (Note that a failed NCQ command will get a more specific
2124	* error when reading the NCQ Command Error log.)
2125	*/
2126	qc->result_tf.status = fis[2];
2127	qc->result_tf.error = fis[3];
2128	return;
2129	}
2130
2131	ata_tf_from_fis(rx_fis + RX_FIS_D2H_REG, &qc->result_tf);
2132	}
2133
2134	static void ahci_qc_ncq_fill_rtf(struct ata_port *ap, u64 done_mask)
2135	{
2136	struct ahci_port_priv *pp = ap->private_data;
2137	const u8 *fis;
2138
2139	/* No outstanding commands. */
2140	if (!ap->qc_active)
2141	return;
2142
2143	/*
2144	* FBS not enabled, so read status and error once, since they are shared
2145	* for all QCs.
2146	*/
2147	if (!pp->fbs_enabled) {
2148	u8 status, error;
2149
2150	/* No outstanding NCQ commands. */
2151	if (!pp->active_link->sactive)
2152	return;
2153
2154	fis = pp->rx_fis + RX_FIS_SDB;
2155	status = fis[2];
2156	error = fis[3];
2157
2158	while (done_mask) {
2159	struct ata_queued_cmd *qc;
2160	unsigned int tag = __ffs64(done_mask);
2161
2162	qc = ata_qc_from_tag(ap, tag);
2163	if (qc && ata_is_ncq(qc->tf.protocol)) {
2164	qc->result_tf.status = status;
2165	qc->result_tf.error = error;
2166	qc->result_tf.flags = qc->tf.flags;
2167	qc->flags \|= ATA_QCFLAG_RTF_FILLED;
2168	}
2169	done_mask &= ~(1ULL << tag);
2170	}
2171
2172	return;
2173	}
2174
2175	/*
2176	* FBS enabled, so read the status and error for each QC, since the QCs
2177	* can belong to different PMP links. (Each PMP link has its own FIS
2178	* Receive Area.)
2179	*/
2180	while (done_mask) {
2181	struct ata_queued_cmd *qc;
2182	unsigned int tag = __ffs64(done_mask);
2183
2184	qc = ata_qc_from_tag(ap, tag);
2185	if (qc && ata_is_ncq(qc->tf.protocol)) {
2186	fis = pp->rx_fis;
2187	fis += qc->dev->link->pmp * AHCI_RX_FIS_SZ;
2188	fis += RX_FIS_SDB;
2189	qc->result_tf.status = fis[2];
2190	qc->result_tf.error = fis[3];
2191	qc->result_tf.flags = qc->tf.flags;
2192	qc->flags \|= ATA_QCFLAG_RTF_FILLED;
2193	}
2194	done_mask &= ~(1ULL << tag);
2195	}
2196	}
2197
2198	static void ahci_freeze(struct ata_port *ap)
2199	{
2200	void __iomem *port_mmio = ahci_port_base(ap);
2201
2202	/* turn IRQ off */
2203	writel(0, port_mmio + PORT_IRQ_MASK);
2204	}
2205
2206	static void ahci_thaw(struct ata_port *ap)
2207	{
2208	struct ahci_host_priv *hpriv = ap->host->private_data;
2209	void __iomem *mmio = hpriv->mmio;
2210	void __iomem *port_mmio = ahci_port_base(ap);
2211	u32 tmp;
2212	struct ahci_port_priv *pp = ap->private_data;
2213
2214	/* clear IRQ */
2215	tmp = readl(port_mmio + PORT_IRQ_STAT);
2216	writel(tmp, port_mmio + PORT_IRQ_STAT);
2217	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
2218
2219	/* turn IRQ back on */
2220	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2221	}
2222
2223	void ahci_error_handler(struct ata_port *ap)
2224	__must_hold(&ap->host->eh_mutex)
2225	{
2226	struct ahci_host_priv *hpriv = ap->host->private_data;
2227
2228	if (!ata_port_is_frozen(ap)) {
2229	/* restart engine */
2230	hpriv->stop_engine(ap);
2231	hpriv->start_engine(ap);
2232	}
2233
2234	sata_pmp_error_handler(ap);
2235
2236	if (!ata_dev_enabled(ap->link.device))
2237	hpriv->stop_engine(ap);
2238	}
2239	EXPORT_SYMBOL_GPL(ahci_error_handler);
2240
2241	static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
2242	{
2243	struct ata_port *ap = qc->ap;
2244
2245	/* make DMA engine forget about the failed command */
2246	if (qc->flags & ATA_QCFLAG_EH)
2247	ahci_kick_engine(ap);
2248	}
2249
2250	static void ahci_set_aggressive_devslp(struct ata_port *ap, bool sleep)
2251	{
2252	struct ahci_host_priv *hpriv = ap->host->private_data;
2253	void __iomem *port_mmio = ahci_port_base(ap);
2254	struct ata_device *dev = ap->link.device;
2255	u32 devslp, dm, dito, mdat, deto, dito_conf;
2256	int rc;
2257	unsigned int err_mask;
2258
2259	devslp = readl(port_mmio + PORT_DEVSLP);
2260	if (!(devslp & PORT_DEVSLP_DSP)) {
2261	dev_info(ap->host->dev, "port does not support device sleep\n");
2262	return;
2263	}
2264
2265	/* disable device sleep */
2266	if (!sleep) {
2267	if (devslp & PORT_DEVSLP_ADSE) {
2268	writel(devslp & ~PORT_DEVSLP_ADSE,
2269	port_mmio + PORT_DEVSLP);
2270	err_mask = ata_dev_set_feature(dev,
2271	SETFEATURES_SATA_DISABLE,
2272	SATA_DEVSLP);
2273	if (err_mask && err_mask != AC_ERR_DEV)
2274	ata_dev_warn(dev, "failed to disable DEVSLP\n");
2275	}
2276	return;
2277	}
2278
2279	dm = (devslp & PORT_DEVSLP_DM_MASK) >> PORT_DEVSLP_DM_OFFSET;
2280	dito = devslp_idle_timeout / (dm + 1);
2281	if (dito > 0x3ff)
2282	dito = 0x3ff;
2283
2284	dito_conf = (devslp >> PORT_DEVSLP_DITO_OFFSET) & 0x3FF;
2285
2286	/* device sleep was already enabled and same dito */
2287	if ((devslp & PORT_DEVSLP_ADSE) && (dito_conf == dito))
2288	return;
2289
2290	/* set DITO, MDAT, DETO and enable DevSlp, need to stop engine first */
2291	rc = hpriv->stop_engine(ap);
2292	if (rc)
2293	return;
2294
2295	/* Use the nominal value 10 ms if the read MDAT is zero,
2296	* the nominal value of DETO is 20 ms.
2297	*/
2298	if (dev->devslp_timing[ATA_LOG_DEVSLP_VALID] &
2299	ATA_LOG_DEVSLP_VALID_MASK) {
2300	mdat = dev->devslp_timing[ATA_LOG_DEVSLP_MDAT] &
2301	ATA_LOG_DEVSLP_MDAT_MASK;
2302	if (!mdat)
2303	mdat = 10;
2304	deto = dev->devslp_timing[ATA_LOG_DEVSLP_DETO];
2305	if (!deto)
2306	deto = 20;
2307	} else {
2308	mdat = 10;
2309	deto = 20;
2310	}
2311
2312	/* Make dito, mdat, deto bits to 0s */
2313	devslp &= ~GENMASK_ULL(24, 2);
2314	devslp \|= ((dito << PORT_DEVSLP_DITO_OFFSET) \|
2315	(mdat << PORT_DEVSLP_MDAT_OFFSET) \|
2316	(deto << PORT_DEVSLP_DETO_OFFSET) \|
2317	PORT_DEVSLP_ADSE);
2318	writel(devslp, port_mmio + PORT_DEVSLP);
2319
2320	hpriv->start_engine(ap);
2321
2322	/* enable device sleep feature for the drive */
2323	err_mask = ata_dev_set_feature(dev,
2324	SETFEATURES_SATA_ENABLE,
2325	SATA_DEVSLP);
2326	if (err_mask && err_mask != AC_ERR_DEV)
2327	ata_dev_warn(dev, "failed to enable DEVSLP\n");
2328	}
2329
2330	static void ahci_enable_fbs(struct ata_port *ap)
2331	{
2332	struct ahci_host_priv *hpriv = ap->host->private_data;
2333	struct ahci_port_priv *pp = ap->private_data;
2334	void __iomem *port_mmio = ahci_port_base(ap);
2335	u32 fbs;
2336	int rc;
2337
2338	if (!pp->fbs_supported)
2339	return;
2340
2341	fbs = readl(port_mmio + PORT_FBS);
2342	if (fbs & PORT_FBS_EN) {
2343	pp->fbs_enabled = true;
2344	pp->fbs_last_dev = -1; /* initialization */
2345	return;
2346	}
2347
2348	rc = hpriv->stop_engine(ap);
2349	if (rc)
2350	return;
2351
2352	writel(fbs \| PORT_FBS_EN, port_mmio + PORT_FBS);
2353	fbs = readl(port_mmio + PORT_FBS);
2354	if (fbs & PORT_FBS_EN) {
2355	dev_info(ap->host->dev, "FBS is enabled\n");
2356	pp->fbs_enabled = true;
2357	pp->fbs_last_dev = -1; /* initialization */
2358	} else
2359	dev_err(ap->host->dev, "Failed to enable FBS\n");
2360
2361	hpriv->start_engine(ap);
2362	}
2363
2364	static void ahci_disable_fbs(struct ata_port *ap)
2365	{
2366	struct ahci_host_priv *hpriv = ap->host->private_data;
2367	struct ahci_port_priv *pp = ap->private_data;
2368	void __iomem *port_mmio = ahci_port_base(ap);
2369	u32 fbs;
2370	int rc;
2371
2372	if (!pp->fbs_supported)
2373	return;
2374
2375	fbs = readl(port_mmio + PORT_FBS);
2376	if ((fbs & PORT_FBS_EN) == 0) {
2377	pp->fbs_enabled = false;
2378	return;
2379	}
2380
2381	rc = hpriv->stop_engine(ap);
2382	if (rc)
2383	return;
2384
2385	writel(fbs & ~PORT_FBS_EN, port_mmio + PORT_FBS);
2386	fbs = readl(port_mmio + PORT_FBS);
2387	if (fbs & PORT_FBS_EN)
2388	dev_err(ap->host->dev, "Failed to disable FBS\n");
2389	else {
2390	dev_info(ap->host->dev, "FBS is disabled\n");
2391	pp->fbs_enabled = false;
2392	}
2393
2394	hpriv->start_engine(ap);
2395	}
2396
2397	static void ahci_pmp_attach(struct ata_port *ap)
2398	{
2399	void __iomem *port_mmio = ahci_port_base(ap);
2400	struct ahci_port_priv *pp = ap->private_data;
2401	u32 cmd;
2402
2403	cmd = readl(port_mmio + PORT_CMD);
2404	cmd \|= PORT_CMD_PMP;
2405	writel(cmd, port_mmio + PORT_CMD);
2406
2407	ahci_enable_fbs(ap);
2408
2409	pp->intr_mask \|= PORT_IRQ_BAD_PMP;
2410
2411	/*
2412	* We must not change the port interrupt mask register if the
2413	* port is marked frozen, the value in pp->intr_mask will be
2414	* restored later when the port is thawed.
2415	*
2416	* Note that during initialization, the port is marked as
2417	* frozen since the irq handler is not yet registered.
2418	*/
2419	if (!ata_port_is_frozen(ap))
2420	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2421	}
2422
2423	static void ahci_pmp_detach(struct ata_port *ap)
2424	{
2425	void __iomem *port_mmio = ahci_port_base(ap);
2426	struct ahci_port_priv *pp = ap->private_data;
2427	u32 cmd;
2428
2429	ahci_disable_fbs(ap);
2430
2431	cmd = readl(port_mmio + PORT_CMD);
2432	cmd &= ~PORT_CMD_PMP;
2433	writel(cmd, port_mmio + PORT_CMD);
2434
2435	pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
2436
2437	/* see comment above in ahci_pmp_attach() */
2438	if (!ata_port_is_frozen(ap))
2439	writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
2440	}
2441
2442	int ahci_port_resume(struct ata_port *ap)
2443	{
2444	ahci_rpm_get_port(ap);
2445
2446	ahci_power_up(ap);
2447	ahci_start_port(ap);
2448
2449	if (sata_pmp_attached(ap))
2450	ahci_pmp_attach(ap);
2451	else
2452	ahci_pmp_detach(ap);
2453
2454	return 0;
2455	}
2456	EXPORT_SYMBOL_GPL(ahci_port_resume);
2457
2458	#ifdef CONFIG_PM
2459	static void ahci_handle_s2idle(struct ata_port *ap)
2460	{
2461	void __iomem *port_mmio = ahci_port_base(ap);
2462	u32 devslp;
2463
2464	if (pm_suspend_via_firmware())
2465	return;
2466	devslp = readl(port_mmio + PORT_DEVSLP);
2467	if ((devslp & PORT_DEVSLP_ADSE))
2468	ata_msleep(ap, devslp_idle_timeout);
2469	}
2470
2471	static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
2472	{
2473	const char *emsg = NULL;
2474	int rc;
2475
2476	rc = ahci_deinit_port(ap, &emsg);
2477	if (rc == 0)
2478	ahci_power_down(ap);
2479	else {
2480	ata_port_err(ap, "%s (%d)\n", emsg, rc);
2481	ata_port_freeze(ap);
2482	}
2483
2484	if (acpi_storage_d3(ap->host->dev))
2485	ahci_handle_s2idle(ap);
2486
2487	ahci_rpm_put_port(ap);
2488	return rc;
2489	}
2490	#endif
2491
2492	static int ahci_port_start(struct ata_port *ap)
2493	{
2494	struct ahci_host_priv *hpriv = ap->host->private_data;
2495	struct device *dev = ap->host->dev;
2496	struct ahci_port_priv *pp;
2497	void *mem;
2498	dma_addr_t mem_dma;
2499	size_t dma_sz, rx_fis_sz;
2500
2501	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2502	if (!pp)
2503	return -ENOMEM;
2504
2505	if (ap->host->n_ports > 1) {
2506	pp->irq_desc = devm_kzalloc(dev, 8, GFP_KERNEL);
2507	if (!pp->irq_desc) {
2508	devm_kfree(dev, pp);
2509	return -ENOMEM;
2510	}
2511	snprintf(pp->irq_desc, 8,
2512	"%s%d", dev_driver_string(dev), ap->port_no);
2513	}
2514
2515	/* check FBS capability */
2516	if ((hpriv->cap & HOST_CAP_FBS) && sata_pmp_supported(ap)) {
2517	void __iomem *port_mmio = ahci_port_base(ap);
2518	u32 cmd = readl(port_mmio + PORT_CMD);
2519	if (cmd & PORT_CMD_FBSCP)
2520	pp->fbs_supported = true;
2521	else if (hpriv->flags & AHCI_HFLAG_YES_FBS) {
2522	dev_info(dev, "port %d can do FBS, forcing FBSCP\n",
2523	ap->port_no);
2524	pp->fbs_supported = true;
2525	} else
2526	dev_warn(dev, "port %d is not capable of FBS\n",
2527	ap->port_no);
2528	}
2529
2530	if (pp->fbs_supported) {
2531	dma_sz = AHCI_PORT_PRIV_FBS_DMA_SZ;
2532	rx_fis_sz = AHCI_RX_FIS_SZ * 16;
2533	} else {
2534	dma_sz = AHCI_PORT_PRIV_DMA_SZ;
2535	rx_fis_sz = AHCI_RX_FIS_SZ;
2536	}
2537
2538	mem = dmam_alloc_coherent(dev, dma_sz, &mem_dma, GFP_KERNEL);
2539	if (!mem)
2540	return -ENOMEM;
2541
2542	/*
2543	* First item in chunk of DMA memory: 32-slot command table,
2544	* 32 bytes each in size
2545	*/
2546	pp->cmd_slot = mem;
2547	pp->cmd_slot_dma = mem_dma;
2548
2549	mem += AHCI_CMD_SLOT_SZ;
2550	mem_dma += AHCI_CMD_SLOT_SZ;
2551
2552	/*
2553	* Second item: Received-FIS area
2554	*/
2555	pp->rx_fis = mem;
2556	pp->rx_fis_dma = mem_dma;
2557
2558	mem += rx_fis_sz;
2559	mem_dma += rx_fis_sz;
2560
2561	/*
2562	* Third item: data area for storing a single command
2563	* and its scatter-gather table
2564	*/
2565	pp->cmd_tbl = mem;
2566	pp->cmd_tbl_dma = mem_dma;
2567
2568	/*
2569	* Save off initial list of interrupts to be enabled.
2570	* This could be changed later
2571	*/
2572	pp->intr_mask = DEF_PORT_IRQ;
2573
2574	/*
2575	* Switch to per-port locking in case each port has its own MSI vector.
2576	*/
2577	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
2578	spin_lock_init(&pp->lock);
2579	ap->lock = &pp->lock;
2580	}
2581
2582	ap->private_data = pp;
2583
2584	/* engage engines, captain */
2585	return ahci_port_resume(ap);
2586	}
2587
2588	static void ahci_port_stop(struct ata_port *ap)
2589	{
2590	const char *emsg = NULL;
2591	struct ahci_host_priv *hpriv = ap->host->private_data;
2592	void __iomem *host_mmio = hpriv->mmio;
2593	int rc;
2594
2595	/* de-initialize port */
2596	rc = ahci_deinit_port(ap, &emsg);
2597	if (rc)
2598	ata_port_warn(ap, "%s (%d)\n", emsg, rc);
2599
2600	/*
2601	* Clear GHC.IS to prevent stuck INTx after disabling MSI and
2602	* re-enabling INTx.
2603	*/
2604	writel(1 << ap->port_no, host_mmio + HOST_IRQ_STAT);
2605
2606	ahci_rpm_put_port(ap);
2607	}
2608
2609	void ahci_print_info(struct ata_host host, const char scc_s)
2610	{
2611	struct ahci_host_priv *hpriv = host->private_data;
2612	u32 vers, cap, cap2, impl, speed;
2613	const char *speed_s;
2614
2615	vers = hpriv->version;
2616	cap = hpriv->cap;
2617	cap2 = hpriv->cap2;
2618	impl = hpriv->port_map;
2619
2620	speed = (cap >> 20) & 0xf;
2621	if (speed == 1)
2622	speed_s = "1.5";
2623	else if (speed == 2)
2624	speed_s = "3";
2625	else if (speed == 3)
2626	speed_s = "6";
2627	else
2628	speed_s = "?";
2629
2630	dev_info(host->dev,
2631	"AHCI vers %02x%02x.%02x%02x, "
2632	"%u command slots, %s Gbps, %s mode\n"
2633	,
2634
2635	(vers >> 24) & 0xff,
2636	(vers >> 16) & 0xff,
2637	(vers >> 8) & 0xff,
2638	vers & 0xff,
2639
2640	((cap >> 8) & 0x1f) + 1,
2641	speed_s,
2642	scc_s);
2643
2644	dev_info(host->dev,
2645	"%u/%u ports implemented (port mask 0x%x)\n"
2646	,
2647
2648	hweight32(impl),
2649	ahci_nr_ports(cap),
2650	impl);
2651
2652	dev_info(host->dev,
2653	"flags: "
2654	"%s%s%s%s%s%s%s"
2655	"%s%s%s%s%s%s%s"
2656	"%s%s%s%s%s%s%s"
2657	"%s%s\n"
2658	,
2659
2660	cap & HOST_CAP_64 ? "64bit " : "",
2661	cap & HOST_CAP_NCQ ? "ncq " : "",
2662	cap & HOST_CAP_SNTF ? "sntf " : "",
2663	cap & HOST_CAP_MPS ? "ilck " : "",
2664	cap & HOST_CAP_SSS ? "stag " : "",
2665	cap & HOST_CAP_ALPM ? "pm " : "",
2666	cap & HOST_CAP_LED ? "led " : "",
2667	cap & HOST_CAP_CLO ? "clo " : "",
2668	cap & HOST_CAP_ONLY ? "only " : "",
2669	cap & HOST_CAP_PMP ? "pmp " : "",
2670	cap & HOST_CAP_FBS ? "fbs " : "",
2671	cap & HOST_CAP_PIO_MULTI ? "pio " : "",
2672	cap & HOST_CAP_SSC ? "slum " : "",
2673	cap & HOST_CAP_PART ? "part " : "",
2674	cap & HOST_CAP_CCC ? "ccc " : "",
2675	cap & HOST_CAP_EMS ? "ems " : "",
2676	cap & HOST_CAP_SXS ? "sxs " : "",
2677	cap2 & HOST_CAP2_DESO ? "deso " : "",
2678	cap2 & HOST_CAP2_SADM ? "sadm " : "",
2679	cap2 & HOST_CAP2_SDS ? "sds " : "",
2680	cap2 & HOST_CAP2_APST ? "apst " : "",
2681	cap2 & HOST_CAP2_NVMHCI ? "nvmp " : "",
2682	cap2 & HOST_CAP2_BOH ? "boh " : ""
2683	);
2684	}
2685	EXPORT_SYMBOL_GPL(ahci_print_info);
2686
2687	void ahci_set_em_messages(struct ahci_host_priv *hpriv,
2688	struct ata_port_info *pi)
2689	{
2690	u8 messages;
2691	void __iomem *mmio = hpriv->mmio;
2692	u32 em_loc = readl(mmio + HOST_EM_LOC);
2693	u32 em_ctl = readl(mmio + HOST_EM_CTL);
2694
2695	if (!ahci_em_messages \|\| !(hpriv->cap & HOST_CAP_EMS))
2696	return;
2697
2698	messages = (em_ctl & EM_CTRL_MSG_TYPE) >> 16;
2699
2700	if (messages) {
2701	/* store em_loc */
2702	hpriv->em_loc = ((em_loc >> 16) * 4);
2703	hpriv->em_buf_sz = ((em_loc & 0xff) * 4);
2704	hpriv->em_msg_type = messages;
2705	pi->flags \|= ATA_FLAG_EM;
2706	if (!(em_ctl & EM_CTL_ALHD))
2707	pi->flags \|= ATA_FLAG_SW_ACTIVITY;
2708	}
2709	}
2710	EXPORT_SYMBOL_GPL(ahci_set_em_messages);
2711
2712	static int ahci_host_activate_multi_irqs(struct ata_host *host,
2713	const struct scsi_host_template *sht)
2714	{
2715	struct ahci_host_priv *hpriv = host->private_data;
2716	int i, rc;
2717
2718	rc = ata_host_start(host);
2719	if (rc)
2720	return rc;
2721	/*
2722	* Requests IRQs according to AHCI-1.1 when multiple MSIs were
2723	* allocated. That is one MSI per port, starting from @irq.
2724	*/
2725	for (i = 0; i < host->n_ports; i++) {
2726	struct ahci_port_priv *pp = host->ports[i]->private_data;
2727	int irq = hpriv->get_irq_vector(host, i);
2728
2729	/* Do not receive interrupts sent by dummy ports */
2730	if (!pp) {
2731	disable_irq(irq);
2732	continue;
2733	}
2734
2735	rc = devm_request_irq(host->dev, irq, ahci_multi_irqs_intr_hard,
2736	0, pp->irq_desc, host->ports[i]);
2737
2738	if (rc)
2739	return rc;
2740	ata_port_desc_misc(host->ports[i], irq);
2741	}
2742
2743	return ata_host_register(host, sht);
2744	}
2745
2746	/**
2747	* ahci_host_activate - start AHCI host, request IRQs and register it
2748	* @host: target ATA host
2749	* @sht: scsi_host_template to use when registering the host
2750	*
2751	* LOCKING:
2752	* Inherited from calling layer (may sleep).
2753	*
2754	* RETURNS:
2755	* 0 on success, -errno otherwise.
2756	*/
2757	int ahci_host_activate(struct ata_host host, const struct scsi_host_template sht)
2758	{
2759	struct ahci_host_priv *hpriv = host->private_data;
2760	int irq = hpriv->irq;
2761	int rc;
2762
2763	if (hpriv->flags & AHCI_HFLAG_MULTI_MSI) {
2764	if (hpriv->irq_handler &&
2765	hpriv->irq_handler != ahci_single_level_irq_intr)
2766	dev_warn(host->dev,
2767	"both AHCI_HFLAG_MULTI_MSI flag set and custom irq handler implemented\n");
2768	if (!hpriv->get_irq_vector) {
2769	dev_err(host->dev,
2770	"AHCI_HFLAG_MULTI_MSI requires ->get_irq_vector!\n");
2771	return -EIO;
2772	}
2773
2774	rc = ahci_host_activate_multi_irqs(host, sht);
2775	} else {
2776	rc = ata_host_activate(host, irq, hpriv->irq_handler,
2777	IRQF_SHARED, sht);
2778	}
2779
2780
2781	return rc;
2782	}
2783	EXPORT_SYMBOL_GPL(ahci_host_activate);
2784
2785	MODULE_AUTHOR("Jeff Garzik");
2786	MODULE_DESCRIPTION("Common AHCI SATA low-level routines");
2787	MODULE_LICENSE("GPL");