projects/linux·blob master · refs

2506 lines · 67.2 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * sata_nv.c - NVIDIA nForce SATA
4 *
5 * Copyright 2004 NVIDIA Corp. All rights reserved.
6 * Copyright 2004 Andrew Chew
7 *
8 * libata documentation is available via 'make {ps|pdf}docs',
9 * as Documentation/driver-api/libata.rst
10 *
11 * No hardware documentation available outside of NVIDIA.
12 * This driver programs the NVIDIA SATA controller in a similar
13 * fashion as with other PCI IDE BMDMA controllers, with a few
14 * NV-specific details such as register offsets, SATA phy location,
15 * hotplug info, etc.
16 *
17 * CK804/MCP04 controllers support an alternate programming interface
18 * similar to the ADMA specification (with some modifications).
19 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
20 * sent through the legacy interface.
21 */
22
23#include <linux/kernel.h>
24#include <linux/module.h>
25#include <linux/gfp.h>
26#include <linux/pci.h>
27#include <linux/blkdev.h>
28#include <linux/delay.h>
29#include <linux/interrupt.h>
30#include <linux/device.h>
31#include <scsi/scsi_host.h>
32#include <scsi/scsi_device.h>
33#include <linux/libata.h>
34#include <trace/events/libata.h>
35
36#define DRV_NAME "sata_nv"
37#define DRV_VERSION "3.5"
38
39#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
40
41enum {
42 NV_MMIO_BAR = 5,
43
44 NV_PORTS = 2,
45 NV_PIO_MASK = ATA_PIO4,
46 NV_MWDMA_MASK = ATA_MWDMA2,
47 NV_UDMA_MASK = ATA_UDMA6,
48 NV_PORT0_SCR_REG_OFFSET = 0x00,
49 NV_PORT1_SCR_REG_OFFSET = 0x40,
50
51 /* INT_STATUS/ENABLE */
52 NV_INT_STATUS = 0x10,
53 NV_INT_ENABLE = 0x11,
54 NV_INT_STATUS_CK804 = 0x440,
55 NV_INT_ENABLE_CK804 = 0x441,
56
57 /* INT_STATUS/ENABLE bits */
58 NV_INT_DEV = 0x01,
59 NV_INT_PM = 0x02,
60 NV_INT_ADDED = 0x04,
61 NV_INT_REMOVED = 0x08,
62
63 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
64
65 NV_INT_ALL = 0x0f,
66 NV_INT_MASK = NV_INT_DEV |
67 NV_INT_ADDED | NV_INT_REMOVED,
68
69 /* INT_CONFIG */
70 NV_INT_CONFIG = 0x12,
71 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
72
73 // For PCI config register 20
74 NV_MCP_SATA_CFG_20 = 0x50,
75 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
76 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
77 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
78 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
79 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
80
81 NV_ADMA_MAX_CPBS = 32,
82 NV_ADMA_CPB_SZ = 128,
83 NV_ADMA_APRD_SZ = 16,
84 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
85 NV_ADMA_APRD_SZ,
86 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
87 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
88 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
89 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
90
91 /* BAR5 offset to ADMA general registers */
92 NV_ADMA_GEN = 0x400,
93 NV_ADMA_GEN_CTL = 0x00,
94 NV_ADMA_NOTIFIER_CLEAR = 0x30,
95
96 /* BAR5 offset to ADMA ports */
97 NV_ADMA_PORT = 0x480,
98
99 /* size of ADMA port register space */
100 NV_ADMA_PORT_SIZE = 0x100,
101
102 /* ADMA port registers */
103 NV_ADMA_CTL = 0x40,
104 NV_ADMA_CPB_COUNT = 0x42,
105 NV_ADMA_NEXT_CPB_IDX = 0x43,
106 NV_ADMA_STAT = 0x44,
107 NV_ADMA_CPB_BASE_LOW = 0x48,
108 NV_ADMA_CPB_BASE_HIGH = 0x4C,
109 NV_ADMA_APPEND = 0x50,
110 NV_ADMA_NOTIFIER = 0x68,
111 NV_ADMA_NOTIFIER_ERROR = 0x6C,
112
113 /* NV_ADMA_CTL register bits */
114 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
115 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
116 NV_ADMA_CTL_GO = (1 << 7),
117 NV_ADMA_CTL_AIEN = (1 << 8),
118 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
119 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
120
121 /* CPB response flag bits */
122 NV_CPB_RESP_DONE = (1 << 0),
123 NV_CPB_RESP_ATA_ERR = (1 << 3),
124 NV_CPB_RESP_CMD_ERR = (1 << 4),
125 NV_CPB_RESP_CPB_ERR = (1 << 7),
126
127 /* CPB control flag bits */
128 NV_CPB_CTL_CPB_VALID = (1 << 0),
129 NV_CPB_CTL_QUEUE = (1 << 1),
130 NV_CPB_CTL_APRD_VALID = (1 << 2),
131 NV_CPB_CTL_IEN = (1 << 3),
132 NV_CPB_CTL_FPDMA = (1 << 4),
133
134 /* APRD flags */
135 NV_APRD_WRITE = (1 << 1),
136 NV_APRD_END = (1 << 2),
137 NV_APRD_CONT = (1 << 3),
138
139 /* NV_ADMA_STAT flags */
140 NV_ADMA_STAT_TIMEOUT = (1 << 0),
141 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
142 NV_ADMA_STAT_HOTPLUG = (1 << 2),
143 NV_ADMA_STAT_CPBERR = (1 << 4),
144 NV_ADMA_STAT_SERROR = (1 << 5),
145 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
146 NV_ADMA_STAT_IDLE = (1 << 8),
147 NV_ADMA_STAT_LEGACY = (1 << 9),
148 NV_ADMA_STAT_STOPPED = (1 << 10),
149 NV_ADMA_STAT_DONE = (1 << 12),
150 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
151 NV_ADMA_STAT_TIMEOUT,
152
153 /* port flags */
154 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
155 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
156
157 /* MCP55 reg offset */
158 NV_CTL_MCP55 = 0x400,
159 NV_INT_STATUS_MCP55 = 0x440,
160 NV_INT_ENABLE_MCP55 = 0x444,
161 NV_NCQ_REG_MCP55 = 0x448,
162
163 /* MCP55 */
164 NV_INT_ALL_MCP55 = 0xffff,
165 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
166 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
167
168 /* SWNCQ ENABLE BITS*/
169 NV_CTL_PRI_SWNCQ = 0x02,
170 NV_CTL_SEC_SWNCQ = 0x04,
171
172 /* SW NCQ status bits*/
173 NV_SWNCQ_IRQ_DEV = (1 << 0),
174 NV_SWNCQ_IRQ_PM = (1 << 1),
175 NV_SWNCQ_IRQ_ADDED = (1 << 2),
176 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
177
178 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
179 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
180 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
181 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
182
183 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
184 NV_SWNCQ_IRQ_REMOVED,
185
186};
187
188/* ADMA Physical Region Descriptor - one SG segment */
189struct nv_adma_prd {
190 __le64 addr;
191 __le32 len;
192 u8 flags;
193 u8 packet_len;
194 __le16 reserved;
195};
196
197enum nv_adma_regbits {
198 CMDEND = (1 << 15), /* end of command list */
199 WNB = (1 << 14), /* wait-not-BSY */
200 IGN = (1 << 13), /* ignore this entry */
201 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
202 DA2 = (1 << (2 + 8)),
203 DA1 = (1 << (1 + 8)),
204 DA0 = (1 << (0 + 8)),
205};
206
207/* ADMA Command Parameter Block
208 The first 5 SG segments are stored inside the Command Parameter Block itself.
209 If there are more than 5 segments the remainder are stored in a separate
210 memory area indicated by next_aprd. */
211struct nv_adma_cpb {
212 u8 resp_flags; /* 0 */
213 u8 reserved1; /* 1 */
214 u8 ctl_flags; /* 2 */
215 /* len is length of taskfile in 64 bit words */
216 u8 len; /* 3 */
217 u8 tag; /* 4 */
218 u8 next_cpb_idx; /* 5 */
219 __le16 reserved2; /* 6-7 */
220 __le16 tf[12]; /* 8-31 */
221 struct nv_adma_prd aprd[5]; /* 32-111 */
222 __le64 next_aprd; /* 112-119 */
223 __le64 reserved3; /* 120-127 */
224};
225
226
227struct nv_adma_port_priv {
228 struct nv_adma_cpb *cpb;
229 dma_addr_t cpb_dma;
230 struct nv_adma_prd *aprd;
231 dma_addr_t aprd_dma;
232 void __iomem *ctl_block;
233 void __iomem *gen_block;
234 void __iomem *notifier_clear_block;
235 u64 adma_dma_mask;
236 u8 flags;
237 int last_issue_ncq;
238};
239
240struct nv_host_priv {
241 unsigned long type;
242};
243
244struct defer_queue {
245 u32 defer_bits;
246 unsigned int head;
247 unsigned int tail;
248 unsigned int tag[ATA_MAX_QUEUE];
249};
250
251enum ncq_saw_flag_list {
252 ncq_saw_d2h = (1U << 0),
253 ncq_saw_dmas = (1U << 1),
254 ncq_saw_sdb = (1U << 2),
255 ncq_saw_backout = (1U << 3),
256};
257
258struct nv_swncq_port_priv {
259 struct ata_bmdma_prd *prd; /* our SG list */
260 dma_addr_t prd_dma; /* and its DMA mapping */
261 void __iomem *sactive_block;
262 void __iomem *irq_block;
263 void __iomem *tag_block;
264 u32 qc_active;
265
266 unsigned int last_issue_tag;
267
268 /* fifo circular queue to store deferral command */
269 struct defer_queue defer_queue;
270
271 /* for NCQ interrupt analysis */
272 u32 dhfis_bits;
273 u32 dmafis_bits;
274 u32 sdbfis_bits;
275
276 unsigned int ncq_flags;
277};
278
279
280#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
281
282static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
283#ifdef CONFIG_PM_SLEEP
284static int nv_pci_device_resume(struct pci_dev *pdev);
285#endif
286static void nv_ck804_host_stop(struct ata_host *host);
287static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
288static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
289static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
290static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
291static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
292
293static int nv_hardreset(struct ata_link *link, unsigned int *class,
294 unsigned long deadline);
295static void nv_nf2_freeze(struct ata_port *ap);
296static void nv_nf2_thaw(struct ata_port *ap);
297static void nv_ck804_freeze(struct ata_port *ap);
298static void nv_ck804_thaw(struct ata_port *ap);
299static int nv_adma_sdev_configure(struct scsi_device *sdev,
300 struct queue_limits *lim);
301static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
302static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc);
303static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
304static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
305static void nv_adma_irq_clear(struct ata_port *ap);
306static int nv_adma_port_start(struct ata_port *ap);
307static void nv_adma_port_stop(struct ata_port *ap);
308#ifdef CONFIG_PM
309static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
310static int nv_adma_port_resume(struct ata_port *ap);
311#endif
312static void nv_adma_freeze(struct ata_port *ap);
313static void nv_adma_thaw(struct ata_port *ap);
314static void nv_adma_error_handler(struct ata_port *ap);
315static void nv_adma_host_stop(struct ata_host *host);
316static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
317static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
318
319static void nv_mcp55_thaw(struct ata_port *ap);
320static void nv_mcp55_freeze(struct ata_port *ap);
321static void nv_swncq_error_handler(struct ata_port *ap);
322static int nv_swncq_sdev_configure(struct scsi_device *sdev,
323 struct queue_limits *lim);
324static int nv_swncq_port_start(struct ata_port *ap);
325static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc);
326static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
327static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
328static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
329static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
330#ifdef CONFIG_PM
331static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
332static int nv_swncq_port_resume(struct ata_port *ap);
333#endif
334
335enum nv_host_type
336{
337 GENERIC,
338 NFORCE2,
339 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
340 CK804,
341 ADMA,
342 MCP5x,
343 SWNCQ,
344};
345
346static const struct pci_device_id nv_pci_tbl[] = {
347 {
348 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA),
349 .driver_data = NFORCE2,
350 }, {
351 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA),
352 .driver_data = NFORCE3,
353 }, {
354 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2),
355 .driver_data = NFORCE3,
356 }, {
357 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA),
358 .driver_data = CK804,
359 }, {
360 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2),
361 .driver_data = CK804,
362 }, {
363 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA),
364 .driver_data = CK804,
365 }, {
366 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2),
367 .driver_data = CK804,
368 }, {
369 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA),
370 .driver_data = MCP5x,
371 }, {
372 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2),
373 .driver_data = MCP5x,
374 }, {
375 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA),
376 .driver_data = MCP5x,
377 }, {
378 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2),
379 .driver_data = MCP5x,
380 }, {
381 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA),
382 .driver_data = GENERIC,
383 }, {
384 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2),
385 .driver_data = GENERIC,
386 }, {
387 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3),
388 .driver_data = GENERIC,
389 },
390 { } /* terminate list */
391};
392
393static struct pci_driver nv_pci_driver = {
394 .name = DRV_NAME,
395 .id_table = nv_pci_tbl,
396 .probe = nv_init_one,
397#ifdef CONFIG_PM_SLEEP
398 .suspend = ata_pci_device_suspend,
399 .resume = nv_pci_device_resume,
400#endif
401 .remove = ata_pci_remove_one,
402};
403
404static const struct scsi_host_template nv_sht = {
405 ATA_BMDMA_SHT(DRV_NAME),
406};
407
408static const struct scsi_host_template nv_adma_sht = {
409 __ATA_BASE_SHT(DRV_NAME),
410 .can_queue = NV_ADMA_MAX_CPBS,
411 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
412 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
413 .sdev_configure = nv_adma_sdev_configure,
414 .sdev_groups = ata_ncq_sdev_groups,
415 .change_queue_depth = ata_scsi_change_queue_depth,
416 .tag_alloc_policy_rr = true,
417};
418
419static const struct scsi_host_template nv_swncq_sht = {
420 __ATA_BASE_SHT(DRV_NAME),
421 .can_queue = ATA_MAX_QUEUE - 1,
422 .sg_tablesize = LIBATA_MAX_PRD,
423 .dma_boundary = ATA_DMA_BOUNDARY,
424 .sdev_configure = nv_swncq_sdev_configure,
425 .sdev_groups = ata_ncq_sdev_groups,
426 .change_queue_depth = ata_scsi_change_queue_depth,
427 .tag_alloc_policy_rr = true,
428};
429
430/*
431 * NV SATA controllers have various different problems with hardreset
432 * protocol depending on the specific controller and device.
433 *
434 * GENERIC:
435 *
436 * bko11195 reports that link doesn't come online after hardreset on
437 * generic nv's and there have been several other similar reports on
438 * linux-ide.
439 *
440 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
441 * softreset.
442 *
443 * NF2/3:
444 *
445 * bko3352 reports nf2/3 controllers can't determine device signature
446 * reliably after hardreset. The following thread reports detection
447 * failure on cold boot with the standard debouncing timing.
448 *
449 * http://thread.gmane.org/gmane.linux.ide/34098
450 *
451 * bko12176 reports that hardreset fails to bring up the link during
452 * boot on nf2.
453 *
454 * CK804:
455 *
456 * For initial probing after boot and hot plugging, hardreset mostly
457 * works fine on CK804 but curiously, reprobing on the initial port
458 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
459 * FIS in somewhat undeterministic way.
460 *
461 * SWNCQ:
462 *
463 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
464 * hardreset should be used and hardreset can't report proper
465 * signature, which suggests that mcp5x is closer to nf2 as long as
466 * reset quirkiness is concerned.
467 *
468 * bko12703 reports that boot probing fails for intel SSD with
469 * hardreset. Link fails to come online. Softreset works fine.
470 *
471 * The failures are varied but the following patterns seem true for
472 * all flavors.
473 *
474 * - Softreset during boot always works.
475 *
476 * - Hardreset during boot sometimes fails to bring up the link on
477 * certain comibnations and device signature acquisition is
478 * unreliable.
479 *
480 * - Hardreset is often necessary after hotplug.
481 *
482 * So, preferring softreset for boot probing and error handling (as
483 * hardreset might bring down the link) but using hardreset for
484 * post-boot probing should work around the above issues in most
485 * cases. Define nv_hardreset() which only kicks in for post-boot
486 * probing and use it for all variants.
487 */
488static struct ata_port_operations nv_generic_ops = {
489 .inherits = &ata_bmdma_port_ops,
490 .lost_interrupt = ATA_OP_NULL,
491 .scr_read = nv_scr_read,
492 .scr_write = nv_scr_write,
493 .reset.hardreset = nv_hardreset,
494};
495
496static struct ata_port_operations nv_nf2_ops = {
497 .inherits = &nv_generic_ops,
498 .freeze = nv_nf2_freeze,
499 .thaw = nv_nf2_thaw,
500};
501
502static struct ata_port_operations nv_ck804_ops = {
503 .inherits = &nv_generic_ops,
504 .freeze = nv_ck804_freeze,
505 .thaw = nv_ck804_thaw,
506 .host_stop = nv_ck804_host_stop,
507};
508
509static struct ata_port_operations nv_adma_ops = {
510 .inherits = &nv_ck804_ops,
511
512 .check_atapi_dma = nv_adma_check_atapi_dma,
513 .sff_tf_read = nv_adma_tf_read,
514 .qc_defer = ata_std_qc_defer,
515 .qc_prep = nv_adma_qc_prep,
516 .qc_issue = nv_adma_qc_issue,
517 .sff_irq_clear = nv_adma_irq_clear,
518
519 .freeze = nv_adma_freeze,
520 .thaw = nv_adma_thaw,
521 .error_handler = nv_adma_error_handler,
522 .post_internal_cmd = nv_adma_post_internal_cmd,
523
524 .port_start = nv_adma_port_start,
525 .port_stop = nv_adma_port_stop,
526#ifdef CONFIG_PM
527 .port_suspend = nv_adma_port_suspend,
528 .port_resume = nv_adma_port_resume,
529#endif
530 .host_stop = nv_adma_host_stop,
531};
532
533static struct ata_port_operations nv_swncq_ops = {
534 .inherits = &nv_generic_ops,
535
536 .qc_defer = ata_std_qc_defer,
537 .qc_prep = nv_swncq_qc_prep,
538 .qc_issue = nv_swncq_qc_issue,
539
540 .freeze = nv_mcp55_freeze,
541 .thaw = nv_mcp55_thaw,
542 .error_handler = nv_swncq_error_handler,
543
544#ifdef CONFIG_PM
545 .port_suspend = nv_swncq_port_suspend,
546 .port_resume = nv_swncq_port_resume,
547#endif
548 .port_start = nv_swncq_port_start,
549};
550
551struct nv_pi_priv {
552 irq_handler_t irq_handler;
553 const struct scsi_host_template *sht;
554};
555
556#define NV_PI_PRIV(_irq_handler, _sht) \
557 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
558
559static const struct ata_port_info nv_port_info[] = {
560 /* generic */
561 {
562 .flags = ATA_FLAG_SATA,
563 .pio_mask = NV_PIO_MASK,
564 .mwdma_mask = NV_MWDMA_MASK,
565 .udma_mask = NV_UDMA_MASK,
566 .port_ops = &nv_generic_ops,
567 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
568 },
569 /* nforce2/3 */
570 {
571 .flags = ATA_FLAG_SATA,
572 .pio_mask = NV_PIO_MASK,
573 .mwdma_mask = NV_MWDMA_MASK,
574 .udma_mask = NV_UDMA_MASK,
575 .port_ops = &nv_nf2_ops,
576 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
577 },
578 /* ck804 */
579 {
580 .flags = ATA_FLAG_SATA,
581 .pio_mask = NV_PIO_MASK,
582 .mwdma_mask = NV_MWDMA_MASK,
583 .udma_mask = NV_UDMA_MASK,
584 .port_ops = &nv_ck804_ops,
585 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
586 },
587 /* ADMA */
588 {
589 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
590 .pio_mask = NV_PIO_MASK,
591 .mwdma_mask = NV_MWDMA_MASK,
592 .udma_mask = NV_UDMA_MASK,
593 .port_ops = &nv_adma_ops,
594 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
595 },
596 /* MCP5x */
597 {
598 .flags = ATA_FLAG_SATA,
599 .pio_mask = NV_PIO_MASK,
600 .mwdma_mask = NV_MWDMA_MASK,
601 .udma_mask = NV_UDMA_MASK,
602 .port_ops = &nv_generic_ops,
603 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
604 },
605 /* SWNCQ */
606 {
607 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
608 .pio_mask = NV_PIO_MASK,
609 .mwdma_mask = NV_MWDMA_MASK,
610 .udma_mask = NV_UDMA_MASK,
611 .port_ops = &nv_swncq_ops,
612 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
613 },
614};
615
616MODULE_AUTHOR("NVIDIA");
617MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
618MODULE_LICENSE("GPL");
619MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
620MODULE_VERSION(DRV_VERSION);
621
622static bool adma_enabled;
623static bool swncq_enabled = true;
624static bool msi_enabled;
625
626static void nv_adma_register_mode(struct ata_port *ap)
627{
628 struct nv_adma_port_priv *pp = ap->private_data;
629 void __iomem *mmio = pp->ctl_block;
630 u16 tmp, status;
631 int count = 0;
632
633 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
634 return;
635
636 status = readw(mmio + NV_ADMA_STAT);
637 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
638 ndelay(50);
639 status = readw(mmio + NV_ADMA_STAT);
640 count++;
641 }
642 if (count == 20)
643 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
644 status);
645
646 tmp = readw(mmio + NV_ADMA_CTL);
647 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
648
649 count = 0;
650 status = readw(mmio + NV_ADMA_STAT);
651 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
652 ndelay(50);
653 status = readw(mmio + NV_ADMA_STAT);
654 count++;
655 }
656 if (count == 20)
657 ata_port_warn(ap,
658 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
659 status);
660
661 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
662}
663
664static void nv_adma_mode(struct ata_port *ap)
665{
666 struct nv_adma_port_priv *pp = ap->private_data;
667 void __iomem *mmio = pp->ctl_block;
668 u16 tmp, status;
669 int count = 0;
670
671 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
672 return;
673
674 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
675
676 tmp = readw(mmio + NV_ADMA_CTL);
677 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
678
679 status = readw(mmio + NV_ADMA_STAT);
680 while (((status & NV_ADMA_STAT_LEGACY) ||
681 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
682 ndelay(50);
683 status = readw(mmio + NV_ADMA_STAT);
684 count++;
685 }
686 if (count == 20)
687 ata_port_warn(ap,
688 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
689 status);
690
691 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
692}
693
694static int nv_adma_sdev_configure(struct scsi_device *sdev,
695 struct queue_limits *lim)
696{
697 struct ata_port *ap = ata_shost_to_port(sdev->host);
698 struct nv_adma_port_priv *pp = ap->private_data;
699 struct nv_adma_port_priv *port0, *port1;
700 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
701 unsigned long segment_boundary, flags;
702 unsigned short sg_tablesize;
703 int rc;
704 int adma_enable;
705 u32 current_reg, new_reg, config_mask;
706
707 rc = ata_scsi_sdev_configure(sdev, lim);
708
709 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
710 /* Not a proper libata device, ignore */
711 return rc;
712
713 spin_lock_irqsave(ap->lock, flags);
714
715 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
716 /*
717 * NVIDIA reports that ADMA mode does not support ATAPI commands.
718 * Therefore ATAPI commands are sent through the legacy interface.
719 * However, the legacy interface only supports 32-bit DMA.
720 * Restrict DMA parameters as required by the legacy interface
721 * when an ATAPI device is connected.
722 */
723 segment_boundary = ATA_DMA_BOUNDARY;
724 /* Subtract 1 since an extra entry may be needed for padding, see
725 libata-scsi.c */
726 sg_tablesize = LIBATA_MAX_PRD - 1;
727
728 /* Since the legacy DMA engine is in use, we need to disable ADMA
729 on the port. */
730 adma_enable = 0;
731 nv_adma_register_mode(ap);
732 } else {
733 segment_boundary = NV_ADMA_DMA_BOUNDARY;
734 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
735 adma_enable = 1;
736 }
737
738 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
739
740 if (ap->port_no == 1)
741 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
742 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
743 else
744 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
745 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
746
747 if (adma_enable) {
748 new_reg = current_reg | config_mask;
749 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
750 } else {
751 new_reg = current_reg & ~config_mask;
752 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
753 }
754
755 if (current_reg != new_reg)
756 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
757
758 port0 = ap->host->ports[0]->private_data;
759 port1 = ap->host->ports[1]->private_data;
760 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
761 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
762 /*
763 * We have to set the DMA mask to 32-bit if either port is in
764 * ATAPI mode, since they are on the same PCI device which is
765 * used for DMA mapping. If either SCSI device is not allocated
766 * yet, it's OK since that port will discover its correct
767 * setting when it does get allocated.
768 */
769 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
770 } else {
771 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
772 }
773
774 lim->seg_boundary_mask = segment_boundary;
775 lim->max_segments = sg_tablesize;
776 ata_port_info(ap,
777 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
778 (unsigned long long)*ap->host->dev->dma_mask,
779 segment_boundary, sg_tablesize);
780
781 spin_unlock_irqrestore(ap->lock, flags);
782
783 return rc;
784}
785
786static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
787{
788 struct nv_adma_port_priv *pp = qc->ap->private_data;
789 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
790}
791
792static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
793{
794 /* Other than when internal or pass-through commands are executed,
795 the only time this function will be called in ADMA mode will be
796 if a command fails. In the failure case we don't care about going
797 into register mode with ADMA commands pending, as the commands will
798 all shortly be aborted anyway. We assume that NCQ commands are not
799 issued via passthrough, which is the only way that switching into
800 ADMA mode could abort outstanding commands. */
801 nv_adma_register_mode(ap);
802
803 ata_sff_tf_read(ap, tf);
804}
805
806static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
807{
808 unsigned int idx = 0;
809
810 if (tf->flags & ATA_TFLAG_ISADDR) {
811 if (tf->flags & ATA_TFLAG_LBA48) {
812 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
813 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
815 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
816 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
818 } else
819 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
820
821 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
823 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
824 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
825 }
826
827 if (tf->flags & ATA_TFLAG_DEVICE)
828 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
829
830 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
831
832 while (idx < 12)
833 cpb[idx++] = cpu_to_le16(IGN);
834
835 return idx;
836}
837
838static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
839{
840 struct nv_adma_port_priv *pp = ap->private_data;
841 u8 flags = pp->cpb[cpb_num].resp_flags;
842
843 ata_port_dbg(ap, "CPB %d, flags=0x%x\n", cpb_num, flags);
844
845 if (unlikely((force_err ||
846 flags & (NV_CPB_RESP_ATA_ERR |
847 NV_CPB_RESP_CMD_ERR |
848 NV_CPB_RESP_CPB_ERR)))) {
849 struct ata_eh_info *ehi = &ap->link.eh_info;
850 int freeze = 0;
851
852 ata_ehi_clear_desc(ehi);
853 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
854 if (flags & NV_CPB_RESP_ATA_ERR) {
855 ata_ehi_push_desc(ehi, "ATA error");
856 ehi->err_mask |= AC_ERR_DEV;
857 } else if (flags & NV_CPB_RESP_CMD_ERR) {
858 ata_ehi_push_desc(ehi, "CMD error");
859 ehi->err_mask |= AC_ERR_DEV;
860 } else if (flags & NV_CPB_RESP_CPB_ERR) {
861 ata_ehi_push_desc(ehi, "CPB error");
862 ehi->err_mask |= AC_ERR_SYSTEM;
863 freeze = 1;
864 } else {
865 /* notifier error, but no error in CPB flags? */
866 ata_ehi_push_desc(ehi, "unknown");
867 ehi->err_mask |= AC_ERR_OTHER;
868 freeze = 1;
869 }
870 /* Kill all commands. EH will determine what actually failed. */
871 if (freeze)
872 ata_port_freeze(ap);
873 else
874 ata_port_abort(ap);
875 return -1;
876 }
877
878 if (likely(flags & NV_CPB_RESP_DONE))
879 return 1;
880 return 0;
881}
882
883static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
884{
885 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
886
887 /* freeze if hotplugged */
888 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
889 ata_port_freeze(ap);
890 return 1;
891 }
892
893 /* bail out if not our interrupt */
894 if (!(irq_stat & NV_INT_DEV))
895 return 0;
896
897 /* DEV interrupt w/ no active qc? */
898 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
899 ata_sff_check_status(ap);
900 return 1;
901 }
902
903 /* handle interrupt */
904 return ata_bmdma_port_intr(ap, qc);
905}
906
907static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
908{
909 struct ata_host *host = dev_instance;
910 int i, handled = 0;
911 u32 notifier_clears[2];
912
913 spin_lock(&host->lock);
914
915 for (i = 0; i < host->n_ports; i++) {
916 struct ata_port *ap = host->ports[i];
917 struct nv_adma_port_priv *pp = ap->private_data;
918 void __iomem *mmio = pp->ctl_block;
919 u16 status;
920 u32 gen_ctl;
921 u32 notifier, notifier_error;
922
923 notifier_clears[i] = 0;
924
925 /* if ADMA is disabled, use standard ata interrupt handler */
926 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
927 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
928 >> (NV_INT_PORT_SHIFT * i);
929 handled += nv_host_intr(ap, irq_stat);
930 continue;
931 }
932
933 /* if in ATA register mode, check for standard interrupts */
934 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
935 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
936 >> (NV_INT_PORT_SHIFT * i);
937 if (ata_tag_valid(ap->link.active_tag))
938 /** NV_INT_DEV indication seems unreliable
939 at times at least in ADMA mode. Force it
940 on always when a command is active, to
941 prevent losing interrupts. */
942 irq_stat |= NV_INT_DEV;
943 handled += nv_host_intr(ap, irq_stat);
944 }
945
946 notifier = readl(mmio + NV_ADMA_NOTIFIER);
947 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
948 notifier_clears[i] = notifier | notifier_error;
949
950 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
951
952 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
953 !notifier_error)
954 /* Nothing to do */
955 continue;
956
957 status = readw(mmio + NV_ADMA_STAT);
958
959 /*
960 * Clear status. Ensure the controller sees the
961 * clearing before we start looking at any of the CPB
962 * statuses, so that any CPB completions after this
963 * point in the handler will raise another interrupt.
964 */
965 writew(status, mmio + NV_ADMA_STAT);
966 readw(mmio + NV_ADMA_STAT); /* flush posted write */
967 rmb();
968
969 handled++; /* irq handled if we got here */
970
971 /* freeze if hotplugged or controller error */
972 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
973 NV_ADMA_STAT_HOTUNPLUG |
974 NV_ADMA_STAT_TIMEOUT |
975 NV_ADMA_STAT_SERROR))) {
976 struct ata_eh_info *ehi = &ap->link.eh_info;
977
978 ata_ehi_clear_desc(ehi);
979 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
980 if (status & NV_ADMA_STAT_TIMEOUT) {
981 ehi->err_mask |= AC_ERR_SYSTEM;
982 ata_ehi_push_desc(ehi, "timeout");
983 } else if (status & NV_ADMA_STAT_HOTPLUG) {
984 ata_ehi_hotplugged(ehi);
985 ata_ehi_push_desc(ehi, "hotplug");
986 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
987 ata_ehi_hotplugged(ehi);
988 ata_ehi_push_desc(ehi, "hot unplug");
989 } else if (status & NV_ADMA_STAT_SERROR) {
990 /* let EH analyze SError and figure out cause */
991 ata_ehi_push_desc(ehi, "SError");
992 } else
993 ata_ehi_push_desc(ehi, "unknown");
994 ata_port_freeze(ap);
995 continue;
996 }
997
998 if (status & (NV_ADMA_STAT_DONE |
999 NV_ADMA_STAT_CPBERR |
1000 NV_ADMA_STAT_CMD_COMPLETE)) {
1001 u32 check_commands = notifier_clears[i];
1002 u32 done_mask = 0;
1003 int pos, rc;
1004
1005 if (status & NV_ADMA_STAT_CPBERR) {
1006 /* check all active commands */
1007 if (ata_tag_valid(ap->link.active_tag))
1008 check_commands = 1 <<
1009 ap->link.active_tag;
1010 else
1011 check_commands = ap->link.sactive;
1012 }
1013
1014 /* check CPBs for completed commands */
1015 while ((pos = ffs(check_commands))) {
1016 pos--;
1017 rc = nv_adma_check_cpb(ap, pos,
1018 notifier_error & (1 << pos));
1019 if (rc > 0)
1020 done_mask |= 1 << pos;
1021 else if (unlikely(rc < 0))
1022 check_commands = 0;
1023 check_commands &= ~(1 << pos);
1024 }
1025 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
1026 }
1027 }
1028
1029 if (notifier_clears[0] || notifier_clears[1]) {
1030 /* Note: Both notifier clear registers must be written
1031 if either is set, even if one is zero, according to NVIDIA. */
1032 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1033 writel(notifier_clears[0], pp->notifier_clear_block);
1034 pp = host->ports[1]->private_data;
1035 writel(notifier_clears[1], pp->notifier_clear_block);
1036 }
1037
1038 spin_unlock(&host->lock);
1039
1040 return IRQ_RETVAL(handled);
1041}
1042
1043static void nv_adma_freeze(struct ata_port *ap)
1044{
1045 struct nv_adma_port_priv *pp = ap->private_data;
1046 void __iomem *mmio = pp->ctl_block;
1047 u16 tmp;
1048
1049 nv_ck804_freeze(ap);
1050
1051 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1052 return;
1053
1054 /* clear any outstanding CK804 notifications */
1055 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1056 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1057
1058 /* Disable interrupt */
1059 tmp = readw(mmio + NV_ADMA_CTL);
1060 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1061 mmio + NV_ADMA_CTL);
1062 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1063}
1064
1065static void nv_adma_thaw(struct ata_port *ap)
1066{
1067 struct nv_adma_port_priv *pp = ap->private_data;
1068 void __iomem *mmio = pp->ctl_block;
1069 u16 tmp;
1070
1071 nv_ck804_thaw(ap);
1072
1073 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1074 return;
1075
1076 /* Enable interrupt */
1077 tmp = readw(mmio + NV_ADMA_CTL);
1078 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1079 mmio + NV_ADMA_CTL);
1080 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1081}
1082
1083static void nv_adma_irq_clear(struct ata_port *ap)
1084{
1085 struct nv_adma_port_priv *pp = ap->private_data;
1086 void __iomem *mmio = pp->ctl_block;
1087 u32 notifier_clears[2];
1088
1089 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1090 ata_bmdma_irq_clear(ap);
1091 return;
1092 }
1093
1094 /* clear any outstanding CK804 notifications */
1095 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1096 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1097
1098 /* clear ADMA status */
1099 writew(0xffff, mmio + NV_ADMA_STAT);
1100
1101 /* clear notifiers - note both ports need to be written with
1102 something even though we are only clearing on one */
1103 if (ap->port_no == 0) {
1104 notifier_clears[0] = 0xFFFFFFFF;
1105 notifier_clears[1] = 0;
1106 } else {
1107 notifier_clears[0] = 0;
1108 notifier_clears[1] = 0xFFFFFFFF;
1109 }
1110 pp = ap->host->ports[0]->private_data;
1111 writel(notifier_clears[0], pp->notifier_clear_block);
1112 pp = ap->host->ports[1]->private_data;
1113 writel(notifier_clears[1], pp->notifier_clear_block);
1114}
1115
1116static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1117{
1118 struct nv_adma_port_priv *pp = qc->ap->private_data;
1119
1120 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1121 ata_bmdma_post_internal_cmd(qc);
1122}
1123
1124static int nv_adma_port_start(struct ata_port *ap)
1125{
1126 struct device *dev = ap->host->dev;
1127 struct nv_adma_port_priv *pp;
1128 int rc;
1129 void *mem;
1130 dma_addr_t mem_dma;
1131 void __iomem *mmio;
1132 struct pci_dev *pdev = to_pci_dev(dev);
1133 u16 tmp;
1134
1135 /*
1136 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1137 * pad buffers.
1138 */
1139 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1140 if (rc)
1141 return rc;
1142
1143 /* we might fallback to bmdma, allocate bmdma resources */
1144 rc = ata_bmdma_port_start(ap);
1145 if (rc)
1146 return rc;
1147
1148 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1149 if (!pp)
1150 return -ENOMEM;
1151
1152 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1153 ap->port_no * NV_ADMA_PORT_SIZE;
1154 pp->ctl_block = mmio;
1155 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1156 pp->notifier_clear_block = pp->gen_block +
1157 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1158
1159 /*
1160 * Now that the legacy PRD and padding buffer are allocated we can
1161 * raise the DMA mask to allocate the CPB/APRD table.
1162 */
1163 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1164
1165 pp->adma_dma_mask = *dev->dma_mask;
1166
1167 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1168 &mem_dma, GFP_KERNEL);
1169 if (!mem)
1170 return -ENOMEM;
1171
1172 /*
1173 * First item in chunk of DMA memory:
1174 * 128-byte command parameter block (CPB)
1175 * one for each command tag
1176 */
1177 pp->cpb = mem;
1178 pp->cpb_dma = mem_dma;
1179
1180 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1181 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1182
1183 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1184 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1185
1186 /*
1187 * Second item: block of ADMA_SGTBL_LEN s/g entries
1188 */
1189 pp->aprd = mem;
1190 pp->aprd_dma = mem_dma;
1191
1192 ap->private_data = pp;
1193
1194 /* clear any outstanding interrupt conditions */
1195 writew(0xffff, mmio + NV_ADMA_STAT);
1196
1197 /* initialize port variables */
1198 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1199
1200 /* clear CPB fetch count */
1201 writew(0, mmio + NV_ADMA_CPB_COUNT);
1202
1203 /* clear GO for register mode, enable interrupt */
1204 tmp = readw(mmio + NV_ADMA_CTL);
1205 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1206 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1207
1208 tmp = readw(mmio + NV_ADMA_CTL);
1209 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1210 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1211 udelay(1);
1212 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1213 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1214
1215 return 0;
1216}
1217
1218static void nv_adma_port_stop(struct ata_port *ap)
1219{
1220 struct nv_adma_port_priv *pp = ap->private_data;
1221 void __iomem *mmio = pp->ctl_block;
1222
1223 writew(0, mmio + NV_ADMA_CTL);
1224}
1225
1226#ifdef CONFIG_PM
1227static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1228{
1229 struct nv_adma_port_priv *pp = ap->private_data;
1230 void __iomem *mmio = pp->ctl_block;
1231
1232 /* Go to register mode - clears GO */
1233 nv_adma_register_mode(ap);
1234
1235 /* clear CPB fetch count */
1236 writew(0, mmio + NV_ADMA_CPB_COUNT);
1237
1238 /* disable interrupt, shut down port */
1239 writew(0, mmio + NV_ADMA_CTL);
1240
1241 return 0;
1242}
1243
1244static int nv_adma_port_resume(struct ata_port *ap)
1245{
1246 struct nv_adma_port_priv *pp = ap->private_data;
1247 void __iomem *mmio = pp->ctl_block;
1248 u16 tmp;
1249
1250 /* set CPB block location */
1251 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1252 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1253
1254 /* clear any outstanding interrupt conditions */
1255 writew(0xffff, mmio + NV_ADMA_STAT);
1256
1257 /* initialize port variables */
1258 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1259
1260 /* clear CPB fetch count */
1261 writew(0, mmio + NV_ADMA_CPB_COUNT);
1262
1263 /* clear GO for register mode, enable interrupt */
1264 tmp = readw(mmio + NV_ADMA_CTL);
1265 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1266 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1267
1268 tmp = readw(mmio + NV_ADMA_CTL);
1269 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1270 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1271 udelay(1);
1272 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1273 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1274
1275 return 0;
1276}
1277#endif
1278
1279static void nv_adma_setup_port(struct ata_port *ap)
1280{
1281 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1282 struct ata_ioports *ioport = &ap->ioaddr;
1283
1284 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1285
1286 ioport->cmd_addr = mmio;
1287 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1288 ioport->error_addr =
1289 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1290 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1291 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1292 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1293 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1294 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1295 ioport->status_addr =
1296 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1297 ioport->altstatus_addr =
1298 ioport->ctl_addr = mmio + 0x20;
1299}
1300
1301static int nv_adma_host_init(struct ata_host *host)
1302{
1303 struct pci_dev *pdev = to_pci_dev(host->dev);
1304 unsigned int i;
1305 u32 tmp32;
1306
1307 /* enable ADMA on the ports */
1308 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1309 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1310 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1311 NV_MCP_SATA_CFG_20_PORT1_EN |
1312 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1313
1314 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1315
1316 for (i = 0; i < host->n_ports; i++)
1317 nv_adma_setup_port(host->ports[i]);
1318
1319 return 0;
1320}
1321
1322static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1323 struct scatterlist *sg,
1324 int idx,
1325 struct nv_adma_prd *aprd)
1326{
1327 u8 flags = 0;
1328 if (qc->tf.flags & ATA_TFLAG_WRITE)
1329 flags |= NV_APRD_WRITE;
1330 if (idx == qc->n_elem - 1)
1331 flags |= NV_APRD_END;
1332 else if (idx != 4)
1333 flags |= NV_APRD_CONT;
1334
1335 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1336 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1337 aprd->flags = flags;
1338 aprd->packet_len = 0;
1339}
1340
1341static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1342{
1343 struct nv_adma_port_priv *pp = qc->ap->private_data;
1344 struct nv_adma_prd *aprd;
1345 struct scatterlist *sg;
1346 unsigned int si;
1347
1348 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1349 aprd = (si < 5) ? &cpb->aprd[si] :
1350 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
1351 nv_adma_fill_aprd(qc, sg, si, aprd);
1352 }
1353 if (si > 5)
1354 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
1355 else
1356 cpb->next_aprd = cpu_to_le64(0);
1357}
1358
1359static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1360{
1361 struct nv_adma_port_priv *pp = qc->ap->private_data;
1362
1363 /* ADMA engine can only be used for non-ATAPI DMA commands,
1364 or interrupt-driven no-data commands. */
1365 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1366 (qc->tf.flags & ATA_TFLAG_POLLING))
1367 return 1;
1368
1369 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1370 (qc->tf.protocol == ATA_PROT_NODATA))
1371 return 0;
1372
1373 return 1;
1374}
1375
1376static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc)
1377{
1378 struct nv_adma_port_priv *pp = qc->ap->private_data;
1379 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
1380 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1381 NV_CPB_CTL_IEN;
1382
1383 if (nv_adma_use_reg_mode(qc)) {
1384 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1385 (qc->flags & ATA_QCFLAG_DMAMAP));
1386 nv_adma_register_mode(qc->ap);
1387 ata_bmdma_qc_prep(qc);
1388 return AC_ERR_OK;
1389 }
1390
1391 cpb->resp_flags = NV_CPB_RESP_DONE;
1392 wmb();
1393 cpb->ctl_flags = 0;
1394 wmb();
1395
1396 cpb->len = 3;
1397 cpb->tag = qc->hw_tag;
1398 cpb->next_cpb_idx = 0;
1399
1400 /* turn on NCQ flags for NCQ commands */
1401 if (qc->tf.protocol == ATA_PROT_NCQ)
1402 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1403
1404 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1405
1406 if (qc->flags & ATA_QCFLAG_DMAMAP) {
1407 nv_adma_fill_sg(qc, cpb);
1408 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1409 } else
1410 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1411
1412 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1413 until we are finished filling in all of the contents */
1414 wmb();
1415 cpb->ctl_flags = ctl_flags;
1416 wmb();
1417 cpb->resp_flags = 0;
1418
1419 return AC_ERR_OK;
1420}
1421
1422static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1423{
1424 struct nv_adma_port_priv *pp = qc->ap->private_data;
1425 void __iomem *mmio = pp->ctl_block;
1426 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1427
1428 /* We can't handle result taskfile with NCQ commands, since
1429 retrieving the taskfile switches us out of ADMA mode and would abort
1430 existing commands. */
1431 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1432 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1433 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
1434 return AC_ERR_SYSTEM;
1435 }
1436
1437 if (nv_adma_use_reg_mode(qc)) {
1438 /* use ATA register mode */
1439 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1440 (qc->flags & ATA_QCFLAG_DMAMAP));
1441 nv_adma_register_mode(qc->ap);
1442 return ata_bmdma_qc_issue(qc);
1443 } else
1444 nv_adma_mode(qc->ap);
1445
1446 /* write append register, command tag in lower 8 bits
1447 and (number of cpbs to append -1) in top 8 bits */
1448 wmb();
1449
1450 if (curr_ncq != pp->last_issue_ncq) {
1451 /* Seems to need some delay before switching between NCQ and
1452 non-NCQ commands, else we get command timeouts and such. */
1453 udelay(20);
1454 pp->last_issue_ncq = curr_ncq;
1455 }
1456
1457 writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
1458
1459 return 0;
1460}
1461
1462static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1463{
1464 struct ata_host *host = dev_instance;
1465 unsigned int i;
1466 unsigned int handled = 0;
1467 unsigned long flags;
1468
1469 spin_lock_irqsave(&host->lock, flags);
1470
1471 for (i = 0; i < host->n_ports; i++) {
1472 struct ata_port *ap = host->ports[i];
1473 struct ata_queued_cmd *qc;
1474
1475 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1476 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
1477 handled += ata_bmdma_port_intr(ap, qc);
1478 } else {
1479 /*
1480 * No request pending? Clear interrupt status
1481 * anyway, in case there's one pending.
1482 */
1483 ap->ops->sff_check_status(ap);
1484 }
1485 }
1486
1487 spin_unlock_irqrestore(&host->lock, flags);
1488
1489 return IRQ_RETVAL(handled);
1490}
1491
1492static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1493{
1494 int i, handled = 0;
1495
1496 for (i = 0; i < host->n_ports; i++) {
1497 handled += nv_host_intr(host->ports[i], irq_stat);
1498 irq_stat >>= NV_INT_PORT_SHIFT;
1499 }
1500
1501 return IRQ_RETVAL(handled);
1502}
1503
1504static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1505{
1506 struct ata_host *host = dev_instance;
1507 u8 irq_stat;
1508 irqreturn_t ret;
1509
1510 spin_lock(&host->lock);
1511 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1512 ret = nv_do_interrupt(host, irq_stat);
1513 spin_unlock(&host->lock);
1514
1515 return ret;
1516}
1517
1518static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1519{
1520 struct ata_host *host = dev_instance;
1521 u8 irq_stat;
1522 irqreturn_t ret;
1523
1524 spin_lock(&host->lock);
1525 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1526 ret = nv_do_interrupt(host, irq_stat);
1527 spin_unlock(&host->lock);
1528
1529 return ret;
1530}
1531
1532static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1533{
1534 if (sc_reg > SCR_CONTROL)
1535 return -EINVAL;
1536
1537 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
1538 return 0;
1539}
1540
1541static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1542{
1543 if (sc_reg > SCR_CONTROL)
1544 return -EINVAL;
1545
1546 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
1547 return 0;
1548}
1549
1550static int nv_hardreset(struct ata_link *link, unsigned int *class,
1551 unsigned long deadline)
1552{
1553 struct ata_eh_context *ehc = &link->eh_context;
1554
1555 /* Do hardreset iff it's post-boot probing, please read the
1556 * comment above port ops for details.
1557 */
1558 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1559 !ata_dev_enabled(link->device))
1560 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1561 NULL, NULL);
1562 else {
1563 const unsigned int *timing = sata_ehc_deb_timing(ehc);
1564 int rc;
1565
1566 if (!(ehc->i.flags & ATA_EHI_QUIET))
1567 ata_link_info(link,
1568 "nv: skipping hardreset on occupied port\n");
1569
1570 /* make sure the link is online */
1571 rc = sata_link_resume(link, timing, deadline);
1572 /* whine about phy resume failure but proceed */
1573 if (rc && rc != -EOPNOTSUPP)
1574 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1575 rc);
1576 }
1577
1578 /* device signature acquisition is unreliable */
1579 return -EAGAIN;
1580}
1581
1582static void nv_nf2_freeze(struct ata_port *ap)
1583{
1584 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1585 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1586 u8 mask;
1587
1588 mask = ioread8(scr_addr + NV_INT_ENABLE);
1589 mask &= ~(NV_INT_ALL << shift);
1590 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1591}
1592
1593static void nv_nf2_thaw(struct ata_port *ap)
1594{
1595 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1596 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1597 u8 mask;
1598
1599 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1600
1601 mask = ioread8(scr_addr + NV_INT_ENABLE);
1602 mask |= (NV_INT_MASK << shift);
1603 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1604}
1605
1606static void nv_ck804_freeze(struct ata_port *ap)
1607{
1608 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1609 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1610 u8 mask;
1611
1612 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1613 mask &= ~(NV_INT_ALL << shift);
1614 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1615}
1616
1617static void nv_ck804_thaw(struct ata_port *ap)
1618{
1619 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1620 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1621 u8 mask;
1622
1623 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1624
1625 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1626 mask |= (NV_INT_MASK << shift);
1627 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1628}
1629
1630static void nv_mcp55_freeze(struct ata_port *ap)
1631{
1632 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1633 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1634 u32 mask;
1635
1636 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1637
1638 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1639 mask &= ~(NV_INT_ALL_MCP55 << shift);
1640 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1641}
1642
1643static void nv_mcp55_thaw(struct ata_port *ap)
1644{
1645 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1646 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1647 u32 mask;
1648
1649 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1650
1651 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1652 mask |= (NV_INT_MASK_MCP55 << shift);
1653 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1654}
1655
1656static void nv_adma_error_handler(struct ata_port *ap)
1657 __must_hold(&ap->host->eh_mutex)
1658{
1659 struct nv_adma_port_priv *pp = ap->private_data;
1660 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1661 void __iomem *mmio = pp->ctl_block;
1662 int i;
1663 u16 tmp;
1664
1665 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1666 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1667 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1668 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1669 u32 status = readw(mmio + NV_ADMA_STAT);
1670 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1671 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1672
1673 ata_port_err(ap,
1674 "EH in ADMA mode, notifier 0x%X "
1675 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1676 "next cpb count 0x%X next cpb idx 0x%x\n",
1677 notifier, notifier_error, gen_ctl, status,
1678 cpb_count, next_cpb_idx);
1679
1680 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1681 struct nv_adma_cpb *cpb = &pp->cpb[i];
1682 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1683 ap->link.sactive & (1 << i))
1684 ata_port_err(ap,
1685 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1686 i, cpb->ctl_flags, cpb->resp_flags);
1687 }
1688 }
1689
1690 /* Push us back into port register mode for error handling. */
1691 nv_adma_register_mode(ap);
1692
1693 /* Mark all of the CPBs as invalid to prevent them from
1694 being executed */
1695 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1696 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1697
1698 /* clear CPB fetch count */
1699 writew(0, mmio + NV_ADMA_CPB_COUNT);
1700
1701 /* Reset channel */
1702 tmp = readw(mmio + NV_ADMA_CTL);
1703 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1704 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1705 udelay(1);
1706 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1707 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1708 }
1709
1710 ata_bmdma_error_handler(ap);
1711}
1712
1713static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1714{
1715 struct nv_swncq_port_priv *pp = ap->private_data;
1716 struct defer_queue *dq = &pp->defer_queue;
1717
1718 /* queue is full */
1719 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1720 dq->defer_bits |= (1 << qc->hw_tag);
1721 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
1722}
1723
1724static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1725{
1726 struct nv_swncq_port_priv *pp = ap->private_data;
1727 struct defer_queue *dq = &pp->defer_queue;
1728 unsigned int tag;
1729
1730 if (dq->head == dq->tail) /* null queue */
1731 return NULL;
1732
1733 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1734 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1735 WARN_ON(!(dq->defer_bits & (1 << tag)));
1736 dq->defer_bits &= ~(1 << tag);
1737
1738 return ata_qc_from_tag(ap, tag);
1739}
1740
1741static void nv_swncq_fis_reinit(struct ata_port *ap)
1742{
1743 struct nv_swncq_port_priv *pp = ap->private_data;
1744
1745 pp->dhfis_bits = 0;
1746 pp->dmafis_bits = 0;
1747 pp->sdbfis_bits = 0;
1748 pp->ncq_flags = 0;
1749}
1750
1751static void nv_swncq_pp_reinit(struct ata_port *ap)
1752{
1753 struct nv_swncq_port_priv *pp = ap->private_data;
1754 struct defer_queue *dq = &pp->defer_queue;
1755
1756 dq->head = 0;
1757 dq->tail = 0;
1758 dq->defer_bits = 0;
1759 pp->qc_active = 0;
1760 pp->last_issue_tag = ATA_TAG_POISON;
1761 nv_swncq_fis_reinit(ap);
1762}
1763
1764static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1765{
1766 struct nv_swncq_port_priv *pp = ap->private_data;
1767
1768 writew(fis, pp->irq_block);
1769}
1770
1771static void __ata_bmdma_stop(struct ata_port *ap)
1772{
1773 struct ata_queued_cmd qc;
1774
1775 qc.ap = ap;
1776 ata_bmdma_stop(&qc);
1777}
1778
1779static void nv_swncq_ncq_stop(struct ata_port *ap)
1780{
1781 struct nv_swncq_port_priv *pp = ap->private_data;
1782 unsigned int i;
1783 u32 sactive;
1784 u32 done_mask;
1785
1786 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
1787 ap->qc_active, ap->link.sactive);
1788 ata_port_err(ap,
1789 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1790 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1791 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1792 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1793
1794 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1795 ap->ops->sff_check_status(ap),
1796 ioread8(ap->ioaddr.error_addr));
1797
1798 sactive = readl(pp->sactive_block);
1799 done_mask = pp->qc_active ^ sactive;
1800
1801 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
1802 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1803 u8 err = 0;
1804 if (pp->qc_active & (1 << i))
1805 err = 0;
1806 else if (done_mask & (1 << i))
1807 err = 1;
1808 else
1809 continue;
1810
1811 ata_port_err(ap,
1812 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1813 (pp->dhfis_bits >> i) & 0x1,
1814 (pp->dmafis_bits >> i) & 0x1,
1815 (pp->sdbfis_bits >> i) & 0x1,
1816 (sactive >> i) & 0x1,
1817 (err ? "error! tag doesn't exit" : " "));
1818 }
1819
1820 nv_swncq_pp_reinit(ap);
1821 ap->ops->sff_irq_clear(ap);
1822 __ata_bmdma_stop(ap);
1823 nv_swncq_irq_clear(ap, 0xffff);
1824}
1825
1826static void nv_swncq_error_handler(struct ata_port *ap)
1827 __must_hold(&ap->host->eh_mutex)
1828{
1829 struct ata_eh_context *ehc = &ap->link.eh_context;
1830
1831 if (ap->link.sactive) {
1832 nv_swncq_ncq_stop(ap);
1833 ehc->i.action |= ATA_EH_RESET;
1834 }
1835
1836 ata_bmdma_error_handler(ap);
1837}
1838
1839#ifdef CONFIG_PM
1840static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1841{
1842 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1843 u32 tmp;
1844
1845 /* clear irq */
1846 writel(~0, mmio + NV_INT_STATUS_MCP55);
1847
1848 /* disable irq */
1849 writel(0, mmio + NV_INT_ENABLE_MCP55);
1850
1851 /* disable swncq */
1852 tmp = readl(mmio + NV_CTL_MCP55);
1853 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1854 writel(tmp, mmio + NV_CTL_MCP55);
1855
1856 return 0;
1857}
1858
1859static int nv_swncq_port_resume(struct ata_port *ap)
1860{
1861 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1862 u32 tmp;
1863
1864 /* clear irq */
1865 writel(~0, mmio + NV_INT_STATUS_MCP55);
1866
1867 /* enable irq */
1868 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1869
1870 /* enable swncq */
1871 tmp = readl(mmio + NV_CTL_MCP55);
1872 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1873
1874 return 0;
1875}
1876#endif
1877
1878static void nv_swncq_host_init(struct ata_host *host)
1879{
1880 u32 tmp;
1881 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1882 struct pci_dev *pdev = to_pci_dev(host->dev);
1883 u8 regval;
1884
1885 /* disable ECO 398 */
1886 pci_read_config_byte(pdev, 0x7f, &regval);
1887 regval &= ~(1 << 7);
1888 pci_write_config_byte(pdev, 0x7f, regval);
1889
1890 /* enable swncq */
1891 tmp = readl(mmio + NV_CTL_MCP55);
1892 dev_dbg(&pdev->dev, "HOST_CTL:0x%X\n", tmp);
1893 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1894
1895 /* enable irq intr */
1896 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1897 dev_dbg(&pdev->dev, "HOST_ENABLE:0x%X\n", tmp);
1898 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1899
1900 /* clear port irq */
1901 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1902}
1903
1904static int nv_swncq_sdev_configure(struct scsi_device *sdev,
1905 struct queue_limits *lim)
1906{
1907 struct ata_port *ap = ata_shost_to_port(sdev->host);
1908 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1909 struct ata_device *dev;
1910 int rc;
1911 u8 rev;
1912 u8 check_maxtor = 0;
1913 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1914
1915 rc = ata_scsi_sdev_configure(sdev, lim);
1916 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1917 /* Not a proper libata device, ignore */
1918 return rc;
1919
1920 dev = &ap->link.device[sdev->id];
1921 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1922 return rc;
1923
1924 /* if MCP51 and Maxtor, then disable ncq */
1925 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1926 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1927 check_maxtor = 1;
1928
1929 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1930 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1931 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1932 pci_read_config_byte(pdev, 0x8, &rev);
1933 if (rev <= 0xa2)
1934 check_maxtor = 1;
1935 }
1936
1937 if (!check_maxtor)
1938 return rc;
1939
1940 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1941
1942 if (strncmp(model_num, "Maxtor", 6) == 0) {
1943 ata_scsi_change_queue_depth(sdev, 1);
1944 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1945 sdev->queue_depth);
1946 }
1947
1948 return rc;
1949}
1950
1951static int nv_swncq_port_start(struct ata_port *ap)
1952{
1953 struct device *dev = ap->host->dev;
1954 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1955 struct nv_swncq_port_priv *pp;
1956 int rc;
1957
1958 /* we might fallback to bmdma, allocate bmdma resources */
1959 rc = ata_bmdma_port_start(ap);
1960 if (rc)
1961 return rc;
1962
1963 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1964 if (!pp)
1965 return -ENOMEM;
1966
1967 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1968 &pp->prd_dma, GFP_KERNEL);
1969 if (!pp->prd)
1970 return -ENOMEM;
1971
1972 ap->private_data = pp;
1973 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1974 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1975 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1976
1977 return 0;
1978}
1979
1980static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1981{
1982 if (qc->tf.protocol != ATA_PROT_NCQ) {
1983 ata_bmdma_qc_prep(qc);
1984 return AC_ERR_OK;
1985 }
1986
1987 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1988 return AC_ERR_OK;
1989
1990 nv_swncq_fill_sg(qc);
1991
1992 return AC_ERR_OK;
1993}
1994
1995static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1996{
1997 struct ata_port *ap = qc->ap;
1998 struct scatterlist *sg;
1999 struct nv_swncq_port_priv *pp = ap->private_data;
2000 struct ata_bmdma_prd *prd;
2001 unsigned int si, idx;
2002
2003 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
2004
2005 idx = 0;
2006 for_each_sg(qc->sg, sg, qc->n_elem, si) {
2007 u32 addr, offset;
2008 u32 sg_len, len;
2009
2010 addr = (u32)sg_dma_address(sg);
2011 sg_len = sg_dma_len(sg);
2012
2013 while (sg_len) {
2014 offset = addr & 0xffff;
2015 len = sg_len;
2016 if ((offset + sg_len) > 0x10000)
2017 len = 0x10000 - offset;
2018
2019 prd[idx].addr = cpu_to_le32(addr);
2020 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2021
2022 idx++;
2023 sg_len -= len;
2024 addr += len;
2025 }
2026 }
2027
2028 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2029}
2030
2031static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2032 struct ata_queued_cmd *qc)
2033{
2034 struct nv_swncq_port_priv *pp = ap->private_data;
2035
2036 if (qc == NULL)
2037 return 0;
2038
2039 writel((1 << qc->hw_tag), pp->sactive_block);
2040 pp->last_issue_tag = qc->hw_tag;
2041 pp->dhfis_bits &= ~(1 << qc->hw_tag);
2042 pp->dmafis_bits &= ~(1 << qc->hw_tag);
2043 pp->qc_active |= (0x1 << qc->hw_tag);
2044
2045 trace_ata_tf_load(ap, &qc->tf);
2046 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2047 trace_ata_exec_command(ap, &qc->tf, qc->hw_tag);
2048 ap->ops->sff_exec_command(ap, &qc->tf);
2049
2050 return 0;
2051}
2052
2053static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2054{
2055 struct ata_port *ap = qc->ap;
2056 struct nv_swncq_port_priv *pp = ap->private_data;
2057
2058 if (qc->tf.protocol != ATA_PROT_NCQ)
2059 return ata_bmdma_qc_issue(qc);
2060
2061 if (!pp->qc_active)
2062 nv_swncq_issue_atacmd(ap, qc);
2063 else
2064 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2065
2066 return 0;
2067}
2068
2069static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2070{
2071 u32 serror;
2072 struct ata_eh_info *ehi = &ap->link.eh_info;
2073
2074 ata_ehi_clear_desc(ehi);
2075
2076 /* AHCI needs SError cleared; otherwise, it might lock up */
2077 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2078 sata_scr_write(&ap->link, SCR_ERROR, serror);
2079
2080 /* analyze @irq_stat */
2081 if (fis & NV_SWNCQ_IRQ_ADDED)
2082 ata_ehi_push_desc(ehi, "hot plug");
2083 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2084 ata_ehi_push_desc(ehi, "hot unplug");
2085
2086 ata_ehi_hotplugged(ehi);
2087
2088 /* okay, let's hand over to EH */
2089 ehi->serror |= serror;
2090
2091 ata_port_freeze(ap);
2092}
2093
2094static int nv_swncq_sdbfis(struct ata_port *ap)
2095{
2096 struct ata_queued_cmd *qc;
2097 struct nv_swncq_port_priv *pp = ap->private_data;
2098 struct ata_eh_info *ehi = &ap->link.eh_info;
2099 u32 sactive;
2100 u32 done_mask;
2101 u8 host_stat;
2102 u8 lack_dhfis = 0;
2103
2104 host_stat = ap->ops->bmdma_status(ap);
2105 trace_ata_bmdma_status(ap, host_stat);
2106 if (unlikely(host_stat & ATA_DMA_ERR)) {
2107 /* error when transferring data to/from memory */
2108 ata_ehi_clear_desc(ehi);
2109 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2110 ehi->err_mask |= AC_ERR_HOST_BUS;
2111 ehi->action |= ATA_EH_RESET;
2112 return -EINVAL;
2113 }
2114
2115 ap->ops->sff_irq_clear(ap);
2116 __ata_bmdma_stop(ap);
2117
2118 sactive = readl(pp->sactive_block);
2119 done_mask = pp->qc_active ^ sactive;
2120
2121 pp->qc_active &= ~done_mask;
2122 pp->dhfis_bits &= ~done_mask;
2123 pp->dmafis_bits &= ~done_mask;
2124 pp->sdbfis_bits |= done_mask;
2125 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
2126
2127 if (!ap->qc_active) {
2128 ata_port_dbg(ap, "over\n");
2129 nv_swncq_pp_reinit(ap);
2130 return 0;
2131 }
2132
2133 if (pp->qc_active & pp->dhfis_bits)
2134 return 0;
2135
2136 if ((pp->ncq_flags & ncq_saw_backout) ||
2137 (pp->qc_active ^ pp->dhfis_bits))
2138 /* if the controller can't get a device to host register FIS,
2139 * The driver needs to reissue the new command.
2140 */
2141 lack_dhfis = 1;
2142
2143 ata_port_dbg(ap, "QC: qc_active 0x%llx,"
2144 "SWNCQ:qc_active 0x%X defer_bits %X "
2145 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2146 ap->qc_active, pp->qc_active,
2147 pp->defer_queue.defer_bits, pp->dhfis_bits,
2148 pp->dmafis_bits, pp->last_issue_tag);
2149
2150 nv_swncq_fis_reinit(ap);
2151
2152 if (lack_dhfis) {
2153 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2154 nv_swncq_issue_atacmd(ap, qc);
2155 return 0;
2156 }
2157
2158 if (pp->defer_queue.defer_bits) {
2159 /* send deferral queue command */
2160 qc = nv_swncq_qc_from_dq(ap);
2161 WARN_ON(qc == NULL);
2162 nv_swncq_issue_atacmd(ap, qc);
2163 }
2164
2165 return 0;
2166}
2167
2168static inline u32 nv_swncq_tag(struct ata_port *ap)
2169{
2170 struct nv_swncq_port_priv *pp = ap->private_data;
2171 u32 tag;
2172
2173 tag = readb(pp->tag_block) >> 2;
2174 return (tag & 0x1f);
2175}
2176
2177static void nv_swncq_dmafis(struct ata_port *ap)
2178{
2179 struct ata_queued_cmd *qc;
2180 unsigned int rw;
2181 u8 dmactl;
2182 u32 tag;
2183 struct nv_swncq_port_priv *pp = ap->private_data;
2184
2185 __ata_bmdma_stop(ap);
2186 tag = nv_swncq_tag(ap);
2187
2188 ata_port_dbg(ap, "dma setup tag 0x%x\n", tag);
2189 qc = ata_qc_from_tag(ap, tag);
2190
2191 if (unlikely(!qc))
2192 return;
2193
2194 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2195
2196 /* load PRD table addr. */
2197 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
2198 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2199
2200 /* specify data direction, triple-check start bit is clear */
2201 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2202 dmactl &= ~ATA_DMA_WR;
2203 if (!rw)
2204 dmactl |= ATA_DMA_WR;
2205
2206 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2207}
2208
2209static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2210{
2211 struct nv_swncq_port_priv *pp = ap->private_data;
2212 struct ata_queued_cmd *qc;
2213 struct ata_eh_info *ehi = &ap->link.eh_info;
2214 u32 serror;
2215 u8 ata_stat;
2216
2217 ata_stat = ap->ops->sff_check_status(ap);
2218 nv_swncq_irq_clear(ap, fis);
2219 if (!fis)
2220 return;
2221
2222 if (ata_port_is_frozen(ap))
2223 return;
2224
2225 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2226 nv_swncq_hotplug(ap, fis);
2227 return;
2228 }
2229
2230 if (!pp->qc_active)
2231 return;
2232
2233 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
2234 return;
2235 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
2236
2237 if (ata_stat & ATA_ERR) {
2238 ata_ehi_clear_desc(ehi);
2239 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2240 ehi->err_mask |= AC_ERR_DEV;
2241 ehi->serror |= serror;
2242 ehi->action |= ATA_EH_RESET;
2243 ata_port_freeze(ap);
2244 return;
2245 }
2246
2247 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2248 /* If the IRQ is backout, driver must issue
2249 * the new command again some time later.
2250 */
2251 pp->ncq_flags |= ncq_saw_backout;
2252 }
2253
2254 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2255 pp->ncq_flags |= ncq_saw_sdb;
2256 ata_port_dbg(ap, "SWNCQ: qc_active 0x%X "
2257 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2258 pp->qc_active, pp->dhfis_bits,
2259 pp->dmafis_bits, readl(pp->sactive_block));
2260 if (nv_swncq_sdbfis(ap) < 0)
2261 goto irq_error;
2262 }
2263
2264 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2265 /* The interrupt indicates the new command
2266 * was transmitted correctly to the drive.
2267 */
2268 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2269 pp->ncq_flags |= ncq_saw_d2h;
2270 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2271 ata_ehi_push_desc(ehi, "illegal fis transaction");
2272 ehi->err_mask |= AC_ERR_HSM;
2273 ehi->action |= ATA_EH_RESET;
2274 goto irq_error;
2275 }
2276
2277 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2278 !(pp->ncq_flags & ncq_saw_dmas)) {
2279 ata_stat = ap->ops->sff_check_status(ap);
2280 if (ata_stat & ATA_BUSY)
2281 goto irq_exit;
2282
2283 if (pp->defer_queue.defer_bits) {
2284 ata_port_dbg(ap, "send next command\n");
2285 qc = nv_swncq_qc_from_dq(ap);
2286 nv_swncq_issue_atacmd(ap, qc);
2287 }
2288 }
2289 }
2290
2291 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2292 /* program the dma controller with appropriate PRD buffers
2293 * and start the DMA transfer for requested command.
2294 */
2295 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2296 pp->ncq_flags |= ncq_saw_dmas;
2297 nv_swncq_dmafis(ap);
2298 }
2299
2300irq_exit:
2301 return;
2302irq_error:
2303 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2304 ata_port_freeze(ap);
2305 return;
2306}
2307
2308static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2309{
2310 struct ata_host *host = dev_instance;
2311 unsigned int i;
2312 unsigned int handled = 0;
2313 unsigned long flags;
2314 u32 irq_stat;
2315
2316 spin_lock_irqsave(&host->lock, flags);
2317
2318 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2319
2320 for (i = 0; i < host->n_ports; i++) {
2321 struct ata_port *ap = host->ports[i];
2322
2323 if (ap->link.sactive) {
2324 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2325 handled = 1;
2326 } else {
2327 if (irq_stat) /* reserve Hotplug */
2328 nv_swncq_irq_clear(ap, 0xfff0);
2329
2330 handled += nv_host_intr(ap, (u8)irq_stat);
2331 }
2332 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2333 }
2334
2335 spin_unlock_irqrestore(&host->lock, flags);
2336
2337 return IRQ_RETVAL(handled);
2338}
2339
2340static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2341{
2342 const struct ata_port_info *ppi[] = { NULL, NULL };
2343 struct nv_pi_priv *ipriv;
2344 struct ata_host *host;
2345 struct nv_host_priv *hpriv;
2346 int rc;
2347 u32 bar;
2348 void __iomem *base;
2349 unsigned long type = ent->driver_data;
2350
2351 // Make sure this is a SATA controller by counting the number of bars
2352 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2353 // it's an IDE controller and we ignore it.
2354 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
2355 if (pci_resource_start(pdev, bar) == 0)
2356 return -ENODEV;
2357
2358 ata_print_version_once(&pdev->dev, DRV_VERSION);
2359
2360 rc = pcim_enable_device(pdev);
2361 if (rc)
2362 return rc;
2363
2364 /* determine type and allocate host */
2365 if (type == CK804 && adma_enabled) {
2366 dev_notice(&pdev->dev, "Using ADMA mode\n");
2367 type = ADMA;
2368 } else if (type == MCP5x && swncq_enabled) {
2369 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2370 type = SWNCQ;
2371 }
2372
2373 ppi[0] = &nv_port_info[type];
2374 ipriv = ppi[0]->private_data;
2375 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
2376 if (rc)
2377 return rc;
2378
2379 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2380 if (!hpriv)
2381 return -ENOMEM;
2382 hpriv->type = type;
2383 host->private_data = hpriv;
2384
2385 /* request and iomap NV_MMIO_BAR */
2386 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2387 if (rc)
2388 return rc;
2389
2390 /* configure SCR access */
2391 base = host->iomap[NV_MMIO_BAR];
2392 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2393 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2394
2395 /* enable SATA space for CK804 */
2396 if (type >= CK804) {
2397 u8 regval;
2398
2399 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2400 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2401 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2402 }
2403
2404 /* init ADMA */
2405 if (type == ADMA) {
2406 rc = nv_adma_host_init(host);
2407 if (rc)
2408 return rc;
2409 } else if (type == SWNCQ)
2410 nv_swncq_host_init(host);
2411
2412 if (msi_enabled) {
2413 dev_notice(&pdev->dev, "Using MSI\n");
2414 pci_enable_msi(pdev);
2415 }
2416
2417 pci_set_master(pdev);
2418 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
2419}
2420
2421#ifdef CONFIG_PM_SLEEP
2422static int nv_pci_device_resume(struct pci_dev *pdev)
2423{
2424 struct ata_host *host = pci_get_drvdata(pdev);
2425 struct nv_host_priv *hpriv = host->private_data;
2426 int rc;
2427
2428 rc = ata_pci_device_do_resume(pdev);
2429 if (rc)
2430 return rc;
2431
2432 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2433 if (hpriv->type >= CK804) {
2434 u8 regval;
2435
2436 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2437 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2438 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2439 }
2440 if (hpriv->type == ADMA) {
2441 u32 tmp32;
2442 struct nv_adma_port_priv *pp;
2443 /* enable/disable ADMA on the ports appropriately */
2444 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2445
2446 pp = host->ports[0]->private_data;
2447 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2448 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2449 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2450 else
2451 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2452 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2453 pp = host->ports[1]->private_data;
2454 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2455 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2456 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2457 else
2458 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2459 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2460
2461 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2462 }
2463 }
2464
2465 ata_host_resume(host);
2466
2467 return 0;
2468}
2469#endif
2470
2471static void nv_ck804_host_stop(struct ata_host *host)
2472{
2473 struct pci_dev *pdev = to_pci_dev(host->dev);
2474 u8 regval;
2475
2476 /* disable SATA space for CK804 */
2477 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2478 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2479 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2480}
2481
2482static void nv_adma_host_stop(struct ata_host *host)
2483{
2484 struct pci_dev *pdev = to_pci_dev(host->dev);
2485 u32 tmp32;
2486
2487 /* disable ADMA on the ports */
2488 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2489 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2490 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2491 NV_MCP_SATA_CFG_20_PORT1_EN |
2492 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2493
2494 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2495
2496 nv_ck804_host_stop(host);
2497}
2498
2499module_pci_driver(nv_pci_driver);
2500
2501module_param_named(adma, adma_enabled, bool, 0444);
2502MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
2503module_param_named(swncq, swncq_enabled, bool, 0444);
2504MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
2505module_param_named(msi, msi_enabled, bool, 0444);
2506MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");