projects/linux·blob master · refs

3361 lines · 92.3 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 *
4 * Bluetooth support for Intel PCIe devices
5 *
6 * Copyright (C) 2024 Intel Corporation
7 */
8
9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/firmware.h>
12#include <linux/overflow.h>
13#include <linux/pci.h>
14#include <linux/string.h>
15#include <linux/wait.h>
16#include <linux/delay.h>
17#include <linux/interrupt.h>
18#include <linux/acpi.h>
19
20#include <linux/unaligned.h>
21#include <linux/devcoredump.h>
22
23#include <net/bluetooth/bluetooth.h>
24#include <net/bluetooth/hci_core.h>
25#include <net/bluetooth/hci_drv.h>
26
27#include "btintel.h"
28#include "btintel_pcie.h"
29
30#define VERSION "0.1"
31
32#define BTINTEL_PCI_DEVICE(dev, subdev) \
33 .vendor = PCI_VENDOR_ID_INTEL, \
34 .device = (dev), \
35 .subvendor = PCI_ANY_ID, \
36 .subdevice = (subdev), \
37 .driver_data = 0
38
39#define POLL_INTERVAL_US 10
40
41#define BTINTEL_PCIE_DMA_ALIGN_128B 128 /* 128 byte aligned */
42
43/* Intel Bluetooth PCIe device id table */
44static const struct pci_device_id btintel_pcie_table[] = {
45 /* BlazarI, Wildcat Lake */
46 { BTINTEL_PCI_DEVICE(0x4D76, PCI_ANY_ID) },
47 /* BlazarI, Lunar Lake */
48 { BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) },
49 /* Scorpious, Panther Lake-H484 */
50 { BTINTEL_PCI_DEVICE(0xE376, PCI_ANY_ID) },
51 /* Scorpious, Panther Lake-H404 */
52 { BTINTEL_PCI_DEVICE(0xE476, PCI_ANY_ID) },
53 /* Scorpious2, Nova Lake-PCD-H */
54 { BTINTEL_PCI_DEVICE(0xD346, PCI_ANY_ID) },
55 /* Scorpious2, Nova Lake-PCD-S */
56 { BTINTEL_PCI_DEVICE(0x6E74, PCI_ANY_ID) },
57 { 0 }
58};
59MODULE_DEVICE_TABLE(pci, btintel_pcie_table);
60
61struct btintel_pcie_dev_recovery {
62 struct list_head list;
63 u8 count;
64 time64_t last_error;
65 char name[];
66};
67
68/* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */
69#define BTINTEL_PCIE_HCI_TYPE_LEN 4
70#define BTINTEL_PCIE_HCI_CMD_PKT 0x00000001
71#define BTINTEL_PCIE_HCI_ACL_PKT 0x00000002
72#define BTINTEL_PCIE_HCI_SCO_PKT 0x00000003
73#define BTINTEL_PCIE_HCI_EVT_PKT 0x00000004
74#define BTINTEL_PCIE_HCI_ISO_PKT 0x00000005
75
76#define BTINTEL_PCIE_MAGIC_NUM 0xA5A5A5A5
77
78#define BTINTEL_PCIE_BLZR_HWEXP_SIZE 1024
79#define BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR 0xB00A7C00
80
81#define BTINTEL_PCIE_SCP_HWEXP_SIZE 4096
82#define BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR 0xB030F800
83
84#define BTINTEL_PCIE_SCP2_HWEXP_SIZE 4096
85#define BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR 0xB031D000
86
87#define BTINTEL_PCIE_MAGIC_NUM 0xA5A5A5A5
88
89#define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER 0x17A2
90#define BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT 0x1E61
91
92#define BTINTEL_PCIE_RESET_WINDOW_SECS 5
93#define BTINTEL_PCIE_FLR_MAX_RETRY 1
94
95/* Alive interrupt context */
96enum {
97 BTINTEL_PCIE_ROM,
98 BTINTEL_PCIE_FW_DL,
99 BTINTEL_PCIE_HCI_RESET,
100 BTINTEL_PCIE_INTEL_HCI_RESET1,
101 BTINTEL_PCIE_INTEL_HCI_RESET2,
102 BTINTEL_PCIE_D0,
103 BTINTEL_PCIE_D3
104};
105
106enum {
107 BTINTEL_PCIE_DSM_SET_RESET_TIMING = 1,
108 BTINTEL_PCIE_DSM_GET_RESET_TIMING = 2,
109 BTINTEL_PCIE_DSM_BT_PLDR_CONFIG = 3,
110 BTINTEL_PCIE_DSM_GET_RESET_TYPE = 4,
111 BTINTEL_PCIE_DSM_DYNAMIC_PLDR = 5,
112 BTINTEL_PCIE_DSM_GET_RESET_METHOD = 6,
113 BTINTEL_PCIE_DSM_SET_PLDR_DELAY = 7,
114};
115
116enum btintel_dsm_internal_product_reset_mode {
117 BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET = BIT(0),
118 BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR = BIT(1),
119 BTINTEL_PCIE_DSM_PLDR_MODE_EN_BT_OFF_ON = BIT(2),
120};
121
122/* Structure for dbgc fragment buffer
123 * @buf_addr_lsb: LSB of the buffer's physical address
124 * @buf_addr_msb: MSB of the buffer's physical address
125 * @buf_size: Total size of the buffer
126 */
127struct btintel_pcie_dbgc_ctxt_buf {
128 u32 buf_addr_lsb;
129 u32 buf_addr_msb;
130 u32 buf_size;
131};
132
133/* Structure for dbgc fragment
134 * @magic_num: 0XA5A5A5A5
135 * @ver: For Driver-FW compatibility
136 * @total_size: Total size of the payload debug info
137 * @num_buf: Num of allocated debug bufs
138 * @bufs: All buffer's addresses and sizes
139 */
140struct btintel_pcie_dbgc_ctxt {
141 u32 magic_num;
142 u32 ver;
143 u32 total_size;
144 u32 num_buf;
145 struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
146};
147
148struct btintel_pcie_trigger_evt {
149 u8 type;
150 u8 len;
151 __le32 addr;
152 __le32 size;
153} __packed;
154
155struct btintel_pcie_fwtrigger_evt {
156 __le32 reserved;
157 u8 type; /* Debug Trigger event */
158 __le16 len;
159 u8 event_type;
160 __le16 event_id;
161 __le16 reserved2;
162} __packed;
163
164static LIST_HEAD(btintel_pcie_recovery_list);
165static DEFINE_SPINLOCK(btintel_pcie_recovery_lock);
166
167static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt)
168{
169 switch (alive_intr_ctxt) {
170 case BTINTEL_PCIE_ROM:
171 return "rom";
172 case BTINTEL_PCIE_FW_DL:
173 return "fw_dl";
174 case BTINTEL_PCIE_D0:
175 return "d0";
176 case BTINTEL_PCIE_D3:
177 return "d3";
178 case BTINTEL_PCIE_HCI_RESET:
179 return "hci_reset";
180 case BTINTEL_PCIE_INTEL_HCI_RESET1:
181 return "intel_reset1";
182 case BTINTEL_PCIE_INTEL_HCI_RESET2:
183 return "intel_reset2";
184 default:
185 return "unknown";
186 }
187}
188
189/* This function initializes the memory for DBGC buffers and formats the
190 * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and
191 * size as the payload
192 */
193static int btintel_pcie_setup_dbgc(struct btintel_pcie_data *data)
194{
195 struct btintel_pcie_dbgc_ctxt db_frag;
196 struct data_buf *buf;
197 int i;
198
199 data->dbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT;
200 data->dbgc.bufs = devm_kcalloc(&data->pdev->dev, data->dbgc.count,
201 sizeof(*buf), GFP_KERNEL);
202 if (!data->dbgc.bufs)
203 return -ENOMEM;
204
205 data->dbgc.buf_v_addr = dmam_alloc_coherent(&data->pdev->dev,
206 data->dbgc.count *
207 BTINTEL_PCIE_DBGC_BUFFER_SIZE,
208 &data->dbgc.buf_p_addr,
209 GFP_KERNEL | __GFP_NOWARN);
210 if (!data->dbgc.buf_v_addr)
211 return -ENOMEM;
212
213 data->dbgc.frag_v_addr = dmam_alloc_coherent(&data->pdev->dev,
214 sizeof(struct btintel_pcie_dbgc_ctxt),
215 &data->dbgc.frag_p_addr,
216 GFP_KERNEL | __GFP_NOWARN);
217 if (!data->dbgc.frag_v_addr)
218 return -ENOMEM;
219
220 data->dbgc.frag_size = sizeof(struct btintel_pcie_dbgc_ctxt);
221
222 db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM;
223 db_frag.ver = BTINTEL_PCIE_DBGC_FRAG_VERSION;
224 db_frag.total_size = BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE;
225 db_frag.num_buf = BTINTEL_PCIE_DBGC_FRAG_BUFFER_COUNT;
226
227 for (i = 0; i < data->dbgc.count; i++) {
228 buf = &data->dbgc.bufs[i];
229 buf->data_p_addr = data->dbgc.buf_p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
230 buf->data = data->dbgc.buf_v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
231 db_frag.bufs[i].buf_addr_lsb = lower_32_bits(buf->data_p_addr);
232 db_frag.bufs[i].buf_addr_msb = upper_32_bits(buf->data_p_addr);
233 db_frag.bufs[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE;
234 }
235
236 memcpy(data->dbgc.frag_v_addr, &db_frag, sizeof(db_frag));
237 return 0;
238}
239
240static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia,
241 u16 queue_num)
242{
243 bt_dev_dbg(hdev, "IA: %s: tr-h:%02u tr-t:%02u cr-h:%02u cr-t:%02u",
244 queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ",
245 ia->tr_hia[queue_num], ia->tr_tia[queue_num],
246 ia->cr_hia[queue_num], ia->cr_tia[queue_num]);
247}
248
249static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1,
250 u16 index)
251{
252 bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x",
253 index, urbd1->frbd_tag, urbd1->status, urbd1->fixed);
254}
255
256static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry)
257{
258 u8 queue = entry->entry;
259 struct msix_entry *entries = entry - queue;
260
261 return container_of(entries, struct btintel_pcie_data, msix_entries[0]);
262}
263
264/* Set the doorbell for TXQ to notify the device that @index (actually index-1)
265 * of the TFD is updated and ready to transmit.
266 */
267static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index)
268{
269 u32 val;
270
271 val = index;
272 val |= (BTINTEL_PCIE_TX_DB_VEC << 16);
273
274 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);
275}
276
277/* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer
278 * descriptor) with the data length and the DMA address of the data buffer.
279 */
280static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index,
281 struct sk_buff *skb)
282{
283 struct data_buf *buf;
284 struct tfd *tfd;
285
286 tfd = &txq->tfds[tfd_index];
287 memset(tfd, 0, sizeof(*tfd));
288
289 buf = &txq->bufs[tfd_index];
290
291 tfd->size = skb->len;
292 tfd->addr = buf->data_p_addr;
293
294 /* Copy the outgoing data to DMA buffer */
295 memcpy(buf->data, skb->data, tfd->size);
296}
297
298static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev)
299{
300 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
301 u16 cr_hia, cr_tia;
302 u32 reg, mbox_reg;
303 struct sk_buff *skb;
304 u8 buf[80];
305
306 skb = alloc_skb(1024, GFP_ATOMIC);
307 if (!skb)
308 return;
309
310 strscpy(buf, "---- Dump of debug registers ---");
311 bt_dev_dbg(hdev, "%s", buf);
312 skb_put_data(skb, buf, strlen(buf));
313
314 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
315 snprintf(buf, sizeof(buf), "boot stage: 0x%8.8x", reg);
316 bt_dev_dbg(hdev, "%s", buf);
317 skb_put_data(skb, buf, strlen(buf));
318 data->boot_stage_cache = reg;
319
320 if (reg & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING)
321 bt_dev_warn(hdev, "Controller device warning (boot_stage: 0x%8.8x)", reg);
322
323 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_STATUS_REG);
324 snprintf(buf, sizeof(buf), "ipc status: 0x%8.8x", reg);
325 skb_put_data(skb, buf, strlen(buf));
326 bt_dev_dbg(hdev, "%s", buf);
327
328 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_CONTROL_REG);
329 snprintf(buf, sizeof(buf), "ipc control: 0x%8.8x", reg);
330 skb_put_data(skb, buf, strlen(buf));
331 bt_dev_dbg(hdev, "%s", buf);
332
333 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG);
334 snprintf(buf, sizeof(buf), "ipc sleep control: 0x%8.8x", reg);
335 skb_put_data(skb, buf, strlen(buf));
336 bt_dev_dbg(hdev, "%s", buf);
337
338 /*Read the Mail box status and registers*/
339 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG);
340 snprintf(buf, sizeof(buf), "mbox status: 0x%8.8x", reg);
341 skb_put_data(skb, buf, strlen(buf));
342 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) {
343 mbox_reg = btintel_pcie_rd_reg32(data,
344 BTINTEL_PCIE_CSR_MBOX_1_REG);
345 snprintf(buf, sizeof(buf), "mbox_1: 0x%8.8x", mbox_reg);
346 skb_put_data(skb, buf, strlen(buf));
347 bt_dev_dbg(hdev, "%s", buf);
348 }
349
350 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) {
351 mbox_reg = btintel_pcie_rd_reg32(data,
352 BTINTEL_PCIE_CSR_MBOX_2_REG);
353 snprintf(buf, sizeof(buf), "mbox_2: 0x%8.8x", mbox_reg);
354 skb_put_data(skb, buf, strlen(buf));
355 bt_dev_dbg(hdev, "%s", buf);
356 }
357
358 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) {
359 mbox_reg = btintel_pcie_rd_reg32(data,
360 BTINTEL_PCIE_CSR_MBOX_3_REG);
361 snprintf(buf, sizeof(buf), "mbox_3: 0x%8.8x", mbox_reg);
362 skb_put_data(skb, buf, strlen(buf));
363 bt_dev_dbg(hdev, "%s", buf);
364 }
365
366 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4) {
367 mbox_reg = btintel_pcie_rd_reg32(data,
368 BTINTEL_PCIE_CSR_MBOX_4_REG);
369 snprintf(buf, sizeof(buf), "mbox_4: 0x%8.8x", mbox_reg);
370 skb_put_data(skb, buf, strlen(buf));
371 bt_dev_dbg(hdev, "%s", buf);
372 }
373
374 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM];
375 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
376 snprintf(buf, sizeof(buf), "rxq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia);
377 skb_put_data(skb, buf, strlen(buf));
378 bt_dev_dbg(hdev, "%s", buf);
379
380 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
381 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM];
382 snprintf(buf, sizeof(buf), "txq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia);
383 skb_put_data(skb, buf, strlen(buf));
384 bt_dev_dbg(hdev, "%s", buf);
385 strscpy(buf, "--------------------------------");
386 bt_dev_dbg(hdev, "%s", buf);
387
388 hci_recv_diag(hdev, skb);
389}
390
391static int btintel_pcie_send_sync(struct btintel_pcie_data *data,
392 struct sk_buff *skb, u32 pkt_type, u16 opcode)
393{
394 int ret;
395 u16 tfd_index;
396 u32 old_ctxt;
397 bool wait_on_alive = false;
398 struct hci_dev *hdev = data->hdev;
399
400 struct txq *txq = &data->txq;
401
402 tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM];
403
404 if (tfd_index >= txq->count)
405 return -ERANGE;
406
407 if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) {
408 bt_dev_err(hdev, "TX skb too large (%u > %u)", skb->len,
409 BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN);
410 return -EMSGSIZE;
411 }
412
413 /* Firmware raises alive interrupt on HCI_OP_RESET or
414 * BTINTEL_HCI_OP_RESET
415 */
416 wait_on_alive = (pkt_type == BTINTEL_PCIE_HCI_CMD_PKT &&
417 (opcode == BTINTEL_HCI_OP_RESET || opcode == HCI_OP_RESET));
418
419 if (wait_on_alive) {
420 data->gp0_received = false;
421 old_ctxt = data->alive_intr_ctxt;
422 data->alive_intr_ctxt =
423 (opcode == BTINTEL_HCI_OP_RESET ? BTINTEL_PCIE_INTEL_HCI_RESET1 :
424 BTINTEL_PCIE_HCI_RESET);
425 bt_dev_dbg(data->hdev, "sending cmd: 0x%4.4x alive context changed: %s -> %s",
426 opcode, btintel_pcie_alivectxt_state2str(old_ctxt),
427 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
428 }
429
430 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &pkt_type,
431 BTINTEL_PCIE_HCI_TYPE_LEN);
432
433 /* Prepare for TX. It updates the TFD with the length of data and
434 * address of the DMA buffer, and copy the data to the DMA buffer
435 */
436 btintel_pcie_prepare_tx(txq, tfd_index, skb);
437
438 tfd_index = (tfd_index + 1) % txq->count;
439 data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index;
440
441 /* Arm wait event condition */
442 data->tx_wait_done = false;
443
444 /* Set the doorbell to notify the device */
445 btintel_pcie_set_tx_db(data, tfd_index);
446
447 /* Wait for the complete interrupt - URBD0 */
448 ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done,
449 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS));
450 if (!ret) {
451 bt_dev_err(data->hdev, "Timeout (%u ms) on tx completion",
452 BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS);
453 btintel_pcie_dump_debug_registers(data->hdev);
454 return -ETIME;
455 }
456
457 if (wait_on_alive) {
458 ret = wait_event_timeout(data->gp0_wait_q,
459 data->gp0_received,
460 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS));
461 if (!ret) {
462 hdev->stat.err_tx++;
463 bt_dev_err(hdev, "Timeout (%u ms) on alive interrupt, alive context: %s",
464 BTINTEL_DEFAULT_INTR_TIMEOUT_MS,
465 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
466 return -ETIME;
467 }
468 }
469 return 0;
470}
471
472/* Set the doorbell for RXQ to notify the device that @index (actually index-1)
473 * is available to receive the data
474 */
475static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index)
476{
477 u32 val;
478
479 val = index;
480 val |= (BTINTEL_PCIE_RX_DB_VEC << 16);
481
482 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);
483}
484
485/* Update the FRBD (free buffer descriptor) with the @frbd_index and the
486 * DMA address of the free buffer.
487 */
488static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index)
489{
490 struct data_buf *buf;
491 struct frbd *frbd;
492
493 /* Get the buffer of the FRBD for DMA */
494 buf = &rxq->bufs[frbd_index];
495
496 frbd = &rxq->frbds[frbd_index];
497 memset(frbd, 0, sizeof(*frbd));
498
499 /* Update FRBD */
500 frbd->tag = frbd_index;
501 frbd->addr = buf->data_p_addr;
502}
503
504static int btintel_pcie_submit_rx(struct btintel_pcie_data *data)
505{
506 u16 frbd_index;
507 struct rxq *rxq = &data->rxq;
508
509 frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM];
510
511 if (frbd_index >= rxq->count)
512 return -ERANGE;
513
514 /* Prepare for RX submit. It updates the FRBD with the address of DMA
515 * buffer
516 */
517 btintel_pcie_prepare_rx(rxq, frbd_index);
518
519 frbd_index = (frbd_index + 1) % rxq->count;
520 data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index;
521 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);
522
523 /* Set the doorbell to notify the device */
524 btintel_pcie_set_rx_db(data, frbd_index);
525
526 return 0;
527}
528
529static int btintel_pcie_start_rx(struct btintel_pcie_data *data)
530{
531 int i, ret;
532 struct rxq *rxq = &data->rxq;
533
534 /* Post (BTINTEL_PCIE_RX_DESCS_COUNT - 3) buffers to overcome the
535 * hardware issues leading to race condition at the firmware.
536 */
537
538 for (i = 0; i < rxq->count - 3; i++) {
539 ret = btintel_pcie_submit_rx(data);
540 if (ret)
541 return ret;
542 }
543
544 return 0;
545}
546
547static void btintel_pcie_reset_ia(struct btintel_pcie_data *data)
548{
549 memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
550 memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
551 memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
552 memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);
553}
554
555static int btintel_pcie_reset_bt(struct btintel_pcie_data *data)
556{
557 u32 reg;
558 int retry = 3;
559
560 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
561
562 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
563 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT |
564 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
565 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON;
566
567 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
568
569 do {
570 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
571 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS)
572 break;
573 usleep_range(10000, 12000);
574
575 } while (--retry > 0);
576 usleep_range(10000, 12000);
577
578 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
579
580 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
581 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT |
582 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
583 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET;
584 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
585 usleep_range(10000, 12000);
586
587 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
588 bt_dev_dbg(data->hdev, "csr register after reset: 0x%8.8x", reg);
589
590 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
591
592 /* If shared hardware reset is success then boot stage register shall be
593 * set to 0
594 */
595 return reg == 0 ? 0 : -ENODEV;
596}
597
598static void btintel_pcie_mac_init(struct btintel_pcie_data *data)
599{
600 u32 reg;
601
602 /* Set MAC_INIT bit to start primary bootloader */
603 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
604 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT |
605 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON |
606 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET);
607 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
608 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT);
609 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
610}
611
612static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data)
613{
614 u32 reg;
615 int retry = 15;
616
617 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
618
619 if (!(reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)) {
620 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
621 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
622 }
623
624 do {
625 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
626 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS)
627 return 0;
628 /* Need delay here for Target Access harwdware to settle down*/
629 usleep_range(1000, 1200);
630
631 } while (--retry > 0);
632
633 return -ETIME;
634}
635
636static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data)
637{
638 u32 reg;
639
640 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
641
642 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ) {
643 reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
644 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
645 }
646}
647
648static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type,
649 void *data, size_t size)
650{
651 struct intel_tlv *tlv;
652
653 tlv = dest;
654 tlv->type = type;
655 tlv->len = size;
656 memcpy(tlv->val, data, tlv->len);
657 return dest + sizeof(*tlv) + size;
658}
659
660static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data)
661{
662 u32 offset, prev_size, wr_ptr_status, dump_size, data_len;
663 u32 status_reg, wrap_reg;
664 struct btintel_pcie_dbgc *dbgc = &data->dbgc;
665 struct hci_dev *hdev = data->hdev;
666 u8 *pdata, *p, buf_idx, hw_variant;
667 struct intel_tlv *tlv;
668 struct timespec64 now;
669 struct tm tm_now;
670 char fw_build[128];
671 char ts[128];
672 char vendor[64];
673 char driver[64];
674
675 if (!IS_ENABLED(CONFIG_DEV_COREDUMP))
676 return -EOPNOTSUPP;
677
678
679 hw_variant = INTEL_HW_VARIANT(data->cnvi);
680 switch (hw_variant) {
681 case BTINTEL_HWID_BZRI:
682 case BTINTEL_HWID_BZRIW:
683 status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS;
684 wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND;
685 break;
686 case BTINTEL_HWID_SCP:
687 case BTINTEL_HWID_SCP2:
688 case BTINTEL_HWID_SCP2F:
689 status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP;
690 wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP;
691 break;
692 default:
693 bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%2.2x)",
694 hw_variant);
695 return -EINVAL;
696 }
697
698 wr_ptr_status = btintel_pcie_rd_dev_mem(data, status_reg);
699 data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data, wrap_reg);
700
701 offset = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK;
702
703 buf_idx = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status);
704 if (buf_idx > dbgc->count) {
705 bt_dev_warn(hdev, "Buffer index is invalid");
706 return -EINVAL;
707 }
708
709 prev_size = buf_idx * BTINTEL_PCIE_DBGC_BUFFER_SIZE;
710 if (prev_size + offset >= prev_size)
711 data->dmp_hdr.write_ptr = prev_size + offset;
712 else
713 return -EINVAL;
714
715 strscpy(vendor, "Vendor: Intel\n");
716 snprintf(driver, sizeof(driver), "Driver: %s\n",
717 data->dmp_hdr.driver_name);
718
719 ktime_get_real_ts64(&now);
720 time64_to_tm(now.tv_sec, 0, &tm_now);
721 snprintf(ts, sizeof(ts), "Dump Time: %02d-%02d-%04ld %02d:%02d:%02d",
722 tm_now.tm_mday, tm_now.tm_mon + 1, tm_now.tm_year + 1900,
723 tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec);
724
725 snprintf(fw_build, sizeof(fw_build),
726 "Firmware Timestamp: Year %u WW %02u buildtype %u build %u",
727 2000 + (data->dmp_hdr.fw_timestamp >> 8),
728 data->dmp_hdr.fw_timestamp & 0xff, data->dmp_hdr.fw_build_type,
729 data->dmp_hdr.fw_build_num);
730
731 data_len = sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_bt) +
732 sizeof(*tlv) + sizeof(data->dmp_hdr.write_ptr) +
733 sizeof(*tlv) + sizeof(data->dmp_hdr.wrap_ctr) +
734 sizeof(*tlv) + sizeof(data->dmp_hdr.trigger_reason) +
735 sizeof(*tlv) + sizeof(data->dmp_hdr.fw_git_sha1) +
736 sizeof(*tlv) + sizeof(data->dmp_hdr.cnvr_top) +
737 sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_top) +
738 sizeof(*tlv) + strlen(ts) +
739 sizeof(*tlv) + strlen(fw_build) +
740 sizeof(*tlv) + strlen(vendor) +
741 sizeof(*tlv) + strlen(driver);
742
743 if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) {
744 data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_type);
745 data_len += sizeof(*tlv) + sizeof(data->dmp_hdr.event_id);
746 }
747
748 /*
749 * sizeof(u32) - signature
750 * sizeof(data_len) - to store tlv data size
751 * data_len - TLV data
752 */
753 dump_size = sizeof(u32) + sizeof(data_len) + data_len;
754
755
756 /* Add debug buffers data length to dump size */
757 dump_size += BTINTEL_PCIE_DBGC_BUFFER_SIZE * dbgc->count;
758
759 pdata = vmalloc(dump_size);
760 if (!pdata)
761 return -ENOMEM;
762 p = pdata;
763
764 *(u32 *)p = BTINTEL_PCIE_MAGIC_NUM;
765 p += sizeof(u32);
766
767 *(u32 *)p = data_len;
768 p += sizeof(u32);
769
770
771 p = btintel_pcie_copy_tlv(p, BTINTEL_VENDOR, vendor, strlen(vendor));
772 p = btintel_pcie_copy_tlv(p, BTINTEL_DRIVER, driver, strlen(driver));
773 p = btintel_pcie_copy_tlv(p, BTINTEL_DUMP_TIME, ts, strlen(ts));
774 p = btintel_pcie_copy_tlv(p, BTINTEL_FW_BUILD, fw_build,
775 strlen(fw_build));
776 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_BT, &data->dmp_hdr.cnvi_bt,
777 sizeof(data->dmp_hdr.cnvi_bt));
778 p = btintel_pcie_copy_tlv(p, BTINTEL_WRITE_PTR, &data->dmp_hdr.write_ptr,
779 sizeof(data->dmp_hdr.write_ptr));
780 p = btintel_pcie_copy_tlv(p, BTINTEL_WRAP_CTR, &data->dmp_hdr.wrap_ctr,
781 sizeof(data->dmp_hdr.wrap_ctr));
782 p = btintel_pcie_copy_tlv(p, BTINTEL_TRIGGER_REASON, &data->dmp_hdr.trigger_reason,
783 sizeof(data->dmp_hdr.trigger_reason));
784 p = btintel_pcie_copy_tlv(p, BTINTEL_FW_SHA, &data->dmp_hdr.fw_git_sha1,
785 sizeof(data->dmp_hdr.fw_git_sha1));
786 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVR_TOP, &data->dmp_hdr.cnvr_top,
787 sizeof(data->dmp_hdr.cnvr_top));
788 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_TOP, &data->dmp_hdr.cnvi_top,
789 sizeof(data->dmp_hdr.cnvi_top));
790
791 if (data->dmp_hdr.event_type && data->dmp_hdr.event_id) {
792 p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_TYPE,
793 &data->dmp_hdr.event_type,
794 sizeof(data->dmp_hdr.event_type));
795 p = btintel_pcie_copy_tlv(p, BTINTEL_EVENT_ID,
796 &data->dmp_hdr.event_id,
797 sizeof(data->dmp_hdr.event_id));
798 data->dmp_hdr.event_type = 0;
799 data->dmp_hdr.event_id = 0;
800 }
801
802 memcpy(p, dbgc->bufs[0].data, dbgc->count * BTINTEL_PCIE_DBGC_BUFFER_SIZE);
803 dev_coredumpv(&hdev->dev, pdata, dump_size, GFP_KERNEL);
804 return 0;
805}
806
807static void btintel_pcie_dump_traces(struct hci_dev *hdev)
808{
809 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
810 int ret = 0;
811
812 ret = btintel_pcie_get_mac_access(data);
813 if (ret) {
814 bt_dev_err(hdev, "Failed to get mac access: (%d)", ret);
815 return;
816 }
817
818 ret = btintel_pcie_read_dram_buffers(data);
819
820 btintel_pcie_release_mac_access(data);
821
822 if (ret)
823 bt_dev_err(hdev, "Failed to dump traces: (%d)", ret);
824}
825
826static bool btintel_pcie_is_blazariw(struct pci_dev *pdev)
827{
828 return pdev->device == 0x4D76;
829}
830
831/* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in
832 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with
833 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0.
834 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage
835 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG.
836 */
837static int btintel_pcie_enable_bt(struct btintel_pcie_data *data)
838{
839 int err;
840 u32 reg;
841
842 data->gp0_received = false;
843
844 /* Update the DMA address of CI struct to CSR */
845 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG,
846 data->ci_p_addr & 0xffffffff);
847 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG,
848 (u64)data->ci_p_addr >> 32);
849
850 /* On BlazarIW, the D0 entry to MAC init does not complete in
851 * time. Wait 50 ms (worst case as per HW analysis) for the
852 * shared hardware reset flow to complete before proceeding with
853 * MAC init.
854 */
855 if (btintel_pcie_is_blazariw(data->pdev))
856 msleep(50);
857
858 /* Reset the cached value of boot stage. it is updated by the MSI-X
859 * gp0 interrupt handler.
860 */
861 data->boot_stage_cache = 0x0;
862
863 /* Set MAC_INIT bit to start primary bootloader */
864 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
865 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT |
866 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON |
867 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET);
868 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |
869 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT);
870
871 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
872
873 /* MAC is ready. Enable BT FUNC */
874 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,
875 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);
876
877 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
878
879 /* wait for interrupt from the device after booting up to primary
880 * bootloader.
881 */
882 data->alive_intr_ctxt = BTINTEL_PCIE_ROM;
883 err = wait_event_timeout(data->gp0_wait_q, data->gp0_received,
884 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS));
885 if (!err)
886 return -ETIME;
887
888 /* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */
889 if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM)
890 return -ENODEV;
891
892 return 0;
893}
894
895static inline bool btintel_pcie_in_op(struct btintel_pcie_data *data)
896{
897 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW;
898}
899
900static inline bool btintel_pcie_in_iml(struct btintel_pcie_data *data)
901{
902 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML &&
903 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW);
904}
905
906static inline bool btintel_pcie_in_d3(struct btintel_pcie_data *data)
907{
908 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY;
909}
910
911static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data)
912{
913 return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY);
914}
915
916static inline bool btintel_pcie_in_device_halt(struct btintel_pcie_data *data)
917{
918 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED;
919}
920
921static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data,
922 u32 dxstate)
923{
924 bt_dev_dbg(data->hdev, "writing sleep_ctl_reg: 0x%8.8x", dxstate);
925 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG, dxstate);
926}
927
928static int btintel_pcie_read_device_mem(struct btintel_pcie_data *data,
929 void *buf, u32 dev_addr, int len)
930{
931 int err;
932 u32 *val = buf;
933
934 /* Get device mac access */
935 err = btintel_pcie_get_mac_access(data);
936 if (err) {
937 bt_dev_err(data->hdev, "Failed to get mac access %d", err);
938 return err;
939 }
940
941 for (; len > 0; len -= 4, dev_addr += 4, val++)
942 *val = btintel_pcie_rd_dev_mem(data, dev_addr);
943
944 btintel_pcie_release_mac_access(data);
945
946 return 0;
947}
948
949static inline bool btintel_pcie_in_lockdown(struct btintel_pcie_data *data)
950{
951 return (data->boot_stage_cache &
952 BTINTEL_PCIE_CSR_BOOT_STAGE_ROM_LOCKDOWN) ||
953 (data->boot_stage_cache &
954 BTINTEL_PCIE_CSR_BOOT_STAGE_IML_LOCKDOWN);
955}
956
957static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data)
958{
959 if (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING)
960 bt_dev_warn(data->hdev, "Controller device warning (boot_stage: 0x%8.8x)",
961 data->boot_stage_cache);
962
963 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER;
964}
965
966static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data)
967{
968 bt_dev_err(data->hdev, "Received gp1 mailbox interrupt");
969 btintel_pcie_dump_debug_registers(data->hdev);
970}
971
972/* This function handles the MSI-X interrupt for gp0 cause (bit 0 in
973 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response.
974 */
975static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)
976{
977 bool submit_rx, signal_waitq;
978 u32 reg, old_ctxt;
979
980 /* This interrupt is for three different causes and it is not easy to
981 * know what causes the interrupt. So, it compares each register value
982 * with cached value and update it before it wake up the queue.
983 */
984 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
985 if (reg != data->boot_stage_cache)
986 data->boot_stage_cache = reg;
987
988 bt_dev_dbg(data->hdev, "Alive context: %s old_boot_stage: 0x%8.8x new_boot_stage: 0x%8.8x",
989 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt),
990 data->boot_stage_cache, reg);
991 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG);
992 if (reg != data->img_resp_cache)
993 data->img_resp_cache = reg;
994
995 if (btintel_pcie_in_error(data)) {
996 bt_dev_err(data->hdev, "Controller in error state (boot_stage: 0x%8.8x)",
997 data->boot_stage_cache);
998 btintel_pcie_dump_debug_registers(data->hdev);
999 return;
1000 }
1001
1002 if (btintel_pcie_in_lockdown(data)) {
1003 bt_dev_err(data->hdev, "Controller in lockdown state");
1004 btintel_pcie_dump_debug_registers(data->hdev);
1005 return;
1006 }
1007
1008 data->gp0_received = true;
1009
1010 old_ctxt = data->alive_intr_ctxt;
1011 submit_rx = false;
1012 signal_waitq = false;
1013
1014 switch (data->alive_intr_ctxt) {
1015 case BTINTEL_PCIE_ROM:
1016 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL;
1017 signal_waitq = true;
1018 break;
1019 case BTINTEL_PCIE_FW_DL:
1020 /* Error case is already handled. Ideally control shall not
1021 * reach here
1022 */
1023 break;
1024 case BTINTEL_PCIE_INTEL_HCI_RESET1:
1025 if (btintel_pcie_in_op(data)) {
1026 submit_rx = true;
1027 signal_waitq = true;
1028 break;
1029 }
1030
1031 if (btintel_pcie_in_iml(data)) {
1032 submit_rx = true;
1033 signal_waitq = true;
1034 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL;
1035 break;
1036 }
1037 break;
1038 case BTINTEL_PCIE_INTEL_HCI_RESET2:
1039 if (btintel_test_and_clear_flag(data->hdev, INTEL_WAIT_FOR_D0)) {
1040 btintel_wake_up_flag(data->hdev, INTEL_WAIT_FOR_D0);
1041 data->alive_intr_ctxt = BTINTEL_PCIE_D0;
1042 }
1043 break;
1044 case BTINTEL_PCIE_D0:
1045 if (btintel_pcie_in_d3(data)) {
1046 data->alive_intr_ctxt = BTINTEL_PCIE_D3;
1047 signal_waitq = true;
1048 break;
1049 }
1050 break;
1051 case BTINTEL_PCIE_D3:
1052 if (btintel_pcie_in_d0(data)) {
1053 data->alive_intr_ctxt = BTINTEL_PCIE_D0;
1054 submit_rx = true;
1055 signal_waitq = true;
1056 break;
1057 }
1058 break;
1059 case BTINTEL_PCIE_HCI_RESET:
1060 data->alive_intr_ctxt = BTINTEL_PCIE_D0;
1061 submit_rx = true;
1062 signal_waitq = true;
1063 break;
1064 default:
1065 bt_dev_err(data->hdev, "Unknown state: 0x%2.2x",
1066 data->alive_intr_ctxt);
1067 break;
1068 }
1069
1070 if (submit_rx) {
1071 btintel_pcie_reset_ia(data);
1072 btintel_pcie_start_rx(data);
1073 }
1074
1075 if (signal_waitq) {
1076 bt_dev_dbg(data->hdev, "wake up gp0 wait_q");
1077 wake_up(&data->gp0_wait_q);
1078 }
1079
1080 if (old_ctxt != data->alive_intr_ctxt)
1081 bt_dev_dbg(data->hdev, "alive context changed: %s -> %s",
1082 btintel_pcie_alivectxt_state2str(old_ctxt),
1083 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt));
1084}
1085
1086/* This function handles the MSX-X interrupt for rx queue 0 which is for TX
1087 */
1088static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data)
1089{
1090 u16 cr_tia, cr_hia;
1091 struct txq *txq;
1092 struct urbd0 *urbd0;
1093
1094 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM];
1095 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
1096
1097 if (cr_tia == cr_hia)
1098 return;
1099
1100 txq = &data->txq;
1101
1102 if (cr_hia >= txq->count) {
1103 bt_dev_err(data->hdev, "TXQ: invalid cr_hia %u", cr_hia);
1104 return;
1105 }
1106
1107 while (cr_tia != cr_hia) {
1108 data->tx_wait_done = true;
1109 wake_up(&data->tx_wait_q);
1110
1111 urbd0 = &txq->urbd0s[cr_tia];
1112
1113 if (urbd0->tfd_index >= txq->count)
1114 return;
1115
1116 cr_tia = (cr_tia + 1) % txq->count;
1117 data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia;
1118 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM);
1119 }
1120}
1121
1122static int btintel_pcie_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
1123{
1124 struct hci_event_hdr *hdr = (void *)skb->data;
1125 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
1126
1127 if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff &&
1128 hdr->plen > 0) {
1129 const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1;
1130 unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1;
1131
1132 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
1133 switch (skb->data[2]) {
1134 case 0x02:
1135 /* When switching to the operational firmware
1136 * the device sends a vendor specific event
1137 * indicating that the bootup completed.
1138 */
1139 btintel_bootup(hdev, ptr, len);
1140
1141 /* If bootup event is from operational image,
1142 * driver needs to write sleep control register to
1143 * move into D0 state
1144 */
1145 if (btintel_pcie_in_op(data)) {
1146 btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0);
1147 data->alive_intr_ctxt = BTINTEL_PCIE_INTEL_HCI_RESET2;
1148 kfree_skb(skb);
1149 return 0;
1150 }
1151
1152 if (btintel_pcie_in_iml(data)) {
1153 /* In case of IML, there is no concept
1154 * of D0 transition. Just mimic as if
1155 * IML moved to D0 by clearing INTEL_WAIT_FOR_D0
1156 * bit and waking up the task waiting on
1157 * INTEL_WAIT_FOR_D0. This is required
1158 * as intel_boot() is common function for
1159 * both IML and OP image loading.
1160 */
1161 if (btintel_test_and_clear_flag(data->hdev,
1162 INTEL_WAIT_FOR_D0))
1163 btintel_wake_up_flag(data->hdev,
1164 INTEL_WAIT_FOR_D0);
1165 }
1166 kfree_skb(skb);
1167 return 0;
1168 case 0x06:
1169 /* When the firmware loading completes the
1170 * device sends out a vendor specific event
1171 * indicating the result of the firmware
1172 * loading.
1173 */
1174 btintel_secure_send_result(hdev, ptr, len);
1175 kfree_skb(skb);
1176 return 0;
1177 }
1178 }
1179
1180 /* This is a debug event that comes from IML and OP image when it
1181 * starts execution. There is no need pass this event to stack.
1182 */
1183 if (skb->data[2] == 0x97) {
1184 hci_recv_diag(hdev, skb);
1185 return 0;
1186 }
1187 }
1188
1189 return hci_recv_frame(hdev, skb);
1190}
1191/* Process the received rx data
1192 * It check the frame header to identify the data type and create skb
1193 * and calling HCI API
1194 */
1195static int btintel_pcie_recv_frame(struct btintel_pcie_data *data,
1196 struct sk_buff *skb)
1197{
1198 int ret;
1199 u8 pkt_type;
1200 u32 plen;
1201 u32 pcie_pkt_type;
1202 void *pdata;
1203 struct hci_dev *hdev = data->hdev;
1204
1205 spin_lock(&data->hci_rx_lock);
1206
1207 /* The first 4 bytes indicates the Intel PCIe specific packet type */
1208 pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN);
1209 if (!pdata) {
1210 bt_dev_err(hdev, "Corrupted packet received");
1211 ret = -EILSEQ;
1212 goto exit_error;
1213 }
1214
1215 pcie_pkt_type = get_unaligned_le32(pdata);
1216
1217 switch (pcie_pkt_type) {
1218 case BTINTEL_PCIE_HCI_ACL_PKT:
1219 if (skb->len >= HCI_ACL_HDR_SIZE) {
1220 plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen);
1221 pkt_type = HCI_ACLDATA_PKT;
1222 } else {
1223 bt_dev_err(hdev, "ACL packet is too short");
1224 ret = -EILSEQ;
1225 goto exit_error;
1226 }
1227 break;
1228
1229 case BTINTEL_PCIE_HCI_SCO_PKT:
1230 if (skb->len >= HCI_SCO_HDR_SIZE) {
1231 plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen;
1232 pkt_type = HCI_SCODATA_PKT;
1233 } else {
1234 bt_dev_err(hdev, "SCO packet is too short");
1235 ret = -EILSEQ;
1236 goto exit_error;
1237 }
1238 break;
1239
1240 case BTINTEL_PCIE_HCI_EVT_PKT:
1241 if (skb->len >= HCI_EVENT_HDR_SIZE) {
1242 plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen;
1243 pkt_type = HCI_EVENT_PKT;
1244 } else {
1245 bt_dev_err(hdev, "Event packet is too short");
1246 ret = -EILSEQ;
1247 goto exit_error;
1248 }
1249 break;
1250
1251 case BTINTEL_PCIE_HCI_ISO_PKT:
1252 if (skb->len >= HCI_ISO_HDR_SIZE) {
1253 plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen);
1254 pkt_type = HCI_ISODATA_PKT;
1255 } else {
1256 bt_dev_err(hdev, "ISO packet is too short");
1257 ret = -EILSEQ;
1258 goto exit_error;
1259 }
1260 break;
1261
1262 default:
1263 bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x",
1264 pcie_pkt_type);
1265 ret = -EINVAL;
1266 goto exit_error;
1267 }
1268
1269 if (skb->len < plen) {
1270 bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x",
1271 pkt_type);
1272 ret = -EILSEQ;
1273 goto exit_error;
1274 }
1275
1276 bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen);
1277
1278 hci_skb_pkt_type(skb) = pkt_type;
1279 hdev->stat.byte_rx += plen;
1280 skb_trim(skb, plen);
1281
1282 if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT)
1283 ret = btintel_pcie_recv_event(hdev, skb);
1284 else
1285 ret = hci_recv_frame(hdev, skb);
1286 skb = NULL; /* skb is freed in the callee */
1287
1288exit_error:
1289 kfree_skb(skb);
1290
1291 if (ret)
1292 hdev->stat.err_rx++;
1293
1294 spin_unlock(&data->hci_rx_lock);
1295
1296 return ret;
1297}
1298
1299static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data)
1300{
1301 int len, err, offset, pending;
1302 struct sk_buff *skb;
1303 u8 *buf, prefix[64];
1304 u32 addr, val;
1305 u16 pkt_len;
1306
1307 struct tlv {
1308 u8 type;
1309 __le16 len;
1310 u8 val[];
1311 } __packed;
1312
1313 struct tlv *tlv;
1314
1315 switch (data->dmp_hdr.cnvi_top & 0xfff) {
1316 case BTINTEL_CNVI_BLAZARI:
1317 case BTINTEL_CNVI_BLAZARIW:
1318 /* only from step B0 onwards */
1319 if (INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top) != 0x01)
1320 return;
1321 len = BTINTEL_PCIE_BLZR_HWEXP_SIZE; /* exception data length */
1322 addr = BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR;
1323 break;
1324 case BTINTEL_CNVI_SCP:
1325 len = BTINTEL_PCIE_SCP_HWEXP_SIZE;
1326 addr = BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR;
1327 break;
1328 case BTINTEL_CNVI_SCP2:
1329 case BTINTEL_CNVI_SCP2F:
1330 len = BTINTEL_PCIE_SCP2_HWEXP_SIZE;
1331 addr = BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR;
1332 break;
1333 default:
1334 bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top);
1335 return;
1336 }
1337
1338 buf = kzalloc(len, GFP_KERNEL);
1339 if (!buf)
1340 goto exit_on_error;
1341
1342 btintel_pcie_mac_init(data);
1343
1344 err = btintel_pcie_read_device_mem(data, buf, addr, len);
1345 if (err)
1346 goto exit_on_error;
1347
1348 val = get_unaligned_le32(buf);
1349 if (val != BTINTEL_PCIE_MAGIC_NUM) {
1350 bt_dev_err(data->hdev, "Invalid exception dump signature: 0x%8.8x",
1351 val);
1352 goto exit_on_error;
1353 }
1354
1355 snprintf(prefix, sizeof(prefix), "Bluetooth: %s: ", bt_dev_name(data->hdev));
1356
1357 offset = 4;
1358 do {
1359 pending = len - offset;
1360 if (pending < sizeof(*tlv))
1361 break;
1362 tlv = (struct tlv *)(buf + offset);
1363
1364 /* If type == 0, then there are no more TLVs to be parsed */
1365 if (!tlv->type) {
1366 bt_dev_dbg(data->hdev, "Invalid TLV type 0");
1367 break;
1368 }
1369 pkt_len = le16_to_cpu(tlv->len);
1370 offset += sizeof(*tlv);
1371 pending = len - offset;
1372 if (pkt_len > pending)
1373 break;
1374
1375 offset += pkt_len;
1376
1377 /* Only TLVs of type == 1 are HCI events, no need to process other
1378 * TLVs
1379 */
1380 if (tlv->type != 1)
1381 continue;
1382
1383 bt_dev_dbg(data->hdev, "TLV packet length: %u", pkt_len);
1384 if (pkt_len > HCI_MAX_EVENT_SIZE)
1385 break;
1386 skb = bt_skb_alloc(pkt_len, GFP_KERNEL);
1387 if (!skb)
1388 goto exit_on_error;
1389 hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
1390 skb_put_data(skb, tlv->val, pkt_len);
1391
1392 /* copy Intel specific pcie packet type */
1393 val = BTINTEL_PCIE_HCI_EVT_PKT;
1394 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &val,
1395 BTINTEL_PCIE_HCI_TYPE_LEN);
1396
1397 print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_OFFSET, 16, 1,
1398 tlv->val, pkt_len, false);
1399
1400 btintel_pcie_recv_frame(data, skb);
1401 } while (offset < len);
1402
1403exit_on_error:
1404 kfree(buf);
1405}
1406
1407static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data)
1408{
1409 struct btintel_pcie_fwtrigger_evt *evt;
1410 struct sk_buff *skb;
1411 unsigned int len;
1412 int err;
1413 u8 *buf;
1414
1415 if (!data->debug_evt_size || !data->debug_evt_addr)
1416 return -EINVAL;
1417
1418 len = data->debug_evt_size;
1419
1420 len = ALIGN_DOWN(len, 4);
1421
1422 if (len < sizeof(*evt) || len > HCI_MAX_EVENT_SIZE) {
1423 bt_dev_err(data->hdev, "Invalid FW trigger data size (%u bytes)", len);
1424 return -EINVAL;
1425 }
1426
1427 buf = kzalloc(len, GFP_KERNEL);
1428 if (!buf)
1429 return -ENOMEM;
1430
1431 btintel_pcie_mac_init(data);
1432
1433 err = btintel_pcie_read_device_mem(data, buf, data->debug_evt_addr,
1434 len);
1435 if (err)
1436 goto exit_on_error;
1437
1438 evt = (void *)buf;
1439 data->dmp_hdr.event_type = evt->event_type;
1440 data->dmp_hdr.event_id = le16_to_cpu(evt->event_id);
1441
1442 bt_dev_dbg(data->hdev, "event type: 0x%2.2x event id: 0x%4.4x len: %u",
1443 data->dmp_hdr.event_type, data->dmp_hdr.event_id, len);
1444
1445 skb = bt_skb_alloc(len, GFP_KERNEL);
1446 if (!skb) {
1447 err = -ENOMEM;
1448 goto exit_on_error;
1449 }
1450 skb_put_data(skb, buf, len);
1451
1452 hci_recv_diag(data->hdev, skb);
1453 err = 0;
1454
1455exit_on_error:
1456 kfree(buf);
1457 return err;
1458}
1459
1460/* Queue a coredump dump_traces() pass.
1461 *
1462 * Returns true if a new coredump was queued, false if one was already
1463 * in-flight (the BTINTEL_PCIE_COREDUMP_INPROGRESS bit serves as the
1464 * single-writer guard for the @coredump_work item) or the workqueue is
1465 * disabled (reset / remove in progress).
1466 *
1467 * Always queue this AFTER any companion event-reader work (hwexp /
1468 * fwtrigger) so that, on the ordered @dump_workqueue, the event reader
1469 * runs first and populates dmp_hdr.event_type / event_id before
1470 * dump_traces consumes them.
1471 */
1472static bool btintel_pcie_queue_coredump(struct btintel_pcie_data *data,
1473 u16 trigger_reason)
1474{
1475 if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags))
1476 return false;
1477
1478 data->dmp_hdr.trigger_reason = trigger_reason;
1479
1480 if (queue_work(data->dump_workqueue, &data->coredump_work))
1481 return true;
1482
1483 /* Workqueue is disabled (reset/remove drained it). Release the
1484 * guard so a later trigger, after re-probe, can succeed.
1485 */
1486 clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
1487 return false;
1488}
1489
1490static void btintel_pcie_msix_fw_trigger_handler(struct btintel_pcie_data *data)
1491{
1492 bt_dev_dbg(data->hdev, "Received firmware smart trigger cause");
1493
1494 /* Per-work guard: deduplicate concurrent FW-trigger interrupts.
1495 * Cleared at the tail of btintel_pcie_fwtrigger_worker().
1496 */
1497 if (test_and_set_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS,
1498 &data->flags))
1499 return;
1500
1501 if (!queue_work(data->dump_workqueue, &data->fwtrigger_work)) {
1502 clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
1503 return;
1504 }
1505
1506 /* Queue coredump after the fwtrigger event reader so dmp_hdr.event_*
1507 * is populated before dump_traces consumes it.
1508 */
1509 btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
1510}
1511
1512static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data)
1513{
1514 bt_dev_err(data->hdev, "Received hw exception interrupt");
1515
1516 /* CORE_HALTED is the single-writer guard for this handler. It is
1517 * set once on first HW exception and cleared only by re-probe
1518 * (data is reallocated), so it also serializes hwexp_work
1519 * scheduling without needing a separate bit.
1520 */
1521 if (test_and_set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags))
1522 return;
1523
1524 /* Queue companion coredump first so it is appended after hwexp_work
1525 * on the ordered @dump_workqueue (preserves the original
1526 * coredump-then-hwexp ordering).
1527 */
1528 btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
1529
1530 queue_work(data->dump_workqueue, &data->hwexp_work);
1531}
1532
1533static void btintel_pcie_coredump_worker(struct work_struct *work)
1534{
1535 struct btintel_pcie_data *data = container_of(work,
1536 struct btintel_pcie_data, coredump_work);
1537
1538 /* hdev is NULL until setup_hdev() succeeds, and is cleared on
1539 * teardown after disable_work_sync() drains us; bail in that case.
1540 */
1541 if (!data->hdev)
1542 goto out;
1543
1544 btintel_pcie_dump_traces(data->hdev);
1545out:
1546 /* Release guard last so a new trigger can run only after this
1547 * pass has fully completed (including dev_coredumpv()).
1548 */
1549 clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags);
1550}
1551
1552static void btintel_pcie_hwexp_worker(struct work_struct *work)
1553{
1554 struct btintel_pcie_data *data = container_of(work,
1555 struct btintel_pcie_data, hwexp_work);
1556
1557 if (!data->hdev)
1558 return;
1559
1560 /* Unlike usb products, controller will not send hardware exception
1561 * event on exception. Instead controller writes the hardware event
1562 * to device memory along with optional debug events, raises MSIX
1563 * and halts. Driver shall read the exception event from device
1564 * memory and passes it to the stack for further processing.
1565 *
1566 * Re-entry is gated by BTINTEL_PCIE_CORE_HALTED in the IRQ
1567 * handler, which is only cleared by re-probe; no per-work bit
1568 * is needed here.
1569 */
1570 btintel_pcie_read_hwexp(data);
1571}
1572
1573static void btintel_pcie_fwtrigger_worker(struct work_struct *work)
1574{
1575 struct btintel_pcie_data *data = container_of(work,
1576 struct btintel_pcie_data, fwtrigger_work);
1577 int err;
1578
1579 if (!data->hdev)
1580 goto out;
1581
1582 err = btintel_pcie_dump_fwtrigger_event(data);
1583 if (err)
1584 bt_dev_warn(data->hdev, "failed to log fwtrigger event");
1585out:
1586 /* Release guard last; matches set in fw_trigger handler. */
1587 clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags);
1588}
1589
1590static void btintel_pcie_rx_work(struct work_struct *work)
1591{
1592 struct btintel_pcie_data *data = container_of(work,
1593 struct btintel_pcie_data, rx_work);
1594 struct sk_buff *skb;
1595
1596 /* Process the sk_buf in queue and send to the HCI layer */
1597 while ((skb = skb_dequeue(&data->rx_skb_q))) {
1598 btintel_pcie_recv_frame(data, skb);
1599 }
1600}
1601
1602/* create sk_buff with data and save it to queue and start RX work */
1603static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status,
1604 void *buf)
1605{
1606 int ret, len;
1607 struct rfh_hdr *rfh_hdr;
1608 struct sk_buff *skb;
1609
1610 rfh_hdr = buf;
1611
1612 len = rfh_hdr->packet_len;
1613 if (len == 0 || len > BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)) {
1614 bt_dev_err(data->hdev, "Invalid packet_len %d (max %zu)", len,
1615 BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr));
1616 ret = -EINVAL;
1617 goto resubmit;
1618 }
1619
1620 /* Remove RFH header */
1621 buf += sizeof(*rfh_hdr);
1622
1623 skb = alloc_skb(len, GFP_ATOMIC);
1624 if (!skb)
1625 goto resubmit;
1626
1627 skb_put_data(skb, buf, len);
1628 skb_queue_tail(&data->rx_skb_q, skb);
1629 queue_work(data->workqueue, &data->rx_work);
1630
1631resubmit:
1632 ret = btintel_pcie_submit_rx(data);
1633
1634 return ret;
1635}
1636
1637/* Handles the MSI-X interrupt for rx queue 1 which is for RX */
1638static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data)
1639{
1640 u16 cr_hia, cr_tia;
1641 struct rxq *rxq;
1642 struct urbd1 *urbd1;
1643 struct data_buf *buf;
1644 int ret;
1645 struct hci_dev *hdev = data->hdev;
1646
1647 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM];
1648 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
1649
1650 bt_dev_dbg(hdev, "RXQ: cr_hia: %u cr_tia: %u", cr_hia, cr_tia);
1651
1652 /* Check CR_TIA and CR_HIA for change */
1653 if (cr_tia == cr_hia)
1654 return;
1655
1656 rxq = &data->rxq;
1657
1658 if (cr_hia >= rxq->count) {
1659 bt_dev_err(hdev, "RXQ: invalid cr_hia %u", cr_hia);
1660 return;
1661 }
1662
1663 /* The firmware sends multiple CD in a single MSI-X and it needs to
1664 * process all received CDs in this interrupt.
1665 */
1666 while (cr_tia != cr_hia) {
1667 urbd1 = &rxq->urbd1s[cr_tia];
1668 ipc_print_urbd1(data->hdev, urbd1, cr_tia);
1669
1670 if (urbd1->frbd_tag >= rxq->count) {
1671 bt_dev_err(hdev, "RXQ: invalid frbd_tag %u",
1672 urbd1->frbd_tag);
1673 return;
1674 }
1675
1676 buf = &rxq->bufs[urbd1->frbd_tag];
1677 if (!buf) {
1678 bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d",
1679 urbd1->frbd_tag);
1680 return;
1681 }
1682
1683 ret = btintel_pcie_submit_rx_work(data, urbd1->status,
1684 buf->data);
1685 if (ret) {
1686 bt_dev_err(hdev, "RXQ: failed to submit rx request");
1687 return;
1688 }
1689
1690 cr_tia = (cr_tia + 1) % rxq->count;
1691 data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia;
1692 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);
1693 }
1694}
1695
1696static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data)
1697{
1698 return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];
1699}
1700
1701static inline bool btintel_pcie_is_txackq_empty(struct btintel_pcie_data *data)
1702{
1703 return data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] == data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];
1704}
1705
1706static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id)
1707{
1708 struct msix_entry *entry = dev_id;
1709 struct btintel_pcie_data *data = btintel_pcie_get_data(entry);
1710 u32 intr_fh, intr_hw;
1711
1712 spin_lock(&data->irq_lock);
1713 intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES);
1714 intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES);
1715
1716 /* Clear causes registers to avoid being handling the same cause */
1717 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh);
1718 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw);
1719 spin_unlock(&data->irq_lock);
1720
1721 if (unlikely(!(intr_fh | intr_hw))) {
1722 /* Ignore interrupt, inta == 0 */
1723 bt_warn_ratelimited("Bluetooth: btintel_pcie: Received spurious interrupt\n");
1724 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST,
1725 BIT(entry->entry));
1726 return IRQ_NONE;
1727 }
1728
1729 /* This interrupt is raised when there is an hardware exception */
1730 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP)
1731 btintel_pcie_msix_hw_exp_handler(data);
1732
1733 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP1)
1734 btintel_pcie_msix_gp1_handler(data);
1735
1736
1737 /* For TX */
1738 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0) {
1739 btintel_pcie_msix_tx_handle(data);
1740 if (!btintel_pcie_is_rxq_empty(data))
1741 btintel_pcie_msix_rx_handle(data);
1742 }
1743
1744 /* For RX */
1745 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1) {
1746 btintel_pcie_msix_rx_handle(data);
1747 if (!btintel_pcie_is_txackq_empty(data))
1748 btintel_pcie_msix_tx_handle(data);
1749 }
1750
1751 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG)
1752 btintel_pcie_msix_fw_trigger_handler(data);
1753
1754 /* This interrupt is triggered by the firmware after updating
1755 * boot_stage register and image_response register
1756 */
1757 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0)
1758 btintel_pcie_msix_gp0_handler(data);
1759
1760 /*
1761 * Before sending the interrupt the HW disables it to prevent a nested
1762 * interrupt. This is done by writing 1 to the corresponding bit in
1763 * the mask register. After handling the interrupt, it should be
1764 * re-enabled by clearing this bit. This register is defined as write 1
1765 * clear (W1C) register, meaning that it's cleared by writing 1
1766 * to the bit.
1767 */
1768 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST,
1769 BIT(entry->entry));
1770
1771 return IRQ_HANDLED;
1772}
1773
1774/* This function requests the irq for MSI-X and registers the handlers per irq.
1775 * Currently, it requests only 1 irq for all interrupt causes.
1776 */
1777static int btintel_pcie_setup_irq(struct btintel_pcie_data *data)
1778{
1779 int err;
1780 int num_irqs, i, j;
1781
1782 for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++)
1783 data->msix_entries[i].entry = i;
1784
1785 num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN,
1786 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX);
1787 if (num_irqs < 0)
1788 return num_irqs;
1789
1790 data->alloc_vecs = num_irqs;
1791 data->msix_enabled = 1;
1792 data->def_irq = 0;
1793
1794 /* setup irq handler */
1795 for (i = 0; i < data->alloc_vecs; i++) {
1796 struct msix_entry *msix_entry;
1797
1798 msix_entry = &data->msix_entries[i];
1799 msix_entry->vector = pci_irq_vector(data->pdev, i);
1800
1801 err = devm_request_threaded_irq(&data->pdev->dev,
1802 msix_entry->vector,
1803 NULL,
1804 btintel_pcie_irq_msix_handler,
1805 IRQF_ONESHOT | IRQF_SHARED,
1806 KBUILD_MODNAME,
1807 msix_entry);
1808 if (err) {
1809 for (j = 0; j < i; j++)
1810 devm_free_irq(&data->pdev->dev,
1811 data->msix_entries[j].vector,
1812 &data->msix_entries[j]);
1813 pci_free_irq_vectors(data->pdev);
1814 data->alloc_vecs = 0;
1815 return err;
1816 }
1817 }
1818 return 0;
1819}
1820
1821struct btintel_pcie_causes_list {
1822 u32 cause;
1823 u32 mask_reg;
1824 u8 cause_num;
1825};
1826
1827static struct btintel_pcie_causes_list causes_list[] = {
1828 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x00 },
1829 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 },
1830 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 },
1831 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x23 },
1832 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x25 },
1833};
1834
1835/* This function configures the interrupt masks for both HW_INT_CAUSES and
1836 * FH_INT_CAUSES which are meaningful to us.
1837 *
1838 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver
1839 * need to call this function again to configure since the masks
1840 * are reset to 0xFFFFFFFF after reset.
1841 */
1842static void btintel_pcie_config_msix(struct btintel_pcie_data *data)
1843{
1844 int i;
1845 int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE;
1846
1847 /* Set Non Auto Clear Cause */
1848 for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
1849 btintel_pcie_wr_reg8(data,
1850 BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num),
1851 val);
1852 btintel_pcie_clr_reg_bits(data,
1853 causes_list[i].mask_reg,
1854 causes_list[i].cause);
1855 }
1856
1857 /* Save the initial interrupt mask */
1858 data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK);
1859 data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK);
1860}
1861
1862static int btintel_pcie_config_pcie(struct pci_dev *pdev,
1863 struct btintel_pcie_data *data)
1864{
1865 int err;
1866
1867 err = pcim_enable_device(pdev);
1868 if (err)
1869 return err;
1870
1871 pci_set_master(pdev);
1872
1873 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1874 if (err) {
1875 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1876 if (err)
1877 return err;
1878 }
1879
1880 data->base_addr = pcim_iomap_region(pdev, 0, KBUILD_MODNAME);
1881 if (IS_ERR(data->base_addr))
1882 return PTR_ERR(data->base_addr);
1883
1884 /* Do shared hardware reset to ensure a clean start before
1885 * configuring interrupts.
1886 */
1887 err = btintel_pcie_reset_bt(data);
1888 if (err)
1889 return err;
1890
1891 err = btintel_pcie_setup_irq(data);
1892 if (err)
1893 return err;
1894
1895 /* Configure MSI-X with causes list */
1896 btintel_pcie_config_msix(data);
1897
1898 return 0;
1899}
1900
1901static void btintel_pcie_init_ci(struct btintel_pcie_data *data,
1902 struct ctx_info *ci)
1903{
1904 ci->version = 0x1;
1905 ci->size = sizeof(*ci);
1906 ci->config = 0x0000;
1907 ci->addr_cr_hia = data->ia.cr_hia_p_addr;
1908 ci->addr_tr_tia = data->ia.tr_tia_p_addr;
1909 ci->addr_cr_tia = data->ia.cr_tia_p_addr;
1910 ci->addr_tr_hia = data->ia.tr_hia_p_addr;
1911 ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES;
1912 ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES;
1913 ci->addr_urbdq0 = data->txq.urbd0s_p_addr;
1914 ci->addr_tfdq = data->txq.tfds_p_addr;
1915 ci->num_tfdq = data->txq.count;
1916 ci->num_urbdq0 = data->txq.count;
1917 ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM;
1918 ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM;
1919 ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K;
1920 ci->addr_frbdq = data->rxq.frbds_p_addr;
1921 ci->num_frbdq = data->rxq.count;
1922 ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;
1923 ci->addr_urbdq1 = data->rxq.urbd1s_p_addr;
1924 ci->num_urbdq1 = data->rxq.count;
1925 ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;
1926
1927 ci->dbg_output_mode = 0x01;
1928 ci->dbgc_addr = data->dbgc.frag_p_addr;
1929 ci->dbgc_size = data->dbgc.frag_size;
1930 ci->dbg_preset = 0x00;
1931}
1932
1933static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data,
1934 struct txq *txq)
1935{
1936 /* Free data buffers first */
1937 dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE,
1938 txq->buf_v_addr, txq->buf_p_addr);
1939 kfree(txq->bufs);
1940}
1941
1942static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data,
1943 struct txq *txq)
1944{
1945 int i;
1946 struct data_buf *buf;
1947
1948 /* Allocate the same number of buffers as the descriptor */
1949 txq->bufs = kmalloc_objs(*buf, txq->count);
1950 if (!txq->bufs)
1951 return -ENOMEM;
1952
1953 /* Allocate full chunk of data buffer for DMA first and do indexing and
1954 * initialization next, so it can be freed easily
1955 */
1956 txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,
1957 txq->count * BTINTEL_PCIE_BUFFER_SIZE,
1958 &txq->buf_p_addr,
1959 GFP_KERNEL | __GFP_NOWARN);
1960 if (!txq->buf_v_addr) {
1961 kfree(txq->bufs);
1962 return -ENOMEM;
1963 }
1964
1965 /* Setup the allocated DMA buffer to bufs. Each data_buf should
1966 * have virtual address and physical address
1967 */
1968 for (i = 0; i < txq->count; i++) {
1969 buf = &txq->bufs[i];
1970 buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1971 buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
1972 }
1973
1974 return 0;
1975}
1976
1977static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data,
1978 struct rxq *rxq)
1979{
1980 /* Free data buffers first */
1981 dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE,
1982 rxq->buf_v_addr, rxq->buf_p_addr);
1983 kfree(rxq->bufs);
1984}
1985
1986static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data,
1987 struct rxq *rxq)
1988{
1989 int i;
1990 struct data_buf *buf;
1991
1992 /* Allocate the same number of buffers as the descriptor */
1993 rxq->bufs = kmalloc_objs(*buf, rxq->count);
1994 if (!rxq->bufs)
1995 return -ENOMEM;
1996
1997 /* Allocate full chunk of data buffer for DMA first and do indexing and
1998 * initialization next, so it can be freed easily
1999 */
2000 rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,
2001 rxq->count * BTINTEL_PCIE_BUFFER_SIZE,
2002 &rxq->buf_p_addr,
2003 GFP_KERNEL | __GFP_NOWARN);
2004 if (!rxq->buf_v_addr) {
2005 kfree(rxq->bufs);
2006 return -ENOMEM;
2007 }
2008
2009 /* Setup the allocated DMA buffer to bufs. Each data_buf should
2010 * have virtual address and physical address
2011 */
2012 for (i = 0; i < rxq->count; i++) {
2013 buf = &rxq->bufs[i];
2014 buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
2015 buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);
2016 }
2017
2018 return 0;
2019}
2020
2021static void btintel_pcie_free(struct btintel_pcie_data *data)
2022{
2023 btintel_pcie_free_rxq_bufs(data, &data->rxq);
2024 btintel_pcie_free_txq_bufs(data, &data->txq);
2025
2026 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);
2027 dma_pool_destroy(data->dma_pool);
2028}
2029
2030/* Allocate tx and rx queues, any related data structures and buffers.
2031 */
2032static int btintel_pcie_alloc(struct btintel_pcie_data *data)
2033{
2034 int err = 0;
2035 size_t total;
2036 dma_addr_t p_addr;
2037 void *v_addr;
2038 size_t tfd_size, frbd_size, ctx_size, ci_size, urbd0_size, urbd1_size;
2039
2040 /* Allocate the chunk of DMA memory for descriptors, index array, and
2041 * context information, instead of allocating individually.
2042 * The DMA memory for data buffer is allocated while setting up the
2043 * each queue.
2044 *
2045 * Total size is sum of the following and each of the individual sizes
2046 * are aligned to 128 bytes before adding up.
2047 *
2048 * + size of TFD * Number of descriptors in queue
2049 * + size of URBD0 * Number of descriptors in queue
2050 * + size of FRBD * Number of descriptors in queue
2051 * + size of URBD1 * Number of descriptors in queue
2052 * + size of index * Number of queues(2) * type of index array(4)
2053 * + size of context information
2054 */
2055 tfd_size = ALIGN(sizeof(struct tfd) * BTINTEL_PCIE_TX_DESCS_COUNT,
2056 BTINTEL_PCIE_DMA_ALIGN_128B);
2057 urbd0_size = ALIGN(sizeof(struct urbd0) * BTINTEL_PCIE_TX_DESCS_COUNT,
2058 BTINTEL_PCIE_DMA_ALIGN_128B);
2059
2060 frbd_size = ALIGN(sizeof(struct frbd) * BTINTEL_PCIE_RX_DESCS_COUNT,
2061 BTINTEL_PCIE_DMA_ALIGN_128B);
2062 urbd1_size = ALIGN(sizeof(struct urbd1) * BTINTEL_PCIE_RX_DESCS_COUNT,
2063 BTINTEL_PCIE_DMA_ALIGN_128B);
2064
2065 ci_size = ALIGN(sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES,
2066 BTINTEL_PCIE_DMA_ALIGN_128B);
2067
2068 ctx_size = ALIGN(sizeof(struct ctx_info), BTINTEL_PCIE_DMA_ALIGN_128B);
2069
2070 total = tfd_size + urbd0_size + frbd_size + urbd1_size + ctx_size + ci_size * 4;
2071
2072 data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev,
2073 total, BTINTEL_PCIE_DMA_ALIGN_128B, 0);
2074 if (!data->dma_pool) {
2075 err = -ENOMEM;
2076 goto exit_error;
2077 }
2078
2079 v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN,
2080 &p_addr);
2081 if (!v_addr) {
2082 dma_pool_destroy(data->dma_pool);
2083 err = -ENOMEM;
2084 goto exit_error;
2085 }
2086
2087 data->dma_p_addr = p_addr;
2088 data->dma_v_addr = v_addr;
2089
2090 /* Setup descriptor count */
2091 data->txq.count = BTINTEL_PCIE_TX_DESCS_COUNT;
2092 data->rxq.count = BTINTEL_PCIE_RX_DESCS_COUNT;
2093
2094 /* Setup tfds */
2095 data->txq.tfds_p_addr = p_addr;
2096 data->txq.tfds = v_addr;
2097
2098 p_addr += tfd_size;
2099 v_addr += tfd_size;
2100
2101 /* Setup urbd0 */
2102 data->txq.urbd0s_p_addr = p_addr;
2103 data->txq.urbd0s = v_addr;
2104
2105 p_addr += urbd0_size;
2106 v_addr += urbd0_size;
2107
2108 /* Setup FRBD*/
2109 data->rxq.frbds_p_addr = p_addr;
2110 data->rxq.frbds = v_addr;
2111
2112 p_addr += frbd_size;
2113 v_addr += frbd_size;
2114
2115 /* Setup urbd1 */
2116 data->rxq.urbd1s_p_addr = p_addr;
2117 data->rxq.urbd1s = v_addr;
2118
2119 p_addr += urbd1_size;
2120 v_addr += urbd1_size;
2121
2122 /* Setup data buffers for txq */
2123 err = btintel_pcie_setup_txq_bufs(data, &data->txq);
2124 if (err)
2125 goto exit_error_pool;
2126
2127 /* Setup data buffers for rxq */
2128 err = btintel_pcie_setup_rxq_bufs(data, &data->rxq);
2129 if (err)
2130 goto exit_error_txq;
2131
2132 /* TR Head Index Array */
2133 data->ia.tr_hia_p_addr = p_addr;
2134 data->ia.tr_hia = v_addr;
2135 p_addr += ci_size;
2136 v_addr += ci_size;
2137
2138 /* TR Tail Index Array */
2139 data->ia.tr_tia_p_addr = p_addr;
2140 data->ia.tr_tia = v_addr;
2141 p_addr += ci_size;
2142 v_addr += ci_size;
2143
2144 /* CR Head index Array */
2145 data->ia.cr_hia_p_addr = p_addr;
2146 data->ia.cr_hia = v_addr;
2147 p_addr += ci_size;
2148 v_addr += ci_size;
2149
2150 /* CR Tail Index Array */
2151 data->ia.cr_tia_p_addr = p_addr;
2152 data->ia.cr_tia = v_addr;
2153 p_addr += ci_size;
2154 v_addr += ci_size;
2155
2156 /* Setup data buffers for dbgc */
2157 err = btintel_pcie_setup_dbgc(data);
2158 if (err)
2159 goto exit_error_txq;
2160
2161 /* Setup Context Information */
2162 data->ci = v_addr;
2163 data->ci_p_addr = p_addr;
2164
2165 /* Initialize the CI */
2166 btintel_pcie_init_ci(data, data->ci);
2167
2168 return 0;
2169
2170exit_error_txq:
2171 btintel_pcie_free_txq_bufs(data, &data->txq);
2172exit_error_pool:
2173 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);
2174 dma_pool_destroy(data->dma_pool);
2175exit_error:
2176 return err;
2177}
2178
2179static int btintel_pcie_open(struct hci_dev *hdev)
2180{
2181 bt_dev_dbg(hdev, "");
2182
2183 return 0;
2184}
2185
2186static int btintel_pcie_close(struct hci_dev *hdev)
2187{
2188 bt_dev_dbg(hdev, "");
2189
2190 return 0;
2191}
2192
2193static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode)
2194{
2195 struct sk_buff *skb;
2196 struct hci_event_hdr *hdr;
2197 struct hci_ev_cmd_complete *evt;
2198
2199 skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);
2200 if (!skb)
2201 return -ENOMEM;
2202
2203 hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr));
2204 hdr->evt = HCI_EV_CMD_COMPLETE;
2205 hdr->plen = sizeof(*evt) + 1;
2206
2207 evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt));
2208 evt->ncmd = 0x01;
2209 evt->opcode = cpu_to_le16(opcode);
2210
2211 *(u8 *)skb_put(skb, 1) = 0x00;
2212
2213 hci_skb_pkt_type(skb) = HCI_EVENT_PKT;
2214
2215 return hci_recv_frame(hdev, skb);
2216}
2217
2218static int btintel_pcie_send_frame(struct hci_dev *hdev,
2219 struct sk_buff *skb)
2220{
2221 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2222 struct hci_command_hdr *cmd;
2223 __u16 opcode = ~0;
2224 int ret;
2225 u32 type;
2226
2227 if (test_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags))
2228 return -ENODEV;
2229
2230 if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
2231 return -ENODEV;
2232
2233 /* Account for the 4-byte PCIe type header prepended before the
2234 * DMA copy. Written as a subtraction to avoid wrap-around on
2235 * attacker-controlled skb->len.
2236 */
2237 if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) {
2238 bt_dev_err(hdev, "Packet too large: %u > %u", skb->len,
2239 BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN);
2240 return -EMSGSIZE;
2241 }
2242
2243 /* Due to the fw limitation, the type header of the packet should be
2244 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read
2245 * the first byte to get the packet type and redirect the rest of data
2246 * packet to the right handler.
2247 *
2248 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data
2249 * from DMA memory and by the time it reads the first 4 bytes, it has
2250 * already consumed some part of packet. Thus the packet type indicator
2251 * for iBT PCIe is 4 bytes.
2252 *
2253 * Luckily, when HCI core creates the skb, it allocates 8 bytes of
2254 * head room for profile and driver use, and before sending the data
2255 * to the device, append the iBT PCIe packet type in the front.
2256 */
2257 switch (hci_skb_pkt_type(skb)) {
2258 case HCI_COMMAND_PKT:
2259 type = BTINTEL_PCIE_HCI_CMD_PKT;
2260 cmd = (void *)skb->data;
2261 opcode = le16_to_cpu(cmd->opcode);
2262 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {
2263 struct hci_command_hdr *cmd = (void *)skb->data;
2264 __u16 opcode = le16_to_cpu(cmd->opcode);
2265
2266 /* When the BTINTEL_HCI_OP_RESET command is issued to
2267 * boot into the operational firmware, it will actually
2268 * not send a command complete event. To keep the flow
2269 * control working inject that event here.
2270 */
2271 if (opcode == BTINTEL_HCI_OP_RESET)
2272 btintel_pcie_inject_cmd_complete(hdev, opcode);
2273 }
2274
2275 hdev->stat.cmd_tx++;
2276 break;
2277 case HCI_ACLDATA_PKT:
2278 type = BTINTEL_PCIE_HCI_ACL_PKT;
2279 hdev->stat.acl_tx++;
2280 break;
2281 case HCI_SCODATA_PKT:
2282 type = BTINTEL_PCIE_HCI_SCO_PKT;
2283 hdev->stat.sco_tx++;
2284 break;
2285 case HCI_ISODATA_PKT:
2286 type = BTINTEL_PCIE_HCI_ISO_PKT;
2287 break;
2288 default:
2289 bt_dev_err(hdev, "Unknown HCI packet type");
2290 return -EILSEQ;
2291 }
2292
2293 ret = btintel_pcie_send_sync(data, skb, type, opcode);
2294 if (ret) {
2295 hdev->stat.err_tx++;
2296 bt_dev_err(hdev, "Failed to send frame (%d)", ret);
2297 goto exit_error;
2298 }
2299
2300 hdev->stat.byte_tx += skb->len;
2301 kfree_skb(skb);
2302
2303exit_error:
2304 return ret;
2305}
2306
2307static void btintel_pcie_release_hdev(struct btintel_pcie_data *data)
2308{
2309 struct hci_dev *hdev = data->hdev;
2310
2311 if (!hdev)
2312 return;
2313
2314 hci_unregister_dev(hdev);
2315 hci_free_dev(hdev);
2316 data->hdev = NULL;
2317}
2318
2319static void btintel_pcie_disable_interrupts(struct btintel_pcie_data *data)
2320{
2321 spin_lock(&data->irq_lock);
2322 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, data->fh_init_mask);
2323 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, data->hw_init_mask);
2324 spin_unlock(&data->irq_lock);
2325}
2326
2327static void btintel_pcie_enable_interrupts(struct btintel_pcie_data *data)
2328{
2329 spin_lock(&data->irq_lock);
2330 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, ~data->fh_init_mask);
2331 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, ~data->hw_init_mask);
2332 spin_unlock(&data->irq_lock);
2333}
2334
2335static void btintel_pcie_synchronize_irqs(struct btintel_pcie_data *data)
2336{
2337 for (int i = 0; i < data->alloc_vecs; i++)
2338 synchronize_irq(data->msix_entries[i].vector);
2339}
2340
2341static int btintel_pcie_get_debug_info_addr(struct hci_dev *hdev)
2342{
2343 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2344 struct btintel_pcie_trigger_evt *evt;
2345 u8 param[1] = {0x10};
2346 struct sk_buff *skb;
2347 int err = 0;
2348
2349 skb = __hci_cmd_sync(hdev, BTINTEL_HCI_OP_DEBUG, 1, param,
2350 HCI_CMD_TIMEOUT);
2351 if (IS_ERR(skb)) {
2352 bt_dev_err(hdev, "Reading Intel read debug info address command failed (%ld)",
2353 PTR_ERR(skb));
2354 /* Not all Intel products supports this command */
2355 if (PTR_ERR(skb) == -EOPNOTSUPP)
2356 return 0;
2357 return PTR_ERR(skb);
2358 }
2359
2360 if (skb->len < (1 + sizeof(*evt))) {
2361 bt_dev_err(hdev, "Debug info response too short (%u bytes)", skb->len);
2362 err = -EIO;
2363 goto exit_error;
2364 }
2365
2366 /* Check the status */
2367 if (skb->data[0]) {
2368 bt_dev_err(hdev, "Reading Intel read debug info command failed (0x%2.2x)",
2369 skb->data[0]);
2370 err = -EIO;
2371 goto exit_error;
2372 }
2373
2374 /* Consume Command Complete Status field */
2375 skb_pull(skb, 1);
2376
2377 evt = (void *)skb->data;
2378
2379 data->debug_evt_addr = le32_to_cpu(evt->addr);
2380 data->debug_evt_size = le32_to_cpu(evt->size);
2381
2382 bt_dev_dbg(hdev, "config type: %u config len: %u debug event addr: 0x%8.8x size: 0x%8.8x",
2383 evt->type, evt->len, data->debug_evt_addr,
2384 data->debug_evt_size);
2385exit_error:
2386 kfree_skb(skb);
2387 return err;
2388}
2389
2390static int btintel_pcie_setup_internal(struct hci_dev *hdev)
2391{
2392 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2393 const u8 param[1] = { 0xFF };
2394 struct intel_version_tlv ver_tlv;
2395 struct sk_buff *skb;
2396 int err;
2397
2398 BT_DBG("%s", hdev->name);
2399
2400 skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT);
2401 if (IS_ERR(skb)) {
2402 bt_dev_err(hdev, "Reading Intel version command failed (%ld)",
2403 PTR_ERR(skb));
2404 return PTR_ERR(skb);
2405 }
2406
2407 /* Check the status */
2408 if (skb->data[0]) {
2409 bt_dev_err(hdev, "Intel Read Version command failed (%02x)",
2410 skb->data[0]);
2411 err = -EIO;
2412 goto exit_error;
2413 }
2414
2415 /* Apply the common HCI quirks for Intel device */
2416 hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER);
2417 hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY);
2418 hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_DIAG);
2419
2420 /* Set up the quality report callback for Intel devices */
2421 hdev->set_quality_report = btintel_set_quality_report;
2422
2423 memset(&ver_tlv, 0, sizeof(ver_tlv));
2424 /* For TLV type device, parse the tlv data */
2425 err = btintel_parse_version_tlv(hdev, &ver_tlv, skb);
2426 if (err) {
2427 bt_dev_err(hdev, "Failed to parse TLV version information");
2428 goto exit_error;
2429 }
2430
2431 switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) {
2432 case 0x37:
2433 break;
2434 default:
2435 bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)",
2436 INTEL_HW_PLATFORM(ver_tlv.cnvi_bt));
2437 err = -EINVAL;
2438 goto exit_error;
2439 }
2440
2441 /* Check for supported iBT hardware variants of this firmware
2442 * loading method.
2443 *
2444 * This check has been put in place to ensure correct forward
2445 * compatibility options when newer hardware variants come
2446 * along.
2447 */
2448 switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) {
2449 case 0x1e: /* BzrI */
2450 case 0x1f: /* ScP */
2451 case 0x20: /* ScP2 */
2452 case 0x21: /* ScP2 F */
2453 case 0x22: /* BzrIW */
2454 /* Display version information of TLV type */
2455 btintel_version_info_tlv(hdev, &ver_tlv);
2456
2457 /* Apply the device specific HCI quirks for TLV based devices
2458 *
2459 * All TLV based devices support WBS
2460 */
2461 hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED);
2462
2463 /* Setup MSFT Extension support */
2464 btintel_set_msft_opcode(hdev,
2465 INTEL_HW_VARIANT(ver_tlv.cnvi_bt));
2466
2467 err = btintel_bootloader_setup_tlv(hdev, &ver_tlv);
2468 if (err)
2469 goto exit_error;
2470 break;
2471 default:
2472 bt_dev_err(hdev, "Unsupported Intel hw variant (%u)",
2473 INTEL_HW_VARIANT(ver_tlv.cnvi_bt));
2474 err = -EINVAL;
2475 goto exit_error;
2476 }
2477
2478 data->dmp_hdr.cnvi_top = ver_tlv.cnvi_top;
2479 data->dmp_hdr.cnvr_top = ver_tlv.cnvr_top;
2480 data->dmp_hdr.fw_timestamp = ver_tlv.timestamp;
2481 data->dmp_hdr.fw_build_type = ver_tlv.build_type;
2482 data->dmp_hdr.fw_build_num = ver_tlv.build_num;
2483 data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt;
2484
2485 if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03)
2486 data->dmp_hdr.fw_git_sha1 = ver_tlv.git_sha1;
2487
2488 err = btintel_pcie_get_debug_info_addr(hdev);
2489 if (err)
2490 goto exit_error;
2491
2492 btintel_print_fseq_info(hdev);
2493exit_error:
2494 kfree_skb(skb);
2495
2496 return err;
2497}
2498
2499static int btintel_pcie_setup(struct hci_dev *hdev)
2500{
2501 int err, fw_dl_retry = 0;
2502 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2503
2504 while ((err = btintel_pcie_setup_internal(hdev)) && fw_dl_retry++ < 1) {
2505 bt_dev_err(hdev, "Firmware download retry count: %d",
2506 fw_dl_retry);
2507 btintel_pcie_dump_debug_registers(hdev);
2508 btintel_pcie_disable_interrupts(data);
2509 btintel_pcie_synchronize_irqs(data);
2510 err = btintel_pcie_reset_bt(data);
2511 if (err) {
2512 bt_dev_err(hdev, "Failed to do shr reset: %d", err);
2513 break;
2514 }
2515 usleep_range(10000, 12000);
2516 btintel_pcie_reset_ia(data);
2517 btintel_pcie_enable_interrupts(data);
2518 btintel_pcie_config_msix(data);
2519 err = btintel_pcie_enable_bt(data);
2520 if (err) {
2521 bt_dev_err(hdev, "Failed to enable hardware: %d", err);
2522 break;
2523 }
2524 btintel_pcie_start_rx(data);
2525 }
2526
2527 if (!err)
2528 set_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags);
2529 return err;
2530}
2531
2532static struct btintel_pcie_dev_recovery *
2533btintel_pcie_get_recovery(struct pci_dev *pdev, struct device *dev)
2534{
2535 struct btintel_pcie_dev_recovery *tmp, *data = NULL;
2536 const char *name = pci_name(pdev);
2537 const size_t name_len = strlen(name) + 1;
2538 struct hci_dev *hdev = to_hci_dev(dev);
2539
2540 spin_lock(&btintel_pcie_recovery_lock);
2541 list_for_each_entry(tmp, &btintel_pcie_recovery_list, list) {
2542 if (strcmp(tmp->name, name))
2543 continue;
2544 data = tmp;
2545 break;
2546 }
2547 spin_unlock(&btintel_pcie_recovery_lock);
2548
2549 if (data) {
2550 bt_dev_dbg(hdev, "Found restart data for BDF: %s", data->name);
2551 return data;
2552 }
2553
2554 data = kzalloc_flex(*data, name, name_len, GFP_ATOMIC);
2555 if (!data)
2556 return NULL;
2557
2558 strscpy(data->name, name, name_len);
2559 spin_lock(&btintel_pcie_recovery_lock);
2560 list_add_tail(&data->list, &btintel_pcie_recovery_list);
2561 spin_unlock(&btintel_pcie_recovery_lock);
2562
2563 return data;
2564}
2565
2566static void btintel_pcie_free_restart_list(void)
2567{
2568 struct btintel_pcie_dev_recovery *tmp;
2569
2570 while ((tmp = list_first_entry_or_null(&btintel_pcie_recovery_list,
2571 typeof(*tmp), list))) {
2572 list_del(&tmp->list);
2573 kfree(tmp);
2574 }
2575}
2576
2577static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev,
2578 struct device *dev)
2579{
2580 struct btintel_pcie_dev_recovery *data;
2581 time64_t retry_window;
2582
2583 data = btintel_pcie_get_recovery(pdev, dev);
2584 if (!data)
2585 return;
2586
2587 retry_window = ktime_get_boottime_seconds() - data->last_error;
2588 if (data->count == 0) {
2589 data->last_error = ktime_get_boottime_seconds();
2590 data->count++;
2591 } else if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
2592 data->count <= BTINTEL_PCIE_FLR_MAX_RETRY) {
2593 data->count++;
2594 } else if (retry_window > BTINTEL_PCIE_RESET_WINDOW_SECS) {
2595 data->last_error = 0;
2596 data->count = 0;
2597 }
2598}
2599
2600static int btintel_pcie_acpi_reset_method(struct btintel_pcie_data *data)
2601{
2602 union acpi_object *obj, argv4;
2603 acpi_handle handle;
2604 int ret;
2605 struct pldr_mode {
2606 __le16 cmd_type;
2607 __le16 cmd_payload;
2608 } __packed;
2609
2610 /* set 1 for _PRR mode
2611 * Product Reset (PLDR Abort flow)
2612 */
2613 static const struct pldr_mode mode = {
2614 .cmd_type = cpu_to_le16(1),
2615 .cmd_payload = cpu_to_le16(BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET |
2616 BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR),
2617 };
2618 struct hci_dev *hdev = data->hdev;
2619
2620 handle = ACPI_HANDLE(GET_HCIDEV_DEV(data->hdev));
2621 if (!handle) {
2622 bt_dev_err(data->hdev, "No support for bluetooth device in ACPI firmware");
2623 return -EACCES;
2624 }
2625
2626 if (!acpi_has_method(handle, "_PRR")) {
2627 bt_dev_err(data->hdev, "No support for _PRR ACPI method, cold boot");
2628 return -ENODEV;
2629 }
2630
2631 argv4.buffer.type = ACPI_TYPE_BUFFER;
2632 argv4.buffer.length = sizeof(mode);
2633 argv4.buffer.pointer = (void *)&mode;
2634
2635 obj = acpi_evaluate_dsm(handle, &btintel_guid_dsm, 0,
2636 BTINTEL_PCIE_DSM_DYNAMIC_PLDR, &argv4);
2637 if (!obj) {
2638 bt_dev_err(data->hdev, "Failed to call dsm to set reset method");
2639 return -EIO;
2640 }
2641 ACPI_FREE(obj);
2642
2643 pci_dev_lock(data->pdev);
2644 pci_save_state(data->pdev);
2645 ret = btintel_acpi_reset_method(hdev);
2646 if (ret)
2647 bt_dev_err(data->hdev, "ACPI _PRR reset failed (%d), PLDR incomplete",
2648 ret);
2649 pci_restore_state(data->pdev);
2650 pci_dev_unlock(data->pdev);
2651 return ret;
2652}
2653
2654static void btintel_pcie_perform_pldr(struct btintel_pcie_data *data)
2655{
2656 struct pci_dev *pdev = data->pdev;
2657 struct pci_dev *wifi = NULL;
2658 struct pci_bus *bus;
2659 int ret;
2660 /* on integrated we have to look up by ID (same bus) */
2661 static const struct pci_device_id wifi_device_ids[] = {
2662 #define WIFI_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) }
2663 WIFI_DEV(0xA840), /* LNL */
2664 WIFI_DEV(0xE440), /* PTL-P */
2665 WIFI_DEV(0xE340), /* PTL-H */
2666 WIFI_DEV(0xD340), /* NVL-H */
2667 WIFI_DEV(0x6E70), /* NVL-S */
2668 WIFI_DEV(0x4D40), /* WCL */
2669 {}
2670 };
2671 struct pci_dev *tmp = NULL;
2672
2673 bus = pdev->bus;
2674 if (!bus)
2675 return;
2676
2677 list_for_each_entry(tmp, &bus->devices, bus_list) {
2678 if (pci_match_id(wifi_device_ids, tmp)) {
2679 wifi = pci_dev_get(tmp);
2680 break;
2681 }
2682 }
2683
2684 if (wifi)
2685 device_release_driver(&wifi->dev);
2686
2687 /* Wi-Fi is fully unbound before the reset and fully reprobed after
2688 * the normal PCI probe path handles all state setup from scratch.
2689 * BT needs pci_save_state()/pci_restore_state() because the BT driver
2690 * is still partially attached when the _PRR runs (it hasn't been unbound yet).
2691 * The PCI device needs to remain minimally functional so that
2692 * device_reprobe(&pdev->dev) can work afterward
2693 */
2694 ret = btintel_pcie_acpi_reset_method(data);
2695
2696 if (wifi) {
2697 if (device_reprobe(&wifi->dev))
2698 BT_ERR("WiFi reprobe failed for BDF:%s", pci_name(wifi));
2699 pci_dev_put(wifi);
2700 }
2701
2702 if (!ret) {
2703 if (device_reprobe(&pdev->dev))
2704 BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev));
2705 }
2706}
2707
2708/*
2709 * Issue a Function Level Reset and hand teardown/re-init off to the PCI
2710 * core via device_reprobe(), mirroring the PLDR path's contract.
2711 *
2712 * Caller must hold pci_lock_rescan_remove() and must have already
2713 * disabled interrupts and drained both rx_work and coredump_work.
2714 */
2715static int btintel_pcie_perform_flr(struct btintel_pcie_data *data)
2716{
2717 struct pci_dev *pdev = data->pdev;
2718 int err;
2719
2720 /* pci_try_reset_function() avoids the device_lock ABBA against
2721 * btintel_pcie_remove(): .remove() runs with device_lock held and
2722 * then waits for this work via disable_work_sync(); the blocking
2723 * pci_reset_function() would deadlock by trying to re-acquire
2724 * device_lock here.
2725 */
2726 err = pci_try_reset_function(pdev);
2727 if (err) {
2728 BT_ERR("Failed resetting the pcie device (%d)", err);
2729 return err;
2730 }
2731
2732 /* device_reprobe() always detaches the driver first (running
2733 * .remove(), which frees 'data'); any re-probe failure leaves the
2734 * device unbound but 'data' is already gone, so just log it.
2735 */
2736 if (device_reprobe(&pdev->dev))
2737 BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev));
2738
2739 return 0;
2740}
2741
2742static void btintel_pcie_reset_work(struct work_struct *wk)
2743{
2744 struct btintel_pcie_data *data =
2745 container_of(wk, struct btintel_pcie_data, reset_work);
2746 struct pci_dev *pdev = data->pdev;
2747
2748 pci_lock_rescan_remove();
2749
2750 if (!pdev->bus)
2751 goto out;
2752
2753 if (!data)
2754 goto out;
2755
2756 btintel_pcie_disable_interrupts(data);
2757 btintel_pcie_synchronize_irqs(data);
2758
2759 flush_work(&data->rx_work);
2760 /* Drain any in-flight dump workers and block new ones across reset.
2761 * Safe from self-deadlock: they all run on a separate wq.
2762 */
2763 disable_work_sync(&data->coredump_work);
2764 disable_work_sync(&data->hwexp_work);
2765 disable_work_sync(&data->fwtrigger_work);
2766
2767 bt_dev_dbg(data->hdev, "Release bluetooth interface");
2768
2769 /* Both reset paths follow the same contract: on success they
2770 * destroy 'data' via device_reprobe() (a fresh probe re-INIT_WORKs
2771 * the dump workers with disable count 0), so enable_work() must
2772 * NOT be called on the success path. Only the FLR path can fail
2773 * with 'data' still alive, in which case we balance the
2774 * disable_work_sync() calls above so a later successful reset is
2775 * not permanently blocked.
2776 *
2777 * pci_lock_rescan_remove() (held above) serializes against PCI
2778 * device addition/removal (hotplug), so no device can be added to
2779 * or removed from the bus list while this code runs.
2780 */
2781 if (data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR) {
2782 btintel_pcie_perform_pldr(data);
2783 goto out;
2784 }
2785
2786 if (btintel_pcie_perform_flr(data)) {
2787 enable_work(&data->coredump_work);
2788 enable_work(&data->hwexp_work);
2789 enable_work(&data->fwtrigger_work);
2790 }
2791
2792out:
2793 pci_dev_put(pdev);
2794 pci_unlock_rescan_remove();
2795}
2796
2797/* Schedule a device reset of the requested type.
2798 *
2799 * BTINTEL_PCIE_RECOVERY_IN_PROGRESS serializes all reset requesters
2800 * (sysfs reset attribute, hci_cmd_timeout(), hw_error, resume error
2801 * path, etc.) so that:
2802 *
2803 * - dev_data->reset_type is written by exactly one caller (the
2804 * thread that wins test_and_set_bit), eliminating the race where
2805 * a second hw_error could clobber an already-scheduled reset's
2806 * type;
2807 * - the write happens AFTER the bit is set, so reset_work observes
2808 * it through schedule_work()'s memory ordering;
2809 * - losers return without touching reset_type or scheduling the
2810 * work, so concurrent triggers are silently coalesced into the
2811 * in-flight one (whose recovery will reinitialize the device
2812 * regardless of the dropped trigger's variant).
2813 *
2814 * The bit is cleared only by .remove() / re-probe via fresh devm
2815 * allocation, which is the intended one-shot semantics: a reset
2816 * tears down and re-probes 'data', so there is no "in-flight"
2817 * reset to follow up after device_reprobe() succeeds.
2818 */
2819static void btintel_pcie_request_reset(struct btintel_pcie_data *data,
2820 enum btintel_pcie_reset_type type)
2821{
2822 if (!test_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags))
2823 return;
2824
2825 if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
2826 return;
2827
2828 data->reset_type = type;
2829
2830 pci_dev_get(data->pdev);
2831 schedule_work(&data->reset_work);
2832}
2833
2834static void btintel_pcie_hci_reset(struct hci_dev *hdev)
2835{
2836 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2837
2838 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
2839}
2840
2841static ssize_t vendor_reset_store(struct device *dev,
2842 struct device_attribute *attr,
2843 const char *buf, size_t count)
2844{
2845 unsigned int val;
2846 struct pci_dev *pdev = to_pci_dev(dev);
2847 struct btintel_pcie_data *data = pci_get_drvdata(pdev);
2848
2849 if (!data || !data->hdev)
2850 return -ENODEV;
2851
2852 if (kstrtouint(buf, 10, &val) || val != 0) {
2853 bt_dev_warn(data->hdev, "PLDR rejected: invalid input");
2854 return -EINVAL;
2855 }
2856
2857 bt_dev_info(data->hdev, "PLDR triggered via sysfs");
2858 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_PLDR);
2859
2860 return count;
2861}
2862
2863static ssize_t vendor_reset_show(struct device *dev,
2864 struct device_attribute *attr, char *buf)
2865{
2866 return sysfs_emit(buf, "0 - PLDR\n");
2867}
2868
2869static DEVICE_ATTR_RW(vendor_reset);
2870
2871static struct attribute *btintel_pcie_attrs[] = {
2872 &dev_attr_vendor_reset.attr,
2873 NULL,
2874};
2875
2876ATTRIBUTE_GROUPS(btintel_pcie);
2877
2878static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code)
2879{
2880 struct btintel_pcie_dev_recovery *rec;
2881 struct btintel_pcie_data *dev_data = hci_get_drvdata(hdev);
2882 struct pci_dev *pdev = dev_data->pdev;
2883 enum btintel_pcie_reset_type type;
2884 time64_t retry_window;
2885
2886 if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &dev_data->flags))
2887 return;
2888
2889 btintel_pcie_dump_debug_registers(hdev);
2890
2891 rec = btintel_pcie_get_recovery(pdev, &hdev->dev);
2892 if (!rec)
2893 return;
2894
2895 type = (code == 0x13) ? BTINTEL_PCIE_IOSF_PRR_PLDR
2896 : BTINTEL_PCIE_IOSF_PRR_FLR;
2897
2898 bt_dev_err(hdev, "Encountered exception err:0x%x triggering: %s", code,
2899 type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR");
2900 retry_window = ktime_get_boottime_seconds() - rec->last_error;
2901
2902 if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
2903 rec->count >= BTINTEL_PCIE_FLR_MAX_RETRY) {
2904 bt_dev_err(hdev, "Exhausted maximum: %d recovery attempts: %d",
2905 BTINTEL_PCIE_FLR_MAX_RETRY, rec->count);
2906 bt_dev_dbg(hdev, "Boot time: %lld seconds",
2907 ktime_get_boottime_seconds());
2908 bt_dev_dbg(hdev, "last error at: %lld seconds",
2909 rec->last_error);
2910 return;
2911 }
2912 btintel_pcie_inc_recovery_count(pdev, &hdev->dev);
2913 btintel_pcie_request_reset(dev_data, type);
2914}
2915
2916static bool btintel_pcie_wakeup(struct hci_dev *hdev)
2917{
2918 struct btintel_pcie_data *data = hci_get_drvdata(hdev);
2919
2920 return device_may_wakeup(&data->pdev->dev);
2921}
2922
2923static const struct {
2924 u16 opcode;
2925 const char *desc;
2926} btintel_pcie_hci_drv_supported_commands[] = {
2927 /* Common commands */
2928 { HCI_DRV_OP_READ_INFO, "Read Info" },
2929};
2930
2931static int btintel_pcie_hci_drv_read_info(struct hci_dev *hdev, void *data,
2932 u16 data_len)
2933{
2934 struct hci_drv_rp_read_info *rp;
2935 size_t rp_size;
2936 int err, i;
2937 u16 opcode, num_supported_commands =
2938 ARRAY_SIZE(btintel_pcie_hci_drv_supported_commands);
2939
2940 rp_size = struct_size(rp, supported_commands, num_supported_commands);
2941
2942 rp = kmalloc(rp_size, GFP_KERNEL);
2943 if (!rp)
2944 return -ENOMEM;
2945
2946 strscpy_pad(rp->driver_name, KBUILD_MODNAME);
2947
2948 rp->num_supported_commands = cpu_to_le16(num_supported_commands);
2949 for (i = 0; i < num_supported_commands; i++) {
2950 opcode = btintel_pcie_hci_drv_supported_commands[i].opcode;
2951 bt_dev_dbg(hdev,
2952 "Supported HCI Drv command (0x%02x|0x%04x): %s",
2953 hci_opcode_ogf(opcode),
2954 hci_opcode_ocf(opcode),
2955 btintel_pcie_hci_drv_supported_commands[i].desc);
2956 rp->supported_commands[i] = cpu_to_le16(opcode);
2957 }
2958
2959 err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO,
2960 HCI_DRV_STATUS_SUCCESS,
2961 rp, rp_size);
2962
2963 kfree(rp);
2964 return err;
2965}
2966
2967static const struct hci_drv_handler btintel_pcie_hci_drv_common_handlers[] = {
2968 { btintel_pcie_hci_drv_read_info, HCI_DRV_READ_INFO_SIZE },
2969};
2970
2971static const struct hci_drv_handler btintel_pcie_hci_drv_specific_handlers[] = {};
2972
2973static struct hci_drv btintel_pcie_hci_drv = {
2974 .common_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_common_handlers),
2975 .common_handlers = btintel_pcie_hci_drv_common_handlers,
2976 .specific_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_specific_handlers),
2977 .specific_handlers = btintel_pcie_hci_drv_specific_handlers,
2978};
2979
2980static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data)
2981{
2982 int err;
2983 struct hci_dev *hdev;
2984
2985 hdev = hci_alloc_dev_priv(sizeof(struct btintel_data));
2986 if (!hdev)
2987 return -ENOMEM;
2988
2989 hdev->bus = HCI_PCI;
2990 hci_set_drvdata(hdev, data);
2991
2992 SET_HCIDEV_DEV(hdev, &data->pdev->dev);
2993
2994 hdev->manufacturer = 2;
2995 hdev->open = btintel_pcie_open;
2996 hdev->close = btintel_pcie_close;
2997 hdev->send = btintel_pcie_send_frame;
2998 hdev->setup = btintel_pcie_setup;
2999 hdev->shutdown = btintel_shutdown_combined;
3000 hdev->hw_error = btintel_pcie_hw_error;
3001 hdev->set_diag = btintel_set_diag;
3002 hdev->set_bdaddr = btintel_set_bdaddr;
3003 hdev->reset = btintel_pcie_hci_reset;
3004 hdev->wakeup = btintel_pcie_wakeup;
3005 hdev->hci_drv = &btintel_pcie_hci_drv;
3006
3007 err = hci_register_dev(hdev);
3008 if (err < 0) {
3009 BT_ERR("Failed to register to hdev (%d)", err);
3010 hci_free_dev(hdev);
3011 return err;
3012 }
3013
3014 /* Publish hdev only after successful registration; the coredump
3015 * worker bails on !data->hdev, so it never observes a half-set-up
3016 * device.
3017 */
3018 data->hdev = hdev;
3019 data->dmp_hdr.driver_name = KBUILD_MODNAME;
3020 return 0;
3021}
3022
3023static int btintel_pcie_probe(struct pci_dev *pdev,
3024 const struct pci_device_id *ent)
3025{
3026 int err;
3027 struct btintel_pcie_data *data;
3028
3029 if (!pdev)
3030 return -ENODEV;
3031
3032 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
3033 if (!data)
3034 return -ENOMEM;
3035
3036 data->pdev = pdev;
3037
3038 spin_lock_init(&data->irq_lock);
3039 spin_lock_init(&data->hci_rx_lock);
3040
3041 init_waitqueue_head(&data->gp0_wait_q);
3042 data->gp0_received = false;
3043
3044 init_waitqueue_head(&data->tx_wait_q);
3045 data->tx_wait_done = false;
3046
3047 data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI);
3048 if (!data->workqueue)
3049 return -ENOMEM;
3050
3051 data->dump_workqueue = alloc_ordered_workqueue(KBUILD_MODNAME "_cd", 0);
3052 if (!data->dump_workqueue) {
3053 destroy_workqueue(data->workqueue);
3054 return -ENOMEM;
3055 }
3056
3057 skb_queue_head_init(&data->rx_skb_q);
3058 INIT_WORK(&data->rx_work, btintel_pcie_rx_work);
3059 INIT_WORK(&data->reset_work, btintel_pcie_reset_work);
3060 INIT_WORK(&data->coredump_work, btintel_pcie_coredump_worker);
3061 INIT_WORK(&data->hwexp_work, btintel_pcie_hwexp_worker);
3062 INIT_WORK(&data->fwtrigger_work, btintel_pcie_fwtrigger_worker);
3063
3064 data->boot_stage_cache = 0x00;
3065 data->img_resp_cache = 0x00;
3066 /* FLR can be invoked by echoing to debugfs path, so explicitly
3067 * initialized
3068 */
3069 data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
3070 err = btintel_pcie_config_pcie(pdev, data);
3071 if (err)
3072 goto exit_error;
3073
3074 pci_set_drvdata(pdev, data);
3075
3076 err = btintel_pcie_alloc(data);
3077 if (err)
3078 goto exit_error;
3079
3080 err = btintel_pcie_enable_bt(data);
3081 if (err)
3082 goto exit_error;
3083
3084 /* CNV information (CNVi and CNVr) is in CSR */
3085 data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG);
3086
3087 data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG);
3088
3089 err = btintel_pcie_start_rx(data);
3090 if (err)
3091 goto exit_error;
3092
3093 err = btintel_pcie_setup_hdev(data);
3094 if (err)
3095 goto exit_error;
3096
3097 bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi,
3098 data->cnvr);
3099 return 0;
3100
3101exit_error:
3102 /* reset device before exit */
3103 btintel_pcie_reset_bt(data);
3104
3105 destroy_workqueue(data->dump_workqueue);
3106
3107 pci_clear_master(pdev);
3108
3109 pci_set_drvdata(pdev, NULL);
3110
3111 return err;
3112}
3113
3114static void btintel_pcie_remove(struct pci_dev *pdev)
3115{
3116 struct btintel_pcie_data *data;
3117
3118 data = pci_get_drvdata(pdev);
3119 if (!data) {
3120 BT_WARN("PCI driver data is NULL, aborting remove");
3121 return;
3122 }
3123
3124 /* Permanently block all dump triggers and drain the workers before
3125 * tearing down. Must run before disable_work_sync(&reset_work) so
3126 * the disable counters stay >= 1 even after reset_work()'s
3127 * balanced enable_work() (counter 2 -> 1, never reaching 0).
3128 */
3129 disable_work_sync(&data->coredump_work);
3130 disable_work_sync(&data->hwexp_work);
3131 disable_work_sync(&data->fwtrigger_work);
3132
3133 /* Cancel pending reset work. Skip only when remove() is called from
3134 * within the reset work itself (PLDR device_reprobe path) to avoid
3135 * deadlock. current_work() returns the work_struct of the caller if
3136 * we are in a workqueue context.
3137 */
3138 if (current_work() != &data->reset_work)
3139 disable_work_sync(&data->reset_work);
3140
3141 btintel_pcie_disable_interrupts(data);
3142
3143 btintel_pcie_synchronize_irqs(data);
3144
3145 flush_work(&data->rx_work);
3146
3147 btintel_pcie_reset_bt(data);
3148 for (int i = 0; i < data->alloc_vecs; i++) {
3149 struct msix_entry *msix_entry;
3150
3151 msix_entry = &data->msix_entries[i];
3152 devm_free_irq(&pdev->dev, msix_entry->vector, msix_entry);
3153 }
3154
3155 pci_free_irq_vectors(pdev);
3156
3157 btintel_pcie_release_hdev(data);
3158
3159 destroy_workqueue(data->dump_workqueue);
3160 destroy_workqueue(data->workqueue);
3161
3162 btintel_pcie_free(data);
3163
3164 pci_clear_master(pdev);
3165
3166 pci_set_drvdata(pdev, NULL);
3167}
3168
3169#ifdef CONFIG_DEV_COREDUMP
3170static void btintel_pcie_coredump(struct device *dev)
3171{
3172 struct pci_dev *pdev = to_pci_dev(dev);
3173 struct btintel_pcie_data *data = pci_get_drvdata(pdev);
3174
3175 if (!data)
3176 return;
3177
3178 btintel_pcie_queue_coredump(data,
3179 BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER);
3180}
3181#endif
3182
3183static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate)
3184{
3185 int retry = 0, status;
3186 u32 dx_intr_timeout_ms = 200;
3187
3188 do {
3189 data->gp0_received = false;
3190
3191 btintel_pcie_wr_sleep_cntrl(data, dxstate);
3192
3193 status = wait_event_timeout(data->gp0_wait_q, data->gp0_received,
3194 msecs_to_jiffies(dx_intr_timeout_ms));
3195
3196 if (status)
3197 return 0;
3198
3199 bt_dev_warn(data->hdev,
3200 "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d",
3201 dx_intr_timeout_ms, dxstate, retry);
3202
3203 /* clear gp0 cause */
3204 btintel_pcie_clr_reg_bits(data,
3205 BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES,
3206 BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0);
3207
3208 /* A hardware bug may cause the alive interrupt to be missed.
3209 * Check if the controller reached the expected state and retry
3210 * the operation only if it hasn't.
3211 */
3212 if (dxstate == BTINTEL_PCIE_STATE_D0) {
3213 if (btintel_pcie_in_d0(data))
3214 return 0;
3215 } else {
3216 if (btintel_pcie_in_d3(data))
3217 return 0;
3218 }
3219
3220 } while (++retry < BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES);
3221
3222 return -EBUSY;
3223}
3224
3225static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg)
3226{
3227 struct pci_dev *pdev = to_pci_dev(dev);
3228 struct btintel_pcie_data *data;
3229 ktime_t start;
3230 u32 dxstate;
3231 int err;
3232
3233 data = pci_get_drvdata(pdev);
3234
3235 dxstate = (mesg.event == PM_EVENT_SUSPEND ?
3236 BTINTEL_PCIE_STATE_D3_HOT : BTINTEL_PCIE_STATE_D3_COLD);
3237
3238 data->pm_sx_event = mesg.event;
3239
3240 start = ktime_get();
3241
3242 /* Refer: 6.4.11.7 -> Platform power management */
3243 err = btintel_pcie_set_dxstate(data, dxstate);
3244
3245 if (err)
3246 return err;
3247
3248 bt_dev_dbg(data->hdev,
3249 "device entered into d3 state from d0 in %lld us",
3250 ktime_to_us(ktime_get() - start));
3251 return err;
3252}
3253
3254static int btintel_pcie_suspend(struct device *dev)
3255{
3256 return btintel_pcie_suspend_late(dev, PMSG_SUSPEND);
3257}
3258
3259static int btintel_pcie_hibernate(struct device *dev)
3260{
3261 return btintel_pcie_suspend_late(dev, PMSG_HIBERNATE);
3262}
3263
3264static int btintel_pcie_freeze(struct device *dev)
3265{
3266 return btintel_pcie_suspend_late(dev, PMSG_FREEZE);
3267}
3268
3269static int btintel_pcie_resume(struct device *dev)
3270{
3271 struct pci_dev *pdev = to_pci_dev(dev);
3272 struct btintel_pcie_data *data;
3273 ktime_t start;
3274 int err;
3275
3276 data = pci_get_drvdata(pdev);
3277 data->gp0_received = false;
3278
3279 start = ktime_get();
3280
3281 /* When the system enters S4 (hibernate) mode, bluetooth device loses
3282 * power, which results in the erasure of its loaded firmware.
3283 * Consequently, function level reset (flr) is required on system
3284 * resume to bring the controller back into an operational state by
3285 * initiating a new firmware download.
3286 */
3287
3288 if (data->pm_sx_event == PM_EVENT_FREEZE ||
3289 data->pm_sx_event == PM_EVENT_HIBERNATE) {
3290 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
3291 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
3292 return 0;
3293 }
3294
3295 /* Refer: 6.4.11.7 -> Platform power management */
3296 err = btintel_pcie_set_dxstate(data, BTINTEL_PCIE_STATE_D0);
3297
3298 if (err == 0) {
3299 bt_dev_dbg(data->hdev,
3300 "device entered into d0 state from d3 in %lld us",
3301 ktime_to_us(ktime_get() - start));
3302 return err;
3303 }
3304
3305 /* Trigger function level reset if the controller is in error
3306 * state during resume() to bring back the controller to
3307 * operational mode
3308 */
3309
3310 data->boot_stage_cache = btintel_pcie_rd_reg32(data,
3311 BTINTEL_PCIE_CSR_BOOT_STAGE_REG);
3312 if (btintel_pcie_in_error(data) ||
3313 btintel_pcie_in_device_halt(data)) {
3314 bt_dev_err(data->hdev, "Controller in error state for D0 entry");
3315 btintel_pcie_queue_coredump(data,
3316 BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT);
3317 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
3318 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR);
3319 }
3320 return err;
3321}
3322
3323static const struct dev_pm_ops btintel_pcie_pm_ops = {
3324 .suspend = btintel_pcie_suspend,
3325 .resume = btintel_pcie_resume,
3326 .freeze = btintel_pcie_freeze,
3327 .thaw = btintel_pcie_resume,
3328 .poweroff = btintel_pcie_hibernate,
3329 .restore = btintel_pcie_resume,
3330};
3331
3332static struct pci_driver btintel_pcie_driver = {
3333 .name = KBUILD_MODNAME,
3334 .id_table = btintel_pcie_table,
3335 .probe = btintel_pcie_probe,
3336 .remove = btintel_pcie_remove,
3337 .driver.pm = pm_sleep_ptr(&btintel_pcie_pm_ops),
3338 .dev_groups = btintel_pcie_groups,
3339#ifdef CONFIG_DEV_COREDUMP
3340 .driver.coredump = btintel_pcie_coredump
3341#endif
3342};
3343
3344static int __init btintel_pcie_init(void)
3345{
3346 return pci_register_driver(&btintel_pcie_driver);
3347}
3348
3349static void __exit btintel_pcie_exit(void)
3350{
3351 pci_unregister_driver(&btintel_pcie_driver);
3352 btintel_pcie_free_restart_list();
3353}
3354
3355module_init(btintel_pcie_init);
3356module_exit(btintel_pcie_exit);
3357
3358MODULE_AUTHOR("Tedd Ho-Jeong An <[email protected]>");
3359MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION);
3360MODULE_VERSION(VERSION);
3361MODULE_LICENSE("GPL");