projects/linux·blob master · refs

521 lines · 12.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * TPH (TLP Processing Hints) support
4 *
5 * Copyright (C) 2024 Advanced Micro Devices, Inc.
6 * Eric Van Tassell <[email protected]>
7 * Wei Huang <[email protected]>
8 */
9#include <linux/pci.h>
10#include <linux/pci-acpi.h>
11#include <linux/msi.h>
12#include <linux/bitfield.h>
13#include <linux/pci-tph.h>
14
15#include "pci.h"
16
17/* System-wide TPH disabled */
18static bool pci_tph_disabled;
19
20#ifdef CONFIG_ACPI
21/*
22 * The st_info struct defines the Steering Tag (ST) info returned by the
23 * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache
24 * Locality TPH Features"), as specified in the approved ECN for PCI Firmware
25 * Spec and available at https://members.pcisig.com/wg/PCI-SIG/document/15470.
26 *
27 * @vm_st_valid: 8-bit ST for volatile memory is valid
28 * @vm_xst_valid: 16-bit extended ST for volatile memory is valid
29 * @vm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
30 * @vm_st: 8-bit ST for volatile mem
31 * @vm_xst: 16-bit extended ST for volatile mem
32 * @pm_st_valid: 8-bit ST for persistent memory is valid
33 * @pm_xst_valid: 16-bit extended ST for persistent memory is valid
34 * @pm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
35 * @pm_st: 8-bit ST for persistent mem
36 * @pm_xst: 16-bit extended ST for persistent mem
37 */
38union st_info {
39 struct {
40 u64 vm_st_valid : 1;
41 u64 vm_xst_valid : 1;
42 u64 vm_ph_ignore : 1;
43 u64 rsvd1 : 5;
44 u64 vm_st : 8;
45 u64 vm_xst : 16;
46 u64 pm_st_valid : 1;
47 u64 pm_xst_valid : 1;
48 u64 pm_ph_ignore : 1;
49 u64 rsvd2 : 5;
50 u64 pm_st : 8;
51 u64 pm_xst : 16;
52 };
53 u64 value;
54};
55
56static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type,
57 union st_info *info)
58{
59 switch (req_type) {
60 case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */
61 switch (mem_type) {
62 case TPH_MEM_TYPE_VM:
63 if (info->vm_st_valid)
64 return info->vm_st;
65 break;
66 case TPH_MEM_TYPE_PM:
67 if (info->pm_st_valid)
68 return info->pm_st;
69 break;
70 }
71 break;
72 case PCI_TPH_REQ_EXT_TPH: /* 16-bit tag */
73 switch (mem_type) {
74 case TPH_MEM_TYPE_VM:
75 if (info->vm_xst_valid)
76 return info->vm_xst;
77 break;
78 case TPH_MEM_TYPE_PM:
79 if (info->pm_xst_valid)
80 return info->pm_xst;
81 break;
82 }
83 break;
84 default:
85 return 0;
86 }
87
88 return 0;
89}
90
91#define TPH_ST_DSM_FUNC_INDEX 0xF
92static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
93 union st_info *st_out)
94{
95 union acpi_object arg3[3], in_obj, *out_obj;
96
97 if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7,
98 BIT(TPH_ST_DSM_FUNC_INDEX)))
99 return AE_ERROR;
100
101 /* DWORD: feature ID (0 for processor cache ST query) */
102 arg3[0].integer.type = ACPI_TYPE_INTEGER;
103 arg3[0].integer.value = 0;
104
105 /* DWORD: target UID */
106 arg3[1].integer.type = ACPI_TYPE_INTEGER;
107 arg3[1].integer.value = cpu_uid;
108
109 /* QWORD: properties, all 0's */
110 arg3[2].integer.type = ACPI_TYPE_INTEGER;
111 arg3[2].integer.value = 0;
112
113 in_obj.type = ACPI_TYPE_PACKAGE;
114 in_obj.package.count = ARRAY_SIZE(arg3);
115 in_obj.package.elements = arg3;
116
117 out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7,
118 TPH_ST_DSM_FUNC_INDEX, &in_obj);
119 if (!out_obj)
120 return AE_ERROR;
121
122 if (out_obj->type != ACPI_TYPE_BUFFER) {
123 ACPI_FREE(out_obj);
124 return AE_ERROR;
125 }
126
127 st_out->value = *((u64 *)(out_obj->buffer.pointer));
128
129 ACPI_FREE(out_obj);
130
131 return AE_OK;
132}
133#endif
134
135/* Update the TPH Requester Enable field of TPH Control Register */
136static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type)
137{
138 u32 reg;
139
140 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
141
142 FIELD_MODIFY(PCI_TPH_CTRL_REQ_EN_MASK, &reg, req_type);
143
144 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
145}
146
147static u8 get_st_modes(struct pci_dev *pdev)
148{
149 u32 reg;
150
151 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
152 reg &= PCI_TPH_CAP_ST_NS | PCI_TPH_CAP_ST_IV | PCI_TPH_CAP_ST_DS;
153
154 return reg;
155}
156
157/**
158 * pcie_tph_get_st_table_loc - Return the device's ST table location
159 * @pdev: PCI device to query
160 *
161 * Return:
162 * PCI_TPH_LOC_NONE - Not present
163 * PCI_TPH_LOC_CAP - Located in the TPH Requester Extended Capability
164 * PCI_TPH_LOC_MSIX - Located in the MSI-X Table
165 */
166u32 pcie_tph_get_st_table_loc(struct pci_dev *pdev)
167{
168 u32 reg;
169
170 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
171
172 return FIELD_GET(PCI_TPH_CAP_LOC_MASK, reg);
173}
174EXPORT_SYMBOL(pcie_tph_get_st_table_loc);
175
176/*
177 * Return the size of ST table. If ST table is not in TPH Requester Extended
178 * Capability space, return 0. Otherwise return the ST Table Size + 1.
179 */
180u16 pcie_tph_get_st_table_size(struct pci_dev *pdev)
181{
182 u32 reg;
183 u32 loc;
184
185 /* Check ST table location first */
186 loc = pcie_tph_get_st_table_loc(pdev);
187
188 /* Convert loc to match with PCI_TPH_LOC_* defined in pci_regs.h */
189 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
190 if (loc != PCI_TPH_LOC_CAP)
191 return 0;
192
193 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
194
195 return FIELD_GET(PCI_TPH_CAP_ST_MASK, reg) + 1;
196}
197EXPORT_SYMBOL(pcie_tph_get_st_table_size);
198
199/* Return device's Root Port completer capability */
200static u8 get_rp_completer_type(struct pci_dev *pdev)
201{
202 struct pci_dev *rp;
203 u32 reg;
204 int ret;
205
206 rp = pcie_find_root_port(pdev);
207 if (!rp)
208 return 0;
209
210 ret = pcie_capability_read_dword(rp, PCI_EXP_DEVCAP2, &reg);
211 if (ret)
212 return 0;
213
214 return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg);
215}
216
217/* Write tag to ST table - Return 0 if OK, otherwise -errno */
218static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag)
219{
220 int st_table_size;
221 int offset;
222
223 /* Check if index is out of bound */
224 st_table_size = pcie_tph_get_st_table_size(pdev);
225 if (index >= st_table_size)
226 return -ENXIO;
227
228 offset = pdev->tph_cap + PCI_TPH_BASE_SIZEOF + index * sizeof(u16);
229
230 return pci_write_config_word(pdev, offset, tag);
231}
232
233/**
234 * pcie_tph_get_cpu_st() - Retrieve Steering Tag for a target memory associated
235 * with a specific CPU
236 * @pdev: PCI device
237 * @mem_type: target memory type (volatile or persistent RAM)
238 * @cpu: associated CPU id
239 * @tag: Steering Tag to be returned
240 *
241 * Return the Steering Tag for a target memory that is associated with a
242 * specific CPU as indicated by cpu.
243 *
244 * Return: 0 if success, otherwise negative value (-errno)
245 */
246int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type,
247 unsigned int cpu, u16 *tag)
248{
249#ifdef CONFIG_ACPI
250 struct pci_dev *rp;
251 acpi_handle rp_acpi_handle;
252 union st_info info;
253 u32 cpu_uid;
254 int ret;
255
256 ret = acpi_get_cpu_uid(cpu, &cpu_uid);
257 if (ret != 0)
258 return ret;
259
260 rp = pcie_find_root_port(pdev);
261 if (!rp || !rp->bus || !rp->bus->bridge)
262 return -ENODEV;
263
264 rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge);
265
266 if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) {
267 *tag = 0;
268 return -EINVAL;
269 }
270
271 *tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info);
272
273 pci_dbg(pdev, "get steering tag: mem_type=%s, cpu=%d, tag=%#04x\n",
274 (mem_type == TPH_MEM_TYPE_VM) ? "volatile" : "persistent",
275 cpu, *tag);
276
277 return 0;
278#else
279 return -ENODEV;
280#endif
281}
282EXPORT_SYMBOL(pcie_tph_get_cpu_st);
283
284/**
285 * pcie_tph_set_st_entry() - Set Steering Tag in the ST table entry
286 * @pdev: PCI device
287 * @index: ST table entry index
288 * @tag: Steering Tag to be written
289 *
290 * Figure out the proper location of ST table, either in the MSI-X table or
291 * in the TPH Extended Capability space, and write the Steering Tag into
292 * the ST entry pointed by index.
293 *
294 * Return: 0 if success, otherwise negative value (-errno)
295 */
296int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag)
297{
298 u32 loc;
299 int err = 0;
300
301 if (!pdev->tph_cap)
302 return -EINVAL;
303
304 if (!pdev->tph_enabled)
305 return -EINVAL;
306
307 /* No need to write tag if device is in "No ST Mode" */
308 if (pdev->tph_mode == PCI_TPH_ST_NS_MODE)
309 return 0;
310
311 /*
312 * Disable TPH before updating ST to avoid potential instability as
313 * cautioned in PCIe r6.2, sec 6.17.3, "ST Modes of Operation"
314 */
315 set_ctrl_reg_req_en(pdev, PCI_TPH_REQ_DISABLE);
316
317 loc = pcie_tph_get_st_table_loc(pdev);
318 /* Convert loc to match with PCI_TPH_LOC_* */
319 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
320
321 switch (loc) {
322 case PCI_TPH_LOC_MSIX:
323 err = pci_msix_write_tph_tag(pdev, index, tag);
324 break;
325 case PCI_TPH_LOC_CAP:
326 err = write_tag_to_st_table(pdev, index, tag);
327 break;
328 default:
329 err = -EINVAL;
330 }
331
332 if (err) {
333 pcie_disable_tph(pdev);
334 return err;
335 }
336
337 set_ctrl_reg_req_en(pdev, pdev->tph_req_type);
338
339 pci_dbg(pdev, "set steering tag: %s table, index=%d, tag=%#04x\n",
340 (loc == PCI_TPH_LOC_MSIX) ? "MSI-X" : "ST", index, tag);
341
342 return 0;
343}
344EXPORT_SYMBOL(pcie_tph_set_st_entry);
345
346/**
347 * pcie_disable_tph - Turn off TPH support for device
348 * @pdev: PCI device
349 *
350 * Return: none
351 */
352void pcie_disable_tph(struct pci_dev *pdev)
353{
354 if (!pdev->tph_cap)
355 return;
356
357 if (!pdev->tph_enabled)
358 return;
359
360 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, 0);
361
362 pdev->tph_mode = 0;
363 pdev->tph_req_type = 0;
364 pdev->tph_enabled = 0;
365}
366EXPORT_SYMBOL(pcie_disable_tph);
367
368/**
369 * pcie_enable_tph - Enable TPH support for device using a specific ST mode
370 * @pdev: PCI device
371 * @mode: ST mode to enable. Current supported modes include:
372 *
373 * - PCI_TPH_ST_NS_MODE: NO ST Mode
374 * - PCI_TPH_ST_IV_MODE: Interrupt Vector Mode
375 * - PCI_TPH_ST_DS_MODE: Device Specific Mode
376 *
377 * Check whether the mode is actually supported by the device before enabling
378 * and return an error if not. Additionally determine what types of requests,
379 * TPH or extended TPH, can be issued by the device based on its TPH requester
380 * capability and the Root Port's completer capability.
381 *
382 * Return: 0 on success, otherwise negative value (-errno)
383 */
384int pcie_enable_tph(struct pci_dev *pdev, int mode)
385{
386 u32 reg;
387 u8 dev_modes;
388 u8 rp_req_type;
389
390 /* Honor "notph" kernel parameter */
391 if (pci_tph_disabled)
392 return -EINVAL;
393
394 if (!pdev->tph_cap)
395 return -EINVAL;
396
397 if (pdev->tph_enabled)
398 return -EBUSY;
399
400 /* Sanitize and check ST mode compatibility */
401 mode &= PCI_TPH_CTRL_MODE_SEL_MASK;
402 dev_modes = get_st_modes(pdev);
403 if (!((1 << mode) & dev_modes))
404 return -EINVAL;
405
406 pdev->tph_mode = mode;
407
408 /* Get req_type supported by device and its Root Port */
409 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
410 if (FIELD_GET(PCI_TPH_CAP_EXT_TPH, reg))
411 pdev->tph_req_type = PCI_TPH_REQ_EXT_TPH;
412 else
413 pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY;
414
415 /* Check if the device is behind a Root Port */
416 if (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_END) {
417 rp_req_type = get_rp_completer_type(pdev);
418
419 /* Final req_type is the smallest value of two */
420 pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type);
421 }
422
423 if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE)
424 return -EINVAL;
425
426 /* Write them into TPH control register */
427 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
428
429 FIELD_MODIFY(PCI_TPH_CTRL_MODE_SEL_MASK, &reg, pdev->tph_mode);
430 FIELD_MODIFY(PCI_TPH_CTRL_REQ_EN_MASK, &reg, pdev->tph_req_type);
431
432 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
433
434 pdev->tph_enabled = 1;
435
436 return 0;
437}
438EXPORT_SYMBOL(pcie_enable_tph);
439
440void pci_restore_tph_state(struct pci_dev *pdev)
441{
442 struct pci_cap_saved_state *save_state;
443 int num_entries, i, offset;
444 u16 *st_entry;
445 u32 *cap;
446
447 if (!pdev->tph_cap)
448 return;
449
450 if (!pdev->tph_enabled)
451 return;
452
453 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
454 if (!save_state)
455 return;
456
457 /* Restore control register and all ST entries */
458 cap = &save_state->cap.data[0];
459 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, *cap++);
460 st_entry = (u16 *)cap;
461 offset = PCI_TPH_BASE_SIZEOF;
462 num_entries = pcie_tph_get_st_table_size(pdev);
463 for (i = 0; i < num_entries; i++) {
464 pci_write_config_word(pdev, pdev->tph_cap + offset,
465 *st_entry++);
466 offset += sizeof(u16);
467 }
468}
469
470void pci_save_tph_state(struct pci_dev *pdev)
471{
472 struct pci_cap_saved_state *save_state;
473 int num_entries, i, offset;
474 u16 *st_entry;
475 u32 *cap;
476
477 if (!pdev->tph_cap)
478 return;
479
480 if (!pdev->tph_enabled)
481 return;
482
483 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
484 if (!save_state)
485 return;
486
487 /* Save control register */
488 cap = &save_state->cap.data[0];
489 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, cap++);
490
491 /* Save all ST entries in extended capability structure */
492 st_entry = (u16 *)cap;
493 offset = PCI_TPH_BASE_SIZEOF;
494 num_entries = pcie_tph_get_st_table_size(pdev);
495 for (i = 0; i < num_entries; i++) {
496 pci_read_config_word(pdev, pdev->tph_cap + offset,
497 st_entry++);
498 offset += sizeof(u16);
499 }
500}
501
502void pci_no_tph(void)
503{
504 pci_tph_disabled = true;
505
506 pr_info("PCIe TPH is disabled\n");
507}
508
509void pci_tph_init(struct pci_dev *pdev)
510{
511 int num_entries;
512 u32 save_size;
513
514 pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH);
515 if (!pdev->tph_cap)
516 return;
517
518 num_entries = pcie_tph_get_st_table_size(pdev);
519 save_size = sizeof(u32) + num_entries * sizeof(u16);
520 pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_TPH, save_size);
521}