projects/linux·blob master · refs

628 lines · 13.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI Express I/O Virtualization (IOV) support
4 * Address Translation Service 1.0
5 * Page Request Interface added by Joerg Roedel <[email protected]>
6 * PASID support added by Joerg Roedel <[email protected]>
7 *
8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <[email protected]>
9 * Copyright (C) 2011 Advanced Micro Devices,
10 */
11
12#include <linux/bitfield.h>
13#include <linux/export.h>
14#include <linux/pci-ats.h>
15#include <linux/pci.h>
16#include <linux/slab.h>
17
18#include "pci.h"
19
20void pci_ats_init(struct pci_dev *dev)
21{
22 int pos;
23
24 if (pci_ats_disabled())
25 return;
26
27 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
28 if (!pos)
29 return;
30
31 dev->ats_cap = pos;
32}
33
34/**
35 * pci_ats_supported - check if the device can use ATS
36 * @dev: the PCI device
37 *
38 * Returns true if the device supports ATS and is allowed to use it, false
39 * otherwise.
40 */
41bool pci_ats_supported(struct pci_dev *dev)
42{
43 if (!dev->ats_cap || dev->untrusted)
44 return false;
45
46 if (dev->is_virtfn)
47 return pci_ats_supported(pci_physfn(dev));
48
49 return true;
50}
51EXPORT_SYMBOL_GPL(pci_ats_supported);
52
53/**
54 * pci_prepare_ats - Setup the PS for ATS
55 * @dev: the PCI device
56 * @ps: the IOMMU page shift
57 *
58 * This must be done by the IOMMU driver on the PF before any VFs are created to
59 * ensure that the VF can have ATS enabled.
60 *
61 * Returns 0 on success, or negative on failure.
62 */
63int pci_prepare_ats(struct pci_dev *dev, int ps)
64{
65 u16 ctrl;
66
67 if (!pci_ats_supported(dev))
68 return -EINVAL;
69
70 if (WARN_ON(dev->ats_enabled))
71 return -EBUSY;
72
73 if (ps < PCI_ATS_MIN_STU)
74 return -EINVAL;
75
76 if (dev->is_virtfn) {
77 if (pci_physfn(dev)->ats_stu != ps)
78 return -EINVAL;
79
80 return 0;
81 }
82
83 dev->ats_stu = ps;
84 ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
85 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
86 return 0;
87}
88EXPORT_SYMBOL_GPL(pci_prepare_ats);
89
90/**
91 * pci_enable_ats - enable the ATS capability
92 * @dev: the PCI device
93 * @ps: the IOMMU page shift
94 *
95 * Returns 0 on success, or negative on failure.
96 */
97int pci_enable_ats(struct pci_dev *dev, int ps)
98{
99 u16 ctrl;
100 struct pci_dev *pdev;
101
102 if (!pci_ats_supported(dev))
103 return -EINVAL;
104
105 if (WARN_ON(dev->ats_enabled))
106 return -EBUSY;
107
108 if (ps < PCI_ATS_MIN_STU)
109 return -EINVAL;
110
111 /*
112 * Note that enabling ATS on a VF fails unless it's already enabled
113 * with the same STU on the PF.
114 */
115 ctrl = PCI_ATS_CTRL_ENABLE;
116 if (dev->is_virtfn) {
117 pdev = pci_physfn(dev);
118 if (pdev->ats_stu != ps)
119 return -EINVAL;
120 } else {
121 dev->ats_stu = ps;
122 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
123 }
124 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
125
126 dev->ats_enabled = 1;
127 return 0;
128}
129EXPORT_SYMBOL_GPL(pci_enable_ats);
130
131/**
132 * pci_disable_ats - disable the ATS capability
133 * @dev: the PCI device
134 */
135void pci_disable_ats(struct pci_dev *dev)
136{
137 u16 ctrl;
138
139 if (WARN_ON(!dev->ats_enabled))
140 return;
141
142 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
143 ctrl &= ~PCI_ATS_CTRL_ENABLE;
144 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
145
146 dev->ats_enabled = 0;
147}
148EXPORT_SYMBOL_GPL(pci_disable_ats);
149
150void pci_restore_ats_state(struct pci_dev *dev)
151{
152 u16 ctrl;
153
154 if (!dev->ats_enabled)
155 return;
156
157 ctrl = PCI_ATS_CTRL_ENABLE;
158 if (!dev->is_virtfn)
159 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
160 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
161}
162
163/**
164 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
165 * @dev: the PCI device
166 *
167 * Returns the queue depth on success, or negative on failure.
168 *
169 * The ATS spec uses 0 in the Invalidate Queue Depth field to
170 * indicate that the function can accept 32 Invalidate Request.
171 * But here we use the `real' values (i.e. 1~32) for the Queue
172 * Depth; and 0 indicates the function shares the Queue with
173 * other functions (doesn't exclusively own a Queue).
174 */
175int pci_ats_queue_depth(struct pci_dev *dev)
176{
177 u16 cap;
178
179 if (!dev->ats_cap)
180 return -EINVAL;
181
182 if (dev->is_virtfn)
183 return 0;
184
185 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
186 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
187}
188
189/**
190 * pci_ats_page_aligned - Return Page Aligned Request bit status.
191 * @pdev: the PCI device
192 *
193 * Returns 1, if the Untranslated Addresses generated by the device
194 * are always aligned or 0 otherwise.
195 *
196 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
197 * is set, it indicates the Untranslated Addresses generated by the
198 * device are always aligned to a 4096 byte boundary.
199 */
200int pci_ats_page_aligned(struct pci_dev *pdev)
201{
202 u16 cap;
203
204 if (!pdev->ats_cap)
205 return 0;
206
207 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
208
209 if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
210 return 1;
211
212 return 0;
213}
214
215/*
216 * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on
217 * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host
218 * by means of an ATS request on CXL.io.
219 *
220 * In other words, CXL.cache devices cannot access host physical memory without
221 * ATS.
222 *
223 * Check Cache_Capable instead of Cache_Enable because CXL.cache may be enabled
224 * after the caller uses this to make its ATS decision.
225 */
226static bool pci_cxl_ats_required(struct pci_dev *pdev)
227{
228 int offset;
229 u16 cap;
230
231 offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
232 PCI_DVSEC_CXL_DEVICE);
233 if (!offset)
234 return false;
235
236 if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap))
237 return false;
238
239 return cap & PCI_DVSEC_CXL_CACHE_CAPABLE;
240}
241
242/**
243 * pci_ats_required - Whether the PCI device requires ATS
244 * @pdev: the PCI device
245 *
246 * Returns true, if the PCI device requires ATS for basic functional operation.
247 */
248bool pci_ats_required(struct pci_dev *pdev)
249{
250 if (!pci_ats_supported(pdev))
251 return false;
252
253 /* A VF inherits its PF's requirement for ATS function */
254 if (pdev->is_virtfn)
255 pdev = pci_physfn(pdev);
256
257 return pci_cxl_ats_required(pdev) ||
258 pci_dev_specific_ats_required(pdev);
259}
260EXPORT_SYMBOL_GPL(pci_ats_required);
261
262#ifdef CONFIG_PCI_PRI
263void pci_pri_init(struct pci_dev *pdev)
264{
265 u16 status;
266
267 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
268
269 if (!pdev->pri_cap)
270 return;
271
272 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
273 if (status & PCI_PRI_STATUS_PASID)
274 pdev->pasid_required = 1;
275}
276
277/**
278 * pci_enable_pri - Enable PRI capability
279 * @pdev: PCI device structure
280 * @reqs: outstanding requests
281 *
282 * Returns 0 on success, negative value on error
283 */
284int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
285{
286 u16 control, status;
287 u32 max_requests;
288 int pri = pdev->pri_cap;
289
290 /*
291 * VFs must not implement the PRI Capability. If their PF
292 * implements PRI, it is shared by the VFs, so if the PF PRI is
293 * enabled, it is also enabled for the VF.
294 */
295 if (pdev->is_virtfn) {
296 if (pci_physfn(pdev)->pri_enabled)
297 return 0;
298 return -EINVAL;
299 }
300
301 if (WARN_ON(pdev->pri_enabled))
302 return -EBUSY;
303
304 if (!pri)
305 return -EINVAL;
306
307 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
308 if (!(status & PCI_PRI_STATUS_STOPPED))
309 return -EBUSY;
310
311 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
312 reqs = min(max_requests, reqs);
313 pdev->pri_reqs_alloc = reqs;
314 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
315
316 control = PCI_PRI_CTRL_ENABLE;
317 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
318
319 pdev->pri_enabled = 1;
320
321 return 0;
322}
323
324/**
325 * pci_disable_pri - Disable PRI capability
326 * @pdev: PCI device structure
327 *
328 * Only clears the enabled-bit, regardless of its former value
329 */
330void pci_disable_pri(struct pci_dev *pdev)
331{
332 u16 control;
333 int pri = pdev->pri_cap;
334
335 /* VFs share the PF PRI */
336 if (pdev->is_virtfn)
337 return;
338
339 if (WARN_ON(!pdev->pri_enabled))
340 return;
341
342 if (!pri)
343 return;
344
345 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
346 control &= ~PCI_PRI_CTRL_ENABLE;
347 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
348
349 pdev->pri_enabled = 0;
350}
351EXPORT_SYMBOL_GPL(pci_disable_pri);
352
353/**
354 * pci_restore_pri_state - Restore PRI
355 * @pdev: PCI device structure
356 */
357void pci_restore_pri_state(struct pci_dev *pdev)
358{
359 u16 control = PCI_PRI_CTRL_ENABLE;
360 u32 reqs = pdev->pri_reqs_alloc;
361 int pri = pdev->pri_cap;
362
363 if (pdev->is_virtfn)
364 return;
365
366 if (!pdev->pri_enabled)
367 return;
368
369 if (!pri)
370 return;
371
372 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
373 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
374}
375
376/**
377 * pci_reset_pri - Resets device's PRI state
378 * @pdev: PCI device structure
379 *
380 * The PRI capability must be disabled before this function is called.
381 * Returns 0 on success, negative value on error.
382 */
383int pci_reset_pri(struct pci_dev *pdev)
384{
385 u16 control;
386 int pri = pdev->pri_cap;
387
388 if (pdev->is_virtfn)
389 return 0;
390
391 if (WARN_ON(pdev->pri_enabled))
392 return -EBUSY;
393
394 if (!pri)
395 return -EINVAL;
396
397 control = PCI_PRI_CTRL_RESET;
398 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
399
400 return 0;
401}
402
403/**
404 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
405 * status.
406 * @pdev: PCI device structure
407 *
408 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
409 */
410int pci_prg_resp_pasid_required(struct pci_dev *pdev)
411{
412 if (pdev->is_virtfn)
413 pdev = pci_physfn(pdev);
414
415 return pdev->pasid_required;
416}
417
418/**
419 * pci_pri_supported - Check if PRI is supported.
420 * @pdev: PCI device structure
421 *
422 * Returns true if PRI capability is present, false otherwise.
423 */
424bool pci_pri_supported(struct pci_dev *pdev)
425{
426 /* VFs share the PF PRI */
427 if (pci_physfn(pdev)->pri_cap)
428 return true;
429 return false;
430}
431EXPORT_SYMBOL_GPL(pci_pri_supported);
432#endif /* CONFIG_PCI_PRI */
433
434#ifdef CONFIG_PCI_PASID
435void pci_pasid_init(struct pci_dev *pdev)
436{
437 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
438}
439
440/**
441 * pci_enable_pasid - Enable the PASID capability
442 * @pdev: PCI device structure
443 * @features: Features to enable
444 *
445 * Returns 0 on success, negative value on error. This function checks
446 * whether the features are actually supported by the device and returns
447 * an error if not.
448 */
449int pci_enable_pasid(struct pci_dev *pdev, int features)
450{
451 u16 control, supported;
452 int pasid = pdev->pasid_cap;
453
454 /*
455 * VFs must not implement the PASID Capability, but if a PF
456 * supports PASID, its VFs share the PF PASID configuration.
457 */
458 if (pdev->is_virtfn) {
459 if (pci_physfn(pdev)->pasid_enabled)
460 return 0;
461 return -EINVAL;
462 }
463
464 if (WARN_ON(pdev->pasid_enabled))
465 return -EBUSY;
466
467 if (!pdev->eetlp_prefix_max && !pdev->pasid_no_tlp)
468 return -EINVAL;
469
470 if (!pasid)
471 return -EINVAL;
472
473 if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))
474 return -EINVAL;
475
476 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
477 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
478
479 /* User wants to enable anything unsupported? */
480 if ((supported & features) != features)
481 return -EINVAL;
482
483 control = PCI_PASID_CTRL_ENABLE | features;
484 pdev->pasid_features = features;
485
486 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
487
488 pdev->pasid_enabled = 1;
489
490 return 0;
491}
492EXPORT_SYMBOL_GPL(pci_enable_pasid);
493
494/**
495 * pci_disable_pasid - Disable the PASID capability
496 * @pdev: PCI device structure
497 */
498void pci_disable_pasid(struct pci_dev *pdev)
499{
500 u16 control = 0;
501 int pasid = pdev->pasid_cap;
502
503 /* VFs share the PF PASID configuration */
504 if (pdev->is_virtfn)
505 return;
506
507 if (WARN_ON(!pdev->pasid_enabled))
508 return;
509
510 if (!pasid)
511 return;
512
513 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
514
515 pdev->pasid_enabled = 0;
516}
517EXPORT_SYMBOL_GPL(pci_disable_pasid);
518
519/**
520 * pci_restore_pasid_state - Restore PASID capabilities
521 * @pdev: PCI device structure
522 */
523void pci_restore_pasid_state(struct pci_dev *pdev)
524{
525 u16 control;
526 int pasid = pdev->pasid_cap;
527
528 if (pdev->is_virtfn)
529 return;
530
531 if (!pdev->pasid_enabled)
532 return;
533
534 if (!pasid)
535 return;
536
537 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
538 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
539}
540
541/**
542 * pci_pasid_features - Check which PASID features are supported
543 * @pdev: PCI device structure
544 *
545 * Return a negative value when no PASID capability is present.
546 * Otherwise return a bitmask with supported features. Current
547 * features reported are:
548 * PCI_PASID_CAP_EXEC - Execute permission supported
549 * PCI_PASID_CAP_PRIV - Privileged mode supported
550 */
551int pci_pasid_features(struct pci_dev *pdev)
552{
553 u16 supported;
554 int pasid;
555
556 if (pdev->is_virtfn)
557 pdev = pci_physfn(pdev);
558
559 pasid = pdev->pasid_cap;
560 if (!pasid)
561 return -EINVAL;
562
563 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
564
565 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
566
567 return supported;
568}
569EXPORT_SYMBOL_GPL(pci_pasid_features);
570
571/**
572 * pci_max_pasids - Get maximum number of PASIDs supported by device
573 * @pdev: PCI device structure
574 *
575 * Returns negative value when PASID capability is not present.
576 * Otherwise it returns the number of supported PASIDs.
577 */
578int pci_max_pasids(struct pci_dev *pdev)
579{
580 u16 supported;
581 int pasid;
582
583 if (pdev->is_virtfn)
584 pdev = pci_physfn(pdev);
585
586 pasid = pdev->pasid_cap;
587 if (!pasid)
588 return -EINVAL;
589
590 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
591
592 return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));
593}
594EXPORT_SYMBOL_GPL(pci_max_pasids);
595
596/**
597 * pci_pasid_status - Check the PASID status
598 * @pdev: PCI device structure
599 *
600 * Returns a negative value when no PASID capability is present.
601 * Otherwise the value of the control register is returned.
602 * Status reported are:
603 *
604 * PCI_PASID_CTRL_ENABLE - PASID enabled
605 * PCI_PASID_CTRL_EXEC - Execute permission enabled
606 * PCI_PASID_CTRL_PRIV - Privileged mode enabled
607 */
608int pci_pasid_status(struct pci_dev *pdev)
609{
610 int pasid;
611 u16 ctrl;
612
613 if (pdev->is_virtfn)
614 pdev = pci_physfn(pdev);
615
616 pasid = pdev->pasid_cap;
617 if (!pasid)
618 return -EINVAL;
619
620 pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl);
621
622 ctrl &= PCI_PASID_CTRL_ENABLE | PCI_PASID_CTRL_EXEC |
623 PCI_PASID_CTRL_PRIV;
624
625 return ctrl;
626}
627EXPORT_SYMBOL_GPL(pci_pasid_status);
628#endif /* CONFIG_PCI_PASID */