projects/linux·blob master · refs

3582 lines · 92.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI detection and setup code
4 */
5
6#include <linux/array_size.h>
7#include <linux/kernel.h>
8#include <linux/delay.h>
9#include <linux/init.h>
10#include <linux/pci.h>
11#include <linux/msi.h>
12#include <linux/of_pci.h>
13#include <linux/of_platform.h>
14#include <linux/platform_device.h>
15#include <linux/pci_hotplug.h>
16#include <linux/slab.h>
17#include <linux/sprintf.h>
18#include <linux/module.h>
19#include <linux/cpumask.h>
20#include <linux/aer.h>
21#include <linux/acpi.h>
22#include <linux/hypervisor.h>
23#include <linux/irqdomain.h>
24#include <linux/pm_runtime.h>
25#include <linux/bitfield.h>
26#include <trace/events/pci.h>
27#include "pci.h"
28
29static struct resource busn_resource = {
30 .name = "PCI busn",
31 .start = 0,
32 .end = 255,
33 .flags = IORESOURCE_BUS,
34};
35
36/* Ugh. Need to stop exporting this to modules. */
37LIST_HEAD(pci_root_buses);
38EXPORT_SYMBOL(pci_root_buses);
39
40static LIST_HEAD(pci_domain_busn_res_list);
41
42struct pci_domain_busn_res {
43 struct list_head list;
44 struct resource res;
45 int domain_nr;
46};
47
48static struct resource *get_pci_domain_busn_res(int domain_nr)
49{
50 struct pci_domain_busn_res *r;
51
52 list_for_each_entry(r, &pci_domain_busn_res_list, list)
53 if (r->domain_nr == domain_nr)
54 return &r->res;
55
56 r = kzalloc_obj(*r);
57 if (!r)
58 return NULL;
59
60 r->domain_nr = domain_nr;
61 r->res.start = 0;
62 r->res.end = 0xff;
63 r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
64
65 list_add_tail(&r->list, &pci_domain_busn_res_list);
66
67 return &r->res;
68}
69
70/*
71 * PCI Bus Class
72 */
73static void release_pcibus_dev(struct device *dev)
74{
75 struct pci_bus *pci_bus = to_pci_bus(dev);
76
77 put_device(pci_bus->bridge);
78 pci_bus_remove_resources(pci_bus);
79 pci_release_bus_of_node(pci_bus);
80 kfree(pci_bus);
81}
82
83static const struct class pcibus_class = {
84 .name = "pci_bus",
85 .dev_release = &release_pcibus_dev,
86 .dev_groups = pcibus_groups,
87};
88
89static int __init pcibus_class_init(void)
90{
91 return class_register(&pcibus_class);
92}
93postcore_initcall(pcibus_class_init);
94
95static u64 pci_size(u64 base, u64 maxbase, u64 mask)
96{
97 u64 size = mask & maxbase; /* Find the significant bits */
98 if (!size)
99 return 0;
100
101 /*
102 * Get the lowest of them to find the decode size, and from that
103 * the extent.
104 */
105 size = size & ~(size-1);
106
107 /*
108 * base == maxbase can be valid only if the BAR has already been
109 * programmed with all 1s.
110 */
111 if (base == maxbase && ((base | (size - 1)) & mask) != mask)
112 return 0;
113
114 return size;
115}
116
117static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
118{
119 u32 mem_type;
120 unsigned long flags;
121
122 if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
123 flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
124 flags |= IORESOURCE_IO;
125 return flags;
126 }
127
128 flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
129 flags |= IORESOURCE_MEM;
130 if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
131 flags |= IORESOURCE_PREFETCH;
132
133 mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
134 switch (mem_type) {
135 case PCI_BASE_ADDRESS_MEM_TYPE_32:
136 break;
137 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
138 /* 1M mem BAR treated as 32-bit BAR */
139 break;
140 case PCI_BASE_ADDRESS_MEM_TYPE_64:
141 flags |= IORESOURCE_MEM_64;
142 break;
143 default:
144 /* mem unknown type treated as 32-bit BAR */
145 break;
146 }
147 return flags;
148}
149
150#define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
151
152/**
153 * __pci_size_bars - Read the raw BAR mask for a range of PCI BARs
154 * @dev: the PCI device
155 * @count: number of BARs to size
156 * @pos: starting config space position
157 * @sizes: array to store mask values
158 * @rom: indicate whether to use ROM mask, which avoids enabling ROM BARs
159 *
160 * Provided @sizes array must be sufficiently sized to store results for
161 * @count u32 BARs. Caller is responsible for disabling decode to specified
162 * BAR range around calling this function. This function is intended to avoid
163 * disabling decode around sizing each BAR individually, which can result in
164 * non-trivial overhead in virtualized environments with very large PCI BARs.
165 */
166static void __pci_size_bars(struct pci_dev *dev, int count,
167 unsigned int pos, u32 *sizes, bool rom)
168{
169 u32 orig, mask = rom ? PCI_ROM_ADDRESS_MASK : ~0;
170 int i;
171
172 for (i = 0; i < count; i++, pos += 4, sizes++) {
173 pci_read_config_dword(dev, pos, &orig);
174 pci_write_config_dword(dev, pos, mask);
175 pci_read_config_dword(dev, pos, sizes);
176 pci_write_config_dword(dev, pos, orig);
177 }
178}
179
180void __pci_size_stdbars(struct pci_dev *dev, int count,
181 unsigned int pos, u32 *sizes)
182{
183 __pci_size_bars(dev, count, pos, sizes, false);
184}
185
186static void __pci_size_rom(struct pci_dev *dev, unsigned int pos, u32 *sizes)
187{
188 __pci_size_bars(dev, 1, pos, sizes, true);
189}
190
191/**
192 * __pci_read_base - Read a PCI BAR
193 * @dev: the PCI device
194 * @type: type of the BAR
195 * @res: resource buffer to be filled in
196 * @pos: BAR position in the config space
197 * @sizes: array of one or more pre-read BAR masks
198 *
199 * Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
200 */
201int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
202 struct resource *res, unsigned int pos, u32 *sizes)
203{
204 u32 l = 0, sz;
205 u64 l64, sz64, mask64;
206 struct pci_bus_region region, inverted_region;
207 const char *res_name = pci_resource_name(dev, res - dev->resource);
208
209 res->name = pci_name(dev);
210
211 pci_read_config_dword(dev, pos, &l);
212 sz = sizes[0];
213
214 /*
215 * All bits set in sz means the device isn't working properly.
216 * If the BAR isn't implemented, all bits must be 0. If it's a
217 * memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
218 * 1 must be clear.
219 */
220 if (PCI_POSSIBLE_ERROR(sz))
221 sz = 0;
222
223 /*
224 * I don't know how l can have all bits set. Copied from old code.
225 * Maybe it fixes a bug on some ancient platform.
226 */
227 if (PCI_POSSIBLE_ERROR(l))
228 l = 0;
229
230 if (type == pci_bar_unknown) {
231 res->flags = decode_bar(dev, l);
232 res->flags |= IORESOURCE_SIZEALIGN;
233 if (res->flags & IORESOURCE_IO) {
234 l64 = l & PCI_BASE_ADDRESS_IO_MASK;
235 sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
236 mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
237 } else {
238 l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
239 sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
240 mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
241 }
242 } else {
243 if (l & PCI_ROM_ADDRESS_ENABLE)
244 res->flags |= IORESOURCE_ROM_ENABLE;
245 l64 = l & PCI_ROM_ADDRESS_MASK;
246 sz64 = sz & PCI_ROM_ADDRESS_MASK;
247 mask64 = PCI_ROM_ADDRESS_MASK;
248 }
249
250 if (res->flags & IORESOURCE_MEM_64) {
251 pci_read_config_dword(dev, pos + 4, &l);
252 sz = sizes[1];
253
254 l64 |= ((u64)l << 32);
255 sz64 |= ((u64)sz << 32);
256 mask64 |= ((u64)~0 << 32);
257 }
258
259 if (!sz64)
260 goto fail;
261
262 sz64 = pci_size(l64, sz64, mask64);
263 if (!sz64) {
264 pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name);
265 goto fail;
266 }
267
268 if (res->flags & IORESOURCE_MEM_64) {
269 if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
270 && sz64 > 0x100000000ULL) {
271 res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
272 resource_set_range(res, 0, 0);
273 pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n",
274 res_name, (unsigned long long)sz64);
275 goto out;
276 }
277
278 if ((sizeof(pci_bus_addr_t) < 8) && l) {
279 /* Above 32-bit boundary; try to reallocate */
280 res->flags |= IORESOURCE_UNSET;
281 resource_set_range(res, 0, sz64);
282 pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n",
283 res_name, (unsigned long long)l64);
284 goto out;
285 }
286 }
287
288 region.start = l64;
289 region.end = l64 + sz64 - 1;
290
291 pcibios_bus_to_resource(dev->bus, res, &region);
292 pcibios_resource_to_bus(dev->bus, &inverted_region, res);
293
294 /*
295 * If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
296 * the corresponding resource address (the physical address used by
297 * the CPU. Converting that resource address back to a bus address
298 * should yield the original BAR value:
299 *
300 * resource_to_bus(bus_to_resource(A)) == A
301 *
302 * If it doesn't, CPU accesses to "bus_to_resource(A)" will not
303 * be claimed by the device.
304 */
305 if (inverted_region.start != region.start) {
306 res->flags |= IORESOURCE_UNSET;
307 res->start = 0;
308 res->end = region.end - region.start;
309 pci_info(dev, "%s: initial BAR value %#010llx invalid\n",
310 res_name, (unsigned long long)region.start);
311 }
312
313 goto out;
314
315
316fail:
317 res->flags = 0;
318out:
319 if (res->flags)
320 pci_info(dev, "%s %pR\n", res_name, res);
321
322 return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
323}
324
325static __always_inline void pci_read_bases(struct pci_dev *dev,
326 unsigned int howmany, int rom)
327{
328 u32 rombar, stdbars[PCI_STD_NUM_BARS];
329 unsigned int pos, reg;
330 u16 orig_cmd;
331
332 BUILD_BUG_ON(statically_true(howmany > PCI_STD_NUM_BARS));
333
334 if (dev->non_compliant_bars)
335 return;
336
337 /* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
338 if (dev->is_virtfn)
339 return;
340
341 /* No printks while decoding is disabled! */
342 if (!dev->mmio_always_on) {
343 pci_read_config_word(dev, PCI_COMMAND, &orig_cmd);
344 if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
345 pci_write_config_word(dev, PCI_COMMAND,
346 orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
347 }
348 }
349
350 __pci_size_stdbars(dev, howmany, PCI_BASE_ADDRESS_0, stdbars);
351 if (rom)
352 __pci_size_rom(dev, rom, &rombar);
353
354 if (!dev->mmio_always_on &&
355 (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
356 pci_write_config_word(dev, PCI_COMMAND, orig_cmd);
357
358 for (pos = 0; pos < howmany; pos++) {
359 struct resource *res = &dev->resource[pos];
360 reg = PCI_BASE_ADDRESS_0 + (pos << 2);
361 pos += __pci_read_base(dev, pci_bar_unknown,
362 res, reg, &stdbars[pos]);
363 }
364
365 if (rom) {
366 struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
367 dev->rom_base_reg = rom;
368 res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
369 IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
370 __pci_read_base(dev, pci_bar_mem32, res, rom, &rombar);
371 }
372}
373
374static void pci_read_bridge_io(struct pci_dev *dev, struct resource *res,
375 bool log)
376{
377 u8 io_base_lo, io_limit_lo;
378 unsigned long io_mask, io_granularity, base, limit;
379 struct pci_bus_region region;
380
381 if (!dev->io_window)
382 return;
383
384 io_mask = PCI_IO_RANGE_MASK;
385 io_granularity = 0x1000;
386 if (dev->io_window_1k) {
387 /* Support 1K I/O space granularity */
388 io_mask = PCI_IO_1K_RANGE_MASK;
389 io_granularity = 0x400;
390 }
391
392 pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo);
393 pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo);
394 base = (io_base_lo & io_mask) << 8;
395 limit = (io_limit_lo & io_mask) << 8;
396
397 if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
398 u16 io_base_hi, io_limit_hi;
399
400 pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi);
401 pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi);
402 base |= ((unsigned long) io_base_hi << 16);
403 limit |= ((unsigned long) io_limit_hi << 16);
404 }
405
406 res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
407
408 if (base <= limit) {
409 region.start = base;
410 region.end = limit + io_granularity - 1;
411 pcibios_bus_to_resource(dev->bus, res, &region);
412 if (log)
413 pci_info(dev, " bridge window %pR\n", res);
414 } else {
415 resource_set_range(res, 0, 0);
416 res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
417 }
418}
419
420static void pci_read_bridge_mmio(struct pci_dev *dev, struct resource *res,
421 bool log)
422{
423 u16 mem_base_lo, mem_limit_lo;
424 unsigned long base, limit;
425 struct pci_bus_region region;
426
427 pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo);
428 pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo);
429 base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
430 limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
431
432 res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
433
434 if (base <= limit) {
435 region.start = base;
436 region.end = limit + 0xfffff;
437 pcibios_bus_to_resource(dev->bus, res, &region);
438 if (log)
439 pci_info(dev, " bridge window %pR\n", res);
440 } else {
441 resource_set_range(res, 0, 0);
442 res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
443 }
444}
445
446static void pci_read_bridge_mmio_pref(struct pci_dev *dev, struct resource *res,
447 bool log)
448{
449 u16 mem_base_lo, mem_limit_lo;
450 u64 base64, limit64;
451 pci_bus_addr_t base, limit;
452 struct pci_bus_region region;
453
454 if (!dev->pref_window)
455 return;
456
457 pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo);
458 pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo);
459 base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
460 limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
461
462 if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
463 u32 mem_base_hi, mem_limit_hi;
464
465 pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi);
466 pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi);
467
468 /*
469 * Some bridges set the base > limit by default, and some
470 * (broken) BIOSes do not initialize them. If we find
471 * this, just assume they are not being used.
472 */
473 if (mem_base_hi <= mem_limit_hi) {
474 base64 |= (u64) mem_base_hi << 32;
475 limit64 |= (u64) mem_limit_hi << 32;
476 }
477 }
478
479 base = (pci_bus_addr_t) base64;
480 limit = (pci_bus_addr_t) limit64;
481
482 if (base != base64) {
483 pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
484 (unsigned long long) base64);
485 return;
486 }
487
488 res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) | IORESOURCE_MEM |
489 IORESOURCE_PREFETCH;
490 if (res->flags & PCI_PREF_RANGE_TYPE_64)
491 res->flags |= IORESOURCE_MEM_64;
492
493 if (base <= limit) {
494 region.start = base;
495 region.end = limit + 0xfffff;
496 pcibios_bus_to_resource(dev->bus, res, &region);
497 if (log)
498 pci_info(dev, " bridge window %pR\n", res);
499 } else {
500 resource_set_range(res, 0, 0);
501 res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
502 }
503}
504
505static void pci_read_bridge_windows(struct pci_dev *bridge)
506{
507 u32 buses;
508 u16 io;
509 u32 pmem, tmp;
510 struct resource res;
511
512 pci_read_config_dword(bridge, PCI_PRIMARY_BUS, &buses);
513 res.flags = IORESOURCE_BUS;
514 res.start = FIELD_GET(PCI_SECONDARY_BUS_MASK, buses);
515 res.end = FIELD_GET(PCI_SUBORDINATE_BUS_MASK, buses);
516 pci_info(bridge, "PCI bridge to %pR%s\n", &res,
517 bridge->transparent ? " (subtractive decode)" : "");
518
519 pci_read_config_word(bridge, PCI_IO_BASE, &io);
520 if (!io) {
521 pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0);
522 pci_read_config_word(bridge, PCI_IO_BASE, &io);
523 pci_write_config_word(bridge, PCI_IO_BASE, 0x0);
524 }
525 if (io) {
526 bridge->io_window = 1;
527 pci_read_bridge_io(bridge, &res, true);
528 }
529
530 pci_read_bridge_mmio(bridge, &res, true);
531
532 /*
533 * DECchip 21050 pass 2 errata: the bridge may miss an address
534 * disconnect boundary by one PCI data phase. Workaround: do not
535 * use prefetching on this device.
536 */
537 if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
538 return;
539
540 pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
541 if (!pmem) {
542 pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE,
543 0xffe0fff0);
544 pci_read_config_dword(bridge, PCI_PREF_MEMORY_BASE, &pmem);
545 pci_write_config_dword(bridge, PCI_PREF_MEMORY_BASE, 0x0);
546 }
547 if (!pmem)
548 return;
549
550 bridge->pref_window = 1;
551
552 if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
553
554 /*
555 * Bridge claims to have a 64-bit prefetchable memory
556 * window; verify that the upper bits are actually
557 * writable.
558 */
559 pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &pmem);
560 pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32,
561 0xffffffff);
562 pci_read_config_dword(bridge, PCI_PREF_BASE_UPPER32, &tmp);
563 pci_write_config_dword(bridge, PCI_PREF_BASE_UPPER32, pmem);
564 if (tmp)
565 bridge->pref_64_window = 1;
566 }
567
568 pci_read_bridge_mmio_pref(bridge, &res, true);
569}
570
571void pci_read_bridge_bases(struct pci_bus *child)
572{
573 struct pci_dev *dev = child->self;
574 struct resource *res;
575 int i;
576
577 if (pci_is_root_bus(child)) /* It's a host bus, nothing to read */
578 return;
579
580 pci_info(dev, "PCI bridge to %pR%s\n",
581 &child->busn_res,
582 dev->transparent ? " (subtractive decode)" : "");
583
584 pci_bus_remove_resources(child);
585 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
586 child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
587
588 pci_read_bridge_io(child->self,
589 child->resource[PCI_BUS_BRIDGE_IO_WINDOW], false);
590 pci_read_bridge_mmio(child->self,
591 child->resource[PCI_BUS_BRIDGE_MEM_WINDOW], false);
592 pci_read_bridge_mmio_pref(child->self,
593 child->resource[PCI_BUS_BRIDGE_PREF_MEM_WINDOW],
594 false);
595
596 if (!dev->transparent)
597 return;
598
599 pci_bus_for_each_resource(child->parent, res) {
600 if (!res || !res->flags)
601 continue;
602
603 pci_bus_add_resource(child, res);
604 pci_info(dev, " bridge window %pR (subtractive decode)\n", res);
605 }
606}
607
608static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
609{
610 struct pci_bus *b;
611
612 b = kzalloc_obj(*b);
613 if (!b)
614 return NULL;
615
616 INIT_LIST_HEAD(&b->node);
617 INIT_LIST_HEAD(&b->children);
618 INIT_LIST_HEAD(&b->devices);
619 INIT_LIST_HEAD(&b->slots);
620 INIT_LIST_HEAD(&b->resources);
621 b->max_bus_speed = PCI_SPEED_UNKNOWN;
622 b->cur_bus_speed = PCI_SPEED_UNKNOWN;
623#ifdef CONFIG_PCI_DOMAINS_GENERIC
624 if (parent)
625 b->domain_nr = parent->domain_nr;
626#endif
627 return b;
628}
629
630static void pci_release_host_bridge_dev(struct device *dev)
631{
632 struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
633
634 if (bridge->release_fn)
635 bridge->release_fn(bridge);
636
637 pci_free_resource_list(&bridge->windows);
638 pci_free_resource_list(&bridge->dma_ranges);
639
640 /* Host bridges only have domain_nr set in the emulation case */
641 if (bridge->domain_nr != PCI_DOMAIN_NR_NOT_SET)
642 pci_bus_release_emul_domain_nr(bridge->domain_nr);
643
644 kfree(bridge);
645}
646
647static const struct attribute_group *pci_host_bridge_groups[] = {
648#ifdef CONFIG_PCI_IDE
649 &pci_ide_attr_group,
650#endif
651 NULL
652};
653
654static const struct device_type pci_host_bridge_type = {
655 .groups = pci_host_bridge_groups,
656 .release = pci_release_host_bridge_dev,
657};
658
659static void pci_init_host_bridge(struct pci_host_bridge *bridge)
660{
661 INIT_LIST_HEAD(&bridge->windows);
662 INIT_LIST_HEAD(&bridge->dma_ranges);
663 INIT_LIST_HEAD(&bridge->ports);
664
665 /*
666 * We assume we can manage these PCIe features. Some systems may
667 * reserve these for use by the platform itself, e.g., an ACPI BIOS
668 * may implement its own AER handling and use _OSC to prevent the
669 * OS from interfering.
670 */
671 bridge->native_aer = 1;
672 bridge->native_pcie_hotplug = 1;
673 bridge->native_shpc_hotplug = 1;
674 bridge->native_pme = 1;
675 bridge->native_ltr = 1;
676 bridge->native_dpc = 1;
677 bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
678 bridge->native_cxl_error = 1;
679 bridge->dev.type = &pci_host_bridge_type;
680 pci_ide_init_host_bridge(bridge);
681
682 device_initialize(&bridge->dev);
683}
684
685struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
686{
687 struct pci_host_bridge *bridge;
688
689 bridge = kzalloc(sizeof(*bridge) + priv, GFP_KERNEL);
690 if (!bridge)
691 return NULL;
692
693 pci_init_host_bridge(bridge);
694
695 return bridge;
696}
697EXPORT_SYMBOL(pci_alloc_host_bridge);
698
699static void devm_pci_alloc_host_bridge_release(void *data)
700{
701 pci_free_host_bridge(data);
702}
703
704struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
705 size_t priv)
706{
707 int ret;
708 struct pci_host_bridge *bridge;
709
710 bridge = pci_alloc_host_bridge(priv);
711 if (!bridge)
712 return NULL;
713
714 bridge->dev.parent = dev;
715
716 ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
717 bridge);
718 if (ret)
719 return NULL;
720
721 ret = devm_of_pci_bridge_init(dev, bridge);
722 if (ret)
723 return NULL;
724
725 return bridge;
726}
727EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
728
729void pci_free_host_bridge(struct pci_host_bridge *bridge)
730{
731 put_device(&bridge->dev);
732}
733EXPORT_SYMBOL(pci_free_host_bridge);
734
735/* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
736static const unsigned char pcix_bus_speed[] = {
737 PCI_SPEED_UNKNOWN, /* 0 */
738 PCI_SPEED_66MHz_PCIX, /* 1 */
739 PCI_SPEED_100MHz_PCIX, /* 2 */
740 PCI_SPEED_133MHz_PCIX, /* 3 */
741 PCI_SPEED_UNKNOWN, /* 4 */
742 PCI_SPEED_66MHz_PCIX_ECC, /* 5 */
743 PCI_SPEED_100MHz_PCIX_ECC, /* 6 */
744 PCI_SPEED_133MHz_PCIX_ECC, /* 7 */
745 PCI_SPEED_UNKNOWN, /* 8 */
746 PCI_SPEED_66MHz_PCIX_266, /* 9 */
747 PCI_SPEED_100MHz_PCIX_266, /* A */
748 PCI_SPEED_133MHz_PCIX_266, /* B */
749 PCI_SPEED_UNKNOWN, /* C */
750 PCI_SPEED_66MHz_PCIX_533, /* D */
751 PCI_SPEED_100MHz_PCIX_533, /* E */
752 PCI_SPEED_133MHz_PCIX_533 /* F */
753};
754
755/* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
756const unsigned char pcie_link_speed[] = {
757 PCI_SPEED_UNKNOWN, /* 0 */
758 PCIE_SPEED_2_5GT, /* 1 */
759 PCIE_SPEED_5_0GT, /* 2 */
760 PCIE_SPEED_8_0GT, /* 3 */
761 PCIE_SPEED_16_0GT, /* 4 */
762 PCIE_SPEED_32_0GT, /* 5 */
763 PCIE_SPEED_64_0GT, /* 6 */
764 PCI_SPEED_UNKNOWN, /* 7 */
765 PCI_SPEED_UNKNOWN, /* 8 */
766 PCI_SPEED_UNKNOWN, /* 9 */
767 PCI_SPEED_UNKNOWN, /* A */
768 PCI_SPEED_UNKNOWN, /* B */
769 PCI_SPEED_UNKNOWN, /* C */
770 PCI_SPEED_UNKNOWN, /* D */
771 PCI_SPEED_UNKNOWN, /* E */
772 PCI_SPEED_UNKNOWN /* F */
773};
774EXPORT_SYMBOL_GPL(pcie_link_speed);
775
776/**
777 * pcie_get_link_speed - Get speed value from PCIe generation number
778 * @speed: PCIe speed (1-based: 1 = 2.5GT, 2 = 5GT, ...)
779 *
780 * Returns the speed value (e.g., PCIE_SPEED_2_5GT) if @speed is valid,
781 * otherwise returns PCI_SPEED_UNKNOWN.
782 */
783unsigned char pcie_get_link_speed(unsigned int speed)
784{
785 if (speed >= ARRAY_SIZE(pcie_link_speed))
786 return PCI_SPEED_UNKNOWN;
787
788 return pcie_link_speed[speed];
789}
790EXPORT_SYMBOL_GPL(pcie_get_link_speed);
791
792const char *pci_speed_string(enum pci_bus_speed speed)
793{
794 /* Indexed by the pci_bus_speed enum */
795 static const char *speed_strings[] = {
796 "33 MHz PCI", /* 0x00 */
797 "66 MHz PCI", /* 0x01 */
798 "66 MHz PCI-X", /* 0x02 */
799 "100 MHz PCI-X", /* 0x03 */
800 "133 MHz PCI-X", /* 0x04 */
801 NULL, /* 0x05 */
802 NULL, /* 0x06 */
803 NULL, /* 0x07 */
804 NULL, /* 0x08 */
805 "66 MHz PCI-X 266", /* 0x09 */
806 "100 MHz PCI-X 266", /* 0x0a */
807 "133 MHz PCI-X 266", /* 0x0b */
808 "Unknown AGP", /* 0x0c */
809 "1x AGP", /* 0x0d */
810 "2x AGP", /* 0x0e */
811 "4x AGP", /* 0x0f */
812 "8x AGP", /* 0x10 */
813 "66 MHz PCI-X 533", /* 0x11 */
814 "100 MHz PCI-X 533", /* 0x12 */
815 "133 MHz PCI-X 533", /* 0x13 */
816 "2.5 GT/s PCIe", /* 0x14 */
817 "5.0 GT/s PCIe", /* 0x15 */
818 "8.0 GT/s PCIe", /* 0x16 */
819 "16.0 GT/s PCIe", /* 0x17 */
820 "32.0 GT/s PCIe", /* 0x18 */
821 "64.0 GT/s PCIe", /* 0x19 */
822 };
823
824 if (speed < ARRAY_SIZE(speed_strings))
825 return speed_strings[speed];
826 return "Unknown";
827}
828EXPORT_SYMBOL_GPL(pci_speed_string);
829
830void pcie_update_link_speed(struct pci_bus *bus,
831 enum pcie_link_change_reason reason)
832{
833 struct pci_dev *bridge = bus->self;
834 u16 linksta, linksta2;
835
836 pcie_capability_read_word(bridge, PCI_EXP_LNKSTA, &linksta);
837 pcie_capability_read_word(bridge, PCI_EXP_LNKSTA2, &linksta2);
838
839 __pcie_update_link_speed(bus, reason, linksta, linksta2);
840}
841EXPORT_SYMBOL_GPL(pcie_update_link_speed);
842
843static unsigned char agp_speeds[] = {
844 AGP_UNKNOWN,
845 AGP_1X,
846 AGP_2X,
847 AGP_4X,
848 AGP_8X
849};
850
851static enum pci_bus_speed agp_speed(int agp3, int agpstat)
852{
853 int index = 0;
854
855 if (agpstat & 4)
856 index = 3;
857 else if (agpstat & 2)
858 index = 2;
859 else if (agpstat & 1)
860 index = 1;
861 else
862 goto out;
863
864 if (agp3) {
865 index += 2;
866 if (index == 5)
867 index = 0;
868 }
869
870 out:
871 return agp_speeds[index];
872}
873
874static void pci_set_bus_speed(struct pci_bus *bus)
875{
876 struct pci_dev *bridge = bus->self;
877 int pos;
878
879 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP);
880 if (!pos)
881 pos = pci_find_capability(bridge, PCI_CAP_ID_AGP3);
882 if (pos) {
883 u32 agpstat, agpcmd;
884
885 pci_read_config_dword(bridge, pos + PCI_AGP_STATUS, &agpstat);
886 bus->max_bus_speed = agp_speed(agpstat & 8, agpstat & 7);
887
888 pci_read_config_dword(bridge, pos + PCI_AGP_COMMAND, &agpcmd);
889 bus->cur_bus_speed = agp_speed(agpstat & 8, agpcmd & 7);
890 }
891
892 pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
893 if (pos) {
894 u16 status;
895 enum pci_bus_speed max;
896
897 pci_read_config_word(bridge, pos + PCI_X_BRIDGE_SSTATUS,
898 &status);
899
900 if (status & PCI_X_SSTATUS_533MHZ) {
901 max = PCI_SPEED_133MHz_PCIX_533;
902 } else if (status & PCI_X_SSTATUS_266MHZ) {
903 max = PCI_SPEED_133MHz_PCIX_266;
904 } else if (status & PCI_X_SSTATUS_133MHZ) {
905 if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
906 max = PCI_SPEED_133MHz_PCIX_ECC;
907 else
908 max = PCI_SPEED_133MHz_PCIX;
909 } else {
910 max = PCI_SPEED_66MHz_PCIX;
911 }
912
913 bus->max_bus_speed = max;
914 bus->cur_bus_speed =
915 pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
916
917 return;
918 }
919
920 if (pci_is_pcie(bridge)) {
921 u32 linkcap;
922
923 pcie_capability_read_dword(bridge, PCI_EXP_LNKCAP, &linkcap);
924 bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
925
926 pcie_update_link_speed(bus, PCIE_ADD_BUS);
927 }
928}
929
930static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
931{
932 struct irq_domain *d;
933
934 /* If the host bridge driver sets a MSI domain of the bridge, use it */
935 d = dev_get_msi_domain(bus->bridge);
936
937 /*
938 * Any firmware interface that can resolve the msi_domain
939 * should be called from here.
940 */
941 if (!d)
942 d = pci_host_bridge_of_msi_domain(bus);
943 if (!d)
944 d = pci_host_bridge_acpi_msi_domain(bus);
945
946 /*
947 * If no IRQ domain was found via the OF tree, try looking it up
948 * directly through the fwnode_handle.
949 */
950 if (!d) {
951 struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
952
953 if (fwnode)
954 d = irq_find_matching_fwnode(fwnode,
955 DOMAIN_BUS_PCI_MSI);
956 }
957
958 return d;
959}
960
961static void pci_set_bus_msi_domain(struct pci_bus *bus)
962{
963 struct irq_domain *d;
964 struct pci_bus *b;
965
966 /*
967 * The bus can be a root bus, a subordinate bus, or a virtual bus
968 * created by an SR-IOV device. Walk up to the first bridge device
969 * found or derive the domain from the host bridge.
970 */
971 for (b = bus, d = NULL; !d && !pci_is_root_bus(b); b = b->parent) {
972 if (b->self)
973 d = dev_get_msi_domain(&b->self->dev);
974 }
975
976 if (!d)
977 d = pci_host_bridge_msi_domain(b);
978
979 dev_set_msi_domain(&bus->dev, d);
980}
981
982static bool pci_preserve_config(struct pci_host_bridge *host_bridge)
983{
984 if (pci_acpi_preserve_config(host_bridge))
985 return true;
986
987 if (host_bridge->dev.parent && host_bridge->dev.parent->of_node)
988 return of_pci_preserve_config(host_bridge->dev.parent->of_node);
989
990 return false;
991}
992
993static int pci_register_host_bridge(struct pci_host_bridge *bridge)
994{
995 struct device *parent = bridge->dev.parent;
996 struct resource_entry *window, *next, *n;
997 struct pci_bus *bus, *b;
998 resource_size_t offset, next_offset;
999 LIST_HEAD(resources);
1000 struct resource *res, *next_res;
1001 bool bus_registered = false;
1002 char addr[64], *fmt;
1003 const char *name;
1004 int err;
1005
1006 bus = pci_alloc_bus(NULL);
1007 if (!bus)
1008 return -ENOMEM;
1009
1010 bridge->bus = bus;
1011
1012 bus->sysdata = bridge->sysdata;
1013 bus->ops = bridge->ops;
1014 bus->number = bus->busn_res.start = bridge->busnr;
1015#ifdef CONFIG_PCI_DOMAINS_GENERIC
1016 if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
1017 bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
1018 else
1019 bus->domain_nr = bridge->domain_nr;
1020 if (bus->domain_nr < 0) {
1021 err = bus->domain_nr;
1022 goto free;
1023 }
1024#endif
1025
1026 b = pci_find_bus(pci_domain_nr(bus), bridge->busnr);
1027 if (b) {
1028 /* Ignore it if we already got here via a different bridge */
1029 dev_dbg(&b->dev, "bus already known\n");
1030 err = -EEXIST;
1031 goto free;
1032 }
1033
1034 dev_set_name(&bridge->dev, "pci%04x:%02x", pci_domain_nr(bus),
1035 bridge->busnr);
1036
1037 err = pcibios_root_bridge_prepare(bridge);
1038 if (err)
1039 goto free;
1040
1041 /* Temporarily move resources off the list */
1042 list_splice_init(&bridge->windows, &resources);
1043 err = device_add(&bridge->dev);
1044 if (err)
1045 goto free;
1046
1047 bus->bridge = get_device(&bridge->dev);
1048 device_enable_async_suspend(bus->bridge);
1049 pci_set_bus_of_node(bus);
1050 pci_set_bus_msi_domain(bus);
1051 if (bridge->msi_domain && !dev_get_msi_domain(&bus->dev) &&
1052 !pci_host_of_has_msi_map(parent))
1053 bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
1054
1055 if (!parent)
1056 set_dev_node(bus->bridge, pcibus_to_node(bus));
1057
1058 bus->dev.class = &pcibus_class;
1059 bus->dev.parent = bus->bridge;
1060
1061 dev_set_name(&bus->dev, "%04x:%02x", pci_domain_nr(bus), bus->number);
1062 name = dev_name(&bus->dev);
1063
1064 err = device_register(&bus->dev);
1065 bus_registered = true;
1066 if (err)
1067 goto unregister;
1068
1069 pcibios_add_bus(bus);
1070
1071 if (bus->ops->add_bus) {
1072 err = bus->ops->add_bus(bus);
1073 if (WARN_ON(err < 0))
1074 dev_err(&bus->dev, "failed to add bus: %d\n", err);
1075 }
1076
1077 if (parent)
1078 dev_info(parent, "PCI host bridge to bus %s\n", name);
1079 else
1080 pr_info("PCI host bridge to bus %s\n", name);
1081
1082 if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
1083 dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
1084
1085 /* Check if the boot configuration by FW needs to be preserved */
1086 bridge->preserve_config = pci_preserve_config(bridge);
1087
1088 /* Coalesce contiguous windows */
1089 resource_list_for_each_entry_safe(window, n, &resources) {
1090 if (list_is_last(&window->node, &resources))
1091 break;
1092
1093 next = list_next_entry(window, node);
1094 offset = window->offset;
1095 res = window->res;
1096 next_offset = next->offset;
1097 next_res = next->res;
1098
1099 if (res->flags != next_res->flags || offset != next_offset)
1100 continue;
1101
1102 if (res->end + 1 == next_res->start) {
1103 next_res->start = res->start;
1104 res->flags = res->start = res->end = 0;
1105 }
1106 }
1107
1108 /* Add initial resources to the bus */
1109 resource_list_for_each_entry_safe(window, n, &resources) {
1110 offset = window->offset;
1111 res = window->res;
1112 if (!res->flags && !res->start && !res->end) {
1113 release_resource(res);
1114 resource_list_destroy_entry(window);
1115 continue;
1116 }
1117
1118 list_move_tail(&window->node, &bridge->windows);
1119
1120 if (res->flags & IORESOURCE_BUS)
1121 pci_bus_insert_busn_res(bus, bus->number, res->end);
1122 else
1123 pci_bus_add_resource(bus, res);
1124
1125 if (offset) {
1126 if (resource_type(res) == IORESOURCE_IO)
1127 fmt = " (bus address [%#06llx-%#06llx])";
1128 else
1129 fmt = " (bus address [%#010llx-%#010llx])";
1130
1131 snprintf(addr, sizeof(addr), fmt,
1132 (unsigned long long)(res->start - offset),
1133 (unsigned long long)(res->end - offset));
1134 } else
1135 addr[0] = '\0';
1136
1137 dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1138 }
1139
1140 of_pci_make_host_bridge_node(bridge);
1141
1142 down_write(&pci_bus_sem);
1143 list_add_tail(&bus->node, &pci_root_buses);
1144 up_write(&pci_bus_sem);
1145
1146 return 0;
1147
1148unregister:
1149 put_device(&bridge->dev);
1150 device_del(&bridge->dev);
1151free:
1152#ifdef CONFIG_PCI_DOMAINS_GENERIC
1153 if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
1154 pci_bus_release_domain_nr(parent, bus->domain_nr);
1155#endif
1156 if (bus_registered)
1157 put_device(&bus->dev);
1158 else
1159 kfree(bus);
1160
1161 return err;
1162}
1163
1164static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1165{
1166 int pos;
1167 u32 status;
1168
1169 /*
1170 * If extended config space isn't accessible on a bridge's primary
1171 * bus, we certainly can't access it on the secondary bus.
1172 */
1173 if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1174 return false;
1175
1176 /*
1177 * PCIe Root Ports and switch ports are PCIe on both sides, so if
1178 * extended config space is accessible on the primary, it's also
1179 * accessible on the secondary.
1180 */
1181 if (pci_is_pcie(bridge) &&
1182 (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1183 pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM ||
1184 pci_pcie_type(bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1185 return true;
1186
1187 /*
1188 * For the other bridge types:
1189 * - PCI-to-PCI bridges
1190 * - PCIe-to-PCI/PCI-X forward bridges
1191 * - PCI/PCI-X-to-PCIe reverse bridges
1192 * extended config space on the secondary side is only accessible
1193 * if the bridge supports PCI-X Mode 2.
1194 */
1195 pos = pci_find_capability(bridge, PCI_CAP_ID_PCIX);
1196 if (!pos)
1197 return false;
1198
1199 pci_read_config_dword(bridge, pos + PCI_X_STATUS, &status);
1200 return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1201}
1202
1203static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1204 struct pci_dev *bridge, int busnr)
1205{
1206 struct pci_bus *child;
1207 struct pci_host_bridge *host;
1208 int i;
1209 int ret;
1210
1211 /* Allocate a new bus and inherit stuff from the parent */
1212 child = pci_alloc_bus(parent);
1213 if (!child)
1214 return NULL;
1215
1216 child->parent = parent;
1217 child->sysdata = parent->sysdata;
1218 child->bus_flags = parent->bus_flags;
1219
1220 host = pci_find_host_bridge(parent);
1221 if (host->child_ops)
1222 child->ops = host->child_ops;
1223 else
1224 child->ops = parent->ops;
1225
1226 /*
1227 * Initialize some portions of the bus device, but don't register
1228 * it now as the parent is not properly set up yet.
1229 */
1230 child->dev.class = &pcibus_class;
1231 dev_set_name(&child->dev, "%04x:%02x", pci_domain_nr(child), busnr);
1232
1233 /* Set up the primary, secondary and subordinate bus numbers */
1234 child->number = child->busn_res.start = busnr;
1235 child->primary = parent->busn_res.start;
1236 child->busn_res.end = 0xff;
1237
1238 if (!bridge) {
1239 child->dev.parent = parent->bridge;
1240 goto add_dev;
1241 }
1242
1243 child->self = bridge;
1244 child->bridge = get_device(&bridge->dev);
1245 child->dev.parent = child->bridge;
1246 pci_set_bus_of_node(child);
1247 pci_set_bus_speed(child);
1248
1249 /*
1250 * Check whether extended config space is accessible on the child
1251 * bus. Note that we currently assume it is always accessible on
1252 * the root bus.
1253 */
1254 if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1255 child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1256 pci_info(child, "extended config space not accessible\n");
1257 }
1258
1259 /* Set up default resource pointers and names */
1260 for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1261 child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1262 child->resource[i]->name = child->name;
1263 }
1264 bridge->subordinate = child;
1265
1266add_dev:
1267 pci_set_bus_msi_domain(child);
1268 ret = device_register(&child->dev);
1269 if (WARN_ON(ret < 0)) {
1270 put_device(&child->dev);
1271 return NULL;
1272 }
1273
1274 pcibios_add_bus(child);
1275
1276 if (child->ops->add_bus) {
1277 ret = child->ops->add_bus(child);
1278 if (WARN_ON(ret < 0))
1279 dev_err(&child->dev, "failed to add bus: %d\n", ret);
1280 }
1281
1282 return child;
1283}
1284
1285struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1286 int busnr)
1287{
1288 struct pci_bus *child;
1289
1290 child = pci_alloc_child_bus(parent, dev, busnr);
1291 if (child) {
1292 down_write(&pci_bus_sem);
1293 list_add_tail(&child->node, &parent->children);
1294 up_write(&pci_bus_sem);
1295 }
1296 return child;
1297}
1298EXPORT_SYMBOL(pci_add_new_bus);
1299
1300static void pci_enable_rrs_sv(struct pci_dev *pdev)
1301{
1302 u16 root_cap = 0;
1303
1304 /* Enable Configuration RRS Software Visibility if supported */
1305 pcie_capability_read_word(pdev, PCI_EXP_RTCAP, &root_cap);
1306 if (root_cap & PCI_EXP_RTCAP_RRS_SV) {
1307 pcie_capability_set_word(pdev, PCI_EXP_RTCTL,
1308 PCI_EXP_RTCTL_RRS_SVE);
1309 pdev->config_rrs_sv = 1;
1310 }
1311}
1312
1313static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1314 unsigned int available_buses);
1315
1316void pbus_validate_busn(struct pci_bus *bus)
1317{
1318 struct pci_bus *upstream = bus->parent;
1319 struct pci_dev *bridge = bus->self;
1320
1321 /* Check that all devices are accessible */
1322 while (upstream->parent) {
1323 if ((bus->busn_res.end > upstream->busn_res.end) ||
1324 (bus->number > upstream->busn_res.end) ||
1325 (bus->number < upstream->number) ||
1326 (bus->busn_res.end < upstream->number)) {
1327 pci_info(bridge, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1328 &bus->busn_res);
1329 break;
1330 }
1331 upstream = upstream->parent;
1332 }
1333}
1334
1335/**
1336 * pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1337 * numbers from EA capability.
1338 * @dev: Bridge
1339 * @sec: updated with secondary bus number from EA
1340 * @sub: updated with subordinate bus number from EA
1341 *
1342 * If @dev is a bridge with EA capability that specifies valid secondary
1343 * and subordinate bus numbers, return true with the bus numbers in @sec
1344 * and @sub. Otherwise return false.
1345 */
1346bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1347{
1348 int ea, offset;
1349 u32 dw;
1350 u8 ea_sec, ea_sub;
1351
1352 if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1353 return false;
1354
1355 /* find PCI EA capability in list */
1356 ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1357 if (!ea)
1358 return false;
1359
1360 offset = ea + PCI_EA_FIRST_ENT;
1361 pci_read_config_dword(dev, offset, &dw);
1362 ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1363 ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1364 if (ea_sec == 0 || ea_sub < ea_sec)
1365 return false;
1366
1367 *sec = ea_sec;
1368 *sub = ea_sub;
1369 return true;
1370}
1371
1372/*
1373 * pci_scan_bridge_extend() - Scan buses behind a bridge
1374 * @bus: Parent bus the bridge is on
1375 * @dev: Bridge itself
1376 * @max: Starting subordinate number of buses behind this bridge
1377 * @available_buses: Total number of buses available for this bridge and
1378 * the devices below. After the minimal bus space has
1379 * been allocated the remaining buses will be
1380 * distributed equally between hotplug-capable bridges.
1381 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1382 * that need to be reconfigured.
1383 *
1384 * If it's a bridge, configure it and scan the bus behind it.
1385 * For CardBus bridges, we don't scan behind as the devices will
1386 * be handled by the bridge driver itself.
1387 *
1388 * We need to process bridges in two passes -- first we scan those
1389 * already configured by the BIOS and after we are done with all of
1390 * them, we proceed to assigning numbers to the remaining buses in
1391 * order to avoid overlaps between old and new bus numbers.
1392 *
1393 * Return: New subordinate number covering all buses behind this bridge.
1394 */
1395static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1396 int max, unsigned int available_buses,
1397 int pass)
1398{
1399 struct pci_bus *child;
1400 u32 buses;
1401 u16 bctl;
1402 u8 primary, secondary, subordinate;
1403 int broken = 0;
1404 bool fixed_buses;
1405 u8 fixed_sec, fixed_sub;
1406 int next_busnr;
1407
1408 /*
1409 * Make sure the bridge is powered on to be able to access config
1410 * space of devices below it.
1411 */
1412 pm_runtime_get_sync(&dev->dev);
1413
1414 pci_read_config_dword(dev, PCI_PRIMARY_BUS, &buses);
1415 primary = FIELD_GET(PCI_PRIMARY_BUS_MASK, buses);
1416 secondary = FIELD_GET(PCI_SECONDARY_BUS_MASK, buses);
1417 subordinate = FIELD_GET(PCI_SUBORDINATE_BUS_MASK, buses);
1418
1419 pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1420 secondary, subordinate, pass);
1421
1422 if (!primary && (primary != bus->number) && secondary && subordinate) {
1423 pci_warn(dev, "Primary bus is hard wired to 0\n");
1424 primary = bus->number;
1425 }
1426
1427 /* Check if setup is sensible at all */
1428 if (!pass &&
1429 (primary != bus->number || secondary <= bus->number ||
1430 secondary > subordinate)) {
1431 pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1432 secondary, subordinate);
1433 broken = 1;
1434 }
1435
1436 /*
1437 * Disable Master-Abort Mode during probing to avoid reporting of
1438 * bus errors in some architectures.
1439 */
1440 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &bctl);
1441 pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1442 bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1443
1444 if (pci_is_cardbus_bridge(dev)) {
1445 max = pci_cardbus_scan_bridge_extend(bus, dev, buses, max,
1446 available_buses,
1447 pass);
1448 goto out;
1449 }
1450
1451 if ((secondary || subordinate) &&
1452 !pcibios_assign_all_busses() && !broken) {
1453 unsigned int cmax, buses;
1454
1455 /*
1456 * Bus already configured by firmware, process it in the
1457 * first pass and just note the configuration.
1458 */
1459 if (pass)
1460 goto out;
1461
1462 /*
1463 * The bus might already exist for two reasons: Either we
1464 * are rescanning the bus or the bus is reachable through
1465 * more than one bridge. The second case can happen with
1466 * the i450NX chipset.
1467 */
1468 child = pci_find_bus(pci_domain_nr(bus), secondary);
1469 if (!child) {
1470 child = pci_add_new_bus(bus, dev, secondary);
1471 if (!child)
1472 goto out;
1473 child->primary = primary;
1474 pci_bus_insert_busn_res(child, secondary, subordinate);
1475 child->bridge_ctl = bctl;
1476 }
1477
1478 buses = subordinate - secondary;
1479 cmax = pci_scan_child_bus_extend(child, buses);
1480 if (cmax > subordinate)
1481 pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1482 subordinate, cmax);
1483
1484 /* Subordinate should equal child->busn_res.end */
1485 if (subordinate > max)
1486 max = subordinate;
1487 } else {
1488
1489 /*
1490 * We need to assign a number to this bus which we always
1491 * do in the second pass.
1492 */
1493 if (!pass) {
1494 if (pcibios_assign_all_busses() || broken)
1495
1496 /*
1497 * Temporarily disable forwarding of the
1498 * configuration cycles on all bridges in
1499 * this bus segment to avoid possible
1500 * conflicts in the second pass between two
1501 * bridges programmed with overlapping bus
1502 * ranges.
1503 */
1504 pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1505 buses & PCI_SEC_LATENCY_TIMER_MASK);
1506 goto out;
1507 }
1508
1509 /* Clear errors */
1510 pci_write_config_word(dev, PCI_STATUS, 0xffff);
1511
1512 /* Read bus numbers from EA Capability (if present) */
1513 fixed_buses = pci_ea_fixed_busnrs(dev, &fixed_sec, &fixed_sub);
1514 if (fixed_buses)
1515 next_busnr = fixed_sec;
1516 else
1517 next_busnr = max + 1;
1518
1519 /*
1520 * Prevent assigning a bus number that already exists.
1521 * This can happen when a bridge is hot-plugged, so in this
1522 * case we only re-scan this bus.
1523 */
1524 child = pci_find_bus(pci_domain_nr(bus), next_busnr);
1525 if (!child) {
1526 child = pci_add_new_bus(bus, dev, next_busnr);
1527 if (!child)
1528 goto out;
1529 pci_bus_insert_busn_res(child, next_busnr,
1530 bus->busn_res.end);
1531 }
1532 max++;
1533 if (available_buses)
1534 available_buses--;
1535
1536 buses = (buses & PCI_SEC_LATENCY_TIMER_MASK) |
1537 FIELD_PREP(PCI_PRIMARY_BUS_MASK, child->primary) |
1538 FIELD_PREP(PCI_SECONDARY_BUS_MASK, child->busn_res.start) |
1539 FIELD_PREP(PCI_SUBORDINATE_BUS_MASK, child->busn_res.end);
1540
1541 /* We need to blast all three values with a single write */
1542 pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
1543
1544 child->bridge_ctl = bctl;
1545 max = pci_scan_child_bus_extend(child, available_buses);
1546
1547 /*
1548 * Set subordinate bus number to its real value.
1549 * If fixed subordinate bus number exists from EA
1550 * capability then use it.
1551 */
1552 if (fixed_buses)
1553 max = fixed_sub;
1554 pci_bus_update_busn_res_end(child, max);
1555 pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, max);
1556 }
1557 scnprintf(child->name, sizeof(child->name), "PCI Bus %04x:%02x",
1558 pci_domain_nr(bus), child->number);
1559
1560 pbus_validate_busn(child);
1561
1562out:
1563 /* Clear errors in the Secondary Status Register */
1564 pci_write_config_word(dev, PCI_SEC_STATUS, 0xffff);
1565
1566 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, bctl);
1567
1568 pm_runtime_put(&dev->dev);
1569
1570 return max;
1571}
1572
1573/*
1574 * pci_scan_bridge() - Scan buses behind a bridge
1575 * @bus: Parent bus the bridge is on
1576 * @dev: Bridge itself
1577 * @max: Starting subordinate number of buses behind this bridge
1578 * @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1579 * that need to be reconfigured.
1580 *
1581 * If it's a bridge, configure it and scan the bus behind it.
1582 * For CardBus bridges, we don't scan behind as the devices will
1583 * be handled by the bridge driver itself.
1584 *
1585 * We need to process bridges in two passes -- first we scan those
1586 * already configured by the BIOS and after we are done with all of
1587 * them, we proceed to assigning numbers to the remaining buses in
1588 * order to avoid overlaps between old and new bus numbers.
1589 *
1590 * Return: New subordinate number covering all buses behind this bridge.
1591 */
1592int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1593{
1594 return pci_scan_bridge_extend(bus, dev, max, 0, pass);
1595}
1596EXPORT_SYMBOL(pci_scan_bridge);
1597
1598/*
1599 * Read interrupt line and base address registers.
1600 * The architecture-dependent code can tweak these, of course.
1601 */
1602static void pci_read_irq(struct pci_dev *dev)
1603{
1604 unsigned char irq;
1605
1606 /* VFs are not allowed to use INTx, so skip the config reads */
1607 if (dev->is_virtfn) {
1608 dev->pin = 0;
1609 dev->irq = 0;
1610 return;
1611 }
1612
1613 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &irq);
1614 dev->pin = irq;
1615 if (irq)
1616 pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &irq);
1617 dev->irq = irq;
1618}
1619
1620void set_pcie_port_type(struct pci_dev *pdev)
1621{
1622 int pos;
1623 u16 reg16;
1624 u32 reg32;
1625 int type;
1626 struct pci_dev *parent;
1627
1628 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
1629 if (!pos)
1630 return;
1631
1632 pdev->pcie_cap = pos;
1633 pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
1634 pdev->pcie_flags_reg = reg16;
1635
1636 type = pci_pcie_type(pdev);
1637 if (type == PCI_EXP_TYPE_ROOT_PORT)
1638 pci_enable_rrs_sv(pdev);
1639
1640 pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->devcap);
1641 pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1642
1643 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &reg32);
1644 if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1645 pdev->link_active_reporting = 1;
1646
1647#ifdef CONFIG_PCIEASPM
1648 if (reg32 & PCI_EXP_LNKCAP_ASPM_L0S)
1649 pdev->aspm_l0s_support = 1;
1650 if (reg32 & PCI_EXP_LNKCAP_ASPM_L1)
1651 pdev->aspm_l1_support = 1;
1652#endif
1653
1654 parent = pci_upstream_bridge(pdev);
1655 if (!parent)
1656 return;
1657
1658 /*
1659 * Some systems do not identify their upstream/downstream ports
1660 * correctly so detect impossible configurations here and correct
1661 * the port type accordingly.
1662 */
1663 if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1664 /*
1665 * If pdev claims to be downstream port but the parent
1666 * device is also downstream port assume pdev is actually
1667 * upstream port.
1668 */
1669 if (pcie_downstream_port(parent)) {
1670 pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1671 pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1672 pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1673 }
1674 } else if (type == PCI_EXP_TYPE_UPSTREAM) {
1675 /*
1676 * If pdev claims to be upstream port but the parent
1677 * device is also upstream port assume pdev is actually
1678 * downstream port.
1679 */
1680 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM) {
1681 pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1682 pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1683 pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1684 }
1685 }
1686}
1687
1688void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1689{
1690 u32 reg32;
1691
1692 pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &reg32);
1693 if (reg32 & PCI_EXP_SLTCAP_HPC)
1694 pdev->is_hotplug_bridge = pdev->is_pciehp = 1;
1695}
1696
1697static void set_pcie_thunderbolt(struct pci_dev *dev)
1698{
1699 u16 vsec;
1700
1701 /* Is the device part of a Thunderbolt controller? */
1702 vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1703 if (vsec)
1704 dev->is_thunderbolt = 1;
1705}
1706
1707static void set_pcie_cxl(struct pci_dev *dev)
1708{
1709 struct pci_dev *bridge;
1710 u16 dvsec, cap;
1711
1712 if (!pci_is_pcie(dev))
1713 return;
1714
1715 /*
1716 * Update parent's CXL state because alternate protocol training
1717 * may have changed
1718 */
1719 bridge = pci_upstream_bridge(dev);
1720 if (bridge)
1721 set_pcie_cxl(bridge);
1722
1723 dvsec = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL,
1724 PCI_DVSEC_CXL_FLEXBUS_PORT);
1725 if (!dvsec)
1726 return;
1727
1728 pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS,
1729 &cap);
1730
1731 dev->is_cxl = FIELD_GET(PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_CACHE, cap) ||
1732 FIELD_GET(PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_MEM, cap);
1733
1734}
1735
1736static void set_pcie_untrusted(struct pci_dev *dev)
1737{
1738 struct pci_dev *parent = pci_upstream_bridge(dev);
1739
1740 if (!parent)
1741 return;
1742 /*
1743 * If the upstream bridge is untrusted we treat this device as
1744 * untrusted as well.
1745 */
1746 if (parent->untrusted) {
1747 dev->untrusted = true;
1748 return;
1749 }
1750
1751 if (arch_pci_dev_is_removable(dev)) {
1752 pci_dbg(dev, "marking as untrusted\n");
1753 dev->untrusted = true;
1754 }
1755}
1756
1757static void pci_set_removable(struct pci_dev *dev)
1758{
1759 struct pci_dev *parent = pci_upstream_bridge(dev);
1760
1761 if (!parent)
1762 return;
1763 /*
1764 * We (only) consider everything tunneled below an external_facing
1765 * device to be removable by the user. We're mainly concerned with
1766 * consumer platforms with user accessible thunderbolt ports that are
1767 * vulnerable to DMA attacks, and we expect those ports to be marked by
1768 * the firmware as external_facing. Devices in traditional hotplug
1769 * slots can technically be removed, but the expectation is that unless
1770 * the port is marked with external_facing, such devices are less
1771 * accessible to user / may not be removed by end user, and thus not
1772 * exposed as "removable" to userspace.
1773 */
1774 if (dev_is_removable(&parent->dev)) {
1775 dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1776 return;
1777 }
1778
1779 if (arch_pci_dev_is_removable(dev)) {
1780 pci_dbg(dev, "marking as removable\n");
1781 dev_set_removable(&dev->dev, DEVICE_REMOVABLE);
1782 }
1783}
1784
1785/**
1786 * pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1787 * @dev: PCI device
1788 *
1789 * PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1790 * when forwarding a type1 configuration request the bridge must check that
1791 * the extended register address field is zero. The bridge is not permitted
1792 * to forward the transactions and must handle it as an Unsupported Request.
1793 * Some bridges do not follow this rule and simply drop the extended register
1794 * bits, resulting in the standard config space being aliased, every 256
1795 * bytes across the entire configuration space. Test for this condition by
1796 * comparing the first dword of each potential alias to the vendor/device ID.
1797 * Known offenders:
1798 * ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1799 * AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1800 */
1801static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1802{
1803#ifdef CONFIG_PCI_QUIRKS
1804 int pos, ret;
1805 u32 header, tmp;
1806
1807 pci_read_config_dword(dev, PCI_VENDOR_ID, &header);
1808
1809 for (pos = PCI_CFG_SPACE_SIZE;
1810 pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1811 ret = pci_read_config_dword(dev, pos, &tmp);
1812 if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp))
1813 return false;
1814 }
1815
1816 return true;
1817#else
1818 return false;
1819#endif
1820}
1821
1822/**
1823 * pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1824 * @dev: PCI device
1825 *
1826 * Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1827 * have 4096 bytes. Even if the device is capable, that doesn't mean we can
1828 * access it. Maybe we don't have a way to generate extended config space
1829 * accesses, or the device is behind a reverse Express bridge. So we try
1830 * reading the dword at 0x100 which must either be 0 or a valid extended
1831 * capability header.
1832 */
1833static int pci_cfg_space_size_ext(struct pci_dev *dev)
1834{
1835 u32 status;
1836 int pos = PCI_CFG_SPACE_SIZE;
1837
1838 if (pci_read_config_dword(dev, pos, &status) != PCIBIOS_SUCCESSFUL)
1839 return PCI_CFG_SPACE_SIZE;
1840 if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev))
1841 return PCI_CFG_SPACE_SIZE;
1842
1843 return PCI_CFG_SPACE_EXP_SIZE;
1844}
1845
1846int pci_cfg_space_size(struct pci_dev *dev)
1847{
1848 int pos;
1849 u32 status;
1850 u16 class;
1851
1852#ifdef CONFIG_PCI_IOV
1853 /*
1854 * Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1855 * implement a PCIe capability and therefore must implement extended
1856 * config space. We can skip the NO_EXTCFG test below and the
1857 * reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1858 * the fact that the SR-IOV capability on the PF resides in extended
1859 * config space and must be accessible and non-aliased to have enabled
1860 * support for this VF. This is a micro performance optimization for
1861 * systems supporting many VFs.
1862 */
1863 if (dev->is_virtfn)
1864 return PCI_CFG_SPACE_EXP_SIZE;
1865#endif
1866
1867 if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1868 return PCI_CFG_SPACE_SIZE;
1869
1870 class = dev->class >> 8;
1871 if (class == PCI_CLASS_BRIDGE_HOST)
1872 return pci_cfg_space_size_ext(dev);
1873
1874 if (pci_is_pcie(dev))
1875 return pci_cfg_space_size_ext(dev);
1876
1877 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1878 if (!pos)
1879 return PCI_CFG_SPACE_SIZE;
1880
1881 pci_read_config_dword(dev, pos + PCI_X_STATUS, &status);
1882 if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1883 return pci_cfg_space_size_ext(dev);
1884
1885 return PCI_CFG_SPACE_SIZE;
1886}
1887
1888static u32 pci_class(struct pci_dev *dev)
1889{
1890 u32 class;
1891
1892#ifdef CONFIG_PCI_IOV
1893 if (dev->is_virtfn)
1894 return dev->physfn->sriov->class;
1895#endif
1896 pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
1897 return class;
1898}
1899
1900static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1901{
1902#ifdef CONFIG_PCI_IOV
1903 if (dev->is_virtfn) {
1904 *vendor = dev->physfn->sriov->subsystem_vendor;
1905 *device = dev->physfn->sriov->subsystem_device;
1906 return;
1907 }
1908#endif
1909 pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, vendor);
1910 pci_read_config_word(dev, PCI_SUBSYSTEM_ID, device);
1911}
1912
1913static u8 pci_hdr_type(struct pci_dev *dev)
1914{
1915 u8 hdr_type;
1916
1917#ifdef CONFIG_PCI_IOV
1918 if (dev->is_virtfn)
1919 return dev->physfn->sriov->hdr_type;
1920#endif
1921 pci_read_config_byte(dev, PCI_HEADER_TYPE, &hdr_type);
1922 return hdr_type;
1923}
1924
1925#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1926
1927/**
1928 * pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1929 * @dev: PCI device
1930 *
1931 * Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev. Check this
1932 * at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1933 */
1934static int pci_intx_mask_broken(struct pci_dev *dev)
1935{
1936 u16 orig, toggle, new;
1937
1938 pci_read_config_word(dev, PCI_COMMAND, &orig);
1939 toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1940 pci_write_config_word(dev, PCI_COMMAND, toggle);
1941 pci_read_config_word(dev, PCI_COMMAND, &new);
1942
1943 pci_write_config_word(dev, PCI_COMMAND, orig);
1944
1945 /*
1946 * PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1947 * r2.3, so strictly speaking, a device is not *broken* if it's not
1948 * writable. But we'll live with the misnomer for now.
1949 */
1950 if (new != toggle)
1951 return 1;
1952 return 0;
1953}
1954
1955static void early_dump_pci_device(struct pci_dev *pdev)
1956{
1957 u32 value[PCI_CFG_SPACE_SIZE / sizeof(u32)];
1958 int i;
1959
1960 pci_info(pdev, "config space:\n");
1961
1962 for (i = 0; i < ARRAY_SIZE(value); i++)
1963 pci_read_config_dword(pdev, i * sizeof(u32), &value[i]);
1964
1965 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
1966 value, ARRAY_SIZE(value) * sizeof(u32), false);
1967}
1968
1969static const char *pci_type_str(struct pci_dev *dev)
1970{
1971 static const char * const str[] = {
1972 "PCIe Endpoint",
1973 "PCIe Legacy Endpoint",
1974 "PCIe unknown",
1975 "PCIe unknown",
1976 "PCIe Root Port",
1977 "PCIe Switch Upstream Port",
1978 "PCIe Switch Downstream Port",
1979 "PCIe to PCI/PCI-X bridge",
1980 "PCI/PCI-X to PCIe bridge",
1981 "PCIe Root Complex Integrated Endpoint",
1982 "PCIe Root Complex Event Collector",
1983 };
1984 int type;
1985
1986 if (pci_is_pcie(dev)) {
1987 type = pci_pcie_type(dev);
1988 if (type < ARRAY_SIZE(str))
1989 return str[type];
1990
1991 return "PCIe unknown";
1992 }
1993
1994 switch (dev->hdr_type) {
1995 case PCI_HEADER_TYPE_NORMAL:
1996 return "conventional PCI endpoint";
1997 case PCI_HEADER_TYPE_BRIDGE:
1998 return "conventional PCI bridge";
1999 case PCI_HEADER_TYPE_CARDBUS:
2000 return "CardBus bridge";
2001 default:
2002 return "conventional PCI";
2003 }
2004}
2005
2006/**
2007 * pci_setup_device - Fill in class and map information of a device
2008 * @dev: the device structure to fill
2009 *
2010 * Initialize the device structure with information about the device's
2011 * vendor,class,memory and IO-space addresses, IRQ lines etc.
2012 * Called at initialisation of the PCI subsystem and by CardBus services.
2013 * Returns 0 on success and negative if unknown type of device (not normal,
2014 * bridge or CardBus).
2015 */
2016int pci_setup_device(struct pci_dev *dev)
2017{
2018 u32 class;
2019 u16 cmd;
2020 u8 hdr_type;
2021 int err, pos = 0;
2022 struct pci_bus_region region;
2023 struct resource *res;
2024
2025 hdr_type = pci_hdr_type(dev);
2026
2027 dev->sysdata = dev->bus->sysdata;
2028 dev->dev.parent = dev->bus->bridge;
2029 dev->dev.bus = &pci_bus_type;
2030 dev->hdr_type = FIELD_GET(PCI_HEADER_TYPE_MASK, hdr_type);
2031 dev->multifunction = FIELD_GET(PCI_HEADER_TYPE_MFD, hdr_type);
2032 dev->error_state = pci_channel_io_normal;
2033 set_pcie_port_type(dev);
2034
2035 err = pci_set_of_node(dev);
2036 if (err)
2037 return err;
2038 pci_set_acpi_fwnode(dev);
2039
2040 pci_dev_assign_slot(dev);
2041
2042 /*
2043 * Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
2044 * set this higher, assuming the system even supports it.
2045 */
2046 dev->dma_mask = 0xffffffff;
2047
2048 /*
2049 * Assume 64-bit addresses for MSI initially. Will be changed to 32-bit
2050 * if MSI (rather than MSI-X) capability does not have
2051 * PCI_MSI_FLAGS_64BIT. Can also be overridden by driver.
2052 */
2053 dev->msi_addr_mask = DMA_BIT_MASK(64);
2054
2055 dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
2056 dev->bus->number, PCI_SLOT(dev->devfn),
2057 PCI_FUNC(dev->devfn));
2058
2059 class = pci_class(dev);
2060
2061 dev->revision = class & 0xff;
2062 dev->class = class >> 8; /* upper 3 bytes */
2063
2064 if (pci_early_dump)
2065 early_dump_pci_device(dev);
2066
2067 /* Need to have dev->class ready */
2068 dev->cfg_size = pci_cfg_space_size(dev);
2069
2070 /* Need to have dev->cfg_size ready */
2071 set_pcie_thunderbolt(dev);
2072
2073 set_pcie_cxl(dev);
2074
2075 set_pcie_untrusted(dev);
2076
2077 if (pci_is_pcie(dev))
2078 dev->supported_speeds = pcie_get_supported_speeds(dev);
2079
2080 /* "Unknown power state" */
2081 dev->current_state = PCI_UNKNOWN;
2082
2083 /* Early fixups, before probing the BARs */
2084 pci_fixup_device(pci_fixup_early, dev);
2085
2086 pci_set_removable(dev);
2087
2088 pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n",
2089 dev->vendor, dev->device, dev->hdr_type, dev->class,
2090 pci_type_str(dev));
2091
2092 /* Device class may be changed after fixup */
2093 class = dev->class >> 8;
2094
2095 if (dev->non_compliant_bars && !dev->mmio_always_on) {
2096 pci_read_config_word(dev, PCI_COMMAND, &cmd);
2097 if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
2098 pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
2099 cmd &= ~PCI_COMMAND_IO;
2100 cmd &= ~PCI_COMMAND_MEMORY;
2101 pci_write_config_word(dev, PCI_COMMAND, cmd);
2102 }
2103 }
2104
2105 dev->broken_intx_masking = pci_intx_mask_broken(dev);
2106
2107 switch (dev->hdr_type) { /* header type */
2108 case PCI_HEADER_TYPE_NORMAL: /* standard header */
2109 if (class == PCI_CLASS_BRIDGE_PCI)
2110 goto bad;
2111 pci_read_irq(dev);
2112 pci_read_bases(dev, PCI_STD_NUM_BARS, PCI_ROM_ADDRESS);
2113
2114 pci_subsystem_ids(dev, &dev->subsystem_vendor, &dev->subsystem_device);
2115
2116 /*
2117 * Do the ugly legacy mode stuff here rather than broken chip
2118 * quirk code. Legacy mode ATA controllers have fixed
2119 * addresses. These are not always echoed in BAR0-3, and
2120 * BAR0-3 in a few cases contain junk!
2121 */
2122 if (class == PCI_CLASS_STORAGE_IDE) {
2123 u8 progif;
2124 pci_read_config_byte(dev, PCI_CLASS_PROG, &progif);
2125 if ((progif & 1) == 0) {
2126 region.start = 0x1F0;
2127 region.end = 0x1F7;
2128 res = &dev->resource[0];
2129 res->flags = LEGACY_IO_RESOURCE;
2130 pcibios_bus_to_resource(dev->bus, res, &region);
2131 pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n",
2132 res);
2133 region.start = 0x3F6;
2134 region.end = 0x3F6;
2135 res = &dev->resource[1];
2136 res->flags = LEGACY_IO_RESOURCE;
2137 pcibios_bus_to_resource(dev->bus, res, &region);
2138 pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n",
2139 res);
2140 }
2141 if ((progif & 4) == 0) {
2142 region.start = 0x170;
2143 region.end = 0x177;
2144 res = &dev->resource[2];
2145 res->flags = LEGACY_IO_RESOURCE;
2146 pcibios_bus_to_resource(dev->bus, res, &region);
2147 pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n",
2148 res);
2149 region.start = 0x376;
2150 region.end = 0x376;
2151 res = &dev->resource[3];
2152 res->flags = LEGACY_IO_RESOURCE;
2153 pcibios_bus_to_resource(dev->bus, res, &region);
2154 pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n",
2155 res);
2156 }
2157 }
2158 break;
2159
2160 case PCI_HEADER_TYPE_BRIDGE: /* bridge header */
2161 /*
2162 * The PCI-to-PCI bridge spec requires that subtractive
2163 * decoding (i.e. transparent) bridge must have programming
2164 * interface code of 0x01.
2165 */
2166 pci_read_irq(dev);
2167 dev->transparent = ((dev->class & 0xff) == 1);
2168 pci_read_bases(dev, 2, PCI_ROM_ADDRESS1);
2169 pci_read_bridge_windows(dev);
2170 set_pcie_hotplug_bridge(dev);
2171 pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
2172 if (pos) {
2173 pci_read_config_word(dev, pos + PCI_SSVID_VENDOR_ID, &dev->subsystem_vendor);
2174 pci_read_config_word(dev, pos + PCI_SSVID_DEVICE_ID, &dev->subsystem_device);
2175 }
2176 break;
2177
2178 case PCI_HEADER_TYPE_CARDBUS: /* CardBus bridge header */
2179 if (class != PCI_CLASS_BRIDGE_CARDBUS)
2180 goto bad;
2181 pci_read_irq(dev);
2182 pci_read_bases(dev, 1, 0);
2183 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, &dev->subsystem_vendor);
2184 pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, &dev->subsystem_device);
2185 break;
2186
2187 default: /* unknown header */
2188 pci_err(dev, "unknown header type %02x, ignoring device\n",
2189 dev->hdr_type);
2190 pci_release_of_node(dev);
2191 return -EIO;
2192
2193 bad:
2194 pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
2195 dev->class, dev->hdr_type);
2196 dev->class = PCI_CLASS_NOT_DEFINED << 8;
2197 }
2198
2199 /* We found a fine healthy device, go go go... */
2200 return 0;
2201}
2202
2203static void pci_configure_mps(struct pci_dev *dev)
2204{
2205 struct pci_dev *bridge = pci_upstream_bridge(dev);
2206 int mps, mpss, p_mps, rc;
2207
2208 if (!pci_is_pcie(dev))
2209 return;
2210
2211 /* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
2212 if (dev->is_virtfn)
2213 return;
2214
2215 /*
2216 * For Root Complex Integrated Endpoints, program the maximum
2217 * supported value unless limited by the PCIE_BUS_PEER2PEER case.
2218 */
2219 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2220 if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2221 mps = 128;
2222 else
2223 mps = 128 << dev->pcie_mpss;
2224 rc = pcie_set_mps(dev, mps);
2225 if (rc) {
2226 pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2227 mps);
2228 }
2229 return;
2230 }
2231
2232 if (!bridge || !pci_is_pcie(bridge))
2233 return;
2234
2235 mps = pcie_get_mps(dev);
2236 p_mps = pcie_get_mps(bridge);
2237
2238 if (mps == p_mps)
2239 return;
2240
2241 if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2242 pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2243 mps, pci_name(bridge), p_mps);
2244 return;
2245 }
2246
2247 /*
2248 * Fancier MPS configuration is done later by
2249 * pcie_bus_configure_settings()
2250 */
2251 if (pcie_bus_config != PCIE_BUS_DEFAULT)
2252 return;
2253
2254 mpss = 128 << dev->pcie_mpss;
2255 if (mpss < p_mps && pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2256 pcie_set_mps(bridge, mpss);
2257 pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2258 mpss, p_mps, 128 << bridge->pcie_mpss);
2259 p_mps = pcie_get_mps(bridge);
2260 }
2261
2262 rc = pcie_set_mps(dev, p_mps);
2263 if (rc) {
2264 pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2265 p_mps);
2266 return;
2267 }
2268
2269 pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2270 p_mps, mps, mpss);
2271}
2272
2273int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2274{
2275 struct pci_host_bridge *host;
2276 u32 cap;
2277 u16 ctl;
2278 int ret;
2279
2280 /* PCI_EXP_DEVCTL_EXT_TAG is RsvdP in VFs */
2281 if (!pci_is_pcie(dev) || dev->is_virtfn)
2282 return 0;
2283
2284 ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
2285 if (ret)
2286 return 0;
2287
2288 if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2289 return 0;
2290
2291 ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
2292 if (ret)
2293 return 0;
2294
2295 host = pci_find_host_bridge(dev->bus);
2296 if (!host)
2297 return 0;
2298
2299 /*
2300 * If some device in the hierarchy doesn't handle Extended Tags
2301 * correctly, make sure they're disabled.
2302 */
2303 if (host->no_ext_tags) {
2304 if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2305 pci_info(dev, "disabling Extended Tags\n");
2306 pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2307 PCI_EXP_DEVCTL_EXT_TAG);
2308 }
2309 return 0;
2310 }
2311
2312 if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2313 pci_info(dev, "enabling Extended Tags\n");
2314 pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2315 PCI_EXP_DEVCTL_EXT_TAG);
2316 }
2317 return 0;
2318}
2319
2320static void pci_dev3_init(struct pci_dev *pdev)
2321{
2322 u16 cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DEV3);
2323 u32 val = 0;
2324
2325 if (!cap)
2326 return;
2327 pci_read_config_dword(pdev, cap + PCI_DEV3_STA, &val);
2328 pdev->fm_enabled = !!(val & PCI_DEV3_STA_SEGMENT);
2329}
2330
2331/**
2332 * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2333 * @dev: PCI device to query
2334 *
2335 * Returns true if the device has enabled relaxed ordering attribute.
2336 */
2337bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2338{
2339 u16 v;
2340
2341 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &v);
2342
2343 return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2344}
2345EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2346
2347static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2348{
2349 struct pci_dev *root;
2350
2351 /* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */
2352 if (dev->is_virtfn)
2353 return;
2354
2355 if (!pcie_relaxed_ordering_enabled(dev))
2356 return;
2357
2358 /*
2359 * For now, we only deal with Relaxed Ordering issues with Root
2360 * Ports. Peer-to-Peer DMA is another can of worms.
2361 */
2362 root = pcie_find_root_port(dev);
2363 if (!root)
2364 return;
2365
2366 if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2367 pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2368 PCI_EXP_DEVCTL_RELAX_EN);
2369 pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2370 }
2371}
2372
2373static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2374{
2375 struct pci_dev *bridge;
2376 unsigned int eetlp_max;
2377 int pcie_type;
2378 u32 cap;
2379
2380 if (!pci_is_pcie(dev))
2381 return;
2382
2383 pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
2384 if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2385 return;
2386
2387 pcie_type = pci_pcie_type(dev);
2388
2389 eetlp_max = FIELD_GET(PCI_EXP_DEVCAP2_EE_PREFIX_MAX, cap);
2390 /* 00b means 4 */
2391 eetlp_max = eetlp_max ?: 4;
2392
2393 if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2394 pcie_type == PCI_EXP_TYPE_RC_END)
2395 dev->eetlp_prefix_max = eetlp_max;
2396 else {
2397 bridge = pci_upstream_bridge(dev);
2398 if (bridge && bridge->eetlp_prefix_max)
2399 dev->eetlp_prefix_max = eetlp_max;
2400 }
2401}
2402
2403static void pci_configure_serr(struct pci_dev *dev)
2404{
2405 u16 control;
2406
2407 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2408
2409 /*
2410 * A bridge will not forward ERR_ messages coming from an
2411 * endpoint unless SERR# forwarding is enabled.
2412 */
2413 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &control);
2414 if (!(control & PCI_BRIDGE_CTL_SERR)) {
2415 control |= PCI_BRIDGE_CTL_SERR;
2416 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, control);
2417 }
2418 }
2419}
2420
2421static void pci_configure_rcb(struct pci_dev *dev)
2422{
2423 struct pci_dev *rp;
2424 u16 rp_lnkctl;
2425
2426 /*
2427 * Per PCIe r7.0, sec 7.5.3.7, RCB is only meaningful in Root Ports
2428 * (where it is read-only), Endpoints, and Bridges. It may only be
2429 * set for Endpoints and Bridges if it is set in the Root Port. For
2430 * Endpoints, it is 'RsvdP' for Virtual Functions.
2431 */
2432 if (!pci_is_pcie(dev) ||
2433 pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT ||
2434 pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM ||
2435 pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
2436 pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC ||
2437 dev->is_virtfn)
2438 return;
2439
2440 /* Root Port often not visible to virtualized guests */
2441 rp = pcie_find_root_port(dev);
2442 if (!rp)
2443 return;
2444
2445 pcie_capability_read_word(rp, PCI_EXP_LNKCTL, &rp_lnkctl);
2446 pcie_capability_clear_and_set_word(dev, PCI_EXP_LNKCTL,
2447 PCI_EXP_LNKCTL_RCB,
2448 (rp_lnkctl & PCI_EXP_LNKCTL_RCB) ?
2449 PCI_EXP_LNKCTL_RCB : 0);
2450}
2451
2452static void pci_configure_device(struct pci_dev *dev)
2453{
2454 pci_configure_mps(dev);
2455 pci_configure_extended_tags(dev, NULL);
2456 pci_configure_relaxed_ordering(dev);
2457 pci_configure_ltr(dev);
2458 pci_configure_aspm_l1ss(dev);
2459 pci_configure_eetlp_prefix(dev);
2460 pci_configure_serr(dev);
2461 pci_configure_rcb(dev);
2462
2463 pci_acpi_program_hp_params(dev);
2464}
2465
2466static void pci_release_capabilities(struct pci_dev *dev)
2467{
2468 pci_aer_exit(dev);
2469 pci_rcec_exit(dev);
2470 pci_iov_release(dev);
2471 pci_free_cap_save_buffers(dev);
2472}
2473
2474/**
2475 * pci_release_dev - Free a PCI device structure when all users of it are
2476 * finished
2477 * @dev: device that's been disconnected
2478 *
2479 * Will be called only by the device core when all users of this PCI device are
2480 * done.
2481 */
2482static void pci_release_dev(struct device *dev)
2483{
2484 struct pci_dev *pci_dev;
2485
2486 pci_dev = to_pci_dev(dev);
2487 pci_release_capabilities(pci_dev);
2488 pci_release_of_node(pci_dev);
2489 pcibios_release_device(pci_dev);
2490 pci_bus_put(pci_dev->bus);
2491 bitmap_free(pci_dev->dma_alias_mask);
2492 dev_dbg(dev, "device released\n");
2493 kfree(pci_dev);
2494}
2495
2496static const struct device_type pci_dev_type = {
2497 .groups = pci_dev_attr_groups,
2498};
2499
2500struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2501{
2502 struct pci_dev *dev;
2503
2504 dev = kzalloc_obj(struct pci_dev);
2505 if (!dev)
2506 return NULL;
2507
2508 INIT_LIST_HEAD(&dev->bus_list);
2509 dev->dev.type = &pci_dev_type;
2510 dev->bus = pci_bus_get(bus);
2511 dev->driver_exclusive_resource = (struct resource) {
2512 .name = "PCI Exclusive",
2513 .start = 0,
2514 .end = -1,
2515 };
2516
2517 spin_lock_init(&dev->pcie_cap_lock);
2518#ifdef CONFIG_PCI_MSI
2519 raw_spin_lock_init(&dev->msi_lock);
2520#endif
2521 return dev;
2522}
2523EXPORT_SYMBOL(pci_alloc_dev);
2524
2525static bool pci_bus_wait_rrs(struct pci_bus *bus, int devfn, u32 *l,
2526 int timeout)
2527{
2528 int delay = 1;
2529
2530 if (!pci_bus_rrs_vendor_id(*l))
2531 return true; /* not a Configuration RRS completion */
2532
2533 if (!timeout)
2534 return false; /* RRS, but caller doesn't want to wait */
2535
2536 /*
2537 * We got the reserved Vendor ID that indicates a completion with
2538 * Configuration Request Retry Status (RRS). Retry until we get a
2539 * valid Vendor ID or we time out.
2540 */
2541 while (pci_bus_rrs_vendor_id(*l)) {
2542 if (delay > timeout) {
2543 pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2544 pci_domain_nr(bus), bus->number,
2545 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2546
2547 return false;
2548 }
2549 if (delay >= 1000)
2550 pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2551 pci_domain_nr(bus), bus->number,
2552 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2553
2554 msleep(delay);
2555 delay *= 2;
2556
2557 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2558 return false;
2559 }
2560
2561 if (delay >= 1000)
2562 pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2563 pci_domain_nr(bus), bus->number,
2564 PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2565
2566 return true;
2567}
2568
2569bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2570 int timeout)
2571{
2572 if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, l))
2573 return false;
2574
2575 /* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */
2576 if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 ||
2577 *l == 0x0000ffff || *l == 0xffff0000)
2578 return false;
2579
2580 if (pci_bus_rrs_vendor_id(*l))
2581 return pci_bus_wait_rrs(bus, devfn, l, timeout);
2582
2583 return true;
2584}
2585
2586bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2587 int timeout)
2588{
2589 return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2590}
2591EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2592
2593/*
2594 * Read the config data for a PCI device, sanity-check it,
2595 * and fill in the dev structure.
2596 */
2597static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2598{
2599 struct pci_dev *dev;
2600 u32 l;
2601
2602 if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2603 return NULL;
2604
2605 dev = pci_alloc_dev(bus);
2606 if (!dev)
2607 return NULL;
2608
2609 dev->devfn = devfn;
2610 dev->vendor = l & 0xffff;
2611 dev->device = (l >> 16) & 0xffff;
2612
2613 if (pci_setup_device(dev)) {
2614 pci_bus_put(dev->bus);
2615 kfree(dev);
2616 return NULL;
2617 }
2618
2619 return dev;
2620}
2621
2622void pcie_report_downtraining(struct pci_dev *dev)
2623{
2624 if (!pci_is_pcie(dev))
2625 return;
2626
2627 /* Look from the device up to avoid downstream ports with no devices */
2628 if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2629 (pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2630 (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2631 return;
2632
2633 /* Multi-function PCIe devices share the same link/status */
2634 if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2635 return;
2636
2637 /* Print link status only if the device is constrained by the fabric */
2638 __pcie_print_link_status(dev, false);
2639}
2640
2641static void pci_imm_ready_init(struct pci_dev *dev)
2642{
2643 u16 status;
2644
2645 pci_read_config_word(dev, PCI_STATUS, &status);
2646 if (status & PCI_STATUS_IMM_READY)
2647 dev->imm_ready = 1;
2648}
2649
2650static void pci_init_capabilities(struct pci_dev *dev)
2651{
2652 pci_ea_init(dev); /* Enhanced Allocation */
2653 pci_msi_init(dev); /* Disable MSI */
2654 pci_msix_init(dev); /* Disable MSI-X */
2655
2656 /* Buffers for saving PCIe and PCI-X capabilities */
2657 pci_allocate_cap_save_buffers(dev);
2658
2659 pci_imm_ready_init(dev); /* Immediate Readiness */
2660 pci_pm_init(dev); /* Power Management */
2661 pci_vpd_init(dev); /* Vital Product Data */
2662 pci_configure_ari(dev); /* Alternative Routing-ID Forwarding */
2663 pci_iov_init(dev); /* Single Root I/O Virtualization */
2664 pci_ats_init(dev); /* Address Translation Services */
2665 pci_pri_init(dev); /* Page Request Interface */
2666 pci_pasid_init(dev); /* Process Address Space ID */
2667 pci_acs_init(dev); /* Access Control Services */
2668 pci_ptm_init(dev); /* Precision Time Measurement */
2669 pci_aer_init(dev); /* Advanced Error Reporting */
2670 pci_dpc_init(dev); /* Downstream Port Containment */
2671 pci_rcec_init(dev); /* Root Complex Event Collector */
2672 pci_doe_init(dev); /* Data Object Exchange */
2673 pci_tph_init(dev); /* TLP Processing Hints */
2674 pci_rebar_init(dev); /* Resizable BAR */
2675 pci_dev3_init(dev); /* Device 3 capabilities */
2676 pci_ide_init(dev); /* Link Integrity and Data Encryption */
2677
2678 pcie_report_downtraining(dev);
2679 pci_init_reset_methods(dev);
2680}
2681
2682/*
2683 * This is the equivalent of pci_host_bridge_msi_domain() that acts on
2684 * devices. Firmware interfaces that can select the MSI domain on a
2685 * per-device basis should be called from here.
2686 */
2687static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2688{
2689 struct irq_domain *d;
2690
2691 /*
2692 * If a domain has been set through the pcibios_device_add()
2693 * callback, then this is the one (platform code knows best).
2694 */
2695 d = dev_get_msi_domain(&dev->dev);
2696 if (d)
2697 return d;
2698
2699 /*
2700 * Let's see if we have a firmware interface able to provide
2701 * the domain.
2702 */
2703 d = pci_msi_get_device_domain(dev);
2704 if (d)
2705 return d;
2706
2707 return NULL;
2708}
2709
2710static void pci_set_msi_domain(struct pci_dev *dev)
2711{
2712 struct irq_domain *d;
2713
2714 /*
2715 * If the platform or firmware interfaces cannot supply a
2716 * device-specific MSI domain, then inherit the default domain
2717 * from the host bridge itself.
2718 */
2719 d = pci_dev_msi_domain(dev);
2720 if (!d)
2721 d = dev_get_msi_domain(&dev->bus->dev);
2722
2723 dev_set_msi_domain(&dev->dev, d);
2724}
2725
2726void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2727{
2728 int ret;
2729
2730 pci_configure_device(dev);
2731
2732 device_initialize(&dev->dev);
2733 dev->dev.release = pci_release_dev;
2734
2735 set_dev_node(&dev->dev, pcibus_to_node(bus));
2736 dev->dev.dma_mask = &dev->dma_mask;
2737 dev->dev.dma_parms = &dev->dma_parms;
2738 dev->dev.coherent_dma_mask = 0xffffffffull;
2739
2740 dma_set_max_seg_size(&dev->dev, 65536);
2741 dma_set_seg_boundary(&dev->dev, 0xffffffff);
2742
2743 pcie_failed_link_retrain(dev);
2744
2745 /* Fix up broken headers */
2746 pci_fixup_device(pci_fixup_header, dev);
2747
2748 pci_reassigndev_resource_alignment(dev);
2749
2750 pci_init_capabilities(dev);
2751
2752 platform_pci_configure_wake(dev);
2753
2754 /*
2755 * Add the device to our list of discovered devices
2756 * and the bus list for fixup functions, etc.
2757 */
2758 down_write(&pci_bus_sem);
2759 list_add_tail(&dev->bus_list, &bus->devices);
2760 up_write(&pci_bus_sem);
2761
2762 ret = pcibios_device_add(dev);
2763 WARN_ON(ret < 0);
2764
2765 /* Set up MSI IRQ domain */
2766 pci_set_msi_domain(dev);
2767
2768 /* Notifier could use PCI capabilities */
2769 ret = device_add(&dev->dev);
2770 WARN_ON(ret < 0);
2771
2772 /* Establish pdev->tsm for newly added (e.g. new SR-IOV VFs) */
2773 pci_tsm_init(dev);
2774
2775 pci_npem_create(dev);
2776
2777 pci_doe_sysfs_init(dev);
2778}
2779
2780struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2781{
2782 struct pci_dev *dev;
2783
2784 dev = pci_get_slot(bus, devfn);
2785 if (dev) {
2786 pci_dev_put(dev);
2787 return dev;
2788 }
2789
2790 dev = pci_scan_device(bus, devfn);
2791 if (!dev)
2792 return NULL;
2793
2794 pci_device_add(dev, bus);
2795
2796 return dev;
2797}
2798EXPORT_SYMBOL(pci_scan_single_device);
2799
2800static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2801{
2802 int pos;
2803 u16 cap = 0;
2804 unsigned int next_fn;
2805
2806 if (!dev)
2807 return -ENODEV;
2808
2809 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2810 if (!pos)
2811 return -ENODEV;
2812
2813 pci_read_config_word(dev, pos + PCI_ARI_CAP, &cap);
2814 next_fn = PCI_ARI_CAP_NFN(cap);
2815 if (next_fn <= fn)
2816 return -ENODEV; /* protect against malformed list */
2817
2818 return next_fn;
2819}
2820
2821static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2822{
2823 if (pci_ari_enabled(bus))
2824 return next_ari_fn(bus, dev, fn);
2825
2826 if (fn >= 7)
2827 return -ENODEV;
2828 /* only multifunction devices may have more functions */
2829 if (dev && !dev->multifunction)
2830 return -ENODEV;
2831
2832 return fn + 1;
2833}
2834
2835static int only_one_child(struct pci_bus *bus)
2836{
2837 struct pci_dev *bridge = bus->self;
2838
2839 /*
2840 * Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2841 * we scan for all possible devices, not just Device 0.
2842 */
2843 if (pci_has_flag(PCI_SCAN_ALL_PCIE_DEVS))
2844 return 0;
2845
2846 /*
2847 * A PCIe Downstream Port normally leads to a Link with only Device
2848 * 0 on it (PCIe spec r3.1, sec 7.3.1). As an optimization, scan
2849 * only for Device 0 in that situation.
2850 */
2851 if (bridge && pci_is_pcie(bridge) && pcie_downstream_port(bridge))
2852 return 1;
2853
2854 return 0;
2855}
2856
2857/**
2858 * pci_scan_slot - Scan a PCI slot on a bus for devices
2859 * @bus: PCI bus to scan
2860 * @devfn: slot number to scan (must have zero function)
2861 *
2862 * Scan a PCI slot on the specified PCI bus for devices, adding
2863 * discovered devices to the @bus->devices list. New devices
2864 * will not have is_added set.
2865 *
2866 * Returns the number of new devices found.
2867 */
2868int pci_scan_slot(struct pci_bus *bus, int devfn)
2869{
2870 struct pci_dev *dev;
2871 int fn = 0, nr = 0;
2872
2873 if (only_one_child(bus) && (devfn > 0))
2874 return 0; /* Already scanned the entire slot */
2875
2876 do {
2877 dev = pci_scan_single_device(bus, devfn + fn);
2878 if (dev) {
2879 if (!pci_dev_is_added(dev))
2880 nr++;
2881 if (fn > 0)
2882 dev->multifunction = 1;
2883 } else if (fn == 0) {
2884 /*
2885 * Function 0 is required unless we are running on
2886 * a hypervisor that passes through individual PCI
2887 * functions.
2888 */
2889 if (!hypervisor_isolated_pci_functions())
2890 break;
2891 }
2892 fn = next_fn(bus, dev, fn);
2893 } while (fn >= 0);
2894
2895 /* Only one slot has PCIe device */
2896 if (bus->self && nr)
2897 pcie_aspm_init_link_state(bus->self);
2898
2899 return nr;
2900}
2901EXPORT_SYMBOL(pci_scan_slot);
2902
2903static int pcie_find_smpss(struct pci_dev *dev, void *data)
2904{
2905 u8 *smpss = data;
2906
2907 if (!pci_is_pcie(dev))
2908 return 0;
2909
2910 /*
2911 * We don't have a way to change MPS settings on devices that have
2912 * drivers attached. A hot-added device might support only the minimum
2913 * MPS setting (MPS=128). Therefore, if the fabric contains a bridge
2914 * where devices may be hot-added, we limit the fabric MPS to 128 so
2915 * hot-added devices will work correctly.
2916 *
2917 * However, if we hot-add a device to a slot directly below a Root
2918 * Port, it's impossible for there to be other existing devices below
2919 * the port. We don't limit the MPS in this case because we can
2920 * reconfigure MPS on both the Root Port and the hot-added device,
2921 * and there are no other devices involved.
2922 *
2923 * Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2924 */
2925 if (dev->is_hotplug_bridge &&
2926 pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2927 *smpss = 0;
2928
2929 if (*smpss > dev->pcie_mpss)
2930 *smpss = dev->pcie_mpss;
2931
2932 return 0;
2933}
2934
2935static void pcie_write_mps(struct pci_dev *dev, int mps)
2936{
2937 int rc;
2938
2939 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2940 mps = 128 << dev->pcie_mpss;
2941
2942 if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2943 dev->bus->self)
2944
2945 /*
2946 * For "Performance", the assumption is made that
2947 * downstream communication will never be larger than
2948 * the MRRS. So, the MPS only needs to be configured
2949 * for the upstream communication. This being the case,
2950 * walk from the top down and set the MPS of the child
2951 * to that of the parent bus.
2952 *
2953 * Configure the device MPS with the smaller of the
2954 * device MPSS or the bridge MPS (which is assumed to be
2955 * properly configured at this point to the largest
2956 * allowable MPS based on its parent bus).
2957 */
2958 mps = min(mps, pcie_get_mps(dev->bus->self));
2959 }
2960
2961 rc = pcie_set_mps(dev, mps);
2962 if (rc)
2963 pci_err(dev, "Failed attempting to set the MPS\n");
2964}
2965
2966static void pcie_write_mrrs(struct pci_dev *dev)
2967{
2968 int rc, mrrs;
2969
2970 /*
2971 * In the "safe" case, do not configure the MRRS. There appear to be
2972 * issues with setting MRRS to 0 on a number of devices.
2973 */
2974 if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2975 return;
2976
2977 /*
2978 * For max performance, the MRRS must be set to the largest supported
2979 * value. However, it cannot be configured larger than the MPS the
2980 * device or the bus can support. This should already be properly
2981 * configured by a prior call to pcie_write_mps().
2982 */
2983 mrrs = pcie_get_mps(dev);
2984
2985 /*
2986 * MRRS is a R/W register. Invalid values can be written, but a
2987 * subsequent read will verify if the value is acceptable or not.
2988 * If the MRRS value provided is not acceptable (e.g., too large),
2989 * shrink the value until it is acceptable to the HW.
2990 */
2991 while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2992 rc = pcie_set_readrq(dev, mrrs);
2993 if (!rc)
2994 break;
2995
2996 pci_warn(dev, "Failed attempting to set the MRRS\n");
2997 mrrs /= 2;
2998 }
2999
3000 if (mrrs < 128)
3001 pci_err(dev, "MRRS was unable to be configured with a safe value. If problems are experienced, try running with pci=pcie_bus_safe\n");
3002}
3003
3004static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
3005{
3006 int mps, orig_mps;
3007
3008 if (!pci_is_pcie(dev))
3009 return 0;
3010
3011 if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
3012 pcie_bus_config == PCIE_BUS_DEFAULT)
3013 return 0;
3014
3015 mps = 128 << *(u8 *)data;
3016 orig_mps = pcie_get_mps(dev);
3017
3018 pcie_write_mps(dev, mps);
3019 pcie_write_mrrs(dev);
3020
3021 pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
3022 pcie_get_mps(dev), 128 << dev->pcie_mpss,
3023 orig_mps, pcie_get_readrq(dev));
3024
3025 return 0;
3026}
3027
3028/*
3029 * pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
3030 * parents then children fashion. If this changes, then this code will not
3031 * work as designed.
3032 */
3033void pcie_bus_configure_settings(struct pci_bus *bus)
3034{
3035 u8 smpss = 0;
3036
3037 if (!bus->self)
3038 return;
3039
3040 if (!pci_is_pcie(bus->self))
3041 return;
3042
3043 /*
3044 * FIXME - Peer to peer DMA is possible, though the endpoint would need
3045 * to be aware of the MPS of the destination. To work around this,
3046 * simply force the MPS of the entire system to the smallest possible.
3047 */
3048 if (pcie_bus_config == PCIE_BUS_PEER2PEER)
3049 smpss = 0;
3050
3051 if (pcie_bus_config == PCIE_BUS_SAFE) {
3052 smpss = bus->self->pcie_mpss;
3053
3054 pcie_find_smpss(bus->self, &smpss);
3055 pci_walk_bus(bus, pcie_find_smpss, &smpss);
3056 }
3057
3058 pcie_bus_configure_set(bus->self, &smpss);
3059 pci_walk_bus(bus, pcie_bus_configure_set, &smpss);
3060}
3061EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
3062
3063/*
3064 * Called after each bus is probed, but before its children are examined. This
3065 * is marked as __weak because multiple architectures define it.
3066 */
3067void __weak pcibios_fixup_bus(struct pci_bus *bus)
3068{
3069 /* nothing to do, expected to be removed in the future */
3070}
3071
3072/**
3073 * pci_scan_child_bus_extend() - Scan devices below a bus
3074 * @bus: Bus to scan for devices
3075 * @available_buses: Total number of buses available (%0 does not try to
3076 * extend beyond the minimal)
3077 *
3078 * Scans devices below @bus including subordinate buses. Returns new
3079 * subordinate number including all the found devices. Passing
3080 * @available_buses causes the remaining bus space to be distributed
3081 * equally between hotplug-capable bridges to allow future extension of the
3082 * hierarchy.
3083 */
3084static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
3085 unsigned int available_buses)
3086{
3087 unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
3088 unsigned int start = bus->busn_res.start;
3089 unsigned int devnr, cmax, max = start;
3090 struct pci_dev *dev;
3091
3092 dev_dbg(&bus->dev, "scanning bus\n");
3093
3094 /* Go find them, Rover! */
3095 for (devnr = 0; devnr < PCI_MAX_NR_DEVS; devnr++)
3096 pci_scan_slot(bus, PCI_DEVFN(devnr, 0));
3097
3098 /* Reserve buses for SR-IOV capability */
3099 used_buses = pci_iov_bus_range(bus);
3100 max += used_buses;
3101
3102 /*
3103 * After performing arch-dependent fixup of the bus, look behind
3104 * all PCI-to-PCI bridges on this bus.
3105 */
3106 if (!bus->is_added) {
3107 dev_dbg(&bus->dev, "fixups for bus\n");
3108 pcibios_fixup_bus(bus);
3109 bus->is_added = 1;
3110 }
3111
3112 /*
3113 * Calculate how many hotplug bridges and normal bridges there
3114 * are on this bus. We will distribute the additional available
3115 * buses between hotplug bridges.
3116 */
3117 for_each_pci_bridge(dev, bus) {
3118 if (dev->is_hotplug_bridge)
3119 hotplug_bridges++;
3120 else
3121 normal_bridges++;
3122 }
3123
3124 /*
3125 * Scan bridges that are already configured. We don't touch them
3126 * unless they are misconfigured (which will be done in the second
3127 * scan below).
3128 */
3129 for_each_pci_bridge(dev, bus) {
3130 cmax = max;
3131 max = pci_scan_bridge_extend(bus, dev, max, 0, 0);
3132
3133 /*
3134 * Reserve one bus for each bridge now to avoid extending
3135 * hotplug bridges too much during the second scan below.
3136 */
3137 used_buses++;
3138 if (max - cmax > 1)
3139 used_buses += max - cmax - 1;
3140 }
3141
3142 /* Scan bridges that need to be reconfigured */
3143 for_each_pci_bridge(dev, bus) {
3144 unsigned int buses = 0;
3145
3146 if (!hotplug_bridges && normal_bridges == 1) {
3147 /*
3148 * There is only one bridge on the bus (upstream
3149 * port) so it gets all available buses which it
3150 * can then distribute to the possible hotplug
3151 * bridges below.
3152 */
3153 buses = available_buses;
3154 } else if (dev->is_hotplug_bridge) {
3155 /*
3156 * Distribute the extra buses between hotplug
3157 * bridges if any.
3158 */
3159 buses = available_buses / hotplug_bridges;
3160 buses = min(buses, available_buses - used_buses + 1);
3161 }
3162
3163 cmax = max;
3164 max = pci_scan_bridge_extend(bus, dev, cmax, buses, 1);
3165 /* One bus is already accounted so don't add it again */
3166 if (max - cmax > 1)
3167 used_buses += max - cmax - 1;
3168 }
3169
3170 /*
3171 * Make sure a hotplug bridge has at least the minimum requested
3172 * number of buses but allow it to grow up to the maximum available
3173 * bus number if there is room.
3174 */
3175 if (bus->self && bus->self->is_hotplug_bridge) {
3176 used_buses = max(available_buses, pci_hotplug_bus_size - 1);
3177 if (max - start < used_buses) {
3178 max = start + used_buses;
3179
3180 /* Do not allocate more buses than we have room left */
3181 if (max > bus->busn_res.end)
3182 max = bus->busn_res.end;
3183
3184 dev_dbg(&bus->dev, "%pR extended by %#02x\n",
3185 &bus->busn_res, max - start);
3186 }
3187 }
3188
3189 /*
3190 * We've scanned the bus and so we know all about what's on
3191 * the other side of any bridges that may be on this bus plus
3192 * any devices.
3193 *
3194 * Return how far we've got finding sub-buses.
3195 */
3196 dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
3197 return max;
3198}
3199
3200/**
3201 * pci_scan_child_bus() - Scan devices below a bus
3202 * @bus: Bus to scan for devices
3203 *
3204 * Scans devices below @bus including subordinate buses. Returns new
3205 * subordinate number including all the found devices.
3206 */
3207unsigned int pci_scan_child_bus(struct pci_bus *bus)
3208{
3209 return pci_scan_child_bus_extend(bus, 0);
3210}
3211EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3212
3213/**
3214 * pcibios_root_bridge_prepare - Platform-specific host bridge setup
3215 * @bridge: Host bridge to set up
3216 *
3217 * Default empty implementation. Replace with an architecture-specific setup
3218 * routine, if necessary.
3219 */
3220int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3221{
3222 return 0;
3223}
3224
3225void __weak pcibios_add_bus(struct pci_bus *bus)
3226{
3227}
3228
3229void __weak pcibios_remove_bus(struct pci_bus *bus)
3230{
3231}
3232
3233struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
3234 struct pci_ops *ops, void *sysdata, struct list_head *resources)
3235{
3236 int error;
3237 struct pci_host_bridge *bridge;
3238
3239 bridge = pci_alloc_host_bridge(0);
3240 if (!bridge)
3241 return NULL;
3242
3243 bridge->dev.parent = parent;
3244
3245 list_splice_init(resources, &bridge->windows);
3246 bridge->sysdata = sysdata;
3247 bridge->busnr = bus;
3248 bridge->ops = ops;
3249
3250 error = pci_register_host_bridge(bridge);
3251 if (error < 0)
3252 goto err_out;
3253
3254 return bridge->bus;
3255
3256err_out:
3257 put_device(&bridge->dev);
3258 return NULL;
3259}
3260EXPORT_SYMBOL_GPL(pci_create_root_bus);
3261
3262int pci_host_probe(struct pci_host_bridge *bridge)
3263{
3264 struct pci_bus *bus, *child;
3265 int ret;
3266
3267 pci_lock_rescan_remove();
3268 ret = pci_scan_root_bus_bridge(bridge);
3269 pci_unlock_rescan_remove();
3270 if (ret < 0) {
3271 dev_err(bridge->dev.parent, "Scanning root bridge failed");
3272 return ret;
3273 }
3274
3275 bus = bridge->bus;
3276
3277 /* If we must preserve the resource configuration, claim now */
3278 if (bridge->preserve_config)
3279 pci_bus_claim_resources(bus);
3280
3281 /*
3282 * Assign whatever was left unassigned. If we didn't claim above,
3283 * this will reassign everything.
3284 */
3285 pci_assign_unassigned_root_bus_resources(bus);
3286
3287 list_for_each_entry(child, &bus->children, node)
3288 pcie_bus_configure_settings(child);
3289
3290 pci_lock_rescan_remove();
3291 pci_bus_add_devices(bus);
3292 pci_unlock_rescan_remove();
3293
3294 /*
3295 * Ensure pm_runtime_enable() is called for the controller drivers
3296 * before calling pci_host_probe(). The PM framework expects that
3297 * if the parent device supports runtime PM, it will be enabled
3298 * before child runtime PM is enabled.
3299 */
3300 pm_runtime_set_active(&bridge->dev);
3301 pm_runtime_no_callbacks(&bridge->dev);
3302 devm_pm_runtime_enable(&bridge->dev);
3303
3304 return 0;
3305}
3306EXPORT_SYMBOL_GPL(pci_host_probe);
3307
3308int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3309{
3310 struct resource *res = &b->busn_res;
3311 struct resource *parent_res, *conflict;
3312
3313 res->start = bus;
3314 res->end = bus_max;
3315 res->flags = IORESOURCE_BUS;
3316
3317 if (!pci_is_root_bus(b))
3318 parent_res = &b->parent->busn_res;
3319 else {
3320 parent_res = get_pci_domain_busn_res(pci_domain_nr(b));
3321 res->flags |= IORESOURCE_PCI_FIXED;
3322 }
3323
3324 conflict = request_resource_conflict(parent_res, res);
3325
3326 if (conflict)
3327 dev_info(&b->dev,
3328 "busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3329 res, pci_is_root_bus(b) ? "domain " : "",
3330 parent_res, conflict->name, conflict);
3331
3332 return conflict == NULL;
3333}
3334
3335int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3336{
3337 struct resource *res = &b->busn_res;
3338 struct resource old_res = *res;
3339 resource_size_t size;
3340 int ret;
3341
3342 if (res->start > bus_max)
3343 return -EINVAL;
3344
3345 size = bus_max - res->start + 1;
3346 ret = adjust_resource(res, res->start, size);
3347 dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3348 &old_res, ret ? "can not be" : "is", bus_max);
3349
3350 if (!ret && !res->parent)
3351 pci_bus_insert_busn_res(b, res->start, res->end);
3352
3353 return ret;
3354}
3355
3356void pci_bus_release_busn_res(struct pci_bus *b)
3357{
3358 struct resource *res = &b->busn_res;
3359 int ret;
3360
3361 if (!res->flags || !res->parent)
3362 return;
3363
3364 ret = release_resource(res);
3365 dev_info(&b->dev, "busn_res: %pR %s released\n",
3366 res, ret ? "can not be" : "is");
3367}
3368
3369int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3370{
3371 struct resource_entry *window;
3372 bool found = false;
3373 struct pci_bus *b;
3374 int max, bus, ret;
3375
3376 if (!bridge)
3377 return -EINVAL;
3378
3379 resource_list_for_each_entry(window, &bridge->windows)
3380 if (window->res->flags & IORESOURCE_BUS) {
3381 bridge->busnr = window->res->start;
3382 found = true;
3383 break;
3384 }
3385
3386 ret = pci_register_host_bridge(bridge);
3387 if (ret < 0)
3388 return ret;
3389
3390 b = bridge->bus;
3391 bus = bridge->busnr;
3392
3393 if (!found) {
3394 dev_info(&b->dev,
3395 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3396 bus);
3397 pci_bus_insert_busn_res(b, bus, 255);
3398 }
3399
3400 max = pci_scan_child_bus(b);
3401
3402 if (!found)
3403 pci_bus_update_busn_res_end(b, max);
3404
3405 return 0;
3406}
3407EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3408
3409struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3410 struct pci_ops *ops, void *sysdata, struct list_head *resources)
3411{
3412 struct resource_entry *window;
3413 bool found = false;
3414 struct pci_bus *b;
3415 int max;
3416
3417 resource_list_for_each_entry(window, resources)
3418 if (window->res->flags & IORESOURCE_BUS) {
3419 found = true;
3420 break;
3421 }
3422
3423 b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3424 if (!b)
3425 return NULL;
3426
3427 if (!found) {
3428 dev_info(&b->dev,
3429 "No busn resource found for root bus, will use [bus %02x-ff]\n",
3430 bus);
3431 pci_bus_insert_busn_res(b, bus, 255);
3432 }
3433
3434 max = pci_scan_child_bus(b);
3435
3436 if (!found)
3437 pci_bus_update_busn_res_end(b, max);
3438
3439 return b;
3440}
3441EXPORT_SYMBOL(pci_scan_root_bus);
3442
3443struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3444 void *sysdata)
3445{
3446 LIST_HEAD(resources);
3447 struct pci_bus *b;
3448
3449 pci_add_resource(&resources, &ioport_resource);
3450 pci_add_resource(&resources, &iomem_resource);
3451 pci_add_resource(&resources, &busn_resource);
3452 b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3453 if (b) {
3454 pci_scan_child_bus(b);
3455 } else {
3456 pci_free_resource_list(&resources);
3457 }
3458 return b;
3459}
3460EXPORT_SYMBOL(pci_scan_bus);
3461
3462/**
3463 * pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3464 * @bridge: PCI bridge for the bus to scan
3465 *
3466 * Scan a PCI bus and child buses for new devices, add them,
3467 * and enable them, resizing bridge mmio/io resource if necessary
3468 * and possible. The caller must ensure the child devices are already
3469 * removed for resizing to occur.
3470 *
3471 * Returns the max number of subordinate bus discovered.
3472 */
3473unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3474{
3475 unsigned int max;
3476 struct pci_bus *bus = bridge->subordinate;
3477
3478 max = pci_scan_child_bus(bus);
3479
3480 pci_assign_unassigned_bridge_resources(bridge);
3481
3482 pci_bus_add_devices(bus);
3483
3484 return max;
3485}
3486
3487/**
3488 * pci_rescan_bus - Scan a PCI bus for devices
3489 * @bus: PCI bus to scan
3490 *
3491 * Scan a PCI bus and child buses for new devices, add them,
3492 * and enable them.
3493 *
3494 * Returns the max number of subordinate bus discovered.
3495 */
3496unsigned int pci_rescan_bus(struct pci_bus *bus)
3497{
3498 unsigned int max;
3499
3500 max = pci_scan_child_bus(bus);
3501 pci_assign_unassigned_bus_resources(bus);
3502 pci_bus_add_devices(bus);
3503
3504 return max;
3505}
3506EXPORT_SYMBOL_GPL(pci_rescan_bus);
3507
3508/*
3509 * pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3510 * routines should always be executed under this mutex.
3511 */
3512DEFINE_MUTEX(pci_rescan_remove_lock);
3513
3514void pci_lock_rescan_remove(void)
3515{
3516 mutex_lock(&pci_rescan_remove_lock);
3517}
3518EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3519
3520void pci_unlock_rescan_remove(void)
3521{
3522 mutex_unlock(&pci_rescan_remove_lock);
3523}
3524EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3525
3526static int __init pci_sort_bf_cmp(const struct device *d_a,
3527 const struct device *d_b)
3528{
3529 const struct pci_dev *a = to_pci_dev(d_a);
3530 const struct pci_dev *b = to_pci_dev(d_b);
3531
3532 if (pci_domain_nr(a->bus) < pci_domain_nr(b->bus)) return -1;
3533 else if (pci_domain_nr(a->bus) > pci_domain_nr(b->bus)) return 1;
3534
3535 if (a->bus->number < b->bus->number) return -1;
3536 else if (a->bus->number > b->bus->number) return 1;
3537
3538 if (a->devfn < b->devfn) return -1;
3539 else if (a->devfn > b->devfn) return 1;
3540
3541 return 0;
3542}
3543
3544void __init pci_sort_breadthfirst(void)
3545{
3546 bus_sort_breadthfirst(&pci_bus_type, &pci_sort_bf_cmp);
3547}
3548
3549int pci_hp_add_bridge(struct pci_dev *dev)
3550{
3551 struct pci_bus *parent = dev->bus;
3552 int busnr, start = parent->busn_res.start;
3553 unsigned int available_buses = 0;
3554 int end = parent->busn_res.end;
3555
3556 for (busnr = start; busnr <= end; busnr++) {
3557 if (!pci_find_bus(pci_domain_nr(parent), busnr))
3558 break;
3559 }
3560 if (busnr-- > end) {
3561 pci_err(dev, "No bus number available for hot-added bridge\n");
3562 return -1;
3563 }
3564
3565 /* Scan bridges that are already configured */
3566 busnr = pci_scan_bridge(parent, dev, busnr, 0);
3567
3568 /*
3569 * Distribute the available bus numbers between hotplug-capable
3570 * bridges to make extending the chain later possible.
3571 */
3572 available_buses = end - busnr;
3573
3574 /* Scan bridges that need to be reconfigured */
3575 pci_scan_bridge_extend(parent, dev, busnr, available_buses, 1);
3576
3577 if (!dev->subordinate)
3578 return -1;
3579
3580 return 0;
3581}
3582EXPORT_SYMBOL_GPL(pci_hp_add_bridge);