projects/linux·blob master · refs

2615 lines · 65.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <[email protected]>
5 */
6
7#define pr_fmt(fmt) "GICv3: " fmt
8
9#include <linux/acpi.h>
10#include <linux/cpu.h>
11#include <linux/cpu_pm.h>
12#include <linux/delay.h>
13#include <linux/interrupt.h>
14#include <linux/irqdomain.h>
15#include <linux/kernel.h>
16#include <linux/kstrtox.h>
17#include <linux/of.h>
18#include <linux/of_address.h>
19#include <linux/of_irq.h>
20#include <linux/percpu.h>
21#include <linux/refcount.h>
22#include <linux/slab.h>
23#include <linux/iopoll.h>
24
25#include <linux/irqchip.h>
26#include <linux/irqchip/arm-gic-common.h>
27#include <linux/irqchip/arm-gic-v3.h>
28#include <linux/irqchip/arm-gic-v3-prio.h>
29#include <linux/bitfield.h>
30#include <linux/bits.h>
31#include <linux/arm-smccc.h>
32
33#include <asm/cputype.h>
34#include <asm/exception.h>
35#include <asm/smp_plat.h>
36#include <asm/virt.h>
37
38#include "irq-gic-common.h"
39
40static u8 dist_prio_irq __ro_after_init = GICV3_PRIO_IRQ;
41static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI;
42
43#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
44#define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
45#define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 2)
46#define FLAGS_WORKAROUND_INSECURE (1ULL << 3)
47
48static struct cpumask broken_rdists __read_mostly __maybe_unused;
49
50struct redist_region {
51 void __iomem *redist_base;
52 phys_addr_t phys_base;
53 bool single_redist;
54};
55
56struct gic_chip_data {
57 struct fwnode_handle *fwnode;
58 phys_addr_t dist_phys_base;
59 void __iomem *dist_base;
60 struct redist_region *redist_regions;
61 struct rdists rdists;
62 struct irq_domain *domain;
63 u64 redist_stride;
64 u32 nr_redist_regions;
65 u64 flags;
66 bool has_rss;
67 unsigned int ppi_nr;
68 struct partition_affinity *parts;
69 unsigned int nr_parts;
70};
71
72struct partition_affinity {
73 cpumask_t mask;
74 struct fwnode_handle *partition_id;
75};
76
77#define T241_CHIPS_MAX 4
78static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly;
79static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum);
80
81static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum);
82
83static struct gic_chip_data gic_data __read_mostly;
84static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
85
86#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
87#define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
88#define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
89
90static bool nmi_support_forbidden;
91
92/*
93 * There are 16 SGIs, though we only actually use 8 in Linux. The other 8 SGIs
94 * are potentially stolen by the secure side. Some code, especially code dealing
95 * with hwirq IDs, is simplified by accounting for all 16.
96 */
97#define SGI_NR 16
98
99/*
100 * The behaviours of RPR and PMR registers differ depending on the value of
101 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
102 * distributor and redistributors depends on whether security is enabled in the
103 * GIC.
104 *
105 * When security is enabled, non-secure priority values from the (re)distributor
106 * are presented to the GIC CPUIF as follow:
107 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
108 *
109 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure
110 * EL1 are subject to a similar operation thus matching the priorities presented
111 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
112 * these values are unchanged by the GIC.
113 *
114 * see GICv3/GICv4 Architecture Specification (IHI0069D):
115 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
116 * priorities.
117 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
118 * interrupt.
119 */
120static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
121
122static u32 gic_get_pribits(void)
123{
124 u32 pribits;
125
126 pribits = gic_read_ctlr();
127 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
128 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
129 pribits++;
130
131 return pribits;
132}
133
134static bool gic_has_group0(void)
135{
136 u32 val;
137 u32 old_pmr;
138
139 old_pmr = gic_read_pmr();
140
141 /*
142 * Let's find out if Group0 is under control of EL3 or not by
143 * setting the highest possible, non-zero priority in PMR.
144 *
145 * If SCR_EL3.FIQ is set, the priority gets shifted down in
146 * order for the CPU interface to set bit 7, and keep the
147 * actual priority in the non-secure range. In the process, it
148 * looses the least significant bit and the actual priority
149 * becomes 0x80. Reading it back returns 0, indicating that
150 * we're don't have access to Group0.
151 */
152 gic_write_pmr(BIT(8 - gic_get_pribits()));
153 val = gic_read_pmr();
154
155 gic_write_pmr(old_pmr);
156
157 return val != 0;
158}
159
160static inline bool gic_dist_security_disabled(void)
161{
162 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
163}
164
165static bool cpus_have_security_disabled __ro_after_init;
166static bool cpus_have_group0 __ro_after_init;
167
168static void __init gic_prio_init(void)
169{
170 bool ds;
171
172 cpus_have_group0 = gic_has_group0();
173
174 ds = gic_dist_security_disabled();
175 if ((gic_data.flags & FLAGS_WORKAROUND_INSECURE) && !ds) {
176 if (cpus_have_group0) {
177 u32 val;
178
179 val = readl_relaxed(gic_data.dist_base + GICD_CTLR);
180 val |= GICD_CTLR_DS;
181 writel_relaxed(val, gic_data.dist_base + GICD_CTLR);
182
183 ds = gic_dist_security_disabled();
184 if (ds)
185 pr_warn("Broken GIC integration, security disabled\n");
186 } else {
187 pr_warn("Broken GIC integration, pNMI forbidden\n");
188 nmi_support_forbidden = true;
189 }
190 }
191
192 cpus_have_security_disabled = ds;
193
194 /*
195 * How priority values are used by the GIC depends on two things:
196 * the security state of the GIC (controlled by the GICD_CTLR.DS bit)
197 * and if Group 0 interrupts can be delivered to Linux in the non-secure
198 * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
199 * way priorities are presented in ICC_PMR_EL1 and in the distributor:
200 *
201 * GICD_CTLR.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Distributor
202 * -------------------------------------------------------
203 * 1 | - | unchanged | unchanged
204 * -------------------------------------------------------
205 * 0 | 1 | non-secure | non-secure
206 * -------------------------------------------------------
207 * 0 | 0 | unchanged | non-secure
208 *
209 * In the non-secure view reads and writes are modified:
210 *
211 * - A value written is right-shifted by one and the MSB is set,
212 * forcing the priority into the non-secure range.
213 *
214 * - A value read is left-shifted by one.
215 *
216 * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
217 * are both either modified or unchanged, we can use the same set of
218 * priorities.
219 *
220 * In the last case, where only the interrupt priorities are modified to
221 * be in the non-secure range, we program the non-secure values into
222 * the distributor to match the PMR values we want.
223 */
224 if (cpus_have_group0 && !cpus_have_security_disabled) {
225 dist_prio_irq = __gicv3_prio_to_ns(dist_prio_irq);
226 dist_prio_nmi = __gicv3_prio_to_ns(dist_prio_nmi);
227 }
228
229 pr_info("GICD_CTLR.DS=%d, SCR_EL3.FIQ=%d\n",
230 cpus_have_security_disabled,
231 !cpus_have_group0);
232}
233
234static struct gic_kvm_info gic_v3_kvm_info __initdata;
235static DEFINE_PER_CPU(bool, has_rss);
236
237#define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4)
238#define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist))
239#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
240#define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K)
241
242/* Our default, arbitrary priority value. Linux only uses one anyway. */
243#define DEFAULT_PMR_VALUE 0xf0
244
245enum gic_intid_range {
246 SGI_RANGE,
247 PPI_RANGE,
248 SPI_RANGE,
249 EPPI_RANGE,
250 ESPI_RANGE,
251 LPI_RANGE,
252 __INVALID_RANGE__
253};
254
255static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
256{
257 switch (hwirq) {
258 case 0 ... 15:
259 return SGI_RANGE;
260 case 16 ... 31:
261 return PPI_RANGE;
262 case 32 ... 1019:
263 return SPI_RANGE;
264 case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
265 return EPPI_RANGE;
266 case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
267 return ESPI_RANGE;
268 case 8192 ... GENMASK(23, 0):
269 return LPI_RANGE;
270 default:
271 return __INVALID_RANGE__;
272 }
273}
274
275static enum gic_intid_range get_intid_range(struct irq_data *d)
276{
277 return __get_intid_range(d->hwirq);
278}
279
280static inline bool gic_irq_in_rdist(struct irq_data *d)
281{
282 switch (get_intid_range(d)) {
283 case SGI_RANGE:
284 case PPI_RANGE:
285 case EPPI_RANGE:
286 return true;
287 default:
288 return false;
289 }
290}
291
292static inline void __iomem *gic_dist_base_alias(struct irq_data *d)
293{
294 if (static_branch_unlikely(&gic_nvidia_t241_erratum)) {
295 irq_hw_number_t hwirq = irqd_to_hwirq(d);
296 u32 chip;
297
298 /*
299 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E}
300 * registers are directed to the chip that owns the SPI. The
301 * the alias region can also be used for writes to the
302 * GICD_In{E} except GICD_ICENABLERn. Each chip has support
303 * for 320 {E}SPIs. Mappings for all 4 chips:
304 * Chip0 = 32-351
305 * Chip1 = 352-671
306 * Chip2 = 672-991
307 * Chip3 = 4096-4415
308 */
309 switch (__get_intid_range(hwirq)) {
310 case SPI_RANGE:
311 chip = (hwirq - 32) / 320;
312 break;
313 case ESPI_RANGE:
314 chip = 3;
315 break;
316 default:
317 unreachable();
318 }
319 return t241_dist_base_alias[chip];
320 }
321
322 return gic_data.dist_base;
323}
324
325static inline void __iomem *gic_dist_base(struct irq_data *d)
326{
327 switch (get_intid_range(d)) {
328 case SGI_RANGE:
329 case PPI_RANGE:
330 case EPPI_RANGE:
331 /* SGI+PPI -> SGI_base for this CPU */
332 return gic_data_rdist_sgi_base();
333
334 case SPI_RANGE:
335 case ESPI_RANGE:
336 /* SPI -> dist_base */
337 return gic_data.dist_base;
338
339 default:
340 return NULL;
341 }
342}
343
344static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
345{
346 u32 val;
347 int ret;
348
349 ret = readl_relaxed_poll_timeout_atomic(base + GICD_CTLR, val, !(val & bit),
350 1, USEC_PER_SEC);
351 if (ret == -ETIMEDOUT)
352 pr_err_ratelimited("RWP timeout, gone fishing\n");
353}
354
355/* Wait for completion of a distributor change */
356static void gic_dist_wait_for_rwp(void)
357{
358 gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
359}
360
361/* Wait for completion of a redistributor change */
362static void gic_redist_wait_for_rwp(void)
363{
364 gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
365}
366
367static void gic_enable_redist(bool enable)
368{
369 void __iomem *rbase;
370 u32 val;
371 int ret;
372
373 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
374 return;
375
376 rbase = gic_data_rdist_rd_base();
377
378 val = readl_relaxed(rbase + GICR_WAKER);
379 if (enable)
380 /* Wake up this CPU redistributor */
381 val &= ~GICR_WAKER_ProcessorSleep;
382 else
383 val |= GICR_WAKER_ProcessorSleep;
384 writel_relaxed(val, rbase + GICR_WAKER);
385
386 if (!enable) { /* Check that GICR_WAKER is writeable */
387 val = readl_relaxed(rbase + GICR_WAKER);
388 if (!(val & GICR_WAKER_ProcessorSleep))
389 return; /* No PM support in this redistributor */
390 }
391
392 ret = readl_relaxed_poll_timeout_atomic(rbase + GICR_WAKER, val,
393 enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep),
394 1, USEC_PER_SEC);
395 if (ret == -ETIMEDOUT) {
396 pr_err_ratelimited("redistributor failed to %s...\n",
397 enable ? "wakeup" : "sleep");
398 }
399}
400
401/*
402 * Routines to disable, enable, EOI and route interrupts
403 */
404static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
405{
406 switch (get_intid_range(d)) {
407 case SGI_RANGE:
408 case PPI_RANGE:
409 case SPI_RANGE:
410 *index = d->hwirq;
411 return offset;
412 case EPPI_RANGE:
413 /*
414 * Contrary to the ESPI range, the EPPI range is contiguous
415 * to the PPI range in the registers, so let's adjust the
416 * displacement accordingly. Consistency is overrated.
417 */
418 *index = d->hwirq - EPPI_BASE_INTID + 32;
419 return offset;
420 case ESPI_RANGE:
421 *index = d->hwirq - ESPI_BASE_INTID;
422 switch (offset) {
423 case GICD_ISENABLER:
424 return GICD_ISENABLERnE;
425 case GICD_ICENABLER:
426 return GICD_ICENABLERnE;
427 case GICD_ISPENDR:
428 return GICD_ISPENDRnE;
429 case GICD_ICPENDR:
430 return GICD_ICPENDRnE;
431 case GICD_ISACTIVER:
432 return GICD_ISACTIVERnE;
433 case GICD_ICACTIVER:
434 return GICD_ICACTIVERnE;
435 case GICD_IPRIORITYR:
436 return GICD_IPRIORITYRnE;
437 case GICD_ICFGR:
438 return GICD_ICFGRnE;
439 case GICD_IROUTER:
440 return GICD_IROUTERnE;
441 default:
442 break;
443 }
444 break;
445 default:
446 break;
447 }
448
449 WARN_ON(1);
450 *index = d->hwirq;
451 return offset;
452}
453
454static int gic_peek_irq(struct irq_data *d, u32 offset)
455{
456 void __iomem *base;
457 u32 index, mask;
458
459 offset = convert_offset_index(d, offset, &index);
460 mask = 1 << (index % 32);
461
462 if (gic_irq_in_rdist(d))
463 base = gic_data_rdist_sgi_base();
464 else
465 base = gic_dist_base_alias(d);
466
467 return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
468}
469
470static void gic_poke_irq(struct irq_data *d, u32 offset)
471{
472 void __iomem *base;
473 u32 index, mask;
474
475 offset = convert_offset_index(d, offset, &index);
476 mask = 1 << (index % 32);
477
478 if (gic_irq_in_rdist(d))
479 base = gic_data_rdist_sgi_base();
480 else
481 base = gic_data.dist_base;
482
483 writel_relaxed(mask, base + offset + (index / 32) * 4);
484}
485
486static void gic_mask_irq(struct irq_data *d)
487{
488 gic_poke_irq(d, GICD_ICENABLER);
489 if (gic_irq_in_rdist(d))
490 gic_redist_wait_for_rwp();
491 else
492 gic_dist_wait_for_rwp();
493}
494
495static void gic_eoimode1_mask_irq(struct irq_data *d)
496{
497 gic_mask_irq(d);
498 /*
499 * When masking a forwarded interrupt, make sure it is
500 * deactivated as well.
501 *
502 * This ensures that an interrupt that is getting
503 * disabled/masked will not get "stuck", because there is
504 * noone to deactivate it (guest is being terminated).
505 */
506 if (irqd_is_forwarded_to_vcpu(d))
507 gic_poke_irq(d, GICD_ICACTIVER);
508}
509
510static void gic_unmask_irq(struct irq_data *d)
511{
512 gic_poke_irq(d, GICD_ISENABLER);
513}
514
515static inline bool gic_supports_nmi(void)
516{
517 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
518 static_branch_likely(&supports_pseudo_nmis);
519}
520
521static int gic_irq_set_irqchip_state(struct irq_data *d,
522 enum irqchip_irq_state which, bool val)
523{
524 u32 reg;
525
526 if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
527 return -EINVAL;
528
529 switch (which) {
530 case IRQCHIP_STATE_PENDING:
531 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
532 break;
533
534 case IRQCHIP_STATE_ACTIVE:
535 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
536 break;
537
538 case IRQCHIP_STATE_MASKED:
539 if (val) {
540 gic_mask_irq(d);
541 return 0;
542 }
543 reg = GICD_ISENABLER;
544 break;
545
546 default:
547 return -EINVAL;
548 }
549
550 gic_poke_irq(d, reg);
551
552 /*
553 * Force read-back to guarantee that the active state has taken
554 * effect, and won't race with a guest-driven deactivation.
555 */
556 if (reg == GICD_ISACTIVER)
557 gic_peek_irq(d, reg);
558 return 0;
559}
560
561static int gic_irq_get_irqchip_state(struct irq_data *d,
562 enum irqchip_irq_state which, bool *val)
563{
564 if (d->hwirq >= 8192) /* PPI/SPI only */
565 return -EINVAL;
566
567 switch (which) {
568 case IRQCHIP_STATE_PENDING:
569 *val = gic_peek_irq(d, GICD_ISPENDR);
570 break;
571
572 case IRQCHIP_STATE_ACTIVE:
573 *val = gic_peek_irq(d, GICD_ISACTIVER);
574 break;
575
576 case IRQCHIP_STATE_MASKED:
577 *val = !gic_peek_irq(d, GICD_ISENABLER);
578 break;
579
580 default:
581 return -EINVAL;
582 }
583
584 return 0;
585}
586
587static void gic_irq_set_prio(struct irq_data *d, u8 prio)
588{
589 void __iomem *base = gic_dist_base(d);
590 u32 offset, index;
591
592 offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
593
594 writeb_relaxed(prio, base + offset + index);
595}
596
597static int gic_irq_nmi_setup(struct irq_data *d)
598{
599 struct irq_desc *desc = irq_to_desc(d->irq);
600
601 if (!gic_supports_nmi())
602 return -EINVAL;
603
604 if (gic_peek_irq(d, GICD_ISENABLER)) {
605 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
606 return -EINVAL;
607 }
608
609 /*
610 * A secondary irq_chip should be in charge of LPI request,
611 * it should not be possible to get there
612 */
613 if (WARN_ON(irqd_to_hwirq(d) >= 8192))
614 return -EINVAL;
615
616 /* desc lock should already be held */
617 if (!gic_irq_in_rdist(d))
618 desc->handle_irq = handle_fasteoi_nmi;
619
620 gic_irq_set_prio(d, dist_prio_nmi);
621
622 return 0;
623}
624
625static void gic_irq_nmi_teardown(struct irq_data *d)
626{
627 struct irq_desc *desc = irq_to_desc(d->irq);
628
629 if (WARN_ON(!gic_supports_nmi()))
630 return;
631
632 if (gic_peek_irq(d, GICD_ISENABLER)) {
633 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
634 return;
635 }
636
637 /*
638 * A secondary irq_chip should be in charge of LPI request,
639 * it should not be possible to get there
640 */
641 if (WARN_ON(irqd_to_hwirq(d) >= 8192))
642 return;
643
644 /* desc lock should already be held */
645 if (!gic_irq_in_rdist(d))
646 desc->handle_irq = handle_fasteoi_irq;
647
648 gic_irq_set_prio(d, dist_prio_irq);
649}
650
651static bool gic_arm64_erratum_2941627_needed(struct irq_data *d)
652{
653 enum gic_intid_range range;
654
655 if (!static_branch_unlikely(&gic_arm64_2941627_erratum))
656 return false;
657
658 range = get_intid_range(d);
659
660 /*
661 * The workaround is needed if the IRQ is an SPI and
662 * the target cpu is different from the one we are
663 * executing on.
664 */
665 return (range == SPI_RANGE || range == ESPI_RANGE) &&
666 !cpumask_test_cpu(raw_smp_processor_id(),
667 irq_data_get_effective_affinity_mask(d));
668}
669
670static void gic_eoi_irq(struct irq_data *d)
671{
672 write_gicreg(irqd_to_hwirq(d), ICC_EOIR1_EL1);
673 isb();
674
675 if (gic_arm64_erratum_2941627_needed(d)) {
676 /*
677 * Make sure the GIC stream deactivate packet
678 * issued by ICC_EOIR1_EL1 has completed before
679 * deactivating through GICD_IACTIVER.
680 */
681 dsb(sy);
682 gic_poke_irq(d, GICD_ICACTIVER);
683 }
684}
685
686static void gic_eoimode1_eoi_irq(struct irq_data *d)
687{
688 /*
689 * No need to deactivate an LPI, or an interrupt that
690 * is is getting forwarded to a vcpu.
691 */
692 if (irqd_to_hwirq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
693 return;
694
695 if (!gic_arm64_erratum_2941627_needed(d))
696 gic_write_dir(irqd_to_hwirq(d));
697 else
698 gic_poke_irq(d, GICD_ICACTIVER);
699}
700
701static int gic_set_type(struct irq_data *d, unsigned int type)
702{
703 irq_hw_number_t irq = irqd_to_hwirq(d);
704 enum gic_intid_range range;
705 void __iomem *base;
706 u32 offset, index;
707 int ret;
708
709 range = get_intid_range(d);
710
711 /* Interrupt configuration for SGIs can't be changed */
712 if (range == SGI_RANGE)
713 return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;
714
715 /* SPIs have restrictions on the supported types */
716 if ((range == SPI_RANGE || range == ESPI_RANGE) &&
717 type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
718 return -EINVAL;
719
720 if (gic_irq_in_rdist(d))
721 base = gic_data_rdist_sgi_base();
722 else
723 base = gic_dist_base_alias(d);
724
725 offset = convert_offset_index(d, GICD_ICFGR, &index);
726
727 ret = gic_configure_irq(index, type, base + offset);
728 if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
729 /* Misconfigured PPIs are usually not fatal */
730 pr_warn("GIC: PPI INTID%ld is secure or misconfigured\n", irq);
731 ret = 0;
732 }
733
734 return ret;
735}
736
737static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
738{
739 if (get_intid_range(d) == SGI_RANGE)
740 return -EINVAL;
741
742 if (vcpu)
743 irqd_set_forwarded_to_vcpu(d);
744 else
745 irqd_clr_forwarded_to_vcpu(d);
746 return 0;
747}
748
749static u64 gic_cpu_to_affinity(int cpu)
750{
751 u64 mpidr = cpu_logical_map(cpu);
752 u64 aff;
753
754 /* ASR8601 needs to have its affinities shifted down... */
755 if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001))
756 mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1) |
757 (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8));
758
759 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
760 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
761 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
762 MPIDR_AFFINITY_LEVEL(mpidr, 0));
763
764 return aff;
765}
766
767static void gic_deactivate_unhandled(u32 irqnr)
768{
769 if (static_branch_likely(&supports_deactivate_key)) {
770 if (irqnr < 8192)
771 gic_write_dir(irqnr);
772 } else {
773 write_gicreg(irqnr, ICC_EOIR1_EL1);
774 isb();
775 }
776}
777
778/*
779 * Follow a read of the IAR with any HW maintenance that needs to happen prior
780 * to invoking the relevant IRQ handler. We must do two things:
781 *
782 * (1) Ensure instruction ordering between a read of IAR and subsequent
783 * instructions in the IRQ handler using an ISB.
784 *
785 * It is possible for the IAR to report an IRQ which was signalled *after*
786 * the CPU took an IRQ exception as multiple interrupts can race to be
787 * recognized by the GIC, earlier interrupts could be withdrawn, and/or
788 * later interrupts could be prioritized by the GIC.
789 *
790 * For devices which are tightly coupled to the CPU, such as PMUs, a
791 * context synchronization event is necessary to ensure that system
792 * register state is not stale, as these may have been indirectly written
793 * *after* exception entry.
794 *
795 * (2) Execute an interrupt priority drop when EOI mode 1 is in use.
796 */
797static inline void gic_complete_ack(u32 irqnr)
798{
799 if (static_branch_likely(&supports_deactivate_key))
800 write_gicreg(irqnr, ICC_EOIR1_EL1);
801
802 isb();
803}
804
805static bool gic_rpr_is_nmi_prio(void)
806{
807 if (!gic_supports_nmi())
808 return false;
809
810 return unlikely(gic_read_rpr() == GICV3_PRIO_NMI);
811}
812
813static bool gic_irqnr_is_special(u32 irqnr)
814{
815 return irqnr >= 1020 && irqnr <= 1023;
816}
817
818static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs)
819{
820 if (gic_irqnr_is_special(irqnr))
821 return;
822
823 gic_complete_ack(irqnr);
824
825 if (generic_handle_domain_irq(gic_data.domain, irqnr)) {
826 WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr);
827 gic_deactivate_unhandled(irqnr);
828 }
829}
830
831static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
832{
833 if (gic_irqnr_is_special(irqnr))
834 return;
835
836 gic_complete_ack(irqnr);
837
838 if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {
839 WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);
840 gic_deactivate_unhandled(irqnr);
841 }
842}
843
844/*
845 * An exception has been taken from a context with IRQs enabled, and this could
846 * be an IRQ or an NMI.
847 *
848 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear
849 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning,
850 * after handling any NMI but before handling any IRQ.
851 *
852 * The entry code has performed IRQ entry, and if an NMI is detected we must
853 * perform NMI entry/exit around invoking the handler.
854 */
855static void __gic_handle_irq_from_irqson(struct pt_regs *regs)
856{
857 bool is_nmi;
858 u32 irqnr;
859
860 irqnr = gic_read_iar();
861
862 is_nmi = gic_rpr_is_nmi_prio();
863
864 if (is_nmi) {
865 nmi_enter();
866 __gic_handle_nmi(irqnr, regs);
867 nmi_exit();
868 }
869
870 gic_unmask_pnmis();
871
872 if (!is_nmi)
873 __gic_handle_irq(irqnr, regs);
874}
875
876/*
877 * An exception has been taken from a context with IRQs disabled, which can only
878 * be an NMI.
879 *
880 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave
881 * DAIF.IF (and ICC_PMR_EL1) unchanged.
882 *
883 * The entry code has performed NMI entry.
884 */
885static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)
886{
887 u64 pmr;
888 u32 irqnr;
889
890 /*
891 * We were in a context with IRQs disabled. However, the
892 * entry code has set PMR to a value that allows any
893 * interrupt to be acknowledged, and not just NMIs. This can
894 * lead to surprising effects if the NMI has been retired in
895 * the meantime, and that there is an IRQ pending. The IRQ
896 * would then be taken in NMI context, something that nobody
897 * wants to debug twice.
898 *
899 * Until we sort this, drop PMR again to a level that will
900 * actually only allow NMIs before reading IAR, and then
901 * restore it to what it was.
902 */
903 pmr = gic_read_pmr();
904 gic_pmr_mask_irqs();
905 isb();
906 irqnr = gic_read_iar();
907 gic_write_pmr(pmr);
908
909 __gic_handle_nmi(irqnr, regs);
910}
911
912static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
913{
914 if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))
915 __gic_handle_irq_from_irqsoff(regs);
916 else
917 __gic_handle_irq_from_irqson(regs);
918}
919
920static void __init gic_dist_init(void)
921{
922 unsigned int i;
923 u64 affinity;
924 void __iomem *base = gic_data.dist_base;
925 u32 val;
926
927 /* Disable the distributor */
928 writel_relaxed(0, base + GICD_CTLR);
929 gic_dist_wait_for_rwp();
930
931 /*
932 * Configure SPIs as non-secure Group-1. This will only matter
933 * if the GIC only has a single security state. This will not
934 * do the right thing if the kernel is running in secure mode,
935 * but that's not the intended use case anyway.
936 */
937 for (i = 32; i < GIC_LINE_NR; i += 32)
938 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
939
940 /* Extended SPI range, not handled by the GICv2/GICv3 common code */
941 for (i = 0; i < GIC_ESPI_NR; i += 32) {
942 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
943 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
944 }
945
946 for (i = 0; i < GIC_ESPI_NR; i += 32)
947 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
948
949 for (i = 0; i < GIC_ESPI_NR; i += 16)
950 writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
951
952 for (i = 0; i < GIC_ESPI_NR; i += 4)
953 writel_relaxed(REPEAT_BYTE_U32(dist_prio_irq),
954 base + GICD_IPRIORITYRnE + i);
955
956 /* Now do the common stuff */
957 gic_dist_config(base, GIC_LINE_NR, dist_prio_irq);
958
959 val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
960 if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
961 pr_info("Enabling SGIs without active state\n");
962 val |= GICD_CTLR_nASSGIreq;
963 }
964
965 /* Enable distributor with ARE, Group1, and wait for it to drain */
966 writel_relaxed(val, base + GICD_CTLR);
967 gic_dist_wait_for_rwp();
968
969 /*
970 * Set all global interrupts to the boot CPU only. ARE must be
971 * enabled.
972 */
973 affinity = gic_cpu_to_affinity(smp_processor_id());
974 for (i = 32; i < GIC_LINE_NR; i++)
975 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
976
977 for (i = 0; i < GIC_ESPI_NR; i++)
978 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
979}
980
981static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
982{
983 int ret = -ENODEV;
984 int i;
985
986 for (i = 0; i < gic_data.nr_redist_regions; i++) {
987 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
988 u64 typer;
989 u32 reg;
990
991 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
992 if (reg != GIC_PIDR2_ARCH_GICv3 &&
993 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
994 pr_warn("No redistributor present @%p\n", ptr);
995 break;
996 }
997
998 do {
999 typer = gic_read_typer(ptr + GICR_TYPER);
1000 ret = fn(gic_data.redist_regions + i, ptr);
1001 if (!ret)
1002 return 0;
1003
1004 if (gic_data.redist_regions[i].single_redist)
1005 break;
1006
1007 if (gic_data.redist_stride) {
1008 ptr += gic_data.redist_stride;
1009 } else {
1010 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
1011 if (typer & GICR_TYPER_VLPIS)
1012 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
1013 }
1014 } while (!(typer & GICR_TYPER_LAST));
1015 }
1016
1017 return ret ? -ENODEV : 0;
1018}
1019
1020static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
1021{
1022 unsigned long mpidr;
1023 u64 typer;
1024 u32 aff;
1025
1026 /*
1027 * Convert affinity to a 32bit value that can be matched to
1028 * GICR_TYPER bits [63:32].
1029 */
1030 mpidr = gic_cpu_to_affinity(smp_processor_id());
1031
1032 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
1033 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
1034 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
1035 MPIDR_AFFINITY_LEVEL(mpidr, 0));
1036
1037 typer = gic_read_typer(ptr + GICR_TYPER);
1038 if ((typer >> 32) == aff) {
1039 u64 offset = ptr - region->redist_base;
1040 raw_spin_lock_init(&gic_data_rdist()->rd_lock);
1041 gic_data_rdist_rd_base() = ptr;
1042 gic_data_rdist()->phys_base = region->phys_base + offset;
1043
1044 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
1045 smp_processor_id(), mpidr,
1046 (int)(region - gic_data.redist_regions),
1047 &gic_data_rdist()->phys_base);
1048 return 0;
1049 }
1050
1051 /* Try next one */
1052 return 1;
1053}
1054
1055static int gic_populate_rdist(void)
1056{
1057 if (gic_iterate_rdists(__gic_populate_rdist) == 0)
1058 return 0;
1059
1060 /* We couldn't even deal with ourselves... */
1061 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
1062 smp_processor_id(),
1063 (unsigned long)cpu_logical_map(smp_processor_id()));
1064 return -ENODEV;
1065}
1066
1067static int __gic_update_rdist_properties(struct redist_region *region,
1068 void __iomem *ptr)
1069{
1070 u64 typer = gic_read_typer(ptr + GICR_TYPER);
1071 u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
1072
1073 /* Boot-time cleanup */
1074 if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
1075 u64 val;
1076
1077 /* Deactivate any present vPE */
1078 val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
1079 if (val & GICR_VPENDBASER_Valid)
1080 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
1081 ptr + SZ_128K + GICR_VPENDBASER);
1082
1083 /* Mark the VPE table as invalid */
1084 val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
1085 val &= ~GICR_VPROPBASER_4_1_VALID;
1086 gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
1087 }
1088
1089 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
1090
1091 /*
1092 * TYPER.RVPEID implies some form of DirectLPI, no matter what the
1093 * doc says... :-/ And CTLR.IR implies another subset of DirectLPI
1094 * that the ITS driver can make use of for LPIs (and not VLPIs).
1095 *
1096 * These are 3 different ways to express the same thing, depending
1097 * on the revision of the architecture and its relaxations over
1098 * time. Just group them under the 'direct_lpi' banner.
1099 */
1100 gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
1101 gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
1102 !!(ctlr & GICR_CTLR_IR) |
1103 gic_data.rdists.has_rvpeid);
1104 gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);
1105
1106 /* Detect non-sensical configurations */
1107 if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
1108 gic_data.rdists.has_direct_lpi = false;
1109 gic_data.rdists.has_vlpis = false;
1110 gic_data.rdists.has_rvpeid = false;
1111 }
1112
1113 gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
1114
1115 return 1;
1116}
1117
1118static void gic_update_rdist_properties(void)
1119{
1120 gic_data.ppi_nr = UINT_MAX;
1121 gic_iterate_rdists(__gic_update_rdist_properties);
1122 if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
1123 gic_data.ppi_nr = 0;
1124 pr_info("GICv3 features: %d PPIs%s%s\n",
1125 gic_data.ppi_nr,
1126 gic_data.has_rss ? ", RSS" : "",
1127 gic_data.rdists.has_direct_lpi ? ", DirectLPI" : "");
1128
1129 if (gic_data.rdists.has_vlpis)
1130 pr_info("GICv4 features: %s%s%s\n",
1131 gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
1132 gic_data.rdists.has_rvpeid ? "RVPEID " : "",
1133 gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
1134}
1135
1136static void gic_cpu_sys_reg_enable(void)
1137{
1138 /*
1139 * Need to check that the SRE bit has actually been set. If
1140 * not, it means that SRE is disabled at EL2. We're going to
1141 * die painfully, and there is nothing we can do about it.
1142 *
1143 * Kindly inform the luser.
1144 */
1145 if (!gic_enable_sre())
1146 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
1147
1148}
1149
1150static void gic_cpu_sys_reg_init(void)
1151{
1152 int i, cpu = smp_processor_id();
1153 u64 mpidr = gic_cpu_to_affinity(cpu);
1154 u64 need_rss = MPIDR_RS(mpidr);
1155 bool group0;
1156 u32 pribits;
1157
1158 pribits = gic_get_pribits();
1159
1160 group0 = gic_has_group0();
1161
1162 /* Set priority mask register */
1163 if (!gic_prio_masking_enabled()) {
1164 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
1165 } else if (gic_supports_nmi()) {
1166 /*
1167 * Check that all CPUs use the same priority space.
1168 *
1169 * If there's a mismatch with the boot CPU, the system is
1170 * likely to die as interrupt masking will not work properly on
1171 * all CPUs.
1172 */
1173 WARN_ON(group0 != cpus_have_group0);
1174 WARN_ON(gic_dist_security_disabled() != cpus_have_security_disabled);
1175 }
1176
1177 /*
1178 * Some firmwares hand over to the kernel with the BPR changed from
1179 * its reset value (and with a value large enough to prevent
1180 * any pre-emptive interrupts from working at all). Writing a zero
1181 * to BPR restores is reset value.
1182 */
1183 gic_write_bpr1(0);
1184
1185 if (static_branch_likely(&supports_deactivate_key)) {
1186 /* EOI drops priority only (mode 1) */
1187 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
1188 } else {
1189 /* EOI deactivates interrupt too (mode 0) */
1190 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
1191 }
1192
1193 /* Always whack Group0 before Group1 */
1194 if (group0) {
1195 switch(pribits) {
1196 case 8:
1197 case 7:
1198 write_gicreg(0, ICC_AP0R3_EL1);
1199 write_gicreg(0, ICC_AP0R2_EL1);
1200 fallthrough;
1201 case 6:
1202 write_gicreg(0, ICC_AP0R1_EL1);
1203 fallthrough;
1204 case 5:
1205 case 4:
1206 write_gicreg(0, ICC_AP0R0_EL1);
1207 }
1208
1209 isb();
1210 }
1211
1212 switch(pribits) {
1213 case 8:
1214 case 7:
1215 write_gicreg(0, ICC_AP1R3_EL1);
1216 write_gicreg(0, ICC_AP1R2_EL1);
1217 fallthrough;
1218 case 6:
1219 write_gicreg(0, ICC_AP1R1_EL1);
1220 fallthrough;
1221 case 5:
1222 case 4:
1223 write_gicreg(0, ICC_AP1R0_EL1);
1224 }
1225
1226 isb();
1227
1228 /* ... and let's hit the road... */
1229 gic_write_grpen1(1);
1230
1231 /* Keep the RSS capability status in per_cpu variable */
1232 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
1233
1234 /* Check all the CPUs have capable of sending SGIs to other CPUs */
1235 for_each_online_cpu(i) {
1236 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
1237
1238 need_rss |= MPIDR_RS(gic_cpu_to_affinity(i));
1239 if (need_rss && (!have_rss))
1240 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
1241 cpu, (unsigned long)mpidr,
1242 i, (unsigned long)gic_cpu_to_affinity(i));
1243 }
1244
1245 /**
1246 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
1247 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
1248 * UNPREDICTABLE choice of :
1249 * - The write is ignored.
1250 * - The RS field is treated as 0.
1251 */
1252 if (need_rss && (!gic_data.has_rss))
1253 pr_crit_once("RSS is required but GICD doesn't support it\n");
1254}
1255
1256static bool gicv3_nolpi;
1257
1258static int __init gicv3_nolpi_cfg(char *buf)
1259{
1260 return kstrtobool(buf, &gicv3_nolpi);
1261}
1262early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1263
1264static int gic_dist_supports_lpis(void)
1265{
1266 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1267 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1268 !gicv3_nolpi);
1269}
1270
1271static void gic_cpu_init(void)
1272{
1273 void __iomem *rbase;
1274 int i;
1275
1276 /* Register ourselves with the rest of the world */
1277 if (gic_populate_rdist())
1278 return;
1279
1280 gic_enable_redist(true);
1281
1282 WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1283 !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1284 "Distributor has extended ranges, but CPU%d doesn't\n",
1285 smp_processor_id());
1286
1287 rbase = gic_data_rdist_sgi_base();
1288
1289 /* Configure SGIs/PPIs as non-secure Group-1 */
1290 for (i = 0; i < gic_data.ppi_nr + SGI_NR; i += 32)
1291 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
1292
1293 gic_cpu_config(rbase, gic_data.ppi_nr + SGI_NR, dist_prio_irq);
1294 gic_redist_wait_for_rwp();
1295
1296 /* initialise system registers */
1297 gic_cpu_sys_reg_init();
1298}
1299
1300#ifdef CONFIG_SMP
1301
1302#define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1303#define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL)
1304
1305/*
1306 * gic_starting_cpu() is called after the last point where cpuhp is allowed
1307 * to fail. So pre check for problems earlier.
1308 */
1309static int gic_check_rdist(unsigned int cpu)
1310{
1311 if (cpumask_test_cpu(cpu, &broken_rdists))
1312 return -EINVAL;
1313
1314 return 0;
1315}
1316
1317static int gic_starting_cpu(unsigned int cpu)
1318{
1319 gic_cpu_sys_reg_enable();
1320 gic_cpu_init();
1321
1322 if (gic_dist_supports_lpis())
1323 its_cpu_init();
1324
1325 return 0;
1326}
1327
1328static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
1329 unsigned long cluster_id)
1330{
1331 int next_cpu, cpu = *base_cpu;
1332 unsigned long mpidr;
1333 u16 tlist = 0;
1334
1335 mpidr = gic_cpu_to_affinity(cpu);
1336
1337 while (cpu < nr_cpu_ids) {
1338 tlist |= 1 << (mpidr & 0xf);
1339
1340 next_cpu = cpumask_next(cpu, mask);
1341 if (next_cpu >= nr_cpu_ids)
1342 goto out;
1343 cpu = next_cpu;
1344
1345 mpidr = gic_cpu_to_affinity(cpu);
1346
1347 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
1348 cpu--;
1349 goto out;
1350 }
1351 }
1352out:
1353 *base_cpu = cpu;
1354 return tlist;
1355}
1356
1357#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1358 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1359 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1360
1361static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1362{
1363 u64 val;
1364
1365 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) |
1366 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) |
1367 irq << ICC_SGI1R_SGI_ID_SHIFT |
1368 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) |
1369 MPIDR_TO_SGI_RS(cluster_id) |
1370 tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
1371
1372 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
1373 gic_write_sgi1r(val);
1374}
1375
1376static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
1377{
1378 int cpu;
1379
1380 if (WARN_ON(d->hwirq >= 16))
1381 return;
1382
1383 /*
1384 * Ensure that stores to Normal memory are visible to the
1385 * other CPUs before issuing the IPI.
1386 */
1387 dsb(ishst);
1388
1389 for_each_cpu(cpu, mask) {
1390 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu));
1391 u16 tlist;
1392
1393 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
1394 gic_send_sgi(cluster_id, tlist, d->hwirq);
1395 }
1396
1397 /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1398 isb();
1399}
1400
1401static void __init gic_smp_init(void)
1402{
1403 struct irq_fwspec sgi_fwspec = {
1404 .fwnode = gic_data.fwnode,
1405 .param_count = 1,
1406 };
1407 int base_sgi;
1408
1409 cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN,
1410 "irqchip/arm/gicv3:checkrdist",
1411 gic_check_rdist, NULL);
1412
1413 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
1414 "irqchip/arm/gicv3:starting",
1415 gic_starting_cpu, NULL);
1416
1417 /* Register all 8 non-secure SGIs */
1418 base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec);
1419 if (WARN_ON(base_sgi <= 0))
1420 return;
1421
1422 set_smp_ipi_range(base_sgi, 8);
1423}
1424
1425static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1426 bool force)
1427{
1428 unsigned int cpu;
1429 u32 offset, index;
1430 void __iomem *reg;
1431 int enabled;
1432 u64 val;
1433
1434 if (force)
1435 cpu = cpumask_first(mask_val);
1436 else
1437 cpu = cpumask_any_and(mask_val, cpu_online_mask);
1438
1439 if (cpu >= nr_cpu_ids)
1440 return -EINVAL;
1441
1442 if (gic_irq_in_rdist(d))
1443 return -EINVAL;
1444
1445 /* If interrupt was enabled, disable it first */
1446 enabled = gic_peek_irq(d, GICD_ISENABLER);
1447 if (enabled)
1448 gic_mask_irq(d);
1449
1450 offset = convert_offset_index(d, GICD_IROUTER, &index);
1451 reg = gic_dist_base(d) + offset + (index * 8);
1452 val = gic_cpu_to_affinity(cpu);
1453
1454 gic_write_irouter(val, reg);
1455
1456 /*
1457 * If the interrupt was enabled, enabled it again. Otherwise,
1458 * just wait for the distributor to have digested our changes.
1459 */
1460 if (enabled)
1461 gic_unmask_irq(d);
1462
1463 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1464
1465 return IRQ_SET_MASK_OK_DONE;
1466}
1467#else
1468#define gic_set_affinity NULL
1469#define gic_ipi_send_mask NULL
1470#define gic_smp_init() do { } while(0)
1471#endif
1472
1473static int gic_retrigger(struct irq_data *data)
1474{
1475 return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
1476}
1477
1478#ifdef CONFIG_CPU_PM
1479static int gic_cpu_pm_notifier(struct notifier_block *self,
1480 unsigned long cmd, void *v)
1481{
1482 if (cmd == CPU_PM_EXIT || cmd == CPU_PM_ENTER_FAILED) {
1483 if (gic_dist_security_disabled())
1484 gic_enable_redist(true);
1485 gic_cpu_sys_reg_enable();
1486 gic_cpu_sys_reg_init();
1487 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
1488 gic_write_grpen1(0);
1489 gic_enable_redist(false);
1490 }
1491 return NOTIFY_OK;
1492}
1493
1494static struct notifier_block gic_cpu_pm_notifier_block = {
1495 .notifier_call = gic_cpu_pm_notifier,
1496};
1497
1498static void gic_cpu_pm_init(void)
1499{
1500 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1501}
1502
1503#else
1504static inline void gic_cpu_pm_init(void) { }
1505#endif /* CONFIG_CPU_PM */
1506
1507static struct irq_chip gic_chip = {
1508 .name = "GICv3",
1509 .irq_mask = gic_mask_irq,
1510 .irq_unmask = gic_unmask_irq,
1511 .irq_eoi = gic_eoi_irq,
1512 .irq_set_type = gic_set_type,
1513 .irq_set_affinity = gic_set_affinity,
1514 .irq_retrigger = gic_retrigger,
1515 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
1516 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
1517 .irq_nmi_setup = gic_irq_nmi_setup,
1518 .irq_nmi_teardown = gic_irq_nmi_teardown,
1519 .ipi_send_mask = gic_ipi_send_mask,
1520 .flags = IRQCHIP_SET_TYPE_MASKED |
1521 IRQCHIP_SKIP_SET_WAKE |
1522 IRQCHIP_MASK_ON_SUSPEND,
1523};
1524
1525static struct irq_chip gic_eoimode1_chip = {
1526 .name = "GICv3",
1527 .irq_mask = gic_eoimode1_mask_irq,
1528 .irq_unmask = gic_unmask_irq,
1529 .irq_eoi = gic_eoimode1_eoi_irq,
1530 .irq_set_type = gic_set_type,
1531 .irq_set_affinity = gic_set_affinity,
1532 .irq_retrigger = gic_retrigger,
1533 .irq_get_irqchip_state = gic_irq_get_irqchip_state,
1534 .irq_set_irqchip_state = gic_irq_set_irqchip_state,
1535 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity,
1536 .irq_nmi_setup = gic_irq_nmi_setup,
1537 .irq_nmi_teardown = gic_irq_nmi_teardown,
1538 .ipi_send_mask = gic_ipi_send_mask,
1539 .flags = IRQCHIP_SET_TYPE_MASKED |
1540 IRQCHIP_SKIP_SET_WAKE |
1541 IRQCHIP_MASK_ON_SUSPEND,
1542};
1543
1544static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1545 irq_hw_number_t hw)
1546{
1547 struct irq_chip *chip = &gic_chip;
1548 struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));
1549
1550 if (static_branch_likely(&supports_deactivate_key))
1551 chip = &gic_eoimode1_chip;
1552
1553 switch (__get_intid_range(hw)) {
1554 case SGI_RANGE:
1555 case PPI_RANGE:
1556 case EPPI_RANGE:
1557 irq_set_percpu_devid(irq);
1558 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1559 handle_percpu_devid_irq, NULL, NULL);
1560 break;
1561
1562 case SPI_RANGE:
1563 case ESPI_RANGE:
1564 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1565 handle_fasteoi_irq, NULL, NULL);
1566 irq_set_probe(irq);
1567 irqd_set_single_target(irqd);
1568 break;
1569
1570 case LPI_RANGE:
1571 if (!gic_dist_supports_lpis())
1572 return -EPERM;
1573 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1574 handle_fasteoi_irq, NULL, NULL);
1575 break;
1576
1577 default:
1578 return -EPERM;
1579 }
1580
1581 /* Prevents SW retriggers which mess up the ACK/EOI ordering */
1582 irqd_set_handle_enforce_irqctx(irqd);
1583 return 0;
1584}
1585
1586static int gic_irq_domain_translate(struct irq_domain *d,
1587 struct irq_fwspec *fwspec,
1588 unsigned long *hwirq,
1589 unsigned int *type)
1590{
1591 if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
1592 *hwirq = fwspec->param[0];
1593 *type = IRQ_TYPE_EDGE_RISING;
1594 return 0;
1595 }
1596
1597 if (is_of_node(fwspec->fwnode)) {
1598 if (fwspec->param_count < 3)
1599 return -EINVAL;
1600
1601 switch (fwspec->param[0]) {
1602 case 0: /* SPI */
1603 if (fwspec->param[1] > 987)
1604 pr_warn_once("SPI %u out of range (use ESPI?)\n", fwspec->param[1]);
1605 *hwirq = fwspec->param[1] + 32;
1606 break;
1607 case 1: /* PPI */
1608 if (fwspec->param[1] > 15)
1609 pr_warn_once("PPI %u out of range (use EPPI?)\n", fwspec->param[1]);
1610 *hwirq = fwspec->param[1] + 16;
1611 break;
1612 case 2: /* ESPI */
1613 if (fwspec->param[1] > 1023)
1614 pr_warn_once("ESPI %u out of range\n", fwspec->param[1]);
1615 *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1616 break;
1617 case 3: /* EPPI */
1618 if (fwspec->param[1] > 63)
1619 pr_warn_once("EPPI %u out of range\n", fwspec->param[1]);
1620 *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
1621 break;
1622 case GIC_IRQ_TYPE_LPI: /* LPI */
1623 *hwirq = fwspec->param[1];
1624 break;
1625 default:
1626 return -EINVAL;
1627 }
1628
1629 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1630
1631 /*
1632 * Make it clear that broken DTs are... broken.
1633 */
1634 WARN_ON(*type == IRQ_TYPE_NONE);
1635 return 0;
1636 }
1637
1638 if (is_fwnode_irqchip(fwspec->fwnode)) {
1639 if(fwspec->param_count != 2)
1640 return -EINVAL;
1641
1642 if (fwspec->param[0] < 16) {
1643 pr_err(FW_BUG "Illegal GSI%d translation request\n",
1644 fwspec->param[0]);
1645 return -EINVAL;
1646 }
1647
1648 *hwirq = fwspec->param[0];
1649 *type = fwspec->param[1];
1650
1651 WARN_ON(*type == IRQ_TYPE_NONE);
1652 return 0;
1653 }
1654
1655 return -EINVAL;
1656}
1657
1658static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1659 unsigned int nr_irqs, void *arg)
1660{
1661 int i, ret;
1662 irq_hw_number_t hwirq;
1663 unsigned int type = IRQ_TYPE_NONE;
1664 struct irq_fwspec *fwspec = arg;
1665
1666 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1667 if (ret)
1668 return ret;
1669
1670 for (i = 0; i < nr_irqs; i++) {
1671 ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1672 if (ret)
1673 return ret;
1674 }
1675
1676 return 0;
1677}
1678
1679static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1680 unsigned int nr_irqs)
1681{
1682 int i;
1683
1684 for (i = 0; i < nr_irqs; i++) {
1685 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1686 irq_set_handler(virq + i, NULL);
1687 irq_domain_reset_irq_data(d);
1688 }
1689}
1690
1691static int gic_irq_domain_select(struct irq_domain *d,
1692 struct irq_fwspec *fwspec,
1693 enum irq_domain_bus_token bus_token)
1694{
1695 irq_hw_number_t hwirq;
1696 unsigned int type;
1697 int ret;
1698
1699 /* Not for us */
1700 if (fwspec->fwnode != d->fwnode)
1701 return 0;
1702
1703 /* Handle pure domain searches */
1704 if (!fwspec->param_count)
1705 return d->bus_token == bus_token;
1706
1707 /* If this is not DT, then we have a single domain */
1708 if (!is_of_node(fwspec->fwnode))
1709 return 1;
1710
1711 ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type);
1712 if (WARN_ON_ONCE(ret))
1713 return 0;
1714
1715 return d == gic_data.domain;
1716}
1717
1718static int gic_irq_get_fwspec_info(struct irq_fwspec *fwspec, struct irq_fwspec_info *info)
1719{
1720 const struct cpumask *mask = NULL;
1721
1722 info->flags = 0;
1723 info->affinity = NULL;
1724
1725 /* ACPI is not capable of describing PPI affinity -- yet */
1726 if (!is_of_node(fwspec->fwnode))
1727 return 0;
1728
1729 /* If the specifier provides an affinity, use it */
1730 if (fwspec->param_count == 4 && fwspec->param[3]) {
1731 struct fwnode_handle *fw;
1732
1733 switch (fwspec->param[0]) {
1734 case 1: /* PPI */
1735 case 3: /* EPPI */
1736 break;
1737 default:
1738 return 0;
1739 }
1740
1741 fw = of_fwnode_handle(of_find_node_by_phandle(fwspec->param[3]));
1742 if (!fw)
1743 return -ENOENT;
1744
1745 for (int i = 0; i < gic_data.nr_parts; i++) {
1746 if (gic_data.parts[i].partition_id == fw) {
1747 mask = &gic_data.parts[i].mask;
1748 break;
1749 }
1750 }
1751
1752 if (!mask)
1753 return -ENOENT;
1754 } else {
1755 mask = cpu_possible_mask;
1756 }
1757
1758 info->affinity = mask;
1759 info->flags = IRQ_FWSPEC_INFO_AFFINITY_VALID;
1760
1761 return 0;
1762}
1763
1764static const struct irq_domain_ops gic_irq_domain_ops = {
1765 .translate = gic_irq_domain_translate,
1766 .alloc = gic_irq_domain_alloc,
1767 .free = gic_irq_domain_free,
1768 .select = gic_irq_domain_select,
1769 .get_fwspec_info = gic_irq_get_fwspec_info,
1770};
1771
1772static bool gic_enable_quirk_msm8996(void *data)
1773{
1774 struct gic_chip_data *d = data;
1775
1776 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1777
1778 return true;
1779}
1780
1781static bool gic_enable_quirk_cavium_38539(void *data)
1782{
1783 struct gic_chip_data *d = data;
1784
1785 d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
1786
1787 return true;
1788}
1789
1790static bool gic_enable_quirk_hip06_07(void *data)
1791{
1792 struct gic_chip_data *d = data;
1793
1794 /*
1795 * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1796 * not being an actual ARM implementation). The saving grace is
1797 * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1798 * HIP07 doesn't even have a proper IIDR, and still pretends to
1799 * have ESPI. In both cases, put them right.
1800 */
1801 if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
1802 /* Zero both ESPI and the RES0 field next to it... */
1803 d->rdists.gicd_typer &= ~GENMASK(9, 8);
1804 return true;
1805 }
1806
1807 return false;
1808}
1809
1810#define T241_CHIPN_MASK GENMASK_ULL(45, 44)
1811#define T241_CHIP_GICDA_OFFSET 0x1580000
1812#define SMCCC_SOC_ID_T241 0x036b0241
1813
1814static bool gic_enable_quirk_nvidia_t241(void *data)
1815{
1816 s32 soc_id = arm_smccc_get_soc_id_version();
1817 unsigned long chip_bmask = 0;
1818 phys_addr_t phys;
1819 u32 i;
1820
1821 /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */
1822 if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241))
1823 return false;
1824
1825 /* Find the chips based on GICR regions PHYS addr */
1826 for (i = 0; i < gic_data.nr_redist_regions; i++) {
1827 chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK,
1828 (u64)gic_data.redist_regions[i].phys_base));
1829 }
1830
1831 if (hweight32(chip_bmask) < 3)
1832 return false;
1833
1834 /* Setup GICD alias regions */
1835 for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) {
1836 if (chip_bmask & BIT(i)) {
1837 phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET;
1838 phys |= FIELD_PREP(T241_CHIPN_MASK, i);
1839 t241_dist_base_alias[i] = ioremap(phys, SZ_64K);
1840 WARN_ON_ONCE(!t241_dist_base_alias[i]);
1841 }
1842 }
1843 static_branch_enable(&gic_nvidia_t241_erratum);
1844 return true;
1845}
1846
1847static bool gic_enable_quirk_asr8601(void *data)
1848{
1849 struct gic_chip_data *d = data;
1850
1851 d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001;
1852
1853 return true;
1854}
1855
1856static bool gic_enable_quirk_arm64_2941627(void *data)
1857{
1858 static_branch_enable(&gic_arm64_2941627_erratum);
1859 return true;
1860}
1861
1862static bool gic_enable_quirk_rk3399(void *data)
1863{
1864 struct gic_chip_data *d = data;
1865
1866 if (of_machine_is_compatible("rockchip,rk3399")) {
1867 d->flags |= FLAGS_WORKAROUND_INSECURE;
1868 return true;
1869 }
1870
1871 return false;
1872}
1873
1874static bool rd_set_non_coherent(void *data)
1875{
1876 struct gic_chip_data *d = data;
1877
1878 d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
1879 return true;
1880}
1881
1882static const struct gic_quirk gic_quirks[] = {
1883 {
1884 .desc = "GICv3: Qualcomm MSM8996 broken firmware",
1885 .compatible = "qcom,msm8996-gic-v3",
1886 .init = gic_enable_quirk_msm8996,
1887 },
1888 {
1889 .desc = "GICv3: ASR erratum 8601001",
1890 .compatible = "asr,asr8601-gic-v3",
1891 .init = gic_enable_quirk_asr8601,
1892 },
1893 {
1894 .desc = "GICv3: HIP06 erratum 161010803",
1895 .iidr = 0x0204043b,
1896 .mask = 0xffffffff,
1897 .init = gic_enable_quirk_hip06_07,
1898 },
1899 {
1900 .desc = "GICv3: HIP07 erratum 161010803",
1901 .iidr = 0x00000000,
1902 .mask = 0xffffffff,
1903 .init = gic_enable_quirk_hip06_07,
1904 },
1905 {
1906 /*
1907 * Reserved register accesses generate a Synchronous
1908 * External Abort. This erratum applies to:
1909 * - ThunderX: CN88xx
1910 * - OCTEON TX: CN83xx, CN81xx
1911 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
1912 */
1913 .desc = "GICv3: Cavium erratum 38539",
1914 .iidr = 0xa000034c,
1915 .mask = 0xe8f00fff,
1916 .init = gic_enable_quirk_cavium_38539,
1917 },
1918 {
1919 .desc = "GICv3: NVIDIA erratum T241-FABRIC-4",
1920 .iidr = 0x0402043b,
1921 .mask = 0xffffffff,
1922 .init = gic_enable_quirk_nvidia_t241,
1923 },
1924 {
1925 /*
1926 * GIC-700: 2941627 workaround - IP variant [0,1]
1927 *
1928 */
1929 .desc = "GICv3: ARM64 erratum 2941627",
1930 .iidr = 0x0400043b,
1931 .mask = 0xff0e0fff,
1932 .init = gic_enable_quirk_arm64_2941627,
1933 },
1934 {
1935 /*
1936 * GIC-700: 2941627 workaround - IP variant [2]
1937 */
1938 .desc = "GICv3: ARM64 erratum 2941627",
1939 .iidr = 0x0402043b,
1940 .mask = 0xff0f0fff,
1941 .init = gic_enable_quirk_arm64_2941627,
1942 },
1943 {
1944 .desc = "GICv3: non-coherent attribute",
1945 .property = "dma-noncoherent",
1946 .init = rd_set_non_coherent,
1947 },
1948 {
1949 .desc = "GICv3: Insecure RK3399 integration",
1950 .iidr = 0x0000043b,
1951 .mask = 0xff000fff,
1952 .init = gic_enable_quirk_rk3399,
1953 },
1954 {
1955 }
1956};
1957
1958static void gic_enable_nmi_support(void)
1959{
1960 if (!gic_prio_masking_enabled() || nmi_support_forbidden)
1961 return;
1962
1963 pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
1964 gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");
1965
1966 static_branch_enable(&supports_pseudo_nmis);
1967
1968 if (static_branch_likely(&supports_deactivate_key))
1969 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1970 else
1971 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1972}
1973
1974static int __init gic_init_bases(phys_addr_t dist_phys_base,
1975 void __iomem *dist_base,
1976 struct redist_region *rdist_regs,
1977 u32 nr_redist_regions,
1978 u64 redist_stride,
1979 struct fwnode_handle *handle)
1980{
1981 u32 typer;
1982 int err;
1983
1984 if (!is_hyp_mode_available())
1985 static_branch_disable(&supports_deactivate_key);
1986
1987 if (static_branch_likely(&supports_deactivate_key))
1988 pr_info("GIC: Using split EOI/Deactivate mode\n");
1989
1990 gic_data.fwnode = handle;
1991 gic_data.dist_phys_base = dist_phys_base;
1992 gic_data.dist_base = dist_base;
1993 gic_data.redist_regions = rdist_regs;
1994 gic_data.nr_redist_regions = nr_redist_regions;
1995 gic_data.redist_stride = redist_stride;
1996
1997 /*
1998 * Find out how many interrupts are supported.
1999 */
2000 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
2001 gic_data.rdists.gicd_typer = typer;
2002
2003 gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
2004 gic_quirks, &gic_data);
2005
2006 pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
2007 pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
2008
2009 /*
2010 * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
2011 * architecture spec (which says that reserved registers are RES0).
2012 */
2013 if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
2014 gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
2015
2016 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
2017 &gic_data);
2018 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
2019 if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) {
2020 /* Disable GICv4.x features for the erratum T241-FABRIC-4 */
2021 gic_data.rdists.has_rvpeid = true;
2022 gic_data.rdists.has_vlpis = true;
2023 gic_data.rdists.has_direct_lpi = true;
2024 gic_data.rdists.has_vpend_valid_dirty = true;
2025 }
2026
2027 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
2028 err = -ENOMEM;
2029 goto out_free;
2030 }
2031
2032 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
2033
2034 gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
2035
2036 if (typer & GICD_TYPER_MBIS) {
2037 err = mbi_init(handle, gic_data.domain);
2038 if (err)
2039 pr_err("Failed to initialize MBIs\n");
2040 }
2041
2042 set_handle_irq(gic_handle_irq);
2043
2044 gic_update_rdist_properties();
2045
2046 gic_cpu_sys_reg_enable();
2047 gic_prio_init();
2048 gic_dist_init();
2049 gic_cpu_init();
2050 gic_enable_nmi_support();
2051 gic_smp_init();
2052 gic_cpu_pm_init();
2053
2054 if (gic_dist_supports_lpis()) {
2055 its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq);
2056 its_cpu_init();
2057 its_lpi_memreserve_init();
2058 } else {
2059 if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
2060 gicv2m_init(handle, gic_data.domain);
2061 }
2062
2063 return 0;
2064
2065out_free:
2066 if (gic_data.domain)
2067 irq_domain_remove(gic_data.domain);
2068 free_percpu(gic_data.rdists.rdist);
2069 return err;
2070}
2071
2072static int __init gic_validate_dist_version(void __iomem *dist_base)
2073{
2074 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2075
2076 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
2077 return -ENODEV;
2078
2079 return 0;
2080}
2081
2082/* Create all possible partitions at boot time */
2083static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
2084{
2085 struct device_node *parts_node, *child_part;
2086 int part_idx = 0, i;
2087 int nr_parts;
2088 struct partition_affinity *parts;
2089
2090 parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
2091 if (!parts_node)
2092 return;
2093
2094 nr_parts = of_get_child_count(parts_node);
2095 if (!nr_parts)
2096 goto out_put_node;
2097
2098 parts = kzalloc_objs(*parts, nr_parts);
2099 if (WARN_ON(!parts))
2100 goto out_put_node;
2101
2102 for_each_child_of_node(parts_node, child_part) {
2103 struct partition_affinity *part;
2104 int n;
2105
2106 part = &parts[part_idx];
2107
2108 part->partition_id = of_fwnode_handle(child_part);
2109
2110 pr_info("GIC: PPI partition %pOFn[%d] { ",
2111 child_part, part_idx);
2112
2113 n = of_property_count_elems_of_size(child_part, "affinity",
2114 sizeof(u32));
2115 WARN_ON(n <= 0);
2116
2117 for (i = 0; i < n; i++) {
2118 int err, cpu;
2119 u32 cpu_phandle;
2120 struct device_node *cpu_node;
2121
2122 err = of_property_read_u32_index(child_part, "affinity",
2123 i, &cpu_phandle);
2124 if (WARN_ON(err))
2125 continue;
2126
2127 cpu_node = of_find_node_by_phandle(cpu_phandle);
2128 if (WARN_ON(!cpu_node))
2129 continue;
2130
2131 cpu = of_cpu_node_to_id(cpu_node);
2132 if (WARN_ON(cpu < 0)) {
2133 of_node_put(cpu_node);
2134 continue;
2135 }
2136
2137 pr_cont("%pOF[%d] ", cpu_node, cpu);
2138
2139 cpumask_set_cpu(cpu, &part->mask);
2140 of_node_put(cpu_node);
2141 }
2142
2143 pr_cont("}\n");
2144 part_idx++;
2145 }
2146
2147 gic_data.parts = parts;
2148 gic_data.nr_parts = nr_parts;
2149
2150out_put_node:
2151 of_node_put(parts_node);
2152}
2153
2154static void __init gic_of_setup_kvm_info(struct device_node *node, u32 nr_redist_regions)
2155{
2156 int ret;
2157 struct resource r;
2158
2159 gic_v3_kvm_info.type = GIC_V3;
2160
2161 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
2162 if (!gic_v3_kvm_info.maint_irq)
2163 return;
2164
2165 /* Also skip GICD, GICC, GICH */
2166 ret = of_address_to_resource(node, nr_redist_regions + 3, &r);
2167 if (!ret)
2168 gic_v3_kvm_info.vcpu = r;
2169
2170 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2171 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
2172 vgic_set_kvm_info(&gic_v3_kvm_info);
2173}
2174
2175static void gic_request_region(resource_size_t base, resource_size_t size,
2176 const char *name)
2177{
2178 if (!request_mem_region(base, size, name))
2179 pr_warn_once(FW_BUG "%s region %pa has overlapping address\n",
2180 name, &base);
2181}
2182
2183static void __iomem *gic_of_iomap(struct device_node *node, int idx,
2184 const char *name, struct resource *res)
2185{
2186 void __iomem *base;
2187 int ret;
2188
2189 ret = of_address_to_resource(node, idx, res);
2190 if (ret)
2191 return IOMEM_ERR_PTR(ret);
2192
2193 gic_request_region(res->start, resource_size(res), name);
2194 base = of_iomap(node, idx);
2195
2196 return base ?: IOMEM_ERR_PTR(-ENOMEM);
2197}
2198
2199static int __init gic_of_init(struct device_node *node, struct device_node *parent)
2200{
2201 phys_addr_t dist_phys_base;
2202 void __iomem *dist_base;
2203 struct redist_region *rdist_regs;
2204 struct resource res;
2205 u64 redist_stride;
2206 u32 nr_redist_regions;
2207 int err, i;
2208
2209 dist_base = gic_of_iomap(node, 0, "GICD", &res);
2210 if (IS_ERR(dist_base)) {
2211 pr_err("%pOF: unable to map gic dist registers\n", node);
2212 return PTR_ERR(dist_base);
2213 }
2214
2215 dist_phys_base = res.start;
2216
2217 err = gic_validate_dist_version(dist_base);
2218 if (err) {
2219 pr_err("%pOF: no distributor detected, giving up\n", node);
2220 goto out_unmap_dist;
2221 }
2222
2223 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
2224 nr_redist_regions = 1;
2225
2226 rdist_regs = kzalloc_objs(*rdist_regs, nr_redist_regions);
2227 if (!rdist_regs) {
2228 err = -ENOMEM;
2229 goto out_unmap_dist;
2230 }
2231
2232 for (i = 0; i < nr_redist_regions; i++) {
2233 rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res);
2234 if (IS_ERR(rdist_regs[i].redist_base)) {
2235 pr_err("%pOF: couldn't map region %d\n", node, i);
2236 err = -ENODEV;
2237 goto out_unmap_rdist;
2238 }
2239 rdist_regs[i].phys_base = res.start;
2240 }
2241
2242 if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
2243 redist_stride = 0;
2244
2245 gic_enable_of_quirks(node, gic_quirks, &gic_data);
2246
2247 err = gic_init_bases(dist_phys_base, dist_base, rdist_regs,
2248 nr_redist_regions, redist_stride, &node->fwnode);
2249 if (err)
2250 goto out_unmap_rdist;
2251
2252 gic_populate_ppi_partitions(node);
2253
2254 if (static_branch_likely(&supports_deactivate_key))
2255 gic_of_setup_kvm_info(node, nr_redist_regions);
2256 return 0;
2257
2258out_unmap_rdist:
2259 for (i = 0; i < nr_redist_regions; i++)
2260 if (!IS_ERR_OR_NULL(rdist_regs[i].redist_base))
2261 iounmap(rdist_regs[i].redist_base);
2262 kfree(rdist_regs);
2263out_unmap_dist:
2264 iounmap(dist_base);
2265 return err;
2266}
2267
2268IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
2269
2270#ifdef CONFIG_ACPI
2271static struct
2272{
2273 void __iomem *dist_base;
2274 struct redist_region *redist_regs;
2275 u32 nr_redist_regions;
2276 bool single_redist;
2277 int enabled_rdists;
2278 u32 maint_irq;
2279 int maint_irq_mode;
2280 phys_addr_t vcpu_base;
2281} acpi_data __initdata;
2282
2283static void __init
2284gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
2285{
2286 static int count = 0;
2287
2288 acpi_data.redist_regs[count].phys_base = phys_base;
2289 acpi_data.redist_regs[count].redist_base = redist_base;
2290 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
2291 count++;
2292}
2293
2294static int __init
2295gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
2296 const unsigned long end)
2297{
2298 struct acpi_madt_generic_redistributor *redist =
2299 (struct acpi_madt_generic_redistributor *)header;
2300 void __iomem *redist_base;
2301
2302 redist_base = ioremap(redist->base_address, redist->length);
2303 if (!redist_base) {
2304 pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
2305 return -ENOMEM;
2306 }
2307
2308 if (acpi_get_madt_revision() >= 7 &&
2309 (redist->flags & ACPI_MADT_GICR_NON_COHERENT))
2310 gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
2311
2312 gic_request_region(redist->base_address, redist->length, "GICR");
2313
2314 gic_acpi_register_redist(redist->base_address, redist_base);
2315 return 0;
2316}
2317
2318static int __init
2319gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
2320 const unsigned long end)
2321{
2322 struct acpi_madt_generic_interrupt *gicc =
2323 (struct acpi_madt_generic_interrupt *)header;
2324 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
2325 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
2326 void __iomem *redist_base;
2327
2328 /* Neither enabled or online capable means it doesn't exist, skip it */
2329 if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
2330 return 0;
2331
2332 /*
2333 * Capable but disabled CPUs can be brought online later. What about
2334 * the redistributor? ACPI doesn't want to say!
2335 * Virtual hotplug systems can use the MADT's "always-on" GICR entries.
2336 * Otherwise, prevent such CPUs from being brought online.
2337 */
2338 if (!(gicc->flags & ACPI_MADT_ENABLED)) {
2339 int cpu = get_cpu_for_acpi_id(gicc->uid);
2340
2341 pr_warn("CPU %u's redistributor is inaccessible: this CPU can't be brought online\n", cpu);
2342 if (cpu >= 0)
2343 cpumask_set_cpu(cpu, &broken_rdists);
2344 return 0;
2345 }
2346
2347 redist_base = ioremap(gicc->gicr_base_address, size);
2348 if (!redist_base)
2349 return -ENOMEM;
2350 gic_request_region(gicc->gicr_base_address, size, "GICR");
2351
2352 if (acpi_get_madt_revision() >= 7 &&
2353 (gicc->flags & ACPI_MADT_GICC_NON_COHERENT))
2354 gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
2355
2356 gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
2357 return 0;
2358}
2359
2360static int __init gic_acpi_collect_gicr_base(void)
2361{
2362 acpi_tbl_entry_handler redist_parser;
2363 enum acpi_madt_type type;
2364
2365 if (acpi_data.single_redist) {
2366 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
2367 redist_parser = gic_acpi_parse_madt_gicc;
2368 } else {
2369 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
2370 redist_parser = gic_acpi_parse_madt_redist;
2371 }
2372
2373 /* Collect redistributor base addresses in GICR entries */
2374 if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
2375 return 0;
2376
2377 pr_info("No valid GICR entries exist\n");
2378 return -ENODEV;
2379}
2380
2381static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
2382 const unsigned long end)
2383{
2384 /* Subtable presence means that redist exists, that's it */
2385 return 0;
2386}
2387
2388static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
2389 const unsigned long end)
2390{
2391 struct acpi_madt_generic_interrupt *gicc =
2392 (struct acpi_madt_generic_interrupt *)header;
2393
2394 /*
2395 * If GICC is enabled and has valid gicr base address, then it means
2396 * GICR base is presented via GICC. The redistributor is only known to
2397 * be accessible if the GICC is marked as enabled. If this bit is not
2398 * set, we'd need to add the redistributor at runtime, which isn't
2399 * supported.
2400 */
2401 if (gicc->flags & ACPI_MADT_ENABLED && gicc->gicr_base_address)
2402 acpi_data.enabled_rdists++;
2403
2404 return 0;
2405}
2406
2407static int __init gic_acpi_count_gicr_regions(void)
2408{
2409 int count;
2410
2411 /*
2412 * Count how many redistributor regions we have. It is not allowed
2413 * to mix redistributor description, GICR and GICC subtables have to be
2414 * mutually exclusive.
2415 */
2416 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
2417 gic_acpi_match_gicr, 0);
2418 if (count > 0) {
2419 acpi_data.single_redist = false;
2420 return count;
2421 }
2422
2423 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2424 gic_acpi_match_gicc, 0);
2425 if (count > 0) {
2426 acpi_data.single_redist = true;
2427 count = acpi_data.enabled_rdists;
2428 }
2429
2430 return count;
2431}
2432
2433static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
2434 struct acpi_probe_entry *ape)
2435{
2436 struct acpi_madt_generic_distributor *dist;
2437 int count;
2438
2439 dist = (struct acpi_madt_generic_distributor *)header;
2440 if (dist->version != ape->driver_data)
2441 return false;
2442
2443 /* We need to do that exercise anyway, the sooner the better */
2444 count = gic_acpi_count_gicr_regions();
2445 if (count <= 0)
2446 return false;
2447
2448 acpi_data.nr_redist_regions = count;
2449 return true;
2450}
2451
2452static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
2453 const unsigned long end)
2454{
2455 struct acpi_madt_generic_interrupt *gicc =
2456 (struct acpi_madt_generic_interrupt *)header;
2457 int maint_irq_mode;
2458 static int first_madt = true;
2459
2460 if (!(gicc->flags &
2461 (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
2462 return 0;
2463
2464 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
2465 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
2466
2467 if (first_madt) {
2468 first_madt = false;
2469
2470 acpi_data.maint_irq = gicc->vgic_interrupt;
2471 acpi_data.maint_irq_mode = maint_irq_mode;
2472 acpi_data.vcpu_base = gicc->gicv_base_address;
2473
2474 return 0;
2475 }
2476
2477 /*
2478 * The maintenance interrupt and GICV should be the same for every CPU
2479 */
2480 if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
2481 (acpi_data.maint_irq_mode != maint_irq_mode) ||
2482 (acpi_data.vcpu_base != gicc->gicv_base_address))
2483 return -EINVAL;
2484
2485 return 0;
2486}
2487
2488static bool __init gic_acpi_collect_virt_info(void)
2489{
2490 int count;
2491
2492 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2493 gic_acpi_parse_virt_madt_gicc, 0);
2494
2495 return (count > 0);
2496}
2497
2498#define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
2499#define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K)
2500#define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K)
2501
2502static void __init gic_acpi_setup_kvm_info(void)
2503{
2504 int irq;
2505
2506 if (!gic_acpi_collect_virt_info()) {
2507 pr_warn("Unable to get hardware information used for virtualization\n");
2508 return;
2509 }
2510
2511 gic_v3_kvm_info.type = GIC_V3;
2512
2513 irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
2514 acpi_data.maint_irq_mode,
2515 ACPI_ACTIVE_HIGH);
2516 if (irq <= 0)
2517 return;
2518
2519 gic_v3_kvm_info.maint_irq = irq;
2520
2521 if (acpi_data.vcpu_base) {
2522 struct resource *vcpu = &gic_v3_kvm_info.vcpu;
2523
2524 vcpu->flags = IORESOURCE_MEM;
2525 vcpu->start = acpi_data.vcpu_base;
2526 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
2527 }
2528
2529 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2530 gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
2531 vgic_set_kvm_info(&gic_v3_kvm_info);
2532}
2533
2534static struct fwnode_handle *gsi_domain_handle;
2535
2536static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi)
2537{
2538 return gsi_domain_handle;
2539}
2540
2541static int __init
2542gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
2543{
2544 struct acpi_madt_generic_distributor *dist;
2545 size_t size;
2546 int i, err;
2547
2548 /* Get distributor base address */
2549 dist = (struct acpi_madt_generic_distributor *)header;
2550 acpi_data.dist_base = ioremap(dist->base_address,
2551 ACPI_GICV3_DIST_MEM_SIZE);
2552 if (!acpi_data.dist_base) {
2553 pr_err("Unable to map GICD registers\n");
2554 return -ENOMEM;
2555 }
2556 gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD");
2557
2558 err = gic_validate_dist_version(acpi_data.dist_base);
2559 if (err) {
2560 pr_err("No distributor detected at @%p, giving up\n",
2561 acpi_data.dist_base);
2562 goto out_dist_unmap;
2563 }
2564
2565 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2566 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2567 if (!acpi_data.redist_regs) {
2568 err = -ENOMEM;
2569 goto out_dist_unmap;
2570 }
2571
2572 err = gic_acpi_collect_gicr_base();
2573 if (err)
2574 goto out_redist_unmap;
2575
2576 gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
2577 if (!gsi_domain_handle) {
2578 err = -ENOMEM;
2579 goto out_redist_unmap;
2580 }
2581
2582 err = gic_init_bases(dist->base_address, acpi_data.dist_base,
2583 acpi_data.redist_regs, acpi_data.nr_redist_regions,
2584 0, gsi_domain_handle);
2585 if (err)
2586 goto out_fwhandle_free;
2587
2588 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id, NULL);
2589
2590 if (static_branch_likely(&supports_deactivate_key))
2591 gic_acpi_setup_kvm_info();
2592
2593 return 0;
2594
2595out_fwhandle_free:
2596 irq_domain_free_fwnode(gsi_domain_handle);
2597out_redist_unmap:
2598 for (i = 0; i < acpi_data.nr_redist_regions; i++)
2599 if (acpi_data.redist_regs[i].redist_base)
2600 iounmap(acpi_data.redist_regs[i].redist_base);
2601 kfree(acpi_data.redist_regs);
2602out_dist_unmap:
2603 iounmap(acpi_data.dist_base);
2604 return err;
2605}
2606IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2607 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2608 gic_acpi_init);
2609IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2610 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2611 gic_acpi_init);
2612IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2613 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2614 gic_acpi_init);
2615#endif