projects/linux·blob master · refs

308 lines · 7.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7#define pr_fmt(fmt) "riscv-imsic: " fmt
8#include <linux/acpi.h>
9#include <linux/cpu.h>
10#include <linux/cpu_pm.h>
11#include <linux/export.h>
12#include <linux/interrupt.h>
13#include <linux/init.h>
14#include <linux/io.h>
15#include <linux/irq.h>
16#include <linux/irqchip.h>
17#include <linux/irqchip/chained_irq.h>
18#include <linux/irqchip/riscv-imsic.h>
19#include <linux/module.h>
20#include <linux/pci.h>
21#include <linux/spinlock.h>
22#include <linux/smp.h>
23
24#include "irq-riscv-imsic-state.h"
25
26static int imsic_parent_irq;
27bool imsic_noipi __ro_after_init;
28
29static int __init imsic_noipi_cfg(char *buf)
30{
31 imsic_noipi = true;
32 return 0;
33}
34early_param("irqchip.riscv_imsic_noipi", imsic_noipi_cfg);
35
36#ifdef CONFIG_SMP
37static void imsic_ipi_send(unsigned int cpu)
38{
39 struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu);
40
41 writel(IMSIC_IPI_ID, local->msi_va);
42}
43
44static void imsic_ipi_starting_cpu(void)
45{
46 if (imsic_noipi)
47 return;
48
49 /* Enable IPIs for current CPU. */
50 __imsic_id_set_enable(IMSIC_IPI_ID);
51}
52
53static void imsic_ipi_dying_cpu(void)
54{
55 if (imsic_noipi)
56 return;
57
58 /* Disable IPIs for current CPU. */
59 __imsic_id_clear_enable(IMSIC_IPI_ID);
60}
61
62static int __init imsic_ipi_domain_init(void)
63{
64 int virq;
65
66 if (imsic_noipi)
67 return 0;
68
69 /* Create IMSIC IPI multiplexing */
70 virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send);
71 if (virq <= 0)
72 return virq < 0 ? virq : -ENOMEM;
73
74 /* Set vIRQ range */
75 riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI);
76
77 /* Announce that IMSIC is providing IPIs */
78 pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID);
79
80 return 0;
81}
82#else
83static void imsic_ipi_starting_cpu(void) { }
84static void imsic_ipi_dying_cpu(void) { }
85static int __init imsic_ipi_domain_init(void) { return 0; }
86#endif
87
88/*
89 * To handle an interrupt, we read the TOPEI CSR and write zero in one
90 * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to
91 * Linux interrupt number and let Linux IRQ subsystem handle it.
92 */
93static void imsic_handle_irq(struct irq_desc *desc)
94{
95 struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv);
96 struct irq_chip *chip = irq_desc_get_chip(desc);
97 unsigned long local_id;
98
99 /*
100 * Process pending local synchronization instead of waiting
101 * for per-CPU local timer to expire.
102 */
103 imsic_local_sync_all(false);
104
105 chained_irq_enter(chip, desc);
106
107 while ((local_id = csr_swap(CSR_TOPEI, 0))) {
108 local_id >>= TOPEI_ID_SHIFT;
109
110 if (!imsic_noipi && local_id == IMSIC_IPI_ID) {
111 if (IS_ENABLED(CONFIG_SMP))
112 ipi_mux_process();
113 continue;
114 }
115
116 if (unlikely(local_id > imsic->global.nr_ids)) {
117 pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id);
118 continue;
119 }
120
121 generic_handle_irq(lpriv->vectors[local_id].irq);
122 }
123
124 chained_irq_exit(chip, desc);
125}
126
127static void imsic_hw_states_init(void)
128{
129 /* Setup IPIs */
130 imsic_ipi_starting_cpu();
131
132 /*
133 * Interrupts identities might have been enabled/disabled while
134 * this CPU was not running so sync-up local enable/disable state.
135 */
136 imsic_local_sync_all(true);
137
138 /* Enable local interrupt delivery */
139 imsic_local_delivery(true);
140}
141
142static int imsic_starting_cpu(unsigned int cpu)
143{
144 /* Mark per-CPU IMSIC state as online */
145 imsic_state_online();
146
147 /* Enable per-CPU parent interrupt */
148 enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq));
149
150 /* Initialize the IMSIC registers to enable the interrupt delivery */
151 imsic_hw_states_init();
152
153 return 0;
154}
155
156static int imsic_dying_cpu(unsigned int cpu)
157{
158 /* Cleanup IPIs */
159 imsic_ipi_dying_cpu();
160
161 imsic_local_sync_all(false);
162
163 /* Mark per-CPU IMSIC state as offline */
164 imsic_state_offline();
165
166 return 0;
167}
168
169static int imsic_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v)
170{
171 switch (cmd) {
172 case CPU_PM_EXIT:
173 /* Initialize the IMSIC registers to enable the interrupt delivery */
174 imsic_hw_states_init();
175 break;
176 }
177
178 return NOTIFY_OK;
179}
180
181static struct notifier_block imsic_pm_notifier_block = {
182 .notifier_call = imsic_pm_notifier,
183};
184
185static int __init imsic_early_probe(struct fwnode_handle *fwnode)
186{
187 struct irq_domain *domain;
188 int rc;
189
190 /* Find parent domain and register chained handler */
191 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY);
192 if (!domain) {
193 pr_err("%pfwP: Failed to find INTC domain\n", fwnode);
194 return -ENOENT;
195 }
196 imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT);
197 if (!imsic_parent_irq) {
198 pr_err("%pfwP: Failed to create INTC mapping\n", fwnode);
199 return -ENOENT;
200 }
201
202 /* Initialize IPI domain */
203 rc = imsic_ipi_domain_init();
204 if (rc) {
205 pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode);
206 return rc;
207 }
208
209 /* Setup chained handler to the parent domain interrupt */
210 irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq);
211
212 /*
213 * Setup cpuhp state (must be done after setting imsic_parent_irq)
214 *
215 * Don't disable per-CPU IMSIC file when CPU goes offline
216 * because this affects IPI and the masking/unmasking of
217 * virtual IPIs is done via generic IPI-Mux
218 */
219 cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting",
220 imsic_starting_cpu, imsic_dying_cpu);
221
222 return cpu_pm_register_notifier(&imsic_pm_notifier_block);
223}
224
225static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent)
226{
227 struct fwnode_handle *fwnode = &node->fwnode;
228 int rc;
229
230 /* Setup IMSIC state */
231 rc = imsic_setup_state(fwnode, NULL);
232 if (rc) {
233 pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc);
234 return rc;
235 }
236
237 /* Do early setup of IPIs */
238 rc = imsic_early_probe(fwnode);
239 if (rc)
240 return rc;
241
242 /* Ensure that OF platform device gets probed */
243 of_node_clear_flag(node, OF_POPULATED);
244 return 0;
245}
246
247IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init);
248
249#ifdef CONFIG_ACPI
250
251static struct fwnode_handle *imsic_acpi_fwnode;
252
253struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev)
254{
255 return imsic_acpi_fwnode;
256}
257EXPORT_SYMBOL_GPL(imsic_acpi_get_fwnode);
258
259static int __init imsic_early_acpi_init(union acpi_subtable_headers *header,
260 const unsigned long end)
261{
262 struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header;
263 int rc;
264
265 imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic");
266 if (!imsic_acpi_fwnode) {
267 pr_err("unable to allocate IMSIC FW node\n");
268 return -ENOMEM;
269 }
270
271 /* Setup IMSIC state */
272 rc = imsic_setup_state(imsic_acpi_fwnode, imsic);
273 if (rc) {
274 pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc);
275 goto cleanup;
276 }
277
278 /* Do early setup of IMSIC state and IPIs */
279 rc = imsic_early_probe(imsic_acpi_fwnode);
280 if (rc)
281 goto cleanup;
282
283 rc = imsic_platform_acpi_probe(imsic_acpi_fwnode);
284
285#ifdef CONFIG_PCI
286 if (!rc)
287 pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode);
288#endif
289
290 if (rc)
291 pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n",
292 imsic_acpi_fwnode, rc);
293
294 /*
295 * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can
296 * continue to work. So, no need to return failure. This is similar to
297 * DT where IPI works but MSI probe fails for some reason.
298 */
299 return 0;
300
301cleanup:
302 irq_domain_free_fwnode(imsic_acpi_fwnode);
303 imsic_acpi_fwnode = NULL;
304 return rc;
305}
306IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL,
307 1, imsic_early_acpi_init);
308#endif