projects/linux·blob master · refs

430 lines · 10.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2020-2024 Loongson Technologies, Inc.
4 */
5
6#include <linux/cpuhotplug.h>
7#include <linux/init.h>
8#include <linux/interrupt.h>
9#include <linux/irq.h>
10#include <linux/irqchip.h>
11#include <linux/irqchip/chained_irq.h>
12#include <linux/irqdomain.h>
13#include <linux/kernel.h>
14#include <linux/msi.h>
15#include <linux/radix-tree.h>
16#include <linux/spinlock.h>
17
18#include <asm/loongarch.h>
19#include <asm/setup.h>
20
21#include <linux/irqchip/irq-msi-lib.h>
22#include "irq-loongson.h"
23
24#define VECTORS_PER_REG 64
25#define IRR_VECTOR_MASK 0xffUL
26#define IRR_INVALID_MASK 0x80000000UL
27
28#ifdef CONFIG_SMP
29struct pending_list {
30 struct list_head head;
31};
32
33static struct cpumask intersect_mask;
34static DEFINE_PER_CPU(struct pending_list, pending_list);
35#endif
36
37static DEFINE_PER_CPU(struct irq_desc * [NR_VECTORS], irq_map);
38
39struct avecintc_chip {
40 raw_spinlock_t lock;
41 struct fwnode_handle *fwnode;
42 struct irq_domain *domain;
43 struct irq_matrix *vector_matrix;
44 phys_addr_t msi_base_addr;
45};
46
47static struct avecintc_chip loongarch_avec;
48
49static inline void avecintc_enable(void)
50{
51#ifdef CONFIG_MACH_LOONGSON64
52 u64 value;
53
54 value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC);
55 value |= IOCSR_MISC_FUNC_AVEC_EN;
56 iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC);
57#endif
58}
59
60static inline void avecintc_ack_irq(struct irq_data *d)
61{
62}
63
64static inline void avecintc_mask_irq(struct irq_data *d)
65{
66}
67
68static inline void avecintc_unmask_irq(struct irq_data *d)
69{
70}
71
72#ifdef CONFIG_SMP
73static inline void pending_list_init(int cpu)
74{
75 struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
76
77 INIT_LIST_HEAD(&plist->head);
78}
79
80void avecintc_sync(struct avecintc_data *adata)
81{
82 struct pending_list *plist;
83
84 if (cpu_online(adata->prev_cpu)) {
85 plist = per_cpu_ptr(&pending_list, adata->prev_cpu);
86 list_add_tail(&adata->entry, &plist->head);
87 adata->moving = 1;
88 mp_ops.send_ipi_single(adata->prev_cpu, ACTION_CLEAR_VECTOR);
89 }
90}
91
92static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
93{
94 int cpu, ret, vector;
95 struct avecintc_data *adata;
96
97 scoped_guard(raw_spinlock, &loongarch_avec.lock) {
98 adata = irq_data_get_irq_chip_data(data);
99
100 if (adata->moving)
101 return -EBUSY;
102
103 if (cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest))
104 return IRQ_SET_MASK_OK_DONE;
105
106 cpumask_and(&intersect_mask, dest, cpu_online_mask);
107
108 ret = irq_matrix_alloc(loongarch_avec.vector_matrix, &intersect_mask, false, &cpu);
109 if (ret < 0)
110 return ret;
111
112 vector = ret;
113 adata->cpu = cpu;
114 adata->vec = vector;
115 per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data);
116 if (!cpu_has_redirectint)
117 avecintc_sync(adata);
118 }
119
120 irq_data_update_effective_affinity(data, cpumask_of(cpu));
121
122 return IRQ_SET_MASK_OK;
123}
124
125static int avecintc_cpu_online(unsigned int cpu)
126{
127 if (!loongarch_avec.vector_matrix)
128 return 0;
129
130 guard(raw_spinlock)(&loongarch_avec.lock);
131
132 avecintc_enable();
133
134 irq_matrix_online(loongarch_avec.vector_matrix);
135
136 pending_list_init(cpu);
137
138 return 0;
139}
140
141static int avecintc_cpu_offline(unsigned int cpu)
142{
143 struct pending_list *plist = per_cpu_ptr(&pending_list, cpu);
144
145 if (!loongarch_avec.vector_matrix)
146 return 0;
147
148 guard(raw_spinlock)(&loongarch_avec.lock);
149
150 if (!list_empty(&plist->head))
151 pr_warn("CPU#%d vector is busy\n", cpu);
152
153 irq_matrix_offline(loongarch_avec.vector_matrix);
154
155 return 0;
156}
157
158void complete_irq_moving(void)
159{
160 struct pending_list *plist = this_cpu_ptr(&pending_list);
161 struct avecintc_data *adata, *tdata;
162 int cpu, vector, bias;
163 unsigned long isr;
164
165 guard(raw_spinlock)(&loongarch_avec.lock);
166
167 list_for_each_entry_safe(adata, tdata, &plist->head, entry) {
168 cpu = adata->prev_cpu;
169 vector = adata->prev_vec;
170 bias = vector / VECTORS_PER_REG;
171 switch (bias) {
172 case 0:
173 isr = csr_read(LOONGARCH_CSR_ISR0);
174 break;
175 case 1:
176 isr = csr_read(LOONGARCH_CSR_ISR1);
177 break;
178 case 2:
179 isr = csr_read(LOONGARCH_CSR_ISR2);
180 break;
181 case 3:
182 isr = csr_read(LOONGARCH_CSR_ISR3);
183 break;
184 }
185
186 if (isr & (1UL << (vector % VECTORS_PER_REG))) {
187 mp_ops.send_ipi_single(cpu, ACTION_CLEAR_VECTOR);
188 continue;
189 }
190 list_del(&adata->entry);
191 irq_matrix_free(loongarch_avec.vector_matrix, cpu, vector, false);
192 this_cpu_write(irq_map[vector], NULL);
193 adata->moving = 0;
194 adata->prev_cpu = adata->cpu;
195 adata->prev_vec = adata->vec;
196 }
197}
198#endif
199
200static void avecintc_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
201{
202 struct avecintc_data *adata = irq_data_get_irq_chip_data(d);
203
204 msg->address_hi = 0x0;
205 msg->address_lo = (loongarch_avec.msi_base_addr |
206 (adata->vec & AVEC_IRQ_MASK) << AVEC_IRQ_SHIFT) |
207 ((cpu_logical_map(adata->cpu & AVEC_CPU_MASK)) << AVEC_CPU_SHIFT);
208 msg->data = 0x0;
209}
210
211static struct irq_chip avec_irq_controller = {
212 .name = "AVECINTC",
213 .irq_ack = avecintc_ack_irq,
214 .irq_mask = avecintc_mask_irq,
215 .irq_unmask = avecintc_unmask_irq,
216#ifdef CONFIG_SMP
217 .irq_set_affinity = avecintc_set_affinity,
218#endif
219 .irq_compose_msi_msg = avecintc_compose_msi_msg,
220};
221
222static void avecintc_irq_dispatch(struct irq_desc *desc)
223{
224 struct irq_chip *chip = irq_desc_get_chip(desc);
225 struct irq_desc *d;
226
227 chained_irq_enter(chip, desc);
228
229 while (true) {
230 unsigned long vector = csr_read(LOONGARCH_CSR_IRR);
231 if (vector & IRR_INVALID_MASK)
232 break;
233
234 vector &= IRR_VECTOR_MASK;
235
236 d = this_cpu_read(irq_map[vector]);
237 if (d) {
238 generic_handle_irq_desc(d);
239 } else {
240 spurious_interrupt();
241 pr_warn("Unexpected IRQ occurs on CPU#%d [vector %ld]\n", smp_processor_id(), vector);
242 }
243 }
244
245 chained_irq_exit(chip, desc);
246}
247
248static int avecintc_alloc_vector(struct irq_data *irqd, struct avecintc_data *adata)
249{
250 int cpu, ret;
251
252 guard(raw_spinlock_irqsave)(&loongarch_avec.lock);
253
254 ret = irq_matrix_alloc(loongarch_avec.vector_matrix, cpu_online_mask, false, &cpu);
255 if (ret < 0)
256 return ret;
257
258 adata->prev_cpu = adata->cpu = cpu;
259 adata->prev_vec = adata->vec = ret;
260 per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(irqd);
261
262 return 0;
263}
264
265static int avecintc_domain_alloc(struct irq_domain *domain, unsigned int virq,
266 unsigned int nr_irqs, void *arg)
267{
268 for (unsigned int i = 0; i < nr_irqs; i++) {
269 struct irq_data *irqd = irq_domain_get_irq_data(domain, virq + i);
270 struct avecintc_data *adata = kzalloc_obj(*adata);
271 int ret;
272
273 if (!adata)
274 return -ENOMEM;
275
276 ret = avecintc_alloc_vector(irqd, adata);
277 if (ret < 0) {
278 kfree(adata);
279 return ret;
280 }
281
282 irq_domain_set_info(domain, virq + i, virq + i, &avec_irq_controller,
283 adata, handle_edge_irq, NULL, NULL);
284 irqd_set_single_target(irqd);
285 irqd_set_affinity_on_activate(irqd);
286 }
287
288 return 0;
289}
290
291static void avecintc_free_vector(struct irq_data *irqd, struct avecintc_data *adata)
292{
293 guard(raw_spinlock_irqsave)(&loongarch_avec.lock);
294
295 per_cpu(irq_map, adata->cpu)[adata->vec] = NULL;
296 irq_matrix_free(loongarch_avec.vector_matrix, adata->cpu, adata->vec, false);
297
298#ifdef CONFIG_SMP
299 if (!adata->moving)
300 return;
301
302 per_cpu(irq_map, adata->prev_cpu)[adata->prev_vec] = NULL;
303 irq_matrix_free(loongarch_avec.vector_matrix, adata->prev_cpu, adata->prev_vec, false);
304 list_del_init(&adata->entry);
305#endif
306}
307
308static void avecintc_domain_free(struct irq_domain *domain, unsigned int virq,
309 unsigned int nr_irqs)
310{
311 for (unsigned int i = 0; i < nr_irqs; i++) {
312 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
313
314 if (d) {
315 struct avecintc_data *adata = irq_data_get_irq_chip_data(d);
316
317 avecintc_free_vector(d, adata);
318 irq_domain_reset_irq_data(d);
319 kfree(adata);
320 }
321 }
322}
323
324static const struct irq_domain_ops avecintc_domain_ops = {
325 .alloc = avecintc_domain_alloc,
326 .free = avecintc_domain_free,
327 .select = msi_lib_irq_domain_select,
328};
329
330static int __init irq_matrix_init(void)
331{
332 loongarch_avec.vector_matrix = irq_alloc_matrix(NR_VECTORS, 0, NR_VECTORS);
333 if (!loongarch_avec.vector_matrix)
334 return -ENOMEM;
335
336 for (int i = 0; i < NR_LEGACY_VECTORS; i++)
337 irq_matrix_assign_system(loongarch_avec.vector_matrix, i, false);
338
339 irq_matrix_online(loongarch_avec.vector_matrix);
340
341 return 0;
342}
343
344static int __init avecintc_init(struct irq_domain *parent)
345{
346 int ret, parent_irq;
347
348 raw_spin_lock_init(&loongarch_avec.lock);
349
350 loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("AVECINTC");
351 if (!loongarch_avec.fwnode) {
352 pr_err("Unable to allocate domain handle\n");
353 ret = -ENOMEM;
354 goto out;
355 }
356
357 loongarch_avec.domain = irq_domain_create_tree(loongarch_avec.fwnode,
358 &avecintc_domain_ops, NULL);
359 if (!loongarch_avec.domain) {
360 pr_err("Unable to create IRQ domain\n");
361 ret = -ENOMEM;
362 goto out_free_handle;
363 }
364
365 parent_irq = irq_create_mapping(parent, INT_AVEC);
366 if (!parent_irq) {
367 pr_err("Failed to mapping hwirq\n");
368 ret = -EINVAL;
369 goto out_remove_domain;
370 }
371
372 ret = irq_matrix_init();
373 if (ret < 0) {
374 pr_err("Failed to init irq matrix\n");
375 goto out_remove_domain;
376 }
377 irq_set_chained_handler_and_data(parent_irq, avecintc_irq_dispatch, NULL);
378
379#ifdef CONFIG_SMP
380 pending_list_init(0);
381 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_AVECINTC_STARTING,
382 "irqchip/loongarch/avecintc:starting",
383 avecintc_cpu_online, avecintc_cpu_offline);
384#endif
385 avecintc_enable();
386
387 return ret;
388
389out_remove_domain:
390 irq_domain_remove(loongarch_avec.domain);
391out_free_handle:
392 irq_domain_free_fwnode(loongarch_avec.fwnode);
393out:
394 return ret;
395}
396
397static int __init pch_msi_parse_madt(union acpi_subtable_headers *header,
398 const unsigned long end)
399{
400 struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
401
402 loongarch_avec.msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
403
404 return pch_msi_acpi_init_avec(loongarch_avec.domain);
405}
406
407static inline int __init acpi_cascade_irqdomain_init(void)
408{
409 if (cpu_has_redirectint)
410 return redirect_acpi_init(loongarch_avec.domain);
411
412 return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
413}
414
415int __init avecintc_acpi_init(struct irq_domain *parent)
416{
417 int ret = avecintc_init(parent);
418 if (ret < 0) {
419 pr_err("Failed to init IRQ domain\n");
420 return ret;
421 }
422
423 ret = acpi_cascade_irqdomain_init();
424 if (ret < 0) {
425 pr_err("Failed to init cascade IRQ domain\n");
426 return ret;
427 }
428
429 return ret;
430}