projects/linux·blob master · refs

537 lines · 12.9 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2024-2026 Loongson Technologies, Inc.
4 */
5#define pr_fmt(fmt) "redirect: " fmt
6
7#include <linux/init.h>
8#include <linux/interrupt.h>
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/irq.h>
11#include <linux/irqchip.h>
12#include <linux/irqchip/irq-msi-lib.h>
13#include <linux/irqdomain.h>
14#include <linux/kernel.h>
15#include <linux/msi.h>
16#include <linux/spinlock.h>
17
18#include <asm/irq.h>
19#include <asm/loongarch.h>
20#include <asm/loongson.h>
21#include <asm/numa.h>
22#include <asm/setup.h>
23
24#include "irq-loongson.h"
25
26#define LOONGARCH_IOCSR_REDIRECT_CFG 0x15e0
27#define LOONGARCH_IOCSR_REDIRECT_TBR 0x15e8 /* IRT BASE REG */
28#define LOONGARCH_IOCSR_REDIRECT_CQB 0x15f0 /* IRT CACHE QUEUE BASE */
29#define LOONGARCH_IOCSR_REDIRECT_CQH 0x15f8 /* IRT CACHE QUEUE HEAD, 32bit */
30#define LOONGARCH_IOCSR_REDIRECT_CQT 0x15fc /* IRT CACHE QUEUE TAIL, 32bit */
31
32#define CQB_ADDR_MASK GENMASK_U64(47, 12)
33#define CQB_SIZE_MASK 0xf
34
35#define GPID_ADDR_MASK GENMASK_U64(47, 6)
36#define GPID_ADDR_SHIFT 6
37
38#define INVALID_INDEX 0
39#define CFG_DISABLE_IDLE 2
40
41#define MAX_IR_ENGINES 16
42
43struct redirect_entry {
44 struct {
45 u64 valid : 1,
46 res1 : 5,
47 gpid : 42,
48 res2 : 8,
49 vector : 8;
50 } lo;
51 u64 hi;
52};
53
54#define IRD_ENTRY_SIZE sizeof(struct redirect_entry)
55#define IRD_ENTRIES SZ_64K
56#define IRD_TABLE_PAGE_ORDER get_order(IRD_ENTRIES * IRD_ENTRY_SIZE)
57
58struct redirect_cmd {
59 union {
60 u64 cmd_info;
61 struct {
62 u64 res1 : 4,
63 type : 1,
64 need_notice : 1,
65 pad1 : 2,
66 index : 16,
67 pad2 : 40;
68 } index;
69 };
70 u64 notice_addr;
71};
72
73#define IRD_CMD_SIZE sizeof(struct redirect_cmd)
74#define INV_QUEUE_SIZE SZ_4K
75#define INV_QUEUE_PAGE_ORDER get_order(INV_QUEUE_SIZE * IRD_CMD_SIZE)
76
77struct redirect_gpid {
78 u64 pir[4]; /* Pending interrupt requested */
79 u8 en : 1, /* Doorbell */
80 res1 : 7;
81 u8 irqnum;
82 u16 res2;
83 u32 dstcpu;
84 u32 rsvd[6];
85};
86
87struct redirect_table {
88 struct redirect_entry *table;
89 unsigned long *bitmap;
90 raw_spinlock_t lock;
91};
92
93struct redirect_queue {
94 struct redirect_cmd *cmd_base;
95 int head;
96 int tail;
97 raw_spinlock_t lock;
98};
99
100struct redirect_desc {
101 struct redirect_table ird_table;
102 struct redirect_queue inv_queue;
103 int node;
104};
105
106struct redirect_item {
107 int index;
108 struct redirect_desc *irde;
109 struct redirect_gpid *gpid;
110};
111
112static struct irq_domain *redirect_domain;
113static struct redirect_desc redirect_descs[MAX_IR_ENGINES];
114
115static phys_addr_t msi_base_addr;
116static phys_addr_t redirect_reg_base = LOONGSON_REG_BASE;
117
118#ifdef CONFIG_32BIT
119
120#define REDIRECT_REG(reg, node) \
121 ((void __iomem *)(IO_BASE | redirect_reg_base | (reg)))
122
123#else
124
125#define REDIRECT_REG(reg, node) \
126 ((void __iomem *)(IO_BASE | redirect_reg_base | (u64)(node) << NODE_ADDRSPACE_SHIFT | (reg)))
127
128#endif
129
130static inline u32 redirect_read_reg32(u32 node, u32 reg)
131{
132 return readl(REDIRECT_REG(reg, node));
133}
134
135static inline void redirect_write_reg32(u32 node, u32 val, u32 reg)
136{
137 writel(val, REDIRECT_REG(reg, node));
138}
139
140static inline void redirect_write_reg64(u32 node, u64 val, u32 reg)
141{
142 writeq(val, REDIRECT_REG(reg, node));
143}
144
145static inline struct redirect_entry *item_get_entry(struct redirect_item *item)
146{
147 return item->irde->ird_table.table + item->index;
148}
149
150static inline bool invalid_queue_is_full(int node, u32 *tail)
151{
152 u32 head = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQH);
153
154 *tail = redirect_read_reg32(node, LOONGARCH_IOCSR_REDIRECT_CQT);
155
156 return head == ((*tail + 1) % INV_QUEUE_SIZE);
157}
158
159static void invalid_enqueue(struct redirect_item *item, struct redirect_cmd *cmd)
160{
161 struct redirect_queue *inv_queue = &item->irde->inv_queue;
162 u32 tail;
163
164 guard(raw_spinlock_irqsave)(&inv_queue->lock);
165
166 while (invalid_queue_is_full(item->irde->node, &tail))
167 cpu_relax();
168
169 memcpy(&inv_queue->cmd_base[tail], cmd, sizeof(*cmd));
170
171 redirect_write_reg32(item->irde->node, (tail + 1) % INV_QUEUE_SIZE, LOONGARCH_IOCSR_REDIRECT_CQT);
172}
173
174static void irde_invalidate_entry(struct redirect_item *item)
175{
176 struct redirect_cmd cmd;
177 u64 raddr = 0;
178
179 cmd.cmd_info = 0;
180 cmd.index.type = INVALID_INDEX;
181 cmd.index.need_notice = 1;
182 cmd.index.index = item->index;
183 cmd.notice_addr = (u64)(__pa(&raddr));
184
185 invalid_enqueue(item, &cmd);
186
187 /*
188 * The CPU needs to wait here for cmd to complete, and it determines this
189 * by checking whether the invalidation queue has already written a valid value
190 * to cmd.notice_addr.
191 */
192 while (!raddr)
193 cpu_relax();
194}
195
196static inline struct avecintc_data *irq_data_get_avec_data(struct irq_data *data)
197{
198 return data->parent_data->chip_data;
199}
200
201static int redirect_table_alloc(int node, u32 nr_irqs)
202{
203 struct redirect_table *ird_table = &redirect_descs[node].ird_table;
204 int index, order = 0;
205
206 if (nr_irqs > 1) {
207 nr_irqs = __roundup_pow_of_two(nr_irqs);
208 order = ilog2(nr_irqs);
209 }
210
211 guard(raw_spinlock_irqsave)(&ird_table->lock);
212
213 index = bitmap_find_free_region(ird_table->bitmap, IRD_ENTRIES, order);
214 if (index < 0) {
215 pr_err("No redirect entry to use\n");
216 return -EINVAL;
217 }
218
219 return index;
220}
221
222static void redirect_table_free(struct redirect_item *item)
223{
224 struct redirect_table *ird_table = &item->irde->ird_table;
225 struct redirect_entry *entry = item_get_entry(item);
226
227 memset(entry, 0, sizeof(*entry));
228
229 scoped_guard(raw_spinlock_irq, &ird_table->lock)
230 clear_bit(item->index, ird_table->bitmap);
231
232 kfree(item->gpid);
233
234 irde_invalidate_entry(item);
235}
236
237static inline void redirect_domain_prepare_entry(struct redirect_item *item,
238 struct avecintc_data *adata)
239{
240 struct redirect_entry *entry = item_get_entry(item);
241
242 item->gpid->en = 1;
243 item->gpid->dstcpu = adata->cpu;
244 item->gpid->irqnum = adata->vec;
245
246 entry->lo.valid = 1;
247 entry->lo.vector = 0xff;
248 entry->lo.gpid = ((unsigned long)item->gpid & GPID_ADDR_MASK) >> GPID_ADDR_SHIFT;
249}
250
251static void redirect_free_resources(struct irq_domain *domain,
252 unsigned int virq, unsigned int nr_irqs)
253{
254 for (int i = 0; i < nr_irqs; i++) {
255 struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
256
257 if (irq_data && irq_data->chip_data) {
258 struct redirect_item *item = irq_data->chip_data;
259
260 redirect_table_free(item);
261 kfree(item);
262 }
263 }
264}
265
266#ifdef CONFIG_SMP
267static int redirect_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force)
268{
269 struct avecintc_data *adata = irq_data_get_avec_data(data);
270 struct redirect_item *item = data->chip_data;
271 int ret;
272
273 ret = irq_chip_set_affinity_parent(data, dest, force);
274 switch (ret) {
275 case IRQ_SET_MASK_OK:
276 break;
277 case IRQ_SET_MASK_OK_DONE:
278 return ret;
279 default:
280 pr_err("IRDE: set_affinity error %d\n", ret);
281 return ret;
282 }
283
284 redirect_domain_prepare_entry(item, adata);
285 irde_invalidate_entry(item);
286 avecintc_sync(adata);
287
288 return IRQ_SET_MASK_OK;
289}
290#endif
291
292static void redirect_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
293{
294 struct redirect_item *item = irq_data_get_irq_chip_data(d);
295
296 msg->address_hi = 0x0;
297 msg->address_lo = (msi_base_addr | 1 << 2);
298 msg->data = item->index;
299}
300
301static struct irq_chip loongarch_redirect_chip = {
302 .name = "REDIRECT",
303 .irq_ack = irq_chip_ack_parent,
304 .irq_mask = irq_chip_mask_parent,
305 .irq_unmask = irq_chip_unmask_parent,
306#ifdef CONFIG_SMP
307 .irq_set_affinity = redirect_set_affinity,
308#endif
309 .irq_compose_msi_msg = redirect_compose_msi_msg,
310};
311
312static int redirect_domain_alloc(struct irq_domain *domain, unsigned int virq,
313 unsigned int nr_irqs, void *arg)
314{
315 msi_alloc_info_t *info = arg;
316 int ret, i, node, index;
317
318 node = dev_to_node(info->desc->dev);
319
320 ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
321 if (ret < 0)
322 return ret;
323
324 index = redirect_table_alloc(node, nr_irqs);
325 if (index < 0) {
326 pr_err("Alloc redirect table entry failed\n");
327 return -EINVAL;
328 }
329
330 for (i = 0; i < nr_irqs; i++) {
331 struct irq_data *irq_data = irq_domain_get_irq_data(domain, virq + i);
332 struct redirect_item *item;
333
334 item = kzalloc_obj(*item);
335 if (!item) {
336 pr_err("Alloc redirect descriptor failed\n");
337 goto out_free_resources;
338 }
339 item->irde = &redirect_descs[node];
340
341 /*
342 * Only bits 47:6 of the GPID are passed to the controller,
343 * 64-byte alignment must be guarantee and make kzalloc can
344 * align to the respective size.
345 */
346 static_assert(sizeof(*item->gpid) == 64);
347 item->gpid = kzalloc_node(sizeof(*item->gpid), GFP_KERNEL, node);
348 if (!item->gpid) {
349 pr_err("Alloc redirect GPID failed\n");
350 goto out_free_resources;
351 }
352 item->index = index + i;
353
354 irq_data->chip_data = item;
355 irq_data->chip = &loongarch_redirect_chip;
356
357 redirect_domain_prepare_entry(item, irq_data_get_avec_data(irq_data));
358 }
359
360 return 0;
361
362out_free_resources:
363 redirect_free_resources(domain, virq, nr_irqs);
364 irq_domain_free_irqs_common(domain, virq, nr_irqs);
365
366 return -ENOMEM;
367}
368
369static void redirect_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
370{
371 redirect_free_resources(domain, virq, nr_irqs);
372 return irq_domain_free_irqs_common(domain, virq, nr_irqs);
373}
374
375static const struct irq_domain_ops redirect_domain_ops = {
376 .alloc = redirect_domain_alloc,
377 .free = redirect_domain_free,
378 .select = msi_lib_irq_domain_select,
379};
380
381static int redirect_table_init(struct redirect_desc *irde)
382{
383 struct redirect_table *ird_table = &irde->ird_table;
384 unsigned long *bitmap;
385 struct folio *folio;
386
387 folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, IRD_TABLE_PAGE_ORDER, irde->node);
388 if (!folio) {
389 pr_err("Node [%d] redirect table alloc pages failed!\n", irde->node);
390 return -ENOMEM;
391 }
392 ird_table->table = folio_address(folio);
393
394 bitmap = bitmap_zalloc(IRD_ENTRIES, GFP_KERNEL);
395 if (!bitmap) {
396 pr_err("Node [%d] redirect table bitmap alloc pages failed!\n", irde->node);
397 folio_put(folio);
398 ird_table->table = NULL;
399 return -ENOMEM;
400 }
401 ird_table->bitmap = bitmap;
402
403 raw_spin_lock_init(&ird_table->lock);
404
405 return 0;
406}
407
408static int redirect_queue_init(struct redirect_desc *irde)
409{
410 struct redirect_queue *inv_queue = &irde->inv_queue;
411 struct folio *folio;
412
413 folio = __folio_alloc_node(GFP_KERNEL | __GFP_ZERO, INV_QUEUE_PAGE_ORDER, irde->node);
414 if (!folio) {
415 pr_err("Node [%d] invalid queue alloc pages failed!\n", irde->node);
416 return -ENOMEM;
417 }
418
419 inv_queue->cmd_base = folio_address(folio);
420 inv_queue->head = 0;
421 inv_queue->tail = 0;
422 raw_spin_lock_init(&inv_queue->lock);
423
424 return 0;
425}
426
427static void redirect_irde_cfg(struct redirect_desc *irde)
428{
429 redirect_write_reg64(irde->node, CFG_DISABLE_IDLE, LOONGARCH_IOCSR_REDIRECT_CFG);
430 redirect_write_reg64(irde->node, __pa(irde->ird_table.table), LOONGARCH_IOCSR_REDIRECT_TBR);
431 redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQH);
432 redirect_write_reg32(irde->node, 0, LOONGARCH_IOCSR_REDIRECT_CQT);
433 redirect_write_reg64(irde->node, ((unsigned long)irde->inv_queue.cmd_base & CQB_ADDR_MASK) |
434 CQB_SIZE_MASK, LOONGARCH_IOCSR_REDIRECT_CQB);
435}
436
437static void __init redirect_irde_free(struct redirect_desc *irde)
438{
439 struct redirect_table *ird_table = &redirect_descs->ird_table;
440 struct redirect_queue *inv_queue = &redirect_descs->inv_queue;
441
442 if (ird_table->table) {
443 folio_put(virt_to_folio(ird_table->table));
444 ird_table->table = NULL;
445 }
446
447 if (ird_table->bitmap) {
448 bitmap_free(ird_table->bitmap);
449 ird_table->bitmap = NULL;
450 }
451
452 if (inv_queue->cmd_base) {
453 folio_put(virt_to_folio(inv_queue->cmd_base));
454 inv_queue->cmd_base = NULL;
455 }
456}
457
458static int __init redirect_irde_init(int node)
459{
460 struct redirect_desc *irde = &redirect_descs[node];
461 int ret;
462
463 irde->node = node;
464
465 ret = redirect_table_init(irde);
466 if (ret)
467 return ret;
468
469 ret = redirect_queue_init(irde);
470 if (ret) {
471 redirect_irde_free(irde);
472 return ret;
473 }
474
475 redirect_irde_cfg(irde);
476
477 return 0;
478}
479
480static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, const unsigned long end)
481{
482 struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header;
483
484 msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET;
485
486 return pch_msi_acpi_init_avec(redirect_domain);
487}
488
489static int __init acpi_cascade_irqdomain_init(void)
490{
491 return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1);
492}
493
494int __init redirect_acpi_init(struct irq_domain *parent)
495{
496 struct fwnode_handle *fwnode;
497 int ret = -EINVAL, node;
498
499 fwnode = irq_domain_alloc_named_fwnode("redirect");
500 if (!fwnode) {
501 pr_err("Unable to alloc redirect domain handle\n");
502 goto fail;
503 }
504
505 redirect_domain = irq_domain_create_hierarchy(parent, 0, IRD_ENTRIES, fwnode,
506 &redirect_domain_ops, redirect_descs);
507 if (!redirect_domain) {
508 pr_err("Unable to alloc redirect domain\n");
509 goto out_free_fwnode;
510 }
511
512 for_each_node_mask(node, node_possible_map) {
513 ret = redirect_irde_init(node);
514 if (ret)
515 goto out_clear_irde;
516 }
517
518 ret = acpi_cascade_irqdomain_init();
519 if (ret < 0) {
520 pr_err("Failed to cascade IRQ domain, ret=%d\n", ret);
521 goto out_clear_irde;
522 }
523
524 pr_info("init succeeded\n");
525
526 return 0;
527
528out_clear_irde:
529 for_each_node_mask(node, node_possible_map) {
530 redirect_irde_free(&redirect_descs[node]);
531 }
532 irq_domain_remove(redirect_domain);
533out_free_fwnode:
534 irq_domain_free_fwnode(fwnode);
535fail:
536 return ret;
537}