projects/linux·blob v3.13 · refs

778 lines · 18.3 KiB

1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 */
10
11#include <linux/export.h>
12#include <linux/kernel_stat.h>
13#include <linux/interrupt.h>
14#include <linux/init.h>
15#include <linux/mm.h>
16#include <linux/notifier.h>
17#include <linux/percpu.h>
18#include <linux/cpu.h>
19#include <linux/freezer.h>
20#include <linux/kthread.h>
21#include <linux/rcupdate.h>
22#include <linux/ftrace.h>
23#include <linux/smp.h>
24#include <linux/smpboot.h>
25#include <linux/tick.h>
26
27#define CREATE_TRACE_POINTS
28#include <trace/events/irq.h>
29
30/*
31 - No shared variables, all the data are CPU local.
32 - If a softirq needs serialization, let it serialize itself
33 by its own spinlocks.
34 - Even if softirq is serialized, only local cpu is marked for
35 execution. Hence, we get something sort of weak cpu binding.
36 Though it is still not clear, will it result in better locality
37 or will not.
38
39 Examples:
40 - NET RX softirq. It is multithreaded and does not require
41 any global serialization.
42 - NET TX softirq. It kicks software netdevice queues, hence
43 it is logically serialized per device, but this serialization
44 is invisible to common code.
45 - Tasklets: serialized wrt itself.
46 */
47
48#ifndef __ARCH_IRQ_STAT
49irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
50EXPORT_SYMBOL(irq_stat);
51#endif
52
53static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
54
55DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
56
57char *softirq_to_name[NR_SOFTIRQS] = {
58 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
59 "TASKLET", "SCHED", "HRTIMER", "RCU"
60};
61
62/*
63 * we cannot loop indefinitely here to avoid userspace starvation,
64 * but we also don't want to introduce a worst case 1/HZ latency
65 * to the pending events, so lets the scheduler to balance
66 * the softirq load for us.
67 */
68static void wakeup_softirqd(void)
69{
70 /* Interrupts are disabled: no need to stop preemption */
71 struct task_struct *tsk = __this_cpu_read(ksoftirqd);
72
73 if (tsk && tsk->state != TASK_RUNNING)
74 wake_up_process(tsk);
75}
76
77/*
78 * preempt_count and SOFTIRQ_OFFSET usage:
79 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
80 * softirq processing.
81 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
82 * on local_bh_disable or local_bh_enable.
83 * This lets us distinguish between whether we are currently processing
84 * softirq and whether we just have bh disabled.
85 */
86
87/*
88 * This one is for softirq.c-internal use,
89 * where hardirqs are disabled legitimately:
90 */
91#ifdef CONFIG_TRACE_IRQFLAGS
92static void __local_bh_disable(unsigned long ip, unsigned int cnt)
93{
94 unsigned long flags;
95
96 WARN_ON_ONCE(in_irq());
97
98 raw_local_irq_save(flags);
99 /*
100 * The preempt tracer hooks into preempt_count_add and will break
101 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
102 * is set and before current->softirq_enabled is cleared.
103 * We must manually increment preempt_count here and manually
104 * call the trace_preempt_off later.
105 */
106 __preempt_count_add(cnt);
107 /*
108 * Were softirqs turned off above:
109 */
110 if (softirq_count() == cnt)
111 trace_softirqs_off(ip);
112 raw_local_irq_restore(flags);
113
114 if (preempt_count() == cnt)
115 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
116}
117#else /* !CONFIG_TRACE_IRQFLAGS */
118static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
119{
120 preempt_count_add(cnt);
121 barrier();
122}
123#endif /* CONFIG_TRACE_IRQFLAGS */
124
125void local_bh_disable(void)
126{
127 __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
128}
129
130EXPORT_SYMBOL(local_bh_disable);
131
132static void __local_bh_enable(unsigned int cnt)
133{
134 WARN_ON_ONCE(!irqs_disabled());
135
136 if (softirq_count() == cnt)
137 trace_softirqs_on(_RET_IP_);
138 preempt_count_sub(cnt);
139}
140
141/*
142 * Special-case - softirqs can safely be enabled in
143 * cond_resched_softirq(), or by __do_softirq(),
144 * without processing still-pending softirqs:
145 */
146void _local_bh_enable(void)
147{
148 WARN_ON_ONCE(in_irq());
149 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
150}
151
152EXPORT_SYMBOL(_local_bh_enable);
153
154static inline void _local_bh_enable_ip(unsigned long ip)
155{
156 WARN_ON_ONCE(in_irq() || irqs_disabled());
157#ifdef CONFIG_TRACE_IRQFLAGS
158 local_irq_disable();
159#endif
160 /*
161 * Are softirqs going to be turned on now:
162 */
163 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
164 trace_softirqs_on(ip);
165 /*
166 * Keep preemption disabled until we are done with
167 * softirq processing:
168 */
169 preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
170
171 if (unlikely(!in_interrupt() && local_softirq_pending())) {
172 /*
173 * Run softirq if any pending. And do it in its own stack
174 * as we may be calling this deep in a task call stack already.
175 */
176 do_softirq();
177 }
178
179 preempt_count_dec();
180#ifdef CONFIG_TRACE_IRQFLAGS
181 local_irq_enable();
182#endif
183 preempt_check_resched();
184}
185
186void local_bh_enable(void)
187{
188 _local_bh_enable_ip(_RET_IP_);
189}
190EXPORT_SYMBOL(local_bh_enable);
191
192void local_bh_enable_ip(unsigned long ip)
193{
194 _local_bh_enable_ip(ip);
195}
196EXPORT_SYMBOL(local_bh_enable_ip);
197
198/*
199 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
200 * but break the loop if need_resched() is set or after 2 ms.
201 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
202 * certain cases, such as stop_machine(), jiffies may cease to
203 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
204 * well to make sure we eventually return from this method.
205 *
206 * These limits have been established via experimentation.
207 * The two things to balance is latency against fairness -
208 * we want to handle softirqs as soon as possible, but they
209 * should not be able to lock up the box.
210 */
211#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
212#define MAX_SOFTIRQ_RESTART 10
213
214asmlinkage void __do_softirq(void)
215{
216 struct softirq_action *h;
217 __u32 pending;
218 unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
219 int cpu;
220 unsigned long old_flags = current->flags;
221 int max_restart = MAX_SOFTIRQ_RESTART;
222
223 /*
224 * Mask out PF_MEMALLOC s current task context is borrowed for the
225 * softirq. A softirq handled such as network RX might set PF_MEMALLOC
226 * again if the socket is related to swap
227 */
228 current->flags &= ~PF_MEMALLOC;
229
230 pending = local_softirq_pending();
231 account_irq_enter_time(current);
232
233 __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
234 lockdep_softirq_enter();
235
236 cpu = smp_processor_id();
237restart:
238 /* Reset the pending bitmask before enabling irqs */
239 set_softirq_pending(0);
240
241 local_irq_enable();
242
243 h = softirq_vec;
244
245 do {
246 if (pending & 1) {
247 unsigned int vec_nr = h - softirq_vec;
248 int prev_count = preempt_count();
249
250 kstat_incr_softirqs_this_cpu(vec_nr);
251
252 trace_softirq_entry(vec_nr);
253 h->action(h);
254 trace_softirq_exit(vec_nr);
255 if (unlikely(prev_count != preempt_count())) {
256 printk(KERN_ERR "huh, entered softirq %u %s %p"
257 "with preempt_count %08x,"
258 " exited with %08x?\n", vec_nr,
259 softirq_to_name[vec_nr], h->action,
260 prev_count, preempt_count());
261 preempt_count_set(prev_count);
262 }
263
264 rcu_bh_qs(cpu);
265 }
266 h++;
267 pending >>= 1;
268 } while (pending);
269
270 local_irq_disable();
271
272 pending = local_softirq_pending();
273 if (pending) {
274 if (time_before(jiffies, end) && !need_resched() &&
275 --max_restart)
276 goto restart;
277
278 wakeup_softirqd();
279 }
280
281 lockdep_softirq_exit();
282
283 account_irq_exit_time(current);
284 __local_bh_enable(SOFTIRQ_OFFSET);
285 WARN_ON_ONCE(in_interrupt());
286 tsk_restore_flags(current, old_flags, PF_MEMALLOC);
287}
288
289
290
291asmlinkage void do_softirq(void)
292{
293 __u32 pending;
294 unsigned long flags;
295
296 if (in_interrupt())
297 return;
298
299 local_irq_save(flags);
300
301 pending = local_softirq_pending();
302
303 if (pending)
304 do_softirq_own_stack();
305
306 local_irq_restore(flags);
307}
308
309/*
310 * Enter an interrupt context.
311 */
312void irq_enter(void)
313{
314 int cpu = smp_processor_id();
315
316 rcu_irq_enter();
317 if (is_idle_task(current) && !in_interrupt()) {
318 /*
319 * Prevent raise_softirq from needlessly waking up ksoftirqd
320 * here, as softirq will be serviced on return from interrupt.
321 */
322 local_bh_disable();
323 tick_check_idle(cpu);
324 _local_bh_enable();
325 }
326
327 __irq_enter();
328}
329
330static inline void invoke_softirq(void)
331{
332 if (!force_irqthreads) {
333#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
334 /*
335 * We can safely execute softirq on the current stack if
336 * it is the irq stack, because it should be near empty
337 * at this stage.
338 */
339 __do_softirq();
340#else
341 /*
342 * Otherwise, irq_exit() is called on the task stack that can
343 * be potentially deep already. So call softirq in its own stack
344 * to prevent from any overrun.
345 */
346 do_softirq_own_stack();
347#endif
348 } else {
349 wakeup_softirqd();
350 }
351}
352
353static inline void tick_irq_exit(void)
354{
355#ifdef CONFIG_NO_HZ_COMMON
356 int cpu = smp_processor_id();
357
358 /* Make sure that timer wheel updates are propagated */
359 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
360 if (!in_interrupt())
361 tick_nohz_irq_exit();
362 }
363#endif
364}
365
366/*
367 * Exit an interrupt context. Process softirqs if needed and possible:
368 */
369void irq_exit(void)
370{
371#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
372 local_irq_disable();
373#else
374 WARN_ON_ONCE(!irqs_disabled());
375#endif
376
377 account_irq_exit_time(current);
378 trace_hardirq_exit();
379 preempt_count_sub(HARDIRQ_OFFSET);
380 if (!in_interrupt() && local_softirq_pending())
381 invoke_softirq();
382
383 tick_irq_exit();
384 rcu_irq_exit();
385}
386
387/*
388 * This function must run with irqs disabled!
389 */
390inline void raise_softirq_irqoff(unsigned int nr)
391{
392 __raise_softirq_irqoff(nr);
393
394 /*
395 * If we're in an interrupt or softirq, we're done
396 * (this also catches softirq-disabled code). We will
397 * actually run the softirq once we return from
398 * the irq or softirq.
399 *
400 * Otherwise we wake up ksoftirqd to make sure we
401 * schedule the softirq soon.
402 */
403 if (!in_interrupt())
404 wakeup_softirqd();
405}
406
407void raise_softirq(unsigned int nr)
408{
409 unsigned long flags;
410
411 local_irq_save(flags);
412 raise_softirq_irqoff(nr);
413 local_irq_restore(flags);
414}
415
416void __raise_softirq_irqoff(unsigned int nr)
417{
418 trace_softirq_raise(nr);
419 or_softirq_pending(1UL << nr);
420}
421
422void open_softirq(int nr, void (*action)(struct softirq_action *))
423{
424 softirq_vec[nr].action = action;
425}
426
427/*
428 * Tasklets
429 */
430struct tasklet_head
431{
432 struct tasklet_struct *head;
433 struct tasklet_struct **tail;
434};
435
436static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
437static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
438
439void __tasklet_schedule(struct tasklet_struct *t)
440{
441 unsigned long flags;
442
443 local_irq_save(flags);
444 t->next = NULL;
445 *__this_cpu_read(tasklet_vec.tail) = t;
446 __this_cpu_write(tasklet_vec.tail, &(t->next));
447 raise_softirq_irqoff(TASKLET_SOFTIRQ);
448 local_irq_restore(flags);
449}
450
451EXPORT_SYMBOL(__tasklet_schedule);
452
453void __tasklet_hi_schedule(struct tasklet_struct *t)
454{
455 unsigned long flags;
456
457 local_irq_save(flags);
458 t->next = NULL;
459 *__this_cpu_read(tasklet_hi_vec.tail) = t;
460 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
461 raise_softirq_irqoff(HI_SOFTIRQ);
462 local_irq_restore(flags);
463}
464
465EXPORT_SYMBOL(__tasklet_hi_schedule);
466
467void __tasklet_hi_schedule_first(struct tasklet_struct *t)
468{
469 BUG_ON(!irqs_disabled());
470
471 t->next = __this_cpu_read(tasklet_hi_vec.head);
472 __this_cpu_write(tasklet_hi_vec.head, t);
473 __raise_softirq_irqoff(HI_SOFTIRQ);
474}
475
476EXPORT_SYMBOL(__tasklet_hi_schedule_first);
477
478static void tasklet_action(struct softirq_action *a)
479{
480 struct tasklet_struct *list;
481
482 local_irq_disable();
483 list = __this_cpu_read(tasklet_vec.head);
484 __this_cpu_write(tasklet_vec.head, NULL);
485 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
486 local_irq_enable();
487
488 while (list) {
489 struct tasklet_struct *t = list;
490
491 list = list->next;
492
493 if (tasklet_trylock(t)) {
494 if (!atomic_read(&t->count)) {
495 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
496 BUG();
497 t->func(t->data);
498 tasklet_unlock(t);
499 continue;
500 }
501 tasklet_unlock(t);
502 }
503
504 local_irq_disable();
505 t->next = NULL;
506 *__this_cpu_read(tasklet_vec.tail) = t;
507 __this_cpu_write(tasklet_vec.tail, &(t->next));
508 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
509 local_irq_enable();
510 }
511}
512
513static void tasklet_hi_action(struct softirq_action *a)
514{
515 struct tasklet_struct *list;
516
517 local_irq_disable();
518 list = __this_cpu_read(tasklet_hi_vec.head);
519 __this_cpu_write(tasklet_hi_vec.head, NULL);
520 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
521 local_irq_enable();
522
523 while (list) {
524 struct tasklet_struct *t = list;
525
526 list = list->next;
527
528 if (tasklet_trylock(t)) {
529 if (!atomic_read(&t->count)) {
530 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
531 BUG();
532 t->func(t->data);
533 tasklet_unlock(t);
534 continue;
535 }
536 tasklet_unlock(t);
537 }
538
539 local_irq_disable();
540 t->next = NULL;
541 *__this_cpu_read(tasklet_hi_vec.tail) = t;
542 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
543 __raise_softirq_irqoff(HI_SOFTIRQ);
544 local_irq_enable();
545 }
546}
547
548
549void tasklet_init(struct tasklet_struct *t,
550 void (*func)(unsigned long), unsigned long data)
551{
552 t->next = NULL;
553 t->state = 0;
554 atomic_set(&t->count, 0);
555 t->func = func;
556 t->data = data;
557}
558
559EXPORT_SYMBOL(tasklet_init);
560
561void tasklet_kill(struct tasklet_struct *t)
562{
563 if (in_interrupt())
564 printk("Attempt to kill tasklet from interrupt\n");
565
566 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
567 do {
568 yield();
569 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
570 }
571 tasklet_unlock_wait(t);
572 clear_bit(TASKLET_STATE_SCHED, &t->state);
573}
574
575EXPORT_SYMBOL(tasklet_kill);
576
577/*
578 * tasklet_hrtimer
579 */
580
581/*
582 * The trampoline is called when the hrtimer expires. It schedules a tasklet
583 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
584 * hrtimer callback, but from softirq context.
585 */
586static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
587{
588 struct tasklet_hrtimer *ttimer =
589 container_of(timer, struct tasklet_hrtimer, timer);
590
591 tasklet_hi_schedule(&ttimer->tasklet);
592 return HRTIMER_NORESTART;
593}
594
595/*
596 * Helper function which calls the hrtimer callback from
597 * tasklet/softirq context
598 */
599static void __tasklet_hrtimer_trampoline(unsigned long data)
600{
601 struct tasklet_hrtimer *ttimer = (void *)data;
602 enum hrtimer_restart restart;
603
604 restart = ttimer->function(&ttimer->timer);
605 if (restart != HRTIMER_NORESTART)
606 hrtimer_restart(&ttimer->timer);
607}
608
609/**
610 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
611 * @ttimer: tasklet_hrtimer which is initialized
612 * @function: hrtimer callback function which gets called from softirq context
613 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
614 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
615 */
616void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
617 enum hrtimer_restart (*function)(struct hrtimer *),
618 clockid_t which_clock, enum hrtimer_mode mode)
619{
620 hrtimer_init(&ttimer->timer, which_clock, mode);
621 ttimer->timer.function = __hrtimer_tasklet_trampoline;
622 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
623 (unsigned long)ttimer);
624 ttimer->function = function;
625}
626EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
627
628void __init softirq_init(void)
629{
630 int cpu;
631
632 for_each_possible_cpu(cpu) {
633 per_cpu(tasklet_vec, cpu).tail =
634 &per_cpu(tasklet_vec, cpu).head;
635 per_cpu(tasklet_hi_vec, cpu).tail =
636 &per_cpu(tasklet_hi_vec, cpu).head;
637 }
638
639 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
640 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
641}
642
643static int ksoftirqd_should_run(unsigned int cpu)
644{
645 return local_softirq_pending();
646}
647
648static void run_ksoftirqd(unsigned int cpu)
649{
650 local_irq_disable();
651 if (local_softirq_pending()) {
652 /*
653 * We can safely run softirq on inline stack, as we are not deep
654 * in the task stack here.
655 */
656 __do_softirq();
657 rcu_note_context_switch(cpu);
658 local_irq_enable();
659 cond_resched();
660 return;
661 }
662 local_irq_enable();
663}
664
665#ifdef CONFIG_HOTPLUG_CPU
666/*
667 * tasklet_kill_immediate is called to remove a tasklet which can already be
668 * scheduled for execution on @cpu.
669 *
670 * Unlike tasklet_kill, this function removes the tasklet
671 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
672 *
673 * When this function is called, @cpu must be in the CPU_DEAD state.
674 */
675void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
676{
677 struct tasklet_struct **i;
678
679 BUG_ON(cpu_online(cpu));
680 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
681
682 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
683 return;
684
685 /* CPU is dead, so no lock needed. */
686 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
687 if (*i == t) {
688 *i = t->next;
689 /* If this was the tail element, move the tail ptr */
690 if (*i == NULL)
691 per_cpu(tasklet_vec, cpu).tail = i;
692 return;
693 }
694 }
695 BUG();
696}
697
698static void takeover_tasklets(unsigned int cpu)
699{
700 /* CPU is dead, so no lock needed. */
701 local_irq_disable();
702
703 /* Find end, append list for that CPU. */
704 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
705 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
706 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
707 per_cpu(tasklet_vec, cpu).head = NULL;
708 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
709 }
710 raise_softirq_irqoff(TASKLET_SOFTIRQ);
711
712 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
713 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
714 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
715 per_cpu(tasklet_hi_vec, cpu).head = NULL;
716 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
717 }
718 raise_softirq_irqoff(HI_SOFTIRQ);
719
720 local_irq_enable();
721}
722#endif /* CONFIG_HOTPLUG_CPU */
723
724static int cpu_callback(struct notifier_block *nfb,
725 unsigned long action,
726 void *hcpu)
727{
728 switch (action) {
729#ifdef CONFIG_HOTPLUG_CPU
730 case CPU_DEAD:
731 case CPU_DEAD_FROZEN:
732 takeover_tasklets((unsigned long)hcpu);
733 break;
734#endif /* CONFIG_HOTPLUG_CPU */
735 }
736 return NOTIFY_OK;
737}
738
739static struct notifier_block cpu_nfb = {
740 .notifier_call = cpu_callback
741};
742
743static struct smp_hotplug_thread softirq_threads = {
744 .store = &ksoftirqd,
745 .thread_should_run = ksoftirqd_should_run,
746 .thread_fn = run_ksoftirqd,
747 .thread_comm = "ksoftirqd/%u",
748};
749
750static __init int spawn_ksoftirqd(void)
751{
752 register_cpu_notifier(&cpu_nfb);
753
754 BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
755
756 return 0;
757}
758early_initcall(spawn_ksoftirqd);
759
760/*
761 * [ These __weak aliases are kept in a separate compilation unit, so that
762 * GCC does not inline them incorrectly. ]
763 */
764
765int __init __weak early_irq_init(void)
766{
767 return 0;
768}
769
770int __init __weak arch_probe_nr_irqs(void)
771{
772 return NR_IRQS_LEGACY;
773}
774
775int __init __weak arch_early_irq_init(void)
776{
777 return 0;
778}