projects/linux·blob v2.6.21 · refs

657 lines · 14.4 KiB

1/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
7 */
8
9#include <linux/module.h>
10#include <linux/kernel_stat.h>
11#include <linux/interrupt.h>
12#include <linux/init.h>
13#include <linux/mm.h>
14#include <linux/notifier.h>
15#include <linux/percpu.h>
16#include <linux/cpu.h>
17#include <linux/kthread.h>
18#include <linux/rcupdate.h>
19#include <linux/smp.h>
20#include <linux/tick.h>
21
22#include <asm/irq.h>
23/*
24 - No shared variables, all the data are CPU local.
25 - If a softirq needs serialization, let it serialize itself
26 by its own spinlocks.
27 - Even if softirq is serialized, only local cpu is marked for
28 execution. Hence, we get something sort of weak cpu binding.
29 Though it is still not clear, will it result in better locality
30 or will not.
31
32 Examples:
33 - NET RX softirq. It is multithreaded and does not require
34 any global serialization.
35 - NET TX softirq. It kicks software netdevice queues, hence
36 it is logically serialized per device, but this serialization
37 is invisible to common code.
38 - Tasklets: serialized wrt itself.
39 */
40
41#ifndef __ARCH_IRQ_STAT
42irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
43EXPORT_SYMBOL(irq_stat);
44#endif
45
46static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
47
48static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
49
50/*
51 * we cannot loop indefinitely here to avoid userspace starvation,
52 * but we also don't want to introduce a worst case 1/HZ latency
53 * to the pending events, so lets the scheduler to balance
54 * the softirq load for us.
55 */
56static inline void wakeup_softirqd(void)
57{
58 /* Interrupts are disabled: no need to stop preemption */
59 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
60
61 if (tsk && tsk->state != TASK_RUNNING)
62 wake_up_process(tsk);
63}
64
65/*
66 * This one is for softirq.c-internal use,
67 * where hardirqs are disabled legitimately:
68 */
69#ifdef CONFIG_TRACE_IRQFLAGS
70static void __local_bh_disable(unsigned long ip)
71{
72 unsigned long flags;
73
74 WARN_ON_ONCE(in_irq());
75
76 raw_local_irq_save(flags);
77 add_preempt_count(SOFTIRQ_OFFSET);
78 /*
79 * Were softirqs turned off above:
80 */
81 if (softirq_count() == SOFTIRQ_OFFSET)
82 trace_softirqs_off(ip);
83 raw_local_irq_restore(flags);
84}
85#else /* !CONFIG_TRACE_IRQFLAGS */
86static inline void __local_bh_disable(unsigned long ip)
87{
88 add_preempt_count(SOFTIRQ_OFFSET);
89 barrier();
90}
91#endif /* CONFIG_TRACE_IRQFLAGS */
92
93void local_bh_disable(void)
94{
95 __local_bh_disable((unsigned long)__builtin_return_address(0));
96}
97
98EXPORT_SYMBOL(local_bh_disable);
99
100void __local_bh_enable(void)
101{
102 WARN_ON_ONCE(in_irq());
103
104 /*
105 * softirqs should never be enabled by __local_bh_enable(),
106 * it always nests inside local_bh_enable() sections:
107 */
108 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
109
110 sub_preempt_count(SOFTIRQ_OFFSET);
111}
112EXPORT_SYMBOL_GPL(__local_bh_enable);
113
114/*
115 * Special-case - softirqs can safely be enabled in
116 * cond_resched_softirq(), or by __do_softirq(),
117 * without processing still-pending softirqs:
118 */
119void _local_bh_enable(void)
120{
121 WARN_ON_ONCE(in_irq());
122 WARN_ON_ONCE(!irqs_disabled());
123
124 if (softirq_count() == SOFTIRQ_OFFSET)
125 trace_softirqs_on((unsigned long)__builtin_return_address(0));
126 sub_preempt_count(SOFTIRQ_OFFSET);
127}
128
129EXPORT_SYMBOL(_local_bh_enable);
130
131void local_bh_enable(void)
132{
133#ifdef CONFIG_TRACE_IRQFLAGS
134 unsigned long flags;
135
136 WARN_ON_ONCE(in_irq());
137#endif
138 WARN_ON_ONCE(irqs_disabled());
139
140#ifdef CONFIG_TRACE_IRQFLAGS
141 local_irq_save(flags);
142#endif
143 /*
144 * Are softirqs going to be turned on now:
145 */
146 if (softirq_count() == SOFTIRQ_OFFSET)
147 trace_softirqs_on((unsigned long)__builtin_return_address(0));
148 /*
149 * Keep preemption disabled until we are done with
150 * softirq processing:
151 */
152 sub_preempt_count(SOFTIRQ_OFFSET - 1);
153
154 if (unlikely(!in_interrupt() && local_softirq_pending()))
155 do_softirq();
156
157 dec_preempt_count();
158#ifdef CONFIG_TRACE_IRQFLAGS
159 local_irq_restore(flags);
160#endif
161 preempt_check_resched();
162}
163EXPORT_SYMBOL(local_bh_enable);
164
165void local_bh_enable_ip(unsigned long ip)
166{
167#ifdef CONFIG_TRACE_IRQFLAGS
168 unsigned long flags;
169
170 WARN_ON_ONCE(in_irq());
171
172 local_irq_save(flags);
173#endif
174 /*
175 * Are softirqs going to be turned on now:
176 */
177 if (softirq_count() == SOFTIRQ_OFFSET)
178 trace_softirqs_on(ip);
179 /*
180 * Keep preemption disabled until we are done with
181 * softirq processing:
182 */
183 sub_preempt_count(SOFTIRQ_OFFSET - 1);
184
185 if (unlikely(!in_interrupt() && local_softirq_pending()))
186 do_softirq();
187
188 dec_preempt_count();
189#ifdef CONFIG_TRACE_IRQFLAGS
190 local_irq_restore(flags);
191#endif
192 preempt_check_resched();
193}
194EXPORT_SYMBOL(local_bh_enable_ip);
195
196/*
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
199 *
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
204 */
205#define MAX_SOFTIRQ_RESTART 10
206
207asmlinkage void __do_softirq(void)
208{
209 struct softirq_action *h;
210 __u32 pending;
211 int max_restart = MAX_SOFTIRQ_RESTART;
212 int cpu;
213
214 pending = local_softirq_pending();
215 account_system_vtime(current);
216
217 __local_bh_disable((unsigned long)__builtin_return_address(0));
218 trace_softirq_enter();
219
220 cpu = smp_processor_id();
221restart:
222 /* Reset the pending bitmask before enabling irqs */
223 set_softirq_pending(0);
224
225 local_irq_enable();
226
227 h = softirq_vec;
228
229 do {
230 if (pending & 1) {
231 h->action(h);
232 rcu_bh_qsctr_inc(cpu);
233 }
234 h++;
235 pending >>= 1;
236 } while (pending);
237
238 local_irq_disable();
239
240 pending = local_softirq_pending();
241 if (pending && --max_restart)
242 goto restart;
243
244 if (pending)
245 wakeup_softirqd();
246
247 trace_softirq_exit();
248
249 account_system_vtime(current);
250 _local_bh_enable();
251}
252
253#ifndef __ARCH_HAS_DO_SOFTIRQ
254
255asmlinkage void do_softirq(void)
256{
257 __u32 pending;
258 unsigned long flags;
259
260 if (in_interrupt())
261 return;
262
263 local_irq_save(flags);
264
265 pending = local_softirq_pending();
266
267 if (pending)
268 __do_softirq();
269
270 local_irq_restore(flags);
271}
272
273EXPORT_SYMBOL(do_softirq);
274
275#endif
276
277/*
278 * Enter an interrupt context.
279 */
280void irq_enter(void)
281{
282 __irq_enter();
283#ifdef CONFIG_NO_HZ
284 if (idle_cpu(smp_processor_id()))
285 tick_nohz_update_jiffies();
286#endif
287}
288
289#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
290# define invoke_softirq() __do_softirq()
291#else
292# define invoke_softirq() do_softirq()
293#endif
294
295/*
296 * Exit an interrupt context. Process softirqs if needed and possible:
297 */
298void irq_exit(void)
299{
300 account_system_vtime(current);
301 trace_hardirq_exit();
302 sub_preempt_count(IRQ_EXIT_OFFSET);
303 if (!in_interrupt() && local_softirq_pending())
304 invoke_softirq();
305
306#ifdef CONFIG_NO_HZ
307 /* Make sure that timer wheel updates are propagated */
308 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
309 tick_nohz_stop_sched_tick();
310#endif
311 preempt_enable_no_resched();
312}
313
314/*
315 * This function must run with irqs disabled!
316 */
317inline fastcall void raise_softirq_irqoff(unsigned int nr)
318{
319 __raise_softirq_irqoff(nr);
320
321 /*
322 * If we're in an interrupt or softirq, we're done
323 * (this also catches softirq-disabled code). We will
324 * actually run the softirq once we return from
325 * the irq or softirq.
326 *
327 * Otherwise we wake up ksoftirqd to make sure we
328 * schedule the softirq soon.
329 */
330 if (!in_interrupt())
331 wakeup_softirqd();
332}
333
334EXPORT_SYMBOL(raise_softirq_irqoff);
335
336void fastcall raise_softirq(unsigned int nr)
337{
338 unsigned long flags;
339
340 local_irq_save(flags);
341 raise_softirq_irqoff(nr);
342 local_irq_restore(flags);
343}
344
345void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
346{
347 softirq_vec[nr].data = data;
348 softirq_vec[nr].action = action;
349}
350
351/* Tasklets */
352struct tasklet_head
353{
354 struct tasklet_struct *list;
355};
356
357/* Some compilers disobey section attribute on statics when not
358 initialized -- RR */
359static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
360static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
361
362void fastcall __tasklet_schedule(struct tasklet_struct *t)
363{
364 unsigned long flags;
365
366 local_irq_save(flags);
367 t->next = __get_cpu_var(tasklet_vec).list;
368 __get_cpu_var(tasklet_vec).list = t;
369 raise_softirq_irqoff(TASKLET_SOFTIRQ);
370 local_irq_restore(flags);
371}
372
373EXPORT_SYMBOL(__tasklet_schedule);
374
375void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
376{
377 unsigned long flags;
378
379 local_irq_save(flags);
380 t->next = __get_cpu_var(tasklet_hi_vec).list;
381 __get_cpu_var(tasklet_hi_vec).list = t;
382 raise_softirq_irqoff(HI_SOFTIRQ);
383 local_irq_restore(flags);
384}
385
386EXPORT_SYMBOL(__tasklet_hi_schedule);
387
388static void tasklet_action(struct softirq_action *a)
389{
390 struct tasklet_struct *list;
391
392 local_irq_disable();
393 list = __get_cpu_var(tasklet_vec).list;
394 __get_cpu_var(tasklet_vec).list = NULL;
395 local_irq_enable();
396
397 while (list) {
398 struct tasklet_struct *t = list;
399
400 list = list->next;
401
402 if (tasklet_trylock(t)) {
403 if (!atomic_read(&t->count)) {
404 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
405 BUG();
406 t->func(t->data);
407 tasklet_unlock(t);
408 continue;
409 }
410 tasklet_unlock(t);
411 }
412
413 local_irq_disable();
414 t->next = __get_cpu_var(tasklet_vec).list;
415 __get_cpu_var(tasklet_vec).list = t;
416 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
417 local_irq_enable();
418 }
419}
420
421static void tasklet_hi_action(struct softirq_action *a)
422{
423 struct tasklet_struct *list;
424
425 local_irq_disable();
426 list = __get_cpu_var(tasklet_hi_vec).list;
427 __get_cpu_var(tasklet_hi_vec).list = NULL;
428 local_irq_enable();
429
430 while (list) {
431 struct tasklet_struct *t = list;
432
433 list = list->next;
434
435 if (tasklet_trylock(t)) {
436 if (!atomic_read(&t->count)) {
437 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438 BUG();
439 t->func(t->data);
440 tasklet_unlock(t);
441 continue;
442 }
443 tasklet_unlock(t);
444 }
445
446 local_irq_disable();
447 t->next = __get_cpu_var(tasklet_hi_vec).list;
448 __get_cpu_var(tasklet_hi_vec).list = t;
449 __raise_softirq_irqoff(HI_SOFTIRQ);
450 local_irq_enable();
451 }
452}
453
454
455void tasklet_init(struct tasklet_struct *t,
456 void (*func)(unsigned long), unsigned long data)
457{
458 t->next = NULL;
459 t->state = 0;
460 atomic_set(&t->count, 0);
461 t->func = func;
462 t->data = data;
463}
464
465EXPORT_SYMBOL(tasklet_init);
466
467void tasklet_kill(struct tasklet_struct *t)
468{
469 if (in_interrupt())
470 printk("Attempt to kill tasklet from interrupt\n");
471
472 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
473 do
474 yield();
475 while (test_bit(TASKLET_STATE_SCHED, &t->state));
476 }
477 tasklet_unlock_wait(t);
478 clear_bit(TASKLET_STATE_SCHED, &t->state);
479}
480
481EXPORT_SYMBOL(tasklet_kill);
482
483void __init softirq_init(void)
484{
485 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
486 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
487}
488
489static int ksoftirqd(void * __bind_cpu)
490{
491 set_user_nice(current, 19);
492 current->flags |= PF_NOFREEZE;
493
494 set_current_state(TASK_INTERRUPTIBLE);
495
496 while (!kthread_should_stop()) {
497 preempt_disable();
498 if (!local_softirq_pending()) {
499 preempt_enable_no_resched();
500 schedule();
501 preempt_disable();
502 }
503
504 __set_current_state(TASK_RUNNING);
505
506 while (local_softirq_pending()) {
507 /* Preempt disable stops cpu going offline.
508 If already offline, we'll be on wrong CPU:
509 don't process */
510 if (cpu_is_offline((long)__bind_cpu))
511 goto wait_to_die;
512 do_softirq();
513 preempt_enable_no_resched();
514 cond_resched();
515 preempt_disable();
516 }
517 preempt_enable();
518 set_current_state(TASK_INTERRUPTIBLE);
519 }
520 __set_current_state(TASK_RUNNING);
521 return 0;
522
523wait_to_die:
524 preempt_enable();
525 /* Wait for kthread_stop */
526 set_current_state(TASK_INTERRUPTIBLE);
527 while (!kthread_should_stop()) {
528 schedule();
529 set_current_state(TASK_INTERRUPTIBLE);
530 }
531 __set_current_state(TASK_RUNNING);
532 return 0;
533}
534
535#ifdef CONFIG_HOTPLUG_CPU
536/*
537 * tasklet_kill_immediate is called to remove a tasklet which can already be
538 * scheduled for execution on @cpu.
539 *
540 * Unlike tasklet_kill, this function removes the tasklet
541 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
542 *
543 * When this function is called, @cpu must be in the CPU_DEAD state.
544 */
545void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
546{
547 struct tasklet_struct **i;
548
549 BUG_ON(cpu_online(cpu));
550 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
551
552 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
553 return;
554
555 /* CPU is dead, so no lock needed. */
556 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
557 if (*i == t) {
558 *i = t->next;
559 return;
560 }
561 }
562 BUG();
563}
564
565static void takeover_tasklets(unsigned int cpu)
566{
567 struct tasklet_struct **i;
568
569 /* CPU is dead, so no lock needed. */
570 local_irq_disable();
571
572 /* Find end, append list for that CPU. */
573 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
574 *i = per_cpu(tasklet_vec, cpu).list;
575 per_cpu(tasklet_vec, cpu).list = NULL;
576 raise_softirq_irqoff(TASKLET_SOFTIRQ);
577
578 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
579 *i = per_cpu(tasklet_hi_vec, cpu).list;
580 per_cpu(tasklet_hi_vec, cpu).list = NULL;
581 raise_softirq_irqoff(HI_SOFTIRQ);
582
583 local_irq_enable();
584}
585#endif /* CONFIG_HOTPLUG_CPU */
586
587static int __cpuinit cpu_callback(struct notifier_block *nfb,
588 unsigned long action,
589 void *hcpu)
590{
591 int hotcpu = (unsigned long)hcpu;
592 struct task_struct *p;
593
594 switch (action) {
595 case CPU_UP_PREPARE:
596 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
597 if (IS_ERR(p)) {
598 printk("ksoftirqd for %i failed\n", hotcpu);
599 return NOTIFY_BAD;
600 }
601 kthread_bind(p, hotcpu);
602 per_cpu(ksoftirqd, hotcpu) = p;
603 break;
604 case CPU_ONLINE:
605 wake_up_process(per_cpu(ksoftirqd, hotcpu));
606 break;
607#ifdef CONFIG_HOTPLUG_CPU
608 case CPU_UP_CANCELED:
609 if (!per_cpu(ksoftirqd, hotcpu))
610 break;
611 /* Unbind so it can run. Fall thru. */
612 kthread_bind(per_cpu(ksoftirqd, hotcpu),
613 any_online_cpu(cpu_online_map));
614 case CPU_DEAD:
615 p = per_cpu(ksoftirqd, hotcpu);
616 per_cpu(ksoftirqd, hotcpu) = NULL;
617 kthread_stop(p);
618 takeover_tasklets(hotcpu);
619 break;
620#endif /* CONFIG_HOTPLUG_CPU */
621 }
622 return NOTIFY_OK;
623}
624
625static struct notifier_block __cpuinitdata cpu_nfb = {
626 .notifier_call = cpu_callback
627};
628
629__init int spawn_ksoftirqd(void)
630{
631 void *cpu = (void *)(long)smp_processor_id();
632 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
633
634 BUG_ON(err == NOTIFY_BAD);
635 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
636 register_cpu_notifier(&cpu_nfb);
637 return 0;
638}
639
640#ifdef CONFIG_SMP
641/*
642 * Call a function on all processors
643 */
644int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
645{
646 int ret = 0;
647
648 preempt_disable();
649 ret = smp_call_function(func, info, retry, wait);
650 local_irq_disable();
651 func(info);
652 local_irq_enable();
653 preempt_enable();
654 return ret;
655}
656EXPORT_SYMBOL(on_each_cpu);
657#endif