Merge branch 'pm-cpufreq'
[firefly-linux-kernel-4.4.55.git] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/export.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/ftrace.h>
25 #include <linux/smp.h>
26 #include <linux/smpboot.h>
27 #include <linux/tick.h>
28
29 #define CREATE_TRACE_POINTS
30 #include <trace/events/irq.h>
31
32 /*
33    - No shared variables, all the data are CPU local.
34    - If a softirq needs serialization, let it serialize itself
35      by its own spinlocks.
36    - Even if softirq is serialized, only local cpu is marked for
37      execution. Hence, we get something sort of weak cpu binding.
38      Though it is still not clear, will it result in better locality
39      or will not.
40
41    Examples:
42    - NET RX softirq. It is multithreaded and does not require
43      any global serialization.
44    - NET TX softirq. It kicks software netdevice queues, hence
45      it is logically serialized per device, but this serialization
46      is invisible to common code.
47    - Tasklets: serialized wrt itself.
48  */
49
50 #ifndef __ARCH_IRQ_STAT
51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52 EXPORT_SYMBOL(irq_stat);
53 #endif
54
55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56
57 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
59 char *softirq_to_name[NR_SOFTIRQS] = {
60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
61         "TASKLET", "SCHED", "HRTIMER", "RCU"
62 };
63
64 /*
65  * we cannot loop indefinitely here to avoid userspace starvation,
66  * but we also don't want to introduce a worst case 1/HZ latency
67  * to the pending events, so lets the scheduler to balance
68  * the softirq load for us.
69  */
70 static void wakeup_softirqd(void)
71 {
72         /* Interrupts are disabled: no need to stop preemption */
73         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74
75         if (tsk && tsk->state != TASK_RUNNING)
76                 wake_up_process(tsk);
77 }
78
79 /*
80  * preempt_count and SOFTIRQ_OFFSET usage:
81  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
82  *   softirq processing.
83  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
84  *   on local_bh_disable or local_bh_enable.
85  * This lets us distinguish between whether we are currently processing
86  * softirq and whether we just have bh disabled.
87  */
88
89 /*
90  * This one is for softirq.c-internal use,
91  * where hardirqs are disabled legitimately:
92  */
93 #ifdef CONFIG_TRACE_IRQFLAGS
94 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
95 {
96         unsigned long flags;
97
98         WARN_ON_ONCE(in_irq());
99
100         raw_local_irq_save(flags);
101         /*
102          * The preempt tracer hooks into preempt_count_add and will break
103          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
104          * is set and before current->softirq_enabled is cleared.
105          * We must manually increment preempt_count here and manually
106          * call the trace_preempt_off later.
107          */
108         __preempt_count_add(cnt);
109         /*
110          * Were softirqs turned off above:
111          */
112         if (softirq_count() == cnt)
113                 trace_softirqs_off(ip);
114         raw_local_irq_restore(flags);
115
116         if (preempt_count() == cnt)
117                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
118 }
119 #else /* !CONFIG_TRACE_IRQFLAGS */
120 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
121 {
122         preempt_count_add(cnt);
123         barrier();
124 }
125 #endif /* CONFIG_TRACE_IRQFLAGS */
126
127 void local_bh_disable(void)
128 {
129         __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
130 }
131
132 EXPORT_SYMBOL(local_bh_disable);
133
134 static void __local_bh_enable(unsigned int cnt)
135 {
136         WARN_ON_ONCE(!irqs_disabled());
137
138         if (softirq_count() == cnt)
139                 trace_softirqs_on(_RET_IP_);
140         preempt_count_sub(cnt);
141 }
142
143 /*
144  * Special-case - softirqs can safely be enabled in
145  * cond_resched_softirq(), or by __do_softirq(),
146  * without processing still-pending softirqs:
147  */
148 void _local_bh_enable(void)
149 {
150         WARN_ON_ONCE(in_irq());
151         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
152 }
153
154 EXPORT_SYMBOL(_local_bh_enable);
155
156 static inline void _local_bh_enable_ip(unsigned long ip)
157 {
158         WARN_ON_ONCE(in_irq() || irqs_disabled());
159 #ifdef CONFIG_TRACE_IRQFLAGS
160         local_irq_disable();
161 #endif
162         /*
163          * Are softirqs going to be turned on now:
164          */
165         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
166                 trace_softirqs_on(ip);
167         /*
168          * Keep preemption disabled until we are done with
169          * softirq processing:
170          */
171         preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
172
173         if (unlikely(!in_interrupt() && local_softirq_pending())) {
174                 /*
175                  * Run softirq if any pending. And do it in its own stack
176                  * as we may be calling this deep in a task call stack already.
177                  */
178                 do_softirq();
179         }
180
181         preempt_count_dec();
182 #ifdef CONFIG_TRACE_IRQFLAGS
183         local_irq_enable();
184 #endif
185         preempt_check_resched();
186 }
187
188 void local_bh_enable(void)
189 {
190         _local_bh_enable_ip(_RET_IP_);
191 }
192 EXPORT_SYMBOL(local_bh_enable);
193
194 void local_bh_enable_ip(unsigned long ip)
195 {
196         _local_bh_enable_ip(ip);
197 }
198 EXPORT_SYMBOL(local_bh_enable_ip);
199
200 /*
201  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
202  * but break the loop if need_resched() is set or after 2 ms.
203  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
204  * certain cases, such as stop_machine(), jiffies may cease to
205  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
206  * well to make sure we eventually return from this method.
207  *
208  * These limits have been established via experimentation.
209  * The two things to balance is latency against fairness -
210  * we want to handle softirqs as soon as possible, but they
211  * should not be able to lock up the box.
212  */
213 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
214 #define MAX_SOFTIRQ_RESTART 10
215
216 asmlinkage void __do_softirq(void)
217 {
218         struct softirq_action *h;
219         __u32 pending;
220         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
221         int cpu;
222         unsigned long old_flags = current->flags;
223         int max_restart = MAX_SOFTIRQ_RESTART;
224
225         /*
226          * Mask out PF_MEMALLOC s current task context is borrowed for the
227          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
228          * again if the socket is related to swap
229          */
230         current->flags &= ~PF_MEMALLOC;
231
232         pending = local_softirq_pending();
233         account_irq_enter_time(current);
234
235         __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
236         lockdep_softirq_enter();
237
238         cpu = smp_processor_id();
239 restart:
240         /* Reset the pending bitmask before enabling irqs */
241         set_softirq_pending(0);
242
243         local_irq_enable();
244
245         h = softirq_vec;
246
247         do {
248                 if (pending & 1) {
249                         unsigned int vec_nr = h - softirq_vec;
250                         int prev_count = preempt_count();
251
252                         kstat_incr_softirqs_this_cpu(vec_nr);
253
254                         trace_softirq_entry(vec_nr);
255                         h->action(h);
256                         trace_softirq_exit(vec_nr);
257                         if (unlikely(prev_count != preempt_count())) {
258                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
259                                        "with preempt_count %08x,"
260                                        " exited with %08x?\n", vec_nr,
261                                        softirq_to_name[vec_nr], h->action,
262                                        prev_count, preempt_count());
263                                 preempt_count_set(prev_count);
264                         }
265
266                         rcu_bh_qs(cpu);
267                 }
268                 h++;
269                 pending >>= 1;
270         } while (pending);
271
272         local_irq_disable();
273
274         pending = local_softirq_pending();
275         if (pending) {
276                 if (time_before(jiffies, end) && !need_resched() &&
277                     --max_restart)
278                         goto restart;
279
280                 wakeup_softirqd();
281         }
282
283         lockdep_softirq_exit();
284
285         account_irq_exit_time(current);
286         __local_bh_enable(SOFTIRQ_OFFSET);
287         WARN_ON_ONCE(in_interrupt());
288         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
289 }
290
291
292
293 asmlinkage void do_softirq(void)
294 {
295         __u32 pending;
296         unsigned long flags;
297
298         if (in_interrupt())
299                 return;
300
301         local_irq_save(flags);
302
303         pending = local_softirq_pending();
304
305         if (pending)
306                 do_softirq_own_stack();
307
308         local_irq_restore(flags);
309 }
310
311 /*
312  * Enter an interrupt context.
313  */
314 void irq_enter(void)
315 {
316         int cpu = smp_processor_id();
317
318         rcu_irq_enter();
319         if (is_idle_task(current) && !in_interrupt()) {
320                 /*
321                  * Prevent raise_softirq from needlessly waking up ksoftirqd
322                  * here, as softirq will be serviced on return from interrupt.
323                  */
324                 local_bh_disable();
325                 tick_check_idle(cpu);
326                 _local_bh_enable();
327         }
328
329         __irq_enter();
330 }
331
332 static inline void invoke_softirq(void)
333 {
334         if (!force_irqthreads) {
335 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
336                 /*
337                  * We can safely execute softirq on the current stack if
338                  * it is the irq stack, because it should be near empty
339                  * at this stage.
340                  */
341                 __do_softirq();
342 #else
343                 /*
344                  * Otherwise, irq_exit() is called on the task stack that can
345                  * be potentially deep already. So call softirq in its own stack
346                  * to prevent from any overrun.
347                  */
348                 do_softirq_own_stack();
349 #endif
350         } else {
351                 wakeup_softirqd();
352         }
353 }
354
355 static inline void tick_irq_exit(void)
356 {
357 #ifdef CONFIG_NO_HZ_COMMON
358         int cpu = smp_processor_id();
359
360         /* Make sure that timer wheel updates are propagated */
361         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
362                 if (!in_interrupt())
363                         tick_nohz_irq_exit();
364         }
365 #endif
366 }
367
368 /*
369  * Exit an interrupt context. Process softirqs if needed and possible:
370  */
371 void irq_exit(void)
372 {
373 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
374         local_irq_disable();
375 #else
376         WARN_ON_ONCE(!irqs_disabled());
377 #endif
378
379         account_irq_exit_time(current);
380         trace_hardirq_exit();
381         preempt_count_sub(HARDIRQ_OFFSET);
382         if (!in_interrupt() && local_softirq_pending())
383                 invoke_softirq();
384
385         tick_irq_exit();
386         rcu_irq_exit();
387 }
388
389 /*
390  * This function must run with irqs disabled!
391  */
392 inline void raise_softirq_irqoff(unsigned int nr)
393 {
394         __raise_softirq_irqoff(nr);
395
396         /*
397          * If we're in an interrupt or softirq, we're done
398          * (this also catches softirq-disabled code). We will
399          * actually run the softirq once we return from
400          * the irq or softirq.
401          *
402          * Otherwise we wake up ksoftirqd to make sure we
403          * schedule the softirq soon.
404          */
405         if (!in_interrupt())
406                 wakeup_softirqd();
407 }
408
409 void raise_softirq(unsigned int nr)
410 {
411         unsigned long flags;
412
413         local_irq_save(flags);
414         raise_softirq_irqoff(nr);
415         local_irq_restore(flags);
416 }
417
418 void __raise_softirq_irqoff(unsigned int nr)
419 {
420         trace_softirq_raise(nr);
421         or_softirq_pending(1UL << nr);
422 }
423
424 void open_softirq(int nr, void (*action)(struct softirq_action *))
425 {
426         softirq_vec[nr].action = action;
427 }
428
429 /*
430  * Tasklets
431  */
432 struct tasklet_head
433 {
434         struct tasklet_struct *head;
435         struct tasklet_struct **tail;
436 };
437
438 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
439 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
440
441 void __tasklet_schedule(struct tasklet_struct *t)
442 {
443         unsigned long flags;
444
445         local_irq_save(flags);
446         t->next = NULL;
447         *__this_cpu_read(tasklet_vec.tail) = t;
448         __this_cpu_write(tasklet_vec.tail, &(t->next));
449         raise_softirq_irqoff(TASKLET_SOFTIRQ);
450         local_irq_restore(flags);
451 }
452
453 EXPORT_SYMBOL(__tasklet_schedule);
454
455 void __tasklet_hi_schedule(struct tasklet_struct *t)
456 {
457         unsigned long flags;
458
459         local_irq_save(flags);
460         t->next = NULL;
461         *__this_cpu_read(tasklet_hi_vec.tail) = t;
462         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
463         raise_softirq_irqoff(HI_SOFTIRQ);
464         local_irq_restore(flags);
465 }
466
467 EXPORT_SYMBOL(__tasklet_hi_schedule);
468
469 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
470 {
471         BUG_ON(!irqs_disabled());
472
473         t->next = __this_cpu_read(tasklet_hi_vec.head);
474         __this_cpu_write(tasklet_hi_vec.head, t);
475         __raise_softirq_irqoff(HI_SOFTIRQ);
476 }
477
478 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
479
480 static void tasklet_action(struct softirq_action *a)
481 {
482         struct tasklet_struct *list;
483
484         local_irq_disable();
485         list = __this_cpu_read(tasklet_vec.head);
486         __this_cpu_write(tasklet_vec.head, NULL);
487         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
488         local_irq_enable();
489
490         while (list) {
491                 struct tasklet_struct *t = list;
492
493                 list = list->next;
494
495                 if (tasklet_trylock(t)) {
496                         if (!atomic_read(&t->count)) {
497                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
498                                         BUG();
499                                 t->func(t->data);
500                                 tasklet_unlock(t);
501                                 continue;
502                         }
503                         tasklet_unlock(t);
504                 }
505
506                 local_irq_disable();
507                 t->next = NULL;
508                 *__this_cpu_read(tasklet_vec.tail) = t;
509                 __this_cpu_write(tasklet_vec.tail, &(t->next));
510                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
511                 local_irq_enable();
512         }
513 }
514
515 static void tasklet_hi_action(struct softirq_action *a)
516 {
517         struct tasklet_struct *list;
518
519         local_irq_disable();
520         list = __this_cpu_read(tasklet_hi_vec.head);
521         __this_cpu_write(tasklet_hi_vec.head, NULL);
522         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
523         local_irq_enable();
524
525         while (list) {
526                 struct tasklet_struct *t = list;
527
528                 list = list->next;
529
530                 if (tasklet_trylock(t)) {
531                         if (!atomic_read(&t->count)) {
532                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
533                                         BUG();
534                                 t->func(t->data);
535                                 tasklet_unlock(t);
536                                 continue;
537                         }
538                         tasklet_unlock(t);
539                 }
540
541                 local_irq_disable();
542                 t->next = NULL;
543                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
544                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
545                 __raise_softirq_irqoff(HI_SOFTIRQ);
546                 local_irq_enable();
547         }
548 }
549
550
551 void tasklet_init(struct tasklet_struct *t,
552                   void (*func)(unsigned long), unsigned long data)
553 {
554         t->next = NULL;
555         t->state = 0;
556         atomic_set(&t->count, 0);
557         t->func = func;
558         t->data = data;
559 }
560
561 EXPORT_SYMBOL(tasklet_init);
562
563 void tasklet_kill(struct tasklet_struct *t)
564 {
565         if (in_interrupt())
566                 printk("Attempt to kill tasklet from interrupt\n");
567
568         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
569                 do {
570                         yield();
571                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
572         }
573         tasklet_unlock_wait(t);
574         clear_bit(TASKLET_STATE_SCHED, &t->state);
575 }
576
577 EXPORT_SYMBOL(tasklet_kill);
578
579 /*
580  * tasklet_hrtimer
581  */
582
583 /*
584  * The trampoline is called when the hrtimer expires. It schedules a tasklet
585  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
586  * hrtimer callback, but from softirq context.
587  */
588 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
589 {
590         struct tasklet_hrtimer *ttimer =
591                 container_of(timer, struct tasklet_hrtimer, timer);
592
593         tasklet_hi_schedule(&ttimer->tasklet);
594         return HRTIMER_NORESTART;
595 }
596
597 /*
598  * Helper function which calls the hrtimer callback from
599  * tasklet/softirq context
600  */
601 static void __tasklet_hrtimer_trampoline(unsigned long data)
602 {
603         struct tasklet_hrtimer *ttimer = (void *)data;
604         enum hrtimer_restart restart;
605
606         restart = ttimer->function(&ttimer->timer);
607         if (restart != HRTIMER_NORESTART)
608                 hrtimer_restart(&ttimer->timer);
609 }
610
611 /**
612  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
613  * @ttimer:      tasklet_hrtimer which is initialized
614  * @function:    hrtimer callback function which gets called from softirq context
615  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
616  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
617  */
618 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
619                           enum hrtimer_restart (*function)(struct hrtimer *),
620                           clockid_t which_clock, enum hrtimer_mode mode)
621 {
622         hrtimer_init(&ttimer->timer, which_clock, mode);
623         ttimer->timer.function = __hrtimer_tasklet_trampoline;
624         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
625                      (unsigned long)ttimer);
626         ttimer->function = function;
627 }
628 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
629
630 /*
631  * Remote softirq bits
632  */
633
634 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
635 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
636
637 static void __local_trigger(struct call_single_data *cp, int softirq)
638 {
639         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
640
641         list_add_tail(&cp->list, head);
642
643         /* Trigger the softirq only if the list was previously empty.  */
644         if (head->next == &cp->list)
645                 raise_softirq_irqoff(softirq);
646 }
647
648 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
649 static void remote_softirq_receive(void *data)
650 {
651         struct call_single_data *cp = data;
652         unsigned long flags;
653         int softirq;
654
655         softirq = *(int *)cp->info;
656         local_irq_save(flags);
657         __local_trigger(cp, softirq);
658         local_irq_restore(flags);
659 }
660
661 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
662 {
663         if (cpu_online(cpu)) {
664                 cp->func = remote_softirq_receive;
665                 cp->info = &softirq;
666                 cp->flags = 0;
667
668                 __smp_call_function_single(cpu, cp, 0);
669                 return 0;
670         }
671         return 1;
672 }
673 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
674 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
675 {
676         return 1;
677 }
678 #endif
679
680 /**
681  * __send_remote_softirq - try to schedule softirq work on a remote cpu
682  * @cp: private SMP call function data area
683  * @cpu: the remote cpu
684  * @this_cpu: the currently executing cpu
685  * @softirq: the softirq for the work
686  *
687  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
688  * done, the work is instead queued up on the local cpu.
689  *
690  * Interrupts must be disabled.
691  */
692 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
693 {
694         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
695                 __local_trigger(cp, softirq);
696 }
697 EXPORT_SYMBOL(__send_remote_softirq);
698
699 /**
700  * send_remote_softirq - try to schedule softirq work on a remote cpu
701  * @cp: private SMP call function data area
702  * @cpu: the remote cpu
703  * @softirq: the softirq for the work
704  *
705  * Like __send_remote_softirq except that disabling interrupts and
706  * computing the current cpu is done for the caller.
707  */
708 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
709 {
710         unsigned long flags;
711         int this_cpu;
712
713         local_irq_save(flags);
714         this_cpu = smp_processor_id();
715         __send_remote_softirq(cp, cpu, this_cpu, softirq);
716         local_irq_restore(flags);
717 }
718 EXPORT_SYMBOL(send_remote_softirq);
719
720 static int remote_softirq_cpu_notify(struct notifier_block *self,
721                                                unsigned long action, void *hcpu)
722 {
723         /*
724          * If a CPU goes away, splice its entries to the current CPU
725          * and trigger a run of the softirq
726          */
727         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
728                 int cpu = (unsigned long) hcpu;
729                 int i;
730
731                 local_irq_disable();
732                 for (i = 0; i < NR_SOFTIRQS; i++) {
733                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
734                         struct list_head *local_head;
735
736                         if (list_empty(head))
737                                 continue;
738
739                         local_head = &__get_cpu_var(softirq_work_list[i]);
740                         list_splice_init(head, local_head);
741                         raise_softirq_irqoff(i);
742                 }
743                 local_irq_enable();
744         }
745
746         return NOTIFY_OK;
747 }
748
749 static struct notifier_block remote_softirq_cpu_notifier = {
750         .notifier_call  = remote_softirq_cpu_notify,
751 };
752
753 void __init softirq_init(void)
754 {
755         int cpu;
756
757         for_each_possible_cpu(cpu) {
758                 int i;
759
760                 per_cpu(tasklet_vec, cpu).tail =
761                         &per_cpu(tasklet_vec, cpu).head;
762                 per_cpu(tasklet_hi_vec, cpu).tail =
763                         &per_cpu(tasklet_hi_vec, cpu).head;
764                 for (i = 0; i < NR_SOFTIRQS; i++)
765                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
766         }
767
768         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
769
770         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
771         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
772 }
773
774 static int ksoftirqd_should_run(unsigned int cpu)
775 {
776         return local_softirq_pending();
777 }
778
779 static void run_ksoftirqd(unsigned int cpu)
780 {
781         local_irq_disable();
782         if (local_softirq_pending()) {
783                 /*
784                  * We can safely run softirq on inline stack, as we are not deep
785                  * in the task stack here.
786                  */
787                 __do_softirq();
788                 rcu_note_context_switch(cpu);
789                 local_irq_enable();
790                 cond_resched();
791                 return;
792         }
793         local_irq_enable();
794 }
795
796 #ifdef CONFIG_HOTPLUG_CPU
797 /*
798  * tasklet_kill_immediate is called to remove a tasklet which can already be
799  * scheduled for execution on @cpu.
800  *
801  * Unlike tasklet_kill, this function removes the tasklet
802  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
803  *
804  * When this function is called, @cpu must be in the CPU_DEAD state.
805  */
806 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
807 {
808         struct tasklet_struct **i;
809
810         BUG_ON(cpu_online(cpu));
811         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
812
813         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
814                 return;
815
816         /* CPU is dead, so no lock needed. */
817         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
818                 if (*i == t) {
819                         *i = t->next;
820                         /* If this was the tail element, move the tail ptr */
821                         if (*i == NULL)
822                                 per_cpu(tasklet_vec, cpu).tail = i;
823                         return;
824                 }
825         }
826         BUG();
827 }
828
829 static void takeover_tasklets(unsigned int cpu)
830 {
831         /* CPU is dead, so no lock needed. */
832         local_irq_disable();
833
834         /* Find end, append list for that CPU. */
835         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
836                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
837                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
838                 per_cpu(tasklet_vec, cpu).head = NULL;
839                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
840         }
841         raise_softirq_irqoff(TASKLET_SOFTIRQ);
842
843         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
844                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
845                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
846                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
847                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
848         }
849         raise_softirq_irqoff(HI_SOFTIRQ);
850
851         local_irq_enable();
852 }
853 #endif /* CONFIG_HOTPLUG_CPU */
854
855 static int cpu_callback(struct notifier_block *nfb,
856                                   unsigned long action,
857                                   void *hcpu)
858 {
859         switch (action) {
860 #ifdef CONFIG_HOTPLUG_CPU
861         case CPU_DEAD:
862         case CPU_DEAD_FROZEN:
863                 takeover_tasklets((unsigned long)hcpu);
864                 break;
865 #endif /* CONFIG_HOTPLUG_CPU */
866         }
867         return NOTIFY_OK;
868 }
869
870 static struct notifier_block cpu_nfb = {
871         .notifier_call = cpu_callback
872 };
873
874 static struct smp_hotplug_thread softirq_threads = {
875         .store                  = &ksoftirqd,
876         .thread_should_run      = ksoftirqd_should_run,
877         .thread_fn              = run_ksoftirqd,
878         .thread_comm            = "ksoftirqd/%u",
879 };
880
881 static __init int spawn_ksoftirqd(void)
882 {
883         register_cpu_notifier(&cpu_nfb);
884
885         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
886
887         return 0;
888 }
889 early_initcall(spawn_ksoftirqd);
890
891 /*
892  * [ These __weak aliases are kept in a separate compilation unit, so that
893  *   GCC does not inline them incorrectly. ]
894  */
895
896 int __init __weak early_irq_init(void)
897 {
898         return 0;
899 }
900
901 int __init __weak arch_probe_nr_irqs(void)
902 {
903         return NR_IRQS_LEGACY;
904 }
905
906 int __init __weak arch_early_irq_init(void)
907 {
908         return 0;
909 }