kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  */
  10
  11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28 #include <linux/irq.h>
  29
  30 #define CREATE_TRACE_POINTS
  31 #include <trace/events/irq.h>
  32
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 const char * const softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into preempt_count_add and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         __preempt_count_add(cnt);
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 EXPORT_SYMBOL(__local_bh_disable_ip);
 121 #endif /* CONFIG_TRACE_IRQFLAGS */
 122
 123 static void __local_bh_enable(unsigned int cnt)
 124 {
 125         WARN_ON_ONCE(!irqs_disabled());
 126
 127         if (softirq_count() == (cnt & SOFTIRQ_MASK))
 128                 trace_softirqs_on(_RET_IP_);
 129         preempt_count_sub(cnt);
 130 }
 131
 132 /*
 133  * Special-case - softirqs can safely be enabled in
 134  * cond_resched_softirq(), or by __do_softirq(),
 135  * without processing still-pending softirqs:
 136  */
 137 void _local_bh_enable(void)
 138 {
 139         WARN_ON_ONCE(in_irq());
 140         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 141 }
 142 EXPORT_SYMBOL(_local_bh_enable);
 143
 144 void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 145 {
 146         WARN_ON_ONCE(in_irq() || irqs_disabled());
 147 #ifdef CONFIG_TRACE_IRQFLAGS
 148         local_irq_disable();
 149 #endif
 150         /*
 151          * Are softirqs going to be turned on now:
 152          */
 153         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 154                 trace_softirqs_on(ip);
 155         /*
 156          * Keep preemption disabled until we are done with
 157          * softirq processing:
 158          */
 159         preempt_count_sub(cnt - 1);
 160
 161         if (unlikely(!in_interrupt() && local_softirq_pending())) {
 162                 /*
 163                  * Run softirq if any pending. And do it in its own stack
 164                  * as we may be calling this deep in a task call stack already.
 165                  */
 166                 do_softirq();
 167         }
 168
 169         preempt_count_dec();
 170 #ifdef CONFIG_TRACE_IRQFLAGS
 171         local_irq_enable();
 172 #endif
 173         preempt_check_resched();
 174 }
 175 EXPORT_SYMBOL(__local_bh_enable_ip);
 176
 177 /*
 178  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 179  * but break the loop if need_resched() is set or after 2 ms.
 180  * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 181  * certain cases, such as stop_machine(), jiffies may cease to
 182  * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 183  * well to make sure we eventually return from this method.
 184  *
 185  * These limits have been established via experimentation.
 186  * The two things to balance is latency against fairness -
 187  * we want to handle softirqs as soon as possible, but they
 188  * should not be able to lock up the box.
 189  */
 190 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 191 #define MAX_SOFTIRQ_RESTART 10
 192
 193 #ifdef CONFIG_TRACE_IRQFLAGS
 194 /*
 195  * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 196  * to keep the lockdep irq context tracking as tight as possible in order to
 197  * not miss-qualify lock contexts and miss possible deadlocks.
 198  */
 199
 200 static inline bool lockdep_softirq_start(void)
 201 {
 202         bool in_hardirq = false;
 203
 204         if (trace_hardirq_context(current)) {
 205                 in_hardirq = true;
 206                 trace_hardirq_exit();
 207         }
 208
 209         lockdep_softirq_enter();
 210
 211         return in_hardirq;
 212 }
 213
 214 static inline void lockdep_softirq_end(bool in_hardirq)
 215 {
 216         lockdep_softirq_exit();
 217
 218         if (in_hardirq)
 219                 trace_hardirq_enter();
 220 }
 221 #else
 222 static inline bool lockdep_softirq_start(void) { return false; }
 223 static inline void lockdep_softirq_end(bool in_hardirq) { }
 224 #endif
 225
 226 asmlinkage __visible void __do_softirq(void)
 227 {
 228         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 229         unsigned long old_flags = current->flags;
 230         int max_restart = MAX_SOFTIRQ_RESTART;
 231         struct softirq_action *h;
 232         bool in_hardirq;
 233         __u32 pending;
 234         int softirq_bit;
 235
 236         /*
 237          * Mask out PF_MEMALLOC s current task context is borrowed for the
 238          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 239          * again if the socket is related to swap
 240          */
 241         current->flags &= ~PF_MEMALLOC;
 242
 243         pending = local_softirq_pending();
 244         account_irq_enter_time(current);
 245
 246         __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 247         in_hardirq = lockdep_softirq_start();
 248
 249 restart:
 250         /* Reset the pending bitmask before enabling irqs */
 251         set_softirq_pending(0);
 252
 253         local_irq_enable();
 254
 255         h = softirq_vec;
 256
 257         while ((softirq_bit = ffs(pending))) {
 258                 unsigned int vec_nr;
 259                 int prev_count;
 260
 261                 h += softirq_bit - 1;
 262
 263                 vec_nr = h - softirq_vec;
 264                 prev_count = preempt_count();
 265
 266                 kstat_incr_softirqs_this_cpu(vec_nr);
 267
 268                 trace_softirq_entry(vec_nr);
 269                 h->action(h);
 270                 trace_softirq_exit(vec_nr);
 271                 if (unlikely(prev_count != preempt_count())) {
 272                         pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
 273                                vec_nr, softirq_to_name[vec_nr], h->action,
 274                                prev_count, preempt_count());
 275                         preempt_count_set(prev_count);
 276                 }
 277                 h++;
 278                 pending >>= softirq_bit;
 279         }
 280
 281         rcu_bh_qs(smp_processor_id());
 282         local_irq_disable();
 283
 284         pending = local_softirq_pending();
 285         if (pending) {
 286                 if (time_before(jiffies, end) && !need_resched() &&
 287                     --max_restart)
 288                         goto restart;
 289
 290                 wakeup_softirqd();
 291         }
 292
 293         lockdep_softirq_end(in_hardirq);
 294         account_irq_exit_time(current);
 295         __local_bh_enable(SOFTIRQ_OFFSET);
 296         WARN_ON_ONCE(in_interrupt());
 297         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 298 }
 299
 300 asmlinkage __visible void do_softirq(void)
 301 {
 302         __u32 pending;
 303         unsigned long flags;
 304
 305         if (in_interrupt())
 306                 return;
 307
 308         local_irq_save(flags);
 309
 310         pending = local_softirq_pending();
 311
 312         if (pending)
 313                 do_softirq_own_stack();
 314
 315         local_irq_restore(flags);
 316 }
 317
 318 /*
 319  * Enter an interrupt context.
 320  */
 321 void irq_enter(void)
 322 {
 323         rcu_irq_enter();
 324         if (is_idle_task(current) && !in_interrupt()) {
 325                 /*
 326                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 327                  * here, as softirq will be serviced on return from interrupt.
 328                  */
 329                 local_bh_disable();
 330                 tick_irq_enter();
 331                 _local_bh_enable();
 332         }
 333
 334         __irq_enter();
 335 }
 336
 337 static inline void invoke_softirq(void)
 338 {
 339         if (!force_irqthreads) {
 340 #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 341                 /*
 342                  * We can safely execute softirq on the current stack if
 343                  * it is the irq stack, because it should be near empty
 344                  * at this stage.
 345                  */
 346                 __do_softirq();
 347 #else
 348                 /*
 349                  * Otherwise, irq_exit() is called on the task stack that can
 350                  * be potentially deep already. So call softirq in its own stack
 351                  * to prevent from any overrun.
 352                  */
 353                 do_softirq_own_stack();
 354 #endif
 355         } else {
 356                 wakeup_softirqd();
 357         }
 358 }
 359
 360 static inline void tick_irq_exit(void)
 361 {
 362 #ifdef CONFIG_NO_HZ_COMMON
 363         int cpu = smp_processor_id();
 364
 365         /* Make sure that timer wheel updates are propagated */
 366         if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 367                 if (!in_interrupt())
 368                         tick_nohz_irq_exit();
 369         }
 370 #endif
 371 }
 372
 373 /*
 374  * Exit an interrupt context. Process softirqs if needed and possible:
 375  */
 376 void irq_exit(void)
 377 {
 378 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 379         local_irq_disable();
 380 #else
 381         WARN_ON_ONCE(!irqs_disabled());
 382 #endif
 383
 384         account_irq_exit_time(current);
 385         preempt_count_sub(HARDIRQ_OFFSET);
 386         if (!in_interrupt() && local_softirq_pending())
 387                 invoke_softirq();
 388
 389         tick_irq_exit();
 390         rcu_irq_exit();
 391         trace_hardirq_exit(); /* must be last! */
 392 }
 393
 394 /*
 395  * This function must run with irqs disabled!
 396  */
 397 inline void raise_softirq_irqoff(unsigned int nr)
 398 {
 399         __raise_softirq_irqoff(nr);
 400
 401         /*
 402          * If we're in an interrupt or softirq, we're done
 403          * (this also catches softirq-disabled code). We will
 404          * actually run the softirq once we return from
 405          * the irq or softirq.
 406          *
 407          * Otherwise we wake up ksoftirqd to make sure we
 408          * schedule the softirq soon.
 409          */
 410         if (!in_interrupt())
 411                 wakeup_softirqd();
 412 }
 413
 414 void raise_softirq(unsigned int nr)
 415 {
 416         unsigned long flags;
 417
 418         local_irq_save(flags);
 419         raise_softirq_irqoff(nr);
 420         local_irq_restore(flags);
 421 }
 422
 423 void __raise_softirq_irqoff(unsigned int nr)
 424 {
 425         trace_softirq_raise(nr);
 426         or_softirq_pending(1UL << nr);
 427 }
 428
 429 void open_softirq(int nr, void (*action)(struct softirq_action *))
 430 {
 431         softirq_vec[nr].action = action;
 432 }
 433
 434 /*
 435  * Tasklets
 436  */
 437 struct tasklet_head {
 438         struct tasklet_struct *head;
 439         struct tasklet_struct **tail;
 440 };
 441
 442 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 443 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 444
 445 void __tasklet_schedule(struct tasklet_struct *t)
 446 {
 447         unsigned long flags;
 448
 449         local_irq_save(flags);
 450         t->next = NULL;
 451         *__this_cpu_read(tasklet_vec.tail) = t;
 452         __this_cpu_write(tasklet_vec.tail, &(t->next));
 453         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 454         local_irq_restore(flags);
 455 }
 456 EXPORT_SYMBOL(__tasklet_schedule);
 457
 458 void __tasklet_hi_schedule(struct tasklet_struct *t)
 459 {
 460         unsigned long flags;
 461
 462         local_irq_save(flags);
 463         t->next = NULL;
 464         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 465         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 466         raise_softirq_irqoff(HI_SOFTIRQ);
 467         local_irq_restore(flags);
 468 }
 469 EXPORT_SYMBOL(__tasklet_hi_schedule);
 470
 471 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 472 {
 473         BUG_ON(!irqs_disabled());
 474
 475         t->next = __this_cpu_read(tasklet_hi_vec.head);
 476         __this_cpu_write(tasklet_hi_vec.head, t);
 477         __raise_softirq_irqoff(HI_SOFTIRQ);
 478 }
 479 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 480
 481 static void tasklet_action(struct softirq_action *a)
 482 {
 483         struct tasklet_struct *list;
 484
 485         local_irq_disable();
 486         list = __this_cpu_read(tasklet_vec.head);
 487         __this_cpu_write(tasklet_vec.head, NULL);
 488         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 489         local_irq_enable();
 490
 491         while (list) {
 492                 struct tasklet_struct *t = list;
 493
 494                 list = list->next;
 495
 496                 if (tasklet_trylock(t)) {
 497                         if (!atomic_read(&t->count)) {
 498                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
 499                                                         &t->state))
 500                                         BUG();
 501                                 t->func(t->data);
 502                                 tasklet_unlock(t);
 503                                 continue;
 504                         }
 505                         tasklet_unlock(t);
 506                 }
 507
 508                 local_irq_disable();
 509                 t->next = NULL;
 510                 *__this_cpu_read(tasklet_vec.tail) = t;
 511                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 512                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 513                 local_irq_enable();
 514         }
 515 }
 516
 517 static void tasklet_hi_action(struct softirq_action *a)
 518 {
 519         struct tasklet_struct *list;
 520
 521         local_irq_disable();
 522         list = __this_cpu_read(tasklet_hi_vec.head);
 523         __this_cpu_write(tasklet_hi_vec.head, NULL);
 524         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 525         local_irq_enable();
 526
 527         while (list) {
 528                 struct tasklet_struct *t = list;
 529
 530                 list = list->next;
 531
 532                 if (tasklet_trylock(t)) {
 533                         if (!atomic_read(&t->count)) {
 534                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
 535                                                         &t->state))
 536                                         BUG();
 537                                 t->func(t->data);
 538                                 tasklet_unlock(t);
 539                                 continue;
 540                         }
 541                         tasklet_unlock(t);
 542                 }
 543
 544                 local_irq_disable();
 545                 t->next = NULL;
 546                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 547                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 548                 __raise_softirq_irqoff(HI_SOFTIRQ);
 549                 local_irq_enable();
 550         }
 551 }
 552
 553 void tasklet_init(struct tasklet_struct *t,
 554                   void (*func)(unsigned long), unsigned long data)
 555 {
 556         t->next = NULL;
 557         t->state = 0;
 558         atomic_set(&t->count, 0);
 559         t->func = func;
 560         t->data = data;
 561 }
 562 EXPORT_SYMBOL(tasklet_init);
 563
 564 void tasklet_kill(struct tasklet_struct *t)
 565 {
 566         if (in_interrupt())
 567                 pr_notice("Attempt to kill tasklet from interrupt\n");
 568
 569         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 570                 do {
 571                         yield();
 572                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 573         }
 574         tasklet_unlock_wait(t);
 575         clear_bit(TASKLET_STATE_SCHED, &t->state);
 576 }
 577 EXPORT_SYMBOL(tasklet_kill);
 578
 579 /*
 580  * tasklet_hrtimer
 581  */
 582
 583 /*
 584  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 585  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 586  * hrtimer callback, but from softirq context.
 587  */
 588 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 589 {
 590         struct tasklet_hrtimer *ttimer =
 591                 container_of(timer, struct tasklet_hrtimer, timer);
 592
 593         tasklet_hi_schedule(&ttimer->tasklet);
 594         return HRTIMER_NORESTART;
 595 }
 596
 597 /*
 598  * Helper function which calls the hrtimer callback from
 599  * tasklet/softirq context
 600  */
 601 static void __tasklet_hrtimer_trampoline(unsigned long data)
 602 {
 603         struct tasklet_hrtimer *ttimer = (void *)data;
 604         enum hrtimer_restart restart;
 605
 606         restart = ttimer->function(&ttimer->timer);
 607         if (restart != HRTIMER_NORESTART)
 608                 hrtimer_restart(&ttimer->timer);
 609 }
 610
 611 /**
 612  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 613  * @ttimer:      tasklet_hrtimer which is initialized
 614  * @function:    hrtimer callback function which gets called from softirq context
 615  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 616  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 617  */
 618 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 619                           enum hrtimer_restart (*function)(struct hrtimer *),
 620                           clockid_t which_clock, enum hrtimer_mode mode)
 621 {
 622         hrtimer_init(&ttimer->timer, which_clock, mode);
 623         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 624         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 625                      (unsigned long)ttimer);
 626         ttimer->function = function;
 627 }
 628 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 629
 630 void __init softirq_init(void)
 631 {
 632         int cpu;
 633
 634         for_each_possible_cpu(cpu) {
 635                 per_cpu(tasklet_vec, cpu).tail =
 636                         &per_cpu(tasklet_vec, cpu).head;
 637                 per_cpu(tasklet_hi_vec, cpu).tail =
 638                         &per_cpu(tasklet_hi_vec, cpu).head;
 639         }
 640
 641         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 642         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 643 }
 644
 645 static int ksoftirqd_should_run(unsigned int cpu)
 646 {
 647         return local_softirq_pending();
 648 }
 649
 650 static void run_ksoftirqd(unsigned int cpu)
 651 {
 652         local_irq_disable();
 653         if (local_softirq_pending()) {
 654                 /*
 655                  * We can safely run softirq on inline stack, as we are not deep
 656                  * in the task stack here.
 657                  */
 658                 __do_softirq();
 659                 rcu_note_context_switch(cpu);
 660                 local_irq_enable();
 661                 cond_resched();
 662                 return;
 663         }
 664         local_irq_enable();
 665 }
 666
 667 #ifdef CONFIG_HOTPLUG_CPU
 668 /*
 669  * tasklet_kill_immediate is called to remove a tasklet which can already be
 670  * scheduled for execution on @cpu.
 671  *
 672  * Unlike tasklet_kill, this function removes the tasklet
 673  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 674  *
 675  * When this function is called, @cpu must be in the CPU_DEAD state.
 676  */
 677 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 678 {
 679         struct tasklet_struct **i;
 680
 681         BUG_ON(cpu_online(cpu));
 682         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 683
 684         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 685                 return;
 686
 687         /* CPU is dead, so no lock needed. */
 688         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 689                 if (*i == t) {
 690                         *i = t->next;
 691                         /* If this was the tail element, move the tail ptr */
 692                         if (*i == NULL)
 693                                 per_cpu(tasklet_vec, cpu).tail = i;
 694                         return;
 695                 }
 696         }
 697         BUG();
 698 }
 699
 700 static void takeover_tasklets(unsigned int cpu)
 701 {
 702         /* CPU is dead, so no lock needed. */
 703         local_irq_disable();
 704
 705         /* Find end, append list for that CPU. */
 706         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 707                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 708                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 709                 per_cpu(tasklet_vec, cpu).head = NULL;
 710                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 711         }
 712         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 713
 714         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 715                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 716                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 717                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 718                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 719         }
 720         raise_softirq_irqoff(HI_SOFTIRQ);
 721
 722         local_irq_enable();
 723 }
 724 #endif /* CONFIG_HOTPLUG_CPU */
 725
 726 static int cpu_callback(struct notifier_block *nfb, unsigned long action,
 727                         void *hcpu)
 728 {
 729         switch (action) {
 730 #ifdef CONFIG_HOTPLUG_CPU
 731         case CPU_DEAD:
 732         case CPU_DEAD_FROZEN:
 733                 takeover_tasklets((unsigned long)hcpu);
 734                 break;
 735 #endif /* CONFIG_HOTPLUG_CPU */
 736         }
 737         return NOTIFY_OK;
 738 }
 739
 740 static struct notifier_block cpu_nfb = {
 741         .notifier_call = cpu_callback
 742 };
 743
 744 static struct smp_hotplug_thread softirq_threads = {
 745         .store                  = &ksoftirqd,
 746         .thread_should_run      = ksoftirqd_should_run,
 747         .thread_fn              = run_ksoftirqd,
 748         .thread_comm            = "ksoftirqd/%u",
 749 };
 750
 751 static __init int spawn_ksoftirqd(void)
 752 {
 753         register_cpu_notifier(&cpu_nfb);
 754
 755         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 756
 757         return 0;
 758 }
 759 early_initcall(spawn_ksoftirqd);
 760
 761 /*
 762  * [ These __weak aliases are kept in a separate compilation unit, so that
 763  *   GCC does not inline them incorrectly. ]
 764  */
 765
 766 int __init __weak early_irq_init(void)
 767 {
 768         return 0;
 769 }
 770
 771 int __init __weak arch_probe_nr_irqs(void)
 772 {
 773         return NR_IRQS_LEGACY;
 774 }
 775
 776 int __init __weak arch_early_irq_init(void)
 777 {
 778         return 0;
 779 }
 780
 781 unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
 782 {
 783         return from;
 784 }