kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <trace/events/sched.h>
  33
  34 #include <asm/ftrace.h>
  35 #include <asm/setup.h>
  36
  37 #include "trace_output.h"
  38 #include "trace_stat.h"
  39
  40 #define FTRACE_WARN_ON(cond)                    \
  41         do {                                    \
  42                 if (WARN_ON(cond))              \
  43                         ftrace_kill();          \
  44         } while (0)
  45
  46 #define FTRACE_WARN_ON_ONCE(cond)               \
  47         do {                                    \
  48                 if (WARN_ON_ONCE(cond))         \
  49                         ftrace_kill();          \
  50         } while (0)
  51
  52 /* hash bits for specific function selection */
  53 #define FTRACE_HASH_BITS 7
  54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  55
  56 /* ftrace_enabled is a method to turn ftrace on or off */
  57 int ftrace_enabled __read_mostly;
  58 static int last_ftrace_enabled;
  59
  60 /* Quick disabling of function tracer. */
  61 int function_trace_stop;
  62
  63 /*
  64  * ftrace_disabled is set when an anomaly is discovered.
  65  * ftrace_disabled is much stronger than ftrace_enabled.
  66  */
  67 static int ftrace_disabled __read_mostly;
  68
  69 static DEFINE_MUTEX(ftrace_lock);
  70
  71 static struct ftrace_ops ftrace_list_end __read_mostly =
  72 {
  73         .func           = ftrace_stub,
  74 };
  75
  76 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  77 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  78 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  79 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  80
  81 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  82 {
  83         struct ftrace_ops *op = ftrace_list;
  84
  85         /* in case someone actually ports this to alpha! */
  86         read_barrier_depends();
  87
  88         while (op != &ftrace_list_end) {
  89                 /* silly alpha */
  90                 read_barrier_depends();
  91                 op->func(ip, parent_ip);
  92                 op = op->next;
  93         };
  94 }
  95
  96 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  97 {
  98         if (!test_tsk_trace_trace(current))
  99                 return;
 100
 101         ftrace_pid_function(ip, parent_ip);
 102 }
 103
 104 static void set_ftrace_pid_function(ftrace_func_t func)
 105 {
 106         /* do not set ftrace_pid_function to itself! */
 107         if (func != ftrace_pid_func)
 108                 ftrace_pid_function = func;
 109 }
 110
 111 /**
 112  * clear_ftrace_function - reset the ftrace function
 113  *
 114  * This NULLs the ftrace function and in essence stops
 115  * tracing.  There may be lag
 116  */
 117 void clear_ftrace_function(void)
 118 {
 119         ftrace_trace_function = ftrace_stub;
 120         __ftrace_trace_function = ftrace_stub;
 121         ftrace_pid_function = ftrace_stub;
 122 }
 123
 124 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 125 /*
 126  * For those archs that do not test ftrace_trace_stop in their
 127  * mcount call site, we need to do it from C.
 128  */
 129 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 130 {
 131         if (function_trace_stop)
 132                 return;
 133
 134         __ftrace_trace_function(ip, parent_ip);
 135 }
 136 #endif
 137
 138 static int __register_ftrace_function(struct ftrace_ops *ops)
 139 {
 140         ops->next = ftrace_list;
 141         /*
 142          * We are entering ops into the ftrace_list but another
 143          * CPU might be walking that list. We need to make sure
 144          * the ops->next pointer is valid before another CPU sees
 145          * the ops pointer included into the ftrace_list.
 146          */
 147         smp_wmb();
 148         ftrace_list = ops;
 149
 150         if (ftrace_enabled) {
 151                 ftrace_func_t func;
 152
 153                 if (ops->next == &ftrace_list_end)
 154                         func = ops->func;
 155                 else
 156                         func = ftrace_list_func;
 157
 158                 if (ftrace_pid_trace) {
 159                         set_ftrace_pid_function(func);
 160                         func = ftrace_pid_func;
 161                 }
 162
 163                 /*
 164                  * For one func, simply call it directly.
 165                  * For more than one func, call the chain.
 166                  */
 167 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 168                 ftrace_trace_function = func;
 169 #else
 170                 __ftrace_trace_function = func;
 171                 ftrace_trace_function = ftrace_test_stop_func;
 172 #endif
 173         }
 174
 175         return 0;
 176 }
 177
 178 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 179 {
 180         struct ftrace_ops **p;
 181
 182         /*
 183          * If we are removing the last function, then simply point
 184          * to the ftrace_stub.
 185          */
 186         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 187                 ftrace_trace_function = ftrace_stub;
 188                 ftrace_list = &ftrace_list_end;
 189                 return 0;
 190         }
 191
 192         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 193                 if (*p == ops)
 194                         break;
 195
 196         if (*p != ops)
 197                 return -1;
 198
 199         *p = (*p)->next;
 200
 201         if (ftrace_enabled) {
 202                 /* If we only have one func left, then call that directly */
 203                 if (ftrace_list->next == &ftrace_list_end) {
 204                         ftrace_func_t func = ftrace_list->func;
 205
 206                         if (ftrace_pid_trace) {
 207                                 set_ftrace_pid_function(func);
 208                                 func = ftrace_pid_func;
 209                         }
 210 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 211                         ftrace_trace_function = func;
 212 #else
 213                         __ftrace_trace_function = func;
 214 #endif
 215                 }
 216         }
 217
 218         return 0;
 219 }
 220
 221 static void ftrace_update_pid_func(void)
 222 {
 223         ftrace_func_t func;
 224
 225         if (ftrace_trace_function == ftrace_stub)
 226                 return;
 227
 228 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 229         func = ftrace_trace_function;
 230 #else
 231         func = __ftrace_trace_function;
 232 #endif
 233
 234         if (ftrace_pid_trace) {
 235                 set_ftrace_pid_function(func);
 236                 func = ftrace_pid_func;
 237         } else {
 238                 if (func == ftrace_pid_func)
 239                         func = ftrace_pid_function;
 240         }
 241
 242 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 243         ftrace_trace_function = func;
 244 #else
 245         __ftrace_trace_function = func;
 246 #endif
 247 }
 248
 249 #ifdef CONFIG_FUNCTION_PROFILER
 250 struct ftrace_profile {
 251         struct hlist_node               node;
 252         unsigned long                   ip;
 253         unsigned long                   counter;
 254 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 255         unsigned long long              time;
 256 #endif
 257 };
 258
 259 struct ftrace_profile_page {
 260         struct ftrace_profile_page      *next;
 261         unsigned long                   index;
 262         struct ftrace_profile           records[];
 263 };
 264
 265 struct ftrace_profile_stat {
 266         atomic_t                        disabled;
 267         struct hlist_head               *hash;
 268         struct ftrace_profile_page      *pages;
 269         struct ftrace_profile_page      *start;
 270         struct tracer_stat              stat;
 271 };
 272
 273 #define PROFILE_RECORDS_SIZE                                            \
 274         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 275
 276 #define PROFILES_PER_PAGE                                       \
 277         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 278
 279 static int ftrace_profile_bits __read_mostly;
 280 static int ftrace_profile_enabled __read_mostly;
 281
 282 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 283 static DEFINE_MUTEX(ftrace_profile_lock);
 284
 285 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 286
 287 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 288
 289 static void *
 290 function_stat_next(void *v, int idx)
 291 {
 292         struct ftrace_profile *rec = v;
 293         struct ftrace_profile_page *pg;
 294
 295         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 296
 297  again:
 298         if (idx != 0)
 299                 rec++;
 300
 301         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 302                 pg = pg->next;
 303                 if (!pg)
 304                         return NULL;
 305                 rec = &pg->records[0];
 306                 if (!rec->counter)
 307                         goto again;
 308         }
 309
 310         return rec;
 311 }
 312
 313 static void *function_stat_start(struct tracer_stat *trace)
 314 {
 315         struct ftrace_profile_stat *stat =
 316                 container_of(trace, struct ftrace_profile_stat, stat);
 317
 318         if (!stat || !stat->start)
 319                 return NULL;
 320
 321         return function_stat_next(&stat->start->records[0], 0);
 322 }
 323
 324 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 325 /* function graph compares on total time */
 326 static int function_stat_cmp(void *p1, void *p2)
 327 {
 328         struct ftrace_profile *a = p1;
 329         struct ftrace_profile *b = p2;
 330
 331         if (a->time < b->time)
 332                 return -1;
 333         if (a->time > b->time)
 334                 return 1;
 335         else
 336                 return 0;
 337 }
 338 #else
 339 /* not function graph compares against hits */
 340 static int function_stat_cmp(void *p1, void *p2)
 341 {
 342         struct ftrace_profile *a = p1;
 343         struct ftrace_profile *b = p2;
 344
 345         if (a->counter < b->counter)
 346                 return -1;
 347         if (a->counter > b->counter)
 348                 return 1;
 349         else
 350                 return 0;
 351 }
 352 #endif
 353
 354 static int function_stat_headers(struct seq_file *m)
 355 {
 356 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 357         seq_printf(m, "  Function                               "
 358                    "Hit    Time            Avg\n"
 359                       "  --------                               "
 360                    "---    ----            ---\n");
 361 #else
 362         seq_printf(m, "  Function                               Hit\n"
 363                       "  --------                               ---\n");
 364 #endif
 365         return 0;
 366 }
 367
 368 static int function_stat_show(struct seq_file *m, void *v)
 369 {
 370         struct ftrace_profile *rec = v;
 371         char str[KSYM_SYMBOL_LEN];
 372         int ret = 0;
 373 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 374         static struct trace_seq s;
 375         unsigned long long avg;
 376 #endif
 377         mutex_lock(&ftrace_profile_lock);
 378
 379         /* we raced with function_profile_reset() */
 380         if (unlikely(rec->counter == 0)) {
 381                 ret = -EBUSY;
 382                 goto out;
 383         }
 384
 385         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 386         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 387
 388 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 389         seq_printf(m, "    ");
 390         avg = rec->time;
 391         do_div(avg, rec->counter);
 392
 393         trace_seq_init(&s);
 394         trace_print_graph_duration(rec->time, &s);
 395         trace_seq_puts(&s, "    ");
 396         trace_print_graph_duration(avg, &s);
 397         trace_print_seq(m, &s);
 398 #endif
 399         seq_putc(m, '\n');
 400 out:
 401         mutex_unlock(&ftrace_profile_lock);
 402
 403         return ret;
 404 }
 405
 406 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 407 {
 408         struct ftrace_profile_page *pg;
 409
 410         pg = stat->pages = stat->start;
 411
 412         while (pg) {
 413                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 414                 pg->index = 0;
 415                 pg = pg->next;
 416         }
 417
 418         memset(stat->hash, 0,
 419                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 420 }
 421
 422 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 423 {
 424         struct ftrace_profile_page *pg;
 425         int functions;
 426         int pages;
 427         int i;
 428
 429         /* If we already allocated, do nothing */
 430         if (stat->pages)
 431                 return 0;
 432
 433         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 434         if (!stat->pages)
 435                 return -ENOMEM;
 436
 437 #ifdef CONFIG_DYNAMIC_FTRACE
 438         functions = ftrace_update_tot_cnt;
 439 #else
 440         /*
 441          * We do not know the number of functions that exist because
 442          * dynamic tracing is what counts them. With past experience
 443          * we have around 20K functions. That should be more than enough.
 444          * It is highly unlikely we will execute every function in
 445          * the kernel.
 446          */
 447         functions = 20000;
 448 #endif
 449
 450         pg = stat->start = stat->pages;
 451
 452         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 453
 454         for (i = 0; i < pages; i++) {
 455                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 456                 if (!pg->next)
 457                         goto out_free;
 458                 pg = pg->next;
 459         }
 460
 461         return 0;
 462
 463  out_free:
 464         pg = stat->start;
 465         while (pg) {
 466                 unsigned long tmp = (unsigned long)pg;
 467
 468                 pg = pg->next;
 469                 free_page(tmp);
 470         }
 471
 472         free_page((unsigned long)stat->pages);
 473         stat->pages = NULL;
 474         stat->start = NULL;
 475
 476         return -ENOMEM;
 477 }
 478
 479 static int ftrace_profile_init_cpu(int cpu)
 480 {
 481         struct ftrace_profile_stat *stat;
 482         int size;
 483
 484         stat = &per_cpu(ftrace_profile_stats, cpu);
 485
 486         if (stat->hash) {
 487                 /* If the profile is already created, simply reset it */
 488                 ftrace_profile_reset(stat);
 489                 return 0;
 490         }
 491
 492         /*
 493          * We are profiling all functions, but usually only a few thousand
 494          * functions are hit. We'll make a hash of 1024 items.
 495          */
 496         size = FTRACE_PROFILE_HASH_SIZE;
 497
 498         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 499
 500         if (!stat->hash)
 501                 return -ENOMEM;
 502
 503         if (!ftrace_profile_bits) {
 504                 size--;
 505
 506                 for (; size; size >>= 1)
 507                         ftrace_profile_bits++;
 508         }
 509
 510         /* Preallocate the function profiling pages */
 511         if (ftrace_profile_pages_init(stat) < 0) {
 512                 kfree(stat->hash);
 513                 stat->hash = NULL;
 514                 return -ENOMEM;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static int ftrace_profile_init(void)
 521 {
 522         int cpu;
 523         int ret = 0;
 524
 525         for_each_online_cpu(cpu) {
 526                 ret = ftrace_profile_init_cpu(cpu);
 527                 if (ret)
 528                         break;
 529         }
 530
 531         return ret;
 532 }
 533
 534 /* interrupts must be disabled */
 535 static struct ftrace_profile *
 536 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 537 {
 538         struct ftrace_profile *rec;
 539         struct hlist_head *hhd;
 540         struct hlist_node *n;
 541         unsigned long key;
 542
 543         key = hash_long(ip, ftrace_profile_bits);
 544         hhd = &stat->hash[key];
 545
 546         if (hlist_empty(hhd))
 547                 return NULL;
 548
 549         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 550                 if (rec->ip == ip)
 551                         return rec;
 552         }
 553
 554         return NULL;
 555 }
 556
 557 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 558                                struct ftrace_profile *rec)
 559 {
 560         unsigned long key;
 561
 562         key = hash_long(rec->ip, ftrace_profile_bits);
 563         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 564 }
 565
 566 /*
 567  * The memory is already allocated, this simply finds a new record to use.
 568  */
 569 static struct ftrace_profile *
 570 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 571 {
 572         struct ftrace_profile *rec = NULL;
 573
 574         /* prevent recursion (from NMIs) */
 575         if (atomic_inc_return(&stat->disabled) != 1)
 576                 goto out;
 577
 578         /*
 579          * Try to find the function again since an NMI
 580          * could have added it
 581          */
 582         rec = ftrace_find_profiled_func(stat, ip);
 583         if (rec)
 584                 goto out;
 585
 586         if (stat->pages->index == PROFILES_PER_PAGE) {
 587                 if (!stat->pages->next)
 588                         goto out;
 589                 stat->pages = stat->pages->next;
 590         }
 591
 592         rec = &stat->pages->records[stat->pages->index++];
 593         rec->ip = ip;
 594         ftrace_add_profile(stat, rec);
 595
 596  out:
 597         atomic_dec(&stat->disabled);
 598
 599         return rec;
 600 }
 601
 602 static void
 603 function_profile_call(unsigned long ip, unsigned long parent_ip)
 604 {
 605         struct ftrace_profile_stat *stat;
 606         struct ftrace_profile *rec;
 607         unsigned long flags;
 608
 609         if (!ftrace_profile_enabled)
 610                 return;
 611
 612         local_irq_save(flags);
 613
 614         stat = &__get_cpu_var(ftrace_profile_stats);
 615         if (!stat->hash || !ftrace_profile_enabled)
 616                 goto out;
 617
 618         rec = ftrace_find_profiled_func(stat, ip);
 619         if (!rec) {
 620                 rec = ftrace_profile_alloc(stat, ip);
 621                 if (!rec)
 622                         goto out;
 623         }
 624
 625         rec->counter++;
 626  out:
 627         local_irq_restore(flags);
 628 }
 629
 630 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 631 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 632 {
 633         function_profile_call(trace->func, 0);
 634         return 1;
 635 }
 636
 637 static void profile_graph_return(struct ftrace_graph_ret *trace)
 638 {
 639         struct ftrace_profile_stat *stat;
 640         unsigned long long calltime;
 641         struct ftrace_profile *rec;
 642         unsigned long flags;
 643
 644         local_irq_save(flags);
 645         stat = &__get_cpu_var(ftrace_profile_stats);
 646         if (!stat->hash || !ftrace_profile_enabled)
 647                 goto out;
 648
 649         calltime = trace->rettime - trace->calltime;
 650
 651         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 652                 int index;
 653
 654                 index = trace->depth;
 655
 656                 /* Append this call time to the parent time to subtract */
 657                 if (index)
 658                         current->ret_stack[index - 1].subtime += calltime;
 659
 660                 if (current->ret_stack[index].subtime < calltime)
 661                         calltime -= current->ret_stack[index].subtime;
 662                 else
 663                         calltime = 0;
 664         }
 665
 666         rec = ftrace_find_profiled_func(stat, trace->func);
 667         if (rec)
 668                 rec->time += calltime;
 669
 670  out:
 671         local_irq_restore(flags);
 672 }
 673
 674 static int register_ftrace_profiler(void)
 675 {
 676         return register_ftrace_graph(&profile_graph_return,
 677                                      &profile_graph_entry);
 678 }
 679
 680 static void unregister_ftrace_profiler(void)
 681 {
 682         unregister_ftrace_graph();
 683 }
 684 #else
 685 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 686 {
 687         .func           = function_profile_call,
 688 };
 689
 690 static int register_ftrace_profiler(void)
 691 {
 692         return register_ftrace_function(&ftrace_profile_ops);
 693 }
 694
 695 static void unregister_ftrace_profiler(void)
 696 {
 697         unregister_ftrace_function(&ftrace_profile_ops);
 698 }
 699 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 700
 701 static ssize_t
 702 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 703                      size_t cnt, loff_t *ppos)
 704 {
 705         unsigned long val;
 706         char buf[64];           /* big enough to hold a number */
 707         int ret;
 708
 709         if (cnt >= sizeof(buf))
 710                 return -EINVAL;
 711
 712         if (copy_from_user(&buf, ubuf, cnt))
 713                 return -EFAULT;
 714
 715         buf[cnt] = 0;
 716
 717         ret = strict_strtoul(buf, 10, &val);
 718         if (ret < 0)
 719                 return ret;
 720
 721         val = !!val;
 722
 723         mutex_lock(&ftrace_profile_lock);
 724         if (ftrace_profile_enabled ^ val) {
 725                 if (val) {
 726                         ret = ftrace_profile_init();
 727                         if (ret < 0) {
 728                                 cnt = ret;
 729                                 goto out;
 730                         }
 731
 732                         ret = register_ftrace_profiler();
 733                         if (ret < 0) {
 734                                 cnt = ret;
 735                                 goto out;
 736                         }
 737                         ftrace_profile_enabled = 1;
 738                 } else {
 739                         ftrace_profile_enabled = 0;
 740                         /*
 741                          * unregister_ftrace_profiler calls stop_machine
 742                          * so this acts like an synchronize_sched.
 743                          */
 744                         unregister_ftrace_profiler();
 745                 }
 746         }
 747  out:
 748         mutex_unlock(&ftrace_profile_lock);
 749
 750         *ppos += cnt;
 751
 752         return cnt;
 753 }
 754
 755 static ssize_t
 756 ftrace_profile_read(struct file *filp, char __user *ubuf,
 757                      size_t cnt, loff_t *ppos)
 758 {
 759         char buf[64];           /* big enough to hold a number */
 760         int r;
 761
 762         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 763         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 764 }
 765
 766 static const struct file_operations ftrace_profile_fops = {
 767         .open           = tracing_open_generic,
 768         .read           = ftrace_profile_read,
 769         .write          = ftrace_profile_write,
 770 };
 771
 772 /* used to initialize the real stat files */
 773 static struct tracer_stat function_stats __initdata = {
 774         .name           = "functions",
 775         .stat_start     = function_stat_start,
 776         .stat_next      = function_stat_next,
 777         .stat_cmp       = function_stat_cmp,
 778         .stat_headers   = function_stat_headers,
 779         .stat_show      = function_stat_show
 780 };
 781
 782 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 783 {
 784         struct ftrace_profile_stat *stat;
 785         struct dentry *entry;
 786         char *name;
 787         int ret;
 788         int cpu;
 789
 790         for_each_possible_cpu(cpu) {
 791                 stat = &per_cpu(ftrace_profile_stats, cpu);
 792
 793                 /* allocate enough for function name + cpu number */
 794                 name = kmalloc(32, GFP_KERNEL);
 795                 if (!name) {
 796                         /*
 797                          * The files created are permanent, if something happens
 798                          * we still do not free memory.
 799                          */
 800                         WARN(1,
 801                              "Could not allocate stat file for cpu %d\n",
 802                              cpu);
 803                         return;
 804                 }
 805                 stat->stat = function_stats;
 806                 snprintf(name, 32, "function%d", cpu);
 807                 stat->stat.name = name;
 808                 ret = register_stat_tracer(&stat->stat);
 809                 if (ret) {
 810                         WARN(1,
 811                              "Could not register function stat for cpu %d\n",
 812                              cpu);
 813                         kfree(name);
 814                         return;
 815                 }
 816         }
 817
 818         entry = debugfs_create_file("function_profile_enabled", 0644,
 819                                     d_tracer, NULL, &ftrace_profile_fops);
 820         if (!entry)
 821                 pr_warning("Could not create debugfs "
 822                            "'function_profile_enabled' entry\n");
 823 }
 824
 825 #else /* CONFIG_FUNCTION_PROFILER */
 826 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 827 {
 828 }
 829 #endif /* CONFIG_FUNCTION_PROFILER */
 830
 831 /* set when tracing only a pid */
 832 struct pid *ftrace_pid_trace;
 833 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 834
 835 #ifdef CONFIG_DYNAMIC_FTRACE
 836
 837 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 838 # error Dynamic ftrace depends on MCOUNT_RECORD
 839 #endif
 840
 841 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 842
 843 struct ftrace_func_probe {
 844         struct hlist_node       node;
 845         struct ftrace_probe_ops *ops;
 846         unsigned long           flags;
 847         unsigned long           ip;
 848         void                    *data;
 849         struct rcu_head         rcu;
 850 };
 851
 852 enum {
 853         FTRACE_ENABLE_CALLS             = (1 << 0),
 854         FTRACE_DISABLE_CALLS            = (1 << 1),
 855         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 856         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 857         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 858         FTRACE_START_FUNC_RET           = (1 << 5),
 859         FTRACE_STOP_FUNC_RET            = (1 << 6),
 860 };
 861
 862 static int ftrace_filtered;
 863
 864 static struct dyn_ftrace *ftrace_new_addrs;
 865
 866 static DEFINE_MUTEX(ftrace_regex_lock);
 867
 868 struct ftrace_page {
 869         struct ftrace_page      *next;
 870         int                     index;
 871         struct dyn_ftrace       records[];
 872 };
 873
 874 #define ENTRIES_PER_PAGE \
 875   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 876
 877 /* estimate from running different kernels */
 878 #define NR_TO_INIT              10000
 879
 880 static struct ftrace_page       *ftrace_pages_start;
 881 static struct ftrace_page       *ftrace_pages;
 882
 883 static struct dyn_ftrace *ftrace_free_records;
 884
 885 /*
 886  * This is a double for. Do not use 'break' to break out of the loop,
 887  * you must use a goto.
 888  */
 889 #define do_for_each_ftrace_rec(pg, rec)                                 \
 890         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 891                 int _____i;                                             \
 892                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 893                         rec = &pg->records[_____i];
 894
 895 #define while_for_each_ftrace_rec()             \
 896                 }                               \
 897         }
 898
 899 #ifdef CONFIG_KPROBES
 900
 901 static int frozen_record_count;
 902
 903 static inline void freeze_record(struct dyn_ftrace *rec)
 904 {
 905         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 906                 rec->flags |= FTRACE_FL_FROZEN;
 907                 frozen_record_count++;
 908         }
 909 }
 910
 911 static inline void unfreeze_record(struct dyn_ftrace *rec)
 912 {
 913         if (rec->flags & FTRACE_FL_FROZEN) {
 914                 rec->flags &= ~FTRACE_FL_FROZEN;
 915                 frozen_record_count--;
 916         }
 917 }
 918
 919 static inline int record_frozen(struct dyn_ftrace *rec)
 920 {
 921         return rec->flags & FTRACE_FL_FROZEN;
 922 }
 923 #else
 924 # define freeze_record(rec)                     ({ 0; })
 925 # define unfreeze_record(rec)                   ({ 0; })
 926 # define record_frozen(rec)                     ({ 0; })
 927 #endif /* CONFIG_KPROBES */
 928
 929 static void ftrace_free_rec(struct dyn_ftrace *rec)
 930 {
 931         rec->freelist = ftrace_free_records;
 932         ftrace_free_records = rec;
 933         rec->flags |= FTRACE_FL_FREE;
 934 }
 935
 936 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 937 {
 938         struct dyn_ftrace *rec;
 939
 940         /* First check for freed records */
 941         if (ftrace_free_records) {
 942                 rec = ftrace_free_records;
 943
 944                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 945                         FTRACE_WARN_ON_ONCE(1);
 946                         ftrace_free_records = NULL;
 947                         return NULL;
 948                 }
 949
 950                 ftrace_free_records = rec->freelist;
 951                 memset(rec, 0, sizeof(*rec));
 952                 return rec;
 953         }
 954
 955         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 956                 if (!ftrace_pages->next) {
 957                         /* allocate another page */
 958                         ftrace_pages->next =
 959                                 (void *)get_zeroed_page(GFP_KERNEL);
 960                         if (!ftrace_pages->next)
 961                                 return NULL;
 962                 }
 963                 ftrace_pages = ftrace_pages->next;
 964         }
 965
 966         return &ftrace_pages->records[ftrace_pages->index++];
 967 }
 968
 969 static struct dyn_ftrace *
 970 ftrace_record_ip(unsigned long ip)
 971 {
 972         struct dyn_ftrace *rec;
 973
 974         if (ftrace_disabled)
 975                 return NULL;
 976
 977         rec = ftrace_alloc_dyn_node(ip);
 978         if (!rec)
 979                 return NULL;
 980
 981         rec->ip = ip;
 982         rec->newlist = ftrace_new_addrs;
 983         ftrace_new_addrs = rec;
 984
 985         return rec;
 986 }
 987
 988 static void print_ip_ins(const char *fmt, unsigned char *p)
 989 {
 990         int i;
 991
 992         printk(KERN_CONT "%s", fmt);
 993
 994         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 995                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 996 }
 997
 998 static void ftrace_bug(int failed, unsigned long ip)
 999 {
1000         switch (failed) {
1001         case -EFAULT:
1002                 FTRACE_WARN_ON_ONCE(1);
1003                 pr_info("ftrace faulted on modifying ");
1004                 print_ip_sym(ip);
1005                 break;
1006         case -EINVAL:
1007                 FTRACE_WARN_ON_ONCE(1);
1008                 pr_info("ftrace failed to modify ");
1009                 print_ip_sym(ip);
1010                 print_ip_ins(" actual: ", (unsigned char *)ip);
1011                 printk(KERN_CONT "\n");
1012                 break;
1013         case -EPERM:
1014                 FTRACE_WARN_ON_ONCE(1);
1015                 pr_info("ftrace faulted on writing ");
1016                 print_ip_sym(ip);
1017                 break;
1018         default:
1019                 FTRACE_WARN_ON_ONCE(1);
1020                 pr_info("ftrace faulted on unknown error ");
1021                 print_ip_sym(ip);
1022         }
1023 }
1024
1025
1026 static int
1027 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1028 {
1029         unsigned long ftrace_addr;
1030         unsigned long flag = 0UL;
1031
1032         ftrace_addr = (unsigned long)FTRACE_ADDR;
1033
1034         /*
1035          * If this record is not to be traced or we want to disable it,
1036          * then disable it.
1037          *
1038          * If we want to enable it and filtering is off, then enable it.
1039          *
1040          * If we want to enable it and filtering is on, enable it only if
1041          * it's filtered
1042          */
1043         if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1044                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1045                         flag = FTRACE_FL_ENABLED;
1046         }
1047
1048         /* If the state of this record hasn't changed, then do nothing */
1049         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1050                 return 0;
1051
1052         if (flag) {
1053                 rec->flags |= FTRACE_FL_ENABLED;
1054                 return ftrace_make_call(rec, ftrace_addr);
1055         }
1056
1057         rec->flags &= ~FTRACE_FL_ENABLED;
1058         return ftrace_make_nop(NULL, rec, ftrace_addr);
1059 }
1060
1061 static void ftrace_replace_code(int enable)
1062 {
1063         struct dyn_ftrace *rec;
1064         struct ftrace_page *pg;
1065         int failed;
1066
1067         do_for_each_ftrace_rec(pg, rec) {
1068                 /*
1069                  * Skip over free records, records that have
1070                  * failed and not converted.
1071                  */
1072                 if (rec->flags & FTRACE_FL_FREE ||
1073                     rec->flags & FTRACE_FL_FAILED ||
1074                     !(rec->flags & FTRACE_FL_CONVERTED))
1075                         continue;
1076
1077                 /* ignore updates to this record's mcount site */
1078                 if (get_kprobe((void *)rec->ip)) {
1079                         freeze_record(rec);
1080                         continue;
1081                 } else {
1082                         unfreeze_record(rec);
1083                 }
1084
1085                 failed = __ftrace_replace_code(rec, enable);
1086                 if (failed) {
1087                         rec->flags |= FTRACE_FL_FAILED;
1088                         ftrace_bug(failed, rec->ip);
1089                         /* Stop processing */
1090                         return;
1091                 }
1092         } while_for_each_ftrace_rec();
1093 }
1094
1095 static int
1096 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1097 {
1098         unsigned long ip;
1099         int ret;
1100
1101         ip = rec->ip;
1102
1103         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1104         if (ret) {
1105                 ftrace_bug(ret, ip);
1106                 rec->flags |= FTRACE_FL_FAILED;
1107                 return 0;
1108         }
1109         return 1;
1110 }
1111
1112 /*
1113  * archs can override this function if they must do something
1114  * before the modifying code is performed.
1115  */
1116 int __weak ftrace_arch_code_modify_prepare(void)
1117 {
1118         return 0;
1119 }
1120
1121 /*
1122  * archs can override this function if they must do something
1123  * after the modifying code is performed.
1124  */
1125 int __weak ftrace_arch_code_modify_post_process(void)
1126 {
1127         return 0;
1128 }
1129
1130 static int __ftrace_modify_code(void *data)
1131 {
1132         int *command = data;
1133
1134         if (*command & FTRACE_ENABLE_CALLS)
1135                 ftrace_replace_code(1);
1136         else if (*command & FTRACE_DISABLE_CALLS)
1137                 ftrace_replace_code(0);
1138
1139         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1140                 ftrace_update_ftrace_func(ftrace_trace_function);
1141
1142         if (*command & FTRACE_START_FUNC_RET)
1143                 ftrace_enable_ftrace_graph_caller();
1144         else if (*command & FTRACE_STOP_FUNC_RET)
1145                 ftrace_disable_ftrace_graph_caller();
1146
1147         return 0;
1148 }
1149
1150 static void ftrace_run_update_code(int command)
1151 {
1152         int ret;
1153
1154         ret = ftrace_arch_code_modify_prepare();
1155         FTRACE_WARN_ON(ret);
1156         if (ret)
1157                 return;
1158
1159         stop_machine(__ftrace_modify_code, &command, NULL);
1160
1161         ret = ftrace_arch_code_modify_post_process();
1162         FTRACE_WARN_ON(ret);
1163 }
1164
1165 static ftrace_func_t saved_ftrace_func;
1166 static int ftrace_start_up;
1167
1168 static void ftrace_startup_enable(int command)
1169 {
1170         if (saved_ftrace_func != ftrace_trace_function) {
1171                 saved_ftrace_func = ftrace_trace_function;
1172                 command |= FTRACE_UPDATE_TRACE_FUNC;
1173         }
1174
1175         if (!command || !ftrace_enabled)
1176                 return;
1177
1178         ftrace_run_update_code(command);
1179 }
1180
1181 static void ftrace_startup(int command)
1182 {
1183         if (unlikely(ftrace_disabled))
1184                 return;
1185
1186         ftrace_start_up++;
1187         command |= FTRACE_ENABLE_CALLS;
1188
1189         ftrace_startup_enable(command);
1190 }
1191
1192 static void ftrace_shutdown(int command)
1193 {
1194         if (unlikely(ftrace_disabled))
1195                 return;
1196
1197         ftrace_start_up--;
1198         /*
1199          * Just warn in case of unbalance, no need to kill ftrace, it's not
1200          * critical but the ftrace_call callers may be never nopped again after
1201          * further ftrace uses.
1202          */
1203         WARN_ON_ONCE(ftrace_start_up < 0);
1204
1205         if (!ftrace_start_up)
1206                 command |= FTRACE_DISABLE_CALLS;
1207
1208         if (saved_ftrace_func != ftrace_trace_function) {
1209                 saved_ftrace_func = ftrace_trace_function;
1210                 command |= FTRACE_UPDATE_TRACE_FUNC;
1211         }
1212
1213         if (!command || !ftrace_enabled)
1214                 return;
1215
1216         ftrace_run_update_code(command);
1217 }
1218
1219 static void ftrace_startup_sysctl(void)
1220 {
1221         int command = FTRACE_ENABLE_MCOUNT;
1222
1223         if (unlikely(ftrace_disabled))
1224                 return;
1225
1226         /* Force update next time */
1227         saved_ftrace_func = NULL;
1228         /* ftrace_start_up is true if we want ftrace running */
1229         if (ftrace_start_up)
1230                 command |= FTRACE_ENABLE_CALLS;
1231
1232         ftrace_run_update_code(command);
1233 }
1234
1235 static void ftrace_shutdown_sysctl(void)
1236 {
1237         int command = FTRACE_DISABLE_MCOUNT;
1238
1239         if (unlikely(ftrace_disabled))
1240                 return;
1241
1242         /* ftrace_start_up is true if ftrace is running */
1243         if (ftrace_start_up)
1244                 command |= FTRACE_DISABLE_CALLS;
1245
1246         ftrace_run_update_code(command);
1247 }
1248
1249 static cycle_t          ftrace_update_time;
1250 static unsigned long    ftrace_update_cnt;
1251 unsigned long           ftrace_update_tot_cnt;
1252
1253 static int ftrace_update_code(struct module *mod)
1254 {
1255         struct dyn_ftrace *p;
1256         cycle_t start, stop;
1257
1258         start = ftrace_now(raw_smp_processor_id());
1259         ftrace_update_cnt = 0;
1260
1261         while (ftrace_new_addrs) {
1262
1263                 /* If something went wrong, bail without enabling anything */
1264                 if (unlikely(ftrace_disabled))
1265                         return -1;
1266
1267                 p = ftrace_new_addrs;
1268                 ftrace_new_addrs = p->newlist;
1269                 p->flags = 0L;
1270
1271                 /* convert record (i.e, patch mcount-call with NOP) */
1272                 if (ftrace_code_disable(mod, p)) {
1273                         p->flags |= FTRACE_FL_CONVERTED;
1274                         ftrace_update_cnt++;
1275                 } else
1276                         ftrace_free_rec(p);
1277         }
1278
1279         stop = ftrace_now(raw_smp_processor_id());
1280         ftrace_update_time = stop - start;
1281         ftrace_update_tot_cnt += ftrace_update_cnt;
1282
1283         return 0;
1284 }
1285
1286 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1287 {
1288         struct ftrace_page *pg;
1289         int cnt;
1290         int i;
1291
1292         /* allocate a few pages */
1293         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1294         if (!ftrace_pages_start)
1295                 return -1;
1296
1297         /*
1298          * Allocate a few more pages.
1299          *
1300          * TODO: have some parser search vmlinux before
1301          *   final linking to find all calls to ftrace.
1302          *   Then we can:
1303          *    a) know how many pages to allocate.
1304          *     and/or
1305          *    b) set up the table then.
1306          *
1307          *  The dynamic code is still necessary for
1308          *  modules.
1309          */
1310
1311         pg = ftrace_pages = ftrace_pages_start;
1312
1313         cnt = num_to_init / ENTRIES_PER_PAGE;
1314         pr_info("ftrace: allocating %ld entries in %d pages\n",
1315                 num_to_init, cnt + 1);
1316
1317         for (i = 0; i < cnt; i++) {
1318                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1319
1320                 /* If we fail, we'll try later anyway */
1321                 if (!pg->next)
1322                         break;
1323
1324                 pg = pg->next;
1325         }
1326
1327         return 0;
1328 }
1329
1330 enum {
1331         FTRACE_ITER_FILTER      = (1 << 0),
1332         FTRACE_ITER_NOTRACE     = (1 << 1),
1333         FTRACE_ITER_FAILURES    = (1 << 2),
1334         FTRACE_ITER_PRINTALL    = (1 << 3),
1335         FTRACE_ITER_HASH        = (1 << 4),
1336 };
1337
1338 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1339
1340 struct ftrace_iterator {
1341         struct ftrace_page      *pg;
1342         int                     hidx;
1343         int                     idx;
1344         unsigned                flags;
1345         struct trace_parser     parser;
1346 };
1347
1348 static void *
1349 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1350 {
1351         struct ftrace_iterator *iter = m->private;
1352         struct hlist_node *hnd = v;
1353         struct hlist_head *hhd;
1354
1355         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1356
1357         (*pos)++;
1358
1359  retry:
1360         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1361                 return NULL;
1362
1363         hhd = &ftrace_func_hash[iter->hidx];
1364
1365         if (hlist_empty(hhd)) {
1366                 iter->hidx++;
1367                 hnd = NULL;
1368                 goto retry;
1369         }
1370
1371         if (!hnd)
1372                 hnd = hhd->first;
1373         else {
1374                 hnd = hnd->next;
1375                 if (!hnd) {
1376                         iter->hidx++;
1377                         goto retry;
1378                 }
1379         }
1380
1381         return hnd;
1382 }
1383
1384 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1385 {
1386         struct ftrace_iterator *iter = m->private;
1387         void *p = NULL;
1388         loff_t l;
1389
1390         if (!(iter->flags & FTRACE_ITER_HASH))
1391                 *pos = 0;
1392
1393         iter->flags |= FTRACE_ITER_HASH;
1394
1395         iter->hidx = 0;
1396         for (l = 0; l <= *pos; ) {
1397                 p = t_hash_next(m, p, &l);
1398                 if (!p)
1399                         break;
1400         }
1401         return p;
1402 }
1403
1404 static int t_hash_show(struct seq_file *m, void *v)
1405 {
1406         struct ftrace_func_probe *rec;
1407         struct hlist_node *hnd = v;
1408
1409         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1410
1411         if (rec->ops->print)
1412                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1413
1414         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1415
1416         if (rec->data)
1417                 seq_printf(m, ":%p", rec->data);
1418         seq_putc(m, '\n');
1419
1420         return 0;
1421 }
1422
1423 static void *
1424 t_next(struct seq_file *m, void *v, loff_t *pos)
1425 {
1426         struct ftrace_iterator *iter = m->private;
1427         struct dyn_ftrace *rec = NULL;
1428
1429         if (iter->flags & FTRACE_ITER_HASH)
1430                 return t_hash_next(m, v, pos);
1431
1432         (*pos)++;
1433
1434         if (iter->flags & FTRACE_ITER_PRINTALL)
1435                 return NULL;
1436
1437  retry:
1438         if (iter->idx >= iter->pg->index) {
1439                 if (iter->pg->next) {
1440                         iter->pg = iter->pg->next;
1441                         iter->idx = 0;
1442                         goto retry;
1443                 }
1444         } else {
1445                 rec = &iter->pg->records[iter->idx++];
1446                 if ((rec->flags & FTRACE_FL_FREE) ||
1447
1448                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1449                      (rec->flags & FTRACE_FL_FAILED)) ||
1450
1451                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1452                      !(rec->flags & FTRACE_FL_FAILED)) ||
1453
1454                     ((iter->flags & FTRACE_ITER_FILTER) &&
1455                      !(rec->flags & FTRACE_FL_FILTER)) ||
1456
1457                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1458                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1459                         rec = NULL;
1460                         goto retry;
1461                 }
1462         }
1463
1464         return rec;
1465 }
1466
1467 static void *t_start(struct seq_file *m, loff_t *pos)
1468 {
1469         struct ftrace_iterator *iter = m->private;
1470         void *p = NULL;
1471         loff_t l;
1472
1473         mutex_lock(&ftrace_lock);
1474         /*
1475          * For set_ftrace_filter reading, if we have the filter
1476          * off, we can short cut and just print out that all
1477          * functions are enabled.
1478          */
1479         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1480                 if (*pos > 0)
1481                         return t_hash_start(m, pos);
1482                 iter->flags |= FTRACE_ITER_PRINTALL;
1483                 /* reset in case of seek/pread */
1484                 iter->flags &= ~FTRACE_ITER_HASH;
1485                 return iter;
1486         }
1487
1488         if (iter->flags & FTRACE_ITER_HASH)
1489                 return t_hash_start(m, pos);
1490
1491         iter->pg = ftrace_pages_start;
1492         iter->idx = 0;
1493         for (l = 0; l <= *pos; ) {
1494                 p = t_next(m, p, &l);
1495                 if (!p)
1496                         break;
1497         }
1498
1499         if (!p && iter->flags & FTRACE_ITER_FILTER)
1500                 return t_hash_start(m, pos);
1501
1502         return p;
1503 }
1504
1505 static void t_stop(struct seq_file *m, void *p)
1506 {
1507         mutex_unlock(&ftrace_lock);
1508 }
1509
1510 static int t_show(struct seq_file *m, void *v)
1511 {
1512         struct ftrace_iterator *iter = m->private;
1513         struct dyn_ftrace *rec = v;
1514
1515         if (iter->flags & FTRACE_ITER_HASH)
1516                 return t_hash_show(m, v);
1517
1518         if (iter->flags & FTRACE_ITER_PRINTALL) {
1519                 seq_printf(m, "#### all functions enabled ####\n");
1520                 return 0;
1521         }
1522
1523         if (!rec)
1524                 return 0;
1525
1526         seq_printf(m, "%ps\n", (void *)rec->ip);
1527
1528         return 0;
1529 }
1530
1531 static const struct seq_operations show_ftrace_seq_ops = {
1532         .start = t_start,
1533         .next = t_next,
1534         .stop = t_stop,
1535         .show = t_show,
1536 };
1537
1538 static int
1539 ftrace_avail_open(struct inode *inode, struct file *file)
1540 {
1541         struct ftrace_iterator *iter;
1542         int ret;
1543
1544         if (unlikely(ftrace_disabled))
1545                 return -ENODEV;
1546
1547         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1548         if (!iter)
1549                 return -ENOMEM;
1550
1551         iter->pg = ftrace_pages_start;
1552
1553         ret = seq_open(file, &show_ftrace_seq_ops);
1554         if (!ret) {
1555                 struct seq_file *m = file->private_data;
1556
1557                 m->private = iter;
1558         } else {
1559                 kfree(iter);
1560         }
1561
1562         return ret;
1563 }
1564
1565 static int
1566 ftrace_failures_open(struct inode *inode, struct file *file)
1567 {
1568         int ret;
1569         struct seq_file *m;
1570         struct ftrace_iterator *iter;
1571
1572         ret = ftrace_avail_open(inode, file);
1573         if (!ret) {
1574                 m = (struct seq_file *)file->private_data;
1575                 iter = (struct ftrace_iterator *)m->private;
1576                 iter->flags = FTRACE_ITER_FAILURES;
1577         }
1578
1579         return ret;
1580 }
1581
1582
1583 static void ftrace_filter_reset(int enable)
1584 {
1585         struct ftrace_page *pg;
1586         struct dyn_ftrace *rec;
1587         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1588
1589         mutex_lock(&ftrace_lock);
1590         if (enable)
1591                 ftrace_filtered = 0;
1592         do_for_each_ftrace_rec(pg, rec) {
1593                 if (rec->flags & FTRACE_FL_FAILED)
1594                         continue;
1595                 rec->flags &= ~type;
1596         } while_for_each_ftrace_rec();
1597         mutex_unlock(&ftrace_lock);
1598 }
1599
1600 static int
1601 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1602 {
1603         struct ftrace_iterator *iter;
1604         int ret = 0;
1605
1606         if (unlikely(ftrace_disabled))
1607                 return -ENODEV;
1608
1609         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1610         if (!iter)
1611                 return -ENOMEM;
1612
1613         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1614                 kfree(iter);
1615                 return -ENOMEM;
1616         }
1617
1618         mutex_lock(&ftrace_regex_lock);
1619         if ((file->f_mode & FMODE_WRITE) &&
1620             (file->f_flags & O_TRUNC))
1621                 ftrace_filter_reset(enable);
1622
1623         if (file->f_mode & FMODE_READ) {
1624                 iter->pg = ftrace_pages_start;
1625                 iter->flags = enable ? FTRACE_ITER_FILTER :
1626                         FTRACE_ITER_NOTRACE;
1627
1628                 ret = seq_open(file, &show_ftrace_seq_ops);
1629                 if (!ret) {
1630                         struct seq_file *m = file->private_data;
1631                         m->private = iter;
1632                 } else {
1633                         trace_parser_put(&iter->parser);
1634                         kfree(iter);
1635                 }
1636         } else
1637                 file->private_data = iter;
1638         mutex_unlock(&ftrace_regex_lock);
1639
1640         return ret;
1641 }
1642
1643 static int
1644 ftrace_filter_open(struct inode *inode, struct file *file)
1645 {
1646         return ftrace_regex_open(inode, file, 1);
1647 }
1648
1649 static int
1650 ftrace_notrace_open(struct inode *inode, struct file *file)
1651 {
1652         return ftrace_regex_open(inode, file, 0);
1653 }
1654
1655 static loff_t
1656 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1657 {
1658         loff_t ret;
1659
1660         if (file->f_mode & FMODE_READ)
1661                 ret = seq_lseek(file, offset, origin);
1662         else
1663                 file->f_pos = ret = 1;
1664
1665         return ret;
1666 }
1667
1668 enum {
1669         MATCH_FULL,
1670         MATCH_FRONT_ONLY,
1671         MATCH_MIDDLE_ONLY,
1672         MATCH_END_ONLY,
1673 };
1674
1675 /*
1676  * (static function - no need for kernel doc)
1677  *
1678  * Pass in a buffer containing a glob and this function will
1679  * set search to point to the search part of the buffer and
1680  * return the type of search it is (see enum above).
1681  * This does modify buff.
1682  *
1683  * Returns enum type.
1684  *  search returns the pointer to use for comparison.
1685  *  not returns 1 if buff started with a '!'
1686  *     0 otherwise.
1687  */
1688 static int
1689 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1690 {
1691         int type = MATCH_FULL;
1692         int i;
1693
1694         if (buff[0] == '!') {
1695                 *not = 1;
1696                 buff++;
1697                 len--;
1698         } else
1699                 *not = 0;
1700
1701         *search = buff;
1702
1703         for (i = 0; i < len; i++) {
1704                 if (buff[i] == '*') {
1705                         if (!i) {
1706                                 *search = buff + 1;
1707                                 type = MATCH_END_ONLY;
1708                         } else {
1709                                 if (type == MATCH_END_ONLY)
1710                                         type = MATCH_MIDDLE_ONLY;
1711                                 else
1712                                         type = MATCH_FRONT_ONLY;
1713                                 buff[i] = 0;
1714                                 break;
1715                         }
1716                 }
1717         }
1718
1719         return type;
1720 }
1721
1722 static int ftrace_match(char *str, char *regex, int len, int type)
1723 {
1724         int matched = 0;
1725         char *ptr;
1726
1727         switch (type) {
1728         case MATCH_FULL:
1729                 if (strcmp(str, regex) == 0)
1730                         matched = 1;
1731                 break;
1732         case MATCH_FRONT_ONLY:
1733                 if (strncmp(str, regex, len) == 0)
1734                         matched = 1;
1735                 break;
1736         case MATCH_MIDDLE_ONLY:
1737                 if (strstr(str, regex))
1738                         matched = 1;
1739                 break;
1740         case MATCH_END_ONLY:
1741                 ptr = strstr(str, regex);
1742                 if (ptr && (ptr[len] == 0))
1743                         matched = 1;
1744                 break;
1745         }
1746
1747         return matched;
1748 }
1749
1750 static int
1751 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1752 {
1753         char str[KSYM_SYMBOL_LEN];
1754
1755         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1756         return ftrace_match(str, regex, len, type);
1757 }
1758
1759 static void ftrace_match_records(char *buff, int len, int enable)
1760 {
1761         unsigned int search_len;
1762         struct ftrace_page *pg;
1763         struct dyn_ftrace *rec;
1764         unsigned long flag;
1765         char *search;
1766         int type;
1767         int not;
1768
1769         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1770         type = ftrace_setup_glob(buff, len, &search, &not);
1771
1772         search_len = strlen(search);
1773
1774         mutex_lock(&ftrace_lock);
1775         do_for_each_ftrace_rec(pg, rec) {
1776
1777                 if (rec->flags & FTRACE_FL_FAILED)
1778                         continue;
1779
1780                 if (ftrace_match_record(rec, search, search_len, type)) {
1781                         if (not)
1782                                 rec->flags &= ~flag;
1783                         else
1784                                 rec->flags |= flag;
1785                 }
1786                 /*
1787                  * Only enable filtering if we have a function that
1788                  * is filtered on.
1789                  */
1790                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1791                         ftrace_filtered = 1;
1792         } while_for_each_ftrace_rec();
1793         mutex_unlock(&ftrace_lock);
1794 }
1795
1796 static int
1797 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1798                            char *regex, int len, int type)
1799 {
1800         char str[KSYM_SYMBOL_LEN];
1801         char *modname;
1802
1803         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1804
1805         if (!modname || strcmp(modname, mod))
1806                 return 0;
1807
1808         /* blank search means to match all funcs in the mod */
1809         if (len)
1810                 return ftrace_match(str, regex, len, type);
1811         else
1812                 return 1;
1813 }
1814
1815 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1816 {
1817         unsigned search_len = 0;
1818         struct ftrace_page *pg;
1819         struct dyn_ftrace *rec;
1820         int type = MATCH_FULL;
1821         char *search = buff;
1822         unsigned long flag;
1823         int not = 0;
1824
1825         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1826
1827         /* blank or '*' mean the same */
1828         if (strcmp(buff, "*") == 0)
1829                 buff[0] = 0;
1830
1831         /* handle the case of 'dont filter this module' */
1832         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1833                 buff[0] = 0;
1834                 not = 1;
1835         }
1836
1837         if (strlen(buff)) {
1838                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1839                 search_len = strlen(search);
1840         }
1841
1842         mutex_lock(&ftrace_lock);
1843         do_for_each_ftrace_rec(pg, rec) {
1844
1845                 if (rec->flags & FTRACE_FL_FAILED)
1846                         continue;
1847
1848                 if (ftrace_match_module_record(rec, mod,
1849                                                search, search_len, type)) {
1850                         if (not)
1851                                 rec->flags &= ~flag;
1852                         else
1853                                 rec->flags |= flag;
1854                 }
1855                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1856                         ftrace_filtered = 1;
1857
1858         } while_for_each_ftrace_rec();
1859         mutex_unlock(&ftrace_lock);
1860 }
1861
1862 /*
1863  * We register the module command as a template to show others how
1864  * to register the a command as well.
1865  */
1866
1867 static int
1868 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1869 {
1870         char *mod;
1871
1872         /*
1873          * cmd == 'mod' because we only registered this func
1874          * for the 'mod' ftrace_func_command.
1875          * But if you register one func with multiple commands,
1876          * you can tell which command was used by the cmd
1877          * parameter.
1878          */
1879
1880         /* we must have a module name */
1881         if (!param)
1882                 return -EINVAL;
1883
1884         mod = strsep(&param, ":");
1885         if (!strlen(mod))
1886                 return -EINVAL;
1887
1888         ftrace_match_module_records(func, mod, enable);
1889         return 0;
1890 }
1891
1892 static struct ftrace_func_command ftrace_mod_cmd = {
1893         .name                   = "mod",
1894         .func                   = ftrace_mod_callback,
1895 };
1896
1897 static int __init ftrace_mod_cmd_init(void)
1898 {
1899         return register_ftrace_command(&ftrace_mod_cmd);
1900 }
1901 device_initcall(ftrace_mod_cmd_init);
1902
1903 static void
1904 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1905 {
1906         struct ftrace_func_probe *entry;
1907         struct hlist_head *hhd;
1908         struct hlist_node *n;
1909         unsigned long key;
1910         int resched;
1911
1912         key = hash_long(ip, FTRACE_HASH_BITS);
1913
1914         hhd = &ftrace_func_hash[key];
1915
1916         if (hlist_empty(hhd))
1917                 return;
1918
1919         /*
1920          * Disable preemption for these calls to prevent a RCU grace
1921          * period. This syncs the hash iteration and freeing of items
1922          * on the hash. rcu_read_lock is too dangerous here.
1923          */
1924         resched = ftrace_preempt_disable();
1925         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1926                 if (entry->ip == ip)
1927                         entry->ops->func(ip, parent_ip, &entry->data);
1928         }
1929         ftrace_preempt_enable(resched);
1930 }
1931
1932 static struct ftrace_ops trace_probe_ops __read_mostly =
1933 {
1934         .func           = function_trace_probe_call,
1935 };
1936
1937 static int ftrace_probe_registered;
1938
1939 static void __enable_ftrace_function_probe(void)
1940 {
1941         int i;
1942
1943         if (ftrace_probe_registered)
1944                 return;
1945
1946         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1947                 struct hlist_head *hhd = &ftrace_func_hash[i];
1948                 if (hhd->first)
1949                         break;
1950         }
1951         /* Nothing registered? */
1952         if (i == FTRACE_FUNC_HASHSIZE)
1953                 return;
1954
1955         __register_ftrace_function(&trace_probe_ops);
1956         ftrace_startup(0);
1957         ftrace_probe_registered = 1;
1958 }
1959
1960 static void __disable_ftrace_function_probe(void)
1961 {
1962         int i;
1963
1964         if (!ftrace_probe_registered)
1965                 return;
1966
1967         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1968                 struct hlist_head *hhd = &ftrace_func_hash[i];
1969                 if (hhd->first)
1970                         return;
1971         }
1972
1973         /* no more funcs left */
1974         __unregister_ftrace_function(&trace_probe_ops);
1975         ftrace_shutdown(0);
1976         ftrace_probe_registered = 0;
1977 }
1978
1979
1980 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1981 {
1982         struct ftrace_func_probe *entry =
1983                 container_of(rhp, struct ftrace_func_probe, rcu);
1984
1985         if (entry->ops->free)
1986                 entry->ops->free(&entry->data);
1987         kfree(entry);
1988 }
1989
1990
1991 int
1992 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1993                               void *data)
1994 {
1995         struct ftrace_func_probe *entry;
1996         struct ftrace_page *pg;
1997         struct dyn_ftrace *rec;
1998         int type, len, not;
1999         unsigned long key;
2000         int count = 0;
2001         char *search;
2002
2003         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2004         len = strlen(search);
2005
2006         /* we do not support '!' for function probes */
2007         if (WARN_ON(not))
2008                 return -EINVAL;
2009
2010         mutex_lock(&ftrace_lock);
2011         do_for_each_ftrace_rec(pg, rec) {
2012
2013                 if (rec->flags & FTRACE_FL_FAILED)
2014                         continue;
2015
2016                 if (!ftrace_match_record(rec, search, len, type))
2017                         continue;
2018
2019                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2020                 if (!entry) {
2021                         /* If we did not process any, then return error */
2022                         if (!count)
2023                                 count = -ENOMEM;
2024                         goto out_unlock;
2025                 }
2026
2027                 count++;
2028
2029                 entry->data = data;
2030
2031                 /*
2032                  * The caller might want to do something special
2033                  * for each function we find. We call the callback
2034                  * to give the caller an opportunity to do so.
2035                  */
2036                 if (ops->callback) {
2037                         if (ops->callback(rec->ip, &entry->data) < 0) {
2038                                 /* caller does not like this func */
2039                                 kfree(entry);
2040                                 continue;
2041                         }
2042                 }
2043
2044                 entry->ops = ops;
2045                 entry->ip = rec->ip;
2046
2047                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2048                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2049
2050         } while_for_each_ftrace_rec();
2051         __enable_ftrace_function_probe();
2052
2053  out_unlock:
2054         mutex_unlock(&ftrace_lock);
2055
2056         return count;
2057 }
2058
2059 enum {
2060         PROBE_TEST_FUNC         = 1,
2061         PROBE_TEST_DATA         = 2
2062 };
2063
2064 static void
2065 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2066                                   void *data, int flags)
2067 {
2068         struct ftrace_func_probe *entry;
2069         struct hlist_node *n, *tmp;
2070         char str[KSYM_SYMBOL_LEN];
2071         int type = MATCH_FULL;
2072         int i, len = 0;
2073         char *search;
2074
2075         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2076                 glob = NULL;
2077         else if (glob) {
2078                 int not;
2079
2080                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2081                 len = strlen(search);
2082
2083                 /* we do not support '!' for function probes */
2084                 if (WARN_ON(not))
2085                         return;
2086         }
2087
2088         mutex_lock(&ftrace_lock);
2089         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2090                 struct hlist_head *hhd = &ftrace_func_hash[i];
2091
2092                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2093
2094                         /* break up if statements for readability */
2095                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2096                                 continue;
2097
2098                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2099                                 continue;
2100
2101                         /* do this last, since it is the most expensive */
2102                         if (glob) {
2103                                 kallsyms_lookup(entry->ip, NULL, NULL,
2104                                                 NULL, str);
2105                                 if (!ftrace_match(str, glob, len, type))
2106                                         continue;
2107                         }
2108
2109                         hlist_del(&entry->node);
2110                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2111                 }
2112         }
2113         __disable_ftrace_function_probe();
2114         mutex_unlock(&ftrace_lock);
2115 }
2116
2117 void
2118 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2119                                 void *data)
2120 {
2121         __unregister_ftrace_function_probe(glob, ops, data,
2122                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2123 }
2124
2125 void
2126 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2127 {
2128         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2129 }
2130
2131 void unregister_ftrace_function_probe_all(char *glob)
2132 {
2133         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2134 }
2135
2136 static LIST_HEAD(ftrace_commands);
2137 static DEFINE_MUTEX(ftrace_cmd_mutex);
2138
2139 int register_ftrace_command(struct ftrace_func_command *cmd)
2140 {
2141         struct ftrace_func_command *p;
2142         int ret = 0;
2143
2144         mutex_lock(&ftrace_cmd_mutex);
2145         list_for_each_entry(p, &ftrace_commands, list) {
2146                 if (strcmp(cmd->name, p->name) == 0) {
2147                         ret = -EBUSY;
2148                         goto out_unlock;
2149                 }
2150         }
2151         list_add(&cmd->list, &ftrace_commands);
2152  out_unlock:
2153         mutex_unlock(&ftrace_cmd_mutex);
2154
2155         return ret;
2156 }
2157
2158 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2159 {
2160         struct ftrace_func_command *p, *n;
2161         int ret = -ENODEV;
2162
2163         mutex_lock(&ftrace_cmd_mutex);
2164         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2165                 if (strcmp(cmd->name, p->name) == 0) {
2166                         ret = 0;
2167                         list_del_init(&p->list);
2168                         goto out_unlock;
2169                 }
2170         }
2171  out_unlock:
2172         mutex_unlock(&ftrace_cmd_mutex);
2173
2174         return ret;
2175 }
2176
2177 static int ftrace_process_regex(char *buff, int len, int enable)
2178 {
2179         char *func, *command, *next = buff;
2180         struct ftrace_func_command *p;
2181         int ret = -EINVAL;
2182
2183         func = strsep(&next, ":");
2184
2185         if (!next) {
2186                 ftrace_match_records(func, len, enable);
2187                 return 0;
2188         }
2189
2190         /* command found */
2191
2192         command = strsep(&next, ":");
2193
2194         mutex_lock(&ftrace_cmd_mutex);
2195         list_for_each_entry(p, &ftrace_commands, list) {
2196                 if (strcmp(p->name, command) == 0) {
2197                         ret = p->func(func, command, next, enable);
2198                         goto out_unlock;
2199                 }
2200         }
2201  out_unlock:
2202         mutex_unlock(&ftrace_cmd_mutex);
2203
2204         return ret;
2205 }
2206
2207 static ssize_t
2208 ftrace_regex_write(struct file *file, const char __user *ubuf,
2209                    size_t cnt, loff_t *ppos, int enable)
2210 {
2211         struct ftrace_iterator *iter;
2212         struct trace_parser *parser;
2213         ssize_t ret, read;
2214
2215         if (!cnt)
2216                 return 0;
2217
2218         mutex_lock(&ftrace_regex_lock);
2219
2220         if (file->f_mode & FMODE_READ) {
2221                 struct seq_file *m = file->private_data;
2222                 iter = m->private;
2223         } else
2224                 iter = file->private_data;
2225
2226         parser = &iter->parser;
2227         read = trace_get_user(parser, ubuf, cnt, ppos);
2228
2229         if (read >= 0 && trace_parser_loaded(parser) &&
2230             !trace_parser_cont(parser)) {
2231                 ret = ftrace_process_regex(parser->buffer,
2232                                            parser->idx, enable);
2233                 if (ret)
2234                         goto out_unlock;
2235
2236                 trace_parser_clear(parser);
2237         }
2238
2239         ret = read;
2240 out_unlock:
2241         mutex_unlock(&ftrace_regex_lock);
2242
2243         return ret;
2244 }
2245
2246 static ssize_t
2247 ftrace_filter_write(struct file *file, const char __user *ubuf,
2248                     size_t cnt, loff_t *ppos)
2249 {
2250         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2251 }
2252
2253 static ssize_t
2254 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2255                      size_t cnt, loff_t *ppos)
2256 {
2257         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2258 }
2259
2260 static void
2261 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2262 {
2263         if (unlikely(ftrace_disabled))
2264                 return;
2265
2266         mutex_lock(&ftrace_regex_lock);
2267         if (reset)
2268                 ftrace_filter_reset(enable);
2269         if (buf)
2270                 ftrace_match_records(buf, len, enable);
2271         mutex_unlock(&ftrace_regex_lock);
2272 }
2273
2274 /**
2275  * ftrace_set_filter - set a function to filter on in ftrace
2276  * @buf - the string that holds the function filter text.
2277  * @len - the length of the string.
2278  * @reset - non zero to reset all filters before applying this filter.
2279  *
2280  * Filters denote which functions should be enabled when tracing is enabled.
2281  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2282  */
2283 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2284 {
2285         ftrace_set_regex(buf, len, reset, 1);
2286 }
2287
2288 /**
2289  * ftrace_set_notrace - set a function to not trace in ftrace
2290  * @buf - the string that holds the function notrace text.
2291  * @len - the length of the string.
2292  * @reset - non zero to reset all filters before applying this filter.
2293  *
2294  * Notrace Filters denote which functions should not be enabled when tracing
2295  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2296  * for tracing.
2297  */
2298 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2299 {
2300         ftrace_set_regex(buf, len, reset, 0);
2301 }
2302
2303 /*
2304  * command line interface to allow users to set filters on boot up.
2305  */
2306 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2307 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2308 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2309
2310 static int __init set_ftrace_notrace(char *str)
2311 {
2312         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2313         return 1;
2314 }
2315 __setup("ftrace_notrace=", set_ftrace_notrace);
2316
2317 static int __init set_ftrace_filter(char *str)
2318 {
2319         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2320         return 1;
2321 }
2322 __setup("ftrace_filter=", set_ftrace_filter);
2323
2324 static void __init set_ftrace_early_filter(char *buf, int enable)
2325 {
2326         char *func;
2327
2328         while (buf) {
2329                 func = strsep(&buf, ",");
2330                 ftrace_set_regex(func, strlen(func), 0, enable);
2331         }
2332 }
2333
2334 static void __init set_ftrace_early_filters(void)
2335 {
2336         if (ftrace_filter_buf[0])
2337                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2338         if (ftrace_notrace_buf[0])
2339                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2340 }
2341
2342 static int
2343 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2344 {
2345         struct seq_file *m = (struct seq_file *)file->private_data;
2346         struct ftrace_iterator *iter;
2347         struct trace_parser *parser;
2348
2349         mutex_lock(&ftrace_regex_lock);
2350         if (file->f_mode & FMODE_READ) {
2351                 iter = m->private;
2352
2353                 seq_release(inode, file);
2354         } else
2355                 iter = file->private_data;
2356
2357         parser = &iter->parser;
2358         if (trace_parser_loaded(parser)) {
2359                 parser->buffer[parser->idx] = 0;
2360                 ftrace_match_records(parser->buffer, parser->idx, enable);
2361         }
2362
2363         mutex_lock(&ftrace_lock);
2364         if (ftrace_start_up && ftrace_enabled)
2365                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2366         mutex_unlock(&ftrace_lock);
2367
2368         trace_parser_put(parser);
2369         kfree(iter);
2370
2371         mutex_unlock(&ftrace_regex_lock);
2372         return 0;
2373 }
2374
2375 static int
2376 ftrace_filter_release(struct inode *inode, struct file *file)
2377 {
2378         return ftrace_regex_release(inode, file, 1);
2379 }
2380
2381 static int
2382 ftrace_notrace_release(struct inode *inode, struct file *file)
2383 {
2384         return ftrace_regex_release(inode, file, 0);
2385 }
2386
2387 static const struct file_operations ftrace_avail_fops = {
2388         .open = ftrace_avail_open,
2389         .read = seq_read,
2390         .llseek = seq_lseek,
2391         .release = seq_release_private,
2392 };
2393
2394 static const struct file_operations ftrace_failures_fops = {
2395         .open = ftrace_failures_open,
2396         .read = seq_read,
2397         .llseek = seq_lseek,
2398         .release = seq_release_private,
2399 };
2400
2401 static const struct file_operations ftrace_filter_fops = {
2402         .open = ftrace_filter_open,
2403         .read = seq_read,
2404         .write = ftrace_filter_write,
2405         .llseek = no_llseek,
2406         .release = ftrace_filter_release,
2407 };
2408
2409 static const struct file_operations ftrace_notrace_fops = {
2410         .open = ftrace_notrace_open,
2411         .read = seq_read,
2412         .write = ftrace_notrace_write,
2413         .llseek = ftrace_regex_lseek,
2414         .release = ftrace_notrace_release,
2415 };
2416
2417 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2418
2419 static DEFINE_MUTEX(graph_lock);
2420
2421 int ftrace_graph_count;
2422 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2423
2424 static void *
2425 __g_next(struct seq_file *m, loff_t *pos)
2426 {
2427         if (*pos >= ftrace_graph_count)
2428                 return NULL;
2429         return &ftrace_graph_funcs[*pos];
2430 }
2431
2432 static void *
2433 g_next(struct seq_file *m, void *v, loff_t *pos)
2434 {
2435         (*pos)++;
2436         return __g_next(m, pos);
2437 }
2438
2439 static void *g_start(struct seq_file *m, loff_t *pos)
2440 {
2441         mutex_lock(&graph_lock);
2442
2443         /* Nothing, tell g_show to print all functions are enabled */
2444         if (!ftrace_graph_count && !*pos)
2445                 return (void *)1;
2446
2447         return __g_next(m, pos);
2448 }
2449
2450 static void g_stop(struct seq_file *m, void *p)
2451 {
2452         mutex_unlock(&graph_lock);
2453 }
2454
2455 static int g_show(struct seq_file *m, void *v)
2456 {
2457         unsigned long *ptr = v;
2458
2459         if (!ptr)
2460                 return 0;
2461
2462         if (ptr == (unsigned long *)1) {
2463                 seq_printf(m, "#### all functions enabled ####\n");
2464                 return 0;
2465         }
2466
2467         seq_printf(m, "%ps\n", (void *)*ptr);
2468
2469         return 0;
2470 }
2471
2472 static const struct seq_operations ftrace_graph_seq_ops = {
2473         .start = g_start,
2474         .next = g_next,
2475         .stop = g_stop,
2476         .show = g_show,
2477 };
2478
2479 static int
2480 ftrace_graph_open(struct inode *inode, struct file *file)
2481 {
2482         int ret = 0;
2483
2484         if (unlikely(ftrace_disabled))
2485                 return -ENODEV;
2486
2487         mutex_lock(&graph_lock);
2488         if ((file->f_mode & FMODE_WRITE) &&
2489             (file->f_flags & O_TRUNC)) {
2490                 ftrace_graph_count = 0;
2491                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2492         }
2493         mutex_unlock(&graph_lock);
2494
2495         if (file->f_mode & FMODE_READ)
2496                 ret = seq_open(file, &ftrace_graph_seq_ops);
2497
2498         return ret;
2499 }
2500
2501 static int
2502 ftrace_graph_release(struct inode *inode, struct file *file)
2503 {
2504         if (file->f_mode & FMODE_READ)
2505                 seq_release(inode, file);
2506         return 0;
2507 }
2508
2509 static int
2510 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2511 {
2512         struct dyn_ftrace *rec;
2513         struct ftrace_page *pg;
2514         int search_len;
2515         int found = 0;
2516         int type, not;
2517         char *search;
2518         bool exists;
2519         int i;
2520
2521         if (ftrace_disabled)
2522                 return -ENODEV;
2523
2524         /* decode regex */
2525         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2526         if (not)
2527                 return -EINVAL;
2528
2529         search_len = strlen(search);
2530
2531         mutex_lock(&ftrace_lock);
2532         do_for_each_ftrace_rec(pg, rec) {
2533
2534                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2535                         break;
2536
2537                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2538                         continue;
2539
2540                 if (ftrace_match_record(rec, search, search_len, type)) {
2541                         /* ensure it is not already in the array */
2542                         exists = false;
2543                         for (i = 0; i < *idx; i++)
2544                                 if (array[i] == rec->ip) {
2545                                         exists = true;
2546                                         break;
2547                                 }
2548                         if (!exists) {
2549                                 array[(*idx)++] = rec->ip;
2550                                 found = 1;
2551                         }
2552                 }
2553         } while_for_each_ftrace_rec();
2554
2555         mutex_unlock(&ftrace_lock);
2556
2557         return found ? 0 : -EINVAL;
2558 }
2559
2560 static ssize_t
2561 ftrace_graph_write(struct file *file, const char __user *ubuf,
2562                    size_t cnt, loff_t *ppos)
2563 {
2564         struct trace_parser parser;
2565         ssize_t read, ret;
2566
2567         if (!cnt || cnt < 0)
2568                 return 0;
2569
2570         mutex_lock(&graph_lock);
2571
2572         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2573                 ret = -EBUSY;
2574                 goto out_unlock;
2575         }
2576
2577         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2578                 ret = -ENOMEM;
2579                 goto out_unlock;
2580         }
2581
2582         read = trace_get_user(&parser, ubuf, cnt, ppos);
2583
2584         if (read >= 0 && trace_parser_loaded((&parser))) {
2585                 parser.buffer[parser.idx] = 0;
2586
2587                 /* we allow only one expression at a time */
2588                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2589                                         parser.buffer);
2590                 if (ret)
2591                         goto out_free;
2592         }
2593
2594         ret = read;
2595
2596 out_free:
2597         trace_parser_put(&parser);
2598 out_unlock:
2599         mutex_unlock(&graph_lock);
2600
2601         return ret;
2602 }
2603
2604 static const struct file_operations ftrace_graph_fops = {
2605         .open           = ftrace_graph_open,
2606         .read           = seq_read,
2607         .write          = ftrace_graph_write,
2608         .release        = ftrace_graph_release,
2609 };
2610 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2611
2612 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2613 {
2614
2615         trace_create_file("available_filter_functions", 0444,
2616                         d_tracer, NULL, &ftrace_avail_fops);
2617
2618         trace_create_file("failures", 0444,
2619                         d_tracer, NULL, &ftrace_failures_fops);
2620
2621         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2622                         NULL, &ftrace_filter_fops);
2623
2624         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2625                                     NULL, &ftrace_notrace_fops);
2626
2627 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2628         trace_create_file("set_graph_function", 0444, d_tracer,
2629                                     NULL,
2630                                     &ftrace_graph_fops);
2631 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2632
2633         return 0;
2634 }
2635
2636 static int ftrace_convert_nops(struct module *mod,
2637                                unsigned long *start,
2638                                unsigned long *end)
2639 {
2640         unsigned long *p;
2641         unsigned long addr;
2642         unsigned long flags;
2643
2644         mutex_lock(&ftrace_lock);
2645         p = start;
2646         while (p < end) {
2647                 addr = ftrace_call_adjust(*p++);
2648                 /*
2649                  * Some architecture linkers will pad between
2650                  * the different mcount_loc sections of different
2651                  * object files to satisfy alignments.
2652                  * Skip any NULL pointers.
2653                  */
2654                 if (!addr)
2655                         continue;
2656                 ftrace_record_ip(addr);
2657         }
2658
2659         /* disable interrupts to prevent kstop machine */
2660         local_irq_save(flags);
2661         ftrace_update_code(mod);
2662         local_irq_restore(flags);
2663         mutex_unlock(&ftrace_lock);
2664
2665         return 0;
2666 }
2667
2668 #ifdef CONFIG_MODULES
2669 void ftrace_release_mod(struct module *mod)
2670 {
2671         struct dyn_ftrace *rec;
2672         struct ftrace_page *pg;
2673
2674         if (ftrace_disabled)
2675                 return;
2676
2677         mutex_lock(&ftrace_lock);
2678         do_for_each_ftrace_rec(pg, rec) {
2679                 if (within_module_core(rec->ip, mod)) {
2680                         /*
2681                          * rec->ip is changed in ftrace_free_rec()
2682                          * It should not between s and e if record was freed.
2683                          */
2684                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2685                         ftrace_free_rec(rec);
2686                 }
2687         } while_for_each_ftrace_rec();
2688         mutex_unlock(&ftrace_lock);
2689 }
2690
2691 static void ftrace_init_module(struct module *mod,
2692                                unsigned long *start, unsigned long *end)
2693 {
2694         if (ftrace_disabled || start == end)
2695                 return;
2696         ftrace_convert_nops(mod, start, end);
2697 }
2698
2699 static int ftrace_module_notify(struct notifier_block *self,
2700                                 unsigned long val, void *data)
2701 {
2702         struct module *mod = data;
2703
2704         switch (val) {
2705         case MODULE_STATE_COMING:
2706                 ftrace_init_module(mod, mod->ftrace_callsites,
2707                                    mod->ftrace_callsites +
2708                                    mod->num_ftrace_callsites);
2709                 break;
2710         case MODULE_STATE_GOING:
2711                 ftrace_release_mod(mod);
2712                 break;
2713         }
2714
2715         return 0;
2716 }
2717 #else
2718 static int ftrace_module_notify(struct notifier_block *self,
2719                                 unsigned long val, void *data)
2720 {
2721         return 0;
2722 }
2723 #endif /* CONFIG_MODULES */
2724
2725 struct notifier_block ftrace_module_nb = {
2726         .notifier_call = ftrace_module_notify,
2727         .priority = 0,
2728 };
2729
2730 extern unsigned long __start_mcount_loc[];
2731 extern unsigned long __stop_mcount_loc[];
2732
2733 void __init ftrace_init(void)
2734 {
2735         unsigned long count, addr, flags;
2736         int ret;
2737
2738         /* Keep the ftrace pointer to the stub */
2739         addr = (unsigned long)ftrace_stub;
2740
2741         local_irq_save(flags);
2742         ftrace_dyn_arch_init(&addr);
2743         local_irq_restore(flags);
2744
2745         /* ftrace_dyn_arch_init places the return code in addr */
2746         if (addr)
2747                 goto failed;
2748
2749         count = __stop_mcount_loc - __start_mcount_loc;
2750
2751         ret = ftrace_dyn_table_alloc(count);
2752         if (ret)
2753                 goto failed;
2754
2755         last_ftrace_enabled = ftrace_enabled = 1;
2756
2757         ret = ftrace_convert_nops(NULL,
2758                                   __start_mcount_loc,
2759                                   __stop_mcount_loc);
2760
2761         ret = register_module_notifier(&ftrace_module_nb);
2762         if (ret)
2763                 pr_warning("Failed to register trace ftrace module notifier\n");
2764
2765         set_ftrace_early_filters();
2766
2767         return;
2768  failed:
2769         ftrace_disabled = 1;
2770 }
2771
2772 #else
2773
2774 static int __init ftrace_nodyn_init(void)
2775 {
2776         ftrace_enabled = 1;
2777         return 0;
2778 }
2779 device_initcall(ftrace_nodyn_init);
2780
2781 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2782 static inline void ftrace_startup_enable(int command) { }
2783 /* Keep as macros so we do not need to define the commands */
2784 # define ftrace_startup(command)        do { } while (0)
2785 # define ftrace_shutdown(command)       do { } while (0)
2786 # define ftrace_startup_sysctl()        do { } while (0)
2787 # define ftrace_shutdown_sysctl()       do { } while (0)
2788 #endif /* CONFIG_DYNAMIC_FTRACE */
2789
2790 static ssize_t
2791 ftrace_pid_read(struct file *file, char __user *ubuf,
2792                        size_t cnt, loff_t *ppos)
2793 {
2794         char buf[64];
2795         int r;
2796
2797         if (ftrace_pid_trace == ftrace_swapper_pid)
2798                 r = sprintf(buf, "swapper tasks\n");
2799         else if (ftrace_pid_trace)
2800                 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2801         else
2802                 r = sprintf(buf, "no pid\n");
2803
2804         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2805 }
2806
2807 static void clear_ftrace_swapper(void)
2808 {
2809         struct task_struct *p;
2810         int cpu;
2811
2812         get_online_cpus();
2813         for_each_online_cpu(cpu) {
2814                 p = idle_task(cpu);
2815                 clear_tsk_trace_trace(p);
2816         }
2817         put_online_cpus();
2818 }
2819
2820 static void set_ftrace_swapper(void)
2821 {
2822         struct task_struct *p;
2823         int cpu;
2824
2825         get_online_cpus();
2826         for_each_online_cpu(cpu) {
2827                 p = idle_task(cpu);
2828                 set_tsk_trace_trace(p);
2829         }
2830         put_online_cpus();
2831 }
2832
2833 static void clear_ftrace_pid(struct pid *pid)
2834 {
2835         struct task_struct *p;
2836
2837         rcu_read_lock();
2838         do_each_pid_task(pid, PIDTYPE_PID, p) {
2839                 clear_tsk_trace_trace(p);
2840         } while_each_pid_task(pid, PIDTYPE_PID, p);
2841         rcu_read_unlock();
2842
2843         put_pid(pid);
2844 }
2845
2846 static void set_ftrace_pid(struct pid *pid)
2847 {
2848         struct task_struct *p;
2849
2850         rcu_read_lock();
2851         do_each_pid_task(pid, PIDTYPE_PID, p) {
2852                 set_tsk_trace_trace(p);
2853         } while_each_pid_task(pid, PIDTYPE_PID, p);
2854         rcu_read_unlock();
2855 }
2856
2857 static void clear_ftrace_pid_task(struct pid **pid)
2858 {
2859         if (*pid == ftrace_swapper_pid)
2860                 clear_ftrace_swapper();
2861         else
2862                 clear_ftrace_pid(*pid);
2863
2864         *pid = NULL;
2865 }
2866
2867 static void set_ftrace_pid_task(struct pid *pid)
2868 {
2869         if (pid == ftrace_swapper_pid)
2870                 set_ftrace_swapper();
2871         else
2872                 set_ftrace_pid(pid);
2873 }
2874
2875 static ssize_t
2876 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2877                    size_t cnt, loff_t *ppos)
2878 {
2879         struct pid *pid;
2880         char buf[64];
2881         long val;
2882         int ret;
2883
2884         if (cnt >= sizeof(buf))
2885                 return -EINVAL;
2886
2887         if (copy_from_user(&buf, ubuf, cnt))
2888                 return -EFAULT;
2889
2890         buf[cnt] = 0;
2891
2892         ret = strict_strtol(buf, 10, &val);
2893         if (ret < 0)
2894                 return ret;
2895
2896         mutex_lock(&ftrace_lock);
2897         if (val < 0) {
2898                 /* disable pid tracing */
2899                 if (!ftrace_pid_trace)
2900                         goto out;
2901
2902                 clear_ftrace_pid_task(&ftrace_pid_trace);
2903
2904         } else {
2905                 /* swapper task is special */
2906                 if (!val) {
2907                         pid = ftrace_swapper_pid;
2908                         if (pid == ftrace_pid_trace)
2909                                 goto out;
2910                 } else {
2911                         pid = find_get_pid(val);
2912
2913                         if (pid == ftrace_pid_trace) {
2914                                 put_pid(pid);
2915                                 goto out;
2916                         }
2917                 }
2918
2919                 if (ftrace_pid_trace)
2920                         clear_ftrace_pid_task(&ftrace_pid_trace);
2921
2922                 if (!pid)
2923                         goto out;
2924
2925                 ftrace_pid_trace = pid;
2926
2927                 set_ftrace_pid_task(ftrace_pid_trace);
2928         }
2929
2930         /* update the function call */
2931         ftrace_update_pid_func();
2932         ftrace_startup_enable(0);
2933
2934  out:
2935         mutex_unlock(&ftrace_lock);
2936
2937         return cnt;
2938 }
2939
2940 static const struct file_operations ftrace_pid_fops = {
2941         .read = ftrace_pid_read,
2942         .write = ftrace_pid_write,
2943 };
2944
2945 static __init int ftrace_init_debugfs(void)
2946 {
2947         struct dentry *d_tracer;
2948
2949         d_tracer = tracing_init_dentry();
2950         if (!d_tracer)
2951                 return 0;
2952
2953         ftrace_init_dyn_debugfs(d_tracer);
2954
2955         trace_create_file("set_ftrace_pid", 0644, d_tracer,
2956                             NULL, &ftrace_pid_fops);
2957
2958         ftrace_profile_debugfs(d_tracer);
2959
2960         return 0;
2961 }
2962 fs_initcall(ftrace_init_debugfs);
2963
2964 /**
2965  * ftrace_kill - kill ftrace
2966  *
2967  * This function should be used by panic code. It stops ftrace
2968  * but in a not so nice way. If you need to simply kill ftrace
2969  * from a non-atomic section, use ftrace_kill.
2970  */
2971 void ftrace_kill(void)
2972 {
2973         ftrace_disabled = 1;
2974         ftrace_enabled = 0;
2975         clear_ftrace_function();
2976 }
2977
2978 /**
2979  * register_ftrace_function - register a function for profiling
2980  * @ops - ops structure that holds the function for profiling.
2981  *
2982  * Register a function to be called by all functions in the
2983  * kernel.
2984  *
2985  * Note: @ops->func and all the functions it calls must be labeled
2986  *       with "notrace", otherwise it will go into a
2987  *       recursive loop.
2988  */
2989 int register_ftrace_function(struct ftrace_ops *ops)
2990 {
2991         int ret;
2992
2993         if (unlikely(ftrace_disabled))
2994                 return -1;
2995
2996         mutex_lock(&ftrace_lock);
2997
2998         ret = __register_ftrace_function(ops);
2999         ftrace_startup(0);
3000
3001         mutex_unlock(&ftrace_lock);
3002         return ret;
3003 }
3004
3005 /**
3006  * unregister_ftrace_function - unregister a function for profiling.
3007  * @ops - ops structure that holds the function to unregister
3008  *
3009  * Unregister a function that was added to be called by ftrace profiling.
3010  */
3011 int unregister_ftrace_function(struct ftrace_ops *ops)
3012 {
3013         int ret;
3014
3015         mutex_lock(&ftrace_lock);
3016         ret = __unregister_ftrace_function(ops);
3017         ftrace_shutdown(0);
3018         mutex_unlock(&ftrace_lock);
3019
3020         return ret;
3021 }
3022
3023 int
3024 ftrace_enable_sysctl(struct ctl_table *table, int write,
3025                      void __user *buffer, size_t *lenp,
3026                      loff_t *ppos)
3027 {
3028         int ret;
3029
3030         if (unlikely(ftrace_disabled))
3031                 return -ENODEV;
3032
3033         mutex_lock(&ftrace_lock);
3034
3035         ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3036
3037         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3038                 goto out;
3039
3040         last_ftrace_enabled = !!ftrace_enabled;
3041
3042         if (ftrace_enabled) {
3043
3044                 ftrace_startup_sysctl();
3045
3046                 /* we are starting ftrace again */
3047                 if (ftrace_list != &ftrace_list_end) {
3048                         if (ftrace_list->next == &ftrace_list_end)
3049                                 ftrace_trace_function = ftrace_list->func;
3050                         else
3051                                 ftrace_trace_function = ftrace_list_func;
3052                 }
3053
3054         } else {
3055                 /* stopping ftrace calls (just send to ftrace_stub) */
3056                 ftrace_trace_function = ftrace_stub;
3057
3058                 ftrace_shutdown_sysctl();
3059         }
3060
3061  out:
3062         mutex_unlock(&ftrace_lock);
3063         return ret;
3064 }
3065
3066 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3067
3068 static int ftrace_graph_active;
3069 static struct notifier_block ftrace_suspend_notifier;
3070
3071 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3072 {
3073         return 0;
3074 }
3075
3076 /* The callbacks that hook a function */
3077 trace_func_graph_ret_t ftrace_graph_return =
3078                         (trace_func_graph_ret_t)ftrace_stub;
3079 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3080
3081 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3082 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3083 {
3084         int i;
3085         int ret = 0;
3086         unsigned long flags;
3087         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3088         struct task_struct *g, *t;
3089
3090         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3091                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3092                                         * sizeof(struct ftrace_ret_stack),
3093                                         GFP_KERNEL);
3094                 if (!ret_stack_list[i]) {
3095                         start = 0;
3096                         end = i;
3097                         ret = -ENOMEM;
3098                         goto free;
3099                 }
3100         }
3101
3102         read_lock_irqsave(&tasklist_lock, flags);
3103         do_each_thread(g, t) {
3104                 if (start == end) {
3105                         ret = -EAGAIN;
3106                         goto unlock;
3107                 }
3108
3109                 if (t->ret_stack == NULL) {
3110                         atomic_set(&t->tracing_graph_pause, 0);
3111                         atomic_set(&t->trace_overrun, 0);
3112                         t->curr_ret_stack = -1;
3113                         /* Make sure the tasks see the -1 first: */
3114                         smp_wmb();
3115                         t->ret_stack = ret_stack_list[start++];
3116                 }
3117         } while_each_thread(g, t);
3118
3119 unlock:
3120         read_unlock_irqrestore(&tasklist_lock, flags);
3121 free:
3122         for (i = start; i < end; i++)
3123                 kfree(ret_stack_list[i]);
3124         return ret;
3125 }
3126
3127 static void
3128 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3129                                 struct task_struct *next)
3130 {
3131         unsigned long long timestamp;
3132         int index;
3133
3134         /*
3135          * Does the user want to count the time a function was asleep.
3136          * If so, do not update the time stamps.
3137          */
3138         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3139                 return;
3140
3141         timestamp = trace_clock_local();
3142
3143         prev->ftrace_timestamp = timestamp;
3144
3145         /* only process tasks that we timestamped */
3146         if (!next->ftrace_timestamp)
3147                 return;
3148
3149         /*
3150          * Update all the counters in next to make up for the
3151          * time next was sleeping.
3152          */
3153         timestamp -= next->ftrace_timestamp;
3154
3155         for (index = next->curr_ret_stack; index >= 0; index--)
3156                 next->ret_stack[index].calltime += timestamp;
3157 }
3158
3159 /* Allocate a return stack for each task */
3160 static int start_graph_tracing(void)
3161 {
3162         struct ftrace_ret_stack **ret_stack_list;
3163         int ret, cpu;
3164
3165         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3166                                 sizeof(struct ftrace_ret_stack *),
3167                                 GFP_KERNEL);
3168
3169         if (!ret_stack_list)
3170                 return -ENOMEM;
3171
3172         /* The cpu_boot init_task->ret_stack will never be freed */
3173         for_each_online_cpu(cpu) {
3174                 if (!idle_task(cpu)->ret_stack)
3175                         ftrace_graph_init_idle_task(idle_task(cpu), cpu);
3176         }
3177
3178         do {
3179                 ret = alloc_retstack_tasklist(ret_stack_list);
3180         } while (ret == -EAGAIN);
3181
3182         if (!ret) {
3183                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3184                 if (ret)
3185                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3186                                 " probe to kernel_sched_switch\n");
3187         }
3188
3189         kfree(ret_stack_list);
3190         return ret;
3191 }
3192
3193 /*
3194  * Hibernation protection.
3195  * The state of the current task is too much unstable during
3196  * suspend/restore to disk. We want to protect against that.
3197  */
3198 static int
3199 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3200                                                         void *unused)
3201 {
3202         switch (state) {
3203         case PM_HIBERNATION_PREPARE:
3204                 pause_graph_tracing();
3205                 break;
3206
3207         case PM_POST_HIBERNATION:
3208                 unpause_graph_tracing();
3209                 break;
3210         }
3211         return NOTIFY_DONE;
3212 }
3213
3214 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3215                         trace_func_graph_ent_t entryfunc)
3216 {
3217         int ret = 0;
3218
3219         mutex_lock(&ftrace_lock);
3220
3221         /* we currently allow only one tracer registered at a time */
3222         if (ftrace_graph_active) {
3223                 ret = -EBUSY;
3224                 goto out;
3225         }
3226
3227         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3228         register_pm_notifier(&ftrace_suspend_notifier);
3229
3230         ftrace_graph_active++;
3231         ret = start_graph_tracing();
3232         if (ret) {
3233                 ftrace_graph_active--;
3234                 goto out;
3235         }
3236
3237         ftrace_graph_return = retfunc;
3238         ftrace_graph_entry = entryfunc;
3239
3240         ftrace_startup(FTRACE_START_FUNC_RET);
3241
3242 out:
3243         mutex_unlock(&ftrace_lock);
3244         return ret;
3245 }
3246
3247 void unregister_ftrace_graph(void)
3248 {
3249         mutex_lock(&ftrace_lock);
3250
3251         if (unlikely(!ftrace_graph_active))
3252                 goto out;
3253
3254         ftrace_graph_active--;
3255         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3256         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3257         ftrace_graph_entry = ftrace_graph_entry_stub;
3258         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3259         unregister_pm_notifier(&ftrace_suspend_notifier);
3260
3261  out:
3262         mutex_unlock(&ftrace_lock);
3263 }
3264
3265 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
3266
3267 static void
3268 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
3269 {
3270         atomic_set(&t->tracing_graph_pause, 0);
3271         atomic_set(&t->trace_overrun, 0);
3272         t->ftrace_timestamp = 0;
3273         /* make curr_ret_stack visable before we add the ret_stack */
3274         smp_wmb();
3275         t->ret_stack = ret_stack;
3276 }
3277
3278 /*
3279  * Allocate a return stack for the idle task. May be the first
3280  * time through, or it may be done by CPU hotplug online.
3281  */
3282 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
3283 {
3284         t->curr_ret_stack = -1;
3285         /*
3286          * The idle task has no parent, it either has its own
3287          * stack or no stack at all.
3288          */
3289         if (t->ret_stack)
3290                 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
3291
3292         if (ftrace_graph_active) {
3293                 struct ftrace_ret_stack *ret_stack;
3294
3295                 ret_stack = per_cpu(idle_ret_stack, cpu);
3296                 if (!ret_stack) {
3297                         ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3298                                             * sizeof(struct ftrace_ret_stack),
3299                                             GFP_KERNEL);
3300                         if (!ret_stack)
3301                                 return;
3302                         per_cpu(idle_ret_stack, cpu) = ret_stack;
3303                 }
3304                 graph_init_task(t, ret_stack);
3305         }
3306 }
3307
3308 /* Allocate a return stack for newly created task */
3309 void ftrace_graph_init_task(struct task_struct *t)
3310 {
3311         /* Make sure we do not use the parent ret_stack */
3312         t->ret_stack = NULL;
3313         t->curr_ret_stack = -1;
3314
3315         if (ftrace_graph_active) {
3316                 struct ftrace_ret_stack *ret_stack;
3317
3318                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3319                                 * sizeof(struct ftrace_ret_stack),
3320                                 GFP_KERNEL);
3321                 if (!ret_stack)
3322                         return;
3323                 graph_init_task(t, ret_stack);
3324         }
3325 }
3326
3327 void ftrace_graph_exit_task(struct task_struct *t)
3328 {
3329         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3330
3331         t->ret_stack = NULL;
3332         /* NULL must become visible to IRQs before we free it: */
3333         barrier();
3334
3335         kfree(ret_stack);
3336 }
3337
3338 void ftrace_graph_stop(void)
3339 {
3340         ftrace_stop();
3341 }
3342 #endif
3343