kernel/trace/trace_functions.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Based on code from the latency_tracer, that is:
   8  *
   9  *  Copyright (C) 2004-2006 Ingo Molnar
  10  *  Copyright (C) 2004 Nadia Yvette Chambers
  11  */
  12 #include <linux/ring_buffer.h>
  13 #include <linux/debugfs.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/ftrace.h>
  16 #include <linux/slab.h>
  17 #include <linux/fs.h>
  18
  19 #include "trace.h"
  20
  21 static void tracing_start_function_trace(struct trace_array *tr);
  22 static void tracing_stop_function_trace(struct trace_array *tr);
  23 static void
  24 function_trace_call(unsigned long ip, unsigned long parent_ip,
  25                     struct ftrace_ops *op, struct pt_regs *pt_regs);
  26 static void
  27 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
  28                           struct ftrace_ops *op, struct pt_regs *pt_regs);
  29 static struct tracer_flags func_flags;
  30
  31 /* Our option */
  32 enum {
  33         TRACE_FUNC_OPT_STACK    = 0x1,
  34 };
  35
  36 static int allocate_ftrace_ops(struct trace_array *tr)
  37 {
  38         struct ftrace_ops *ops;
  39
  40         ops = kzalloc(sizeof(*ops), GFP_KERNEL);
  41         if (!ops)
  42                 return -ENOMEM;
  43
  44         /* Currently only the non stack verision is supported */
  45         ops->func = function_trace_call;
  46         ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;
  47
  48         tr->ops = ops;
  49         ops->private = tr;
  50         return 0;
  51 }
  52
  53
  54 int ftrace_create_function_files(struct trace_array *tr,
  55                                  struct dentry *parent)
  56 {
  57         int ret;
  58
  59         /*
  60          * The top level array uses the "global_ops", and the files are
  61          * created on boot up.
  62          */
  63         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  64                 return 0;
  65
  66         ret = allocate_ftrace_ops(tr);
  67         if (ret)
  68                 return ret;
  69
  70         ftrace_create_filter_files(tr->ops, parent);
  71
  72         return 0;
  73 }
  74
  75 void ftrace_destroy_function_files(struct trace_array *tr)
  76 {
  77         ftrace_destroy_filter_files(tr->ops);
  78         kfree(tr->ops);
  79         tr->ops = NULL;
  80 }
  81
  82 static int function_trace_init(struct trace_array *tr)
  83 {
  84         ftrace_func_t func;
  85
  86         /*
  87          * Instance trace_arrays get their ops allocated
  88          * at instance creation. Unless it failed
  89          * the allocation.
  90          */
  91         if (!tr->ops)
  92                 return -ENOMEM;
  93
  94         /* Currently only the global instance can do stack tracing */
  95         if (tr->flags & TRACE_ARRAY_FL_GLOBAL &&
  96             func_flags.val & TRACE_FUNC_OPT_STACK)
  97                 func = function_stack_trace_call;
  98         else
  99                 func = function_trace_call;
 100
 101         ftrace_init_array_ops(tr, func);
 102
 103         tr->trace_buffer.cpu = get_cpu();
 104         put_cpu();
 105
 106         tracing_start_cmdline_record();
 107         tracing_start_function_trace(tr);
 108         return 0;
 109 }
 110
 111 static void function_trace_reset(struct trace_array *tr)
 112 {
 113         tracing_stop_function_trace(tr);
 114         tracing_stop_cmdline_record();
 115         ftrace_reset_array_ops(tr);
 116 }
 117
 118 static void function_trace_start(struct trace_array *tr)
 119 {
 120         tracing_reset_online_cpus(&tr->trace_buffer);
 121 }
 122
 123 static void
 124 function_trace_call(unsigned long ip, unsigned long parent_ip,
 125                     struct ftrace_ops *op, struct pt_regs *pt_regs)
 126 {
 127         struct trace_array *tr = op->private;
 128         struct trace_array_cpu *data;
 129         unsigned long flags;
 130         int bit;
 131         int cpu;
 132         int pc;
 133
 134         if (unlikely(!tr->function_enabled))
 135                 return;
 136
 137         pc = preempt_count();
 138         preempt_disable_notrace();
 139
 140         bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX);
 141         if (bit < 0)
 142                 goto out;
 143
 144         cpu = smp_processor_id();
 145         data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 146         if (!atomic_read(&data->disabled)) {
 147                 local_save_flags(flags);
 148                 trace_function(tr, ip, parent_ip, flags, pc);
 149         }
 150         trace_clear_recursion(bit);
 151
 152  out:
 153         preempt_enable_notrace();
 154 }
 155
 156 static void
 157 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 158                           struct ftrace_ops *op, struct pt_regs *pt_regs)
 159 {
 160         struct trace_array *tr = op->private;
 161         struct trace_array_cpu *data;
 162         unsigned long flags;
 163         long disabled;
 164         int cpu;
 165         int pc;
 166
 167         if (unlikely(!tr->function_enabled))
 168                 return;
 169
 170         /*
 171          * Need to use raw, since this must be called before the
 172          * recursive protection is performed.
 173          */
 174         local_irq_save(flags);
 175         cpu = raw_smp_processor_id();
 176         data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 177         disabled = atomic_inc_return(&data->disabled);
 178
 179         if (likely(disabled == 1)) {
 180                 pc = preempt_count();
 181                 trace_function(tr, ip, parent_ip, flags, pc);
 182                 /*
 183                  * skip over 5 funcs:
 184                  *    __ftrace_trace_stack,
 185                  *    __trace_stack,
 186                  *    function_stack_trace_call
 187                  *    ftrace_list_func
 188                  *    ftrace_call
 189                  */
 190                 __trace_stack(tr, flags, 5, pc);
 191         }
 192
 193         atomic_dec(&data->disabled);
 194         local_irq_restore(flags);
 195 }
 196
 197 static struct tracer_opt func_opts[] = {
 198 #ifdef CONFIG_STACKTRACE
 199         { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
 200 #endif
 201         { } /* Always set a last empty entry */
 202 };
 203
 204 static struct tracer_flags func_flags = {
 205         .val = 0, /* By default: all flags disabled */
 206         .opts = func_opts
 207 };
 208
 209 static void tracing_start_function_trace(struct trace_array *tr)
 210 {
 211         tr->function_enabled = 0;
 212         register_ftrace_function(tr->ops);
 213         tr->function_enabled = 1;
 214 }
 215
 216 static void tracing_stop_function_trace(struct trace_array *tr)
 217 {
 218         tr->function_enabled = 0;
 219         unregister_ftrace_function(tr->ops);
 220 }
 221
 222 static int
 223 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
 224 {
 225         switch (bit) {
 226         case TRACE_FUNC_OPT_STACK:
 227                 /* do nothing if already set */
 228                 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
 229                         break;
 230
 231                 unregister_ftrace_function(tr->ops);
 232
 233                 if (set) {
 234                         tr->ops->func = function_stack_trace_call;
 235                         register_ftrace_function(tr->ops);
 236                 } else {
 237                         tr->ops->func = function_trace_call;
 238                         register_ftrace_function(tr->ops);
 239                 }
 240
 241                 break;
 242         default:
 243                 return -EINVAL;
 244         }
 245
 246         return 0;
 247 }
 248
 249 static struct tracer function_trace __tracer_data =
 250 {
 251         .name           = "function",
 252         .init           = function_trace_init,
 253         .reset          = function_trace_reset,
 254         .start          = function_trace_start,
 255         .wait_pipe      = poll_wait_pipe,
 256         .flags          = &func_flags,
 257         .set_flag       = func_set_flag,
 258         .allow_instances = true,
 259 #ifdef CONFIG_FTRACE_SELFTEST
 260         .selftest       = trace_selftest_startup_function,
 261 #endif
 262 };
 263
 264 #ifdef CONFIG_DYNAMIC_FTRACE
 265 static int update_count(void **data)
 266 {
 267         unsigned long *count = (long *)data;
 268
 269         if (!*count)
 270                 return 0;
 271
 272         if (*count != -1)
 273                 (*count)--;
 274
 275         return 1;
 276 }
 277
 278 static void
 279 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data)
 280 {
 281         if (tracing_is_on())
 282                 return;
 283
 284         if (update_count(data))
 285                 tracing_on();
 286 }
 287
 288 static void
 289 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data)
 290 {
 291         if (!tracing_is_on())
 292                 return;
 293
 294         if (update_count(data))
 295                 tracing_off();
 296 }
 297
 298 static void
 299 ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
 300 {
 301         if (tracing_is_on())
 302                 return;
 303
 304         tracing_on();
 305 }
 306
 307 static void
 308 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
 309 {
 310         if (!tracing_is_on())
 311                 return;
 312
 313         tracing_off();
 314 }
 315
 316 /*
 317  * Skip 4:
 318  *   ftrace_stacktrace()
 319  *   function_trace_probe_call()
 320  *   ftrace_ops_list_func()
 321  *   ftrace_call()
 322  */
 323 #define STACK_SKIP 4
 324
 325 static void
 326 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data)
 327 {
 328         trace_dump_stack(STACK_SKIP);
 329 }
 330
 331 static void
 332 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data)
 333 {
 334         if (!tracing_is_on())
 335                 return;
 336
 337         if (update_count(data))
 338                 trace_dump_stack(STACK_SKIP);
 339 }
 340
 341 static void
 342 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data)
 343 {
 344         if (update_count(data))
 345                 ftrace_dump(DUMP_ALL);
 346 }
 347
 348 /* Only dump the current CPU buffer. */
 349 static void
 350 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data)
 351 {
 352         if (update_count(data))
 353                 ftrace_dump(DUMP_ORIG);
 354 }
 355
 356 static int
 357 ftrace_probe_print(const char *name, struct seq_file *m,
 358                    unsigned long ip, void *data)
 359 {
 360         long count = (long)data;
 361
 362         seq_printf(m, "%ps:%s", (void *)ip, name);
 363
 364         if (count == -1)
 365                 seq_printf(m, ":unlimited\n");
 366         else
 367                 seq_printf(m, ":count=%ld\n", count);
 368
 369         return 0;
 370 }
 371
 372 static int
 373 ftrace_traceon_print(struct seq_file *m, unsigned long ip,
 374                          struct ftrace_probe_ops *ops, void *data)
 375 {
 376         return ftrace_probe_print("traceon", m, ip, data);
 377 }
 378
 379 static int
 380 ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
 381                          struct ftrace_probe_ops *ops, void *data)
 382 {
 383         return ftrace_probe_print("traceoff", m, ip, data);
 384 }
 385
 386 static int
 387 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
 388                         struct ftrace_probe_ops *ops, void *data)
 389 {
 390         return ftrace_probe_print("stacktrace", m, ip, data);
 391 }
 392
 393 static int
 394 ftrace_dump_print(struct seq_file *m, unsigned long ip,
 395                         struct ftrace_probe_ops *ops, void *data)
 396 {
 397         return ftrace_probe_print("dump", m, ip, data);
 398 }
 399
 400 static int
 401 ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
 402                         struct ftrace_probe_ops *ops, void *data)
 403 {
 404         return ftrace_probe_print("cpudump", m, ip, data);
 405 }
 406
 407 static struct ftrace_probe_ops traceon_count_probe_ops = {
 408         .func                   = ftrace_traceon_count,
 409         .print                  = ftrace_traceon_print,
 410 };
 411
 412 static struct ftrace_probe_ops traceoff_count_probe_ops = {
 413         .func                   = ftrace_traceoff_count,
 414         .print                  = ftrace_traceoff_print,
 415 };
 416
 417 static struct ftrace_probe_ops stacktrace_count_probe_ops = {
 418         .func                   = ftrace_stacktrace_count,
 419         .print                  = ftrace_stacktrace_print,
 420 };
 421
 422 static struct ftrace_probe_ops dump_probe_ops = {
 423         .func                   = ftrace_dump_probe,
 424         .print                  = ftrace_dump_print,
 425 };
 426
 427 static struct ftrace_probe_ops cpudump_probe_ops = {
 428         .func                   = ftrace_cpudump_probe,
 429         .print                  = ftrace_cpudump_print,
 430 };
 431
 432 static struct ftrace_probe_ops traceon_probe_ops = {
 433         .func                   = ftrace_traceon,
 434         .print                  = ftrace_traceon_print,
 435 };
 436
 437 static struct ftrace_probe_ops traceoff_probe_ops = {
 438         .func                   = ftrace_traceoff,
 439         .print                  = ftrace_traceoff_print,
 440 };
 441
 442 static struct ftrace_probe_ops stacktrace_probe_ops = {
 443         .func                   = ftrace_stacktrace,
 444         .print                  = ftrace_stacktrace_print,
 445 };
 446
 447 static int
 448 ftrace_trace_probe_callback(struct ftrace_probe_ops *ops,
 449                             struct ftrace_hash *hash, char *glob,
 450                             char *cmd, char *param, int enable)
 451 {
 452         void *count = (void *)-1;
 453         char *number;
 454         int ret;
 455
 456         /* hash funcs only work with set_ftrace_filter */
 457         if (!enable)
 458                 return -EINVAL;
 459
 460         if (glob[0] == '!') {
 461                 unregister_ftrace_function_probe_func(glob+1, ops);
 462                 return 0;
 463         }
 464
 465         if (!param)
 466                 goto out_reg;
 467
 468         number = strsep(&param, ":");
 469
 470         if (!strlen(number))
 471                 goto out_reg;
 472
 473         /*
 474          * We use the callback data field (which is a pointer)
 475          * as our counter.
 476          */
 477         ret = kstrtoul(number, 0, (unsigned long *)&count);
 478         if (ret)
 479                 return ret;
 480
 481  out_reg:
 482         ret = register_ftrace_function_probe(glob, ops, count);
 483
 484         return ret < 0 ? ret : 0;
 485 }
 486
 487 static int
 488 ftrace_trace_onoff_callback(struct ftrace_hash *hash,
 489                             char *glob, char *cmd, char *param, int enable)
 490 {
 491         struct ftrace_probe_ops *ops;
 492
 493         /* we register both traceon and traceoff to this callback */
 494         if (strcmp(cmd, "traceon") == 0)
 495                 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
 496         else
 497                 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
 498
 499         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 500                                            param, enable);
 501 }
 502
 503 static int
 504 ftrace_stacktrace_callback(struct ftrace_hash *hash,
 505                            char *glob, char *cmd, char *param, int enable)
 506 {
 507         struct ftrace_probe_ops *ops;
 508
 509         ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
 510
 511         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 512                                            param, enable);
 513 }
 514
 515 static int
 516 ftrace_dump_callback(struct ftrace_hash *hash,
 517                            char *glob, char *cmd, char *param, int enable)
 518 {
 519         struct ftrace_probe_ops *ops;
 520
 521         ops = &dump_probe_ops;
 522
 523         /* Only dump once. */
 524         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 525                                            "1", enable);
 526 }
 527
 528 static int
 529 ftrace_cpudump_callback(struct ftrace_hash *hash,
 530                            char *glob, char *cmd, char *param, int enable)
 531 {
 532         struct ftrace_probe_ops *ops;
 533
 534         ops = &cpudump_probe_ops;
 535
 536         /* Only dump once. */
 537         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 538                                            "1", enable);
 539 }
 540
 541 static struct ftrace_func_command ftrace_traceon_cmd = {
 542         .name                   = "traceon",
 543         .func                   = ftrace_trace_onoff_callback,
 544 };
 545
 546 static struct ftrace_func_command ftrace_traceoff_cmd = {
 547         .name                   = "traceoff",
 548         .func                   = ftrace_trace_onoff_callback,
 549 };
 550
 551 static struct ftrace_func_command ftrace_stacktrace_cmd = {
 552         .name                   = "stacktrace",
 553         .func                   = ftrace_stacktrace_callback,
 554 };
 555
 556 static struct ftrace_func_command ftrace_dump_cmd = {
 557         .name                   = "dump",
 558         .func                   = ftrace_dump_callback,
 559 };
 560
 561 static struct ftrace_func_command ftrace_cpudump_cmd = {
 562         .name                   = "cpudump",
 563         .func                   = ftrace_cpudump_callback,
 564 };
 565
 566 static int __init init_func_cmd_traceon(void)
 567 {
 568         int ret;
 569
 570         ret = register_ftrace_command(&ftrace_traceoff_cmd);
 571         if (ret)
 572                 return ret;
 573
 574         ret = register_ftrace_command(&ftrace_traceon_cmd);
 575         if (ret)
 576                 goto out_free_traceoff;
 577
 578         ret = register_ftrace_command(&ftrace_stacktrace_cmd);
 579         if (ret)
 580                 goto out_free_traceon;
 581
 582         ret = register_ftrace_command(&ftrace_dump_cmd);
 583         if (ret)
 584                 goto out_free_stacktrace;
 585
 586         ret = register_ftrace_command(&ftrace_cpudump_cmd);
 587         if (ret)
 588                 goto out_free_dump;
 589
 590         return 0;
 591
 592  out_free_dump:
 593         unregister_ftrace_command(&ftrace_dump_cmd);
 594  out_free_stacktrace:
 595         unregister_ftrace_command(&ftrace_stacktrace_cmd);
 596  out_free_traceon:
 597         unregister_ftrace_command(&ftrace_traceon_cmd);
 598  out_free_traceoff:
 599         unregister_ftrace_command(&ftrace_traceoff_cmd);
 600
 601         return ret;
 602 }
 603 #else
 604 static inline int init_func_cmd_traceon(void)
 605 {
 606         return 0;
 607 }
 608 #endif /* CONFIG_DYNAMIC_FTRACE */
 609
 610 static __init int init_function_trace(void)
 611 {
 612         init_func_cmd_traceon();
 613         return register_tracer(&function_trace);
 614 }
 615 core_initcall(init_function_trace);