4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace;
25 static int branch_tracing_enabled __read_mostly;
26 static DEFINE_MUTEX(branch_tracing_mutex);
28 static struct trace_array *branch_tracer;
31 probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
33 struct ftrace_event_call *call = &event_branch;
34 struct trace_array *tr = branch_tracer;
35 struct trace_array_cpu *data;
36 struct ring_buffer_event *event;
37 struct trace_branch *entry;
38 struct ring_buffer *buffer;
43 if (current->trace_recursion & TRACE_BRANCH_BIT)
47 * I would love to save just the ftrace_likely_data pointer, but
48 * this code can also be used by modules. Ugly things can happen
49 * if the module is unloaded, and then we go and read the
50 * pointer. This is slower, but much safer.
56 raw_local_irq_save(flags);
57 current->trace_recursion |= TRACE_BRANCH_BIT;
58 data = this_cpu_ptr(tr->trace_buffer.data);
59 if (atomic_read(&data->disabled))
63 buffer = tr->trace_buffer.buffer;
64 event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
65 sizeof(*entry), flags, pc);
69 entry = ring_buffer_event_data(event);
71 /* Strip off the path, only save the file */
72 p = f->file + strlen(f->file);
73 while (p >= f->file && *p != '/')
77 strncpy(entry->func, f->func, TRACE_FUNC_SIZE);
78 strncpy(entry->file, p, TRACE_FILE_SIZE);
79 entry->func[TRACE_FUNC_SIZE] = 0;
80 entry->file[TRACE_FILE_SIZE] = 0;
81 entry->line = f->line;
82 entry->correct = val == expect;
84 if (!filter_check_discard(call, entry, buffer, event))
85 __buffer_unlock_commit(buffer, event);
88 current->trace_recursion &= ~TRACE_BRANCH_BIT;
89 raw_local_irq_restore(flags);
93 void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
95 if (!branch_tracing_enabled)
98 probe_likely_condition(f, val, expect);
101 int enable_branch_tracing(struct trace_array *tr)
103 mutex_lock(&branch_tracing_mutex);
106 * Must be seen before enabling. The reader is a condition
107 * where we do not need a matching rmb()
110 branch_tracing_enabled++;
111 mutex_unlock(&branch_tracing_mutex);
116 void disable_branch_tracing(void)
118 mutex_lock(&branch_tracing_mutex);
120 if (!branch_tracing_enabled)
123 branch_tracing_enabled--;
126 mutex_unlock(&branch_tracing_mutex);
129 static void start_branch_trace(struct trace_array *tr)
131 enable_branch_tracing(tr);
134 static void stop_branch_trace(struct trace_array *tr)
136 disable_branch_tracing();
139 static int branch_trace_init(struct trace_array *tr)
141 start_branch_trace(tr);
145 static void branch_trace_reset(struct trace_array *tr)
147 stop_branch_trace(tr);
150 static enum print_line_t trace_branch_print(struct trace_iterator *iter,
151 int flags, struct trace_event *event)
153 struct trace_branch *field;
155 trace_assign_type(field, iter->ent);
157 if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
158 field->correct ? " ok " : " MISS ",
162 return TRACE_TYPE_PARTIAL_LINE;
164 return TRACE_TYPE_HANDLED;
167 static void branch_print_header(struct seq_file *s)
169 seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT"
170 " FUNC:FILE:LINE\n");
171 seq_puts(s, "# | | | | | "
175 static struct trace_event_functions trace_branch_funcs = {
176 .trace = trace_branch_print,
179 static struct trace_event trace_branch_event = {
180 .type = TRACE_BRANCH,
181 .funcs = &trace_branch_funcs,
184 static struct tracer branch_trace __read_mostly =
187 .init = branch_trace_init,
188 .reset = branch_trace_reset,
189 #ifdef CONFIG_FTRACE_SELFTEST
190 .selftest = trace_selftest_startup_branch,
191 #endif /* CONFIG_FTRACE_SELFTEST */
192 .print_header = branch_print_header,
195 __init static int init_branch_tracer(void)
199 ret = register_ftrace_event(&trace_branch_event);
201 printk(KERN_WARNING "Warning: could not register "
205 return register_tracer(&branch_trace);
207 core_initcall(init_branch_tracer);
211 void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
214 #endif /* CONFIG_BRANCH_TRACER */
216 void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
219 * I would love to have a trace point here instead, but the
220 * trace point code is so inundated with unlikely and likely
221 * conditions that the recursive nightmare that exists is too
222 * much to try to get working. At least for now.
224 trace_likely_condition(f, val, expect);
226 /* FIXME: Make this atomic! */
232 EXPORT_SYMBOL(ftrace_likely_update);
234 extern unsigned long __start_annotated_branch_profile[];
235 extern unsigned long __stop_annotated_branch_profile[];
237 static int annotated_branch_stat_headers(struct seq_file *m)
239 seq_printf(m, " correct incorrect %% ");
240 seq_printf(m, " Function "
242 " ------- --------- - "
248 static inline long get_incorrect_percent(struct ftrace_branch_data *p)
253 percent = p->incorrect * 100;
254 percent /= p->correct + p->incorrect;
256 percent = p->incorrect ? 100 : -1;
261 static int branch_stat_show(struct seq_file *m, void *v)
263 struct ftrace_branch_data *p = v;
267 /* Only print the file, not the path */
268 f = p->file + strlen(p->file);
269 while (f >= p->file && *f != '/')
274 * The miss is overlayed on correct, and hit on incorrect.
276 percent = get_incorrect_percent(p);
278 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
280 seq_printf(m, " X ");
282 seq_printf(m, "%3ld ", percent);
283 seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
287 static void *annotated_branch_stat_start(struct tracer_stat *trace)
289 return __start_annotated_branch_profile;
293 annotated_branch_stat_next(void *v, int idx)
295 struct ftrace_branch_data *p = v;
299 if ((void *)p >= (void *)__stop_annotated_branch_profile)
305 static int annotated_branch_stat_cmp(void *p1, void *p2)
307 struct ftrace_branch_data *a = p1;
308 struct ftrace_branch_data *b = p2;
310 long percent_a, percent_b;
312 percent_a = get_incorrect_percent(a);
313 percent_b = get_incorrect_percent(b);
315 if (percent_a < percent_b)
317 if (percent_a > percent_b)
320 if (a->incorrect < b->incorrect)
322 if (a->incorrect > b->incorrect)
326 * Since the above shows worse (incorrect) cases
327 * first, we continue that by showing best (correct)
330 if (a->correct > b->correct)
332 if (a->correct < b->correct)
338 static struct tracer_stat annotated_branch_stats = {
339 .name = "branch_annotated",
340 .stat_start = annotated_branch_stat_start,
341 .stat_next = annotated_branch_stat_next,
342 .stat_cmp = annotated_branch_stat_cmp,
343 .stat_headers = annotated_branch_stat_headers,
344 .stat_show = branch_stat_show
347 __init static int init_annotated_branch_stats(void)
351 ret = register_stat_tracer(&annotated_branch_stats);
353 printk(KERN_WARNING "Warning: could not register "
354 "annotated branches stats\n");
359 fs_initcall(init_annotated_branch_stats);
361 #ifdef CONFIG_PROFILE_ALL_BRANCHES
363 extern unsigned long __start_branch_profile[];
364 extern unsigned long __stop_branch_profile[];
366 static int all_branch_stat_headers(struct seq_file *m)
368 seq_printf(m, " miss hit %% ");
369 seq_printf(m, " Function "
371 " ------- --------- - "
377 static void *all_branch_stat_start(struct tracer_stat *trace)
379 return __start_branch_profile;
383 all_branch_stat_next(void *v, int idx)
385 struct ftrace_branch_data *p = v;
389 if ((void *)p >= (void *)__stop_branch_profile)
395 static struct tracer_stat all_branch_stats = {
396 .name = "branch_all",
397 .stat_start = all_branch_stat_start,
398 .stat_next = all_branch_stat_next,
399 .stat_headers = all_branch_stat_headers,
400 .stat_show = branch_stat_show
403 __init static int all_annotated_branch_stats(void)
407 ret = register_stat_tracer(&all_branch_stats);
409 printk(KERN_WARNING "Warning: could not register "
410 "all branches stats\n");
415 fs_initcall(all_annotated_branch_stats);
416 #endif /* CONFIG_PROFILE_ALL_BRANCHES */