2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* Pipe tracepoints to printk */
67 struct trace_iterator *tracepoint_print_iter;
68 int tracepoint_printk;
70 /* For tracers that don't implement custom flags */
71 static struct tracer_opt dummy_tracer_opt[] = {
75 static struct tracer_flags dummy_tracer_flags = {
77 .opts = dummy_tracer_opt
81 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
87 * To prevent the comm cache from being overwritten when no
88 * tracing is active, only save the comm when a trace event
91 static DEFINE_PER_CPU(bool, trace_cmdline_save);
94 * Kill all tracing for good (never come back).
95 * It is initialized to 1 but will turn to zero if the initialization
96 * of the tracer is successful. But that is the only place that sets
99 static int tracing_disabled = 1;
101 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
103 cpumask_var_t __read_mostly tracing_buffer_mask;
106 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
108 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
109 * is set, then ftrace_dump is called. This will output the contents
110 * of the ftrace buffers to the console. This is very useful for
111 * capturing traces that lead to crashes and outputing it to a
114 * It is default off, but you can enable it with either specifying
115 * "ftrace_dump_on_oops" in the kernel command line, or setting
116 * /proc/sys/kernel/ftrace_dump_on_oops
117 * Set 1 if you want to dump buffers of all CPUs
118 * Set 2 if you want to dump the buffer of the CPU that triggered oops
121 enum ftrace_dump_mode ftrace_dump_on_oops;
123 /* When set, tracing will stop when a WARN*() is hit */
124 int __disable_trace_on_warning;
126 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
128 #define MAX_TRACER_SIZE 100
129 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
130 static char *default_bootup_tracer;
132 static bool allocate_snapshot;
134 static int __init set_cmdline_ftrace(char *str)
136 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
137 default_bootup_tracer = bootup_tracer_buf;
138 /* We are using ftrace early, expand it */
139 ring_buffer_expanded = true;
142 __setup("ftrace=", set_cmdline_ftrace);
144 static int __init set_ftrace_dump_on_oops(char *str)
146 if (*str++ != '=' || !*str) {
147 ftrace_dump_on_oops = DUMP_ALL;
151 if (!strcmp("orig_cpu", str)) {
152 ftrace_dump_on_oops = DUMP_ORIG;
158 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
160 static int __init stop_trace_on_warning(char *str)
162 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
163 __disable_trace_on_warning = 1;
166 __setup("traceoff_on_warning", stop_trace_on_warning);
168 static int __init boot_alloc_snapshot(char *str)
170 allocate_snapshot = true;
171 /* We also need the main ring buffer expanded */
172 ring_buffer_expanded = true;
175 __setup("alloc_snapshot", boot_alloc_snapshot);
178 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
179 static char *trace_boot_options __initdata;
181 static int __init set_trace_boot_options(char *str)
183 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
184 trace_boot_options = trace_boot_options_buf;
187 __setup("trace_options=", set_trace_boot_options);
189 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
190 static char *trace_boot_clock __initdata;
192 static int __init set_trace_boot_clock(char *str)
194 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
195 trace_boot_clock = trace_boot_clock_buf;
198 __setup("trace_clock=", set_trace_boot_clock);
200 static int __init set_tracepoint_printk(char *str)
202 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
203 tracepoint_printk = 1;
206 __setup("tp_printk", set_tracepoint_printk);
208 unsigned long long ns2usecs(cycle_t nsec)
216 * The global_trace is the descriptor that holds the tracing
217 * buffers for the live tracing. For each CPU, it contains
218 * a link list of pages that will store trace entries. The
219 * page descriptor of the pages in the memory is used to hold
220 * the link list by linking the lru item in the page descriptor
221 * to each of the pages in the buffer per CPU.
223 * For each active CPU there is a data field that holds the
224 * pages for the buffer for that CPU. Each CPU has the same number
225 * of pages allocated for its buffer.
227 static struct trace_array global_trace;
229 LIST_HEAD(ftrace_trace_arrays);
231 int trace_array_get(struct trace_array *this_tr)
233 struct trace_array *tr;
236 mutex_lock(&trace_types_lock);
237 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
244 mutex_unlock(&trace_types_lock);
249 static void __trace_array_put(struct trace_array *this_tr)
251 WARN_ON(!this_tr->ref);
255 void trace_array_put(struct trace_array *this_tr)
257 mutex_lock(&trace_types_lock);
258 __trace_array_put(this_tr);
259 mutex_unlock(&trace_types_lock);
262 int filter_check_discard(struct ftrace_event_file *file, void *rec,
263 struct ring_buffer *buffer,
264 struct ring_buffer_event *event)
266 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
267 !filter_match_preds(file->filter, rec)) {
268 ring_buffer_discard_commit(buffer, event);
274 EXPORT_SYMBOL_GPL(filter_check_discard);
276 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
277 struct ring_buffer *buffer,
278 struct ring_buffer_event *event)
280 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
281 !filter_match_preds(call->filter, rec)) {
282 ring_buffer_discard_commit(buffer, event);
288 EXPORT_SYMBOL_GPL(call_filter_check_discard);
290 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
294 /* Early boot up does not have a buffer yet */
296 return trace_clock_local();
298 ts = ring_buffer_time_stamp(buf->buffer, cpu);
299 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
304 cycle_t ftrace_now(int cpu)
306 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
310 * tracing_is_enabled - Show if global_trace has been disabled
312 * Shows if the global trace has been enabled or not. It uses the
313 * mirror flag "buffer_disabled" to be used in fast paths such as for
314 * the irqsoff tracer. But it may be inaccurate due to races. If you
315 * need to know the accurate state, use tracing_is_on() which is a little
316 * slower, but accurate.
318 int tracing_is_enabled(void)
321 * For quick access (irqsoff uses this in fast path), just
322 * return the mirror variable of the state of the ring buffer.
323 * It's a little racy, but we don't really care.
326 return !global_trace.buffer_disabled;
330 * trace_buf_size is the size in bytes that is allocated
331 * for a buffer. Note, the number of bytes is always rounded
334 * This number is purposely set to a low number of 16384.
335 * If the dump on oops happens, it will be much appreciated
336 * to not have to wait for all that output. Anyway this can be
337 * boot time and run time configurable.
339 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
341 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
343 /* trace_types holds a link list of available tracers. */
344 static struct tracer *trace_types __read_mostly;
347 * trace_types_lock is used to protect the trace_types list.
349 DEFINE_MUTEX(trace_types_lock);
352 * serialize the access of the ring buffer
354 * ring buffer serializes readers, but it is low level protection.
355 * The validity of the events (which returns by ring_buffer_peek() ..etc)
356 * are not protected by ring buffer.
358 * The content of events may become garbage if we allow other process consumes
359 * these events concurrently:
360 * A) the page of the consumed events may become a normal page
361 * (not reader page) in ring buffer, and this page will be rewrited
362 * by events producer.
363 * B) The page of the consumed events may become a page for splice_read,
364 * and this page will be returned to system.
366 * These primitives allow multi process access to different cpu ring buffer
369 * These primitives don't distinguish read-only and read-consume access.
370 * Multi read-only access are also serialized.
374 static DECLARE_RWSEM(all_cpu_access_lock);
375 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
377 static inline void trace_access_lock(int cpu)
379 if (cpu == RING_BUFFER_ALL_CPUS) {
380 /* gain it for accessing the whole ring buffer. */
381 down_write(&all_cpu_access_lock);
383 /* gain it for accessing a cpu ring buffer. */
385 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
386 down_read(&all_cpu_access_lock);
388 /* Secondly block other access to this @cpu ring buffer. */
389 mutex_lock(&per_cpu(cpu_access_lock, cpu));
393 static inline void trace_access_unlock(int cpu)
395 if (cpu == RING_BUFFER_ALL_CPUS) {
396 up_write(&all_cpu_access_lock);
398 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
399 up_read(&all_cpu_access_lock);
403 static inline void trace_access_lock_init(void)
407 for_each_possible_cpu(cpu)
408 mutex_init(&per_cpu(cpu_access_lock, cpu));
413 static DEFINE_MUTEX(access_lock);
415 static inline void trace_access_lock(int cpu)
418 mutex_lock(&access_lock);
421 static inline void trace_access_unlock(int cpu)
424 mutex_unlock(&access_lock);
427 static inline void trace_access_lock_init(void)
433 /* trace_flags holds trace_options default values */
434 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
435 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
436 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
437 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
439 static void tracer_tracing_on(struct trace_array *tr)
441 if (tr->trace_buffer.buffer)
442 ring_buffer_record_on(tr->trace_buffer.buffer);
444 * This flag is looked at when buffers haven't been allocated
445 * yet, or by some tracers (like irqsoff), that just want to
446 * know if the ring buffer has been disabled, but it can handle
447 * races of where it gets disabled but we still do a record.
448 * As the check is in the fast path of the tracers, it is more
449 * important to be fast than accurate.
451 tr->buffer_disabled = 0;
452 /* Make the flag seen by readers */
457 * tracing_on - enable tracing buffers
459 * This function enables tracing buffers that may have been
460 * disabled with tracing_off.
462 void tracing_on(void)
464 tracer_tracing_on(&global_trace);
466 EXPORT_SYMBOL_GPL(tracing_on);
469 * __trace_puts - write a constant string into the trace buffer.
470 * @ip: The address of the caller
471 * @str: The constant string to write
472 * @size: The size of the string.
474 int __trace_puts(unsigned long ip, const char *str, int size)
476 struct ring_buffer_event *event;
477 struct ring_buffer *buffer;
478 struct print_entry *entry;
479 unsigned long irq_flags;
483 if (!(trace_flags & TRACE_ITER_PRINTK))
486 pc = preempt_count();
488 if (unlikely(tracing_selftest_running || tracing_disabled))
491 alloc = sizeof(*entry) + size + 2; /* possible \n added */
493 local_save_flags(irq_flags);
494 buffer = global_trace.trace_buffer.buffer;
495 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
500 entry = ring_buffer_event_data(event);
503 memcpy(&entry->buf, str, size);
505 /* Add a newline if necessary */
506 if (entry->buf[size - 1] != '\n') {
507 entry->buf[size] = '\n';
508 entry->buf[size + 1] = '\0';
510 entry->buf[size] = '\0';
512 __buffer_unlock_commit(buffer, event);
513 ftrace_trace_stack(buffer, irq_flags, 4, pc);
517 EXPORT_SYMBOL_GPL(__trace_puts);
520 * __trace_bputs - write the pointer to a constant string into trace buffer
521 * @ip: The address of the caller
522 * @str: The constant string to write to the buffer to
524 int __trace_bputs(unsigned long ip, const char *str)
526 struct ring_buffer_event *event;
527 struct ring_buffer *buffer;
528 struct bputs_entry *entry;
529 unsigned long irq_flags;
530 int size = sizeof(struct bputs_entry);
533 if (!(trace_flags & TRACE_ITER_PRINTK))
536 pc = preempt_count();
538 if (unlikely(tracing_selftest_running || tracing_disabled))
541 local_save_flags(irq_flags);
542 buffer = global_trace.trace_buffer.buffer;
543 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
548 entry = ring_buffer_event_data(event);
552 __buffer_unlock_commit(buffer, event);
553 ftrace_trace_stack(buffer, irq_flags, 4, pc);
557 EXPORT_SYMBOL_GPL(__trace_bputs);
559 #ifdef CONFIG_TRACER_SNAPSHOT
561 * trace_snapshot - take a snapshot of the current buffer.
563 * This causes a swap between the snapshot buffer and the current live
564 * tracing buffer. You can use this to take snapshots of the live
565 * trace when some condition is triggered, but continue to trace.
567 * Note, make sure to allocate the snapshot with either
568 * a tracing_snapshot_alloc(), or by doing it manually
569 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
571 * If the snapshot buffer is not allocated, it will stop tracing.
572 * Basically making a permanent snapshot.
574 void tracing_snapshot(void)
576 struct trace_array *tr = &global_trace;
577 struct tracer *tracer = tr->current_trace;
581 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
582 internal_trace_puts("*** snapshot is being ignored ***\n");
586 if (!tr->allocated_snapshot) {
587 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
588 internal_trace_puts("*** stopping trace here! ***\n");
593 /* Note, snapshot can not be used when the tracer uses it */
594 if (tracer->use_max_tr) {
595 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
596 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
600 local_irq_save(flags);
601 update_max_tr(tr, current, smp_processor_id());
602 local_irq_restore(flags);
604 EXPORT_SYMBOL_GPL(tracing_snapshot);
606 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
607 struct trace_buffer *size_buf, int cpu_id);
608 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
610 static int alloc_snapshot(struct trace_array *tr)
614 if (!tr->allocated_snapshot) {
616 /* allocate spare buffer */
617 ret = resize_buffer_duplicate_size(&tr->max_buffer,
618 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
622 tr->allocated_snapshot = true;
628 static void free_snapshot(struct trace_array *tr)
631 * We don't free the ring buffer. instead, resize it because
632 * The max_tr ring buffer has some state (e.g. ring->clock) and
633 * we want preserve it.
635 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
636 set_buffer_entries(&tr->max_buffer, 1);
637 tracing_reset_online_cpus(&tr->max_buffer);
638 tr->allocated_snapshot = false;
642 * tracing_alloc_snapshot - allocate snapshot buffer.
644 * This only allocates the snapshot buffer if it isn't already
645 * allocated - it doesn't also take a snapshot.
647 * This is meant to be used in cases where the snapshot buffer needs
648 * to be set up for events that can't sleep but need to be able to
649 * trigger a snapshot.
651 int tracing_alloc_snapshot(void)
653 struct trace_array *tr = &global_trace;
656 ret = alloc_snapshot(tr);
661 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
664 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
666 * This is similar to trace_snapshot(), but it will allocate the
667 * snapshot buffer if it isn't already allocated. Use this only
668 * where it is safe to sleep, as the allocation may sleep.
670 * This causes a swap between the snapshot buffer and the current live
671 * tracing buffer. You can use this to take snapshots of the live
672 * trace when some condition is triggered, but continue to trace.
674 void tracing_snapshot_alloc(void)
678 ret = tracing_alloc_snapshot();
684 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
686 void tracing_snapshot(void)
688 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
690 EXPORT_SYMBOL_GPL(tracing_snapshot);
691 int tracing_alloc_snapshot(void)
693 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
696 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
697 void tracing_snapshot_alloc(void)
702 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
703 #endif /* CONFIG_TRACER_SNAPSHOT */
705 static void tracer_tracing_off(struct trace_array *tr)
707 if (tr->trace_buffer.buffer)
708 ring_buffer_record_off(tr->trace_buffer.buffer);
710 * This flag is looked at when buffers haven't been allocated
711 * yet, or by some tracers (like irqsoff), that just want to
712 * know if the ring buffer has been disabled, but it can handle
713 * races of where it gets disabled but we still do a record.
714 * As the check is in the fast path of the tracers, it is more
715 * important to be fast than accurate.
717 tr->buffer_disabled = 1;
718 /* Make the flag seen by readers */
723 * tracing_off - turn off tracing buffers
725 * This function stops the tracing buffers from recording data.
726 * It does not disable any overhead the tracers themselves may
727 * be causing. This function simply causes all recording to
728 * the ring buffers to fail.
730 void tracing_off(void)
732 tracer_tracing_off(&global_trace);
734 EXPORT_SYMBOL_GPL(tracing_off);
736 void disable_trace_on_warning(void)
738 if (__disable_trace_on_warning)
743 * tracer_tracing_is_on - show real state of ring buffer enabled
744 * @tr : the trace array to know if ring buffer is enabled
746 * Shows real state of the ring buffer if it is enabled or not.
748 static int tracer_tracing_is_on(struct trace_array *tr)
750 if (tr->trace_buffer.buffer)
751 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
752 return !tr->buffer_disabled;
756 * tracing_is_on - show state of ring buffers enabled
758 int tracing_is_on(void)
760 return tracer_tracing_is_on(&global_trace);
762 EXPORT_SYMBOL_GPL(tracing_is_on);
764 static int __init set_buf_size(char *str)
766 unsigned long buf_size;
770 buf_size = memparse(str, &str);
771 /* nr_entries can not be zero */
774 trace_buf_size = buf_size;
777 __setup("trace_buf_size=", set_buf_size);
779 static int __init set_tracing_thresh(char *str)
781 unsigned long threshold;
786 ret = kstrtoul(str, 0, &threshold);
789 tracing_thresh = threshold * 1000;
792 __setup("tracing_thresh=", set_tracing_thresh);
794 unsigned long nsecs_to_usecs(unsigned long nsecs)
799 /* These must match the bit postions in trace_iterator_flags */
800 static const char *trace_options[] = {
833 int in_ns; /* is this clock in nanoseconds? */
835 { trace_clock_local, "local", 1 },
836 { trace_clock_global, "global", 1 },
837 { trace_clock_counter, "counter", 0 },
838 { trace_clock_jiffies, "uptime", 0 },
839 { trace_clock, "perf", 1 },
840 { ktime_get_mono_fast_ns, "mono", 1 },
845 * trace_parser_get_init - gets the buffer for trace parser
847 int trace_parser_get_init(struct trace_parser *parser, int size)
849 memset(parser, 0, sizeof(*parser));
851 parser->buffer = kmalloc(size, GFP_KERNEL);
860 * trace_parser_put - frees the buffer for trace parser
862 void trace_parser_put(struct trace_parser *parser)
864 kfree(parser->buffer);
868 * trace_get_user - reads the user input string separated by space
869 * (matched by isspace(ch))
871 * For each string found the 'struct trace_parser' is updated,
872 * and the function returns.
874 * Returns number of bytes read.
876 * See kernel/trace/trace.h for 'struct trace_parser' details.
878 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
879 size_t cnt, loff_t *ppos)
886 trace_parser_clear(parser);
888 ret = get_user(ch, ubuf++);
896 * The parser is not finished with the last write,
897 * continue reading the user input without skipping spaces.
900 /* skip white space */
901 while (cnt && isspace(ch)) {
902 ret = get_user(ch, ubuf++);
909 /* only spaces were written */
919 /* read the non-space input */
920 while (cnt && !isspace(ch)) {
921 if (parser->idx < parser->size - 1)
922 parser->buffer[parser->idx++] = ch;
927 ret = get_user(ch, ubuf++);
934 /* We either got finished input or we have to wait for another call. */
936 parser->buffer[parser->idx] = 0;
937 parser->cont = false;
938 } else if (parser->idx < parser->size - 1) {
940 parser->buffer[parser->idx++] = ch;
953 /* TODO add a seq_buf_to_buffer() */
954 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
958 if (trace_seq_used(s) <= s->seq.readpos)
961 len = trace_seq_used(s) - s->seq.readpos;
964 memcpy(buf, s->buffer + s->seq.readpos, cnt);
966 s->seq.readpos += cnt;
970 unsigned long __read_mostly tracing_thresh;
972 #ifdef CONFIG_TRACER_MAX_TRACE
974 * Copy the new maximum trace into the separate maximum-trace
975 * structure. (this way the maximum trace is permanently saved,
976 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
979 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
981 struct trace_buffer *trace_buf = &tr->trace_buffer;
982 struct trace_buffer *max_buf = &tr->max_buffer;
983 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
984 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
987 max_buf->time_start = data->preempt_timestamp;
989 max_data->saved_latency = tr->max_latency;
990 max_data->critical_start = data->critical_start;
991 max_data->critical_end = data->critical_end;
993 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
994 max_data->pid = tsk->pid;
996 * If tsk == current, then use current_uid(), as that does not use
997 * RCU. The irq tracer can be called out of RCU scope.
1000 max_data->uid = current_uid();
1002 max_data->uid = task_uid(tsk);
1004 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1005 max_data->policy = tsk->policy;
1006 max_data->rt_priority = tsk->rt_priority;
1008 /* record this tasks comm */
1009 tracing_record_cmdline(tsk);
1013 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1015 * @tsk: the task with the latency
1016 * @cpu: The cpu that initiated the trace.
1018 * Flip the buffers between the @tr and the max_tr and record information
1019 * about which task was the cause of this latency.
1022 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1024 struct ring_buffer *buf;
1029 WARN_ON_ONCE(!irqs_disabled());
1031 if (!tr->allocated_snapshot) {
1032 /* Only the nop tracer should hit this when disabling */
1033 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1037 arch_spin_lock(&tr->max_lock);
1039 buf = tr->trace_buffer.buffer;
1040 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1041 tr->max_buffer.buffer = buf;
1043 __update_max_tr(tr, tsk, cpu);
1044 arch_spin_unlock(&tr->max_lock);
1048 * update_max_tr_single - only copy one trace over, and reset the rest
1050 * @tsk - task with the latency
1051 * @cpu - the cpu of the buffer to copy.
1053 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1056 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1063 WARN_ON_ONCE(!irqs_disabled());
1064 if (!tr->allocated_snapshot) {
1065 /* Only the nop tracer should hit this when disabling */
1066 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1070 arch_spin_lock(&tr->max_lock);
1072 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1074 if (ret == -EBUSY) {
1076 * We failed to swap the buffer due to a commit taking
1077 * place on this CPU. We fail to record, but we reset
1078 * the max trace buffer (no one writes directly to it)
1079 * and flag that it failed.
1081 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1082 "Failed to swap buffers due to commit in progress\n");
1085 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1087 __update_max_tr(tr, tsk, cpu);
1088 arch_spin_unlock(&tr->max_lock);
1090 #endif /* CONFIG_TRACER_MAX_TRACE */
1092 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1094 /* Iterators are static, they should be filled or empty */
1095 if (trace_buffer_iter(iter, iter->cpu_file))
1098 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1102 #ifdef CONFIG_FTRACE_STARTUP_TEST
1103 static int run_tracer_selftest(struct tracer *type)
1105 struct trace_array *tr = &global_trace;
1106 struct tracer *saved_tracer = tr->current_trace;
1109 if (!type->selftest || tracing_selftest_disabled)
1113 * Run a selftest on this tracer.
1114 * Here we reset the trace buffer, and set the current
1115 * tracer to be this tracer. The tracer can then run some
1116 * internal tracing to verify that everything is in order.
1117 * If we fail, we do not register this tracer.
1119 tracing_reset_online_cpus(&tr->trace_buffer);
1121 tr->current_trace = type;
1123 #ifdef CONFIG_TRACER_MAX_TRACE
1124 if (type->use_max_tr) {
1125 /* If we expanded the buffers, make sure the max is expanded too */
1126 if (ring_buffer_expanded)
1127 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1128 RING_BUFFER_ALL_CPUS);
1129 tr->allocated_snapshot = true;
1133 /* the test is responsible for initializing and enabling */
1134 pr_info("Testing tracer %s: ", type->name);
1135 ret = type->selftest(type, tr);
1136 /* the test is responsible for resetting too */
1137 tr->current_trace = saved_tracer;
1139 printk(KERN_CONT "FAILED!\n");
1140 /* Add the warning after printing 'FAILED' */
1144 /* Only reset on passing, to avoid touching corrupted buffers */
1145 tracing_reset_online_cpus(&tr->trace_buffer);
1147 #ifdef CONFIG_TRACER_MAX_TRACE
1148 if (type->use_max_tr) {
1149 tr->allocated_snapshot = false;
1151 /* Shrink the max buffer again */
1152 if (ring_buffer_expanded)
1153 ring_buffer_resize(tr->max_buffer.buffer, 1,
1154 RING_BUFFER_ALL_CPUS);
1158 printk(KERN_CONT "PASSED\n");
1162 static inline int run_tracer_selftest(struct tracer *type)
1166 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1169 * register_tracer - register a tracer with the ftrace system.
1170 * @type - the plugin for the tracer
1172 * Register a new plugin tracer.
1174 int register_tracer(struct tracer *type)
1180 pr_info("Tracer must have a name\n");
1184 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1185 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1189 mutex_lock(&trace_types_lock);
1191 tracing_selftest_running = true;
1193 for (t = trace_types; t; t = t->next) {
1194 if (strcmp(type->name, t->name) == 0) {
1196 pr_info("Tracer %s already registered\n",
1203 if (!type->set_flag)
1204 type->set_flag = &dummy_set_flag;
1206 type->flags = &dummy_tracer_flags;
1208 if (!type->flags->opts)
1209 type->flags->opts = dummy_tracer_opt;
1211 ret = run_tracer_selftest(type);
1215 type->next = trace_types;
1219 tracing_selftest_running = false;
1220 mutex_unlock(&trace_types_lock);
1222 if (ret || !default_bootup_tracer)
1225 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1228 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1229 /* Do we want this tracer to start on bootup? */
1230 tracing_set_tracer(&global_trace, type->name);
1231 default_bootup_tracer = NULL;
1232 /* disable other selftests, since this will break it. */
1233 tracing_selftest_disabled = true;
1234 #ifdef CONFIG_FTRACE_STARTUP_TEST
1235 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1243 void tracing_reset(struct trace_buffer *buf, int cpu)
1245 struct ring_buffer *buffer = buf->buffer;
1250 ring_buffer_record_disable(buffer);
1252 /* Make sure all commits have finished */
1253 synchronize_sched();
1254 ring_buffer_reset_cpu(buffer, cpu);
1256 ring_buffer_record_enable(buffer);
1259 void tracing_reset_online_cpus(struct trace_buffer *buf)
1261 struct ring_buffer *buffer = buf->buffer;
1267 ring_buffer_record_disable(buffer);
1269 /* Make sure all commits have finished */
1270 synchronize_sched();
1272 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1274 for_each_online_cpu(cpu)
1275 ring_buffer_reset_cpu(buffer, cpu);
1277 ring_buffer_record_enable(buffer);
1280 /* Must have trace_types_lock held */
1281 void tracing_reset_all_online_cpus(void)
1283 struct trace_array *tr;
1285 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1286 tracing_reset_online_cpus(&tr->trace_buffer);
1287 #ifdef CONFIG_TRACER_MAX_TRACE
1288 tracing_reset_online_cpus(&tr->max_buffer);
1293 #define SAVED_CMDLINES_DEFAULT 128
1294 #define NO_CMDLINE_MAP UINT_MAX
1295 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1296 struct saved_cmdlines_buffer {
1297 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1298 unsigned *map_cmdline_to_pid;
1299 unsigned cmdline_num;
1301 char *saved_cmdlines;
1303 static struct saved_cmdlines_buffer *savedcmd;
1305 /* temporary disable recording */
1306 static atomic_t trace_record_cmdline_disabled __read_mostly;
1308 static inline char *get_saved_cmdlines(int idx)
1310 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1313 static inline void set_cmdline(int idx, const char *cmdline)
1315 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1318 static int allocate_cmdlines_buffer(unsigned int val,
1319 struct saved_cmdlines_buffer *s)
1321 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1323 if (!s->map_cmdline_to_pid)
1326 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1327 if (!s->saved_cmdlines) {
1328 kfree(s->map_cmdline_to_pid);
1333 s->cmdline_num = val;
1334 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1335 sizeof(s->map_pid_to_cmdline));
1336 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1337 val * sizeof(*s->map_cmdline_to_pid));
1342 static int trace_create_savedcmd(void)
1346 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1350 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1360 int is_tracing_stopped(void)
1362 return global_trace.stop_count;
1366 * tracing_start - quick start of the tracer
1368 * If tracing is enabled but was stopped by tracing_stop,
1369 * this will start the tracer back up.
1371 void tracing_start(void)
1373 struct ring_buffer *buffer;
1374 unsigned long flags;
1376 if (tracing_disabled)
1379 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1380 if (--global_trace.stop_count) {
1381 if (global_trace.stop_count < 0) {
1382 /* Someone screwed up their debugging */
1384 global_trace.stop_count = 0;
1389 /* Prevent the buffers from switching */
1390 arch_spin_lock(&global_trace.max_lock);
1392 buffer = global_trace.trace_buffer.buffer;
1394 ring_buffer_record_enable(buffer);
1396 #ifdef CONFIG_TRACER_MAX_TRACE
1397 buffer = global_trace.max_buffer.buffer;
1399 ring_buffer_record_enable(buffer);
1402 arch_spin_unlock(&global_trace.max_lock);
1405 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1408 static void tracing_start_tr(struct trace_array *tr)
1410 struct ring_buffer *buffer;
1411 unsigned long flags;
1413 if (tracing_disabled)
1416 /* If global, we need to also start the max tracer */
1417 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1418 return tracing_start();
1420 raw_spin_lock_irqsave(&tr->start_lock, flags);
1422 if (--tr->stop_count) {
1423 if (tr->stop_count < 0) {
1424 /* Someone screwed up their debugging */
1431 buffer = tr->trace_buffer.buffer;
1433 ring_buffer_record_enable(buffer);
1436 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1440 * tracing_stop - quick stop of the tracer
1442 * Light weight way to stop tracing. Use in conjunction with
1445 void tracing_stop(void)
1447 struct ring_buffer *buffer;
1448 unsigned long flags;
1450 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1451 if (global_trace.stop_count++)
1454 /* Prevent the buffers from switching */
1455 arch_spin_lock(&global_trace.max_lock);
1457 buffer = global_trace.trace_buffer.buffer;
1459 ring_buffer_record_disable(buffer);
1461 #ifdef CONFIG_TRACER_MAX_TRACE
1462 buffer = global_trace.max_buffer.buffer;
1464 ring_buffer_record_disable(buffer);
1467 arch_spin_unlock(&global_trace.max_lock);
1470 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1473 static void tracing_stop_tr(struct trace_array *tr)
1475 struct ring_buffer *buffer;
1476 unsigned long flags;
1478 /* If global, we need to also stop the max tracer */
1479 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1480 return tracing_stop();
1482 raw_spin_lock_irqsave(&tr->start_lock, flags);
1483 if (tr->stop_count++)
1486 buffer = tr->trace_buffer.buffer;
1488 ring_buffer_record_disable(buffer);
1491 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1494 void trace_stop_cmdline_recording(void);
1496 static int trace_save_cmdline(struct task_struct *tsk)
1500 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1504 * It's not the end of the world if we don't get
1505 * the lock, but we also don't want to spin
1506 * nor do we want to disable interrupts,
1507 * so if we miss here, then better luck next time.
1509 if (!arch_spin_trylock(&trace_cmdline_lock))
1512 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1513 if (idx == NO_CMDLINE_MAP) {
1514 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1517 * Check whether the cmdline buffer at idx has a pid
1518 * mapped. We are going to overwrite that entry so we
1519 * need to clear the map_pid_to_cmdline. Otherwise we
1520 * would read the new comm for the old pid.
1522 pid = savedcmd->map_cmdline_to_pid[idx];
1523 if (pid != NO_CMDLINE_MAP)
1524 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1526 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1527 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1529 savedcmd->cmdline_idx = idx;
1532 set_cmdline(idx, tsk->comm);
1534 arch_spin_unlock(&trace_cmdline_lock);
1539 static void __trace_find_cmdline(int pid, char comm[])
1544 strcpy(comm, "<idle>");
1548 if (WARN_ON_ONCE(pid < 0)) {
1549 strcpy(comm, "<XXX>");
1553 if (pid > PID_MAX_DEFAULT) {
1554 strcpy(comm, "<...>");
1558 map = savedcmd->map_pid_to_cmdline[pid];
1559 if (map != NO_CMDLINE_MAP)
1560 strcpy(comm, get_saved_cmdlines(map));
1562 strcpy(comm, "<...>");
1565 void trace_find_cmdline(int pid, char comm[])
1568 arch_spin_lock(&trace_cmdline_lock);
1570 __trace_find_cmdline(pid, comm);
1572 arch_spin_unlock(&trace_cmdline_lock);
1576 void tracing_record_cmdline(struct task_struct *tsk)
1578 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1581 if (!__this_cpu_read(trace_cmdline_save))
1584 if (trace_save_cmdline(tsk))
1585 __this_cpu_write(trace_cmdline_save, false);
1589 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1592 struct task_struct *tsk = current;
1594 entry->preempt_count = pc & 0xff;
1595 entry->pid = (tsk) ? tsk->pid : 0;
1597 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1598 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1600 TRACE_FLAG_IRQS_NOSUPPORT |
1602 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1603 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1604 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1605 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1607 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1609 struct ring_buffer_event *
1610 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1613 unsigned long flags, int pc)
1615 struct ring_buffer_event *event;
1617 event = ring_buffer_lock_reserve(buffer, len);
1618 if (event != NULL) {
1619 struct trace_entry *ent = ring_buffer_event_data(event);
1621 tracing_generic_entry_update(ent, flags, pc);
1629 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1631 __this_cpu_write(trace_cmdline_save, true);
1632 ring_buffer_unlock_commit(buffer, event);
1636 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1637 struct ring_buffer_event *event,
1638 unsigned long flags, int pc)
1640 __buffer_unlock_commit(buffer, event);
1642 ftrace_trace_stack(buffer, flags, 6, pc);
1643 ftrace_trace_userstack(buffer, flags, pc);
1646 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1647 struct ring_buffer_event *event,
1648 unsigned long flags, int pc)
1650 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1652 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1654 static struct ring_buffer *temp_buffer;
1656 struct ring_buffer_event *
1657 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1658 struct ftrace_event_file *ftrace_file,
1659 int type, unsigned long len,
1660 unsigned long flags, int pc)
1662 struct ring_buffer_event *entry;
1664 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1665 entry = trace_buffer_lock_reserve(*current_rb,
1666 type, len, flags, pc);
1668 * If tracing is off, but we have triggers enabled
1669 * we still need to look at the event data. Use the temp_buffer
1670 * to store the trace event for the tigger to use. It's recusive
1671 * safe and will not be recorded anywhere.
1673 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1674 *current_rb = temp_buffer;
1675 entry = trace_buffer_lock_reserve(*current_rb,
1676 type, len, flags, pc);
1680 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1682 struct ring_buffer_event *
1683 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1684 int type, unsigned long len,
1685 unsigned long flags, int pc)
1687 *current_rb = global_trace.trace_buffer.buffer;
1688 return trace_buffer_lock_reserve(*current_rb,
1689 type, len, flags, pc);
1691 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1693 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1694 struct ring_buffer_event *event,
1695 unsigned long flags, int pc)
1697 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1699 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1701 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1702 struct ring_buffer_event *event,
1703 unsigned long flags, int pc,
1704 struct pt_regs *regs)
1706 __buffer_unlock_commit(buffer, event);
1708 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1709 ftrace_trace_userstack(buffer, flags, pc);
1711 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1713 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1714 struct ring_buffer_event *event)
1716 ring_buffer_discard_commit(buffer, event);
1718 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1721 trace_function(struct trace_array *tr,
1722 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1725 struct ftrace_event_call *call = &event_function;
1726 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1727 struct ring_buffer_event *event;
1728 struct ftrace_entry *entry;
1730 /* If we are reading the ring buffer, don't trace */
1731 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1734 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1738 entry = ring_buffer_event_data(event);
1740 entry->parent_ip = parent_ip;
1742 if (!call_filter_check_discard(call, entry, buffer, event))
1743 __buffer_unlock_commit(buffer, event);
1746 #ifdef CONFIG_STACKTRACE
1748 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1749 struct ftrace_stack {
1750 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1753 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1754 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1756 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1757 unsigned long flags,
1758 int skip, int pc, struct pt_regs *regs)
1760 struct ftrace_event_call *call = &event_kernel_stack;
1761 struct ring_buffer_event *event;
1762 struct stack_entry *entry;
1763 struct stack_trace trace;
1765 int size = FTRACE_STACK_ENTRIES;
1767 trace.nr_entries = 0;
1771 * Since events can happen in NMIs there's no safe way to
1772 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1773 * or NMI comes in, it will just have to use the default
1774 * FTRACE_STACK_SIZE.
1776 preempt_disable_notrace();
1778 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1780 * We don't need any atomic variables, just a barrier.
1781 * If an interrupt comes in, we don't care, because it would
1782 * have exited and put the counter back to what we want.
1783 * We just need a barrier to keep gcc from moving things
1787 if (use_stack == 1) {
1788 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1789 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1792 save_stack_trace_regs(regs, &trace);
1794 save_stack_trace(&trace);
1796 if (trace.nr_entries > size)
1797 size = trace.nr_entries;
1799 /* From now on, use_stack is a boolean */
1802 size *= sizeof(unsigned long);
1804 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1805 sizeof(*entry) + size, flags, pc);
1808 entry = ring_buffer_event_data(event);
1810 memset(&entry->caller, 0, size);
1813 memcpy(&entry->caller, trace.entries,
1814 trace.nr_entries * sizeof(unsigned long));
1816 trace.max_entries = FTRACE_STACK_ENTRIES;
1817 trace.entries = entry->caller;
1819 save_stack_trace_regs(regs, &trace);
1821 save_stack_trace(&trace);
1824 entry->size = trace.nr_entries;
1826 if (!call_filter_check_discard(call, entry, buffer, event))
1827 __buffer_unlock_commit(buffer, event);
1830 /* Again, don't let gcc optimize things here */
1832 __this_cpu_dec(ftrace_stack_reserve);
1833 preempt_enable_notrace();
1837 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1838 int skip, int pc, struct pt_regs *regs)
1840 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1843 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1846 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1849 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1852 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1855 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1858 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1862 * trace_dump_stack - record a stack back trace in the trace buffer
1863 * @skip: Number of functions to skip (helper handlers)
1865 void trace_dump_stack(int skip)
1867 unsigned long flags;
1869 if (tracing_disabled || tracing_selftest_running)
1872 local_save_flags(flags);
1875 * Skip 3 more, seems to get us at the caller of
1879 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1880 flags, skip, preempt_count(), NULL);
1883 static DEFINE_PER_CPU(int, user_stack_count);
1886 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1888 struct ftrace_event_call *call = &event_user_stack;
1889 struct ring_buffer_event *event;
1890 struct userstack_entry *entry;
1891 struct stack_trace trace;
1893 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1897 * NMIs can not handle page faults, even with fix ups.
1898 * The save user stack can (and often does) fault.
1900 if (unlikely(in_nmi()))
1904 * prevent recursion, since the user stack tracing may
1905 * trigger other kernel events.
1908 if (__this_cpu_read(user_stack_count))
1911 __this_cpu_inc(user_stack_count);
1913 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1914 sizeof(*entry), flags, pc);
1916 goto out_drop_count;
1917 entry = ring_buffer_event_data(event);
1919 entry->tgid = current->tgid;
1920 memset(&entry->caller, 0, sizeof(entry->caller));
1922 trace.nr_entries = 0;
1923 trace.max_entries = FTRACE_STACK_ENTRIES;
1925 trace.entries = entry->caller;
1927 save_stack_trace_user(&trace);
1928 if (!call_filter_check_discard(call, entry, buffer, event))
1929 __buffer_unlock_commit(buffer, event);
1932 __this_cpu_dec(user_stack_count);
1938 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1940 ftrace_trace_userstack(tr, flags, preempt_count());
1944 #endif /* CONFIG_STACKTRACE */
1946 /* created for use with alloc_percpu */
1947 struct trace_buffer_struct {
1948 char buffer[TRACE_BUF_SIZE];
1951 static struct trace_buffer_struct *trace_percpu_buffer;
1952 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1953 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1954 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1957 * The buffer used is dependent on the context. There is a per cpu
1958 * buffer for normal context, softirq contex, hard irq context and
1959 * for NMI context. Thise allows for lockless recording.
1961 * Note, if the buffers failed to be allocated, then this returns NULL
1963 static char *get_trace_buf(void)
1965 struct trace_buffer_struct *percpu_buffer;
1968 * If we have allocated per cpu buffers, then we do not
1969 * need to do any locking.
1972 percpu_buffer = trace_percpu_nmi_buffer;
1974 percpu_buffer = trace_percpu_irq_buffer;
1975 else if (in_softirq())
1976 percpu_buffer = trace_percpu_sirq_buffer;
1978 percpu_buffer = trace_percpu_buffer;
1983 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1986 static int alloc_percpu_trace_buffer(void)
1988 struct trace_buffer_struct *buffers;
1989 struct trace_buffer_struct *sirq_buffers;
1990 struct trace_buffer_struct *irq_buffers;
1991 struct trace_buffer_struct *nmi_buffers;
1993 buffers = alloc_percpu(struct trace_buffer_struct);
1997 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2001 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2005 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2009 trace_percpu_buffer = buffers;
2010 trace_percpu_sirq_buffer = sirq_buffers;
2011 trace_percpu_irq_buffer = irq_buffers;
2012 trace_percpu_nmi_buffer = nmi_buffers;
2017 free_percpu(irq_buffers);
2019 free_percpu(sirq_buffers);
2021 free_percpu(buffers);
2023 WARN(1, "Could not allocate percpu trace_printk buffer");
2027 static int buffers_allocated;
2029 void trace_printk_init_buffers(void)
2031 if (buffers_allocated)
2034 if (alloc_percpu_trace_buffer())
2037 /* trace_printk() is for debug use only. Don't use it in production. */
2040 pr_warning("**********************************************************\n");
2041 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2042 pr_warning("** **\n");
2043 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2044 pr_warning("** **\n");
2045 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2046 pr_warning("** unsafe for production use. **\n");
2047 pr_warning("** **\n");
2048 pr_warning("** If you see this message and you are not debugging **\n");
2049 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2050 pr_warning("** **\n");
2051 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2052 pr_warning("**********************************************************\n");
2054 /* Expand the buffers to set size */
2055 tracing_update_buffers();
2057 buffers_allocated = 1;
2060 * trace_printk_init_buffers() can be called by modules.
2061 * If that happens, then we need to start cmdline recording
2062 * directly here. If the global_trace.buffer is already
2063 * allocated here, then this was called by module code.
2065 if (global_trace.trace_buffer.buffer)
2066 tracing_start_cmdline_record();
2069 void trace_printk_start_comm(void)
2071 /* Start tracing comms if trace printk is set */
2072 if (!buffers_allocated)
2074 tracing_start_cmdline_record();
2077 static void trace_printk_start_stop_comm(int enabled)
2079 if (!buffers_allocated)
2083 tracing_start_cmdline_record();
2085 tracing_stop_cmdline_record();
2089 * trace_vbprintk - write binary msg to tracing buffer
2092 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2094 struct ftrace_event_call *call = &event_bprint;
2095 struct ring_buffer_event *event;
2096 struct ring_buffer *buffer;
2097 struct trace_array *tr = &global_trace;
2098 struct bprint_entry *entry;
2099 unsigned long flags;
2101 int len = 0, size, pc;
2103 if (unlikely(tracing_selftest_running || tracing_disabled))
2106 /* Don't pollute graph traces with trace_vprintk internals */
2107 pause_graph_tracing();
2109 pc = preempt_count();
2110 preempt_disable_notrace();
2112 tbuffer = get_trace_buf();
2118 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2120 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2123 local_save_flags(flags);
2124 size = sizeof(*entry) + sizeof(u32) * len;
2125 buffer = tr->trace_buffer.buffer;
2126 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2130 entry = ring_buffer_event_data(event);
2134 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2135 if (!call_filter_check_discard(call, entry, buffer, event)) {
2136 __buffer_unlock_commit(buffer, event);
2137 ftrace_trace_stack(buffer, flags, 6, pc);
2141 preempt_enable_notrace();
2142 unpause_graph_tracing();
2146 EXPORT_SYMBOL_GPL(trace_vbprintk);
2149 __trace_array_vprintk(struct ring_buffer *buffer,
2150 unsigned long ip, const char *fmt, va_list args)
2152 struct ftrace_event_call *call = &event_print;
2153 struct ring_buffer_event *event;
2154 int len = 0, size, pc;
2155 struct print_entry *entry;
2156 unsigned long flags;
2159 if (tracing_disabled || tracing_selftest_running)
2162 /* Don't pollute graph traces with trace_vprintk internals */
2163 pause_graph_tracing();
2165 pc = preempt_count();
2166 preempt_disable_notrace();
2169 tbuffer = get_trace_buf();
2175 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2177 local_save_flags(flags);
2178 size = sizeof(*entry) + len + 1;
2179 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2183 entry = ring_buffer_event_data(event);
2186 memcpy(&entry->buf, tbuffer, len + 1);
2187 if (!call_filter_check_discard(call, entry, buffer, event)) {
2188 __buffer_unlock_commit(buffer, event);
2189 ftrace_trace_stack(buffer, flags, 6, pc);
2192 preempt_enable_notrace();
2193 unpause_graph_tracing();
2198 int trace_array_vprintk(struct trace_array *tr,
2199 unsigned long ip, const char *fmt, va_list args)
2201 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2204 int trace_array_printk(struct trace_array *tr,
2205 unsigned long ip, const char *fmt, ...)
2210 if (!(trace_flags & TRACE_ITER_PRINTK))
2214 ret = trace_array_vprintk(tr, ip, fmt, ap);
2219 int trace_array_printk_buf(struct ring_buffer *buffer,
2220 unsigned long ip, const char *fmt, ...)
2225 if (!(trace_flags & TRACE_ITER_PRINTK))
2229 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2234 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2236 return trace_array_vprintk(&global_trace, ip, fmt, args);
2238 EXPORT_SYMBOL_GPL(trace_vprintk);
2240 static void trace_iterator_increment(struct trace_iterator *iter)
2242 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2246 ring_buffer_read(buf_iter, NULL);
2249 static struct trace_entry *
2250 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2251 unsigned long *lost_events)
2253 struct ring_buffer_event *event;
2254 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2257 event = ring_buffer_iter_peek(buf_iter, ts);
2259 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2263 iter->ent_size = ring_buffer_event_length(event);
2264 return ring_buffer_event_data(event);
2270 static struct trace_entry *
2271 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2272 unsigned long *missing_events, u64 *ent_ts)
2274 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2275 struct trace_entry *ent, *next = NULL;
2276 unsigned long lost_events = 0, next_lost = 0;
2277 int cpu_file = iter->cpu_file;
2278 u64 next_ts = 0, ts;
2284 * If we are in a per_cpu trace file, don't bother by iterating over
2285 * all cpu and peek directly.
2287 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2288 if (ring_buffer_empty_cpu(buffer, cpu_file))
2290 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2292 *ent_cpu = cpu_file;
2297 for_each_tracing_cpu(cpu) {
2299 if (ring_buffer_empty_cpu(buffer, cpu))
2302 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2305 * Pick the entry with the smallest timestamp:
2307 if (ent && (!next || ts < next_ts)) {
2311 next_lost = lost_events;
2312 next_size = iter->ent_size;
2316 iter->ent_size = next_size;
2319 *ent_cpu = next_cpu;
2325 *missing_events = next_lost;
2330 /* Find the next real entry, without updating the iterator itself */
2331 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2332 int *ent_cpu, u64 *ent_ts)
2334 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2337 /* Find the next real entry, and increment the iterator to the next entry */
2338 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2340 iter->ent = __find_next_entry(iter, &iter->cpu,
2341 &iter->lost_events, &iter->ts);
2344 trace_iterator_increment(iter);
2346 return iter->ent ? iter : NULL;
2349 static void trace_consume(struct trace_iterator *iter)
2351 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2352 &iter->lost_events);
2355 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2357 struct trace_iterator *iter = m->private;
2361 WARN_ON_ONCE(iter->leftover);
2365 /* can't go backwards */
2370 ent = trace_find_next_entry_inc(iter);
2374 while (ent && iter->idx < i)
2375 ent = trace_find_next_entry_inc(iter);
2382 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2384 struct ring_buffer_event *event;
2385 struct ring_buffer_iter *buf_iter;
2386 unsigned long entries = 0;
2389 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2391 buf_iter = trace_buffer_iter(iter, cpu);
2395 ring_buffer_iter_reset(buf_iter);
2398 * We could have the case with the max latency tracers
2399 * that a reset never took place on a cpu. This is evident
2400 * by the timestamp being before the start of the buffer.
2402 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2403 if (ts >= iter->trace_buffer->time_start)
2406 ring_buffer_read(buf_iter, NULL);
2409 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2413 * The current tracer is copied to avoid a global locking
2416 static void *s_start(struct seq_file *m, loff_t *pos)
2418 struct trace_iterator *iter = m->private;
2419 struct trace_array *tr = iter->tr;
2420 int cpu_file = iter->cpu_file;
2426 * copy the tracer to avoid using a global lock all around.
2427 * iter->trace is a copy of current_trace, the pointer to the
2428 * name may be used instead of a strcmp(), as iter->trace->name
2429 * will point to the same string as current_trace->name.
2431 mutex_lock(&trace_types_lock);
2432 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2433 *iter->trace = *tr->current_trace;
2434 mutex_unlock(&trace_types_lock);
2436 #ifdef CONFIG_TRACER_MAX_TRACE
2437 if (iter->snapshot && iter->trace->use_max_tr)
2438 return ERR_PTR(-EBUSY);
2441 if (!iter->snapshot)
2442 atomic_inc(&trace_record_cmdline_disabled);
2444 if (*pos != iter->pos) {
2449 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2450 for_each_tracing_cpu(cpu)
2451 tracing_iter_reset(iter, cpu);
2453 tracing_iter_reset(iter, cpu_file);
2456 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2461 * If we overflowed the seq_file before, then we want
2462 * to just reuse the trace_seq buffer again.
2468 p = s_next(m, p, &l);
2472 trace_event_read_lock();
2473 trace_access_lock(cpu_file);
2477 static void s_stop(struct seq_file *m, void *p)
2479 struct trace_iterator *iter = m->private;
2481 #ifdef CONFIG_TRACER_MAX_TRACE
2482 if (iter->snapshot && iter->trace->use_max_tr)
2486 if (!iter->snapshot)
2487 atomic_dec(&trace_record_cmdline_disabled);
2489 trace_access_unlock(iter->cpu_file);
2490 trace_event_read_unlock();
2494 get_total_entries(struct trace_buffer *buf,
2495 unsigned long *total, unsigned long *entries)
2497 unsigned long count;
2503 for_each_tracing_cpu(cpu) {
2504 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2506 * If this buffer has skipped entries, then we hold all
2507 * entries for the trace and we need to ignore the
2508 * ones before the time stamp.
2510 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2511 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2512 /* total is the same as the entries */
2516 ring_buffer_overrun_cpu(buf->buffer, cpu);
2521 static void print_lat_help_header(struct seq_file *m)
2523 seq_puts(m, "# _------=> CPU# \n"
2524 "# / _-----=> irqs-off \n"
2525 "# | / _----=> need-resched \n"
2526 "# || / _---=> hardirq/softirq \n"
2527 "# ||| / _--=> preempt-depth \n"
2529 "# cmd pid ||||| time | caller \n"
2530 "# \\ / ||||| \\ | / \n");
2533 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2535 unsigned long total;
2536 unsigned long entries;
2538 get_total_entries(buf, &total, &entries);
2539 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2540 entries, total, num_online_cpus());
2544 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2546 print_event_info(buf, m);
2547 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2551 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2553 print_event_info(buf, m);
2554 seq_puts(m, "# _-----=> irqs-off\n"
2555 "# / _----=> need-resched\n"
2556 "# | / _---=> hardirq/softirq\n"
2557 "# || / _--=> preempt-depth\n"
2559 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2560 "# | | | |||| | |\n");
2564 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2566 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2567 struct trace_buffer *buf = iter->trace_buffer;
2568 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2569 struct tracer *type = iter->trace;
2570 unsigned long entries;
2571 unsigned long total;
2572 const char *name = "preemption";
2576 get_total_entries(buf, &total, &entries);
2578 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2580 seq_puts(m, "# -----------------------------------"
2581 "---------------------------------\n");
2582 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2583 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2584 nsecs_to_usecs(data->saved_latency),
2588 #if defined(CONFIG_PREEMPT_NONE)
2590 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2592 #elif defined(CONFIG_PREEMPT)
2597 /* These are reserved for later use */
2600 seq_printf(m, " #P:%d)\n", num_online_cpus());
2604 seq_puts(m, "# -----------------\n");
2605 seq_printf(m, "# | task: %.16s-%d "
2606 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2607 data->comm, data->pid,
2608 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2609 data->policy, data->rt_priority);
2610 seq_puts(m, "# -----------------\n");
2612 if (data->critical_start) {
2613 seq_puts(m, "# => started at: ");
2614 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2615 trace_print_seq(m, &iter->seq);
2616 seq_puts(m, "\n# => ended at: ");
2617 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2618 trace_print_seq(m, &iter->seq);
2619 seq_puts(m, "\n#\n");
2625 static void test_cpu_buff_start(struct trace_iterator *iter)
2627 struct trace_seq *s = &iter->seq;
2629 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2632 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2635 if (cpumask_test_cpu(iter->cpu, iter->started))
2638 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2641 cpumask_set_cpu(iter->cpu, iter->started);
2643 /* Don't print started cpu buffer for the first entry of the trace */
2645 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2649 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2651 struct trace_seq *s = &iter->seq;
2652 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2653 struct trace_entry *entry;
2654 struct trace_event *event;
2658 test_cpu_buff_start(iter);
2660 event = ftrace_find_event(entry->type);
2662 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2663 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2664 trace_print_lat_context(iter);
2666 trace_print_context(iter);
2669 if (trace_seq_has_overflowed(s))
2670 return TRACE_TYPE_PARTIAL_LINE;
2673 return event->funcs->trace(iter, sym_flags, event);
2675 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2677 return trace_handle_return(s);
2680 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2682 struct trace_seq *s = &iter->seq;
2683 struct trace_entry *entry;
2684 struct trace_event *event;
2688 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2689 trace_seq_printf(s, "%d %d %llu ",
2690 entry->pid, iter->cpu, iter->ts);
2692 if (trace_seq_has_overflowed(s))
2693 return TRACE_TYPE_PARTIAL_LINE;
2695 event = ftrace_find_event(entry->type);
2697 return event->funcs->raw(iter, 0, event);
2699 trace_seq_printf(s, "%d ?\n", entry->type);
2701 return trace_handle_return(s);
2704 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2706 struct trace_seq *s = &iter->seq;
2707 unsigned char newline = '\n';
2708 struct trace_entry *entry;
2709 struct trace_event *event;
2713 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2714 SEQ_PUT_HEX_FIELD(s, entry->pid);
2715 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2716 SEQ_PUT_HEX_FIELD(s, iter->ts);
2717 if (trace_seq_has_overflowed(s))
2718 return TRACE_TYPE_PARTIAL_LINE;
2721 event = ftrace_find_event(entry->type);
2723 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2724 if (ret != TRACE_TYPE_HANDLED)
2728 SEQ_PUT_FIELD(s, newline);
2730 return trace_handle_return(s);
2733 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2735 struct trace_seq *s = &iter->seq;
2736 struct trace_entry *entry;
2737 struct trace_event *event;
2741 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2742 SEQ_PUT_FIELD(s, entry->pid);
2743 SEQ_PUT_FIELD(s, iter->cpu);
2744 SEQ_PUT_FIELD(s, iter->ts);
2745 if (trace_seq_has_overflowed(s))
2746 return TRACE_TYPE_PARTIAL_LINE;
2749 event = ftrace_find_event(entry->type);
2750 return event ? event->funcs->binary(iter, 0, event) :
2754 int trace_empty(struct trace_iterator *iter)
2756 struct ring_buffer_iter *buf_iter;
2759 /* If we are looking at one CPU buffer, only check that one */
2760 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2761 cpu = iter->cpu_file;
2762 buf_iter = trace_buffer_iter(iter, cpu);
2764 if (!ring_buffer_iter_empty(buf_iter))
2767 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2773 for_each_tracing_cpu(cpu) {
2774 buf_iter = trace_buffer_iter(iter, cpu);
2776 if (!ring_buffer_iter_empty(buf_iter))
2779 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2787 /* Called with trace_event_read_lock() held. */
2788 enum print_line_t print_trace_line(struct trace_iterator *iter)
2790 enum print_line_t ret;
2792 if (iter->lost_events) {
2793 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2794 iter->cpu, iter->lost_events);
2795 if (trace_seq_has_overflowed(&iter->seq))
2796 return TRACE_TYPE_PARTIAL_LINE;
2799 if (iter->trace && iter->trace->print_line) {
2800 ret = iter->trace->print_line(iter);
2801 if (ret != TRACE_TYPE_UNHANDLED)
2805 if (iter->ent->type == TRACE_BPUTS &&
2806 trace_flags & TRACE_ITER_PRINTK &&
2807 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2808 return trace_print_bputs_msg_only(iter);
2810 if (iter->ent->type == TRACE_BPRINT &&
2811 trace_flags & TRACE_ITER_PRINTK &&
2812 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2813 return trace_print_bprintk_msg_only(iter);
2815 if (iter->ent->type == TRACE_PRINT &&
2816 trace_flags & TRACE_ITER_PRINTK &&
2817 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2818 return trace_print_printk_msg_only(iter);
2820 if (trace_flags & TRACE_ITER_BIN)
2821 return print_bin_fmt(iter);
2823 if (trace_flags & TRACE_ITER_HEX)
2824 return print_hex_fmt(iter);
2826 if (trace_flags & TRACE_ITER_RAW)
2827 return print_raw_fmt(iter);
2829 return print_trace_fmt(iter);
2832 void trace_latency_header(struct seq_file *m)
2834 struct trace_iterator *iter = m->private;
2836 /* print nothing if the buffers are empty */
2837 if (trace_empty(iter))
2840 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2841 print_trace_header(m, iter);
2843 if (!(trace_flags & TRACE_ITER_VERBOSE))
2844 print_lat_help_header(m);
2847 void trace_default_header(struct seq_file *m)
2849 struct trace_iterator *iter = m->private;
2851 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2854 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2855 /* print nothing if the buffers are empty */
2856 if (trace_empty(iter))
2858 print_trace_header(m, iter);
2859 if (!(trace_flags & TRACE_ITER_VERBOSE))
2860 print_lat_help_header(m);
2862 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2863 if (trace_flags & TRACE_ITER_IRQ_INFO)
2864 print_func_help_header_irq(iter->trace_buffer, m);
2866 print_func_help_header(iter->trace_buffer, m);
2871 static void test_ftrace_alive(struct seq_file *m)
2873 if (!ftrace_is_dead())
2875 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2876 "# MAY BE MISSING FUNCTION EVENTS\n");
2879 #ifdef CONFIG_TRACER_MAX_TRACE
2880 static void show_snapshot_main_help(struct seq_file *m)
2882 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2883 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2884 "# Takes a snapshot of the main buffer.\n"
2885 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2886 "# (Doesn't have to be '2' works with any number that\n"
2887 "# is not a '0' or '1')\n");
2890 static void show_snapshot_percpu_help(struct seq_file *m)
2892 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2893 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2894 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2895 "# Takes a snapshot of the main buffer for this cpu.\n");
2897 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2898 "# Must use main snapshot file to allocate.\n");
2900 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2901 "# (Doesn't have to be '2' works with any number that\n"
2902 "# is not a '0' or '1')\n");
2905 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2907 if (iter->tr->allocated_snapshot)
2908 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2910 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2912 seq_puts(m, "# Snapshot commands:\n");
2913 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2914 show_snapshot_main_help(m);
2916 show_snapshot_percpu_help(m);
2919 /* Should never be called */
2920 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2923 static int s_show(struct seq_file *m, void *v)
2925 struct trace_iterator *iter = v;
2928 if (iter->ent == NULL) {
2930 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2932 test_ftrace_alive(m);
2934 if (iter->snapshot && trace_empty(iter))
2935 print_snapshot_help(m, iter);
2936 else if (iter->trace && iter->trace->print_header)
2937 iter->trace->print_header(m);
2939 trace_default_header(m);
2941 } else if (iter->leftover) {
2943 * If we filled the seq_file buffer earlier, we
2944 * want to just show it now.
2946 ret = trace_print_seq(m, &iter->seq);
2948 /* ret should this time be zero, but you never know */
2949 iter->leftover = ret;
2952 print_trace_line(iter);
2953 ret = trace_print_seq(m, &iter->seq);
2955 * If we overflow the seq_file buffer, then it will
2956 * ask us for this data again at start up.
2958 * ret is 0 if seq_file write succeeded.
2961 iter->leftover = ret;
2968 * Should be used after trace_array_get(), trace_types_lock
2969 * ensures that i_cdev was already initialized.
2971 static inline int tracing_get_cpu(struct inode *inode)
2973 if (inode->i_cdev) /* See trace_create_cpu_file() */
2974 return (long)inode->i_cdev - 1;
2975 return RING_BUFFER_ALL_CPUS;
2978 static const struct seq_operations tracer_seq_ops = {
2985 static struct trace_iterator *
2986 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2988 struct trace_array *tr = inode->i_private;
2989 struct trace_iterator *iter;
2992 if (tracing_disabled)
2993 return ERR_PTR(-ENODEV);
2995 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2997 return ERR_PTR(-ENOMEM);
2999 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
3001 if (!iter->buffer_iter)
3005 * We make a copy of the current tracer to avoid concurrent
3006 * changes on it while we are reading.
3008 mutex_lock(&trace_types_lock);
3009 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3013 *iter->trace = *tr->current_trace;
3015 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3020 #ifdef CONFIG_TRACER_MAX_TRACE
3021 /* Currently only the top directory has a snapshot */
3022 if (tr->current_trace->print_max || snapshot)
3023 iter->trace_buffer = &tr->max_buffer;
3026 iter->trace_buffer = &tr->trace_buffer;
3027 iter->snapshot = snapshot;
3029 iter->cpu_file = tracing_get_cpu(inode);
3030 mutex_init(&iter->mutex);
3032 /* Notify the tracer early; before we stop tracing. */
3033 if (iter->trace && iter->trace->open)
3034 iter->trace->open(iter);
3036 /* Annotate start of buffers if we had overruns */
3037 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3038 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3040 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3041 if (trace_clocks[tr->clock_id].in_ns)
3042 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3044 /* stop the trace while dumping if we are not opening "snapshot" */
3045 if (!iter->snapshot)
3046 tracing_stop_tr(tr);
3048 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3049 for_each_tracing_cpu(cpu) {
3050 iter->buffer_iter[cpu] =
3051 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3053 ring_buffer_read_prepare_sync();
3054 for_each_tracing_cpu(cpu) {
3055 ring_buffer_read_start(iter->buffer_iter[cpu]);
3056 tracing_iter_reset(iter, cpu);
3059 cpu = iter->cpu_file;
3060 iter->buffer_iter[cpu] =
3061 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3062 ring_buffer_read_prepare_sync();
3063 ring_buffer_read_start(iter->buffer_iter[cpu]);
3064 tracing_iter_reset(iter, cpu);
3067 mutex_unlock(&trace_types_lock);
3072 mutex_unlock(&trace_types_lock);
3074 kfree(iter->buffer_iter);
3076 seq_release_private(inode, file);
3077 return ERR_PTR(-ENOMEM);
3080 int tracing_open_generic(struct inode *inode, struct file *filp)
3082 if (tracing_disabled)
3085 filp->private_data = inode->i_private;
3089 bool tracing_is_disabled(void)
3091 return (tracing_disabled) ? true: false;
3095 * Open and update trace_array ref count.
3096 * Must have the current trace_array passed to it.
3098 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3100 struct trace_array *tr = inode->i_private;
3102 if (tracing_disabled)
3105 if (trace_array_get(tr) < 0)
3108 filp->private_data = inode->i_private;
3113 static int tracing_release(struct inode *inode, struct file *file)
3115 struct trace_array *tr = inode->i_private;
3116 struct seq_file *m = file->private_data;
3117 struct trace_iterator *iter;
3120 if (!(file->f_mode & FMODE_READ)) {
3121 trace_array_put(tr);
3125 /* Writes do not use seq_file */
3127 mutex_lock(&trace_types_lock);
3129 for_each_tracing_cpu(cpu) {
3130 if (iter->buffer_iter[cpu])
3131 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3134 if (iter->trace && iter->trace->close)
3135 iter->trace->close(iter);
3137 if (!iter->snapshot)
3138 /* reenable tracing if it was previously enabled */
3139 tracing_start_tr(tr);
3141 __trace_array_put(tr);
3143 mutex_unlock(&trace_types_lock);
3145 mutex_destroy(&iter->mutex);
3146 free_cpumask_var(iter->started);
3148 kfree(iter->buffer_iter);
3149 seq_release_private(inode, file);
3154 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3156 struct trace_array *tr = inode->i_private;
3158 trace_array_put(tr);
3162 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3164 struct trace_array *tr = inode->i_private;
3166 trace_array_put(tr);
3168 return single_release(inode, file);
3171 static int tracing_open(struct inode *inode, struct file *file)
3173 struct trace_array *tr = inode->i_private;
3174 struct trace_iterator *iter;
3177 if (trace_array_get(tr) < 0)
3180 /* If this file was open for write, then erase contents */
3181 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3182 int cpu = tracing_get_cpu(inode);
3184 if (cpu == RING_BUFFER_ALL_CPUS)
3185 tracing_reset_online_cpus(&tr->trace_buffer);
3187 tracing_reset(&tr->trace_buffer, cpu);
3190 if (file->f_mode & FMODE_READ) {
3191 iter = __tracing_open(inode, file, false);
3193 ret = PTR_ERR(iter);
3194 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3195 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3199 trace_array_put(tr);
3205 * Some tracers are not suitable for instance buffers.
3206 * A tracer is always available for the global array (toplevel)
3207 * or if it explicitly states that it is.
3210 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3212 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3215 /* Find the next tracer that this trace array may use */
3216 static struct tracer *
3217 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3219 while (t && !trace_ok_for_array(t, tr))
3226 t_next(struct seq_file *m, void *v, loff_t *pos)
3228 struct trace_array *tr = m->private;
3229 struct tracer *t = v;
3234 t = get_tracer_for_array(tr, t->next);
3239 static void *t_start(struct seq_file *m, loff_t *pos)
3241 struct trace_array *tr = m->private;
3245 mutex_lock(&trace_types_lock);
3247 t = get_tracer_for_array(tr, trace_types);
3248 for (; t && l < *pos; t = t_next(m, t, &l))
3254 static void t_stop(struct seq_file *m, void *p)
3256 mutex_unlock(&trace_types_lock);
3259 static int t_show(struct seq_file *m, void *v)
3261 struct tracer *t = v;
3266 seq_puts(m, t->name);
3275 static const struct seq_operations show_traces_seq_ops = {
3282 static int show_traces_open(struct inode *inode, struct file *file)
3284 struct trace_array *tr = inode->i_private;
3288 if (tracing_disabled)
3291 ret = seq_open(file, &show_traces_seq_ops);
3295 m = file->private_data;
3302 tracing_write_stub(struct file *filp, const char __user *ubuf,
3303 size_t count, loff_t *ppos)
3308 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3312 if (file->f_mode & FMODE_READ)
3313 ret = seq_lseek(file, offset, whence);
3315 file->f_pos = ret = 0;
3320 static const struct file_operations tracing_fops = {
3321 .open = tracing_open,
3323 .write = tracing_write_stub,
3324 .llseek = tracing_lseek,
3325 .release = tracing_release,
3328 static const struct file_operations show_traces_fops = {
3329 .open = show_traces_open,
3331 .release = seq_release,
3332 .llseek = seq_lseek,
3336 * The tracer itself will not take this lock, but still we want
3337 * to provide a consistent cpumask to user-space:
3339 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3342 * Temporary storage for the character representation of the
3343 * CPU bitmask (and one more byte for the newline):
3345 static char mask_str[NR_CPUS + 1];
3348 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3349 size_t count, loff_t *ppos)
3351 struct trace_array *tr = file_inode(filp)->i_private;
3354 mutex_lock(&tracing_cpumask_update_lock);
3356 len = snprintf(mask_str, count, "%*pb\n",
3357 cpumask_pr_args(tr->tracing_cpumask));
3362 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3365 mutex_unlock(&tracing_cpumask_update_lock);
3371 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3372 size_t count, loff_t *ppos)
3374 struct trace_array *tr = file_inode(filp)->i_private;
3375 cpumask_var_t tracing_cpumask_new;
3378 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3381 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3385 mutex_lock(&tracing_cpumask_update_lock);
3387 local_irq_disable();
3388 arch_spin_lock(&tr->max_lock);
3389 for_each_tracing_cpu(cpu) {
3391 * Increase/decrease the disabled counter if we are
3392 * about to flip a bit in the cpumask:
3394 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3395 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3396 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3397 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3399 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3400 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3401 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3402 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3405 arch_spin_unlock(&tr->max_lock);
3408 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3410 mutex_unlock(&tracing_cpumask_update_lock);
3411 free_cpumask_var(tracing_cpumask_new);
3416 free_cpumask_var(tracing_cpumask_new);
3421 static const struct file_operations tracing_cpumask_fops = {
3422 .open = tracing_open_generic_tr,
3423 .read = tracing_cpumask_read,
3424 .write = tracing_cpumask_write,
3425 .release = tracing_release_generic_tr,
3426 .llseek = generic_file_llseek,
3429 static int tracing_trace_options_show(struct seq_file *m, void *v)
3431 struct tracer_opt *trace_opts;
3432 struct trace_array *tr = m->private;
3436 mutex_lock(&trace_types_lock);
3437 tracer_flags = tr->current_trace->flags->val;
3438 trace_opts = tr->current_trace->flags->opts;
3440 for (i = 0; trace_options[i]; i++) {
3441 if (trace_flags & (1 << i))
3442 seq_printf(m, "%s\n", trace_options[i]);
3444 seq_printf(m, "no%s\n", trace_options[i]);
3447 for (i = 0; trace_opts[i].name; i++) {
3448 if (tracer_flags & trace_opts[i].bit)
3449 seq_printf(m, "%s\n", trace_opts[i].name);
3451 seq_printf(m, "no%s\n", trace_opts[i].name);
3453 mutex_unlock(&trace_types_lock);
3458 static int __set_tracer_option(struct trace_array *tr,
3459 struct tracer_flags *tracer_flags,
3460 struct tracer_opt *opts, int neg)
3462 struct tracer *trace = tr->current_trace;
3465 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3470 tracer_flags->val &= ~opts->bit;
3472 tracer_flags->val |= opts->bit;
3476 /* Try to assign a tracer specific option */
3477 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3479 struct tracer *trace = tr->current_trace;
3480 struct tracer_flags *tracer_flags = trace->flags;
3481 struct tracer_opt *opts = NULL;
3484 for (i = 0; tracer_flags->opts[i].name; i++) {
3485 opts = &tracer_flags->opts[i];
3487 if (strcmp(cmp, opts->name) == 0)
3488 return __set_tracer_option(tr, trace->flags, opts, neg);
3494 /* Some tracers require overwrite to stay enabled */
3495 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3497 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3503 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3505 /* do nothing if flag is already set */
3506 if (!!(trace_flags & mask) == !!enabled)
3509 /* Give the tracer a chance to approve the change */
3510 if (tr->current_trace->flag_changed)
3511 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3515 trace_flags |= mask;
3517 trace_flags &= ~mask;
3519 if (mask == TRACE_ITER_RECORD_CMD)
3520 trace_event_enable_cmd_record(enabled);
3522 if (mask == TRACE_ITER_OVERWRITE) {
3523 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3524 #ifdef CONFIG_TRACER_MAX_TRACE
3525 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3529 if (mask == TRACE_ITER_PRINTK)
3530 trace_printk_start_stop_comm(enabled);
3535 static int trace_set_options(struct trace_array *tr, char *option)
3542 cmp = strstrip(option);
3544 if (strncmp(cmp, "no", 2) == 0) {
3549 mutex_lock(&trace_types_lock);
3551 for (i = 0; trace_options[i]; i++) {
3552 if (strcmp(cmp, trace_options[i]) == 0) {
3553 ret = set_tracer_flag(tr, 1 << i, !neg);
3558 /* If no option could be set, test the specific tracer options */
3559 if (!trace_options[i])
3560 ret = set_tracer_option(tr, cmp, neg);
3562 mutex_unlock(&trace_types_lock);
3568 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3569 size_t cnt, loff_t *ppos)
3571 struct seq_file *m = filp->private_data;
3572 struct trace_array *tr = m->private;
3576 if (cnt >= sizeof(buf))
3579 if (copy_from_user(&buf, ubuf, cnt))
3584 ret = trace_set_options(tr, buf);
3593 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3595 struct trace_array *tr = inode->i_private;
3598 if (tracing_disabled)
3601 if (trace_array_get(tr) < 0)
3604 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3606 trace_array_put(tr);
3611 static const struct file_operations tracing_iter_fops = {
3612 .open = tracing_trace_options_open,
3614 .llseek = seq_lseek,
3615 .release = tracing_single_release_tr,
3616 .write = tracing_trace_options_write,
3619 static const char readme_msg[] =
3620 "tracing mini-HOWTO:\n\n"
3621 "# echo 0 > tracing_on : quick way to disable tracing\n"
3622 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3623 " Important files:\n"
3624 " trace\t\t\t- The static contents of the buffer\n"
3625 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3626 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3627 " current_tracer\t- function and latency tracers\n"
3628 " available_tracers\t- list of configured tracers for current_tracer\n"
3629 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3630 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3631 " trace_clock\t\t-change the clock used to order events\n"
3632 " local: Per cpu clock but may not be synced across CPUs\n"
3633 " global: Synced across CPUs but slows tracing down.\n"
3634 " counter: Not a clock, but just an increment\n"
3635 " uptime: Jiffy counter from time of boot\n"
3636 " perf: Same clock that perf events use\n"
3637 #ifdef CONFIG_X86_64
3638 " x86-tsc: TSC cycle counter\n"
3640 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3641 " tracing_cpumask\t- Limit which CPUs to trace\n"
3642 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3643 "\t\t\t Remove sub-buffer with rmdir\n"
3644 " trace_options\t\t- Set format or modify how tracing happens\n"
3645 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3646 "\t\t\t option name\n"
3647 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3648 #ifdef CONFIG_DYNAMIC_FTRACE
3649 "\n available_filter_functions - list of functions that can be filtered on\n"
3650 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3651 "\t\t\t functions\n"
3652 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3653 "\t modules: Can select a group via module\n"
3654 "\t Format: :mod:<module-name>\n"
3655 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3656 "\t triggers: a command to perform when function is hit\n"
3657 "\t Format: <function>:<trigger>[:count]\n"
3658 "\t trigger: traceon, traceoff\n"
3659 "\t\t enable_event:<system>:<event>\n"
3660 "\t\t disable_event:<system>:<event>\n"
3661 #ifdef CONFIG_STACKTRACE
3664 #ifdef CONFIG_TRACER_SNAPSHOT
3669 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3670 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3671 "\t The first one will disable tracing every time do_fault is hit\n"
3672 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3673 "\t The first time do trap is hit and it disables tracing, the\n"
3674 "\t counter will decrement to 2. If tracing is already disabled,\n"
3675 "\t the counter will not decrement. It only decrements when the\n"
3676 "\t trigger did work\n"
3677 "\t To remove trigger without count:\n"
3678 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3679 "\t To remove trigger with a count:\n"
3680 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3681 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3682 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3683 "\t modules: Can select a group via module command :mod:\n"
3684 "\t Does not accept triggers\n"
3685 #endif /* CONFIG_DYNAMIC_FTRACE */
3686 #ifdef CONFIG_FUNCTION_TRACER
3687 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3690 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3691 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3692 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3693 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3695 #ifdef CONFIG_TRACER_SNAPSHOT
3696 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3697 "\t\t\t snapshot buffer. Read the contents for more\n"
3698 "\t\t\t information\n"
3700 #ifdef CONFIG_STACK_TRACER
3701 " stack_trace\t\t- Shows the max stack trace when active\n"
3702 " stack_max_size\t- Shows current max stack size that was traced\n"
3703 "\t\t\t Write into this file to reset the max size (trigger a\n"
3704 "\t\t\t new trace)\n"
3705 #ifdef CONFIG_DYNAMIC_FTRACE
3706 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3709 #endif /* CONFIG_STACK_TRACER */
3710 " events/\t\t- Directory containing all trace event subsystems:\n"
3711 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3712 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3713 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3715 " filter\t\t- If set, only events passing filter are traced\n"
3716 " events/<system>/<event>/\t- Directory containing control files for\n"
3718 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3719 " filter\t\t- If set, only events passing filter are traced\n"
3720 " trigger\t\t- If set, a command to perform when event is hit\n"
3721 "\t Format: <trigger>[:count][if <filter>]\n"
3722 "\t trigger: traceon, traceoff\n"
3723 "\t enable_event:<system>:<event>\n"
3724 "\t disable_event:<system>:<event>\n"
3725 #ifdef CONFIG_STACKTRACE
3728 #ifdef CONFIG_TRACER_SNAPSHOT
3731 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3732 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3733 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3734 "\t events/block/block_unplug/trigger\n"
3735 "\t The first disables tracing every time block_unplug is hit.\n"
3736 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3737 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3738 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3739 "\t Like function triggers, the counter is only decremented if it\n"
3740 "\t enabled or disabled tracing.\n"
3741 "\t To remove a trigger without a count:\n"
3742 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3743 "\t To remove a trigger with a count:\n"
3744 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3745 "\t Filters can be ignored when removing a trigger.\n"
3749 tracing_readme_read(struct file *filp, char __user *ubuf,
3750 size_t cnt, loff_t *ppos)
3752 return simple_read_from_buffer(ubuf, cnt, ppos,
3753 readme_msg, strlen(readme_msg));
3756 static const struct file_operations tracing_readme_fops = {
3757 .open = tracing_open_generic,
3758 .read = tracing_readme_read,
3759 .llseek = generic_file_llseek,
3762 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3764 unsigned int *ptr = v;
3766 if (*pos || m->count)
3771 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3773 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3782 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3788 arch_spin_lock(&trace_cmdline_lock);
3790 v = &savedcmd->map_cmdline_to_pid[0];
3792 v = saved_cmdlines_next(m, v, &l);
3800 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3802 arch_spin_unlock(&trace_cmdline_lock);
3806 static int saved_cmdlines_show(struct seq_file *m, void *v)
3808 char buf[TASK_COMM_LEN];
3809 unsigned int *pid = v;
3811 __trace_find_cmdline(*pid, buf);
3812 seq_printf(m, "%d %s\n", *pid, buf);
3816 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3817 .start = saved_cmdlines_start,
3818 .next = saved_cmdlines_next,
3819 .stop = saved_cmdlines_stop,
3820 .show = saved_cmdlines_show,
3823 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3825 if (tracing_disabled)
3828 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3831 static const struct file_operations tracing_saved_cmdlines_fops = {
3832 .open = tracing_saved_cmdlines_open,
3834 .llseek = seq_lseek,
3835 .release = seq_release,
3839 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3840 size_t cnt, loff_t *ppos)
3845 arch_spin_lock(&trace_cmdline_lock);
3846 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3847 arch_spin_unlock(&trace_cmdline_lock);
3849 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3852 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3854 kfree(s->saved_cmdlines);
3855 kfree(s->map_cmdline_to_pid);
3859 static int tracing_resize_saved_cmdlines(unsigned int val)
3861 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3863 s = kmalloc(sizeof(*s), GFP_KERNEL);
3867 if (allocate_cmdlines_buffer(val, s) < 0) {
3872 arch_spin_lock(&trace_cmdline_lock);
3873 savedcmd_temp = savedcmd;
3875 arch_spin_unlock(&trace_cmdline_lock);
3876 free_saved_cmdlines_buffer(savedcmd_temp);
3882 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3883 size_t cnt, loff_t *ppos)
3888 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3892 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3893 if (!val || val > PID_MAX_DEFAULT)
3896 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3905 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3906 .open = tracing_open_generic,
3907 .read = tracing_saved_cmdlines_size_read,
3908 .write = tracing_saved_cmdlines_size_write,
3911 static void trace_insert_enum_map(struct trace_enum_map **start, int len)
3913 struct trace_enum_map **map;
3920 trace_event_enum_update(map, len);
3924 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3925 size_t cnt, loff_t *ppos)
3927 struct trace_array *tr = filp->private_data;
3928 char buf[MAX_TRACER_SIZE+2];
3931 mutex_lock(&trace_types_lock);
3932 r = sprintf(buf, "%s\n", tr->current_trace->name);
3933 mutex_unlock(&trace_types_lock);
3935 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3938 int tracer_init(struct tracer *t, struct trace_array *tr)
3940 tracing_reset_online_cpus(&tr->trace_buffer);
3944 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3948 for_each_tracing_cpu(cpu)
3949 per_cpu_ptr(buf->data, cpu)->entries = val;
3952 #ifdef CONFIG_TRACER_MAX_TRACE
3953 /* resize @tr's buffer to the size of @size_tr's entries */
3954 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3955 struct trace_buffer *size_buf, int cpu_id)
3959 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3960 for_each_tracing_cpu(cpu) {
3961 ret = ring_buffer_resize(trace_buf->buffer,
3962 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3965 per_cpu_ptr(trace_buf->data, cpu)->entries =
3966 per_cpu_ptr(size_buf->data, cpu)->entries;
3969 ret = ring_buffer_resize(trace_buf->buffer,
3970 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3972 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3973 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3978 #endif /* CONFIG_TRACER_MAX_TRACE */
3980 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3981 unsigned long size, int cpu)
3986 * If kernel or user changes the size of the ring buffer
3987 * we use the size that was given, and we can forget about
3988 * expanding it later.
3990 ring_buffer_expanded = true;
3992 /* May be called before buffers are initialized */
3993 if (!tr->trace_buffer.buffer)
3996 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4000 #ifdef CONFIG_TRACER_MAX_TRACE
4001 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4002 !tr->current_trace->use_max_tr)
4005 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4007 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4008 &tr->trace_buffer, cpu);
4011 * AARGH! We are left with different
4012 * size max buffer!!!!
4013 * The max buffer is our "snapshot" buffer.
4014 * When a tracer needs a snapshot (one of the
4015 * latency tracers), it swaps the max buffer
4016 * with the saved snap shot. We succeeded to
4017 * update the size of the main buffer, but failed to
4018 * update the size of the max buffer. But when we tried
4019 * to reset the main buffer to the original size, we
4020 * failed there too. This is very unlikely to
4021 * happen, but if it does, warn and kill all
4025 tracing_disabled = 1;
4030 if (cpu == RING_BUFFER_ALL_CPUS)
4031 set_buffer_entries(&tr->max_buffer, size);
4033 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4036 #endif /* CONFIG_TRACER_MAX_TRACE */
4038 if (cpu == RING_BUFFER_ALL_CPUS)
4039 set_buffer_entries(&tr->trace_buffer, size);
4041 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4046 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4047 unsigned long size, int cpu_id)
4051 mutex_lock(&trace_types_lock);
4053 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4054 /* make sure, this cpu is enabled in the mask */
4055 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4061 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4066 mutex_unlock(&trace_types_lock);
4073 * tracing_update_buffers - used by tracing facility to expand ring buffers
4075 * To save on memory when the tracing is never used on a system with it
4076 * configured in. The ring buffers are set to a minimum size. But once
4077 * a user starts to use the tracing facility, then they need to grow
4078 * to their default size.
4080 * This function is to be called when a tracer is about to be used.
4082 int tracing_update_buffers(void)
4086 mutex_lock(&trace_types_lock);
4087 if (!ring_buffer_expanded)
4088 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4089 RING_BUFFER_ALL_CPUS);
4090 mutex_unlock(&trace_types_lock);
4095 struct trace_option_dentry;
4097 static struct trace_option_dentry *
4098 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4101 destroy_trace_option_files(struct trace_option_dentry *topts);
4104 * Used to clear out the tracer before deletion of an instance.
4105 * Must have trace_types_lock held.
4107 static void tracing_set_nop(struct trace_array *tr)
4109 if (tr->current_trace == &nop_trace)
4112 tr->current_trace->enabled--;
4114 if (tr->current_trace->reset)
4115 tr->current_trace->reset(tr);
4117 tr->current_trace = &nop_trace;
4120 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4122 static struct trace_option_dentry *topts;
4124 #ifdef CONFIG_TRACER_MAX_TRACE
4129 mutex_lock(&trace_types_lock);
4131 if (!ring_buffer_expanded) {
4132 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4133 RING_BUFFER_ALL_CPUS);
4139 for (t = trace_types; t; t = t->next) {
4140 if (strcmp(t->name, buf) == 0)
4147 if (t == tr->current_trace)
4150 /* Some tracers are only allowed for the top level buffer */
4151 if (!trace_ok_for_array(t, tr)) {
4156 /* If trace pipe files are being read, we can't change the tracer */
4157 if (tr->current_trace->ref) {
4162 trace_branch_disable();
4164 tr->current_trace->enabled--;
4166 if (tr->current_trace->reset)
4167 tr->current_trace->reset(tr);
4169 /* Current trace needs to be nop_trace before synchronize_sched */
4170 tr->current_trace = &nop_trace;
4172 #ifdef CONFIG_TRACER_MAX_TRACE
4173 had_max_tr = tr->allocated_snapshot;
4175 if (had_max_tr && !t->use_max_tr) {
4177 * We need to make sure that the update_max_tr sees that
4178 * current_trace changed to nop_trace to keep it from
4179 * swapping the buffers after we resize it.
4180 * The update_max_tr is called from interrupts disabled
4181 * so a synchronized_sched() is sufficient.
4183 synchronize_sched();
4187 /* Currently, only the top instance has options */
4188 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4189 destroy_trace_option_files(topts);
4190 topts = create_trace_option_files(tr, t);
4193 #ifdef CONFIG_TRACER_MAX_TRACE
4194 if (t->use_max_tr && !had_max_tr) {
4195 ret = alloc_snapshot(tr);
4202 ret = tracer_init(t, tr);
4207 tr->current_trace = t;
4208 tr->current_trace->enabled++;
4209 trace_branch_enable(tr);
4211 mutex_unlock(&trace_types_lock);
4217 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4218 size_t cnt, loff_t *ppos)
4220 struct trace_array *tr = filp->private_data;
4221 char buf[MAX_TRACER_SIZE+1];
4228 if (cnt > MAX_TRACER_SIZE)
4229 cnt = MAX_TRACER_SIZE;
4231 if (copy_from_user(&buf, ubuf, cnt))
4236 /* strip ending whitespace. */
4237 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4240 err = tracing_set_tracer(tr, buf);
4250 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4251 size_t cnt, loff_t *ppos)
4256 r = snprintf(buf, sizeof(buf), "%ld\n",
4257 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4258 if (r > sizeof(buf))
4260 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4264 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4265 size_t cnt, loff_t *ppos)
4270 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4280 tracing_thresh_read(struct file *filp, char __user *ubuf,
4281 size_t cnt, loff_t *ppos)
4283 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4287 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4288 size_t cnt, loff_t *ppos)
4290 struct trace_array *tr = filp->private_data;
4293 mutex_lock(&trace_types_lock);
4294 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4298 if (tr->current_trace->update_thresh) {
4299 ret = tr->current_trace->update_thresh(tr);
4306 mutex_unlock(&trace_types_lock);
4312 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4313 size_t cnt, loff_t *ppos)
4315 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4319 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4320 size_t cnt, loff_t *ppos)
4322 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4325 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4327 struct trace_array *tr = inode->i_private;
4328 struct trace_iterator *iter;
4331 if (tracing_disabled)
4334 if (trace_array_get(tr) < 0)
4337 mutex_lock(&trace_types_lock);
4339 /* create a buffer to store the information to pass to userspace */
4340 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4343 __trace_array_put(tr);
4347 trace_seq_init(&iter->seq);
4348 iter->trace = tr->current_trace;
4350 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4355 /* trace pipe does not show start of buffer */
4356 cpumask_setall(iter->started);
4358 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4359 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4361 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4362 if (trace_clocks[tr->clock_id].in_ns)
4363 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4366 iter->trace_buffer = &tr->trace_buffer;
4367 iter->cpu_file = tracing_get_cpu(inode);
4368 mutex_init(&iter->mutex);
4369 filp->private_data = iter;
4371 if (iter->trace->pipe_open)
4372 iter->trace->pipe_open(iter);
4374 nonseekable_open(inode, filp);
4376 tr->current_trace->ref++;
4378 mutex_unlock(&trace_types_lock);
4384 __trace_array_put(tr);
4385 mutex_unlock(&trace_types_lock);
4389 static int tracing_release_pipe(struct inode *inode, struct file *file)
4391 struct trace_iterator *iter = file->private_data;
4392 struct trace_array *tr = inode->i_private;
4394 mutex_lock(&trace_types_lock);
4396 tr->current_trace->ref--;
4398 if (iter->trace->pipe_close)
4399 iter->trace->pipe_close(iter);
4401 mutex_unlock(&trace_types_lock);
4403 free_cpumask_var(iter->started);
4404 mutex_destroy(&iter->mutex);
4407 trace_array_put(tr);
4413 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4415 /* Iterators are static, they should be filled or empty */
4416 if (trace_buffer_iter(iter, iter->cpu_file))
4417 return POLLIN | POLLRDNORM;
4419 if (trace_flags & TRACE_ITER_BLOCK)
4421 * Always select as readable when in blocking mode
4423 return POLLIN | POLLRDNORM;
4425 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4430 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4432 struct trace_iterator *iter = filp->private_data;
4434 return trace_poll(iter, filp, poll_table);
4437 /* Must be called with iter->mutex held. */
4438 static int tracing_wait_pipe(struct file *filp)
4440 struct trace_iterator *iter = filp->private_data;
4443 while (trace_empty(iter)) {
4445 if ((filp->f_flags & O_NONBLOCK)) {
4450 * We block until we read something and tracing is disabled.
4451 * We still block if tracing is disabled, but we have never
4452 * read anything. This allows a user to cat this file, and
4453 * then enable tracing. But after we have read something,
4454 * we give an EOF when tracing is again disabled.
4456 * iter->pos will be 0 if we haven't read anything.
4458 if (!tracing_is_on() && iter->pos)
4461 mutex_unlock(&iter->mutex);
4463 ret = wait_on_pipe(iter, false);
4465 mutex_lock(&iter->mutex);
4478 tracing_read_pipe(struct file *filp, char __user *ubuf,
4479 size_t cnt, loff_t *ppos)
4481 struct trace_iterator *iter = filp->private_data;
4484 /* return any leftover data */
4485 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4489 trace_seq_init(&iter->seq);
4492 * Avoid more than one consumer on a single file descriptor
4493 * This is just a matter of traces coherency, the ring buffer itself
4496 mutex_lock(&iter->mutex);
4497 if (iter->trace->read) {
4498 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4504 sret = tracing_wait_pipe(filp);
4508 /* stop when tracing is finished */
4509 if (trace_empty(iter)) {
4514 if (cnt >= PAGE_SIZE)
4515 cnt = PAGE_SIZE - 1;
4517 /* reset all but tr, trace, and overruns */
4518 memset(&iter->seq, 0,
4519 sizeof(struct trace_iterator) -
4520 offsetof(struct trace_iterator, seq));
4521 cpumask_clear(iter->started);
4524 trace_event_read_lock();
4525 trace_access_lock(iter->cpu_file);
4526 while (trace_find_next_entry_inc(iter) != NULL) {
4527 enum print_line_t ret;
4528 int save_len = iter->seq.seq.len;
4530 ret = print_trace_line(iter);
4531 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4532 /* don't print partial lines */
4533 iter->seq.seq.len = save_len;
4536 if (ret != TRACE_TYPE_NO_CONSUME)
4537 trace_consume(iter);
4539 if (trace_seq_used(&iter->seq) >= cnt)
4543 * Setting the full flag means we reached the trace_seq buffer
4544 * size and we should leave by partial output condition above.
4545 * One of the trace_seq_* functions is not used properly.
4547 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4550 trace_access_unlock(iter->cpu_file);
4551 trace_event_read_unlock();
4553 /* Now copy what we have to the user */
4554 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4555 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4556 trace_seq_init(&iter->seq);
4559 * If there was nothing to send to user, in spite of consuming trace
4560 * entries, go back to wait for more entries.
4566 mutex_unlock(&iter->mutex);
4571 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4574 __free_page(spd->pages[idx]);
4577 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4579 .confirm = generic_pipe_buf_confirm,
4580 .release = generic_pipe_buf_release,
4581 .steal = generic_pipe_buf_steal,
4582 .get = generic_pipe_buf_get,
4586 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4592 /* Seq buffer is page-sized, exactly what we need. */
4594 save_len = iter->seq.seq.len;
4595 ret = print_trace_line(iter);
4597 if (trace_seq_has_overflowed(&iter->seq)) {
4598 iter->seq.seq.len = save_len;
4603 * This should not be hit, because it should only
4604 * be set if the iter->seq overflowed. But check it
4605 * anyway to be safe.
4607 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4608 iter->seq.seq.len = save_len;
4612 count = trace_seq_used(&iter->seq) - save_len;
4615 iter->seq.seq.len = save_len;
4619 if (ret != TRACE_TYPE_NO_CONSUME)
4620 trace_consume(iter);
4622 if (!trace_find_next_entry_inc(iter)) {
4632 static ssize_t tracing_splice_read_pipe(struct file *filp,
4634 struct pipe_inode_info *pipe,
4638 struct page *pages_def[PIPE_DEF_BUFFERS];
4639 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4640 struct trace_iterator *iter = filp->private_data;
4641 struct splice_pipe_desc spd = {
4643 .partial = partial_def,
4644 .nr_pages = 0, /* This gets updated below. */
4645 .nr_pages_max = PIPE_DEF_BUFFERS,
4647 .ops = &tracing_pipe_buf_ops,
4648 .spd_release = tracing_spd_release_pipe,
4654 if (splice_grow_spd(pipe, &spd))
4657 mutex_lock(&iter->mutex);
4659 if (iter->trace->splice_read) {
4660 ret = iter->trace->splice_read(iter, filp,
4661 ppos, pipe, len, flags);
4666 ret = tracing_wait_pipe(filp);
4670 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4675 trace_event_read_lock();
4676 trace_access_lock(iter->cpu_file);
4678 /* Fill as many pages as possible. */
4679 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4680 spd.pages[i] = alloc_page(GFP_KERNEL);
4684 rem = tracing_fill_pipe_page(rem, iter);
4686 /* Copy the data into the page, so we can start over. */
4687 ret = trace_seq_to_buffer(&iter->seq,
4688 page_address(spd.pages[i]),
4689 trace_seq_used(&iter->seq));
4691 __free_page(spd.pages[i]);
4694 spd.partial[i].offset = 0;
4695 spd.partial[i].len = trace_seq_used(&iter->seq);
4697 trace_seq_init(&iter->seq);
4700 trace_access_unlock(iter->cpu_file);
4701 trace_event_read_unlock();
4702 mutex_unlock(&iter->mutex);
4706 ret = splice_to_pipe(pipe, &spd);
4708 splice_shrink_spd(&spd);
4712 mutex_unlock(&iter->mutex);
4717 tracing_entries_read(struct file *filp, char __user *ubuf,
4718 size_t cnt, loff_t *ppos)
4720 struct inode *inode = file_inode(filp);
4721 struct trace_array *tr = inode->i_private;
4722 int cpu = tracing_get_cpu(inode);
4727 mutex_lock(&trace_types_lock);
4729 if (cpu == RING_BUFFER_ALL_CPUS) {
4730 int cpu, buf_size_same;
4735 /* check if all cpu sizes are same */
4736 for_each_tracing_cpu(cpu) {
4737 /* fill in the size from first enabled cpu */
4739 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4740 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4746 if (buf_size_same) {
4747 if (!ring_buffer_expanded)
4748 r = sprintf(buf, "%lu (expanded: %lu)\n",
4750 trace_buf_size >> 10);
4752 r = sprintf(buf, "%lu\n", size >> 10);
4754 r = sprintf(buf, "X\n");
4756 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4758 mutex_unlock(&trace_types_lock);
4760 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4765 tracing_entries_write(struct file *filp, const char __user *ubuf,
4766 size_t cnt, loff_t *ppos)
4768 struct inode *inode = file_inode(filp);
4769 struct trace_array *tr = inode->i_private;
4773 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4777 /* must have at least 1 entry */
4781 /* value is in KB */
4783 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4793 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4794 size_t cnt, loff_t *ppos)
4796 struct trace_array *tr = filp->private_data;
4799 unsigned long size = 0, expanded_size = 0;
4801 mutex_lock(&trace_types_lock);
4802 for_each_tracing_cpu(cpu) {
4803 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4804 if (!ring_buffer_expanded)
4805 expanded_size += trace_buf_size >> 10;
4807 if (ring_buffer_expanded)
4808 r = sprintf(buf, "%lu\n", size);
4810 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4811 mutex_unlock(&trace_types_lock);
4813 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4817 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4818 size_t cnt, loff_t *ppos)
4821 * There is no need to read what the user has written, this function
4822 * is just to make sure that there is no error when "echo" is used
4831 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4833 struct trace_array *tr = inode->i_private;
4835 /* disable tracing ? */
4836 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4837 tracer_tracing_off(tr);
4838 /* resize the ring buffer to 0 */
4839 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4841 trace_array_put(tr);
4847 tracing_mark_write(struct file *filp, const char __user *ubuf,
4848 size_t cnt, loff_t *fpos)
4850 unsigned long addr = (unsigned long)ubuf;
4851 struct trace_array *tr = filp->private_data;
4852 struct ring_buffer_event *event;
4853 struct ring_buffer *buffer;
4854 struct print_entry *entry;
4855 unsigned long irq_flags;
4856 struct page *pages[2];
4866 if (tracing_disabled)
4869 if (!(trace_flags & TRACE_ITER_MARKERS))
4872 if (cnt > TRACE_BUF_SIZE)
4873 cnt = TRACE_BUF_SIZE;
4876 * Userspace is injecting traces into the kernel trace buffer.
4877 * We want to be as non intrusive as possible.
4878 * To do so, we do not want to allocate any special buffers
4879 * or take any locks, but instead write the userspace data
4880 * straight into the ring buffer.
4882 * First we need to pin the userspace buffer into memory,
4883 * which, most likely it is, because it just referenced it.
4884 * But there's no guarantee that it is. By using get_user_pages_fast()
4885 * and kmap_atomic/kunmap_atomic() we can get access to the
4886 * pages directly. We then write the data directly into the
4889 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4891 /* check if we cross pages */
4892 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4895 offset = addr & (PAGE_SIZE - 1);
4898 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4899 if (ret < nr_pages) {
4901 put_page(pages[ret]);
4906 for (i = 0; i < nr_pages; i++)
4907 map_page[i] = kmap_atomic(pages[i]);
4909 local_save_flags(irq_flags);
4910 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4911 buffer = tr->trace_buffer.buffer;
4912 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4913 irq_flags, preempt_count());
4915 /* Ring buffer disabled, return as if not open for write */
4920 entry = ring_buffer_event_data(event);
4921 entry->ip = _THIS_IP_;
4923 if (nr_pages == 2) {
4924 len = PAGE_SIZE - offset;
4925 memcpy(&entry->buf, map_page[0] + offset, len);
4926 memcpy(&entry->buf[len], map_page[1], cnt - len);
4928 memcpy(&entry->buf, map_page[0] + offset, cnt);
4930 if (entry->buf[cnt - 1] != '\n') {
4931 entry->buf[cnt] = '\n';
4932 entry->buf[cnt + 1] = '\0';
4934 entry->buf[cnt] = '\0';
4936 __buffer_unlock_commit(buffer, event);
4943 for (i = nr_pages - 1; i >= 0; i--) {
4944 kunmap_atomic(map_page[i]);
4951 static int tracing_clock_show(struct seq_file *m, void *v)
4953 struct trace_array *tr = m->private;
4956 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4958 "%s%s%s%s", i ? " " : "",
4959 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4960 i == tr->clock_id ? "]" : "");
4966 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4970 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4971 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4974 if (i == ARRAY_SIZE(trace_clocks))
4977 mutex_lock(&trace_types_lock);
4981 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4984 * New clock may not be consistent with the previous clock.
4985 * Reset the buffer so that it doesn't have incomparable timestamps.
4987 tracing_reset_online_cpus(&tr->trace_buffer);
4989 #ifdef CONFIG_TRACER_MAX_TRACE
4990 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4991 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4992 tracing_reset_online_cpus(&tr->max_buffer);
4995 mutex_unlock(&trace_types_lock);
5000 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5001 size_t cnt, loff_t *fpos)
5003 struct seq_file *m = filp->private_data;
5004 struct trace_array *tr = m->private;
5006 const char *clockstr;
5009 if (cnt >= sizeof(buf))
5012 if (copy_from_user(&buf, ubuf, cnt))
5017 clockstr = strstrip(buf);
5019 ret = tracing_set_clock(tr, clockstr);
5028 static int tracing_clock_open(struct inode *inode, struct file *file)
5030 struct trace_array *tr = inode->i_private;
5033 if (tracing_disabled)
5036 if (trace_array_get(tr))
5039 ret = single_open(file, tracing_clock_show, inode->i_private);
5041 trace_array_put(tr);
5046 struct ftrace_buffer_info {
5047 struct trace_iterator iter;
5052 #ifdef CONFIG_TRACER_SNAPSHOT
5053 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5055 struct trace_array *tr = inode->i_private;
5056 struct trace_iterator *iter;
5060 if (trace_array_get(tr) < 0)
5063 if (file->f_mode & FMODE_READ) {
5064 iter = __tracing_open(inode, file, true);
5066 ret = PTR_ERR(iter);
5068 /* Writes still need the seq_file to hold the private data */
5070 m = kzalloc(sizeof(*m), GFP_KERNEL);
5073 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5081 iter->trace_buffer = &tr->max_buffer;
5082 iter->cpu_file = tracing_get_cpu(inode);
5084 file->private_data = m;
5088 trace_array_put(tr);
5094 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5097 struct seq_file *m = filp->private_data;
5098 struct trace_iterator *iter = m->private;
5099 struct trace_array *tr = iter->tr;
5103 ret = tracing_update_buffers();
5107 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5111 mutex_lock(&trace_types_lock);
5113 if (tr->current_trace->use_max_tr) {
5120 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5124 if (tr->allocated_snapshot)
5128 /* Only allow per-cpu swap if the ring buffer supports it */
5129 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5130 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5135 if (!tr->allocated_snapshot) {
5136 ret = alloc_snapshot(tr);
5140 local_irq_disable();
5141 /* Now, we're going to swap */
5142 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5143 update_max_tr(tr, current, smp_processor_id());
5145 update_max_tr_single(tr, current, iter->cpu_file);
5149 if (tr->allocated_snapshot) {
5150 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5151 tracing_reset_online_cpus(&tr->max_buffer);
5153 tracing_reset(&tr->max_buffer, iter->cpu_file);
5163 mutex_unlock(&trace_types_lock);
5167 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5169 struct seq_file *m = file->private_data;
5172 ret = tracing_release(inode, file);
5174 if (file->f_mode & FMODE_READ)
5177 /* If write only, the seq_file is just a stub */
5185 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5186 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5187 size_t count, loff_t *ppos);
5188 static int tracing_buffers_release(struct inode *inode, struct file *file);
5189 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5190 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5192 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5194 struct ftrace_buffer_info *info;
5197 ret = tracing_buffers_open(inode, filp);
5201 info = filp->private_data;
5203 if (info->iter.trace->use_max_tr) {
5204 tracing_buffers_release(inode, filp);
5208 info->iter.snapshot = true;
5209 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5214 #endif /* CONFIG_TRACER_SNAPSHOT */
5217 static const struct file_operations tracing_thresh_fops = {
5218 .open = tracing_open_generic,
5219 .read = tracing_thresh_read,
5220 .write = tracing_thresh_write,
5221 .llseek = generic_file_llseek,
5224 static const struct file_operations tracing_max_lat_fops = {
5225 .open = tracing_open_generic,
5226 .read = tracing_max_lat_read,
5227 .write = tracing_max_lat_write,
5228 .llseek = generic_file_llseek,
5231 static const struct file_operations set_tracer_fops = {
5232 .open = tracing_open_generic,
5233 .read = tracing_set_trace_read,
5234 .write = tracing_set_trace_write,
5235 .llseek = generic_file_llseek,
5238 static const struct file_operations tracing_pipe_fops = {
5239 .open = tracing_open_pipe,
5240 .poll = tracing_poll_pipe,
5241 .read = tracing_read_pipe,
5242 .splice_read = tracing_splice_read_pipe,
5243 .release = tracing_release_pipe,
5244 .llseek = no_llseek,
5247 static const struct file_operations tracing_entries_fops = {
5248 .open = tracing_open_generic_tr,
5249 .read = tracing_entries_read,
5250 .write = tracing_entries_write,
5251 .llseek = generic_file_llseek,
5252 .release = tracing_release_generic_tr,
5255 static const struct file_operations tracing_total_entries_fops = {
5256 .open = tracing_open_generic_tr,
5257 .read = tracing_total_entries_read,
5258 .llseek = generic_file_llseek,
5259 .release = tracing_release_generic_tr,
5262 static const struct file_operations tracing_free_buffer_fops = {
5263 .open = tracing_open_generic_tr,
5264 .write = tracing_free_buffer_write,
5265 .release = tracing_free_buffer_release,
5268 static const struct file_operations tracing_mark_fops = {
5269 .open = tracing_open_generic_tr,
5270 .write = tracing_mark_write,
5271 .llseek = generic_file_llseek,
5272 .release = tracing_release_generic_tr,
5275 static const struct file_operations trace_clock_fops = {
5276 .open = tracing_clock_open,
5278 .llseek = seq_lseek,
5279 .release = tracing_single_release_tr,
5280 .write = tracing_clock_write,
5283 #ifdef CONFIG_TRACER_SNAPSHOT
5284 static const struct file_operations snapshot_fops = {
5285 .open = tracing_snapshot_open,
5287 .write = tracing_snapshot_write,
5288 .llseek = tracing_lseek,
5289 .release = tracing_snapshot_release,
5292 static const struct file_operations snapshot_raw_fops = {
5293 .open = snapshot_raw_open,
5294 .read = tracing_buffers_read,
5295 .release = tracing_buffers_release,
5296 .splice_read = tracing_buffers_splice_read,
5297 .llseek = no_llseek,
5300 #endif /* CONFIG_TRACER_SNAPSHOT */
5302 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5304 struct trace_array *tr = inode->i_private;
5305 struct ftrace_buffer_info *info;
5308 if (tracing_disabled)
5311 if (trace_array_get(tr) < 0)
5314 info = kzalloc(sizeof(*info), GFP_KERNEL);
5316 trace_array_put(tr);
5320 mutex_lock(&trace_types_lock);
5323 info->iter.cpu_file = tracing_get_cpu(inode);
5324 info->iter.trace = tr->current_trace;
5325 info->iter.trace_buffer = &tr->trace_buffer;
5327 /* Force reading ring buffer for first read */
5328 info->read = (unsigned int)-1;
5330 filp->private_data = info;
5332 tr->current_trace->ref++;
5334 mutex_unlock(&trace_types_lock);
5336 ret = nonseekable_open(inode, filp);
5338 trace_array_put(tr);
5344 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5346 struct ftrace_buffer_info *info = filp->private_data;
5347 struct trace_iterator *iter = &info->iter;
5349 return trace_poll(iter, filp, poll_table);
5353 tracing_buffers_read(struct file *filp, char __user *ubuf,
5354 size_t count, loff_t *ppos)
5356 struct ftrace_buffer_info *info = filp->private_data;
5357 struct trace_iterator *iter = &info->iter;
5364 #ifdef CONFIG_TRACER_MAX_TRACE
5365 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5370 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5375 /* Do we have previous read data to read? */
5376 if (info->read < PAGE_SIZE)
5380 trace_access_lock(iter->cpu_file);
5381 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5385 trace_access_unlock(iter->cpu_file);
5388 if (trace_empty(iter)) {
5389 if ((filp->f_flags & O_NONBLOCK))
5392 ret = wait_on_pipe(iter, false);
5403 size = PAGE_SIZE - info->read;
5407 ret = copy_to_user(ubuf, info->spare + info->read, size);
5419 static int tracing_buffers_release(struct inode *inode, struct file *file)
5421 struct ftrace_buffer_info *info = file->private_data;
5422 struct trace_iterator *iter = &info->iter;
5424 mutex_lock(&trace_types_lock);
5426 iter->tr->current_trace->ref--;
5428 __trace_array_put(iter->tr);
5431 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5434 mutex_unlock(&trace_types_lock);
5440 struct ring_buffer *buffer;
5445 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5446 struct pipe_buffer *buf)
5448 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5453 ring_buffer_free_read_page(ref->buffer, ref->page);
5458 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5459 struct pipe_buffer *buf)
5461 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5466 /* Pipe buffer operations for a buffer. */
5467 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5469 .confirm = generic_pipe_buf_confirm,
5470 .release = buffer_pipe_buf_release,
5471 .steal = generic_pipe_buf_steal,
5472 .get = buffer_pipe_buf_get,
5476 * Callback from splice_to_pipe(), if we need to release some pages
5477 * at the end of the spd in case we error'ed out in filling the pipe.
5479 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5481 struct buffer_ref *ref =
5482 (struct buffer_ref *)spd->partial[i].private;
5487 ring_buffer_free_read_page(ref->buffer, ref->page);
5489 spd->partial[i].private = 0;
5493 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5494 struct pipe_inode_info *pipe, size_t len,
5497 struct ftrace_buffer_info *info = file->private_data;
5498 struct trace_iterator *iter = &info->iter;
5499 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5500 struct page *pages_def[PIPE_DEF_BUFFERS];
5501 struct splice_pipe_desc spd = {
5503 .partial = partial_def,
5504 .nr_pages_max = PIPE_DEF_BUFFERS,
5506 .ops = &buffer_pipe_buf_ops,
5507 .spd_release = buffer_spd_release,
5509 struct buffer_ref *ref;
5510 int entries, size, i;
5513 #ifdef CONFIG_TRACER_MAX_TRACE
5514 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5518 if (splice_grow_spd(pipe, &spd))
5521 if (*ppos & (PAGE_SIZE - 1))
5524 if (len & (PAGE_SIZE - 1)) {
5525 if (len < PAGE_SIZE)
5531 trace_access_lock(iter->cpu_file);
5532 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5534 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5538 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5545 ref->buffer = iter->trace_buffer->buffer;
5546 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5553 r = ring_buffer_read_page(ref->buffer, &ref->page,
5554 len, iter->cpu_file, 1);
5556 ring_buffer_free_read_page(ref->buffer, ref->page);
5562 * zero out any left over data, this is going to
5565 size = ring_buffer_page_len(ref->page);
5566 if (size < PAGE_SIZE)
5567 memset(ref->page + size, 0, PAGE_SIZE - size);
5569 page = virt_to_page(ref->page);
5571 spd.pages[i] = page;
5572 spd.partial[i].len = PAGE_SIZE;
5573 spd.partial[i].offset = 0;
5574 spd.partial[i].private = (unsigned long)ref;
5578 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5581 trace_access_unlock(iter->cpu_file);
5584 /* did we read anything? */
5585 if (!spd.nr_pages) {
5589 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5592 ret = wait_on_pipe(iter, true);
5599 ret = splice_to_pipe(pipe, &spd);
5600 splice_shrink_spd(&spd);
5605 static const struct file_operations tracing_buffers_fops = {
5606 .open = tracing_buffers_open,
5607 .read = tracing_buffers_read,
5608 .poll = tracing_buffers_poll,
5609 .release = tracing_buffers_release,
5610 .splice_read = tracing_buffers_splice_read,
5611 .llseek = no_llseek,
5615 tracing_stats_read(struct file *filp, char __user *ubuf,
5616 size_t count, loff_t *ppos)
5618 struct inode *inode = file_inode(filp);
5619 struct trace_array *tr = inode->i_private;
5620 struct trace_buffer *trace_buf = &tr->trace_buffer;
5621 int cpu = tracing_get_cpu(inode);
5622 struct trace_seq *s;
5624 unsigned long long t;
5625 unsigned long usec_rem;
5627 s = kmalloc(sizeof(*s), GFP_KERNEL);
5633 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5634 trace_seq_printf(s, "entries: %ld\n", cnt);
5636 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5637 trace_seq_printf(s, "overrun: %ld\n", cnt);
5639 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5640 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5642 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5643 trace_seq_printf(s, "bytes: %ld\n", cnt);
5645 if (trace_clocks[tr->clock_id].in_ns) {
5646 /* local or global for trace_clock */
5647 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5648 usec_rem = do_div(t, USEC_PER_SEC);
5649 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5652 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5653 usec_rem = do_div(t, USEC_PER_SEC);
5654 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5656 /* counter or tsc mode for trace_clock */
5657 trace_seq_printf(s, "oldest event ts: %llu\n",
5658 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5660 trace_seq_printf(s, "now ts: %llu\n",
5661 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5664 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5665 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5667 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5668 trace_seq_printf(s, "read events: %ld\n", cnt);
5670 count = simple_read_from_buffer(ubuf, count, ppos,
5671 s->buffer, trace_seq_used(s));
5678 static const struct file_operations tracing_stats_fops = {
5679 .open = tracing_open_generic_tr,
5680 .read = tracing_stats_read,
5681 .llseek = generic_file_llseek,
5682 .release = tracing_release_generic_tr,
5685 #ifdef CONFIG_DYNAMIC_FTRACE
5687 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5693 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5694 size_t cnt, loff_t *ppos)
5696 static char ftrace_dyn_info_buffer[1024];
5697 static DEFINE_MUTEX(dyn_info_mutex);
5698 unsigned long *p = filp->private_data;
5699 char *buf = ftrace_dyn_info_buffer;
5700 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5703 mutex_lock(&dyn_info_mutex);
5704 r = sprintf(buf, "%ld ", *p);
5706 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5709 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5711 mutex_unlock(&dyn_info_mutex);
5716 static const struct file_operations tracing_dyn_info_fops = {
5717 .open = tracing_open_generic,
5718 .read = tracing_read_dyn_info,
5719 .llseek = generic_file_llseek,
5721 #endif /* CONFIG_DYNAMIC_FTRACE */
5723 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5725 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5731 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5733 unsigned long *count = (long *)data;
5745 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5746 struct ftrace_probe_ops *ops, void *data)
5748 long count = (long)data;
5750 seq_printf(m, "%ps:", (void *)ip);
5752 seq_puts(m, "snapshot");
5755 seq_puts(m, ":unlimited\n");
5757 seq_printf(m, ":count=%ld\n", count);
5762 static struct ftrace_probe_ops snapshot_probe_ops = {
5763 .func = ftrace_snapshot,
5764 .print = ftrace_snapshot_print,
5767 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5768 .func = ftrace_count_snapshot,
5769 .print = ftrace_snapshot_print,
5773 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5774 char *glob, char *cmd, char *param, int enable)
5776 struct ftrace_probe_ops *ops;
5777 void *count = (void *)-1;
5781 /* hash funcs only work with set_ftrace_filter */
5785 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5787 if (glob[0] == '!') {
5788 unregister_ftrace_function_probe_func(glob+1, ops);
5795 number = strsep(¶m, ":");
5797 if (!strlen(number))
5801 * We use the callback data field (which is a pointer)
5804 ret = kstrtoul(number, 0, (unsigned long *)&count);
5809 ret = register_ftrace_function_probe(glob, ops, count);
5812 alloc_snapshot(&global_trace);
5814 return ret < 0 ? ret : 0;
5817 static struct ftrace_func_command ftrace_snapshot_cmd = {
5819 .func = ftrace_trace_snapshot_callback,
5822 static __init int register_snapshot_cmd(void)
5824 return register_ftrace_command(&ftrace_snapshot_cmd);
5827 static inline __init int register_snapshot_cmd(void) { return 0; }
5828 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5830 static struct dentry *tracing_get_dentry(struct trace_array *tr)
5835 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5837 struct dentry *d_tracer;
5840 return tr->percpu_dir;
5842 d_tracer = tracing_get_dentry(tr);
5843 if (IS_ERR(d_tracer))
5846 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5848 WARN_ONCE(!tr->percpu_dir,
5849 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5851 return tr->percpu_dir;
5854 static struct dentry *
5855 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5856 void *data, long cpu, const struct file_operations *fops)
5858 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5860 if (ret) /* See tracing_get_cpu() */
5861 ret->d_inode->i_cdev = (void *)(cpu + 1);
5866 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5868 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5869 struct dentry *d_cpu;
5870 char cpu_dir[30]; /* 30 characters should be more than enough */
5875 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5876 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5878 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5882 /* per cpu trace_pipe */
5883 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5884 tr, cpu, &tracing_pipe_fops);
5887 trace_create_cpu_file("trace", 0644, d_cpu,
5888 tr, cpu, &tracing_fops);
5890 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5891 tr, cpu, &tracing_buffers_fops);
5893 trace_create_cpu_file("stats", 0444, d_cpu,
5894 tr, cpu, &tracing_stats_fops);
5896 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5897 tr, cpu, &tracing_entries_fops);
5899 #ifdef CONFIG_TRACER_SNAPSHOT
5900 trace_create_cpu_file("snapshot", 0644, d_cpu,
5901 tr, cpu, &snapshot_fops);
5903 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5904 tr, cpu, &snapshot_raw_fops);
5908 #ifdef CONFIG_FTRACE_SELFTEST
5909 /* Let selftest have access to static functions in this file */
5910 #include "trace_selftest.c"
5913 struct trace_option_dentry {
5914 struct tracer_opt *opt;
5915 struct tracer_flags *flags;
5916 struct trace_array *tr;
5917 struct dentry *entry;
5921 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5924 struct trace_option_dentry *topt = filp->private_data;
5927 if (topt->flags->val & topt->opt->bit)
5932 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5936 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5939 struct trace_option_dentry *topt = filp->private_data;
5943 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5947 if (val != 0 && val != 1)
5950 if (!!(topt->flags->val & topt->opt->bit) != val) {
5951 mutex_lock(&trace_types_lock);
5952 ret = __set_tracer_option(topt->tr, topt->flags,
5954 mutex_unlock(&trace_types_lock);
5965 static const struct file_operations trace_options_fops = {
5966 .open = tracing_open_generic,
5967 .read = trace_options_read,
5968 .write = trace_options_write,
5969 .llseek = generic_file_llseek,
5973 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5976 long index = (long)filp->private_data;
5979 if (trace_flags & (1 << index))
5984 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5988 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5991 struct trace_array *tr = &global_trace;
5992 long index = (long)filp->private_data;
5996 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6000 if (val != 0 && val != 1)
6003 mutex_lock(&trace_types_lock);
6004 ret = set_tracer_flag(tr, 1 << index, val);
6005 mutex_unlock(&trace_types_lock);
6015 static const struct file_operations trace_options_core_fops = {
6016 .open = tracing_open_generic,
6017 .read = trace_options_core_read,
6018 .write = trace_options_core_write,
6019 .llseek = generic_file_llseek,
6022 struct dentry *trace_create_file(const char *name,
6024 struct dentry *parent,
6026 const struct file_operations *fops)
6030 ret = debugfs_create_file(name, mode, parent, data, fops);
6032 pr_warning("Could not create debugfs '%s' entry\n", name);
6038 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6040 struct dentry *d_tracer;
6045 d_tracer = tracing_get_dentry(tr);
6046 if (IS_ERR(d_tracer))
6049 tr->options = debugfs_create_dir("options", d_tracer);
6051 pr_warning("Could not create debugfs directory 'options'\n");
6059 create_trace_option_file(struct trace_array *tr,
6060 struct trace_option_dentry *topt,
6061 struct tracer_flags *flags,
6062 struct tracer_opt *opt)
6064 struct dentry *t_options;
6066 t_options = trace_options_init_dentry(tr);
6070 topt->flags = flags;
6074 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6075 &trace_options_fops);
6079 static struct trace_option_dentry *
6080 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6082 struct trace_option_dentry *topts;
6083 struct tracer_flags *flags;
6084 struct tracer_opt *opts;
6090 flags = tracer->flags;
6092 if (!flags || !flags->opts)
6097 for (cnt = 0; opts[cnt].name; cnt++)
6100 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6104 for (cnt = 0; opts[cnt].name; cnt++)
6105 create_trace_option_file(tr, &topts[cnt], flags,
6112 destroy_trace_option_files(struct trace_option_dentry *topts)
6119 for (cnt = 0; topts[cnt].opt; cnt++)
6120 debugfs_remove(topts[cnt].entry);
6125 static struct dentry *
6126 create_trace_option_core_file(struct trace_array *tr,
6127 const char *option, long index)
6129 struct dentry *t_options;
6131 t_options = trace_options_init_dentry(tr);
6135 return trace_create_file(option, 0644, t_options, (void *)index,
6136 &trace_options_core_fops);
6139 static __init void create_trace_options_dir(struct trace_array *tr)
6141 struct dentry *t_options;
6144 t_options = trace_options_init_dentry(tr);
6148 for (i = 0; trace_options[i]; i++)
6149 create_trace_option_core_file(tr, trace_options[i], i);
6153 rb_simple_read(struct file *filp, char __user *ubuf,
6154 size_t cnt, loff_t *ppos)
6156 struct trace_array *tr = filp->private_data;
6160 r = tracer_tracing_is_on(tr);
6161 r = sprintf(buf, "%d\n", r);
6163 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6167 rb_simple_write(struct file *filp, const char __user *ubuf,
6168 size_t cnt, loff_t *ppos)
6170 struct trace_array *tr = filp->private_data;
6171 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6175 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6180 mutex_lock(&trace_types_lock);
6182 tracer_tracing_on(tr);
6183 if (tr->current_trace->start)
6184 tr->current_trace->start(tr);
6186 tracer_tracing_off(tr);
6187 if (tr->current_trace->stop)
6188 tr->current_trace->stop(tr);
6190 mutex_unlock(&trace_types_lock);
6198 static const struct file_operations rb_simple_fops = {
6199 .open = tracing_open_generic_tr,
6200 .read = rb_simple_read,
6201 .write = rb_simple_write,
6202 .release = tracing_release_generic_tr,
6203 .llseek = default_llseek,
6206 struct dentry *trace_instance_dir;
6209 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6212 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6214 enum ring_buffer_flags rb_flags;
6216 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6220 buf->buffer = ring_buffer_alloc(size, rb_flags);
6224 buf->data = alloc_percpu(struct trace_array_cpu);
6226 ring_buffer_free(buf->buffer);
6230 /* Allocate the first page for all buffers */
6231 set_buffer_entries(&tr->trace_buffer,
6232 ring_buffer_size(tr->trace_buffer.buffer, 0));
6237 static int allocate_trace_buffers(struct trace_array *tr, int size)
6241 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6245 #ifdef CONFIG_TRACER_MAX_TRACE
6246 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6247 allocate_snapshot ? size : 1);
6249 ring_buffer_free(tr->trace_buffer.buffer);
6250 free_percpu(tr->trace_buffer.data);
6253 tr->allocated_snapshot = allocate_snapshot;
6256 * Only the top level trace array gets its snapshot allocated
6257 * from the kernel command line.
6259 allocate_snapshot = false;
6264 static void free_trace_buffer(struct trace_buffer *buf)
6267 ring_buffer_free(buf->buffer);
6269 free_percpu(buf->data);
6274 static void free_trace_buffers(struct trace_array *tr)
6279 free_trace_buffer(&tr->trace_buffer);
6281 #ifdef CONFIG_TRACER_MAX_TRACE
6282 free_trace_buffer(&tr->max_buffer);
6286 static int new_instance_create(const char *name)
6288 struct trace_array *tr;
6291 mutex_lock(&trace_types_lock);
6294 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6295 if (tr->name && strcmp(tr->name, name) == 0)
6300 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6304 tr->name = kstrdup(name, GFP_KERNEL);
6308 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6311 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6313 raw_spin_lock_init(&tr->start_lock);
6315 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6317 tr->current_trace = &nop_trace;
6319 INIT_LIST_HEAD(&tr->systems);
6320 INIT_LIST_HEAD(&tr->events);
6322 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6325 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6329 ret = event_trace_add_tracer(tr->dir, tr);
6331 debugfs_remove_recursive(tr->dir);
6335 init_tracer_debugfs(tr, tr->dir);
6337 list_add(&tr->list, &ftrace_trace_arrays);
6339 mutex_unlock(&trace_types_lock);
6344 free_trace_buffers(tr);
6345 free_cpumask_var(tr->tracing_cpumask);
6350 mutex_unlock(&trace_types_lock);
6356 static int instance_delete(const char *name)
6358 struct trace_array *tr;
6362 mutex_lock(&trace_types_lock);
6365 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6366 if (tr->name && strcmp(tr->name, name) == 0) {
6375 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6378 list_del(&tr->list);
6380 tracing_set_nop(tr);
6381 event_trace_del_tracer(tr);
6382 ftrace_destroy_function_files(tr);
6383 debugfs_remove_recursive(tr->dir);
6384 free_trace_buffers(tr);
6392 mutex_unlock(&trace_types_lock);
6397 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6399 struct dentry *parent;
6402 /* Paranoid: Make sure the parent is the "instances" directory */
6403 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6404 if (WARN_ON_ONCE(parent != trace_instance_dir))
6408 * The inode mutex is locked, but debugfs_create_dir() will also
6409 * take the mutex. As the instances directory can not be destroyed
6410 * or changed in any other way, it is safe to unlock it, and
6411 * let the dentry try. If two users try to make the same dir at
6412 * the same time, then the new_instance_create() will determine the
6415 mutex_unlock(&inode->i_mutex);
6417 ret = new_instance_create(dentry->d_iname);
6419 mutex_lock(&inode->i_mutex);
6424 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6426 struct dentry *parent;
6429 /* Paranoid: Make sure the parent is the "instances" directory */
6430 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6431 if (WARN_ON_ONCE(parent != trace_instance_dir))
6434 /* The caller did a dget() on dentry */
6435 mutex_unlock(&dentry->d_inode->i_mutex);
6438 * The inode mutex is locked, but debugfs_create_dir() will also
6439 * take the mutex. As the instances directory can not be destroyed
6440 * or changed in any other way, it is safe to unlock it, and
6441 * let the dentry try. If two users try to make the same dir at
6442 * the same time, then the instance_delete() will determine the
6445 mutex_unlock(&inode->i_mutex);
6447 ret = instance_delete(dentry->d_iname);
6449 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6450 mutex_lock(&dentry->d_inode->i_mutex);
6455 static const struct inode_operations instance_dir_inode_operations = {
6456 .lookup = simple_lookup,
6457 .mkdir = instance_mkdir,
6458 .rmdir = instance_rmdir,
6461 static __init void create_trace_instances(struct dentry *d_tracer)
6463 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6464 if (WARN_ON(!trace_instance_dir))
6467 /* Hijack the dir inode operations, to allow mkdir */
6468 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6472 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6476 trace_create_file("available_tracers", 0444, d_tracer,
6477 tr, &show_traces_fops);
6479 trace_create_file("current_tracer", 0644, d_tracer,
6480 tr, &set_tracer_fops);
6482 trace_create_file("tracing_cpumask", 0644, d_tracer,
6483 tr, &tracing_cpumask_fops);
6485 trace_create_file("trace_options", 0644, d_tracer,
6486 tr, &tracing_iter_fops);
6488 trace_create_file("trace", 0644, d_tracer,
6491 trace_create_file("trace_pipe", 0444, d_tracer,
6492 tr, &tracing_pipe_fops);
6494 trace_create_file("buffer_size_kb", 0644, d_tracer,
6495 tr, &tracing_entries_fops);
6497 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6498 tr, &tracing_total_entries_fops);
6500 trace_create_file("free_buffer", 0200, d_tracer,
6501 tr, &tracing_free_buffer_fops);
6503 trace_create_file("trace_marker", 0220, d_tracer,
6504 tr, &tracing_mark_fops);
6506 trace_create_file("trace_clock", 0644, d_tracer, tr,
6509 trace_create_file("tracing_on", 0644, d_tracer,
6510 tr, &rb_simple_fops);
6512 #ifdef CONFIG_TRACER_MAX_TRACE
6513 trace_create_file("tracing_max_latency", 0644, d_tracer,
6514 &tr->max_latency, &tracing_max_lat_fops);
6517 if (ftrace_create_function_files(tr, d_tracer))
6518 WARN(1, "Could not allocate function filter files");
6520 #ifdef CONFIG_TRACER_SNAPSHOT
6521 trace_create_file("snapshot", 0644, d_tracer,
6522 tr, &snapshot_fops);
6525 for_each_tracing_cpu(cpu)
6526 tracing_init_debugfs_percpu(tr, cpu);
6531 * tracing_init_dentry - initialize top level trace array
6533 * This is called when creating files or directories in the tracing
6534 * directory. It is called via fs_initcall() by any of the boot up code
6535 * and expects to return the dentry of the top level tracing directory.
6537 struct dentry *tracing_init_dentry(void)
6539 struct trace_array *tr = &global_trace;
6544 if (WARN_ON(!debugfs_initialized()))
6545 return ERR_PTR(-ENODEV);
6547 tr->dir = debugfs_create_dir("tracing", NULL);
6550 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6551 return ERR_PTR(-ENOMEM);
6557 extern struct trace_enum_map *__start_ftrace_enum_maps[];
6558 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6560 static void __init trace_enum_init(void)
6564 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6565 trace_insert_enum_map(__start_ftrace_enum_maps, len);
6568 #ifdef CONFIG_MODULES
6569 static void trace_module_add_enums(struct module *mod)
6571 if (!mod->num_trace_enums)
6575 * Modules with bad taint do not have events created, do
6576 * not bother with enums either.
6578 if (trace_module_has_bad_taint(mod))
6581 trace_insert_enum_map(mod->trace_enums, mod->num_trace_enums);
6584 static int trace_module_notify(struct notifier_block *self,
6585 unsigned long val, void *data)
6587 struct module *mod = data;
6590 case MODULE_STATE_COMING:
6591 trace_module_add_enums(mod);
6598 static struct notifier_block trace_module_nb = {
6599 .notifier_call = trace_module_notify,
6604 static __init int tracer_init_debugfs(void)
6606 struct dentry *d_tracer;
6608 trace_access_lock_init();
6610 d_tracer = tracing_init_dentry();
6611 if (IS_ERR(d_tracer))
6614 init_tracer_debugfs(&global_trace, d_tracer);
6616 trace_create_file("tracing_thresh", 0644, d_tracer,
6617 &global_trace, &tracing_thresh_fops);
6619 trace_create_file("README", 0444, d_tracer,
6620 NULL, &tracing_readme_fops);
6622 trace_create_file("saved_cmdlines", 0444, d_tracer,
6623 NULL, &tracing_saved_cmdlines_fops);
6625 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6626 NULL, &tracing_saved_cmdlines_size_fops);
6630 #ifdef CONFIG_MODULES
6631 register_module_notifier(&trace_module_nb);
6634 #ifdef CONFIG_DYNAMIC_FTRACE
6635 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6636 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6639 create_trace_instances(d_tracer);
6641 create_trace_options_dir(&global_trace);
6646 static int trace_panic_handler(struct notifier_block *this,
6647 unsigned long event, void *unused)
6649 if (ftrace_dump_on_oops)
6650 ftrace_dump(ftrace_dump_on_oops);
6654 static struct notifier_block trace_panic_notifier = {
6655 .notifier_call = trace_panic_handler,
6657 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6660 static int trace_die_handler(struct notifier_block *self,
6666 if (ftrace_dump_on_oops)
6667 ftrace_dump(ftrace_dump_on_oops);
6675 static struct notifier_block trace_die_notifier = {
6676 .notifier_call = trace_die_handler,
6681 * printk is set to max of 1024, we really don't need it that big.
6682 * Nothing should be printing 1000 characters anyway.
6684 #define TRACE_MAX_PRINT 1000
6687 * Define here KERN_TRACE so that we have one place to modify
6688 * it if we decide to change what log level the ftrace dump
6691 #define KERN_TRACE KERN_EMERG
6694 trace_printk_seq(struct trace_seq *s)
6696 /* Probably should print a warning here. */
6697 if (s->seq.len >= TRACE_MAX_PRINT)
6698 s->seq.len = TRACE_MAX_PRINT;
6701 * More paranoid code. Although the buffer size is set to
6702 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6703 * an extra layer of protection.
6705 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6706 s->seq.len = s->seq.size - 1;
6708 /* should be zero ended, but we are paranoid. */
6709 s->buffer[s->seq.len] = 0;
6711 printk(KERN_TRACE "%s", s->buffer);
6716 void trace_init_global_iter(struct trace_iterator *iter)
6718 iter->tr = &global_trace;
6719 iter->trace = iter->tr->current_trace;
6720 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6721 iter->trace_buffer = &global_trace.trace_buffer;
6723 if (iter->trace && iter->trace->open)
6724 iter->trace->open(iter);
6726 /* Annotate start of buffers if we had overruns */
6727 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6728 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6730 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6731 if (trace_clocks[iter->tr->clock_id].in_ns)
6732 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6735 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6737 /* use static because iter can be a bit big for the stack */
6738 static struct trace_iterator iter;
6739 static atomic_t dump_running;
6740 unsigned int old_userobj;
6741 unsigned long flags;
6744 /* Only allow one dump user at a time. */
6745 if (atomic_inc_return(&dump_running) != 1) {
6746 atomic_dec(&dump_running);
6751 * Always turn off tracing when we dump.
6752 * We don't need to show trace output of what happens
6753 * between multiple crashes.
6755 * If the user does a sysrq-z, then they can re-enable
6756 * tracing with echo 1 > tracing_on.
6760 local_irq_save(flags);
6762 /* Simulate the iterator */
6763 trace_init_global_iter(&iter);
6765 for_each_tracing_cpu(cpu) {
6766 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6769 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6771 /* don't look at user memory in panic mode */
6772 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6774 switch (oops_dump_mode) {
6776 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6779 iter.cpu_file = raw_smp_processor_id();
6784 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6785 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6788 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6790 /* Did function tracer already get disabled? */
6791 if (ftrace_is_dead()) {
6792 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6793 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6797 * We need to stop all tracing on all CPUS to read the
6798 * the next buffer. This is a bit expensive, but is
6799 * not done often. We fill all what we can read,
6800 * and then release the locks again.
6803 while (!trace_empty(&iter)) {
6806 printk(KERN_TRACE "---------------------------------\n");
6810 /* reset all but tr, trace, and overruns */
6811 memset(&iter.seq, 0,
6812 sizeof(struct trace_iterator) -
6813 offsetof(struct trace_iterator, seq));
6814 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6817 if (trace_find_next_entry_inc(&iter) != NULL) {
6820 ret = print_trace_line(&iter);
6821 if (ret != TRACE_TYPE_NO_CONSUME)
6822 trace_consume(&iter);
6824 touch_nmi_watchdog();
6826 trace_printk_seq(&iter.seq);
6830 printk(KERN_TRACE " (ftrace buffer empty)\n");
6832 printk(KERN_TRACE "---------------------------------\n");
6835 trace_flags |= old_userobj;
6837 for_each_tracing_cpu(cpu) {
6838 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6840 atomic_dec(&dump_running);
6841 local_irq_restore(flags);
6843 EXPORT_SYMBOL_GPL(ftrace_dump);
6845 __init static int tracer_alloc_buffers(void)
6850 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6853 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6854 goto out_free_buffer_mask;
6856 /* Only allocate trace_printk buffers if a trace_printk exists */
6857 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6858 /* Must be called before global_trace.buffer is allocated */
6859 trace_printk_init_buffers();
6861 /* To save memory, keep the ring buffer size to its minimum */
6862 if (ring_buffer_expanded)
6863 ring_buf_size = trace_buf_size;
6867 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6868 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6870 raw_spin_lock_init(&global_trace.start_lock);
6872 /* Used for event triggers */
6873 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6875 goto out_free_cpumask;
6877 if (trace_create_savedcmd() < 0)
6878 goto out_free_temp_buffer;
6880 /* TODO: make the number of buffers hot pluggable with CPUS */
6881 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6882 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6884 goto out_free_savedcmd;
6887 if (global_trace.buffer_disabled)
6890 if (trace_boot_clock) {
6891 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6893 pr_warning("Trace clock %s not defined, going back to default\n",
6898 * register_tracer() might reference current_trace, so it
6899 * needs to be set before we register anything. This is
6900 * just a bootstrap of current_trace anyway.
6902 global_trace.current_trace = &nop_trace;
6904 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6906 ftrace_init_global_array_ops(&global_trace);
6908 register_tracer(&nop_trace);
6910 /* All seems OK, enable tracing */
6911 tracing_disabled = 0;
6913 atomic_notifier_chain_register(&panic_notifier_list,
6914 &trace_panic_notifier);
6916 register_die_notifier(&trace_die_notifier);
6918 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6920 INIT_LIST_HEAD(&global_trace.systems);
6921 INIT_LIST_HEAD(&global_trace.events);
6922 list_add(&global_trace.list, &ftrace_trace_arrays);
6924 while (trace_boot_options) {
6927 option = strsep(&trace_boot_options, ",");
6928 trace_set_options(&global_trace, option);
6931 register_snapshot_cmd();
6936 free_saved_cmdlines_buffer(savedcmd);
6937 out_free_temp_buffer:
6938 ring_buffer_free(temp_buffer);
6940 free_cpumask_var(global_trace.tracing_cpumask);
6941 out_free_buffer_mask:
6942 free_cpumask_var(tracing_buffer_mask);
6947 void __init trace_init(void)
6949 if (tracepoint_printk) {
6950 tracepoint_print_iter =
6951 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
6952 if (WARN_ON(!tracepoint_print_iter))
6953 tracepoint_printk = 0;
6955 tracer_alloc_buffers();
6959 __init static int clear_boot_tracer(void)
6962 * The default tracer at boot buffer is an init section.
6963 * This function is called in lateinit. If we did not
6964 * find the boot tracer, then clear it out, to prevent
6965 * later registration from accessing the buffer that is
6966 * about to be freed.
6968 if (!default_bootup_tracer)
6971 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6972 default_bootup_tracer);
6973 default_bootup_tracer = NULL;
6978 fs_initcall(tracer_init_debugfs);
6979 late_initcall(clear_boot_tracer);