2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 __disable_trace_on_warning = 1;
161 __setup("traceoff_on_warning=", stop_trace_on_warning);
163 static int __init boot_alloc_snapshot(char *str)
165 allocate_snapshot = true;
166 /* We also need the main ring buffer expanded */
167 ring_buffer_expanded = true;
170 __setup("alloc_snapshot", boot_alloc_snapshot);
173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
174 static char *trace_boot_options __initdata;
176 static int __init set_trace_boot_options(char *str)
178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 trace_boot_options = trace_boot_options_buf;
182 __setup("trace_options=", set_trace_boot_options);
184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
185 static char *trace_boot_clock __initdata;
187 static int __init set_trace_boot_clock(char *str)
189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
190 trace_boot_clock = trace_boot_clock_buf;
193 __setup("trace_clock=", set_trace_boot_clock);
196 unsigned long long ns2usecs(cycle_t nsec)
204 * The global_trace is the descriptor that holds the tracing
205 * buffers for the live tracing. For each CPU, it contains
206 * a link list of pages that will store trace entries. The
207 * page descriptor of the pages in the memory is used to hold
208 * the link list by linking the lru item in the page descriptor
209 * to each of the pages in the buffer per CPU.
211 * For each active CPU there is a data field that holds the
212 * pages for the buffer for that CPU. Each CPU has the same number
213 * of pages allocated for its buffer.
215 static struct trace_array global_trace;
217 LIST_HEAD(ftrace_trace_arrays);
219 int trace_array_get(struct trace_array *this_tr)
221 struct trace_array *tr;
224 mutex_lock(&trace_types_lock);
225 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
232 mutex_unlock(&trace_types_lock);
237 static void __trace_array_put(struct trace_array *this_tr)
239 WARN_ON(!this_tr->ref);
243 void trace_array_put(struct trace_array *this_tr)
245 mutex_lock(&trace_types_lock);
246 __trace_array_put(this_tr);
247 mutex_unlock(&trace_types_lock);
250 int filter_check_discard(struct ftrace_event_file *file, void *rec,
251 struct ring_buffer *buffer,
252 struct ring_buffer_event *event)
254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
255 !filter_match_preds(file->filter, rec)) {
256 ring_buffer_discard_commit(buffer, event);
262 EXPORT_SYMBOL_GPL(filter_check_discard);
264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
265 struct ring_buffer *buffer,
266 struct ring_buffer_event *event)
268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
269 !filter_match_preds(call->filter, rec)) {
270 ring_buffer_discard_commit(buffer, event);
276 EXPORT_SYMBOL_GPL(call_filter_check_discard);
278 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
282 /* Early boot up does not have a buffer yet */
284 return trace_clock_local();
286 ts = ring_buffer_time_stamp(buf->buffer, cpu);
287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
292 cycle_t ftrace_now(int cpu)
294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
298 * tracing_is_enabled - Show if global_trace has been disabled
300 * Shows if the global trace has been enabled or not. It uses the
301 * mirror flag "buffer_disabled" to be used in fast paths such as for
302 * the irqsoff tracer. But it may be inaccurate due to races. If you
303 * need to know the accurate state, use tracing_is_on() which is a little
304 * slower, but accurate.
306 int tracing_is_enabled(void)
309 * For quick access (irqsoff uses this in fast path), just
310 * return the mirror variable of the state of the ring buffer.
311 * It's a little racy, but we don't really care.
314 return !global_trace.buffer_disabled;
318 * trace_buf_size is the size in bytes that is allocated
319 * for a buffer. Note, the number of bytes is always rounded
322 * This number is purposely set to a low number of 16384.
323 * If the dump on oops happens, it will be much appreciated
324 * to not have to wait for all that output. Anyway this can be
325 * boot time and run time configurable.
327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
331 /* trace_types holds a link list of available tracers. */
332 static struct tracer *trace_types __read_mostly;
335 * trace_types_lock is used to protect the trace_types list.
337 DEFINE_MUTEX(trace_types_lock);
340 * serialize the access of the ring buffer
342 * ring buffer serializes readers, but it is low level protection.
343 * The validity of the events (which returns by ring_buffer_peek() ..etc)
344 * are not protected by ring buffer.
346 * The content of events may become garbage if we allow other process consumes
347 * these events concurrently:
348 * A) the page of the consumed events may become a normal page
349 * (not reader page) in ring buffer, and this page will be rewrited
350 * by events producer.
351 * B) The page of the consumed events may become a page for splice_read,
352 * and this page will be returned to system.
354 * These primitives allow multi process access to different cpu ring buffer
357 * These primitives don't distinguish read-only and read-consume access.
358 * Multi read-only access are also serialized.
362 static DECLARE_RWSEM(all_cpu_access_lock);
363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365 static inline void trace_access_lock(int cpu)
367 if (cpu == RING_BUFFER_ALL_CPUS) {
368 /* gain it for accessing the whole ring buffer. */
369 down_write(&all_cpu_access_lock);
371 /* gain it for accessing a cpu ring buffer. */
373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 down_read(&all_cpu_access_lock);
376 /* Secondly block other access to this @cpu ring buffer. */
377 mutex_lock(&per_cpu(cpu_access_lock, cpu));
381 static inline void trace_access_unlock(int cpu)
383 if (cpu == RING_BUFFER_ALL_CPUS) {
384 up_write(&all_cpu_access_lock);
386 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
387 up_read(&all_cpu_access_lock);
391 static inline void trace_access_lock_init(void)
395 for_each_possible_cpu(cpu)
396 mutex_init(&per_cpu(cpu_access_lock, cpu));
401 static DEFINE_MUTEX(access_lock);
403 static inline void trace_access_lock(int cpu)
406 mutex_lock(&access_lock);
409 static inline void trace_access_unlock(int cpu)
412 mutex_unlock(&access_lock);
415 static inline void trace_access_lock_init(void)
421 /* trace_flags holds trace_options default values */
422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
427 static void tracer_tracing_on(struct trace_array *tr)
429 if (tr->trace_buffer.buffer)
430 ring_buffer_record_on(tr->trace_buffer.buffer);
432 * This flag is looked at when buffers haven't been allocated
433 * yet, or by some tracers (like irqsoff), that just want to
434 * know if the ring buffer has been disabled, but it can handle
435 * races of where it gets disabled but we still do a record.
436 * As the check is in the fast path of the tracers, it is more
437 * important to be fast than accurate.
439 tr->buffer_disabled = 0;
440 /* Make the flag seen by readers */
445 * tracing_on - enable tracing buffers
447 * This function enables tracing buffers that may have been
448 * disabled with tracing_off.
450 void tracing_on(void)
452 tracer_tracing_on(&global_trace);
454 EXPORT_SYMBOL_GPL(tracing_on);
457 * __trace_puts - write a constant string into the trace buffer.
458 * @ip: The address of the caller
459 * @str: The constant string to write
460 * @size: The size of the string.
462 int __trace_puts(unsigned long ip, const char *str, int size)
464 struct ring_buffer_event *event;
465 struct ring_buffer *buffer;
466 struct print_entry *entry;
467 unsigned long irq_flags;
471 if (!(trace_flags & TRACE_ITER_PRINTK))
474 pc = preempt_count();
476 if (unlikely(tracing_selftest_running || tracing_disabled))
479 alloc = sizeof(*entry) + size + 2; /* possible \n added */
481 local_save_flags(irq_flags);
482 buffer = global_trace.trace_buffer.buffer;
483 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
488 entry = ring_buffer_event_data(event);
491 memcpy(&entry->buf, str, size);
493 /* Add a newline if necessary */
494 if (entry->buf[size - 1] != '\n') {
495 entry->buf[size] = '\n';
496 entry->buf[size + 1] = '\0';
498 entry->buf[size] = '\0';
500 __buffer_unlock_commit(buffer, event);
501 ftrace_trace_stack(buffer, irq_flags, 4, pc);
505 EXPORT_SYMBOL_GPL(__trace_puts);
508 * __trace_bputs - write the pointer to a constant string into trace buffer
509 * @ip: The address of the caller
510 * @str: The constant string to write to the buffer to
512 int __trace_bputs(unsigned long ip, const char *str)
514 struct ring_buffer_event *event;
515 struct ring_buffer *buffer;
516 struct bputs_entry *entry;
517 unsigned long irq_flags;
518 int size = sizeof(struct bputs_entry);
521 if (!(trace_flags & TRACE_ITER_PRINTK))
524 pc = preempt_count();
526 if (unlikely(tracing_selftest_running || tracing_disabled))
529 local_save_flags(irq_flags);
530 buffer = global_trace.trace_buffer.buffer;
531 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
536 entry = ring_buffer_event_data(event);
540 __buffer_unlock_commit(buffer, event);
541 ftrace_trace_stack(buffer, irq_flags, 4, pc);
545 EXPORT_SYMBOL_GPL(__trace_bputs);
547 #ifdef CONFIG_TRACER_SNAPSHOT
549 * trace_snapshot - take a snapshot of the current buffer.
551 * This causes a swap between the snapshot buffer and the current live
552 * tracing buffer. You can use this to take snapshots of the live
553 * trace when some condition is triggered, but continue to trace.
555 * Note, make sure to allocate the snapshot with either
556 * a tracing_snapshot_alloc(), or by doing it manually
557 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
559 * If the snapshot buffer is not allocated, it will stop tracing.
560 * Basically making a permanent snapshot.
562 void tracing_snapshot(void)
564 struct trace_array *tr = &global_trace;
565 struct tracer *tracer = tr->current_trace;
569 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
570 internal_trace_puts("*** snapshot is being ignored ***\n");
574 if (!tr->allocated_snapshot) {
575 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
576 internal_trace_puts("*** stopping trace here! ***\n");
581 /* Note, snapshot can not be used when the tracer uses it */
582 if (tracer->use_max_tr) {
583 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
584 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
588 local_irq_save(flags);
589 update_max_tr(tr, current, smp_processor_id());
590 local_irq_restore(flags);
592 EXPORT_SYMBOL_GPL(tracing_snapshot);
594 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
595 struct trace_buffer *size_buf, int cpu_id);
596 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
598 static int alloc_snapshot(struct trace_array *tr)
602 if (!tr->allocated_snapshot) {
604 /* allocate spare buffer */
605 ret = resize_buffer_duplicate_size(&tr->max_buffer,
606 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
610 tr->allocated_snapshot = true;
616 static void free_snapshot(struct trace_array *tr)
619 * We don't free the ring buffer. instead, resize it because
620 * The max_tr ring buffer has some state (e.g. ring->clock) and
621 * we want preserve it.
623 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
624 set_buffer_entries(&tr->max_buffer, 1);
625 tracing_reset_online_cpus(&tr->max_buffer);
626 tr->allocated_snapshot = false;
630 * tracing_alloc_snapshot - allocate snapshot buffer.
632 * This only allocates the snapshot buffer if it isn't already
633 * allocated - it doesn't also take a snapshot.
635 * This is meant to be used in cases where the snapshot buffer needs
636 * to be set up for events that can't sleep but need to be able to
637 * trigger a snapshot.
639 int tracing_alloc_snapshot(void)
641 struct trace_array *tr = &global_trace;
644 ret = alloc_snapshot(tr);
649 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
652 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
654 * This is similar to trace_snapshot(), but it will allocate the
655 * snapshot buffer if it isn't already allocated. Use this only
656 * where it is safe to sleep, as the allocation may sleep.
658 * This causes a swap between the snapshot buffer and the current live
659 * tracing buffer. You can use this to take snapshots of the live
660 * trace when some condition is triggered, but continue to trace.
662 void tracing_snapshot_alloc(void)
666 ret = tracing_alloc_snapshot();
672 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
674 void tracing_snapshot(void)
676 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
678 EXPORT_SYMBOL_GPL(tracing_snapshot);
679 int tracing_alloc_snapshot(void)
681 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
684 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
685 void tracing_snapshot_alloc(void)
690 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
691 #endif /* CONFIG_TRACER_SNAPSHOT */
693 static void tracer_tracing_off(struct trace_array *tr)
695 if (tr->trace_buffer.buffer)
696 ring_buffer_record_off(tr->trace_buffer.buffer);
698 * This flag is looked at when buffers haven't been allocated
699 * yet, or by some tracers (like irqsoff), that just want to
700 * know if the ring buffer has been disabled, but it can handle
701 * races of where it gets disabled but we still do a record.
702 * As the check is in the fast path of the tracers, it is more
703 * important to be fast than accurate.
705 tr->buffer_disabled = 1;
706 /* Make the flag seen by readers */
711 * tracing_off - turn off tracing buffers
713 * This function stops the tracing buffers from recording data.
714 * It does not disable any overhead the tracers themselves may
715 * be causing. This function simply causes all recording to
716 * the ring buffers to fail.
718 void tracing_off(void)
720 tracer_tracing_off(&global_trace);
722 EXPORT_SYMBOL_GPL(tracing_off);
724 void disable_trace_on_warning(void)
726 if (__disable_trace_on_warning)
731 * tracer_tracing_is_on - show real state of ring buffer enabled
732 * @tr : the trace array to know if ring buffer is enabled
734 * Shows real state of the ring buffer if it is enabled or not.
736 static int tracer_tracing_is_on(struct trace_array *tr)
738 if (tr->trace_buffer.buffer)
739 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
740 return !tr->buffer_disabled;
744 * tracing_is_on - show state of ring buffers enabled
746 int tracing_is_on(void)
748 return tracer_tracing_is_on(&global_trace);
750 EXPORT_SYMBOL_GPL(tracing_is_on);
752 static int __init set_buf_size(char *str)
754 unsigned long buf_size;
758 buf_size = memparse(str, &str);
759 /* nr_entries can not be zero */
762 trace_buf_size = buf_size;
765 __setup("trace_buf_size=", set_buf_size);
767 static int __init set_tracing_thresh(char *str)
769 unsigned long threshold;
774 ret = kstrtoul(str, 0, &threshold);
777 tracing_thresh = threshold * 1000;
780 __setup("tracing_thresh=", set_tracing_thresh);
782 unsigned long nsecs_to_usecs(unsigned long nsecs)
787 /* These must match the bit postions in trace_iterator_flags */
788 static const char *trace_options[] = {
821 int in_ns; /* is this clock in nanoseconds? */
823 { trace_clock_local, "local", 1 },
824 { trace_clock_global, "global", 1 },
825 { trace_clock_counter, "counter", 0 },
826 { trace_clock_jiffies, "uptime", 0 },
827 { trace_clock, "perf", 1 },
828 { ktime_get_mono_fast_ns, "mono", 1 },
833 * trace_parser_get_init - gets the buffer for trace parser
835 int trace_parser_get_init(struct trace_parser *parser, int size)
837 memset(parser, 0, sizeof(*parser));
839 parser->buffer = kmalloc(size, GFP_KERNEL);
848 * trace_parser_put - frees the buffer for trace parser
850 void trace_parser_put(struct trace_parser *parser)
852 kfree(parser->buffer);
856 * trace_get_user - reads the user input string separated by space
857 * (matched by isspace(ch))
859 * For each string found the 'struct trace_parser' is updated,
860 * and the function returns.
862 * Returns number of bytes read.
864 * See kernel/trace/trace.h for 'struct trace_parser' details.
866 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
867 size_t cnt, loff_t *ppos)
874 trace_parser_clear(parser);
876 ret = get_user(ch, ubuf++);
884 * The parser is not finished with the last write,
885 * continue reading the user input without skipping spaces.
888 /* skip white space */
889 while (cnt && isspace(ch)) {
890 ret = get_user(ch, ubuf++);
897 /* only spaces were written */
907 /* read the non-space input */
908 while (cnt && !isspace(ch)) {
909 if (parser->idx < parser->size - 1)
910 parser->buffer[parser->idx++] = ch;
915 ret = get_user(ch, ubuf++);
922 /* We either got finished input or we have to wait for another call. */
924 parser->buffer[parser->idx] = 0;
925 parser->cont = false;
926 } else if (parser->idx < parser->size - 1) {
928 parser->buffer[parser->idx++] = ch;
941 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
945 if (s->len <= s->readpos)
948 len = s->len - s->readpos;
951 memcpy(buf, s->buffer + s->readpos, cnt);
957 unsigned long __read_mostly tracing_thresh;
959 #ifdef CONFIG_TRACER_MAX_TRACE
961 * Copy the new maximum trace into the separate maximum-trace
962 * structure. (this way the maximum trace is permanently saved,
963 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
966 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
968 struct trace_buffer *trace_buf = &tr->trace_buffer;
969 struct trace_buffer *max_buf = &tr->max_buffer;
970 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
971 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
974 max_buf->time_start = data->preempt_timestamp;
976 max_data->saved_latency = tr->max_latency;
977 max_data->critical_start = data->critical_start;
978 max_data->critical_end = data->critical_end;
980 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
981 max_data->pid = tsk->pid;
983 * If tsk == current, then use current_uid(), as that does not use
984 * RCU. The irq tracer can be called out of RCU scope.
987 max_data->uid = current_uid();
989 max_data->uid = task_uid(tsk);
991 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
992 max_data->policy = tsk->policy;
993 max_data->rt_priority = tsk->rt_priority;
995 /* record this tasks comm */
996 tracing_record_cmdline(tsk);
1000 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1002 * @tsk: the task with the latency
1003 * @cpu: The cpu that initiated the trace.
1005 * Flip the buffers between the @tr and the max_tr and record information
1006 * about which task was the cause of this latency.
1009 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1011 struct ring_buffer *buf;
1016 WARN_ON_ONCE(!irqs_disabled());
1018 if (!tr->allocated_snapshot) {
1019 /* Only the nop tracer should hit this when disabling */
1020 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1024 arch_spin_lock(&tr->max_lock);
1026 buf = tr->trace_buffer.buffer;
1027 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1028 tr->max_buffer.buffer = buf;
1030 __update_max_tr(tr, tsk, cpu);
1031 arch_spin_unlock(&tr->max_lock);
1035 * update_max_tr_single - only copy one trace over, and reset the rest
1037 * @tsk - task with the latency
1038 * @cpu - the cpu of the buffer to copy.
1040 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1043 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1050 WARN_ON_ONCE(!irqs_disabled());
1051 if (!tr->allocated_snapshot) {
1052 /* Only the nop tracer should hit this when disabling */
1053 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1057 arch_spin_lock(&tr->max_lock);
1059 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1061 if (ret == -EBUSY) {
1063 * We failed to swap the buffer due to a commit taking
1064 * place on this CPU. We fail to record, but we reset
1065 * the max trace buffer (no one writes directly to it)
1066 * and flag that it failed.
1068 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1069 "Failed to swap buffers due to commit in progress\n");
1072 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1074 __update_max_tr(tr, tsk, cpu);
1075 arch_spin_unlock(&tr->max_lock);
1077 #endif /* CONFIG_TRACER_MAX_TRACE */
1079 static int wait_on_pipe(struct trace_iterator *iter)
1081 /* Iterators are static, they should be filled or empty */
1082 if (trace_buffer_iter(iter, iter->cpu_file))
1085 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1088 #ifdef CONFIG_FTRACE_STARTUP_TEST
1089 static int run_tracer_selftest(struct tracer *type)
1091 struct trace_array *tr = &global_trace;
1092 struct tracer *saved_tracer = tr->current_trace;
1095 if (!type->selftest || tracing_selftest_disabled)
1099 * Run a selftest on this tracer.
1100 * Here we reset the trace buffer, and set the current
1101 * tracer to be this tracer. The tracer can then run some
1102 * internal tracing to verify that everything is in order.
1103 * If we fail, we do not register this tracer.
1105 tracing_reset_online_cpus(&tr->trace_buffer);
1107 tr->current_trace = type;
1109 #ifdef CONFIG_TRACER_MAX_TRACE
1110 if (type->use_max_tr) {
1111 /* If we expanded the buffers, make sure the max is expanded too */
1112 if (ring_buffer_expanded)
1113 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1114 RING_BUFFER_ALL_CPUS);
1115 tr->allocated_snapshot = true;
1119 /* the test is responsible for initializing and enabling */
1120 pr_info("Testing tracer %s: ", type->name);
1121 ret = type->selftest(type, tr);
1122 /* the test is responsible for resetting too */
1123 tr->current_trace = saved_tracer;
1125 printk(KERN_CONT "FAILED!\n");
1126 /* Add the warning after printing 'FAILED' */
1130 /* Only reset on passing, to avoid touching corrupted buffers */
1131 tracing_reset_online_cpus(&tr->trace_buffer);
1133 #ifdef CONFIG_TRACER_MAX_TRACE
1134 if (type->use_max_tr) {
1135 tr->allocated_snapshot = false;
1137 /* Shrink the max buffer again */
1138 if (ring_buffer_expanded)
1139 ring_buffer_resize(tr->max_buffer.buffer, 1,
1140 RING_BUFFER_ALL_CPUS);
1144 printk(KERN_CONT "PASSED\n");
1148 static inline int run_tracer_selftest(struct tracer *type)
1152 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1155 * register_tracer - register a tracer with the ftrace system.
1156 * @type - the plugin for the tracer
1158 * Register a new plugin tracer.
1160 int register_tracer(struct tracer *type)
1166 pr_info("Tracer must have a name\n");
1170 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1171 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1175 mutex_lock(&trace_types_lock);
1177 tracing_selftest_running = true;
1179 for (t = trace_types; t; t = t->next) {
1180 if (strcmp(type->name, t->name) == 0) {
1182 pr_info("Tracer %s already registered\n",
1189 if (!type->set_flag)
1190 type->set_flag = &dummy_set_flag;
1192 type->flags = &dummy_tracer_flags;
1194 if (!type->flags->opts)
1195 type->flags->opts = dummy_tracer_opt;
1197 ret = run_tracer_selftest(type);
1201 type->next = trace_types;
1205 tracing_selftest_running = false;
1206 mutex_unlock(&trace_types_lock);
1208 if (ret || !default_bootup_tracer)
1211 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1214 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1215 /* Do we want this tracer to start on bootup? */
1216 tracing_set_tracer(&global_trace, type->name);
1217 default_bootup_tracer = NULL;
1218 /* disable other selftests, since this will break it. */
1219 tracing_selftest_disabled = true;
1220 #ifdef CONFIG_FTRACE_STARTUP_TEST
1221 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1229 void tracing_reset(struct trace_buffer *buf, int cpu)
1231 struct ring_buffer *buffer = buf->buffer;
1236 ring_buffer_record_disable(buffer);
1238 /* Make sure all commits have finished */
1239 synchronize_sched();
1240 ring_buffer_reset_cpu(buffer, cpu);
1242 ring_buffer_record_enable(buffer);
1245 void tracing_reset_online_cpus(struct trace_buffer *buf)
1247 struct ring_buffer *buffer = buf->buffer;
1253 ring_buffer_record_disable(buffer);
1255 /* Make sure all commits have finished */
1256 synchronize_sched();
1258 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1260 for_each_online_cpu(cpu)
1261 ring_buffer_reset_cpu(buffer, cpu);
1263 ring_buffer_record_enable(buffer);
1266 /* Must have trace_types_lock held */
1267 void tracing_reset_all_online_cpus(void)
1269 struct trace_array *tr;
1271 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1272 tracing_reset_online_cpus(&tr->trace_buffer);
1273 #ifdef CONFIG_TRACER_MAX_TRACE
1274 tracing_reset_online_cpus(&tr->max_buffer);
1279 #define SAVED_CMDLINES_DEFAULT 128
1280 #define NO_CMDLINE_MAP UINT_MAX
1281 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1282 struct saved_cmdlines_buffer {
1283 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1284 unsigned *map_cmdline_to_pid;
1285 unsigned cmdline_num;
1287 char *saved_cmdlines;
1289 static struct saved_cmdlines_buffer *savedcmd;
1291 /* temporary disable recording */
1292 static atomic_t trace_record_cmdline_disabled __read_mostly;
1294 static inline char *get_saved_cmdlines(int idx)
1296 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1299 static inline void set_cmdline(int idx, const char *cmdline)
1301 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1304 static int allocate_cmdlines_buffer(unsigned int val,
1305 struct saved_cmdlines_buffer *s)
1307 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1309 if (!s->map_cmdline_to_pid)
1312 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1313 if (!s->saved_cmdlines) {
1314 kfree(s->map_cmdline_to_pid);
1319 s->cmdline_num = val;
1320 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1321 sizeof(s->map_pid_to_cmdline));
1322 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1323 val * sizeof(*s->map_cmdline_to_pid));
1328 static int trace_create_savedcmd(void)
1332 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1336 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1346 int is_tracing_stopped(void)
1348 return global_trace.stop_count;
1352 * tracing_start - quick start of the tracer
1354 * If tracing is enabled but was stopped by tracing_stop,
1355 * this will start the tracer back up.
1357 void tracing_start(void)
1359 struct ring_buffer *buffer;
1360 unsigned long flags;
1362 if (tracing_disabled)
1365 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1366 if (--global_trace.stop_count) {
1367 if (global_trace.stop_count < 0) {
1368 /* Someone screwed up their debugging */
1370 global_trace.stop_count = 0;
1375 /* Prevent the buffers from switching */
1376 arch_spin_lock(&global_trace.max_lock);
1378 buffer = global_trace.trace_buffer.buffer;
1380 ring_buffer_record_enable(buffer);
1382 #ifdef CONFIG_TRACER_MAX_TRACE
1383 buffer = global_trace.max_buffer.buffer;
1385 ring_buffer_record_enable(buffer);
1388 arch_spin_unlock(&global_trace.max_lock);
1391 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1394 static void tracing_start_tr(struct trace_array *tr)
1396 struct ring_buffer *buffer;
1397 unsigned long flags;
1399 if (tracing_disabled)
1402 /* If global, we need to also start the max tracer */
1403 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1404 return tracing_start();
1406 raw_spin_lock_irqsave(&tr->start_lock, flags);
1408 if (--tr->stop_count) {
1409 if (tr->stop_count < 0) {
1410 /* Someone screwed up their debugging */
1417 buffer = tr->trace_buffer.buffer;
1419 ring_buffer_record_enable(buffer);
1422 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1426 * tracing_stop - quick stop of the tracer
1428 * Light weight way to stop tracing. Use in conjunction with
1431 void tracing_stop(void)
1433 struct ring_buffer *buffer;
1434 unsigned long flags;
1436 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1437 if (global_trace.stop_count++)
1440 /* Prevent the buffers from switching */
1441 arch_spin_lock(&global_trace.max_lock);
1443 buffer = global_trace.trace_buffer.buffer;
1445 ring_buffer_record_disable(buffer);
1447 #ifdef CONFIG_TRACER_MAX_TRACE
1448 buffer = global_trace.max_buffer.buffer;
1450 ring_buffer_record_disable(buffer);
1453 arch_spin_unlock(&global_trace.max_lock);
1456 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1459 static void tracing_stop_tr(struct trace_array *tr)
1461 struct ring_buffer *buffer;
1462 unsigned long flags;
1464 /* If global, we need to also stop the max tracer */
1465 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1466 return tracing_stop();
1468 raw_spin_lock_irqsave(&tr->start_lock, flags);
1469 if (tr->stop_count++)
1472 buffer = tr->trace_buffer.buffer;
1474 ring_buffer_record_disable(buffer);
1477 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1480 void trace_stop_cmdline_recording(void);
1482 static int trace_save_cmdline(struct task_struct *tsk)
1486 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1490 * It's not the end of the world if we don't get
1491 * the lock, but we also don't want to spin
1492 * nor do we want to disable interrupts,
1493 * so if we miss here, then better luck next time.
1495 if (!arch_spin_trylock(&trace_cmdline_lock))
1498 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1499 if (idx == NO_CMDLINE_MAP) {
1500 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1503 * Check whether the cmdline buffer at idx has a pid
1504 * mapped. We are going to overwrite that entry so we
1505 * need to clear the map_pid_to_cmdline. Otherwise we
1506 * would read the new comm for the old pid.
1508 pid = savedcmd->map_cmdline_to_pid[idx];
1509 if (pid != NO_CMDLINE_MAP)
1510 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1512 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1513 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1515 savedcmd->cmdline_idx = idx;
1518 set_cmdline(idx, tsk->comm);
1520 arch_spin_unlock(&trace_cmdline_lock);
1525 static void __trace_find_cmdline(int pid, char comm[])
1530 strcpy(comm, "<idle>");
1534 if (WARN_ON_ONCE(pid < 0)) {
1535 strcpy(comm, "<XXX>");
1539 if (pid > PID_MAX_DEFAULT) {
1540 strcpy(comm, "<...>");
1544 map = savedcmd->map_pid_to_cmdline[pid];
1545 if (map != NO_CMDLINE_MAP)
1546 strcpy(comm, get_saved_cmdlines(map));
1548 strcpy(comm, "<...>");
1551 void trace_find_cmdline(int pid, char comm[])
1554 arch_spin_lock(&trace_cmdline_lock);
1556 __trace_find_cmdline(pid, comm);
1558 arch_spin_unlock(&trace_cmdline_lock);
1562 void tracing_record_cmdline(struct task_struct *tsk)
1564 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1567 if (!__this_cpu_read(trace_cmdline_save))
1570 if (trace_save_cmdline(tsk))
1571 __this_cpu_write(trace_cmdline_save, false);
1575 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1578 struct task_struct *tsk = current;
1580 entry->preempt_count = pc & 0xff;
1581 entry->pid = (tsk) ? tsk->pid : 0;
1583 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1584 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1586 TRACE_FLAG_IRQS_NOSUPPORT |
1588 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1589 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1590 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1591 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1593 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1595 struct ring_buffer_event *
1596 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1599 unsigned long flags, int pc)
1601 struct ring_buffer_event *event;
1603 event = ring_buffer_lock_reserve(buffer, len);
1604 if (event != NULL) {
1605 struct trace_entry *ent = ring_buffer_event_data(event);
1607 tracing_generic_entry_update(ent, flags, pc);
1615 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1617 __this_cpu_write(trace_cmdline_save, true);
1618 ring_buffer_unlock_commit(buffer, event);
1622 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1623 struct ring_buffer_event *event,
1624 unsigned long flags, int pc)
1626 __buffer_unlock_commit(buffer, event);
1628 ftrace_trace_stack(buffer, flags, 6, pc);
1629 ftrace_trace_userstack(buffer, flags, pc);
1632 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1633 struct ring_buffer_event *event,
1634 unsigned long flags, int pc)
1636 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1638 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1640 static struct ring_buffer *temp_buffer;
1642 struct ring_buffer_event *
1643 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1644 struct ftrace_event_file *ftrace_file,
1645 int type, unsigned long len,
1646 unsigned long flags, int pc)
1648 struct ring_buffer_event *entry;
1650 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1651 entry = trace_buffer_lock_reserve(*current_rb,
1652 type, len, flags, pc);
1654 * If tracing is off, but we have triggers enabled
1655 * we still need to look at the event data. Use the temp_buffer
1656 * to store the trace event for the tigger to use. It's recusive
1657 * safe and will not be recorded anywhere.
1659 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1660 *current_rb = temp_buffer;
1661 entry = trace_buffer_lock_reserve(*current_rb,
1662 type, len, flags, pc);
1666 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1668 struct ring_buffer_event *
1669 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1670 int type, unsigned long len,
1671 unsigned long flags, int pc)
1673 *current_rb = global_trace.trace_buffer.buffer;
1674 return trace_buffer_lock_reserve(*current_rb,
1675 type, len, flags, pc);
1677 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1679 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1680 struct ring_buffer_event *event,
1681 unsigned long flags, int pc)
1683 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1685 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1687 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1688 struct ring_buffer_event *event,
1689 unsigned long flags, int pc,
1690 struct pt_regs *regs)
1692 __buffer_unlock_commit(buffer, event);
1694 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1695 ftrace_trace_userstack(buffer, flags, pc);
1697 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1699 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1700 struct ring_buffer_event *event)
1702 ring_buffer_discard_commit(buffer, event);
1704 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1707 trace_function(struct trace_array *tr,
1708 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1711 struct ftrace_event_call *call = &event_function;
1712 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1713 struct ring_buffer_event *event;
1714 struct ftrace_entry *entry;
1716 /* If we are reading the ring buffer, don't trace */
1717 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1720 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1724 entry = ring_buffer_event_data(event);
1726 entry->parent_ip = parent_ip;
1728 if (!call_filter_check_discard(call, entry, buffer, event))
1729 __buffer_unlock_commit(buffer, event);
1732 #ifdef CONFIG_STACKTRACE
1734 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1735 struct ftrace_stack {
1736 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1739 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1740 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1742 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1743 unsigned long flags,
1744 int skip, int pc, struct pt_regs *regs)
1746 struct ftrace_event_call *call = &event_kernel_stack;
1747 struct ring_buffer_event *event;
1748 struct stack_entry *entry;
1749 struct stack_trace trace;
1751 int size = FTRACE_STACK_ENTRIES;
1753 trace.nr_entries = 0;
1757 * Since events can happen in NMIs there's no safe way to
1758 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1759 * or NMI comes in, it will just have to use the default
1760 * FTRACE_STACK_SIZE.
1762 preempt_disable_notrace();
1764 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1766 * We don't need any atomic variables, just a barrier.
1767 * If an interrupt comes in, we don't care, because it would
1768 * have exited and put the counter back to what we want.
1769 * We just need a barrier to keep gcc from moving things
1773 if (use_stack == 1) {
1774 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1775 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1778 save_stack_trace_regs(regs, &trace);
1780 save_stack_trace(&trace);
1782 if (trace.nr_entries > size)
1783 size = trace.nr_entries;
1785 /* From now on, use_stack is a boolean */
1788 size *= sizeof(unsigned long);
1790 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1791 sizeof(*entry) + size, flags, pc);
1794 entry = ring_buffer_event_data(event);
1796 memset(&entry->caller, 0, size);
1799 memcpy(&entry->caller, trace.entries,
1800 trace.nr_entries * sizeof(unsigned long));
1802 trace.max_entries = FTRACE_STACK_ENTRIES;
1803 trace.entries = entry->caller;
1805 save_stack_trace_regs(regs, &trace);
1807 save_stack_trace(&trace);
1810 entry->size = trace.nr_entries;
1812 if (!call_filter_check_discard(call, entry, buffer, event))
1813 __buffer_unlock_commit(buffer, event);
1816 /* Again, don't let gcc optimize things here */
1818 __this_cpu_dec(ftrace_stack_reserve);
1819 preempt_enable_notrace();
1823 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1824 int skip, int pc, struct pt_regs *regs)
1826 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1829 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1832 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1835 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1838 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1841 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1844 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1848 * trace_dump_stack - record a stack back trace in the trace buffer
1849 * @skip: Number of functions to skip (helper handlers)
1851 void trace_dump_stack(int skip)
1853 unsigned long flags;
1855 if (tracing_disabled || tracing_selftest_running)
1858 local_save_flags(flags);
1861 * Skip 3 more, seems to get us at the caller of
1865 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1866 flags, skip, preempt_count(), NULL);
1869 static DEFINE_PER_CPU(int, user_stack_count);
1872 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1874 struct ftrace_event_call *call = &event_user_stack;
1875 struct ring_buffer_event *event;
1876 struct userstack_entry *entry;
1877 struct stack_trace trace;
1879 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1883 * NMIs can not handle page faults, even with fix ups.
1884 * The save user stack can (and often does) fault.
1886 if (unlikely(in_nmi()))
1890 * prevent recursion, since the user stack tracing may
1891 * trigger other kernel events.
1894 if (__this_cpu_read(user_stack_count))
1897 __this_cpu_inc(user_stack_count);
1899 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1900 sizeof(*entry), flags, pc);
1902 goto out_drop_count;
1903 entry = ring_buffer_event_data(event);
1905 entry->tgid = current->tgid;
1906 memset(&entry->caller, 0, sizeof(entry->caller));
1908 trace.nr_entries = 0;
1909 trace.max_entries = FTRACE_STACK_ENTRIES;
1911 trace.entries = entry->caller;
1913 save_stack_trace_user(&trace);
1914 if (!call_filter_check_discard(call, entry, buffer, event))
1915 __buffer_unlock_commit(buffer, event);
1918 __this_cpu_dec(user_stack_count);
1924 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1926 ftrace_trace_userstack(tr, flags, preempt_count());
1930 #endif /* CONFIG_STACKTRACE */
1932 /* created for use with alloc_percpu */
1933 struct trace_buffer_struct {
1934 char buffer[TRACE_BUF_SIZE];
1937 static struct trace_buffer_struct *trace_percpu_buffer;
1938 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1939 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1940 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1943 * The buffer used is dependent on the context. There is a per cpu
1944 * buffer for normal context, softirq contex, hard irq context and
1945 * for NMI context. Thise allows for lockless recording.
1947 * Note, if the buffers failed to be allocated, then this returns NULL
1949 static char *get_trace_buf(void)
1951 struct trace_buffer_struct *percpu_buffer;
1954 * If we have allocated per cpu buffers, then we do not
1955 * need to do any locking.
1958 percpu_buffer = trace_percpu_nmi_buffer;
1960 percpu_buffer = trace_percpu_irq_buffer;
1961 else if (in_softirq())
1962 percpu_buffer = trace_percpu_sirq_buffer;
1964 percpu_buffer = trace_percpu_buffer;
1969 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1972 static int alloc_percpu_trace_buffer(void)
1974 struct trace_buffer_struct *buffers;
1975 struct trace_buffer_struct *sirq_buffers;
1976 struct trace_buffer_struct *irq_buffers;
1977 struct trace_buffer_struct *nmi_buffers;
1979 buffers = alloc_percpu(struct trace_buffer_struct);
1983 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1987 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1991 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1995 trace_percpu_buffer = buffers;
1996 trace_percpu_sirq_buffer = sirq_buffers;
1997 trace_percpu_irq_buffer = irq_buffers;
1998 trace_percpu_nmi_buffer = nmi_buffers;
2003 free_percpu(irq_buffers);
2005 free_percpu(sirq_buffers);
2007 free_percpu(buffers);
2009 WARN(1, "Could not allocate percpu trace_printk buffer");
2013 static int buffers_allocated;
2015 void trace_printk_init_buffers(void)
2017 if (buffers_allocated)
2020 if (alloc_percpu_trace_buffer())
2023 /* trace_printk() is for debug use only. Don't use it in production. */
2025 pr_warning("\n**********************************************************\n");
2026 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2027 pr_warning("** **\n");
2028 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2029 pr_warning("** **\n");
2030 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2031 pr_warning("** unsafe for produciton use. **\n");
2032 pr_warning("** **\n");
2033 pr_warning("** If you see this message and you are not debugging **\n");
2034 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2035 pr_warning("** **\n");
2036 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2037 pr_warning("**********************************************************\n");
2039 /* Expand the buffers to set size */
2040 tracing_update_buffers();
2042 buffers_allocated = 1;
2045 * trace_printk_init_buffers() can be called by modules.
2046 * If that happens, then we need to start cmdline recording
2047 * directly here. If the global_trace.buffer is already
2048 * allocated here, then this was called by module code.
2050 if (global_trace.trace_buffer.buffer)
2051 tracing_start_cmdline_record();
2054 void trace_printk_start_comm(void)
2056 /* Start tracing comms if trace printk is set */
2057 if (!buffers_allocated)
2059 tracing_start_cmdline_record();
2062 static void trace_printk_start_stop_comm(int enabled)
2064 if (!buffers_allocated)
2068 tracing_start_cmdline_record();
2070 tracing_stop_cmdline_record();
2074 * trace_vbprintk - write binary msg to tracing buffer
2077 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2079 struct ftrace_event_call *call = &event_bprint;
2080 struct ring_buffer_event *event;
2081 struct ring_buffer *buffer;
2082 struct trace_array *tr = &global_trace;
2083 struct bprint_entry *entry;
2084 unsigned long flags;
2086 int len = 0, size, pc;
2088 if (unlikely(tracing_selftest_running || tracing_disabled))
2091 /* Don't pollute graph traces with trace_vprintk internals */
2092 pause_graph_tracing();
2094 pc = preempt_count();
2095 preempt_disable_notrace();
2097 tbuffer = get_trace_buf();
2103 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2105 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2108 local_save_flags(flags);
2109 size = sizeof(*entry) + sizeof(u32) * len;
2110 buffer = tr->trace_buffer.buffer;
2111 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2115 entry = ring_buffer_event_data(event);
2119 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2120 if (!call_filter_check_discard(call, entry, buffer, event)) {
2121 __buffer_unlock_commit(buffer, event);
2122 ftrace_trace_stack(buffer, flags, 6, pc);
2126 preempt_enable_notrace();
2127 unpause_graph_tracing();
2131 EXPORT_SYMBOL_GPL(trace_vbprintk);
2134 __trace_array_vprintk(struct ring_buffer *buffer,
2135 unsigned long ip, const char *fmt, va_list args)
2137 struct ftrace_event_call *call = &event_print;
2138 struct ring_buffer_event *event;
2139 int len = 0, size, pc;
2140 struct print_entry *entry;
2141 unsigned long flags;
2144 if (tracing_disabled || tracing_selftest_running)
2147 /* Don't pollute graph traces with trace_vprintk internals */
2148 pause_graph_tracing();
2150 pc = preempt_count();
2151 preempt_disable_notrace();
2154 tbuffer = get_trace_buf();
2160 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2161 if (len > TRACE_BUF_SIZE)
2164 local_save_flags(flags);
2165 size = sizeof(*entry) + len + 1;
2166 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2170 entry = ring_buffer_event_data(event);
2173 memcpy(&entry->buf, tbuffer, len);
2174 entry->buf[len] = '\0';
2175 if (!call_filter_check_discard(call, entry, buffer, event)) {
2176 __buffer_unlock_commit(buffer, event);
2177 ftrace_trace_stack(buffer, flags, 6, pc);
2180 preempt_enable_notrace();
2181 unpause_graph_tracing();
2186 int trace_array_vprintk(struct trace_array *tr,
2187 unsigned long ip, const char *fmt, va_list args)
2189 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2192 int trace_array_printk(struct trace_array *tr,
2193 unsigned long ip, const char *fmt, ...)
2198 if (!(trace_flags & TRACE_ITER_PRINTK))
2202 ret = trace_array_vprintk(tr, ip, fmt, ap);
2207 int trace_array_printk_buf(struct ring_buffer *buffer,
2208 unsigned long ip, const char *fmt, ...)
2213 if (!(trace_flags & TRACE_ITER_PRINTK))
2217 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2222 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2224 return trace_array_vprintk(&global_trace, ip, fmt, args);
2226 EXPORT_SYMBOL_GPL(trace_vprintk);
2228 static void trace_iterator_increment(struct trace_iterator *iter)
2230 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2234 ring_buffer_read(buf_iter, NULL);
2237 static struct trace_entry *
2238 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2239 unsigned long *lost_events)
2241 struct ring_buffer_event *event;
2242 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2245 event = ring_buffer_iter_peek(buf_iter, ts);
2247 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2251 iter->ent_size = ring_buffer_event_length(event);
2252 return ring_buffer_event_data(event);
2258 static struct trace_entry *
2259 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2260 unsigned long *missing_events, u64 *ent_ts)
2262 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2263 struct trace_entry *ent, *next = NULL;
2264 unsigned long lost_events = 0, next_lost = 0;
2265 int cpu_file = iter->cpu_file;
2266 u64 next_ts = 0, ts;
2272 * If we are in a per_cpu trace file, don't bother by iterating over
2273 * all cpu and peek directly.
2275 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2276 if (ring_buffer_empty_cpu(buffer, cpu_file))
2278 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2280 *ent_cpu = cpu_file;
2285 for_each_tracing_cpu(cpu) {
2287 if (ring_buffer_empty_cpu(buffer, cpu))
2290 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2293 * Pick the entry with the smallest timestamp:
2295 if (ent && (!next || ts < next_ts)) {
2299 next_lost = lost_events;
2300 next_size = iter->ent_size;
2304 iter->ent_size = next_size;
2307 *ent_cpu = next_cpu;
2313 *missing_events = next_lost;
2318 /* Find the next real entry, without updating the iterator itself */
2319 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2320 int *ent_cpu, u64 *ent_ts)
2322 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2325 /* Find the next real entry, and increment the iterator to the next entry */
2326 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2328 iter->ent = __find_next_entry(iter, &iter->cpu,
2329 &iter->lost_events, &iter->ts);
2332 trace_iterator_increment(iter);
2334 return iter->ent ? iter : NULL;
2337 static void trace_consume(struct trace_iterator *iter)
2339 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2340 &iter->lost_events);
2343 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2345 struct trace_iterator *iter = m->private;
2349 WARN_ON_ONCE(iter->leftover);
2353 /* can't go backwards */
2358 ent = trace_find_next_entry_inc(iter);
2362 while (ent && iter->idx < i)
2363 ent = trace_find_next_entry_inc(iter);
2370 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2372 struct ring_buffer_event *event;
2373 struct ring_buffer_iter *buf_iter;
2374 unsigned long entries = 0;
2377 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2379 buf_iter = trace_buffer_iter(iter, cpu);
2383 ring_buffer_iter_reset(buf_iter);
2386 * We could have the case with the max latency tracers
2387 * that a reset never took place on a cpu. This is evident
2388 * by the timestamp being before the start of the buffer.
2390 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2391 if (ts >= iter->trace_buffer->time_start)
2394 ring_buffer_read(buf_iter, NULL);
2397 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2401 * The current tracer is copied to avoid a global locking
2404 static void *s_start(struct seq_file *m, loff_t *pos)
2406 struct trace_iterator *iter = m->private;
2407 struct trace_array *tr = iter->tr;
2408 int cpu_file = iter->cpu_file;
2414 * copy the tracer to avoid using a global lock all around.
2415 * iter->trace is a copy of current_trace, the pointer to the
2416 * name may be used instead of a strcmp(), as iter->trace->name
2417 * will point to the same string as current_trace->name.
2419 mutex_lock(&trace_types_lock);
2420 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2421 *iter->trace = *tr->current_trace;
2422 mutex_unlock(&trace_types_lock);
2424 #ifdef CONFIG_TRACER_MAX_TRACE
2425 if (iter->snapshot && iter->trace->use_max_tr)
2426 return ERR_PTR(-EBUSY);
2429 if (!iter->snapshot)
2430 atomic_inc(&trace_record_cmdline_disabled);
2432 if (*pos != iter->pos) {
2437 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2438 for_each_tracing_cpu(cpu)
2439 tracing_iter_reset(iter, cpu);
2441 tracing_iter_reset(iter, cpu_file);
2444 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2449 * If we overflowed the seq_file before, then we want
2450 * to just reuse the trace_seq buffer again.
2456 p = s_next(m, p, &l);
2460 trace_event_read_lock();
2461 trace_access_lock(cpu_file);
2465 static void s_stop(struct seq_file *m, void *p)
2467 struct trace_iterator *iter = m->private;
2469 #ifdef CONFIG_TRACER_MAX_TRACE
2470 if (iter->snapshot && iter->trace->use_max_tr)
2474 if (!iter->snapshot)
2475 atomic_dec(&trace_record_cmdline_disabled);
2477 trace_access_unlock(iter->cpu_file);
2478 trace_event_read_unlock();
2482 get_total_entries(struct trace_buffer *buf,
2483 unsigned long *total, unsigned long *entries)
2485 unsigned long count;
2491 for_each_tracing_cpu(cpu) {
2492 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2494 * If this buffer has skipped entries, then we hold all
2495 * entries for the trace and we need to ignore the
2496 * ones before the time stamp.
2498 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2499 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2500 /* total is the same as the entries */
2504 ring_buffer_overrun_cpu(buf->buffer, cpu);
2509 static void print_lat_help_header(struct seq_file *m)
2511 seq_puts(m, "# _------=> CPU# \n");
2512 seq_puts(m, "# / _-----=> irqs-off \n");
2513 seq_puts(m, "# | / _----=> need-resched \n");
2514 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2515 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2516 seq_puts(m, "# |||| / delay \n");
2517 seq_puts(m, "# cmd pid ||||| time | caller \n");
2518 seq_puts(m, "# \\ / ||||| \\ | / \n");
2521 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2523 unsigned long total;
2524 unsigned long entries;
2526 get_total_entries(buf, &total, &entries);
2527 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2528 entries, total, num_online_cpus());
2532 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2534 print_event_info(buf, m);
2535 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2536 seq_puts(m, "# | | | | |\n");
2539 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2541 print_event_info(buf, m);
2542 seq_puts(m, "# _-----=> irqs-off\n");
2543 seq_puts(m, "# / _----=> need-resched\n");
2544 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2545 seq_puts(m, "# || / _--=> preempt-depth\n");
2546 seq_puts(m, "# ||| / delay\n");
2547 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2548 seq_puts(m, "# | | | |||| | |\n");
2552 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2554 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2555 struct trace_buffer *buf = iter->trace_buffer;
2556 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2557 struct tracer *type = iter->trace;
2558 unsigned long entries;
2559 unsigned long total;
2560 const char *name = "preemption";
2564 get_total_entries(buf, &total, &entries);
2566 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2568 seq_puts(m, "# -----------------------------------"
2569 "---------------------------------\n");
2570 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2571 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2572 nsecs_to_usecs(data->saved_latency),
2576 #if defined(CONFIG_PREEMPT_NONE)
2578 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2580 #elif defined(CONFIG_PREEMPT)
2585 /* These are reserved for later use */
2588 seq_printf(m, " #P:%d)\n", num_online_cpus());
2592 seq_puts(m, "# -----------------\n");
2593 seq_printf(m, "# | task: %.16s-%d "
2594 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2595 data->comm, data->pid,
2596 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2597 data->policy, data->rt_priority);
2598 seq_puts(m, "# -----------------\n");
2600 if (data->critical_start) {
2601 seq_puts(m, "# => started at: ");
2602 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2603 trace_print_seq(m, &iter->seq);
2604 seq_puts(m, "\n# => ended at: ");
2605 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2606 trace_print_seq(m, &iter->seq);
2607 seq_puts(m, "\n#\n");
2613 static void test_cpu_buff_start(struct trace_iterator *iter)
2615 struct trace_seq *s = &iter->seq;
2617 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2620 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2623 if (cpumask_test_cpu(iter->cpu, iter->started))
2626 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2629 cpumask_set_cpu(iter->cpu, iter->started);
2631 /* Don't print started cpu buffer for the first entry of the trace */
2633 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2637 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2639 struct trace_seq *s = &iter->seq;
2640 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2641 struct trace_entry *entry;
2642 struct trace_event *event;
2646 test_cpu_buff_start(iter);
2648 event = ftrace_find_event(entry->type);
2650 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2651 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2652 if (!trace_print_lat_context(iter))
2655 if (!trace_print_context(iter))
2661 return event->funcs->trace(iter, sym_flags, event);
2663 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2666 return TRACE_TYPE_HANDLED;
2668 return TRACE_TYPE_PARTIAL_LINE;
2671 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2673 struct trace_seq *s = &iter->seq;
2674 struct trace_entry *entry;
2675 struct trace_event *event;
2679 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2680 if (!trace_seq_printf(s, "%d %d %llu ",
2681 entry->pid, iter->cpu, iter->ts))
2685 event = ftrace_find_event(entry->type);
2687 return event->funcs->raw(iter, 0, event);
2689 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2692 return TRACE_TYPE_HANDLED;
2694 return TRACE_TYPE_PARTIAL_LINE;
2697 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2699 struct trace_seq *s = &iter->seq;
2700 unsigned char newline = '\n';
2701 struct trace_entry *entry;
2702 struct trace_event *event;
2706 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2707 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2708 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2709 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2712 event = ftrace_find_event(entry->type);
2714 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2715 if (ret != TRACE_TYPE_HANDLED)
2719 SEQ_PUT_FIELD_RET(s, newline);
2721 return TRACE_TYPE_HANDLED;
2724 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2726 struct trace_seq *s = &iter->seq;
2727 struct trace_entry *entry;
2728 struct trace_event *event;
2732 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2733 SEQ_PUT_FIELD_RET(s, entry->pid);
2734 SEQ_PUT_FIELD_RET(s, iter->cpu);
2735 SEQ_PUT_FIELD_RET(s, iter->ts);
2738 event = ftrace_find_event(entry->type);
2739 return event ? event->funcs->binary(iter, 0, event) :
2743 int trace_empty(struct trace_iterator *iter)
2745 struct ring_buffer_iter *buf_iter;
2748 /* If we are looking at one CPU buffer, only check that one */
2749 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2750 cpu = iter->cpu_file;
2751 buf_iter = trace_buffer_iter(iter, cpu);
2753 if (!ring_buffer_iter_empty(buf_iter))
2756 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2762 for_each_tracing_cpu(cpu) {
2763 buf_iter = trace_buffer_iter(iter, cpu);
2765 if (!ring_buffer_iter_empty(buf_iter))
2768 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2776 /* Called with trace_event_read_lock() held. */
2777 enum print_line_t print_trace_line(struct trace_iterator *iter)
2779 enum print_line_t ret;
2781 if (iter->lost_events &&
2782 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2783 iter->cpu, iter->lost_events))
2784 return TRACE_TYPE_PARTIAL_LINE;
2786 if (iter->trace && iter->trace->print_line) {
2787 ret = iter->trace->print_line(iter);
2788 if (ret != TRACE_TYPE_UNHANDLED)
2792 if (iter->ent->type == TRACE_BPUTS &&
2793 trace_flags & TRACE_ITER_PRINTK &&
2794 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2795 return trace_print_bputs_msg_only(iter);
2797 if (iter->ent->type == TRACE_BPRINT &&
2798 trace_flags & TRACE_ITER_PRINTK &&
2799 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2800 return trace_print_bprintk_msg_only(iter);
2802 if (iter->ent->type == TRACE_PRINT &&
2803 trace_flags & TRACE_ITER_PRINTK &&
2804 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2805 return trace_print_printk_msg_only(iter);
2807 if (trace_flags & TRACE_ITER_BIN)
2808 return print_bin_fmt(iter);
2810 if (trace_flags & TRACE_ITER_HEX)
2811 return print_hex_fmt(iter);
2813 if (trace_flags & TRACE_ITER_RAW)
2814 return print_raw_fmt(iter);
2816 return print_trace_fmt(iter);
2819 void trace_latency_header(struct seq_file *m)
2821 struct trace_iterator *iter = m->private;
2823 /* print nothing if the buffers are empty */
2824 if (trace_empty(iter))
2827 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2828 print_trace_header(m, iter);
2830 if (!(trace_flags & TRACE_ITER_VERBOSE))
2831 print_lat_help_header(m);
2834 void trace_default_header(struct seq_file *m)
2836 struct trace_iterator *iter = m->private;
2838 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2841 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2842 /* print nothing if the buffers are empty */
2843 if (trace_empty(iter))
2845 print_trace_header(m, iter);
2846 if (!(trace_flags & TRACE_ITER_VERBOSE))
2847 print_lat_help_header(m);
2849 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2850 if (trace_flags & TRACE_ITER_IRQ_INFO)
2851 print_func_help_header_irq(iter->trace_buffer, m);
2853 print_func_help_header(iter->trace_buffer, m);
2858 static void test_ftrace_alive(struct seq_file *m)
2860 if (!ftrace_is_dead())
2862 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2863 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2866 #ifdef CONFIG_TRACER_MAX_TRACE
2867 static void show_snapshot_main_help(struct seq_file *m)
2869 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2870 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2871 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2872 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2873 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2874 seq_printf(m, "# is not a '0' or '1')\n");
2877 static void show_snapshot_percpu_help(struct seq_file *m)
2879 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2880 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2881 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2882 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2884 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2885 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2887 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2888 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2889 seq_printf(m, "# is not a '0' or '1')\n");
2892 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2894 if (iter->tr->allocated_snapshot)
2895 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2897 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2899 seq_printf(m, "# Snapshot commands:\n");
2900 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2901 show_snapshot_main_help(m);
2903 show_snapshot_percpu_help(m);
2906 /* Should never be called */
2907 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2910 static int s_show(struct seq_file *m, void *v)
2912 struct trace_iterator *iter = v;
2915 if (iter->ent == NULL) {
2917 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2919 test_ftrace_alive(m);
2921 if (iter->snapshot && trace_empty(iter))
2922 print_snapshot_help(m, iter);
2923 else if (iter->trace && iter->trace->print_header)
2924 iter->trace->print_header(m);
2926 trace_default_header(m);
2928 } else if (iter->leftover) {
2930 * If we filled the seq_file buffer earlier, we
2931 * want to just show it now.
2933 ret = trace_print_seq(m, &iter->seq);
2935 /* ret should this time be zero, but you never know */
2936 iter->leftover = ret;
2939 print_trace_line(iter);
2940 ret = trace_print_seq(m, &iter->seq);
2942 * If we overflow the seq_file buffer, then it will
2943 * ask us for this data again at start up.
2945 * ret is 0 if seq_file write succeeded.
2948 iter->leftover = ret;
2955 * Should be used after trace_array_get(), trace_types_lock
2956 * ensures that i_cdev was already initialized.
2958 static inline int tracing_get_cpu(struct inode *inode)
2960 if (inode->i_cdev) /* See trace_create_cpu_file() */
2961 return (long)inode->i_cdev - 1;
2962 return RING_BUFFER_ALL_CPUS;
2965 static const struct seq_operations tracer_seq_ops = {
2972 static struct trace_iterator *
2973 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2975 struct trace_array *tr = inode->i_private;
2976 struct trace_iterator *iter;
2979 if (tracing_disabled)
2980 return ERR_PTR(-ENODEV);
2982 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2984 return ERR_PTR(-ENOMEM);
2986 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2988 if (!iter->buffer_iter)
2992 * We make a copy of the current tracer to avoid concurrent
2993 * changes on it while we are reading.
2995 mutex_lock(&trace_types_lock);
2996 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3000 *iter->trace = *tr->current_trace;
3002 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3007 #ifdef CONFIG_TRACER_MAX_TRACE
3008 /* Currently only the top directory has a snapshot */
3009 if (tr->current_trace->print_max || snapshot)
3010 iter->trace_buffer = &tr->max_buffer;
3013 iter->trace_buffer = &tr->trace_buffer;
3014 iter->snapshot = snapshot;
3016 iter->cpu_file = tracing_get_cpu(inode);
3017 mutex_init(&iter->mutex);
3019 /* Notify the tracer early; before we stop tracing. */
3020 if (iter->trace && iter->trace->open)
3021 iter->trace->open(iter);
3023 /* Annotate start of buffers if we had overruns */
3024 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3025 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3027 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3028 if (trace_clocks[tr->clock_id].in_ns)
3029 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3031 /* stop the trace while dumping if we are not opening "snapshot" */
3032 if (!iter->snapshot)
3033 tracing_stop_tr(tr);
3035 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3036 for_each_tracing_cpu(cpu) {
3037 iter->buffer_iter[cpu] =
3038 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3040 ring_buffer_read_prepare_sync();
3041 for_each_tracing_cpu(cpu) {
3042 ring_buffer_read_start(iter->buffer_iter[cpu]);
3043 tracing_iter_reset(iter, cpu);
3046 cpu = iter->cpu_file;
3047 iter->buffer_iter[cpu] =
3048 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3049 ring_buffer_read_prepare_sync();
3050 ring_buffer_read_start(iter->buffer_iter[cpu]);
3051 tracing_iter_reset(iter, cpu);
3054 mutex_unlock(&trace_types_lock);
3059 mutex_unlock(&trace_types_lock);
3061 kfree(iter->buffer_iter);
3063 seq_release_private(inode, file);
3064 return ERR_PTR(-ENOMEM);
3067 int tracing_open_generic(struct inode *inode, struct file *filp)
3069 if (tracing_disabled)
3072 filp->private_data = inode->i_private;
3076 bool tracing_is_disabled(void)
3078 return (tracing_disabled) ? true: false;
3082 * Open and update trace_array ref count.
3083 * Must have the current trace_array passed to it.
3085 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3087 struct trace_array *tr = inode->i_private;
3089 if (tracing_disabled)
3092 if (trace_array_get(tr) < 0)
3095 filp->private_data = inode->i_private;
3100 static int tracing_release(struct inode *inode, struct file *file)
3102 struct trace_array *tr = inode->i_private;
3103 struct seq_file *m = file->private_data;
3104 struct trace_iterator *iter;
3107 if (!(file->f_mode & FMODE_READ)) {
3108 trace_array_put(tr);
3112 /* Writes do not use seq_file */
3114 mutex_lock(&trace_types_lock);
3116 for_each_tracing_cpu(cpu) {
3117 if (iter->buffer_iter[cpu])
3118 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3121 if (iter->trace && iter->trace->close)
3122 iter->trace->close(iter);
3124 if (!iter->snapshot)
3125 /* reenable tracing if it was previously enabled */
3126 tracing_start_tr(tr);
3128 __trace_array_put(tr);
3130 mutex_unlock(&trace_types_lock);
3132 mutex_destroy(&iter->mutex);
3133 free_cpumask_var(iter->started);
3135 kfree(iter->buffer_iter);
3136 seq_release_private(inode, file);
3141 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3143 struct trace_array *tr = inode->i_private;
3145 trace_array_put(tr);
3149 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3151 struct trace_array *tr = inode->i_private;
3153 trace_array_put(tr);
3155 return single_release(inode, file);
3158 static int tracing_open(struct inode *inode, struct file *file)
3160 struct trace_array *tr = inode->i_private;
3161 struct trace_iterator *iter;
3164 if (trace_array_get(tr) < 0)
3167 /* If this file was open for write, then erase contents */
3168 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3169 int cpu = tracing_get_cpu(inode);
3171 if (cpu == RING_BUFFER_ALL_CPUS)
3172 tracing_reset_online_cpus(&tr->trace_buffer);
3174 tracing_reset(&tr->trace_buffer, cpu);
3177 if (file->f_mode & FMODE_READ) {
3178 iter = __tracing_open(inode, file, false);
3180 ret = PTR_ERR(iter);
3181 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3182 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3186 trace_array_put(tr);
3192 * Some tracers are not suitable for instance buffers.
3193 * A tracer is always available for the global array (toplevel)
3194 * or if it explicitly states that it is.
3197 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3199 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3202 /* Find the next tracer that this trace array may use */
3203 static struct tracer *
3204 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3206 while (t && !trace_ok_for_array(t, tr))
3213 t_next(struct seq_file *m, void *v, loff_t *pos)
3215 struct trace_array *tr = m->private;
3216 struct tracer *t = v;
3221 t = get_tracer_for_array(tr, t->next);
3226 static void *t_start(struct seq_file *m, loff_t *pos)
3228 struct trace_array *tr = m->private;
3232 mutex_lock(&trace_types_lock);
3234 t = get_tracer_for_array(tr, trace_types);
3235 for (; t && l < *pos; t = t_next(m, t, &l))
3241 static void t_stop(struct seq_file *m, void *p)
3243 mutex_unlock(&trace_types_lock);
3246 static int t_show(struct seq_file *m, void *v)
3248 struct tracer *t = v;
3253 seq_printf(m, "%s", t->name);
3262 static const struct seq_operations show_traces_seq_ops = {
3269 static int show_traces_open(struct inode *inode, struct file *file)
3271 struct trace_array *tr = inode->i_private;
3275 if (tracing_disabled)
3278 ret = seq_open(file, &show_traces_seq_ops);
3282 m = file->private_data;
3289 tracing_write_stub(struct file *filp, const char __user *ubuf,
3290 size_t count, loff_t *ppos)
3295 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3299 if (file->f_mode & FMODE_READ)
3300 ret = seq_lseek(file, offset, whence);
3302 file->f_pos = ret = 0;
3307 static const struct file_operations tracing_fops = {
3308 .open = tracing_open,
3310 .write = tracing_write_stub,
3311 .llseek = tracing_lseek,
3312 .release = tracing_release,
3315 static const struct file_operations show_traces_fops = {
3316 .open = show_traces_open,
3318 .release = seq_release,
3319 .llseek = seq_lseek,
3323 * The tracer itself will not take this lock, but still we want
3324 * to provide a consistent cpumask to user-space:
3326 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3329 * Temporary storage for the character representation of the
3330 * CPU bitmask (and one more byte for the newline):
3332 static char mask_str[NR_CPUS + 1];
3335 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3336 size_t count, loff_t *ppos)
3338 struct trace_array *tr = file_inode(filp)->i_private;
3341 mutex_lock(&tracing_cpumask_update_lock);
3343 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3344 if (count - len < 2) {
3348 len += sprintf(mask_str + len, "\n");
3349 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3352 mutex_unlock(&tracing_cpumask_update_lock);
3358 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3359 size_t count, loff_t *ppos)
3361 struct trace_array *tr = file_inode(filp)->i_private;
3362 cpumask_var_t tracing_cpumask_new;
3365 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3368 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3372 mutex_lock(&tracing_cpumask_update_lock);
3374 local_irq_disable();
3375 arch_spin_lock(&tr->max_lock);
3376 for_each_tracing_cpu(cpu) {
3378 * Increase/decrease the disabled counter if we are
3379 * about to flip a bit in the cpumask:
3381 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3382 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3383 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3384 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3386 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3387 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3388 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3389 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3392 arch_spin_unlock(&tr->max_lock);
3395 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3397 mutex_unlock(&tracing_cpumask_update_lock);
3398 free_cpumask_var(tracing_cpumask_new);
3403 free_cpumask_var(tracing_cpumask_new);
3408 static const struct file_operations tracing_cpumask_fops = {
3409 .open = tracing_open_generic_tr,
3410 .read = tracing_cpumask_read,
3411 .write = tracing_cpumask_write,
3412 .release = tracing_release_generic_tr,
3413 .llseek = generic_file_llseek,
3416 static int tracing_trace_options_show(struct seq_file *m, void *v)
3418 struct tracer_opt *trace_opts;
3419 struct trace_array *tr = m->private;
3423 mutex_lock(&trace_types_lock);
3424 tracer_flags = tr->current_trace->flags->val;
3425 trace_opts = tr->current_trace->flags->opts;
3427 for (i = 0; trace_options[i]; i++) {
3428 if (trace_flags & (1 << i))
3429 seq_printf(m, "%s\n", trace_options[i]);
3431 seq_printf(m, "no%s\n", trace_options[i]);
3434 for (i = 0; trace_opts[i].name; i++) {
3435 if (tracer_flags & trace_opts[i].bit)
3436 seq_printf(m, "%s\n", trace_opts[i].name);
3438 seq_printf(m, "no%s\n", trace_opts[i].name);
3440 mutex_unlock(&trace_types_lock);
3445 static int __set_tracer_option(struct trace_array *tr,
3446 struct tracer_flags *tracer_flags,
3447 struct tracer_opt *opts, int neg)
3449 struct tracer *trace = tr->current_trace;
3452 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3457 tracer_flags->val &= ~opts->bit;
3459 tracer_flags->val |= opts->bit;
3463 /* Try to assign a tracer specific option */
3464 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3466 struct tracer *trace = tr->current_trace;
3467 struct tracer_flags *tracer_flags = trace->flags;
3468 struct tracer_opt *opts = NULL;
3471 for (i = 0; tracer_flags->opts[i].name; i++) {
3472 opts = &tracer_flags->opts[i];
3474 if (strcmp(cmp, opts->name) == 0)
3475 return __set_tracer_option(tr, trace->flags, opts, neg);
3481 /* Some tracers require overwrite to stay enabled */
3482 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3484 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3490 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3492 /* do nothing if flag is already set */
3493 if (!!(trace_flags & mask) == !!enabled)
3496 /* Give the tracer a chance to approve the change */
3497 if (tr->current_trace->flag_changed)
3498 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3502 trace_flags |= mask;
3504 trace_flags &= ~mask;
3506 if (mask == TRACE_ITER_RECORD_CMD)
3507 trace_event_enable_cmd_record(enabled);
3509 if (mask == TRACE_ITER_OVERWRITE) {
3510 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3511 #ifdef CONFIG_TRACER_MAX_TRACE
3512 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3516 if (mask == TRACE_ITER_PRINTK)
3517 trace_printk_start_stop_comm(enabled);
3522 static int trace_set_options(struct trace_array *tr, char *option)
3529 cmp = strstrip(option);
3531 if (strncmp(cmp, "no", 2) == 0) {
3536 mutex_lock(&trace_types_lock);
3538 for (i = 0; trace_options[i]; i++) {
3539 if (strcmp(cmp, trace_options[i]) == 0) {
3540 ret = set_tracer_flag(tr, 1 << i, !neg);
3545 /* If no option could be set, test the specific tracer options */
3546 if (!trace_options[i])
3547 ret = set_tracer_option(tr, cmp, neg);
3549 mutex_unlock(&trace_types_lock);
3555 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3556 size_t cnt, loff_t *ppos)
3558 struct seq_file *m = filp->private_data;
3559 struct trace_array *tr = m->private;
3563 if (cnt >= sizeof(buf))
3566 if (copy_from_user(&buf, ubuf, cnt))
3571 ret = trace_set_options(tr, buf);
3580 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3582 struct trace_array *tr = inode->i_private;
3585 if (tracing_disabled)
3588 if (trace_array_get(tr) < 0)
3591 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3593 trace_array_put(tr);
3598 static const struct file_operations tracing_iter_fops = {
3599 .open = tracing_trace_options_open,
3601 .llseek = seq_lseek,
3602 .release = tracing_single_release_tr,
3603 .write = tracing_trace_options_write,
3606 static const char readme_msg[] =
3607 "tracing mini-HOWTO:\n\n"
3608 "# echo 0 > tracing_on : quick way to disable tracing\n"
3609 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3610 " Important files:\n"
3611 " trace\t\t\t- The static contents of the buffer\n"
3612 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3613 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3614 " current_tracer\t- function and latency tracers\n"
3615 " available_tracers\t- list of configured tracers for current_tracer\n"
3616 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3617 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3618 " trace_clock\t\t-change the clock used to order events\n"
3619 " local: Per cpu clock but may not be synced across CPUs\n"
3620 " global: Synced across CPUs but slows tracing down.\n"
3621 " counter: Not a clock, but just an increment\n"
3622 " uptime: Jiffy counter from time of boot\n"
3623 " perf: Same clock that perf events use\n"
3624 #ifdef CONFIG_X86_64
3625 " x86-tsc: TSC cycle counter\n"
3627 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3628 " tracing_cpumask\t- Limit which CPUs to trace\n"
3629 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3630 "\t\t\t Remove sub-buffer with rmdir\n"
3631 " trace_options\t\t- Set format or modify how tracing happens\n"
3632 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3633 "\t\t\t option name\n"
3634 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3635 #ifdef CONFIG_DYNAMIC_FTRACE
3636 "\n available_filter_functions - list of functions that can be filtered on\n"
3637 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3638 "\t\t\t functions\n"
3639 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3640 "\t modules: Can select a group via module\n"
3641 "\t Format: :mod:<module-name>\n"
3642 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3643 "\t triggers: a command to perform when function is hit\n"
3644 "\t Format: <function>:<trigger>[:count]\n"
3645 "\t trigger: traceon, traceoff\n"
3646 "\t\t enable_event:<system>:<event>\n"
3647 "\t\t disable_event:<system>:<event>\n"
3648 #ifdef CONFIG_STACKTRACE
3651 #ifdef CONFIG_TRACER_SNAPSHOT
3656 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3657 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3658 "\t The first one will disable tracing every time do_fault is hit\n"
3659 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3660 "\t The first time do trap is hit and it disables tracing, the\n"
3661 "\t counter will decrement to 2. If tracing is already disabled,\n"
3662 "\t the counter will not decrement. It only decrements when the\n"
3663 "\t trigger did work\n"
3664 "\t To remove trigger without count:\n"
3665 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3666 "\t To remove trigger with a count:\n"
3667 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3668 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3669 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3670 "\t modules: Can select a group via module command :mod:\n"
3671 "\t Does not accept triggers\n"
3672 #endif /* CONFIG_DYNAMIC_FTRACE */
3673 #ifdef CONFIG_FUNCTION_TRACER
3674 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3677 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3678 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3679 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3680 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3682 #ifdef CONFIG_TRACER_SNAPSHOT
3683 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3684 "\t\t\t snapshot buffer. Read the contents for more\n"
3685 "\t\t\t information\n"
3687 #ifdef CONFIG_STACK_TRACER
3688 " stack_trace\t\t- Shows the max stack trace when active\n"
3689 " stack_max_size\t- Shows current max stack size that was traced\n"
3690 "\t\t\t Write into this file to reset the max size (trigger a\n"
3691 "\t\t\t new trace)\n"
3692 #ifdef CONFIG_DYNAMIC_FTRACE
3693 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3696 #endif /* CONFIG_STACK_TRACER */
3697 " events/\t\t- Directory containing all trace event subsystems:\n"
3698 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3699 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3700 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3702 " filter\t\t- If set, only events passing filter are traced\n"
3703 " events/<system>/<event>/\t- Directory containing control files for\n"
3705 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3706 " filter\t\t- If set, only events passing filter are traced\n"
3707 " trigger\t\t- If set, a command to perform when event is hit\n"
3708 "\t Format: <trigger>[:count][if <filter>]\n"
3709 "\t trigger: traceon, traceoff\n"
3710 "\t enable_event:<system>:<event>\n"
3711 "\t disable_event:<system>:<event>\n"
3712 #ifdef CONFIG_STACKTRACE
3715 #ifdef CONFIG_TRACER_SNAPSHOT
3718 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3719 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3720 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3721 "\t events/block/block_unplug/trigger\n"
3722 "\t The first disables tracing every time block_unplug is hit.\n"
3723 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3724 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3725 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3726 "\t Like function triggers, the counter is only decremented if it\n"
3727 "\t enabled or disabled tracing.\n"
3728 "\t To remove a trigger without a count:\n"
3729 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3730 "\t To remove a trigger with a count:\n"
3731 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3732 "\t Filters can be ignored when removing a trigger.\n"
3736 tracing_readme_read(struct file *filp, char __user *ubuf,
3737 size_t cnt, loff_t *ppos)
3739 return simple_read_from_buffer(ubuf, cnt, ppos,
3740 readme_msg, strlen(readme_msg));
3743 static const struct file_operations tracing_readme_fops = {
3744 .open = tracing_open_generic,
3745 .read = tracing_readme_read,
3746 .llseek = generic_file_llseek,
3749 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3751 unsigned int *ptr = v;
3753 if (*pos || m->count)
3758 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3760 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3769 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3775 arch_spin_lock(&trace_cmdline_lock);
3777 v = &savedcmd->map_cmdline_to_pid[0];
3779 v = saved_cmdlines_next(m, v, &l);
3787 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3789 arch_spin_unlock(&trace_cmdline_lock);
3793 static int saved_cmdlines_show(struct seq_file *m, void *v)
3795 char buf[TASK_COMM_LEN];
3796 unsigned int *pid = v;
3798 __trace_find_cmdline(*pid, buf);
3799 seq_printf(m, "%d %s\n", *pid, buf);
3803 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3804 .start = saved_cmdlines_start,
3805 .next = saved_cmdlines_next,
3806 .stop = saved_cmdlines_stop,
3807 .show = saved_cmdlines_show,
3810 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3812 if (tracing_disabled)
3815 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3818 static const struct file_operations tracing_saved_cmdlines_fops = {
3819 .open = tracing_saved_cmdlines_open,
3821 .llseek = seq_lseek,
3822 .release = seq_release,
3826 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3827 size_t cnt, loff_t *ppos)
3832 arch_spin_lock(&trace_cmdline_lock);
3833 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3834 arch_spin_unlock(&trace_cmdline_lock);
3836 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3839 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3841 kfree(s->saved_cmdlines);
3842 kfree(s->map_cmdline_to_pid);
3846 static int tracing_resize_saved_cmdlines(unsigned int val)
3848 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3850 s = kmalloc(sizeof(*s), GFP_KERNEL);
3854 if (allocate_cmdlines_buffer(val, s) < 0) {
3859 arch_spin_lock(&trace_cmdline_lock);
3860 savedcmd_temp = savedcmd;
3862 arch_spin_unlock(&trace_cmdline_lock);
3863 free_saved_cmdlines_buffer(savedcmd_temp);
3869 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3870 size_t cnt, loff_t *ppos)
3875 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3879 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3880 if (!val || val > PID_MAX_DEFAULT)
3883 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3892 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3893 .open = tracing_open_generic,
3894 .read = tracing_saved_cmdlines_size_read,
3895 .write = tracing_saved_cmdlines_size_write,
3899 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3900 size_t cnt, loff_t *ppos)
3902 struct trace_array *tr = filp->private_data;
3903 char buf[MAX_TRACER_SIZE+2];
3906 mutex_lock(&trace_types_lock);
3907 r = sprintf(buf, "%s\n", tr->current_trace->name);
3908 mutex_unlock(&trace_types_lock);
3910 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3913 int tracer_init(struct tracer *t, struct trace_array *tr)
3915 tracing_reset_online_cpus(&tr->trace_buffer);
3919 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3923 for_each_tracing_cpu(cpu)
3924 per_cpu_ptr(buf->data, cpu)->entries = val;
3927 #ifdef CONFIG_TRACER_MAX_TRACE
3928 /* resize @tr's buffer to the size of @size_tr's entries */
3929 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3930 struct trace_buffer *size_buf, int cpu_id)
3934 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3935 for_each_tracing_cpu(cpu) {
3936 ret = ring_buffer_resize(trace_buf->buffer,
3937 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3940 per_cpu_ptr(trace_buf->data, cpu)->entries =
3941 per_cpu_ptr(size_buf->data, cpu)->entries;
3944 ret = ring_buffer_resize(trace_buf->buffer,
3945 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3947 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3948 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3953 #endif /* CONFIG_TRACER_MAX_TRACE */
3955 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3956 unsigned long size, int cpu)
3961 * If kernel or user changes the size of the ring buffer
3962 * we use the size that was given, and we can forget about
3963 * expanding it later.
3965 ring_buffer_expanded = true;
3967 /* May be called before buffers are initialized */
3968 if (!tr->trace_buffer.buffer)
3971 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3975 #ifdef CONFIG_TRACER_MAX_TRACE
3976 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3977 !tr->current_trace->use_max_tr)
3980 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3982 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3983 &tr->trace_buffer, cpu);
3986 * AARGH! We are left with different
3987 * size max buffer!!!!
3988 * The max buffer is our "snapshot" buffer.
3989 * When a tracer needs a snapshot (one of the
3990 * latency tracers), it swaps the max buffer
3991 * with the saved snap shot. We succeeded to
3992 * update the size of the main buffer, but failed to
3993 * update the size of the max buffer. But when we tried
3994 * to reset the main buffer to the original size, we
3995 * failed there too. This is very unlikely to
3996 * happen, but if it does, warn and kill all
4000 tracing_disabled = 1;
4005 if (cpu == RING_BUFFER_ALL_CPUS)
4006 set_buffer_entries(&tr->max_buffer, size);
4008 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4011 #endif /* CONFIG_TRACER_MAX_TRACE */
4013 if (cpu == RING_BUFFER_ALL_CPUS)
4014 set_buffer_entries(&tr->trace_buffer, size);
4016 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4021 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4022 unsigned long size, int cpu_id)
4026 mutex_lock(&trace_types_lock);
4028 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4029 /* make sure, this cpu is enabled in the mask */
4030 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4036 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4041 mutex_unlock(&trace_types_lock);
4048 * tracing_update_buffers - used by tracing facility to expand ring buffers
4050 * To save on memory when the tracing is never used on a system with it
4051 * configured in. The ring buffers are set to a minimum size. But once
4052 * a user starts to use the tracing facility, then they need to grow
4053 * to their default size.
4055 * This function is to be called when a tracer is about to be used.
4057 int tracing_update_buffers(void)
4061 mutex_lock(&trace_types_lock);
4062 if (!ring_buffer_expanded)
4063 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4064 RING_BUFFER_ALL_CPUS);
4065 mutex_unlock(&trace_types_lock);
4070 struct trace_option_dentry;
4072 static struct trace_option_dentry *
4073 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4076 destroy_trace_option_files(struct trace_option_dentry *topts);
4079 * Used to clear out the tracer before deletion of an instance.
4080 * Must have trace_types_lock held.
4082 static void tracing_set_nop(struct trace_array *tr)
4084 if (tr->current_trace == &nop_trace)
4087 tr->current_trace->enabled--;
4089 if (tr->current_trace->reset)
4090 tr->current_trace->reset(tr);
4092 tr->current_trace = &nop_trace;
4095 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4097 static struct trace_option_dentry *topts;
4099 #ifdef CONFIG_TRACER_MAX_TRACE
4104 mutex_lock(&trace_types_lock);
4106 if (!ring_buffer_expanded) {
4107 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4108 RING_BUFFER_ALL_CPUS);
4114 for (t = trace_types; t; t = t->next) {
4115 if (strcmp(t->name, buf) == 0)
4122 if (t == tr->current_trace)
4125 /* Some tracers are only allowed for the top level buffer */
4126 if (!trace_ok_for_array(t, tr)) {
4131 trace_branch_disable();
4133 tr->current_trace->enabled--;
4135 if (tr->current_trace->reset)
4136 tr->current_trace->reset(tr);
4138 /* Current trace needs to be nop_trace before synchronize_sched */
4139 tr->current_trace = &nop_trace;
4141 #ifdef CONFIG_TRACER_MAX_TRACE
4142 had_max_tr = tr->allocated_snapshot;
4144 if (had_max_tr && !t->use_max_tr) {
4146 * We need to make sure that the update_max_tr sees that
4147 * current_trace changed to nop_trace to keep it from
4148 * swapping the buffers after we resize it.
4149 * The update_max_tr is called from interrupts disabled
4150 * so a synchronized_sched() is sufficient.
4152 synchronize_sched();
4156 /* Currently, only the top instance has options */
4157 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4158 destroy_trace_option_files(topts);
4159 topts = create_trace_option_files(tr, t);
4162 #ifdef CONFIG_TRACER_MAX_TRACE
4163 if (t->use_max_tr && !had_max_tr) {
4164 ret = alloc_snapshot(tr);
4171 ret = tracer_init(t, tr);
4176 tr->current_trace = t;
4177 tr->current_trace->enabled++;
4178 trace_branch_enable(tr);
4180 mutex_unlock(&trace_types_lock);
4186 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4187 size_t cnt, loff_t *ppos)
4189 struct trace_array *tr = filp->private_data;
4190 char buf[MAX_TRACER_SIZE+1];
4197 if (cnt > MAX_TRACER_SIZE)
4198 cnt = MAX_TRACER_SIZE;
4200 if (copy_from_user(&buf, ubuf, cnt))
4205 /* strip ending whitespace. */
4206 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4209 err = tracing_set_tracer(tr, buf);
4219 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4220 size_t cnt, loff_t *ppos)
4225 r = snprintf(buf, sizeof(buf), "%ld\n",
4226 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4227 if (r > sizeof(buf))
4229 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4233 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4234 size_t cnt, loff_t *ppos)
4239 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4249 tracing_thresh_read(struct file *filp, char __user *ubuf,
4250 size_t cnt, loff_t *ppos)
4252 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4256 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4257 size_t cnt, loff_t *ppos)
4259 struct trace_array *tr = filp->private_data;
4262 mutex_lock(&trace_types_lock);
4263 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4267 if (tr->current_trace->update_thresh) {
4268 ret = tr->current_trace->update_thresh(tr);
4275 mutex_unlock(&trace_types_lock);
4281 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4282 size_t cnt, loff_t *ppos)
4284 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4288 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4289 size_t cnt, loff_t *ppos)
4291 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4294 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4296 struct trace_array *tr = inode->i_private;
4297 struct trace_iterator *iter;
4300 if (tracing_disabled)
4303 if (trace_array_get(tr) < 0)
4306 mutex_lock(&trace_types_lock);
4308 /* create a buffer to store the information to pass to userspace */
4309 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4312 __trace_array_put(tr);
4317 * We make a copy of the current tracer to avoid concurrent
4318 * changes on it while we are reading.
4320 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4325 *iter->trace = *tr->current_trace;
4327 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4332 /* trace pipe does not show start of buffer */
4333 cpumask_setall(iter->started);
4335 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4336 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4338 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4339 if (trace_clocks[tr->clock_id].in_ns)
4340 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4343 iter->trace_buffer = &tr->trace_buffer;
4344 iter->cpu_file = tracing_get_cpu(inode);
4345 mutex_init(&iter->mutex);
4346 filp->private_data = iter;
4348 if (iter->trace->pipe_open)
4349 iter->trace->pipe_open(iter);
4351 nonseekable_open(inode, filp);
4353 mutex_unlock(&trace_types_lock);
4359 __trace_array_put(tr);
4360 mutex_unlock(&trace_types_lock);
4364 static int tracing_release_pipe(struct inode *inode, struct file *file)
4366 struct trace_iterator *iter = file->private_data;
4367 struct trace_array *tr = inode->i_private;
4369 mutex_lock(&trace_types_lock);
4371 if (iter->trace->pipe_close)
4372 iter->trace->pipe_close(iter);
4374 mutex_unlock(&trace_types_lock);
4376 free_cpumask_var(iter->started);
4377 mutex_destroy(&iter->mutex);
4381 trace_array_put(tr);
4387 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4389 /* Iterators are static, they should be filled or empty */
4390 if (trace_buffer_iter(iter, iter->cpu_file))
4391 return POLLIN | POLLRDNORM;
4393 if (trace_flags & TRACE_ITER_BLOCK)
4395 * Always select as readable when in blocking mode
4397 return POLLIN | POLLRDNORM;
4399 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4404 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4406 struct trace_iterator *iter = filp->private_data;
4408 return trace_poll(iter, filp, poll_table);
4411 /* Must be called with trace_types_lock mutex held. */
4412 static int tracing_wait_pipe(struct file *filp)
4414 struct trace_iterator *iter = filp->private_data;
4417 while (trace_empty(iter)) {
4419 if ((filp->f_flags & O_NONBLOCK)) {
4424 * We block until we read something and tracing is disabled.
4425 * We still block if tracing is disabled, but we have never
4426 * read anything. This allows a user to cat this file, and
4427 * then enable tracing. But after we have read something,
4428 * we give an EOF when tracing is again disabled.
4430 * iter->pos will be 0 if we haven't read anything.
4432 if (!tracing_is_on() && iter->pos)
4435 mutex_unlock(&iter->mutex);
4437 ret = wait_on_pipe(iter);
4439 mutex_lock(&iter->mutex);
4444 if (signal_pending(current))
4455 tracing_read_pipe(struct file *filp, char __user *ubuf,
4456 size_t cnt, loff_t *ppos)
4458 struct trace_iterator *iter = filp->private_data;
4459 struct trace_array *tr = iter->tr;
4462 /* return any leftover data */
4463 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4467 trace_seq_init(&iter->seq);
4469 /* copy the tracer to avoid using a global lock all around */
4470 mutex_lock(&trace_types_lock);
4471 if (unlikely(iter->trace->name != tr->current_trace->name))
4472 *iter->trace = *tr->current_trace;
4473 mutex_unlock(&trace_types_lock);
4476 * Avoid more than one consumer on a single file descriptor
4477 * This is just a matter of traces coherency, the ring buffer itself
4480 mutex_lock(&iter->mutex);
4481 if (iter->trace->read) {
4482 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4488 sret = tracing_wait_pipe(filp);
4492 /* stop when tracing is finished */
4493 if (trace_empty(iter)) {
4498 if (cnt >= PAGE_SIZE)
4499 cnt = PAGE_SIZE - 1;
4501 /* reset all but tr, trace, and overruns */
4502 memset(&iter->seq, 0,
4503 sizeof(struct trace_iterator) -
4504 offsetof(struct trace_iterator, seq));
4505 cpumask_clear(iter->started);
4508 trace_event_read_lock();
4509 trace_access_lock(iter->cpu_file);
4510 while (trace_find_next_entry_inc(iter) != NULL) {
4511 enum print_line_t ret;
4512 int len = iter->seq.len;
4514 ret = print_trace_line(iter);
4515 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4516 /* don't print partial lines */
4517 iter->seq.len = len;
4520 if (ret != TRACE_TYPE_NO_CONSUME)
4521 trace_consume(iter);
4523 if (iter->seq.len >= cnt)
4527 * Setting the full flag means we reached the trace_seq buffer
4528 * size and we should leave by partial output condition above.
4529 * One of the trace_seq_* functions is not used properly.
4531 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4534 trace_access_unlock(iter->cpu_file);
4535 trace_event_read_unlock();
4537 /* Now copy what we have to the user */
4538 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4539 if (iter->seq.readpos >= iter->seq.len)
4540 trace_seq_init(&iter->seq);
4543 * If there was nothing to send to user, in spite of consuming trace
4544 * entries, go back to wait for more entries.
4550 mutex_unlock(&iter->mutex);
4555 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4558 __free_page(spd->pages[idx]);
4561 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4563 .confirm = generic_pipe_buf_confirm,
4564 .release = generic_pipe_buf_release,
4565 .steal = generic_pipe_buf_steal,
4566 .get = generic_pipe_buf_get,
4570 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4575 /* Seq buffer is page-sized, exactly what we need. */
4577 count = iter->seq.len;
4578 ret = print_trace_line(iter);
4579 count = iter->seq.len - count;
4582 iter->seq.len -= count;
4585 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4586 iter->seq.len -= count;
4590 if (ret != TRACE_TYPE_NO_CONSUME)
4591 trace_consume(iter);
4593 if (!trace_find_next_entry_inc(iter)) {
4603 static ssize_t tracing_splice_read_pipe(struct file *filp,
4605 struct pipe_inode_info *pipe,
4609 struct page *pages_def[PIPE_DEF_BUFFERS];
4610 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4611 struct trace_iterator *iter = filp->private_data;
4612 struct splice_pipe_desc spd = {
4614 .partial = partial_def,
4615 .nr_pages = 0, /* This gets updated below. */
4616 .nr_pages_max = PIPE_DEF_BUFFERS,
4618 .ops = &tracing_pipe_buf_ops,
4619 .spd_release = tracing_spd_release_pipe,
4621 struct trace_array *tr = iter->tr;
4626 if (splice_grow_spd(pipe, &spd))
4629 /* copy the tracer to avoid using a global lock all around */
4630 mutex_lock(&trace_types_lock);
4631 if (unlikely(iter->trace->name != tr->current_trace->name))
4632 *iter->trace = *tr->current_trace;
4633 mutex_unlock(&trace_types_lock);
4635 mutex_lock(&iter->mutex);
4637 if (iter->trace->splice_read) {
4638 ret = iter->trace->splice_read(iter, filp,
4639 ppos, pipe, len, flags);
4644 ret = tracing_wait_pipe(filp);
4648 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4653 trace_event_read_lock();
4654 trace_access_lock(iter->cpu_file);
4656 /* Fill as many pages as possible. */
4657 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4658 spd.pages[i] = alloc_page(GFP_KERNEL);
4662 rem = tracing_fill_pipe_page(rem, iter);
4664 /* Copy the data into the page, so we can start over. */
4665 ret = trace_seq_to_buffer(&iter->seq,
4666 page_address(spd.pages[i]),
4669 __free_page(spd.pages[i]);
4672 spd.partial[i].offset = 0;
4673 spd.partial[i].len = iter->seq.len;
4675 trace_seq_init(&iter->seq);
4678 trace_access_unlock(iter->cpu_file);
4679 trace_event_read_unlock();
4680 mutex_unlock(&iter->mutex);
4684 ret = splice_to_pipe(pipe, &spd);
4686 splice_shrink_spd(&spd);
4690 mutex_unlock(&iter->mutex);
4695 tracing_entries_read(struct file *filp, char __user *ubuf,
4696 size_t cnt, loff_t *ppos)
4698 struct inode *inode = file_inode(filp);
4699 struct trace_array *tr = inode->i_private;
4700 int cpu = tracing_get_cpu(inode);
4705 mutex_lock(&trace_types_lock);
4707 if (cpu == RING_BUFFER_ALL_CPUS) {
4708 int cpu, buf_size_same;
4713 /* check if all cpu sizes are same */
4714 for_each_tracing_cpu(cpu) {
4715 /* fill in the size from first enabled cpu */
4717 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4718 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4724 if (buf_size_same) {
4725 if (!ring_buffer_expanded)
4726 r = sprintf(buf, "%lu (expanded: %lu)\n",
4728 trace_buf_size >> 10);
4730 r = sprintf(buf, "%lu\n", size >> 10);
4732 r = sprintf(buf, "X\n");
4734 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4736 mutex_unlock(&trace_types_lock);
4738 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4743 tracing_entries_write(struct file *filp, const char __user *ubuf,
4744 size_t cnt, loff_t *ppos)
4746 struct inode *inode = file_inode(filp);
4747 struct trace_array *tr = inode->i_private;
4751 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4755 /* must have at least 1 entry */
4759 /* value is in KB */
4761 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4771 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4772 size_t cnt, loff_t *ppos)
4774 struct trace_array *tr = filp->private_data;
4777 unsigned long size = 0, expanded_size = 0;
4779 mutex_lock(&trace_types_lock);
4780 for_each_tracing_cpu(cpu) {
4781 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4782 if (!ring_buffer_expanded)
4783 expanded_size += trace_buf_size >> 10;
4785 if (ring_buffer_expanded)
4786 r = sprintf(buf, "%lu\n", size);
4788 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4789 mutex_unlock(&trace_types_lock);
4791 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4795 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4796 size_t cnt, loff_t *ppos)
4799 * There is no need to read what the user has written, this function
4800 * is just to make sure that there is no error when "echo" is used
4809 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4811 struct trace_array *tr = inode->i_private;
4813 /* disable tracing ? */
4814 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4815 tracer_tracing_off(tr);
4816 /* resize the ring buffer to 0 */
4817 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4819 trace_array_put(tr);
4825 tracing_mark_write(struct file *filp, const char __user *ubuf,
4826 size_t cnt, loff_t *fpos)
4828 unsigned long addr = (unsigned long)ubuf;
4829 struct trace_array *tr = filp->private_data;
4830 struct ring_buffer_event *event;
4831 struct ring_buffer *buffer;
4832 struct print_entry *entry;
4833 unsigned long irq_flags;
4834 struct page *pages[2];
4844 if (tracing_disabled)
4847 if (!(trace_flags & TRACE_ITER_MARKERS))
4850 if (cnt > TRACE_BUF_SIZE)
4851 cnt = TRACE_BUF_SIZE;
4854 * Userspace is injecting traces into the kernel trace buffer.
4855 * We want to be as non intrusive as possible.
4856 * To do so, we do not want to allocate any special buffers
4857 * or take any locks, but instead write the userspace data
4858 * straight into the ring buffer.
4860 * First we need to pin the userspace buffer into memory,
4861 * which, most likely it is, because it just referenced it.
4862 * But there's no guarantee that it is. By using get_user_pages_fast()
4863 * and kmap_atomic/kunmap_atomic() we can get access to the
4864 * pages directly. We then write the data directly into the
4867 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4869 /* check if we cross pages */
4870 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4873 offset = addr & (PAGE_SIZE - 1);
4876 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4877 if (ret < nr_pages) {
4879 put_page(pages[ret]);
4884 for (i = 0; i < nr_pages; i++)
4885 map_page[i] = kmap_atomic(pages[i]);
4887 local_save_flags(irq_flags);
4888 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4889 buffer = tr->trace_buffer.buffer;
4890 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4891 irq_flags, preempt_count());
4893 /* Ring buffer disabled, return as if not open for write */
4898 entry = ring_buffer_event_data(event);
4899 entry->ip = _THIS_IP_;
4901 if (nr_pages == 2) {
4902 len = PAGE_SIZE - offset;
4903 memcpy(&entry->buf, map_page[0] + offset, len);
4904 memcpy(&entry->buf[len], map_page[1], cnt - len);
4906 memcpy(&entry->buf, map_page[0] + offset, cnt);
4908 if (entry->buf[cnt - 1] != '\n') {
4909 entry->buf[cnt] = '\n';
4910 entry->buf[cnt + 1] = '\0';
4912 entry->buf[cnt] = '\0';
4914 __buffer_unlock_commit(buffer, event);
4921 for (i = 0; i < nr_pages; i++){
4922 kunmap_atomic(map_page[i]);
4929 static int tracing_clock_show(struct seq_file *m, void *v)
4931 struct trace_array *tr = m->private;
4934 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4936 "%s%s%s%s", i ? " " : "",
4937 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4938 i == tr->clock_id ? "]" : "");
4944 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4948 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4949 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4952 if (i == ARRAY_SIZE(trace_clocks))
4955 mutex_lock(&trace_types_lock);
4959 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4962 * New clock may not be consistent with the previous clock.
4963 * Reset the buffer so that it doesn't have incomparable timestamps.
4965 tracing_reset_online_cpus(&tr->trace_buffer);
4967 #ifdef CONFIG_TRACER_MAX_TRACE
4968 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4969 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4970 tracing_reset_online_cpus(&tr->max_buffer);
4973 mutex_unlock(&trace_types_lock);
4978 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4979 size_t cnt, loff_t *fpos)
4981 struct seq_file *m = filp->private_data;
4982 struct trace_array *tr = m->private;
4984 const char *clockstr;
4987 if (cnt >= sizeof(buf))
4990 if (copy_from_user(&buf, ubuf, cnt))
4995 clockstr = strstrip(buf);
4997 ret = tracing_set_clock(tr, clockstr);
5006 static int tracing_clock_open(struct inode *inode, struct file *file)
5008 struct trace_array *tr = inode->i_private;
5011 if (tracing_disabled)
5014 if (trace_array_get(tr))
5017 ret = single_open(file, tracing_clock_show, inode->i_private);
5019 trace_array_put(tr);
5024 struct ftrace_buffer_info {
5025 struct trace_iterator iter;
5030 #ifdef CONFIG_TRACER_SNAPSHOT
5031 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5033 struct trace_array *tr = inode->i_private;
5034 struct trace_iterator *iter;
5038 if (trace_array_get(tr) < 0)
5041 if (file->f_mode & FMODE_READ) {
5042 iter = __tracing_open(inode, file, true);
5044 ret = PTR_ERR(iter);
5046 /* Writes still need the seq_file to hold the private data */
5048 m = kzalloc(sizeof(*m), GFP_KERNEL);
5051 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5059 iter->trace_buffer = &tr->max_buffer;
5060 iter->cpu_file = tracing_get_cpu(inode);
5062 file->private_data = m;
5066 trace_array_put(tr);
5072 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5075 struct seq_file *m = filp->private_data;
5076 struct trace_iterator *iter = m->private;
5077 struct trace_array *tr = iter->tr;
5081 ret = tracing_update_buffers();
5085 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5089 mutex_lock(&trace_types_lock);
5091 if (tr->current_trace->use_max_tr) {
5098 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5102 if (tr->allocated_snapshot)
5106 /* Only allow per-cpu swap if the ring buffer supports it */
5107 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5108 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5113 if (!tr->allocated_snapshot) {
5114 ret = alloc_snapshot(tr);
5118 local_irq_disable();
5119 /* Now, we're going to swap */
5120 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5121 update_max_tr(tr, current, smp_processor_id());
5123 update_max_tr_single(tr, current, iter->cpu_file);
5127 if (tr->allocated_snapshot) {
5128 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5129 tracing_reset_online_cpus(&tr->max_buffer);
5131 tracing_reset(&tr->max_buffer, iter->cpu_file);
5141 mutex_unlock(&trace_types_lock);
5145 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5147 struct seq_file *m = file->private_data;
5150 ret = tracing_release(inode, file);
5152 if (file->f_mode & FMODE_READ)
5155 /* If write only, the seq_file is just a stub */
5163 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5164 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5165 size_t count, loff_t *ppos);
5166 static int tracing_buffers_release(struct inode *inode, struct file *file);
5167 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5168 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5170 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5172 struct ftrace_buffer_info *info;
5175 ret = tracing_buffers_open(inode, filp);
5179 info = filp->private_data;
5181 if (info->iter.trace->use_max_tr) {
5182 tracing_buffers_release(inode, filp);
5186 info->iter.snapshot = true;
5187 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5192 #endif /* CONFIG_TRACER_SNAPSHOT */
5195 static const struct file_operations tracing_thresh_fops = {
5196 .open = tracing_open_generic,
5197 .read = tracing_thresh_read,
5198 .write = tracing_thresh_write,
5199 .llseek = generic_file_llseek,
5202 static const struct file_operations tracing_max_lat_fops = {
5203 .open = tracing_open_generic,
5204 .read = tracing_max_lat_read,
5205 .write = tracing_max_lat_write,
5206 .llseek = generic_file_llseek,
5209 static const struct file_operations set_tracer_fops = {
5210 .open = tracing_open_generic,
5211 .read = tracing_set_trace_read,
5212 .write = tracing_set_trace_write,
5213 .llseek = generic_file_llseek,
5216 static const struct file_operations tracing_pipe_fops = {
5217 .open = tracing_open_pipe,
5218 .poll = tracing_poll_pipe,
5219 .read = tracing_read_pipe,
5220 .splice_read = tracing_splice_read_pipe,
5221 .release = tracing_release_pipe,
5222 .llseek = no_llseek,
5225 static const struct file_operations tracing_entries_fops = {
5226 .open = tracing_open_generic_tr,
5227 .read = tracing_entries_read,
5228 .write = tracing_entries_write,
5229 .llseek = generic_file_llseek,
5230 .release = tracing_release_generic_tr,
5233 static const struct file_operations tracing_total_entries_fops = {
5234 .open = tracing_open_generic_tr,
5235 .read = tracing_total_entries_read,
5236 .llseek = generic_file_llseek,
5237 .release = tracing_release_generic_tr,
5240 static const struct file_operations tracing_free_buffer_fops = {
5241 .open = tracing_open_generic_tr,
5242 .write = tracing_free_buffer_write,
5243 .release = tracing_free_buffer_release,
5246 static const struct file_operations tracing_mark_fops = {
5247 .open = tracing_open_generic_tr,
5248 .write = tracing_mark_write,
5249 .llseek = generic_file_llseek,
5250 .release = tracing_release_generic_tr,
5253 static const struct file_operations trace_clock_fops = {
5254 .open = tracing_clock_open,
5256 .llseek = seq_lseek,
5257 .release = tracing_single_release_tr,
5258 .write = tracing_clock_write,
5261 #ifdef CONFIG_TRACER_SNAPSHOT
5262 static const struct file_operations snapshot_fops = {
5263 .open = tracing_snapshot_open,
5265 .write = tracing_snapshot_write,
5266 .llseek = tracing_lseek,
5267 .release = tracing_snapshot_release,
5270 static const struct file_operations snapshot_raw_fops = {
5271 .open = snapshot_raw_open,
5272 .read = tracing_buffers_read,
5273 .release = tracing_buffers_release,
5274 .splice_read = tracing_buffers_splice_read,
5275 .llseek = no_llseek,
5278 #endif /* CONFIG_TRACER_SNAPSHOT */
5280 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5282 struct trace_array *tr = inode->i_private;
5283 struct ftrace_buffer_info *info;
5286 if (tracing_disabled)
5289 if (trace_array_get(tr) < 0)
5292 info = kzalloc(sizeof(*info), GFP_KERNEL);
5294 trace_array_put(tr);
5298 mutex_lock(&trace_types_lock);
5301 info->iter.cpu_file = tracing_get_cpu(inode);
5302 info->iter.trace = tr->current_trace;
5303 info->iter.trace_buffer = &tr->trace_buffer;
5305 /* Force reading ring buffer for first read */
5306 info->read = (unsigned int)-1;
5308 filp->private_data = info;
5310 mutex_unlock(&trace_types_lock);
5312 ret = nonseekable_open(inode, filp);
5314 trace_array_put(tr);
5320 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5322 struct ftrace_buffer_info *info = filp->private_data;
5323 struct trace_iterator *iter = &info->iter;
5325 return trace_poll(iter, filp, poll_table);
5329 tracing_buffers_read(struct file *filp, char __user *ubuf,
5330 size_t count, loff_t *ppos)
5332 struct ftrace_buffer_info *info = filp->private_data;
5333 struct trace_iterator *iter = &info->iter;
5340 mutex_lock(&trace_types_lock);
5342 #ifdef CONFIG_TRACER_MAX_TRACE
5343 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5350 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5356 /* Do we have previous read data to read? */
5357 if (info->read < PAGE_SIZE)
5361 trace_access_lock(iter->cpu_file);
5362 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5366 trace_access_unlock(iter->cpu_file);
5369 if (trace_empty(iter)) {
5370 if ((filp->f_flags & O_NONBLOCK)) {
5374 mutex_unlock(&trace_types_lock);
5375 ret = wait_on_pipe(iter);
5376 mutex_lock(&trace_types_lock);
5381 if (signal_pending(current)) {
5393 size = PAGE_SIZE - info->read;
5397 ret = copy_to_user(ubuf, info->spare + info->read, size);
5408 mutex_unlock(&trace_types_lock);
5413 static int tracing_buffers_release(struct inode *inode, struct file *file)
5415 struct ftrace_buffer_info *info = file->private_data;
5416 struct trace_iterator *iter = &info->iter;
5418 mutex_lock(&trace_types_lock);
5420 __trace_array_put(iter->tr);
5423 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5426 mutex_unlock(&trace_types_lock);
5432 struct ring_buffer *buffer;
5437 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5438 struct pipe_buffer *buf)
5440 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5445 ring_buffer_free_read_page(ref->buffer, ref->page);
5450 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5451 struct pipe_buffer *buf)
5453 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5458 /* Pipe buffer operations for a buffer. */
5459 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5461 .confirm = generic_pipe_buf_confirm,
5462 .release = buffer_pipe_buf_release,
5463 .steal = generic_pipe_buf_steal,
5464 .get = buffer_pipe_buf_get,
5468 * Callback from splice_to_pipe(), if we need to release some pages
5469 * at the end of the spd in case we error'ed out in filling the pipe.
5471 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5473 struct buffer_ref *ref =
5474 (struct buffer_ref *)spd->partial[i].private;
5479 ring_buffer_free_read_page(ref->buffer, ref->page);
5481 spd->partial[i].private = 0;
5485 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5486 struct pipe_inode_info *pipe, size_t len,
5489 struct ftrace_buffer_info *info = file->private_data;
5490 struct trace_iterator *iter = &info->iter;
5491 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5492 struct page *pages_def[PIPE_DEF_BUFFERS];
5493 struct splice_pipe_desc spd = {
5495 .partial = partial_def,
5496 .nr_pages_max = PIPE_DEF_BUFFERS,
5498 .ops = &buffer_pipe_buf_ops,
5499 .spd_release = buffer_spd_release,
5501 struct buffer_ref *ref;
5502 int entries, size, i;
5505 mutex_lock(&trace_types_lock);
5507 #ifdef CONFIG_TRACER_MAX_TRACE
5508 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5514 if (splice_grow_spd(pipe, &spd)) {
5519 if (*ppos & (PAGE_SIZE - 1)) {
5524 if (len & (PAGE_SIZE - 1)) {
5525 if (len < PAGE_SIZE) {
5533 trace_access_lock(iter->cpu_file);
5534 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5536 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5540 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5545 ref->buffer = iter->trace_buffer->buffer;
5546 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5552 r = ring_buffer_read_page(ref->buffer, &ref->page,
5553 len, iter->cpu_file, 1);
5555 ring_buffer_free_read_page(ref->buffer, ref->page);
5561 * zero out any left over data, this is going to
5564 size = ring_buffer_page_len(ref->page);
5565 if (size < PAGE_SIZE)
5566 memset(ref->page + size, 0, PAGE_SIZE - size);
5568 page = virt_to_page(ref->page);
5570 spd.pages[i] = page;
5571 spd.partial[i].len = PAGE_SIZE;
5572 spd.partial[i].offset = 0;
5573 spd.partial[i].private = (unsigned long)ref;
5577 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5580 trace_access_unlock(iter->cpu_file);
5583 /* did we read anything? */
5584 if (!spd.nr_pages) {
5585 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5589 mutex_unlock(&trace_types_lock);
5590 ret = wait_on_pipe(iter);
5591 mutex_lock(&trace_types_lock);
5594 if (signal_pending(current)) {
5601 ret = splice_to_pipe(pipe, &spd);
5602 splice_shrink_spd(&spd);
5604 mutex_unlock(&trace_types_lock);
5609 static const struct file_operations tracing_buffers_fops = {
5610 .open = tracing_buffers_open,
5611 .read = tracing_buffers_read,
5612 .poll = tracing_buffers_poll,
5613 .release = tracing_buffers_release,
5614 .splice_read = tracing_buffers_splice_read,
5615 .llseek = no_llseek,
5619 tracing_stats_read(struct file *filp, char __user *ubuf,
5620 size_t count, loff_t *ppos)
5622 struct inode *inode = file_inode(filp);
5623 struct trace_array *tr = inode->i_private;
5624 struct trace_buffer *trace_buf = &tr->trace_buffer;
5625 int cpu = tracing_get_cpu(inode);
5626 struct trace_seq *s;
5628 unsigned long long t;
5629 unsigned long usec_rem;
5631 s = kmalloc(sizeof(*s), GFP_KERNEL);
5637 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5638 trace_seq_printf(s, "entries: %ld\n", cnt);
5640 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5641 trace_seq_printf(s, "overrun: %ld\n", cnt);
5643 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5644 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5646 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5647 trace_seq_printf(s, "bytes: %ld\n", cnt);
5649 if (trace_clocks[tr->clock_id].in_ns) {
5650 /* local or global for trace_clock */
5651 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5652 usec_rem = do_div(t, USEC_PER_SEC);
5653 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5656 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5657 usec_rem = do_div(t, USEC_PER_SEC);
5658 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5660 /* counter or tsc mode for trace_clock */
5661 trace_seq_printf(s, "oldest event ts: %llu\n",
5662 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5664 trace_seq_printf(s, "now ts: %llu\n",
5665 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5668 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5669 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5671 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5672 trace_seq_printf(s, "read events: %ld\n", cnt);
5674 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5681 static const struct file_operations tracing_stats_fops = {
5682 .open = tracing_open_generic_tr,
5683 .read = tracing_stats_read,
5684 .llseek = generic_file_llseek,
5685 .release = tracing_release_generic_tr,
5688 #ifdef CONFIG_DYNAMIC_FTRACE
5690 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5696 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5697 size_t cnt, loff_t *ppos)
5699 static char ftrace_dyn_info_buffer[1024];
5700 static DEFINE_MUTEX(dyn_info_mutex);
5701 unsigned long *p = filp->private_data;
5702 char *buf = ftrace_dyn_info_buffer;
5703 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5706 mutex_lock(&dyn_info_mutex);
5707 r = sprintf(buf, "%ld ", *p);
5709 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5712 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5714 mutex_unlock(&dyn_info_mutex);
5719 static const struct file_operations tracing_dyn_info_fops = {
5720 .open = tracing_open_generic,
5721 .read = tracing_read_dyn_info,
5722 .llseek = generic_file_llseek,
5724 #endif /* CONFIG_DYNAMIC_FTRACE */
5726 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5728 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5734 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5736 unsigned long *count = (long *)data;
5748 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5749 struct ftrace_probe_ops *ops, void *data)
5751 long count = (long)data;
5753 seq_printf(m, "%ps:", (void *)ip);
5755 seq_printf(m, "snapshot");
5758 seq_printf(m, ":unlimited\n");
5760 seq_printf(m, ":count=%ld\n", count);
5765 static struct ftrace_probe_ops snapshot_probe_ops = {
5766 .func = ftrace_snapshot,
5767 .print = ftrace_snapshot_print,
5770 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5771 .func = ftrace_count_snapshot,
5772 .print = ftrace_snapshot_print,
5776 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5777 char *glob, char *cmd, char *param, int enable)
5779 struct ftrace_probe_ops *ops;
5780 void *count = (void *)-1;
5784 /* hash funcs only work with set_ftrace_filter */
5788 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5790 if (glob[0] == '!') {
5791 unregister_ftrace_function_probe_func(glob+1, ops);
5798 number = strsep(¶m, ":");
5800 if (!strlen(number))
5804 * We use the callback data field (which is a pointer)
5807 ret = kstrtoul(number, 0, (unsigned long *)&count);
5812 ret = register_ftrace_function_probe(glob, ops, count);
5815 alloc_snapshot(&global_trace);
5817 return ret < 0 ? ret : 0;
5820 static struct ftrace_func_command ftrace_snapshot_cmd = {
5822 .func = ftrace_trace_snapshot_callback,
5825 static __init int register_snapshot_cmd(void)
5827 return register_ftrace_command(&ftrace_snapshot_cmd);
5830 static inline __init int register_snapshot_cmd(void) { return 0; }
5831 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5833 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5838 if (!debugfs_initialized())
5841 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5842 tr->dir = debugfs_create_dir("tracing", NULL);
5845 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5850 struct dentry *tracing_init_dentry(void)
5852 return tracing_init_dentry_tr(&global_trace);
5855 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5857 struct dentry *d_tracer;
5860 return tr->percpu_dir;
5862 d_tracer = tracing_init_dentry_tr(tr);
5866 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5868 WARN_ONCE(!tr->percpu_dir,
5869 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5871 return tr->percpu_dir;
5874 static struct dentry *
5875 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5876 void *data, long cpu, const struct file_operations *fops)
5878 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5880 if (ret) /* See tracing_get_cpu() */
5881 ret->d_inode->i_cdev = (void *)(cpu + 1);
5886 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5888 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5889 struct dentry *d_cpu;
5890 char cpu_dir[30]; /* 30 characters should be more than enough */
5895 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5896 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5898 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5902 /* per cpu trace_pipe */
5903 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5904 tr, cpu, &tracing_pipe_fops);
5907 trace_create_cpu_file("trace", 0644, d_cpu,
5908 tr, cpu, &tracing_fops);
5910 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5911 tr, cpu, &tracing_buffers_fops);
5913 trace_create_cpu_file("stats", 0444, d_cpu,
5914 tr, cpu, &tracing_stats_fops);
5916 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5917 tr, cpu, &tracing_entries_fops);
5919 #ifdef CONFIG_TRACER_SNAPSHOT
5920 trace_create_cpu_file("snapshot", 0644, d_cpu,
5921 tr, cpu, &snapshot_fops);
5923 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5924 tr, cpu, &snapshot_raw_fops);
5928 #ifdef CONFIG_FTRACE_SELFTEST
5929 /* Let selftest have access to static functions in this file */
5930 #include "trace_selftest.c"
5933 struct trace_option_dentry {
5934 struct tracer_opt *opt;
5935 struct tracer_flags *flags;
5936 struct trace_array *tr;
5937 struct dentry *entry;
5941 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5944 struct trace_option_dentry *topt = filp->private_data;
5947 if (topt->flags->val & topt->opt->bit)
5952 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5956 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5959 struct trace_option_dentry *topt = filp->private_data;
5963 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5967 if (val != 0 && val != 1)
5970 if (!!(topt->flags->val & topt->opt->bit) != val) {
5971 mutex_lock(&trace_types_lock);
5972 ret = __set_tracer_option(topt->tr, topt->flags,
5974 mutex_unlock(&trace_types_lock);
5985 static const struct file_operations trace_options_fops = {
5986 .open = tracing_open_generic,
5987 .read = trace_options_read,
5988 .write = trace_options_write,
5989 .llseek = generic_file_llseek,
5993 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5996 long index = (long)filp->private_data;
5999 if (trace_flags & (1 << index))
6004 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6008 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6011 struct trace_array *tr = &global_trace;
6012 long index = (long)filp->private_data;
6016 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6020 if (val != 0 && val != 1)
6023 mutex_lock(&trace_types_lock);
6024 ret = set_tracer_flag(tr, 1 << index, val);
6025 mutex_unlock(&trace_types_lock);
6035 static const struct file_operations trace_options_core_fops = {
6036 .open = tracing_open_generic,
6037 .read = trace_options_core_read,
6038 .write = trace_options_core_write,
6039 .llseek = generic_file_llseek,
6042 struct dentry *trace_create_file(const char *name,
6044 struct dentry *parent,
6046 const struct file_operations *fops)
6050 ret = debugfs_create_file(name, mode, parent, data, fops);
6052 pr_warning("Could not create debugfs '%s' entry\n", name);
6058 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6060 struct dentry *d_tracer;
6065 d_tracer = tracing_init_dentry_tr(tr);
6069 tr->options = debugfs_create_dir("options", d_tracer);
6071 pr_warning("Could not create debugfs directory 'options'\n");
6079 create_trace_option_file(struct trace_array *tr,
6080 struct trace_option_dentry *topt,
6081 struct tracer_flags *flags,
6082 struct tracer_opt *opt)
6084 struct dentry *t_options;
6086 t_options = trace_options_init_dentry(tr);
6090 topt->flags = flags;
6094 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6095 &trace_options_fops);
6099 static struct trace_option_dentry *
6100 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6102 struct trace_option_dentry *topts;
6103 struct tracer_flags *flags;
6104 struct tracer_opt *opts;
6110 flags = tracer->flags;
6112 if (!flags || !flags->opts)
6117 for (cnt = 0; opts[cnt].name; cnt++)
6120 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6124 for (cnt = 0; opts[cnt].name; cnt++)
6125 create_trace_option_file(tr, &topts[cnt], flags,
6132 destroy_trace_option_files(struct trace_option_dentry *topts)
6139 for (cnt = 0; topts[cnt].opt; cnt++)
6140 debugfs_remove(topts[cnt].entry);
6145 static struct dentry *
6146 create_trace_option_core_file(struct trace_array *tr,
6147 const char *option, long index)
6149 struct dentry *t_options;
6151 t_options = trace_options_init_dentry(tr);
6155 return trace_create_file(option, 0644, t_options, (void *)index,
6156 &trace_options_core_fops);
6159 static __init void create_trace_options_dir(struct trace_array *tr)
6161 struct dentry *t_options;
6164 t_options = trace_options_init_dentry(tr);
6168 for (i = 0; trace_options[i]; i++)
6169 create_trace_option_core_file(tr, trace_options[i], i);
6173 rb_simple_read(struct file *filp, char __user *ubuf,
6174 size_t cnt, loff_t *ppos)
6176 struct trace_array *tr = filp->private_data;
6180 r = tracer_tracing_is_on(tr);
6181 r = sprintf(buf, "%d\n", r);
6183 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6187 rb_simple_write(struct file *filp, const char __user *ubuf,
6188 size_t cnt, loff_t *ppos)
6190 struct trace_array *tr = filp->private_data;
6191 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6195 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6200 mutex_lock(&trace_types_lock);
6202 tracer_tracing_on(tr);
6203 if (tr->current_trace->start)
6204 tr->current_trace->start(tr);
6206 tracer_tracing_off(tr);
6207 if (tr->current_trace->stop)
6208 tr->current_trace->stop(tr);
6210 mutex_unlock(&trace_types_lock);
6218 static const struct file_operations rb_simple_fops = {
6219 .open = tracing_open_generic_tr,
6220 .read = rb_simple_read,
6221 .write = rb_simple_write,
6222 .release = tracing_release_generic_tr,
6223 .llseek = default_llseek,
6226 struct dentry *trace_instance_dir;
6229 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6232 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6234 enum ring_buffer_flags rb_flags;
6236 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6240 buf->buffer = ring_buffer_alloc(size, rb_flags);
6244 buf->data = alloc_percpu(struct trace_array_cpu);
6246 ring_buffer_free(buf->buffer);
6250 /* Allocate the first page for all buffers */
6251 set_buffer_entries(&tr->trace_buffer,
6252 ring_buffer_size(tr->trace_buffer.buffer, 0));
6257 static int allocate_trace_buffers(struct trace_array *tr, int size)
6261 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6265 #ifdef CONFIG_TRACER_MAX_TRACE
6266 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6267 allocate_snapshot ? size : 1);
6269 ring_buffer_free(tr->trace_buffer.buffer);
6270 free_percpu(tr->trace_buffer.data);
6273 tr->allocated_snapshot = allocate_snapshot;
6276 * Only the top level trace array gets its snapshot allocated
6277 * from the kernel command line.
6279 allocate_snapshot = false;
6284 static void free_trace_buffer(struct trace_buffer *buf)
6287 ring_buffer_free(buf->buffer);
6289 free_percpu(buf->data);
6294 static void free_trace_buffers(struct trace_array *tr)
6299 free_trace_buffer(&tr->trace_buffer);
6301 #ifdef CONFIG_TRACER_MAX_TRACE
6302 free_trace_buffer(&tr->max_buffer);
6306 static int new_instance_create(const char *name)
6308 struct trace_array *tr;
6311 mutex_lock(&trace_types_lock);
6314 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6315 if (tr->name && strcmp(tr->name, name) == 0)
6320 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6324 tr->name = kstrdup(name, GFP_KERNEL);
6328 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6331 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6333 raw_spin_lock_init(&tr->start_lock);
6335 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6337 tr->current_trace = &nop_trace;
6339 INIT_LIST_HEAD(&tr->systems);
6340 INIT_LIST_HEAD(&tr->events);
6342 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6345 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6349 ret = event_trace_add_tracer(tr->dir, tr);
6351 debugfs_remove_recursive(tr->dir);
6355 init_tracer_debugfs(tr, tr->dir);
6357 list_add(&tr->list, &ftrace_trace_arrays);
6359 mutex_unlock(&trace_types_lock);
6364 free_trace_buffers(tr);
6365 free_cpumask_var(tr->tracing_cpumask);
6370 mutex_unlock(&trace_types_lock);
6376 static int instance_delete(const char *name)
6378 struct trace_array *tr;
6382 mutex_lock(&trace_types_lock);
6385 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6386 if (tr->name && strcmp(tr->name, name) == 0) {
6398 list_del(&tr->list);
6400 tracing_set_nop(tr);
6401 event_trace_del_tracer(tr);
6402 ftrace_destroy_function_files(tr);
6403 debugfs_remove_recursive(tr->dir);
6404 free_trace_buffers(tr);
6412 mutex_unlock(&trace_types_lock);
6417 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6419 struct dentry *parent;
6422 /* Paranoid: Make sure the parent is the "instances" directory */
6423 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6424 if (WARN_ON_ONCE(parent != trace_instance_dir))
6428 * The inode mutex is locked, but debugfs_create_dir() will also
6429 * take the mutex. As the instances directory can not be destroyed
6430 * or changed in any other way, it is safe to unlock it, and
6431 * let the dentry try. If two users try to make the same dir at
6432 * the same time, then the new_instance_create() will determine the
6435 mutex_unlock(&inode->i_mutex);
6437 ret = new_instance_create(dentry->d_iname);
6439 mutex_lock(&inode->i_mutex);
6444 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6446 struct dentry *parent;
6449 /* Paranoid: Make sure the parent is the "instances" directory */
6450 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6451 if (WARN_ON_ONCE(parent != trace_instance_dir))
6454 /* The caller did a dget() on dentry */
6455 mutex_unlock(&dentry->d_inode->i_mutex);
6458 * The inode mutex is locked, but debugfs_create_dir() will also
6459 * take the mutex. As the instances directory can not be destroyed
6460 * or changed in any other way, it is safe to unlock it, and
6461 * let the dentry try. If two users try to make the same dir at
6462 * the same time, then the instance_delete() will determine the
6465 mutex_unlock(&inode->i_mutex);
6467 ret = instance_delete(dentry->d_iname);
6469 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6470 mutex_lock(&dentry->d_inode->i_mutex);
6475 static const struct inode_operations instance_dir_inode_operations = {
6476 .lookup = simple_lookup,
6477 .mkdir = instance_mkdir,
6478 .rmdir = instance_rmdir,
6481 static __init void create_trace_instances(struct dentry *d_tracer)
6483 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6484 if (WARN_ON(!trace_instance_dir))
6487 /* Hijack the dir inode operations, to allow mkdir */
6488 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6492 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6496 trace_create_file("available_tracers", 0444, d_tracer,
6497 tr, &show_traces_fops);
6499 trace_create_file("current_tracer", 0644, d_tracer,
6500 tr, &set_tracer_fops);
6502 trace_create_file("tracing_cpumask", 0644, d_tracer,
6503 tr, &tracing_cpumask_fops);
6505 trace_create_file("trace_options", 0644, d_tracer,
6506 tr, &tracing_iter_fops);
6508 trace_create_file("trace", 0644, d_tracer,
6511 trace_create_file("trace_pipe", 0444, d_tracer,
6512 tr, &tracing_pipe_fops);
6514 trace_create_file("buffer_size_kb", 0644, d_tracer,
6515 tr, &tracing_entries_fops);
6517 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6518 tr, &tracing_total_entries_fops);
6520 trace_create_file("free_buffer", 0200, d_tracer,
6521 tr, &tracing_free_buffer_fops);
6523 trace_create_file("trace_marker", 0220, d_tracer,
6524 tr, &tracing_mark_fops);
6526 trace_create_file("trace_clock", 0644, d_tracer, tr,
6529 trace_create_file("tracing_on", 0644, d_tracer,
6530 tr, &rb_simple_fops);
6532 #ifdef CONFIG_TRACER_MAX_TRACE
6533 trace_create_file("tracing_max_latency", 0644, d_tracer,
6534 &tr->max_latency, &tracing_max_lat_fops);
6537 if (ftrace_create_function_files(tr, d_tracer))
6538 WARN(1, "Could not allocate function filter files");
6540 #ifdef CONFIG_TRACER_SNAPSHOT
6541 trace_create_file("snapshot", 0644, d_tracer,
6542 tr, &snapshot_fops);
6545 for_each_tracing_cpu(cpu)
6546 tracing_init_debugfs_percpu(tr, cpu);
6550 static __init int tracer_init_debugfs(void)
6552 struct dentry *d_tracer;
6554 trace_access_lock_init();
6556 d_tracer = tracing_init_dentry();
6560 init_tracer_debugfs(&global_trace, d_tracer);
6562 trace_create_file("tracing_thresh", 0644, d_tracer,
6563 &global_trace, &tracing_thresh_fops);
6565 trace_create_file("README", 0444, d_tracer,
6566 NULL, &tracing_readme_fops);
6568 trace_create_file("saved_cmdlines", 0444, d_tracer,
6569 NULL, &tracing_saved_cmdlines_fops);
6571 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6572 NULL, &tracing_saved_cmdlines_size_fops);
6574 #ifdef CONFIG_DYNAMIC_FTRACE
6575 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6576 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6579 create_trace_instances(d_tracer);
6581 create_trace_options_dir(&global_trace);
6586 static int trace_panic_handler(struct notifier_block *this,
6587 unsigned long event, void *unused)
6589 if (ftrace_dump_on_oops)
6590 ftrace_dump(ftrace_dump_on_oops);
6594 static struct notifier_block trace_panic_notifier = {
6595 .notifier_call = trace_panic_handler,
6597 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6600 static int trace_die_handler(struct notifier_block *self,
6606 if (ftrace_dump_on_oops)
6607 ftrace_dump(ftrace_dump_on_oops);
6615 static struct notifier_block trace_die_notifier = {
6616 .notifier_call = trace_die_handler,
6621 * printk is set to max of 1024, we really don't need it that big.
6622 * Nothing should be printing 1000 characters anyway.
6624 #define TRACE_MAX_PRINT 1000
6627 * Define here KERN_TRACE so that we have one place to modify
6628 * it if we decide to change what log level the ftrace dump
6631 #define KERN_TRACE KERN_EMERG
6634 trace_printk_seq(struct trace_seq *s)
6636 /* Probably should print a warning here. */
6637 if (s->len >= TRACE_MAX_PRINT)
6638 s->len = TRACE_MAX_PRINT;
6640 /* should be zero ended, but we are paranoid. */
6641 s->buffer[s->len] = 0;
6643 printk(KERN_TRACE "%s", s->buffer);
6648 void trace_init_global_iter(struct trace_iterator *iter)
6650 iter->tr = &global_trace;
6651 iter->trace = iter->tr->current_trace;
6652 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6653 iter->trace_buffer = &global_trace.trace_buffer;
6655 if (iter->trace && iter->trace->open)
6656 iter->trace->open(iter);
6658 /* Annotate start of buffers if we had overruns */
6659 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6660 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6662 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6663 if (trace_clocks[iter->tr->clock_id].in_ns)
6664 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6667 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6669 /* use static because iter can be a bit big for the stack */
6670 static struct trace_iterator iter;
6671 static atomic_t dump_running;
6672 unsigned int old_userobj;
6673 unsigned long flags;
6676 /* Only allow one dump user at a time. */
6677 if (atomic_inc_return(&dump_running) != 1) {
6678 atomic_dec(&dump_running);
6683 * Always turn off tracing when we dump.
6684 * We don't need to show trace output of what happens
6685 * between multiple crashes.
6687 * If the user does a sysrq-z, then they can re-enable
6688 * tracing with echo 1 > tracing_on.
6692 local_irq_save(flags);
6694 /* Simulate the iterator */
6695 trace_init_global_iter(&iter);
6697 for_each_tracing_cpu(cpu) {
6698 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6701 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6703 /* don't look at user memory in panic mode */
6704 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6706 switch (oops_dump_mode) {
6708 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6711 iter.cpu_file = raw_smp_processor_id();
6716 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6717 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6720 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6722 /* Did function tracer already get disabled? */
6723 if (ftrace_is_dead()) {
6724 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6725 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6729 * We need to stop all tracing on all CPUS to read the
6730 * the next buffer. This is a bit expensive, but is
6731 * not done often. We fill all what we can read,
6732 * and then release the locks again.
6735 while (!trace_empty(&iter)) {
6738 printk(KERN_TRACE "---------------------------------\n");
6742 /* reset all but tr, trace, and overruns */
6743 memset(&iter.seq, 0,
6744 sizeof(struct trace_iterator) -
6745 offsetof(struct trace_iterator, seq));
6746 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6749 if (trace_find_next_entry_inc(&iter) != NULL) {
6752 ret = print_trace_line(&iter);
6753 if (ret != TRACE_TYPE_NO_CONSUME)
6754 trace_consume(&iter);
6756 touch_nmi_watchdog();
6758 trace_printk_seq(&iter.seq);
6762 printk(KERN_TRACE " (ftrace buffer empty)\n");
6764 printk(KERN_TRACE "---------------------------------\n");
6767 trace_flags |= old_userobj;
6769 for_each_tracing_cpu(cpu) {
6770 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6772 atomic_dec(&dump_running);
6773 local_irq_restore(flags);
6775 EXPORT_SYMBOL_GPL(ftrace_dump);
6777 __init static int tracer_alloc_buffers(void)
6783 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6786 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6787 goto out_free_buffer_mask;
6789 /* Only allocate trace_printk buffers if a trace_printk exists */
6790 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6791 /* Must be called before global_trace.buffer is allocated */
6792 trace_printk_init_buffers();
6794 /* To save memory, keep the ring buffer size to its minimum */
6795 if (ring_buffer_expanded)
6796 ring_buf_size = trace_buf_size;
6800 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6801 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6803 raw_spin_lock_init(&global_trace.start_lock);
6805 /* Used for event triggers */
6806 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6808 goto out_free_cpumask;
6810 if (trace_create_savedcmd() < 0)
6811 goto out_free_temp_buffer;
6813 /* TODO: make the number of buffers hot pluggable with CPUS */
6814 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6815 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6817 goto out_free_savedcmd;
6820 if (global_trace.buffer_disabled)
6823 if (trace_boot_clock) {
6824 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6826 pr_warning("Trace clock %s not defined, going back to default\n",
6831 * register_tracer() might reference current_trace, so it
6832 * needs to be set before we register anything. This is
6833 * just a bootstrap of current_trace anyway.
6835 global_trace.current_trace = &nop_trace;
6837 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6839 ftrace_init_global_array_ops(&global_trace);
6841 register_tracer(&nop_trace);
6843 /* All seems OK, enable tracing */
6844 tracing_disabled = 0;
6846 atomic_notifier_chain_register(&panic_notifier_list,
6847 &trace_panic_notifier);
6849 register_die_notifier(&trace_die_notifier);
6851 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6853 INIT_LIST_HEAD(&global_trace.systems);
6854 INIT_LIST_HEAD(&global_trace.events);
6855 list_add(&global_trace.list, &ftrace_trace_arrays);
6857 while (trace_boot_options) {
6860 option = strsep(&trace_boot_options, ",");
6861 trace_set_options(&global_trace, option);
6864 register_snapshot_cmd();
6869 free_saved_cmdlines_buffer(savedcmd);
6870 out_free_temp_buffer:
6871 ring_buffer_free(temp_buffer);
6873 free_cpumask_var(global_trace.tracing_cpumask);
6874 out_free_buffer_mask:
6875 free_cpumask_var(tracing_buffer_mask);
6880 __init static int clear_boot_tracer(void)
6883 * The default tracer at boot buffer is an init section.
6884 * This function is called in lateinit. If we did not
6885 * find the boot tracer, then clear it out, to prevent
6886 * later registration from accessing the buffer that is
6887 * about to be freed.
6889 if (!default_bootup_tracer)
6892 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6893 default_bootup_tracer);
6894 default_bootup_tracer = NULL;
6899 early_initcall(tracer_alloc_buffers);
6900 fs_initcall(tracer_init_debugfs);
6901 late_initcall(clear_boot_tracer);