2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
77 static struct tracer_flags dummy_tracer_flags = {
79 .opts = dummy_tracer_opt
83 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
89 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
93 static DEFINE_PER_CPU(bool, trace_cmdline_save);
96 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
101 static int tracing_disabled = 1;
103 cpumask_var_t __read_mostly tracing_buffer_mask;
106 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
108 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
109 * is set, then ftrace_dump is called. This will output the contents
110 * of the ftrace buffers to the console. This is very useful for
111 * capturing traces that lead to crashes and outputing it to a
114 * It is default off, but you can enable it with either specifying
115 * "ftrace_dump_on_oops" in the kernel command line, or setting
116 * /proc/sys/kernel/ftrace_dump_on_oops
117 * Set 1 if you want to dump buffers of all CPUs
118 * Set 2 if you want to dump the buffer of the CPU that triggered oops
121 enum ftrace_dump_mode ftrace_dump_on_oops;
123 /* When set, tracing will stop when a WARN*() is hit */
124 int __disable_trace_on_warning;
126 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
127 /* Map of enums to their values, for "enum_map" file */
128 struct trace_enum_map_head {
130 unsigned long length;
133 union trace_enum_map_item;
135 struct trace_enum_map_tail {
137 * "end" is first and points to NULL as it must be different
138 * than "mod" or "enum_string"
140 union trace_enum_map_item *next;
141 const char *end; /* points to NULL */
144 static DEFINE_MUTEX(trace_enum_mutex);
147 * The trace_enum_maps are saved in an array with two extra elements,
148 * one at the beginning, and one at the end. The beginning item contains
149 * the count of the saved maps (head.length), and the module they
150 * belong to if not built in (head.mod). The ending item contains a
151 * pointer to the next array of saved enum_map items.
153 union trace_enum_map_item {
154 struct trace_enum_map map;
155 struct trace_enum_map_head head;
156 struct trace_enum_map_tail tail;
159 static union trace_enum_map_item *trace_enum_maps;
160 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
162 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
164 #define MAX_TRACER_SIZE 100
165 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
166 static char *default_bootup_tracer;
168 static bool allocate_snapshot;
170 static int __init set_cmdline_ftrace(char *str)
172 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
173 default_bootup_tracer = bootup_tracer_buf;
174 /* We are using ftrace early, expand it */
175 ring_buffer_expanded = true;
178 __setup("ftrace=", set_cmdline_ftrace);
180 static int __init set_ftrace_dump_on_oops(char *str)
182 if (*str++ != '=' || !*str) {
183 ftrace_dump_on_oops = DUMP_ALL;
187 if (!strcmp("orig_cpu", str)) {
188 ftrace_dump_on_oops = DUMP_ORIG;
194 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
196 static int __init stop_trace_on_warning(char *str)
198 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
199 __disable_trace_on_warning = 1;
202 __setup("traceoff_on_warning", stop_trace_on_warning);
204 static int __init boot_alloc_snapshot(char *str)
206 allocate_snapshot = true;
207 /* We also need the main ring buffer expanded */
208 ring_buffer_expanded = true;
211 __setup("alloc_snapshot", boot_alloc_snapshot);
214 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
216 static int __init set_trace_boot_options(char *str)
218 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
221 __setup("trace_options=", set_trace_boot_options);
223 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
224 static char *trace_boot_clock __initdata;
226 static int __init set_trace_boot_clock(char *str)
228 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
229 trace_boot_clock = trace_boot_clock_buf;
232 __setup("trace_clock=", set_trace_boot_clock);
234 static int __init set_tracepoint_printk(char *str)
236 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
237 tracepoint_printk = 1;
240 __setup("tp_printk", set_tracepoint_printk);
242 unsigned long long ns2usecs(cycle_t nsec)
249 /* trace_flags holds trace_options default values */
250 #define TRACE_DEFAULT_FLAGS \
251 (FUNCTION_DEFAULT_FLAGS | \
252 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
253 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
254 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
255 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
257 /* trace_options that are only supported by global_trace */
258 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
259 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
263 * The global_trace is the descriptor that holds the tracing
264 * buffers for the live tracing. For each CPU, it contains
265 * a link list of pages that will store trace entries. The
266 * page descriptor of the pages in the memory is used to hold
267 * the link list by linking the lru item in the page descriptor
268 * to each of the pages in the buffer per CPU.
270 * For each active CPU there is a data field that holds the
271 * pages for the buffer for that CPU. Each CPU has the same number
272 * of pages allocated for its buffer.
274 static struct trace_array global_trace = {
275 .trace_flags = TRACE_DEFAULT_FLAGS,
278 LIST_HEAD(ftrace_trace_arrays);
280 int trace_array_get(struct trace_array *this_tr)
282 struct trace_array *tr;
285 mutex_lock(&trace_types_lock);
286 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
293 mutex_unlock(&trace_types_lock);
298 static void __trace_array_put(struct trace_array *this_tr)
300 WARN_ON(!this_tr->ref);
304 void trace_array_put(struct trace_array *this_tr)
306 mutex_lock(&trace_types_lock);
307 __trace_array_put(this_tr);
308 mutex_unlock(&trace_types_lock);
311 int filter_check_discard(struct trace_event_file *file, void *rec,
312 struct ring_buffer *buffer,
313 struct ring_buffer_event *event)
315 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
316 !filter_match_preds(file->filter, rec)) {
317 ring_buffer_discard_commit(buffer, event);
323 EXPORT_SYMBOL_GPL(filter_check_discard);
325 int call_filter_check_discard(struct trace_event_call *call, void *rec,
326 struct ring_buffer *buffer,
327 struct ring_buffer_event *event)
329 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
330 !filter_match_preds(call->filter, rec)) {
331 ring_buffer_discard_commit(buffer, event);
337 EXPORT_SYMBOL_GPL(call_filter_check_discard);
339 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
343 /* Early boot up does not have a buffer yet */
345 return trace_clock_local();
347 ts = ring_buffer_time_stamp(buf->buffer, cpu);
348 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
353 cycle_t ftrace_now(int cpu)
355 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
359 * tracing_is_enabled - Show if global_trace has been disabled
361 * Shows if the global trace has been enabled or not. It uses the
362 * mirror flag "buffer_disabled" to be used in fast paths such as for
363 * the irqsoff tracer. But it may be inaccurate due to races. If you
364 * need to know the accurate state, use tracing_is_on() which is a little
365 * slower, but accurate.
367 int tracing_is_enabled(void)
370 * For quick access (irqsoff uses this in fast path), just
371 * return the mirror variable of the state of the ring buffer.
372 * It's a little racy, but we don't really care.
375 return !global_trace.buffer_disabled;
379 * trace_buf_size is the size in bytes that is allocated
380 * for a buffer. Note, the number of bytes is always rounded
383 * This number is purposely set to a low number of 16384.
384 * If the dump on oops happens, it will be much appreciated
385 * to not have to wait for all that output. Anyway this can be
386 * boot time and run time configurable.
388 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
390 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
392 /* trace_types holds a link list of available tracers. */
393 static struct tracer *trace_types __read_mostly;
396 * trace_types_lock is used to protect the trace_types list.
398 DEFINE_MUTEX(trace_types_lock);
401 * serialize the access of the ring buffer
403 * ring buffer serializes readers, but it is low level protection.
404 * The validity of the events (which returns by ring_buffer_peek() ..etc)
405 * are not protected by ring buffer.
407 * The content of events may become garbage if we allow other process consumes
408 * these events concurrently:
409 * A) the page of the consumed events may become a normal page
410 * (not reader page) in ring buffer, and this page will be rewrited
411 * by events producer.
412 * B) The page of the consumed events may become a page for splice_read,
413 * and this page will be returned to system.
415 * These primitives allow multi process access to different cpu ring buffer
418 * These primitives don't distinguish read-only and read-consume access.
419 * Multi read-only access are also serialized.
423 static DECLARE_RWSEM(all_cpu_access_lock);
424 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
426 static inline void trace_access_lock(int cpu)
428 if (cpu == RING_BUFFER_ALL_CPUS) {
429 /* gain it for accessing the whole ring buffer. */
430 down_write(&all_cpu_access_lock);
432 /* gain it for accessing a cpu ring buffer. */
434 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
435 down_read(&all_cpu_access_lock);
437 /* Secondly block other access to this @cpu ring buffer. */
438 mutex_lock(&per_cpu(cpu_access_lock, cpu));
442 static inline void trace_access_unlock(int cpu)
444 if (cpu == RING_BUFFER_ALL_CPUS) {
445 up_write(&all_cpu_access_lock);
447 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
448 up_read(&all_cpu_access_lock);
452 static inline void trace_access_lock_init(void)
456 for_each_possible_cpu(cpu)
457 mutex_init(&per_cpu(cpu_access_lock, cpu));
462 static DEFINE_MUTEX(access_lock);
464 static inline void trace_access_lock(int cpu)
467 mutex_lock(&access_lock);
470 static inline void trace_access_unlock(int cpu)
473 mutex_unlock(&access_lock);
476 static inline void trace_access_lock_init(void)
482 #ifdef CONFIG_STACKTRACE
483 static void __ftrace_trace_stack(struct ring_buffer *buffer,
485 int skip, int pc, struct pt_regs *regs);
486 static inline void ftrace_trace_stack(struct trace_array *tr,
487 struct ring_buffer *buffer,
489 int skip, int pc, struct pt_regs *regs);
492 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
494 int skip, int pc, struct pt_regs *regs)
497 static inline void ftrace_trace_stack(struct trace_array *tr,
498 struct ring_buffer *buffer,
500 int skip, int pc, struct pt_regs *regs)
506 static void tracer_tracing_on(struct trace_array *tr)
508 if (tr->trace_buffer.buffer)
509 ring_buffer_record_on(tr->trace_buffer.buffer);
511 * This flag is looked at when buffers haven't been allocated
512 * yet, or by some tracers (like irqsoff), that just want to
513 * know if the ring buffer has been disabled, but it can handle
514 * races of where it gets disabled but we still do a record.
515 * As the check is in the fast path of the tracers, it is more
516 * important to be fast than accurate.
518 tr->buffer_disabled = 0;
519 /* Make the flag seen by readers */
524 * tracing_on - enable tracing buffers
526 * This function enables tracing buffers that may have been
527 * disabled with tracing_off.
529 void tracing_on(void)
531 tracer_tracing_on(&global_trace);
533 EXPORT_SYMBOL_GPL(tracing_on);
536 * __trace_puts - write a constant string into the trace buffer.
537 * @ip: The address of the caller
538 * @str: The constant string to write
539 * @size: The size of the string.
541 int __trace_puts(unsigned long ip, const char *str, int size)
543 struct ring_buffer_event *event;
544 struct ring_buffer *buffer;
545 struct print_entry *entry;
546 unsigned long irq_flags;
550 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
553 pc = preempt_count();
555 if (unlikely(tracing_selftest_running || tracing_disabled))
558 alloc = sizeof(*entry) + size + 2; /* possible \n added */
560 local_save_flags(irq_flags);
561 buffer = global_trace.trace_buffer.buffer;
562 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
567 entry = ring_buffer_event_data(event);
570 memcpy(&entry->buf, str, size);
572 /* Add a newline if necessary */
573 if (entry->buf[size - 1] != '\n') {
574 entry->buf[size] = '\n';
575 entry->buf[size + 1] = '\0';
577 entry->buf[size] = '\0';
579 __buffer_unlock_commit(buffer, event);
580 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
584 EXPORT_SYMBOL_GPL(__trace_puts);
587 * __trace_bputs - write the pointer to a constant string into trace buffer
588 * @ip: The address of the caller
589 * @str: The constant string to write to the buffer to
591 int __trace_bputs(unsigned long ip, const char *str)
593 struct ring_buffer_event *event;
594 struct ring_buffer *buffer;
595 struct bputs_entry *entry;
596 unsigned long irq_flags;
597 int size = sizeof(struct bputs_entry);
600 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
603 pc = preempt_count();
605 if (unlikely(tracing_selftest_running || tracing_disabled))
608 local_save_flags(irq_flags);
609 buffer = global_trace.trace_buffer.buffer;
610 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
615 entry = ring_buffer_event_data(event);
619 __buffer_unlock_commit(buffer, event);
620 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
624 EXPORT_SYMBOL_GPL(__trace_bputs);
626 #ifdef CONFIG_TRACER_SNAPSHOT
628 * trace_snapshot - take a snapshot of the current buffer.
630 * This causes a swap between the snapshot buffer and the current live
631 * tracing buffer. You can use this to take snapshots of the live
632 * trace when some condition is triggered, but continue to trace.
634 * Note, make sure to allocate the snapshot with either
635 * a tracing_snapshot_alloc(), or by doing it manually
636 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
638 * If the snapshot buffer is not allocated, it will stop tracing.
639 * Basically making a permanent snapshot.
641 void tracing_snapshot(void)
643 struct trace_array *tr = &global_trace;
644 struct tracer *tracer = tr->current_trace;
648 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
649 internal_trace_puts("*** snapshot is being ignored ***\n");
653 if (!tr->allocated_snapshot) {
654 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
655 internal_trace_puts("*** stopping trace here! ***\n");
660 /* Note, snapshot can not be used when the tracer uses it */
661 if (tracer->use_max_tr) {
662 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
663 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
667 local_irq_save(flags);
668 update_max_tr(tr, current, smp_processor_id());
669 local_irq_restore(flags);
671 EXPORT_SYMBOL_GPL(tracing_snapshot);
673 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
674 struct trace_buffer *size_buf, int cpu_id);
675 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
677 static int alloc_snapshot(struct trace_array *tr)
681 if (!tr->allocated_snapshot) {
683 /* allocate spare buffer */
684 ret = resize_buffer_duplicate_size(&tr->max_buffer,
685 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
689 tr->allocated_snapshot = true;
695 static void free_snapshot(struct trace_array *tr)
698 * We don't free the ring buffer. instead, resize it because
699 * The max_tr ring buffer has some state (e.g. ring->clock) and
700 * we want preserve it.
702 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
703 set_buffer_entries(&tr->max_buffer, 1);
704 tracing_reset_online_cpus(&tr->max_buffer);
705 tr->allocated_snapshot = false;
709 * tracing_alloc_snapshot - allocate snapshot buffer.
711 * This only allocates the snapshot buffer if it isn't already
712 * allocated - it doesn't also take a snapshot.
714 * This is meant to be used in cases where the snapshot buffer needs
715 * to be set up for events that can't sleep but need to be able to
716 * trigger a snapshot.
718 int tracing_alloc_snapshot(void)
720 struct trace_array *tr = &global_trace;
723 ret = alloc_snapshot(tr);
728 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
731 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
733 * This is similar to trace_snapshot(), but it will allocate the
734 * snapshot buffer if it isn't already allocated. Use this only
735 * where it is safe to sleep, as the allocation may sleep.
737 * This causes a swap between the snapshot buffer and the current live
738 * tracing buffer. You can use this to take snapshots of the live
739 * trace when some condition is triggered, but continue to trace.
741 void tracing_snapshot_alloc(void)
745 ret = tracing_alloc_snapshot();
751 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
753 void tracing_snapshot(void)
755 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
757 EXPORT_SYMBOL_GPL(tracing_snapshot);
758 int tracing_alloc_snapshot(void)
760 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
763 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
764 void tracing_snapshot_alloc(void)
769 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
770 #endif /* CONFIG_TRACER_SNAPSHOT */
772 static void tracer_tracing_off(struct trace_array *tr)
774 if (tr->trace_buffer.buffer)
775 ring_buffer_record_off(tr->trace_buffer.buffer);
777 * This flag is looked at when buffers haven't been allocated
778 * yet, or by some tracers (like irqsoff), that just want to
779 * know if the ring buffer has been disabled, but it can handle
780 * races of where it gets disabled but we still do a record.
781 * As the check is in the fast path of the tracers, it is more
782 * important to be fast than accurate.
784 tr->buffer_disabled = 1;
785 /* Make the flag seen by readers */
790 * tracing_off - turn off tracing buffers
792 * This function stops the tracing buffers from recording data.
793 * It does not disable any overhead the tracers themselves may
794 * be causing. This function simply causes all recording to
795 * the ring buffers to fail.
797 void tracing_off(void)
799 tracer_tracing_off(&global_trace);
801 EXPORT_SYMBOL_GPL(tracing_off);
803 void disable_trace_on_warning(void)
805 if (__disable_trace_on_warning)
810 * tracer_tracing_is_on - show real state of ring buffer enabled
811 * @tr : the trace array to know if ring buffer is enabled
813 * Shows real state of the ring buffer if it is enabled or not.
815 static int tracer_tracing_is_on(struct trace_array *tr)
817 if (tr->trace_buffer.buffer)
818 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
819 return !tr->buffer_disabled;
823 * tracing_is_on - show state of ring buffers enabled
825 int tracing_is_on(void)
827 return tracer_tracing_is_on(&global_trace);
829 EXPORT_SYMBOL_GPL(tracing_is_on);
831 static int __init set_buf_size(char *str)
833 unsigned long buf_size;
837 buf_size = memparse(str, &str);
838 /* nr_entries can not be zero */
841 trace_buf_size = buf_size;
844 __setup("trace_buf_size=", set_buf_size);
846 static int __init set_tracing_thresh(char *str)
848 unsigned long threshold;
853 ret = kstrtoul(str, 0, &threshold);
856 tracing_thresh = threshold * 1000;
859 __setup("tracing_thresh=", set_tracing_thresh);
861 unsigned long nsecs_to_usecs(unsigned long nsecs)
867 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
868 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
869 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
870 * of strings in the order that the enums were defined.
875 /* These must match the bit postions in trace_iterator_flags */
876 static const char *trace_options[] = {
884 int in_ns; /* is this clock in nanoseconds? */
886 { trace_clock_local, "local", 1 },
887 { trace_clock_global, "global", 1 },
888 { trace_clock_counter, "counter", 0 },
889 { trace_clock_jiffies, "uptime", 0 },
890 { trace_clock, "perf", 1 },
891 { ktime_get_mono_fast_ns, "mono", 1 },
892 { ktime_get_raw_fast_ns, "mono_raw", 1 },
897 * trace_parser_get_init - gets the buffer for trace parser
899 int trace_parser_get_init(struct trace_parser *parser, int size)
901 memset(parser, 0, sizeof(*parser));
903 parser->buffer = kmalloc(size, GFP_KERNEL);
912 * trace_parser_put - frees the buffer for trace parser
914 void trace_parser_put(struct trace_parser *parser)
916 kfree(parser->buffer);
920 * trace_get_user - reads the user input string separated by space
921 * (matched by isspace(ch))
923 * For each string found the 'struct trace_parser' is updated,
924 * and the function returns.
926 * Returns number of bytes read.
928 * See kernel/trace/trace.h for 'struct trace_parser' details.
930 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
931 size_t cnt, loff_t *ppos)
938 trace_parser_clear(parser);
940 ret = get_user(ch, ubuf++);
948 * The parser is not finished with the last write,
949 * continue reading the user input without skipping spaces.
952 /* skip white space */
953 while (cnt && isspace(ch)) {
954 ret = get_user(ch, ubuf++);
961 /* only spaces were written */
971 /* read the non-space input */
972 while (cnt && !isspace(ch)) {
973 if (parser->idx < parser->size - 1)
974 parser->buffer[parser->idx++] = ch;
979 ret = get_user(ch, ubuf++);
986 /* We either got finished input or we have to wait for another call. */
988 parser->buffer[parser->idx] = 0;
989 parser->cont = false;
990 } else if (parser->idx < parser->size - 1) {
992 parser->buffer[parser->idx++] = ch;
1005 /* TODO add a seq_buf_to_buffer() */
1006 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1010 if (trace_seq_used(s) <= s->seq.readpos)
1013 len = trace_seq_used(s) - s->seq.readpos;
1016 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1018 s->seq.readpos += cnt;
1022 unsigned long __read_mostly tracing_thresh;
1024 #ifdef CONFIG_TRACER_MAX_TRACE
1026 * Copy the new maximum trace into the separate maximum-trace
1027 * structure. (this way the maximum trace is permanently saved,
1028 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1031 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1033 struct trace_buffer *trace_buf = &tr->trace_buffer;
1034 struct trace_buffer *max_buf = &tr->max_buffer;
1035 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1036 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1039 max_buf->time_start = data->preempt_timestamp;
1041 max_data->saved_latency = tr->max_latency;
1042 max_data->critical_start = data->critical_start;
1043 max_data->critical_end = data->critical_end;
1045 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1046 max_data->pid = tsk->pid;
1048 * If tsk == current, then use current_uid(), as that does not use
1049 * RCU. The irq tracer can be called out of RCU scope.
1052 max_data->uid = current_uid();
1054 max_data->uid = task_uid(tsk);
1056 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1057 max_data->policy = tsk->policy;
1058 max_data->rt_priority = tsk->rt_priority;
1060 /* record this tasks comm */
1061 tracing_record_cmdline(tsk);
1065 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1067 * @tsk: the task with the latency
1068 * @cpu: The cpu that initiated the trace.
1070 * Flip the buffers between the @tr and the max_tr and record information
1071 * about which task was the cause of this latency.
1074 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1076 struct ring_buffer *buf;
1081 WARN_ON_ONCE(!irqs_disabled());
1083 if (!tr->allocated_snapshot) {
1084 /* Only the nop tracer should hit this when disabling */
1085 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1089 arch_spin_lock(&tr->max_lock);
1091 buf = tr->trace_buffer.buffer;
1092 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1093 tr->max_buffer.buffer = buf;
1095 __update_max_tr(tr, tsk, cpu);
1096 arch_spin_unlock(&tr->max_lock);
1100 * update_max_tr_single - only copy one trace over, and reset the rest
1102 * @tsk - task with the latency
1103 * @cpu - the cpu of the buffer to copy.
1105 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1108 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1115 WARN_ON_ONCE(!irqs_disabled());
1116 if (!tr->allocated_snapshot) {
1117 /* Only the nop tracer should hit this when disabling */
1118 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1122 arch_spin_lock(&tr->max_lock);
1124 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1126 if (ret == -EBUSY) {
1128 * We failed to swap the buffer due to a commit taking
1129 * place on this CPU. We fail to record, but we reset
1130 * the max trace buffer (no one writes directly to it)
1131 * and flag that it failed.
1133 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1134 "Failed to swap buffers due to commit in progress\n");
1137 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1139 __update_max_tr(tr, tsk, cpu);
1140 arch_spin_unlock(&tr->max_lock);
1142 #endif /* CONFIG_TRACER_MAX_TRACE */
1144 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1146 /* Iterators are static, they should be filled or empty */
1147 if (trace_buffer_iter(iter, iter->cpu_file))
1150 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1154 #ifdef CONFIG_FTRACE_STARTUP_TEST
1155 static int run_tracer_selftest(struct tracer *type)
1157 struct trace_array *tr = &global_trace;
1158 struct tracer *saved_tracer = tr->current_trace;
1161 if (!type->selftest || tracing_selftest_disabled)
1165 * Run a selftest on this tracer.
1166 * Here we reset the trace buffer, and set the current
1167 * tracer to be this tracer. The tracer can then run some
1168 * internal tracing to verify that everything is in order.
1169 * If we fail, we do not register this tracer.
1171 tracing_reset_online_cpus(&tr->trace_buffer);
1173 tr->current_trace = type;
1175 #ifdef CONFIG_TRACER_MAX_TRACE
1176 if (type->use_max_tr) {
1177 /* If we expanded the buffers, make sure the max is expanded too */
1178 if (ring_buffer_expanded)
1179 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1180 RING_BUFFER_ALL_CPUS);
1181 tr->allocated_snapshot = true;
1185 /* the test is responsible for initializing and enabling */
1186 pr_info("Testing tracer %s: ", type->name);
1187 ret = type->selftest(type, tr);
1188 /* the test is responsible for resetting too */
1189 tr->current_trace = saved_tracer;
1191 printk(KERN_CONT "FAILED!\n");
1192 /* Add the warning after printing 'FAILED' */
1196 /* Only reset on passing, to avoid touching corrupted buffers */
1197 tracing_reset_online_cpus(&tr->trace_buffer);
1199 #ifdef CONFIG_TRACER_MAX_TRACE
1200 if (type->use_max_tr) {
1201 tr->allocated_snapshot = false;
1203 /* Shrink the max buffer again */
1204 if (ring_buffer_expanded)
1205 ring_buffer_resize(tr->max_buffer.buffer, 1,
1206 RING_BUFFER_ALL_CPUS);
1210 printk(KERN_CONT "PASSED\n");
1214 static inline int run_tracer_selftest(struct tracer *type)
1218 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1220 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1222 static void __init apply_trace_boot_options(void);
1225 * register_tracer - register a tracer with the ftrace system.
1226 * @type - the plugin for the tracer
1228 * Register a new plugin tracer.
1230 int __init register_tracer(struct tracer *type)
1236 pr_info("Tracer must have a name\n");
1240 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1241 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1245 mutex_lock(&trace_types_lock);
1247 tracing_selftest_running = true;
1249 for (t = trace_types; t; t = t->next) {
1250 if (strcmp(type->name, t->name) == 0) {
1252 pr_info("Tracer %s already registered\n",
1259 if (!type->set_flag)
1260 type->set_flag = &dummy_set_flag;
1262 type->flags = &dummy_tracer_flags;
1264 if (!type->flags->opts)
1265 type->flags->opts = dummy_tracer_opt;
1267 ret = run_tracer_selftest(type);
1271 type->next = trace_types;
1273 add_tracer_options(&global_trace, type);
1276 tracing_selftest_running = false;
1277 mutex_unlock(&trace_types_lock);
1279 if (ret || !default_bootup_tracer)
1282 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1285 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1286 /* Do we want this tracer to start on bootup? */
1287 tracing_set_tracer(&global_trace, type->name);
1288 default_bootup_tracer = NULL;
1290 apply_trace_boot_options();
1292 /* disable other selftests, since this will break it. */
1293 tracing_selftest_disabled = true;
1294 #ifdef CONFIG_FTRACE_STARTUP_TEST
1295 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1303 void tracing_reset(struct trace_buffer *buf, int cpu)
1305 struct ring_buffer *buffer = buf->buffer;
1310 ring_buffer_record_disable(buffer);
1312 /* Make sure all commits have finished */
1313 synchronize_sched();
1314 ring_buffer_reset_cpu(buffer, cpu);
1316 ring_buffer_record_enable(buffer);
1319 void tracing_reset_online_cpus(struct trace_buffer *buf)
1321 struct ring_buffer *buffer = buf->buffer;
1327 ring_buffer_record_disable(buffer);
1329 /* Make sure all commits have finished */
1330 synchronize_sched();
1332 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1334 for_each_online_cpu(cpu)
1335 ring_buffer_reset_cpu(buffer, cpu);
1337 ring_buffer_record_enable(buffer);
1340 /* Must have trace_types_lock held */
1341 void tracing_reset_all_online_cpus(void)
1343 struct trace_array *tr;
1345 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1346 tracing_reset_online_cpus(&tr->trace_buffer);
1347 #ifdef CONFIG_TRACER_MAX_TRACE
1348 tracing_reset_online_cpus(&tr->max_buffer);
1353 #define SAVED_CMDLINES_DEFAULT 128
1354 #define NO_CMDLINE_MAP UINT_MAX
1355 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1356 struct saved_cmdlines_buffer {
1357 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1358 unsigned *map_cmdline_to_pid;
1359 unsigned cmdline_num;
1361 char *saved_cmdlines;
1363 static struct saved_cmdlines_buffer *savedcmd;
1365 /* temporary disable recording */
1366 static atomic_t trace_record_cmdline_disabled __read_mostly;
1368 static inline char *get_saved_cmdlines(int idx)
1370 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1373 static inline void set_cmdline(int idx, const char *cmdline)
1375 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1378 static int allocate_cmdlines_buffer(unsigned int val,
1379 struct saved_cmdlines_buffer *s)
1381 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1383 if (!s->map_cmdline_to_pid)
1386 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1387 if (!s->saved_cmdlines) {
1388 kfree(s->map_cmdline_to_pid);
1393 s->cmdline_num = val;
1394 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1395 sizeof(s->map_pid_to_cmdline));
1396 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1397 val * sizeof(*s->map_cmdline_to_pid));
1402 static int trace_create_savedcmd(void)
1406 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1410 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1420 int is_tracing_stopped(void)
1422 return global_trace.stop_count;
1426 * tracing_start - quick start of the tracer
1428 * If tracing is enabled but was stopped by tracing_stop,
1429 * this will start the tracer back up.
1431 void tracing_start(void)
1433 struct ring_buffer *buffer;
1434 unsigned long flags;
1436 if (tracing_disabled)
1439 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1440 if (--global_trace.stop_count) {
1441 if (global_trace.stop_count < 0) {
1442 /* Someone screwed up their debugging */
1444 global_trace.stop_count = 0;
1449 /* Prevent the buffers from switching */
1450 arch_spin_lock(&global_trace.max_lock);
1452 buffer = global_trace.trace_buffer.buffer;
1454 ring_buffer_record_enable(buffer);
1456 #ifdef CONFIG_TRACER_MAX_TRACE
1457 buffer = global_trace.max_buffer.buffer;
1459 ring_buffer_record_enable(buffer);
1462 arch_spin_unlock(&global_trace.max_lock);
1465 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1468 static void tracing_start_tr(struct trace_array *tr)
1470 struct ring_buffer *buffer;
1471 unsigned long flags;
1473 if (tracing_disabled)
1476 /* If global, we need to also start the max tracer */
1477 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1478 return tracing_start();
1480 raw_spin_lock_irqsave(&tr->start_lock, flags);
1482 if (--tr->stop_count) {
1483 if (tr->stop_count < 0) {
1484 /* Someone screwed up their debugging */
1491 buffer = tr->trace_buffer.buffer;
1493 ring_buffer_record_enable(buffer);
1496 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1500 * tracing_stop - quick stop of the tracer
1502 * Light weight way to stop tracing. Use in conjunction with
1505 void tracing_stop(void)
1507 struct ring_buffer *buffer;
1508 unsigned long flags;
1510 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1511 if (global_trace.stop_count++)
1514 /* Prevent the buffers from switching */
1515 arch_spin_lock(&global_trace.max_lock);
1517 buffer = global_trace.trace_buffer.buffer;
1519 ring_buffer_record_disable(buffer);
1521 #ifdef CONFIG_TRACER_MAX_TRACE
1522 buffer = global_trace.max_buffer.buffer;
1524 ring_buffer_record_disable(buffer);
1527 arch_spin_unlock(&global_trace.max_lock);
1530 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1533 static void tracing_stop_tr(struct trace_array *tr)
1535 struct ring_buffer *buffer;
1536 unsigned long flags;
1538 /* If global, we need to also stop the max tracer */
1539 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1540 return tracing_stop();
1542 raw_spin_lock_irqsave(&tr->start_lock, flags);
1543 if (tr->stop_count++)
1546 buffer = tr->trace_buffer.buffer;
1548 ring_buffer_record_disable(buffer);
1551 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1554 void trace_stop_cmdline_recording(void);
1556 static int trace_save_cmdline(struct task_struct *tsk)
1560 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1564 * It's not the end of the world if we don't get
1565 * the lock, but we also don't want to spin
1566 * nor do we want to disable interrupts,
1567 * so if we miss here, then better luck next time.
1569 if (!arch_spin_trylock(&trace_cmdline_lock))
1572 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1573 if (idx == NO_CMDLINE_MAP) {
1574 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1577 * Check whether the cmdline buffer at idx has a pid
1578 * mapped. We are going to overwrite that entry so we
1579 * need to clear the map_pid_to_cmdline. Otherwise we
1580 * would read the new comm for the old pid.
1582 pid = savedcmd->map_cmdline_to_pid[idx];
1583 if (pid != NO_CMDLINE_MAP)
1584 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1586 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1587 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1589 savedcmd->cmdline_idx = idx;
1592 set_cmdline(idx, tsk->comm);
1594 arch_spin_unlock(&trace_cmdline_lock);
1599 static void __trace_find_cmdline(int pid, char comm[])
1604 strcpy(comm, "<idle>");
1608 if (WARN_ON_ONCE(pid < 0)) {
1609 strcpy(comm, "<XXX>");
1613 if (pid > PID_MAX_DEFAULT) {
1614 strcpy(comm, "<...>");
1618 map = savedcmd->map_pid_to_cmdline[pid];
1619 if (map != NO_CMDLINE_MAP)
1620 strcpy(comm, get_saved_cmdlines(map));
1622 strcpy(comm, "<...>");
1625 void trace_find_cmdline(int pid, char comm[])
1628 arch_spin_lock(&trace_cmdline_lock);
1630 __trace_find_cmdline(pid, comm);
1632 arch_spin_unlock(&trace_cmdline_lock);
1636 void tracing_record_cmdline(struct task_struct *tsk)
1638 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1641 if (!__this_cpu_read(trace_cmdline_save))
1644 if (trace_save_cmdline(tsk))
1645 __this_cpu_write(trace_cmdline_save, false);
1649 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1652 struct task_struct *tsk = current;
1654 entry->preempt_count = pc & 0xff;
1655 entry->pid = (tsk) ? tsk->pid : 0;
1657 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1658 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1660 TRACE_FLAG_IRQS_NOSUPPORT |
1662 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1663 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1664 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1665 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1667 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1669 struct ring_buffer_event *
1670 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1673 unsigned long flags, int pc)
1675 struct ring_buffer_event *event;
1677 event = ring_buffer_lock_reserve(buffer, len);
1678 if (event != NULL) {
1679 struct trace_entry *ent = ring_buffer_event_data(event);
1681 tracing_generic_entry_update(ent, flags, pc);
1689 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1691 __this_cpu_write(trace_cmdline_save, true);
1692 ring_buffer_unlock_commit(buffer, event);
1695 void trace_buffer_unlock_commit(struct trace_array *tr,
1696 struct ring_buffer *buffer,
1697 struct ring_buffer_event *event,
1698 unsigned long flags, int pc)
1700 __buffer_unlock_commit(buffer, event);
1702 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
1703 ftrace_trace_userstack(buffer, flags, pc);
1705 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1707 static struct ring_buffer *temp_buffer;
1709 struct ring_buffer_event *
1710 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1711 struct trace_event_file *trace_file,
1712 int type, unsigned long len,
1713 unsigned long flags, int pc)
1715 struct ring_buffer_event *entry;
1717 *current_rb = trace_file->tr->trace_buffer.buffer;
1718 entry = trace_buffer_lock_reserve(*current_rb,
1719 type, len, flags, pc);
1721 * If tracing is off, but we have triggers enabled
1722 * we still need to look at the event data. Use the temp_buffer
1723 * to store the trace event for the tigger to use. It's recusive
1724 * safe and will not be recorded anywhere.
1726 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1727 *current_rb = temp_buffer;
1728 entry = trace_buffer_lock_reserve(*current_rb,
1729 type, len, flags, pc);
1733 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1735 struct ring_buffer_event *
1736 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1737 int type, unsigned long len,
1738 unsigned long flags, int pc)
1740 *current_rb = global_trace.trace_buffer.buffer;
1741 return trace_buffer_lock_reserve(*current_rb,
1742 type, len, flags, pc);
1744 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1746 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1747 struct ring_buffer *buffer,
1748 struct ring_buffer_event *event,
1749 unsigned long flags, int pc,
1750 struct pt_regs *regs)
1752 __buffer_unlock_commit(buffer, event);
1754 ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
1755 ftrace_trace_userstack(buffer, flags, pc);
1757 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1759 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1760 struct ring_buffer_event *event)
1762 ring_buffer_discard_commit(buffer, event);
1764 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1767 trace_function(struct trace_array *tr,
1768 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1771 struct trace_event_call *call = &event_function;
1772 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1773 struct ring_buffer_event *event;
1774 struct ftrace_entry *entry;
1776 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1780 entry = ring_buffer_event_data(event);
1782 entry->parent_ip = parent_ip;
1784 if (!call_filter_check_discard(call, entry, buffer, event))
1785 __buffer_unlock_commit(buffer, event);
1788 #ifdef CONFIG_STACKTRACE
1790 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1791 struct ftrace_stack {
1792 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1795 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1796 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1798 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1799 unsigned long flags,
1800 int skip, int pc, struct pt_regs *regs)
1802 struct trace_event_call *call = &event_kernel_stack;
1803 struct ring_buffer_event *event;
1804 struct stack_entry *entry;
1805 struct stack_trace trace;
1807 int size = FTRACE_STACK_ENTRIES;
1809 trace.nr_entries = 0;
1813 * Since events can happen in NMIs there's no safe way to
1814 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1815 * or NMI comes in, it will just have to use the default
1816 * FTRACE_STACK_SIZE.
1818 preempt_disable_notrace();
1820 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1822 * We don't need any atomic variables, just a barrier.
1823 * If an interrupt comes in, we don't care, because it would
1824 * have exited and put the counter back to what we want.
1825 * We just need a barrier to keep gcc from moving things
1829 if (use_stack == 1) {
1830 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1831 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1834 save_stack_trace_regs(regs, &trace);
1836 save_stack_trace(&trace);
1838 if (trace.nr_entries > size)
1839 size = trace.nr_entries;
1841 /* From now on, use_stack is a boolean */
1844 size *= sizeof(unsigned long);
1846 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1847 sizeof(*entry) + size, flags, pc);
1850 entry = ring_buffer_event_data(event);
1852 memset(&entry->caller, 0, size);
1855 memcpy(&entry->caller, trace.entries,
1856 trace.nr_entries * sizeof(unsigned long));
1858 trace.max_entries = FTRACE_STACK_ENTRIES;
1859 trace.entries = entry->caller;
1861 save_stack_trace_regs(regs, &trace);
1863 save_stack_trace(&trace);
1866 entry->size = trace.nr_entries;
1868 if (!call_filter_check_discard(call, entry, buffer, event))
1869 __buffer_unlock_commit(buffer, event);
1872 /* Again, don't let gcc optimize things here */
1874 __this_cpu_dec(ftrace_stack_reserve);
1875 preempt_enable_notrace();
1879 static inline void ftrace_trace_stack(struct trace_array *tr,
1880 struct ring_buffer *buffer,
1881 unsigned long flags,
1882 int skip, int pc, struct pt_regs *regs)
1884 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
1887 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1890 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1893 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1897 * trace_dump_stack - record a stack back trace in the trace buffer
1898 * @skip: Number of functions to skip (helper handlers)
1900 void trace_dump_stack(int skip)
1902 unsigned long flags;
1904 if (tracing_disabled || tracing_selftest_running)
1907 local_save_flags(flags);
1910 * Skip 3 more, seems to get us at the caller of
1914 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1915 flags, skip, preempt_count(), NULL);
1918 static DEFINE_PER_CPU(int, user_stack_count);
1921 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1923 struct trace_event_call *call = &event_user_stack;
1924 struct ring_buffer_event *event;
1925 struct userstack_entry *entry;
1926 struct stack_trace trace;
1928 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
1932 * NMIs can not handle page faults, even with fix ups.
1933 * The save user stack can (and often does) fault.
1935 if (unlikely(in_nmi()))
1939 * prevent recursion, since the user stack tracing may
1940 * trigger other kernel events.
1943 if (__this_cpu_read(user_stack_count))
1946 __this_cpu_inc(user_stack_count);
1948 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1949 sizeof(*entry), flags, pc);
1951 goto out_drop_count;
1952 entry = ring_buffer_event_data(event);
1954 entry->tgid = current->tgid;
1955 memset(&entry->caller, 0, sizeof(entry->caller));
1957 trace.nr_entries = 0;
1958 trace.max_entries = FTRACE_STACK_ENTRIES;
1960 trace.entries = entry->caller;
1962 save_stack_trace_user(&trace);
1963 if (!call_filter_check_discard(call, entry, buffer, event))
1964 __buffer_unlock_commit(buffer, event);
1967 __this_cpu_dec(user_stack_count);
1973 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1975 ftrace_trace_userstack(tr, flags, preempt_count());
1979 #endif /* CONFIG_STACKTRACE */
1981 /* created for use with alloc_percpu */
1982 struct trace_buffer_struct {
1983 char buffer[TRACE_BUF_SIZE];
1986 static struct trace_buffer_struct *trace_percpu_buffer;
1987 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1988 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1989 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1992 * The buffer used is dependent on the context. There is a per cpu
1993 * buffer for normal context, softirq contex, hard irq context and
1994 * for NMI context. Thise allows for lockless recording.
1996 * Note, if the buffers failed to be allocated, then this returns NULL
1998 static char *get_trace_buf(void)
2000 struct trace_buffer_struct *percpu_buffer;
2003 * If we have allocated per cpu buffers, then we do not
2004 * need to do any locking.
2007 percpu_buffer = trace_percpu_nmi_buffer;
2009 percpu_buffer = trace_percpu_irq_buffer;
2010 else if (in_softirq())
2011 percpu_buffer = trace_percpu_sirq_buffer;
2013 percpu_buffer = trace_percpu_buffer;
2018 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2021 static int alloc_percpu_trace_buffer(void)
2023 struct trace_buffer_struct *buffers;
2024 struct trace_buffer_struct *sirq_buffers;
2025 struct trace_buffer_struct *irq_buffers;
2026 struct trace_buffer_struct *nmi_buffers;
2028 buffers = alloc_percpu(struct trace_buffer_struct);
2032 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2036 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2040 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2044 trace_percpu_buffer = buffers;
2045 trace_percpu_sirq_buffer = sirq_buffers;
2046 trace_percpu_irq_buffer = irq_buffers;
2047 trace_percpu_nmi_buffer = nmi_buffers;
2052 free_percpu(irq_buffers);
2054 free_percpu(sirq_buffers);
2056 free_percpu(buffers);
2058 WARN(1, "Could not allocate percpu trace_printk buffer");
2062 static int buffers_allocated;
2064 void trace_printk_init_buffers(void)
2066 if (buffers_allocated)
2069 if (alloc_percpu_trace_buffer())
2072 /* trace_printk() is for debug use only. Don't use it in production. */
2075 pr_warning("**********************************************************\n");
2076 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2077 pr_warning("** **\n");
2078 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2079 pr_warning("** **\n");
2080 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2081 pr_warning("** unsafe for production use. **\n");
2082 pr_warning("** **\n");
2083 pr_warning("** If you see this message and you are not debugging **\n");
2084 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2085 pr_warning("** **\n");
2086 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2087 pr_warning("**********************************************************\n");
2089 /* Expand the buffers to set size */
2090 tracing_update_buffers();
2092 buffers_allocated = 1;
2095 * trace_printk_init_buffers() can be called by modules.
2096 * If that happens, then we need to start cmdline recording
2097 * directly here. If the global_trace.buffer is already
2098 * allocated here, then this was called by module code.
2100 if (global_trace.trace_buffer.buffer)
2101 tracing_start_cmdline_record();
2104 void trace_printk_start_comm(void)
2106 /* Start tracing comms if trace printk is set */
2107 if (!buffers_allocated)
2109 tracing_start_cmdline_record();
2112 static void trace_printk_start_stop_comm(int enabled)
2114 if (!buffers_allocated)
2118 tracing_start_cmdline_record();
2120 tracing_stop_cmdline_record();
2124 * trace_vbprintk - write binary msg to tracing buffer
2127 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2129 struct trace_event_call *call = &event_bprint;
2130 struct ring_buffer_event *event;
2131 struct ring_buffer *buffer;
2132 struct trace_array *tr = &global_trace;
2133 struct bprint_entry *entry;
2134 unsigned long flags;
2136 int len = 0, size, pc;
2138 if (unlikely(tracing_selftest_running || tracing_disabled))
2141 /* Don't pollute graph traces with trace_vprintk internals */
2142 pause_graph_tracing();
2144 pc = preempt_count();
2145 preempt_disable_notrace();
2147 tbuffer = get_trace_buf();
2153 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2155 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2158 local_save_flags(flags);
2159 size = sizeof(*entry) + sizeof(u32) * len;
2160 buffer = tr->trace_buffer.buffer;
2161 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2165 entry = ring_buffer_event_data(event);
2169 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2170 if (!call_filter_check_discard(call, entry, buffer, event)) {
2171 __buffer_unlock_commit(buffer, event);
2172 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2176 preempt_enable_notrace();
2177 unpause_graph_tracing();
2181 EXPORT_SYMBOL_GPL(trace_vbprintk);
2184 __trace_array_vprintk(struct ring_buffer *buffer,
2185 unsigned long ip, const char *fmt, va_list args)
2187 struct trace_event_call *call = &event_print;
2188 struct ring_buffer_event *event;
2189 int len = 0, size, pc;
2190 struct print_entry *entry;
2191 unsigned long flags;
2194 if (tracing_disabled || tracing_selftest_running)
2197 /* Don't pollute graph traces with trace_vprintk internals */
2198 pause_graph_tracing();
2200 pc = preempt_count();
2201 preempt_disable_notrace();
2204 tbuffer = get_trace_buf();
2210 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2212 local_save_flags(flags);
2213 size = sizeof(*entry) + len + 1;
2214 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2218 entry = ring_buffer_event_data(event);
2221 memcpy(&entry->buf, tbuffer, len + 1);
2222 if (!call_filter_check_discard(call, entry, buffer, event)) {
2223 __buffer_unlock_commit(buffer, event);
2224 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2227 preempt_enable_notrace();
2228 unpause_graph_tracing();
2233 int trace_array_vprintk(struct trace_array *tr,
2234 unsigned long ip, const char *fmt, va_list args)
2236 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2239 int trace_array_printk(struct trace_array *tr,
2240 unsigned long ip, const char *fmt, ...)
2245 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2249 ret = trace_array_vprintk(tr, ip, fmt, ap);
2254 int trace_array_printk_buf(struct ring_buffer *buffer,
2255 unsigned long ip, const char *fmt, ...)
2260 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2264 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2269 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2271 return trace_array_vprintk(&global_trace, ip, fmt, args);
2273 EXPORT_SYMBOL_GPL(trace_vprintk);
2275 static void trace_iterator_increment(struct trace_iterator *iter)
2277 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2281 ring_buffer_read(buf_iter, NULL);
2284 static struct trace_entry *
2285 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2286 unsigned long *lost_events)
2288 struct ring_buffer_event *event;
2289 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2292 event = ring_buffer_iter_peek(buf_iter, ts);
2294 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2298 iter->ent_size = ring_buffer_event_length(event);
2299 return ring_buffer_event_data(event);
2305 static struct trace_entry *
2306 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2307 unsigned long *missing_events, u64 *ent_ts)
2309 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2310 struct trace_entry *ent, *next = NULL;
2311 unsigned long lost_events = 0, next_lost = 0;
2312 int cpu_file = iter->cpu_file;
2313 u64 next_ts = 0, ts;
2319 * If we are in a per_cpu trace file, don't bother by iterating over
2320 * all cpu and peek directly.
2322 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2323 if (ring_buffer_empty_cpu(buffer, cpu_file))
2325 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2327 *ent_cpu = cpu_file;
2332 for_each_tracing_cpu(cpu) {
2334 if (ring_buffer_empty_cpu(buffer, cpu))
2337 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2340 * Pick the entry with the smallest timestamp:
2342 if (ent && (!next || ts < next_ts)) {
2346 next_lost = lost_events;
2347 next_size = iter->ent_size;
2351 iter->ent_size = next_size;
2354 *ent_cpu = next_cpu;
2360 *missing_events = next_lost;
2365 /* Find the next real entry, without updating the iterator itself */
2366 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2367 int *ent_cpu, u64 *ent_ts)
2369 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2372 /* Find the next real entry, and increment the iterator to the next entry */
2373 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2375 iter->ent = __find_next_entry(iter, &iter->cpu,
2376 &iter->lost_events, &iter->ts);
2379 trace_iterator_increment(iter);
2381 return iter->ent ? iter : NULL;
2384 static void trace_consume(struct trace_iterator *iter)
2386 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2387 &iter->lost_events);
2390 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2392 struct trace_iterator *iter = m->private;
2396 WARN_ON_ONCE(iter->leftover);
2400 /* can't go backwards */
2405 ent = trace_find_next_entry_inc(iter);
2409 while (ent && iter->idx < i)
2410 ent = trace_find_next_entry_inc(iter);
2417 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2419 struct ring_buffer_event *event;
2420 struct ring_buffer_iter *buf_iter;
2421 unsigned long entries = 0;
2424 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2426 buf_iter = trace_buffer_iter(iter, cpu);
2430 ring_buffer_iter_reset(buf_iter);
2433 * We could have the case with the max latency tracers
2434 * that a reset never took place on a cpu. This is evident
2435 * by the timestamp being before the start of the buffer.
2437 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2438 if (ts >= iter->trace_buffer->time_start)
2441 ring_buffer_read(buf_iter, NULL);
2444 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2448 * The current tracer is copied to avoid a global locking
2451 static void *s_start(struct seq_file *m, loff_t *pos)
2453 struct trace_iterator *iter = m->private;
2454 struct trace_array *tr = iter->tr;
2455 int cpu_file = iter->cpu_file;
2461 * copy the tracer to avoid using a global lock all around.
2462 * iter->trace is a copy of current_trace, the pointer to the
2463 * name may be used instead of a strcmp(), as iter->trace->name
2464 * will point to the same string as current_trace->name.
2466 mutex_lock(&trace_types_lock);
2467 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2468 *iter->trace = *tr->current_trace;
2469 mutex_unlock(&trace_types_lock);
2471 #ifdef CONFIG_TRACER_MAX_TRACE
2472 if (iter->snapshot && iter->trace->use_max_tr)
2473 return ERR_PTR(-EBUSY);
2476 if (!iter->snapshot)
2477 atomic_inc(&trace_record_cmdline_disabled);
2479 if (*pos != iter->pos) {
2484 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2485 for_each_tracing_cpu(cpu)
2486 tracing_iter_reset(iter, cpu);
2488 tracing_iter_reset(iter, cpu_file);
2491 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2496 * If we overflowed the seq_file before, then we want
2497 * to just reuse the trace_seq buffer again.
2503 p = s_next(m, p, &l);
2507 trace_event_read_lock();
2508 trace_access_lock(cpu_file);
2512 static void s_stop(struct seq_file *m, void *p)
2514 struct trace_iterator *iter = m->private;
2516 #ifdef CONFIG_TRACER_MAX_TRACE
2517 if (iter->snapshot && iter->trace->use_max_tr)
2521 if (!iter->snapshot)
2522 atomic_dec(&trace_record_cmdline_disabled);
2524 trace_access_unlock(iter->cpu_file);
2525 trace_event_read_unlock();
2529 get_total_entries(struct trace_buffer *buf,
2530 unsigned long *total, unsigned long *entries)
2532 unsigned long count;
2538 for_each_tracing_cpu(cpu) {
2539 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2541 * If this buffer has skipped entries, then we hold all
2542 * entries for the trace and we need to ignore the
2543 * ones before the time stamp.
2545 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2546 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2547 /* total is the same as the entries */
2551 ring_buffer_overrun_cpu(buf->buffer, cpu);
2556 static void print_lat_help_header(struct seq_file *m)
2558 seq_puts(m, "# _------=> CPU# \n"
2559 "# / _-----=> irqs-off \n"
2560 "# | / _----=> need-resched \n"
2561 "# || / _---=> hardirq/softirq \n"
2562 "# ||| / _--=> preempt-depth \n"
2564 "# cmd pid ||||| time | caller \n"
2565 "# \\ / ||||| \\ | / \n");
2568 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2570 unsigned long total;
2571 unsigned long entries;
2573 get_total_entries(buf, &total, &entries);
2574 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2575 entries, total, num_online_cpus());
2579 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2581 print_event_info(buf, m);
2582 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2586 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2588 print_event_info(buf, m);
2589 seq_puts(m, "# _-----=> irqs-off\n"
2590 "# / _----=> need-resched\n"
2591 "# | / _---=> hardirq/softirq\n"
2592 "# || / _--=> preempt-depth\n"
2594 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2595 "# | | | |||| | |\n");
2599 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2601 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2602 struct trace_buffer *buf = iter->trace_buffer;
2603 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2604 struct tracer *type = iter->trace;
2605 unsigned long entries;
2606 unsigned long total;
2607 const char *name = "preemption";
2611 get_total_entries(buf, &total, &entries);
2613 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2615 seq_puts(m, "# -----------------------------------"
2616 "---------------------------------\n");
2617 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2618 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2619 nsecs_to_usecs(data->saved_latency),
2623 #if defined(CONFIG_PREEMPT_NONE)
2625 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2627 #elif defined(CONFIG_PREEMPT)
2632 /* These are reserved for later use */
2635 seq_printf(m, " #P:%d)\n", num_online_cpus());
2639 seq_puts(m, "# -----------------\n");
2640 seq_printf(m, "# | task: %.16s-%d "
2641 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2642 data->comm, data->pid,
2643 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2644 data->policy, data->rt_priority);
2645 seq_puts(m, "# -----------------\n");
2647 if (data->critical_start) {
2648 seq_puts(m, "# => started at: ");
2649 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2650 trace_print_seq(m, &iter->seq);
2651 seq_puts(m, "\n# => ended at: ");
2652 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2653 trace_print_seq(m, &iter->seq);
2654 seq_puts(m, "\n#\n");
2660 static void test_cpu_buff_start(struct trace_iterator *iter)
2662 struct trace_seq *s = &iter->seq;
2663 struct trace_array *tr = iter->tr;
2665 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
2668 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2671 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
2674 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2678 cpumask_set_cpu(iter->cpu, iter->started);
2680 /* Don't print started cpu buffer for the first entry of the trace */
2682 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2686 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2688 struct trace_array *tr = iter->tr;
2689 struct trace_seq *s = &iter->seq;
2690 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
2691 struct trace_entry *entry;
2692 struct trace_event *event;
2696 test_cpu_buff_start(iter);
2698 event = ftrace_find_event(entry->type);
2700 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2701 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2702 trace_print_lat_context(iter);
2704 trace_print_context(iter);
2707 if (trace_seq_has_overflowed(s))
2708 return TRACE_TYPE_PARTIAL_LINE;
2711 return event->funcs->trace(iter, sym_flags, event);
2713 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2715 return trace_handle_return(s);
2718 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2720 struct trace_array *tr = iter->tr;
2721 struct trace_seq *s = &iter->seq;
2722 struct trace_entry *entry;
2723 struct trace_event *event;
2727 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
2728 trace_seq_printf(s, "%d %d %llu ",
2729 entry->pid, iter->cpu, iter->ts);
2731 if (trace_seq_has_overflowed(s))
2732 return TRACE_TYPE_PARTIAL_LINE;
2734 event = ftrace_find_event(entry->type);
2736 return event->funcs->raw(iter, 0, event);
2738 trace_seq_printf(s, "%d ?\n", entry->type);
2740 return trace_handle_return(s);
2743 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2745 struct trace_array *tr = iter->tr;
2746 struct trace_seq *s = &iter->seq;
2747 unsigned char newline = '\n';
2748 struct trace_entry *entry;
2749 struct trace_event *event;
2753 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2754 SEQ_PUT_HEX_FIELD(s, entry->pid);
2755 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2756 SEQ_PUT_HEX_FIELD(s, iter->ts);
2757 if (trace_seq_has_overflowed(s))
2758 return TRACE_TYPE_PARTIAL_LINE;
2761 event = ftrace_find_event(entry->type);
2763 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2764 if (ret != TRACE_TYPE_HANDLED)
2768 SEQ_PUT_FIELD(s, newline);
2770 return trace_handle_return(s);
2773 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2775 struct trace_array *tr = iter->tr;
2776 struct trace_seq *s = &iter->seq;
2777 struct trace_entry *entry;
2778 struct trace_event *event;
2782 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2783 SEQ_PUT_FIELD(s, entry->pid);
2784 SEQ_PUT_FIELD(s, iter->cpu);
2785 SEQ_PUT_FIELD(s, iter->ts);
2786 if (trace_seq_has_overflowed(s))
2787 return TRACE_TYPE_PARTIAL_LINE;
2790 event = ftrace_find_event(entry->type);
2791 return event ? event->funcs->binary(iter, 0, event) :
2795 int trace_empty(struct trace_iterator *iter)
2797 struct ring_buffer_iter *buf_iter;
2800 /* If we are looking at one CPU buffer, only check that one */
2801 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2802 cpu = iter->cpu_file;
2803 buf_iter = trace_buffer_iter(iter, cpu);
2805 if (!ring_buffer_iter_empty(buf_iter))
2808 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2814 for_each_tracing_cpu(cpu) {
2815 buf_iter = trace_buffer_iter(iter, cpu);
2817 if (!ring_buffer_iter_empty(buf_iter))
2820 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2828 /* Called with trace_event_read_lock() held. */
2829 enum print_line_t print_trace_line(struct trace_iterator *iter)
2831 struct trace_array *tr = iter->tr;
2832 unsigned long trace_flags = tr->trace_flags;
2833 enum print_line_t ret;
2835 if (iter->lost_events) {
2836 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2837 iter->cpu, iter->lost_events);
2838 if (trace_seq_has_overflowed(&iter->seq))
2839 return TRACE_TYPE_PARTIAL_LINE;
2842 if (iter->trace && iter->trace->print_line) {
2843 ret = iter->trace->print_line(iter);
2844 if (ret != TRACE_TYPE_UNHANDLED)
2848 if (iter->ent->type == TRACE_BPUTS &&
2849 trace_flags & TRACE_ITER_PRINTK &&
2850 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2851 return trace_print_bputs_msg_only(iter);
2853 if (iter->ent->type == TRACE_BPRINT &&
2854 trace_flags & TRACE_ITER_PRINTK &&
2855 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2856 return trace_print_bprintk_msg_only(iter);
2858 if (iter->ent->type == TRACE_PRINT &&
2859 trace_flags & TRACE_ITER_PRINTK &&
2860 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2861 return trace_print_printk_msg_only(iter);
2863 if (trace_flags & TRACE_ITER_BIN)
2864 return print_bin_fmt(iter);
2866 if (trace_flags & TRACE_ITER_HEX)
2867 return print_hex_fmt(iter);
2869 if (trace_flags & TRACE_ITER_RAW)
2870 return print_raw_fmt(iter);
2872 return print_trace_fmt(iter);
2875 void trace_latency_header(struct seq_file *m)
2877 struct trace_iterator *iter = m->private;
2878 struct trace_array *tr = iter->tr;
2880 /* print nothing if the buffers are empty */
2881 if (trace_empty(iter))
2884 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2885 print_trace_header(m, iter);
2887 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
2888 print_lat_help_header(m);
2891 void trace_default_header(struct seq_file *m)
2893 struct trace_iterator *iter = m->private;
2894 struct trace_array *tr = iter->tr;
2895 unsigned long trace_flags = tr->trace_flags;
2897 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2900 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2901 /* print nothing if the buffers are empty */
2902 if (trace_empty(iter))
2904 print_trace_header(m, iter);
2905 if (!(trace_flags & TRACE_ITER_VERBOSE))
2906 print_lat_help_header(m);
2908 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2909 if (trace_flags & TRACE_ITER_IRQ_INFO)
2910 print_func_help_header_irq(iter->trace_buffer, m);
2912 print_func_help_header(iter->trace_buffer, m);
2917 static void test_ftrace_alive(struct seq_file *m)
2919 if (!ftrace_is_dead())
2921 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2922 "# MAY BE MISSING FUNCTION EVENTS\n");
2925 #ifdef CONFIG_TRACER_MAX_TRACE
2926 static void show_snapshot_main_help(struct seq_file *m)
2928 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2929 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2930 "# Takes a snapshot of the main buffer.\n"
2931 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2932 "# (Doesn't have to be '2' works with any number that\n"
2933 "# is not a '0' or '1')\n");
2936 static void show_snapshot_percpu_help(struct seq_file *m)
2938 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2939 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2940 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2941 "# Takes a snapshot of the main buffer for this cpu.\n");
2943 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2944 "# Must use main snapshot file to allocate.\n");
2946 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2947 "# (Doesn't have to be '2' works with any number that\n"
2948 "# is not a '0' or '1')\n");
2951 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2953 if (iter->tr->allocated_snapshot)
2954 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2956 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2958 seq_puts(m, "# Snapshot commands:\n");
2959 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2960 show_snapshot_main_help(m);
2962 show_snapshot_percpu_help(m);
2965 /* Should never be called */
2966 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2969 static int s_show(struct seq_file *m, void *v)
2971 struct trace_iterator *iter = v;
2974 if (iter->ent == NULL) {
2976 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2978 test_ftrace_alive(m);
2980 if (iter->snapshot && trace_empty(iter))
2981 print_snapshot_help(m, iter);
2982 else if (iter->trace && iter->trace->print_header)
2983 iter->trace->print_header(m);
2985 trace_default_header(m);
2987 } else if (iter->leftover) {
2989 * If we filled the seq_file buffer earlier, we
2990 * want to just show it now.
2992 ret = trace_print_seq(m, &iter->seq);
2994 /* ret should this time be zero, but you never know */
2995 iter->leftover = ret;
2998 print_trace_line(iter);
2999 ret = trace_print_seq(m, &iter->seq);
3001 * If we overflow the seq_file buffer, then it will
3002 * ask us for this data again at start up.
3004 * ret is 0 if seq_file write succeeded.
3007 iter->leftover = ret;
3014 * Should be used after trace_array_get(), trace_types_lock
3015 * ensures that i_cdev was already initialized.
3017 static inline int tracing_get_cpu(struct inode *inode)
3019 if (inode->i_cdev) /* See trace_create_cpu_file() */
3020 return (long)inode->i_cdev - 1;
3021 return RING_BUFFER_ALL_CPUS;
3024 static const struct seq_operations tracer_seq_ops = {
3031 static struct trace_iterator *
3032 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3034 struct trace_array *tr = inode->i_private;
3035 struct trace_iterator *iter;
3038 if (tracing_disabled)
3039 return ERR_PTR(-ENODEV);
3041 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3043 return ERR_PTR(-ENOMEM);
3045 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3047 if (!iter->buffer_iter)
3051 * We make a copy of the current tracer to avoid concurrent
3052 * changes on it while we are reading.
3054 mutex_lock(&trace_types_lock);
3055 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3059 *iter->trace = *tr->current_trace;
3061 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3066 #ifdef CONFIG_TRACER_MAX_TRACE
3067 /* Currently only the top directory has a snapshot */
3068 if (tr->current_trace->print_max || snapshot)
3069 iter->trace_buffer = &tr->max_buffer;
3072 iter->trace_buffer = &tr->trace_buffer;
3073 iter->snapshot = snapshot;
3075 iter->cpu_file = tracing_get_cpu(inode);
3076 mutex_init(&iter->mutex);
3078 /* Notify the tracer early; before we stop tracing. */
3079 if (iter->trace && iter->trace->open)
3080 iter->trace->open(iter);
3082 /* Annotate start of buffers if we had overruns */
3083 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3084 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3086 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3087 if (trace_clocks[tr->clock_id].in_ns)
3088 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3090 /* stop the trace while dumping if we are not opening "snapshot" */
3091 if (!iter->snapshot)
3092 tracing_stop_tr(tr);
3094 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3095 for_each_tracing_cpu(cpu) {
3096 iter->buffer_iter[cpu] =
3097 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3099 ring_buffer_read_prepare_sync();
3100 for_each_tracing_cpu(cpu) {
3101 ring_buffer_read_start(iter->buffer_iter[cpu]);
3102 tracing_iter_reset(iter, cpu);
3105 cpu = iter->cpu_file;
3106 iter->buffer_iter[cpu] =
3107 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3108 ring_buffer_read_prepare_sync();
3109 ring_buffer_read_start(iter->buffer_iter[cpu]);
3110 tracing_iter_reset(iter, cpu);
3113 mutex_unlock(&trace_types_lock);
3118 mutex_unlock(&trace_types_lock);
3120 kfree(iter->buffer_iter);
3122 seq_release_private(inode, file);
3123 return ERR_PTR(-ENOMEM);
3126 int tracing_open_generic(struct inode *inode, struct file *filp)
3128 if (tracing_disabled)
3131 filp->private_data = inode->i_private;
3135 bool tracing_is_disabled(void)
3137 return (tracing_disabled) ? true: false;
3141 * Open and update trace_array ref count.
3142 * Must have the current trace_array passed to it.
3144 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3146 struct trace_array *tr = inode->i_private;
3148 if (tracing_disabled)
3151 if (trace_array_get(tr) < 0)
3154 filp->private_data = inode->i_private;
3159 static int tracing_release(struct inode *inode, struct file *file)
3161 struct trace_array *tr = inode->i_private;
3162 struct seq_file *m = file->private_data;
3163 struct trace_iterator *iter;
3166 if (!(file->f_mode & FMODE_READ)) {
3167 trace_array_put(tr);
3171 /* Writes do not use seq_file */
3173 mutex_lock(&trace_types_lock);
3175 for_each_tracing_cpu(cpu) {
3176 if (iter->buffer_iter[cpu])
3177 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3180 if (iter->trace && iter->trace->close)
3181 iter->trace->close(iter);
3183 if (!iter->snapshot)
3184 /* reenable tracing if it was previously enabled */
3185 tracing_start_tr(tr);
3187 __trace_array_put(tr);
3189 mutex_unlock(&trace_types_lock);
3191 mutex_destroy(&iter->mutex);
3192 free_cpumask_var(iter->started);
3194 kfree(iter->buffer_iter);
3195 seq_release_private(inode, file);
3200 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3202 struct trace_array *tr = inode->i_private;
3204 trace_array_put(tr);
3208 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3210 struct trace_array *tr = inode->i_private;
3212 trace_array_put(tr);
3214 return single_release(inode, file);
3217 static int tracing_open(struct inode *inode, struct file *file)
3219 struct trace_array *tr = inode->i_private;
3220 struct trace_iterator *iter;
3223 if (trace_array_get(tr) < 0)
3226 /* If this file was open for write, then erase contents */
3227 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3228 int cpu = tracing_get_cpu(inode);
3230 if (cpu == RING_BUFFER_ALL_CPUS)
3231 tracing_reset_online_cpus(&tr->trace_buffer);
3233 tracing_reset(&tr->trace_buffer, cpu);
3236 if (file->f_mode & FMODE_READ) {
3237 iter = __tracing_open(inode, file, false);
3239 ret = PTR_ERR(iter);
3240 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3241 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3245 trace_array_put(tr);
3251 * Some tracers are not suitable for instance buffers.
3252 * A tracer is always available for the global array (toplevel)
3253 * or if it explicitly states that it is.
3256 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3258 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3261 /* Find the next tracer that this trace array may use */
3262 static struct tracer *
3263 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3265 while (t && !trace_ok_for_array(t, tr))
3272 t_next(struct seq_file *m, void *v, loff_t *pos)
3274 struct trace_array *tr = m->private;
3275 struct tracer *t = v;
3280 t = get_tracer_for_array(tr, t->next);
3285 static void *t_start(struct seq_file *m, loff_t *pos)
3287 struct trace_array *tr = m->private;
3291 mutex_lock(&trace_types_lock);
3293 t = get_tracer_for_array(tr, trace_types);
3294 for (; t && l < *pos; t = t_next(m, t, &l))
3300 static void t_stop(struct seq_file *m, void *p)
3302 mutex_unlock(&trace_types_lock);
3305 static int t_show(struct seq_file *m, void *v)
3307 struct tracer *t = v;
3312 seq_puts(m, t->name);
3321 static const struct seq_operations show_traces_seq_ops = {
3328 static int show_traces_open(struct inode *inode, struct file *file)
3330 struct trace_array *tr = inode->i_private;
3334 if (tracing_disabled)
3337 ret = seq_open(file, &show_traces_seq_ops);
3341 m = file->private_data;
3348 tracing_write_stub(struct file *filp, const char __user *ubuf,
3349 size_t count, loff_t *ppos)
3354 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3358 if (file->f_mode & FMODE_READ)
3359 ret = seq_lseek(file, offset, whence);
3361 file->f_pos = ret = 0;
3366 static const struct file_operations tracing_fops = {
3367 .open = tracing_open,
3369 .write = tracing_write_stub,
3370 .llseek = tracing_lseek,
3371 .release = tracing_release,
3374 static const struct file_operations show_traces_fops = {
3375 .open = show_traces_open,
3377 .release = seq_release,
3378 .llseek = seq_lseek,
3382 * The tracer itself will not take this lock, but still we want
3383 * to provide a consistent cpumask to user-space:
3385 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3388 * Temporary storage for the character representation of the
3389 * CPU bitmask (and one more byte for the newline):
3391 static char mask_str[NR_CPUS + 1];
3394 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3395 size_t count, loff_t *ppos)
3397 struct trace_array *tr = file_inode(filp)->i_private;
3400 mutex_lock(&tracing_cpumask_update_lock);
3402 len = snprintf(mask_str, count, "%*pb\n",
3403 cpumask_pr_args(tr->tracing_cpumask));
3408 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3411 mutex_unlock(&tracing_cpumask_update_lock);
3417 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3418 size_t count, loff_t *ppos)
3420 struct trace_array *tr = file_inode(filp)->i_private;
3421 cpumask_var_t tracing_cpumask_new;
3424 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3427 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3431 mutex_lock(&tracing_cpumask_update_lock);
3433 local_irq_disable();
3434 arch_spin_lock(&tr->max_lock);
3435 for_each_tracing_cpu(cpu) {
3437 * Increase/decrease the disabled counter if we are
3438 * about to flip a bit in the cpumask:
3440 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3441 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3442 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3443 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3445 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3446 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3447 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3448 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3451 arch_spin_unlock(&tr->max_lock);
3454 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3456 mutex_unlock(&tracing_cpumask_update_lock);
3457 free_cpumask_var(tracing_cpumask_new);
3462 free_cpumask_var(tracing_cpumask_new);
3467 static const struct file_operations tracing_cpumask_fops = {
3468 .open = tracing_open_generic_tr,
3469 .read = tracing_cpumask_read,
3470 .write = tracing_cpumask_write,
3471 .release = tracing_release_generic_tr,
3472 .llseek = generic_file_llseek,
3475 static int tracing_trace_options_show(struct seq_file *m, void *v)
3477 struct tracer_opt *trace_opts;
3478 struct trace_array *tr = m->private;
3482 mutex_lock(&trace_types_lock);
3483 tracer_flags = tr->current_trace->flags->val;
3484 trace_opts = tr->current_trace->flags->opts;
3486 for (i = 0; trace_options[i]; i++) {
3487 if (tr->trace_flags & (1 << i))
3488 seq_printf(m, "%s\n", trace_options[i]);
3490 seq_printf(m, "no%s\n", trace_options[i]);
3493 for (i = 0; trace_opts[i].name; i++) {
3494 if (tracer_flags & trace_opts[i].bit)
3495 seq_printf(m, "%s\n", trace_opts[i].name);
3497 seq_printf(m, "no%s\n", trace_opts[i].name);
3499 mutex_unlock(&trace_types_lock);
3504 static int __set_tracer_option(struct trace_array *tr,
3505 struct tracer_flags *tracer_flags,
3506 struct tracer_opt *opts, int neg)
3508 struct tracer *trace = tr->current_trace;
3511 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3516 tracer_flags->val &= ~opts->bit;
3518 tracer_flags->val |= opts->bit;
3522 /* Try to assign a tracer specific option */
3523 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3525 struct tracer *trace = tr->current_trace;
3526 struct tracer_flags *tracer_flags = trace->flags;
3527 struct tracer_opt *opts = NULL;
3530 for (i = 0; tracer_flags->opts[i].name; i++) {
3531 opts = &tracer_flags->opts[i];
3533 if (strcmp(cmp, opts->name) == 0)
3534 return __set_tracer_option(tr, trace->flags, opts, neg);
3540 /* Some tracers require overwrite to stay enabled */
3541 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3543 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3549 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3551 /* do nothing if flag is already set */
3552 if (!!(tr->trace_flags & mask) == !!enabled)
3555 /* Give the tracer a chance to approve the change */
3556 if (tr->current_trace->flag_changed)
3557 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3561 tr->trace_flags |= mask;
3563 tr->trace_flags &= ~mask;
3565 if (mask == TRACE_ITER_RECORD_CMD)
3566 trace_event_enable_cmd_record(enabled);
3568 if (mask == TRACE_ITER_OVERWRITE) {
3569 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3570 #ifdef CONFIG_TRACER_MAX_TRACE
3571 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3575 if (mask == TRACE_ITER_PRINTK) {
3576 trace_printk_start_stop_comm(enabled);
3577 trace_printk_control(enabled);
3583 static int trace_set_options(struct trace_array *tr, char *option)
3589 size_t orig_len = strlen(option);
3591 cmp = strstrip(option);
3593 if (strncmp(cmp, "no", 2) == 0) {
3598 mutex_lock(&trace_types_lock);
3600 for (i = 0; trace_options[i]; i++) {
3601 if (strcmp(cmp, trace_options[i]) == 0) {
3602 ret = set_tracer_flag(tr, 1 << i, !neg);
3607 /* If no option could be set, test the specific tracer options */
3608 if (!trace_options[i])
3609 ret = set_tracer_option(tr, cmp, neg);
3611 mutex_unlock(&trace_types_lock);
3614 * If the first trailing whitespace is replaced with '\0' by strstrip,
3615 * turn it back into a space.
3617 if (orig_len > strlen(option))
3618 option[strlen(option)] = ' ';
3623 static void __init apply_trace_boot_options(void)
3625 char *buf = trace_boot_options_buf;
3629 option = strsep(&buf, ",");
3635 trace_set_options(&global_trace, option);
3637 /* Put back the comma to allow this to be called again */
3644 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3645 size_t cnt, loff_t *ppos)
3647 struct seq_file *m = filp->private_data;
3648 struct trace_array *tr = m->private;
3652 if (cnt >= sizeof(buf))
3655 if (copy_from_user(&buf, ubuf, cnt))
3660 ret = trace_set_options(tr, buf);
3669 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3671 struct trace_array *tr = inode->i_private;
3674 if (tracing_disabled)
3677 if (trace_array_get(tr) < 0)
3680 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3682 trace_array_put(tr);
3687 static const struct file_operations tracing_iter_fops = {
3688 .open = tracing_trace_options_open,
3690 .llseek = seq_lseek,
3691 .release = tracing_single_release_tr,
3692 .write = tracing_trace_options_write,
3695 static const char readme_msg[] =
3696 "tracing mini-HOWTO:\n\n"
3697 "# echo 0 > tracing_on : quick way to disable tracing\n"
3698 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3699 " Important files:\n"
3700 " trace\t\t\t- The static contents of the buffer\n"
3701 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3702 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3703 " current_tracer\t- function and latency tracers\n"
3704 " available_tracers\t- list of configured tracers for current_tracer\n"
3705 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3706 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3707 " trace_clock\t\t-change the clock used to order events\n"
3708 " local: Per cpu clock but may not be synced across CPUs\n"
3709 " global: Synced across CPUs but slows tracing down.\n"
3710 " counter: Not a clock, but just an increment\n"
3711 " uptime: Jiffy counter from time of boot\n"
3712 " perf: Same clock that perf events use\n"
3713 #ifdef CONFIG_X86_64
3714 " x86-tsc: TSC cycle counter\n"
3716 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3717 " tracing_cpumask\t- Limit which CPUs to trace\n"
3718 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3719 "\t\t\t Remove sub-buffer with rmdir\n"
3720 " trace_options\t\t- Set format or modify how tracing happens\n"
3721 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3722 "\t\t\t option name\n"
3723 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3724 #ifdef CONFIG_DYNAMIC_FTRACE
3725 "\n available_filter_functions - list of functions that can be filtered on\n"
3726 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3727 "\t\t\t functions\n"
3728 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3729 "\t modules: Can select a group via module\n"
3730 "\t Format: :mod:<module-name>\n"
3731 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3732 "\t triggers: a command to perform when function is hit\n"
3733 "\t Format: <function>:<trigger>[:count]\n"
3734 "\t trigger: traceon, traceoff\n"
3735 "\t\t enable_event:<system>:<event>\n"
3736 "\t\t disable_event:<system>:<event>\n"
3737 #ifdef CONFIG_STACKTRACE
3740 #ifdef CONFIG_TRACER_SNAPSHOT
3745 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3746 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3747 "\t The first one will disable tracing every time do_fault is hit\n"
3748 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3749 "\t The first time do trap is hit and it disables tracing, the\n"
3750 "\t counter will decrement to 2. If tracing is already disabled,\n"
3751 "\t the counter will not decrement. It only decrements when the\n"
3752 "\t trigger did work\n"
3753 "\t To remove trigger without count:\n"
3754 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3755 "\t To remove trigger with a count:\n"
3756 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3757 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3758 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3759 "\t modules: Can select a group via module command :mod:\n"
3760 "\t Does not accept triggers\n"
3761 #endif /* CONFIG_DYNAMIC_FTRACE */
3762 #ifdef CONFIG_FUNCTION_TRACER
3763 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3766 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3767 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3768 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3769 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3771 #ifdef CONFIG_TRACER_SNAPSHOT
3772 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3773 "\t\t\t snapshot buffer. Read the contents for more\n"
3774 "\t\t\t information\n"
3776 #ifdef CONFIG_STACK_TRACER
3777 " stack_trace\t\t- Shows the max stack trace when active\n"
3778 " stack_max_size\t- Shows current max stack size that was traced\n"
3779 "\t\t\t Write into this file to reset the max size (trigger a\n"
3780 "\t\t\t new trace)\n"
3781 #ifdef CONFIG_DYNAMIC_FTRACE
3782 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3785 #endif /* CONFIG_STACK_TRACER */
3786 " events/\t\t- Directory containing all trace event subsystems:\n"
3787 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3788 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3789 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3791 " filter\t\t- If set, only events passing filter are traced\n"
3792 " events/<system>/<event>/\t- Directory containing control files for\n"
3794 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3795 " filter\t\t- If set, only events passing filter are traced\n"
3796 " trigger\t\t- If set, a command to perform when event is hit\n"
3797 "\t Format: <trigger>[:count][if <filter>]\n"
3798 "\t trigger: traceon, traceoff\n"
3799 "\t enable_event:<system>:<event>\n"
3800 "\t disable_event:<system>:<event>\n"
3801 #ifdef CONFIG_STACKTRACE
3804 #ifdef CONFIG_TRACER_SNAPSHOT
3807 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3808 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3809 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3810 "\t events/block/block_unplug/trigger\n"
3811 "\t The first disables tracing every time block_unplug is hit.\n"
3812 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3813 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3814 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3815 "\t Like function triggers, the counter is only decremented if it\n"
3816 "\t enabled or disabled tracing.\n"
3817 "\t To remove a trigger without a count:\n"
3818 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3819 "\t To remove a trigger with a count:\n"
3820 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3821 "\t Filters can be ignored when removing a trigger.\n"
3825 tracing_readme_read(struct file *filp, char __user *ubuf,
3826 size_t cnt, loff_t *ppos)
3828 return simple_read_from_buffer(ubuf, cnt, ppos,
3829 readme_msg, strlen(readme_msg));
3832 static const struct file_operations tracing_readme_fops = {
3833 .open = tracing_open_generic,
3834 .read = tracing_readme_read,
3835 .llseek = generic_file_llseek,
3838 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3840 unsigned int *ptr = v;
3842 if (*pos || m->count)
3847 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3849 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3858 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3864 arch_spin_lock(&trace_cmdline_lock);
3866 v = &savedcmd->map_cmdline_to_pid[0];
3868 v = saved_cmdlines_next(m, v, &l);
3876 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3878 arch_spin_unlock(&trace_cmdline_lock);
3882 static int saved_cmdlines_show(struct seq_file *m, void *v)
3884 char buf[TASK_COMM_LEN];
3885 unsigned int *pid = v;
3887 __trace_find_cmdline(*pid, buf);
3888 seq_printf(m, "%d %s\n", *pid, buf);
3892 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3893 .start = saved_cmdlines_start,
3894 .next = saved_cmdlines_next,
3895 .stop = saved_cmdlines_stop,
3896 .show = saved_cmdlines_show,
3899 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3901 if (tracing_disabled)
3904 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3907 static const struct file_operations tracing_saved_cmdlines_fops = {
3908 .open = tracing_saved_cmdlines_open,
3910 .llseek = seq_lseek,
3911 .release = seq_release,
3915 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3916 size_t cnt, loff_t *ppos)
3921 arch_spin_lock(&trace_cmdline_lock);
3922 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3923 arch_spin_unlock(&trace_cmdline_lock);
3925 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3928 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3930 kfree(s->saved_cmdlines);
3931 kfree(s->map_cmdline_to_pid);
3935 static int tracing_resize_saved_cmdlines(unsigned int val)
3937 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3939 s = kmalloc(sizeof(*s), GFP_KERNEL);
3943 if (allocate_cmdlines_buffer(val, s) < 0) {
3948 arch_spin_lock(&trace_cmdline_lock);
3949 savedcmd_temp = savedcmd;
3951 arch_spin_unlock(&trace_cmdline_lock);
3952 free_saved_cmdlines_buffer(savedcmd_temp);
3958 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3959 size_t cnt, loff_t *ppos)
3964 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3968 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3969 if (!val || val > PID_MAX_DEFAULT)
3972 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3981 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3982 .open = tracing_open_generic,
3983 .read = tracing_saved_cmdlines_size_read,
3984 .write = tracing_saved_cmdlines_size_write,
3987 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
3988 static union trace_enum_map_item *
3989 update_enum_map(union trace_enum_map_item *ptr)
3991 if (!ptr->map.enum_string) {
3992 if (ptr->tail.next) {
3993 ptr = ptr->tail.next;
3994 /* Set ptr to the next real item (skip head) */
4002 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4004 union trace_enum_map_item *ptr = v;
4007 * Paranoid! If ptr points to end, we don't want to increment past it.
4008 * This really should never happen.
4010 ptr = update_enum_map(ptr);
4011 if (WARN_ON_ONCE(!ptr))
4018 ptr = update_enum_map(ptr);
4023 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4025 union trace_enum_map_item *v;
4028 mutex_lock(&trace_enum_mutex);
4030 v = trace_enum_maps;
4034 while (v && l < *pos) {
4035 v = enum_map_next(m, v, &l);
4041 static void enum_map_stop(struct seq_file *m, void *v)
4043 mutex_unlock(&trace_enum_mutex);
4046 static int enum_map_show(struct seq_file *m, void *v)
4048 union trace_enum_map_item *ptr = v;
4050 seq_printf(m, "%s %ld (%s)\n",
4051 ptr->map.enum_string, ptr->map.enum_value,
4057 static const struct seq_operations tracing_enum_map_seq_ops = {
4058 .start = enum_map_start,
4059 .next = enum_map_next,
4060 .stop = enum_map_stop,
4061 .show = enum_map_show,
4064 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4066 if (tracing_disabled)
4069 return seq_open(filp, &tracing_enum_map_seq_ops);
4072 static const struct file_operations tracing_enum_map_fops = {
4073 .open = tracing_enum_map_open,
4075 .llseek = seq_lseek,
4076 .release = seq_release,
4079 static inline union trace_enum_map_item *
4080 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4082 /* Return tail of array given the head */
4083 return ptr + ptr->head.length + 1;
4087 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4090 struct trace_enum_map **stop;
4091 struct trace_enum_map **map;
4092 union trace_enum_map_item *map_array;
4093 union trace_enum_map_item *ptr;
4098 * The trace_enum_maps contains the map plus a head and tail item,
4099 * where the head holds the module and length of array, and the
4100 * tail holds a pointer to the next list.
4102 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4104 pr_warning("Unable to allocate trace enum mapping\n");
4108 mutex_lock(&trace_enum_mutex);
4110 if (!trace_enum_maps)
4111 trace_enum_maps = map_array;
4113 ptr = trace_enum_maps;
4115 ptr = trace_enum_jmp_to_tail(ptr);
4116 if (!ptr->tail.next)
4118 ptr = ptr->tail.next;
4121 ptr->tail.next = map_array;
4123 map_array->head.mod = mod;
4124 map_array->head.length = len;
4127 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4128 map_array->map = **map;
4131 memset(map_array, 0, sizeof(*map_array));
4133 mutex_unlock(&trace_enum_mutex);
4136 static void trace_create_enum_file(struct dentry *d_tracer)
4138 trace_create_file("enum_map", 0444, d_tracer,
4139 NULL, &tracing_enum_map_fops);
4142 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4143 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4144 static inline void trace_insert_enum_map_file(struct module *mod,
4145 struct trace_enum_map **start, int len) { }
4146 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4148 static void trace_insert_enum_map(struct module *mod,
4149 struct trace_enum_map **start, int len)
4151 struct trace_enum_map **map;
4158 trace_event_enum_update(map, len);
4160 trace_insert_enum_map_file(mod, start, len);
4164 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4165 size_t cnt, loff_t *ppos)
4167 struct trace_array *tr = filp->private_data;
4168 char buf[MAX_TRACER_SIZE+2];
4171 mutex_lock(&trace_types_lock);
4172 r = sprintf(buf, "%s\n", tr->current_trace->name);
4173 mutex_unlock(&trace_types_lock);
4175 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4178 int tracer_init(struct tracer *t, struct trace_array *tr)
4180 tracing_reset_online_cpus(&tr->trace_buffer);
4184 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4188 for_each_tracing_cpu(cpu)
4189 per_cpu_ptr(buf->data, cpu)->entries = val;
4192 #ifdef CONFIG_TRACER_MAX_TRACE
4193 /* resize @tr's buffer to the size of @size_tr's entries */
4194 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4195 struct trace_buffer *size_buf, int cpu_id)
4199 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4200 for_each_tracing_cpu(cpu) {
4201 ret = ring_buffer_resize(trace_buf->buffer,
4202 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4205 per_cpu_ptr(trace_buf->data, cpu)->entries =
4206 per_cpu_ptr(size_buf->data, cpu)->entries;
4209 ret = ring_buffer_resize(trace_buf->buffer,
4210 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4212 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4213 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4218 #endif /* CONFIG_TRACER_MAX_TRACE */
4220 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4221 unsigned long size, int cpu)
4226 * If kernel or user changes the size of the ring buffer
4227 * we use the size that was given, and we can forget about
4228 * expanding it later.
4230 ring_buffer_expanded = true;
4232 /* May be called before buffers are initialized */
4233 if (!tr->trace_buffer.buffer)
4236 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4240 #ifdef CONFIG_TRACER_MAX_TRACE
4241 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4242 !tr->current_trace->use_max_tr)
4245 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4247 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4248 &tr->trace_buffer, cpu);
4251 * AARGH! We are left with different
4252 * size max buffer!!!!
4253 * The max buffer is our "snapshot" buffer.
4254 * When a tracer needs a snapshot (one of the
4255 * latency tracers), it swaps the max buffer
4256 * with the saved snap shot. We succeeded to
4257 * update the size of the main buffer, but failed to
4258 * update the size of the max buffer. But when we tried
4259 * to reset the main buffer to the original size, we
4260 * failed there too. This is very unlikely to
4261 * happen, but if it does, warn and kill all
4265 tracing_disabled = 1;
4270 if (cpu == RING_BUFFER_ALL_CPUS)
4271 set_buffer_entries(&tr->max_buffer, size);
4273 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4276 #endif /* CONFIG_TRACER_MAX_TRACE */
4278 if (cpu == RING_BUFFER_ALL_CPUS)
4279 set_buffer_entries(&tr->trace_buffer, size);
4281 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4286 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4287 unsigned long size, int cpu_id)
4291 mutex_lock(&trace_types_lock);
4293 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4294 /* make sure, this cpu is enabled in the mask */
4295 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4301 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4306 mutex_unlock(&trace_types_lock);
4313 * tracing_update_buffers - used by tracing facility to expand ring buffers
4315 * To save on memory when the tracing is never used on a system with it
4316 * configured in. The ring buffers are set to a minimum size. But once
4317 * a user starts to use the tracing facility, then they need to grow
4318 * to their default size.
4320 * This function is to be called when a tracer is about to be used.
4322 int tracing_update_buffers(void)
4326 mutex_lock(&trace_types_lock);
4327 if (!ring_buffer_expanded)
4328 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4329 RING_BUFFER_ALL_CPUS);
4330 mutex_unlock(&trace_types_lock);
4335 struct trace_option_dentry;
4338 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4341 * Used to clear out the tracer before deletion of an instance.
4342 * Must have trace_types_lock held.
4344 static void tracing_set_nop(struct trace_array *tr)
4346 if (tr->current_trace == &nop_trace)
4349 tr->current_trace->enabled--;
4351 if (tr->current_trace->reset)
4352 tr->current_trace->reset(tr);
4354 tr->current_trace = &nop_trace;
4357 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4359 /* Only enable if the directory has been created already. */
4363 create_trace_option_files(tr, t);
4366 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4369 #ifdef CONFIG_TRACER_MAX_TRACE
4374 mutex_lock(&trace_types_lock);
4376 if (!ring_buffer_expanded) {
4377 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4378 RING_BUFFER_ALL_CPUS);
4384 for (t = trace_types; t; t = t->next) {
4385 if (strcmp(t->name, buf) == 0)
4392 if (t == tr->current_trace)
4395 /* Some tracers are only allowed for the top level buffer */
4396 if (!trace_ok_for_array(t, tr)) {
4401 /* If trace pipe files are being read, we can't change the tracer */
4402 if (tr->current_trace->ref) {
4407 trace_branch_disable();
4409 tr->current_trace->enabled--;
4411 if (tr->current_trace->reset)
4412 tr->current_trace->reset(tr);
4414 /* Current trace needs to be nop_trace before synchronize_sched */
4415 tr->current_trace = &nop_trace;
4417 #ifdef CONFIG_TRACER_MAX_TRACE
4418 had_max_tr = tr->allocated_snapshot;
4420 if (had_max_tr && !t->use_max_tr) {
4422 * We need to make sure that the update_max_tr sees that
4423 * current_trace changed to nop_trace to keep it from
4424 * swapping the buffers after we resize it.
4425 * The update_max_tr is called from interrupts disabled
4426 * so a synchronized_sched() is sufficient.
4428 synchronize_sched();
4433 #ifdef CONFIG_TRACER_MAX_TRACE
4434 if (t->use_max_tr && !had_max_tr) {
4435 ret = alloc_snapshot(tr);
4442 ret = tracer_init(t, tr);
4447 tr->current_trace = t;
4448 tr->current_trace->enabled++;
4449 trace_branch_enable(tr);
4451 mutex_unlock(&trace_types_lock);
4457 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4458 size_t cnt, loff_t *ppos)
4460 struct trace_array *tr = filp->private_data;
4461 char buf[MAX_TRACER_SIZE+1];
4468 if (cnt > MAX_TRACER_SIZE)
4469 cnt = MAX_TRACER_SIZE;
4471 if (copy_from_user(&buf, ubuf, cnt))
4476 /* strip ending whitespace. */
4477 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4480 err = tracing_set_tracer(tr, buf);
4490 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4491 size_t cnt, loff_t *ppos)
4496 r = snprintf(buf, sizeof(buf), "%ld\n",
4497 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4498 if (r > sizeof(buf))
4500 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4504 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4505 size_t cnt, loff_t *ppos)
4510 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4520 tracing_thresh_read(struct file *filp, char __user *ubuf,
4521 size_t cnt, loff_t *ppos)
4523 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4527 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4528 size_t cnt, loff_t *ppos)
4530 struct trace_array *tr = filp->private_data;
4533 mutex_lock(&trace_types_lock);
4534 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4538 if (tr->current_trace->update_thresh) {
4539 ret = tr->current_trace->update_thresh(tr);
4546 mutex_unlock(&trace_types_lock);
4551 #ifdef CONFIG_TRACER_MAX_TRACE
4554 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4555 size_t cnt, loff_t *ppos)
4557 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4561 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4562 size_t cnt, loff_t *ppos)
4564 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4569 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4571 struct trace_array *tr = inode->i_private;
4572 struct trace_iterator *iter;
4575 if (tracing_disabled)
4578 if (trace_array_get(tr) < 0)
4581 mutex_lock(&trace_types_lock);
4583 /* create a buffer to store the information to pass to userspace */
4584 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4587 __trace_array_put(tr);
4591 trace_seq_init(&iter->seq);
4592 iter->trace = tr->current_trace;
4594 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4599 /* trace pipe does not show start of buffer */
4600 cpumask_setall(iter->started);
4602 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4603 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4605 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4606 if (trace_clocks[tr->clock_id].in_ns)
4607 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4610 iter->trace_buffer = &tr->trace_buffer;
4611 iter->cpu_file = tracing_get_cpu(inode);
4612 mutex_init(&iter->mutex);
4613 filp->private_data = iter;
4615 if (iter->trace->pipe_open)
4616 iter->trace->pipe_open(iter);
4618 nonseekable_open(inode, filp);
4620 tr->current_trace->ref++;
4622 mutex_unlock(&trace_types_lock);
4628 __trace_array_put(tr);
4629 mutex_unlock(&trace_types_lock);
4633 static int tracing_release_pipe(struct inode *inode, struct file *file)
4635 struct trace_iterator *iter = file->private_data;
4636 struct trace_array *tr = inode->i_private;
4638 mutex_lock(&trace_types_lock);
4640 tr->current_trace->ref--;
4642 if (iter->trace->pipe_close)
4643 iter->trace->pipe_close(iter);
4645 mutex_unlock(&trace_types_lock);
4647 free_cpumask_var(iter->started);
4648 mutex_destroy(&iter->mutex);
4651 trace_array_put(tr);
4657 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4659 struct trace_array *tr = iter->tr;
4661 /* Iterators are static, they should be filled or empty */
4662 if (trace_buffer_iter(iter, iter->cpu_file))
4663 return POLLIN | POLLRDNORM;
4665 if (tr->trace_flags & TRACE_ITER_BLOCK)
4667 * Always select as readable when in blocking mode
4669 return POLLIN | POLLRDNORM;
4671 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4676 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4678 struct trace_iterator *iter = filp->private_data;
4680 return trace_poll(iter, filp, poll_table);
4683 /* Must be called with iter->mutex held. */
4684 static int tracing_wait_pipe(struct file *filp)
4686 struct trace_iterator *iter = filp->private_data;
4689 while (trace_empty(iter)) {
4691 if ((filp->f_flags & O_NONBLOCK)) {
4696 * We block until we read something and tracing is disabled.
4697 * We still block if tracing is disabled, but we have never
4698 * read anything. This allows a user to cat this file, and
4699 * then enable tracing. But after we have read something,
4700 * we give an EOF when tracing is again disabled.
4702 * iter->pos will be 0 if we haven't read anything.
4704 if (!tracing_is_on() && iter->pos)
4707 mutex_unlock(&iter->mutex);
4709 ret = wait_on_pipe(iter, false);
4711 mutex_lock(&iter->mutex);
4724 tracing_read_pipe(struct file *filp, char __user *ubuf,
4725 size_t cnt, loff_t *ppos)
4727 struct trace_iterator *iter = filp->private_data;
4731 * Avoid more than one consumer on a single file descriptor
4732 * This is just a matter of traces coherency, the ring buffer itself
4735 mutex_lock(&iter->mutex);
4737 /* return any leftover data */
4738 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4742 trace_seq_init(&iter->seq);
4744 if (iter->trace->read) {
4745 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4751 sret = tracing_wait_pipe(filp);
4755 /* stop when tracing is finished */
4756 if (trace_empty(iter)) {
4761 if (cnt >= PAGE_SIZE)
4762 cnt = PAGE_SIZE - 1;
4764 /* reset all but tr, trace, and overruns */
4765 memset(&iter->seq, 0,
4766 sizeof(struct trace_iterator) -
4767 offsetof(struct trace_iterator, seq));
4768 cpumask_clear(iter->started);
4771 trace_event_read_lock();
4772 trace_access_lock(iter->cpu_file);
4773 while (trace_find_next_entry_inc(iter) != NULL) {
4774 enum print_line_t ret;
4775 int save_len = iter->seq.seq.len;
4777 ret = print_trace_line(iter);
4778 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4779 /* don't print partial lines */
4780 iter->seq.seq.len = save_len;
4783 if (ret != TRACE_TYPE_NO_CONSUME)
4784 trace_consume(iter);
4786 if (trace_seq_used(&iter->seq) >= cnt)
4790 * Setting the full flag means we reached the trace_seq buffer
4791 * size and we should leave by partial output condition above.
4792 * One of the trace_seq_* functions is not used properly.
4794 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4797 trace_access_unlock(iter->cpu_file);
4798 trace_event_read_unlock();
4800 /* Now copy what we have to the user */
4801 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4802 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4803 trace_seq_init(&iter->seq);
4806 * If there was nothing to send to user, in spite of consuming trace
4807 * entries, go back to wait for more entries.
4813 mutex_unlock(&iter->mutex);
4818 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4821 __free_page(spd->pages[idx]);
4824 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4826 .confirm = generic_pipe_buf_confirm,
4827 .release = generic_pipe_buf_release,
4828 .steal = generic_pipe_buf_steal,
4829 .get = generic_pipe_buf_get,
4833 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4839 /* Seq buffer is page-sized, exactly what we need. */
4841 save_len = iter->seq.seq.len;
4842 ret = print_trace_line(iter);
4844 if (trace_seq_has_overflowed(&iter->seq)) {
4845 iter->seq.seq.len = save_len;
4850 * This should not be hit, because it should only
4851 * be set if the iter->seq overflowed. But check it
4852 * anyway to be safe.
4854 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4855 iter->seq.seq.len = save_len;
4859 count = trace_seq_used(&iter->seq) - save_len;
4862 iter->seq.seq.len = save_len;
4866 if (ret != TRACE_TYPE_NO_CONSUME)
4867 trace_consume(iter);
4869 if (!trace_find_next_entry_inc(iter)) {
4879 static ssize_t tracing_splice_read_pipe(struct file *filp,
4881 struct pipe_inode_info *pipe,
4885 struct page *pages_def[PIPE_DEF_BUFFERS];
4886 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4887 struct trace_iterator *iter = filp->private_data;
4888 struct splice_pipe_desc spd = {
4890 .partial = partial_def,
4891 .nr_pages = 0, /* This gets updated below. */
4892 .nr_pages_max = PIPE_DEF_BUFFERS,
4894 .ops = &tracing_pipe_buf_ops,
4895 .spd_release = tracing_spd_release_pipe,
4901 if (splice_grow_spd(pipe, &spd))
4904 mutex_lock(&iter->mutex);
4906 if (iter->trace->splice_read) {
4907 ret = iter->trace->splice_read(iter, filp,
4908 ppos, pipe, len, flags);
4913 ret = tracing_wait_pipe(filp);
4917 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4922 trace_event_read_lock();
4923 trace_access_lock(iter->cpu_file);
4925 /* Fill as many pages as possible. */
4926 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4927 spd.pages[i] = alloc_page(GFP_KERNEL);
4931 rem = tracing_fill_pipe_page(rem, iter);
4933 /* Copy the data into the page, so we can start over. */
4934 ret = trace_seq_to_buffer(&iter->seq,
4935 page_address(spd.pages[i]),
4936 trace_seq_used(&iter->seq));
4938 __free_page(spd.pages[i]);
4941 spd.partial[i].offset = 0;
4942 spd.partial[i].len = trace_seq_used(&iter->seq);
4944 trace_seq_init(&iter->seq);
4947 trace_access_unlock(iter->cpu_file);
4948 trace_event_read_unlock();
4949 mutex_unlock(&iter->mutex);
4954 ret = splice_to_pipe(pipe, &spd);
4958 splice_shrink_spd(&spd);
4962 mutex_unlock(&iter->mutex);
4967 tracing_entries_read(struct file *filp, char __user *ubuf,
4968 size_t cnt, loff_t *ppos)
4970 struct inode *inode = file_inode(filp);
4971 struct trace_array *tr = inode->i_private;
4972 int cpu = tracing_get_cpu(inode);
4977 mutex_lock(&trace_types_lock);
4979 if (cpu == RING_BUFFER_ALL_CPUS) {
4980 int cpu, buf_size_same;
4985 /* check if all cpu sizes are same */
4986 for_each_tracing_cpu(cpu) {
4987 /* fill in the size from first enabled cpu */
4989 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4990 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4996 if (buf_size_same) {
4997 if (!ring_buffer_expanded)
4998 r = sprintf(buf, "%lu (expanded: %lu)\n",
5000 trace_buf_size >> 10);
5002 r = sprintf(buf, "%lu\n", size >> 10);
5004 r = sprintf(buf, "X\n");
5006 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5008 mutex_unlock(&trace_types_lock);
5010 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5015 tracing_entries_write(struct file *filp, const char __user *ubuf,
5016 size_t cnt, loff_t *ppos)
5018 struct inode *inode = file_inode(filp);
5019 struct trace_array *tr = inode->i_private;
5023 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5027 /* must have at least 1 entry */
5031 /* value is in KB */
5033 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5043 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5044 size_t cnt, loff_t *ppos)
5046 struct trace_array *tr = filp->private_data;
5049 unsigned long size = 0, expanded_size = 0;
5051 mutex_lock(&trace_types_lock);
5052 for_each_tracing_cpu(cpu) {
5053 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5054 if (!ring_buffer_expanded)
5055 expanded_size += trace_buf_size >> 10;
5057 if (ring_buffer_expanded)
5058 r = sprintf(buf, "%lu\n", size);
5060 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5061 mutex_unlock(&trace_types_lock);
5063 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5067 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5068 size_t cnt, loff_t *ppos)
5071 * There is no need to read what the user has written, this function
5072 * is just to make sure that there is no error when "echo" is used
5081 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5083 struct trace_array *tr = inode->i_private;
5085 /* disable tracing ? */
5086 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5087 tracer_tracing_off(tr);
5088 /* resize the ring buffer to 0 */
5089 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5091 trace_array_put(tr);
5097 tracing_mark_write(struct file *filp, const char __user *ubuf,
5098 size_t cnt, loff_t *fpos)
5100 unsigned long addr = (unsigned long)ubuf;
5101 struct trace_array *tr = filp->private_data;
5102 struct ring_buffer_event *event;
5103 struct ring_buffer *buffer;
5104 struct print_entry *entry;
5105 unsigned long irq_flags;
5106 struct page *pages[2];
5116 if (tracing_disabled)
5119 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5122 if (cnt > TRACE_BUF_SIZE)
5123 cnt = TRACE_BUF_SIZE;
5126 * Userspace is injecting traces into the kernel trace buffer.
5127 * We want to be as non intrusive as possible.
5128 * To do so, we do not want to allocate any special buffers
5129 * or take any locks, but instead write the userspace data
5130 * straight into the ring buffer.
5132 * First we need to pin the userspace buffer into memory,
5133 * which, most likely it is, because it just referenced it.
5134 * But there's no guarantee that it is. By using get_user_pages_fast()
5135 * and kmap_atomic/kunmap_atomic() we can get access to the
5136 * pages directly. We then write the data directly into the
5139 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5141 /* check if we cross pages */
5142 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5145 offset = addr & (PAGE_SIZE - 1);
5148 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5149 if (ret < nr_pages) {
5151 put_page(pages[ret]);
5156 for (i = 0; i < nr_pages; i++)
5157 map_page[i] = kmap_atomic(pages[i]);
5159 local_save_flags(irq_flags);
5160 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5161 buffer = tr->trace_buffer.buffer;
5162 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5163 irq_flags, preempt_count());
5165 /* Ring buffer disabled, return as if not open for write */
5170 entry = ring_buffer_event_data(event);
5171 entry->ip = _THIS_IP_;
5173 if (nr_pages == 2) {
5174 len = PAGE_SIZE - offset;
5175 memcpy(&entry->buf, map_page[0] + offset, len);
5176 memcpy(&entry->buf[len], map_page[1], cnt - len);
5178 memcpy(&entry->buf, map_page[0] + offset, cnt);
5180 if (entry->buf[cnt - 1] != '\n') {
5181 entry->buf[cnt] = '\n';
5182 entry->buf[cnt + 1] = '\0';
5184 entry->buf[cnt] = '\0';
5186 __buffer_unlock_commit(buffer, event);
5193 for (i = nr_pages - 1; i >= 0; i--) {
5194 kunmap_atomic(map_page[i]);
5201 static int tracing_clock_show(struct seq_file *m, void *v)
5203 struct trace_array *tr = m->private;
5206 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5208 "%s%s%s%s", i ? " " : "",
5209 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5210 i == tr->clock_id ? "]" : "");
5216 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5220 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5221 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5224 if (i == ARRAY_SIZE(trace_clocks))
5227 mutex_lock(&trace_types_lock);
5231 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5234 * New clock may not be consistent with the previous clock.
5235 * Reset the buffer so that it doesn't have incomparable timestamps.
5237 tracing_reset_online_cpus(&tr->trace_buffer);
5239 #ifdef CONFIG_TRACER_MAX_TRACE
5240 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5241 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5242 tracing_reset_online_cpus(&tr->max_buffer);
5245 mutex_unlock(&trace_types_lock);
5250 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5251 size_t cnt, loff_t *fpos)
5253 struct seq_file *m = filp->private_data;
5254 struct trace_array *tr = m->private;
5256 const char *clockstr;
5259 if (cnt >= sizeof(buf))
5262 if (copy_from_user(&buf, ubuf, cnt))
5267 clockstr = strstrip(buf);
5269 ret = tracing_set_clock(tr, clockstr);
5278 static int tracing_clock_open(struct inode *inode, struct file *file)
5280 struct trace_array *tr = inode->i_private;
5283 if (tracing_disabled)
5286 if (trace_array_get(tr))
5289 ret = single_open(file, tracing_clock_show, inode->i_private);
5291 trace_array_put(tr);
5296 struct ftrace_buffer_info {
5297 struct trace_iterator iter;
5302 #ifdef CONFIG_TRACER_SNAPSHOT
5303 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5305 struct trace_array *tr = inode->i_private;
5306 struct trace_iterator *iter;
5310 if (trace_array_get(tr) < 0)
5313 if (file->f_mode & FMODE_READ) {
5314 iter = __tracing_open(inode, file, true);
5316 ret = PTR_ERR(iter);
5318 /* Writes still need the seq_file to hold the private data */
5320 m = kzalloc(sizeof(*m), GFP_KERNEL);
5323 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5331 iter->trace_buffer = &tr->max_buffer;
5332 iter->cpu_file = tracing_get_cpu(inode);
5334 file->private_data = m;
5338 trace_array_put(tr);
5344 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5347 struct seq_file *m = filp->private_data;
5348 struct trace_iterator *iter = m->private;
5349 struct trace_array *tr = iter->tr;
5353 ret = tracing_update_buffers();
5357 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5361 mutex_lock(&trace_types_lock);
5363 if (tr->current_trace->use_max_tr) {
5370 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5374 if (tr->allocated_snapshot)
5378 /* Only allow per-cpu swap if the ring buffer supports it */
5379 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5380 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5385 if (!tr->allocated_snapshot) {
5386 ret = alloc_snapshot(tr);
5390 local_irq_disable();
5391 /* Now, we're going to swap */
5392 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5393 update_max_tr(tr, current, smp_processor_id());
5395 update_max_tr_single(tr, current, iter->cpu_file);
5399 if (tr->allocated_snapshot) {
5400 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5401 tracing_reset_online_cpus(&tr->max_buffer);
5403 tracing_reset(&tr->max_buffer, iter->cpu_file);
5413 mutex_unlock(&trace_types_lock);
5417 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5419 struct seq_file *m = file->private_data;
5422 ret = tracing_release(inode, file);
5424 if (file->f_mode & FMODE_READ)
5427 /* If write only, the seq_file is just a stub */
5435 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5436 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5437 size_t count, loff_t *ppos);
5438 static int tracing_buffers_release(struct inode *inode, struct file *file);
5439 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5440 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5442 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5444 struct ftrace_buffer_info *info;
5447 ret = tracing_buffers_open(inode, filp);
5451 info = filp->private_data;
5453 if (info->iter.trace->use_max_tr) {
5454 tracing_buffers_release(inode, filp);
5458 info->iter.snapshot = true;
5459 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5464 #endif /* CONFIG_TRACER_SNAPSHOT */
5467 static const struct file_operations tracing_thresh_fops = {
5468 .open = tracing_open_generic,
5469 .read = tracing_thresh_read,
5470 .write = tracing_thresh_write,
5471 .llseek = generic_file_llseek,
5474 #ifdef CONFIG_TRACER_MAX_TRACE
5475 static const struct file_operations tracing_max_lat_fops = {
5476 .open = tracing_open_generic,
5477 .read = tracing_max_lat_read,
5478 .write = tracing_max_lat_write,
5479 .llseek = generic_file_llseek,
5483 static const struct file_operations set_tracer_fops = {
5484 .open = tracing_open_generic,
5485 .read = tracing_set_trace_read,
5486 .write = tracing_set_trace_write,
5487 .llseek = generic_file_llseek,
5490 static const struct file_operations tracing_pipe_fops = {
5491 .open = tracing_open_pipe,
5492 .poll = tracing_poll_pipe,
5493 .read = tracing_read_pipe,
5494 .splice_read = tracing_splice_read_pipe,
5495 .release = tracing_release_pipe,
5496 .llseek = no_llseek,
5499 static const struct file_operations tracing_entries_fops = {
5500 .open = tracing_open_generic_tr,
5501 .read = tracing_entries_read,
5502 .write = tracing_entries_write,
5503 .llseek = generic_file_llseek,
5504 .release = tracing_release_generic_tr,
5507 static const struct file_operations tracing_total_entries_fops = {
5508 .open = tracing_open_generic_tr,
5509 .read = tracing_total_entries_read,
5510 .llseek = generic_file_llseek,
5511 .release = tracing_release_generic_tr,
5514 static const struct file_operations tracing_free_buffer_fops = {
5515 .open = tracing_open_generic_tr,
5516 .write = tracing_free_buffer_write,
5517 .release = tracing_free_buffer_release,
5520 static const struct file_operations tracing_mark_fops = {
5521 .open = tracing_open_generic_tr,
5522 .write = tracing_mark_write,
5523 .llseek = generic_file_llseek,
5524 .release = tracing_release_generic_tr,
5527 static const struct file_operations trace_clock_fops = {
5528 .open = tracing_clock_open,
5530 .llseek = seq_lseek,
5531 .release = tracing_single_release_tr,
5532 .write = tracing_clock_write,
5535 #ifdef CONFIG_TRACER_SNAPSHOT
5536 static const struct file_operations snapshot_fops = {
5537 .open = tracing_snapshot_open,
5539 .write = tracing_snapshot_write,
5540 .llseek = tracing_lseek,
5541 .release = tracing_snapshot_release,
5544 static const struct file_operations snapshot_raw_fops = {
5545 .open = snapshot_raw_open,
5546 .read = tracing_buffers_read,
5547 .release = tracing_buffers_release,
5548 .splice_read = tracing_buffers_splice_read,
5549 .llseek = no_llseek,
5552 #endif /* CONFIG_TRACER_SNAPSHOT */
5554 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5556 struct trace_array *tr = inode->i_private;
5557 struct ftrace_buffer_info *info;
5560 if (tracing_disabled)
5563 if (trace_array_get(tr) < 0)
5566 info = kzalloc(sizeof(*info), GFP_KERNEL);
5568 trace_array_put(tr);
5572 mutex_lock(&trace_types_lock);
5575 info->iter.cpu_file = tracing_get_cpu(inode);
5576 info->iter.trace = tr->current_trace;
5577 info->iter.trace_buffer = &tr->trace_buffer;
5579 /* Force reading ring buffer for first read */
5580 info->read = (unsigned int)-1;
5582 filp->private_data = info;
5584 tr->current_trace->ref++;
5586 mutex_unlock(&trace_types_lock);
5588 ret = nonseekable_open(inode, filp);
5590 trace_array_put(tr);
5596 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5598 struct ftrace_buffer_info *info = filp->private_data;
5599 struct trace_iterator *iter = &info->iter;
5601 return trace_poll(iter, filp, poll_table);
5605 tracing_buffers_read(struct file *filp, char __user *ubuf,
5606 size_t count, loff_t *ppos)
5608 struct ftrace_buffer_info *info = filp->private_data;
5609 struct trace_iterator *iter = &info->iter;
5616 #ifdef CONFIG_TRACER_MAX_TRACE
5617 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5622 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5627 /* Do we have previous read data to read? */
5628 if (info->read < PAGE_SIZE)
5632 trace_access_lock(iter->cpu_file);
5633 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5637 trace_access_unlock(iter->cpu_file);
5640 if (trace_empty(iter)) {
5641 if ((filp->f_flags & O_NONBLOCK))
5644 ret = wait_on_pipe(iter, false);
5655 size = PAGE_SIZE - info->read;
5659 ret = copy_to_user(ubuf, info->spare + info->read, size);
5671 static int tracing_buffers_release(struct inode *inode, struct file *file)
5673 struct ftrace_buffer_info *info = file->private_data;
5674 struct trace_iterator *iter = &info->iter;
5676 mutex_lock(&trace_types_lock);
5678 iter->tr->current_trace->ref--;
5680 __trace_array_put(iter->tr);
5683 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5686 mutex_unlock(&trace_types_lock);
5692 struct ring_buffer *buffer;
5697 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5698 struct pipe_buffer *buf)
5700 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5705 ring_buffer_free_read_page(ref->buffer, ref->page);
5710 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5711 struct pipe_buffer *buf)
5713 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5718 /* Pipe buffer operations for a buffer. */
5719 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5721 .confirm = generic_pipe_buf_confirm,
5722 .release = buffer_pipe_buf_release,
5723 .steal = generic_pipe_buf_steal,
5724 .get = buffer_pipe_buf_get,
5728 * Callback from splice_to_pipe(), if we need to release some pages
5729 * at the end of the spd in case we error'ed out in filling the pipe.
5731 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5733 struct buffer_ref *ref =
5734 (struct buffer_ref *)spd->partial[i].private;
5739 ring_buffer_free_read_page(ref->buffer, ref->page);
5741 spd->partial[i].private = 0;
5745 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5746 struct pipe_inode_info *pipe, size_t len,
5749 struct ftrace_buffer_info *info = file->private_data;
5750 struct trace_iterator *iter = &info->iter;
5751 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5752 struct page *pages_def[PIPE_DEF_BUFFERS];
5753 struct splice_pipe_desc spd = {
5755 .partial = partial_def,
5756 .nr_pages_max = PIPE_DEF_BUFFERS,
5758 .ops = &buffer_pipe_buf_ops,
5759 .spd_release = buffer_spd_release,
5761 struct buffer_ref *ref;
5762 int entries, size, i;
5765 #ifdef CONFIG_TRACER_MAX_TRACE
5766 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5770 if (*ppos & (PAGE_SIZE - 1))
5773 if (len & (PAGE_SIZE - 1)) {
5774 if (len < PAGE_SIZE)
5779 if (splice_grow_spd(pipe, &spd))
5783 trace_access_lock(iter->cpu_file);
5784 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5786 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5790 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5797 ref->buffer = iter->trace_buffer->buffer;
5798 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5805 r = ring_buffer_read_page(ref->buffer, &ref->page,
5806 len, iter->cpu_file, 1);
5808 ring_buffer_free_read_page(ref->buffer, ref->page);
5814 * zero out any left over data, this is going to
5817 size = ring_buffer_page_len(ref->page);
5818 if (size < PAGE_SIZE)
5819 memset(ref->page + size, 0, PAGE_SIZE - size);
5821 page = virt_to_page(ref->page);
5823 spd.pages[i] = page;
5824 spd.partial[i].len = PAGE_SIZE;
5825 spd.partial[i].offset = 0;
5826 spd.partial[i].private = (unsigned long)ref;
5830 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5833 trace_access_unlock(iter->cpu_file);
5836 /* did we read anything? */
5837 if (!spd.nr_pages) {
5842 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5845 ret = wait_on_pipe(iter, true);
5852 ret = splice_to_pipe(pipe, &spd);
5854 splice_shrink_spd(&spd);
5859 static const struct file_operations tracing_buffers_fops = {
5860 .open = tracing_buffers_open,
5861 .read = tracing_buffers_read,
5862 .poll = tracing_buffers_poll,
5863 .release = tracing_buffers_release,
5864 .splice_read = tracing_buffers_splice_read,
5865 .llseek = no_llseek,
5869 tracing_stats_read(struct file *filp, char __user *ubuf,
5870 size_t count, loff_t *ppos)
5872 struct inode *inode = file_inode(filp);
5873 struct trace_array *tr = inode->i_private;
5874 struct trace_buffer *trace_buf = &tr->trace_buffer;
5875 int cpu = tracing_get_cpu(inode);
5876 struct trace_seq *s;
5878 unsigned long long t;
5879 unsigned long usec_rem;
5881 s = kmalloc(sizeof(*s), GFP_KERNEL);
5887 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5888 trace_seq_printf(s, "entries: %ld\n", cnt);
5890 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5891 trace_seq_printf(s, "overrun: %ld\n", cnt);
5893 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5894 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5896 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5897 trace_seq_printf(s, "bytes: %ld\n", cnt);
5899 if (trace_clocks[tr->clock_id].in_ns) {
5900 /* local or global for trace_clock */
5901 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5902 usec_rem = do_div(t, USEC_PER_SEC);
5903 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5906 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5907 usec_rem = do_div(t, USEC_PER_SEC);
5908 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5910 /* counter or tsc mode for trace_clock */
5911 trace_seq_printf(s, "oldest event ts: %llu\n",
5912 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5914 trace_seq_printf(s, "now ts: %llu\n",
5915 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5918 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5919 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5921 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5922 trace_seq_printf(s, "read events: %ld\n", cnt);
5924 count = simple_read_from_buffer(ubuf, count, ppos,
5925 s->buffer, trace_seq_used(s));
5932 static const struct file_operations tracing_stats_fops = {
5933 .open = tracing_open_generic_tr,
5934 .read = tracing_stats_read,
5935 .llseek = generic_file_llseek,
5936 .release = tracing_release_generic_tr,
5939 #ifdef CONFIG_DYNAMIC_FTRACE
5941 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5947 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5948 size_t cnt, loff_t *ppos)
5950 static char ftrace_dyn_info_buffer[1024];
5951 static DEFINE_MUTEX(dyn_info_mutex);
5952 unsigned long *p = filp->private_data;
5953 char *buf = ftrace_dyn_info_buffer;
5954 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5957 mutex_lock(&dyn_info_mutex);
5958 r = sprintf(buf, "%ld ", *p);
5960 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5963 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5965 mutex_unlock(&dyn_info_mutex);
5970 static const struct file_operations tracing_dyn_info_fops = {
5971 .open = tracing_open_generic,
5972 .read = tracing_read_dyn_info,
5973 .llseek = generic_file_llseek,
5975 #endif /* CONFIG_DYNAMIC_FTRACE */
5977 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5979 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5985 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5987 unsigned long *count = (long *)data;
5999 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6000 struct ftrace_probe_ops *ops, void *data)
6002 long count = (long)data;
6004 seq_printf(m, "%ps:", (void *)ip);
6006 seq_puts(m, "snapshot");
6009 seq_puts(m, ":unlimited\n");
6011 seq_printf(m, ":count=%ld\n", count);
6016 static struct ftrace_probe_ops snapshot_probe_ops = {
6017 .func = ftrace_snapshot,
6018 .print = ftrace_snapshot_print,
6021 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6022 .func = ftrace_count_snapshot,
6023 .print = ftrace_snapshot_print,
6027 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6028 char *glob, char *cmd, char *param, int enable)
6030 struct ftrace_probe_ops *ops;
6031 void *count = (void *)-1;
6035 /* hash funcs only work with set_ftrace_filter */
6039 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6041 if (glob[0] == '!') {
6042 unregister_ftrace_function_probe_func(glob+1, ops);
6049 number = strsep(¶m, ":");
6051 if (!strlen(number))
6055 * We use the callback data field (which is a pointer)
6058 ret = kstrtoul(number, 0, (unsigned long *)&count);
6063 ret = alloc_snapshot(&global_trace);
6067 ret = register_ftrace_function_probe(glob, ops, count);
6070 return ret < 0 ? ret : 0;
6073 static struct ftrace_func_command ftrace_snapshot_cmd = {
6075 .func = ftrace_trace_snapshot_callback,
6078 static __init int register_snapshot_cmd(void)
6080 return register_ftrace_command(&ftrace_snapshot_cmd);
6083 static inline __init int register_snapshot_cmd(void) { return 0; }
6084 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6086 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6088 if (WARN_ON(!tr->dir))
6089 return ERR_PTR(-ENODEV);
6091 /* Top directory uses NULL as the parent */
6092 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6095 /* All sub buffers have a descriptor */
6099 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6101 struct dentry *d_tracer;
6104 return tr->percpu_dir;
6106 d_tracer = tracing_get_dentry(tr);
6107 if (IS_ERR(d_tracer))
6110 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6112 WARN_ONCE(!tr->percpu_dir,
6113 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6115 return tr->percpu_dir;
6118 static struct dentry *
6119 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6120 void *data, long cpu, const struct file_operations *fops)
6122 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6124 if (ret) /* See tracing_get_cpu() */
6125 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6130 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6132 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6133 struct dentry *d_cpu;
6134 char cpu_dir[30]; /* 30 characters should be more than enough */
6139 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6140 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6142 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
6146 /* per cpu trace_pipe */
6147 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6148 tr, cpu, &tracing_pipe_fops);
6151 trace_create_cpu_file("trace", 0644, d_cpu,
6152 tr, cpu, &tracing_fops);
6154 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6155 tr, cpu, &tracing_buffers_fops);
6157 trace_create_cpu_file("stats", 0444, d_cpu,
6158 tr, cpu, &tracing_stats_fops);
6160 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6161 tr, cpu, &tracing_entries_fops);
6163 #ifdef CONFIG_TRACER_SNAPSHOT
6164 trace_create_cpu_file("snapshot", 0644, d_cpu,
6165 tr, cpu, &snapshot_fops);
6167 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6168 tr, cpu, &snapshot_raw_fops);
6172 #ifdef CONFIG_FTRACE_SELFTEST
6173 /* Let selftest have access to static functions in this file */
6174 #include "trace_selftest.c"
6178 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6181 struct trace_option_dentry *topt = filp->private_data;
6184 if (topt->flags->val & topt->opt->bit)
6189 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6193 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6196 struct trace_option_dentry *topt = filp->private_data;
6200 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6204 if (val != 0 && val != 1)
6207 if (!!(topt->flags->val & topt->opt->bit) != val) {
6208 mutex_lock(&trace_types_lock);
6209 ret = __set_tracer_option(topt->tr, topt->flags,
6211 mutex_unlock(&trace_types_lock);
6222 static const struct file_operations trace_options_fops = {
6223 .open = tracing_open_generic,
6224 .read = trace_options_read,
6225 .write = trace_options_write,
6226 .llseek = generic_file_llseek,
6230 * In order to pass in both the trace_array descriptor as well as the index
6231 * to the flag that the trace option file represents, the trace_array
6232 * has a character array of trace_flags_index[], which holds the index
6233 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6234 * The address of this character array is passed to the flag option file
6235 * read/write callbacks.
6237 * In order to extract both the index and the trace_array descriptor,
6238 * get_tr_index() uses the following algorithm.
6242 * As the pointer itself contains the address of the index (remember
6245 * Then to get the trace_array descriptor, by subtracting that index
6246 * from the ptr, we get to the start of the index itself.
6248 * ptr - idx == &index[0]
6250 * Then a simple container_of() from that pointer gets us to the
6251 * trace_array descriptor.
6253 static void get_tr_index(void *data, struct trace_array **ptr,
6254 unsigned int *pindex)
6256 *pindex = *(unsigned char *)data;
6258 *ptr = container_of(data - *pindex, struct trace_array,
6263 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6266 void *tr_index = filp->private_data;
6267 struct trace_array *tr;
6271 get_tr_index(tr_index, &tr, &index);
6273 if (tr->trace_flags & (1 << index))
6278 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6282 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6285 void *tr_index = filp->private_data;
6286 struct trace_array *tr;
6291 get_tr_index(tr_index, &tr, &index);
6293 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6297 if (val != 0 && val != 1)
6300 mutex_lock(&trace_types_lock);
6301 ret = set_tracer_flag(tr, 1 << index, val);
6302 mutex_unlock(&trace_types_lock);
6312 static const struct file_operations trace_options_core_fops = {
6313 .open = tracing_open_generic,
6314 .read = trace_options_core_read,
6315 .write = trace_options_core_write,
6316 .llseek = generic_file_llseek,
6319 struct dentry *trace_create_file(const char *name,
6321 struct dentry *parent,
6323 const struct file_operations *fops)
6327 ret = tracefs_create_file(name, mode, parent, data, fops);
6329 pr_warning("Could not create tracefs '%s' entry\n", name);
6335 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6337 struct dentry *d_tracer;
6342 d_tracer = tracing_get_dentry(tr);
6343 if (IS_ERR(d_tracer))
6346 tr->options = tracefs_create_dir("options", d_tracer);
6348 pr_warning("Could not create tracefs directory 'options'\n");
6356 create_trace_option_file(struct trace_array *tr,
6357 struct trace_option_dentry *topt,
6358 struct tracer_flags *flags,
6359 struct tracer_opt *opt)
6361 struct dentry *t_options;
6363 t_options = trace_options_init_dentry(tr);
6367 topt->flags = flags;
6371 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6372 &trace_options_fops);
6377 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6379 struct trace_option_dentry *topts;
6380 struct trace_options *tr_topts;
6381 struct tracer_flags *flags;
6382 struct tracer_opt *opts;
6389 flags = tracer->flags;
6391 if (!flags || !flags->opts)
6395 * If this is an instance, only create flags for tracers
6396 * the instance may have.
6398 if (!trace_ok_for_array(tracer, tr))
6401 for (i = 0; i < tr->nr_topts; i++) {
6403 * Check if these flags have already been added.
6404 * Some tracers share flags.
6406 if (tr->topts[i].tracer->flags == tracer->flags)
6412 for (cnt = 0; opts[cnt].name; cnt++)
6415 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6419 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6426 tr->topts = tr_topts;
6427 tr->topts[tr->nr_topts].tracer = tracer;
6428 tr->topts[tr->nr_topts].topts = topts;
6431 for (cnt = 0; opts[cnt].name; cnt++) {
6432 create_trace_option_file(tr, &topts[cnt], flags,
6434 WARN_ONCE(topts[cnt].entry == NULL,
6435 "Failed to create trace option: %s",
6440 static struct dentry *
6441 create_trace_option_core_file(struct trace_array *tr,
6442 const char *option, long index)
6444 struct dentry *t_options;
6446 t_options = trace_options_init_dentry(tr);
6450 return trace_create_file(option, 0644, t_options,
6451 (void *)&tr->trace_flags_index[index],
6452 &trace_options_core_fops);
6455 static void create_trace_options_dir(struct trace_array *tr)
6457 struct dentry *t_options;
6458 bool top_level = tr == &global_trace;
6461 t_options = trace_options_init_dentry(tr);
6465 for (i = 0; trace_options[i]; i++) {
6467 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6468 create_trace_option_core_file(tr, trace_options[i], i);
6473 rb_simple_read(struct file *filp, char __user *ubuf,
6474 size_t cnt, loff_t *ppos)
6476 struct trace_array *tr = filp->private_data;
6480 r = tracer_tracing_is_on(tr);
6481 r = sprintf(buf, "%d\n", r);
6483 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6487 rb_simple_write(struct file *filp, const char __user *ubuf,
6488 size_t cnt, loff_t *ppos)
6490 struct trace_array *tr = filp->private_data;
6491 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6495 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6500 mutex_lock(&trace_types_lock);
6502 tracer_tracing_on(tr);
6503 if (tr->current_trace->start)
6504 tr->current_trace->start(tr);
6506 tracer_tracing_off(tr);
6507 if (tr->current_trace->stop)
6508 tr->current_trace->stop(tr);
6510 mutex_unlock(&trace_types_lock);
6518 static const struct file_operations rb_simple_fops = {
6519 .open = tracing_open_generic_tr,
6520 .read = rb_simple_read,
6521 .write = rb_simple_write,
6522 .release = tracing_release_generic_tr,
6523 .llseek = default_llseek,
6526 struct dentry *trace_instance_dir;
6529 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6532 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6534 enum ring_buffer_flags rb_flags;
6536 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6540 buf->buffer = ring_buffer_alloc(size, rb_flags);
6544 buf->data = alloc_percpu(struct trace_array_cpu);
6546 ring_buffer_free(buf->buffer);
6550 /* Allocate the first page for all buffers */
6551 set_buffer_entries(&tr->trace_buffer,
6552 ring_buffer_size(tr->trace_buffer.buffer, 0));
6557 static int allocate_trace_buffers(struct trace_array *tr, int size)
6561 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6565 #ifdef CONFIG_TRACER_MAX_TRACE
6566 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6567 allocate_snapshot ? size : 1);
6569 ring_buffer_free(tr->trace_buffer.buffer);
6570 free_percpu(tr->trace_buffer.data);
6573 tr->allocated_snapshot = allocate_snapshot;
6576 * Only the top level trace array gets its snapshot allocated
6577 * from the kernel command line.
6579 allocate_snapshot = false;
6584 static void free_trace_buffer(struct trace_buffer *buf)
6587 ring_buffer_free(buf->buffer);
6589 free_percpu(buf->data);
6594 static void free_trace_buffers(struct trace_array *tr)
6599 free_trace_buffer(&tr->trace_buffer);
6601 #ifdef CONFIG_TRACER_MAX_TRACE
6602 free_trace_buffer(&tr->max_buffer);
6606 static void init_trace_flags_index(struct trace_array *tr)
6610 /* Used by the trace options files */
6611 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6612 tr->trace_flags_index[i] = i;
6615 static void __update_tracer_options(struct trace_array *tr)
6619 for (t = trace_types; t; t = t->next)
6620 add_tracer_options(tr, t);
6623 static void update_tracer_options(struct trace_array *tr)
6625 mutex_lock(&trace_types_lock);
6626 __update_tracer_options(tr);
6627 mutex_unlock(&trace_types_lock);
6630 static int instance_mkdir(const char *name)
6632 struct trace_array *tr;
6635 mutex_lock(&trace_types_lock);
6638 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6639 if (tr->name && strcmp(tr->name, name) == 0)
6644 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6648 tr->name = kstrdup(name, GFP_KERNEL);
6652 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6655 tr->trace_flags = global_trace.trace_flags;
6657 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6659 raw_spin_lock_init(&tr->start_lock);
6661 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6663 tr->current_trace = &nop_trace;
6665 INIT_LIST_HEAD(&tr->systems);
6666 INIT_LIST_HEAD(&tr->events);
6668 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6671 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6675 ret = event_trace_add_tracer(tr->dir, tr);
6677 tracefs_remove_recursive(tr->dir);
6681 init_tracer_tracefs(tr, tr->dir);
6682 init_trace_flags_index(tr);
6683 __update_tracer_options(tr);
6685 list_add(&tr->list, &ftrace_trace_arrays);
6687 mutex_unlock(&trace_types_lock);
6692 free_trace_buffers(tr);
6693 free_cpumask_var(tr->tracing_cpumask);
6698 mutex_unlock(&trace_types_lock);
6704 static int instance_rmdir(const char *name)
6706 struct trace_array *tr;
6711 mutex_lock(&trace_types_lock);
6714 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6715 if (tr->name && strcmp(tr->name, name) == 0) {
6724 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6727 list_del(&tr->list);
6729 tracing_set_nop(tr);
6730 event_trace_del_tracer(tr);
6731 ftrace_destroy_function_files(tr);
6732 tracefs_remove_recursive(tr->dir);
6733 free_trace_buffers(tr);
6735 for (i = 0; i < tr->nr_topts; i++) {
6736 kfree(tr->topts[i].topts);
6746 mutex_unlock(&trace_types_lock);
6751 static __init void create_trace_instances(struct dentry *d_tracer)
6753 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6756 if (WARN_ON(!trace_instance_dir))
6761 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6765 trace_create_file("available_tracers", 0444, d_tracer,
6766 tr, &show_traces_fops);
6768 trace_create_file("current_tracer", 0644, d_tracer,
6769 tr, &set_tracer_fops);
6771 trace_create_file("tracing_cpumask", 0644, d_tracer,
6772 tr, &tracing_cpumask_fops);
6774 trace_create_file("trace_options", 0644, d_tracer,
6775 tr, &tracing_iter_fops);
6777 trace_create_file("trace", 0644, d_tracer,
6780 trace_create_file("trace_pipe", 0444, d_tracer,
6781 tr, &tracing_pipe_fops);
6783 trace_create_file("buffer_size_kb", 0644, d_tracer,
6784 tr, &tracing_entries_fops);
6786 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6787 tr, &tracing_total_entries_fops);
6789 trace_create_file("free_buffer", 0200, d_tracer,
6790 tr, &tracing_free_buffer_fops);
6792 trace_create_file("trace_marker", 0220, d_tracer,
6793 tr, &tracing_mark_fops);
6795 trace_create_file("trace_clock", 0644, d_tracer, tr,
6798 trace_create_file("tracing_on", 0644, d_tracer,
6799 tr, &rb_simple_fops);
6801 create_trace_options_dir(tr);
6803 #ifdef CONFIG_TRACER_MAX_TRACE
6804 trace_create_file("tracing_max_latency", 0644, d_tracer,
6805 &tr->max_latency, &tracing_max_lat_fops);
6808 if (ftrace_create_function_files(tr, d_tracer))
6809 WARN(1, "Could not allocate function filter files");
6811 #ifdef CONFIG_TRACER_SNAPSHOT
6812 trace_create_file("snapshot", 0644, d_tracer,
6813 tr, &snapshot_fops);
6816 for_each_tracing_cpu(cpu)
6817 tracing_init_tracefs_percpu(tr, cpu);
6821 static struct vfsmount *trace_automount(void *ingore)
6823 struct vfsmount *mnt;
6824 struct file_system_type *type;
6827 * To maintain backward compatibility for tools that mount
6828 * debugfs to get to the tracing facility, tracefs is automatically
6829 * mounted to the debugfs/tracing directory.
6831 type = get_fs_type("tracefs");
6834 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6835 put_filesystem(type);
6844 * tracing_init_dentry - initialize top level trace array
6846 * This is called when creating files or directories in the tracing
6847 * directory. It is called via fs_initcall() by any of the boot up code
6848 * and expects to return the dentry of the top level tracing directory.
6850 struct dentry *tracing_init_dentry(void)
6852 struct trace_array *tr = &global_trace;
6854 /* The top level trace array uses NULL as parent */
6858 if (WARN_ON(!tracefs_initialized()) ||
6859 (IS_ENABLED(CONFIG_DEBUG_FS) &&
6860 WARN_ON(!debugfs_initialized())))
6861 return ERR_PTR(-ENODEV);
6864 * As there may still be users that expect the tracing
6865 * files to exist in debugfs/tracing, we must automount
6866 * the tracefs file system there, so older tools still
6867 * work with the newer kerenl.
6869 tr->dir = debugfs_create_automount("tracing", NULL,
6870 trace_automount, NULL);
6872 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6873 return ERR_PTR(-ENOMEM);
6879 extern struct trace_enum_map *__start_ftrace_enum_maps[];
6880 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6882 static void __init trace_enum_init(void)
6886 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6887 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
6890 #ifdef CONFIG_MODULES
6891 static void trace_module_add_enums(struct module *mod)
6893 if (!mod->num_trace_enums)
6897 * Modules with bad taint do not have events created, do
6898 * not bother with enums either.
6900 if (trace_module_has_bad_taint(mod))
6903 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
6906 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
6907 static void trace_module_remove_enums(struct module *mod)
6909 union trace_enum_map_item *map;
6910 union trace_enum_map_item **last = &trace_enum_maps;
6912 if (!mod->num_trace_enums)
6915 mutex_lock(&trace_enum_mutex);
6917 map = trace_enum_maps;
6920 if (map->head.mod == mod)
6922 map = trace_enum_jmp_to_tail(map);
6923 last = &map->tail.next;
6924 map = map->tail.next;
6929 *last = trace_enum_jmp_to_tail(map)->tail.next;
6932 mutex_unlock(&trace_enum_mutex);
6935 static inline void trace_module_remove_enums(struct module *mod) { }
6936 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6938 static int trace_module_notify(struct notifier_block *self,
6939 unsigned long val, void *data)
6941 struct module *mod = data;
6944 case MODULE_STATE_COMING:
6945 trace_module_add_enums(mod);
6947 case MODULE_STATE_GOING:
6948 trace_module_remove_enums(mod);
6955 static struct notifier_block trace_module_nb = {
6956 .notifier_call = trace_module_notify,
6959 #endif /* CONFIG_MODULES */
6961 static __init int tracer_init_tracefs(void)
6963 struct dentry *d_tracer;
6965 trace_access_lock_init();
6967 d_tracer = tracing_init_dentry();
6968 if (IS_ERR(d_tracer))
6971 init_tracer_tracefs(&global_trace, d_tracer);
6973 trace_create_file("tracing_thresh", 0644, d_tracer,
6974 &global_trace, &tracing_thresh_fops);
6976 trace_create_file("README", 0444, d_tracer,
6977 NULL, &tracing_readme_fops);
6979 trace_create_file("saved_cmdlines", 0444, d_tracer,
6980 NULL, &tracing_saved_cmdlines_fops);
6982 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6983 NULL, &tracing_saved_cmdlines_size_fops);
6987 trace_create_enum_file(d_tracer);
6989 #ifdef CONFIG_MODULES
6990 register_module_notifier(&trace_module_nb);
6993 #ifdef CONFIG_DYNAMIC_FTRACE
6994 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6995 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6998 create_trace_instances(d_tracer);
7000 update_tracer_options(&global_trace);
7005 static int trace_panic_handler(struct notifier_block *this,
7006 unsigned long event, void *unused)
7008 if (ftrace_dump_on_oops)
7009 ftrace_dump(ftrace_dump_on_oops);
7013 static struct notifier_block trace_panic_notifier = {
7014 .notifier_call = trace_panic_handler,
7016 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7019 static int trace_die_handler(struct notifier_block *self,
7025 if (ftrace_dump_on_oops)
7026 ftrace_dump(ftrace_dump_on_oops);
7034 static struct notifier_block trace_die_notifier = {
7035 .notifier_call = trace_die_handler,
7040 * printk is set to max of 1024, we really don't need it that big.
7041 * Nothing should be printing 1000 characters anyway.
7043 #define TRACE_MAX_PRINT 1000
7046 * Define here KERN_TRACE so that we have one place to modify
7047 * it if we decide to change what log level the ftrace dump
7050 #define KERN_TRACE KERN_EMERG
7053 trace_printk_seq(struct trace_seq *s)
7055 /* Probably should print a warning here. */
7056 if (s->seq.len >= TRACE_MAX_PRINT)
7057 s->seq.len = TRACE_MAX_PRINT;
7060 * More paranoid code. Although the buffer size is set to
7061 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7062 * an extra layer of protection.
7064 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7065 s->seq.len = s->seq.size - 1;
7067 /* should be zero ended, but we are paranoid. */
7068 s->buffer[s->seq.len] = 0;
7070 printk(KERN_TRACE "%s", s->buffer);
7075 void trace_init_global_iter(struct trace_iterator *iter)
7077 iter->tr = &global_trace;
7078 iter->trace = iter->tr->current_trace;
7079 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7080 iter->trace_buffer = &global_trace.trace_buffer;
7082 if (iter->trace && iter->trace->open)
7083 iter->trace->open(iter);
7085 /* Annotate start of buffers if we had overruns */
7086 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7087 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7089 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7090 if (trace_clocks[iter->tr->clock_id].in_ns)
7091 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7094 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7096 /* use static because iter can be a bit big for the stack */
7097 static struct trace_iterator iter;
7098 static atomic_t dump_running;
7099 struct trace_array *tr = &global_trace;
7100 unsigned int old_userobj;
7101 unsigned long flags;
7104 /* Only allow one dump user at a time. */
7105 if (atomic_inc_return(&dump_running) != 1) {
7106 atomic_dec(&dump_running);
7111 * Always turn off tracing when we dump.
7112 * We don't need to show trace output of what happens
7113 * between multiple crashes.
7115 * If the user does a sysrq-z, then they can re-enable
7116 * tracing with echo 1 > tracing_on.
7120 local_irq_save(flags);
7122 /* Simulate the iterator */
7123 trace_init_global_iter(&iter);
7125 for_each_tracing_cpu(cpu) {
7126 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7129 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7131 /* don't look at user memory in panic mode */
7132 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7134 switch (oops_dump_mode) {
7136 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7139 iter.cpu_file = raw_smp_processor_id();
7144 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7145 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7148 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7150 /* Did function tracer already get disabled? */
7151 if (ftrace_is_dead()) {
7152 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7153 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7157 * We need to stop all tracing on all CPUS to read the
7158 * the next buffer. This is a bit expensive, but is
7159 * not done often. We fill all what we can read,
7160 * and then release the locks again.
7163 while (!trace_empty(&iter)) {
7166 printk(KERN_TRACE "---------------------------------\n");
7170 /* reset all but tr, trace, and overruns */
7171 memset(&iter.seq, 0,
7172 sizeof(struct trace_iterator) -
7173 offsetof(struct trace_iterator, seq));
7174 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7177 if (trace_find_next_entry_inc(&iter) != NULL) {
7180 ret = print_trace_line(&iter);
7181 if (ret != TRACE_TYPE_NO_CONSUME)
7182 trace_consume(&iter);
7184 touch_nmi_watchdog();
7186 trace_printk_seq(&iter.seq);
7190 printk(KERN_TRACE " (ftrace buffer empty)\n");
7192 printk(KERN_TRACE "---------------------------------\n");
7195 tr->trace_flags |= old_userobj;
7197 for_each_tracing_cpu(cpu) {
7198 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7200 atomic_dec(&dump_running);
7201 local_irq_restore(flags);
7203 EXPORT_SYMBOL_GPL(ftrace_dump);
7205 __init static int tracer_alloc_buffers(void)
7211 * Make sure we don't accidently add more trace options
7212 * than we have bits for.
7214 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7216 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7219 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7220 goto out_free_buffer_mask;
7222 /* Only allocate trace_printk buffers if a trace_printk exists */
7223 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7224 /* Must be called before global_trace.buffer is allocated */
7225 trace_printk_init_buffers();
7227 /* To save memory, keep the ring buffer size to its minimum */
7228 if (ring_buffer_expanded)
7229 ring_buf_size = trace_buf_size;
7233 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7234 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7236 raw_spin_lock_init(&global_trace.start_lock);
7238 /* Used for event triggers */
7239 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7241 goto out_free_cpumask;
7243 if (trace_create_savedcmd() < 0)
7244 goto out_free_temp_buffer;
7246 /* TODO: make the number of buffers hot pluggable with CPUS */
7247 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7248 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7250 goto out_free_savedcmd;
7253 if (global_trace.buffer_disabled)
7256 if (trace_boot_clock) {
7257 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7259 pr_warning("Trace clock %s not defined, going back to default\n",
7264 * register_tracer() might reference current_trace, so it
7265 * needs to be set before we register anything. This is
7266 * just a bootstrap of current_trace anyway.
7268 global_trace.current_trace = &nop_trace;
7270 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7272 ftrace_init_global_array_ops(&global_trace);
7274 init_trace_flags_index(&global_trace);
7276 register_tracer(&nop_trace);
7278 /* All seems OK, enable tracing */
7279 tracing_disabled = 0;
7281 atomic_notifier_chain_register(&panic_notifier_list,
7282 &trace_panic_notifier);
7284 register_die_notifier(&trace_die_notifier);
7286 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7288 INIT_LIST_HEAD(&global_trace.systems);
7289 INIT_LIST_HEAD(&global_trace.events);
7290 list_add(&global_trace.list, &ftrace_trace_arrays);
7292 apply_trace_boot_options();
7294 register_snapshot_cmd();
7299 free_saved_cmdlines_buffer(savedcmd);
7300 out_free_temp_buffer:
7301 ring_buffer_free(temp_buffer);
7303 free_cpumask_var(global_trace.tracing_cpumask);
7304 out_free_buffer_mask:
7305 free_cpumask_var(tracing_buffer_mask);
7310 void __init trace_init(void)
7312 if (tracepoint_printk) {
7313 tracepoint_print_iter =
7314 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7315 if (WARN_ON(!tracepoint_print_iter))
7316 tracepoint_printk = 0;
7318 tracer_alloc_buffers();
7322 __init static int clear_boot_tracer(void)
7325 * The default tracer at boot buffer is an init section.
7326 * This function is called in lateinit. If we did not
7327 * find the boot tracer, then clear it out, to prevent
7328 * later registration from accessing the buffer that is
7329 * about to be freed.
7331 if (!default_bootup_tracer)
7334 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7335 default_bootup_tracer);
7336 default_bootup_tracer = NULL;
7341 fs_initcall(tracer_init_tracefs);
7342 late_initcall(clear_boot_tracer);