2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 // This version must match the gator daemon version
11 #define PROTOCOL_VERSION 17
12 static unsigned long gator_protocol_version = PROTOCOL_VERSION;
14 #include <linux/slab.h>
15 #include <linux/cpu.h>
16 #include <linux/sched.h>
17 #include <linux/irq.h>
18 #include <linux/vmalloc.h>
19 #include <linux/hardirq.h>
20 #include <linux/highmem.h>
21 #include <linux/pagemap.h>
22 #include <linux/suspend.h>
23 #include <linux/module.h>
24 #include <linux/perf_event.h>
25 #include <linux/utsname.h>
26 #include <linux/kthread.h>
27 #include <asm/stacktrace.h>
28 #include <asm/uaccess.h>
32 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
33 #error kernels prior to 2.6.32 are not supported
36 #if defined(MODULE) && !defined(CONFIG_MODULES)
37 #error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel.
40 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
41 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
44 #ifndef CONFIG_PROFILING
45 #error gator requires the kernel to have CONFIG_PROFILING defined
48 #ifndef CONFIG_HIGH_RES_TIMERS
49 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
53 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
56 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
57 #ifndef CONFIG_PERF_EVENTS
58 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
59 #elif !defined CONFIG_HW_PERF_EVENTS
60 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
64 /******************************************************************************
66 ******************************************************************************/
67 #define SUMMARY_BUFFER_SIZE (1*1024)
68 #define BACKTRACE_BUFFER_SIZE (128*1024)
69 #define NAME_BUFFER_SIZE (64*1024)
70 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
71 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
72 #define ANNOTATE_BUFFER_SIZE (128*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
73 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
74 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
75 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
78 #define UNRESOLVED_COOKIE ~0U
80 #define FRAME_SUMMARY 1
81 #define FRAME_BACKTRACE 2
83 #define FRAME_COUNTER 4
84 #define FRAME_BLOCK_COUNTER 5
85 #define FRAME_ANNOTATE 6
86 #define FRAME_SCHED_TRACE 7
87 #define FRAME_GPU_TRACE 8
90 #define MESSAGE_END_BACKTRACE 1
92 #define MESSAGE_COOKIE 1
93 #define MESSAGE_THREAD_NAME 2
94 #define HRTIMER_CORE_NAME 3
95 #define MESSAGE_LINK 4
97 #define MESSAGE_GPU_START 1
98 #define MESSAGE_GPU_STOP 2
100 #define MESSAGE_SCHED_SWITCH 1
101 #define MESSAGE_SCHED_EXIT 2
102 #define MESSAGE_SCHED_START 3
104 #define MESSAGE_IDLE_ENTER 1
105 #define MESSAGE_IDLE_EXIT 2
107 #define MAXSIZE_PACK32 5
108 #define MAXSIZE_PACK64 10
110 #define FRAME_HEADER_SIZE 3
113 #define PC_REG regs->ARM_pc
114 #elif defined(__aarch64__)
115 #define PC_REG regs->pc
117 #define PC_REG regs->ip
133 /******************************************************************************
135 ******************************************************************************/
136 static unsigned long gator_cpu_cores;
137 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
138 static unsigned long userspace_buffer_size;
139 static unsigned long gator_backtrace_depth;
140 // How often to commit the buffers for live in nanoseconds
141 static u64 gator_live_rate;
143 static unsigned long gator_started;
144 static u64 gator_monotonic_started;
145 static u64 gator_hibernate_time;
146 static unsigned long gator_buffer_opened;
147 static unsigned long gator_timer_count;
148 static unsigned long gator_response_type;
149 static DEFINE_MUTEX(start_mutex);
150 static DEFINE_MUTEX(gator_buffer_mutex);
152 bool event_based_sampling;
154 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
155 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
156 static struct timer_list gator_buffer_wake_up_timer;
157 static bool gator_buffer_wake_stop;
158 static struct task_struct *gator_buffer_wake_thread;
159 static LIST_HEAD(gator_events);
161 static DEFINE_PER_CPU(u64, last_timestamp);
163 static bool printed_monotonic_warning;
165 static bool sent_core_name[NR_CPUS];
167 /******************************************************************************
169 ******************************************************************************/
170 static void buffer_check(int cpu, int buftype, u64 time);
171 static void gator_commit_buffer(int cpu, int buftype, u64 time);
172 static int buffer_bytes_available(int cpu, int buftype);
173 static bool buffer_check_space(int cpu, int buftype, int bytes);
174 static int contiguous_space_available(int cpu, int bufytpe);
175 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
176 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
177 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
178 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
179 static void gator_add_trace(int cpu, unsigned long address);
180 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
181 static u64 gator_get_time(void);
183 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
184 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
185 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
186 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
187 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
188 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
189 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
190 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
191 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
192 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
193 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
194 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
195 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
196 // The buffer. Allocated in gator_op_setup
197 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
198 // The time after which the buffer should be committed for live display
199 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
201 // List of all gator events - new events must be added to this list
202 #define GATOR_EVENTS_LIST \
203 GATOR_EVENT(gator_events_armv6_init) \
204 GATOR_EVENT(gator_events_armv7_init) \
205 GATOR_EVENT(gator_events_block_init) \
206 GATOR_EVENT(gator_events_ccn504_init) \
207 GATOR_EVENT(gator_events_irq_init) \
208 GATOR_EVENT(gator_events_l2c310_init) \
209 GATOR_EVENT(gator_events_mali_init) \
210 GATOR_EVENT(gator_events_mali_t6xx_hw_init) \
211 GATOR_EVENT(gator_events_mali_t6xx_init) \
212 GATOR_EVENT(gator_events_meminfo_init) \
213 GATOR_EVENT(gator_events_mmapped_init) \
214 GATOR_EVENT(gator_events_net_init) \
215 GATOR_EVENT(gator_events_perf_pmu_init) \
216 GATOR_EVENT(gator_events_sched_init) \
217 GATOR_EVENT(gator_events_scorpion_init) \
219 #define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void);
223 static int (*gator_events_list[])(void) = {
224 #define GATOR_EVENT(EVENT_INIT) EVENT_INIT,
229 /******************************************************************************
230 * Application Includes
231 ******************************************************************************/
232 #include "gator_marshaling.c"
233 #include "gator_hrtimer_perf.c"
234 #include "gator_hrtimer_gator.c"
235 #include "gator_cookies.c"
236 #include "gator_annotate.c"
237 #include "gator_trace_sched.c"
238 #include "gator_trace_power.c"
239 #include "gator_trace_gpu.c"
240 #include "gator_backtrace.c"
241 #include "gator_fs.c"
242 #include "gator_pack.c"
244 /******************************************************************************
246 ******************************************************************************/
248 const struct gator_cpu gator_cpus[] = {
251 .core_name = "ARM1136",
252 .pmnc_name = "ARM_ARM11",
253 .dt_name = "arm,arm1136",
258 .core_name = "ARM1156",
259 .pmnc_name = "ARM_ARM11",
260 .dt_name = "arm,arm1156",
265 .core_name = "ARM1176",
266 .pmnc_name = "ARM_ARM11",
267 .dt_name = "arm,arm1176",
271 .cpuid = ARM11MPCORE,
272 .core_name = "ARM11MPCore",
273 .pmnc_name = "ARM_ARM11MPCore",
274 .dt_name = "arm,arm11mpcore",
279 .core_name = "Cortex-A5",
280 .pmu_name = "ARMv7_Cortex_A5",
281 .pmnc_name = "ARM_Cortex-A5",
282 .dt_name = "arm,cortex-a5",
287 .core_name = "Cortex-A7",
288 .pmu_name = "ARMv7_Cortex_A7",
289 .pmnc_name = "ARM_Cortex-A7",
290 .dt_name = "arm,cortex-a7",
295 .core_name = "Cortex-A8",
296 .pmu_name = "ARMv7_Cortex_A8",
297 .pmnc_name = "ARM_Cortex-A8",
298 .dt_name = "arm,cortex-a8",
303 .core_name = "Cortex-A9",
304 .pmu_name = "ARMv7_Cortex_A9",
305 .pmnc_name = "ARM_Cortex-A9",
306 .dt_name = "arm,cortex-a9",
311 .core_name = "Cortex-A12",
312 .pmu_name = "ARMv7_Cortex_A12",
313 .pmnc_name = "ARM_Cortex-A12",
314 .dt_name = "arm,cortex-a12",
319 .core_name = "Cortex-A15",
320 .pmu_name = "ARMv7_Cortex_A15",
321 .pmnc_name = "ARM_Cortex-A15",
322 .dt_name = "arm,cortex-a15",
327 .core_name = "Scorpion",
328 .pmnc_name = "Scorpion",
333 .core_name = "ScorpionMP",
334 .pmnc_name = "ScorpionMP",
339 .core_name = "KraitSIM",
340 .pmnc_name = "Krait",
345 .core_name = "Krait",
346 .pmnc_name = "Krait",
350 .cpuid = KRAIT_S4_PRO,
351 .core_name = "Krait S4 Pro",
352 .pmnc_name = "Krait",
357 .core_name = "Cortex-A53",
358 .pmnc_name = "ARM_Cortex-A53",
359 .dt_name = "arm,cortex-a53",
364 .core_name = "Cortex-A57",
365 .pmnc_name = "ARM_Cortex-A57",
366 .dt_name = "arm,cortex-a57",
371 .core_name = "AArch64",
372 .pmnc_name = "ARM_AArch64",
377 .core_name = "Other",
378 .pmnc_name = "Other",
384 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
388 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
389 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
390 if (gator_cpu->cpuid == cpuid) {
398 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
402 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
403 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
404 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
412 u32 gator_cpuid(void)
414 #if defined(__arm__) || defined(__aarch64__)
416 #if !defined(__aarch64__)
417 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
419 asm volatile("mrs %0, midr_el1" : "=r" (val));
421 return (val >> 4) & 0xfff;
427 static void gator_buffer_wake_up(unsigned long data)
429 wake_up(&gator_buffer_wait);
432 static int gator_buffer_wake_func(void *data)
434 while (!gator_buffer_wake_stop) {
435 set_current_state(TASK_INTERRUPTIBLE);
437 if (gator_buffer_wake_stop) {
441 gator_buffer_wake_up(0);
447 /******************************************************************************
449 ******************************************************************************/
450 static bool buffer_commit_ready(int *cpu, int *buftype)
453 for_each_present_cpu(cpu_x) {
454 for (x = 0; x < NUM_GATOR_BUFS; x++)
455 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
466 /******************************************************************************
468 ******************************************************************************/
469 static int buffer_bytes_available(int cpu, int buftype)
471 int remaining, filled;
473 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
475 filled += gator_buffer_size[buftype];
478 remaining = gator_buffer_size[buftype] - filled;
480 if (per_cpu(buffer_space_available, cpu)[buftype]) {
481 // Give some extra room; also allows space to insert the overflow error packet
484 // Hysteresis, prevents multiple overflow messages
491 static int contiguous_space_available(int cpu, int buftype)
493 int remaining = buffer_bytes_available(cpu, buftype);
494 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
495 if (remaining < contiguous)
501 static bool buffer_check_space(int cpu, int buftype, int bytes)
503 int remaining = buffer_bytes_available(cpu, buftype);
505 if (remaining < bytes) {
506 per_cpu(buffer_space_available, cpu)[buftype] = false;
508 per_cpu(buffer_space_available, cpu)[buftype] = true;
511 return per_cpu(buffer_space_available, cpu)[buftype];
514 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
517 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
518 u32 mask = gator_buffer_mask[buftype];
519 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
521 for (i = 0; i < len; i++) {
522 buffer[write] = x[i];
523 write = (write + 1) & mask;
526 per_cpu(gator_buffer_write, cpu)[buftype] = write;
529 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
532 gator_buffer_write_packed_int(cpu, buftype, len);
533 gator_buffer_write_bytes(cpu, buftype, x, len);
536 static void gator_commit_buffer(int cpu, int buftype, u64 time)
538 int type_length, commit, length, byte;
540 if (!per_cpu(gator_buffer, cpu)[buftype])
543 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
544 type_length = gator_response_type ? 1 : 0;
545 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
546 length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
548 length += gator_buffer_size[buftype];
550 length = length - type_length - sizeof(s32);
552 if (length <= FRAME_HEADER_SIZE) {
553 // Nothing to write, only the frame header is present
557 for (byte = 0; byte < sizeof(s32); byte++) {
558 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
561 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
563 if (gator_live_rate > 0) {
564 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
565 per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
569 marshal_frame(cpu, buftype);
571 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
572 if (per_cpu(in_scheduler_context, cpu)) {
573 #ifndef CONFIG_PREEMPT_RT_FULL
574 // mod_timer can not be used in interrupt context in RT-Preempt full
575 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
578 wake_up_process(gator_buffer_wake_thread);
582 static void buffer_check(int cpu, int buftype, u64 time)
584 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
586 filled += gator_buffer_size[buftype];
588 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
589 gator_commit_buffer(cpu, buftype, time);
593 static void gator_add_trace(int cpu, unsigned long address)
596 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
598 if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
602 marshal_backtrace(offset & ~1, cookie);
605 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
608 unsigned long exec_cookie;
613 inKernel = !user_mode(regs);
614 exec_cookie = get_exec_cookie(cpu, current);
616 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
620 kernel_backtrace(cpu, regs);
623 gator_add_trace(cpu, PC_REG);
626 if (gator_backtrace_depth)
627 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
630 marshal_backtrace_footer(time);
633 /******************************************************************************
634 * hrtimer interrupt processing
635 ******************************************************************************/
636 static void gator_timer_interrupt(void)
638 struct pt_regs *const regs = get_irq_regs();
639 gator_backtrace_handler(regs);
642 void gator_backtrace_handler(struct pt_regs *const regs)
644 u64 time = gator_get_time();
645 int cpu = get_physical_cpu();
648 gator_add_sample(cpu, regs, time);
651 if (!per_cpu(collecting, cpu)) {
652 collect_counters(time, NULL);
655 // No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed
656 #ifdef CONFIG_PREEMPT_RT_FULL
657 buffer_check(cpu, SCHED_TRACE_BUF, time);
661 static int gator_running;
663 // This function runs in interrupt context and on the appropriate core
664 static void gator_timer_offline(void *migrate)
666 struct gator_interface *gi;
667 int i, len, cpu = get_physical_cpu();
671 gator_trace_sched_offline();
672 gator_trace_power_offline();
675 gator_hrtimer_offline();
678 // Offline any events and output counters
679 time = gator_get_time();
680 if (marshal_event_header(time)) {
681 list_for_each_entry(gi, &gator_events, list) {
683 len = gi->offline(&buffer, migrate);
684 marshal_event(len, buffer);
687 // Only check after writing all counters so that time and corresponding counters appear in the same frame
688 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
691 // Flush all buffers on this core
692 for (i = 0; i < NUM_GATOR_BUFS; i++)
693 gator_commit_buffer(cpu, i, time);
696 // This function runs in interrupt context and may be running on a core other than core 'cpu'
697 static void gator_timer_offline_dispatch(int cpu, bool migrate)
699 struct gator_interface *gi;
701 list_for_each_entry(gi, &gator_events, list) {
702 if (gi->offline_dispatch) {
703 gi->offline_dispatch(cpu, migrate);
708 static void gator_timer_stop(void)
713 on_each_cpu(gator_timer_offline, NULL, 1);
714 for_each_online_cpu(cpu) {
715 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
719 gator_hrtimer_shutdown();
723 #if defined(__arm__) || defined(__aarch64__)
724 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
725 const char *core_name = NULL;
726 char core_name_buf[32];
728 if (!sent_core_name[cpu]) {
729 if (gator_cpu != NULL) {
730 core_name = gator_cpu->core_name;
732 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
733 core_name = core_name_buf;
736 marshal_core_name(cpu, cpuid, core_name);
737 sent_core_name[cpu] = true;
742 // This function runs in interrupt context and on the appropriate core
743 static void gator_timer_online(void *migrate)
745 struct gator_interface *gi;
746 int len, cpu = get_physical_cpu();
750 gator_trace_power_online();
752 // online any events and output counters
753 time = gator_get_time();
754 if (marshal_event_header(time)) {
755 list_for_each_entry(gi, &gator_events, list) {
757 len = gi->online(&buffer, migrate);
758 marshal_event(len, buffer);
761 // Only check after writing all counters so that time and corresponding counters appear in the same frame
762 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
766 gator_hrtimer_online();
769 #if defined(__arm__) || defined(__aarch64__)
770 if (!sent_core_name[cpu]) {
771 const u32 cpuid = gator_cpuid();
772 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
777 // This function runs in interrupt context and may be running on a core other than core 'cpu'
778 static void gator_timer_online_dispatch(int cpu, bool migrate)
780 struct gator_interface *gi;
782 list_for_each_entry(gi, &gator_events, list) {
783 if (gi->online_dispatch) {
784 gi->online_dispatch(cpu, migrate);
789 #include "gator_iks.c"
791 int gator_timer_start(unsigned long sample_rate)
796 pr_notice("gator: already running\n");
802 // event based sampling trumps hr timer based sampling
803 if (event_based_sampling) {
807 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
810 gator_send_iks_core_names();
811 for_each_online_cpu(cpu) {
812 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
814 on_each_cpu(gator_timer_online, NULL, 1);
819 static u64 gator_get_time(void)
825 int cpu = smp_processor_id();
827 // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
828 getrawmonotonic(&ts);
829 timestamp = timespec_to_ns(&ts);
831 // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
832 // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
833 // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
834 prev_timestamp = per_cpu(last_timestamp, cpu);
835 if (prev_timestamp <= timestamp) {
836 per_cpu(last_timestamp, cpu) = timestamp;
838 delta = prev_timestamp - timestamp;
839 // Log the error once
840 if (!printed_monotonic_warning && delta > 500000) {
841 printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
842 printed_monotonic_warning = true;
844 timestamp = prev_timestamp;
847 return timestamp - gator_monotonic_started;
850 /******************************************************************************
851 * cpu hotplug and pm notifiers
852 ******************************************************************************/
853 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
855 int cpu = lcpu_to_pcpu((long)hcpu);
858 case CPU_DOWN_PREPARE:
859 case CPU_DOWN_PREPARE_FROZEN:
860 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
861 gator_timer_offline_dispatch(cpu, false);
864 case CPU_ONLINE_FROZEN:
865 gator_timer_online_dispatch(cpu, false);
866 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
873 static struct notifier_block __refdata gator_hotcpu_notifier = {
874 .notifier_call = gator_hotcpu_notify,
877 // n.b. calling "on_each_cpu" only runs on those that are online
878 // Registered linux events are not disabled, so their counters will continue to collect
879 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
885 case PM_HIBERNATION_PREPARE:
886 case PM_SUSPEND_PREPARE:
887 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
888 unregister_scheduler_tracepoints();
889 on_each_cpu(gator_timer_offline, NULL, 1);
890 for_each_online_cpu(cpu) {
891 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
894 // Record the wallclock hibernate time
896 gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time();
898 case PM_POST_HIBERNATION:
899 case PM_POST_SUSPEND:
900 // Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it
901 if (gator_hibernate_time > 0) {
903 gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts);
904 gator_hibernate_time = 0;
907 for_each_online_cpu(cpu) {
908 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
910 on_each_cpu(gator_timer_online, NULL, 1);
911 register_scheduler_tracepoints();
912 register_hotcpu_notifier(&gator_hotcpu_notifier);
919 static struct notifier_block gator_pm_notifier = {
920 .notifier_call = gator_pm_notify,
923 static int gator_notifier_start(void)
926 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
928 retval = register_pm_notifier(&gator_pm_notifier);
932 static void gator_notifier_stop(void)
934 unregister_pm_notifier(&gator_pm_notifier);
935 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
938 /******************************************************************************
940 ******************************************************************************/
941 static void gator_summary(void)
943 u64 timestamp, uptime;
946 void (*m2b)(struct timespec *ts);
949 snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
952 timestamp = timespec_to_ns(&ts);
954 do_posix_clock_monotonic_gettime(&ts);
955 // monotonic_to_bootbased is not defined for some versions of Android
956 m2b = symbol_get(monotonic_to_bootbased);
960 uptime = timespec_to_ns(&ts);
962 // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
963 local_irq_save(flags);
964 // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
965 gator_monotonic_started = 0;
966 gator_monotonic_started = gator_get_time();
967 local_irq_restore(flags);
969 marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
972 int gator_events_install(struct gator_interface *interface)
974 list_add_tail(&interface->list, &gator_events);
979 int gator_events_get_key(void)
981 // key 0 is reserved as a timestamp
982 // key 1 is reserved as the marker for thread specific counters
983 // Odd keys are assigned by the driver, even keys by the daemon
991 static int gator_init(void)
995 calc_first_cluster_size();
998 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
999 if (gator_events_list[i])
1000 gator_events_list[i]();
1002 gator_trace_sched_init();
1003 gator_trace_power_init();
1008 static void gator_exit(void)
1010 struct gator_interface *gi;
1012 list_for_each_entry(gi, &gator_events, list)
1017 static int gator_start(void)
1019 unsigned long cpu, i;
1020 struct gator_interface *gi;
1022 gator_buffer_wake_stop = false;
1023 if (IS_ERR(gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake"))) {
1027 if (gator_migrate_start())
1028 goto migrate_failure;
1030 // Initialize the buffer with the frame type and core
1031 for_each_present_cpu(cpu) {
1032 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1033 marshal_frame(cpu, i);
1035 per_cpu(last_timestamp, cpu) = 0;
1037 printed_monotonic_warning = false;
1039 // Capture the start time
1043 list_for_each_entry(gi, &gator_events, list) {
1044 if (gi->start && gi->start() != 0) {
1045 struct list_head *ptr = gi->list.prev;
1047 while (ptr != &gator_events) {
1048 gi = list_entry(ptr, struct gator_interface, list);
1055 goto events_failure;
1059 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
1060 if (cookies_initialize())
1061 goto cookies_failure;
1062 if (gator_annotate_start())
1063 goto annotate_failure;
1064 if (gator_trace_sched_start())
1066 if (gator_trace_power_start())
1068 if (gator_trace_gpu_start())
1070 if (gator_timer_start(gator_timer_count))
1072 if (gator_notifier_start())
1073 goto notifier_failure;
1080 gator_trace_gpu_stop();
1082 gator_trace_power_stop();
1084 gator_trace_sched_stop();
1086 gator_annotate_stop();
1091 list_for_each_entry(gi, &gator_events, list)
1095 gator_migrate_stop();
1097 gator_buffer_wake_stop = true;
1098 wake_up_process(gator_buffer_wake_thread);
1104 static void gator_stop(void)
1106 struct gator_interface *gi;
1108 gator_annotate_stop();
1109 gator_trace_sched_stop();
1110 gator_trace_power_stop();
1111 gator_trace_gpu_stop();
1113 // stop all interrupt callback reads before tearing down other interfaces
1114 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1118 list_for_each_entry(gi, &gator_events, list)
1122 gator_migrate_stop();
1124 gator_buffer_wake_stop = true;
1125 wake_up_process(gator_buffer_wake_thread);
1128 /******************************************************************************
1130 ******************************************************************************/
1131 /* fopen("buffer") */
1132 static int gator_op_setup(void)
1137 mutex_lock(&start_mutex);
1139 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1140 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1142 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1143 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1145 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1146 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1148 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1149 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1151 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1152 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1154 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1155 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1157 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1158 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1160 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1161 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1163 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1164 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1166 // Initialize percpu per buffer variables
1167 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1168 // Verify buffers are a power of 2
1169 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1174 for_each_present_cpu(cpu) {
1175 per_cpu(gator_buffer_read, cpu)[i] = 0;
1176 per_cpu(gator_buffer_write, cpu)[i] = 0;
1177 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1178 per_cpu(buffer_space_available, cpu)[i] = true;
1179 per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1181 // Annotation is a special case that only uses a single buffer
1182 if (cpu > 0 && i == ANNOTATE_BUF) {
1183 per_cpu(gator_buffer, cpu)[i] = NULL;
1187 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1188 if (!per_cpu(gator_buffer, cpu)[i]) {
1196 mutex_unlock(&start_mutex);
1200 /* Actually start profiling (echo 1>/dev/gator/enable) */
1201 static int gator_op_start(void)
1205 mutex_lock(&start_mutex);
1207 if (gator_started || gator_start())
1212 mutex_unlock(&start_mutex);
1217 /* echo 0>/dev/gator/enable */
1218 static void gator_op_stop(void)
1220 mutex_lock(&start_mutex);
1222 if (gator_started) {
1225 mutex_lock(&gator_buffer_mutex);
1228 gator_monotonic_started = 0;
1230 wake_up(&gator_buffer_wait);
1232 mutex_unlock(&gator_buffer_mutex);
1235 mutex_unlock(&start_mutex);
1238 static void gator_shutdown(void)
1242 mutex_lock(&start_mutex);
1244 for_each_present_cpu(cpu) {
1245 mutex_lock(&gator_buffer_mutex);
1246 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1247 vfree(per_cpu(gator_buffer, cpu)[i]);
1248 per_cpu(gator_buffer, cpu)[i] = NULL;
1249 per_cpu(gator_buffer_read, cpu)[i] = 0;
1250 per_cpu(gator_buffer_write, cpu)[i] = 0;
1251 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1252 per_cpu(buffer_space_available, cpu)[i] = true;
1253 per_cpu(gator_buffer_commit_time, cpu) = 0;
1255 mutex_unlock(&gator_buffer_mutex);
1258 memset(&sent_core_name, 0, sizeof(sent_core_name));
1260 mutex_unlock(&start_mutex);
1263 static int gator_set_backtrace(unsigned long val)
1267 mutex_lock(&start_mutex);
1272 gator_backtrace_depth = val;
1274 mutex_unlock(&start_mutex);
1279 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1281 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1284 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1292 retval = gatorfs_ulong_from_user(&val, buf, count);
1297 retval = gator_op_start();
1306 static const struct file_operations enable_fops = {
1307 .read = enable_read,
1308 .write = enable_write,
1311 static int userspace_buffer_open(struct inode *inode, struct file *file)
1315 if (!capable(CAP_SYS_ADMIN))
1318 if (test_and_set_bit_lock(0, &gator_buffer_opened))
1321 if ((err = gator_op_setup()))
1324 /* NB: the actual start happens from userspace
1325 * echo 1 >/dev/gator/enable
1331 __clear_bit_unlock(0, &gator_buffer_opened);
1335 static int userspace_buffer_release(struct inode *inode, struct file *file)
1339 __clear_bit_unlock(0, &gator_buffer_opened);
1343 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1345 int commit, length1, length2, read;
1351 // ensure there is enough space for a whole frame
1352 if (count < userspace_buffer_size || *offset) {
1356 // sleep until the condition is true or a signal is received
1357 // the condition is checked each time gator_buffer_wait is woken up
1358 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1360 if (signal_pending(current)) {
1364 if (buftype == -1 || cpu == -1) {
1368 mutex_lock(&gator_buffer_mutex);
1371 read = per_cpu(gator_buffer_read, cpu)[buftype];
1372 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1374 // May happen if the buffer is freed during pending reads.
1375 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1379 // determine the size of two halves
1380 length1 = commit - read;
1382 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1383 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1385 length1 = gator_buffer_size[buftype] - read;
1389 if (length1 + length2 > count - written) {
1393 // start, middle or end
1394 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1398 // possible wrap around
1399 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1403 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1404 written += length1 + length2;
1406 // Wake up annotate_write if more space is available
1407 if (buftype == ANNOTATE_BUF) {
1408 wake_up(&gator_annotate_wait);
1410 } while (buffer_commit_ready(&cpu, &buftype));
1412 mutex_unlock(&gator_buffer_mutex);
1414 // kick just in case we've lost an SMP event
1415 wake_up(&gator_buffer_wait);
1417 return written > 0 ? written : -EFAULT;
1420 const struct file_operations gator_event_buffer_fops = {
1421 .open = userspace_buffer_open,
1422 .release = userspace_buffer_release,
1423 .read = userspace_buffer_read,
1426 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1428 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1431 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1439 retval = gatorfs_ulong_from_user(&val, buf, count);
1443 retval = gator_set_backtrace(val);
1450 static const struct file_operations depth_fops = {
1452 .write = depth_write
1455 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1458 struct gator_interface *gi;
1461 /* reinitialize default values */
1462 gator_cpu_cores = 0;
1463 for_each_present_cpu(cpu) {
1466 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1467 gator_response_type = 1;
1468 gator_live_rate = 0;
1470 gatorfs_create_file(sb, root, "enable", &enable_fops);
1471 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1472 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1473 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1474 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1475 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1476 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1477 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1478 gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1479 gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1481 // Annotate interface
1482 gator_annotate_create_files(sb, root);
1485 dir = gatorfs_mkdir(sb, root, "events");
1486 list_for_each_entry(gi, &gator_events, list)
1487 if (gi->create_files)
1488 gi->create_files(sb, dir);
1491 sched_trace_create_files(sb, dir);
1494 gator_trace_power_create_files(sb, dir);
1497 /******************************************************************************
1499 ******************************************************************************/
1500 static int __init gator_module_init(void)
1502 if (gatorfs_register()) {
1507 gatorfs_unregister();
1511 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1516 static void __exit gator_module_exit(void)
1518 del_timer_sync(&gator_buffer_wake_up_timer);
1519 tracepoint_synchronize_unregister();
1521 gatorfs_unregister();
1524 module_init(gator_module_init);
1525 module_exit(gator_module_exit);
1527 MODULE_LICENSE("GPL");
1528 MODULE_AUTHOR("ARM Ltd");
1529 MODULE_DESCRIPTION("Gator system profiler");
1530 #define STRIFY2(ARG) #ARG
1531 #define STRIFY(ARG) STRIFY2(ARG)
1532 MODULE_VERSION(STRIFY(PROTOCOL_VERSION));