2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 16;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
59 /******************************************************************************
61 ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE (1*1024)
63 #define BACKTRACE_BUFFER_SIZE (128*1024)
64 #define NAME_BUFFER_SIZE (64*1024)
65 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE (128*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
73 #define UNRESOLVED_COOKIE ~0U
75 #define FRAME_SUMMARY 1
76 #define FRAME_BACKTRACE 2
78 #define FRAME_COUNTER 4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE 6
81 #define FRAME_SCHED_TRACE 7
82 #define FRAME_GPU_TRACE 8
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE 1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME 3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP 2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT 2
96 #define MESSAGE_SCHED_START 3
98 #define MESSAGE_IDLE_ENTER 1
99 #define MESSAGE_IDLE_EXIT 2
101 #define MAXSIZE_PACK32 5
102 #define MAXSIZE_PACK64 10
104 #define FRAME_HEADER_SIZE 3
107 #define PC_REG regs->ARM_pc
108 #elif defined(__aarch64__)
109 #define PC_REG regs->pc
111 #define PC_REG regs->ip
127 /******************************************************************************
129 ******************************************************************************/
130 static unsigned long gator_cpu_cores;
131 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
132 static unsigned long userspace_buffer_size;
133 static unsigned long gator_backtrace_depth;
134 // How often to commit the buffers for live in nanoseconds
135 static u64 gator_live_rate;
137 static unsigned long gator_started;
138 static u64 gator_monotonic_started;
139 static unsigned long gator_buffer_opened;
140 static unsigned long gator_timer_count;
141 static unsigned long gator_response_type;
142 static DEFINE_MUTEX(start_mutex);
143 static DEFINE_MUTEX(gator_buffer_mutex);
145 bool event_based_sampling;
147 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
148 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
149 static struct timer_list gator_buffer_wake_up_timer;
150 static LIST_HEAD(gator_events);
152 static DEFINE_PER_CPU(u64, last_timestamp);
154 static bool printed_monotonic_warning;
156 static bool sent_core_name[NR_CPUS];
158 /******************************************************************************
160 ******************************************************************************/
161 static void buffer_check(int cpu, int buftype, u64 time);
162 static void gator_commit_buffer(int cpu, int buftype, u64 time);
163 static int buffer_bytes_available(int cpu, int buftype);
164 static bool buffer_check_space(int cpu, int buftype, int bytes);
165 static int contiguous_space_available(int cpu, int bufytpe);
166 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
167 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
168 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
169 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
170 static void gator_add_trace(int cpu, unsigned long address);
171 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
172 static u64 gator_get_time(void);
174 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
175 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
176 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
177 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
178 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
179 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
180 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
181 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
182 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
183 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
184 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
185 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
186 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
187 // The buffer. Allocated in gator_op_setup
188 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 // The time after which the buffer should be committed for live display
190 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 /******************************************************************************
193 * Application Includes
194 ******************************************************************************/
195 #include "gator_marshaling.c"
196 #include "gator_hrtimer_perf.c"
197 #include "gator_hrtimer_gator.c"
198 #include "gator_cookies.c"
199 #include "gator_annotate.c"
200 #include "gator_trace_sched.c"
201 #include "gator_trace_power.c"
202 #include "gator_trace_gpu.c"
203 #include "gator_backtrace.c"
204 #include "gator_fs.c"
205 #include "gator_pack.c"
207 /******************************************************************************
209 ******************************************************************************/
211 const struct gator_cpu gator_cpus[] = {
214 .core_name = "ARM1136",
215 .pmnc_name = "ARM_ARM11",
216 .dt_name = "arm,arm1136",
221 .core_name = "ARM1156",
222 .pmnc_name = "ARM_ARM11",
223 .dt_name = "arm,arm1156",
228 .core_name = "ARM1176",
229 .pmnc_name = "ARM_ARM11",
230 .dt_name = "arm,arm1176",
234 .cpuid = ARM11MPCORE,
235 .core_name = "ARM11MPCore",
236 .pmnc_name = "ARM_ARM11MPCore",
237 .dt_name = "arm,arm11mpcore",
242 .core_name = "Cortex-A5",
243 .pmu_name = "ARMv7_Cortex_A5",
244 .pmnc_name = "ARM_Cortex-A5",
245 .dt_name = "arm,cortex-a5",
250 .core_name = "Cortex-A7",
251 .pmu_name = "ARMv7_Cortex_A7",
252 .pmnc_name = "ARM_Cortex-A7",
253 .dt_name = "arm,cortex-a7",
258 .core_name = "Cortex-A8",
259 .pmu_name = "ARMv7_Cortex_A8",
260 .pmnc_name = "ARM_Cortex-A8",
261 .dt_name = "arm,cortex-a8",
266 .core_name = "Cortex-A9",
267 .pmu_name = "ARMv7_Cortex_A9",
268 .pmnc_name = "ARM_Cortex-A9",
269 .dt_name = "arm,cortex-a9",
274 .core_name = "Cortex-A12",
275 .pmu_name = "ARMv7_Cortex_A12",
276 .pmnc_name = "ARM_Cortex-A12",
277 .dt_name = "arm,cortex-a12",
282 .core_name = "Cortex-A15",
283 .pmu_name = "ARMv7_Cortex_A15",
284 .pmnc_name = "ARM_Cortex-A15",
285 .dt_name = "arm,cortex-a15",
290 .core_name = "Scorpion",
291 .pmnc_name = "Scorpion",
296 .core_name = "ScorpionMP",
297 .pmnc_name = "ScorpionMP",
302 .core_name = "KraitSIM",
303 .pmnc_name = "Krait",
308 .core_name = "Krait",
309 .pmnc_name = "Krait",
313 .cpuid = KRAIT_S4_PRO,
314 .core_name = "Krait S4 Pro",
315 .pmnc_name = "Krait",
320 .core_name = "Cortex-A53",
321 .pmnc_name = "ARM_Cortex-A53",
322 .dt_name = "arm,cortex-a53",
327 .core_name = "Cortex-A57",
328 .pmnc_name = "ARM_Cortex-A57",
329 .dt_name = "arm,cortex-a57",
334 .core_name = "AArch64",
335 .pmnc_name = "ARM_AArch64",
340 .core_name = "Other",
341 .pmnc_name = "Other",
347 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
351 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
352 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
353 if (gator_cpu->cpuid == cpuid) {
361 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
365 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
366 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
367 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
375 u32 gator_cpuid(void)
377 #if defined(__arm__) || defined(__aarch64__)
379 #if !defined(__aarch64__)
380 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
382 asm volatile("mrs %0, midr_el1" : "=r" (val));
384 return (val >> 4) & 0xfff;
390 static void gator_buffer_wake_up(unsigned long data)
392 wake_up(&gator_buffer_wait);
395 /******************************************************************************
397 ******************************************************************************/
398 static bool buffer_commit_ready(int *cpu, int *buftype)
401 for_each_present_cpu(cpu_x) {
402 for (x = 0; x < NUM_GATOR_BUFS; x++)
403 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
414 /******************************************************************************
416 ******************************************************************************/
417 static int buffer_bytes_available(int cpu, int buftype)
419 int remaining, filled;
421 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
423 filled += gator_buffer_size[buftype];
426 remaining = gator_buffer_size[buftype] - filled;
428 if (per_cpu(buffer_space_available, cpu)[buftype]) {
429 // Give some extra room; also allows space to insert the overflow error packet
432 // Hysteresis, prevents multiple overflow messages
439 static int contiguous_space_available(int cpu, int buftype)
441 int remaining = buffer_bytes_available(cpu, buftype);
442 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
443 if (remaining < contiguous)
449 static bool buffer_check_space(int cpu, int buftype, int bytes)
451 int remaining = buffer_bytes_available(cpu, buftype);
453 if (remaining < bytes) {
454 per_cpu(buffer_space_available, cpu)[buftype] = false;
456 per_cpu(buffer_space_available, cpu)[buftype] = true;
459 return per_cpu(buffer_space_available, cpu)[buftype];
462 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
465 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
466 u32 mask = gator_buffer_mask[buftype];
467 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
469 for (i = 0; i < len; i++) {
470 buffer[write] = x[i];
471 write = (write + 1) & mask;
474 per_cpu(gator_buffer_write, cpu)[buftype] = write;
477 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
480 gator_buffer_write_packed_int(cpu, buftype, len);
481 gator_buffer_write_bytes(cpu, buftype, x, len);
484 static void gator_commit_buffer(int cpu, int buftype, u64 time)
486 int type_length, commit, length, byte;
488 if (!per_cpu(gator_buffer, cpu)[buftype])
491 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
492 type_length = gator_response_type ? 1 : 0;
493 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
494 length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
496 length += gator_buffer_size[buftype];
498 length = length - type_length - sizeof(s32);
500 if (length <= FRAME_HEADER_SIZE) {
501 // Nothing to write, only the frame header is present
505 for (byte = 0; byte < sizeof(s32); byte++) {
506 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
509 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
511 if (gator_live_rate > 0) {
512 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
513 per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
517 marshal_frame(cpu, buftype);
519 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
520 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
523 static void buffer_check(int cpu, int buftype, u64 time)
525 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
527 filled += gator_buffer_size[buftype];
529 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
530 gator_commit_buffer(cpu, buftype, time);
534 static void gator_add_trace(int cpu, unsigned long address)
537 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
539 if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
543 marshal_backtrace(offset & ~1, cookie);
546 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
549 unsigned long exec_cookie;
554 inKernel = !user_mode(regs);
555 exec_cookie = get_exec_cookie(cpu, current);
557 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
561 kernel_backtrace(cpu, regs);
564 gator_add_trace(cpu, PC_REG);
567 if (gator_backtrace_depth)
568 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
571 marshal_backtrace_footer(time);
574 /******************************************************************************
575 * hrtimer interrupt processing
576 ******************************************************************************/
577 static void gator_timer_interrupt(void)
579 struct pt_regs *const regs = get_irq_regs();
580 gator_backtrace_handler(regs);
583 void gator_backtrace_handler(struct pt_regs *const regs)
585 u64 time = gator_get_time();
586 int cpu = get_physical_cpu();
589 gator_add_sample(cpu, regs, time);
592 if (!per_cpu(collecting, cpu)) {
593 collect_counters(time);
597 static int gator_running;
599 // This function runs in interrupt context and on the appropriate core
600 static void gator_timer_offline(void *migrate)
602 struct gator_interface *gi;
603 int i, len, cpu = get_physical_cpu();
607 gator_trace_sched_offline();
608 gator_trace_power_offline();
611 gator_hrtimer_offline();
614 // Offline any events and output counters
615 time = gator_get_time();
616 if (marshal_event_header(time)) {
617 list_for_each_entry(gi, &gator_events, list) {
619 len = gi->offline(&buffer, migrate);
620 marshal_event(len, buffer);
623 // Only check after writing all counters so that time and corresponding counters appear in the same frame
624 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
627 // Flush all buffers on this core
628 for (i = 0; i < NUM_GATOR_BUFS; i++)
629 gator_commit_buffer(cpu, i, time);
632 // This function runs in interrupt context and may be running on a core other than core 'cpu'
633 static void gator_timer_offline_dispatch(int cpu, bool migrate)
635 struct gator_interface *gi;
637 list_for_each_entry(gi, &gator_events, list) {
638 if (gi->offline_dispatch) {
639 gi->offline_dispatch(cpu, migrate);
644 static void gator_timer_stop(void)
649 on_each_cpu(gator_timer_offline, NULL, 1);
650 for_each_online_cpu(cpu) {
651 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
655 gator_hrtimer_shutdown();
659 #if defined(__arm__) || defined(__aarch64__)
660 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
661 const char *core_name = NULL;
662 char core_name_buf[32];
664 if (!sent_core_name[cpu]) {
665 if (gator_cpu != NULL) {
666 core_name = gator_cpu->core_name;
668 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
669 core_name = core_name_buf;
672 marshal_core_name(cpu, cpuid, core_name);
673 sent_core_name[cpu] = true;
678 // This function runs in interrupt context and on the appropriate core
679 static void gator_timer_online(void *migrate)
681 struct gator_interface *gi;
682 int len, cpu = get_physical_cpu();
686 gator_trace_power_online();
688 // online any events and output counters
689 time = gator_get_time();
690 if (marshal_event_header(time)) {
691 list_for_each_entry(gi, &gator_events, list) {
693 len = gi->online(&buffer, migrate);
694 marshal_event(len, buffer);
697 // Only check after writing all counters so that time and corresponding counters appear in the same frame
698 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
702 gator_hrtimer_online();
705 #if defined(__arm__) || defined(__aarch64__)
706 if (!sent_core_name[cpu]) {
707 const u32 cpuid = gator_cpuid();
708 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
713 // This function runs in interrupt context and may be running on a core other than core 'cpu'
714 static void gator_timer_online_dispatch(int cpu, bool migrate)
716 struct gator_interface *gi;
718 list_for_each_entry(gi, &gator_events, list) {
719 if (gi->online_dispatch) {
720 gi->online_dispatch(cpu, migrate);
725 #include "gator_iks.c"
727 int gator_timer_start(unsigned long sample_rate)
732 pr_notice("gator: already running\n");
738 // event based sampling trumps hr timer based sampling
739 if (event_based_sampling) {
743 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
746 gator_send_iks_core_names();
747 for_each_online_cpu(cpu) {
748 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
750 on_each_cpu(gator_timer_online, NULL, 1);
755 static u64 gator_get_time(void)
761 int cpu = smp_processor_id();
763 // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
764 getrawmonotonic(&ts);
765 timestamp = timespec_to_ns(&ts);
767 // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
768 // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
769 // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
770 prev_timestamp = per_cpu(last_timestamp, cpu);
771 if (prev_timestamp <= timestamp) {
772 per_cpu(last_timestamp, cpu) = timestamp;
774 delta = prev_timestamp - timestamp;
775 // Log the error once
776 if (!printed_monotonic_warning && delta > 500000) {
777 printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
778 printed_monotonic_warning = true;
780 timestamp = prev_timestamp;
783 return timestamp - gator_monotonic_started;
786 /******************************************************************************
787 * cpu hotplug and pm notifiers
788 ******************************************************************************/
789 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
791 int cpu = lcpu_to_pcpu((long)hcpu);
794 case CPU_DOWN_PREPARE:
795 case CPU_DOWN_PREPARE_FROZEN:
796 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
797 gator_timer_offline_dispatch(cpu, false);
800 case CPU_ONLINE_FROZEN:
801 gator_timer_online_dispatch(cpu, false);
802 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
809 static struct notifier_block __refdata gator_hotcpu_notifier = {
810 .notifier_call = gator_hotcpu_notify,
813 // n.b. calling "on_each_cpu" only runs on those that are online
814 // Registered linux events are not disabled, so their counters will continue to collect
815 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
820 case PM_HIBERNATION_PREPARE:
821 case PM_SUSPEND_PREPARE:
822 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
823 unregister_scheduler_tracepoints();
824 on_each_cpu(gator_timer_offline, NULL, 1);
825 for_each_online_cpu(cpu) {
826 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
829 case PM_POST_HIBERNATION:
830 case PM_POST_SUSPEND:
831 for_each_online_cpu(cpu) {
832 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
834 on_each_cpu(gator_timer_online, NULL, 1);
835 register_scheduler_tracepoints();
836 register_hotcpu_notifier(&gator_hotcpu_notifier);
843 static struct notifier_block gator_pm_notifier = {
844 .notifier_call = gator_pm_notify,
847 static int gator_notifier_start(void)
850 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
852 retval = register_pm_notifier(&gator_pm_notifier);
856 static void gator_notifier_stop(void)
858 unregister_pm_notifier(&gator_pm_notifier);
859 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
862 /******************************************************************************
864 ******************************************************************************/
865 static void gator_summary(void)
867 u64 timestamp, uptime;
870 void (*m2b)(struct timespec *ts);
873 snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
876 timestamp = timespec_to_ns(&ts);
878 do_posix_clock_monotonic_gettime(&ts);
879 // monotonic_to_bootbased is not defined for some versions of Android
880 m2b = symbol_get(monotonic_to_bootbased);
884 uptime = timespec_to_ns(&ts);
886 // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
887 local_irq_save(flags);
888 // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
889 gator_monotonic_started = 0;
890 gator_monotonic_started = gator_get_time();
891 local_irq_restore(flags);
893 marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
896 int gator_events_install(struct gator_interface *interface)
898 list_add_tail(&interface->list, &gator_events);
903 int gator_events_get_key(void)
905 // key of zero is reserved as a timestamp
913 static int gator_init(void)
917 calc_first_cluster_size();
919 // events sources (gator_events.h, generated by gator_events.sh)
920 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
921 if (gator_events_list[i])
922 gator_events_list[i]();
924 gator_trace_sched_init();
925 gator_trace_power_init();
930 static void gator_exit(void)
932 struct gator_interface *gi;
934 list_for_each_entry(gi, &gator_events, list)
939 static int gator_start(void)
941 unsigned long cpu, i;
942 struct gator_interface *gi;
944 if (gator_migrate_start())
945 goto migrate_failure;
947 // Initialize the buffer with the frame type and core
948 for_each_present_cpu(cpu) {
949 for (i = 0; i < NUM_GATOR_BUFS; i++) {
950 marshal_frame(cpu, i);
952 per_cpu(last_timestamp, cpu) = 0;
954 printed_monotonic_warning = false;
956 // Capture the start time
960 list_for_each_entry(gi, &gator_events, list) {
961 if (gi->start && gi->start() != 0) {
962 struct list_head *ptr = gi->list.prev;
964 while (ptr != &gator_events) {
965 gi = list_entry(ptr, struct gator_interface, list);
976 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
977 if (cookies_initialize())
978 goto cookies_failure;
979 if (gator_annotate_start())
980 goto annotate_failure;
981 if (gator_trace_sched_start())
983 if (gator_trace_power_start())
985 if (gator_trace_gpu_start())
987 if (gator_timer_start(gator_timer_count))
989 if (gator_notifier_start())
990 goto notifier_failure;
997 gator_trace_gpu_stop();
999 gator_trace_power_stop();
1001 gator_trace_sched_stop();
1003 gator_annotate_stop();
1008 list_for_each_entry(gi, &gator_events, list)
1012 gator_migrate_stop();
1018 static void gator_stop(void)
1020 struct gator_interface *gi;
1022 gator_annotate_stop();
1023 gator_trace_sched_stop();
1024 gator_trace_power_stop();
1025 gator_trace_gpu_stop();
1027 // stop all interrupt callback reads before tearing down other interfaces
1028 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1032 list_for_each_entry(gi, &gator_events, list)
1036 gator_migrate_stop();
1039 /******************************************************************************
1041 ******************************************************************************/
1042 /* fopen("buffer") */
1043 static int gator_op_setup(void)
1048 mutex_lock(&start_mutex);
1050 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1051 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1053 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1054 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1056 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1057 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1059 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1060 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1062 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1063 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1065 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1066 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1068 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1069 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1071 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1072 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1074 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1075 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1077 // Initialize percpu per buffer variables
1078 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1079 // Verify buffers are a power of 2
1080 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1085 for_each_present_cpu(cpu) {
1086 per_cpu(gator_buffer_read, cpu)[i] = 0;
1087 per_cpu(gator_buffer_write, cpu)[i] = 0;
1088 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1089 per_cpu(buffer_space_available, cpu)[i] = true;
1090 per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1092 // Annotation is a special case that only uses a single buffer
1093 if (cpu > 0 && i == ANNOTATE_BUF) {
1094 per_cpu(gator_buffer, cpu)[i] = NULL;
1098 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1099 if (!per_cpu(gator_buffer, cpu)[i]) {
1107 mutex_unlock(&start_mutex);
1111 /* Actually start profiling (echo 1>/dev/gator/enable) */
1112 static int gator_op_start(void)
1116 mutex_lock(&start_mutex);
1118 if (gator_started || gator_start())
1123 mutex_unlock(&start_mutex);
1128 /* echo 0>/dev/gator/enable */
1129 static void gator_op_stop(void)
1131 mutex_lock(&start_mutex);
1133 if (gator_started) {
1136 mutex_lock(&gator_buffer_mutex);
1139 gator_monotonic_started = 0;
1141 wake_up(&gator_buffer_wait);
1143 mutex_unlock(&gator_buffer_mutex);
1146 mutex_unlock(&start_mutex);
1149 static void gator_shutdown(void)
1153 mutex_lock(&start_mutex);
1155 for_each_present_cpu(cpu) {
1156 mutex_lock(&gator_buffer_mutex);
1157 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1158 vfree(per_cpu(gator_buffer, cpu)[i]);
1159 per_cpu(gator_buffer, cpu)[i] = NULL;
1160 per_cpu(gator_buffer_read, cpu)[i] = 0;
1161 per_cpu(gator_buffer_write, cpu)[i] = 0;
1162 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1163 per_cpu(buffer_space_available, cpu)[i] = true;
1164 per_cpu(gator_buffer_commit_time, cpu) = 0;
1166 mutex_unlock(&gator_buffer_mutex);
1169 memset(&sent_core_name, 0, sizeof(sent_core_name));
1171 mutex_unlock(&start_mutex);
1174 static int gator_set_backtrace(unsigned long val)
1178 mutex_lock(&start_mutex);
1183 gator_backtrace_depth = val;
1185 mutex_unlock(&start_mutex);
1190 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1192 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1195 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1203 retval = gatorfs_ulong_from_user(&val, buf, count);
1208 retval = gator_op_start();
1217 static const struct file_operations enable_fops = {
1218 .read = enable_read,
1219 .write = enable_write,
1222 static int userspace_buffer_open(struct inode *inode, struct file *file)
1226 if (!capable(CAP_SYS_ADMIN))
1229 if (test_and_set_bit_lock(0, &gator_buffer_opened))
1232 if ((err = gator_op_setup()))
1235 /* NB: the actual start happens from userspace
1236 * echo 1 >/dev/gator/enable
1242 __clear_bit_unlock(0, &gator_buffer_opened);
1246 static int userspace_buffer_release(struct inode *inode, struct file *file)
1250 __clear_bit_unlock(0, &gator_buffer_opened);
1254 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1256 int commit, length1, length2, read;
1262 // ensure there is enough space for a whole frame
1263 if (count < userspace_buffer_size || *offset) {
1267 // sleep until the condition is true or a signal is received
1268 // the condition is checked each time gator_buffer_wait is woken up
1269 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1271 if (signal_pending(current)) {
1275 if (buftype == -1 || cpu == -1) {
1279 mutex_lock(&gator_buffer_mutex);
1282 read = per_cpu(gator_buffer_read, cpu)[buftype];
1283 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1285 // May happen if the buffer is freed during pending reads.
1286 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1290 // determine the size of two halves
1291 length1 = commit - read;
1293 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1294 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1296 length1 = gator_buffer_size[buftype] - read;
1300 if (length1 + length2 > count - written) {
1304 // start, middle or end
1305 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1309 // possible wrap around
1310 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1314 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1315 written += length1 + length2;
1317 // Wake up annotate_write if more space is available
1318 if (buftype == ANNOTATE_BUF) {
1319 wake_up(&gator_annotate_wait);
1321 } while (buffer_commit_ready(&cpu, &buftype));
1323 mutex_unlock(&gator_buffer_mutex);
1325 // kick just in case we've lost an SMP event
1326 wake_up(&gator_buffer_wait);
1328 return written > 0 ? written : -EFAULT;
1331 const struct file_operations gator_event_buffer_fops = {
1332 .open = userspace_buffer_open,
1333 .release = userspace_buffer_release,
1334 .read = userspace_buffer_read,
1337 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1339 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1342 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1350 retval = gatorfs_ulong_from_user(&val, buf, count);
1354 retval = gator_set_backtrace(val);
1361 static const struct file_operations depth_fops = {
1363 .write = depth_write
1366 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1369 struct gator_interface *gi;
1372 /* reinitialize default values */
1373 gator_cpu_cores = 0;
1374 for_each_present_cpu(cpu) {
1377 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1378 gator_response_type = 1;
1379 gator_live_rate = 0;
1381 gatorfs_create_file(sb, root, "enable", &enable_fops);
1382 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1383 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1384 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1385 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1386 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1387 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1388 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1389 gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1390 gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1392 // Annotate interface
1393 gator_annotate_create_files(sb, root);
1396 dir = gatorfs_mkdir(sb, root, "events");
1397 list_for_each_entry(gi, &gator_events, list)
1398 if (gi->create_files)
1399 gi->create_files(sb, dir);
1402 sched_trace_create_files(sb, dir);
1405 gator_trace_power_create_files(sb, dir);
1408 /******************************************************************************
1410 ******************************************************************************/
1411 static int __init gator_module_init(void)
1413 if (gatorfs_register()) {
1418 gatorfs_unregister();
1422 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1427 static void __exit gator_module_exit(void)
1429 del_timer_sync(&gator_buffer_wake_up_timer);
1430 tracepoint_synchronize_unregister();
1432 gatorfs_unregister();
1435 module_init(gator_module_init);
1436 module_exit(gator_module_exit);
1438 MODULE_LICENSE("GPL");
1439 MODULE_AUTHOR("ARM Ltd");
1440 MODULE_DESCRIPTION("Gator system profiler");