drivers/gator/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 // This version must match the gator daemon version
  11 #define PROTOCOL_VERSION 17
  12 static unsigned long gator_protocol_version = PROTOCOL_VERSION;
  13
  14 #include <linux/slab.h>
  15 #include <linux/cpu.h>
  16 #include <linux/sched.h>
  17 #include <linux/irq.h>
  18 #include <linux/vmalloc.h>
  19 #include <linux/hardirq.h>
  20 #include <linux/highmem.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/suspend.h>
  23 #include <linux/module.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/utsname.h>
  26 #include <linux/kthread.h>
  27 #include <asm/stacktrace.h>
  28 #include <asm/uaccess.h>
  29
  30 #include "gator.h"
  31
  32 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
  33 #error kernels prior to 2.6.32 are not supported
  34 #endif
  35
  36 #if defined(MODULE) && !defined(CONFIG_MODULES)
  37 #error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel.
  38 #endif
  39
  40 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  41 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  42 #endif
  43
  44 #ifndef CONFIG_PROFILING
  45 #error gator requires the kernel to have CONFIG_PROFILING defined
  46 #endif
  47
  48 #ifndef CONFIG_HIGH_RES_TIMERS
  49 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  50 #endif
  51
  52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  53 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  54 #endif
  55
  56 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
  57 #ifndef CONFIG_PERF_EVENTS
  58 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  59 #elif !defined CONFIG_HW_PERF_EVENTS
  60 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  61 #endif
  62 #endif
  63
  64 /******************************************************************************
  65  * DEFINES
  66  ******************************************************************************/
  67 #define SUMMARY_BUFFER_SIZE       (1*1024)
  68 #define BACKTRACE_BUFFER_SIZE     (128*1024)
  69 #define NAME_BUFFER_SIZE          (64*1024)
  70 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
  71 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
  72 #define ANNOTATE_BUFFER_SIZE      (128*1024)    // annotate counters have the core as part of the data and the core value in the frame header may be discarded
  73 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
  74 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
  75 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
  76
  77 #define NO_COOKIE      0U
  78 #define UNRESOLVED_COOKIE ~0U
  79
  80 #define FRAME_SUMMARY       1
  81 #define FRAME_BACKTRACE     2
  82 #define FRAME_NAME          3
  83 #define FRAME_COUNTER       4
  84 #define FRAME_BLOCK_COUNTER 5
  85 #define FRAME_ANNOTATE      6
  86 #define FRAME_SCHED_TRACE   7
  87 #define FRAME_GPU_TRACE     8
  88 #define FRAME_IDLE          9
  89
  90 #define MESSAGE_END_BACKTRACE 1
  91
  92 #define MESSAGE_COOKIE      1
  93 #define MESSAGE_THREAD_NAME 2
  94 #define HRTIMER_CORE_NAME   3
  95 #define MESSAGE_LINK        4
  96
  97 #define MESSAGE_GPU_START 1
  98 #define MESSAGE_GPU_STOP  2
  99
 100 #define MESSAGE_SCHED_SWITCH 1
 101 #define MESSAGE_SCHED_EXIT   2
 102 #define MESSAGE_SCHED_START  3
 103
 104 #define MESSAGE_IDLE_ENTER 1
 105 #define MESSAGE_IDLE_EXIT 2
 106
 107 #define MAXSIZE_PACK32     5
 108 #define MAXSIZE_PACK64    10
 109
 110 #define FRAME_HEADER_SIZE 3
 111
 112 #if defined(__arm__)
 113 #define PC_REG regs->ARM_pc
 114 #elif defined(__aarch64__)
 115 #define PC_REG regs->pc
 116 #else
 117 #define PC_REG regs->ip
 118 #endif
 119
 120 enum {
 121         SUMMARY_BUF,
 122         BACKTRACE_BUF,
 123         NAME_BUF,
 124         COUNTER_BUF,
 125         BLOCK_COUNTER_BUF,
 126         ANNOTATE_BUF,
 127         SCHED_TRACE_BUF,
 128         GPU_TRACE_BUF,
 129         IDLE_BUF,
 130         NUM_GATOR_BUFS
 131 };
 132
 133 /******************************************************************************
 134  * Globals
 135  ******************************************************************************/
 136 static unsigned long gator_cpu_cores;
 137 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
 138 static unsigned long userspace_buffer_size;
 139 static unsigned long gator_backtrace_depth;
 140 // How often to commit the buffers for live in nanoseconds
 141 static u64 gator_live_rate;
 142
 143 static unsigned long gator_started;
 144 static u64 gator_monotonic_started;
 145 static u64 gator_hibernate_time;
 146 static unsigned long gator_buffer_opened;
 147 static unsigned long gator_timer_count;
 148 static unsigned long gator_response_type;
 149 static DEFINE_MUTEX(start_mutex);
 150 static DEFINE_MUTEX(gator_buffer_mutex);
 151
 152 bool event_based_sampling;
 153
 154 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 155 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
 156 static struct timer_list gator_buffer_wake_up_timer;
 157 static bool gator_buffer_wake_stop;
 158 static struct task_struct *gator_buffer_wake_thread;
 159 static LIST_HEAD(gator_events);
 160
 161 static DEFINE_PER_CPU(u64, last_timestamp);
 162
 163 static bool printed_monotonic_warning;
 164
 165 static bool sent_core_name[NR_CPUS];
 166
 167 /******************************************************************************
 168  * Prototypes
 169  ******************************************************************************/
 170 static void buffer_check(int cpu, int buftype, u64 time);
 171 static void gator_commit_buffer(int cpu, int buftype, u64 time);
 172 static int buffer_bytes_available(int cpu, int buftype);
 173 static bool buffer_check_space(int cpu, int buftype, int bytes);
 174 static int contiguous_space_available(int cpu, int bufytpe);
 175 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
 176 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
 177 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
 178 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
 179 static void gator_add_trace(int cpu, unsigned long address);
 180 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
 181 static u64 gator_get_time(void);
 182
 183 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
 184 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 185 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
 186 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 187 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
 188 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 189 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
 190 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 191 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
 192 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 193 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
 194 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
 195 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 196 // The buffer. Allocated in gator_op_setup
 197 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 198 // The time after which the buffer should be committed for live display
 199 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
 200
 201 // List of all gator events - new events must be added to this list
 202 #define GATOR_EVENTS_LIST \
 203         GATOR_EVENT(gator_events_armv6_init) \
 204         GATOR_EVENT(gator_events_armv7_init) \
 205         GATOR_EVENT(gator_events_block_init) \
 206         GATOR_EVENT(gator_events_ccn504_init) \
 207         GATOR_EVENT(gator_events_irq_init) \
 208         GATOR_EVENT(gator_events_l2c310_init) \
 209         GATOR_EVENT(gator_events_mali_init) \
 210         GATOR_EVENT(gator_events_mali_t6xx_hw_init) \
 211         GATOR_EVENT(gator_events_mali_t6xx_init) \
 212         GATOR_EVENT(gator_events_meminfo_init) \
 213         GATOR_EVENT(gator_events_mmapped_init) \
 214         GATOR_EVENT(gator_events_net_init) \
 215         GATOR_EVENT(gator_events_perf_pmu_init) \
 216         GATOR_EVENT(gator_events_sched_init) \
 217         GATOR_EVENT(gator_events_scorpion_init) \
 218
 219 #define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void);
 220 GATOR_EVENTS_LIST
 221 #undef GATOR_EVENT
 222
 223 static int (*gator_events_list[])(void) = {
 224 #define GATOR_EVENT(EVENT_INIT) EVENT_INIT,
 225 GATOR_EVENTS_LIST
 226 #undef GATOR_EVENT
 227 };
 228
 229 /******************************************************************************
 230  * Application Includes
 231  ******************************************************************************/
 232 #include "gator_marshaling.c"
 233 #include "gator_hrtimer_perf.c"
 234 #include "gator_hrtimer_gator.c"
 235 #include "gator_cookies.c"
 236 #include "gator_annotate.c"
 237 #include "gator_trace_sched.c"
 238 #include "gator_trace_power.c"
 239 #include "gator_trace_gpu.c"
 240 #include "gator_backtrace.c"
 241 #include "gator_fs.c"
 242 #include "gator_pack.c"
 243
 244 /******************************************************************************
 245  * Misc
 246  ******************************************************************************/
 247
 248 const struct gator_cpu gator_cpus[] = {
 249         {
 250                 .cpuid = ARM1136,
 251                 .core_name = "ARM1136",
 252                 .pmnc_name = "ARM_ARM11",
 253                 .dt_name = "arm,arm1136",
 254                 .pmnc_counters = 3,
 255         },
 256         {
 257                 .cpuid = ARM1156,
 258                 .core_name = "ARM1156",
 259                 .pmnc_name = "ARM_ARM11",
 260                 .dt_name = "arm,arm1156",
 261                 .pmnc_counters = 3,
 262         },
 263         {
 264                 .cpuid = ARM1176,
 265                 .core_name = "ARM1176",
 266                 .pmnc_name = "ARM_ARM11",
 267                 .dt_name = "arm,arm1176",
 268                 .pmnc_counters = 3,
 269         },
 270         {
 271                 .cpuid = ARM11MPCORE,
 272                 .core_name = "ARM11MPCore",
 273                 .pmnc_name = "ARM_ARM11MPCore",
 274                 .dt_name = "arm,arm11mpcore",
 275                 .pmnc_counters = 3,
 276         },
 277         {
 278                 .cpuid = CORTEX_A5,
 279                 .core_name = "Cortex-A5",
 280                 .pmu_name = "ARMv7_Cortex_A5",
 281                 .pmnc_name = "ARM_Cortex-A5",
 282                 .dt_name = "arm,cortex-a5",
 283                 .pmnc_counters = 2,
 284         },
 285         {
 286                 .cpuid = CORTEX_A7,
 287                 .core_name = "Cortex-A7",
 288                 .pmu_name = "ARMv7_Cortex_A7",
 289                 .pmnc_name = "ARM_Cortex-A7",
 290                 .dt_name = "arm,cortex-a7",
 291                 .pmnc_counters = 4,
 292         },
 293         {
 294                 .cpuid = CORTEX_A8,
 295                 .core_name = "Cortex-A8",
 296                 .pmu_name = "ARMv7_Cortex_A8",
 297                 .pmnc_name = "ARM_Cortex-A8",
 298                 .dt_name = "arm,cortex-a8",
 299                 .pmnc_counters = 4,
 300         },
 301         {
 302                 .cpuid = CORTEX_A9,
 303                 .core_name = "Cortex-A9",
 304                 .pmu_name = "ARMv7_Cortex_A9",
 305                 .pmnc_name = "ARM_Cortex-A9",
 306                 .dt_name = "arm,cortex-a9",
 307                 .pmnc_counters = 6,
 308         },
 309         {
 310                 .cpuid = CORTEX_A12,
 311                 .core_name = "Cortex-A12",
 312                 .pmu_name = "ARMv7_Cortex_A12",
 313                 .pmnc_name = "ARM_Cortex-A12",
 314                 .dt_name = "arm,cortex-a12",
 315                 .pmnc_counters = 6,
 316         },
 317         {
 318                 .cpuid = CORTEX_A15,
 319                 .core_name = "Cortex-A15",
 320                 .pmu_name = "ARMv7_Cortex_A15",
 321                 .pmnc_name = "ARM_Cortex-A15",
 322                 .dt_name = "arm,cortex-a15",
 323                 .pmnc_counters = 6,
 324         },
 325         {
 326                 .cpuid = SCORPION,
 327                 .core_name = "Scorpion",
 328                 .pmnc_name = "Scorpion",
 329                 .pmnc_counters = 4,
 330         },
 331         {
 332                 .cpuid = SCORPIONMP,
 333                 .core_name = "ScorpionMP",
 334                 .pmnc_name = "ScorpionMP",
 335                 .pmnc_counters = 4,
 336         },
 337         {
 338                 .cpuid = KRAITSIM,
 339                 .core_name = "KraitSIM",
 340                 .pmnc_name = "Krait",
 341                 .pmnc_counters = 4,
 342         },
 343         {
 344                 .cpuid = KRAIT,
 345                 .core_name = "Krait",
 346                 .pmnc_name = "Krait",
 347                 .pmnc_counters = 4,
 348         },
 349         {
 350                 .cpuid = KRAIT_S4_PRO,
 351                 .core_name = "Krait S4 Pro",
 352                 .pmnc_name = "Krait",
 353                 .pmnc_counters = 4,
 354         },
 355         {
 356                 .cpuid = CORTEX_A53,
 357                 .core_name = "Cortex-A53",
 358                 .pmnc_name = "ARM_Cortex-A53",
 359                 .dt_name = "arm,cortex-a53",
 360                 .pmnc_counters = 6,
 361         },
 362         {
 363                 .cpuid = CORTEX_A57,
 364                 .core_name = "Cortex-A57",
 365                 .pmnc_name = "ARM_Cortex-A57",
 366                 .dt_name = "arm,cortex-a57",
 367                 .pmnc_counters = 6,
 368         },
 369         {
 370                 .cpuid = AARCH64,
 371                 .core_name = "AArch64",
 372                 .pmnc_name = "ARM_AArch64",
 373                 .pmnc_counters = 6,
 374         },
 375         {
 376                 .cpuid = OTHER,
 377                 .core_name = "Other",
 378                 .pmnc_name = "Other",
 379                 .pmnc_counters = 6,
 380         },
 381         {}
 382 };
 383
 384 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
 385 {
 386         int i;
 387
 388         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 389                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 390                 if (gator_cpu->cpuid == cpuid) {
 391                         return gator_cpu;
 392                 }
 393         }
 394
 395         return NULL;
 396 }
 397
 398 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
 399 {
 400         int i;
 401
 402         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 403                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 404                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
 405                         return gator_cpu;
 406                 }
 407         }
 408
 409         return NULL;
 410 }
 411
 412 u32 gator_cpuid(void)
 413 {
 414 #if defined(__arm__) || defined(__aarch64__)
 415         u32 val;
 416 #if !defined(__aarch64__)
 417         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 418 #else
 419         asm volatile("mrs %0, midr_el1" : "=r" (val));
 420 #endif
 421         return (val >> 4) & 0xfff;
 422 #else
 423         return OTHER;
 424 #endif
 425 }
 426
 427 static void gator_buffer_wake_up(unsigned long data)
 428 {
 429         wake_up(&gator_buffer_wait);
 430 }
 431
 432 static int gator_buffer_wake_func(void *data)
 433 {
 434         while (!gator_buffer_wake_stop) {
 435                 set_current_state(TASK_INTERRUPTIBLE);
 436                 schedule();
 437                 if (gator_buffer_wake_stop) {
 438                         break;
 439                 }
 440
 441                 gator_buffer_wake_up(0);
 442         }
 443
 444         return 0;
 445 }
 446
 447 /******************************************************************************
 448  * Commit interface
 449  ******************************************************************************/
 450 static bool buffer_commit_ready(int *cpu, int *buftype)
 451 {
 452         int cpu_x, x;
 453         for_each_present_cpu(cpu_x) {
 454                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 455                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 456                                 *cpu = cpu_x;
 457                                 *buftype = x;
 458                                 return true;
 459                         }
 460         }
 461         *cpu = -1;
 462         *buftype = -1;
 463         return false;
 464 }
 465
 466 /******************************************************************************
 467  * Buffer management
 468  ******************************************************************************/
 469 static int buffer_bytes_available(int cpu, int buftype)
 470 {
 471         int remaining, filled;
 472
 473         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
 474         if (filled < 0) {
 475                 filled += gator_buffer_size[buftype];
 476         }
 477
 478         remaining = gator_buffer_size[buftype] - filled;
 479
 480         if (per_cpu(buffer_space_available, cpu)[buftype]) {
 481                 // Give some extra room; also allows space to insert the overflow error packet
 482                 remaining -= 200;
 483         } else {
 484                 // Hysteresis, prevents multiple overflow messages
 485                 remaining -= 2000;
 486         }
 487
 488         return remaining;
 489 }
 490
 491 static int contiguous_space_available(int cpu, int buftype)
 492 {
 493         int remaining = buffer_bytes_available(cpu, buftype);
 494         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
 495         if (remaining < contiguous)
 496                 return remaining;
 497         else
 498                 return contiguous;
 499 }
 500
 501 static bool buffer_check_space(int cpu, int buftype, int bytes)
 502 {
 503         int remaining = buffer_bytes_available(cpu, buftype);
 504
 505         if (remaining < bytes) {
 506                 per_cpu(buffer_space_available, cpu)[buftype] = false;
 507         } else {
 508                 per_cpu(buffer_space_available, cpu)[buftype] = true;
 509         }
 510
 511         return per_cpu(buffer_space_available, cpu)[buftype];
 512 }
 513
 514 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
 515 {
 516         int i;
 517         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
 518         u32 mask = gator_buffer_mask[buftype];
 519         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
 520
 521         for (i = 0; i < len; i++) {
 522                 buffer[write] = x[i];
 523                 write = (write + 1) & mask;
 524         }
 525
 526         per_cpu(gator_buffer_write, cpu)[buftype] = write;
 527 }
 528
 529 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
 530 {
 531         int len = strlen(x);
 532         gator_buffer_write_packed_int(cpu, buftype, len);
 533         gator_buffer_write_bytes(cpu, buftype, x, len);
 534 }
 535
 536 static void gator_commit_buffer(int cpu, int buftype, u64 time)
 537 {
 538         int type_length, commit, length, byte;
 539
 540         if (!per_cpu(gator_buffer, cpu)[buftype])
 541                 return;
 542
 543         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
 544         type_length = gator_response_type ? 1 : 0;
 545         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
 546         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
 547         if (length < 0) {
 548                 length += gator_buffer_size[buftype];
 549         }
 550         length = length - type_length - sizeof(s32);
 551
 552         if (length <= FRAME_HEADER_SIZE) {
 553                 // Nothing to write, only the frame header is present
 554                 return;
 555         }
 556
 557         for (byte = 0; byte < sizeof(s32); byte++) {
 558                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
 559         }
 560
 561         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
 562
 563         if (gator_live_rate > 0) {
 564                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
 565                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
 566                 }
 567         }
 568
 569         marshal_frame(cpu, buftype);
 570
 571         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 572         if (per_cpu(in_scheduler_context, cpu)) {
 573 #ifndef CONFIG_PREEMPT_RT_FULL
 574                 // mod_timer can not be used in interrupt context in RT-Preempt full
 575                 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
 576 #endif
 577         } else {
 578                 wake_up_process(gator_buffer_wake_thread);
 579         }
 580 }
 581
 582 static void buffer_check(int cpu, int buftype, u64 time)
 583 {
 584         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
 585         if (filled < 0) {
 586                 filled += gator_buffer_size[buftype];
 587         }
 588         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
 589                 gator_commit_buffer(cpu, buftype, time);
 590         }
 591 }
 592
 593 static void gator_add_trace(int cpu, unsigned long address)
 594 {
 595         off_t offset = 0;
 596         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
 597
 598         if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
 599                 offset = address;
 600         }
 601
 602         marshal_backtrace(offset & ~1, cookie);
 603 }
 604
 605 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
 606 {
 607         bool inKernel;
 608         unsigned long exec_cookie;
 609
 610         if (!regs)
 611                 return;
 612
 613         inKernel = !user_mode(regs);
 614         exec_cookie = get_exec_cookie(cpu, current);
 615
 616         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
 617                 return;
 618
 619         if (inKernel) {
 620                 kernel_backtrace(cpu, regs);
 621         } else {
 622                 // Cookie+PC
 623                 gator_add_trace(cpu, PC_REG);
 624
 625                 // Backtrace
 626                 if (gator_backtrace_depth)
 627                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
 628         }
 629
 630         marshal_backtrace_footer(time);
 631 }
 632
 633 /******************************************************************************
 634  * hrtimer interrupt processing
 635  ******************************************************************************/
 636 static void gator_timer_interrupt(void)
 637 {
 638         struct pt_regs *const regs = get_irq_regs();
 639         gator_backtrace_handler(regs);
 640 }
 641
 642 void gator_backtrace_handler(struct pt_regs *const regs)
 643 {
 644         u64 time = gator_get_time();
 645         int cpu = get_physical_cpu();
 646
 647         // Output backtrace
 648         gator_add_sample(cpu, regs, time);
 649
 650         // Collect counters
 651         if (!per_cpu(collecting, cpu)) {
 652                 collect_counters(time, NULL);
 653         }
 654
 655         // No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed
 656 #ifdef CONFIG_PREEMPT_RT_FULL
 657         buffer_check(cpu, SCHED_TRACE_BUF, time);
 658 #endif
 659 }
 660
 661 static int gator_running;
 662
 663 // This function runs in interrupt context and on the appropriate core
 664 static void gator_timer_offline(void *migrate)
 665 {
 666         struct gator_interface *gi;
 667         int i, len, cpu = get_physical_cpu();
 668         int *buffer;
 669         u64 time;
 670
 671         gator_trace_sched_offline();
 672         gator_trace_power_offline();
 673
 674         if (!migrate) {
 675                 gator_hrtimer_offline();
 676         }
 677
 678         // Offline any events and output counters
 679         time = gator_get_time();
 680         if (marshal_event_header(time)) {
 681                 list_for_each_entry(gi, &gator_events, list) {
 682                         if (gi->offline) {
 683                                 len = gi->offline(&buffer, migrate);
 684                                 marshal_event(len, buffer);
 685                         }
 686                 }
 687                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 688                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 689         }
 690
 691         // Flush all buffers on this core
 692         for (i = 0; i < NUM_GATOR_BUFS; i++)
 693                 gator_commit_buffer(cpu, i, time);
 694 }
 695
 696 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 697 static void gator_timer_offline_dispatch(int cpu, bool migrate)
 698 {
 699         struct gator_interface *gi;
 700
 701         list_for_each_entry(gi, &gator_events, list) {
 702                 if (gi->offline_dispatch) {
 703                         gi->offline_dispatch(cpu, migrate);
 704                 }
 705         }
 706 }
 707
 708 static void gator_timer_stop(void)
 709 {
 710         int cpu;
 711
 712         if (gator_running) {
 713                 on_each_cpu(gator_timer_offline, NULL, 1);
 714                 for_each_online_cpu(cpu) {
 715                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 716                 }
 717
 718                 gator_running = 0;
 719                 gator_hrtimer_shutdown();
 720         }
 721 }
 722
 723 #if defined(__arm__) || defined(__aarch64__)
 724 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
 725         const char *core_name = NULL;
 726         char core_name_buf[32];
 727
 728         if (!sent_core_name[cpu]) {
 729                 if (gator_cpu != NULL) {
 730                         core_name = gator_cpu->core_name;
 731                 } else {
 732                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
 733                         core_name = core_name_buf;
 734                 }
 735
 736                 marshal_core_name(cpu, cpuid, core_name);
 737                 sent_core_name[cpu] = true;
 738         }
 739 }
 740 #endif
 741
 742 // This function runs in interrupt context and on the appropriate core
 743 static void gator_timer_online(void *migrate)
 744 {
 745         struct gator_interface *gi;
 746         int len, cpu = get_physical_cpu();
 747         int *buffer;
 748         u64 time;
 749
 750         gator_trace_power_online();
 751
 752         // online any events and output counters
 753         time = gator_get_time();
 754         if (marshal_event_header(time)) {
 755                 list_for_each_entry(gi, &gator_events, list) {
 756                         if (gi->online) {
 757                                 len = gi->online(&buffer, migrate);
 758                                 marshal_event(len, buffer);
 759                         }
 760                 }
 761                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 762                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 763         }
 764
 765         if (!migrate) {
 766                 gator_hrtimer_online();
 767         }
 768
 769 #if defined(__arm__) || defined(__aarch64__)
 770         if (!sent_core_name[cpu]) {
 771                 const u32 cpuid = gator_cpuid();
 772                 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
 773         }
 774 #endif
 775 }
 776
 777 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 778 static void gator_timer_online_dispatch(int cpu, bool migrate)
 779 {
 780         struct gator_interface *gi;
 781
 782         list_for_each_entry(gi, &gator_events, list) {
 783                 if (gi->online_dispatch) {
 784                         gi->online_dispatch(cpu, migrate);
 785                 }
 786         }
 787 }
 788
 789 #include "gator_iks.c"
 790
 791 int gator_timer_start(unsigned long sample_rate)
 792 {
 793         int cpu;
 794
 795         if (gator_running) {
 796                 pr_notice("gator: already running\n");
 797                 return 0;
 798         }
 799
 800         gator_running = 1;
 801
 802         // event based sampling trumps hr timer based sampling
 803         if (event_based_sampling) {
 804                 sample_rate = 0;
 805         }
 806
 807         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 808                 return -1;
 809
 810         gator_send_iks_core_names();
 811         for_each_online_cpu(cpu) {
 812                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 813         }
 814         on_each_cpu(gator_timer_online, NULL, 1);
 815
 816         return 0;
 817 }
 818
 819 static u64 gator_get_time(void)
 820 {
 821         struct timespec ts;
 822         u64 timestamp;
 823         u64 prev_timestamp;
 824         u64 delta;
 825         int cpu = smp_processor_id();
 826
 827         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
 828         getrawmonotonic(&ts);
 829         timestamp = timespec_to_ns(&ts);
 830
 831         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
 832         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
 833         // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
 834         prev_timestamp = per_cpu(last_timestamp, cpu);
 835         if (prev_timestamp <= timestamp) {
 836                 per_cpu(last_timestamp, cpu) = timestamp;
 837         } else {
 838                 delta = prev_timestamp - timestamp;
 839                 // Log the error once
 840                 if (!printed_monotonic_warning && delta > 500000) {
 841                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
 842                         printed_monotonic_warning = true;
 843                 }
 844                 timestamp = prev_timestamp;
 845         }
 846
 847         return timestamp - gator_monotonic_started;
 848 }
 849
 850 /******************************************************************************
 851  * cpu hotplug and pm notifiers
 852  ******************************************************************************/
 853 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 854 {
 855         int cpu = lcpu_to_pcpu((long)hcpu);
 856
 857         switch (action) {
 858         case CPU_DOWN_PREPARE:
 859         case CPU_DOWN_PREPARE_FROZEN:
 860                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 861                 gator_timer_offline_dispatch(cpu, false);
 862                 break;
 863         case CPU_ONLINE:
 864         case CPU_ONLINE_FROZEN:
 865                 gator_timer_online_dispatch(cpu, false);
 866                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 867                 break;
 868         }
 869
 870         return NOTIFY_OK;
 871 }
 872
 873 static struct notifier_block __refdata gator_hotcpu_notifier = {
 874         .notifier_call = gator_hotcpu_notify,
 875 };
 876
 877 // n.b. calling "on_each_cpu" only runs on those that are online
 878 // Registered linux events are not disabled, so their counters will continue to collect
 879 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 880 {
 881         int cpu;
 882         struct timespec ts;
 883
 884         switch (event) {
 885         case PM_HIBERNATION_PREPARE:
 886         case PM_SUSPEND_PREPARE:
 887                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 888                 unregister_scheduler_tracepoints();
 889                 on_each_cpu(gator_timer_offline, NULL, 1);
 890                 for_each_online_cpu(cpu) {
 891                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 892                 }
 893
 894                 // Record the wallclock hibernate time
 895                 getnstimeofday(&ts);
 896                 gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time();
 897                 break;
 898         case PM_POST_HIBERNATION:
 899         case PM_POST_SUSPEND:
 900                 // Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it
 901                 if (gator_hibernate_time > 0) {
 902                         getnstimeofday(&ts);
 903                         gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts);
 904                         gator_hibernate_time = 0;
 905                 }
 906
 907                 for_each_online_cpu(cpu) {
 908                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 909                 }
 910                 on_each_cpu(gator_timer_online, NULL, 1);
 911                 register_scheduler_tracepoints();
 912                 register_hotcpu_notifier(&gator_hotcpu_notifier);
 913                 break;
 914         }
 915
 916         return NOTIFY_OK;
 917 }
 918
 919 static struct notifier_block gator_pm_notifier = {
 920         .notifier_call = gator_pm_notify,
 921 };
 922
 923 static int gator_notifier_start(void)
 924 {
 925         int retval;
 926         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 927         if (retval == 0)
 928                 retval = register_pm_notifier(&gator_pm_notifier);
 929         return retval;
 930 }
 931
 932 static void gator_notifier_stop(void)
 933 {
 934         unregister_pm_notifier(&gator_pm_notifier);
 935         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 936 }
 937
 938 /******************************************************************************
 939  * Main
 940  ******************************************************************************/
 941 static void gator_summary(void)
 942 {
 943         u64 timestamp, uptime;
 944         struct timespec ts;
 945         char uname_buf[512];
 946         void (*m2b)(struct timespec *ts);
 947         unsigned long flags;
 948
 949         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
 950
 951         getnstimeofday(&ts);
 952         timestamp = timespec_to_ns(&ts);
 953
 954         do_posix_clock_monotonic_gettime(&ts);
 955         // monotonic_to_bootbased is not defined for some versions of Android
 956         m2b = symbol_get(monotonic_to_bootbased);
 957         if (m2b) {
 958                 m2b(&ts);
 959         }
 960         uptime = timespec_to_ns(&ts);
 961
 962         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
 963         local_irq_save(flags);
 964         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
 965         gator_monotonic_started = 0;
 966         gator_monotonic_started = gator_get_time();
 967         local_irq_restore(flags);
 968
 969         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
 970 }
 971
 972 int gator_events_install(struct gator_interface *interface)
 973 {
 974         list_add_tail(&interface->list, &gator_events);
 975
 976         return 0;
 977 }
 978
 979 int gator_events_get_key(void)
 980 {
 981         // key 0 is reserved as a timestamp
 982         // key 1 is reserved as the marker for thread specific counters
 983         // Odd keys are assigned by the driver, even keys by the daemon
 984         static int key = 3;
 985
 986         const int ret = key;
 987         key += 2;
 988         return ret;
 989 }
 990
 991 static int gator_init(void)
 992 {
 993         int i;
 994
 995         calc_first_cluster_size();
 996
 997         // events sources
 998         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 999                 if (gator_events_list[i])
1000                         gator_events_list[i]();
1001
1002         gator_trace_sched_init();
1003         gator_trace_power_init();
1004
1005         return 0;
1006 }
1007
1008 static void gator_exit(void)
1009 {
1010         struct gator_interface *gi;
1011
1012         list_for_each_entry(gi, &gator_events, list)
1013                 if (gi->shutdown)
1014                         gi->shutdown();
1015 }
1016
1017 static int gator_start(void)
1018 {
1019         unsigned long cpu, i;
1020         struct gator_interface *gi;
1021
1022         gator_buffer_wake_stop = false;
1023         if (IS_ERR(gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake"))) {
1024                 goto bwake_failure;
1025         }
1026
1027         if (gator_migrate_start())
1028                 goto migrate_failure;
1029
1030         // Initialize the buffer with the frame type and core
1031         for_each_present_cpu(cpu) {
1032                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1033                         marshal_frame(cpu, i);
1034                 }
1035                 per_cpu(last_timestamp, cpu) = 0;
1036         }
1037         printed_monotonic_warning = false;
1038
1039         // Capture the start time
1040         gator_summary();
1041
1042         // start all events
1043         list_for_each_entry(gi, &gator_events, list) {
1044                 if (gi->start && gi->start() != 0) {
1045                         struct list_head *ptr = gi->list.prev;
1046
1047                         while (ptr != &gator_events) {
1048                                 gi = list_entry(ptr, struct gator_interface, list);
1049
1050                                 if (gi->stop)
1051                                         gi->stop();
1052
1053                                 ptr = ptr->prev;
1054                         }
1055                         goto events_failure;
1056                 }
1057         }
1058
1059         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
1060         if (cookies_initialize())
1061                 goto cookies_failure;
1062         if (gator_annotate_start())
1063                 goto annotate_failure;
1064         if (gator_trace_sched_start())
1065                 goto sched_failure;
1066         if (gator_trace_power_start())
1067                 goto power_failure;
1068         if (gator_trace_gpu_start())
1069                 goto gpu_failure;
1070         if (gator_timer_start(gator_timer_count))
1071                 goto timer_failure;
1072         if (gator_notifier_start())
1073                 goto notifier_failure;
1074
1075         return 0;
1076
1077 notifier_failure:
1078         gator_timer_stop();
1079 timer_failure:
1080         gator_trace_gpu_stop();
1081 gpu_failure:
1082         gator_trace_power_stop();
1083 power_failure:
1084         gator_trace_sched_stop();
1085 sched_failure:
1086         gator_annotate_stop();
1087 annotate_failure:
1088         cookies_release();
1089 cookies_failure:
1090         // stop all events
1091         list_for_each_entry(gi, &gator_events, list)
1092                 if (gi->stop)
1093                         gi->stop();
1094 events_failure:
1095         gator_migrate_stop();
1096 migrate_failure:
1097         gator_buffer_wake_stop = true;
1098         wake_up_process(gator_buffer_wake_thread);
1099 bwake_failure:
1100
1101         return -1;
1102 }
1103
1104 static void gator_stop(void)
1105 {
1106         struct gator_interface *gi;
1107
1108         gator_annotate_stop();
1109         gator_trace_sched_stop();
1110         gator_trace_power_stop();
1111         gator_trace_gpu_stop();
1112
1113         // stop all interrupt callback reads before tearing down other interfaces
1114         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1115         gator_timer_stop();
1116
1117         // stop all events
1118         list_for_each_entry(gi, &gator_events, list)
1119                 if (gi->stop)
1120                         gi->stop();
1121
1122         gator_migrate_stop();
1123
1124         gator_buffer_wake_stop = true;
1125         wake_up_process(gator_buffer_wake_thread);
1126 }
1127
1128 /******************************************************************************
1129  * Filesystem
1130  ******************************************************************************/
1131 /* fopen("buffer") */
1132 static int gator_op_setup(void)
1133 {
1134         int err = 0;
1135         int cpu, i;
1136
1137         mutex_lock(&start_mutex);
1138
1139         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1140         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1141
1142         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1143         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1144
1145         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1146         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1147
1148         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1149         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1150
1151         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1152         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1153
1154         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1155         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1156
1157         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1158         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1159
1160         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1161         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1162
1163         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1164         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1165
1166         // Initialize percpu per buffer variables
1167         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1168                 // Verify buffers are a power of 2
1169                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1170                         err = -ENOEXEC;
1171                         goto setup_error;
1172                 }
1173
1174                 for_each_present_cpu(cpu) {
1175                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1176                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1177                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1178                         per_cpu(buffer_space_available, cpu)[i] = true;
1179                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1180
1181                         // Annotation is a special case that only uses a single buffer
1182                         if (cpu > 0 && i == ANNOTATE_BUF) {
1183                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1184                                 continue;
1185                         }
1186
1187                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1188                         if (!per_cpu(gator_buffer, cpu)[i]) {
1189                                 err = -ENOMEM;
1190                                 goto setup_error;
1191                         }
1192                 }
1193         }
1194
1195 setup_error:
1196         mutex_unlock(&start_mutex);
1197         return err;
1198 }
1199
1200 /* Actually start profiling (echo 1>/dev/gator/enable) */
1201 static int gator_op_start(void)
1202 {
1203         int err = 0;
1204
1205         mutex_lock(&start_mutex);
1206
1207         if (gator_started || gator_start())
1208                 err = -EINVAL;
1209         else
1210                 gator_started = 1;
1211
1212         mutex_unlock(&start_mutex);
1213
1214         return err;
1215 }
1216
1217 /* echo 0>/dev/gator/enable */
1218 static void gator_op_stop(void)
1219 {
1220         mutex_lock(&start_mutex);
1221
1222         if (gator_started) {
1223                 gator_stop();
1224
1225                 mutex_lock(&gator_buffer_mutex);
1226
1227                 gator_started = 0;
1228                 gator_monotonic_started = 0;
1229                 cookies_release();
1230                 wake_up(&gator_buffer_wait);
1231
1232                 mutex_unlock(&gator_buffer_mutex);
1233         }
1234
1235         mutex_unlock(&start_mutex);
1236 }
1237
1238 static void gator_shutdown(void)
1239 {
1240         int cpu, i;
1241
1242         mutex_lock(&start_mutex);
1243
1244         for_each_present_cpu(cpu) {
1245                 mutex_lock(&gator_buffer_mutex);
1246                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1247                         vfree(per_cpu(gator_buffer, cpu)[i]);
1248                         per_cpu(gator_buffer, cpu)[i] = NULL;
1249                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1250                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1251                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1252                         per_cpu(buffer_space_available, cpu)[i] = true;
1253                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1254                 }
1255                 mutex_unlock(&gator_buffer_mutex);
1256         }
1257
1258         memset(&sent_core_name, 0, sizeof(sent_core_name));
1259
1260         mutex_unlock(&start_mutex);
1261 }
1262
1263 static int gator_set_backtrace(unsigned long val)
1264 {
1265         int err = 0;
1266
1267         mutex_lock(&start_mutex);
1268
1269         if (gator_started)
1270                 err = -EBUSY;
1271         else
1272                 gator_backtrace_depth = val;
1273
1274         mutex_unlock(&start_mutex);
1275
1276         return err;
1277 }
1278
1279 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1280 {
1281         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1282 }
1283
1284 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1285 {
1286         unsigned long val;
1287         int retval;
1288
1289         if (*offset)
1290                 return -EINVAL;
1291
1292         retval = gatorfs_ulong_from_user(&val, buf, count);
1293         if (retval)
1294                 return retval;
1295
1296         if (val)
1297                 retval = gator_op_start();
1298         else
1299                 gator_op_stop();
1300
1301         if (retval)
1302                 return retval;
1303         return count;
1304 }
1305
1306 static const struct file_operations enable_fops = {
1307         .read = enable_read,
1308         .write = enable_write,
1309 };
1310
1311 static int userspace_buffer_open(struct inode *inode, struct file *file)
1312 {
1313         int err = -EPERM;
1314
1315         if (!capable(CAP_SYS_ADMIN))
1316                 return -EPERM;
1317
1318         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1319                 return -EBUSY;
1320
1321         if ((err = gator_op_setup()))
1322                 goto fail;
1323
1324         /* NB: the actual start happens from userspace
1325          * echo 1 >/dev/gator/enable
1326          */
1327
1328         return 0;
1329
1330 fail:
1331         __clear_bit_unlock(0, &gator_buffer_opened);
1332         return err;
1333 }
1334
1335 static int userspace_buffer_release(struct inode *inode, struct file *file)
1336 {
1337         gator_op_stop();
1338         gator_shutdown();
1339         __clear_bit_unlock(0, &gator_buffer_opened);
1340         return 0;
1341 }
1342
1343 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1344 {
1345         int commit, length1, length2, read;
1346         char *buffer1;
1347         char *buffer2;
1348         int cpu, buftype;
1349         int written = 0;
1350
1351         // ensure there is enough space for a whole frame
1352         if (count < userspace_buffer_size || *offset) {
1353                 return -EINVAL;
1354         }
1355
1356         // sleep until the condition is true or a signal is received
1357         // the condition is checked each time gator_buffer_wait is woken up
1358         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1359
1360         if (signal_pending(current)) {
1361                 return -EINTR;
1362         }
1363
1364         if (buftype == -1 || cpu == -1) {
1365                 return 0;
1366         }
1367
1368         mutex_lock(&gator_buffer_mutex);
1369
1370         do {
1371                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1372                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1373
1374                 // May happen if the buffer is freed during pending reads.
1375                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1376                         break;
1377                 }
1378
1379                 // determine the size of two halves
1380                 length1 = commit - read;
1381                 length2 = 0;
1382                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1383                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1384                 if (length1 < 0) {
1385                         length1 = gator_buffer_size[buftype] - read;
1386                         length2 = commit;
1387                 }
1388
1389                 if (length1 + length2 > count - written) {
1390                         break;
1391                 }
1392
1393                 // start, middle or end
1394                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1395                         break;
1396                 }
1397
1398                 // possible wrap around
1399                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1400                         break;
1401                 }
1402
1403                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1404                 written += length1 + length2;
1405
1406                 // Wake up annotate_write if more space is available
1407                 if (buftype == ANNOTATE_BUF) {
1408                         wake_up(&gator_annotate_wait);
1409                 }
1410         } while (buffer_commit_ready(&cpu, &buftype));
1411
1412         mutex_unlock(&gator_buffer_mutex);
1413
1414         // kick just in case we've lost an SMP event
1415         wake_up(&gator_buffer_wait);
1416
1417         return written > 0 ? written : -EFAULT;
1418 }
1419
1420 const struct file_operations gator_event_buffer_fops = {
1421         .open = userspace_buffer_open,
1422         .release = userspace_buffer_release,
1423         .read = userspace_buffer_read,
1424 };
1425
1426 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1427 {
1428         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1429 }
1430
1431 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1432 {
1433         unsigned long val;
1434         int retval;
1435
1436         if (*offset)
1437                 return -EINVAL;
1438
1439         retval = gatorfs_ulong_from_user(&val, buf, count);
1440         if (retval)
1441                 return retval;
1442
1443         retval = gator_set_backtrace(val);
1444
1445         if (retval)
1446                 return retval;
1447         return count;
1448 }
1449
1450 static const struct file_operations depth_fops = {
1451         .read = depth_read,
1452         .write = depth_write
1453 };
1454
1455 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1456 {
1457         struct dentry *dir;
1458         struct gator_interface *gi;
1459         int cpu;
1460
1461         /* reinitialize default values */
1462         gator_cpu_cores = 0;
1463         for_each_present_cpu(cpu) {
1464                 gator_cpu_cores++;
1465         }
1466         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1467         gator_response_type = 1;
1468         gator_live_rate = 0;
1469
1470         gatorfs_create_file(sb, root, "enable", &enable_fops);
1471         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1472         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1473         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1474         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1475         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1476         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1477         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1478         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1479         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1480
1481         // Annotate interface
1482         gator_annotate_create_files(sb, root);
1483
1484         // Linux Events
1485         dir = gatorfs_mkdir(sb, root, "events");
1486         list_for_each_entry(gi, &gator_events, list)
1487                 if (gi->create_files)
1488                         gi->create_files(sb, dir);
1489
1490         // Sched Events
1491         sched_trace_create_files(sb, dir);
1492
1493         // Power interface
1494         gator_trace_power_create_files(sb, dir);
1495 }
1496
1497 /******************************************************************************
1498  * Module
1499  ******************************************************************************/
1500 static int __init gator_module_init(void)
1501 {
1502         if (gatorfs_register()) {
1503                 return -1;
1504         }
1505
1506         if (gator_init()) {
1507                 gatorfs_unregister();
1508                 return -1;
1509         }
1510
1511         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1512
1513         return 0;
1514 }
1515
1516 static void __exit gator_module_exit(void)
1517 {
1518         del_timer_sync(&gator_buffer_wake_up_timer);
1519         tracepoint_synchronize_unregister();
1520         gator_exit();
1521         gatorfs_unregister();
1522 }
1523
1524 module_init(gator_module_init);
1525 module_exit(gator_module_exit);
1526
1527 MODULE_LICENSE("GPL");
1528 MODULE_AUTHOR("ARM Ltd");
1529 MODULE_DESCRIPTION("Gator system profiler");
1530 #define STRIFY2(ARG) #ARG
1531 #define STRIFY(ARG) STRIFY2(ARG)
1532 MODULE_VERSION(STRIFY(PROTOCOL_VERSION));