driver/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 // This version must match the gator daemon version
  11 static unsigned long gator_protocol_version = 16;
  12
  13 #include <linux/slab.h>
  14 #include <linux/cpu.h>
  15 #include <linux/sched.h>
  16 #include <linux/irq.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/hardirq.h>
  19 #include <linux/highmem.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/suspend.h>
  22 #include <linux/module.h>
  23 #include <linux/perf_event.h>
  24 #include <linux/utsname.h>
  25 #include <asm/stacktrace.h>
  26 #include <asm/uaccess.h>
  27
  28 #include "gator.h"
  29 #include "gator_events.h"
  30
  31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
  32 #error kernels prior to 2.6.32 are not supported
  33 #endif
  34
  35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  37 #endif
  38
  39 #ifndef CONFIG_PROFILING
  40 #error gator requires the kernel to have CONFIG_PROFILING defined
  41 #endif
  42
  43 #ifndef CONFIG_HIGH_RES_TIMERS
  44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  45 #endif
  46
  47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  49 #endif
  50
  51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
  52 #ifndef CONFIG_PERF_EVENTS
  53 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  54 #elif !defined CONFIG_HW_PERF_EVENTS
  55 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  56 #endif
  57 #endif
  58
  59 /******************************************************************************
  60  * DEFINES
  61  ******************************************************************************/
  62 #define SUMMARY_BUFFER_SIZE       (1*1024)
  63 #define BACKTRACE_BUFFER_SIZE     (128*1024)
  64 #define NAME_BUFFER_SIZE          (64*1024)
  65 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
  66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
  67 #define ANNOTATE_BUFFER_SIZE      (128*1024)    // annotate counters have the core as part of the data and the core value in the frame header may be discarded
  68 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
  69 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
  70 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
  71
  72 #define NO_COOKIE      0U
  73 #define UNRESOLVED_COOKIE ~0U
  74
  75 #define FRAME_SUMMARY       1
  76 #define FRAME_BACKTRACE     2
  77 #define FRAME_NAME          3
  78 #define FRAME_COUNTER       4
  79 #define FRAME_BLOCK_COUNTER 5
  80 #define FRAME_ANNOTATE      6
  81 #define FRAME_SCHED_TRACE   7
  82 #define FRAME_GPU_TRACE     8
  83 #define FRAME_IDLE          9
  84
  85 #define MESSAGE_END_BACKTRACE 1
  86
  87 #define MESSAGE_COOKIE      1
  88 #define MESSAGE_THREAD_NAME 2
  89 #define HRTIMER_CORE_NAME   3
  90
  91 #define MESSAGE_GPU_START 1
  92 #define MESSAGE_GPU_STOP  2
  93
  94 #define MESSAGE_SCHED_SWITCH 1
  95 #define MESSAGE_SCHED_EXIT   2
  96 #define MESSAGE_SCHED_START  3
  97
  98 #define MESSAGE_IDLE_ENTER 1
  99 #define MESSAGE_IDLE_EXIT 2
 100
 101 #define MAXSIZE_PACK32     5
 102 #define MAXSIZE_PACK64    10
 103
 104 #define FRAME_HEADER_SIZE 3
 105
 106 #if defined(__arm__)
 107 #define PC_REG regs->ARM_pc
 108 #elif defined(__aarch64__)
 109 #define PC_REG regs->pc
 110 #else
 111 #define PC_REG regs->ip
 112 #endif
 113
 114 enum {
 115         SUMMARY_BUF,
 116         BACKTRACE_BUF,
 117         NAME_BUF,
 118         COUNTER_BUF,
 119         BLOCK_COUNTER_BUF,
 120         ANNOTATE_BUF,
 121         SCHED_TRACE_BUF,
 122         GPU_TRACE_BUF,
 123         IDLE_BUF,
 124         NUM_GATOR_BUFS
 125 };
 126
 127 /******************************************************************************
 128  * Globals
 129  ******************************************************************************/
 130 static unsigned long gator_cpu_cores;
 131 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
 132 static unsigned long userspace_buffer_size;
 133 static unsigned long gator_backtrace_depth;
 134 // How often to commit the buffers for live in nanoseconds
 135 static u64 gator_live_rate;
 136
 137 static unsigned long gator_started;
 138 static u64 gator_monotonic_started;
 139 static unsigned long gator_buffer_opened;
 140 static unsigned long gator_timer_count;
 141 static unsigned long gator_response_type;
 142 static DEFINE_MUTEX(start_mutex);
 143 static DEFINE_MUTEX(gator_buffer_mutex);
 144
 145 bool event_based_sampling;
 146
 147 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 148 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
 149 static struct timer_list gator_buffer_wake_up_timer;
 150 static LIST_HEAD(gator_events);
 151
 152 static DEFINE_PER_CPU(u64, last_timestamp);
 153
 154 static bool printed_monotonic_warning;
 155
 156 static bool sent_core_name[NR_CPUS];
 157
 158 /******************************************************************************
 159  * Prototypes
 160  ******************************************************************************/
 161 static void buffer_check(int cpu, int buftype, u64 time);
 162 static void gator_commit_buffer(int cpu, int buftype, u64 time);
 163 static int buffer_bytes_available(int cpu, int buftype);
 164 static bool buffer_check_space(int cpu, int buftype, int bytes);
 165 static int contiguous_space_available(int cpu, int bufytpe);
 166 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
 167 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
 168 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
 169 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
 170 static void gator_add_trace(int cpu, unsigned long address);
 171 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
 172 static u64 gator_get_time(void);
 173
 174 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
 175 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 176 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
 177 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 178 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
 179 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 180 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
 181 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 182 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
 183 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 184 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
 185 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
 186 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 187 // The buffer. Allocated in gator_op_setup
 188 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 189 // The time after which the buffer should be committed for live display
 190 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
 191
 192 /******************************************************************************
 193  * Application Includes
 194  ******************************************************************************/
 195 #include "gator_marshaling.c"
 196 #include "gator_hrtimer_perf.c"
 197 #include "gator_hrtimer_gator.c"
 198 #include "gator_cookies.c"
 199 #include "gator_annotate.c"
 200 #include "gator_trace_sched.c"
 201 #include "gator_trace_power.c"
 202 #include "gator_trace_gpu.c"
 203 #include "gator_backtrace.c"
 204 #include "gator_fs.c"
 205 #include "gator_pack.c"
 206
 207 /******************************************************************************
 208  * Misc
 209  ******************************************************************************/
 210
 211 const struct gator_cpu gator_cpus[] = {
 212         {
 213                 .cpuid = ARM1136,
 214                 .core_name = "ARM1136",
 215                 .pmnc_name = "ARM_ARM11",
 216                 .dt_name = "arm,arm1136",
 217                 .pmnc_counters = 3,
 218         },
 219         {
 220                 .cpuid = ARM1156,
 221                 .core_name = "ARM1156",
 222                 .pmnc_name = "ARM_ARM11",
 223                 .dt_name = "arm,arm1156",
 224                 .pmnc_counters = 3,
 225         },
 226         {
 227                 .cpuid = ARM1176,
 228                 .core_name = "ARM1176",
 229                 .pmnc_name = "ARM_ARM11",
 230                 .dt_name = "arm,arm1176",
 231                 .pmnc_counters = 3,
 232         },
 233         {
 234                 .cpuid = ARM11MPCORE,
 235                 .core_name = "ARM11MPCore",
 236                 .pmnc_name = "ARM_ARM11MPCore",
 237                 .dt_name = "arm,arm11mpcore",
 238                 .pmnc_counters = 3,
 239         },
 240         {
 241                 .cpuid = CORTEX_A5,
 242                 .core_name = "Cortex-A5",
 243                 .pmu_name = "ARMv7_Cortex_A5",
 244                 .pmnc_name = "ARM_Cortex-A5",
 245                 .dt_name = "arm,cortex-a5",
 246                 .pmnc_counters = 2,
 247         },
 248         {
 249                 .cpuid = CORTEX_A7,
 250                 .core_name = "Cortex-A7",
 251                 .pmu_name = "ARMv7_Cortex_A7",
 252                 .pmnc_name = "ARM_Cortex-A7",
 253                 .dt_name = "arm,cortex-a7",
 254                 .pmnc_counters = 4,
 255         },
 256         {
 257                 .cpuid = CORTEX_A8,
 258                 .core_name = "Cortex-A8",
 259                 .pmu_name = "ARMv7_Cortex_A8",
 260                 .pmnc_name = "ARM_Cortex-A8",
 261                 .dt_name = "arm,cortex-a8",
 262                 .pmnc_counters = 4,
 263         },
 264         {
 265                 .cpuid = CORTEX_A9,
 266                 .core_name = "Cortex-A9",
 267                 .pmu_name = "ARMv7_Cortex_A9",
 268                 .pmnc_name = "ARM_Cortex-A9",
 269                 .dt_name = "arm,cortex-a9",
 270                 .pmnc_counters = 6,
 271         },
 272         {
 273                 .cpuid = CORTEX_A12,
 274                 .core_name = "Cortex-A12",
 275                 .pmu_name = "ARMv7_Cortex_A12",
 276                 .pmnc_name = "ARM_Cortex-A12",
 277                 .dt_name = "arm,cortex-a12",
 278                 .pmnc_counters = 6,
 279         },
 280         {
 281                 .cpuid = CORTEX_A15,
 282                 .core_name = "Cortex-A15",
 283                 .pmu_name = "ARMv7_Cortex_A15",
 284                 .pmnc_name = "ARM_Cortex-A15",
 285                 .dt_name = "arm,cortex-a15",
 286                 .pmnc_counters = 6,
 287         },
 288         {
 289                 .cpuid = SCORPION,
 290                 .core_name = "Scorpion",
 291                 .pmnc_name = "Scorpion",
 292                 .pmnc_counters = 4,
 293         },
 294         {
 295                 .cpuid = SCORPIONMP,
 296                 .core_name = "ScorpionMP",
 297                 .pmnc_name = "ScorpionMP",
 298                 .pmnc_counters = 4,
 299         },
 300         {
 301                 .cpuid = KRAITSIM,
 302                 .core_name = "KraitSIM",
 303                 .pmnc_name = "Krait",
 304                 .pmnc_counters = 4,
 305         },
 306         {
 307                 .cpuid = KRAIT,
 308                 .core_name = "Krait",
 309                 .pmnc_name = "Krait",
 310                 .pmnc_counters = 4,
 311         },
 312         {
 313                 .cpuid = KRAIT_S4_PRO,
 314                 .core_name = "Krait S4 Pro",
 315                 .pmnc_name = "Krait",
 316                 .pmnc_counters = 4,
 317         },
 318         {
 319                 .cpuid = CORTEX_A53,
 320                 .core_name = "Cortex-A53",
 321                 .pmnc_name = "ARM_Cortex-A53",
 322                 .dt_name = "arm,cortex-a53",
 323                 .pmnc_counters = 6,
 324         },
 325         {
 326                 .cpuid = CORTEX_A57,
 327                 .core_name = "Cortex-A57",
 328                 .pmnc_name = "ARM_Cortex-A57",
 329                 .dt_name = "arm,cortex-a57",
 330                 .pmnc_counters = 6,
 331         },
 332         {
 333                 .cpuid = AARCH64,
 334                 .core_name = "AArch64",
 335                 .pmnc_name = "ARM_AArch64",
 336                 .pmnc_counters = 6,
 337         },
 338         {
 339                 .cpuid = OTHER,
 340                 .core_name = "Other",
 341                 .pmnc_name = "Other",
 342                 .pmnc_counters = 6,
 343         },
 344         {}
 345 };
 346
 347 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
 348 {
 349         int i;
 350
 351         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 352                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 353                 if (gator_cpu->cpuid == cpuid) {
 354                         return gator_cpu;
 355                 }
 356         }
 357
 358         return NULL;
 359 }
 360
 361 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
 362 {
 363         int i;
 364
 365         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 366                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 367                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
 368                         return gator_cpu;
 369                 }
 370         }
 371
 372         return NULL;
 373 }
 374
 375 u32 gator_cpuid(void)
 376 {
 377 #if defined(__arm__) || defined(__aarch64__)
 378         u32 val;
 379 #if !defined(__aarch64__)
 380         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 381 #else
 382         asm volatile("mrs %0, midr_el1" : "=r" (val));
 383 #endif
 384         return (val >> 4) & 0xfff;
 385 #else
 386         return OTHER;
 387 #endif
 388 }
 389
 390 static void gator_buffer_wake_up(unsigned long data)
 391 {
 392         wake_up(&gator_buffer_wait);
 393 }
 394
 395 /******************************************************************************
 396  * Commit interface
 397  ******************************************************************************/
 398 static bool buffer_commit_ready(int *cpu, int *buftype)
 399 {
 400         int cpu_x, x;
 401         for_each_present_cpu(cpu_x) {
 402                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 403                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 404                                 *cpu = cpu_x;
 405                                 *buftype = x;
 406                                 return true;
 407                         }
 408         }
 409         *cpu = -1;
 410         *buftype = -1;
 411         return false;
 412 }
 413
 414 /******************************************************************************
 415  * Buffer management
 416  ******************************************************************************/
 417 static int buffer_bytes_available(int cpu, int buftype)
 418 {
 419         int remaining, filled;
 420
 421         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
 422         if (filled < 0) {
 423                 filled += gator_buffer_size[buftype];
 424         }
 425
 426         remaining = gator_buffer_size[buftype] - filled;
 427
 428         if (per_cpu(buffer_space_available, cpu)[buftype]) {
 429                 // Give some extra room; also allows space to insert the overflow error packet
 430                 remaining -= 200;
 431         } else {
 432                 // Hysteresis, prevents multiple overflow messages
 433                 remaining -= 2000;
 434         }
 435
 436         return remaining;
 437 }
 438
 439 static int contiguous_space_available(int cpu, int buftype)
 440 {
 441         int remaining = buffer_bytes_available(cpu, buftype);
 442         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
 443         if (remaining < contiguous)
 444                 return remaining;
 445         else
 446                 return contiguous;
 447 }
 448
 449 static bool buffer_check_space(int cpu, int buftype, int bytes)
 450 {
 451         int remaining = buffer_bytes_available(cpu, buftype);
 452
 453         if (remaining < bytes) {
 454                 per_cpu(buffer_space_available, cpu)[buftype] = false;
 455         } else {
 456                 per_cpu(buffer_space_available, cpu)[buftype] = true;
 457         }
 458
 459         return per_cpu(buffer_space_available, cpu)[buftype];
 460 }
 461
 462 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
 463 {
 464         int i;
 465         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
 466         u32 mask = gator_buffer_mask[buftype];
 467         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
 468
 469         for (i = 0; i < len; i++) {
 470                 buffer[write] = x[i];
 471                 write = (write + 1) & mask;
 472         }
 473
 474         per_cpu(gator_buffer_write, cpu)[buftype] = write;
 475 }
 476
 477 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
 478 {
 479         int len = strlen(x);
 480         gator_buffer_write_packed_int(cpu, buftype, len);
 481         gator_buffer_write_bytes(cpu, buftype, x, len);
 482 }
 483
 484 static void gator_commit_buffer(int cpu, int buftype, u64 time)
 485 {
 486         int type_length, commit, length, byte;
 487
 488         if (!per_cpu(gator_buffer, cpu)[buftype])
 489                 return;
 490
 491         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
 492         type_length = gator_response_type ? 1 : 0;
 493         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
 494         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
 495         if (length < 0) {
 496                 length += gator_buffer_size[buftype];
 497         }
 498         length = length - type_length - sizeof(s32);
 499
 500         if (length <= FRAME_HEADER_SIZE) {
 501                 // Nothing to write, only the frame header is present
 502                 return;
 503         }
 504
 505         for (byte = 0; byte < sizeof(s32); byte++) {
 506                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
 507         }
 508
 509         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
 510
 511         if (gator_live_rate > 0) {
 512                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
 513                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
 514                 }
 515         }
 516
 517         marshal_frame(cpu, buftype);
 518
 519         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 520         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
 521 }
 522
 523 static void buffer_check(int cpu, int buftype, u64 time)
 524 {
 525         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
 526         if (filled < 0) {
 527                 filled += gator_buffer_size[buftype];
 528         }
 529         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
 530                 gator_commit_buffer(cpu, buftype, time);
 531         }
 532 }
 533
 534 static void gator_add_trace(int cpu, unsigned long address)
 535 {
 536         off_t offset = 0;
 537         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
 538
 539         if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
 540                 offset = address;
 541         }
 542
 543         marshal_backtrace(offset & ~1, cookie);
 544 }
 545
 546 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
 547 {
 548         bool inKernel;
 549         unsigned long exec_cookie;
 550
 551         if (!regs)
 552                 return;
 553
 554         inKernel = !user_mode(regs);
 555         exec_cookie = get_exec_cookie(cpu, current);
 556
 557         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
 558                 return;
 559
 560         if (inKernel) {
 561                 kernel_backtrace(cpu, regs);
 562         } else {
 563                 // Cookie+PC
 564                 gator_add_trace(cpu, PC_REG);
 565
 566                 // Backtrace
 567                 if (gator_backtrace_depth)
 568                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
 569         }
 570
 571         marshal_backtrace_footer(time);
 572 }
 573
 574 /******************************************************************************
 575  * hrtimer interrupt processing
 576  ******************************************************************************/
 577 static void gator_timer_interrupt(void)
 578 {
 579         struct pt_regs *const regs = get_irq_regs();
 580         gator_backtrace_handler(regs);
 581 }
 582
 583 void gator_backtrace_handler(struct pt_regs *const regs)
 584 {
 585         u64 time = gator_get_time();
 586         int cpu = get_physical_cpu();
 587
 588         // Output backtrace
 589         gator_add_sample(cpu, regs, time);
 590
 591         // Collect counters
 592         if (!per_cpu(collecting, cpu)) {
 593                 collect_counters(time);
 594         }
 595 }
 596
 597 static int gator_running;
 598
 599 // This function runs in interrupt context and on the appropriate core
 600 static void gator_timer_offline(void *migrate)
 601 {
 602         struct gator_interface *gi;
 603         int i, len, cpu = get_physical_cpu();
 604         int *buffer;
 605         u64 time;
 606
 607         gator_trace_sched_offline();
 608         gator_trace_power_offline();
 609
 610         if (!migrate) {
 611                 gator_hrtimer_offline();
 612         }
 613
 614         // Offline any events and output counters
 615         time = gator_get_time();
 616         if (marshal_event_header(time)) {
 617                 list_for_each_entry(gi, &gator_events, list) {
 618                         if (gi->offline) {
 619                                 len = gi->offline(&buffer, migrate);
 620                                 marshal_event(len, buffer);
 621                         }
 622                 }
 623                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 624                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 625         }
 626
 627         // Flush all buffers on this core
 628         for (i = 0; i < NUM_GATOR_BUFS; i++)
 629                 gator_commit_buffer(cpu, i, time);
 630 }
 631
 632 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 633 static void gator_timer_offline_dispatch(int cpu, bool migrate)
 634 {
 635         struct gator_interface *gi;
 636
 637         list_for_each_entry(gi, &gator_events, list) {
 638                 if (gi->offline_dispatch) {
 639                         gi->offline_dispatch(cpu, migrate);
 640                 }
 641         }
 642 }
 643
 644 static void gator_timer_stop(void)
 645 {
 646         int cpu;
 647
 648         if (gator_running) {
 649                 on_each_cpu(gator_timer_offline, NULL, 1);
 650                 for_each_online_cpu(cpu) {
 651                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 652                 }
 653
 654                 gator_running = 0;
 655                 gator_hrtimer_shutdown();
 656         }
 657 }
 658
 659 #if defined(__arm__) || defined(__aarch64__)
 660 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
 661         const char *core_name = NULL;
 662         char core_name_buf[32];
 663
 664         if (!sent_core_name[cpu]) {
 665                 if (gator_cpu != NULL) {
 666                         core_name = gator_cpu->core_name;
 667                 } else {
 668                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
 669                         core_name = core_name_buf;
 670                 }
 671
 672                 marshal_core_name(cpu, cpuid, core_name);
 673                 sent_core_name[cpu] = true;
 674         }
 675 }
 676 #endif
 677
 678 // This function runs in interrupt context and on the appropriate core
 679 static void gator_timer_online(void *migrate)
 680 {
 681         struct gator_interface *gi;
 682         int len, cpu = get_physical_cpu();
 683         int *buffer;
 684         u64 time;
 685
 686         gator_trace_power_online();
 687
 688         // online any events and output counters
 689         time = gator_get_time();
 690         if (marshal_event_header(time)) {
 691                 list_for_each_entry(gi, &gator_events, list) {
 692                         if (gi->online) {
 693                                 len = gi->online(&buffer, migrate);
 694                                 marshal_event(len, buffer);
 695                         }
 696                 }
 697                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 698                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 699         }
 700
 701         if (!migrate) {
 702                 gator_hrtimer_online();
 703         }
 704
 705 #if defined(__arm__) || defined(__aarch64__)
 706         if (!sent_core_name[cpu]) {
 707                 const u32 cpuid = gator_cpuid();
 708                 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
 709         }
 710 #endif
 711 }
 712
 713 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 714 static void gator_timer_online_dispatch(int cpu, bool migrate)
 715 {
 716         struct gator_interface *gi;
 717
 718         list_for_each_entry(gi, &gator_events, list) {
 719                 if (gi->online_dispatch) {
 720                         gi->online_dispatch(cpu, migrate);
 721                 }
 722         }
 723 }
 724
 725 #include "gator_iks.c"
 726
 727 int gator_timer_start(unsigned long sample_rate)
 728 {
 729         int cpu;
 730
 731         if (gator_running) {
 732                 pr_notice("gator: already running\n");
 733                 return 0;
 734         }
 735
 736         gator_running = 1;
 737
 738         // event based sampling trumps hr timer based sampling
 739         if (event_based_sampling) {
 740                 sample_rate = 0;
 741         }
 742
 743         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 744                 return -1;
 745
 746         gator_send_iks_core_names();
 747         for_each_online_cpu(cpu) {
 748                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 749         }
 750         on_each_cpu(gator_timer_online, NULL, 1);
 751
 752         return 0;
 753 }
 754
 755 static u64 gator_get_time(void)
 756 {
 757         struct timespec ts;
 758         u64 timestamp;
 759         u64 prev_timestamp;
 760         u64 delta;
 761         int cpu = smp_processor_id();
 762
 763         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
 764         getrawmonotonic(&ts);
 765         timestamp = timespec_to_ns(&ts);
 766
 767         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
 768         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
 769         // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
 770         prev_timestamp = per_cpu(last_timestamp, cpu);
 771         if (prev_timestamp <= timestamp) {
 772                 per_cpu(last_timestamp, cpu) = timestamp;
 773         } else {
 774                 delta = prev_timestamp - timestamp;
 775                 // Log the error once
 776                 if (!printed_monotonic_warning && delta > 500000) {
 777                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
 778                         printed_monotonic_warning = true;
 779                 }
 780                 timestamp = prev_timestamp;
 781         }
 782
 783         return timestamp - gator_monotonic_started;
 784 }
 785
 786 /******************************************************************************
 787  * cpu hotplug and pm notifiers
 788  ******************************************************************************/
 789 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 790 {
 791         int cpu = lcpu_to_pcpu((long)hcpu);
 792
 793         switch (action) {
 794         case CPU_DOWN_PREPARE:
 795         case CPU_DOWN_PREPARE_FROZEN:
 796                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 797                 gator_timer_offline_dispatch(cpu, false);
 798                 break;
 799         case CPU_ONLINE:
 800         case CPU_ONLINE_FROZEN:
 801                 gator_timer_online_dispatch(cpu, false);
 802                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 803                 break;
 804         }
 805
 806         return NOTIFY_OK;
 807 }
 808
 809 static struct notifier_block __refdata gator_hotcpu_notifier = {
 810         .notifier_call = gator_hotcpu_notify,
 811 };
 812
 813 // n.b. calling "on_each_cpu" only runs on those that are online
 814 // Registered linux events are not disabled, so their counters will continue to collect
 815 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 816 {
 817         int cpu;
 818
 819         switch (event) {
 820         case PM_HIBERNATION_PREPARE:
 821         case PM_SUSPEND_PREPARE:
 822                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 823                 unregister_scheduler_tracepoints();
 824                 on_each_cpu(gator_timer_offline, NULL, 1);
 825                 for_each_online_cpu(cpu) {
 826                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 827                 }
 828                 break;
 829         case PM_POST_HIBERNATION:
 830         case PM_POST_SUSPEND:
 831                 for_each_online_cpu(cpu) {
 832                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 833                 }
 834                 on_each_cpu(gator_timer_online, NULL, 1);
 835                 register_scheduler_tracepoints();
 836                 register_hotcpu_notifier(&gator_hotcpu_notifier);
 837                 break;
 838         }
 839
 840         return NOTIFY_OK;
 841 }
 842
 843 static struct notifier_block gator_pm_notifier = {
 844         .notifier_call = gator_pm_notify,
 845 };
 846
 847 static int gator_notifier_start(void)
 848 {
 849         int retval;
 850         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 851         if (retval == 0)
 852                 retval = register_pm_notifier(&gator_pm_notifier);
 853         return retval;
 854 }
 855
 856 static void gator_notifier_stop(void)
 857 {
 858         unregister_pm_notifier(&gator_pm_notifier);
 859         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 860 }
 861
 862 /******************************************************************************
 863  * Main
 864  ******************************************************************************/
 865 static void gator_summary(void)
 866 {
 867         u64 timestamp, uptime;
 868         struct timespec ts;
 869         char uname_buf[512];
 870         void (*m2b)(struct timespec *ts);
 871         unsigned long flags;
 872
 873         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
 874
 875         getnstimeofday(&ts);
 876         timestamp = timespec_to_ns(&ts);
 877
 878         do_posix_clock_monotonic_gettime(&ts);
 879         // monotonic_to_bootbased is not defined for some versions of Android
 880         m2b = symbol_get(monotonic_to_bootbased);
 881         if (m2b) {
 882                 m2b(&ts);
 883         }
 884         uptime = timespec_to_ns(&ts);
 885
 886         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
 887         local_irq_save(flags);
 888         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
 889         gator_monotonic_started = 0;
 890         gator_monotonic_started = gator_get_time();
 891         local_irq_restore(flags);
 892
 893         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
 894 }
 895
 896 int gator_events_install(struct gator_interface *interface)
 897 {
 898         list_add_tail(&interface->list, &gator_events);
 899
 900         return 0;
 901 }
 902
 903 int gator_events_get_key(void)
 904 {
 905         // key of zero is reserved as a timestamp
 906         static int key = 1;
 907
 908         const int ret = key;
 909         key += 2;
 910         return ret;
 911 }
 912
 913 static int gator_init(void)
 914 {
 915         int i;
 916
 917         calc_first_cluster_size();
 918
 919         // events sources (gator_events.h, generated by gator_events.sh)
 920         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 921                 if (gator_events_list[i])
 922                         gator_events_list[i]();
 923
 924         gator_trace_sched_init();
 925         gator_trace_power_init();
 926
 927         return 0;
 928 }
 929
 930 static void gator_exit(void)
 931 {
 932         struct gator_interface *gi;
 933
 934         list_for_each_entry(gi, &gator_events, list)
 935                 if (gi->shutdown)
 936                         gi->shutdown();
 937 }
 938
 939 static int gator_start(void)
 940 {
 941         unsigned long cpu, i;
 942         struct gator_interface *gi;
 943
 944         if (gator_migrate_start())
 945                 goto migrate_failure;
 946
 947         // Initialize the buffer with the frame type and core
 948         for_each_present_cpu(cpu) {
 949                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 950                         marshal_frame(cpu, i);
 951                 }
 952                 per_cpu(last_timestamp, cpu) = 0;
 953         }
 954         printed_monotonic_warning = false;
 955
 956         // Capture the start time
 957         gator_summary();
 958
 959         // start all events
 960         list_for_each_entry(gi, &gator_events, list) {
 961                 if (gi->start && gi->start() != 0) {
 962                         struct list_head *ptr = gi->list.prev;
 963
 964                         while (ptr != &gator_events) {
 965                                 gi = list_entry(ptr, struct gator_interface, list);
 966
 967                                 if (gi->stop)
 968                                         gi->stop();
 969
 970                                 ptr = ptr->prev;
 971                         }
 972                         goto events_failure;
 973                 }
 974         }
 975
 976         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
 977         if (cookies_initialize())
 978                 goto cookies_failure;
 979         if (gator_annotate_start())
 980                 goto annotate_failure;
 981         if (gator_trace_sched_start())
 982                 goto sched_failure;
 983         if (gator_trace_power_start())
 984                 goto power_failure;
 985         if (gator_trace_gpu_start())
 986                 goto gpu_failure;
 987         if (gator_timer_start(gator_timer_count))
 988                 goto timer_failure;
 989         if (gator_notifier_start())
 990                 goto notifier_failure;
 991
 992         return 0;
 993
 994 notifier_failure:
 995         gator_timer_stop();
 996 timer_failure:
 997         gator_trace_gpu_stop();
 998 gpu_failure:
 999         gator_trace_power_stop();
1000 power_failure:
1001         gator_trace_sched_stop();
1002 sched_failure:
1003         gator_annotate_stop();
1004 annotate_failure:
1005         cookies_release();
1006 cookies_failure:
1007         // stop all events
1008         list_for_each_entry(gi, &gator_events, list)
1009                 if (gi->stop)
1010                         gi->stop();
1011 events_failure:
1012         gator_migrate_stop();
1013 migrate_failure:
1014
1015         return -1;
1016 }
1017
1018 static void gator_stop(void)
1019 {
1020         struct gator_interface *gi;
1021
1022         gator_annotate_stop();
1023         gator_trace_sched_stop();
1024         gator_trace_power_stop();
1025         gator_trace_gpu_stop();
1026
1027         // stop all interrupt callback reads before tearing down other interfaces
1028         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1029         gator_timer_stop();
1030
1031         // stop all events
1032         list_for_each_entry(gi, &gator_events, list)
1033                 if (gi->stop)
1034                         gi->stop();
1035
1036         gator_migrate_stop();
1037 }
1038
1039 /******************************************************************************
1040  * Filesystem
1041  ******************************************************************************/
1042 /* fopen("buffer") */
1043 static int gator_op_setup(void)
1044 {
1045         int err = 0;
1046         int cpu, i;
1047
1048         mutex_lock(&start_mutex);
1049
1050         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1051         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1052
1053         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1054         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1055
1056         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1057         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1058
1059         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1060         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1061
1062         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1063         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1064
1065         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1066         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1067
1068         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1069         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1070
1071         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1072         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1073
1074         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1075         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1076
1077         // Initialize percpu per buffer variables
1078         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1079                 // Verify buffers are a power of 2
1080                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1081                         err = -ENOEXEC;
1082                         goto setup_error;
1083                 }
1084
1085                 for_each_present_cpu(cpu) {
1086                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1087                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1088                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1089                         per_cpu(buffer_space_available, cpu)[i] = true;
1090                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1091
1092                         // Annotation is a special case that only uses a single buffer
1093                         if (cpu > 0 && i == ANNOTATE_BUF) {
1094                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1095                                 continue;
1096                         }
1097
1098                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1099                         if (!per_cpu(gator_buffer, cpu)[i]) {
1100                                 err = -ENOMEM;
1101                                 goto setup_error;
1102                         }
1103                 }
1104         }
1105
1106 setup_error:
1107         mutex_unlock(&start_mutex);
1108         return err;
1109 }
1110
1111 /* Actually start profiling (echo 1>/dev/gator/enable) */
1112 static int gator_op_start(void)
1113 {
1114         int err = 0;
1115
1116         mutex_lock(&start_mutex);
1117
1118         if (gator_started || gator_start())
1119                 err = -EINVAL;
1120         else
1121                 gator_started = 1;
1122
1123         mutex_unlock(&start_mutex);
1124
1125         return err;
1126 }
1127
1128 /* echo 0>/dev/gator/enable */
1129 static void gator_op_stop(void)
1130 {
1131         mutex_lock(&start_mutex);
1132
1133         if (gator_started) {
1134                 gator_stop();
1135
1136                 mutex_lock(&gator_buffer_mutex);
1137
1138                 gator_started = 0;
1139                 gator_monotonic_started = 0;
1140                 cookies_release();
1141                 wake_up(&gator_buffer_wait);
1142
1143                 mutex_unlock(&gator_buffer_mutex);
1144         }
1145
1146         mutex_unlock(&start_mutex);
1147 }
1148
1149 static void gator_shutdown(void)
1150 {
1151         int cpu, i;
1152
1153         mutex_lock(&start_mutex);
1154
1155         for_each_present_cpu(cpu) {
1156                 mutex_lock(&gator_buffer_mutex);
1157                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1158                         vfree(per_cpu(gator_buffer, cpu)[i]);
1159                         per_cpu(gator_buffer, cpu)[i] = NULL;
1160                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1161                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1162                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1163                         per_cpu(buffer_space_available, cpu)[i] = true;
1164                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1165                 }
1166                 mutex_unlock(&gator_buffer_mutex);
1167         }
1168
1169         memset(&sent_core_name, 0, sizeof(sent_core_name));
1170
1171         mutex_unlock(&start_mutex);
1172 }
1173
1174 static int gator_set_backtrace(unsigned long val)
1175 {
1176         int err = 0;
1177
1178         mutex_lock(&start_mutex);
1179
1180         if (gator_started)
1181                 err = -EBUSY;
1182         else
1183                 gator_backtrace_depth = val;
1184
1185         mutex_unlock(&start_mutex);
1186
1187         return err;
1188 }
1189
1190 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1191 {
1192         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1193 }
1194
1195 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1196 {
1197         unsigned long val;
1198         int retval;
1199
1200         if (*offset)
1201                 return -EINVAL;
1202
1203         retval = gatorfs_ulong_from_user(&val, buf, count);
1204         if (retval)
1205                 return retval;
1206
1207         if (val)
1208                 retval = gator_op_start();
1209         else
1210                 gator_op_stop();
1211
1212         if (retval)
1213                 return retval;
1214         return count;
1215 }
1216
1217 static const struct file_operations enable_fops = {
1218         .read = enable_read,
1219         .write = enable_write,
1220 };
1221
1222 static int userspace_buffer_open(struct inode *inode, struct file *file)
1223 {
1224         int err = -EPERM;
1225
1226         if (!capable(CAP_SYS_ADMIN))
1227                 return -EPERM;
1228
1229         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1230                 return -EBUSY;
1231
1232         if ((err = gator_op_setup()))
1233                 goto fail;
1234
1235         /* NB: the actual start happens from userspace
1236          * echo 1 >/dev/gator/enable
1237          */
1238
1239         return 0;
1240
1241 fail:
1242         __clear_bit_unlock(0, &gator_buffer_opened);
1243         return err;
1244 }
1245
1246 static int userspace_buffer_release(struct inode *inode, struct file *file)
1247 {
1248         gator_op_stop();
1249         gator_shutdown();
1250         __clear_bit_unlock(0, &gator_buffer_opened);
1251         return 0;
1252 }
1253
1254 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1255 {
1256         int commit, length1, length2, read;
1257         char *buffer1;
1258         char *buffer2;
1259         int cpu, buftype;
1260         int written = 0;
1261
1262         // ensure there is enough space for a whole frame
1263         if (count < userspace_buffer_size || *offset) {
1264                 return -EINVAL;
1265         }
1266
1267         // sleep until the condition is true or a signal is received
1268         // the condition is checked each time gator_buffer_wait is woken up
1269         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1270
1271         if (signal_pending(current)) {
1272                 return -EINTR;
1273         }
1274
1275         if (buftype == -1 || cpu == -1) {
1276                 return 0;
1277         }
1278
1279         mutex_lock(&gator_buffer_mutex);
1280
1281         do {
1282                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1283                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1284
1285                 // May happen if the buffer is freed during pending reads.
1286                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1287                         break;
1288                 }
1289
1290                 // determine the size of two halves
1291                 length1 = commit - read;
1292                 length2 = 0;
1293                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1294                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1295                 if (length1 < 0) {
1296                         length1 = gator_buffer_size[buftype] - read;
1297                         length2 = commit;
1298                 }
1299
1300                 if (length1 + length2 > count - written) {
1301                         break;
1302                 }
1303
1304                 // start, middle or end
1305                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1306                         break;
1307                 }
1308
1309                 // possible wrap around
1310                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1311                         break;
1312                 }
1313
1314                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1315                 written += length1 + length2;
1316
1317                 // Wake up annotate_write if more space is available
1318                 if (buftype == ANNOTATE_BUF) {
1319                         wake_up(&gator_annotate_wait);
1320                 }
1321         } while (buffer_commit_ready(&cpu, &buftype));
1322
1323         mutex_unlock(&gator_buffer_mutex);
1324
1325         // kick just in case we've lost an SMP event
1326         wake_up(&gator_buffer_wait);
1327
1328         return written > 0 ? written : -EFAULT;
1329 }
1330
1331 const struct file_operations gator_event_buffer_fops = {
1332         .open = userspace_buffer_open,
1333         .release = userspace_buffer_release,
1334         .read = userspace_buffer_read,
1335 };
1336
1337 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1338 {
1339         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1340 }
1341
1342 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1343 {
1344         unsigned long val;
1345         int retval;
1346
1347         if (*offset)
1348                 return -EINVAL;
1349
1350         retval = gatorfs_ulong_from_user(&val, buf, count);
1351         if (retval)
1352                 return retval;
1353
1354         retval = gator_set_backtrace(val);
1355
1356         if (retval)
1357                 return retval;
1358         return count;
1359 }
1360
1361 static const struct file_operations depth_fops = {
1362         .read = depth_read,
1363         .write = depth_write
1364 };
1365
1366 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1367 {
1368         struct dentry *dir;
1369         struct gator_interface *gi;
1370         int cpu;
1371
1372         /* reinitialize default values */
1373         gator_cpu_cores = 0;
1374         for_each_present_cpu(cpu) {
1375                 gator_cpu_cores++;
1376         }
1377         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1378         gator_response_type = 1;
1379         gator_live_rate = 0;
1380
1381         gatorfs_create_file(sb, root, "enable", &enable_fops);
1382         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1383         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1384         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1385         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1386         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1387         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1388         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1389         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1390         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1391
1392         // Annotate interface
1393         gator_annotate_create_files(sb, root);
1394
1395         // Linux Events
1396         dir = gatorfs_mkdir(sb, root, "events");
1397         list_for_each_entry(gi, &gator_events, list)
1398                 if (gi->create_files)
1399                         gi->create_files(sb, dir);
1400
1401         // Sched Events
1402         sched_trace_create_files(sb, dir);
1403
1404         // Power interface
1405         gator_trace_power_create_files(sb, dir);
1406 }
1407
1408 /******************************************************************************
1409  * Module
1410  ******************************************************************************/
1411 static int __init gator_module_init(void)
1412 {
1413         if (gatorfs_register()) {
1414                 return -1;
1415         }
1416
1417         if (gator_init()) {
1418                 gatorfs_unregister();
1419                 return -1;
1420         }
1421
1422         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1423
1424         return 0;
1425 }
1426
1427 static void __exit gator_module_exit(void)
1428 {
1429         del_timer_sync(&gator_buffer_wake_up_timer);
1430         tracepoint_synchronize_unregister();
1431         gator_exit();
1432         gatorfs_unregister();
1433 }
1434
1435 module_init(gator_module_init);
1436 module_exit(gator_module_exit);
1437
1438 MODULE_LICENSE("GPL");
1439 MODULE_AUTHOR("ARM Ltd");
1440 MODULE_DESCRIPTION("Gator system profiler");