driver/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 static unsigned long gator_protocol_version = 10;
  11
  12 #include <linux/slab.h>
  13 #include <linux/cpu.h>
  14 #include <linux/sched.h>
  15 #include <linux/irq.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/highmem.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/suspend.h>
  21 #include <linux/module.h>
  22 #include <linux/perf_event.h>
  23 #include <asm/stacktrace.h>
  24 #include <asm/uaccess.h>
  25
  26 #include "gator.h"
  27 #include "gator_events.h"
  28
  29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
  30 #error kernels prior to 2.6.32 are not supported
  31 #endif
  32
  33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  35 #endif
  36
  37 #ifndef CONFIG_PROFILING
  38 #error gator requires the kernel to have CONFIG_PROFILING defined
  39 #endif
  40
  41 #ifndef CONFIG_HIGH_RES_TIMERS
  42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  43 #endif
  44
  45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  47 #endif
  48
  49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
  50 #ifndef CONFIG_PERF_EVENTS
  51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  52 #elif !defined CONFIG_HW_PERF_EVENTS
  53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  54 #endif
  55 #endif
  56
  57 /******************************************************************************
  58  * DEFINES
  59  ******************************************************************************/
  60 #define BACKTRACE_BUFFER_SIZE    (128*1024)
  61 #define COUNTER_BUFFER_SIZE      (128*1024)
  62 #define ANNOTATE_BUFFER_SIZE     (64*1024) // annotate  counters have the core as part of the data and the core value in the frame header may be discarded
  63 #define SCHED_TRACE_BUFFER_SIZE  (128*1024)
  64 #define GPU_TRACE_BUFFER_SIZE    (64*1024)
  65 #define COUNTER2_BUFFER_SIZE     (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
  66 #define WFI_BUFFER_SIZE          (32*1024) // wfi       counters have the core as part of the data and the core value in the frame header may be discarded
  67
  68 #define NO_COOKIE                               0UL
  69 #define INVALID_COOKIE                  ~0UL
  70
  71 #define FRAME_BACKTRACE     1
  72 #define FRAME_COUNTER       2
  73 #define FRAME_ANNOTATE      3
  74 #define FRAME_SCHED_TRACE   4
  75 #define FRAME_GPU_TRACE     5
  76 #define FRAME_COUNTER2      6
  77 #define FRAME_WFI           7
  78
  79 #define MESSAGE_COOKIE              1
  80 #define MESSAGE_START_BACKTRACE     5
  81 #define MESSAGE_END_BACKTRACE       7
  82 #define MESSAGE_SUMMARY             9
  83 #define MESSAGE_PID_NAME            11
  84
  85 #define MAXSIZE_PACK32          5
  86 #define MAXSIZE_PACK64          9
  87
  88 #if defined(__arm__)
  89 #define PC_REG regs->ARM_pc
  90 #else
  91 #define PC_REG regs->ip
  92 #endif
  93
  94 enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
  95
  96 /******************************************************************************
  97  * Globals
  98  ******************************************************************************/
  99 static unsigned long gator_cpu_cores;
 100 static unsigned long userspace_buffer_size;
 101 static unsigned long gator_backtrace_depth;
 102
 103 static unsigned long gator_started;
 104 static unsigned long gator_buffer_opened;
 105 static unsigned long gator_timer_count;
 106 static unsigned long gator_response_type;
 107 static DEFINE_MUTEX(start_mutex);
 108 static DEFINE_MUTEX(gator_buffer_mutex);
 109
 110 bool event_based_sampling;
 111 #if defined(__arm__) && (GATOR_PERF_PMU_SUPPORT)
 112 DEFINE_PER_CPU(struct perf_event *, pevent_ebs);
 113 #endif
 114
 115 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 116 static struct timer_list gator_buffer_wake_up_timer;
 117 static LIST_HEAD(gator_events);
 118
 119 /******************************************************************************
 120  * Prototypes
 121  ******************************************************************************/
 122 static void buffer_check(int cpu, int buftype);
 123 static int buffer_bytes_available(int cpu, int buftype);
 124 static bool buffer_check_space(int cpu, int buftype, int bytes);
 125 static int contiguous_space_available(int cpu, int bufytpe);
 126 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
 127 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
 128 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
 129 static void gator_buffer_write_string(int cpu, int buftype, char *x);
 130 static void gator_add_trace(int cpu, int buftype, unsigned int address);
 131 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
 132 static uint64_t gator_get_time(void);
 133
 134 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 135 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 136 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 137 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 138 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 139 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 140 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 141
 142 /******************************************************************************
 143  * Application Includes
 144  ******************************************************************************/
 145 #include "gator_marshaling.c"
 146 #include "gator_hrtimer_perf.c"
 147 #include "gator_hrtimer_gator.c"
 148 #include "gator_cookies.c"
 149 #include "gator_trace_sched.c"
 150 #include "gator_trace_power.c"
 151 #include "gator_trace_gpu.c"
 152 #include "gator_backtrace.c"
 153 #include "gator_annotate.c"
 154 #include "gator_fs.c"
 155 #include "gator_ebs.c"
 156 #include "gator_pack.c"
 157
 158 /******************************************************************************
 159  * Misc
 160  ******************************************************************************/
 161 #if defined(__arm__)
 162 u32 gator_cpuid(void)
 163 {
 164         u32 val;
 165         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 166         return (val >> 4) & 0xfff;
 167 }
 168 #endif
 169
 170 static void gator_buffer_wake_up(unsigned long data)
 171 {
 172         wake_up(&gator_buffer_wait);
 173 }
 174
 175 /******************************************************************************
 176  * Commit interface
 177  ******************************************************************************/
 178 static bool buffer_commit_ready(int* cpu, int* buftype)
 179 {
 180         int cpu_x, x;
 181         for_each_present_cpu(cpu_x) {
 182                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 183                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 184                                 *cpu = cpu_x;
 185                                 *buftype = x;
 186                                 return true;
 187                         }
 188         }
 189         return false;
 190 }
 191
 192 /******************************************************************************
 193  * Buffer management
 194  ******************************************************************************/
 195 static int buffer_bytes_available(int cpu, int buftype)
 196 {
 197         int remaining, filled;
 198
 199         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
 200         if (filled < 0) {
 201                 filled += gator_buffer_size[buftype];
 202         }
 203
 204         remaining = gator_buffer_size[buftype] - filled;
 205
 206         if (per_cpu(buffer_space_available, cpu)[buftype]) {
 207                 // Give some extra room; also allows space to insert the overflow error packet
 208                 remaining -= 200;
 209         } else {
 210                 // Hysteresis, prevents multiple overflow messages
 211                 remaining -= 2000;
 212         }
 213
 214         return remaining;
 215 }
 216
 217 static int contiguous_space_available(int cpu, int buftype)
 218 {
 219         int remaining = buffer_bytes_available(cpu, buftype);
 220         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
 221         if (remaining < contiguous)
 222                 return remaining;
 223         else
 224                 return contiguous;
 225 }
 226
 227 static bool buffer_check_space(int cpu, int buftype, int bytes)
 228 {
 229         int remaining = buffer_bytes_available(cpu, buftype);
 230
 231         if (remaining < bytes) {
 232                 per_cpu(buffer_space_available, cpu)[buftype] = false;
 233         } else {
 234                 per_cpu(buffer_space_available, cpu)[buftype] = true;
 235         }
 236
 237         return per_cpu(buffer_space_available, cpu)[buftype];
 238 }
 239
 240 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
 241 {
 242         int i;
 243         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
 244         u32 mask = gator_buffer_mask[buftype];
 245         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
 246
 247         for (i = 0; i < len; i++) {
 248                 buffer[write] = x[i];
 249                 write = (write + 1) & mask;
 250         }
 251
 252         per_cpu(gator_buffer_write, cpu)[buftype] = write;
 253 }
 254
 255 static void gator_buffer_write_string(int cpu, int buftype, char *x)
 256 {
 257         int len = strlen(x);
 258         gator_buffer_write_packed_int(cpu, buftype, len);
 259         gator_buffer_write_bytes(cpu, buftype, x, len);
 260 }
 261
 262 static void gator_buffer_header(int cpu, int buftype)
 263 {
 264         int frame;
 265
 266         if (buftype == BACKTRACE_BUF)
 267                 frame = FRAME_BACKTRACE;
 268         else if (buftype == COUNTER_BUF)
 269                 frame = FRAME_COUNTER;
 270         else if (buftype == ANNOTATE_BUF)
 271                 frame = FRAME_ANNOTATE;
 272         else if (buftype == SCHED_TRACE_BUF)
 273                 frame = FRAME_SCHED_TRACE;
 274         else if (buftype == GPU_TRACE_BUF)
 275                 frame = FRAME_GPU_TRACE;
 276         else if (buftype == COUNTER2_BUF)
 277                 frame = FRAME_COUNTER2;
 278         else if (buftype == WFI_BUF)
 279                 frame = FRAME_WFI;
 280         else
 281                 frame = -1;
 282
 283         if (per_cpu(gator_buffer, cpu)[buftype]) {
 284                 marshal_frame(cpu, buftype, frame);
 285         }
 286 }
 287
 288 static void gator_commit_buffer(int cpu, int buftype)
 289 {
 290         if (!per_cpu(gator_buffer, cpu)[buftype])
 291                 return;
 292
 293         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
 294         gator_buffer_header(cpu, buftype);
 295
 296         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 297         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
 298 }
 299
 300 static void buffer_check(int cpu, int buftype)
 301 {
 302         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
 303         if (filled < 0) {
 304                 filled += gator_buffer_size[buftype];
 305         }
 306         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
 307                 gator_commit_buffer(cpu, buftype);
 308         }
 309 }
 310
 311 static void gator_add_trace(int cpu, int buftype, unsigned int address)
 312 {
 313         off_t offset = 0;
 314         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
 315
 316         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
 317                 offset = address;
 318         }
 319
 320         marshal_backtrace(offset & ~1, cookie);
 321 }
 322
 323 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
 324 {
 325         int inKernel = regs ? !user_mode(regs) : 1;
 326         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
 327
 328         if (!regs)
 329                 return;
 330
 331         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
 332                 return;
 333
 334         if (inKernel) {
 335                 kernel_backtrace(cpu, buftype, regs);
 336         } else {
 337                 // Cookie+PC
 338                 gator_add_trace(cpu, buftype, PC_REG);
 339
 340                 // Backtrace
 341                 if (gator_backtrace_depth)
 342                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
 343         }
 344
 345         marshal_backtrace_footer();
 346 }
 347
 348 /******************************************************************************
 349  * hrtimer interrupt processing
 350  ******************************************************************************/
 351 static void gator_timer_interrupt(void)
 352 {
 353         struct pt_regs * const regs = get_irq_regs();
 354         int cpu = smp_processor_id();
 355
 356         // Output backtrace
 357         gator_add_sample(cpu, BACKTRACE_BUF, regs);
 358
 359         // Collect counters
 360         collect_counters();
 361 }
 362
 363 static int gator_running;
 364
 365 // This function runs in interrupt context and on the appropriate core
 366 static void gator_timer_offline(void* unused)
 367 {
 368         struct gator_interface *gi;
 369         int i, len, cpu = smp_processor_id();
 370         int* buffer;
 371
 372         gator_trace_sched_offline();
 373         gator_trace_power_offline();
 374
 375         gator_hrtimer_offline(cpu);
 376
 377         // Offline any events and output counters
 378         if (marshal_event_header()) {
 379                 list_for_each_entry(gi, &gator_events, list) {
 380                         if (gi->offline) {
 381                                 len = gi->offline(&buffer);
 382                                 marshal_event(len, buffer);
 383                         }
 384                 }
 385         }
 386
 387         // Flush all buffers on this core
 388         for (i = 0; i < NUM_GATOR_BUFS; i++)
 389                 gator_commit_buffer(cpu, i);
 390 }
 391
 392 // This function runs in process context and may be running on a core other than core 'cpu'
 393 static void gator_timer_offline_dispatch(int cpu)
 394 {
 395         struct gator_interface *gi;
 396
 397         list_for_each_entry(gi, &gator_events, list)
 398                 if (gi->offline_dispatch)
 399                         gi->offline_dispatch(cpu);
 400
 401         gator_event_sampling_offline_dispatch(cpu);
 402 }
 403
 404 static void gator_timer_stop(void)
 405 {
 406         int cpu;
 407
 408         if (gator_running) {
 409                 on_each_cpu(gator_timer_offline, NULL, 1);
 410                 for_each_online_cpu(cpu) {
 411                         gator_timer_offline_dispatch(cpu);
 412                 }
 413
 414                 gator_running = 0;
 415                 gator_hrtimer_shutdown();
 416         }
 417 }
 418
 419 // This function runs in interrupt context and on the appropriate core
 420 static void gator_timer_online(void* unused)
 421 {
 422         struct gator_interface *gi;
 423         int len, cpu = smp_processor_id();
 424         int* buffer;
 425
 426         gator_trace_power_online();
 427
 428         // online any events and output counters
 429         if (marshal_event_header()) {
 430                 list_for_each_entry(gi, &gator_events, list) {
 431                         if (gi->online) {
 432                                 len = gi->online(&buffer);
 433                                 marshal_event(len, buffer);
 434                         }
 435                 }
 436         }
 437
 438         gator_hrtimer_online(cpu);
 439 }
 440
 441 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 442 static void gator_timer_online_dispatch(int cpu)
 443 {
 444         struct gator_interface *gi;
 445
 446         list_for_each_entry(gi, &gator_events, list)
 447                 if (gi->online_dispatch)
 448                         gi->online_dispatch(cpu);
 449
 450         gator_event_sampling_online_dispatch(cpu);
 451 }
 452
 453 int gator_timer_start(unsigned long sample_rate)
 454 {
 455         int cpu;
 456
 457         if (gator_running) {
 458                 pr_notice("gator: already running\n");
 459                 return 0;
 460         }
 461
 462         gator_running = 1;
 463
 464         // event based sampling trumps hr timer based sampling
 465         if (event_based_sampling)
 466                 sample_rate = 0;
 467
 468         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 469                 return -1;
 470
 471         for_each_online_cpu(cpu) {
 472                 gator_timer_online_dispatch(cpu);
 473         }
 474         on_each_cpu(gator_timer_online, NULL, 1);
 475
 476         return 0;
 477 }
 478
 479 static uint64_t gator_get_time(void)
 480 {
 481         struct timespec ts;
 482         uint64_t timestamp;
 483
 484         getnstimeofday(&ts);
 485         timestamp = timespec_to_ns(&ts);
 486
 487         return timestamp;
 488 }
 489
 490 /******************************************************************************
 491  * cpu hotplug and pm notifiers
 492  ******************************************************************************/
 493 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 494 {
 495         long cpu = (long)hcpu;
 496
 497         switch (action) {
 498                 case CPU_DOWN_PREPARE:
 499                 case CPU_DOWN_PREPARE_FROZEN:
 500                         smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 501                         gator_timer_offline_dispatch(cpu);
 502                         break;
 503                 case CPU_ONLINE:
 504                 case CPU_ONLINE_FROZEN:
 505                         gator_timer_online_dispatch(cpu);
 506                         smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 507                         break;
 508         }
 509
 510         return NOTIFY_OK;
 511 }
 512
 513 static struct notifier_block __refdata gator_hotcpu_notifier = {
 514         .notifier_call = gator_hotcpu_notify,
 515 };
 516
 517 // n.b. calling "on_each_cpu" only runs on those that are online
 518 // Registered linux events are not disabled, so their counters will continue to collect
 519 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 520 {
 521         int cpu;
 522
 523         switch (event) {
 524                 case PM_HIBERNATION_PREPARE:
 525                 case PM_SUSPEND_PREPARE:
 526                         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 527                         unregister_scheduler_tracepoints();
 528                         on_each_cpu(gator_timer_offline, NULL, 1);
 529                         for_each_online_cpu(cpu) {
 530                                 gator_timer_offline_dispatch(cpu);
 531                         }
 532                         break;
 533                 case PM_POST_HIBERNATION:
 534                 case PM_POST_SUSPEND:
 535                         for_each_online_cpu(cpu) {
 536                                 gator_timer_online_dispatch(cpu);
 537                         }
 538                         on_each_cpu(gator_timer_online, NULL, 1);
 539                         register_scheduler_tracepoints();
 540                         register_hotcpu_notifier(&gator_hotcpu_notifier);
 541                         break;
 542         }
 543
 544         return NOTIFY_OK;
 545 }
 546
 547 static struct notifier_block gator_pm_notifier = {
 548         .notifier_call = gator_pm_notify,
 549 };
 550
 551 static int gator_notifier_start(void)
 552 {
 553         int retval;
 554         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 555         if (retval == 0)
 556                 retval = register_pm_notifier(&gator_pm_notifier);
 557         return retval;
 558 }
 559
 560 static void gator_notifier_stop(void)
 561 {
 562         unregister_pm_notifier(&gator_pm_notifier);
 563         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 564 }
 565
 566 /******************************************************************************
 567  * Main
 568  ******************************************************************************/
 569 static void gator_summary(void)
 570 {
 571         uint64_t timestamp, uptime = 0;
 572         struct timespec uptime_ts;
 573         void (*m2b)(struct timespec *ts);
 574
 575         timestamp = gator_get_time();
 576
 577         do_posix_clock_monotonic_gettime(&uptime_ts);
 578         m2b = symbol_get(monotonic_to_bootbased);
 579         if (m2b) {
 580                 m2b(&uptime_ts);
 581                 uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
 582         }
 583
 584         marshal_summary(timestamp, uptime);
 585 }
 586
 587 int gator_events_install(struct gator_interface *interface)
 588 {
 589         list_add_tail(&interface->list, &gator_events);
 590
 591         return 0;
 592 }
 593
 594 int gator_events_get_key(void)
 595 {
 596         // key of zero is reserved as a timestamp
 597         static int key = 1;
 598
 599         return key++;
 600 }
 601
 602 static int gator_init(void)
 603 {
 604         int i;
 605
 606         // events sources (gator_events.h, generated by gator_events.sh)
 607         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 608                 if (gator_events_list[i])
 609                         gator_events_list[i]();
 610
 611         gator_trace_power_init();
 612
 613         return 0;
 614 }
 615
 616 static int gator_start(void)
 617 {
 618         unsigned long cpu, i;
 619         struct gator_interface *gi;
 620
 621         // Initialize the buffer with the frame type and core
 622         for_each_present_cpu(cpu) {
 623                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 624                         gator_buffer_header(cpu, i);
 625                 }
 626         }
 627
 628         // Capture the start time
 629         gator_summary();
 630
 631         // start all events
 632         list_for_each_entry(gi, &gator_events, list) {
 633                 if (gi->start && gi->start() != 0) {
 634                         struct list_head *ptr = gi->list.prev;
 635
 636                         while (ptr != &gator_events) {
 637                                 gi = list_entry(ptr, struct gator_interface, list);
 638
 639                                 if (gi->stop)
 640                                         gi->stop();
 641
 642                                 ptr = ptr->prev;
 643                         }
 644                         goto events_failure;
 645                 }
 646         }
 647
 648         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
 649         if (cookies_initialize())
 650                 goto cookies_failure;
 651         if (gator_annotate_start())
 652                 goto annotate_failure;
 653         if (gator_trace_sched_start())
 654                 goto sched_failure;
 655         if (gator_trace_power_start())
 656                 goto power_failure;
 657         if (gator_trace_gpu_start())
 658                 goto gpu_failure;
 659         if (gator_event_sampling_start())
 660                 goto event_sampling_failure;
 661         if (gator_timer_start(gator_timer_count))
 662                 goto timer_failure;
 663         if (gator_notifier_start())
 664                 goto notifier_failure;
 665
 666         return 0;
 667
 668 notifier_failure:
 669         gator_timer_stop();
 670 timer_failure:
 671         gator_event_sampling_stop();
 672 event_sampling_failure:
 673         gator_trace_gpu_stop();
 674 gpu_failure:
 675         gator_trace_power_stop();
 676 power_failure:
 677         gator_trace_sched_stop();
 678 sched_failure:
 679         gator_annotate_stop();
 680 annotate_failure:
 681         cookies_release();
 682 cookies_failure:
 683         // stop all events
 684         list_for_each_entry(gi, &gator_events, list)
 685                 if (gi->stop)
 686                         gi->stop();
 687 events_failure:
 688
 689         return -1;
 690 }
 691
 692 static void gator_stop(void)
 693 {
 694         struct gator_interface *gi;
 695
 696         gator_annotate_stop();
 697         gator_trace_sched_stop();
 698         gator_trace_power_stop();
 699         gator_trace_gpu_stop();
 700         gator_event_sampling_stop();
 701
 702         // stop all interrupt callback reads before tearing down other interfaces
 703         gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
 704         gator_timer_stop();
 705
 706         // stop all events
 707         list_for_each_entry(gi, &gator_events, list)
 708                 if (gi->stop)
 709                         gi->stop();
 710 }
 711
 712 /******************************************************************************
 713  * Filesystem
 714  ******************************************************************************/
 715 /* fopen("buffer") */
 716 static int gator_op_setup(void)
 717 {
 718         int err = 0;
 719         int cpu, i;
 720
 721         mutex_lock(&start_mutex);
 722
 723         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
 724         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
 725
 726         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
 727         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
 728
 729         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
 730         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
 731
 732         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
 733         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
 734
 735         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
 736         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
 737
 738         gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
 739         gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
 740
 741         gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
 742         gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
 743
 744         // Initialize percpu per buffer variables
 745         for (i = 0; i < NUM_GATOR_BUFS; i++) {
 746                 // Verify buffers are a power of 2
 747                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
 748                         err = -ENOEXEC;
 749                         goto setup_error;
 750                 }
 751
 752                 for_each_present_cpu(cpu) {
 753                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 754                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 755                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 756                         per_cpu(buffer_space_available, cpu)[i] = true;
 757
 758                         // Annotation is a special case that only uses a single buffer
 759                         if (cpu > 0 && i == ANNOTATE_BUF) {
 760                                 per_cpu(gator_buffer, cpu)[i] = NULL;
 761                                 continue;
 762                         }
 763
 764                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
 765                         if (!per_cpu(gator_buffer, cpu)[i]) {
 766                                 err = -ENOMEM;
 767                                 goto setup_error;
 768                         }
 769                 }
 770         }
 771
 772 setup_error:
 773         mutex_unlock(&start_mutex);
 774         return err;
 775 }
 776
 777 /* Actually start profiling (echo 1>/dev/gator/enable) */
 778 static int gator_op_start(void)
 779 {
 780         int err = 0;
 781
 782         mutex_lock(&start_mutex);
 783
 784         if (gator_started || gator_start())
 785                 err = -EINVAL;
 786         else
 787                 gator_started = 1;
 788
 789         mutex_unlock(&start_mutex);
 790
 791         return err;
 792 }
 793
 794 /* echo 0>/dev/gator/enable */
 795 static void gator_op_stop(void)
 796 {
 797         mutex_lock(&start_mutex);
 798
 799         if (gator_started) {
 800                 gator_stop();
 801
 802                 mutex_lock(&gator_buffer_mutex);
 803
 804                 gator_started = 0;
 805                 cookies_release();
 806                 wake_up(&gator_buffer_wait);
 807
 808                 mutex_unlock(&gator_buffer_mutex);
 809         }
 810
 811         mutex_unlock(&start_mutex);
 812 }
 813
 814 static void gator_shutdown(void)
 815 {
 816         int cpu, i;
 817
 818         mutex_lock(&start_mutex);
 819
 820         for_each_present_cpu(cpu) {
 821                 mutex_lock(&gator_buffer_mutex);
 822                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 823                         vfree(per_cpu(gator_buffer, cpu)[i]);
 824                         per_cpu(gator_buffer, cpu)[i] = NULL;
 825                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 826                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 827                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 828                         per_cpu(buffer_space_available, cpu)[i] = true;
 829                 }
 830                 mutex_unlock(&gator_buffer_mutex);
 831         }
 832
 833         mutex_unlock(&start_mutex);
 834 }
 835
 836 static int gator_set_backtrace(unsigned long val)
 837 {
 838         int err = 0;
 839
 840         mutex_lock(&start_mutex);
 841
 842         if (gator_started)
 843                 err = -EBUSY;
 844         else
 845                 gator_backtrace_depth = val;
 846
 847         mutex_unlock(&start_mutex);
 848
 849         return err;
 850 }
 851
 852 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
 853 {
 854         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
 855 }
 856
 857 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
 858 {
 859         unsigned long val;
 860         int retval;
 861
 862         if (*offset)
 863                 return -EINVAL;
 864
 865         retval = gatorfs_ulong_from_user(&val, buf, count);
 866         if (retval)
 867                 return retval;
 868
 869         if (val)
 870                 retval = gator_op_start();
 871         else
 872                 gator_op_stop();
 873
 874         if (retval)
 875                 return retval;
 876         return count;
 877 }
 878
 879 static const struct file_operations enable_fops = {
 880         .read           = enable_read,
 881         .write          = enable_write,
 882 };
 883
 884 static int userspace_buffer_open(struct inode *inode, struct file *file)
 885 {
 886         int err = -EPERM;
 887
 888         if (!capable(CAP_SYS_ADMIN))
 889                 return -EPERM;
 890
 891         if (test_and_set_bit_lock(0, &gator_buffer_opened))
 892                 return -EBUSY;
 893
 894         if ((err = gator_op_setup()))
 895                 goto fail;
 896
 897         /* NB: the actual start happens from userspace
 898          * echo 1 >/dev/gator/enable
 899          */
 900
 901         return 0;
 902
 903 fail:
 904         __clear_bit_unlock(0, &gator_buffer_opened);
 905         return err;
 906 }
 907
 908 static int userspace_buffer_release(struct inode *inode, struct file *file)
 909 {
 910         gator_op_stop();
 911         gator_shutdown();
 912         __clear_bit_unlock(0, &gator_buffer_opened);
 913         return 0;
 914 }
 915
 916 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
 917                                  size_t count, loff_t *offset)
 918 {
 919         int retval = -EINVAL;
 920         int commit = 0, length, length1, length2, read, byte, type_length;
 921         char *buffer1;
 922         char *buffer2 = NULL;
 923         int cpu, buftype;
 924
 925         /* do not handle partial reads */
 926         if (count != userspace_buffer_size || *offset)
 927                 return -EINVAL;
 928
 929         // sleep until the condition is true or a signal is received
 930         // the condition is checked each time gator_buffer_wait is woken up
 931         buftype = cpu = -1;
 932         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
 933
 934         if (signal_pending(current))
 935                 return -EINTR;
 936
 937         length2 = 0;
 938         retval = -EFAULT;
 939
 940         mutex_lock(&gator_buffer_mutex);
 941
 942         if (buftype == -1 || cpu == -1) {
 943                 retval = 0;
 944                 goto out;
 945         }
 946
 947         read = per_cpu(gator_buffer_read, cpu)[buftype];
 948         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
 949
 950         /* May happen if the buffer is freed during pending reads. */
 951         if (!per_cpu(gator_buffer, cpu)[buftype]) {
 952                 retval = -EFAULT;
 953                 goto out;
 954         }
 955
 956         /* determine the size of two halves */
 957         length1 = commit - read;
 958         buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
 959         buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
 960         if (length1 < 0) {
 961                 length1 = gator_buffer_size[buftype] - read;
 962                 length2 = commit;
 963         }
 964
 965         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
 966         type_length = gator_response_type ? 1 : 0;
 967         length = length1 + length2 - type_length - sizeof(int);
 968         for (byte = 0; byte < sizeof(int); byte++) {
 969                 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
 970         }
 971
 972         /* start, middle or end */
 973         if (length1 > 0) {
 974                 if (copy_to_user(&buf[0], buffer1, length1)) {
 975                         goto out;
 976                 }
 977         }
 978
 979         /* possible wrap around */
 980         if (length2 > 0) {
 981                 if (copy_to_user(&buf[length1], buffer2, length2)) {
 982                         goto out;
 983                 }
 984         }
 985
 986         per_cpu(gator_buffer_read, cpu)[buftype] = commit;
 987         retval = length1 + length2;
 988
 989         /* kick just in case we've lost an SMP event */
 990         wake_up(&gator_buffer_wait);
 991
 992 out:
 993         mutex_unlock(&gator_buffer_mutex);
 994         return retval;
 995 }
 996
 997 const struct file_operations gator_event_buffer_fops = {
 998         .open           = userspace_buffer_open,
 999         .release        = userspace_buffer_release,
1000         .read           = userspace_buffer_read,
1001 };
1002
1003 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1004 {
1005         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
1006                                         offset);
1007 }
1008
1009 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1010 {
1011         unsigned long val;
1012         int retval;
1013
1014         if (*offset)
1015                 return -EINVAL;
1016
1017         retval = gatorfs_ulong_from_user(&val, buf, count);
1018         if (retval)
1019                 return retval;
1020
1021         retval = gator_set_backtrace(val);
1022
1023         if (retval)
1024                 return retval;
1025         return count;
1026 }
1027
1028 static const struct file_operations depth_fops = {
1029         .read           = depth_read,
1030         .write          = depth_write
1031 };
1032
1033 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1034 {
1035         struct dentry *dir;
1036         struct gator_interface *gi;
1037         int cpu;
1038
1039         /* reinitialize default values */
1040         gator_cpu_cores = 0;
1041         for_each_present_cpu(cpu) {
1042                 gator_cpu_cores++;
1043         }
1044         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1045         gator_response_type = 1;
1046
1047         gatorfs_create_file(sb, root, "enable", &enable_fops);
1048         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1049         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1050         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1051         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1052         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1053         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1054         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1055
1056         // Annotate interface
1057         gator_annotate_create_files(sb, root);
1058
1059         // Linux Events
1060         dir = gatorfs_mkdir(sb, root, "events");
1061         list_for_each_entry(gi, &gator_events, list)
1062                 if (gi->create_files)
1063                         gi->create_files(sb, dir);
1064
1065         // Power interface
1066         gator_trace_power_create_files(sb, dir);
1067 }
1068
1069 /******************************************************************************
1070  * Module
1071  ******************************************************************************/
1072 static int __init gator_module_init(void)
1073 {
1074         if (gatorfs_register()) {
1075                 return -1;
1076         }
1077
1078         if (gator_init()) {
1079                 gatorfs_unregister();
1080                 return -1;
1081         }
1082
1083         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1084
1085         return 0;
1086 }
1087
1088 static void __exit gator_module_exit(void)
1089 {
1090         del_timer_sync(&gator_buffer_wake_up_timer);
1091         tracepoint_synchronize_unregister();
1092         gatorfs_unregister();
1093 }
1094
1095 module_init(gator_module_init);
1096 module_exit(gator_module_exit);
1097
1098 MODULE_LICENSE("GPL");
1099 MODULE_AUTHOR("ARM Ltd");
1100 MODULE_DESCRIPTION("Gator system profiler");