driver/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 static unsigned long gator_protocol_version = 9;
  11
  12 #include <linux/slab.h>
  13 #include <linux/cpu.h>
  14 #include <linux/sched.h>
  15 #include <linux/irq.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/highmem.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/suspend.h>
  21 #include <linux/module.h>
  22 #include <linux/perf_event.h>
  23 #include <asm/stacktrace.h>
  24 #include <asm/uaccess.h>
  25
  26 #include "gator.h"
  27 #include "gator_events.h"
  28
  29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
  30 #error kernels prior to 2.6.32 are not supported
  31 #endif
  32
  33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  35 #endif
  36
  37 #ifndef CONFIG_PROFILING
  38 #error gator requires the kernel to have CONFIG_PROFILING defined
  39 #endif
  40
  41 #ifndef CONFIG_HIGH_RES_TIMERS
  42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  43 #endif
  44
  45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  47 #endif
  48
  49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
  50 #ifndef CONFIG_PERF_EVENTS
  51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  52 #elif !defined CONFIG_HW_PERF_EVENTS
  53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  54 #endif
  55 #endif
  56
  57 /******************************************************************************
  58  * DEFINES
  59  ******************************************************************************/
  60 #define BACKTRACE_BUFFER_SIZE    (128*1024)
  61 #define COUNTER_BUFFER_SIZE      (128*1024)
  62 #define ANNOTATE_BUFFER_SIZE     (64*1024) // annotate  counters have the core as part of the data and the core value in the frame header may be discarded
  63 #define SCHED_TRACE_BUFFER_SIZE  (128*1024)
  64 #define GPU_TRACE_BUFFER_SIZE    (64*1024)
  65 #define COUNTER2_BUFFER_SIZE     (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
  66 #define WFI_BUFFER_SIZE          (32*1024) // wfi       counters have the core as part of the data and the core value in the frame header may be discarded
  67
  68 #define NO_COOKIE                               0UL
  69 #define INVALID_COOKIE                  ~0UL
  70
  71 #define FRAME_BACKTRACE     1
  72 #define FRAME_COUNTER       2
  73 #define FRAME_ANNOTATE      3
  74 #define FRAME_SCHED_TRACE   4
  75 #define FRAME_GPU_TRACE     5
  76 #define FRAME_COUNTER2      6
  77 #define FRAME_WFI           7
  78
  79 #define MESSAGE_COOKIE              1
  80 #define MESSAGE_START_BACKTRACE     5
  81 #define MESSAGE_END_BACKTRACE       7
  82 #define MESSAGE_SUMMARY             9
  83 #define MESSAGE_PID_NAME            11
  84
  85 #define MAXSIZE_PACK32          5
  86 #define MAXSIZE_PACK64          9
  87
  88 #if defined(__arm__)
  89 #define PC_REG regs->ARM_pc
  90 #else
  91 #define PC_REG regs->ip
  92 #endif
  93
  94 enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
  95
  96 /******************************************************************************
  97  * Globals
  98  ******************************************************************************/
  99 static unsigned long gator_cpu_cores;
 100 static unsigned long userspace_buffer_size;
 101 static unsigned long gator_backtrace_depth;
 102
 103 static unsigned long gator_started;
 104 static unsigned long gator_buffer_opened;
 105 static unsigned long gator_timer_count;
 106 static unsigned long gator_response_type;
 107 static DEFINE_MUTEX(start_mutex);
 108 static DEFINE_MUTEX(gator_buffer_mutex);
 109
 110 bool event_based_sampling;
 111
 112 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 113 static LIST_HEAD(gator_events);
 114
 115 /******************************************************************************
 116  * Prototypes
 117  ******************************************************************************/
 118 static void buffer_check(int cpu, int buftype);
 119 static int buffer_bytes_available(int cpu, int buftype);
 120 static bool buffer_check_space(int cpu, int buftype, int bytes);
 121 static int contiguous_space_available(int cpu, int bufytpe);
 122 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
 123 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
 124 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
 125 static void gator_buffer_write_string(int cpu, int buftype, char *x);
 126 static void gator_add_trace(int cpu, int buftype, unsigned int address);
 127 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
 128 static uint64_t gator_get_time(void);
 129
 130 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 131 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 132 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 133 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 134 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 135 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 136 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 137
 138 /******************************************************************************
 139  * Application Includes
 140  ******************************************************************************/
 141 #include "gator_marshaling.c"
 142 #include "gator_hrtimer_perf.c"
 143 #include "gator_hrtimer_gator.c"
 144 #include "gator_cookies.c"
 145 #include "gator_trace_sched.c"
 146 #include "gator_trace_power.c"
 147 #include "gator_trace_gpu.c"
 148 #include "gator_backtrace.c"
 149 #include "gator_annotate.c"
 150 #include "gator_fs.c"
 151 #include "gator_ebs.c"
 152 #include "gator_pack.c"
 153
 154 /******************************************************************************
 155  * Misc
 156  ******************************************************************************/
 157 #if defined(__arm__)
 158 u32 gator_cpuid(void)
 159 {
 160         u32 val;
 161         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 162         return (val >> 4) & 0xfff;
 163 }
 164 #endif
 165
 166 /******************************************************************************
 167  * Commit interface
 168  ******************************************************************************/
 169 static bool buffer_commit_ready(int* cpu, int* buftype)
 170 {
 171         int cpu_x, x;
 172         for_each_present_cpu(cpu_x) {
 173                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 174                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 175                                 *cpu = cpu_x;
 176                                 *buftype = x;
 177                                 return true;
 178                         }
 179         }
 180         return false;
 181 }
 182
 183 /******************************************************************************
 184  * Buffer management
 185  ******************************************************************************/
 186 static int buffer_bytes_available(int cpu, int buftype)
 187 {
 188         int remaining, filled;
 189
 190         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
 191         if (filled < 0) {
 192                 filled += gator_buffer_size[buftype];
 193         }
 194
 195         remaining = gator_buffer_size[buftype] - filled;
 196
 197         if (per_cpu(buffer_space_available, cpu)[buftype]) {
 198                 // Give some extra room; also allows space to insert the overflow error packet
 199                 remaining -= 200;
 200         } else {
 201                 // Hysteresis, prevents multiple overflow messages
 202                 remaining -= 2000;
 203         }
 204
 205         return remaining;
 206 }
 207
 208 static int contiguous_space_available(int cpu, int buftype)
 209 {
 210         int remaining = buffer_bytes_available(cpu, buftype);
 211         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
 212         if (remaining < contiguous)
 213                 return remaining;
 214         else
 215                 return contiguous;
 216 }
 217
 218 static bool buffer_check_space(int cpu, int buftype, int bytes)
 219 {
 220         int remaining = buffer_bytes_available(cpu, buftype);
 221
 222         if (remaining < bytes) {
 223                 per_cpu(buffer_space_available, cpu)[buftype] = false;
 224         } else {
 225                 per_cpu(buffer_space_available, cpu)[buftype] = true;
 226         }
 227
 228         return per_cpu(buffer_space_available, cpu)[buftype];
 229 }
 230
 231 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
 232 {
 233         int i;
 234         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
 235         u32 mask = gator_buffer_mask[buftype];
 236         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
 237
 238         for (i = 0; i < len; i++) {
 239                 buffer[write] = x[i];
 240                 write = (write + 1) & mask;
 241         }
 242
 243         per_cpu(gator_buffer_write, cpu)[buftype] = write;
 244 }
 245
 246 static void gator_buffer_write_string(int cpu, int buftype, char *x)
 247 {
 248         int len = strlen(x);
 249         gator_buffer_write_packed_int(cpu, buftype, len);
 250         gator_buffer_write_bytes(cpu, buftype, x, len);
 251 }
 252
 253 static void gator_buffer_header(int cpu, int buftype)
 254 {
 255         int frame;
 256
 257         if (buftype == BACKTRACE_BUF)
 258                 frame = FRAME_BACKTRACE;
 259         else if (buftype == COUNTER_BUF)
 260                 frame = FRAME_COUNTER;
 261         else if (buftype == ANNOTATE_BUF)
 262                 frame = FRAME_ANNOTATE;
 263         else if (buftype == SCHED_TRACE_BUF)
 264                 frame = FRAME_SCHED_TRACE;
 265         else if (buftype == GPU_TRACE_BUF)
 266                 frame = FRAME_GPU_TRACE;
 267         else if (buftype == COUNTER2_BUF)
 268                 frame = FRAME_COUNTER2;
 269         else if (buftype == WFI_BUF)
 270                 frame = FRAME_WFI;
 271         else
 272                 frame = -1;
 273
 274         if (per_cpu(gator_buffer, cpu)[buftype]) {
 275                 marshal_frame(cpu, buftype, frame);
 276         }
 277 }
 278
 279 static void gator_commit_buffer(int cpu, int buftype)
 280 {
 281         if (!per_cpu(gator_buffer, cpu)[buftype])
 282                 return;
 283
 284         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
 285         gator_buffer_header(cpu, buftype);
 286         wake_up(&gator_buffer_wait);
 287 }
 288
 289 static void buffer_check(int cpu, int buftype)
 290 {
 291         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
 292         if (filled < 0) {
 293                 filled += gator_buffer_size[buftype];
 294         }
 295         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
 296                 gator_commit_buffer(cpu, buftype);
 297         }
 298 }
 299
 300 static void gator_add_trace(int cpu, int buftype, unsigned int address)
 301 {
 302         off_t offset = 0;
 303         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
 304
 305         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
 306                 offset = address;
 307         }
 308
 309         marshal_backtrace(offset & ~1, cookie);
 310 }
 311
 312 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
 313 {
 314         int inKernel = regs ? !user_mode(regs) : 1;
 315         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
 316
 317         if (!regs)
 318                 return;
 319
 320         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
 321                 return;
 322
 323         if (inKernel) {
 324                 kernel_backtrace(cpu, buftype, regs);
 325         } else {
 326                 // Cookie+PC
 327                 gator_add_trace(cpu, buftype, PC_REG);
 328
 329                 // Backtrace
 330                 if (gator_backtrace_depth)
 331                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
 332         }
 333
 334         marshal_backtrace_footer();
 335 }
 336
 337 /******************************************************************************
 338  * hrtimer interrupt processing
 339  ******************************************************************************/
 340 static void gator_timer_interrupt(void)
 341 {
 342         struct pt_regs * const regs = get_irq_regs();
 343         int cpu = smp_processor_id();
 344
 345         // Output backtrace
 346         gator_add_sample(cpu, BACKTRACE_BUF, regs);
 347
 348         // Collect counters
 349         collect_counters();
 350 }
 351
 352 static int gator_running;
 353
 354 // This function runs in interrupt context and on the appropriate core
 355 static void gator_timer_offline(void* unused)
 356 {
 357         struct gator_interface *gi;
 358         int i, len, cpu = smp_processor_id();
 359         int* buffer;
 360
 361         gator_trace_sched_offline();
 362         gator_trace_power_offline();
 363
 364         gator_hrtimer_offline(cpu);
 365
 366         // Offline any events and output counters
 367         if (marshal_event_header()) {
 368                 list_for_each_entry(gi, &gator_events, list) {
 369                         if (gi->offline) {
 370                                 len = gi->offline(&buffer);
 371                                 marshal_event(len, buffer);
 372                         }
 373                 }
 374         }
 375
 376         // Flush all buffers on this core
 377         for (i = 0; i < NUM_GATOR_BUFS; i++)
 378                 gator_commit_buffer(cpu, i);
 379 }
 380
 381 // This function runs in process context and may be running on a core other than core 'cpu'
 382 static void gator_timer_offline_dispatch(int cpu)
 383 {
 384         struct gator_interface *gi;
 385
 386         list_for_each_entry(gi, &gator_events, list)
 387                 if (gi->offline_dispatch)
 388                         gi->offline_dispatch(cpu);
 389
 390         gator_event_sampling_offline_dispatch(cpu);
 391 }
 392
 393 static void gator_timer_stop(void)
 394 {
 395         int cpu;
 396
 397         if (gator_running) {
 398                 on_each_cpu(gator_timer_offline, NULL, 1);
 399                 for_each_online_cpu(cpu) {
 400                         gator_timer_offline_dispatch(cpu);
 401                 }
 402
 403                 gator_running = 0;
 404                 gator_hrtimer_shutdown();
 405         }
 406 }
 407
 408 // This function runs in interrupt context and on the appropriate core
 409 static void gator_timer_online(void* unused)
 410 {
 411         struct gator_interface *gi;
 412         int len, cpu = smp_processor_id();
 413         int* buffer;
 414
 415         gator_trace_power_online();
 416
 417         // online any events and output counters
 418         if (marshal_event_header()) {
 419                 list_for_each_entry(gi, &gator_events, list) {
 420                         if (gi->online) {
 421                                 len = gi->online(&buffer);
 422                                 marshal_event(len, buffer);
 423                         }
 424                 }
 425         }
 426
 427         gator_hrtimer_online(cpu);
 428 }
 429
 430 // This function runs in interrupt context and may be running on a core other than core 'cpu'
 431 static void gator_timer_online_dispatch(int cpu)
 432 {
 433         struct gator_interface *gi;
 434
 435         list_for_each_entry(gi, &gator_events, list)
 436                 if (gi->online_dispatch)
 437                         gi->online_dispatch(cpu);
 438
 439         gator_event_sampling_online_dispatch(cpu);
 440 }
 441
 442 int gator_timer_start(unsigned long sample_rate)
 443 {
 444         int cpu;
 445
 446         if (gator_running) {
 447                 pr_notice("gator: already running\n");
 448                 return 0;
 449         }
 450
 451         gator_running = 1;
 452
 453         // event based sampling trumps hr timer based sampling
 454         if (event_based_sampling)
 455                 sample_rate = 0;
 456
 457         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 458                 return -1;
 459
 460         for_each_online_cpu(cpu) {
 461                 gator_timer_online_dispatch(cpu);
 462         }
 463         on_each_cpu(gator_timer_online, NULL, 1);
 464
 465         return 0;
 466 }
 467
 468 static uint64_t gator_get_time(void)
 469 {
 470         struct timespec ts;
 471         uint64_t timestamp;
 472
 473         getnstimeofday(&ts);
 474         timestamp = timespec_to_ns(&ts);
 475
 476         return timestamp;
 477 }
 478
 479 /******************************************************************************
 480  * cpu hotplug and pm notifiers
 481  ******************************************************************************/
 482 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 483 {
 484         long cpu = (long)hcpu;
 485
 486         switch (action) {
 487                 case CPU_DOWN_PREPARE:
 488                 case CPU_DOWN_PREPARE_FROZEN:
 489                         smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 490                         gator_timer_offline_dispatch(cpu);
 491                         break;
 492                 case CPU_ONLINE:
 493                 case CPU_ONLINE_FROZEN:
 494                         gator_timer_online_dispatch(cpu);
 495                         smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 496                         break;
 497         }
 498
 499         return NOTIFY_OK;
 500 }
 501
 502 static struct notifier_block __refdata gator_hotcpu_notifier = {
 503         .notifier_call = gator_hotcpu_notify,
 504 };
 505
 506 // n.b. calling "on_each_cpu" only runs on those that are online
 507 // Registered linux events are not disabled, so their counters will continue to collect
 508 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 509 {
 510         int cpu;
 511
 512         switch (event) {
 513                 case PM_HIBERNATION_PREPARE:
 514                 case PM_SUSPEND_PREPARE:
 515                         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 516                         unregister_scheduler_tracepoints();
 517                         on_each_cpu(gator_timer_offline, NULL, 1);
 518                         for_each_online_cpu(cpu) {
 519                                 gator_timer_offline_dispatch(cpu);
 520                         }
 521                         break;
 522                 case PM_POST_HIBERNATION:
 523                 case PM_POST_SUSPEND:
 524                         for_each_online_cpu(cpu) {
 525                                 gator_timer_online_dispatch(cpu);
 526                         }
 527                         on_each_cpu(gator_timer_online, NULL, 1);
 528                         register_scheduler_tracepoints();
 529                         register_hotcpu_notifier(&gator_hotcpu_notifier);
 530                         break;
 531         }
 532
 533         return NOTIFY_OK;
 534 }
 535
 536 static struct notifier_block gator_pm_notifier = {
 537         .notifier_call = gator_pm_notify,
 538 };
 539
 540 static int gator_notifier_start(void)
 541 {
 542         int retval;
 543         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 544         if (retval == 0)
 545                 retval = register_pm_notifier(&gator_pm_notifier);
 546         return retval;
 547 }
 548
 549 static void gator_notifier_stop(void)
 550 {
 551         unregister_pm_notifier(&gator_pm_notifier);
 552         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 553 }
 554
 555 /******************************************************************************
 556  * Main
 557  ******************************************************************************/
 558 static void gator_summary(void)
 559 {
 560         uint64_t timestamp, uptime = 0;
 561         struct timespec uptime_ts;
 562         void (*m2b)(struct timespec *ts);
 563
 564         timestamp = gator_get_time();
 565
 566         do_posix_clock_monotonic_gettime(&uptime_ts);
 567         m2b = symbol_get(monotonic_to_bootbased);
 568         if (m2b) {
 569                 m2b(&uptime_ts);
 570                 uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
 571         }
 572
 573         marshal_summary(timestamp, uptime);
 574 }
 575
 576 int gator_events_install(struct gator_interface *interface)
 577 {
 578         list_add_tail(&interface->list, &gator_events);
 579
 580         return 0;
 581 }
 582
 583 int gator_events_get_key(void)
 584 {
 585         // key of zero is reserved as a timestamp
 586         static int key = 1;
 587
 588         return key++;
 589 }
 590
 591 static int gator_init(void)
 592 {
 593         int i;
 594
 595         // events sources (gator_events.h, generated by gator_events.sh)
 596         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 597                 if (gator_events_list[i])
 598                         gator_events_list[i]();
 599
 600         gator_trace_power_init();
 601
 602         return 0;
 603 }
 604
 605 static int gator_start(void)
 606 {
 607         unsigned long cpu, i;
 608         struct gator_interface *gi;
 609
 610         // Initialize the buffer with the frame type and core
 611         for_each_present_cpu(cpu) {
 612                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 613                         gator_buffer_header(cpu, i);
 614                 }
 615         }
 616
 617         // Capture the start time
 618         gator_summary();
 619
 620         // start all events
 621         list_for_each_entry(gi, &gator_events, list) {
 622                 if (gi->start && gi->start() != 0) {
 623                         struct list_head *ptr = gi->list.prev;
 624
 625                         while (ptr != &gator_events) {
 626                                 gi = list_entry(ptr, struct gator_interface, list);
 627
 628                                 if (gi->stop)
 629                                         gi->stop();
 630
 631                                 ptr = ptr->prev;
 632                         }
 633                         goto events_failure;
 634                 }
 635         }
 636
 637         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
 638         if (cookies_initialize())
 639                 goto cookies_failure;
 640         if (gator_annotate_start())
 641                 goto annotate_failure;
 642         if (gator_trace_sched_start())
 643                 goto sched_failure;
 644         if (gator_trace_power_start())
 645                 goto power_failure;
 646         if (gator_trace_gpu_start())
 647                 goto gpu_failure;
 648         if (gator_event_sampling_start())
 649                 goto event_sampling_failure;
 650         if (gator_timer_start(gator_timer_count))
 651                 goto timer_failure;
 652         if (gator_notifier_start())
 653                 goto notifier_failure;
 654
 655         return 0;
 656
 657 notifier_failure:
 658         gator_timer_stop();
 659 timer_failure:
 660         gator_event_sampling_stop();
 661 event_sampling_failure:
 662         gator_trace_gpu_stop();
 663 gpu_failure:
 664         gator_trace_power_stop();
 665 power_failure:
 666         gator_trace_sched_stop();
 667 sched_failure:
 668         gator_annotate_stop();
 669 annotate_failure:
 670         cookies_release();
 671 cookies_failure:
 672         // stop all events
 673         list_for_each_entry(gi, &gator_events, list)
 674                 if (gi->stop)
 675                         gi->stop();
 676 events_failure:
 677
 678         return -1;
 679 }
 680
 681 static void gator_stop(void)
 682 {
 683         struct gator_interface *gi;
 684
 685         // stop all events
 686         list_for_each_entry(gi, &gator_events, list)
 687                 if (gi->stop)
 688                         gi->stop();
 689
 690         gator_annotate_stop();
 691         gator_trace_sched_stop();
 692         gator_trace_power_stop();
 693         gator_trace_gpu_stop();
 694         gator_event_sampling_stop();
 695
 696         // stop all interrupt callback reads before tearing down other interfaces
 697         gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
 698         gator_timer_stop();
 699 }
 700
 701 /******************************************************************************
 702  * Filesystem
 703  ******************************************************************************/
 704 /* fopen("buffer") */
 705 static int gator_op_setup(void)
 706 {
 707         int err = 0;
 708         int cpu, i;
 709
 710         mutex_lock(&start_mutex);
 711
 712         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
 713         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
 714
 715         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
 716         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
 717
 718         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
 719         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
 720
 721         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
 722         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
 723
 724         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
 725         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
 726
 727         gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
 728         gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
 729
 730         gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
 731         gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
 732
 733         // Initialize percpu per buffer variables
 734         for (i = 0; i < NUM_GATOR_BUFS; i++) {
 735                 // Verify buffers are a power of 2
 736                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
 737                         err = -ENOEXEC;
 738                         goto setup_error;
 739                 }
 740
 741                 for_each_present_cpu(cpu) {
 742                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 743                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 744                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 745                         per_cpu(buffer_space_available, cpu)[i] = true;
 746
 747                         // Annotation is a special case that only uses a single buffer
 748                         if (cpu > 0 && i == ANNOTATE_BUF) {
 749                                 per_cpu(gator_buffer, cpu)[i] = NULL;
 750                                 continue;
 751                         }
 752
 753                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
 754                         if (!per_cpu(gator_buffer, cpu)[i]) {
 755                                 err = -ENOMEM;
 756                                 goto setup_error;
 757                         }
 758                 }
 759         }
 760
 761 setup_error:
 762         mutex_unlock(&start_mutex);
 763         return err;
 764 }
 765
 766 /* Actually start profiling (echo 1>/dev/gator/enable) */
 767 static int gator_op_start(void)
 768 {
 769         int err = 0;
 770
 771         mutex_lock(&start_mutex);
 772
 773         if (gator_started || gator_start())
 774                 err = -EINVAL;
 775         else
 776                 gator_started = 1;
 777
 778         mutex_unlock(&start_mutex);
 779
 780         return err;
 781 }
 782
 783 /* echo 0>/dev/gator/enable */
 784 static void gator_op_stop(void)
 785 {
 786         mutex_lock(&start_mutex);
 787
 788         if (gator_started) {
 789                 gator_stop();
 790
 791                 mutex_lock(&gator_buffer_mutex);
 792
 793                 gator_started = 0;
 794                 cookies_release();
 795                 wake_up(&gator_buffer_wait);
 796
 797                 mutex_unlock(&gator_buffer_mutex);
 798         }
 799
 800         mutex_unlock(&start_mutex);
 801 }
 802
 803 static void gator_shutdown(void)
 804 {
 805         int cpu, i;
 806
 807         mutex_lock(&start_mutex);
 808
 809         for_each_present_cpu(cpu) {
 810                 mutex_lock(&gator_buffer_mutex);
 811                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
 812                         vfree(per_cpu(gator_buffer, cpu)[i]);
 813                         per_cpu(gator_buffer, cpu)[i] = NULL;
 814                         per_cpu(gator_buffer_read, cpu)[i] = 0;
 815                         per_cpu(gator_buffer_write, cpu)[i] = 0;
 816                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
 817                         per_cpu(buffer_space_available, cpu)[i] = true;
 818                 }
 819                 mutex_unlock(&gator_buffer_mutex);
 820         }
 821
 822         mutex_unlock(&start_mutex);
 823 }
 824
 825 static int gator_set_backtrace(unsigned long val)
 826 {
 827         int err = 0;
 828
 829         mutex_lock(&start_mutex);
 830
 831         if (gator_started)
 832                 err = -EBUSY;
 833         else
 834                 gator_backtrace_depth = val;
 835
 836         mutex_unlock(&start_mutex);
 837
 838         return err;
 839 }
 840
 841 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
 842 {
 843         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
 844 }
 845
 846 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
 847 {
 848         unsigned long val;
 849         int retval;
 850
 851         if (*offset)
 852                 return -EINVAL;
 853
 854         retval = gatorfs_ulong_from_user(&val, buf, count);
 855         if (retval)
 856                 return retval;
 857
 858         if (val)
 859                 retval = gator_op_start();
 860         else
 861                 gator_op_stop();
 862
 863         if (retval)
 864                 return retval;
 865         return count;
 866 }
 867
 868 static const struct file_operations enable_fops = {
 869         .read           = enable_read,
 870         .write          = enable_write,
 871 };
 872
 873 static int userspace_buffer_open(struct inode *inode, struct file *file)
 874 {
 875         int err = -EPERM;
 876
 877         if (!capable(CAP_SYS_ADMIN))
 878                 return -EPERM;
 879
 880         if (test_and_set_bit_lock(0, &gator_buffer_opened))
 881                 return -EBUSY;
 882
 883         if ((err = gator_op_setup()))
 884                 goto fail;
 885
 886         /* NB: the actual start happens from userspace
 887          * echo 1 >/dev/gator/enable
 888          */
 889
 890         return 0;
 891
 892 fail:
 893         __clear_bit_unlock(0, &gator_buffer_opened);
 894         return err;
 895 }
 896
 897 static int userspace_buffer_release(struct inode *inode, struct file *file)
 898 {
 899         gator_op_stop();
 900         gator_shutdown();
 901         __clear_bit_unlock(0, &gator_buffer_opened);
 902         return 0;
 903 }
 904
 905 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
 906                                  size_t count, loff_t *offset)
 907 {
 908         int retval = -EINVAL;
 909         int commit = 0, length, length1, length2, read, byte, type_length;
 910         char *buffer1;
 911         char *buffer2 = NULL;
 912         int cpu, buftype;
 913
 914         /* do not handle partial reads */
 915         if (count != userspace_buffer_size || *offset)
 916                 return -EINVAL;
 917
 918         // sleep until the condition is true or a signal is received
 919         // the condition is checked each time gator_buffer_wait is woken up
 920         buftype = cpu = -1;
 921         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
 922
 923         if (signal_pending(current))
 924                 return -EINTR;
 925
 926         length2 = 0;
 927         retval = -EFAULT;
 928
 929         mutex_lock(&gator_buffer_mutex);
 930
 931         if (buftype == -1 || cpu == -1) {
 932                 retval = 0;
 933                 goto out;
 934         }
 935
 936         read = per_cpu(gator_buffer_read, cpu)[buftype];
 937         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
 938
 939         /* May happen if the buffer is freed during pending reads. */
 940         if (!per_cpu(gator_buffer, cpu)[buftype]) {
 941                 retval = -EFAULT;
 942                 goto out;
 943         }
 944
 945         /* determine the size of two halves */
 946         length1 = commit - read;
 947         buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
 948         buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
 949         if (length1 < 0) {
 950                 length1 = gator_buffer_size[buftype] - read;
 951                 length2 = commit;
 952         }
 953
 954         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
 955         type_length = gator_response_type ? 1 : 0;
 956         length = length1 + length2 - type_length - sizeof(int);
 957         for (byte = 0; byte < sizeof(int); byte++) {
 958                 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
 959         }
 960
 961         /* start, middle or end */
 962         if (length1 > 0) {
 963                 if (copy_to_user(&buf[0], buffer1, length1)) {
 964                         goto out;
 965                 }
 966         }
 967
 968         /* possible wrap around */
 969         if (length2 > 0) {
 970                 if (copy_to_user(&buf[length1], buffer2, length2)) {
 971                         goto out;
 972                 }
 973         }
 974
 975         per_cpu(gator_buffer_read, cpu)[buftype] = commit;
 976         retval = length1 + length2;
 977
 978         /* kick just in case we've lost an SMP event */
 979         wake_up(&gator_buffer_wait);
 980
 981 out:
 982         mutex_unlock(&gator_buffer_mutex);
 983         return retval;
 984 }
 985
 986 const struct file_operations gator_event_buffer_fops = {
 987         .open           = userspace_buffer_open,
 988         .release        = userspace_buffer_release,
 989         .read           = userspace_buffer_read,
 990 };
 991
 992 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
 993 {
 994         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
 995                                         offset);
 996 }
 997
 998 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
 999 {
1000         unsigned long val;
1001         int retval;
1002
1003         if (*offset)
1004                 return -EINVAL;
1005
1006         retval = gatorfs_ulong_from_user(&val, buf, count);
1007         if (retval)
1008                 return retval;
1009
1010         retval = gator_set_backtrace(val);
1011
1012         if (retval)
1013                 return retval;
1014         return count;
1015 }
1016
1017 static const struct file_operations depth_fops = {
1018         .read           = depth_read,
1019         .write          = depth_write
1020 };
1021
1022 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1023 {
1024         struct dentry *dir;
1025         struct gator_interface *gi;
1026         int cpu;
1027
1028         /* reinitialize default values */
1029         gator_cpu_cores = 0;
1030         for_each_present_cpu(cpu) {
1031                 gator_cpu_cores++;
1032         }
1033         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1034         gator_response_type = 1;
1035
1036         gatorfs_create_file(sb, root, "enable", &enable_fops);
1037         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1038         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1039         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1040         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1041         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1042         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1043         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1044
1045         // Annotate interface
1046         gator_annotate_create_files(sb, root);
1047
1048         // Linux Events
1049         dir = gatorfs_mkdir(sb, root, "events");
1050         list_for_each_entry(gi, &gator_events, list)
1051                 if (gi->create_files)
1052                         gi->create_files(sb, dir);
1053
1054         // Power interface
1055         gator_trace_power_create_files(sb, dir);
1056 }
1057
1058 /******************************************************************************
1059  * Module
1060  ******************************************************************************/
1061 static int __init gator_module_init(void)
1062 {
1063         if (gatorfs_register()) {
1064                 return -1;
1065         }
1066
1067         if (gator_init()) {
1068                 gatorfs_unregister();
1069                 return -1;
1070         }
1071
1072         return 0;
1073 }
1074
1075 static void __exit gator_module_exit(void)
1076 {
1077         tracepoint_synchronize_unregister();
1078         gatorfs_unregister();
1079 }
1080
1081 module_init(gator_module_init);
1082 module_exit(gator_module_exit);
1083
1084 MODULE_LICENSE("GPL");
1085 MODULE_AUTHOR("ARM Ltd");
1086 MODULE_DESCRIPTION("Gator system profiler");