gator-driver: Revert #error about lack of CONFIG_PERF_EVENTS
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 13;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
49 #endif
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
56 #endif
57 #endif
59 /******************************************************************************
60  * DEFINES
61  ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE       (1*1024)
63 #define BACKTRACE_BUFFER_SIZE     (128*1024)
64 #define NAME_BUFFER_SIZE          (64*1024)
65 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE      (64*1024)     // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
72 #define NO_COOKIE      0U
73 #define INVALID_COOKIE ~0U
75 #define FRAME_SUMMARY       1
76 #define FRAME_BACKTRACE     2
77 #define FRAME_NAME          3
78 #define FRAME_COUNTER       4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE      6
81 #define FRAME_SCHED_TRACE   7
82 #define FRAME_GPU_TRACE     8
83 #define FRAME_IDLE          9
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE      1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME   3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP  2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT   2
97 #define MESSAGE_IDLE_ENTER 1
98 #define MESSAGE_IDLE_EXIT 2
100 #define MAXSIZE_PACK32     5
101 #define MAXSIZE_PACK64    10
103 #define FRAME_HEADER_SIZE 3
105 #if defined(__arm__)
106 #define PC_REG regs->ARM_pc
107 #elif defined(__aarch64__)
108 #define PC_REG regs->pc
109 #else
110 #define PC_REG regs->ip
111 #endif
113 enum {
114         SUMMARY_BUF,
115         BACKTRACE_BUF,
116         NAME_BUF,
117         COUNTER_BUF,
118         BLOCK_COUNTER_BUF,
119         ANNOTATE_BUF,
120         SCHED_TRACE_BUF,
121         GPU_TRACE_BUF,
122         IDLE_BUF,
123         NUM_GATOR_BUFS
124 };
126 /******************************************************************************
127  * Globals
128  ******************************************************************************/
129 static unsigned long gator_cpu_cores;
130 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
131 static unsigned long userspace_buffer_size;
132 static unsigned long gator_backtrace_depth;
133 // How often to commit the buffers for live in nanoseconds
134 static u64 gator_live_rate;
136 static unsigned long gator_started;
137 static u64 gator_monotonic_started;
138 static unsigned long gator_buffer_opened;
139 static unsigned long gator_timer_count;
140 static unsigned long gator_response_type;
141 static DEFINE_MUTEX(start_mutex);
142 static DEFINE_MUTEX(gator_buffer_mutex);
144 bool event_based_sampling;
146 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
147 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
148 static struct timer_list gator_buffer_wake_up_timer;
149 static LIST_HEAD(gator_events);
151 static DEFINE_PER_CPU(u64, last_timestamp);
153 static bool printed_monotonic_warning;
155 static bool sent_core_name[NR_CPUS];
157 /******************************************************************************
158  * Prototypes
159  ******************************************************************************/
160 static void buffer_check(int cpu, int buftype, u64 time);
161 static void gator_commit_buffer(int cpu, int buftype, u64 time);
162 static int buffer_bytes_available(int cpu, int buftype);
163 static bool buffer_check_space(int cpu, int buftype, int bytes);
164 static int contiguous_space_available(int cpu, int bufytpe);
165 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
166 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
167 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
168 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
169 static void gator_add_trace(int cpu, unsigned long address);
170 static void gator_add_sample(int cpu, struct pt_regs *const regs);
171 static u64 gator_get_time(void);
173 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
174 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
175 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
176 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
177 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
178 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
179 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
182 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
183 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
184 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
186 // The buffer. Allocated in gator_op_setup
187 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 #if GATOR_LIVE
190 // The time after which the buffer should be committed for live display
191 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 #endif
194 /******************************************************************************
195  * Application Includes
196  ******************************************************************************/
197 #include "gator_marshaling.c"
198 #include "gator_hrtimer_perf.c"
199 #include "gator_hrtimer_gator.c"
200 #include "gator_cookies.c"
201 #include "gator_trace_sched.c"
202 #include "gator_trace_power.c"
203 #include "gator_trace_gpu.c"
204 #include "gator_backtrace.c"
205 #include "gator_annotate.c"
206 #include "gator_fs.c"
207 #include "gator_pack.c"
209 /******************************************************************************
210  * Misc
211  ******************************************************************************/
213 const struct gator_cpu gator_cpus[] = {
214         {
215                 .cpuid = ARM1136,
216                 .core_name = "ARM1136",
217                 .pmnc_name = "ARM_ARM11",
218                 .pmnc_counters = 3,
219         },
220         {
221                 .cpuid = ARM1156,
222                 .core_name = "ARM1156",
223                 .pmnc_name = "ARM_ARM11",
224                 .pmnc_counters = 3,
225         },
226         {
227                 .cpuid = ARM1176,
228                 .core_name = "ARM1176",
229                 .pmnc_name = "ARM_ARM11",
230                 .pmnc_counters = 3,
231         },
232         {
233                 .cpuid = ARM11MPCORE,
234                 .core_name = "ARM11MPCore",
235                 .pmnc_name = "ARM_ARM11MPCore",
236                 .pmnc_counters = 3,
237         },
238         {
239                 .cpuid = CORTEX_A5,
240                 .core_name = "Cortex-A5",
241                 .pmu_name = "ARMv7_Cortex_A5",
242                 .pmnc_name = "ARM_Cortex-A5",
243                 .pmnc_counters = 2,
244         },
245         {
246                 .cpuid = CORTEX_A7,
247                 .core_name = "Cortex-A7",
248                 .pmu_name = "ARMv7_Cortex_A7",
249                 .pmnc_name = "ARM_Cortex-A7",
250                 .pmnc_counters = 4,
251         },
252         {
253                 .cpuid = CORTEX_A8,
254                 .core_name = "Cortex-A8",
255                 .pmu_name = "ARMv7_Cortex_A8",
256                 .pmnc_name = "ARM_Cortex-A8",
257                 .pmnc_counters = 4,
258         },
259         {
260                 .cpuid = CORTEX_A9,
261                 .core_name = "Cortex-A9",
262                 .pmu_name = "ARMv7_Cortex_A9",
263                 .pmnc_name = "ARM_Cortex-A9",
264                 .pmnc_counters = 6,
265         },
266         {
267                 .cpuid = CORTEX_A15,
268                 .core_name = "Cortex-A15",
269                 .pmu_name = "ARMv7_Cortex_A15",
270                 .pmnc_name = "ARM_Cortex-A15",
271                 .pmnc_counters = 6,
272         },
273         {
274                 .cpuid = SCORPION,
275                 .core_name = "Scorpion",
276                 .pmnc_name = "Scorpion",
277                 .pmnc_counters = 4,
278         },
279         {
280                 .cpuid = SCORPIONMP,
281                 .core_name = "ScorpionMP",
282                 .pmnc_name = "ScorpionMP",
283                 .pmnc_counters = 4,
284         },
285         {
286                 .cpuid = KRAITSIM,
287                 .core_name = "KraitSIM",
288                 .pmnc_name = "Krait",
289                 .pmnc_counters = 4,
290         },
291         {
292                 .cpuid = KRAIT,
293                 .core_name = "Krait",
294                 .pmnc_name = "Krait",
295                 .pmnc_counters = 4,
296         },
297         {
298                 .cpuid = KRAIT_S4_PRO,
299                 .core_name = "Krait S4 Pro",
300                 .pmnc_name = "Krait",
301                 .pmnc_counters = 4,
302         },
303         {
304                 .cpuid = CORTEX_A53,
305                 .core_name = "Cortex-A53",
306                 .pmnc_name = "ARM_Cortex-A53",
307                 .pmnc_counters = 6,
308         },
309         {
310                 .cpuid = CORTEX_A57,
311                 .core_name = "Cortex-A57",
312                 .pmnc_name = "ARM_Cortex-A57",
313                 .pmnc_counters = 6,
314         },
315         {
316                 .cpuid = AARCH64,
317                 .core_name = "AArch64",
318                 .pmnc_name = "ARM_AArch64",
319                 .pmnc_counters = 6,
320         },
321         {
322                 .cpuid = OTHER,
323                 .core_name = "Other",
324                 .pmnc_name = "Other",
325                 .pmnc_counters = 6,
326         },
327         {}
328 };
330 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
332         int i;
334         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
335                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
336                 if (gator_cpu->cpuid == cpuid) {
337                         return gator_cpu;
338                 }
339         }
341         return NULL;
344 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
346         int i;
348         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
349                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
350                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
351                         return gator_cpu;
352                 }
353         }
355         return NULL;
358 u32 gator_cpuid(void)
360 #if defined(__arm__) || defined(__aarch64__)
361         u32 val;
362 #if !defined(__aarch64__)
363         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
364 #else
365         asm volatile("mrs %0, midr_el1" : "=r" (val));
366 #endif
367         return (val >> 4) & 0xfff;
368 #else
369         return OTHER;
370 #endif
373 static void gator_buffer_wake_up(unsigned long data)
375         wake_up(&gator_buffer_wait);
378 /******************************************************************************
379  * Commit interface
380  ******************************************************************************/
381 static bool buffer_commit_ready(int *cpu, int *buftype)
383         int cpu_x, x;
384         for_each_present_cpu(cpu_x) {
385                 for (x = 0; x < NUM_GATOR_BUFS; x++)
386                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
387                                 *cpu = cpu_x;
388                                 *buftype = x;
389                                 return true;
390                         }
391         }
392         return false;
395 /******************************************************************************
396  * Buffer management
397  ******************************************************************************/
398 static int buffer_bytes_available(int cpu, int buftype)
400         int remaining, filled;
402         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
403         if (filled < 0) {
404                 filled += gator_buffer_size[buftype];
405         }
407         remaining = gator_buffer_size[buftype] - filled;
409         if (per_cpu(buffer_space_available, cpu)[buftype]) {
410                 // Give some extra room; also allows space to insert the overflow error packet
411                 remaining -= 200;
412         } else {
413                 // Hysteresis, prevents multiple overflow messages
414                 remaining -= 2000;
415         }
417         return remaining;
420 static int contiguous_space_available(int cpu, int buftype)
422         int remaining = buffer_bytes_available(cpu, buftype);
423         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
424         if (remaining < contiguous)
425                 return remaining;
426         else
427                 return contiguous;
430 static bool buffer_check_space(int cpu, int buftype, int bytes)
432         int remaining = buffer_bytes_available(cpu, buftype);
434         if (remaining < bytes) {
435                 per_cpu(buffer_space_available, cpu)[buftype] = false;
436         } else {
437                 per_cpu(buffer_space_available, cpu)[buftype] = true;
438         }
440         return per_cpu(buffer_space_available, cpu)[buftype];
443 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
445         int i;
446         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
447         u32 mask = gator_buffer_mask[buftype];
448         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
450         for (i = 0; i < len; i++) {
451                 buffer[write] = x[i];
452                 write = (write + 1) & mask;
453         }
455         per_cpu(gator_buffer_write, cpu)[buftype] = write;
458 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
460         int len = strlen(x);
461         gator_buffer_write_packed_int(cpu, buftype, len);
462         gator_buffer_write_bytes(cpu, buftype, x, len);
465 static void gator_commit_buffer(int cpu, int buftype, u64 time)
467         int type_length, commit, length, byte;
469         if (!per_cpu(gator_buffer, cpu)[buftype])
470                 return;
472         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
473         type_length = gator_response_type ? 1 : 0;
474         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
475         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
476         if (length < 0) {
477                 length += gator_buffer_size[buftype];
478         }
479         length = length - type_length - sizeof(int);
481         if (length <= FRAME_HEADER_SIZE) {
482                 // Nothing to write, only the frame header is present
483                 return;
484         }
486         for (byte = 0; byte < sizeof(int); byte++) {
487                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
488         }
490         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
492 #if GATOR_LIVE
493         if (gator_live_rate > 0) {
494                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
495                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
496                 }
497         }
498 #endif
500         marshal_frame(cpu, buftype);
502         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
503         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
506 static void buffer_check(int cpu, int buftype, u64 time)
508         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
509         if (filled < 0) {
510                 filled += gator_buffer_size[buftype];
511         }
512         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
513                 gator_commit_buffer(cpu, buftype, time);
514         }
517 static void gator_add_trace(int cpu, unsigned long address)
519         off_t offset = 0;
520         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
522         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
523                 offset = address;
524         }
526         marshal_backtrace(offset & ~1, cookie);
529 static void gator_add_sample(int cpu, struct pt_regs *const regs)
531         bool inKernel;
532         unsigned long exec_cookie;
534         if (!regs)
535                 return;
537         inKernel = !user_mode(regs);
538         exec_cookie = get_exec_cookie(cpu, current);
540         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
541                 return;
543         if (inKernel) {
544                 kernel_backtrace(cpu, regs);
545         } else {
546                 // Cookie+PC
547                 gator_add_trace(cpu, PC_REG);
549                 // Backtrace
550                 if (gator_backtrace_depth)
551                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
552         }
554         marshal_backtrace_footer();
557 /******************************************************************************
558  * hrtimer interrupt processing
559  ******************************************************************************/
560 static void gator_timer_interrupt(void)
562         struct pt_regs *const regs = get_irq_regs();
563         gator_backtrace_handler(regs);
566 void gator_backtrace_handler(struct pt_regs *const regs)
568         int cpu = get_physical_cpu();
570         // Output backtrace
571         gator_add_sample(cpu, regs);
573         // Collect counters
574         if (!per_cpu(collecting, cpu)) {
575                 collect_counters();
576         }
579 static int gator_running;
581 // This function runs in interrupt context and on the appropriate core
582 static void gator_timer_offline(void *migrate)
584         struct gator_interface *gi;
585         int i, len, cpu = get_physical_cpu();
586         int *buffer;
587         u64 time;
589         gator_trace_sched_offline();
590         gator_trace_power_offline();
592         if (!migrate) {
593                 gator_hrtimer_offline();
594         }
596         // Offline any events and output counters
597         time = gator_get_time();
598         if (marshal_event_header()) {
599                 list_for_each_entry(gi, &gator_events, list) {
600                         if (gi->offline) {
601                                 len = gi->offline(&buffer, migrate);
602                                 marshal_event(len, buffer);
603                         }
604                 }
605                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
606                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
607         }
609         // Flush all buffers on this core
610         for (i = 0; i < NUM_GATOR_BUFS; i++)
611                 gator_commit_buffer(cpu, i, time);
614 // This function runs in interrupt context and may be running on a core other than core 'cpu'
615 static void gator_timer_offline_dispatch(int cpu, bool migrate)
617         struct gator_interface *gi;
619         list_for_each_entry(gi, &gator_events, list) {
620                 if (gi->offline_dispatch) {
621                         gi->offline_dispatch(cpu, migrate);
622                 }
623         }
626 static void gator_timer_stop(void)
628         int cpu;
630         if (gator_running) {
631                 on_each_cpu(gator_timer_offline, NULL, 1);
632                 for_each_online_cpu(cpu) {
633                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
634                 }
636                 gator_running = 0;
637                 gator_hrtimer_shutdown();
638         }
641 // This function runs in interrupt context and on the appropriate core
642 static void gator_timer_online(void *migrate)
644         struct gator_interface *gi;
645         int len, cpu = get_physical_cpu();
646         int *buffer;
648         gator_trace_power_online();
650         // online any events and output counters
651         if (marshal_event_header()) {
652                 list_for_each_entry(gi, &gator_events, list) {
653                         if (gi->online) {
654                                 len = gi->online(&buffer, migrate);
655                                 marshal_event(len, buffer);
656                         }
657                 }
658                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
659                 buffer_check(cpu, BLOCK_COUNTER_BUF, gator_get_time());
660         }
662         if (!migrate) {
663                 gator_hrtimer_online();
664         }
666 #if defined(__arm__) || defined(__aarch64__)
667         if (!sent_core_name[cpu]) {
668                 const char *core_name = NULL;
669                 const u32 cpuid = gator_cpuid();
670                 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
671                 char core_name_buf[32];
673                 if (gator_cpu != NULL) {
674                         core_name = gator_cpu->core_name;
675                 } else {
676                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
677                         core_name = core_name_buf;
678                 }
680                 marshal_core_name(cpuid, core_name);
681                 sent_core_name[cpu] = true;
682         }
683 #endif
686 // This function runs in interrupt context and may be running on a core other than core 'cpu'
687 static void gator_timer_online_dispatch(int cpu, bool migrate)
689         struct gator_interface *gi;
691         list_for_each_entry(gi, &gator_events, list) {
692                 if (gi->online_dispatch) {
693                         gi->online_dispatch(cpu, migrate);
694                 }
695         }
698 int gator_timer_start(unsigned long sample_rate)
700         int cpu;
702         if (gator_running) {
703                 pr_notice("gator: already running\n");
704                 return 0;
705         }
707         gator_running = 1;
709         // event based sampling trumps hr timer based sampling
710         if (event_based_sampling) {
711                 sample_rate = 0;
712         }
714         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
715                 return -1;
717         for_each_online_cpu(cpu) {
718                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
719         }
720         on_each_cpu(gator_timer_online, NULL, 1);
722         return 0;
725 static u64 gator_get_time(void)
727         struct timespec ts;
728         u64 timestamp;
729         u64 prev_timestamp;
730         u64 delta;
731         int cpu = smp_processor_id();
733         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
734         getrawmonotonic(&ts);
735         timestamp = timespec_to_ns(&ts);
737         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
738         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
739         prev_timestamp = per_cpu(last_timestamp, cpu);
740         if (prev_timestamp <= timestamp) {
741                 per_cpu(last_timestamp, cpu) = timestamp;
742         } else {
743                 delta = prev_timestamp - timestamp;
744                 // Log the error once
745                 if (!printed_monotonic_warning && delta > 500000) {
746                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
747                         printed_monotonic_warning = true;
748                 } else {
749                         pr_debug("%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\n", __FUNCTION__, cpu, delta);
750                 }
751                 timestamp = prev_timestamp;
752         }
754         return timestamp - gator_monotonic_started;
757 /******************************************************************************
758  * cpu hotplug and pm notifiers
759  ******************************************************************************/
760 #include "gator_iks.c"
762 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
764         int cpu = lcpu_to_pcpu((long)hcpu);
766         switch (action) {
767         case CPU_DOWN_PREPARE:
768         case CPU_DOWN_PREPARE_FROZEN:
769                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
770                 gator_timer_offline_dispatch(cpu, false);
771                 break;
772         case CPU_ONLINE:
773         case CPU_ONLINE_FROZEN:
774                 gator_timer_online_dispatch(cpu, false);
775                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
776                 break;
777         }
779         return NOTIFY_OK;
782 static struct notifier_block __refdata gator_hotcpu_notifier = {
783         .notifier_call = gator_hotcpu_notify,
784 };
786 // n.b. calling "on_each_cpu" only runs on those that are online
787 // Registered linux events are not disabled, so their counters will continue to collect
788 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
790         int cpu;
792         switch (event) {
793         case PM_HIBERNATION_PREPARE:
794         case PM_SUSPEND_PREPARE:
795                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
796                 unregister_scheduler_tracepoints();
797                 on_each_cpu(gator_timer_offline, NULL, 1);
798                 for_each_online_cpu(cpu) {
799                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
800                 }
801                 break;
802         case PM_POST_HIBERNATION:
803         case PM_POST_SUSPEND:
804                 for_each_online_cpu(cpu) {
805                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
806                 }
807                 on_each_cpu(gator_timer_online, NULL, 1);
808                 register_scheduler_tracepoints();
809                 register_hotcpu_notifier(&gator_hotcpu_notifier);
810                 break;
811         }
813         return NOTIFY_OK;
816 static struct notifier_block gator_pm_notifier = {
817         .notifier_call = gator_pm_notify,
818 };
820 static int gator_notifier_start(void)
822         int retval;
823         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
824         if (retval == 0)
825                 retval = register_pm_notifier(&gator_pm_notifier);
826         return retval;
829 static void gator_notifier_stop(void)
831         unregister_pm_notifier(&gator_pm_notifier);
832         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
835 /******************************************************************************
836  * Main
837  ******************************************************************************/
838 static void gator_summary(void)
840         u64 timestamp, uptime;
841         struct timespec ts;
842         char uname_buf[512];
843         void (*m2b)(struct timespec *ts);
844         unsigned long flags;
846         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
848         getnstimeofday(&ts);
849         timestamp = timespec_to_ns(&ts);
851         do_posix_clock_monotonic_gettime(&ts);
852         // monotonic_to_bootbased is not defined for some versions of Android
853         m2b = symbol_get(monotonic_to_bootbased);
854         if (m2b) {
855                 m2b(&ts);
856         }
857         uptime = timespec_to_ns(&ts);
859         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
860         local_irq_save(flags);
861         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
862         gator_monotonic_started = 0;
863         gator_monotonic_started = gator_get_time();
864         local_irq_restore(flags);
866         marshal_summary(timestamp, uptime, uname_buf);
869 int gator_events_install(struct gator_interface *interface)
871         list_add_tail(&interface->list, &gator_events);
873         return 0;
876 int gator_events_get_key(void)
878         // key of zero is reserved as a timestamp
879         static int key = 1;
881         const int ret = key;
882         key += 2;
883         return ret;
886 static int gator_init(void)
888         int i;
890         calc_first_cluster_size();
892         // events sources (gator_events.h, generated by gator_events.sh)
893         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
894                 if (gator_events_list[i])
895                         gator_events_list[i]();
897         gator_trace_power_init();
899         return 0;
902 static void gator_exit(void)
904         struct gator_interface *gi;
906         list_for_each_entry(gi, &gator_events, list)
907                 if (gi->shutdown)
908                         gi->shutdown();
911 static int gator_start(void)
913         unsigned long cpu, i;
914         struct gator_interface *gi;
916         if (gator_migrate_start())
917                 goto migrate_failure;
919         // Initialize the buffer with the frame type and core
920         for_each_present_cpu(cpu) {
921                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
922                         marshal_frame(cpu, i);
923                 }
924                 per_cpu(last_timestamp, cpu) = 0;
925         }
926         printed_monotonic_warning = false;
928         // Capture the start time
929         gator_summary();
931         // start all events
932         list_for_each_entry(gi, &gator_events, list) {
933                 if (gi->start && gi->start() != 0) {
934                         struct list_head *ptr = gi->list.prev;
936                         while (ptr != &gator_events) {
937                                 gi = list_entry(ptr, struct gator_interface, list);
939                                 if (gi->stop)
940                                         gi->stop();
942                                 ptr = ptr->prev;
943                         }
944                         goto events_failure;
945                 }
946         }
948         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
949         if (cookies_initialize())
950                 goto cookies_failure;
951         if (gator_annotate_start())
952                 goto annotate_failure;
953         if (gator_trace_sched_start())
954                 goto sched_failure;
955         if (gator_trace_power_start())
956                 goto power_failure;
957         if (gator_trace_gpu_start())
958                 goto gpu_failure;
959         if (gator_timer_start(gator_timer_count))
960                 goto timer_failure;
961         if (gator_notifier_start())
962                 goto notifier_failure;
964         return 0;
966 notifier_failure:
967         gator_timer_stop();
968 timer_failure:
969         gator_trace_gpu_stop();
970 gpu_failure:
971         gator_trace_power_stop();
972 power_failure:
973         gator_trace_sched_stop();
974 sched_failure:
975         gator_annotate_stop();
976 annotate_failure:
977         cookies_release();
978 cookies_failure:
979         // stop all events
980         list_for_each_entry(gi, &gator_events, list)
981                 if (gi->stop)
982                         gi->stop();
983 events_failure:
984         gator_migrate_stop();
985 migrate_failure:
987         return -1;
990 static void gator_stop(void)
992         struct gator_interface *gi;
994         gator_annotate_stop();
995         gator_trace_sched_stop();
996         gator_trace_power_stop();
997         gator_trace_gpu_stop();
999         // stop all interrupt callback reads before tearing down other interfaces
1000         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1001         gator_timer_stop();
1003         // stop all events
1004         list_for_each_entry(gi, &gator_events, list)
1005                 if (gi->stop)
1006                         gi->stop();
1008         gator_migrate_stop();
1011 /******************************************************************************
1012  * Filesystem
1013  ******************************************************************************/
1014 /* fopen("buffer") */
1015 static int gator_op_setup(void)
1017         int err = 0;
1018         int cpu, i;
1020         mutex_lock(&start_mutex);
1022         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1023         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1025         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1026         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1028         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1029         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1031         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1032         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1034         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1035         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1037         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1038         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1040         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1041         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1043         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1044         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1046         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1047         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1049         // Initialize percpu per buffer variables
1050         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1051                 // Verify buffers are a power of 2
1052                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1053                         err = -ENOEXEC;
1054                         goto setup_error;
1055                 }
1057                 for_each_present_cpu(cpu) {
1058                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1059                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1060                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1061                         per_cpu(buffer_space_available, cpu)[i] = true;
1062 #if GATOR_LIVE
1063                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1064 #endif
1066                         // Annotation is a special case that only uses a single buffer
1067                         if (cpu > 0 && i == ANNOTATE_BUF) {
1068                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1069                                 continue;
1070                         }
1072                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1073                         if (!per_cpu(gator_buffer, cpu)[i]) {
1074                                 err = -ENOMEM;
1075                                 goto setup_error;
1076                         }
1077                 }
1078         }
1080 setup_error:
1081         mutex_unlock(&start_mutex);
1082         return err;
1085 /* Actually start profiling (echo 1>/dev/gator/enable) */
1086 static int gator_op_start(void)
1088         int err = 0;
1090         mutex_lock(&start_mutex);
1092         if (gator_started || gator_start())
1093                 err = -EINVAL;
1094         else
1095                 gator_started = 1;
1097         mutex_unlock(&start_mutex);
1099         return err;
1102 /* echo 0>/dev/gator/enable */
1103 static void gator_op_stop(void)
1105         mutex_lock(&start_mutex);
1107         if (gator_started) {
1108                 gator_stop();
1110                 mutex_lock(&gator_buffer_mutex);
1112                 gator_started = 0;
1113                 gator_monotonic_started = 0;
1114                 cookies_release();
1115                 wake_up(&gator_buffer_wait);
1117                 mutex_unlock(&gator_buffer_mutex);
1118         }
1120         mutex_unlock(&start_mutex);
1123 static void gator_shutdown(void)
1125         int cpu, i;
1127         mutex_lock(&start_mutex);
1129         for_each_present_cpu(cpu) {
1130                 mutex_lock(&gator_buffer_mutex);
1131                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1132                         vfree(per_cpu(gator_buffer, cpu)[i]);
1133                         per_cpu(gator_buffer, cpu)[i] = NULL;
1134                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1135                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1136                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1137                         per_cpu(buffer_space_available, cpu)[i] = true;
1138 #if GATOR_LIVE
1139                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1140 #endif
1141                 }
1142                 mutex_unlock(&gator_buffer_mutex);
1143         }
1145         memset(&sent_core_name, 0, sizeof(sent_core_name));
1147         mutex_unlock(&start_mutex);
1150 static int gator_set_backtrace(unsigned long val)
1152         int err = 0;
1154         mutex_lock(&start_mutex);
1156         if (gator_started)
1157                 err = -EBUSY;
1158         else
1159                 gator_backtrace_depth = val;
1161         mutex_unlock(&start_mutex);
1163         return err;
1166 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1168         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1171 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1173         unsigned long val;
1174         int retval;
1176         if (*offset)
1177                 return -EINVAL;
1179         retval = gatorfs_ulong_from_user(&val, buf, count);
1180         if (retval)
1181                 return retval;
1183         if (val)
1184                 retval = gator_op_start();
1185         else
1186                 gator_op_stop();
1188         if (retval)
1189                 return retval;
1190         return count;
1193 static const struct file_operations enable_fops = {
1194         .read = enable_read,
1195         .write = enable_write,
1196 };
1198 static int userspace_buffer_open(struct inode *inode, struct file *file)
1200         int err = -EPERM;
1202         if (!capable(CAP_SYS_ADMIN))
1203                 return -EPERM;
1205         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1206                 return -EBUSY;
1208         if ((err = gator_op_setup()))
1209                 goto fail;
1211         /* NB: the actual start happens from userspace
1212          * echo 1 >/dev/gator/enable
1213          */
1215         return 0;
1217 fail:
1218         __clear_bit_unlock(0, &gator_buffer_opened);
1219         return err;
1222 static int userspace_buffer_release(struct inode *inode, struct file *file)
1224         gator_op_stop();
1225         gator_shutdown();
1226         __clear_bit_unlock(0, &gator_buffer_opened);
1227         return 0;
1230 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
1231                                      size_t count, loff_t *offset)
1233         int retval = -EINVAL;
1234         int commit = 0, length1, length2, read;
1235         char *buffer1;
1236         char *buffer2 = NULL;
1237         int cpu, buftype;
1239         /* do not handle partial reads */
1240         if (count != userspace_buffer_size || *offset)
1241                 return -EINVAL;
1243         // sleep until the condition is true or a signal is received
1244         // the condition is checked each time gator_buffer_wait is woken up
1245         buftype = cpu = -1;
1246         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1248         if (signal_pending(current))
1249                 return -EINTR;
1251         length2 = 0;
1252         retval = -EFAULT;
1254         mutex_lock(&gator_buffer_mutex);
1256         if (buftype == -1 || cpu == -1) {
1257                 retval = 0;
1258                 goto out;
1259         }
1261         read = per_cpu(gator_buffer_read, cpu)[buftype];
1262         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1264         /* May happen if the buffer is freed during pending reads. */
1265         if (!per_cpu(gator_buffer, cpu)[buftype]) {
1266                 retval = -EFAULT;
1267                 goto out;
1268         }
1270         /* determine the size of two halves */
1271         length1 = commit - read;
1272         buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1273         buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1274         if (length1 < 0) {
1275                 length1 = gator_buffer_size[buftype] - read;
1276                 length2 = commit;
1277         }
1279         /* start, middle or end */
1280         if (length1 > 0) {
1281                 if (copy_to_user(&buf[0], buffer1, length1)) {
1282                         goto out;
1283                 }
1284         }
1286         /* possible wrap around */
1287         if (length2 > 0) {
1288                 if (copy_to_user(&buf[length1], buffer2, length2)) {
1289                         goto out;
1290                 }
1291         }
1293         per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1294         retval = length1 + length2;
1296         /* kick just in case we've lost an SMP event */
1297         wake_up(&gator_buffer_wait);
1299         // Wake up annotate_write if more space is available
1300         if (buftype == ANNOTATE_BUF) {
1301                 wake_up(&gator_annotate_wait);
1302         }
1304 out:
1305         mutex_unlock(&gator_buffer_mutex);
1306         return retval;
1309 const struct file_operations gator_event_buffer_fops = {
1310         .open = userspace_buffer_open,
1311         .release = userspace_buffer_release,
1312         .read = userspace_buffer_read,
1313 };
1315 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1317         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1320 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1322         unsigned long val;
1323         int retval;
1325         if (*offset)
1326                 return -EINVAL;
1328         retval = gatorfs_ulong_from_user(&val, buf, count);
1329         if (retval)
1330                 return retval;
1332         retval = gator_set_backtrace(val);
1334         if (retval)
1335                 return retval;
1336         return count;
1339 static const struct file_operations depth_fops = {
1340         .read = depth_read,
1341         .write = depth_write
1342 };
1344 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1346         struct dentry *dir;
1347         struct gator_interface *gi;
1348         int cpu;
1350         /* reinitialize default values */
1351         gator_cpu_cores = 0;
1352         for_each_present_cpu(cpu) {
1353                 gator_cpu_cores++;
1354         }
1355         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1356         gator_response_type = 1;
1357         gator_live_rate = 0;
1359         gatorfs_create_file(sb, root, "enable", &enable_fops);
1360         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1361         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1362         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1363         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1364         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1365         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1366         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1367         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1368         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1370         // Annotate interface
1371         gator_annotate_create_files(sb, root);
1373         // Linux Events
1374         dir = gatorfs_mkdir(sb, root, "events");
1375         list_for_each_entry(gi, &gator_events, list)
1376                 if (gi->create_files)
1377                         gi->create_files(sb, dir);
1379         // Power interface
1380         gator_trace_power_create_files(sb, dir);
1383 /******************************************************************************
1384  * Module
1385  ******************************************************************************/
1386 static int __init gator_module_init(void)
1388         if (gatorfs_register()) {
1389                 return -1;
1390         }
1392         if (gator_init()) {
1393                 gatorfs_unregister();
1394                 return -1;
1395         }
1397         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1399         return 0;
1402 static void __exit gator_module_exit(void)
1404         del_timer_sync(&gator_buffer_wake_up_timer);
1405         tracepoint_synchronize_unregister();
1406         gator_exit();
1407         gatorfs_unregister();
1410 module_init(gator_module_init);
1411 module_exit(gator_module_exit);
1413 MODULE_LICENSE("GPL");
1414 MODULE_AUTHOR("ARM Ltd");
1415 MODULE_DESCRIPTION("Gator system profiler");