e0db6559669715670fa66305ed0cf9a969ae584c
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 16;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
49 #endif
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
56 #endif
57 #endif
59 /******************************************************************************
60  * DEFINES
61  ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE       (1*1024)
63 #define BACKTRACE_BUFFER_SIZE     (128*1024)
64 #define NAME_BUFFER_SIZE          (64*1024)
65 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE      (128*1024)    // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
72 #define NO_COOKIE      0U
73 #define UNRESOLVED_COOKIE ~0U
75 #define FRAME_SUMMARY       1
76 #define FRAME_BACKTRACE     2
77 #define FRAME_NAME          3
78 #define FRAME_COUNTER       4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE      6
81 #define FRAME_SCHED_TRACE   7
82 #define FRAME_GPU_TRACE     8
83 #define FRAME_IDLE          9
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE      1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME   3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP  2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT   2
96 #define MESSAGE_SCHED_START  3
98 #define MESSAGE_IDLE_ENTER 1
99 #define MESSAGE_IDLE_EXIT 2
101 #define MAXSIZE_PACK32     5
102 #define MAXSIZE_PACK64    10
104 #define FRAME_HEADER_SIZE 3
106 #if defined(__arm__)
107 #define PC_REG regs->ARM_pc
108 #elif defined(__aarch64__)
109 #define PC_REG regs->pc
110 #else
111 #define PC_REG regs->ip
112 #endif
114 enum {
115         SUMMARY_BUF,
116         BACKTRACE_BUF,
117         NAME_BUF,
118         COUNTER_BUF,
119         BLOCK_COUNTER_BUF,
120         ANNOTATE_BUF,
121         SCHED_TRACE_BUF,
122         GPU_TRACE_BUF,
123         IDLE_BUF,
124         NUM_GATOR_BUFS
125 };
127 /******************************************************************************
128  * Globals
129  ******************************************************************************/
130 static unsigned long gator_cpu_cores;
131 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
132 static unsigned long userspace_buffer_size;
133 static unsigned long gator_backtrace_depth;
134 // How often to commit the buffers for live in nanoseconds
135 static u64 gator_live_rate;
137 static unsigned long gator_started;
138 static u64 gator_monotonic_started;
139 static unsigned long gator_buffer_opened;
140 static unsigned long gator_timer_count;
141 static unsigned long gator_response_type;
142 static DEFINE_MUTEX(start_mutex);
143 static DEFINE_MUTEX(gator_buffer_mutex);
145 bool event_based_sampling;
147 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
148 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
149 static struct timer_list gator_buffer_wake_up_timer;
150 static LIST_HEAD(gator_events);
152 static DEFINE_PER_CPU(u64, last_timestamp);
154 static bool printed_monotonic_warning;
156 static bool sent_core_name[NR_CPUS];
158 /******************************************************************************
159  * Prototypes
160  ******************************************************************************/
161 static void buffer_check(int cpu, int buftype, u64 time);
162 static void gator_commit_buffer(int cpu, int buftype, u64 time);
163 static int buffer_bytes_available(int cpu, int buftype);
164 static bool buffer_check_space(int cpu, int buftype, int bytes);
165 static int contiguous_space_available(int cpu, int bufytpe);
166 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
167 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
168 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
169 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
170 static void gator_add_trace(int cpu, unsigned long address);
171 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
172 static u64 gator_get_time(void);
174 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
175 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
176 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
177 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
178 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
179 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
180 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
181 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
182 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
183 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
184 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
185 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
186 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
187 // The buffer. Allocated in gator_op_setup
188 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 // The time after which the buffer should be committed for live display
190 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 /******************************************************************************
193  * Application Includes
194  ******************************************************************************/
195 #include "gator_marshaling.c"
196 #include "gator_hrtimer_perf.c"
197 #include "gator_hrtimer_gator.c"
198 #include "gator_cookies.c"
199 #include "gator_annotate.c"
200 #include "gator_trace_sched.c"
201 #include "gator_trace_power.c"
202 #include "gator_trace_gpu.c"
203 #include "gator_backtrace.c"
204 #include "gator_fs.c"
205 #include "gator_pack.c"
207 /******************************************************************************
208  * Misc
209  ******************************************************************************/
211 const struct gator_cpu gator_cpus[] = {
212         {
213                 .cpuid = ARM1136,
214                 .core_name = "ARM1136",
215                 .pmnc_name = "ARM_ARM11",
216                 .dt_name = "arm,arm1136",
217                 .pmnc_counters = 3,
218         },
219         {
220                 .cpuid = ARM1156,
221                 .core_name = "ARM1156",
222                 .pmnc_name = "ARM_ARM11",
223                 .dt_name = "arm,arm1156",
224                 .pmnc_counters = 3,
225         },
226         {
227                 .cpuid = ARM1176,
228                 .core_name = "ARM1176",
229                 .pmnc_name = "ARM_ARM11",
230                 .dt_name = "arm,arm1176",
231                 .pmnc_counters = 3,
232         },
233         {
234                 .cpuid = ARM11MPCORE,
235                 .core_name = "ARM11MPCore",
236                 .pmnc_name = "ARM_ARM11MPCore",
237                 .dt_name = "arm,arm11mpcore",
238                 .pmnc_counters = 3,
239         },
240         {
241                 .cpuid = CORTEX_A5,
242                 .core_name = "Cortex-A5",
243                 .pmu_name = "ARMv7_Cortex_A5",
244                 .pmnc_name = "ARM_Cortex-A5",
245                 .dt_name = "arm,cortex-a5",
246                 .pmnc_counters = 2,
247         },
248         {
249                 .cpuid = CORTEX_A7,
250                 .core_name = "Cortex-A7",
251                 .pmu_name = "ARMv7_Cortex_A7",
252                 .pmnc_name = "ARM_Cortex-A7",
253                 .dt_name = "arm,cortex-a7",
254                 .pmnc_counters = 4,
255         },
256         {
257                 .cpuid = CORTEX_A8,
258                 .core_name = "Cortex-A8",
259                 .pmu_name = "ARMv7_Cortex_A8",
260                 .pmnc_name = "ARM_Cortex-A8",
261                 .dt_name = "arm,cortex-a8",
262                 .pmnc_counters = 4,
263         },
264         {
265                 .cpuid = CORTEX_A9,
266                 .core_name = "Cortex-A9",
267                 .pmu_name = "ARMv7_Cortex_A9",
268                 .pmnc_name = "ARM_Cortex-A9",
269                 .dt_name = "arm,cortex-a9",
270                 .pmnc_counters = 6,
271         },
272         {
273                 .cpuid = CORTEX_A12,
274                 .core_name = "Cortex-A12",
275                 .pmu_name = "ARMv7_Cortex_A12",
276                 .pmnc_name = "ARM_Cortex-A12",
277                 .dt_name = "arm,cortex-a12",
278                 .pmnc_counters = 6,
279         },
280         {
281                 .cpuid = CORTEX_A15,
282                 .core_name = "Cortex-A15",
283                 .pmu_name = "ARMv7_Cortex_A15",
284                 .pmnc_name = "ARM_Cortex-A15",
285                 .dt_name = "arm,cortex-a15",
286                 .pmnc_counters = 6,
287         },
288         {
289                 .cpuid = SCORPION,
290                 .core_name = "Scorpion",
291                 .pmnc_name = "Scorpion",
292                 .pmnc_counters = 4,
293         },
294         {
295                 .cpuid = SCORPIONMP,
296                 .core_name = "ScorpionMP",
297                 .pmnc_name = "ScorpionMP",
298                 .pmnc_counters = 4,
299         },
300         {
301                 .cpuid = KRAITSIM,
302                 .core_name = "KraitSIM",
303                 .pmnc_name = "Krait",
304                 .pmnc_counters = 4,
305         },
306         {
307                 .cpuid = KRAIT,
308                 .core_name = "Krait",
309                 .pmnc_name = "Krait",
310                 .pmnc_counters = 4,
311         },
312         {
313                 .cpuid = KRAIT_S4_PRO,
314                 .core_name = "Krait S4 Pro",
315                 .pmnc_name = "Krait",
316                 .pmnc_counters = 4,
317         },
318         {
319                 .cpuid = CORTEX_A53,
320                 .core_name = "Cortex-A53",
321                 .pmnc_name = "ARM_Cortex-A53",
322                 .dt_name = "arm,cortex-a53",
323                 .pmnc_counters = 6,
324         },
325         {
326                 .cpuid = CORTEX_A57,
327                 .core_name = "Cortex-A57",
328                 .pmnc_name = "ARM_Cortex-A57",
329                 .dt_name = "arm,cortex-a57",
330                 .pmnc_counters = 6,
331         },
332         {
333                 .cpuid = AARCH64,
334                 .core_name = "AArch64",
335                 .pmnc_name = "ARM_AArch64",
336                 .pmnc_counters = 6,
337         },
338         {
339                 .cpuid = OTHER,
340                 .core_name = "Other",
341                 .pmnc_name = "Other",
342                 .pmnc_counters = 6,
343         },
344         {}
345 };
347 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
349         int i;
351         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
352                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
353                 if (gator_cpu->cpuid == cpuid) {
354                         return gator_cpu;
355                 }
356         }
358         return NULL;
361 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
363         int i;
365         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
366                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
367                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
368                         return gator_cpu;
369                 }
370         }
372         return NULL;
375 u32 gator_cpuid(void)
377 #if defined(__arm__) || defined(__aarch64__)
378         u32 val;
379 #if !defined(__aarch64__)
380         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
381 #else
382         asm volatile("mrs %0, midr_el1" : "=r" (val));
383 #endif
384         return (val >> 4) & 0xfff;
385 #else
386         return OTHER;
387 #endif
390 static void gator_buffer_wake_up(unsigned long data)
392         wake_up(&gator_buffer_wait);
395 /******************************************************************************
396  * Commit interface
397  ******************************************************************************/
398 static bool buffer_commit_ready(int *cpu, int *buftype)
400         int cpu_x, x;
401         for_each_present_cpu(cpu_x) {
402                 for (x = 0; x < NUM_GATOR_BUFS; x++)
403                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
404                                 *cpu = cpu_x;
405                                 *buftype = x;
406                                 return true;
407                         }
408         }
409         *cpu = -1;
410         *buftype = -1;
411         return false;
414 /******************************************************************************
415  * Buffer management
416  ******************************************************************************/
417 static int buffer_bytes_available(int cpu, int buftype)
419         int remaining, filled;
421         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
422         if (filled < 0) {
423                 filled += gator_buffer_size[buftype];
424         }
426         remaining = gator_buffer_size[buftype] - filled;
428         if (per_cpu(buffer_space_available, cpu)[buftype]) {
429                 // Give some extra room; also allows space to insert the overflow error packet
430                 remaining -= 200;
431         } else {
432                 // Hysteresis, prevents multiple overflow messages
433                 remaining -= 2000;
434         }
436         return remaining;
439 static int contiguous_space_available(int cpu, int buftype)
441         int remaining = buffer_bytes_available(cpu, buftype);
442         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
443         if (remaining < contiguous)
444                 return remaining;
445         else
446                 return contiguous;
449 static bool buffer_check_space(int cpu, int buftype, int bytes)
451         int remaining = buffer_bytes_available(cpu, buftype);
453         if (remaining < bytes) {
454                 per_cpu(buffer_space_available, cpu)[buftype] = false;
455         } else {
456                 per_cpu(buffer_space_available, cpu)[buftype] = true;
457         }
459         return per_cpu(buffer_space_available, cpu)[buftype];
462 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
464         int i;
465         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
466         u32 mask = gator_buffer_mask[buftype];
467         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
469         for (i = 0; i < len; i++) {
470                 buffer[write] = x[i];
471                 write = (write + 1) & mask;
472         }
474         per_cpu(gator_buffer_write, cpu)[buftype] = write;
477 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
479         int len = strlen(x);
480         gator_buffer_write_packed_int(cpu, buftype, len);
481         gator_buffer_write_bytes(cpu, buftype, x, len);
484 static void gator_commit_buffer(int cpu, int buftype, u64 time)
486         int type_length, commit, length, byte;
488         if (!per_cpu(gator_buffer, cpu)[buftype])
489                 return;
491         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
492         type_length = gator_response_type ? 1 : 0;
493         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
494         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
495         if (length < 0) {
496                 length += gator_buffer_size[buftype];
497         }
498         length = length - type_length - sizeof(s32);
500         if (length <= FRAME_HEADER_SIZE) {
501                 // Nothing to write, only the frame header is present
502                 return;
503         }
505         for (byte = 0; byte < sizeof(s32); byte++) {
506                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
507         }
509         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
511         if (gator_live_rate > 0) {
512                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
513                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
514                 }
515         }
517         marshal_frame(cpu, buftype);
519         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
520         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
523 static void buffer_check(int cpu, int buftype, u64 time)
525         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
526         if (filled < 0) {
527                 filled += gator_buffer_size[buftype];
528         }
529         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
530                 gator_commit_buffer(cpu, buftype, time);
531         }
534 static void gator_add_trace(int cpu, unsigned long address)
536         off_t offset = 0;
537         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
539         if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
540                 offset = address;
541         }
543         marshal_backtrace(offset & ~1, cookie);
546 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
548         bool inKernel;
549         unsigned long exec_cookie;
551         if (!regs)
552                 return;
554         inKernel = !user_mode(regs);
555         exec_cookie = get_exec_cookie(cpu, current);
557         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
558                 return;
560         if (inKernel) {
561                 kernel_backtrace(cpu, regs);
562         } else {
563                 // Cookie+PC
564                 gator_add_trace(cpu, PC_REG);
566                 // Backtrace
567                 if (gator_backtrace_depth)
568                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
569         }
571         marshal_backtrace_footer(time);
574 /******************************************************************************
575  * hrtimer interrupt processing
576  ******************************************************************************/
577 static void gator_timer_interrupt(void)
579         struct pt_regs *const regs = get_irq_regs();
580         gator_backtrace_handler(regs);
583 void gator_backtrace_handler(struct pt_regs *const regs)
585         u64 time = gator_get_time();
586         int cpu = get_physical_cpu();
588         // Output backtrace
589         gator_add_sample(cpu, regs, time);
591         // Collect counters
592         if (!per_cpu(collecting, cpu)) {
593                 collect_counters(time);
594         }
597 static int gator_running;
599 // This function runs in interrupt context and on the appropriate core
600 static void gator_timer_offline(void *migrate)
602         struct gator_interface *gi;
603         int i, len, cpu = get_physical_cpu();
604         int *buffer;
605         u64 time;
607         gator_trace_sched_offline();
608         gator_trace_power_offline();
610         if (!migrate) {
611                 gator_hrtimer_offline();
612         }
614         // Offline any events and output counters
615         time = gator_get_time();
616         if (marshal_event_header(time)) {
617                 list_for_each_entry(gi, &gator_events, list) {
618                         if (gi->offline) {
619                                 len = gi->offline(&buffer, migrate);
620                                 marshal_event(len, buffer);
621                         }
622                 }
623                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
624                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
625         }
627         // Flush all buffers on this core
628         for (i = 0; i < NUM_GATOR_BUFS; i++)
629                 gator_commit_buffer(cpu, i, time);
632 // This function runs in interrupt context and may be running on a core other than core 'cpu'
633 static void gator_timer_offline_dispatch(int cpu, bool migrate)
635         struct gator_interface *gi;
637         list_for_each_entry(gi, &gator_events, list) {
638                 if (gi->offline_dispatch) {
639                         gi->offline_dispatch(cpu, migrate);
640                 }
641         }
644 static void gator_timer_stop(void)
646         int cpu;
648         if (gator_running) {
649                 on_each_cpu(gator_timer_offline, NULL, 1);
650                 for_each_online_cpu(cpu) {
651                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
652                 }
654                 gator_running = 0;
655                 gator_hrtimer_shutdown();
656         }
659 #if defined(__arm__) || defined(__aarch64__)
660 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
661         const char *core_name = NULL;
662         char core_name_buf[32];
664         if (!sent_core_name[cpu]) {
665                 if (gator_cpu != NULL) {
666                         core_name = gator_cpu->core_name;
667                 } else {
668                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
669                         core_name = core_name_buf;
670                 }
672                 marshal_core_name(cpu, cpuid, core_name);
673                 sent_core_name[cpu] = true;
674         }
676 #endif
678 // This function runs in interrupt context and on the appropriate core
679 static void gator_timer_online(void *migrate)
681         struct gator_interface *gi;
682         int len, cpu = get_physical_cpu();
683         int *buffer;
684         u64 time;
686         gator_trace_power_online();
688         // online any events and output counters
689         time = gator_get_time();
690         if (marshal_event_header(time)) {
691                 list_for_each_entry(gi, &gator_events, list) {
692                         if (gi->online) {
693                                 len = gi->online(&buffer, migrate);
694                                 marshal_event(len, buffer);
695                         }
696                 }
697                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
698                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
699         }
701         if (!migrate) {
702                 gator_hrtimer_online();
703         }
705 #if defined(__arm__) || defined(__aarch64__)
706         if (!sent_core_name[cpu]) {
707                 const u32 cpuid = gator_cpuid();
708                 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
709         }
710 #endif
713 // This function runs in interrupt context and may be running on a core other than core 'cpu'
714 static void gator_timer_online_dispatch(int cpu, bool migrate)
716         struct gator_interface *gi;
718         list_for_each_entry(gi, &gator_events, list) {
719                 if (gi->online_dispatch) {
720                         gi->online_dispatch(cpu, migrate);
721                 }
722         }
725 #include "gator_iks.c"
727 int gator_timer_start(unsigned long sample_rate)
729         int cpu;
731         if (gator_running) {
732                 pr_notice("gator: already running\n");
733                 return 0;
734         }
736         gator_running = 1;
738         // event based sampling trumps hr timer based sampling
739         if (event_based_sampling) {
740                 sample_rate = 0;
741         }
743         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
744                 return -1;
746         gator_send_iks_core_names();
747         for_each_online_cpu(cpu) {
748                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
749         }
750         on_each_cpu(gator_timer_online, NULL, 1);
752         return 0;
755 static u64 gator_get_time(void)
757         struct timespec ts;
758         u64 timestamp;
759         u64 prev_timestamp;
760         u64 delta;
761         int cpu = smp_processor_id();
763         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
764         getrawmonotonic(&ts);
765         timestamp = timespec_to_ns(&ts);
767         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
768         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
769         // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
770         prev_timestamp = per_cpu(last_timestamp, cpu);
771         if (prev_timestamp <= timestamp) {
772                 per_cpu(last_timestamp, cpu) = timestamp;
773         } else {
774                 delta = prev_timestamp - timestamp;
775                 // Log the error once
776                 if (!printed_monotonic_warning && delta > 500000) {
777                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
778                         printed_monotonic_warning = true;
779                 }
780                 timestamp = prev_timestamp;
781         }
783         return timestamp - gator_monotonic_started;
786 /******************************************************************************
787  * cpu hotplug and pm notifiers
788  ******************************************************************************/
789 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
791         int cpu = lcpu_to_pcpu((long)hcpu);
793         switch (action) {
794         case CPU_DOWN_PREPARE:
795         case CPU_DOWN_PREPARE_FROZEN:
796                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
797                 gator_timer_offline_dispatch(cpu, false);
798                 break;
799         case CPU_ONLINE:
800         case CPU_ONLINE_FROZEN:
801                 gator_timer_online_dispatch(cpu, false);
802                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
803                 break;
804         }
806         return NOTIFY_OK;
809 static struct notifier_block __refdata gator_hotcpu_notifier = {
810         .notifier_call = gator_hotcpu_notify,
811 };
813 // n.b. calling "on_each_cpu" only runs on those that are online
814 // Registered linux events are not disabled, so their counters will continue to collect
815 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
817         int cpu;
819         switch (event) {
820         case PM_HIBERNATION_PREPARE:
821         case PM_SUSPEND_PREPARE:
822                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
823                 unregister_scheduler_tracepoints();
824                 on_each_cpu(gator_timer_offline, NULL, 1);
825                 for_each_online_cpu(cpu) {
826                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
827                 }
828                 break;
829         case PM_POST_HIBERNATION:
830         case PM_POST_SUSPEND:
831                 for_each_online_cpu(cpu) {
832                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
833                 }
834                 on_each_cpu(gator_timer_online, NULL, 1);
835                 register_scheduler_tracepoints();
836                 register_hotcpu_notifier(&gator_hotcpu_notifier);
837                 break;
838         }
840         return NOTIFY_OK;
843 static struct notifier_block gator_pm_notifier = {
844         .notifier_call = gator_pm_notify,
845 };
847 static int gator_notifier_start(void)
849         int retval;
850         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
851         if (retval == 0)
852                 retval = register_pm_notifier(&gator_pm_notifier);
853         return retval;
856 static void gator_notifier_stop(void)
858         unregister_pm_notifier(&gator_pm_notifier);
859         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
862 /******************************************************************************
863  * Main
864  ******************************************************************************/
865 static void gator_summary(void)
867         u64 timestamp, uptime;
868         struct timespec ts;
869         char uname_buf[512];
870         void (*m2b)(struct timespec *ts);
871         unsigned long flags;
873         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
875         getnstimeofday(&ts);
876         timestamp = timespec_to_ns(&ts);
878         do_posix_clock_monotonic_gettime(&ts);
879         // monotonic_to_bootbased is not defined for some versions of Android
880         m2b = symbol_get(monotonic_to_bootbased);
881         if (m2b) {
882                 m2b(&ts);
883         }
884         uptime = timespec_to_ns(&ts);
886         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
887         local_irq_save(flags);
888         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
889         gator_monotonic_started = 0;
890         gator_monotonic_started = gator_get_time();
891         local_irq_restore(flags);
893         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
896 int gator_events_install(struct gator_interface *interface)
898         list_add_tail(&interface->list, &gator_events);
900         return 0;
903 int gator_events_get_key(void)
905         // key of zero is reserved as a timestamp
906         static int key = 1;
908         const int ret = key;
909         key += 2;
910         return ret;
913 static int gator_init(void)
915         int i;
917         calc_first_cluster_size();
919         // events sources (gator_events.h, generated by gator_events.sh)
920         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
921                 if (gator_events_list[i])
922                         gator_events_list[i]();
924         gator_trace_sched_init();
925         gator_trace_power_init();
927         return 0;
930 static void gator_exit(void)
932         struct gator_interface *gi;
934         list_for_each_entry(gi, &gator_events, list)
935                 if (gi->shutdown)
936                         gi->shutdown();
939 static int gator_start(void)
941         unsigned long cpu, i;
942         struct gator_interface *gi;
944         if (gator_migrate_start())
945                 goto migrate_failure;
947         // Initialize the buffer with the frame type and core
948         for_each_present_cpu(cpu) {
949                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
950                         marshal_frame(cpu, i);
951                 }
952                 per_cpu(last_timestamp, cpu) = 0;
953         }
954         printed_monotonic_warning = false;
956         // Capture the start time
957         gator_summary();
959         // start all events
960         list_for_each_entry(gi, &gator_events, list) {
961                 if (gi->start && gi->start() != 0) {
962                         struct list_head *ptr = gi->list.prev;
964                         while (ptr != &gator_events) {
965                                 gi = list_entry(ptr, struct gator_interface, list);
967                                 if (gi->stop)
968                                         gi->stop();
970                                 ptr = ptr->prev;
971                         }
972                         goto events_failure;
973                 }
974         }
976         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
977         if (cookies_initialize())
978                 goto cookies_failure;
979         if (gator_annotate_start())
980                 goto annotate_failure;
981         if (gator_trace_sched_start())
982                 goto sched_failure;
983         if (gator_trace_power_start())
984                 goto power_failure;
985         if (gator_trace_gpu_start())
986                 goto gpu_failure;
987         if (gator_timer_start(gator_timer_count))
988                 goto timer_failure;
989         if (gator_notifier_start())
990                 goto notifier_failure;
992         return 0;
994 notifier_failure:
995         gator_timer_stop();
996 timer_failure:
997         gator_trace_gpu_stop();
998 gpu_failure:
999         gator_trace_power_stop();
1000 power_failure:
1001         gator_trace_sched_stop();
1002 sched_failure:
1003         gator_annotate_stop();
1004 annotate_failure:
1005         cookies_release();
1006 cookies_failure:
1007         // stop all events
1008         list_for_each_entry(gi, &gator_events, list)
1009                 if (gi->stop)
1010                         gi->stop();
1011 events_failure:
1012         gator_migrate_stop();
1013 migrate_failure:
1015         return -1;
1018 static void gator_stop(void)
1020         struct gator_interface *gi;
1022         gator_annotate_stop();
1023         gator_trace_sched_stop();
1024         gator_trace_power_stop();
1025         gator_trace_gpu_stop();
1027         // stop all interrupt callback reads before tearing down other interfaces
1028         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1029         gator_timer_stop();
1031         // stop all events
1032         list_for_each_entry(gi, &gator_events, list)
1033                 if (gi->stop)
1034                         gi->stop();
1036         gator_migrate_stop();
1039 /******************************************************************************
1040  * Filesystem
1041  ******************************************************************************/
1042 /* fopen("buffer") */
1043 static int gator_op_setup(void)
1045         int err = 0;
1046         int cpu, i;
1048         mutex_lock(&start_mutex);
1050         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1051         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1053         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1054         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1056         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1057         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1059         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1060         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1062         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1063         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1065         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1066         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1068         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1069         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1071         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1072         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1074         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1075         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1077         // Initialize percpu per buffer variables
1078         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1079                 // Verify buffers are a power of 2
1080                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1081                         err = -ENOEXEC;
1082                         goto setup_error;
1083                 }
1085                 for_each_present_cpu(cpu) {
1086                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1087                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1088                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1089                         per_cpu(buffer_space_available, cpu)[i] = true;
1090                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1092                         // Annotation is a special case that only uses a single buffer
1093                         if (cpu > 0 && i == ANNOTATE_BUF) {
1094                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1095                                 continue;
1096                         }
1098                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1099                         if (!per_cpu(gator_buffer, cpu)[i]) {
1100                                 err = -ENOMEM;
1101                                 goto setup_error;
1102                         }
1103                 }
1104         }
1106 setup_error:
1107         mutex_unlock(&start_mutex);
1108         return err;
1111 /* Actually start profiling (echo 1>/dev/gator/enable) */
1112 static int gator_op_start(void)
1114         int err = 0;
1116         mutex_lock(&start_mutex);
1118         if (gator_started || gator_start())
1119                 err = -EINVAL;
1120         else
1121                 gator_started = 1;
1123         mutex_unlock(&start_mutex);
1125         return err;
1128 /* echo 0>/dev/gator/enable */
1129 static void gator_op_stop(void)
1131         mutex_lock(&start_mutex);
1133         if (gator_started) {
1134                 gator_stop();
1136                 mutex_lock(&gator_buffer_mutex);
1138                 gator_started = 0;
1139                 gator_monotonic_started = 0;
1140                 cookies_release();
1141                 wake_up(&gator_buffer_wait);
1143                 mutex_unlock(&gator_buffer_mutex);
1144         }
1146         mutex_unlock(&start_mutex);
1149 static void gator_shutdown(void)
1151         int cpu, i;
1153         mutex_lock(&start_mutex);
1155         for_each_present_cpu(cpu) {
1156                 mutex_lock(&gator_buffer_mutex);
1157                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1158                         vfree(per_cpu(gator_buffer, cpu)[i]);
1159                         per_cpu(gator_buffer, cpu)[i] = NULL;
1160                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1161                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1162                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1163                         per_cpu(buffer_space_available, cpu)[i] = true;
1164                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1165                 }
1166                 mutex_unlock(&gator_buffer_mutex);
1167         }
1169         memset(&sent_core_name, 0, sizeof(sent_core_name));
1171         mutex_unlock(&start_mutex);
1174 static int gator_set_backtrace(unsigned long val)
1176         int err = 0;
1178         mutex_lock(&start_mutex);
1180         if (gator_started)
1181                 err = -EBUSY;
1182         else
1183                 gator_backtrace_depth = val;
1185         mutex_unlock(&start_mutex);
1187         return err;
1190 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1192         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1195 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1197         unsigned long val;
1198         int retval;
1200         if (*offset)
1201                 return -EINVAL;
1203         retval = gatorfs_ulong_from_user(&val, buf, count);
1204         if (retval)
1205                 return retval;
1207         if (val)
1208                 retval = gator_op_start();
1209         else
1210                 gator_op_stop();
1212         if (retval)
1213                 return retval;
1214         return count;
1217 static const struct file_operations enable_fops = {
1218         .read = enable_read,
1219         .write = enable_write,
1220 };
1222 static int userspace_buffer_open(struct inode *inode, struct file *file)
1224         int err = -EPERM;
1226         if (!capable(CAP_SYS_ADMIN))
1227                 return -EPERM;
1229         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1230                 return -EBUSY;
1232         if ((err = gator_op_setup()))
1233                 goto fail;
1235         /* NB: the actual start happens from userspace
1236          * echo 1 >/dev/gator/enable
1237          */
1239         return 0;
1241 fail:
1242         __clear_bit_unlock(0, &gator_buffer_opened);
1243         return err;
1246 static int userspace_buffer_release(struct inode *inode, struct file *file)
1248         gator_op_stop();
1249         gator_shutdown();
1250         __clear_bit_unlock(0, &gator_buffer_opened);
1251         return 0;
1254 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1256         int commit, length1, length2, read;
1257         char *buffer1;
1258         char *buffer2;
1259         int cpu, buftype;
1260         int written = 0;
1262         // ensure there is enough space for a whole frame
1263         if (count < userspace_buffer_size || *offset) {
1264                 return -EINVAL;
1265         }
1267         // sleep until the condition is true or a signal is received
1268         // the condition is checked each time gator_buffer_wait is woken up
1269         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1271         if (signal_pending(current)) {
1272                 return -EINTR;
1273         }
1275         if (buftype == -1 || cpu == -1) {
1276                 return 0;
1277         }
1279         mutex_lock(&gator_buffer_mutex);
1281         do {
1282                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1283                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1285                 // May happen if the buffer is freed during pending reads.
1286                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1287                         break;
1288                 }
1290                 // determine the size of two halves
1291                 length1 = commit - read;
1292                 length2 = 0;
1293                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1294                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1295                 if (length1 < 0) {
1296                         length1 = gator_buffer_size[buftype] - read;
1297                         length2 = commit;
1298                 }
1300                 if (length1 + length2 > count - written) {
1301                         break;
1302                 }
1304                 // start, middle or end
1305                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1306                         break;
1307                 }
1309                 // possible wrap around
1310                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1311                         break;
1312                 }
1314                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1315                 written += length1 + length2;
1317                 // Wake up annotate_write if more space is available
1318                 if (buftype == ANNOTATE_BUF) {
1319                         wake_up(&gator_annotate_wait);
1320                 }
1321         } while (buffer_commit_ready(&cpu, &buftype));
1323         mutex_unlock(&gator_buffer_mutex);
1325         // kick just in case we've lost an SMP event
1326         wake_up(&gator_buffer_wait);
1328         return written > 0 ? written : -EFAULT;
1331 const struct file_operations gator_event_buffer_fops = {
1332         .open = userspace_buffer_open,
1333         .release = userspace_buffer_release,
1334         .read = userspace_buffer_read,
1335 };
1337 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1339         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1342 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1344         unsigned long val;
1345         int retval;
1347         if (*offset)
1348                 return -EINVAL;
1350         retval = gatorfs_ulong_from_user(&val, buf, count);
1351         if (retval)
1352                 return retval;
1354         retval = gator_set_backtrace(val);
1356         if (retval)
1357                 return retval;
1358         return count;
1361 static const struct file_operations depth_fops = {
1362         .read = depth_read,
1363         .write = depth_write
1364 };
1366 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1368         struct dentry *dir;
1369         struct gator_interface *gi;
1370         int cpu;
1372         /* reinitialize default values */
1373         gator_cpu_cores = 0;
1374         for_each_present_cpu(cpu) {
1375                 gator_cpu_cores++;
1376         }
1377         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1378         gator_response_type = 1;
1379         gator_live_rate = 0;
1381         gatorfs_create_file(sb, root, "enable", &enable_fops);
1382         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1383         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1384         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1385         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1386         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1387         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1388         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1389         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1390         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1392         // Annotate interface
1393         gator_annotate_create_files(sb, root);
1395         // Linux Events
1396         dir = gatorfs_mkdir(sb, root, "events");
1397         list_for_each_entry(gi, &gator_events, list)
1398                 if (gi->create_files)
1399                         gi->create_files(sb, dir);
1401         // Sched Events
1402         sched_trace_create_files(sb, dir);
1404         // Power interface
1405         gator_trace_power_create_files(sb, dir);
1408 /******************************************************************************
1409  * Module
1410  ******************************************************************************/
1411 static int __init gator_module_init(void)
1413         if (gatorfs_register()) {
1414                 return -1;
1415         }
1417         if (gator_init()) {
1418                 gatorfs_unregister();
1419                 return -1;
1420         }
1422         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1424         return 0;
1427 static void __exit gator_module_exit(void)
1429         del_timer_sync(&gator_buffer_wake_up_timer);
1430         tracepoint_synchronize_unregister();
1431         gator_exit();
1432         gatorfs_unregister();
1435 module_init(gator_module_init);
1436 module_exit(gator_module_exit);
1438 MODULE_LICENSE("GPL");
1439 MODULE_AUTHOR("ARM Ltd");
1440 MODULE_DESCRIPTION("Gator system profiler");