4a1f0566fe1f97f0c9faadf006193fed945c1152
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 // This version must match the gator daemon version
11 #define PROTOCOL_VERSION 18
12 static unsigned long gator_protocol_version = PROTOCOL_VERSION;
14 #include <linux/slab.h>
15 #include <linux/cpu.h>
16 #include <linux/sched.h>
17 #include <linux/irq.h>
18 #include <linux/vmalloc.h>
19 #include <linux/hardirq.h>
20 #include <linux/highmem.h>
21 #include <linux/pagemap.h>
22 #include <linux/suspend.h>
23 #include <linux/module.h>
24 #include <linux/perf_event.h>
25 #include <linux/utsname.h>
26 #include <linux/kthread.h>
27 #include <asm/stacktrace.h>
28 #include <asm/uaccess.h>
30 #include "gator.h"
32 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
33 #error kernels prior to 2.6.32 are not supported
34 #endif
36 #if defined(MODULE) && !defined(CONFIG_MODULES)
37 #error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel.
38 #endif
40 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
41 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
42 #endif
44 #ifndef CONFIG_PROFILING
45 #error gator requires the kernel to have CONFIG_PROFILING defined
46 #endif
48 #ifndef CONFIG_HIGH_RES_TIMERS
49 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
50 #endif
52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
53 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
54 #endif
56 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
57 #ifndef CONFIG_PERF_EVENTS
58 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
59 #elif !defined CONFIG_HW_PERF_EVENTS
60 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
61 #endif
62 #endif
64 /******************************************************************************
65  * DEFINES
66  ******************************************************************************/
67 #define SUMMARY_BUFFER_SIZE       (1*1024)
68 #define BACKTRACE_BUFFER_SIZE     (128*1024)
69 #define NAME_BUFFER_SIZE          (64*1024)
70 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
71 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
72 #define ANNOTATE_BUFFER_SIZE      (128*1024)    // annotate counters have the core as part of the data and the core value in the frame header may be discarded
73 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
74 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
75 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
77 #define NO_COOKIE      0U
78 #define UNRESOLVED_COOKIE ~0U
80 #define FRAME_SUMMARY       1
81 #define FRAME_BACKTRACE     2
82 #define FRAME_NAME          3
83 #define FRAME_COUNTER       4
84 #define FRAME_BLOCK_COUNTER 5
85 #define FRAME_ANNOTATE      6
86 #define FRAME_SCHED_TRACE   7
87 #define FRAME_GPU_TRACE     8
88 #define FRAME_IDLE          9
90 #define MESSAGE_END_BACKTRACE 1
92 // Name Frame Messages
93 #define MESSAGE_COOKIE      1
94 #define MESSAGE_THREAD_NAME 2
95 #define MESSAGE_LINK        4
97 // GPU Trace Frame Messages
98 #define MESSAGE_GPU_START 1
99 #define MESSAGE_GPU_STOP  2
101 // Scheduler Trace Frame Messages
102 #define MESSAGE_SCHED_SWITCH 1
103 #define MESSAGE_SCHED_EXIT   2
104 #define MESSAGE_SCHED_START  3
106 // Idle Frame Messages
107 #define MESSAGE_IDLE_ENTER 1
108 #define MESSAGE_IDLE_EXIT  2
110 // Summary Frame Messages
111 #define MESSAGE_SUMMARY   1
112 #define MESSAGE_CORE_NAME 3
114 #define MAXSIZE_PACK32     5
115 #define MAXSIZE_PACK64    10
117 #define FRAME_HEADER_SIZE 3
119 #if defined(__arm__)
120 #define PC_REG regs->ARM_pc
121 #elif defined(__aarch64__)
122 #define PC_REG regs->pc
123 #else
124 #define PC_REG regs->ip
125 #endif
127 enum {
128         SUMMARY_BUF,
129         BACKTRACE_BUF,
130         NAME_BUF,
131         COUNTER_BUF,
132         BLOCK_COUNTER_BUF,
133         ANNOTATE_BUF,
134         SCHED_TRACE_BUF,
135         GPU_TRACE_BUF,
136         IDLE_BUF,
137         NUM_GATOR_BUFS
138 };
140 /******************************************************************************
141  * Globals
142  ******************************************************************************/
143 static unsigned long gator_cpu_cores;
144 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
145 static unsigned long userspace_buffer_size;
146 static unsigned long gator_backtrace_depth;
147 // How often to commit the buffers for live in nanoseconds
148 static u64 gator_live_rate;
150 static unsigned long gator_started;
151 static u64 gator_monotonic_started;
152 static u64 gator_hibernate_time;
153 static unsigned long gator_buffer_opened;
154 static unsigned long gator_timer_count;
155 static unsigned long gator_response_type;
156 static DEFINE_MUTEX(start_mutex);
157 static DEFINE_MUTEX(gator_buffer_mutex);
159 bool event_based_sampling;
161 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
162 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
163 static struct timer_list gator_buffer_wake_up_timer;
164 static bool gator_buffer_wake_run;
165 // Initialize semaphore unlocked to initialize memory values
166 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36)
167 static DECLARE_MUTEX(gator_buffer_wake_sem);
168 #else
169 static DEFINE_SEMAPHORE(gator_buffer_wake_sem);
170 #endif
171 static struct task_struct *gator_buffer_wake_thread;
172 static LIST_HEAD(gator_events);
174 static DEFINE_PER_CPU(u64, last_timestamp);
176 static bool printed_monotonic_warning;
178 static bool sent_core_name[NR_CPUS];
180 static DEFINE_PER_CPU(bool, in_scheduler_context);
182 /******************************************************************************
183  * Prototypes
184  ******************************************************************************/
185 static u64 gator_get_time(void);
186 static void gator_op_create_files(struct super_block *sb, struct dentry *root);
188 // gator_buffer is protected by being per_cpu and by having IRQs disabled when writing to it.
189 // Most marshal_* calls take care of this except for marshal_cookie*, marshal_backtrace* and marshal_frame where the caller is responsible for doing so.
190 // No synchronization is needed with the backtrace buffer as it is per cpu and is only used from the hrtimer.
191 // The annotate_lock must be held when using the annotation buffer as it is not per cpu.
192 // collect_counters which is the sole writer to the block counter frame is additionally protected by the per cpu collecting flag
194 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
195 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
196 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
197 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
198 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
199 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
200 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
201 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
202 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
203 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
204 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
205 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
206 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
207 // The buffer. Allocated in gator_op_setup
208 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
209 // The time after which the buffer should be committed for live display
210 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
212 // List of all gator events - new events must be added to this list
213 #define GATOR_EVENTS_LIST \
214         GATOR_EVENT(gator_events_armv6_init) \
215         GATOR_EVENT(gator_events_armv7_init) \
216         GATOR_EVENT(gator_events_block_init) \
217         GATOR_EVENT(gator_events_ccn504_init) \
218         GATOR_EVENT(gator_events_irq_init) \
219         GATOR_EVENT(gator_events_l2c310_init) \
220         GATOR_EVENT(gator_events_mali_init) \
221         GATOR_EVENT(gator_events_mali_t6xx_hw_init) \
222         GATOR_EVENT(gator_events_mali_t6xx_init) \
223         GATOR_EVENT(gator_events_meminfo_init) \
224         GATOR_EVENT(gator_events_mmapped_init) \
225         GATOR_EVENT(gator_events_net_init) \
226         GATOR_EVENT(gator_events_perf_pmu_init) \
227         GATOR_EVENT(gator_events_sched_init) \
228         GATOR_EVENT(gator_events_scorpion_init) \
230 #define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void);
231 GATOR_EVENTS_LIST
232 #undef GATOR_EVENT
234 static int (*gator_events_list[])(void) = {
235 #define GATOR_EVENT(EVENT_INIT) EVENT_INIT,
236 GATOR_EVENTS_LIST
237 #undef GATOR_EVENT
238 };
240 /******************************************************************************
241  * Application Includes
242  ******************************************************************************/
243 #include "gator_fs.c"
244 #include "gator_buffer_write.c"
245 #include "gator_buffer.c"
246 #include "gator_marshaling.c"
247 #include "gator_hrtimer_gator.c"
248 #include "gator_cookies.c"
249 #include "gator_annotate.c"
250 #include "gator_trace_sched.c"
251 #include "gator_trace_power.c"
252 #include "gator_trace_gpu.c"
253 #include "gator_backtrace.c"
255 /******************************************************************************
256  * Misc
257  ******************************************************************************/
259 static const struct gator_cpu gator_cpus[] = {
260         {
261                 .cpuid = ARM1136,
262                 .core_name = "ARM1136",
263                 .pmnc_name = "ARM_ARM11",
264                 .dt_name = "arm,arm1136",
265                 .pmnc_counters = 3,
266         },
267         {
268                 .cpuid = ARM1156,
269                 .core_name = "ARM1156",
270                 .pmnc_name = "ARM_ARM11",
271                 .dt_name = "arm,arm1156",
272                 .pmnc_counters = 3,
273         },
274         {
275                 .cpuid = ARM1176,
276                 .core_name = "ARM1176",
277                 .pmnc_name = "ARM_ARM11",
278                 .dt_name = "arm,arm1176",
279                 .pmnc_counters = 3,
280         },
281         {
282                 .cpuid = ARM11MPCORE,
283                 .core_name = "ARM11MPCore",
284                 .pmnc_name = "ARM_ARM11MPCore",
285                 .dt_name = "arm,arm11mpcore",
286                 .pmnc_counters = 3,
287         },
288         {
289                 .cpuid = CORTEX_A5,
290                 .core_name = "Cortex-A5",
291                 .pmnc_name = "ARMv7_Cortex_A5",
292                 .dt_name = "arm,cortex-a5",
293                 .pmnc_counters = 2,
294         },
295         {
296                 .cpuid = CORTEX_A7,
297                 .core_name = "Cortex-A7",
298                 .pmnc_name = "ARMv7_Cortex_A7",
299                 .dt_name = "arm,cortex-a7",
300                 .pmnc_counters = 4,
301         },
302         {
303                 .cpuid = CORTEX_A8,
304                 .core_name = "Cortex-A8",
305                 .pmnc_name = "ARMv7_Cortex_A8",
306                 .dt_name = "arm,cortex-a8",
307                 .pmnc_counters = 4,
308         },
309         {
310                 .cpuid = CORTEX_A9,
311                 .core_name = "Cortex-A9",
312                 .pmnc_name = "ARMv7_Cortex_A9",
313                 .dt_name = "arm,cortex-a9",
314                 .pmnc_counters = 6,
315         },
316         {
317                 .cpuid = CORTEX_A12,
318                 .core_name = "Cortex-A12",
319                 .pmnc_name = "ARMv7_Cortex_A12",
320                 .dt_name = "arm,cortex-a12",
321                 .pmnc_counters = 6,
322         },
323         {
324                 .cpuid = CORTEX_A15,
325                 .core_name = "Cortex-A15",
326                 .pmnc_name = "ARMv7_Cortex_A15",
327                 .dt_name = "arm,cortex-a15",
328                 .pmnc_counters = 6,
329         },
330         {
331                 .cpuid = CORTEX_A17,
332                 .core_name = "Cortex-A17",
333                 .pmnc_name = "ARMv7_Cortex_A17",
334                 .dt_name = "arm,cortex-a17",
335                 .pmnc_counters = 6,
336         },
337         {
338                 .cpuid = SCORPION,
339                 .core_name = "Scorpion",
340                 .pmnc_name = "Scorpion",
341                 .pmnc_counters = 4,
342         },
343         {
344                 .cpuid = SCORPIONMP,
345                 .core_name = "ScorpionMP",
346                 .pmnc_name = "ScorpionMP",
347                 .pmnc_counters = 4,
348         },
349         {
350                 .cpuid = KRAITSIM,
351                 .core_name = "KraitSIM",
352                 .pmnc_name = "Krait",
353                 .pmnc_counters = 4,
354         },
355         {
356                 .cpuid = KRAIT,
357                 .core_name = "Krait",
358                 .pmnc_name = "Krait",
359                 .pmnc_counters = 4,
360         },
361         {
362                 .cpuid = KRAIT_S4_PRO,
363                 .core_name = "Krait S4 Pro",
364                 .pmnc_name = "Krait",
365                 .pmnc_counters = 4,
366         },
367         {
368                 .cpuid = CORTEX_A53,
369                 .core_name = "Cortex-A53",
370                 .pmnc_name = "ARM_Cortex-A53",
371                 .dt_name = "arm,cortex-a53",
372                 .pmnc_counters = 6,
373         },
374         {
375                 .cpuid = CORTEX_A57,
376                 .core_name = "Cortex-A57",
377                 .pmnc_name = "ARM_Cortex-A57",
378                 .dt_name = "arm,cortex-a57",
379                 .pmnc_counters = 6,
380         },
381         {
382                 .cpuid = AARCH64,
383                 .core_name = "AArch64",
384                 .pmnc_name = "ARM_AArch64",
385                 .pmnc_counters = 6,
386         },
387         {
388                 .cpuid = OTHER,
389                 .core_name = "Other",
390                 .pmnc_name = "Other",
391                 .pmnc_counters = 6,
392         },
393         {}
394 };
396 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
398         int i;
400         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
401                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
402                 if (gator_cpu->cpuid == cpuid) {
403                         return gator_cpu;
404                 }
405         }
407         return NULL;
410 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
412         int i;
414         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
415                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
416                 if (gator_cpu->pmnc_name != NULL && strcmp(gator_cpu->pmnc_name, name) == 0) {
417                         return gator_cpu;
418                 }
419         }
421         return NULL;
424 u32 gator_cpuid(void)
426 #if defined(__arm__) || defined(__aarch64__)
427         u32 val;
428 #if !defined(__aarch64__)
429         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
430 #else
431         asm volatile("mrs %0, midr_el1" : "=r" (val));
432 #endif
433         return (val >> 4) & 0xfff;
434 #else
435         return OTHER;
436 #endif
439 static void gator_buffer_wake_up(unsigned long data)
441         wake_up(&gator_buffer_wait);
444 static int gator_buffer_wake_func(void *data)
446         for (;;) {
447                 if (down_killable(&gator_buffer_wake_sem)) {
448                         break;
449                 }
451                 // Eat up any pending events
452                 while (!down_trylock(&gator_buffer_wake_sem));
454                 if (!gator_buffer_wake_run) {
455                         break;
456                 }
458                 gator_buffer_wake_up(0);
459         }
461         return 0;
464 /******************************************************************************
465  * Commit interface
466  ******************************************************************************/
467 static bool buffer_commit_ready(int *cpu, int *buftype)
469         int cpu_x, x;
470         for_each_present_cpu(cpu_x) {
471                 for (x = 0; x < NUM_GATOR_BUFS; x++)
472                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
473                                 *cpu = cpu_x;
474                                 *buftype = x;
475                                 return true;
476                         }
477         }
478         *cpu = -1;
479         *buftype = -1;
480         return false;
483 /******************************************************************************
484  * hrtimer interrupt processing
485  ******************************************************************************/
486 static void gator_timer_interrupt(void)
488         struct pt_regs *const regs = get_irq_regs();
489         gator_backtrace_handler(regs);
492 void gator_backtrace_handler(struct pt_regs *const regs)
494         u64 time = gator_get_time();
495         int cpu = get_physical_cpu();
497         // Output backtrace
498         gator_add_sample(cpu, regs, time);
500         // Collect counters
501         if (!per_cpu(collecting, cpu)) {
502                 collect_counters(time, NULL);
503         }
505         // No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed
506 #ifdef CONFIG_PREEMPT_RT_FULL
507         buffer_check(cpu, SCHED_TRACE_BUF, time);
508 #endif
511 static int gator_running;
513 // This function runs in interrupt context and on the appropriate core
514 static void gator_timer_offline(void *migrate)
516         struct gator_interface *gi;
517         int i, len, cpu = get_physical_cpu();
518         int *buffer;
519         u64 time;
521         gator_trace_sched_offline();
522         gator_trace_power_offline();
524         if (!migrate) {
525                 gator_hrtimer_offline();
526         }
528         // Offline any events and output counters
529         time = gator_get_time();
530         if (marshal_event_header(time)) {
531                 list_for_each_entry(gi, &gator_events, list) {
532                         if (gi->offline) {
533                                 len = gi->offline(&buffer, migrate);
534                                 marshal_event(len, buffer);
535                         }
536                 }
537                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
538                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
539         }
541         // Flush all buffers on this core
542         for (i = 0; i < NUM_GATOR_BUFS; i++)
543                 gator_commit_buffer(cpu, i, time);
546 // This function runs in interrupt context and may be running on a core other than core 'cpu'
547 static void gator_timer_offline_dispatch(int cpu, bool migrate)
549         struct gator_interface *gi;
551         list_for_each_entry(gi, &gator_events, list) {
552                 if (gi->offline_dispatch) {
553                         gi->offline_dispatch(cpu, migrate);
554                 }
555         }
558 static void gator_timer_stop(void)
560         int cpu;
562         if (gator_running) {
563                 on_each_cpu(gator_timer_offline, NULL, 1);
564                 for_each_online_cpu(cpu) {
565                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
566                 }
568                 gator_running = 0;
569                 gator_hrtimer_shutdown();
570         }
573 #if defined(__arm__) || defined(__aarch64__)
574 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu)
576         const char *core_name = NULL;
577         char core_name_buf[32];
579         if (!sent_core_name[cpu]) {
580                 if (gator_cpu != NULL) {
581                         core_name = gator_cpu->core_name;
582                 } else {
583                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
584                         core_name = core_name_buf;
585                 }
587                 marshal_core_name(cpu, cpuid, core_name);
588                 sent_core_name[cpu] = true;
589         }
591 #endif
593 // This function runs in interrupt context and on the appropriate core
594 static void gator_timer_online(void *migrate)
596         struct gator_interface *gi;
597         int len, cpu = get_physical_cpu();
598         int *buffer;
599         u64 time;
601         gator_trace_power_online();
603         // online any events and output counters
604         time = gator_get_time();
605         if (marshal_event_header(time)) {
606                 list_for_each_entry(gi, &gator_events, list) {
607                         if (gi->online) {
608                                 len = gi->online(&buffer, migrate);
609                                 marshal_event(len, buffer);
610                         }
611                 }
612                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
613                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
614         }
616         if (!migrate) {
617                 gator_hrtimer_online();
618         }
620 #if defined(__arm__) || defined(__aarch64__)
621         if (!sent_core_name[cpu]) {
622                 const u32 cpuid = gator_cpuid();
623                 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
624         }
625 #endif
628 // This function runs in interrupt context and may be running on a core other than core 'cpu'
629 static void gator_timer_online_dispatch(int cpu, bool migrate)
631         struct gator_interface *gi;
633         list_for_each_entry(gi, &gator_events, list) {
634                 if (gi->online_dispatch) {
635                         gi->online_dispatch(cpu, migrate);
636                 }
637         }
640 #include "gator_iks.c"
642 static int gator_timer_start(unsigned long sample_rate)
644         int cpu;
646         if (gator_running) {
647                 pr_notice("gator: already running\n");
648                 return 0;
649         }
651         gator_running = 1;
653         // event based sampling trumps hr timer based sampling
654         if (event_based_sampling) {
655                 sample_rate = 0;
656         }
658         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
659                 return -1;
661         gator_send_iks_core_names();
662         for_each_online_cpu(cpu) {
663                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
664         }
665         on_each_cpu(gator_timer_online, NULL, 1);
667         return 0;
670 static u64 gator_get_time(void)
672         struct timespec ts;
673         u64 timestamp;
674         u64 prev_timestamp;
675         u64 delta;
676         int cpu = smp_processor_id();
678         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
679         getrawmonotonic(&ts);
680         timestamp = timespec_to_ns(&ts);
682         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
683         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
684         // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
685         prev_timestamp = per_cpu(last_timestamp, cpu);
686         if (prev_timestamp <= timestamp) {
687                 per_cpu(last_timestamp, cpu) = timestamp;
688         } else {
689                 delta = prev_timestamp - timestamp;
690                 // Log the error once
691                 if (!printed_monotonic_warning && delta > 500000) {
692                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
693                         printed_monotonic_warning = true;
694                 }
695                 timestamp = prev_timestamp;
696         }
698         return timestamp - gator_monotonic_started;
701 /******************************************************************************
702  * cpu hotplug and pm notifiers
703  ******************************************************************************/
704 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
706         int cpu = lcpu_to_pcpu((long)hcpu);
708         switch (action) {
709         case CPU_DOWN_PREPARE:
710         case CPU_DOWN_PREPARE_FROZEN:
711                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
712                 gator_timer_offline_dispatch(cpu, false);
713                 break;
714         case CPU_ONLINE:
715         case CPU_ONLINE_FROZEN:
716                 gator_timer_online_dispatch(cpu, false);
717                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
718                 break;
719         }
721         return NOTIFY_OK;
724 static struct notifier_block __refdata gator_hotcpu_notifier = {
725         .notifier_call = gator_hotcpu_notify,
726 };
728 // n.b. calling "on_each_cpu" only runs on those that are online
729 // Registered linux events are not disabled, so their counters will continue to collect
730 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
732         int cpu;
733         struct timespec ts;
735         switch (event) {
736         case PM_HIBERNATION_PREPARE:
737         case PM_SUSPEND_PREPARE:
738                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
739                 unregister_scheduler_tracepoints();
740                 on_each_cpu(gator_timer_offline, NULL, 1);
741                 for_each_online_cpu(cpu) {
742                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
743                 }
745                 // Record the wallclock hibernate time
746                 getnstimeofday(&ts);
747                 gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time();
748                 break;
749         case PM_POST_HIBERNATION:
750         case PM_POST_SUSPEND:
751                 // Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it
752                 if (gator_hibernate_time > 0) {
753                         getnstimeofday(&ts);
754                         gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts);
755                         gator_hibernate_time = 0;
756                 }
758                 for_each_online_cpu(cpu) {
759                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
760                 }
761                 on_each_cpu(gator_timer_online, NULL, 1);
762                 register_scheduler_tracepoints();
763                 register_hotcpu_notifier(&gator_hotcpu_notifier);
764                 break;
765         }
767         return NOTIFY_OK;
770 static struct notifier_block gator_pm_notifier = {
771         .notifier_call = gator_pm_notify,
772 };
774 static int gator_notifier_start(void)
776         int retval;
777         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
778         if (retval == 0)
779                 retval = register_pm_notifier(&gator_pm_notifier);
780         return retval;
783 static void gator_notifier_stop(void)
785         unregister_pm_notifier(&gator_pm_notifier);
786         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
789 /******************************************************************************
790  * Main
791  ******************************************************************************/
792 static void gator_summary(void)
794         u64 timestamp, uptime;
795         struct timespec ts;
796         char uname_buf[512];
797         void (*m2b)(struct timespec *ts);
799         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
801         getnstimeofday(&ts);
802         timestamp = timespec_to_ns(&ts);
804         do_posix_clock_monotonic_gettime(&ts);
805         // monotonic_to_bootbased is not defined for some versions of Android
806         m2b = symbol_get(monotonic_to_bootbased);
807         if (m2b) {
808                 m2b(&ts);
809         }
810         uptime = timespec_to_ns(&ts);
812         // Disable preemption as gator_get_time calls smp_processor_id to verify time is monotonic
813         preempt_disable();
814         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
815         gator_monotonic_started = 0;
816         gator_monotonic_started = gator_get_time();
818         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
819         preempt_enable();
822 int gator_events_install(struct gator_interface *interface)
824         list_add_tail(&interface->list, &gator_events);
826         return 0;
829 int gator_events_get_key(void)
831         // key 0 is reserved as a timestamp
832         // key 1 is reserved as the marker for thread specific counters
833         // Odd keys are assigned by the driver, even keys by the daemon
834         static int key = 3;
836         const int ret = key;
837         key += 2;
838         return ret;
841 static int gator_init(void)
843         int i;
845         calc_first_cluster_size();
847         // events sources
848         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
849                 if (gator_events_list[i])
850                         gator_events_list[i]();
852         gator_trace_sched_init();
853         gator_trace_power_init();
855         return 0;
858 static void gator_exit(void)
860         struct gator_interface *gi;
862         list_for_each_entry(gi, &gator_events, list)
863                 if (gi->shutdown)
864                         gi->shutdown();
867 static int gator_start(void)
869         unsigned long cpu, i;
870         struct gator_interface *gi;
872         gator_buffer_wake_run = true;
873         if (IS_ERR(gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake"))) {
874                 goto bwake_failure;
875         }
877         if (gator_migrate_start())
878                 goto migrate_failure;
880         // Initialize the buffer with the frame type and core
881         for_each_present_cpu(cpu) {
882                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
883                         marshal_frame(cpu, i);
884                 }
885                 per_cpu(last_timestamp, cpu) = 0;
886         }
887         printed_monotonic_warning = false;
889         // Capture the start time
890         gator_summary();
892         // start all events
893         list_for_each_entry(gi, &gator_events, list) {
894                 if (gi->start && gi->start() != 0) {
895                         struct list_head *ptr = gi->list.prev;
897                         while (ptr != &gator_events) {
898                                 gi = list_entry(ptr, struct gator_interface, list);
900                                 if (gi->stop)
901                                         gi->stop();
903                                 ptr = ptr->prev;
904                         }
905                         goto events_failure;
906                 }
907         }
909         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
910         if (cookies_initialize())
911                 goto cookies_failure;
912         if (gator_annotate_start())
913                 goto annotate_failure;
914         if (gator_trace_sched_start())
915                 goto sched_failure;
916         if (gator_trace_power_start())
917                 goto power_failure;
918         if (gator_trace_gpu_start())
919                 goto gpu_failure;
920         if (gator_timer_start(gator_timer_count))
921                 goto timer_failure;
922         if (gator_notifier_start())
923                 goto notifier_failure;
925         return 0;
927 notifier_failure:
928         gator_timer_stop();
929 timer_failure:
930         gator_trace_gpu_stop();
931 gpu_failure:
932         gator_trace_power_stop();
933 power_failure:
934         gator_trace_sched_stop();
935 sched_failure:
936         gator_annotate_stop();
937 annotate_failure:
938         cookies_release();
939 cookies_failure:
940         // stop all events
941         list_for_each_entry(gi, &gator_events, list)
942                 if (gi->stop)
943                         gi->stop();
944 events_failure:
945         gator_migrate_stop();
946 migrate_failure:
947         gator_buffer_wake_run = false;
948         up(&gator_buffer_wake_sem);
949         gator_buffer_wake_thread = NULL;
950 bwake_failure:
952         return -1;
955 static void gator_stop(void)
957         struct gator_interface *gi;
959         gator_annotate_stop();
960         gator_trace_sched_stop();
961         gator_trace_power_stop();
962         gator_trace_gpu_stop();
964         // stop all interrupt callback reads before tearing down other interfaces
965         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
966         gator_timer_stop();
968         // stop all events
969         list_for_each_entry(gi, &gator_events, list)
970                 if (gi->stop)
971                         gi->stop();
973         gator_migrate_stop();
975         gator_buffer_wake_run = false;
976         up(&gator_buffer_wake_sem);
977         gator_buffer_wake_thread = NULL;
980 /******************************************************************************
981  * Filesystem
982  ******************************************************************************/
983 /* fopen("buffer") */
984 static int gator_op_setup(void)
986         int err = 0;
987         int cpu, i;
989         mutex_lock(&start_mutex);
991         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
992         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
994         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
995         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
997         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
998         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1000         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1001         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1003         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1004         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1006         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1007         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1009         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1010         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1012         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1013         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1015         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1016         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1018         // Initialize percpu per buffer variables
1019         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1020                 // Verify buffers are a power of 2
1021                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1022                         err = -ENOEXEC;
1023                         goto setup_error;
1024                 }
1026                 for_each_present_cpu(cpu) {
1027                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1028                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1029                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1030                         per_cpu(buffer_space_available, cpu)[i] = true;
1031                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1033                         // Annotation is a special case that only uses a single buffer
1034                         if (cpu > 0 && i == ANNOTATE_BUF) {
1035                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1036                                 continue;
1037                         }
1039                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1040                         if (!per_cpu(gator_buffer, cpu)[i]) {
1041                                 err = -ENOMEM;
1042                                 goto setup_error;
1043                         }
1044                 }
1045         }
1047 setup_error:
1048         mutex_unlock(&start_mutex);
1049         return err;
1052 /* Actually start profiling (echo 1>/dev/gator/enable) */
1053 static int gator_op_start(void)
1055         int err = 0;
1057         mutex_lock(&start_mutex);
1059         if (gator_started || gator_start())
1060                 err = -EINVAL;
1061         else
1062                 gator_started = 1;
1064         mutex_unlock(&start_mutex);
1066         return err;
1069 /* echo 0>/dev/gator/enable */
1070 static void gator_op_stop(void)
1072         mutex_lock(&start_mutex);
1074         if (gator_started) {
1075                 gator_stop();
1077                 mutex_lock(&gator_buffer_mutex);
1079                 gator_started = 0;
1080                 gator_monotonic_started = 0;
1081                 cookies_release();
1082                 wake_up(&gator_buffer_wait);
1084                 mutex_unlock(&gator_buffer_mutex);
1085         }
1087         mutex_unlock(&start_mutex);
1090 static void gator_shutdown(void)
1092         int cpu, i;
1094         mutex_lock(&start_mutex);
1096         for_each_present_cpu(cpu) {
1097                 mutex_lock(&gator_buffer_mutex);
1098                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1099                         vfree(per_cpu(gator_buffer, cpu)[i]);
1100                         per_cpu(gator_buffer, cpu)[i] = NULL;
1101                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1102                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1103                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1104                         per_cpu(buffer_space_available, cpu)[i] = true;
1105                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1106                 }
1107                 mutex_unlock(&gator_buffer_mutex);
1108         }
1110         memset(&sent_core_name, 0, sizeof(sent_core_name));
1112         mutex_unlock(&start_mutex);
1115 static int gator_set_backtrace(unsigned long val)
1117         int err = 0;
1119         mutex_lock(&start_mutex);
1121         if (gator_started)
1122                 err = -EBUSY;
1123         else
1124                 gator_backtrace_depth = val;
1126         mutex_unlock(&start_mutex);
1128         return err;
1131 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1133         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1136 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1138         unsigned long val;
1139         int retval;
1141         if (*offset)
1142                 return -EINVAL;
1144         retval = gatorfs_ulong_from_user(&val, buf, count);
1145         if (retval)
1146                 return retval;
1148         if (val)
1149                 retval = gator_op_start();
1150         else
1151                 gator_op_stop();
1153         if (retval)
1154                 return retval;
1155         return count;
1158 static const struct file_operations enable_fops = {
1159         .read = enable_read,
1160         .write = enable_write,
1161 };
1163 static int userspace_buffer_open(struct inode *inode, struct file *file)
1165         int err = -EPERM;
1167         if (!capable(CAP_SYS_ADMIN))
1168                 return -EPERM;
1170         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1171                 return -EBUSY;
1173         if ((err = gator_op_setup()))
1174                 goto fail;
1176         /* NB: the actual start happens from userspace
1177          * echo 1 >/dev/gator/enable
1178          */
1180         return 0;
1182 fail:
1183         __clear_bit_unlock(0, &gator_buffer_opened);
1184         return err;
1187 static int userspace_buffer_release(struct inode *inode, struct file *file)
1189         gator_op_stop();
1190         gator_shutdown();
1191         __clear_bit_unlock(0, &gator_buffer_opened);
1192         return 0;
1195 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1197         int commit, length1, length2, read;
1198         char *buffer1;
1199         char *buffer2;
1200         int cpu, buftype;
1201         int written = 0;
1203         // ensure there is enough space for a whole frame
1204         if (count < userspace_buffer_size || *offset) {
1205                 return -EINVAL;
1206         }
1208         // sleep until the condition is true or a signal is received
1209         // the condition is checked each time gator_buffer_wait is woken up
1210         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1212         if (signal_pending(current)) {
1213                 return -EINTR;
1214         }
1216         if (buftype == -1 || cpu == -1) {
1217                 return 0;
1218         }
1220         mutex_lock(&gator_buffer_mutex);
1222         do {
1223                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1224                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1226                 // May happen if the buffer is freed during pending reads.
1227                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1228                         break;
1229                 }
1231                 // determine the size of two halves
1232                 length1 = commit - read;
1233                 length2 = 0;
1234                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1235                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1236                 if (length1 < 0) {
1237                         length1 = gator_buffer_size[buftype] - read;
1238                         length2 = commit;
1239                 }
1241                 if (length1 + length2 > count - written) {
1242                         break;
1243                 }
1245                 // start, middle or end
1246                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1247                         break;
1248                 }
1250                 // possible wrap around
1251                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1252                         break;
1253                 }
1255                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1256                 written += length1 + length2;
1258                 // Wake up annotate_write if more space is available
1259                 if (buftype == ANNOTATE_BUF) {
1260                         wake_up(&gator_annotate_wait);
1261                 }
1262         } while (buffer_commit_ready(&cpu, &buftype));
1264         mutex_unlock(&gator_buffer_mutex);
1266         // kick just in case we've lost an SMP event
1267         wake_up(&gator_buffer_wait);
1269         return written > 0 ? written : -EFAULT;
1272 static const struct file_operations gator_event_buffer_fops = {
1273         .open = userspace_buffer_open,
1274         .release = userspace_buffer_release,
1275         .read = userspace_buffer_read,
1276 };
1278 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1280         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1283 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1285         unsigned long val;
1286         int retval;
1288         if (*offset)
1289                 return -EINVAL;
1291         retval = gatorfs_ulong_from_user(&val, buf, count);
1292         if (retval)
1293                 return retval;
1295         retval = gator_set_backtrace(val);
1297         if (retval)
1298                 return retval;
1299         return count;
1302 static const struct file_operations depth_fops = {
1303         .read = depth_read,
1304         .write = depth_write
1305 };
1307 static void gator_op_create_files(struct super_block *sb, struct dentry *root)
1309         struct dentry *dir;
1310         struct gator_interface *gi;
1311         int cpu;
1313         /* reinitialize default values */
1314         gator_cpu_cores = 0;
1315         for_each_present_cpu(cpu) {
1316                 gator_cpu_cores++;
1317         }
1318         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1319         gator_response_type = 1;
1320         gator_live_rate = 0;
1322         gatorfs_create_file(sb, root, "enable", &enable_fops);
1323         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1324         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1325         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1326         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1327         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1328         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1329         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1330         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1331         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1333         // Annotate interface
1334         gator_annotate_create_files(sb, root);
1336         // Linux Events
1337         dir = gatorfs_mkdir(sb, root, "events");
1338         list_for_each_entry(gi, &gator_events, list)
1339                 if (gi->create_files)
1340                         gi->create_files(sb, dir);
1342         // Sched Events
1343         sched_trace_create_files(sb, dir);
1345         // Power interface
1346         gator_trace_power_create_files(sb, dir);
1349 /******************************************************************************
1350  * Module
1351  ******************************************************************************/
1352 static int __init gator_module_init(void)
1354         if (gatorfs_register()) {
1355                 return -1;
1356         }
1358         if (gator_init()) {
1359                 gatorfs_unregister();
1360                 return -1;
1361         }
1363         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1365         return 0;
1368 static void __exit gator_module_exit(void)
1370         del_timer_sync(&gator_buffer_wake_up_timer);
1371         tracepoint_synchronize_unregister();
1372         gator_exit();
1373         gatorfs_unregister();
1376 module_init(gator_module_init);
1377 module_exit(gator_module_exit);
1379 MODULE_LICENSE("GPL");
1380 MODULE_AUTHOR("ARM Ltd");
1381 MODULE_DESCRIPTION("Gator system profiler");
1382 #define STRIFY2(ARG) #ARG
1383 #define STRIFY(ARG) STRIFY2(ARG)
1384 MODULE_VERSION(STRIFY(PROTOCOL_VERSION));