gator-driver: Fix building on Linux 3.12
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 16;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) /* Because CONFIG_LOCAL_TIMERS was removed in 3.12 */
48 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
49 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
50 #endif
51 #endif
53 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
54 #ifndef CONFIG_PERF_EVENTS
55 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
56 #elif !defined CONFIG_HW_PERF_EVENTS
57 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
58 #endif
59 #endif
61 /******************************************************************************
62  * DEFINES
63  ******************************************************************************/
64 #define SUMMARY_BUFFER_SIZE       (1*1024)
65 #define BACKTRACE_BUFFER_SIZE     (128*1024)
66 #define NAME_BUFFER_SIZE          (64*1024)
67 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
68 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
69 #define ANNOTATE_BUFFER_SIZE      (128*1024)    // annotate counters have the core as part of the data and the core value in the frame header may be discarded
70 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
71 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
72 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
74 #define NO_COOKIE      0U
75 #define UNRESOLVED_COOKIE ~0U
77 #define FRAME_SUMMARY       1
78 #define FRAME_BACKTRACE     2
79 #define FRAME_NAME          3
80 #define FRAME_COUNTER       4
81 #define FRAME_BLOCK_COUNTER 5
82 #define FRAME_ANNOTATE      6
83 #define FRAME_SCHED_TRACE   7
84 #define FRAME_GPU_TRACE     8
85 #define FRAME_IDLE          9
87 #define MESSAGE_END_BACKTRACE 1
89 #define MESSAGE_COOKIE      1
90 #define MESSAGE_THREAD_NAME 2
91 #define HRTIMER_CORE_NAME   3
93 #define MESSAGE_GPU_START 1
94 #define MESSAGE_GPU_STOP  2
96 #define MESSAGE_SCHED_SWITCH 1
97 #define MESSAGE_SCHED_EXIT   2
98 #define MESSAGE_SCHED_START  3
100 #define MESSAGE_IDLE_ENTER 1
101 #define MESSAGE_IDLE_EXIT 2
103 #define MAXSIZE_PACK32     5
104 #define MAXSIZE_PACK64    10
106 #define FRAME_HEADER_SIZE 3
108 #if defined(__arm__)
109 #define PC_REG regs->ARM_pc
110 #elif defined(__aarch64__)
111 #define PC_REG regs->pc
112 #else
113 #define PC_REG regs->ip
114 #endif
116 enum {
117         SUMMARY_BUF,
118         BACKTRACE_BUF,
119         NAME_BUF,
120         COUNTER_BUF,
121         BLOCK_COUNTER_BUF,
122         ANNOTATE_BUF,
123         SCHED_TRACE_BUF,
124         GPU_TRACE_BUF,
125         IDLE_BUF,
126         NUM_GATOR_BUFS
127 };
129 /******************************************************************************
130  * Globals
131  ******************************************************************************/
132 static unsigned long gator_cpu_cores;
133 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
134 static unsigned long userspace_buffer_size;
135 static unsigned long gator_backtrace_depth;
136 // How often to commit the buffers for live in nanoseconds
137 static u64 gator_live_rate;
139 static unsigned long gator_started;
140 static u64 gator_monotonic_started;
141 static unsigned long gator_buffer_opened;
142 static unsigned long gator_timer_count;
143 static unsigned long gator_response_type;
144 static DEFINE_MUTEX(start_mutex);
145 static DEFINE_MUTEX(gator_buffer_mutex);
147 bool event_based_sampling;
149 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
150 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
151 static struct timer_list gator_buffer_wake_up_timer;
152 static LIST_HEAD(gator_events);
154 static DEFINE_PER_CPU(u64, last_timestamp);
156 static bool printed_monotonic_warning;
158 static bool sent_core_name[NR_CPUS];
160 /******************************************************************************
161  * Prototypes
162  ******************************************************************************/
163 static void buffer_check(int cpu, int buftype, u64 time);
164 static void gator_commit_buffer(int cpu, int buftype, u64 time);
165 static int buffer_bytes_available(int cpu, int buftype);
166 static bool buffer_check_space(int cpu, int buftype, int bytes);
167 static int contiguous_space_available(int cpu, int bufytpe);
168 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
169 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
170 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
171 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
172 static void gator_add_trace(int cpu, unsigned long address);
173 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time);
174 static u64 gator_get_time(void);
176 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
177 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
178 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
179 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
180 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
181 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
182 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
183 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
184 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
186 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
187 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
188 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
189 // The buffer. Allocated in gator_op_setup
190 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
191 // The time after which the buffer should be committed for live display
192 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
194 /******************************************************************************
195  * Application Includes
196  ******************************************************************************/
197 #include "gator_marshaling.c"
198 #include "gator_hrtimer_perf.c"
199 #include "gator_hrtimer_gator.c"
200 #include "gator_cookies.c"
201 #include "gator_annotate.c"
202 #include "gator_trace_sched.c"
203 #include "gator_trace_power.c"
204 #include "gator_trace_gpu.c"
205 #include "gator_backtrace.c"
206 #include "gator_fs.c"
207 #include "gator_pack.c"
209 /******************************************************************************
210  * Misc
211  ******************************************************************************/
213 const struct gator_cpu gator_cpus[] = {
214         {
215                 .cpuid = ARM1136,
216                 .core_name = "ARM1136",
217                 .pmnc_name = "ARM_ARM11",
218                 .dt_name = "arm,arm1136",
219                 .pmnc_counters = 3,
220         },
221         {
222                 .cpuid = ARM1156,
223                 .core_name = "ARM1156",
224                 .pmnc_name = "ARM_ARM11",
225                 .dt_name = "arm,arm1156",
226                 .pmnc_counters = 3,
227         },
228         {
229                 .cpuid = ARM1176,
230                 .core_name = "ARM1176",
231                 .pmnc_name = "ARM_ARM11",
232                 .dt_name = "arm,arm1176",
233                 .pmnc_counters = 3,
234         },
235         {
236                 .cpuid = ARM11MPCORE,
237                 .core_name = "ARM11MPCore",
238                 .pmnc_name = "ARM_ARM11MPCore",
239                 .dt_name = "arm,arm11mpcore",
240                 .pmnc_counters = 3,
241         },
242         {
243                 .cpuid = CORTEX_A5,
244                 .core_name = "Cortex-A5",
245                 .pmu_name = "ARMv7_Cortex_A5",
246                 .pmnc_name = "ARM_Cortex-A5",
247                 .dt_name = "arm,cortex-a5",
248                 .pmnc_counters = 2,
249         },
250         {
251                 .cpuid = CORTEX_A7,
252                 .core_name = "Cortex-A7",
253                 .pmu_name = "ARMv7_Cortex_A7",
254                 .pmnc_name = "ARM_Cortex-A7",
255                 .dt_name = "arm,cortex-a7",
256                 .pmnc_counters = 4,
257         },
258         {
259                 .cpuid = CORTEX_A8,
260                 .core_name = "Cortex-A8",
261                 .pmu_name = "ARMv7_Cortex_A8",
262                 .pmnc_name = "ARM_Cortex-A8",
263                 .dt_name = "arm,cortex-a8",
264                 .pmnc_counters = 4,
265         },
266         {
267                 .cpuid = CORTEX_A9,
268                 .core_name = "Cortex-A9",
269                 .pmu_name = "ARMv7_Cortex_A9",
270                 .pmnc_name = "ARM_Cortex-A9",
271                 .dt_name = "arm,cortex-a9",
272                 .pmnc_counters = 6,
273         },
274         {
275                 .cpuid = CORTEX_A12,
276                 .core_name = "Cortex-A12",
277                 .pmu_name = "ARMv7_Cortex_A12",
278                 .pmnc_name = "ARM_Cortex-A12",
279                 .dt_name = "arm,cortex-a12",
280                 .pmnc_counters = 6,
281         },
282         {
283                 .cpuid = CORTEX_A15,
284                 .core_name = "Cortex-A15",
285                 .pmu_name = "ARMv7_Cortex_A15",
286                 .pmnc_name = "ARM_Cortex-A15",
287                 .dt_name = "arm,cortex-a15",
288                 .pmnc_counters = 6,
289         },
290         {
291                 .cpuid = SCORPION,
292                 .core_name = "Scorpion",
293                 .pmnc_name = "Scorpion",
294                 .pmnc_counters = 4,
295         },
296         {
297                 .cpuid = SCORPIONMP,
298                 .core_name = "ScorpionMP",
299                 .pmnc_name = "ScorpionMP",
300                 .pmnc_counters = 4,
301         },
302         {
303                 .cpuid = KRAITSIM,
304                 .core_name = "KraitSIM",
305                 .pmnc_name = "Krait",
306                 .pmnc_counters = 4,
307         },
308         {
309                 .cpuid = KRAIT,
310                 .core_name = "Krait",
311                 .pmnc_name = "Krait",
312                 .pmnc_counters = 4,
313         },
314         {
315                 .cpuid = KRAIT_S4_PRO,
316                 .core_name = "Krait S4 Pro",
317                 .pmnc_name = "Krait",
318                 .pmnc_counters = 4,
319         },
320         {
321                 .cpuid = CORTEX_A53,
322                 .core_name = "Cortex-A53",
323                 .pmnc_name = "ARM_Cortex-A53",
324                 .dt_name = "arm,cortex-a53",
325                 .pmnc_counters = 6,
326         },
327         {
328                 .cpuid = CORTEX_A57,
329                 .core_name = "Cortex-A57",
330                 .pmnc_name = "ARM_Cortex-A57",
331                 .dt_name = "arm,cortex-a57",
332                 .pmnc_counters = 6,
333         },
334         {
335                 .cpuid = AARCH64,
336                 .core_name = "AArch64",
337                 .pmnc_name = "ARM_AArch64",
338                 .pmnc_counters = 6,
339         },
340         {
341                 .cpuid = OTHER,
342                 .core_name = "Other",
343                 .pmnc_name = "Other",
344                 .pmnc_counters = 6,
345         },
346         {}
347 };
349 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
351         int i;
353         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
354                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
355                 if (gator_cpu->cpuid == cpuid) {
356                         return gator_cpu;
357                 }
358         }
360         return NULL;
363 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
365         int i;
367         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
368                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
369                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
370                         return gator_cpu;
371                 }
372         }
374         return NULL;
377 u32 gator_cpuid(void)
379 #if defined(__arm__) || defined(__aarch64__)
380         u32 val;
381 #if !defined(__aarch64__)
382         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
383 #else
384         asm volatile("mrs %0, midr_el1" : "=r" (val));
385 #endif
386         return (val >> 4) & 0xfff;
387 #else
388         return OTHER;
389 #endif
392 static void gator_buffer_wake_up(unsigned long data)
394         wake_up(&gator_buffer_wait);
397 /******************************************************************************
398  * Commit interface
399  ******************************************************************************/
400 static bool buffer_commit_ready(int *cpu, int *buftype)
402         int cpu_x, x;
403         for_each_present_cpu(cpu_x) {
404                 for (x = 0; x < NUM_GATOR_BUFS; x++)
405                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
406                                 *cpu = cpu_x;
407                                 *buftype = x;
408                                 return true;
409                         }
410         }
411         *cpu = -1;
412         *buftype = -1;
413         return false;
416 /******************************************************************************
417  * Buffer management
418  ******************************************************************************/
419 static int buffer_bytes_available(int cpu, int buftype)
421         int remaining, filled;
423         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
424         if (filled < 0) {
425                 filled += gator_buffer_size[buftype];
426         }
428         remaining = gator_buffer_size[buftype] - filled;
430         if (per_cpu(buffer_space_available, cpu)[buftype]) {
431                 // Give some extra room; also allows space to insert the overflow error packet
432                 remaining -= 200;
433         } else {
434                 // Hysteresis, prevents multiple overflow messages
435                 remaining -= 2000;
436         }
438         return remaining;
441 static int contiguous_space_available(int cpu, int buftype)
443         int remaining = buffer_bytes_available(cpu, buftype);
444         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
445         if (remaining < contiguous)
446                 return remaining;
447         else
448                 return contiguous;
451 static bool buffer_check_space(int cpu, int buftype, int bytes)
453         int remaining = buffer_bytes_available(cpu, buftype);
455         if (remaining < bytes) {
456                 per_cpu(buffer_space_available, cpu)[buftype] = false;
457         } else {
458                 per_cpu(buffer_space_available, cpu)[buftype] = true;
459         }
461         return per_cpu(buffer_space_available, cpu)[buftype];
464 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
466         int i;
467         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
468         u32 mask = gator_buffer_mask[buftype];
469         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
471         for (i = 0; i < len; i++) {
472                 buffer[write] = x[i];
473                 write = (write + 1) & mask;
474         }
476         per_cpu(gator_buffer_write, cpu)[buftype] = write;
479 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
481         int len = strlen(x);
482         gator_buffer_write_packed_int(cpu, buftype, len);
483         gator_buffer_write_bytes(cpu, buftype, x, len);
486 static void gator_commit_buffer(int cpu, int buftype, u64 time)
488         int type_length, commit, length, byte;
490         if (!per_cpu(gator_buffer, cpu)[buftype])
491                 return;
493         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
494         type_length = gator_response_type ? 1 : 0;
495         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
496         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
497         if (length < 0) {
498                 length += gator_buffer_size[buftype];
499         }
500         length = length - type_length - sizeof(s32);
502         if (length <= FRAME_HEADER_SIZE) {
503                 // Nothing to write, only the frame header is present
504                 return;
505         }
507         for (byte = 0; byte < sizeof(s32); byte++) {
508                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
509         }
511         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
513         if (gator_live_rate > 0) {
514                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
515                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
516                 }
517         }
519         marshal_frame(cpu, buftype);
521         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
522         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
525 static void buffer_check(int cpu, int buftype, u64 time)
527         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
528         if (filled < 0) {
529                 filled += gator_buffer_size[buftype];
530         }
531         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
532                 gator_commit_buffer(cpu, buftype, time);
533         }
536 static void gator_add_trace(int cpu, unsigned long address)
538         off_t offset = 0;
539         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
541         if (cookie == NO_COOKIE || cookie == UNRESOLVED_COOKIE) {
542                 offset = address;
543         }
545         marshal_backtrace(offset & ~1, cookie);
548 static void gator_add_sample(int cpu, struct pt_regs *const regs, u64 time)
550         bool inKernel;
551         unsigned long exec_cookie;
553         if (!regs)
554                 return;
556         inKernel = !user_mode(regs);
557         exec_cookie = get_exec_cookie(cpu, current);
559         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel, time))
560                 return;
562         if (inKernel) {
563                 kernel_backtrace(cpu, regs);
564         } else {
565                 // Cookie+PC
566                 gator_add_trace(cpu, PC_REG);
568                 // Backtrace
569                 if (gator_backtrace_depth)
570                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
571         }
573         marshal_backtrace_footer(time);
576 /******************************************************************************
577  * hrtimer interrupt processing
578  ******************************************************************************/
579 static void gator_timer_interrupt(void)
581         struct pt_regs *const regs = get_irq_regs();
582         gator_backtrace_handler(regs);
585 void gator_backtrace_handler(struct pt_regs *const regs)
587         u64 time = gator_get_time();
588         int cpu = get_physical_cpu();
590         // Output backtrace
591         gator_add_sample(cpu, regs, time);
593         // Collect counters
594         if (!per_cpu(collecting, cpu)) {
595                 collect_counters(time);
596         }
599 static int gator_running;
601 // This function runs in interrupt context and on the appropriate core
602 static void gator_timer_offline(void *migrate)
604         struct gator_interface *gi;
605         int i, len, cpu = get_physical_cpu();
606         int *buffer;
607         u64 time;
609         gator_trace_sched_offline();
610         gator_trace_power_offline();
612         if (!migrate) {
613                 gator_hrtimer_offline();
614         }
616         // Offline any events and output counters
617         time = gator_get_time();
618         if (marshal_event_header(time)) {
619                 list_for_each_entry(gi, &gator_events, list) {
620                         if (gi->offline) {
621                                 len = gi->offline(&buffer, migrate);
622                                 marshal_event(len, buffer);
623                         }
624                 }
625                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
626                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
627         }
629         // Flush all buffers on this core
630         for (i = 0; i < NUM_GATOR_BUFS; i++)
631                 gator_commit_buffer(cpu, i, time);
634 // This function runs in interrupt context and may be running on a core other than core 'cpu'
635 static void gator_timer_offline_dispatch(int cpu, bool migrate)
637         struct gator_interface *gi;
639         list_for_each_entry(gi, &gator_events, list) {
640                 if (gi->offline_dispatch) {
641                         gi->offline_dispatch(cpu, migrate);
642                 }
643         }
646 static void gator_timer_stop(void)
648         int cpu;
650         if (gator_running) {
651                 on_each_cpu(gator_timer_offline, NULL, 1);
652                 for_each_online_cpu(cpu) {
653                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
654                 }
656                 gator_running = 0;
657                 gator_hrtimer_shutdown();
658         }
661 #if defined(__arm__) || defined(__aarch64__)
662 static void gator_send_core_name(int cpu, const u32 cpuid, const struct gator_cpu *const gator_cpu) {
663         const char *core_name = NULL;
664         char core_name_buf[32];
666         if (!sent_core_name[cpu]) {
667                 if (gator_cpu != NULL) {
668                         core_name = gator_cpu->core_name;
669                 } else {
670                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
671                         core_name = core_name_buf;
672                 }
674                 marshal_core_name(cpu, cpuid, core_name);
675                 sent_core_name[cpu] = true;
676         }
678 #endif
680 // This function runs in interrupt context and on the appropriate core
681 static void gator_timer_online(void *migrate)
683         struct gator_interface *gi;
684         int len, cpu = get_physical_cpu();
685         int *buffer;
686         u64 time;
688         gator_trace_power_online();
690         // online any events and output counters
691         time = gator_get_time();
692         if (marshal_event_header(time)) {
693                 list_for_each_entry(gi, &gator_events, list) {
694                         if (gi->online) {
695                                 len = gi->online(&buffer, migrate);
696                                 marshal_event(len, buffer);
697                         }
698                 }
699                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
700                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
701         }
703         if (!migrate) {
704                 gator_hrtimer_online();
705         }
707 #if defined(__arm__) || defined(__aarch64__)
708         if (!sent_core_name[cpu]) {
709                 const u32 cpuid = gator_cpuid();
710                 gator_send_core_name(cpu, cpuid, gator_find_cpu_by_cpuid(cpuid));
711         }
712 #endif
715 // This function runs in interrupt context and may be running on a core other than core 'cpu'
716 static void gator_timer_online_dispatch(int cpu, bool migrate)
718         struct gator_interface *gi;
720         list_for_each_entry(gi, &gator_events, list) {
721                 if (gi->online_dispatch) {
722                         gi->online_dispatch(cpu, migrate);
723                 }
724         }
727 #include "gator_iks.c"
729 int gator_timer_start(unsigned long sample_rate)
731         int cpu;
733         if (gator_running) {
734                 pr_notice("gator: already running\n");
735                 return 0;
736         }
738         gator_running = 1;
740         // event based sampling trumps hr timer based sampling
741         if (event_based_sampling) {
742                 sample_rate = 0;
743         }
745         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
746                 return -1;
748         gator_send_iks_core_names();
749         for_each_online_cpu(cpu) {
750                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
751         }
752         on_each_cpu(gator_timer_online, NULL, 1);
754         return 0;
757 static u64 gator_get_time(void)
759         struct timespec ts;
760         u64 timestamp;
761         u64 prev_timestamp;
762         u64 delta;
763         int cpu = smp_processor_id();
765         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
766         getrawmonotonic(&ts);
767         timestamp = timespec_to_ns(&ts);
769         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
770         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
771         // This doesn't work well with interrupts, but that it's OK - the real concern is to catch big jumps in time
772         prev_timestamp = per_cpu(last_timestamp, cpu);
773         if (prev_timestamp <= timestamp) {
774                 per_cpu(last_timestamp, cpu) = timestamp;
775         } else {
776                 delta = prev_timestamp - timestamp;
777                 // Log the error once
778                 if (!printed_monotonic_warning && delta > 500000) {
779                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
780                         printed_monotonic_warning = true;
781                 }
782                 timestamp = prev_timestamp;
783         }
785         return timestamp - gator_monotonic_started;
788 /******************************************************************************
789  * cpu hotplug and pm notifiers
790  ******************************************************************************/
791 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
793         int cpu = lcpu_to_pcpu((long)hcpu);
795         switch (action) {
796         case CPU_DOWN_PREPARE:
797         case CPU_DOWN_PREPARE_FROZEN:
798                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
799                 gator_timer_offline_dispatch(cpu, false);
800                 break;
801         case CPU_ONLINE:
802         case CPU_ONLINE_FROZEN:
803                 gator_timer_online_dispatch(cpu, false);
804                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
805                 break;
806         }
808         return NOTIFY_OK;
811 static struct notifier_block __refdata gator_hotcpu_notifier = {
812         .notifier_call = gator_hotcpu_notify,
813 };
815 // n.b. calling "on_each_cpu" only runs on those that are online
816 // Registered linux events are not disabled, so their counters will continue to collect
817 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
819         int cpu;
821         switch (event) {
822         case PM_HIBERNATION_PREPARE:
823         case PM_SUSPEND_PREPARE:
824                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
825                 unregister_scheduler_tracepoints();
826                 on_each_cpu(gator_timer_offline, NULL, 1);
827                 for_each_online_cpu(cpu) {
828                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
829                 }
830                 break;
831         case PM_POST_HIBERNATION:
832         case PM_POST_SUSPEND:
833                 for_each_online_cpu(cpu) {
834                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
835                 }
836                 on_each_cpu(gator_timer_online, NULL, 1);
837                 register_scheduler_tracepoints();
838                 register_hotcpu_notifier(&gator_hotcpu_notifier);
839                 break;
840         }
842         return NOTIFY_OK;
845 static struct notifier_block gator_pm_notifier = {
846         .notifier_call = gator_pm_notify,
847 };
849 static int gator_notifier_start(void)
851         int retval;
852         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
853         if (retval == 0)
854                 retval = register_pm_notifier(&gator_pm_notifier);
855         return retval;
858 static void gator_notifier_stop(void)
860         unregister_pm_notifier(&gator_pm_notifier);
861         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
864 /******************************************************************************
865  * Main
866  ******************************************************************************/
867 static void gator_summary(void)
869         u64 timestamp, uptime;
870         struct timespec ts;
871         char uname_buf[512];
872         void (*m2b)(struct timespec *ts);
873         unsigned long flags;
875         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
877         getnstimeofday(&ts);
878         timestamp = timespec_to_ns(&ts);
880         do_posix_clock_monotonic_gettime(&ts);
881         // monotonic_to_bootbased is not defined for some versions of Android
882         m2b = symbol_get(monotonic_to_bootbased);
883         if (m2b) {
884                 m2b(&ts);
885         }
886         uptime = timespec_to_ns(&ts);
888         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
889         local_irq_save(flags);
890         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
891         gator_monotonic_started = 0;
892         gator_monotonic_started = gator_get_time();
893         local_irq_restore(flags);
895         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
898 int gator_events_install(struct gator_interface *interface)
900         list_add_tail(&interface->list, &gator_events);
902         return 0;
905 int gator_events_get_key(void)
907         // key of zero is reserved as a timestamp
908         static int key = 1;
910         const int ret = key;
911         key += 2;
912         return ret;
915 static int gator_init(void)
917         int i;
919         calc_first_cluster_size();
921         // events sources (gator_events.h, generated by gator_events.sh)
922         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
923                 if (gator_events_list[i])
924                         gator_events_list[i]();
926         gator_trace_sched_init();
927         gator_trace_power_init();
929         return 0;
932 static void gator_exit(void)
934         struct gator_interface *gi;
936         list_for_each_entry(gi, &gator_events, list)
937                 if (gi->shutdown)
938                         gi->shutdown();
941 static int gator_start(void)
943         unsigned long cpu, i;
944         struct gator_interface *gi;
946         if (gator_migrate_start())
947                 goto migrate_failure;
949         // Initialize the buffer with the frame type and core
950         for_each_present_cpu(cpu) {
951                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
952                         marshal_frame(cpu, i);
953                 }
954                 per_cpu(last_timestamp, cpu) = 0;
955         }
956         printed_monotonic_warning = false;
958         // Capture the start time
959         gator_summary();
961         // start all events
962         list_for_each_entry(gi, &gator_events, list) {
963                 if (gi->start && gi->start() != 0) {
964                         struct list_head *ptr = gi->list.prev;
966                         while (ptr != &gator_events) {
967                                 gi = list_entry(ptr, struct gator_interface, list);
969                                 if (gi->stop)
970                                         gi->stop();
972                                 ptr = ptr->prev;
973                         }
974                         goto events_failure;
975                 }
976         }
978         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
979         if (cookies_initialize())
980                 goto cookies_failure;
981         if (gator_annotate_start())
982                 goto annotate_failure;
983         if (gator_trace_sched_start())
984                 goto sched_failure;
985         if (gator_trace_power_start())
986                 goto power_failure;
987         if (gator_trace_gpu_start())
988                 goto gpu_failure;
989         if (gator_timer_start(gator_timer_count))
990                 goto timer_failure;
991         if (gator_notifier_start())
992                 goto notifier_failure;
994         return 0;
996 notifier_failure:
997         gator_timer_stop();
998 timer_failure:
999         gator_trace_gpu_stop();
1000 gpu_failure:
1001         gator_trace_power_stop();
1002 power_failure:
1003         gator_trace_sched_stop();
1004 sched_failure:
1005         gator_annotate_stop();
1006 annotate_failure:
1007         cookies_release();
1008 cookies_failure:
1009         // stop all events
1010         list_for_each_entry(gi, &gator_events, list)
1011                 if (gi->stop)
1012                         gi->stop();
1013 events_failure:
1014         gator_migrate_stop();
1015 migrate_failure:
1017         return -1;
1020 static void gator_stop(void)
1022         struct gator_interface *gi;
1024         gator_annotate_stop();
1025         gator_trace_sched_stop();
1026         gator_trace_power_stop();
1027         gator_trace_gpu_stop();
1029         // stop all interrupt callback reads before tearing down other interfaces
1030         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1031         gator_timer_stop();
1033         // stop all events
1034         list_for_each_entry(gi, &gator_events, list)
1035                 if (gi->stop)
1036                         gi->stop();
1038         gator_migrate_stop();
1041 /******************************************************************************
1042  * Filesystem
1043  ******************************************************************************/
1044 /* fopen("buffer") */
1045 static int gator_op_setup(void)
1047         int err = 0;
1048         int cpu, i;
1050         mutex_lock(&start_mutex);
1052         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1053         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1055         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1056         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1058         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1059         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1061         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1062         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1064         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1065         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1067         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1068         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1070         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1071         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1073         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1074         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1076         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1077         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1079         // Initialize percpu per buffer variables
1080         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1081                 // Verify buffers are a power of 2
1082                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1083                         err = -ENOEXEC;
1084                         goto setup_error;
1085                 }
1087                 for_each_present_cpu(cpu) {
1088                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1089                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1090                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1091                         per_cpu(buffer_space_available, cpu)[i] = true;
1092                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1094                         // Annotation is a special case that only uses a single buffer
1095                         if (cpu > 0 && i == ANNOTATE_BUF) {
1096                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1097                                 continue;
1098                         }
1100                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1101                         if (!per_cpu(gator_buffer, cpu)[i]) {
1102                                 err = -ENOMEM;
1103                                 goto setup_error;
1104                         }
1105                 }
1106         }
1108 setup_error:
1109         mutex_unlock(&start_mutex);
1110         return err;
1113 /* Actually start profiling (echo 1>/dev/gator/enable) */
1114 static int gator_op_start(void)
1116         int err = 0;
1118         mutex_lock(&start_mutex);
1120         if (gator_started || gator_start())
1121                 err = -EINVAL;
1122         else
1123                 gator_started = 1;
1125         mutex_unlock(&start_mutex);
1127         return err;
1130 /* echo 0>/dev/gator/enable */
1131 static void gator_op_stop(void)
1133         mutex_lock(&start_mutex);
1135         if (gator_started) {
1136                 gator_stop();
1138                 mutex_lock(&gator_buffer_mutex);
1140                 gator_started = 0;
1141                 gator_monotonic_started = 0;
1142                 cookies_release();
1143                 wake_up(&gator_buffer_wait);
1145                 mutex_unlock(&gator_buffer_mutex);
1146         }
1148         mutex_unlock(&start_mutex);
1151 static void gator_shutdown(void)
1153         int cpu, i;
1155         mutex_lock(&start_mutex);
1157         for_each_present_cpu(cpu) {
1158                 mutex_lock(&gator_buffer_mutex);
1159                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1160                         vfree(per_cpu(gator_buffer, cpu)[i]);
1161                         per_cpu(gator_buffer, cpu)[i] = NULL;
1162                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1163                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1164                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1165                         per_cpu(buffer_space_available, cpu)[i] = true;
1166                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1167                 }
1168                 mutex_unlock(&gator_buffer_mutex);
1169         }
1171         memset(&sent_core_name, 0, sizeof(sent_core_name));
1173         mutex_unlock(&start_mutex);
1176 static int gator_set_backtrace(unsigned long val)
1178         int err = 0;
1180         mutex_lock(&start_mutex);
1182         if (gator_started)
1183                 err = -EBUSY;
1184         else
1185                 gator_backtrace_depth = val;
1187         mutex_unlock(&start_mutex);
1189         return err;
1192 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1194         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1197 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1199         unsigned long val;
1200         int retval;
1202         if (*offset)
1203                 return -EINVAL;
1205         retval = gatorfs_ulong_from_user(&val, buf, count);
1206         if (retval)
1207                 return retval;
1209         if (val)
1210                 retval = gator_op_start();
1211         else
1212                 gator_op_stop();
1214         if (retval)
1215                 return retval;
1216         return count;
1219 static const struct file_operations enable_fops = {
1220         .read = enable_read,
1221         .write = enable_write,
1222 };
1224 static int userspace_buffer_open(struct inode *inode, struct file *file)
1226         int err = -EPERM;
1228         if (!capable(CAP_SYS_ADMIN))
1229                 return -EPERM;
1231         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1232                 return -EBUSY;
1234         if ((err = gator_op_setup()))
1235                 goto fail;
1237         /* NB: the actual start happens from userspace
1238          * echo 1 >/dev/gator/enable
1239          */
1241         return 0;
1243 fail:
1244         __clear_bit_unlock(0, &gator_buffer_opened);
1245         return err;
1248 static int userspace_buffer_release(struct inode *inode, struct file *file)
1250         gator_op_stop();
1251         gator_shutdown();
1252         __clear_bit_unlock(0, &gator_buffer_opened);
1253         return 0;
1256 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1258         int commit, length1, length2, read;
1259         char *buffer1;
1260         char *buffer2;
1261         int cpu, buftype;
1262         int written = 0;
1264         // ensure there is enough space for a whole frame
1265         if (count < userspace_buffer_size || *offset) {
1266                 return -EINVAL;
1267         }
1269         // sleep until the condition is true or a signal is received
1270         // the condition is checked each time gator_buffer_wait is woken up
1271         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1273         if (signal_pending(current)) {
1274                 return -EINTR;
1275         }
1277         if (buftype == -1 || cpu == -1) {
1278                 return 0;
1279         }
1281         mutex_lock(&gator_buffer_mutex);
1283         do {
1284                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1285                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1287                 // May happen if the buffer is freed during pending reads.
1288                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1289                         break;
1290                 }
1292                 // determine the size of two halves
1293                 length1 = commit - read;
1294                 length2 = 0;
1295                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1296                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1297                 if (length1 < 0) {
1298                         length1 = gator_buffer_size[buftype] - read;
1299                         length2 = commit;
1300                 }
1302                 if (length1 + length2 > count - written) {
1303                         break;
1304                 }
1306                 // start, middle or end
1307                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1308                         break;
1309                 }
1311                 // possible wrap around
1312                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1313                         break;
1314                 }
1316                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1317                 written += length1 + length2;
1319                 // Wake up annotate_write if more space is available
1320                 if (buftype == ANNOTATE_BUF) {
1321                         wake_up(&gator_annotate_wait);
1322                 }
1323         } while (buffer_commit_ready(&cpu, &buftype));
1325         mutex_unlock(&gator_buffer_mutex);
1327         // kick just in case we've lost an SMP event
1328         wake_up(&gator_buffer_wait);
1330         return written > 0 ? written : -EFAULT;
1333 const struct file_operations gator_event_buffer_fops = {
1334         .open = userspace_buffer_open,
1335         .release = userspace_buffer_release,
1336         .read = userspace_buffer_read,
1337 };
1339 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1341         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1344 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1346         unsigned long val;
1347         int retval;
1349         if (*offset)
1350                 return -EINVAL;
1352         retval = gatorfs_ulong_from_user(&val, buf, count);
1353         if (retval)
1354                 return retval;
1356         retval = gator_set_backtrace(val);
1358         if (retval)
1359                 return retval;
1360         return count;
1363 static const struct file_operations depth_fops = {
1364         .read = depth_read,
1365         .write = depth_write
1366 };
1368 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1370         struct dentry *dir;
1371         struct gator_interface *gi;
1372         int cpu;
1374         /* reinitialize default values */
1375         gator_cpu_cores = 0;
1376         for_each_present_cpu(cpu) {
1377                 gator_cpu_cores++;
1378         }
1379         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1380         gator_response_type = 1;
1381         gator_live_rate = 0;
1383         gatorfs_create_file(sb, root, "enable", &enable_fops);
1384         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1385         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1386         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1387         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1388         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1389         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1390         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1391         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1392         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1394         // Annotate interface
1395         gator_annotate_create_files(sb, root);
1397         // Linux Events
1398         dir = gatorfs_mkdir(sb, root, "events");
1399         list_for_each_entry(gi, &gator_events, list)
1400                 if (gi->create_files)
1401                         gi->create_files(sb, dir);
1403         // Sched Events
1404         sched_trace_create_files(sb, dir);
1406         // Power interface
1407         gator_trace_power_create_files(sb, dir);
1410 /******************************************************************************
1411  * Module
1412  ******************************************************************************/
1413 static int __init gator_module_init(void)
1415         if (gatorfs_register()) {
1416                 return -1;
1417         }
1419         if (gator_init()) {
1420                 gatorfs_unregister();
1421                 return -1;
1422         }
1424         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1426         return 0;
1429 static void __exit gator_module_exit(void)
1431         del_timer_sync(&gator_buffer_wake_up_timer);
1432         tracepoint_synchronize_unregister();
1433         gator_exit();
1434         gatorfs_unregister();
1437 module_init(gator_module_init);
1438 module_exit(gator_module_exit);
1440 MODULE_LICENSE("GPL");
1441 MODULE_AUTHOR("ARM Ltd");
1442 MODULE_DESCRIPTION("Gator system profiler");