46fe31d2505dbe4f49cfbfaba3c127e914b9d190
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 14;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
49 #endif
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
56 #endif
57 #endif
59 /******************************************************************************
60  * DEFINES
61  ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE       (1*1024)
63 #define BACKTRACE_BUFFER_SIZE     (128*1024)
64 #define NAME_BUFFER_SIZE          (64*1024)
65 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE      (64*1024)     // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
72 #define NO_COOKIE      0U
73 #define INVALID_COOKIE ~0U
75 #define FRAME_SUMMARY       1
76 #define FRAME_BACKTRACE     2
77 #define FRAME_NAME          3
78 #define FRAME_COUNTER       4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE      6
81 #define FRAME_SCHED_TRACE   7
82 #define FRAME_GPU_TRACE     8
83 #define FRAME_IDLE          9
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE      1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME   3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP  2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT   2
97 #define MESSAGE_IDLE_ENTER 1
98 #define MESSAGE_IDLE_EXIT 2
100 #define MAXSIZE_PACK32     5
101 #define MAXSIZE_PACK64    10
103 #define FRAME_HEADER_SIZE 3
105 #if defined(__arm__)
106 #define PC_REG regs->ARM_pc
107 #elif defined(__aarch64__)
108 #define PC_REG regs->pc
109 #else
110 #define PC_REG regs->ip
111 #endif
113 enum {
114         SUMMARY_BUF,
115         BACKTRACE_BUF,
116         NAME_BUF,
117         COUNTER_BUF,
118         BLOCK_COUNTER_BUF,
119         ANNOTATE_BUF,
120         SCHED_TRACE_BUF,
121         GPU_TRACE_BUF,
122         IDLE_BUF,
123         NUM_GATOR_BUFS
124 };
126 /******************************************************************************
127  * Globals
128  ******************************************************************************/
129 static unsigned long gator_cpu_cores;
130 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
131 static unsigned long userspace_buffer_size;
132 static unsigned long gator_backtrace_depth;
133 // How often to commit the buffers for live in nanoseconds
134 static u64 gator_live_rate;
136 static unsigned long gator_started;
137 static u64 gator_monotonic_started;
138 static unsigned long gator_buffer_opened;
139 static unsigned long gator_timer_count;
140 static unsigned long gator_response_type;
141 static DEFINE_MUTEX(start_mutex);
142 static DEFINE_MUTEX(gator_buffer_mutex);
144 bool event_based_sampling;
146 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
147 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
148 static struct timer_list gator_buffer_wake_up_timer;
149 static LIST_HEAD(gator_events);
151 static DEFINE_PER_CPU(u64, last_timestamp);
153 static bool printed_monotonic_warning;
155 static bool sent_core_name[NR_CPUS];
157 /******************************************************************************
158  * Prototypes
159  ******************************************************************************/
160 static void buffer_check(int cpu, int buftype, u64 time);
161 static void gator_commit_buffer(int cpu, int buftype, u64 time);
162 static int buffer_bytes_available(int cpu, int buftype);
163 static bool buffer_check_space(int cpu, int buftype, int bytes);
164 static int contiguous_space_available(int cpu, int bufytpe);
165 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
166 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
167 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
168 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
169 static void gator_add_trace(int cpu, unsigned long address);
170 static void gator_add_sample(int cpu, struct pt_regs *const regs);
171 static u64 gator_get_time(void);
173 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
174 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
175 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
176 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
177 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
178 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
179 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
182 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
183 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
184 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
186 // The buffer. Allocated in gator_op_setup
187 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 #if GATOR_LIVE
190 // The time after which the buffer should be committed for live display
191 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 #endif
194 /******************************************************************************
195  * Application Includes
196  ******************************************************************************/
197 #include "gator_marshaling.c"
198 #include "gator_hrtimer_perf.c"
199 #include "gator_hrtimer_gator.c"
200 #include "gator_cookies.c"
201 #include "gator_trace_sched.c"
202 #include "gator_trace_power.c"
203 #include "gator_trace_gpu.c"
204 #include "gator_backtrace.c"
205 #include "gator_annotate.c"
206 #include "gator_fs.c"
207 #include "gator_pack.c"
209 /******************************************************************************
210  * Misc
211  ******************************************************************************/
213 const struct gator_cpu gator_cpus[] = {
214         {
215                 .cpuid = ARM1136,
216                 .core_name = "ARM1136",
217                 .pmnc_name = "ARM_ARM11",
218                 .pmnc_counters = 3,
219         },
220         {
221                 .cpuid = ARM1156,
222                 .core_name = "ARM1156",
223                 .pmnc_name = "ARM_ARM11",
224                 .pmnc_counters = 3,
225         },
226         {
227                 .cpuid = ARM1176,
228                 .core_name = "ARM1176",
229                 .pmnc_name = "ARM_ARM11",
230                 .pmnc_counters = 3,
231         },
232         {
233                 .cpuid = ARM11MPCORE,
234                 .core_name = "ARM11MPCore",
235                 .pmnc_name = "ARM_ARM11MPCore",
236                 .pmnc_counters = 3,
237         },
238         {
239                 .cpuid = CORTEX_A5,
240                 .core_name = "Cortex-A5",
241                 .pmu_name = "ARMv7_Cortex_A5",
242                 .pmnc_name = "ARM_Cortex-A5",
243                 .pmnc_counters = 2,
244         },
245         {
246                 .cpuid = CORTEX_A7,
247                 .core_name = "Cortex-A7",
248                 .pmu_name = "ARMv7_Cortex_A7",
249                 .pmnc_name = "ARM_Cortex-A7",
250                 .pmnc_counters = 4,
251         },
252         {
253                 .cpuid = CORTEX_A8,
254                 .core_name = "Cortex-A8",
255                 .pmu_name = "ARMv7_Cortex_A8",
256                 .pmnc_name = "ARM_Cortex-A8",
257                 .pmnc_counters = 4,
258         },
259         {
260                 .cpuid = CORTEX_A9,
261                 .core_name = "Cortex-A9",
262                 .pmu_name = "ARMv7_Cortex_A9",
263                 .pmnc_name = "ARM_Cortex-A9",
264                 .pmnc_counters = 6,
265         },
266         {
267                 .cpuid = CORTEX_A15,
268                 .core_name = "Cortex-A15",
269                 .pmu_name = "ARMv7_Cortex_A15",
270                 .pmnc_name = "ARM_Cortex-A15",
271                 .pmnc_counters = 6,
272         },
273         {
274                 .cpuid = SCORPION,
275                 .core_name = "Scorpion",
276                 .pmnc_name = "Scorpion",
277                 .pmnc_counters = 4,
278         },
279         {
280                 .cpuid = SCORPIONMP,
281                 .core_name = "ScorpionMP",
282                 .pmnc_name = "ScorpionMP",
283                 .pmnc_counters = 4,
284         },
285         {
286                 .cpuid = KRAITSIM,
287                 .core_name = "KraitSIM",
288                 .pmnc_name = "Krait",
289                 .pmnc_counters = 4,
290         },
291         {
292                 .cpuid = KRAIT,
293                 .core_name = "Krait",
294                 .pmnc_name = "Krait",
295                 .pmnc_counters = 4,
296         },
297         {
298                 .cpuid = KRAIT_S4_PRO,
299                 .core_name = "Krait S4 Pro",
300                 .pmnc_name = "Krait",
301                 .pmnc_counters = 4,
302         },
303         {
304                 .cpuid = CORTEX_A53,
305                 .core_name = "Cortex-A53",
306                 .pmnc_name = "ARM_Cortex-A53",
307                 .pmnc_counters = 6,
308         },
309         {
310                 .cpuid = CORTEX_A57,
311                 .core_name = "Cortex-A57",
312                 .pmnc_name = "ARM_Cortex-A57",
313                 .pmnc_counters = 6,
314         },
315         {
316                 .cpuid = AARCH64,
317                 .core_name = "AArch64",
318                 .pmnc_name = "ARM_AArch64",
319                 .pmnc_counters = 6,
320         },
321         {
322                 .cpuid = OTHER,
323                 .core_name = "Other",
324                 .pmnc_name = "Other",
325                 .pmnc_counters = 6,
326         },
327         {}
328 };
330 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
332         int i;
334         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
335                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
336                 if (gator_cpu->cpuid == cpuid) {
337                         return gator_cpu;
338                 }
339         }
341         return NULL;
344 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
346         int i;
348         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
349                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
350                 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
351                         return gator_cpu;
352                 }
353         }
355         return NULL;
358 u32 gator_cpuid(void)
360 #if defined(__arm__) || defined(__aarch64__)
361         u32 val;
362 #if !defined(__aarch64__)
363         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
364 #else
365         asm volatile("mrs %0, midr_el1" : "=r" (val));
366 #endif
367         return (val >> 4) & 0xfff;
368 #else
369         return OTHER;
370 #endif
373 static void gator_buffer_wake_up(unsigned long data)
375         wake_up(&gator_buffer_wait);
378 /******************************************************************************
379  * Commit interface
380  ******************************************************************************/
381 static bool buffer_commit_ready(int *cpu, int *buftype)
383         int cpu_x, x;
384         for_each_present_cpu(cpu_x) {
385                 for (x = 0; x < NUM_GATOR_BUFS; x++)
386                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
387                                 *cpu = cpu_x;
388                                 *buftype = x;
389                                 return true;
390                         }
391         }
392         *cpu = -1;
393         *buftype = -1;
394         return false;
397 /******************************************************************************
398  * Buffer management
399  ******************************************************************************/
400 static int buffer_bytes_available(int cpu, int buftype)
402         int remaining, filled;
404         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
405         if (filled < 0) {
406                 filled += gator_buffer_size[buftype];
407         }
409         remaining = gator_buffer_size[buftype] - filled;
411         if (per_cpu(buffer_space_available, cpu)[buftype]) {
412                 // Give some extra room; also allows space to insert the overflow error packet
413                 remaining -= 200;
414         } else {
415                 // Hysteresis, prevents multiple overflow messages
416                 remaining -= 2000;
417         }
419         return remaining;
422 static int contiguous_space_available(int cpu, int buftype)
424         int remaining = buffer_bytes_available(cpu, buftype);
425         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
426         if (remaining < contiguous)
427                 return remaining;
428         else
429                 return contiguous;
432 static bool buffer_check_space(int cpu, int buftype, int bytes)
434         int remaining = buffer_bytes_available(cpu, buftype);
436         if (remaining < bytes) {
437                 per_cpu(buffer_space_available, cpu)[buftype] = false;
438         } else {
439                 per_cpu(buffer_space_available, cpu)[buftype] = true;
440         }
442         return per_cpu(buffer_space_available, cpu)[buftype];
445 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
447         int i;
448         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
449         u32 mask = gator_buffer_mask[buftype];
450         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
452         for (i = 0; i < len; i++) {
453                 buffer[write] = x[i];
454                 write = (write + 1) & mask;
455         }
457         per_cpu(gator_buffer_write, cpu)[buftype] = write;
460 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
462         int len = strlen(x);
463         gator_buffer_write_packed_int(cpu, buftype, len);
464         gator_buffer_write_bytes(cpu, buftype, x, len);
467 static void gator_commit_buffer(int cpu, int buftype, u64 time)
469         int type_length, commit, length, byte;
471         if (!per_cpu(gator_buffer, cpu)[buftype])
472                 return;
474         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
475         type_length = gator_response_type ? 1 : 0;
476         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
477         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
478         if (length < 0) {
479                 length += gator_buffer_size[buftype];
480         }
481         length = length - type_length - sizeof(s32);
483         if (length <= FRAME_HEADER_SIZE) {
484                 // Nothing to write, only the frame header is present
485                 return;
486         }
488         for (byte = 0; byte < sizeof(s32); byte++) {
489                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
490         }
492         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
494 #if GATOR_LIVE
495         if (gator_live_rate > 0) {
496                 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
497                         per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
498                 }
499         }
500 #endif
502         marshal_frame(cpu, buftype);
504         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
505         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
508 static void buffer_check(int cpu, int buftype, u64 time)
510         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
511         if (filled < 0) {
512                 filled += gator_buffer_size[buftype];
513         }
514         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
515                 gator_commit_buffer(cpu, buftype, time);
516         }
519 static void gator_add_trace(int cpu, unsigned long address)
521         off_t offset = 0;
522         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
524         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
525                 offset = address;
526         }
528         marshal_backtrace(offset & ~1, cookie);
531 static void gator_add_sample(int cpu, struct pt_regs *const regs)
533         bool inKernel;
534         unsigned long exec_cookie;
536         if (!regs)
537                 return;
539         inKernel = !user_mode(regs);
540         exec_cookie = get_exec_cookie(cpu, current);
542         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
543                 return;
545         if (inKernel) {
546                 kernel_backtrace(cpu, regs);
547         } else {
548                 // Cookie+PC
549                 gator_add_trace(cpu, PC_REG);
551                 // Backtrace
552                 if (gator_backtrace_depth)
553                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
554         }
556         marshal_backtrace_footer();
559 /******************************************************************************
560  * hrtimer interrupt processing
561  ******************************************************************************/
562 static void gator_timer_interrupt(void)
564         struct pt_regs *const regs = get_irq_regs();
565         gator_backtrace_handler(regs);
568 void gator_backtrace_handler(struct pt_regs *const regs)
570         int cpu = get_physical_cpu();
572         // Output backtrace
573         gator_add_sample(cpu, regs);
575         // Collect counters
576         if (!per_cpu(collecting, cpu)) {
577                 collect_counters();
578         }
581 static int gator_running;
583 // This function runs in interrupt context and on the appropriate core
584 static void gator_timer_offline(void *migrate)
586         struct gator_interface *gi;
587         int i, len, cpu = get_physical_cpu();
588         int *buffer;
589         u64 time;
591         gator_trace_sched_offline();
592         gator_trace_power_offline();
594         if (!migrate) {
595                 gator_hrtimer_offline();
596         }
598         // Offline any events and output counters
599         time = gator_get_time();
600         if (marshal_event_header(time)) {
601                 list_for_each_entry(gi, &gator_events, list) {
602                         if (gi->offline) {
603                                 len = gi->offline(&buffer, migrate);
604                                 marshal_event(len, buffer);
605                         }
606                 }
607                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
608                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
609         }
611         // Flush all buffers on this core
612         for (i = 0; i < NUM_GATOR_BUFS; i++)
613                 gator_commit_buffer(cpu, i, time);
616 // This function runs in interrupt context and may be running on a core other than core 'cpu'
617 static void gator_timer_offline_dispatch(int cpu, bool migrate)
619         struct gator_interface *gi;
621         list_for_each_entry(gi, &gator_events, list) {
622                 if (gi->offline_dispatch) {
623                         gi->offline_dispatch(cpu, migrate);
624                 }
625         }
628 static void gator_timer_stop(void)
630         int cpu;
632         if (gator_running) {
633                 on_each_cpu(gator_timer_offline, NULL, 1);
634                 for_each_online_cpu(cpu) {
635                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
636                 }
638                 gator_running = 0;
639                 gator_hrtimer_shutdown();
640         }
643 // This function runs in interrupt context and on the appropriate core
644 static void gator_timer_online(void *migrate)
646         struct gator_interface *gi;
647         int len, cpu = get_physical_cpu();
648         int *buffer;
649         u64 time;
651         gator_trace_power_online();
653         // online any events and output counters
654         time = gator_get_time();
655         if (marshal_event_header(time)) {
656                 list_for_each_entry(gi, &gator_events, list) {
657                         if (gi->online) {
658                                 len = gi->online(&buffer, migrate);
659                                 marshal_event(len, buffer);
660                         }
661                 }
662                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
663                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
664         }
666         if (!migrate) {
667                 gator_hrtimer_online();
668         }
670 #if defined(__arm__) || defined(__aarch64__)
671         if (!sent_core_name[cpu]) {
672                 const char *core_name = NULL;
673                 const u32 cpuid = gator_cpuid();
674                 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
675                 char core_name_buf[32];
677                 if (gator_cpu != NULL) {
678                         core_name = gator_cpu->core_name;
679                 } else {
680                         snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
681                         core_name = core_name_buf;
682                 }
684                 marshal_core_name(cpuid, core_name);
685                 sent_core_name[cpu] = true;
686         }
687 #endif
690 // This function runs in interrupt context and may be running on a core other than core 'cpu'
691 static void gator_timer_online_dispatch(int cpu, bool migrate)
693         struct gator_interface *gi;
695         list_for_each_entry(gi, &gator_events, list) {
696                 if (gi->online_dispatch) {
697                         gi->online_dispatch(cpu, migrate);
698                 }
699         }
702 int gator_timer_start(unsigned long sample_rate)
704         int cpu;
706         if (gator_running) {
707                 pr_notice("gator: already running\n");
708                 return 0;
709         }
711         gator_running = 1;
713         // event based sampling trumps hr timer based sampling
714         if (event_based_sampling) {
715                 sample_rate = 0;
716         }
718         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
719                 return -1;
721         for_each_online_cpu(cpu) {
722                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
723         }
724         on_each_cpu(gator_timer_online, NULL, 1);
726         return 0;
729 static u64 gator_get_time(void)
731         struct timespec ts;
732         u64 timestamp;
733         u64 prev_timestamp;
734         u64 delta;
735         int cpu = smp_processor_id();
737         // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
738         getrawmonotonic(&ts);
739         timestamp = timespec_to_ns(&ts);
741         // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
742         // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
743         prev_timestamp = per_cpu(last_timestamp, cpu);
744         if (prev_timestamp <= timestamp) {
745                 per_cpu(last_timestamp, cpu) = timestamp;
746         } else {
747                 delta = prev_timestamp - timestamp;
748                 // Log the error once
749                 if (!printed_monotonic_warning && delta > 500000) {
750                         printk(KERN_ERR "%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
751                         printed_monotonic_warning = true;
752                 }
753                 timestamp = prev_timestamp;
754         }
756         return timestamp - gator_monotonic_started;
759 /******************************************************************************
760  * cpu hotplug and pm notifiers
761  ******************************************************************************/
762 #include "gator_iks.c"
764 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
766         int cpu = lcpu_to_pcpu((long)hcpu);
768         switch (action) {
769         case CPU_DOWN_PREPARE:
770         case CPU_DOWN_PREPARE_FROZEN:
771                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
772                 gator_timer_offline_dispatch(cpu, false);
773                 break;
774         case CPU_ONLINE:
775         case CPU_ONLINE_FROZEN:
776                 gator_timer_online_dispatch(cpu, false);
777                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
778                 break;
779         }
781         return NOTIFY_OK;
784 static struct notifier_block __refdata gator_hotcpu_notifier = {
785         .notifier_call = gator_hotcpu_notify,
786 };
788 // n.b. calling "on_each_cpu" only runs on those that are online
789 // Registered linux events are not disabled, so their counters will continue to collect
790 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
792         int cpu;
794         switch (event) {
795         case PM_HIBERNATION_PREPARE:
796         case PM_SUSPEND_PREPARE:
797                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
798                 unregister_scheduler_tracepoints();
799                 on_each_cpu(gator_timer_offline, NULL, 1);
800                 for_each_online_cpu(cpu) {
801                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
802                 }
803                 break;
804         case PM_POST_HIBERNATION:
805         case PM_POST_SUSPEND:
806                 for_each_online_cpu(cpu) {
807                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
808                 }
809                 on_each_cpu(gator_timer_online, NULL, 1);
810                 register_scheduler_tracepoints();
811                 register_hotcpu_notifier(&gator_hotcpu_notifier);
812                 break;
813         }
815         return NOTIFY_OK;
818 static struct notifier_block gator_pm_notifier = {
819         .notifier_call = gator_pm_notify,
820 };
822 static int gator_notifier_start(void)
824         int retval;
825         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
826         if (retval == 0)
827                 retval = register_pm_notifier(&gator_pm_notifier);
828         return retval;
831 static void gator_notifier_stop(void)
833         unregister_pm_notifier(&gator_pm_notifier);
834         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
837 /******************************************************************************
838  * Main
839  ******************************************************************************/
840 static void gator_summary(void)
842         u64 timestamp, uptime;
843         struct timespec ts;
844         char uname_buf[512];
845         void (*m2b)(struct timespec *ts);
846         unsigned long flags;
848         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
850         getnstimeofday(&ts);
851         timestamp = timespec_to_ns(&ts);
853         do_posix_clock_monotonic_gettime(&ts);
854         // monotonic_to_bootbased is not defined for some versions of Android
855         m2b = symbol_get(monotonic_to_bootbased);
856         if (m2b) {
857                 m2b(&ts);
858         }
859         uptime = timespec_to_ns(&ts);
861         // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
862         local_irq_save(flags);
863         // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
864         gator_monotonic_started = 0;
865         gator_monotonic_started = gator_get_time();
866         local_irq_restore(flags);
868         marshal_summary(timestamp, uptime, uname_buf);
871 int gator_events_install(struct gator_interface *interface)
873         list_add_tail(&interface->list, &gator_events);
875         return 0;
878 int gator_events_get_key(void)
880         // key of zero is reserved as a timestamp
881         static int key = 1;
883         const int ret = key;
884         key += 2;
885         return ret;
888 static int gator_init(void)
890         int i;
892         calc_first_cluster_size();
894         // events sources (gator_events.h, generated by gator_events.sh)
895         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
896                 if (gator_events_list[i])
897                         gator_events_list[i]();
899         gator_trace_sched_init();
900         gator_trace_power_init();
902         return 0;
905 static void gator_exit(void)
907         struct gator_interface *gi;
909         list_for_each_entry(gi, &gator_events, list)
910                 if (gi->shutdown)
911                         gi->shutdown();
914 static int gator_start(void)
916         unsigned long cpu, i;
917         struct gator_interface *gi;
919         if (gator_migrate_start())
920                 goto migrate_failure;
922         // Initialize the buffer with the frame type and core
923         for_each_present_cpu(cpu) {
924                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
925                         marshal_frame(cpu, i);
926                 }
927                 per_cpu(last_timestamp, cpu) = 0;
928         }
929         printed_monotonic_warning = false;
931         // Capture the start time
932         gator_summary();
934         // start all events
935         list_for_each_entry(gi, &gator_events, list) {
936                 if (gi->start && gi->start() != 0) {
937                         struct list_head *ptr = gi->list.prev;
939                         while (ptr != &gator_events) {
940                                 gi = list_entry(ptr, struct gator_interface, list);
942                                 if (gi->stop)
943                                         gi->stop();
945                                 ptr = ptr->prev;
946                         }
947                         goto events_failure;
948                 }
949         }
951         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
952         if (cookies_initialize())
953                 goto cookies_failure;
954         if (gator_annotate_start())
955                 goto annotate_failure;
956         if (gator_trace_sched_start())
957                 goto sched_failure;
958         if (gator_trace_power_start())
959                 goto power_failure;
960         if (gator_trace_gpu_start())
961                 goto gpu_failure;
962         if (gator_timer_start(gator_timer_count))
963                 goto timer_failure;
964         if (gator_notifier_start())
965                 goto notifier_failure;
967         return 0;
969 notifier_failure:
970         gator_timer_stop();
971 timer_failure:
972         gator_trace_gpu_stop();
973 gpu_failure:
974         gator_trace_power_stop();
975 power_failure:
976         gator_trace_sched_stop();
977 sched_failure:
978         gator_annotate_stop();
979 annotate_failure:
980         cookies_release();
981 cookies_failure:
982         // stop all events
983         list_for_each_entry(gi, &gator_events, list)
984                 if (gi->stop)
985                         gi->stop();
986 events_failure:
987         gator_migrate_stop();
988 migrate_failure:
990         return -1;
993 static void gator_stop(void)
995         struct gator_interface *gi;
997         gator_annotate_stop();
998         gator_trace_sched_stop();
999         gator_trace_power_stop();
1000         gator_trace_gpu_stop();
1002         // stop all interrupt callback reads before tearing down other interfaces
1003         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1004         gator_timer_stop();
1006         // stop all events
1007         list_for_each_entry(gi, &gator_events, list)
1008                 if (gi->stop)
1009                         gi->stop();
1011         gator_migrate_stop();
1014 /******************************************************************************
1015  * Filesystem
1016  ******************************************************************************/
1017 /* fopen("buffer") */
1018 static int gator_op_setup(void)
1020         int err = 0;
1021         int cpu, i;
1023         mutex_lock(&start_mutex);
1025         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1026         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1028         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1029         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1031         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1032         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1034         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1035         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1037         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1038         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1040         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1041         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1043         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1044         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1046         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1047         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1049         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1050         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1052         // Initialize percpu per buffer variables
1053         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1054                 // Verify buffers are a power of 2
1055                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1056                         err = -ENOEXEC;
1057                         goto setup_error;
1058                 }
1060                 for_each_present_cpu(cpu) {
1061                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1062                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1063                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1064                         per_cpu(buffer_space_available, cpu)[i] = true;
1065 #if GATOR_LIVE
1066                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1067 #endif
1069                         // Annotation is a special case that only uses a single buffer
1070                         if (cpu > 0 && i == ANNOTATE_BUF) {
1071                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1072                                 continue;
1073                         }
1075                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1076                         if (!per_cpu(gator_buffer, cpu)[i]) {
1077                                 err = -ENOMEM;
1078                                 goto setup_error;
1079                         }
1080                 }
1081         }
1083 setup_error:
1084         mutex_unlock(&start_mutex);
1085         return err;
1088 /* Actually start profiling (echo 1>/dev/gator/enable) */
1089 static int gator_op_start(void)
1091         int err = 0;
1093         mutex_lock(&start_mutex);
1095         if (gator_started || gator_start())
1096                 err = -EINVAL;
1097         else
1098                 gator_started = 1;
1100         mutex_unlock(&start_mutex);
1102         return err;
1105 /* echo 0>/dev/gator/enable */
1106 static void gator_op_stop(void)
1108         mutex_lock(&start_mutex);
1110         if (gator_started) {
1111                 gator_stop();
1113                 mutex_lock(&gator_buffer_mutex);
1115                 gator_started = 0;
1116                 gator_monotonic_started = 0;
1117                 cookies_release();
1118                 wake_up(&gator_buffer_wait);
1120                 mutex_unlock(&gator_buffer_mutex);
1121         }
1123         mutex_unlock(&start_mutex);
1126 static void gator_shutdown(void)
1128         int cpu, i;
1130         mutex_lock(&start_mutex);
1132         for_each_present_cpu(cpu) {
1133                 mutex_lock(&gator_buffer_mutex);
1134                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1135                         vfree(per_cpu(gator_buffer, cpu)[i]);
1136                         per_cpu(gator_buffer, cpu)[i] = NULL;
1137                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1138                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1139                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1140                         per_cpu(buffer_space_available, cpu)[i] = true;
1141 #if GATOR_LIVE
1142                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1143 #endif
1144                 }
1145                 mutex_unlock(&gator_buffer_mutex);
1146         }
1148         memset(&sent_core_name, 0, sizeof(sent_core_name));
1150         mutex_unlock(&start_mutex);
1153 static int gator_set_backtrace(unsigned long val)
1155         int err = 0;
1157         mutex_lock(&start_mutex);
1159         if (gator_started)
1160                 err = -EBUSY;
1161         else
1162                 gator_backtrace_depth = val;
1164         mutex_unlock(&start_mutex);
1166         return err;
1169 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1171         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1174 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1176         unsigned long val;
1177         int retval;
1179         if (*offset)
1180                 return -EINVAL;
1182         retval = gatorfs_ulong_from_user(&val, buf, count);
1183         if (retval)
1184                 return retval;
1186         if (val)
1187                 retval = gator_op_start();
1188         else
1189                 gator_op_stop();
1191         if (retval)
1192                 return retval;
1193         return count;
1196 static const struct file_operations enable_fops = {
1197         .read = enable_read,
1198         .write = enable_write,
1199 };
1201 static int userspace_buffer_open(struct inode *inode, struct file *file)
1203         int err = -EPERM;
1205         if (!capable(CAP_SYS_ADMIN))
1206                 return -EPERM;
1208         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1209                 return -EBUSY;
1211         if ((err = gator_op_setup()))
1212                 goto fail;
1214         /* NB: the actual start happens from userspace
1215          * echo 1 >/dev/gator/enable
1216          */
1218         return 0;
1220 fail:
1221         __clear_bit_unlock(0, &gator_buffer_opened);
1222         return err;
1225 static int userspace_buffer_release(struct inode *inode, struct file *file)
1227         gator_op_stop();
1228         gator_shutdown();
1229         __clear_bit_unlock(0, &gator_buffer_opened);
1230         return 0;
1233 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1235         int commit, length1, length2, read;
1236         char *buffer1;
1237         char *buffer2;
1238         int cpu, buftype;
1239         int written = 0;
1241         // ensure there is enough space for a whole frame
1242         if (count < userspace_buffer_size || *offset) {
1243                 return -EINVAL;
1244         }
1246         // sleep until the condition is true or a signal is received
1247         // the condition is checked each time gator_buffer_wait is woken up
1248         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1250         if (signal_pending(current)) {
1251                 return -EINTR;
1252         }
1254         if (buftype == -1 || cpu == -1) {
1255                 return 0;
1256         }
1258         mutex_lock(&gator_buffer_mutex);
1260         do {
1261                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1262                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1264                 // May happen if the buffer is freed during pending reads.
1265                 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1266                         break;
1267                 }
1269                 // determine the size of two halves
1270                 length1 = commit - read;
1271                 length2 = 0;
1272                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1273                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1274                 if (length1 < 0) {
1275                         length1 = gator_buffer_size[buftype] - read;
1276                         length2 = commit;
1277                 }
1279                 if (length1 + length2 > count - written) {
1280                         break;
1281                 }
1283                 // start, middle or end
1284                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1285                         break;
1286                 }
1288                 // possible wrap around
1289                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1290                         break;
1291                 }
1293                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1294                 written += length1 + length2;
1296                 // Wake up annotate_write if more space is available
1297                 if (buftype == ANNOTATE_BUF) {
1298                         wake_up(&gator_annotate_wait);
1299                 }
1300         }       while (buffer_commit_ready(&cpu, &buftype));
1302         mutex_unlock(&gator_buffer_mutex);
1304         // kick just in case we've lost an SMP event
1305         wake_up(&gator_buffer_wait);
1307         return written > 0 ? written : -EFAULT;
1310 const struct file_operations gator_event_buffer_fops = {
1311         .open = userspace_buffer_open,
1312         .release = userspace_buffer_release,
1313         .read = userspace_buffer_read,
1314 };
1316 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1318         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1321 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1323         unsigned long val;
1324         int retval;
1326         if (*offset)
1327                 return -EINVAL;
1329         retval = gatorfs_ulong_from_user(&val, buf, count);
1330         if (retval)
1331                 return retval;
1333         retval = gator_set_backtrace(val);
1335         if (retval)
1336                 return retval;
1337         return count;
1340 static const struct file_operations depth_fops = {
1341         .read = depth_read,
1342         .write = depth_write
1343 };
1345 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1347         struct dentry *dir;
1348         struct gator_interface *gi;
1349         int cpu;
1351         /* reinitialize default values */
1352         gator_cpu_cores = 0;
1353         for_each_present_cpu(cpu) {
1354                 gator_cpu_cores++;
1355         }
1356         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1357         gator_response_type = 1;
1358         gator_live_rate = 0;
1360         gatorfs_create_file(sb, root, "enable", &enable_fops);
1361         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1362         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1363         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1364         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1365         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1366         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1367         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1368         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1369         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1371         // Annotate interface
1372         gator_annotate_create_files(sb, root);
1374         // Linux Events
1375         dir = gatorfs_mkdir(sb, root, "events");
1376         list_for_each_entry(gi, &gator_events, list)
1377                 if (gi->create_files)
1378                         gi->create_files(sb, dir);
1380         // Sched Events
1381         sched_trace_create_files(sb, dir);
1383         // Power interface
1384         gator_trace_power_create_files(sb, dir);
1387 /******************************************************************************
1388  * Module
1389  ******************************************************************************/
1390 static int __init gator_module_init(void)
1392         if (gatorfs_register()) {
1393                 return -1;
1394         }
1396         if (gator_init()) {
1397                 gatorfs_unregister();
1398                 return -1;
1399         }
1401         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1403         return 0;
1406 static void __exit gator_module_exit(void)
1408         del_timer_sync(&gator_buffer_wake_up_timer);
1409         tracepoint_synchronize_unregister();
1410         gator_exit();
1411         gatorfs_unregister();
1414 module_init(gator_module_init);
1415 module_exit(gator_module_exit);
1417 MODULE_LICENSE("GPL");
1418 MODULE_AUTHOR("ARM Ltd");
1419 MODULE_DESCRIPTION("Gator system profiler");