1 /**
2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 13;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
49 #endif
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
56 #endif
57 #endif
59 /******************************************************************************
60 * DEFINES
61 ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE (1*1024)
63 #define BACKTRACE_BUFFER_SIZE (128*1024)
64 #define NAME_BUFFER_SIZE (64*1024)
65 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
72 #define NO_COOKIE 0U
73 #define INVALID_COOKIE ~0U
75 #define FRAME_SUMMARY 1
76 #define FRAME_BACKTRACE 2
77 #define FRAME_NAME 3
78 #define FRAME_COUNTER 4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE 6
81 #define FRAME_SCHED_TRACE 7
82 #define FRAME_GPU_TRACE 8
83 #define FRAME_IDLE 9
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE 1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME 3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP 2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT 2
97 #define MESSAGE_IDLE_ENTER 1
98 #define MESSAGE_IDLE_EXIT 2
100 #define MAXSIZE_PACK32 5
101 #define MAXSIZE_PACK64 10
103 #define FRAME_HEADER_SIZE 3
105 #if defined(__arm__)
106 #define PC_REG regs->ARM_pc
107 #elif defined(__aarch64__)
108 #define PC_REG regs->pc
109 #else
110 #define PC_REG regs->ip
111 #endif
113 enum {
114 SUMMARY_BUF,
115 BACKTRACE_BUF,
116 NAME_BUF,
117 COUNTER_BUF,
118 BLOCK_COUNTER_BUF,
119 ANNOTATE_BUF,
120 SCHED_TRACE_BUF,
121 GPU_TRACE_BUF,
122 IDLE_BUF,
123 NUM_GATOR_BUFS
124 };
126 /******************************************************************************
127 * Globals
128 ******************************************************************************/
129 static unsigned long gator_cpu_cores;
130 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
131 static unsigned long userspace_buffer_size;
132 static unsigned long gator_backtrace_depth;
133 // How often to commit the buffers for live in nanoseconds
134 static u64 gator_live_rate;
136 static unsigned long gator_started;
137 static u64 gator_monotonic_started;
138 static unsigned long gator_buffer_opened;
139 static unsigned long gator_timer_count;
140 static unsigned long gator_response_type;
141 static DEFINE_MUTEX(start_mutex);
142 static DEFINE_MUTEX(gator_buffer_mutex);
144 bool event_based_sampling;
146 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
147 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
148 static struct timer_list gator_buffer_wake_up_timer;
149 static LIST_HEAD(gator_events);
151 static DEFINE_PER_CPU(u64, last_timestamp);
153 static bool printed_monotonic_warning;
155 static bool sent_core_name[NR_CPUS];
157 /******************************************************************************
158 * Prototypes
159 ******************************************************************************/
160 static void buffer_check(int cpu, int buftype, u64 time);
161 static void gator_commit_buffer(int cpu, int buftype, u64 time);
162 static int buffer_bytes_available(int cpu, int buftype);
163 static bool buffer_check_space(int cpu, int buftype, int bytes);
164 static int contiguous_space_available(int cpu, int bufytpe);
165 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
166 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
167 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
168 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
169 static void gator_add_trace(int cpu, unsigned long address);
170 static void gator_add_sample(int cpu, struct pt_regs *const regs);
171 static u64 gator_get_time(void);
173 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
174 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
175 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
176 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
177 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
178 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
179 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
182 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
183 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
184 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
186 // The buffer. Allocated in gator_op_setup
187 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 #if GATOR_LIVE
190 // The time after which the buffer should be committed for live display
191 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 #endif
194 /******************************************************************************
195 * Application Includes
196 ******************************************************************************/
197 #include "gator_marshaling.c"
198 #include "gator_hrtimer_perf.c"
199 #include "gator_hrtimer_gator.c"
200 #include "gator_cookies.c"
201 #include "gator_trace_sched.c"
202 #include "gator_trace_power.c"
203 #include "gator_trace_gpu.c"
204 #include "gator_backtrace.c"
205 #include "gator_annotate.c"
206 #include "gator_fs.c"
207 #include "gator_pack.c"
209 /******************************************************************************
210 * Misc
211 ******************************************************************************/
213 const struct gator_cpu gator_cpus[] = {
214 {
215 .cpuid = ARM1136,
216 .core_name = "ARM1136",
217 .pmnc_name = "ARM_ARM11",
218 .pmnc_counters = 3,
219 },
220 {
221 .cpuid = ARM1156,
222 .core_name = "ARM1156",
223 .pmnc_name = "ARM_ARM11",
224 .pmnc_counters = 3,
225 },
226 {
227 .cpuid = ARM1176,
228 .core_name = "ARM1176",
229 .pmnc_name = "ARM_ARM11",
230 .pmnc_counters = 3,
231 },
232 {
233 .cpuid = ARM11MPCORE,
234 .core_name = "ARM11MPCore",
235 .pmnc_name = "ARM_ARM11MPCore",
236 .pmnc_counters = 3,
237 },
238 {
239 .cpuid = CORTEX_A5,
240 .core_name = "Cortex-A5",
241 .pmu_name = "ARMv7_Cortex_A5",
242 .pmnc_name = "ARM_Cortex-A5",
243 .pmnc_counters = 2,
244 },
245 {
246 .cpuid = CORTEX_A7,
247 .core_name = "Cortex-A7",
248 .pmu_name = "ARMv7_Cortex_A7",
249 .pmnc_name = "ARM_Cortex-A7",
250 .pmnc_counters = 4,
251 },
252 {
253 .cpuid = CORTEX_A8,
254 .core_name = "Cortex-A8",
255 .pmu_name = "ARMv7_Cortex_A8",
256 .pmnc_name = "ARM_Cortex-A8",
257 .pmnc_counters = 4,
258 },
259 {
260 .cpuid = CORTEX_A9,
261 .core_name = "Cortex-A9",
262 .pmu_name = "ARMv7_Cortex_A9",
263 .pmnc_name = "ARM_Cortex-A9",
264 .pmnc_counters = 6,
265 },
266 {
267 .cpuid = CORTEX_A15,
268 .core_name = "Cortex-A15",
269 .pmu_name = "ARMv7_Cortex_A15",
270 .pmnc_name = "ARM_Cortex-A15",
271 .pmnc_counters = 6,
272 },
273 {
274 .cpuid = SCORPION,
275 .core_name = "Scorpion",
276 .pmnc_name = "Scorpion",
277 .pmnc_counters = 4,
278 },
279 {
280 .cpuid = SCORPIONMP,
281 .core_name = "ScorpionMP",
282 .pmnc_name = "ScorpionMP",
283 .pmnc_counters = 4,
284 },
285 {
286 .cpuid = KRAITSIM,
287 .core_name = "KraitSIM",
288 .pmnc_name = "Krait",
289 .pmnc_counters = 4,
290 },
291 {
292 .cpuid = KRAIT,
293 .core_name = "Krait",
294 .pmnc_name = "Krait",
295 .pmnc_counters = 4,
296 },
297 {
298 .cpuid = KRAIT_S4_PRO,
299 .core_name = "Krait S4 Pro",
300 .pmnc_name = "Krait",
301 .pmnc_counters = 4,
302 },
303 {
304 .cpuid = CORTEX_A53,
305 .core_name = "Cortex-A53",
306 .pmnc_name = "ARM_Cortex-A53",
307 .pmnc_counters = 6,
308 },
309 {
310 .cpuid = CORTEX_A57,
311 .core_name = "Cortex-A57",
312 .pmnc_name = "ARM_Cortex-A57",
313 .pmnc_counters = 6,
314 },
315 {
316 .cpuid = AARCH64,
317 .core_name = "AArch64",
318 .pmnc_name = "ARM_AArch64",
319 .pmnc_counters = 6,
320 },
321 {
322 .cpuid = OTHER,
323 .core_name = "Other",
324 .pmnc_name = "Other",
325 .pmnc_counters = 6,
326 },
327 {}
328 };
330 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
331 {
332 int i;
334 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
335 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
336 if (gator_cpu->cpuid == cpuid) {
337 return gator_cpu;
338 }
339 }
341 return NULL;
342 }
344 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
345 {
346 int i;
348 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
349 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
350 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
351 return gator_cpu;
352 }
353 }
355 return NULL;
356 }
358 u32 gator_cpuid(void)
359 {
360 #if defined(__arm__) || defined(__aarch64__)
361 u32 val;
362 #if !defined(__aarch64__)
363 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
364 #else
365 asm volatile("mrs %0, midr_el1" : "=r" (val));
366 #endif
367 return (val >> 4) & 0xfff;
368 #else
369 return OTHER;
370 #endif
371 }
373 static void gator_buffer_wake_up(unsigned long data)
374 {
375 wake_up(&gator_buffer_wait);
376 }
378 /******************************************************************************
379 * Commit interface
380 ******************************************************************************/
381 static bool buffer_commit_ready(int *cpu, int *buftype)
382 {
383 int cpu_x, x;
384 for_each_present_cpu(cpu_x) {
385 for (x = 0; x < NUM_GATOR_BUFS; x++)
386 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
387 *cpu = cpu_x;
388 *buftype = x;
389 return true;
390 }
391 }
392 return false;
393 }
395 /******************************************************************************
396 * Buffer management
397 ******************************************************************************/
398 static int buffer_bytes_available(int cpu, int buftype)
399 {
400 int remaining, filled;
402 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
403 if (filled < 0) {
404 filled += gator_buffer_size[buftype];
405 }
407 remaining = gator_buffer_size[buftype] - filled;
409 if (per_cpu(buffer_space_available, cpu)[buftype]) {
410 // Give some extra room; also allows space to insert the overflow error packet
411 remaining -= 200;
412 } else {
413 // Hysteresis, prevents multiple overflow messages
414 remaining -= 2000;
415 }
417 return remaining;
418 }
420 static int contiguous_space_available(int cpu, int buftype)
421 {
422 int remaining = buffer_bytes_available(cpu, buftype);
423 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
424 if (remaining < contiguous)
425 return remaining;
426 else
427 return contiguous;
428 }
430 static bool buffer_check_space(int cpu, int buftype, int bytes)
431 {
432 int remaining = buffer_bytes_available(cpu, buftype);
434 if (remaining < bytes) {
435 per_cpu(buffer_space_available, cpu)[buftype] = false;
436 } else {
437 per_cpu(buffer_space_available, cpu)[buftype] = true;
438 }
440 return per_cpu(buffer_space_available, cpu)[buftype];
441 }
443 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
444 {
445 int i;
446 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
447 u32 mask = gator_buffer_mask[buftype];
448 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
450 for (i = 0; i < len; i++) {
451 buffer[write] = x[i];
452 write = (write + 1) & mask;
453 }
455 per_cpu(gator_buffer_write, cpu)[buftype] = write;
456 }
458 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
459 {
460 int len = strlen(x);
461 gator_buffer_write_packed_int(cpu, buftype, len);
462 gator_buffer_write_bytes(cpu, buftype, x, len);
463 }
465 static void gator_commit_buffer(int cpu, int buftype, u64 time)
466 {
467 int type_length, commit, length, byte;
469 if (!per_cpu(gator_buffer, cpu)[buftype])
470 return;
472 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
473 type_length = gator_response_type ? 1 : 0;
474 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
475 length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
476 if (length < 0) {
477 length += gator_buffer_size[buftype];
478 }
479 length = length - type_length - sizeof(int);
481 if (length <= FRAME_HEADER_SIZE) {
482 // Nothing to write, only the frame header is present
483 return;
484 }
486 for (byte = 0; byte < sizeof(int); byte++) {
487 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
488 }
490 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
492 #if GATOR_LIVE
493 if (gator_live_rate > 0) {
494 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
495 per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
496 }
497 }
498 #endif
500 marshal_frame(cpu, buftype);
502 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
503 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
504 }
506 static void buffer_check(int cpu, int buftype, u64 time)
507 {
508 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
509 if (filled < 0) {
510 filled += gator_buffer_size[buftype];
511 }
512 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
513 gator_commit_buffer(cpu, buftype, time);
514 }
515 }
517 static void gator_add_trace(int cpu, unsigned long address)
518 {
519 off_t offset = 0;
520 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
522 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
523 offset = address;
524 }
526 marshal_backtrace(offset & ~1, cookie);
527 }
529 static void gator_add_sample(int cpu, struct pt_regs *const regs)
530 {
531 bool inKernel;
532 unsigned long exec_cookie;
534 if (!regs)
535 return;
537 inKernel = !user_mode(regs);
538 exec_cookie = get_exec_cookie(cpu, current);
540 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
541 return;
543 if (inKernel) {
544 kernel_backtrace(cpu, regs);
545 } else {
546 // Cookie+PC
547 gator_add_trace(cpu, PC_REG);
549 // Backtrace
550 if (gator_backtrace_depth)
551 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
552 }
554 marshal_backtrace_footer();
555 }
557 /******************************************************************************
558 * hrtimer interrupt processing
559 ******************************************************************************/
560 static void gator_timer_interrupt(void)
561 {
562 struct pt_regs *const regs = get_irq_regs();
563 gator_backtrace_handler(regs);
564 }
566 void gator_backtrace_handler(struct pt_regs *const regs)
567 {
568 int cpu = get_physical_cpu();
570 // Output backtrace
571 gator_add_sample(cpu, regs);
573 // Collect counters
574 if (!per_cpu(collecting, cpu)) {
575 collect_counters();
576 }
577 }
579 static int gator_running;
581 // This function runs in interrupt context and on the appropriate core
582 static void gator_timer_offline(void *migrate)
583 {
584 struct gator_interface *gi;
585 int i, len, cpu = get_physical_cpu();
586 int *buffer;
587 u64 time;
589 gator_trace_sched_offline();
590 gator_trace_power_offline();
592 if (!migrate) {
593 gator_hrtimer_offline();
594 }
596 // Offline any events and output counters
597 time = gator_get_time();
598 if (marshal_event_header()) {
599 list_for_each_entry(gi, &gator_events, list) {
600 if (gi->offline) {
601 len = gi->offline(&buffer, migrate);
602 marshal_event(len, buffer);
603 }
604 }
605 // Only check after writing all counters so that time and corresponding counters appear in the same frame
606 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
607 }
609 // Flush all buffers on this core
610 for (i = 0; i < NUM_GATOR_BUFS; i++)
611 gator_commit_buffer(cpu, i, time);
612 }
614 // This function runs in interrupt context and may be running on a core other than core 'cpu'
615 static void gator_timer_offline_dispatch(int cpu, bool migrate)
616 {
617 struct gator_interface *gi;
619 list_for_each_entry(gi, &gator_events, list) {
620 if (gi->offline_dispatch) {
621 gi->offline_dispatch(cpu, migrate);
622 }
623 }
624 }
626 static void gator_timer_stop(void)
627 {
628 int cpu;
630 if (gator_running) {
631 on_each_cpu(gator_timer_offline, NULL, 1);
632 for_each_online_cpu(cpu) {
633 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
634 }
636 gator_running = 0;
637 gator_hrtimer_shutdown();
638 }
639 }
641 // This function runs in interrupt context and on the appropriate core
642 static void gator_timer_online(void *migrate)
643 {
644 struct gator_interface *gi;
645 int len, cpu = get_physical_cpu();
646 int *buffer;
648 gator_trace_power_online();
650 // online any events and output counters
651 if (marshal_event_header()) {
652 list_for_each_entry(gi, &gator_events, list) {
653 if (gi->online) {
654 len = gi->online(&buffer, migrate);
655 marshal_event(len, buffer);
656 }
657 }
658 // Only check after writing all counters so that time and corresponding counters appear in the same frame
659 buffer_check(cpu, BLOCK_COUNTER_BUF, gator_get_time());
660 }
662 if (!migrate) {
663 gator_hrtimer_online();
664 }
666 #if defined(__arm__) || defined(__aarch64__)
667 if (!sent_core_name[cpu]) {
668 const char *core_name = NULL;
669 const u32 cpuid = gator_cpuid();
670 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
671 char core_name_buf[32];
673 if (gator_cpu != NULL) {
674 core_name = gator_cpu->core_name;
675 } else {
676 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
677 core_name = core_name_buf;
678 }
680 marshal_core_name(cpuid, core_name);
681 sent_core_name[cpu] = true;
682 }
683 #endif
684 }
686 // This function runs in interrupt context and may be running on a core other than core 'cpu'
687 static void gator_timer_online_dispatch(int cpu, bool migrate)
688 {
689 struct gator_interface *gi;
691 list_for_each_entry(gi, &gator_events, list) {
692 if (gi->online_dispatch) {
693 gi->online_dispatch(cpu, migrate);
694 }
695 }
696 }
698 int gator_timer_start(unsigned long sample_rate)
699 {
700 int cpu;
702 if (gator_running) {
703 pr_notice("gator: already running\n");
704 return 0;
705 }
707 gator_running = 1;
709 // event based sampling trumps hr timer based sampling
710 if (event_based_sampling) {
711 sample_rate = 0;
712 }
714 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
715 return -1;
717 for_each_online_cpu(cpu) {
718 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
719 }
720 on_each_cpu(gator_timer_online, NULL, 1);
722 return 0;
723 }
725 static u64 gator_get_time(void)
726 {
727 struct timespec ts;
728 u64 timestamp;
729 u64 prev_timestamp;
730 u64 delta;
731 int cpu = smp_processor_id();
733 // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
734 getrawmonotonic(&ts);
735 timestamp = timespec_to_ns(&ts);
737 // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
738 // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
739 prev_timestamp = per_cpu(last_timestamp, cpu);
740 if (prev_timestamp <= timestamp) {
741 per_cpu(last_timestamp, cpu) = timestamp;
742 } else {
743 delta = prev_timestamp - timestamp;
744 // Log the error once
745 if (!printed_monotonic_warning && delta > 500000) {
746 printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
747 printed_monotonic_warning = true;
748 } else {
749 pr_debug("%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\n", __FUNCTION__, cpu, delta);
750 }
751 timestamp = prev_timestamp;
752 }
754 return timestamp - gator_monotonic_started;
755 }
757 /******************************************************************************
758 * cpu hotplug and pm notifiers
759 ******************************************************************************/
760 #include "gator_iks.c"
762 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
763 {
764 int cpu = lcpu_to_pcpu((long)hcpu);
766 switch (action) {
767 case CPU_DOWN_PREPARE:
768 case CPU_DOWN_PREPARE_FROZEN:
769 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
770 gator_timer_offline_dispatch(cpu, false);
771 break;
772 case CPU_ONLINE:
773 case CPU_ONLINE_FROZEN:
774 gator_timer_online_dispatch(cpu, false);
775 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
776 break;
777 }
779 return NOTIFY_OK;
780 }
782 static struct notifier_block __refdata gator_hotcpu_notifier = {
783 .notifier_call = gator_hotcpu_notify,
784 };
786 // n.b. calling "on_each_cpu" only runs on those that are online
787 // Registered linux events are not disabled, so their counters will continue to collect
788 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
789 {
790 int cpu;
792 switch (event) {
793 case PM_HIBERNATION_PREPARE:
794 case PM_SUSPEND_PREPARE:
795 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
796 unregister_scheduler_tracepoints();
797 on_each_cpu(gator_timer_offline, NULL, 1);
798 for_each_online_cpu(cpu) {
799 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
800 }
801 break;
802 case PM_POST_HIBERNATION:
803 case PM_POST_SUSPEND:
804 for_each_online_cpu(cpu) {
805 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
806 }
807 on_each_cpu(gator_timer_online, NULL, 1);
808 register_scheduler_tracepoints();
809 register_hotcpu_notifier(&gator_hotcpu_notifier);
810 break;
811 }
813 return NOTIFY_OK;
814 }
816 static struct notifier_block gator_pm_notifier = {
817 .notifier_call = gator_pm_notify,
818 };
820 static int gator_notifier_start(void)
821 {
822 int retval;
823 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
824 if (retval == 0)
825 retval = register_pm_notifier(&gator_pm_notifier);
826 return retval;
827 }
829 static void gator_notifier_stop(void)
830 {
831 unregister_pm_notifier(&gator_pm_notifier);
832 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
833 }
835 /******************************************************************************
836 * Main
837 ******************************************************************************/
838 static void gator_summary(void)
839 {
840 u64 timestamp, uptime;
841 struct timespec ts;
842 char uname_buf[512];
843 void (*m2b)(struct timespec *ts);
844 unsigned long flags;
846 snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
848 getnstimeofday(&ts);
849 timestamp = timespec_to_ns(&ts);
851 do_posix_clock_monotonic_gettime(&ts);
852 // monotonic_to_bootbased is not defined for some versions of Android
853 m2b = symbol_get(monotonic_to_bootbased);
854 if (m2b) {
855 m2b(&ts);
856 }
857 uptime = timespec_to_ns(&ts);
859 // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
860 local_irq_save(flags);
861 // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
862 gator_monotonic_started = 0;
863 gator_monotonic_started = gator_get_time();
864 local_irq_restore(flags);
866 marshal_summary(timestamp, uptime, uname_buf);
867 }
869 int gator_events_install(struct gator_interface *interface)
870 {
871 list_add_tail(&interface->list, &gator_events);
873 return 0;
874 }
876 int gator_events_get_key(void)
877 {
878 // key of zero is reserved as a timestamp
879 static int key = 1;
881 const int ret = key;
882 key += 2;
883 return ret;
884 }
886 static int gator_init(void)
887 {
888 int i;
890 calc_first_cluster_size();
892 // events sources (gator_events.h, generated by gator_events.sh)
893 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
894 if (gator_events_list[i])
895 gator_events_list[i]();
897 gator_trace_power_init();
899 return 0;
900 }
902 static void gator_exit(void)
903 {
904 struct gator_interface *gi;
906 list_for_each_entry(gi, &gator_events, list)
907 if (gi->shutdown)
908 gi->shutdown();
909 }
911 static int gator_start(void)
912 {
913 unsigned long cpu, i;
914 struct gator_interface *gi;
916 if (gator_migrate_start())
917 goto migrate_failure;
919 // Initialize the buffer with the frame type and core
920 for_each_present_cpu(cpu) {
921 for (i = 0; i < NUM_GATOR_BUFS; i++) {
922 marshal_frame(cpu, i);
923 }
924 per_cpu(last_timestamp, cpu) = 0;
925 }
926 printed_monotonic_warning = false;
928 // Capture the start time
929 gator_summary();
931 // start all events
932 list_for_each_entry(gi, &gator_events, list) {
933 if (gi->start && gi->start() != 0) {
934 struct list_head *ptr = gi->list.prev;
936 while (ptr != &gator_events) {
937 gi = list_entry(ptr, struct gator_interface, list);
939 if (gi->stop)
940 gi->stop();
942 ptr = ptr->prev;
943 }
944 goto events_failure;
945 }
946 }
948 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
949 if (cookies_initialize())
950 goto cookies_failure;
951 if (gator_annotate_start())
952 goto annotate_failure;
953 if (gator_trace_sched_start())
954 goto sched_failure;
955 if (gator_trace_power_start())
956 goto power_failure;
957 if (gator_trace_gpu_start())
958 goto gpu_failure;
959 if (gator_timer_start(gator_timer_count))
960 goto timer_failure;
961 if (gator_notifier_start())
962 goto notifier_failure;
964 return 0;
966 notifier_failure:
967 gator_timer_stop();
968 timer_failure:
969 gator_trace_gpu_stop();
970 gpu_failure:
971 gator_trace_power_stop();
972 power_failure:
973 gator_trace_sched_stop();
974 sched_failure:
975 gator_annotate_stop();
976 annotate_failure:
977 cookies_release();
978 cookies_failure:
979 // stop all events
980 list_for_each_entry(gi, &gator_events, list)
981 if (gi->stop)
982 gi->stop();
983 events_failure:
984 gator_migrate_stop();
985 migrate_failure:
987 return -1;
988 }
990 static void gator_stop(void)
991 {
992 struct gator_interface *gi;
994 gator_annotate_stop();
995 gator_trace_sched_stop();
996 gator_trace_power_stop();
997 gator_trace_gpu_stop();
999 // stop all interrupt callback reads before tearing down other interfaces
1000 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1001 gator_timer_stop();
1003 // stop all events
1004 list_for_each_entry(gi, &gator_events, list)
1005 if (gi->stop)
1006 gi->stop();
1008 gator_migrate_stop();
1009 }
1011 /******************************************************************************
1012 * Filesystem
1013 ******************************************************************************/
1014 /* fopen("buffer") */
1015 static int gator_op_setup(void)
1016 {
1017 int err = 0;
1018 int cpu, i;
1020 mutex_lock(&start_mutex);
1022 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1023 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1025 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1026 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1028 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1029 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1031 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1032 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1034 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1035 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1037 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1038 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1040 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1041 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1043 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1044 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1046 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1047 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1049 // Initialize percpu per buffer variables
1050 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1051 // Verify buffers are a power of 2
1052 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1053 err = -ENOEXEC;
1054 goto setup_error;
1055 }
1057 for_each_present_cpu(cpu) {
1058 per_cpu(gator_buffer_read, cpu)[i] = 0;
1059 per_cpu(gator_buffer_write, cpu)[i] = 0;
1060 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1061 per_cpu(buffer_space_available, cpu)[i] = true;
1062 #if GATOR_LIVE
1063 per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1064 #endif
1066 // Annotation is a special case that only uses a single buffer
1067 if (cpu > 0 && i == ANNOTATE_BUF) {
1068 per_cpu(gator_buffer, cpu)[i] = NULL;
1069 continue;
1070 }
1072 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1073 if (!per_cpu(gator_buffer, cpu)[i]) {
1074 err = -ENOMEM;
1075 goto setup_error;
1076 }
1077 }
1078 }
1080 setup_error:
1081 mutex_unlock(&start_mutex);
1082 return err;
1083 }
1085 /* Actually start profiling (echo 1>/dev/gator/enable) */
1086 static int gator_op_start(void)
1087 {
1088 int err = 0;
1090 mutex_lock(&start_mutex);
1092 if (gator_started || gator_start())
1093 err = -EINVAL;
1094 else
1095 gator_started = 1;
1097 mutex_unlock(&start_mutex);
1099 return err;
1100 }
1102 /* echo 0>/dev/gator/enable */
1103 static void gator_op_stop(void)
1104 {
1105 mutex_lock(&start_mutex);
1107 if (gator_started) {
1108 gator_stop();
1110 mutex_lock(&gator_buffer_mutex);
1112 gator_started = 0;
1113 gator_monotonic_started = 0;
1114 cookies_release();
1115 wake_up(&gator_buffer_wait);
1117 mutex_unlock(&gator_buffer_mutex);
1118 }
1120 mutex_unlock(&start_mutex);
1121 }
1123 static void gator_shutdown(void)
1124 {
1125 int cpu, i;
1127 mutex_lock(&start_mutex);
1129 for_each_present_cpu(cpu) {
1130 mutex_lock(&gator_buffer_mutex);
1131 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1132 vfree(per_cpu(gator_buffer, cpu)[i]);
1133 per_cpu(gator_buffer, cpu)[i] = NULL;
1134 per_cpu(gator_buffer_read, cpu)[i] = 0;
1135 per_cpu(gator_buffer_write, cpu)[i] = 0;
1136 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1137 per_cpu(buffer_space_available, cpu)[i] = true;
1138 #if GATOR_LIVE
1139 per_cpu(gator_buffer_commit_time, cpu) = 0;
1140 #endif
1141 }
1142 mutex_unlock(&gator_buffer_mutex);
1143 }
1145 memset(&sent_core_name, 0, sizeof(sent_core_name));
1147 mutex_unlock(&start_mutex);
1148 }
1150 static int gator_set_backtrace(unsigned long val)
1151 {
1152 int err = 0;
1154 mutex_lock(&start_mutex);
1156 if (gator_started)
1157 err = -EBUSY;
1158 else
1159 gator_backtrace_depth = val;
1161 mutex_unlock(&start_mutex);
1163 return err;
1164 }
1166 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1167 {
1168 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1169 }
1171 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1172 {
1173 unsigned long val;
1174 int retval;
1176 if (*offset)
1177 return -EINVAL;
1179 retval = gatorfs_ulong_from_user(&val, buf, count);
1180 if (retval)
1181 return retval;
1183 if (val)
1184 retval = gator_op_start();
1185 else
1186 gator_op_stop();
1188 if (retval)
1189 return retval;
1190 return count;
1191 }
1193 static const struct file_operations enable_fops = {
1194 .read = enable_read,
1195 .write = enable_write,
1196 };
1198 static int userspace_buffer_open(struct inode *inode, struct file *file)
1199 {
1200 int err = -EPERM;
1202 if (!capable(CAP_SYS_ADMIN))
1203 return -EPERM;
1205 if (test_and_set_bit_lock(0, &gator_buffer_opened))
1206 return -EBUSY;
1208 if ((err = gator_op_setup()))
1209 goto fail;
1211 /* NB: the actual start happens from userspace
1212 * echo 1 >/dev/gator/enable
1213 */
1215 return 0;
1217 fail:
1218 __clear_bit_unlock(0, &gator_buffer_opened);
1219 return err;
1220 }
1222 static int userspace_buffer_release(struct inode *inode, struct file *file)
1223 {
1224 gator_op_stop();
1225 gator_shutdown();
1226 __clear_bit_unlock(0, &gator_buffer_opened);
1227 return 0;
1228 }
1230 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
1231 size_t count, loff_t *offset)
1232 {
1233 int retval = -EINVAL;
1234 int commit = 0, length1, length2, read;
1235 char *buffer1;
1236 char *buffer2 = NULL;
1237 int cpu, buftype;
1239 /* do not handle partial reads */
1240 if (count != userspace_buffer_size || *offset)
1241 return -EINVAL;
1243 // sleep until the condition is true or a signal is received
1244 // the condition is checked each time gator_buffer_wait is woken up
1245 buftype = cpu = -1;
1246 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1248 if (signal_pending(current))
1249 return -EINTR;
1251 length2 = 0;
1252 retval = -EFAULT;
1254 mutex_lock(&gator_buffer_mutex);
1256 if (buftype == -1 || cpu == -1) {
1257 retval = 0;
1258 goto out;
1259 }
1261 read = per_cpu(gator_buffer_read, cpu)[buftype];
1262 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1264 /* May happen if the buffer is freed during pending reads. */
1265 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1266 retval = -EFAULT;
1267 goto out;
1268 }
1270 /* determine the size of two halves */
1271 length1 = commit - read;
1272 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1273 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1274 if (length1 < 0) {
1275 length1 = gator_buffer_size[buftype] - read;
1276 length2 = commit;
1277 }
1279 /* start, middle or end */
1280 if (length1 > 0) {
1281 if (copy_to_user(&buf[0], buffer1, length1)) {
1282 goto out;
1283 }
1284 }
1286 /* possible wrap around */
1287 if (length2 > 0) {
1288 if (copy_to_user(&buf[length1], buffer2, length2)) {
1289 goto out;
1290 }
1291 }
1293 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1294 retval = length1 + length2;
1296 /* kick just in case we've lost an SMP event */
1297 wake_up(&gator_buffer_wait);
1299 // Wake up annotate_write if more space is available
1300 if (buftype == ANNOTATE_BUF) {
1301 wake_up(&gator_annotate_wait);
1302 }
1304 out:
1305 mutex_unlock(&gator_buffer_mutex);
1306 return retval;
1307 }
1309 const struct file_operations gator_event_buffer_fops = {
1310 .open = userspace_buffer_open,
1311 .release = userspace_buffer_release,
1312 .read = userspace_buffer_read,
1313 };
1315 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1316 {
1317 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1318 }
1320 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1321 {
1322 unsigned long val;
1323 int retval;
1325 if (*offset)
1326 return -EINVAL;
1328 retval = gatorfs_ulong_from_user(&val, buf, count);
1329 if (retval)
1330 return retval;
1332 retval = gator_set_backtrace(val);
1334 if (retval)
1335 return retval;
1336 return count;
1337 }
1339 static const struct file_operations depth_fops = {
1340 .read = depth_read,
1341 .write = depth_write
1342 };
1344 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1345 {
1346 struct dentry *dir;
1347 struct gator_interface *gi;
1348 int cpu;
1350 /* reinitialize default values */
1351 gator_cpu_cores = 0;
1352 for_each_present_cpu(cpu) {
1353 gator_cpu_cores++;
1354 }
1355 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1356 gator_response_type = 1;
1357 gator_live_rate = 0;
1359 gatorfs_create_file(sb, root, "enable", &enable_fops);
1360 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1361 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1362 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1363 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1364 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1365 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1366 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1367 gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1368 gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1370 // Annotate interface
1371 gator_annotate_create_files(sb, root);
1373 // Linux Events
1374 dir = gatorfs_mkdir(sb, root, "events");
1375 list_for_each_entry(gi, &gator_events, list)
1376 if (gi->create_files)
1377 gi->create_files(sb, dir);
1379 // Power interface
1380 gator_trace_power_create_files(sb, dir);
1381 }
1383 /******************************************************************************
1384 * Module
1385 ******************************************************************************/
1386 static int __init gator_module_init(void)
1387 {
1388 if (gatorfs_register()) {
1389 return -1;
1390 }
1392 if (gator_init()) {
1393 gatorfs_unregister();
1394 return -1;
1395 }
1397 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1399 return 0;
1400 }
1402 static void __exit gator_module_exit(void)
1403 {
1404 del_timer_sync(&gator_buffer_wake_up_timer);
1405 tracepoint_synchronize_unregister();
1406 gator_exit();
1407 gatorfs_unregister();
1408 }
1410 module_init(gator_module_init);
1411 module_exit(gator_module_exit);
1413 MODULE_LICENSE("GPL");
1414 MODULE_AUTHOR("ARM Ltd");
1415 MODULE_DESCRIPTION("Gator system profiler");