1 /**
2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 // This version must match the gator daemon version
11 static unsigned long gator_protocol_version = 14;
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <linux/suspend.h>
22 #include <linux/module.h>
23 #include <linux/perf_event.h>
24 #include <linux/utsname.h>
25 #include <asm/stacktrace.h>
26 #include <asm/uaccess.h>
28 #include "gator.h"
29 #include "gator_events.h"
31 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
32 #error kernels prior to 2.6.32 are not supported
33 #endif
35 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
36 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
37 #endif
39 #ifndef CONFIG_PROFILING
40 #error gator requires the kernel to have CONFIG_PROFILING defined
41 #endif
43 #ifndef CONFIG_HIGH_RES_TIMERS
44 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
45 #endif
47 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
48 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
49 #endif
51 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
52 #ifndef CONFIG_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
54 #elif !defined CONFIG_HW_PERF_EVENTS
55 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
56 #endif
57 #endif
59 /******************************************************************************
60 * DEFINES
61 ******************************************************************************/
62 #define SUMMARY_BUFFER_SIZE (1*1024)
63 #define BACKTRACE_BUFFER_SIZE (128*1024)
64 #define NAME_BUFFER_SIZE (64*1024)
65 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
66 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
67 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
68 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
69 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
70 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
72 #define NO_COOKIE 0U
73 #define INVALID_COOKIE ~0U
75 #define FRAME_SUMMARY 1
76 #define FRAME_BACKTRACE 2
77 #define FRAME_NAME 3
78 #define FRAME_COUNTER 4
79 #define FRAME_BLOCK_COUNTER 5
80 #define FRAME_ANNOTATE 6
81 #define FRAME_SCHED_TRACE 7
82 #define FRAME_GPU_TRACE 8
83 #define FRAME_IDLE 9
85 #define MESSAGE_END_BACKTRACE 1
87 #define MESSAGE_COOKIE 1
88 #define MESSAGE_THREAD_NAME 2
89 #define HRTIMER_CORE_NAME 3
91 #define MESSAGE_GPU_START 1
92 #define MESSAGE_GPU_STOP 2
94 #define MESSAGE_SCHED_SWITCH 1
95 #define MESSAGE_SCHED_EXIT 2
97 #define MESSAGE_IDLE_ENTER 1
98 #define MESSAGE_IDLE_EXIT 2
100 #define MAXSIZE_PACK32 5
101 #define MAXSIZE_PACK64 10
103 #define FRAME_HEADER_SIZE 3
105 #if defined(__arm__)
106 #define PC_REG regs->ARM_pc
107 #elif defined(__aarch64__)
108 #define PC_REG regs->pc
109 #else
110 #define PC_REG regs->ip
111 #endif
113 enum {
114 SUMMARY_BUF,
115 BACKTRACE_BUF,
116 NAME_BUF,
117 COUNTER_BUF,
118 BLOCK_COUNTER_BUF,
119 ANNOTATE_BUF,
120 SCHED_TRACE_BUF,
121 GPU_TRACE_BUF,
122 IDLE_BUF,
123 NUM_GATOR_BUFS
124 };
126 /******************************************************************************
127 * Globals
128 ******************************************************************************/
129 static unsigned long gator_cpu_cores;
130 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
131 static unsigned long userspace_buffer_size;
132 static unsigned long gator_backtrace_depth;
133 // How often to commit the buffers for live in nanoseconds
134 static u64 gator_live_rate;
136 static unsigned long gator_started;
137 static u64 gator_monotonic_started;
138 static unsigned long gator_buffer_opened;
139 static unsigned long gator_timer_count;
140 static unsigned long gator_response_type;
141 static DEFINE_MUTEX(start_mutex);
142 static DEFINE_MUTEX(gator_buffer_mutex);
144 bool event_based_sampling;
146 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
147 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
148 static struct timer_list gator_buffer_wake_up_timer;
149 static LIST_HEAD(gator_events);
151 static DEFINE_PER_CPU(u64, last_timestamp);
153 static bool printed_monotonic_warning;
155 static bool sent_core_name[NR_CPUS];
157 /******************************************************************************
158 * Prototypes
159 ******************************************************************************/
160 static void buffer_check(int cpu, int buftype, u64 time);
161 static void gator_commit_buffer(int cpu, int buftype, u64 time);
162 static int buffer_bytes_available(int cpu, int buftype);
163 static bool buffer_check_space(int cpu, int buftype, int bytes);
164 static int contiguous_space_available(int cpu, int bufytpe);
165 static void gator_buffer_write_packed_int(int cpu, int buftype, int x);
166 static void gator_buffer_write_packed_int64(int cpu, int buftype, long long x);
167 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
168 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
169 static void gator_add_trace(int cpu, unsigned long address);
170 static void gator_add_sample(int cpu, struct pt_regs *const regs);
171 static u64 gator_get_time(void);
173 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
174 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
175 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
176 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
177 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
178 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
179 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
182 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
183 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
184 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
186 // The buffer. Allocated in gator_op_setup
187 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
189 #if GATOR_LIVE
190 // The time after which the buffer should be committed for live display
191 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192 #endif
194 /******************************************************************************
195 * Application Includes
196 ******************************************************************************/
197 #include "gator_marshaling.c"
198 #include "gator_hrtimer_perf.c"
199 #include "gator_hrtimer_gator.c"
200 #include "gator_cookies.c"
201 #include "gator_trace_sched.c"
202 #include "gator_trace_power.c"
203 #include "gator_trace_gpu.c"
204 #include "gator_backtrace.c"
205 #include "gator_annotate.c"
206 #include "gator_fs.c"
207 #include "gator_pack.c"
209 /******************************************************************************
210 * Misc
211 ******************************************************************************/
213 const struct gator_cpu gator_cpus[] = {
214 {
215 .cpuid = ARM1136,
216 .core_name = "ARM1136",
217 .pmnc_name = "ARM_ARM11",
218 .pmnc_counters = 3,
219 },
220 {
221 .cpuid = ARM1156,
222 .core_name = "ARM1156",
223 .pmnc_name = "ARM_ARM11",
224 .pmnc_counters = 3,
225 },
226 {
227 .cpuid = ARM1176,
228 .core_name = "ARM1176",
229 .pmnc_name = "ARM_ARM11",
230 .pmnc_counters = 3,
231 },
232 {
233 .cpuid = ARM11MPCORE,
234 .core_name = "ARM11MPCore",
235 .pmnc_name = "ARM_ARM11MPCore",
236 .pmnc_counters = 3,
237 },
238 {
239 .cpuid = CORTEX_A5,
240 .core_name = "Cortex-A5",
241 .pmu_name = "ARMv7_Cortex_A5",
242 .pmnc_name = "ARM_Cortex-A5",
243 .pmnc_counters = 2,
244 },
245 {
246 .cpuid = CORTEX_A7,
247 .core_name = "Cortex-A7",
248 .pmu_name = "ARMv7_Cortex_A7",
249 .pmnc_name = "ARM_Cortex-A7",
250 .pmnc_counters = 4,
251 },
252 {
253 .cpuid = CORTEX_A8,
254 .core_name = "Cortex-A8",
255 .pmu_name = "ARMv7_Cortex_A8",
256 .pmnc_name = "ARM_Cortex-A8",
257 .pmnc_counters = 4,
258 },
259 {
260 .cpuid = CORTEX_A9,
261 .core_name = "Cortex-A9",
262 .pmu_name = "ARMv7_Cortex_A9",
263 .pmnc_name = "ARM_Cortex-A9",
264 .pmnc_counters = 6,
265 },
266 {
267 .cpuid = CORTEX_A15,
268 .core_name = "Cortex-A15",
269 .pmu_name = "ARMv7_Cortex_A15",
270 .pmnc_name = "ARM_Cortex-A15",
271 .pmnc_counters = 6,
272 },
273 {
274 .cpuid = SCORPION,
275 .core_name = "Scorpion",
276 .pmnc_name = "Scorpion",
277 .pmnc_counters = 4,
278 },
279 {
280 .cpuid = SCORPIONMP,
281 .core_name = "ScorpionMP",
282 .pmnc_name = "ScorpionMP",
283 .pmnc_counters = 4,
284 },
285 {
286 .cpuid = KRAITSIM,
287 .core_name = "KraitSIM",
288 .pmnc_name = "Krait",
289 .pmnc_counters = 4,
290 },
291 {
292 .cpuid = KRAIT,
293 .core_name = "Krait",
294 .pmnc_name = "Krait",
295 .pmnc_counters = 4,
296 },
297 {
298 .cpuid = KRAIT_S4_PRO,
299 .core_name = "Krait S4 Pro",
300 .pmnc_name = "Krait",
301 .pmnc_counters = 4,
302 },
303 {
304 .cpuid = CORTEX_A53,
305 .core_name = "Cortex-A53",
306 .pmnc_name = "ARM_Cortex-A53",
307 .pmnc_counters = 6,
308 },
309 {
310 .cpuid = CORTEX_A57,
311 .core_name = "Cortex-A57",
312 .pmnc_name = "ARM_Cortex-A57",
313 .pmnc_counters = 6,
314 },
315 {
316 .cpuid = AARCH64,
317 .core_name = "AArch64",
318 .pmnc_name = "ARM_AArch64",
319 .pmnc_counters = 6,
320 },
321 {
322 .cpuid = OTHER,
323 .core_name = "Other",
324 .pmnc_name = "Other",
325 .pmnc_counters = 6,
326 },
327 {}
328 };
330 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
331 {
332 int i;
334 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
335 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
336 if (gator_cpu->cpuid == cpuid) {
337 return gator_cpu;
338 }
339 }
341 return NULL;
342 }
344 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
345 {
346 int i;
348 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
349 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
350 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
351 return gator_cpu;
352 }
353 }
355 return NULL;
356 }
358 u32 gator_cpuid(void)
359 {
360 #if defined(__arm__) || defined(__aarch64__)
361 u32 val;
362 #if !defined(__aarch64__)
363 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
364 #else
365 asm volatile("mrs %0, midr_el1" : "=r" (val));
366 #endif
367 return (val >> 4) & 0xfff;
368 #else
369 return OTHER;
370 #endif
371 }
373 static void gator_buffer_wake_up(unsigned long data)
374 {
375 wake_up(&gator_buffer_wait);
376 }
378 /******************************************************************************
379 * Commit interface
380 ******************************************************************************/
381 static bool buffer_commit_ready(int *cpu, int *buftype)
382 {
383 int cpu_x, x;
384 for_each_present_cpu(cpu_x) {
385 for (x = 0; x < NUM_GATOR_BUFS; x++)
386 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
387 *cpu = cpu_x;
388 *buftype = x;
389 return true;
390 }
391 }
392 *cpu = -1;
393 *buftype = -1;
394 return false;
395 }
397 /******************************************************************************
398 * Buffer management
399 ******************************************************************************/
400 static int buffer_bytes_available(int cpu, int buftype)
401 {
402 int remaining, filled;
404 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
405 if (filled < 0) {
406 filled += gator_buffer_size[buftype];
407 }
409 remaining = gator_buffer_size[buftype] - filled;
411 if (per_cpu(buffer_space_available, cpu)[buftype]) {
412 // Give some extra room; also allows space to insert the overflow error packet
413 remaining -= 200;
414 } else {
415 // Hysteresis, prevents multiple overflow messages
416 remaining -= 2000;
417 }
419 return remaining;
420 }
422 static int contiguous_space_available(int cpu, int buftype)
423 {
424 int remaining = buffer_bytes_available(cpu, buftype);
425 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
426 if (remaining < contiguous)
427 return remaining;
428 else
429 return contiguous;
430 }
432 static bool buffer_check_space(int cpu, int buftype, int bytes)
433 {
434 int remaining = buffer_bytes_available(cpu, buftype);
436 if (remaining < bytes) {
437 per_cpu(buffer_space_available, cpu)[buftype] = false;
438 } else {
439 per_cpu(buffer_space_available, cpu)[buftype] = true;
440 }
442 return per_cpu(buffer_space_available, cpu)[buftype];
443 }
445 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
446 {
447 int i;
448 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
449 u32 mask = gator_buffer_mask[buftype];
450 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
452 for (i = 0; i < len; i++) {
453 buffer[write] = x[i];
454 write = (write + 1) & mask;
455 }
457 per_cpu(gator_buffer_write, cpu)[buftype] = write;
458 }
460 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
461 {
462 int len = strlen(x);
463 gator_buffer_write_packed_int(cpu, buftype, len);
464 gator_buffer_write_bytes(cpu, buftype, x, len);
465 }
467 static void gator_commit_buffer(int cpu, int buftype, u64 time)
468 {
469 int type_length, commit, length, byte;
471 if (!per_cpu(gator_buffer, cpu)[buftype])
472 return;
474 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
475 type_length = gator_response_type ? 1 : 0;
476 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
477 length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
478 if (length < 0) {
479 length += gator_buffer_size[buftype];
480 }
481 length = length - type_length - sizeof(s32);
483 if (length <= FRAME_HEADER_SIZE) {
484 // Nothing to write, only the frame header is present
485 return;
486 }
488 for (byte = 0; byte < sizeof(s32); byte++) {
489 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
490 }
492 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
494 #if GATOR_LIVE
495 if (gator_live_rate > 0) {
496 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
497 per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
498 }
499 }
500 #endif
502 marshal_frame(cpu, buftype);
504 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
505 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
506 }
508 static void buffer_check(int cpu, int buftype, u64 time)
509 {
510 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
511 if (filled < 0) {
512 filled += gator_buffer_size[buftype];
513 }
514 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
515 gator_commit_buffer(cpu, buftype, time);
516 }
517 }
519 static void gator_add_trace(int cpu, unsigned long address)
520 {
521 off_t offset = 0;
522 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
524 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
525 offset = address;
526 }
528 marshal_backtrace(offset & ~1, cookie);
529 }
531 static void gator_add_sample(int cpu, struct pt_regs *const regs)
532 {
533 bool inKernel;
534 unsigned long exec_cookie;
536 if (!regs)
537 return;
539 inKernel = !user_mode(regs);
540 exec_cookie = get_exec_cookie(cpu, current);
542 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
543 return;
545 if (inKernel) {
546 kernel_backtrace(cpu, regs);
547 } else {
548 // Cookie+PC
549 gator_add_trace(cpu, PC_REG);
551 // Backtrace
552 if (gator_backtrace_depth)
553 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
554 }
556 marshal_backtrace_footer();
557 }
559 /******************************************************************************
560 * hrtimer interrupt processing
561 ******************************************************************************/
562 static void gator_timer_interrupt(void)
563 {
564 struct pt_regs *const regs = get_irq_regs();
565 gator_backtrace_handler(regs);
566 }
568 void gator_backtrace_handler(struct pt_regs *const regs)
569 {
570 int cpu = get_physical_cpu();
572 // Output backtrace
573 gator_add_sample(cpu, regs);
575 // Collect counters
576 if (!per_cpu(collecting, cpu)) {
577 collect_counters();
578 }
579 }
581 static int gator_running;
583 // This function runs in interrupt context and on the appropriate core
584 static void gator_timer_offline(void *migrate)
585 {
586 struct gator_interface *gi;
587 int i, len, cpu = get_physical_cpu();
588 int *buffer;
589 u64 time;
591 gator_trace_sched_offline();
592 gator_trace_power_offline();
594 if (!migrate) {
595 gator_hrtimer_offline();
596 }
598 // Offline any events and output counters
599 time = gator_get_time();
600 if (marshal_event_header(time)) {
601 list_for_each_entry(gi, &gator_events, list) {
602 if (gi->offline) {
603 len = gi->offline(&buffer, migrate);
604 marshal_event(len, buffer);
605 }
606 }
607 // Only check after writing all counters so that time and corresponding counters appear in the same frame
608 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
609 }
611 // Flush all buffers on this core
612 for (i = 0; i < NUM_GATOR_BUFS; i++)
613 gator_commit_buffer(cpu, i, time);
614 }
616 // This function runs in interrupt context and may be running on a core other than core 'cpu'
617 static void gator_timer_offline_dispatch(int cpu, bool migrate)
618 {
619 struct gator_interface *gi;
621 list_for_each_entry(gi, &gator_events, list) {
622 if (gi->offline_dispatch) {
623 gi->offline_dispatch(cpu, migrate);
624 }
625 }
626 }
628 static void gator_timer_stop(void)
629 {
630 int cpu;
632 if (gator_running) {
633 on_each_cpu(gator_timer_offline, NULL, 1);
634 for_each_online_cpu(cpu) {
635 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
636 }
638 gator_running = 0;
639 gator_hrtimer_shutdown();
640 }
641 }
643 // This function runs in interrupt context and on the appropriate core
644 static void gator_timer_online(void *migrate)
645 {
646 struct gator_interface *gi;
647 int len, cpu = get_physical_cpu();
648 int *buffer;
649 u64 time;
651 gator_trace_power_online();
653 // online any events and output counters
654 time = gator_get_time();
655 if (marshal_event_header(time)) {
656 list_for_each_entry(gi, &gator_events, list) {
657 if (gi->online) {
658 len = gi->online(&buffer, migrate);
659 marshal_event(len, buffer);
660 }
661 }
662 // Only check after writing all counters so that time and corresponding counters appear in the same frame
663 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
664 }
666 if (!migrate) {
667 gator_hrtimer_online();
668 }
670 #if defined(__arm__) || defined(__aarch64__)
671 if (!sent_core_name[cpu]) {
672 const char *core_name = NULL;
673 const u32 cpuid = gator_cpuid();
674 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
675 char core_name_buf[32];
677 if (gator_cpu != NULL) {
678 core_name = gator_cpu->core_name;
679 } else {
680 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
681 core_name = core_name_buf;
682 }
684 marshal_core_name(cpuid, core_name);
685 sent_core_name[cpu] = true;
686 }
687 #endif
688 }
690 // This function runs in interrupt context and may be running on a core other than core 'cpu'
691 static void gator_timer_online_dispatch(int cpu, bool migrate)
692 {
693 struct gator_interface *gi;
695 list_for_each_entry(gi, &gator_events, list) {
696 if (gi->online_dispatch) {
697 gi->online_dispatch(cpu, migrate);
698 }
699 }
700 }
702 int gator_timer_start(unsigned long sample_rate)
703 {
704 int cpu;
706 if (gator_running) {
707 pr_notice("gator: already running\n");
708 return 0;
709 }
711 gator_running = 1;
713 // event based sampling trumps hr timer based sampling
714 if (event_based_sampling) {
715 sample_rate = 0;
716 }
718 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
719 return -1;
721 for_each_online_cpu(cpu) {
722 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
723 }
724 on_each_cpu(gator_timer_online, NULL, 1);
726 return 0;
727 }
729 static u64 gator_get_time(void)
730 {
731 struct timespec ts;
732 u64 timestamp;
733 u64 prev_timestamp;
734 u64 delta;
735 int cpu = smp_processor_id();
737 // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
738 getrawmonotonic(&ts);
739 timestamp = timespec_to_ns(&ts);
741 // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
742 // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
743 prev_timestamp = per_cpu(last_timestamp, cpu);
744 if (prev_timestamp <= timestamp) {
745 per_cpu(last_timestamp, cpu) = timestamp;
746 } else {
747 delta = prev_timestamp - timestamp;
748 // Log the error once
749 if (!printed_monotonic_warning && delta > 500000) {
750 printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
751 printed_monotonic_warning = true;
752 }
753 timestamp = prev_timestamp;
754 }
756 return timestamp - gator_monotonic_started;
757 }
759 /******************************************************************************
760 * cpu hotplug and pm notifiers
761 ******************************************************************************/
762 #include "gator_iks.c"
764 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
765 {
766 int cpu = lcpu_to_pcpu((long)hcpu);
768 switch (action) {
769 case CPU_DOWN_PREPARE:
770 case CPU_DOWN_PREPARE_FROZEN:
771 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
772 gator_timer_offline_dispatch(cpu, false);
773 break;
774 case CPU_ONLINE:
775 case CPU_ONLINE_FROZEN:
776 gator_timer_online_dispatch(cpu, false);
777 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
778 break;
779 }
781 return NOTIFY_OK;
782 }
784 static struct notifier_block __refdata gator_hotcpu_notifier = {
785 .notifier_call = gator_hotcpu_notify,
786 };
788 // n.b. calling "on_each_cpu" only runs on those that are online
789 // Registered linux events are not disabled, so their counters will continue to collect
790 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
791 {
792 int cpu;
794 switch (event) {
795 case PM_HIBERNATION_PREPARE:
796 case PM_SUSPEND_PREPARE:
797 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
798 unregister_scheduler_tracepoints();
799 on_each_cpu(gator_timer_offline, NULL, 1);
800 for_each_online_cpu(cpu) {
801 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
802 }
803 break;
804 case PM_POST_HIBERNATION:
805 case PM_POST_SUSPEND:
806 for_each_online_cpu(cpu) {
807 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
808 }
809 on_each_cpu(gator_timer_online, NULL, 1);
810 register_scheduler_tracepoints();
811 register_hotcpu_notifier(&gator_hotcpu_notifier);
812 break;
813 }
815 return NOTIFY_OK;
816 }
818 static struct notifier_block gator_pm_notifier = {
819 .notifier_call = gator_pm_notify,
820 };
822 static int gator_notifier_start(void)
823 {
824 int retval;
825 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
826 if (retval == 0)
827 retval = register_pm_notifier(&gator_pm_notifier);
828 return retval;
829 }
831 static void gator_notifier_stop(void)
832 {
833 unregister_pm_notifier(&gator_pm_notifier);
834 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
835 }
837 /******************************************************************************
838 * Main
839 ******************************************************************************/
840 static void gator_summary(void)
841 {
842 u64 timestamp, uptime;
843 struct timespec ts;
844 char uname_buf[512];
845 void (*m2b)(struct timespec *ts);
846 unsigned long flags;
848 snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
850 getnstimeofday(&ts);
851 timestamp = timespec_to_ns(&ts);
853 do_posix_clock_monotonic_gettime(&ts);
854 // monotonic_to_bootbased is not defined for some versions of Android
855 m2b = symbol_get(monotonic_to_bootbased);
856 if (m2b) {
857 m2b(&ts);
858 }
859 uptime = timespec_to_ns(&ts);
861 // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
862 local_irq_save(flags);
863 // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
864 gator_monotonic_started = 0;
865 gator_monotonic_started = gator_get_time();
866 local_irq_restore(flags);
868 marshal_summary(timestamp, uptime, uname_buf);
869 }
871 int gator_events_install(struct gator_interface *interface)
872 {
873 list_add_tail(&interface->list, &gator_events);
875 return 0;
876 }
878 int gator_events_get_key(void)
879 {
880 // key of zero is reserved as a timestamp
881 static int key = 1;
883 const int ret = key;
884 key += 2;
885 return ret;
886 }
888 static int gator_init(void)
889 {
890 int i;
892 calc_first_cluster_size();
894 // events sources (gator_events.h, generated by gator_events.sh)
895 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
896 if (gator_events_list[i])
897 gator_events_list[i]();
899 gator_trace_sched_init();
900 gator_trace_power_init();
902 return 0;
903 }
905 static void gator_exit(void)
906 {
907 struct gator_interface *gi;
909 list_for_each_entry(gi, &gator_events, list)
910 if (gi->shutdown)
911 gi->shutdown();
912 }
914 static int gator_start(void)
915 {
916 unsigned long cpu, i;
917 struct gator_interface *gi;
919 if (gator_migrate_start())
920 goto migrate_failure;
922 // Initialize the buffer with the frame type and core
923 for_each_present_cpu(cpu) {
924 for (i = 0; i < NUM_GATOR_BUFS; i++) {
925 marshal_frame(cpu, i);
926 }
927 per_cpu(last_timestamp, cpu) = 0;
928 }
929 printed_monotonic_warning = false;
931 // Capture the start time
932 gator_summary();
934 // start all events
935 list_for_each_entry(gi, &gator_events, list) {
936 if (gi->start && gi->start() != 0) {
937 struct list_head *ptr = gi->list.prev;
939 while (ptr != &gator_events) {
940 gi = list_entry(ptr, struct gator_interface, list);
942 if (gi->stop)
943 gi->stop();
945 ptr = ptr->prev;
946 }
947 goto events_failure;
948 }
949 }
951 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
952 if (cookies_initialize())
953 goto cookies_failure;
954 if (gator_annotate_start())
955 goto annotate_failure;
956 if (gator_trace_sched_start())
957 goto sched_failure;
958 if (gator_trace_power_start())
959 goto power_failure;
960 if (gator_trace_gpu_start())
961 goto gpu_failure;
962 if (gator_timer_start(gator_timer_count))
963 goto timer_failure;
964 if (gator_notifier_start())
965 goto notifier_failure;
967 return 0;
969 notifier_failure:
970 gator_timer_stop();
971 timer_failure:
972 gator_trace_gpu_stop();
973 gpu_failure:
974 gator_trace_power_stop();
975 power_failure:
976 gator_trace_sched_stop();
977 sched_failure:
978 gator_annotate_stop();
979 annotate_failure:
980 cookies_release();
981 cookies_failure:
982 // stop all events
983 list_for_each_entry(gi, &gator_events, list)
984 if (gi->stop)
985 gi->stop();
986 events_failure:
987 gator_migrate_stop();
988 migrate_failure:
990 return -1;
991 }
993 static void gator_stop(void)
994 {
995 struct gator_interface *gi;
997 gator_annotate_stop();
998 gator_trace_sched_stop();
999 gator_trace_power_stop();
1000 gator_trace_gpu_stop();
1002 // stop all interrupt callback reads before tearing down other interfaces
1003 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
1004 gator_timer_stop();
1006 // stop all events
1007 list_for_each_entry(gi, &gator_events, list)
1008 if (gi->stop)
1009 gi->stop();
1011 gator_migrate_stop();
1012 }
1014 /******************************************************************************
1015 * Filesystem
1016 ******************************************************************************/
1017 /* fopen("buffer") */
1018 static int gator_op_setup(void)
1019 {
1020 int err = 0;
1021 int cpu, i;
1023 mutex_lock(&start_mutex);
1025 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1026 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1028 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1029 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1031 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1032 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1034 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1035 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1037 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1038 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1040 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1041 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1043 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1044 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1046 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
1047 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
1049 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1050 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1052 // Initialize percpu per buffer variables
1053 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1054 // Verify buffers are a power of 2
1055 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1056 err = -ENOEXEC;
1057 goto setup_error;
1058 }
1060 for_each_present_cpu(cpu) {
1061 per_cpu(gator_buffer_read, cpu)[i] = 0;
1062 per_cpu(gator_buffer_write, cpu)[i] = 0;
1063 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1064 per_cpu(buffer_space_available, cpu)[i] = true;
1065 #if GATOR_LIVE
1066 per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1067 #endif
1069 // Annotation is a special case that only uses a single buffer
1070 if (cpu > 0 && i == ANNOTATE_BUF) {
1071 per_cpu(gator_buffer, cpu)[i] = NULL;
1072 continue;
1073 }
1075 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1076 if (!per_cpu(gator_buffer, cpu)[i]) {
1077 err = -ENOMEM;
1078 goto setup_error;
1079 }
1080 }
1081 }
1083 setup_error:
1084 mutex_unlock(&start_mutex);
1085 return err;
1086 }
1088 /* Actually start profiling (echo 1>/dev/gator/enable) */
1089 static int gator_op_start(void)
1090 {
1091 int err = 0;
1093 mutex_lock(&start_mutex);
1095 if (gator_started || gator_start())
1096 err = -EINVAL;
1097 else
1098 gator_started = 1;
1100 mutex_unlock(&start_mutex);
1102 return err;
1103 }
1105 /* echo 0>/dev/gator/enable */
1106 static void gator_op_stop(void)
1107 {
1108 mutex_lock(&start_mutex);
1110 if (gator_started) {
1111 gator_stop();
1113 mutex_lock(&gator_buffer_mutex);
1115 gator_started = 0;
1116 gator_monotonic_started = 0;
1117 cookies_release();
1118 wake_up(&gator_buffer_wait);
1120 mutex_unlock(&gator_buffer_mutex);
1121 }
1123 mutex_unlock(&start_mutex);
1124 }
1126 static void gator_shutdown(void)
1127 {
1128 int cpu, i;
1130 mutex_lock(&start_mutex);
1132 for_each_present_cpu(cpu) {
1133 mutex_lock(&gator_buffer_mutex);
1134 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1135 vfree(per_cpu(gator_buffer, cpu)[i]);
1136 per_cpu(gator_buffer, cpu)[i] = NULL;
1137 per_cpu(gator_buffer_read, cpu)[i] = 0;
1138 per_cpu(gator_buffer_write, cpu)[i] = 0;
1139 per_cpu(gator_buffer_commit, cpu)[i] = 0;
1140 per_cpu(buffer_space_available, cpu)[i] = true;
1141 #if GATOR_LIVE
1142 per_cpu(gator_buffer_commit_time, cpu) = 0;
1143 #endif
1144 }
1145 mutex_unlock(&gator_buffer_mutex);
1146 }
1148 memset(&sent_core_name, 0, sizeof(sent_core_name));
1150 mutex_unlock(&start_mutex);
1151 }
1153 static int gator_set_backtrace(unsigned long val)
1154 {
1155 int err = 0;
1157 mutex_lock(&start_mutex);
1159 if (gator_started)
1160 err = -EBUSY;
1161 else
1162 gator_backtrace_depth = val;
1164 mutex_unlock(&start_mutex);
1166 return err;
1167 }
1169 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1170 {
1171 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1172 }
1174 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1175 {
1176 unsigned long val;
1177 int retval;
1179 if (*offset)
1180 return -EINVAL;
1182 retval = gatorfs_ulong_from_user(&val, buf, count);
1183 if (retval)
1184 return retval;
1186 if (val)
1187 retval = gator_op_start();
1188 else
1189 gator_op_stop();
1191 if (retval)
1192 return retval;
1193 return count;
1194 }
1196 static const struct file_operations enable_fops = {
1197 .read = enable_read,
1198 .write = enable_write,
1199 };
1201 static int userspace_buffer_open(struct inode *inode, struct file *file)
1202 {
1203 int err = -EPERM;
1205 if (!capable(CAP_SYS_ADMIN))
1206 return -EPERM;
1208 if (test_and_set_bit_lock(0, &gator_buffer_opened))
1209 return -EBUSY;
1211 if ((err = gator_op_setup()))
1212 goto fail;
1214 /* NB: the actual start happens from userspace
1215 * echo 1 >/dev/gator/enable
1216 */
1218 return 0;
1220 fail:
1221 __clear_bit_unlock(0, &gator_buffer_opened);
1222 return err;
1223 }
1225 static int userspace_buffer_release(struct inode *inode, struct file *file)
1226 {
1227 gator_op_stop();
1228 gator_shutdown();
1229 __clear_bit_unlock(0, &gator_buffer_opened);
1230 return 0;
1231 }
1233 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1234 {
1235 int commit, length1, length2, read;
1236 char *buffer1;
1237 char *buffer2;
1238 int cpu, buftype;
1239 int written = 0;
1241 // ensure there is enough space for a whole frame
1242 if (count < userspace_buffer_size || *offset) {
1243 return -EINVAL;
1244 }
1246 // sleep until the condition is true or a signal is received
1247 // the condition is checked each time gator_buffer_wait is woken up
1248 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1250 if (signal_pending(current)) {
1251 return -EINTR;
1252 }
1254 if (buftype == -1 || cpu == -1) {
1255 return 0;
1256 }
1258 mutex_lock(&gator_buffer_mutex);
1260 do {
1261 read = per_cpu(gator_buffer_read, cpu)[buftype];
1262 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1264 // May happen if the buffer is freed during pending reads.
1265 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1266 break;
1267 }
1269 // determine the size of two halves
1270 length1 = commit - read;
1271 length2 = 0;
1272 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1273 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1274 if (length1 < 0) {
1275 length1 = gator_buffer_size[buftype] - read;
1276 length2 = commit;
1277 }
1279 if (length1 + length2 > count - written) {
1280 break;
1281 }
1283 // start, middle or end
1284 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1)) {
1285 break;
1286 }
1288 // possible wrap around
1289 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2)) {
1290 break;
1291 }
1293 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1294 written += length1 + length2;
1296 // Wake up annotate_write if more space is available
1297 if (buftype == ANNOTATE_BUF) {
1298 wake_up(&gator_annotate_wait);
1299 }
1300 } while (buffer_commit_ready(&cpu, &buftype));
1302 mutex_unlock(&gator_buffer_mutex);
1304 // kick just in case we've lost an SMP event
1305 wake_up(&gator_buffer_wait);
1307 return written > 0 ? written : -EFAULT;
1308 }
1310 const struct file_operations gator_event_buffer_fops = {
1311 .open = userspace_buffer_open,
1312 .release = userspace_buffer_release,
1313 .read = userspace_buffer_read,
1314 };
1316 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1317 {
1318 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1319 }
1321 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1322 {
1323 unsigned long val;
1324 int retval;
1326 if (*offset)
1327 return -EINVAL;
1329 retval = gatorfs_ulong_from_user(&val, buf, count);
1330 if (retval)
1331 return retval;
1333 retval = gator_set_backtrace(val);
1335 if (retval)
1336 return retval;
1337 return count;
1338 }
1340 static const struct file_operations depth_fops = {
1341 .read = depth_read,
1342 .write = depth_write
1343 };
1345 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1346 {
1347 struct dentry *dir;
1348 struct gator_interface *gi;
1349 int cpu;
1351 /* reinitialize default values */
1352 gator_cpu_cores = 0;
1353 for_each_present_cpu(cpu) {
1354 gator_cpu_cores++;
1355 }
1356 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1357 gator_response_type = 1;
1358 gator_live_rate = 0;
1360 gatorfs_create_file(sb, root, "enable", &enable_fops);
1361 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1362 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1363 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1364 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1365 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1366 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1367 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1368 gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1369 gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1371 // Annotate interface
1372 gator_annotate_create_files(sb, root);
1374 // Linux Events
1375 dir = gatorfs_mkdir(sb, root, "events");
1376 list_for_each_entry(gi, &gator_events, list)
1377 if (gi->create_files)
1378 gi->create_files(sb, dir);
1380 // Sched Events
1381 sched_trace_create_files(sb, dir);
1383 // Power interface
1384 gator_trace_power_create_files(sb, dir);
1385 }
1387 /******************************************************************************
1388 * Module
1389 ******************************************************************************/
1390 static int __init gator_module_init(void)
1391 {
1392 if (gatorfs_register()) {
1393 return -1;
1394 }
1396 if (gator_init()) {
1397 gatorfs_unregister();
1398 return -1;
1399 }
1401 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1403 return 0;
1404 }
1406 static void __exit gator_module_exit(void)
1407 {
1408 del_timer_sync(&gator_buffer_wake_up_timer);
1409 tracepoint_synchronize_unregister();
1410 gator_exit();
1411 gatorfs_unregister();
1412 }
1414 module_init(gator_module_init);
1415 module_exit(gator_module_exit);
1417 MODULE_LICENSE("GPL");
1418 MODULE_AUTHOR("ARM Ltd");
1419 MODULE_DESCRIPTION("Gator system profiler");