1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 12;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58 * DEFINES
59 ******************************************************************************/
60 #define SUMMARY_BUFFER_SIZE (1*1024)
61 #define BACKTRACE_BUFFER_SIZE (128*1024)
62 #define NAME_BUFFER_SIZE (64*1024)
63 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
64 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
65 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
66 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
67 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
68 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
70 #define NO_COOKIE 0U
71 #define INVALID_COOKIE ~0U
73 #define FRAME_SUMMARY 1
74 #define FRAME_BACKTRACE 2
75 #define FRAME_NAME 3
76 #define FRAME_COUNTER 4
77 #define FRAME_BLOCK_COUNTER 5
78 #define FRAME_ANNOTATE 6
79 #define FRAME_SCHED_TRACE 7
80 #define FRAME_GPU_TRACE 8
81 #define FRAME_IDLE 9
83 #define MESSAGE_END_BACKTRACE 1
85 #define MESSAGE_COOKIE 1
86 #define MESSAGE_THREAD_NAME 2
87 #define HRTIMER_CORE_NAME 3
89 #define MESSAGE_GPU_START 1
90 #define MESSAGE_GPU_STOP 2
92 #define MESSAGE_SCHED_SWITCH 1
93 #define MESSAGE_SCHED_EXIT 2
95 #define MAXSIZE_PACK32 5
96 #define MAXSIZE_PACK64 10
98 #if defined(__arm__)
99 #define PC_REG regs->ARM_pc
100 #elif defined(__aarch64__)
101 #define PC_REG regs->pc
102 #else
103 #define PC_REG regs->ip
104 #endif
106 enum {
107 SUMMARY_BUF,
108 BACKTRACE_BUF,
109 NAME_BUF,
110 COUNTER_BUF,
111 BLOCK_COUNTER_BUF,
112 ANNOTATE_BUF,
113 SCHED_TRACE_BUF,
114 GPU_TRACE_BUF,
115 IDLE_BUF,
116 NUM_GATOR_BUFS
117 };
119 /******************************************************************************
120 * Globals
121 ******************************************************************************/
122 static unsigned long gator_cpu_cores;
123 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
124 static unsigned long userspace_buffer_size;
125 static unsigned long gator_backtrace_depth;
127 static unsigned long gator_started;
128 static uint64_t monotonic_started;
129 static unsigned long gator_buffer_opened;
130 static unsigned long gator_timer_count;
131 static unsigned long gator_response_type;
132 static DEFINE_MUTEX(start_mutex);
133 static DEFINE_MUTEX(gator_buffer_mutex);
135 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
136 static struct timer_list gator_buffer_wake_up_timer;
137 static LIST_HEAD(gator_events);
139 /******************************************************************************
140 * Prototypes
141 ******************************************************************************/
142 static void buffer_check(int cpu, int buftype);
143 static int buffer_bytes_available(int cpu, int buftype);
144 static bool buffer_check_space(int cpu, int buftype, int bytes);
145 static int contiguous_space_available(int cpu, int bufytpe);
146 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
147 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
148 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
149 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
150 static void gator_add_trace(int cpu, unsigned long address);
151 static void gator_add_sample(int cpu, struct pt_regs *const regs);
152 static uint64_t gator_get_time(void);
154 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
155 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
156 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
157 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
158 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
159 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
160 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
161 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
162 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
163 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
164 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
165 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
166 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
167 // The buffer. Allocated in gator_op_setup
168 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
170 /******************************************************************************
171 * Application Includes
172 ******************************************************************************/
173 #include "gator_marshaling.c"
174 #include "gator_hrtimer_perf.c"
175 #include "gator_hrtimer_gator.c"
176 #include "gator_cookies.c"
177 #include "gator_trace_sched.c"
178 #include "gator_trace_power.c"
179 #include "gator_trace_gpu.c"
180 #include "gator_backtrace.c"
181 #include "gator_annotate.c"
182 #include "gator_fs.c"
183 #include "gator_pack.c"
185 /******************************************************************************
186 * Misc
187 ******************************************************************************/
189 struct gator_cpu gator_cpus[] = {
190 {
191 .cpuid = ARM1136,
192 .core_name = "ARM1136",
193 .pmnc_name = "ARM_ARM11",
194 .pmnc_counters = 3,
195 .ccnt = 2,
196 },
197 {
198 .cpuid = ARM1156,
199 .core_name = "ARM1156",
200 .pmnc_name = "ARM_ARM11",
201 .pmnc_counters = 3,
202 .ccnt = 2,
203 },
204 {
205 .cpuid = ARM1176,
206 .core_name = "ARM1176",
207 .pmnc_name = "ARM_ARM11",
208 .pmnc_counters = 3,
209 .ccnt = 2,
210 },
211 {
212 .cpuid = ARM11MPCORE,
213 .core_name = "ARM11MPCore",
214 .pmnc_name = "ARM_ARM11MPCore",
215 .pmnc_counters = 3,
216 },
217 {
218 .cpuid = CORTEX_A5,
219 .core_name = "Cortex-A5",
220 .pmnc_name = "ARM_Cortex-A5",
221 .pmnc_counters = 2,
222 },
223 {
224 .cpuid = CORTEX_A7,
225 .core_name = "Cortex-A7",
226 .pmnc_name = "ARM_Cortex-A7",
227 .pmnc_counters = 4,
228 },
229 {
230 .cpuid = CORTEX_A8,
231 .core_name = "Cortex-A8",
232 .pmnc_name = "ARM_Cortex-A8",
233 .pmnc_counters = 4,
234 },
235 {
236 .cpuid = CORTEX_A9,
237 .core_name = "Cortex-A9",
238 .pmnc_name = "ARM_Cortex-A9",
239 .pmnc_counters = 6,
240 },
241 {
242 .cpuid = CORTEX_A15,
243 .core_name = "Cortex-A15",
244 .pmnc_name = "ARM_Cortex-A15",
245 .pmnc_counters = 6,
246 },
247 {
248 .cpuid = SCORPION,
249 .core_name = "Scorpion",
250 .pmnc_name = "Scorpion",
251 .pmnc_counters = 4,
252 },
253 {
254 .cpuid = SCORPIONMP,
255 .core_name = "ScorpionMP",
256 .pmnc_name = "ScorpionMP",
257 .pmnc_counters = 4,
258 },
259 {
260 .cpuid = KRAITSIM,
261 .core_name = "KraitSIM",
262 .pmnc_name = "Krait",
263 .pmnc_counters = 4,
264 },
265 {
266 .cpuid = KRAIT,
267 .core_name = "Krait",
268 .pmnc_name = "Krait",
269 .pmnc_counters = 4,
270 },
271 {
272 .cpuid = KRAIT_S4_PRO,
273 .core_name = "Krait S4 Pro",
274 .pmnc_name = "Krait",
275 .pmnc_counters = 4,
276 },
277 {
278 .cpuid = CORTEX_A53,
279 .core_name = "Cortex-A53",
280 .pmnc_name = "ARM_Cortex-A53",
281 .pmnc_counters = 6,
282 },
283 {
284 .cpuid = CORTEX_A57,
285 .core_name = "Cortex-A57",
286 .pmnc_name = "ARM_Cortex-A57",
287 .pmnc_counters = 6,
288 },
289 {
290 .cpuid = AARCH64,
291 .core_name = "AArch64",
292 .pmnc_name = "ARM_AArch64",
293 .pmnc_counters = 6,
294 },
295 {
296 .cpuid = OTHER,
297 .core_name = "Other",
298 .pmnc_name = "Other",
299 .pmnc_counters = 6,
300 },
301 {}
302 };
304 u32 gator_cpuid(void)
305 {
306 #if defined(__arm__) || defined(__aarch64__)
307 u32 val;
308 #if !defined(__aarch64__)
309 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
310 #else
311 asm volatile("mrs %0, midr_el1" : "=r" (val));
312 #endif
313 return (val >> 4) & 0xfff;
314 #else
315 return OTHER;
316 #endif
317 }
319 static void gator_buffer_wake_up(unsigned long data)
320 {
321 wake_up(&gator_buffer_wait);
322 }
324 /******************************************************************************
325 * Commit interface
326 ******************************************************************************/
327 static bool buffer_commit_ready(int *cpu, int *buftype)
328 {
329 int cpu_x, x;
330 for_each_present_cpu(cpu_x) {
331 for (x = 0; x < NUM_GATOR_BUFS; x++)
332 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
333 *cpu = cpu_x;
334 *buftype = x;
335 return true;
336 }
337 }
338 return false;
339 }
341 /******************************************************************************
342 * Buffer management
343 ******************************************************************************/
344 static int buffer_bytes_available(int cpu, int buftype)
345 {
346 int remaining, filled;
348 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
349 if (filled < 0) {
350 filled += gator_buffer_size[buftype];
351 }
353 remaining = gator_buffer_size[buftype] - filled;
355 if (per_cpu(buffer_space_available, cpu)[buftype]) {
356 // Give some extra room; also allows space to insert the overflow error packet
357 remaining -= 200;
358 } else {
359 // Hysteresis, prevents multiple overflow messages
360 remaining -= 2000;
361 }
363 return remaining;
364 }
366 static int contiguous_space_available(int cpu, int buftype)
367 {
368 int remaining = buffer_bytes_available(cpu, buftype);
369 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
370 if (remaining < contiguous)
371 return remaining;
372 else
373 return contiguous;
374 }
376 static bool buffer_check_space(int cpu, int buftype, int bytes)
377 {
378 int remaining = buffer_bytes_available(cpu, buftype);
380 if (remaining < bytes) {
381 per_cpu(buffer_space_available, cpu)[buftype] = false;
382 } else {
383 per_cpu(buffer_space_available, cpu)[buftype] = true;
384 }
386 return per_cpu(buffer_space_available, cpu)[buftype];
387 }
389 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
390 {
391 int i;
392 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
393 u32 mask = gator_buffer_mask[buftype];
394 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
396 for (i = 0; i < len; i++) {
397 buffer[write] = x[i];
398 write = (write + 1) & mask;
399 }
401 per_cpu(gator_buffer_write, cpu)[buftype] = write;
402 }
404 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
405 {
406 int len = strlen(x);
407 gator_buffer_write_packed_int(cpu, buftype, len);
408 gator_buffer_write_bytes(cpu, buftype, x, len);
409 }
411 static void gator_commit_buffer(int cpu, int buftype)
412 {
413 int type_length, commit, length, byte;
415 if (!per_cpu(gator_buffer, cpu)[buftype])
416 return;
418 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
419 type_length = gator_response_type ? 1 : 0;
420 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
421 length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
422 if (length < 0) {
423 length += gator_buffer_size[buftype];
424 }
425 length = length - type_length - sizeof(int);
426 for (byte = 0; byte < sizeof(int); byte++) {
427 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
428 }
430 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
431 marshal_frame(cpu, buftype);
433 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
434 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
435 }
437 static void buffer_check(int cpu, int buftype)
438 {
439 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
440 if (filled < 0) {
441 filled += gator_buffer_size[buftype];
442 }
443 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
444 gator_commit_buffer(cpu, buftype);
445 }
446 }
448 static void gator_add_trace(int cpu, unsigned long address)
449 {
450 off_t offset = 0;
451 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
453 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
454 offset = address;
455 }
457 marshal_backtrace(offset & ~1, cookie);
458 }
460 static void gator_add_sample(int cpu, struct pt_regs *const regs)
461 {
462 bool inKernel;
463 unsigned long exec_cookie;
465 if (!regs)
466 return;
468 inKernel = !user_mode(regs);
469 exec_cookie = get_exec_cookie(cpu, current);
471 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
472 return;
474 if (inKernel) {
475 kernel_backtrace(cpu, regs);
476 } else {
477 // Cookie+PC
478 gator_add_trace(cpu, PC_REG);
480 // Backtrace
481 if (gator_backtrace_depth)
482 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
483 }
485 marshal_backtrace_footer();
486 }
488 /******************************************************************************
489 * hrtimer interrupt processing
490 ******************************************************************************/
491 static void gator_timer_interrupt(void)
492 {
493 struct pt_regs *const regs = get_irq_regs();
494 gator_backtrace_handler(regs);
495 }
497 void gator_backtrace_handler(struct pt_regs *const regs)
498 {
499 int cpu = smp_processor_id();
501 // Output backtrace
502 gator_add_sample(cpu, regs);
504 // Collect counters
505 if (!per_cpu(collecting, cpu)) {
506 collect_counters();
507 }
508 }
510 static int gator_running;
512 // This function runs in interrupt context and on the appropriate core
513 static void gator_timer_offline(void *unused)
514 {
515 struct gator_interface *gi;
516 int i, len, cpu = smp_processor_id();
517 int *buffer;
519 gator_trace_sched_offline();
520 gator_trace_power_offline();
522 gator_hrtimer_offline(cpu);
524 // Offline any events and output counters
525 if (marshal_event_header()) {
526 list_for_each_entry(gi, &gator_events, list) {
527 if (gi->offline) {
528 len = gi->offline(&buffer);
529 marshal_event(len, buffer);
530 }
531 }
532 }
534 // Flush all buffers on this core
535 for (i = 0; i < NUM_GATOR_BUFS; i++)
536 gator_commit_buffer(cpu, i);
537 }
539 // This function runs in process context and may be running on a core other than core 'cpu'
540 static void gator_timer_offline_dispatch(int cpu)
541 {
542 struct gator_interface *gi;
544 list_for_each_entry(gi, &gator_events, list)
545 if (gi->offline_dispatch)
546 gi->offline_dispatch(cpu);
547 }
549 static void gator_timer_stop(void)
550 {
551 int cpu;
553 if (gator_running) {
554 on_each_cpu(gator_timer_offline, NULL, 1);
555 for_each_online_cpu(cpu) {
556 gator_timer_offline_dispatch(cpu);
557 }
559 gator_running = 0;
560 gator_hrtimer_shutdown();
561 }
562 }
564 // This function runs in interrupt context and on the appropriate core
565 static void gator_timer_online(void *unused)
566 {
567 struct gator_interface *gi;
568 int len, cpu = smp_processor_id();
569 int *buffer;
571 gator_trace_power_online();
573 // online any events and output counters
574 if (marshal_event_header()) {
575 list_for_each_entry(gi, &gator_events, list) {
576 if (gi->online) {
577 len = gi->online(&buffer);
578 marshal_event(len, buffer);
579 }
580 }
581 }
583 gator_hrtimer_online(cpu);
584 #if defined(__arm__) || defined(__aarch64__)
585 {
586 const char *core_name = "Unknown";
587 const u32 cpuid = gator_cpuid();
588 int i;
590 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
591 if (gator_cpus[i].cpuid == cpuid) {
592 core_name = gator_cpus[i].core_name;
593 break;
594 }
595 }
597 marshal_core_name(core_name);
598 }
599 #endif
600 }
602 // This function runs in interrupt context and may be running on a core other than core 'cpu'
603 static void gator_timer_online_dispatch(int cpu)
604 {
605 struct gator_interface *gi;
607 list_for_each_entry(gi, &gator_events, list)
608 if (gi->online_dispatch)
609 gi->online_dispatch(cpu);
610 }
612 int gator_timer_start(unsigned long sample_rate)
613 {
614 int cpu;
616 if (gator_running) {
617 pr_notice("gator: already running\n");
618 return 0;
619 }
621 gator_running = 1;
623 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
624 return -1;
626 for_each_online_cpu(cpu) {
627 gator_timer_online_dispatch(cpu);
628 }
629 on_each_cpu(gator_timer_online, NULL, 1);
631 return 0;
632 }
634 static uint64_t gator_get_time(void)
635 {
636 struct timespec ts;
637 uint64_t timestamp;
639 //getnstimeofday(&ts);
640 do_posix_clock_monotonic_gettime(&ts);
641 timestamp = timespec_to_ns(&ts) - monotonic_started;
643 return timestamp;
644 }
646 /******************************************************************************
647 * cpu hotplug and pm notifiers
648 ******************************************************************************/
649 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
650 {
651 long cpu = (long)hcpu;
653 switch (action) {
654 case CPU_DOWN_PREPARE:
655 case CPU_DOWN_PREPARE_FROZEN:
656 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
657 gator_timer_offline_dispatch(cpu);
658 break;
659 case CPU_ONLINE:
660 case CPU_ONLINE_FROZEN:
661 gator_timer_online_dispatch(cpu);
662 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
663 break;
664 }
666 return NOTIFY_OK;
667 }
669 static struct notifier_block __refdata gator_hotcpu_notifier = {
670 .notifier_call = gator_hotcpu_notify,
671 };
673 // n.b. calling "on_each_cpu" only runs on those that are online
674 // Registered linux events are not disabled, so their counters will continue to collect
675 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
676 {
677 int cpu;
679 switch (event) {
680 case PM_HIBERNATION_PREPARE:
681 case PM_SUSPEND_PREPARE:
682 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
683 unregister_scheduler_tracepoints();
684 on_each_cpu(gator_timer_offline, NULL, 1);
685 for_each_online_cpu(cpu) {
686 gator_timer_offline_dispatch(cpu);
687 }
688 break;
689 case PM_POST_HIBERNATION:
690 case PM_POST_SUSPEND:
691 for_each_online_cpu(cpu) {
692 gator_timer_online_dispatch(cpu);
693 }
694 on_each_cpu(gator_timer_online, NULL, 1);
695 register_scheduler_tracepoints();
696 register_hotcpu_notifier(&gator_hotcpu_notifier);
697 break;
698 }
700 return NOTIFY_OK;
701 }
703 static struct notifier_block gator_pm_notifier = {
704 .notifier_call = gator_pm_notify,
705 };
707 static int gator_notifier_start(void)
708 {
709 int retval;
710 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
711 if (retval == 0)
712 retval = register_pm_notifier(&gator_pm_notifier);
713 return retval;
714 }
716 static void gator_notifier_stop(void)
717 {
718 unregister_pm_notifier(&gator_pm_notifier);
719 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
720 }
722 /******************************************************************************
723 * Main
724 ******************************************************************************/
725 static void gator_summary(void)
726 {
727 uint64_t timestamp;
728 struct timespec uptime_ts;
730 timestamp = gator_get_time();
732 do_posix_clock_monotonic_gettime(&uptime_ts);
733 monotonic_started = timespec_to_ns(&uptime_ts);
735 marshal_summary(timestamp, monotonic_started);
736 }
738 int gator_events_install(struct gator_interface *interface)
739 {
740 list_add_tail(&interface->list, &gator_events);
742 return 0;
743 }
745 int gator_events_get_key(void)
746 {
747 // key of zero is reserved as a timestamp
748 static int key = 1;
750 return key++;
751 }
753 static int gator_init(void)
754 {
755 int i;
757 // events sources (gator_events.h, generated by gator_events.sh)
758 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
759 if (gator_events_list[i])
760 gator_events_list[i]();
762 gator_trace_power_init();
764 return 0;
765 }
767 static void gator_exit(void)
768 {
769 struct gator_interface *gi;
771 list_for_each_entry(gi, &gator_events, list)
772 if (gi->shutdown)
773 gi->shutdown();
774 }
776 static int gator_start(void)
777 {
778 unsigned long cpu, i;
779 struct gator_interface *gi;
781 // Initialize the buffer with the frame type and core
782 for_each_present_cpu(cpu) {
783 for (i = 0; i < NUM_GATOR_BUFS; i++) {
784 marshal_frame(cpu, i);
785 }
786 }
788 // Capture the start time
789 gator_summary();
791 // start all events
792 list_for_each_entry(gi, &gator_events, list) {
793 if (gi->start && gi->start() != 0) {
794 struct list_head *ptr = gi->list.prev;
796 while (ptr != &gator_events) {
797 gi = list_entry(ptr, struct gator_interface, list);
799 if (gi->stop)
800 gi->stop();
802 ptr = ptr->prev;
803 }
804 goto events_failure;
805 }
806 }
808 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
809 if (cookies_initialize())
810 goto cookies_failure;
811 if (gator_annotate_start())
812 goto annotate_failure;
813 if (gator_trace_sched_start())
814 goto sched_failure;
815 if (gator_trace_power_start())
816 goto power_failure;
817 if (gator_trace_gpu_start())
818 goto gpu_failure;
819 if (gator_timer_start(gator_timer_count))
820 goto timer_failure;
821 if (gator_notifier_start())
822 goto notifier_failure;
824 return 0;
826 notifier_failure:
827 gator_timer_stop();
828 timer_failure:
829 gator_trace_gpu_stop();
830 gpu_failure:
831 gator_trace_power_stop();
832 power_failure:
833 gator_trace_sched_stop();
834 sched_failure:
835 gator_annotate_stop();
836 annotate_failure:
837 cookies_release();
838 cookies_failure:
839 // stop all events
840 list_for_each_entry(gi, &gator_events, list)
841 if (gi->stop)
842 gi->stop();
843 events_failure:
845 return -1;
846 }
848 static void gator_stop(void)
849 {
850 struct gator_interface *gi;
852 gator_annotate_stop();
853 gator_trace_sched_stop();
854 gator_trace_power_stop();
855 gator_trace_gpu_stop();
857 // stop all interrupt callback reads before tearing down other interfaces
858 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
859 gator_timer_stop();
861 // stop all events
862 list_for_each_entry(gi, &gator_events, list)
863 if (gi->stop)
864 gi->stop();
865 }
867 /******************************************************************************
868 * Filesystem
869 ******************************************************************************/
870 /* fopen("buffer") */
871 static int gator_op_setup(void)
872 {
873 int err = 0;
874 int cpu, i;
876 mutex_lock(&start_mutex);
878 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
879 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
881 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
882 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
884 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
885 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
887 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
888 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
890 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
891 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
893 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
894 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
896 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
897 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
899 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
900 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
902 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
903 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
905 // Initialize percpu per buffer variables
906 for (i = 0; i < NUM_GATOR_BUFS; i++) {
907 // Verify buffers are a power of 2
908 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
909 err = -ENOEXEC;
910 goto setup_error;
911 }
913 for_each_present_cpu(cpu) {
914 per_cpu(gator_buffer_read, cpu)[i] = 0;
915 per_cpu(gator_buffer_write, cpu)[i] = 0;
916 per_cpu(gator_buffer_commit, cpu)[i] = 0;
917 per_cpu(buffer_space_available, cpu)[i] = true;
919 // Annotation is a special case that only uses a single buffer
920 if (cpu > 0 && i == ANNOTATE_BUF) {
921 per_cpu(gator_buffer, cpu)[i] = NULL;
922 continue;
923 }
925 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
926 if (!per_cpu(gator_buffer, cpu)[i]) {
927 err = -ENOMEM;
928 goto setup_error;
929 }
930 }
931 }
933 setup_error:
934 mutex_unlock(&start_mutex);
935 return err;
936 }
938 /* Actually start profiling (echo 1>/dev/gator/enable) */
939 static int gator_op_start(void)
940 {
941 int err = 0;
943 mutex_lock(&start_mutex);
945 if (gator_started || gator_start())
946 err = -EINVAL;
947 else
948 gator_started = 1;
950 mutex_unlock(&start_mutex);
952 return err;
953 }
955 /* echo 0>/dev/gator/enable */
956 static void gator_op_stop(void)
957 {
958 mutex_lock(&start_mutex);
960 if (gator_started) {
961 gator_stop();
963 mutex_lock(&gator_buffer_mutex);
965 gator_started = 0;
966 cookies_release();
967 wake_up(&gator_buffer_wait);
969 mutex_unlock(&gator_buffer_mutex);
970 }
972 mutex_unlock(&start_mutex);
973 }
975 static void gator_shutdown(void)
976 {
977 int cpu, i;
979 mutex_lock(&start_mutex);
981 for_each_present_cpu(cpu) {
982 mutex_lock(&gator_buffer_mutex);
983 for (i = 0; i < NUM_GATOR_BUFS; i++) {
984 vfree(per_cpu(gator_buffer, cpu)[i]);
985 per_cpu(gator_buffer, cpu)[i] = NULL;
986 per_cpu(gator_buffer_read, cpu)[i] = 0;
987 per_cpu(gator_buffer_write, cpu)[i] = 0;
988 per_cpu(gator_buffer_commit, cpu)[i] = 0;
989 per_cpu(buffer_space_available, cpu)[i] = true;
990 }
991 mutex_unlock(&gator_buffer_mutex);
992 }
994 mutex_unlock(&start_mutex);
995 }
997 static int gator_set_backtrace(unsigned long val)
998 {
999 int err = 0;
1001 mutex_lock(&start_mutex);
1003 if (gator_started)
1004 err = -EBUSY;
1005 else
1006 gator_backtrace_depth = val;
1008 mutex_unlock(&start_mutex);
1010 return err;
1011 }
1013 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1014 {
1015 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1016 }
1018 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1019 {
1020 unsigned long val;
1021 int retval;
1023 if (*offset)
1024 return -EINVAL;
1026 retval = gatorfs_ulong_from_user(&val, buf, count);
1027 if (retval)
1028 return retval;
1030 if (val)
1031 retval = gator_op_start();
1032 else
1033 gator_op_stop();
1035 if (retval)
1036 return retval;
1037 return count;
1038 }
1040 static const struct file_operations enable_fops = {
1041 .read = enable_read,
1042 .write = enable_write,
1043 };
1045 static int userspace_buffer_open(struct inode *inode, struct file *file)
1046 {
1047 int err = -EPERM;
1049 if (!capable(CAP_SYS_ADMIN))
1050 return -EPERM;
1052 if (test_and_set_bit_lock(0, &gator_buffer_opened))
1053 return -EBUSY;
1055 if ((err = gator_op_setup()))
1056 goto fail;
1058 /* NB: the actual start happens from userspace
1059 * echo 1 >/dev/gator/enable
1060 */
1062 return 0;
1064 fail:
1065 __clear_bit_unlock(0, &gator_buffer_opened);
1066 return err;
1067 }
1069 static int userspace_buffer_release(struct inode *inode, struct file *file)
1070 {
1071 gator_op_stop();
1072 gator_shutdown();
1073 __clear_bit_unlock(0, &gator_buffer_opened);
1074 return 0;
1075 }
1077 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
1078 size_t count, loff_t *offset)
1079 {
1080 int retval = -EINVAL;
1081 int commit = 0, length1, length2, read;
1082 char *buffer1;
1083 char *buffer2 = NULL;
1084 int cpu, buftype;
1086 /* do not handle partial reads */
1087 if (count != userspace_buffer_size || *offset)
1088 return -EINVAL;
1090 // sleep until the condition is true or a signal is received
1091 // the condition is checked each time gator_buffer_wait is woken up
1092 buftype = cpu = -1;
1093 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1095 if (signal_pending(current))
1096 return -EINTR;
1098 length2 = 0;
1099 retval = -EFAULT;
1101 mutex_lock(&gator_buffer_mutex);
1103 if (buftype == -1 || cpu == -1) {
1104 retval = 0;
1105 goto out;
1106 }
1108 read = per_cpu(gator_buffer_read, cpu)[buftype];
1109 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1111 /* May happen if the buffer is freed during pending reads. */
1112 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1113 retval = -EFAULT;
1114 goto out;
1115 }
1117 /* determine the size of two halves */
1118 length1 = commit - read;
1119 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1120 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1121 if (length1 < 0) {
1122 length1 = gator_buffer_size[buftype] - read;
1123 length2 = commit;
1124 }
1126 /* start, middle or end */
1127 if (length1 > 0) {
1128 if (copy_to_user(&buf[0], buffer1, length1)) {
1129 goto out;
1130 }
1131 }
1133 /* possible wrap around */
1134 if (length2 > 0) {
1135 if (copy_to_user(&buf[length1], buffer2, length2)) {
1136 goto out;
1137 }
1138 }
1140 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1141 retval = length1 + length2;
1143 /* kick just in case we've lost an SMP event */
1144 wake_up(&gator_buffer_wait);
1146 out:
1147 mutex_unlock(&gator_buffer_mutex);
1148 return retval;
1149 }
1151 const struct file_operations gator_event_buffer_fops = {
1152 .open = userspace_buffer_open,
1153 .release = userspace_buffer_release,
1154 .read = userspace_buffer_read,
1155 };
1157 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1158 {
1159 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1160 }
1162 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1163 {
1164 unsigned long val;
1165 int retval;
1167 if (*offset)
1168 return -EINVAL;
1170 retval = gatorfs_ulong_from_user(&val, buf, count);
1171 if (retval)
1172 return retval;
1174 retval = gator_set_backtrace(val);
1176 if (retval)
1177 return retval;
1178 return count;
1179 }
1181 static const struct file_operations depth_fops = {
1182 .read = depth_read,
1183 .write = depth_write
1184 };
1186 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1187 {
1188 struct dentry *dir;
1189 struct gator_interface *gi;
1190 int cpu;
1192 /* reinitialize default values */
1193 gator_cpu_cores = 0;
1194 for_each_present_cpu(cpu) {
1195 gator_cpu_cores++;
1196 }
1197 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1198 gator_response_type = 1;
1200 gatorfs_create_file(sb, root, "enable", &enable_fops);
1201 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1202 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1203 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1204 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1205 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1206 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1207 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1209 // Annotate interface
1210 gator_annotate_create_files(sb, root);
1212 // Linux Events
1213 dir = gatorfs_mkdir(sb, root, "events");
1214 list_for_each_entry(gi, &gator_events, list)
1215 if (gi->create_files)
1216 gi->create_files(sb, dir);
1218 // Power interface
1219 gator_trace_power_create_files(sb, dir);
1220 }
1222 /******************************************************************************
1223 * Module
1224 ******************************************************************************/
1225 static int __init gator_module_init(void)
1226 {
1227 if (gatorfs_register()) {
1228 return -1;
1229 }
1231 if (gator_init()) {
1232 gatorfs_unregister();
1233 return -1;
1234 }
1236 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1238 return 0;
1239 }
1241 static void __exit gator_module_exit(void)
1242 {
1243 del_timer_sync(&gator_buffer_wake_up_timer);
1244 tracepoint_synchronize_unregister();
1245 gator_exit();
1246 gatorfs_unregister();
1247 }
1249 module_init(gator_module_init);
1250 module_exit(gator_module_exit);
1252 MODULE_LICENSE("GPL");
1253 MODULE_AUTHOR("ARM Ltd");
1254 MODULE_DESCRIPTION("Gator system profiler");