1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 8;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 #if (!(GATOR_CPU_FREQ_SUPPORT))
58 #warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart
59 #endif
61 /******************************************************************************
62 * DEFINES
63 ******************************************************************************/
64 #define TIMER_BUFFER_SIZE_DEFAULT (512*1024)
65 #define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
67 #define NO_COOKIE 0UL
68 #define INVALID_COOKIE ~0UL
70 #define FRAME_HRTIMER 1
71 #define FRAME_EVENT 2
72 #define FRAME_ANNOTATE 3
74 #define MESSAGE_COOKIE 1
75 #define MESSAGE_COUNTERS 3
76 #define MESSAGE_START_BACKTRACE 5
77 #define MESSAGE_END_BACKTRACE 7
78 #define MESSAGE_SCHEDULER_TRACE 9
79 #define MESSAGE_PID_NAME 11
80 #define MESSAGE_GPU_TRACE 13
81 #define MESSAGE_OVERFLOW 127
83 #define MAXSIZE_PACK32 5
84 #define MAXSIZE_PACK64 9
86 #if defined(__arm__)
87 #define PC_REG regs->ARM_pc
88 #else
89 #define PC_REG regs->ip
90 #endif
92 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
94 /******************************************************************************
95 * Globals
96 ******************************************************************************/
97 static unsigned long gator_cpu_cores;
98 static unsigned long userspace_buffer_size;
99 static unsigned long gator_backtrace_depth;
101 static unsigned long gator_started;
102 static unsigned long gator_buffer_opened;
103 static unsigned long gator_timer_count;
104 static unsigned long gator_streaming;
105 static DEFINE_MUTEX(start_mutex);
106 static DEFINE_MUTEX(gator_buffer_mutex);
108 bool event_based_sampling;
110 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
112 static void buffer_check(int cpu, int buftype);
114 /******************************************************************************
115 * Prototypes
116 ******************************************************************************/
117 static bool buffer_check_space(int cpu, int buftype, int bytes);
118 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
119 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
120 static void gator_buffer_write_string(int cpu, int buftype, char *x);
121 static int gator_write_packed_int(char *buffer, unsigned int x);
122 static int gator_write_packed_int64(char *buffer, unsigned long long x);
123 static void gator_add_trace(int cpu, int buftype, unsigned int address);
124 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
125 static uint64_t gator_get_time(void);
127 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
128 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
129 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
130 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
131 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
132 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
133 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
134 static DEFINE_PER_CPU(uint64_t, emit_overflow);
136 /******************************************************************************
137 * Application Includes
138 ******************************************************************************/
139 #include "gator_hrtimer_perf.c"
140 #include "gator_hrtimer_gator.c"
141 #include "gator_cookies.c"
142 #include "gator_trace_sched.c"
143 #include "gator_trace_gpu.c"
144 #include "gator_backtrace.c"
145 #include "gator_annotate.c"
146 #include "gator_fs.c"
147 #include "gator_ebs.c"
148 #include "gator_pack.c"
150 /******************************************************************************
151 * Misc
152 ******************************************************************************/
153 #if defined(__arm__)
154 u32 gator_cpuid(void)
155 {
156 u32 val;
157 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
158 return (val >> 4) & 0xfff;
159 }
160 #endif
162 /******************************************************************************
163 * Commit interface
164 ******************************************************************************/
165 static bool buffer_commit_ready(int* cpu, int* buftype)
166 {
167 int cpu_x, x;
168 for_each_present_cpu(cpu_x) {
169 for (x = 0; x < NUM_GATOR_BUFS; x++)
170 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
171 *cpu = cpu_x;
172 *buftype = x;
173 return true;
174 }
175 }
176 return false;
177 }
179 /******************************************************************************
180 * Buffer management
181 ******************************************************************************/
182 static bool buffer_check_space(int cpu, int buftype, int bytes)
183 {
184 int remaining, filled;
186 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
187 if (filled < 0) {
188 filled += gator_buffer_size[buftype];
189 }
191 remaining = gator_buffer_size[buftype] - filled;
193 if (per_cpu(buffer_space_available, cpu)[buftype]) {
194 // Give some extra room; also allows space to insert the overflow error packet
195 remaining -= 200;
196 } else {
197 // Hysteresis, prevents multiple overflow messages
198 remaining -= 2000;
199 }
201 if (remaining < bytes) {
202 if (per_cpu(buffer_space_available, cpu)[buftype] == true) {
203 // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters
204 per_cpu(emit_overflow, cpu) = gator_get_time();
205 pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled);
206 }
207 per_cpu(buffer_space_available, cpu)[buftype] = false;
208 } else {
209 per_cpu(buffer_space_available, cpu)[buftype] = true;
210 }
212 return per_cpu(buffer_space_available, cpu)[buftype];
213 }
215 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
216 {
217 int i;
218 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
219 u32 mask = gator_buffer_mask[buftype];
220 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
222 for (i = 0; i < len; i++) {
223 buffer[write] = x[i];
224 write = (write + 1) & mask;
225 }
227 per_cpu(gator_buffer_write, cpu)[buftype] = write;
228 }
230 static void gator_buffer_write_string(int cpu, int buftype, char *x)
231 {
232 int len = strlen(x);
233 gator_buffer_write_packed_int(cpu, buftype, len);
234 gator_buffer_write_bytes(cpu, buftype, x, len);
235 }
237 static void gator_buffer_header(int cpu, int buftype)
238 {
239 int frame;
241 if (buftype == TIMER_BUF)
242 frame = FRAME_HRTIMER;
243 else if (buftype == EVENT_BUF)
244 frame = FRAME_EVENT;
245 else
246 frame = -1;
248 gator_buffer_write_packed_int(cpu, buftype, frame);
249 gator_buffer_write_packed_int(cpu, buftype, cpu);
250 }
252 static void gator_commit_buffer(int cpu, int buftype)
253 {
254 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
255 gator_buffer_header(cpu, buftype);
256 wake_up(&gator_buffer_wait);
257 }
259 static void buffer_check(int cpu, int buftype)
260 {
261 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
262 if (filled < 0) {
263 filled += gator_buffer_size[buftype];
264 }
265 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
266 gator_commit_buffer(cpu, buftype);
267 }
268 }
270 static void gator_add_trace(int cpu, int buftype, unsigned int address)
271 {
272 off_t offset = 0;
273 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
275 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
276 offset = address;
277 }
279 gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
280 gator_buffer_write_packed_int(cpu, buftype, cookie);
281 }
283 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
284 {
285 int inKernel = regs ? !user_mode(regs) : 1;
286 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
288 if (!regs)
289 return;
291 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
292 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
293 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
294 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
295 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
296 gator_buffer_write_packed_int(cpu, buftype, inKernel);
298 if (inKernel) {
299 kernel_backtrace(cpu, buftype, regs);
300 } else {
301 // Cookie+PC
302 gator_add_trace(cpu, buftype, PC_REG);
304 // Backtrace
305 if (gator_backtrace_depth)
306 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
307 }
309 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
310 }
312 /******************************************************************************
313 * hrtimer interrupt processing
314 ******************************************************************************/
315 static LIST_HEAD(gator_events);
317 static void gator_timer_interrupt(void)
318 {
319 struct pt_regs * const regs = get_irq_regs();
320 int cpu = smp_processor_id();
321 int *buffer, len, i, buftype = TIMER_BUF;
322 long long *buffer64;
323 struct gator_interface *gi;
325 // Output scheduler trace
326 len = gator_trace_sched_read(&buffer64);
327 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
328 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
329 gator_buffer_write_packed_int(cpu, buftype, len);
330 for (i = 0; i < len; i++) {
331 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
332 }
333 }
335 // Output GPU trace
336 len = gator_trace_gpu_read(&buffer64);
337 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
338 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE);
339 gator_buffer_write_packed_int(cpu, buftype, len);
340 for (i = 0; i < len; i++) {
341 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
342 }
343 }
345 // Output counters
346 if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) {
347 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
348 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
349 list_for_each_entry(gi, &gator_events, list) {
350 if (gi->read) {
351 len = gi->read(&buffer);
352 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
353 gator_buffer_write_packed_int(cpu, buftype, len);
354 for (i = 0; i < len; i++) {
355 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
356 }
357 }
358 } else if (gi->read64) {
359 len = gi->read64(&buffer64);
360 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) {
361 gator_buffer_write_packed_int(cpu, buftype, len);
362 for (i = 0; i < len; i++) {
363 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
364 }
365 }
366 }
367 }
368 gator_buffer_write_packed_int(cpu, buftype, 0);
369 }
371 // Output backtrace
372 if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
373 gator_add_sample(cpu, buftype, regs);
375 // Overflow message
376 if (per_cpu(emit_overflow, cpu)) {
377 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW);
378 gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu));
379 per_cpu(emit_overflow, cpu) = 0;
380 }
382 // Check and commit; generally, commit is set to occur once per second
383 buffer_check(cpu, buftype);
384 }
386 DEFINE_PER_CPU(int, hrtimer_is_active);
387 static int hrtimer_running;
389 // This function runs in interrupt context and on the appropriate core
390 static void gator_timer_offline(void* unused)
391 {
392 int i, len, cpu = smp_processor_id();
393 int* buffer;
394 long long* buffer64;
396 if (per_cpu(hrtimer_is_active, cpu)) {
397 struct gator_interface *gi;
398 gator_hrtimer_offline(cpu);
399 per_cpu(hrtimer_is_active, cpu) = 0;
401 // Output scheduler trace
402 len = gator_trace_sched_offline(&buffer64);
403 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
404 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE);
405 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
406 for (i = 0; i < len; i++) {
407 gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
408 }
409 }
411 // Output GPU trace
412 len = gator_trace_gpu_offline(&buffer64);
413 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
414 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE);
415 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
416 for (i = 0; i < len; i++) {
417 gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
418 }
419 }
421 // offline any events and output counters
422 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
423 gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
424 list_for_each_entry(gi, &gator_events, list) {
425 if (gi->offline) {
426 len = gi->offline(&buffer);
427 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
428 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
429 for (i = 0; i < len; i++)
430 gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
431 }
432 }
433 }
434 gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
436 gator_commit_buffer(cpu, TIMER_BUF);
437 }
439 if (event_based_sampling) {
440 gator_commit_buffer(cpu, EVENT_BUF);
441 }
442 }
444 // This function runs in interrupt context and may be running on a core other than core 'cpu'
445 static void gator_timer_offline_dispatch(int cpu)
446 {
447 struct gator_interface *gi;
449 list_for_each_entry(gi, &gator_events, list)
450 if (gi->offline_dispatch)
451 gi->offline_dispatch(cpu);
453 gator_event_sampling_offline_dispatch(cpu);
454 }
456 static void gator_timer_stop(void)
457 {
458 int cpu;
460 if (hrtimer_running) {
461 on_each_cpu(gator_timer_offline, NULL, 1);
462 for_each_online_cpu(cpu) {
463 gator_timer_offline_dispatch(cpu);
464 }
466 hrtimer_running = 0;
467 gator_hrtimer_shutdown();
468 }
469 }
471 // This function runs in interrupt context and on the appropriate core
472 static void gator_timer_online(void* unused)
473 {
474 int i, len, cpu = smp_processor_id();
475 int* buffer;
477 if (!per_cpu(hrtimer_is_active, cpu)) {
478 struct gator_interface *gi;
480 // online any events and output counters
481 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
482 gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
483 list_for_each_entry(gi, &gator_events, list) {
484 if (gi->online) {
485 len = gi->online(&buffer);
486 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
487 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
488 for (i = 0; i < len; i++)
489 gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
490 }
491 }
492 }
493 gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
495 gator_event_sampling_online();
497 gator_hrtimer_online(cpu);
498 per_cpu(hrtimer_is_active, cpu) = 1;
499 }
500 }
502 // This function runs in interrupt context and may be running on a core other than core 'cpu'
503 static void gator_timer_online_dispatch(int cpu)
504 {
505 struct gator_interface *gi;
507 list_for_each_entry(gi, &gator_events, list)
508 if (gi->online_dispatch)
509 gi->online_dispatch(cpu);
511 gator_event_sampling_online_dispatch(cpu);
512 }
514 int gator_timer_start(unsigned long setup)
515 {
516 int cpu;
518 if (!setup) {
519 pr_err("gator: cannot start due to a system tick value of zero\n");
520 return -1;
521 } else if (hrtimer_running) {
522 pr_notice("gator: high res timer already running\n");
523 return 0;
524 }
526 hrtimer_running = 1;
528 if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1)
529 return -1;
531 for_each_online_cpu(cpu) {
532 gator_timer_online_dispatch(cpu);
533 }
534 on_each_cpu(gator_timer_online, NULL, 1);
536 return 0;
537 }
539 static uint64_t gator_get_time(void)
540 {
541 struct timespec ts;
542 uint64_t timestamp;
544 getnstimeofday(&ts);
545 timestamp = timespec_to_ns(&ts);
547 return timestamp;
548 }
550 /******************************************************************************
551 * cpu hotplug and pm notifiers
552 ******************************************************************************/
553 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
554 {
555 long cpu = (long)hcpu;
557 switch (action) {
558 case CPU_DOWN_PREPARE:
559 case CPU_DOWN_PREPARE_FROZEN:
560 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
561 gator_timer_offline_dispatch(cpu);
562 break;
563 case CPU_ONLINE:
564 case CPU_ONLINE_FROZEN:
565 gator_timer_online_dispatch(cpu);
566 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
567 break;
568 }
570 return NOTIFY_OK;
571 }
573 static struct notifier_block __refdata gator_cpu_notifier = {
574 .notifier_call = gator_cpu_notify,
575 };
577 // n.b. calling "on_each_cpu" only runs on those that are online
578 // Registered linux events are not disabled, so their counters will continue to collect
579 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
580 {
581 int cpu;
583 switch (event) {
584 case PM_HIBERNATION_PREPARE:
585 case PM_SUSPEND_PREPARE:
586 unregister_hotcpu_notifier(&gator_cpu_notifier);
587 unregister_scheduler_tracepoints();
588 on_each_cpu(gator_timer_offline, NULL, 1);
589 for_each_online_cpu(cpu) {
590 gator_timer_offline_dispatch(cpu);
591 }
592 break;
593 case PM_POST_HIBERNATION:
594 case PM_POST_SUSPEND:
595 for_each_online_cpu(cpu) {
596 gator_timer_online_dispatch(cpu);
597 }
598 on_each_cpu(gator_timer_online, NULL, 1);
599 register_scheduler_tracepoints();
600 register_hotcpu_notifier(&gator_cpu_notifier);
601 break;
602 }
604 return NOTIFY_OK;
605 }
607 static struct notifier_block gator_pm_notifier = {
608 .notifier_call = gator_pm_notify,
609 };
611 static int gator_notifier_start(void)
612 {
613 int retval;
614 retval = register_hotcpu_notifier(&gator_cpu_notifier);
615 if (retval == 0)
616 retval = register_pm_notifier(&gator_pm_notifier);
617 return retval;
618 }
620 static void gator_notifier_stop(void)
621 {
622 unregister_pm_notifier(&gator_pm_notifier);
623 unregister_hotcpu_notifier(&gator_cpu_notifier);
624 }
626 /******************************************************************************
627 * Main
628 ******************************************************************************/
629 int gator_events_install(struct gator_interface *interface)
630 {
631 list_add_tail(&interface->list, &gator_events);
633 return 0;
634 }
636 int gator_events_get_key(void)
637 {
638 static int key;
640 return key++;
641 }
643 static int gator_init(void)
644 {
645 int i;
647 if (gator_annotate_init())
648 return -1;
650 // events sources (gator_events.h, generated by gator_events.sh)
651 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
652 if (gator_events_list[i])
653 gator_events_list[i]();
655 return 0;
656 }
658 static int gator_start(void)
659 {
660 struct gator_interface *gi;
662 // start all events
663 list_for_each_entry(gi, &gator_events, list) {
664 if (gi->start && gi->start() != 0) {
665 struct list_head *ptr = gi->list.prev;
667 while (ptr != &gator_events) {
668 gi = list_entry(ptr, struct gator_interface, list);
670 if (gi->stop)
671 gi->stop();
673 ptr = ptr->prev;
674 }
675 goto events_failure;
676 }
677 }
679 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
680 if (cookies_initialize())
681 goto cookies_failure;
682 if (gator_annotate_start())
683 goto annotate_failure;
684 if (gator_trace_sched_start())
685 goto sched_failure;
686 if (gator_trace_gpu_start())
687 goto gpu_failure;
688 if (gator_event_sampling_start())
689 goto event_sampling_failure;
690 if (gator_timer_start(gator_timer_count))
691 goto timer_failure;
692 if (gator_notifier_start())
693 goto notifier_failure;
695 return 0;
697 notifier_failure:
698 gator_timer_stop();
699 timer_failure:
700 gator_event_sampling_stop();
701 event_sampling_failure:
702 gator_trace_gpu_stop();
703 gpu_failure:
704 gator_trace_sched_stop();
705 sched_failure:
706 gator_annotate_stop();
707 annotate_failure:
708 cookies_release();
709 cookies_failure:
710 // stop all events
711 list_for_each_entry(gi, &gator_events, list)
712 if (gi->stop)
713 gi->stop();
714 events_failure:
716 return -1;
717 }
719 static void gator_stop(void)
720 {
721 struct gator_interface *gi;
723 // stop all events
724 list_for_each_entry(gi, &gator_events, list)
725 if (gi->stop)
726 gi->stop();
728 gator_annotate_stop();
729 gator_trace_sched_stop();
730 gator_trace_gpu_stop();
731 gator_event_sampling_stop();
733 // stop all interrupt callback reads before tearing down other interfaces
734 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
735 gator_timer_stop();
736 }
738 static void gator_exit(void)
739 {
740 gator_annotate_exit();
741 }
743 /******************************************************************************
744 * Filesystem
745 ******************************************************************************/
746 /* fopen("buffer") */
747 static int gator_op_setup(void)
748 {
749 int err = 0;
750 int cpu, i;
752 mutex_lock(&start_mutex);
754 gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
755 gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
757 // must be a power of 2
758 if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
759 err = -ENOEXEC;
760 goto setup_error;
761 }
763 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
764 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
766 // Initialize percpu per buffer variables
767 for (i = 0; i < NUM_GATOR_BUFS; i++) {
768 for_each_present_cpu(cpu) {
769 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
770 if (!per_cpu(gator_buffer, cpu)[i]) {
771 err = -ENOMEM;
772 goto setup_error;
773 }
775 per_cpu(gator_buffer_read, cpu)[i] = 0;
776 per_cpu(gator_buffer_write, cpu)[i] = 0;
777 per_cpu(gator_buffer_commit, cpu)[i] = 0;
778 per_cpu(buffer_space_available, cpu)[i] = true;
779 per_cpu(emit_overflow, cpu) = 0;
780 gator_buffer_header(cpu, i);
781 }
782 }
784 setup_error:
785 mutex_unlock(&start_mutex);
786 return err;
787 }
789 /* Actually start profiling (echo 1>/dev/gator/enable) */
790 static int gator_op_start(void)
791 {
792 int err = 0;
794 mutex_lock(&start_mutex);
796 if (gator_started || gator_start())
797 err = -EINVAL;
798 else
799 gator_started = 1;
801 mutex_unlock(&start_mutex);
803 return err;
804 }
806 /* echo 0>/dev/gator/enable */
807 static void gator_op_stop(void)
808 {
809 mutex_lock(&start_mutex);
811 if (gator_started) {
812 gator_stop();
814 mutex_lock(&gator_buffer_mutex);
816 gator_started = 0;
817 cookies_release();
818 wake_up(&gator_buffer_wait);
820 mutex_unlock(&gator_buffer_mutex);
821 }
823 mutex_unlock(&start_mutex);
824 }
826 static void gator_shutdown(void)
827 {
828 int cpu, i;
830 mutex_lock(&start_mutex);
832 gator_annotate_shutdown();
834 for_each_present_cpu(cpu) {
835 mutex_lock(&gator_buffer_mutex);
836 for (i = 0; i < NUM_GATOR_BUFS; i++) {
837 vfree(per_cpu(gator_buffer, cpu)[i]);
838 per_cpu(gator_buffer, cpu)[i] = NULL;
839 per_cpu(gator_buffer_read, cpu)[i] = 0;
840 per_cpu(gator_buffer_write, cpu)[i] = 0;
841 per_cpu(gator_buffer_commit, cpu)[i] = 0;
842 per_cpu(buffer_space_available, cpu)[i] = true;
843 per_cpu(emit_overflow, cpu) = 0;
844 }
845 mutex_unlock(&gator_buffer_mutex);
846 }
848 mutex_unlock(&start_mutex);
849 }
851 static int gator_set_backtrace(unsigned long val)
852 {
853 int err = 0;
855 mutex_lock(&start_mutex);
857 if (gator_started)
858 err = -EBUSY;
859 else
860 gator_backtrace_depth = val;
862 mutex_unlock(&start_mutex);
864 return err;
865 }
867 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
868 {
869 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
870 }
872 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
873 {
874 unsigned long val;
875 int retval;
877 if (*offset)
878 return -EINVAL;
880 retval = gatorfs_ulong_from_user(&val, buf, count);
881 if (retval)
882 return retval;
884 if (val)
885 retval = gator_op_start();
886 else
887 gator_op_stop();
889 if (retval)
890 return retval;
891 return count;
892 }
894 static const struct file_operations enable_fops = {
895 .read = enable_read,
896 .write = enable_write,
897 };
899 static int userspace_buffer_open(struct inode *inode, struct file *file)
900 {
901 int err = -EPERM;
903 if (!capable(CAP_SYS_ADMIN))
904 return -EPERM;
906 if (test_and_set_bit_lock(0, &gator_buffer_opened))
907 return -EBUSY;
909 if ((err = gator_op_setup()))
910 goto fail;
912 /* NB: the actual start happens from userspace
913 * echo 1 >/dev/gator/enable
914 */
916 return 0;
918 fail:
919 __clear_bit_unlock(0, &gator_buffer_opened);
920 return err;
921 }
923 static int userspace_buffer_release(struct inode *inode, struct file *file)
924 {
925 gator_op_stop();
926 gator_shutdown();
927 __clear_bit_unlock(0, &gator_buffer_opened);
928 return 0;
929 }
931 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
932 size_t count, loff_t *offset)
933 {
934 int retval = -EINVAL;
935 int commit = 0, length1, length2, read;
936 char *buffer1;
937 char *buffer2 = NULL;
938 int cpu, buftype;
940 /* do not handle partial reads */
941 if (count != userspace_buffer_size || *offset)
942 return -EINVAL;
944 // sleep until the condition is true or a signal is received
945 // the condition is checked each time gator_buffer_wait is woken up
946 buftype = cpu = -1;
947 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started);
949 if (signal_pending(current))
950 return -EINTR;
952 length2 = 0;
953 retval = -EFAULT;
955 mutex_lock(&gator_buffer_mutex);
957 if (buftype != -1 && cpu != -1) {
958 read = per_cpu(gator_buffer_read, cpu)[buftype];
959 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
961 /* May happen if the buffer is freed during pending reads. */
962 if (!per_cpu(gator_buffer, cpu)[buftype]) {
963 retval = -EFAULT;
964 goto out;
965 }
967 /* determine the size of two halves */
968 length1 = commit - read;
969 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
970 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
971 if (length1 < 0) {
972 length1 = gator_buffer_size[buftype] - read;
973 length2 = commit;
974 }
975 } else if (gator_annotate_ready()) {
976 length1 = gator_annotate_read(&buffer1);
977 if (!length1)
978 goto out;
979 } else {
980 retval = 0;
981 goto out;
982 }
984 /* start, middle or end */
985 if (length1 > 0) {
986 if (copy_to_user(&buf[0], buffer1, length1)) {
987 goto out;
988 }
989 }
991 /* possible wrap around */
992 if (length2 > 0) {
993 if (copy_to_user(&buf[length1], buffer2, length2)) {
994 goto out;
995 }
996 }
998 if (buftype != -1 && cpu != -1)
999 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1001 retval = length1 + length2;
1003 /* kick just in case we've lost an SMP event */
1004 wake_up(&gator_buffer_wait);
1006 out:
1007 mutex_unlock(&gator_buffer_mutex);
1008 return retval;
1009 }
1011 const struct file_operations gator_event_buffer_fops = {
1012 .open = userspace_buffer_open,
1013 .release = userspace_buffer_release,
1014 .read = userspace_buffer_read,
1015 };
1017 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1018 {
1019 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
1020 offset);
1021 }
1023 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1024 {
1025 unsigned long val;
1026 int retval;
1028 if (*offset)
1029 return -EINVAL;
1031 retval = gatorfs_ulong_from_user(&val, buf, count);
1032 if (retval)
1033 return retval;
1035 retval = gator_set_backtrace(val);
1037 if (retval)
1038 return retval;
1039 return count;
1040 }
1042 static const struct file_operations depth_fops = {
1043 .read = depth_read,
1044 .write = depth_write
1045 };
1047 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1048 {
1049 struct dentry *dir;
1050 struct gator_interface *gi;
1051 int cpu;
1053 /* reinitialize default values */
1054 gator_cpu_cores = 0;
1055 for_each_present_cpu(cpu) {
1056 gator_cpu_cores++;
1057 }
1058 userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
1059 gator_streaming = 1;
1061 gatorfs_create_file(sb, root, "enable", &enable_fops);
1062 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1063 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1064 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1065 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1066 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1067 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
1068 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1070 // Annotate interface
1071 gator_annotate_create_files(sb, root);
1073 // Linux Events
1074 dir = gatorfs_mkdir(sb, root, "events");
1075 list_for_each_entry(gi, &gator_events, list)
1076 if (gi->create_files)
1077 gi->create_files(sb, dir);
1078 }
1080 /******************************************************************************
1081 * Module
1082 ******************************************************************************/
1083 static int __init gator_module_init(void)
1084 {
1085 if (gatorfs_register()) {
1086 return -1;
1087 }
1089 if (gator_init()) {
1090 gatorfs_unregister();
1091 return -1;
1092 }
1094 return 0;
1095 }
1097 static void __exit gator_module_exit(void)
1098 {
1099 tracepoint_synchronize_unregister();
1100 gatorfs_unregister();
1101 gator_exit();
1102 }
1104 module_init(gator_module_init);
1105 module_exit(gator_module_exit);
1107 MODULE_LICENSE("GPL");
1108 MODULE_AUTHOR("ARM Ltd");
1109 MODULE_DESCRIPTION("Gator system profiler");