1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 11;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58 * DEFINES
59 ******************************************************************************/
60 #define SUMMARY_BUFFER_SIZE (1*1024)
61 #define BACKTRACE_BUFFER_SIZE (128*1024)
62 #define NAME_BUFFER_SIZE (64*1024)
63 #define COUNTER_BUFFER_SIZE (64*1024) // counters have the core as part of the data and the core value in the frame header may be discarded
64 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
65 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
66 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
67 #define GPU_TRACE_BUFFER_SIZE (64*1024) // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
68 #define IDLE_BUFFER_SIZE (32*1024) // idle counters have the core as part of the data and the core value in the frame header may be discarded
70 #define NO_COOKIE 0U
71 #define INVALID_COOKIE ~0U
73 #define FRAME_SUMMARY 1
74 #define FRAME_BACKTRACE 2
75 #define FRAME_NAME 3
76 #define FRAME_COUNTER 4
77 #define FRAME_BLOCK_COUNTER 5
78 #define FRAME_ANNOTATE 6
79 #define FRAME_SCHED_TRACE 7
80 #define FRAME_GPU_TRACE 8
81 #define FRAME_IDLE 9
83 #define MESSAGE_END_BACKTRACE 1
85 #define MESSAGE_COOKIE 1
86 #define MESSAGE_THREAD_NAME 2
87 #define HRTIMER_CORE_NAME 3
89 #define MESSAGE_GPU_START 1
90 #define MESSAGE_GPU_STOP 2
92 #define MESSAGE_SCHED_SWITCH 1
93 #define MESSAGE_SCHED_EXIT 2
95 #define MAXSIZE_PACK32 5
96 #define MAXSIZE_PACK64 10
97 #define MAXSIZE_CORE_NAME 32
99 #if defined(__arm__)
100 #define PC_REG regs->ARM_pc
101 #elif defined(__aarch64__)
102 #define PC_REG regs->pc
103 #else
104 #define PC_REG regs->ip
105 #endif
107 enum {
108 SUMMARY_BUF,
109 BACKTRACE_BUF,
110 NAME_BUF,
111 COUNTER_BUF,
112 BLOCK_COUNTER_BUF,
113 ANNOTATE_BUF,
114 SCHED_TRACE_BUF,
115 GPU_TRACE_BUF,
116 IDLE_BUF,
117 NUM_GATOR_BUFS
118 };
120 /******************************************************************************
121 * Globals
122 ******************************************************************************/
123 static unsigned long gator_cpu_cores;
124 static unsigned long userspace_buffer_size;
125 static unsigned long gator_backtrace_depth;
127 static unsigned long gator_started;
128 static uint64_t monotonic_started;
129 static unsigned long gator_buffer_opened;
130 static unsigned long gator_timer_count;
131 static unsigned long gator_response_type;
132 static DEFINE_MUTEX(start_mutex);
133 static DEFINE_MUTEX(gator_buffer_mutex);
135 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
136 static struct timer_list gator_buffer_wake_up_timer;
137 static LIST_HEAD(gator_events);
139 /******************************************************************************
140 * Prototypes
141 ******************************************************************************/
142 static void buffer_check(int cpu, int buftype);
143 static int buffer_bytes_available(int cpu, int buftype);
144 static bool buffer_check_space(int cpu, int buftype, int bytes);
145 static int contiguous_space_available(int cpu, int bufytpe);
146 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
147 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
148 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
149 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
150 static void gator_add_trace(int cpu, unsigned long address);
151 static void gator_add_sample(int cpu, struct pt_regs * const regs);
152 static uint64_t gator_get_time(void);
154 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
155 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
156 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
157 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
158 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
159 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
160 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
162 /******************************************************************************
163 * Application Includes
164 ******************************************************************************/
165 #include "gator_marshaling.c"
166 #include "gator_hrtimer_perf.c"
167 #include "gator_hrtimer_gator.c"
168 #include "gator_cookies.c"
169 #include "gator_trace_sched.c"
170 #include "gator_trace_power.c"
171 #include "gator_trace_gpu.c"
172 #include "gator_backtrace.c"
173 #include "gator_annotate.c"
174 #include "gator_fs.c"
175 #include "gator_pack.c"
177 /******************************************************************************
178 * Misc
179 ******************************************************************************/
180 #if defined(__arm__) || defined(__aarch64__)
181 u32 gator_cpuid(void)
182 {
183 u32 val;
184 #if !defined(__aarch64__)
185 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
186 #else
187 asm volatile("mrs %0, midr_el1" : "=r" (val));
188 #endif
189 return (val >> 4) & 0xfff;
190 }
191 #endif
193 static void gator_buffer_wake_up(unsigned long data)
194 {
195 wake_up(&gator_buffer_wait);
196 }
198 /******************************************************************************
199 * Commit interface
200 ******************************************************************************/
201 static bool buffer_commit_ready(int* cpu, int* buftype)
202 {
203 int cpu_x, x;
204 for_each_present_cpu(cpu_x) {
205 for (x = 0; x < NUM_GATOR_BUFS; x++)
206 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
207 *cpu = cpu_x;
208 *buftype = x;
209 return true;
210 }
211 }
212 return false;
213 }
215 /******************************************************************************
216 * Buffer management
217 ******************************************************************************/
218 static int buffer_bytes_available(int cpu, int buftype)
219 {
220 int remaining, filled;
222 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
223 if (filled < 0) {
224 filled += gator_buffer_size[buftype];
225 }
227 remaining = gator_buffer_size[buftype] - filled;
229 if (per_cpu(buffer_space_available, cpu)[buftype]) {
230 // Give some extra room; also allows space to insert the overflow error packet
231 remaining -= 200;
232 } else {
233 // Hysteresis, prevents multiple overflow messages
234 remaining -= 2000;
235 }
237 return remaining;
238 }
240 static int contiguous_space_available(int cpu, int buftype)
241 {
242 int remaining = buffer_bytes_available(cpu, buftype);
243 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
244 if (remaining < contiguous)
245 return remaining;
246 else
247 return contiguous;
248 }
250 static bool buffer_check_space(int cpu, int buftype, int bytes)
251 {
252 int remaining = buffer_bytes_available(cpu, buftype);
254 if (remaining < bytes) {
255 per_cpu(buffer_space_available, cpu)[buftype] = false;
256 } else {
257 per_cpu(buffer_space_available, cpu)[buftype] = true;
258 }
260 return per_cpu(buffer_space_available, cpu)[buftype];
261 }
263 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
264 {
265 int i;
266 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
267 u32 mask = gator_buffer_mask[buftype];
268 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
270 for (i = 0; i < len; i++) {
271 buffer[write] = x[i];
272 write = (write + 1) & mask;
273 }
275 per_cpu(gator_buffer_write, cpu)[buftype] = write;
276 }
278 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
279 {
280 int len = strlen(x);
281 gator_buffer_write_packed_int(cpu, buftype, len);
282 gator_buffer_write_bytes(cpu, buftype, x, len);
283 }
285 static void gator_buffer_header(int cpu, int buftype)
286 {
287 int frame;
289 switch (buftype) {
290 case SUMMARY_BUF:
291 frame = FRAME_SUMMARY;
292 break;
293 case BACKTRACE_BUF:
294 frame = FRAME_BACKTRACE;
295 break;
296 case NAME_BUF:
297 frame = FRAME_NAME;
298 break;
299 case COUNTER_BUF:
300 frame = FRAME_COUNTER;
301 break;
302 case BLOCK_COUNTER_BUF:
303 frame = FRAME_BLOCK_COUNTER;
304 break;
305 case ANNOTATE_BUF:
306 frame = FRAME_ANNOTATE;
307 break;
308 case SCHED_TRACE_BUF:
309 frame = FRAME_SCHED_TRACE;
310 break;
311 case GPU_TRACE_BUF:
312 frame = FRAME_GPU_TRACE;
313 break;
314 case IDLE_BUF:
315 frame = FRAME_IDLE;
316 break;
317 default:
318 frame = -1;
319 break;
320 }
322 if (per_cpu(gator_buffer, cpu)[buftype]) {
323 marshal_frame(cpu, buftype, frame);
324 }
325 }
327 static void gator_commit_buffer(int cpu, int buftype)
328 {
329 if (!per_cpu(gator_buffer, cpu)[buftype])
330 return;
332 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
333 gator_buffer_header(cpu, buftype);
335 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
336 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
337 }
339 static void buffer_check(int cpu, int buftype)
340 {
341 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
342 if (filled < 0) {
343 filled += gator_buffer_size[buftype];
344 }
345 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
346 gator_commit_buffer(cpu, buftype);
347 }
348 }
350 static void gator_add_trace(int cpu, unsigned long address)
351 {
352 off_t offset = 0;
353 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
355 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
356 offset = address;
357 }
359 marshal_backtrace(offset & ~1, cookie);
360 }
362 static void gator_add_sample(int cpu, struct pt_regs * const regs)
363 {
364 bool inKernel;
365 unsigned long exec_cookie;
367 if (!regs)
368 return;
370 inKernel = !user_mode(regs);
371 exec_cookie = get_exec_cookie(cpu, current);
373 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
374 return;
376 if (inKernel) {
377 kernel_backtrace(cpu, regs);
378 } else {
379 // Cookie+PC
380 gator_add_trace(cpu, PC_REG);
382 // Backtrace
383 if (gator_backtrace_depth)
384 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
385 }
387 marshal_backtrace_footer();
388 }
390 /******************************************************************************
391 * hrtimer interrupt processing
392 ******************************************************************************/
393 static void gator_timer_interrupt(void)
394 {
395 struct pt_regs * const regs = get_irq_regs();
396 gator_backtrace_handler(regs);
397 }
399 void gator_backtrace_handler(struct pt_regs * const regs)
400 {
401 int cpu = smp_processor_id();
403 // Output backtrace
404 gator_add_sample(cpu, regs);
406 // Collect counters
407 if (!per_cpu(collecting, cpu)) {
408 collect_counters();
409 }
410 }
412 static int gator_running;
414 // This function runs in interrupt context and on the appropriate core
415 static void gator_timer_offline(void* unused)
416 {
417 struct gator_interface *gi;
418 int i, len, cpu = smp_processor_id();
419 int* buffer;
421 gator_trace_sched_offline();
422 gator_trace_power_offline();
424 gator_hrtimer_offline(cpu);
426 // Offline any events and output counters
427 if (marshal_event_header()) {
428 list_for_each_entry(gi, &gator_events, list) {
429 if (gi->offline) {
430 len = gi->offline(&buffer);
431 marshal_event(len, buffer);
432 }
433 }
434 }
436 // Flush all buffers on this core
437 for (i = 0; i < NUM_GATOR_BUFS; i++)
438 gator_commit_buffer(cpu, i);
439 }
441 // This function runs in process context and may be running on a core other than core 'cpu'
442 static void gator_timer_offline_dispatch(int cpu)
443 {
444 struct gator_interface *gi;
446 list_for_each_entry(gi, &gator_events, list)
447 if (gi->offline_dispatch)
448 gi->offline_dispatch(cpu);
449 }
451 static void gator_timer_stop(void)
452 {
453 int cpu;
455 if (gator_running) {
456 on_each_cpu(gator_timer_offline, NULL, 1);
457 for_each_online_cpu(cpu) {
458 gator_timer_offline_dispatch(cpu);
459 }
461 gator_running = 0;
462 gator_hrtimer_shutdown();
463 }
464 }
466 // This function runs in interrupt context and on the appropriate core
467 static void gator_timer_online(void* unused)
468 {
469 struct gator_interface *gi;
470 int len, cpu = smp_processor_id();
471 int* buffer;
473 gator_trace_power_online();
475 // online any events and output counters
476 if (marshal_event_header()) {
477 list_for_each_entry(gi, &gator_events, list) {
478 if (gi->online) {
479 len = gi->online(&buffer);
480 marshal_event(len, buffer);
481 }
482 }
483 }
485 gator_hrtimer_online(cpu);
486 #if defined(__arm__) || defined(__aarch64__)
487 {
488 const char * core_name = NULL;
490 // String lengths must be less than MAXSIZE_CORE_NAME
491 switch (gator_cpuid()) {
492 case ARM1136: core_name = "ARM1136"; break;
493 case ARM1156: core_name = "ARM1156"; break;
494 case ARM1176: core_name = "ARM1176"; break;
495 case ARM11MPCORE: core_name = "ARM11MPCore"; break;
496 case CORTEX_A5: core_name = "Cortex-A5"; break;
497 case CORTEX_A7: core_name = "Cortex-A7"; break;
498 case CORTEX_A8: core_name = "Cortex-A8"; break;
499 case CORTEX_A9: core_name = "Cortex-A9"; break;
500 case CORTEX_A15: core_name = "Cortex-A15"; break;
501 case SCORPION: core_name = "Scorpion"; break;
502 case SCORPIONMP: core_name = "ScorpionMP"; break;
503 case KRAITSIM: core_name = "KraitSIM"; break;
504 case KRAIT: core_name = "Krait"; break;
505 case AARCH64: core_name = "AArch64"; break;
506 default: core_name = "Unknown"; break;
507 }
509 marshal_core_name(core_name);
510 }
511 #endif
512 }
514 // This function runs in interrupt context and may be running on a core other than core 'cpu'
515 static void gator_timer_online_dispatch(int cpu)
516 {
517 struct gator_interface *gi;
519 list_for_each_entry(gi, &gator_events, list)
520 if (gi->online_dispatch)
521 gi->online_dispatch(cpu);
522 }
524 int gator_timer_start(unsigned long sample_rate)
525 {
526 int cpu;
528 if (gator_running) {
529 pr_notice("gator: already running\n");
530 return 0;
531 }
533 gator_running = 1;
535 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
536 return -1;
538 for_each_online_cpu(cpu) {
539 gator_timer_online_dispatch(cpu);
540 }
541 on_each_cpu(gator_timer_online, NULL, 1);
543 return 0;
544 }
546 static uint64_t gator_get_time(void)
547 {
548 struct timespec ts;
549 uint64_t timestamp;
551 //getnstimeofday(&ts);
552 do_posix_clock_monotonic_gettime(&ts);
553 timestamp = timespec_to_ns(&ts) - monotonic_started;
555 return timestamp;
556 }
558 /******************************************************************************
559 * cpu hotplug and pm notifiers
560 ******************************************************************************/
561 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
562 {
563 long cpu = (long)hcpu;
565 switch (action) {
566 case CPU_DOWN_PREPARE:
567 case CPU_DOWN_PREPARE_FROZEN:
568 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
569 gator_timer_offline_dispatch(cpu);
570 break;
571 case CPU_ONLINE:
572 case CPU_ONLINE_FROZEN:
573 gator_timer_online_dispatch(cpu);
574 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
575 break;
576 }
578 return NOTIFY_OK;
579 }
581 static struct notifier_block __refdata gator_hotcpu_notifier = {
582 .notifier_call = gator_hotcpu_notify,
583 };
585 // n.b. calling "on_each_cpu" only runs on those that are online
586 // Registered linux events are not disabled, so their counters will continue to collect
587 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
588 {
589 int cpu;
591 switch (event) {
592 case PM_HIBERNATION_PREPARE:
593 case PM_SUSPEND_PREPARE:
594 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
595 unregister_scheduler_tracepoints();
596 on_each_cpu(gator_timer_offline, NULL, 1);
597 for_each_online_cpu(cpu) {
598 gator_timer_offline_dispatch(cpu);
599 }
600 break;
601 case PM_POST_HIBERNATION:
602 case PM_POST_SUSPEND:
603 for_each_online_cpu(cpu) {
604 gator_timer_online_dispatch(cpu);
605 }
606 on_each_cpu(gator_timer_online, NULL, 1);
607 register_scheduler_tracepoints();
608 register_hotcpu_notifier(&gator_hotcpu_notifier);
609 break;
610 }
612 return NOTIFY_OK;
613 }
615 static struct notifier_block gator_pm_notifier = {
616 .notifier_call = gator_pm_notify,
617 };
619 static int gator_notifier_start(void)
620 {
621 int retval;
622 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
623 if (retval == 0)
624 retval = register_pm_notifier(&gator_pm_notifier);
625 return retval;
626 }
628 static void gator_notifier_stop(void)
629 {
630 unregister_pm_notifier(&gator_pm_notifier);
631 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
632 }
634 /******************************************************************************
635 * Main
636 ******************************************************************************/
637 static void gator_summary(void)
638 {
639 uint64_t timestamp;
640 struct timespec uptime_ts;
642 timestamp = gator_get_time();
644 do_posix_clock_monotonic_gettime(&uptime_ts);
645 monotonic_started = timespec_to_ns(&uptime_ts);
647 marshal_summary(timestamp, monotonic_started);
648 }
650 int gator_events_install(struct gator_interface *interface)
651 {
652 list_add_tail(&interface->list, &gator_events);
654 return 0;
655 }
657 int gator_events_get_key(void)
658 {
659 // key of zero is reserved as a timestamp
660 static int key = 1;
662 return key++;
663 }
665 static int gator_init(void)
666 {
667 int i;
669 // events sources (gator_events.h, generated by gator_events.sh)
670 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
671 if (gator_events_list[i])
672 gator_events_list[i]();
674 gator_trace_power_init();
676 return 0;
677 }
679 static int gator_start(void)
680 {
681 unsigned long cpu, i;
682 struct gator_interface *gi;
684 // Initialize the buffer with the frame type and core
685 for_each_present_cpu(cpu) {
686 for (i = 0; i < NUM_GATOR_BUFS; i++) {
687 gator_buffer_header(cpu, i);
688 }
689 }
691 // Capture the start time
692 gator_summary();
694 // start all events
695 list_for_each_entry(gi, &gator_events, list) {
696 if (gi->start && gi->start() != 0) {
697 struct list_head *ptr = gi->list.prev;
699 while (ptr != &gator_events) {
700 gi = list_entry(ptr, struct gator_interface, list);
702 if (gi->stop)
703 gi->stop();
705 ptr = ptr->prev;
706 }
707 goto events_failure;
708 }
709 }
711 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
712 if (cookies_initialize())
713 goto cookies_failure;
714 if (gator_annotate_start())
715 goto annotate_failure;
716 if (gator_trace_sched_start())
717 goto sched_failure;
718 if (gator_trace_power_start())
719 goto power_failure;
720 if (gator_trace_gpu_start())
721 goto gpu_failure;
722 if (gator_timer_start(gator_timer_count))
723 goto timer_failure;
724 if (gator_notifier_start())
725 goto notifier_failure;
727 return 0;
729 notifier_failure:
730 gator_timer_stop();
731 timer_failure:
732 gator_trace_gpu_stop();
733 gpu_failure:
734 gator_trace_power_stop();
735 power_failure:
736 gator_trace_sched_stop();
737 sched_failure:
738 gator_annotate_stop();
739 annotate_failure:
740 cookies_release();
741 cookies_failure:
742 // stop all events
743 list_for_each_entry(gi, &gator_events, list)
744 if (gi->stop)
745 gi->stop();
746 events_failure:
748 return -1;
749 }
751 static void gator_stop(void)
752 {
753 struct gator_interface *gi;
755 gator_annotate_stop();
756 gator_trace_sched_stop();
757 gator_trace_power_stop();
758 gator_trace_gpu_stop();
760 // stop all interrupt callback reads before tearing down other interfaces
761 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
762 gator_timer_stop();
764 // stop all events
765 list_for_each_entry(gi, &gator_events, list)
766 if (gi->stop)
767 gi->stop();
768 }
770 /******************************************************************************
771 * Filesystem
772 ******************************************************************************/
773 /* fopen("buffer") */
774 static int gator_op_setup(void)
775 {
776 int err = 0;
777 int cpu, i;
779 mutex_lock(&start_mutex);
781 gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
782 gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
784 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
785 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
787 gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
788 gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
790 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
791 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
793 gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
794 gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
796 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
797 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
799 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
800 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
802 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
803 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
805 gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
806 gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
808 // Initialize percpu per buffer variables
809 for (i = 0; i < NUM_GATOR_BUFS; i++) {
810 // Verify buffers are a power of 2
811 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
812 err = -ENOEXEC;
813 goto setup_error;
814 }
816 for_each_present_cpu(cpu) {
817 per_cpu(gator_buffer_read, cpu)[i] = 0;
818 per_cpu(gator_buffer_write, cpu)[i] = 0;
819 per_cpu(gator_buffer_commit, cpu)[i] = 0;
820 per_cpu(buffer_space_available, cpu)[i] = true;
822 // Annotation is a special case that only uses a single buffer
823 if (cpu > 0 && i == ANNOTATE_BUF) {
824 per_cpu(gator_buffer, cpu)[i] = NULL;
825 continue;
826 }
828 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
829 if (!per_cpu(gator_buffer, cpu)[i]) {
830 err = -ENOMEM;
831 goto setup_error;
832 }
833 }
834 }
836 setup_error:
837 mutex_unlock(&start_mutex);
838 return err;
839 }
841 /* Actually start profiling (echo 1>/dev/gator/enable) */
842 static int gator_op_start(void)
843 {
844 int err = 0;
846 mutex_lock(&start_mutex);
848 if (gator_started || gator_start())
849 err = -EINVAL;
850 else
851 gator_started = 1;
853 mutex_unlock(&start_mutex);
855 return err;
856 }
858 /* echo 0>/dev/gator/enable */
859 static void gator_op_stop(void)
860 {
861 mutex_lock(&start_mutex);
863 if (gator_started) {
864 gator_stop();
866 mutex_lock(&gator_buffer_mutex);
868 gator_started = 0;
869 cookies_release();
870 wake_up(&gator_buffer_wait);
872 mutex_unlock(&gator_buffer_mutex);
873 }
875 mutex_unlock(&start_mutex);
876 }
878 static void gator_shutdown(void)
879 {
880 int cpu, i;
882 mutex_lock(&start_mutex);
884 for_each_present_cpu(cpu) {
885 mutex_lock(&gator_buffer_mutex);
886 for (i = 0; i < NUM_GATOR_BUFS; i++) {
887 vfree(per_cpu(gator_buffer, cpu)[i]);
888 per_cpu(gator_buffer, cpu)[i] = NULL;
889 per_cpu(gator_buffer_read, cpu)[i] = 0;
890 per_cpu(gator_buffer_write, cpu)[i] = 0;
891 per_cpu(gator_buffer_commit, cpu)[i] = 0;
892 per_cpu(buffer_space_available, cpu)[i] = true;
893 }
894 mutex_unlock(&gator_buffer_mutex);
895 }
897 mutex_unlock(&start_mutex);
898 }
900 static int gator_set_backtrace(unsigned long val)
901 {
902 int err = 0;
904 mutex_lock(&start_mutex);
906 if (gator_started)
907 err = -EBUSY;
908 else
909 gator_backtrace_depth = val;
911 mutex_unlock(&start_mutex);
913 return err;
914 }
916 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
917 {
918 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
919 }
921 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
922 {
923 unsigned long val;
924 int retval;
926 if (*offset)
927 return -EINVAL;
929 retval = gatorfs_ulong_from_user(&val, buf, count);
930 if (retval)
931 return retval;
933 if (val)
934 retval = gator_op_start();
935 else
936 gator_op_stop();
938 if (retval)
939 return retval;
940 return count;
941 }
943 static const struct file_operations enable_fops = {
944 .read = enable_read,
945 .write = enable_write,
946 };
948 static int userspace_buffer_open(struct inode *inode, struct file *file)
949 {
950 int err = -EPERM;
952 if (!capable(CAP_SYS_ADMIN))
953 return -EPERM;
955 if (test_and_set_bit_lock(0, &gator_buffer_opened))
956 return -EBUSY;
958 if ((err = gator_op_setup()))
959 goto fail;
961 /* NB: the actual start happens from userspace
962 * echo 1 >/dev/gator/enable
963 */
965 return 0;
967 fail:
968 __clear_bit_unlock(0, &gator_buffer_opened);
969 return err;
970 }
972 static int userspace_buffer_release(struct inode *inode, struct file *file)
973 {
974 gator_op_stop();
975 gator_shutdown();
976 __clear_bit_unlock(0, &gator_buffer_opened);
977 return 0;
978 }
980 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
981 size_t count, loff_t *offset)
982 {
983 int retval = -EINVAL;
984 int commit = 0, length, length1, length2, read, byte, type_length;
985 char *buffer1;
986 char *buffer2 = NULL;
987 int cpu, buftype;
989 /* do not handle partial reads */
990 if (count != userspace_buffer_size || *offset)
991 return -EINVAL;
993 // sleep until the condition is true or a signal is received
994 // the condition is checked each time gator_buffer_wait is woken up
995 buftype = cpu = -1;
996 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
998 if (signal_pending(current))
999 return -EINTR;
1001 length2 = 0;
1002 retval = -EFAULT;
1004 mutex_lock(&gator_buffer_mutex);
1006 if (buftype == -1 || cpu == -1) {
1007 retval = 0;
1008 goto out;
1009 }
1011 read = per_cpu(gator_buffer_read, cpu)[buftype];
1012 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1014 /* May happen if the buffer is freed during pending reads. */
1015 if (!per_cpu(gator_buffer, cpu)[buftype]) {
1016 retval = -EFAULT;
1017 goto out;
1018 }
1020 /* determine the size of two halves */
1021 length1 = commit - read;
1022 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1023 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1024 if (length1 < 0) {
1025 length1 = gator_buffer_size[buftype] - read;
1026 length2 = commit;
1027 }
1029 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
1030 type_length = gator_response_type ? 1 : 0;
1031 length = length1 + length2 - type_length - sizeof(int);
1032 for (byte = 0; byte < sizeof(int); byte++) {
1033 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
1034 }
1036 /* start, middle or end */
1037 if (length1 > 0) {
1038 if (copy_to_user(&buf[0], buffer1, length1)) {
1039 goto out;
1040 }
1041 }
1043 /* possible wrap around */
1044 if (length2 > 0) {
1045 if (copy_to_user(&buf[length1], buffer2, length2)) {
1046 goto out;
1047 }
1048 }
1050 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1051 retval = length1 + length2;
1053 /* kick just in case we've lost an SMP event */
1054 wake_up(&gator_buffer_wait);
1056 out:
1057 mutex_unlock(&gator_buffer_mutex);
1058 return retval;
1059 }
1061 const struct file_operations gator_event_buffer_fops = {
1062 .open = userspace_buffer_open,
1063 .release = userspace_buffer_release,
1064 .read = userspace_buffer_read,
1065 };
1067 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1068 {
1069 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
1070 offset);
1071 }
1073 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1074 {
1075 unsigned long val;
1076 int retval;
1078 if (*offset)
1079 return -EINVAL;
1081 retval = gatorfs_ulong_from_user(&val, buf, count);
1082 if (retval)
1083 return retval;
1085 retval = gator_set_backtrace(val);
1087 if (retval)
1088 return retval;
1089 return count;
1090 }
1092 static const struct file_operations depth_fops = {
1093 .read = depth_read,
1094 .write = depth_write
1095 };
1097 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1098 {
1099 struct dentry *dir;
1100 struct gator_interface *gi;
1101 int cpu;
1103 /* reinitialize default values */
1104 gator_cpu_cores = 0;
1105 for_each_present_cpu(cpu) {
1106 gator_cpu_cores++;
1107 }
1108 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1109 gator_response_type = 1;
1111 gatorfs_create_file(sb, root, "enable", &enable_fops);
1112 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1113 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1114 gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1115 gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1116 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1117 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1118 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1120 // Annotate interface
1121 gator_annotate_create_files(sb, root);
1123 // Linux Events
1124 dir = gatorfs_mkdir(sb, root, "events");
1125 list_for_each_entry(gi, &gator_events, list)
1126 if (gi->create_files)
1127 gi->create_files(sb, dir);
1129 // Power interface
1130 gator_trace_power_create_files(sb, dir);
1131 }
1133 /******************************************************************************
1134 * Module
1135 ******************************************************************************/
1136 static int __init gator_module_init(void)
1137 {
1138 if (gatorfs_register()) {
1139 return -1;
1140 }
1142 if (gator_init()) {
1143 gatorfs_unregister();
1144 return -1;
1145 }
1147 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1149 return 0;
1150 }
1152 static void __exit gator_module_exit(void)
1153 {
1154 del_timer_sync(&gator_buffer_wake_up_timer);
1155 tracepoint_synchronize_unregister();
1156 gatorfs_unregister();
1157 }
1159 module_init(gator_module_init);
1160 module_exit(gator_module_exit);
1162 MODULE_LICENSE("GPL");
1163 MODULE_AUTHOR("ARM Ltd");
1164 MODULE_DESCRIPTION("Gator system profiler");