1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 9;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58 * DEFINES
59 ******************************************************************************/
60 #define BACKTRACE_BUFFER_SIZE (128*1024)
61 #define COUNTER_BUFFER_SIZE (128*1024)
62 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
63 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
64 #define GPU_TRACE_BUFFER_SIZE (64*1024)
65 #define COUNTER2_BUFFER_SIZE (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
66 #define WFI_BUFFER_SIZE (32*1024) // wfi counters have the core as part of the data and the core value in the frame header may be discarded
68 #define NO_COOKIE 0UL
69 #define INVALID_COOKIE ~0UL
71 #define FRAME_BACKTRACE 1
72 #define FRAME_COUNTER 2
73 #define FRAME_ANNOTATE 3
74 #define FRAME_SCHED_TRACE 4
75 #define FRAME_GPU_TRACE 5
76 #define FRAME_COUNTER2 6
77 #define FRAME_WFI 7
79 #define MESSAGE_COOKIE 1
80 #define MESSAGE_START_BACKTRACE 5
81 #define MESSAGE_END_BACKTRACE 7
82 #define MESSAGE_SUMMARY 9
83 #define MESSAGE_PID_NAME 11
85 #define MAXSIZE_PACK32 5
86 #define MAXSIZE_PACK64 9
88 #if defined(__arm__)
89 #define PC_REG regs->ARM_pc
90 #else
91 #define PC_REG regs->ip
92 #endif
94 enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
96 /******************************************************************************
97 * Globals
98 ******************************************************************************/
99 static unsigned long gator_cpu_cores;
100 static unsigned long userspace_buffer_size;
101 static unsigned long gator_backtrace_depth;
103 static unsigned long gator_started;
104 static unsigned long gator_buffer_opened;
105 static unsigned long gator_timer_count;
106 static unsigned long gator_response_type;
107 static DEFINE_MUTEX(start_mutex);
108 static DEFINE_MUTEX(gator_buffer_mutex);
110 bool event_based_sampling;
112 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
113 static struct timer_list gator_buffer_wake_up_timer;
114 static LIST_HEAD(gator_events);
116 /******************************************************************************
117 * Prototypes
118 ******************************************************************************/
119 static void buffer_check(int cpu, int buftype);
120 static int buffer_bytes_available(int cpu, int buftype);
121 static bool buffer_check_space(int cpu, int buftype, int bytes);
122 static int contiguous_space_available(int cpu, int bufytpe);
123 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
124 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
125 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
126 static void gator_buffer_write_string(int cpu, int buftype, char *x);
127 static void gator_add_trace(int cpu, int buftype, unsigned int address);
128 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
129 static uint64_t gator_get_time(void);
131 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
132 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
133 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
134 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
135 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
136 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
137 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
139 /******************************************************************************
140 * Application Includes
141 ******************************************************************************/
142 #include "gator_marshaling.c"
143 #include "gator_hrtimer_perf.c"
144 #include "gator_hrtimer_gator.c"
145 #include "gator_cookies.c"
146 #include "gator_trace_sched.c"
147 #include "gator_trace_power.c"
148 #include "gator_trace_gpu.c"
149 #include "gator_backtrace.c"
150 #include "gator_annotate.c"
151 #include "gator_fs.c"
152 #include "gator_ebs.c"
153 #include "gator_pack.c"
155 /******************************************************************************
156 * Misc
157 ******************************************************************************/
158 #if defined(__arm__)
159 u32 gator_cpuid(void)
160 {
161 u32 val;
162 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
163 return (val >> 4) & 0xfff;
164 }
165 #endif
167 static void gator_buffer_wake_up(unsigned long data)
168 {
169 wake_up(&gator_buffer_wait);
170 }
172 /******************************************************************************
173 * Commit interface
174 ******************************************************************************/
175 static bool buffer_commit_ready(int* cpu, int* buftype)
176 {
177 int cpu_x, x;
178 for_each_present_cpu(cpu_x) {
179 for (x = 0; x < NUM_GATOR_BUFS; x++)
180 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
181 *cpu = cpu_x;
182 *buftype = x;
183 return true;
184 }
185 }
186 return false;
187 }
189 /******************************************************************************
190 * Buffer management
191 ******************************************************************************/
192 static int buffer_bytes_available(int cpu, int buftype)
193 {
194 int remaining, filled;
196 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
197 if (filled < 0) {
198 filled += gator_buffer_size[buftype];
199 }
201 remaining = gator_buffer_size[buftype] - filled;
203 if (per_cpu(buffer_space_available, cpu)[buftype]) {
204 // Give some extra room; also allows space to insert the overflow error packet
205 remaining -= 200;
206 } else {
207 // Hysteresis, prevents multiple overflow messages
208 remaining -= 2000;
209 }
211 return remaining;
212 }
214 static int contiguous_space_available(int cpu, int buftype)
215 {
216 int remaining = buffer_bytes_available(cpu, buftype);
217 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
218 if (remaining < contiguous)
219 return remaining;
220 else
221 return contiguous;
222 }
224 static bool buffer_check_space(int cpu, int buftype, int bytes)
225 {
226 int remaining = buffer_bytes_available(cpu, buftype);
228 if (remaining < bytes) {
229 per_cpu(buffer_space_available, cpu)[buftype] = false;
230 } else {
231 per_cpu(buffer_space_available, cpu)[buftype] = true;
232 }
234 return per_cpu(buffer_space_available, cpu)[buftype];
235 }
237 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
238 {
239 int i;
240 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
241 u32 mask = gator_buffer_mask[buftype];
242 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
244 for (i = 0; i < len; i++) {
245 buffer[write] = x[i];
246 write = (write + 1) & mask;
247 }
249 per_cpu(gator_buffer_write, cpu)[buftype] = write;
250 }
252 static void gator_buffer_write_string(int cpu, int buftype, char *x)
253 {
254 int len = strlen(x);
255 gator_buffer_write_packed_int(cpu, buftype, len);
256 gator_buffer_write_bytes(cpu, buftype, x, len);
257 }
259 static void gator_buffer_header(int cpu, int buftype)
260 {
261 int frame;
263 if (buftype == BACKTRACE_BUF)
264 frame = FRAME_BACKTRACE;
265 else if (buftype == COUNTER_BUF)
266 frame = FRAME_COUNTER;
267 else if (buftype == ANNOTATE_BUF)
268 frame = FRAME_ANNOTATE;
269 else if (buftype == SCHED_TRACE_BUF)
270 frame = FRAME_SCHED_TRACE;
271 else if (buftype == GPU_TRACE_BUF)
272 frame = FRAME_GPU_TRACE;
273 else if (buftype == COUNTER2_BUF)
274 frame = FRAME_COUNTER2;
275 else if (buftype == WFI_BUF)
276 frame = FRAME_WFI;
277 else
278 frame = -1;
280 if (per_cpu(gator_buffer, cpu)[buftype]) {
281 marshal_frame(cpu, buftype, frame);
282 }
283 }
285 static void gator_commit_buffer(int cpu, int buftype)
286 {
287 if (!per_cpu(gator_buffer, cpu)[buftype])
288 return;
290 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
291 gator_buffer_header(cpu, buftype);
293 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
294 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
295 }
297 static void buffer_check(int cpu, int buftype)
298 {
299 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
300 if (filled < 0) {
301 filled += gator_buffer_size[buftype];
302 }
303 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
304 gator_commit_buffer(cpu, buftype);
305 }
306 }
308 static void gator_add_trace(int cpu, int buftype, unsigned int address)
309 {
310 off_t offset = 0;
311 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
313 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
314 offset = address;
315 }
317 marshal_backtrace(offset & ~1, cookie);
318 }
320 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
321 {
322 int inKernel = regs ? !user_mode(regs) : 1;
323 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
325 if (!regs)
326 return;
328 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
329 return;
331 if (inKernel) {
332 kernel_backtrace(cpu, buftype, regs);
333 } else {
334 // Cookie+PC
335 gator_add_trace(cpu, buftype, PC_REG);
337 // Backtrace
338 if (gator_backtrace_depth)
339 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
340 }
342 marshal_backtrace_footer();
343 }
345 /******************************************************************************
346 * hrtimer interrupt processing
347 ******************************************************************************/
348 static void gator_timer_interrupt(void)
349 {
350 struct pt_regs * const regs = get_irq_regs();
351 int cpu = smp_processor_id();
353 // Output backtrace
354 gator_add_sample(cpu, BACKTRACE_BUF, regs);
356 // Collect counters
357 collect_counters();
358 }
360 static int gator_running;
362 // This function runs in interrupt context and on the appropriate core
363 static void gator_timer_offline(void* unused)
364 {
365 struct gator_interface *gi;
366 int i, len, cpu = smp_processor_id();
367 int* buffer;
369 gator_trace_sched_offline();
370 gator_trace_power_offline();
372 gator_hrtimer_offline(cpu);
374 // Offline any events and output counters
375 if (marshal_event_header()) {
376 list_for_each_entry(gi, &gator_events, list) {
377 if (gi->offline) {
378 len = gi->offline(&buffer);
379 marshal_event(len, buffer);
380 }
381 }
382 }
384 // Flush all buffers on this core
385 for (i = 0; i < NUM_GATOR_BUFS; i++)
386 gator_commit_buffer(cpu, i);
387 }
389 // This function runs in process context and may be running on a core other than core 'cpu'
390 static void gator_timer_offline_dispatch(int cpu)
391 {
392 struct gator_interface *gi;
394 list_for_each_entry(gi, &gator_events, list)
395 if (gi->offline_dispatch)
396 gi->offline_dispatch(cpu);
398 gator_event_sampling_offline_dispatch(cpu);
399 }
401 static void gator_timer_stop(void)
402 {
403 int cpu;
405 if (gator_running) {
406 on_each_cpu(gator_timer_offline, NULL, 1);
407 for_each_online_cpu(cpu) {
408 gator_timer_offline_dispatch(cpu);
409 }
411 gator_running = 0;
412 gator_hrtimer_shutdown();
413 }
414 }
416 // This function runs in interrupt context and on the appropriate core
417 static void gator_timer_online(void* unused)
418 {
419 struct gator_interface *gi;
420 int len, cpu = smp_processor_id();
421 int* buffer;
423 gator_trace_power_online();
425 // online any events and output counters
426 if (marshal_event_header()) {
427 list_for_each_entry(gi, &gator_events, list) {
428 if (gi->online) {
429 len = gi->online(&buffer);
430 marshal_event(len, buffer);
431 }
432 }
433 }
435 gator_hrtimer_online(cpu);
436 }
438 // This function runs in interrupt context and may be running on a core other than core 'cpu'
439 static void gator_timer_online_dispatch(int cpu)
440 {
441 struct gator_interface *gi;
443 list_for_each_entry(gi, &gator_events, list)
444 if (gi->online_dispatch)
445 gi->online_dispatch(cpu);
447 gator_event_sampling_online_dispatch(cpu);
448 }
450 int gator_timer_start(unsigned long sample_rate)
451 {
452 int cpu;
454 if (gator_running) {
455 pr_notice("gator: already running\n");
456 return 0;
457 }
459 gator_running = 1;
461 // event based sampling trumps hr timer based sampling
462 if (event_based_sampling)
463 sample_rate = 0;
465 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
466 return -1;
468 for_each_online_cpu(cpu) {
469 gator_timer_online_dispatch(cpu);
470 }
471 on_each_cpu(gator_timer_online, NULL, 1);
473 return 0;
474 }
476 static uint64_t gator_get_time(void)
477 {
478 struct timespec ts;
479 uint64_t timestamp;
481 getnstimeofday(&ts);
482 timestamp = timespec_to_ns(&ts);
484 return timestamp;
485 }
487 /******************************************************************************
488 * cpu hotplug and pm notifiers
489 ******************************************************************************/
490 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
491 {
492 long cpu = (long)hcpu;
494 switch (action) {
495 case CPU_DOWN_PREPARE:
496 case CPU_DOWN_PREPARE_FROZEN:
497 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
498 gator_timer_offline_dispatch(cpu);
499 break;
500 case CPU_ONLINE:
501 case CPU_ONLINE_FROZEN:
502 gator_timer_online_dispatch(cpu);
503 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
504 break;
505 }
507 return NOTIFY_OK;
508 }
510 static struct notifier_block __refdata gator_hotcpu_notifier = {
511 .notifier_call = gator_hotcpu_notify,
512 };
514 // n.b. calling "on_each_cpu" only runs on those that are online
515 // Registered linux events are not disabled, so their counters will continue to collect
516 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
517 {
518 int cpu;
520 switch (event) {
521 case PM_HIBERNATION_PREPARE:
522 case PM_SUSPEND_PREPARE:
523 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
524 unregister_scheduler_tracepoints();
525 on_each_cpu(gator_timer_offline, NULL, 1);
526 for_each_online_cpu(cpu) {
527 gator_timer_offline_dispatch(cpu);
528 }
529 break;
530 case PM_POST_HIBERNATION:
531 case PM_POST_SUSPEND:
532 for_each_online_cpu(cpu) {
533 gator_timer_online_dispatch(cpu);
534 }
535 on_each_cpu(gator_timer_online, NULL, 1);
536 register_scheduler_tracepoints();
537 register_hotcpu_notifier(&gator_hotcpu_notifier);
538 break;
539 }
541 return NOTIFY_OK;
542 }
544 static struct notifier_block gator_pm_notifier = {
545 .notifier_call = gator_pm_notify,
546 };
548 static int gator_notifier_start(void)
549 {
550 int retval;
551 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
552 if (retval == 0)
553 retval = register_pm_notifier(&gator_pm_notifier);
554 return retval;
555 }
557 static void gator_notifier_stop(void)
558 {
559 unregister_pm_notifier(&gator_pm_notifier);
560 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
561 }
563 /******************************************************************************
564 * Main
565 ******************************************************************************/
566 static void gator_summary(void)
567 {
568 uint64_t timestamp, uptime = 0;
569 struct timespec uptime_ts;
570 void (*m2b)(struct timespec *ts);
572 timestamp = gator_get_time();
574 do_posix_clock_monotonic_gettime(&uptime_ts);
575 m2b = symbol_get(monotonic_to_bootbased);
576 if (m2b) {
577 m2b(&uptime_ts);
578 uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
579 }
581 marshal_summary(timestamp, uptime);
582 }
584 int gator_events_install(struct gator_interface *interface)
585 {
586 list_add_tail(&interface->list, &gator_events);
588 return 0;
589 }
591 int gator_events_get_key(void)
592 {
593 // key of zero is reserved as a timestamp
594 static int key = 1;
596 return key++;
597 }
599 static int gator_init(void)
600 {
601 int i;
603 // events sources (gator_events.h, generated by gator_events.sh)
604 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
605 if (gator_events_list[i])
606 gator_events_list[i]();
608 gator_trace_power_init();
610 return 0;
611 }
613 static int gator_start(void)
614 {
615 unsigned long cpu, i;
616 struct gator_interface *gi;
618 // Initialize the buffer with the frame type and core
619 for_each_present_cpu(cpu) {
620 for (i = 0; i < NUM_GATOR_BUFS; i++) {
621 gator_buffer_header(cpu, i);
622 }
623 }
625 // Capture the start time
626 gator_summary();
628 // start all events
629 list_for_each_entry(gi, &gator_events, list) {
630 if (gi->start && gi->start() != 0) {
631 struct list_head *ptr = gi->list.prev;
633 while (ptr != &gator_events) {
634 gi = list_entry(ptr, struct gator_interface, list);
636 if (gi->stop)
637 gi->stop();
639 ptr = ptr->prev;
640 }
641 goto events_failure;
642 }
643 }
645 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
646 if (cookies_initialize())
647 goto cookies_failure;
648 if (gator_annotate_start())
649 goto annotate_failure;
650 if (gator_trace_sched_start())
651 goto sched_failure;
652 if (gator_trace_power_start())
653 goto power_failure;
654 if (gator_trace_gpu_start())
655 goto gpu_failure;
656 if (gator_event_sampling_start())
657 goto event_sampling_failure;
658 if (gator_timer_start(gator_timer_count))
659 goto timer_failure;
660 if (gator_notifier_start())
661 goto notifier_failure;
663 return 0;
665 notifier_failure:
666 gator_timer_stop();
667 timer_failure:
668 gator_event_sampling_stop();
669 event_sampling_failure:
670 gator_trace_gpu_stop();
671 gpu_failure:
672 gator_trace_power_stop();
673 power_failure:
674 gator_trace_sched_stop();
675 sched_failure:
676 gator_annotate_stop();
677 annotate_failure:
678 cookies_release();
679 cookies_failure:
680 // stop all events
681 list_for_each_entry(gi, &gator_events, list)
682 if (gi->stop)
683 gi->stop();
684 events_failure:
686 return -1;
687 }
689 static void gator_stop(void)
690 {
691 struct gator_interface *gi;
693 // stop all events
694 list_for_each_entry(gi, &gator_events, list)
695 if (gi->stop)
696 gi->stop();
698 gator_annotate_stop();
699 gator_trace_sched_stop();
700 gator_trace_power_stop();
701 gator_trace_gpu_stop();
702 gator_event_sampling_stop();
704 // stop all interrupt callback reads before tearing down other interfaces
705 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
706 gator_timer_stop();
707 }
709 /******************************************************************************
710 * Filesystem
711 ******************************************************************************/
712 /* fopen("buffer") */
713 static int gator_op_setup(void)
714 {
715 int err = 0;
716 int cpu, i;
718 mutex_lock(&start_mutex);
720 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
721 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
723 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
724 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
726 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
727 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
729 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
730 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
732 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
733 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
735 gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
736 gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
738 gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
739 gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
741 // Initialize percpu per buffer variables
742 for (i = 0; i < NUM_GATOR_BUFS; i++) {
743 // Verify buffers are a power of 2
744 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
745 err = -ENOEXEC;
746 goto setup_error;
747 }
749 for_each_present_cpu(cpu) {
750 per_cpu(gator_buffer_read, cpu)[i] = 0;
751 per_cpu(gator_buffer_write, cpu)[i] = 0;
752 per_cpu(gator_buffer_commit, cpu)[i] = 0;
753 per_cpu(buffer_space_available, cpu)[i] = true;
755 // Annotation is a special case that only uses a single buffer
756 if (cpu > 0 && i == ANNOTATE_BUF) {
757 per_cpu(gator_buffer, cpu)[i] = NULL;
758 continue;
759 }
761 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
762 if (!per_cpu(gator_buffer, cpu)[i]) {
763 err = -ENOMEM;
764 goto setup_error;
765 }
766 }
767 }
769 setup_error:
770 mutex_unlock(&start_mutex);
771 return err;
772 }
774 /* Actually start profiling (echo 1>/dev/gator/enable) */
775 static int gator_op_start(void)
776 {
777 int err = 0;
779 mutex_lock(&start_mutex);
781 if (gator_started || gator_start())
782 err = -EINVAL;
783 else
784 gator_started = 1;
786 mutex_unlock(&start_mutex);
788 return err;
789 }
791 /* echo 0>/dev/gator/enable */
792 static void gator_op_stop(void)
793 {
794 mutex_lock(&start_mutex);
796 if (gator_started) {
797 gator_stop();
799 mutex_lock(&gator_buffer_mutex);
801 gator_started = 0;
802 cookies_release();
803 wake_up(&gator_buffer_wait);
805 mutex_unlock(&gator_buffer_mutex);
806 }
808 mutex_unlock(&start_mutex);
809 }
811 static void gator_shutdown(void)
812 {
813 int cpu, i;
815 mutex_lock(&start_mutex);
817 for_each_present_cpu(cpu) {
818 mutex_lock(&gator_buffer_mutex);
819 for (i = 0; i < NUM_GATOR_BUFS; i++) {
820 vfree(per_cpu(gator_buffer, cpu)[i]);
821 per_cpu(gator_buffer, cpu)[i] = NULL;
822 per_cpu(gator_buffer_read, cpu)[i] = 0;
823 per_cpu(gator_buffer_write, cpu)[i] = 0;
824 per_cpu(gator_buffer_commit, cpu)[i] = 0;
825 per_cpu(buffer_space_available, cpu)[i] = true;
826 }
827 mutex_unlock(&gator_buffer_mutex);
828 }
830 mutex_unlock(&start_mutex);
831 }
833 static int gator_set_backtrace(unsigned long val)
834 {
835 int err = 0;
837 mutex_lock(&start_mutex);
839 if (gator_started)
840 err = -EBUSY;
841 else
842 gator_backtrace_depth = val;
844 mutex_unlock(&start_mutex);
846 return err;
847 }
849 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
850 {
851 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
852 }
854 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
855 {
856 unsigned long val;
857 int retval;
859 if (*offset)
860 return -EINVAL;
862 retval = gatorfs_ulong_from_user(&val, buf, count);
863 if (retval)
864 return retval;
866 if (val)
867 retval = gator_op_start();
868 else
869 gator_op_stop();
871 if (retval)
872 return retval;
873 return count;
874 }
876 static const struct file_operations enable_fops = {
877 .read = enable_read,
878 .write = enable_write,
879 };
881 static int userspace_buffer_open(struct inode *inode, struct file *file)
882 {
883 int err = -EPERM;
885 if (!capable(CAP_SYS_ADMIN))
886 return -EPERM;
888 if (test_and_set_bit_lock(0, &gator_buffer_opened))
889 return -EBUSY;
891 if ((err = gator_op_setup()))
892 goto fail;
894 /* NB: the actual start happens from userspace
895 * echo 1 >/dev/gator/enable
896 */
898 return 0;
900 fail:
901 __clear_bit_unlock(0, &gator_buffer_opened);
902 return err;
903 }
905 static int userspace_buffer_release(struct inode *inode, struct file *file)
906 {
907 gator_op_stop();
908 gator_shutdown();
909 __clear_bit_unlock(0, &gator_buffer_opened);
910 return 0;
911 }
913 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
914 size_t count, loff_t *offset)
915 {
916 int retval = -EINVAL;
917 int commit = 0, length, length1, length2, read, byte, type_length;
918 char *buffer1;
919 char *buffer2 = NULL;
920 int cpu, buftype;
922 /* do not handle partial reads */
923 if (count != userspace_buffer_size || *offset)
924 return -EINVAL;
926 // sleep until the condition is true or a signal is received
927 // the condition is checked each time gator_buffer_wait is woken up
928 buftype = cpu = -1;
929 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
931 if (signal_pending(current))
932 return -EINTR;
934 length2 = 0;
935 retval = -EFAULT;
937 mutex_lock(&gator_buffer_mutex);
939 if (buftype == -1 || cpu == -1) {
940 retval = 0;
941 goto out;
942 }
944 read = per_cpu(gator_buffer_read, cpu)[buftype];
945 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
947 /* May happen if the buffer is freed during pending reads. */
948 if (!per_cpu(gator_buffer, cpu)[buftype]) {
949 retval = -EFAULT;
950 goto out;
951 }
953 /* determine the size of two halves */
954 length1 = commit - read;
955 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
956 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
957 if (length1 < 0) {
958 length1 = gator_buffer_size[buftype] - read;
959 length2 = commit;
960 }
962 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
963 type_length = gator_response_type ? 1 : 0;
964 length = length1 + length2 - type_length - sizeof(int);
965 for (byte = 0; byte < sizeof(int); byte++) {
966 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
967 }
969 /* start, middle or end */
970 if (length1 > 0) {
971 if (copy_to_user(&buf[0], buffer1, length1)) {
972 goto out;
973 }
974 }
976 /* possible wrap around */
977 if (length2 > 0) {
978 if (copy_to_user(&buf[length1], buffer2, length2)) {
979 goto out;
980 }
981 }
983 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
984 retval = length1 + length2;
986 /* kick just in case we've lost an SMP event */
987 wake_up(&gator_buffer_wait);
989 out:
990 mutex_unlock(&gator_buffer_mutex);
991 return retval;
992 }
994 const struct file_operations gator_event_buffer_fops = {
995 .open = userspace_buffer_open,
996 .release = userspace_buffer_release,
997 .read = userspace_buffer_read,
998 };
1000 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1001 {
1002 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
1003 offset);
1004 }
1006 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1007 {
1008 unsigned long val;
1009 int retval;
1011 if (*offset)
1012 return -EINVAL;
1014 retval = gatorfs_ulong_from_user(&val, buf, count);
1015 if (retval)
1016 return retval;
1018 retval = gator_set_backtrace(val);
1020 if (retval)
1021 return retval;
1022 return count;
1023 }
1025 static const struct file_operations depth_fops = {
1026 .read = depth_read,
1027 .write = depth_write
1028 };
1030 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1031 {
1032 struct dentry *dir;
1033 struct gator_interface *gi;
1034 int cpu;
1036 /* reinitialize default values */
1037 gator_cpu_cores = 0;
1038 for_each_present_cpu(cpu) {
1039 gator_cpu_cores++;
1040 }
1041 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1042 gator_response_type = 1;
1044 gatorfs_create_file(sb, root, "enable", &enable_fops);
1045 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1046 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1047 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1048 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1049 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1050 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1051 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1053 // Annotate interface
1054 gator_annotate_create_files(sb, root);
1056 // Linux Events
1057 dir = gatorfs_mkdir(sb, root, "events");
1058 list_for_each_entry(gi, &gator_events, list)
1059 if (gi->create_files)
1060 gi->create_files(sb, dir);
1062 // Power interface
1063 gator_trace_power_create_files(sb, dir);
1064 }
1066 /******************************************************************************
1067 * Module
1068 ******************************************************************************/
1069 static int __init gator_module_init(void)
1070 {
1071 if (gatorfs_register()) {
1072 return -1;
1073 }
1075 if (gator_init()) {
1076 gatorfs_unregister();
1077 return -1;
1078 }
1080 setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1082 return 0;
1083 }
1085 static void __exit gator_module_exit(void)
1086 {
1087 del_timer_sync(&gator_buffer_wake_up_timer);
1088 tracepoint_synchronize_unregister();
1089 gatorfs_unregister();
1090 }
1092 module_init(gator_module_init);
1093 module_exit(gator_module_exit);
1095 MODULE_LICENSE("GPL");
1096 MODULE_AUTHOR("ARM Ltd");
1097 MODULE_DESCRIPTION("Gator system profiler");