1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 9;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58 * DEFINES
59 ******************************************************************************/
60 #define BACKTRACE_BUFFER_SIZE (128*1024)
61 #define COUNTER_BUFFER_SIZE (128*1024)
62 #define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
63 #define SCHED_TRACE_BUFFER_SIZE (128*1024)
64 #define GPU_TRACE_BUFFER_SIZE (64*1024)
65 #define COUNTER2_BUFFER_SIZE (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
66 #define WFI_BUFFER_SIZE (32*1024) // wfi counters have the core as part of the data and the core value in the frame header may be discarded
68 #define NO_COOKIE 0UL
69 #define INVALID_COOKIE ~0UL
71 #define FRAME_BACKTRACE 1
72 #define FRAME_COUNTER 2
73 #define FRAME_ANNOTATE 3
74 #define FRAME_SCHED_TRACE 4
75 #define FRAME_GPU_TRACE 5
76 #define FRAME_COUNTER2 6
77 #define FRAME_WFI 7
79 #define MESSAGE_COOKIE 1
80 #define MESSAGE_START_BACKTRACE 5
81 #define MESSAGE_END_BACKTRACE 7
82 #define MESSAGE_SUMMARY 9
83 #define MESSAGE_PID_NAME 11
85 #define MAXSIZE_PACK32 5
86 #define MAXSIZE_PACK64 9
88 #if defined(__arm__)
89 #define PC_REG regs->ARM_pc
90 #else
91 #define PC_REG regs->ip
92 #endif
94 enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
96 /******************************************************************************
97 * Globals
98 ******************************************************************************/
99 static unsigned long gator_cpu_cores;
100 static unsigned long userspace_buffer_size;
101 static unsigned long gator_backtrace_depth;
103 static unsigned long gator_started;
104 static unsigned long gator_buffer_opened;
105 static unsigned long gator_timer_count;
106 static unsigned long gator_response_type;
107 static DEFINE_MUTEX(start_mutex);
108 static DEFINE_MUTEX(gator_buffer_mutex);
110 bool event_based_sampling;
112 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
113 static LIST_HEAD(gator_events);
115 /******************************************************************************
116 * Prototypes
117 ******************************************************************************/
118 static void buffer_check(int cpu, int buftype);
119 static int buffer_bytes_available(int cpu, int buftype);
120 static bool buffer_check_space(int cpu, int buftype, int bytes);
121 static int contiguous_space_available(int cpu, int bufytpe);
122 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
123 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
124 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
125 static void gator_buffer_write_string(int cpu, int buftype, char *x);
126 static void gator_add_trace(int cpu, int buftype, unsigned int address);
127 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
128 static uint64_t gator_get_time(void);
130 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
131 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
132 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
133 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
134 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
135 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
136 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
138 /******************************************************************************
139 * Application Includes
140 ******************************************************************************/
141 #include "gator_marshaling.c"
142 #include "gator_hrtimer_perf.c"
143 #include "gator_hrtimer_gator.c"
144 #include "gator_cookies.c"
145 #include "gator_trace_sched.c"
146 #include "gator_trace_power.c"
147 #include "gator_trace_gpu.c"
148 #include "gator_backtrace.c"
149 #include "gator_annotate.c"
150 #include "gator_fs.c"
151 #include "gator_ebs.c"
152 #include "gator_pack.c"
154 /******************************************************************************
155 * Misc
156 ******************************************************************************/
157 #if defined(__arm__)
158 u32 gator_cpuid(void)
159 {
160 u32 val;
161 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
162 return (val >> 4) & 0xfff;
163 }
164 #endif
166 /******************************************************************************
167 * Commit interface
168 ******************************************************************************/
169 static bool buffer_commit_ready(int* cpu, int* buftype)
170 {
171 int cpu_x, x;
172 for_each_present_cpu(cpu_x) {
173 for (x = 0; x < NUM_GATOR_BUFS; x++)
174 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
175 *cpu = cpu_x;
176 *buftype = x;
177 return true;
178 }
179 }
180 return false;
181 }
183 /******************************************************************************
184 * Buffer management
185 ******************************************************************************/
186 static int buffer_bytes_available(int cpu, int buftype)
187 {
188 int remaining, filled;
190 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
191 if (filled < 0) {
192 filled += gator_buffer_size[buftype];
193 }
195 remaining = gator_buffer_size[buftype] - filled;
197 if (per_cpu(buffer_space_available, cpu)[buftype]) {
198 // Give some extra room; also allows space to insert the overflow error packet
199 remaining -= 200;
200 } else {
201 // Hysteresis, prevents multiple overflow messages
202 remaining -= 2000;
203 }
205 return remaining;
206 }
208 static int contiguous_space_available(int cpu, int buftype)
209 {
210 int remaining = buffer_bytes_available(cpu, buftype);
211 int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
212 if (remaining < contiguous)
213 return remaining;
214 else
215 return contiguous;
216 }
218 static bool buffer_check_space(int cpu, int buftype, int bytes)
219 {
220 int remaining = buffer_bytes_available(cpu, buftype);
222 if (remaining < bytes) {
223 per_cpu(buffer_space_available, cpu)[buftype] = false;
224 } else {
225 per_cpu(buffer_space_available, cpu)[buftype] = true;
226 }
228 return per_cpu(buffer_space_available, cpu)[buftype];
229 }
231 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
232 {
233 int i;
234 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
235 u32 mask = gator_buffer_mask[buftype];
236 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
238 for (i = 0; i < len; i++) {
239 buffer[write] = x[i];
240 write = (write + 1) & mask;
241 }
243 per_cpu(gator_buffer_write, cpu)[buftype] = write;
244 }
246 static void gator_buffer_write_string(int cpu, int buftype, char *x)
247 {
248 int len = strlen(x);
249 gator_buffer_write_packed_int(cpu, buftype, len);
250 gator_buffer_write_bytes(cpu, buftype, x, len);
251 }
253 static void gator_buffer_header(int cpu, int buftype)
254 {
255 int frame;
257 if (buftype == BACKTRACE_BUF)
258 frame = FRAME_BACKTRACE;
259 else if (buftype == COUNTER_BUF)
260 frame = FRAME_COUNTER;
261 else if (buftype == ANNOTATE_BUF)
262 frame = FRAME_ANNOTATE;
263 else if (buftype == SCHED_TRACE_BUF)
264 frame = FRAME_SCHED_TRACE;
265 else if (buftype == GPU_TRACE_BUF)
266 frame = FRAME_GPU_TRACE;
267 else if (buftype == COUNTER2_BUF)
268 frame = FRAME_COUNTER2;
269 else if (buftype == WFI_BUF)
270 frame = FRAME_WFI;
271 else
272 frame = -1;
274 if (per_cpu(gator_buffer, cpu)[buftype]) {
275 marshal_frame(cpu, buftype, frame);
276 }
277 }
279 static void gator_commit_buffer(int cpu, int buftype)
280 {
281 if (!per_cpu(gator_buffer, cpu)[buftype])
282 return;
284 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
285 gator_buffer_header(cpu, buftype);
286 wake_up(&gator_buffer_wait);
287 }
289 static void buffer_check(int cpu, int buftype)
290 {
291 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
292 if (filled < 0) {
293 filled += gator_buffer_size[buftype];
294 }
295 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
296 gator_commit_buffer(cpu, buftype);
297 }
298 }
300 static void gator_add_trace(int cpu, int buftype, unsigned int address)
301 {
302 off_t offset = 0;
303 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
305 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
306 offset = address;
307 }
309 marshal_backtrace(offset & ~1, cookie);
310 }
312 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
313 {
314 int inKernel = regs ? !user_mode(regs) : 1;
315 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
317 if (!regs)
318 return;
320 if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
321 return;
323 if (inKernel) {
324 kernel_backtrace(cpu, buftype, regs);
325 } else {
326 // Cookie+PC
327 gator_add_trace(cpu, buftype, PC_REG);
329 // Backtrace
330 if (gator_backtrace_depth)
331 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
332 }
334 marshal_backtrace_footer();
335 }
337 /******************************************************************************
338 * hrtimer interrupt processing
339 ******************************************************************************/
340 static void gator_timer_interrupt(void)
341 {
342 struct pt_regs * const regs = get_irq_regs();
343 int cpu = smp_processor_id();
345 // Output backtrace
346 gator_add_sample(cpu, BACKTRACE_BUF, regs);
348 // Collect counters
349 collect_counters();
350 }
352 static int gator_running;
354 // This function runs in interrupt context and on the appropriate core
355 static void gator_timer_offline(void* unused)
356 {
357 struct gator_interface *gi;
358 int i, len, cpu = smp_processor_id();
359 int* buffer;
361 gator_trace_sched_offline();
362 gator_trace_power_offline();
364 gator_hrtimer_offline(cpu);
366 // Offline any events and output counters
367 if (marshal_event_header()) {
368 list_for_each_entry(gi, &gator_events, list) {
369 if (gi->offline) {
370 len = gi->offline(&buffer);
371 marshal_event(len, buffer);
372 }
373 }
374 }
376 // Flush all buffers on this core
377 for (i = 0; i < NUM_GATOR_BUFS; i++)
378 gator_commit_buffer(cpu, i);
379 }
381 // This function runs in process context and may be running on a core other than core 'cpu'
382 static void gator_timer_offline_dispatch(int cpu)
383 {
384 struct gator_interface *gi;
386 list_for_each_entry(gi, &gator_events, list)
387 if (gi->offline_dispatch)
388 gi->offline_dispatch(cpu);
390 gator_event_sampling_offline_dispatch(cpu);
391 }
393 static void gator_timer_stop(void)
394 {
395 int cpu;
397 if (gator_running) {
398 on_each_cpu(gator_timer_offline, NULL, 1);
399 for_each_online_cpu(cpu) {
400 gator_timer_offline_dispatch(cpu);
401 }
403 gator_running = 0;
404 gator_hrtimer_shutdown();
405 }
406 }
408 // This function runs in interrupt context and on the appropriate core
409 static void gator_timer_online(void* unused)
410 {
411 struct gator_interface *gi;
412 int len, cpu = smp_processor_id();
413 int* buffer;
415 gator_trace_power_online();
417 // online any events and output counters
418 if (marshal_event_header()) {
419 list_for_each_entry(gi, &gator_events, list) {
420 if (gi->online) {
421 len = gi->online(&buffer);
422 marshal_event(len, buffer);
423 }
424 }
425 }
427 gator_hrtimer_online(cpu);
428 }
430 // This function runs in interrupt context and may be running on a core other than core 'cpu'
431 static void gator_timer_online_dispatch(int cpu)
432 {
433 struct gator_interface *gi;
435 list_for_each_entry(gi, &gator_events, list)
436 if (gi->online_dispatch)
437 gi->online_dispatch(cpu);
439 gator_event_sampling_online_dispatch(cpu);
440 }
442 int gator_timer_start(unsigned long sample_rate)
443 {
444 int cpu;
446 if (gator_running) {
447 pr_notice("gator: already running\n");
448 return 0;
449 }
451 gator_running = 1;
453 // event based sampling trumps hr timer based sampling
454 if (event_based_sampling)
455 sample_rate = 0;
457 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
458 return -1;
460 for_each_online_cpu(cpu) {
461 gator_timer_online_dispatch(cpu);
462 }
463 on_each_cpu(gator_timer_online, NULL, 1);
465 return 0;
466 }
468 static uint64_t gator_get_time(void)
469 {
470 struct timespec ts;
471 uint64_t timestamp;
473 getnstimeofday(&ts);
474 timestamp = timespec_to_ns(&ts);
476 return timestamp;
477 }
479 /******************************************************************************
480 * cpu hotplug and pm notifiers
481 ******************************************************************************/
482 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
483 {
484 long cpu = (long)hcpu;
486 switch (action) {
487 case CPU_DOWN_PREPARE:
488 case CPU_DOWN_PREPARE_FROZEN:
489 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
490 gator_timer_offline_dispatch(cpu);
491 break;
492 case CPU_ONLINE:
493 case CPU_ONLINE_FROZEN:
494 gator_timer_online_dispatch(cpu);
495 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
496 break;
497 }
499 return NOTIFY_OK;
500 }
502 static struct notifier_block __refdata gator_hotcpu_notifier = {
503 .notifier_call = gator_hotcpu_notify,
504 };
506 // n.b. calling "on_each_cpu" only runs on those that are online
507 // Registered linux events are not disabled, so their counters will continue to collect
508 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
509 {
510 int cpu;
512 switch (event) {
513 case PM_HIBERNATION_PREPARE:
514 case PM_SUSPEND_PREPARE:
515 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
516 unregister_scheduler_tracepoints();
517 on_each_cpu(gator_timer_offline, NULL, 1);
518 for_each_online_cpu(cpu) {
519 gator_timer_offline_dispatch(cpu);
520 }
521 break;
522 case PM_POST_HIBERNATION:
523 case PM_POST_SUSPEND:
524 for_each_online_cpu(cpu) {
525 gator_timer_online_dispatch(cpu);
526 }
527 on_each_cpu(gator_timer_online, NULL, 1);
528 register_scheduler_tracepoints();
529 register_hotcpu_notifier(&gator_hotcpu_notifier);
530 break;
531 }
533 return NOTIFY_OK;
534 }
536 static struct notifier_block gator_pm_notifier = {
537 .notifier_call = gator_pm_notify,
538 };
540 static int gator_notifier_start(void)
541 {
542 int retval;
543 retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
544 if (retval == 0)
545 retval = register_pm_notifier(&gator_pm_notifier);
546 return retval;
547 }
549 static void gator_notifier_stop(void)
550 {
551 unregister_pm_notifier(&gator_pm_notifier);
552 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
553 }
555 /******************************************************************************
556 * Main
557 ******************************************************************************/
558 static void gator_summary(void)
559 {
560 uint64_t timestamp, uptime = 0;
561 struct timespec uptime_ts;
562 void (*m2b)(struct timespec *ts);
564 timestamp = gator_get_time();
566 do_posix_clock_monotonic_gettime(&uptime_ts);
567 m2b = symbol_get(monotonic_to_bootbased);
568 if (m2b) {
569 m2b(&uptime_ts);
570 uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
571 }
573 marshal_summary(timestamp, uptime);
574 }
576 int gator_events_install(struct gator_interface *interface)
577 {
578 list_add_tail(&interface->list, &gator_events);
580 return 0;
581 }
583 int gator_events_get_key(void)
584 {
585 // key of zero is reserved as a timestamp
586 static int key = 1;
588 return key++;
589 }
591 static int gator_init(void)
592 {
593 int i;
595 // events sources (gator_events.h, generated by gator_events.sh)
596 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
597 if (gator_events_list[i])
598 gator_events_list[i]();
600 gator_trace_power_init();
602 return 0;
603 }
605 static int gator_start(void)
606 {
607 unsigned long cpu, i;
608 struct gator_interface *gi;
610 // Initialize the buffer with the frame type and core
611 for_each_present_cpu(cpu) {
612 for (i = 0; i < NUM_GATOR_BUFS; i++) {
613 gator_buffer_header(cpu, i);
614 }
615 }
617 // Capture the start time
618 gator_summary();
620 // start all events
621 list_for_each_entry(gi, &gator_events, list) {
622 if (gi->start && gi->start() != 0) {
623 struct list_head *ptr = gi->list.prev;
625 while (ptr != &gator_events) {
626 gi = list_entry(ptr, struct gator_interface, list);
628 if (gi->stop)
629 gi->stop();
631 ptr = ptr->prev;
632 }
633 goto events_failure;
634 }
635 }
637 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
638 if (cookies_initialize())
639 goto cookies_failure;
640 if (gator_annotate_start())
641 goto annotate_failure;
642 if (gator_trace_sched_start())
643 goto sched_failure;
644 if (gator_trace_power_start())
645 goto power_failure;
646 if (gator_trace_gpu_start())
647 goto gpu_failure;
648 if (gator_event_sampling_start())
649 goto event_sampling_failure;
650 if (gator_timer_start(gator_timer_count))
651 goto timer_failure;
652 if (gator_notifier_start())
653 goto notifier_failure;
655 return 0;
657 notifier_failure:
658 gator_timer_stop();
659 timer_failure:
660 gator_event_sampling_stop();
661 event_sampling_failure:
662 gator_trace_gpu_stop();
663 gpu_failure:
664 gator_trace_power_stop();
665 power_failure:
666 gator_trace_sched_stop();
667 sched_failure:
668 gator_annotate_stop();
669 annotate_failure:
670 cookies_release();
671 cookies_failure:
672 // stop all events
673 list_for_each_entry(gi, &gator_events, list)
674 if (gi->stop)
675 gi->stop();
676 events_failure:
678 return -1;
679 }
681 static void gator_stop(void)
682 {
683 struct gator_interface *gi;
685 // stop all events
686 list_for_each_entry(gi, &gator_events, list)
687 if (gi->stop)
688 gi->stop();
690 gator_annotate_stop();
691 gator_trace_sched_stop();
692 gator_trace_power_stop();
693 gator_trace_gpu_stop();
694 gator_event_sampling_stop();
696 // stop all interrupt callback reads before tearing down other interfaces
697 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
698 gator_timer_stop();
699 }
701 /******************************************************************************
702 * Filesystem
703 ******************************************************************************/
704 /* fopen("buffer") */
705 static int gator_op_setup(void)
706 {
707 int err = 0;
708 int cpu, i;
710 mutex_lock(&start_mutex);
712 gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
713 gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
715 gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
716 gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
718 gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
719 gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
721 gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
722 gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
724 gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
725 gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
727 gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
728 gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
730 gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
731 gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
733 // Initialize percpu per buffer variables
734 for (i = 0; i < NUM_GATOR_BUFS; i++) {
735 // Verify buffers are a power of 2
736 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
737 err = -ENOEXEC;
738 goto setup_error;
739 }
741 for_each_present_cpu(cpu) {
742 per_cpu(gator_buffer_read, cpu)[i] = 0;
743 per_cpu(gator_buffer_write, cpu)[i] = 0;
744 per_cpu(gator_buffer_commit, cpu)[i] = 0;
745 per_cpu(buffer_space_available, cpu)[i] = true;
747 // Annotation is a special case that only uses a single buffer
748 if (cpu > 0 && i == ANNOTATE_BUF) {
749 per_cpu(gator_buffer, cpu)[i] = NULL;
750 continue;
751 }
753 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
754 if (!per_cpu(gator_buffer, cpu)[i]) {
755 err = -ENOMEM;
756 goto setup_error;
757 }
758 }
759 }
761 setup_error:
762 mutex_unlock(&start_mutex);
763 return err;
764 }
766 /* Actually start profiling (echo 1>/dev/gator/enable) */
767 static int gator_op_start(void)
768 {
769 int err = 0;
771 mutex_lock(&start_mutex);
773 if (gator_started || gator_start())
774 err = -EINVAL;
775 else
776 gator_started = 1;
778 mutex_unlock(&start_mutex);
780 return err;
781 }
783 /* echo 0>/dev/gator/enable */
784 static void gator_op_stop(void)
785 {
786 mutex_lock(&start_mutex);
788 if (gator_started) {
789 gator_stop();
791 mutex_lock(&gator_buffer_mutex);
793 gator_started = 0;
794 cookies_release();
795 wake_up(&gator_buffer_wait);
797 mutex_unlock(&gator_buffer_mutex);
798 }
800 mutex_unlock(&start_mutex);
801 }
803 static void gator_shutdown(void)
804 {
805 int cpu, i;
807 mutex_lock(&start_mutex);
809 for_each_present_cpu(cpu) {
810 mutex_lock(&gator_buffer_mutex);
811 for (i = 0; i < NUM_GATOR_BUFS; i++) {
812 vfree(per_cpu(gator_buffer, cpu)[i]);
813 per_cpu(gator_buffer, cpu)[i] = NULL;
814 per_cpu(gator_buffer_read, cpu)[i] = 0;
815 per_cpu(gator_buffer_write, cpu)[i] = 0;
816 per_cpu(gator_buffer_commit, cpu)[i] = 0;
817 per_cpu(buffer_space_available, cpu)[i] = true;
818 }
819 mutex_unlock(&gator_buffer_mutex);
820 }
822 mutex_unlock(&start_mutex);
823 }
825 static int gator_set_backtrace(unsigned long val)
826 {
827 int err = 0;
829 mutex_lock(&start_mutex);
831 if (gator_started)
832 err = -EBUSY;
833 else
834 gator_backtrace_depth = val;
836 mutex_unlock(&start_mutex);
838 return err;
839 }
841 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
842 {
843 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
844 }
846 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
847 {
848 unsigned long val;
849 int retval;
851 if (*offset)
852 return -EINVAL;
854 retval = gatorfs_ulong_from_user(&val, buf, count);
855 if (retval)
856 return retval;
858 if (val)
859 retval = gator_op_start();
860 else
861 gator_op_stop();
863 if (retval)
864 return retval;
865 return count;
866 }
868 static const struct file_operations enable_fops = {
869 .read = enable_read,
870 .write = enable_write,
871 };
873 static int userspace_buffer_open(struct inode *inode, struct file *file)
874 {
875 int err = -EPERM;
877 if (!capable(CAP_SYS_ADMIN))
878 return -EPERM;
880 if (test_and_set_bit_lock(0, &gator_buffer_opened))
881 return -EBUSY;
883 if ((err = gator_op_setup()))
884 goto fail;
886 /* NB: the actual start happens from userspace
887 * echo 1 >/dev/gator/enable
888 */
890 return 0;
892 fail:
893 __clear_bit_unlock(0, &gator_buffer_opened);
894 return err;
895 }
897 static int userspace_buffer_release(struct inode *inode, struct file *file)
898 {
899 gator_op_stop();
900 gator_shutdown();
901 __clear_bit_unlock(0, &gator_buffer_opened);
902 return 0;
903 }
905 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
906 size_t count, loff_t *offset)
907 {
908 int retval = -EINVAL;
909 int commit = 0, length, length1, length2, read, byte, type_length;
910 char *buffer1;
911 char *buffer2 = NULL;
912 int cpu, buftype;
914 /* do not handle partial reads */
915 if (count != userspace_buffer_size || *offset)
916 return -EINVAL;
918 // sleep until the condition is true or a signal is received
919 // the condition is checked each time gator_buffer_wait is woken up
920 buftype = cpu = -1;
921 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
923 if (signal_pending(current))
924 return -EINTR;
926 length2 = 0;
927 retval = -EFAULT;
929 mutex_lock(&gator_buffer_mutex);
931 if (buftype == -1 || cpu == -1) {
932 retval = 0;
933 goto out;
934 }
936 read = per_cpu(gator_buffer_read, cpu)[buftype];
937 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
939 /* May happen if the buffer is freed during pending reads. */
940 if (!per_cpu(gator_buffer, cpu)[buftype]) {
941 retval = -EFAULT;
942 goto out;
943 }
945 /* determine the size of two halves */
946 length1 = commit - read;
947 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
948 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
949 if (length1 < 0) {
950 length1 = gator_buffer_size[buftype] - read;
951 length2 = commit;
952 }
954 // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
955 type_length = gator_response_type ? 1 : 0;
956 length = length1 + length2 - type_length - sizeof(int);
957 for (byte = 0; byte < sizeof(int); byte++) {
958 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
959 }
961 /* start, middle or end */
962 if (length1 > 0) {
963 if (copy_to_user(&buf[0], buffer1, length1)) {
964 goto out;
965 }
966 }
968 /* possible wrap around */
969 if (length2 > 0) {
970 if (copy_to_user(&buf[length1], buffer2, length2)) {
971 goto out;
972 }
973 }
975 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
976 retval = length1 + length2;
978 /* kick just in case we've lost an SMP event */
979 wake_up(&gator_buffer_wait);
981 out:
982 mutex_unlock(&gator_buffer_mutex);
983 return retval;
984 }
986 const struct file_operations gator_event_buffer_fops = {
987 .open = userspace_buffer_open,
988 .release = userspace_buffer_release,
989 .read = userspace_buffer_read,
990 };
992 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
993 {
994 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
995 offset);
996 }
998 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
999 {
1000 unsigned long val;
1001 int retval;
1003 if (*offset)
1004 return -EINVAL;
1006 retval = gatorfs_ulong_from_user(&val, buf, count);
1007 if (retval)
1008 return retval;
1010 retval = gator_set_backtrace(val);
1012 if (retval)
1013 return retval;
1014 return count;
1015 }
1017 static const struct file_operations depth_fops = {
1018 .read = depth_read,
1019 .write = depth_write
1020 };
1022 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1023 {
1024 struct dentry *dir;
1025 struct gator_interface *gi;
1026 int cpu;
1028 /* reinitialize default values */
1029 gator_cpu_cores = 0;
1030 for_each_present_cpu(cpu) {
1031 gator_cpu_cores++;
1032 }
1033 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1034 gator_response_type = 1;
1036 gatorfs_create_file(sb, root, "enable", &enable_fops);
1037 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1038 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1039 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1040 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1041 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1042 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1043 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1045 // Annotate interface
1046 gator_annotate_create_files(sb, root);
1048 // Linux Events
1049 dir = gatorfs_mkdir(sb, root, "events");
1050 list_for_each_entry(gi, &gator_events, list)
1051 if (gi->create_files)
1052 gi->create_files(sb, dir);
1054 // Power interface
1055 gator_trace_power_create_files(sb, dir);
1056 }
1058 /******************************************************************************
1059 * Module
1060 ******************************************************************************/
1061 static int __init gator_module_init(void)
1062 {
1063 if (gatorfs_register()) {
1064 return -1;
1065 }
1067 if (gator_init()) {
1068 gatorfs_unregister();
1069 return -1;
1070 }
1072 return 0;
1073 }
1075 static void __exit gator_module_exit(void)
1076 {
1077 tracepoint_synchronize_unregister();
1078 gatorfs_unregister();
1079 }
1081 module_init(gator_module_init);
1082 module_exit(gator_module_exit);
1084 MODULE_LICENSE("GPL");
1085 MODULE_AUTHOR("ARM Ltd");
1086 MODULE_DESCRIPTION("Gator system profiler");