1 /**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 7;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <asm/stacktrace.h>
23 #include <asm/uaccess.h>
25 #include "gator.h"
26 #include "gator_events.h"
28 #ifndef CONFIG_GENERIC_TRACER
29 #ifndef CONFIG_TRACING
30 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
31 #endif
32 #endif
34 #ifndef CONFIG_PROFILING
35 #error gator requires the kernel to have CONFIG_PROFILING defined
36 #endif
38 #ifndef CONFIG_HIGH_RES_TIMERS
39 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
40 #endif
42 #if defined (__arm__)
43 #ifdef CONFIG_SMP
44 #ifndef CONFIG_LOCAL_TIMERS
45 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
46 #endif
47 #endif
48 #endif
50 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
51 #error kernels prior to 2.6.32 are not supported
52 #endif
54 /******************************************************************************
55 * DEFINES
56 ******************************************************************************/
57 #define TIMER_BUFFER_SIZE_DEFAULT (512*1024)
58 #define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
60 #define NO_COOKIE 0UL
61 #define INVALID_COOKIE ~0UL
63 #define FRAME_HRTIMER 1
64 #define FRAME_EVENT 2
65 #define FRAME_ANNOTATE 3
67 #define MESSAGE_COOKIE 1
68 #define MESSAGE_COUNTERS 3
69 #define MESSAGE_START_BACKTRACE 5
70 #define MESSAGE_END_BACKTRACE 7
71 #define MESSAGE_SCHEDULER_TRACE 9
72 #define MESSAGE_PID_NAME 11
74 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
76 #if defined(__arm__)
77 #define PC_REG regs->ARM_pc
78 #else
79 #define PC_REG regs->ip
80 #endif
82 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
84 /******************************************************************************
85 * Globals
86 ******************************************************************************/
87 static unsigned long gator_cpu_cores;
88 static unsigned long userspace_buffer_size;
89 static unsigned long gator_backtrace_depth;
91 static unsigned long gator_started;
92 static unsigned long gator_buffer_opened;
93 static unsigned long gator_timer_count;
94 static unsigned long gator_streaming;
95 static DEFINE_MUTEX(start_mutex);
96 static DEFINE_MUTEX(gator_buffer_mutex);
98 unsigned long gator_net_traffic;
99 bool event_based_sampling;
101 #define COMMIT_SIZE 128
102 #define COMMIT_MASK (COMMIT_SIZE-1)
103 static DEFINE_SPINLOCK(timer_commit_lock);
104 static int *gator_commit[NUM_GATOR_BUFS];
105 static int gator_commit_read[NUM_GATOR_BUFS];
106 static int gator_commit_write[NUM_GATOR_BUFS];
108 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
109 static DEFINE_PER_CPU(int, gator_first_time);
111 #if LINUX_PMU_SUPPORT
112 static void event_buffer_check(int cpu);
113 static DEFINE_SPINLOCK(event_commit_lock);
114 #endif
116 /******************************************************************************
117 * Prototypes
118 ******************************************************************************/
119 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
120 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
121 static void gator_buffer_write_string(int cpu, int buftype, char *x);
122 static int gator_write_packed_int(char *buffer, unsigned int x);
123 static int gator_write_packed_int64(char *buffer, unsigned long long x);
124 static void gator_add_trace(int cpu, int buftype, unsigned int address);
125 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
126 static uint64_t gator_get_time(void);
128 /******************************************************************************
129 * Application Includes
130 ******************************************************************************/
131 #include "gator_cookies.c"
132 #include "gator_trace_sched.c"
133 #include "gator_backtrace.c"
134 #include "gator_annotate.c"
135 #include "gator_fs.c"
136 #include "gator_ebs.c"
138 /******************************************************************************
139 * Misc
140 ******************************************************************************/
141 #if defined(__arm__)
142 u32 gator_cpuid(void)
143 {
144 u32 val;
145 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
146 return (val >> 4) & 0xfff;
147 }
148 #endif
150 /******************************************************************************
151 * Commit interface
152 ******************************************************************************/
153 static int buffer_commit_ready(int buftype)
154 {
155 return gator_commit_read[buftype] != gator_commit_write[buftype];
156 }
158 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
159 {
160 int read = gator_commit_read[buftype];
161 *cpu = gator_commit[buftype][read+0];
162 *readval = gator_commit[buftype][read+1];
163 *writeval = gator_commit[buftype][read+2];
164 gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
165 }
167 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
168 int write = gator_commit_write[buftype];
169 gator_commit[buftype][write+0] = cpu;
170 gator_commit[buftype][write+1] = readval;
171 gator_commit[buftype][write+2] = writeval;
172 gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
173 }
175 /******************************************************************************
176 * Buffer management
177 ******************************************************************************/
178 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
179 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
181 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
182 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
183 #include "gator_pack.c"
185 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
186 {
187 int i;
188 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
189 u32 mask = gator_buffer_mask[buftype];
190 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
192 for (i = 0; i < len; i++) {
193 buffer[write] = x[i];
194 write = (write + 1) & mask;
195 }
197 per_cpu(gator_buffer_write, cpu)[buftype] = write;
198 }
200 static void gator_buffer_write_string(int cpu, int buftype, char *x)
201 {
202 int len = strlen(x);
203 gator_buffer_write_packed_int(cpu, buftype, len);
204 gator_buffer_write_bytes(cpu, buftype, x, len);
205 }
207 static void gator_buffer_header(int cpu, int buftype)
208 {
209 int frame;
211 if (buftype == TIMER_BUF)
212 frame = FRAME_HRTIMER;
213 else if (buftype == EVENT_BUF)
214 frame = FRAME_EVENT;
215 else
216 frame = -1;
218 gator_buffer_write_packed_int(cpu, buftype, frame);
219 gator_buffer_write_packed_int(cpu, buftype, cpu);
220 }
222 static void gator_buffer_commit(int cpu, int buftype)
223 {
224 buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
225 per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
226 gator_buffer_header(cpu, buftype);
227 wake_up(&gator_buffer_wait);
228 }
230 static void timer_buffer_check(int cpu)
231 {
232 int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
233 if (available < 0) {
234 available += gator_buffer_size[TIMER_BUF];
235 }
236 if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
237 spin_lock(&timer_commit_lock);
238 gator_buffer_commit(cpu, TIMER_BUF);
239 spin_unlock(&timer_commit_lock);
240 }
241 }
243 #if LINUX_PMU_SUPPORT
244 static void event_buffer_check(int cpu)
245 {
246 int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
247 if (available < 0) {
248 available += gator_buffer_size[EVENT_BUF];
249 }
250 if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
251 spin_lock(&event_commit_lock);
252 gator_buffer_commit(cpu, EVENT_BUF);
253 spin_unlock(&event_commit_lock);
254 }
255 }
256 #endif
258 static void gator_add_trace(int cpu, int buftype, unsigned int address)
259 {
260 off_t offset = 0;
261 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
263 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
264 offset = address;
265 }
267 gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
268 gator_buffer_write_packed_int(cpu, buftype, cookie);
269 }
271 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
272 {
273 int inKernel = regs ? !user_mode(regs) : 1;
274 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
276 if (!regs)
277 return;
279 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
280 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
281 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
282 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
283 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
284 gator_buffer_write_packed_int(cpu, buftype, inKernel);
286 if (inKernel) {
287 kernel_backtrace(cpu, buftype, regs);
288 } else {
289 // Cookie+PC
290 gator_add_trace(cpu, buftype, PC_REG);
292 // Backtrace
293 if (gator_backtrace_depth)
294 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
295 }
297 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
298 }
300 /******************************************************************************
301 * hrtimer interrupt processing
302 ******************************************************************************/
303 static LIST_HEAD(gator_events);
305 static void gator_timer_interrupt(void)
306 {
307 struct pt_regs * const regs = get_irq_regs();
308 int cpu = smp_processor_id();
309 int *buffer, len, i, buftype = TIMER_BUF;
310 long long *buffer64;
311 struct gator_interface *gi;
313 // check full backtrace has enough space, otherwise may
314 // have breaks between samples in the same callstack
315 if (per_cpu(gator_first_time, cpu)) {
316 per_cpu(gator_first_time, cpu) = 0;
318 list_for_each_entry(gi, &gator_events, list)
319 if (gi->read)
320 gi->read(NULL);
322 return;
323 }
325 // Output scheduler
326 len = gator_trace_sched_read(&buffer64);
327 if (len > 0) {
328 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
329 gator_buffer_write_packed_int(cpu, buftype, len);
330 for (i = 0; i < len; i++) {
331 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
332 }
333 }
335 // Output counters
336 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
337 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
338 list_for_each_entry(gi, &gator_events, list) {
339 if (gi->read) {
340 len = gi->read(&buffer);
341 if (len > 0) {
342 gator_buffer_write_packed_int(cpu, buftype, len);
343 for (i = 0; i < len; i++) {
344 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
345 }
346 }
347 } else if (gi->read64) {
348 len = gi->read64(&buffer64);
349 if (len > 0) {
350 gator_buffer_write_packed_int(cpu, buftype, len);
351 for (i = 0; i < len; i++) {
352 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
353 }
354 }
355 }
356 }
357 gator_buffer_write_packed_int(cpu, buftype, 0);
359 // Output backtrace
360 if (!event_based_sampling) {
361 gator_add_sample(cpu, buftype, regs);
362 }
364 // Check and commit; generally, commit is set to occur once per second
365 timer_buffer_check(cpu);
366 }
368 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
369 DEFINE_PER_CPU(int, hrtimer_is_active);
370 static int hrtimer_running;
371 static ktime_t profiling_interval;
373 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
374 {
375 hrtimer_forward_now(hrtimer, profiling_interval);
376 gator_timer_interrupt();
377 return HRTIMER_RESTART;
378 }
380 static int gator_timer_init(void)
381 {
382 return 0;
383 }
385 static void __gator_timer_offline(void *unused)
386 {
387 int cpu = smp_processor_id();
388 if (per_cpu(hrtimer_is_active, cpu)) {
389 struct gator_interface *gi;
390 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
391 hrtimer_cancel(hrtimer);
392 per_cpu(hrtimer_is_active, cpu) = 0;
393 gator_buffer_commit(cpu, TIMER_BUF);
394 if (event_based_sampling)
395 gator_buffer_commit(cpu, EVENT_BUF);
397 // offline any events
398 list_for_each_entry(gi, &gator_events, list)
399 if (gi->offline)
400 gi->offline();
401 }
402 }
404 static void gator_timer_offline(void)
405 {
406 if (hrtimer_running) {
407 hrtimer_running = 0;
409 on_each_cpu(__gator_timer_offline, NULL, 1);
410 }
411 }
413 static void __gator_timer_online(void *unused)
414 {
415 int cpu = smp_processor_id();
416 if (!per_cpu(hrtimer_is_active, cpu)) {
417 struct gator_interface *gi;
418 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
419 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
420 hrtimer->function = gator_hrtimer_notify;
421 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
422 per_cpu(gator_first_time, cpu) = 1;
423 per_cpu(hrtimer_is_active, cpu) = 1;
425 // online any events
426 list_for_each_entry(gi, &gator_events, list)
427 if (gi->online)
428 gi->online();
429 }
430 }
432 int gator_timer_online(unsigned long setup)
433 {
434 if (!setup) {
435 pr_err("gator: cannot start due to a system tick value of zero\n");
436 return -1;
437 } else if (hrtimer_running) {
438 pr_notice("gator: high res timer already running\n");
439 return 0;
440 }
442 hrtimer_running = 1;
444 // calculate profiling interval
445 profiling_interval = ns_to_ktime(1000000000UL / setup);
447 // timer interrupt
448 on_each_cpu(__gator_timer_online, NULL, 1);
450 return 0;
451 }
453 static uint64_t gator_get_time(void)
454 {
455 struct timespec ts;
456 uint64_t timestamp;
458 getnstimeofday(&ts);
459 timestamp = timespec_to_ns(&ts);
461 return timestamp;
462 }
464 /******************************************************************************
465 * cpu online and pm notifiers
466 ******************************************************************************/
467 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
468 {
469 long cpu = (long)hcpu;
471 switch (action) {
472 case CPU_DOWN_PREPARE:
473 case CPU_DOWN_PREPARE_FROZEN:
474 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
475 break;
476 case CPU_ONLINE:
477 case CPU_ONLINE_FROZEN:
478 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
479 break;
480 }
482 return NOTIFY_OK;
483 }
485 static struct notifier_block __refdata gator_cpu_notifier = {
486 .notifier_call = gator_cpu_notify,
487 };
489 // n.b. calling "on_each_cpu" only runs on those that are online
490 // Registered linux events are not disabled, so their counters will continue to collect
491 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
492 {
493 switch (event) {
494 case PM_HIBERNATION_PREPARE:
495 case PM_SUSPEND_PREPARE:
496 unregister_hotcpu_notifier(&gator_cpu_notifier);
497 unregister_scheduler_tracepoints();
498 on_each_cpu(trace_sched_insert_idle, NULL, 1);
499 on_each_cpu(__gator_timer_offline, NULL, 1);
500 break;
501 case PM_POST_HIBERNATION:
502 case PM_POST_SUSPEND:
503 on_each_cpu(__gator_timer_online, NULL, 1);
504 register_scheduler_tracepoints();
505 register_hotcpu_notifier(&gator_cpu_notifier);
506 break;
507 }
509 return NOTIFY_OK;
510 }
512 static struct notifier_block gator_pm_notifier = {
513 .notifier_call = gator_pm_notify,
514 };
516 static int gator_notifier_start(void)
517 {
518 int retval;
519 retval = register_hotcpu_notifier(&gator_cpu_notifier);
520 if (retval == 0)
521 retval = register_pm_notifier(&gator_pm_notifier);
522 return retval;
523 }
525 static void gator_notifier_stop(void)
526 {
527 unregister_pm_notifier(&gator_pm_notifier);
528 unregister_hotcpu_notifier(&gator_cpu_notifier);
529 }
531 /******************************************************************************
532 * Main
533 ******************************************************************************/
534 int gator_events_install(struct gator_interface *interface)
535 {
536 list_add_tail(&interface->list, &gator_events);
538 return 0;
539 }
541 int gator_events_get_key(void)
542 {
543 static int key;
545 return key++;
546 }
548 static int gator_init(void)
549 {
550 int i;
552 if (gator_timer_init())
553 return -1;
554 if (gator_trace_sched_init())
555 return -1;
556 if (gator_annotate_init())
557 return -1;
559 // events sources (gator_events.h, generated by gator_events.sh)
560 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
561 if (gator_events_list[i])
562 gator_events_list[i]();
564 return 0;
565 }
567 static int gator_start(void)
568 {
569 struct gator_interface *gi;
571 // start all events
572 list_for_each_entry(gi, &gator_events, list) {
573 if (gi->start && gi->start() != 0) {
574 struct list_head *ptr = gi->list.prev;
576 while (ptr != &gator_events) {
577 gi = list_entry(ptr, struct gator_interface,
578 list);
580 if (gi->stop)
581 gi->stop();
583 ptr = ptr->prev;
584 }
585 goto events_failure;
586 }
587 }
589 // cookies shall be initialized before trace_sched_start() and gator_timer_online()
590 if (cookies_initialize())
591 goto cookies_failure;
592 if (gator_annotate_start())
593 goto annotate_failure;
594 if (gator_trace_sched_start())
595 goto sched_failure;
596 if (gator_event_sampling_start())
597 goto event_sampling_failure;
598 if (gator_timer_online(gator_timer_count))
599 goto timer_failure;
600 if (gator_notifier_start())
601 goto notifier_failure;
603 return 0;
605 notifier_failure:
606 gator_timer_offline();
607 timer_failure:
608 gator_event_sampling_stop();
609 event_sampling_failure:
610 gator_trace_sched_stop();
611 sched_failure:
612 gator_annotate_stop();
613 annotate_failure:
614 cookies_release();
615 cookies_failure:
616 // stop all events
617 list_for_each_entry(gi, &gator_events, list)
618 if (gi->stop)
619 gi->stop();
620 events_failure:
622 return -1;
623 }
625 static void gator_stop(void)
626 {
627 struct gator_interface *gi;
629 // stop all events
630 list_for_each_entry(gi, &gator_events, list)
631 if (gi->stop)
632 gi->stop();
634 gator_annotate_stop();
635 gator_trace_sched_stop();
636 gator_event_sampling_stop();
638 // stop all interrupt callback reads before tearing down other interfaces
639 gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
640 gator_timer_offline();
641 }
643 static void gator_exit(void)
644 {
645 gator_annotate_exit();
646 }
648 /******************************************************************************
649 * Filesystem
650 ******************************************************************************/
651 /* fopen("buffer") */
652 static int gator_op_setup(void)
653 {
654 int err = 0;
655 int cpu, i;
657 mutex_lock(&start_mutex);
659 gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
660 gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
662 // must be a power of 2
663 if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
664 err = -ENOEXEC;
665 goto setup_error;
666 }
668 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
669 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
671 gator_net_traffic = 0;
673 // Initialize per buffer variables
674 for (i = 0; i < NUM_GATOR_BUFS; i++) {
675 gator_commit_read[i] = gator_commit_write[i] = 0;
676 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
677 if (!gator_commit[i]) {
678 err = -ENOMEM;
679 goto setup_error;
680 }
682 // Initialize percpu per buffer variables
683 for_each_present_cpu(cpu) {
684 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
685 if (!per_cpu(gator_buffer, cpu)[i]) {
686 err = -ENOMEM;
687 goto setup_error;
688 }
690 per_cpu(gator_buffer_read, cpu)[i] = 0;
691 per_cpu(gator_buffer_write, cpu)[i] = 0;
692 gator_buffer_header(cpu, i);
693 }
694 }
696 setup_error:
697 mutex_unlock(&start_mutex);
698 return err;
699 }
701 /* Actually start profiling (echo 1>/dev/gator/enable) */
702 static int gator_op_start(void)
703 {
704 int err = 0;
706 mutex_lock(&start_mutex);
708 if (gator_started || gator_start())
709 err = -EINVAL;
710 else
711 gator_started = 1;
713 mutex_unlock(&start_mutex);
715 return err;
716 }
718 /* echo 0>/dev/gator/enable */
719 static void gator_op_stop(void)
720 {
721 mutex_lock(&start_mutex);
723 if (gator_started) {
724 gator_stop();
726 mutex_lock(&gator_buffer_mutex);
728 gator_started = 0;
729 cookies_release();
730 wake_up(&gator_buffer_wait);
732 mutex_unlock(&gator_buffer_mutex);
733 }
735 mutex_unlock(&start_mutex);
736 }
738 static void gator_shutdown(void)
739 {
740 int cpu, i;
742 mutex_lock(&start_mutex);
744 gator_annotate_shutdown();
746 for (i = 0; i < NUM_GATOR_BUFS; i++) {
747 vfree(gator_commit[i]);
748 gator_commit[i] = NULL;
749 }
751 for_each_present_cpu(cpu) {
752 mutex_lock(&gator_buffer_mutex);
753 for (i = 0; i < NUM_GATOR_BUFS; i++) {
754 vfree(per_cpu(gator_buffer, cpu)[i]);
755 per_cpu(gator_buffer, cpu)[i] = NULL;
756 per_cpu(gator_buffer_read, cpu)[i] = 0;
757 per_cpu(gator_buffer_write, cpu)[i] = 0;
758 }
759 mutex_unlock(&gator_buffer_mutex);
760 }
762 mutex_unlock(&start_mutex);
763 }
765 static int gator_set_backtrace(unsigned long val)
766 {
767 int err = 0;
769 mutex_lock(&start_mutex);
771 if (gator_started)
772 err = -EBUSY;
773 else
774 gator_backtrace_depth = val;
776 mutex_unlock(&start_mutex);
778 return err;
779 }
781 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
782 {
783 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
784 }
786 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
787 {
788 unsigned long val;
789 int retval;
791 if (*offset)
792 return -EINVAL;
794 retval = gatorfs_ulong_from_user(&val, buf, count);
795 if (retval)
796 return retval;
798 if (val)
799 retval = gator_op_start();
800 else
801 gator_op_stop();
803 if (retval)
804 return retval;
805 return count;
806 }
808 static const struct file_operations enable_fops = {
809 .read = enable_read,
810 .write = enable_write,
811 };
813 static int userspace_buffer_open(struct inode *inode, struct file *file)
814 {
815 int err = -EPERM;
817 if (!capable(CAP_SYS_ADMIN))
818 return -EPERM;
820 if (test_and_set_bit_lock(0, &gator_buffer_opened))
821 return -EBUSY;
823 if ((err = gator_op_setup()))
824 goto fail;
826 /* NB: the actual start happens from userspace
827 * echo 1 >/dev/gator/enable
828 */
830 return 0;
832 fail:
833 __clear_bit_unlock(0, &gator_buffer_opened);
834 return err;
835 }
837 static int userspace_buffer_release(struct inode *inode, struct file *file)
838 {
839 gator_op_stop();
840 gator_shutdown();
841 __clear_bit_unlock(0, &gator_buffer_opened);
842 return 0;
843 }
845 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
846 size_t count, loff_t *offset)
847 {
848 int retval = -EINVAL;
849 int commit, length1, length2, read;
850 char *buffer1;
851 char *buffer2 = NULL;
852 int cpu, i;
854 /* do not handle partial reads */
855 if (count != userspace_buffer_size || *offset)
856 return -EINVAL;
858 // sleep until the condition is true or a signal is received
859 // the condition is checked each time gator_buffer_wait is woken up
860 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
862 if (signal_pending(current))
863 return -EINTR;
865 length2 = 0;
866 retval = -EFAULT;
868 mutex_lock(&gator_buffer_mutex);
870 i = -1;
871 if (buffer_commit_ready(TIMER_BUF)) {
872 i = TIMER_BUF;
873 } else if (buffer_commit_ready(EVENT_BUF)) {
874 i = EVENT_BUF;
875 }
877 if (i != -1) {
878 buffer_commit_read(&cpu, i, &read, &commit);
880 /* May happen if the buffer is freed during pending reads. */
881 if (!per_cpu(gator_buffer, cpu)[i]) {
882 retval = -EFAULT;
883 goto out;
884 }
886 /* determine the size of two halves */
887 length1 = commit - read;
888 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
889 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
890 if (length1 < 0) {
891 length1 = gator_buffer_size[i] - read;
892 length2 = commit;
893 }
894 } else if (gator_annotate_ready()) {
895 length1 = gator_annotate_read(&buffer1);
896 if (!length1)
897 goto out;
898 } else {
899 retval = 0;
900 goto out;
901 }
903 /* start, middle or end */
904 if (length1 > 0) {
905 if (copy_to_user(&buf[0], buffer1, length1)) {
906 goto out;
907 }
908 }
910 /* possible wrap around */
911 if (length2 > 0) {
912 if (copy_to_user(&buf[length1], buffer2, length2)) {
913 goto out;
914 }
915 }
917 retval = length1 + length2;
919 /* kick just in case we've lost an SMP event */
920 wake_up(&gator_buffer_wait);
922 out:
923 // only adjust network stats if in streaming mode
924 if (gator_streaming)
925 gator_net_traffic += retval;
926 mutex_unlock(&gator_buffer_mutex);
927 return retval;
928 }
930 const struct file_operations gator_event_buffer_fops = {
931 .open = userspace_buffer_open,
932 .release = userspace_buffer_release,
933 .read = userspace_buffer_read,
934 };
936 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
937 {
938 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
939 offset);
940 }
942 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
943 {
944 unsigned long val;
945 int retval;
947 if (*offset)
948 return -EINVAL;
950 retval = gatorfs_ulong_from_user(&val, buf, count);
951 if (retval)
952 return retval;
954 retval = gator_set_backtrace(val);
956 if (retval)
957 return retval;
958 return count;
959 }
961 static const struct file_operations depth_fops = {
962 .read = depth_read,
963 .write = depth_write
964 };
966 void gator_op_create_files(struct super_block *sb, struct dentry *root)
967 {
968 struct dentry *dir;
969 struct gator_interface *gi;
970 int cpu;
972 /* reinitialize default values */
973 gator_cpu_cores = 0;
974 for_each_present_cpu(cpu) {
975 gator_cpu_cores++;
976 }
977 userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
978 gator_streaming = 1;
980 gatorfs_create_file(sb, root, "enable", &enable_fops);
981 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
982 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
983 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
984 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
985 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
986 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
987 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
989 // Annotate interface
990 gator_annotate_create_files(sb, root);
992 // Linux Events
993 dir = gatorfs_mkdir(sb, root, "events");
994 list_for_each_entry(gi, &gator_events, list)
995 if (gi->create_files)
996 gi->create_files(sb, dir);
997 }
999 /******************************************************************************
1000 * Module
1001 ******************************************************************************/
1002 static int __init gator_module_init(void)
1003 {
1004 if (gatorfs_register()) {
1005 return -1;
1006 }
1008 if (gator_init()) {
1009 gatorfs_unregister();
1010 return -1;
1011 }
1013 return 0;
1014 }
1016 static void __exit gator_module_exit(void)
1017 {
1018 tracepoint_synchronize_unregister();
1019 gatorfs_unregister();
1020 gator_exit();
1021 }
1023 module_init(gator_module_init);
1024 module_exit(gator_module_exit);
1026 MODULE_LICENSE("GPL");
1027 MODULE_AUTHOR("ARM Ltd");
1028 MODULE_DESCRIPTION("Gator system profiler");