1 /**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 7;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <asm/stacktrace.h>
22 #include <asm/uaccess.h>
24 #include "gator.h"
25 #include "gator_events.h"
27 #ifndef CONFIG_GENERIC_TRACER
28 #ifndef CONFIG_TRACING
29 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
30 #endif
31 #endif
33 #ifndef CONFIG_PROFILING
34 #error gator requires the kernel to have CONFIG_PROFILING defined
35 #endif
37 #ifndef CONFIG_HIGH_RES_TIMERS
38 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
39 #endif
41 #if defined (__arm__)
42 #ifdef CONFIG_SMP
43 #ifndef CONFIG_LOCAL_TIMERS
44 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
45 #endif
46 #endif
47 #endif
49 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
50 #error kernels prior to 2.6.32 are not supported
51 #endif
53 /******************************************************************************
54 * DEFINES
55 ******************************************************************************/
56 #define TIMER_BUFFER_SIZE_DEFAULT (512*1024)
57 #define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
59 #define NO_COOKIE 0UL
60 #define INVALID_COOKIE ~0UL
62 #define FRAME_HRTIMER 1
63 #define FRAME_EVENT 2
64 #define FRAME_ANNOTATE 3
66 #define MESSAGE_COOKIE 1
67 #define MESSAGE_COUNTERS 3
68 #define MESSAGE_START_BACKTRACE 5
69 #define MESSAGE_END_BACKTRACE 7
70 #define MESSAGE_SCHEDULER_TRACE 9
71 #define MESSAGE_PID_NAME 11
73 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
75 #if defined(__arm__)
76 #define PC_REG regs->ARM_pc
77 #else
78 #define PC_REG regs->ip
79 #endif
81 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
83 /******************************************************************************
84 * Globals
85 ******************************************************************************/
86 static unsigned long gator_cpu_cores;
87 static unsigned long userspace_buffer_size;
88 static unsigned long gator_backtrace_depth;
90 static unsigned long gator_started;
91 static unsigned long gator_buffer_opened;
92 static unsigned long gator_timer_count;
93 static unsigned long gator_streaming;
94 static DEFINE_MUTEX(start_mutex);
95 static DEFINE_MUTEX(gator_buffer_mutex);
97 unsigned long gator_net_traffic;
98 bool event_based_sampling;
100 #define COMMIT_SIZE 128
101 #define COMMIT_MASK (COMMIT_SIZE-1)
102 static DEFINE_SPINLOCK(timer_commit_lock);
103 static int *gator_commit[NUM_GATOR_BUFS];
104 static int gator_commit_read[NUM_GATOR_BUFS];
105 static int gator_commit_write[NUM_GATOR_BUFS];
107 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
108 static DEFINE_PER_CPU(int, gator_first_time);
110 #if LINUX_PMU_SUPPORT
111 static void event_buffer_check(int cpu);
112 static DEFINE_SPINLOCK(event_commit_lock);
113 #endif
115 /******************************************************************************
116 * Prototypes
117 ******************************************************************************/
118 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
119 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
120 static void gator_buffer_write_string(int cpu, int buftype, char *x);
121 static int gator_write_packed_int(char *buffer, unsigned int x);
122 static int gator_write_packed_int64(char *buffer, unsigned long long x);
123 static void gator_add_trace(int cpu, int buftype, unsigned int address);
124 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
125 static uint64_t gator_get_time(void);
127 /******************************************************************************
128 * Application Includes
129 ******************************************************************************/
130 #include "gator_cookies.c"
131 #include "gator_trace_sched.c"
132 #include "gator_backtrace.c"
133 #include "gator_annotate.c"
134 #include "gator_fs.c"
135 #include "gator_ebs.c"
137 /******************************************************************************
138 * Misc
139 ******************************************************************************/
140 #if defined(__arm__)
141 u32 gator_cpuid(void)
142 {
143 u32 val;
144 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
145 return (val >> 4) & 0xfff;
146 }
147 #endif
149 /******************************************************************************
150 * Commit interface
151 ******************************************************************************/
152 static int buffer_commit_ready(int buftype)
153 {
154 return gator_commit_read[buftype] != gator_commit_write[buftype];
155 }
157 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
158 {
159 int read = gator_commit_read[buftype];
160 *cpu = gator_commit[buftype][read+0];
161 *readval = gator_commit[buftype][read+1];
162 *writeval = gator_commit[buftype][read+2];
163 gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
164 }
166 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
167 int write = gator_commit_write[buftype];
168 gator_commit[buftype][write+0] = cpu;
169 gator_commit[buftype][write+1] = readval;
170 gator_commit[buftype][write+2] = writeval;
171 gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
172 }
174 /******************************************************************************
175 * Buffer management
176 ******************************************************************************/
177 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
178 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
179 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
182 #include "gator_pack.c"
184 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
185 {
186 int i;
187 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
188 u32 mask = gator_buffer_mask[buftype];
189 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
191 for (i = 0; i < len; i++) {
192 buffer[write] = x[i];
193 write = (write + 1) & mask;
194 }
196 per_cpu(gator_buffer_write, cpu)[buftype] = write;
197 }
199 static void gator_buffer_write_string(int cpu, int buftype, char *x)
200 {
201 int len = strlen(x);
202 gator_buffer_write_packed_int(cpu, buftype, len);
203 gator_buffer_write_bytes(cpu, buftype, x, len);
204 }
206 static void gator_buffer_header(int cpu, int buftype)
207 {
208 int frame;
210 if (buftype == TIMER_BUF)
211 frame = FRAME_HRTIMER;
212 else if (buftype == EVENT_BUF)
213 frame = FRAME_EVENT;
214 else
215 frame = -1;
217 gator_buffer_write_packed_int(cpu, buftype, frame);
218 gator_buffer_write_packed_int(cpu, buftype, cpu);
219 }
221 static void gator_buffer_commit(int cpu, int buftype)
222 {
223 buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
224 per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
225 gator_buffer_header(cpu, buftype);
226 wake_up(&gator_buffer_wait);
227 }
229 static void timer_buffer_check(int cpu)
230 {
231 int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
232 if (available < 0) {
233 available += gator_buffer_size[TIMER_BUF];
234 }
235 if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
236 spin_lock(&timer_commit_lock);
237 gator_buffer_commit(cpu, TIMER_BUF);
238 spin_unlock(&timer_commit_lock);
239 }
240 }
242 #if LINUX_PMU_SUPPORT
243 static void event_buffer_check(int cpu)
244 {
245 int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
246 if (available < 0) {
247 available += gator_buffer_size[EVENT_BUF];
248 }
249 if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
250 spin_lock(&event_commit_lock);
251 gator_buffer_commit(cpu, EVENT_BUF);
252 spin_unlock(&event_commit_lock);
253 }
254 }
255 #endif
257 static void gator_add_trace(int cpu, int buftype, unsigned int address)
258 {
259 off_t offset = 0;
260 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
262 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
263 offset = address;
264 }
266 gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
267 gator_buffer_write_packed_int(cpu, buftype, cookie);
268 }
270 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
271 {
272 int inKernel = regs ? !user_mode(regs) : 1;
273 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
275 if (!regs)
276 return;
278 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
279 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
280 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
281 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
282 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
283 gator_buffer_write_packed_int(cpu, buftype, inKernel);
285 if (inKernel) {
286 kernel_backtrace(cpu, buftype, regs);
287 } else {
288 // Cookie+PC
289 gator_add_trace(cpu, buftype, PC_REG);
291 // Backtrace
292 if (gator_backtrace_depth)
293 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
294 }
296 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
297 }
299 /******************************************************************************
300 * hrtimer interrupt processing
301 ******************************************************************************/
302 static LIST_HEAD(gator_events);
304 static void gator_timer_interrupt(void)
305 {
306 struct pt_regs * const regs = get_irq_regs();
307 int cpu = smp_processor_id();
308 int *buffer, len, i, buftype = TIMER_BUF;
309 long long *buffer64;
310 struct gator_interface *gi;
312 // check full backtrace has enough space, otherwise may
313 // have breaks between samples in the same callstack
314 if (per_cpu(gator_first_time, cpu)) {
315 per_cpu(gator_first_time, cpu) = 0;
317 list_for_each_entry(gi, &gator_events, list)
318 if (gi->read)
319 gi->read(NULL);
321 return;
322 }
324 // Output scheduler
325 len = gator_trace_sched_read(&buffer64);
326 if (len > 0) {
327 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
328 gator_buffer_write_packed_int(cpu, buftype, len);
329 for (i = 0; i < len; i++) {
330 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
331 }
332 }
334 // Output counters
335 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
336 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
337 list_for_each_entry(gi, &gator_events, list) {
338 if (gi->read) {
339 len = gi->read(&buffer);
340 if (len > 0) {
341 gator_buffer_write_packed_int(cpu, buftype, len);
342 for (i = 0; i < len; i++) {
343 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
344 }
345 }
346 } else if (gi->read64) {
347 len = gi->read64(&buffer64);
348 if (len > 0) {
349 gator_buffer_write_packed_int(cpu, buftype, len);
350 for (i = 0; i < len; i++) {
351 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
352 }
353 }
354 }
355 }
356 gator_buffer_write_packed_int(cpu, buftype, 0);
358 // Output backtrace
359 if (!event_based_sampling) {
360 gator_add_sample(cpu, buftype, regs);
361 }
363 // Check and commit; generally, commit is set to occur once per second
364 timer_buffer_check(cpu);
365 }
367 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
368 DEFINE_PER_CPU(int, hrtimer_is_active);
369 static int hrtimer_running;
370 static ktime_t profiling_interval;
372 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
373 {
374 hrtimer_forward_now(hrtimer, profiling_interval);
375 gator_timer_interrupt();
376 return HRTIMER_RESTART;
377 }
379 static int gator_timer_init(void)
380 {
381 return 0;
382 }
384 static void __gator_timer_offline(void *unused)
385 {
386 int cpu = smp_processor_id();
387 if (per_cpu(hrtimer_is_active, cpu)) {
388 struct gator_interface *gi;
389 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
390 hrtimer_cancel(hrtimer);
391 per_cpu(hrtimer_is_active, cpu) = 0;
392 gator_buffer_commit(cpu, TIMER_BUF);
393 if (event_based_sampling)
394 gator_buffer_commit(cpu, EVENT_BUF);
396 // offline any events
397 list_for_each_entry(gi, &gator_events, list)
398 if (gi->offline)
399 gi->offline();
400 }
401 }
403 static void gator_timer_offline(void)
404 {
405 if (hrtimer_running) {
406 hrtimer_running = 0;
408 on_each_cpu(__gator_timer_offline, NULL, 1);
409 }
410 }
412 static void __gator_timer_online(void *unused)
413 {
414 int cpu = smp_processor_id();
415 if (!per_cpu(hrtimer_is_active, cpu)) {
416 struct gator_interface *gi;
417 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
418 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
419 hrtimer->function = gator_hrtimer_notify;
420 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
421 per_cpu(gator_first_time, cpu) = 1;
422 per_cpu(hrtimer_is_active, cpu) = 1;
424 // online any events
425 list_for_each_entry(gi, &gator_events, list)
426 if (gi->online)
427 gi->online();
428 }
429 }
431 int gator_timer_online(unsigned long setup)
432 {
433 if (!setup) {
434 pr_err("gator: cannot start due to a system tick value of zero\n");
435 return -1;
436 } else if (hrtimer_running) {
437 pr_notice("gator: high res timer already running\n");
438 return 0;
439 }
441 hrtimer_running = 1;
443 // calculate profiling interval
444 profiling_interval = ns_to_ktime(1000000000UL / setup);
446 // timer interrupt
447 on_each_cpu(__gator_timer_online, NULL, 1);
449 return 0;
450 }
452 static uint64_t gator_get_time(void)
453 {
454 struct timespec ts;
455 uint64_t timestamp;
457 getnstimeofday(&ts);
458 timestamp = timespec_to_ns(&ts);
460 return timestamp;
461 }
463 /******************************************************************************
464 * cpu online and pm notifiers
465 ******************************************************************************/
466 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
467 {
468 long cpu = (long)hcpu;
470 switch (action) {
471 case CPU_DOWN_PREPARE:
472 case CPU_DOWN_PREPARE_FROZEN:
473 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
474 break;
475 case CPU_ONLINE:
476 case CPU_ONLINE_FROZEN:
477 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
478 break;
479 }
481 return NOTIFY_OK;
482 }
484 static struct notifier_block __refdata gator_cpu_notifier = {
485 .notifier_call = gator_cpu_notify,
486 };
488 // n.b. calling "on_each_cpu" only runs on those that are online
489 // Registered linux events are not disabled, so their counters will continue to collect
490 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
491 {
492 switch (event) {
493 case PM_HIBERNATION_PREPARE:
494 case PM_SUSPEND_PREPARE:
495 unregister_hotcpu_notifier(&gator_cpu_notifier);
496 unregister_scheduler_tracepoints();
497 on_each_cpu(trace_sched_insert_idle, NULL, 1);
498 on_each_cpu(__gator_timer_offline, NULL, 1);
499 break;
500 case PM_POST_HIBERNATION:
501 case PM_POST_SUSPEND:
502 on_each_cpu(__gator_timer_online, NULL, 1);
503 register_scheduler_tracepoints();
504 register_hotcpu_notifier(&gator_cpu_notifier);
505 break;
506 }
508 return NOTIFY_OK;
509 }
511 static struct notifier_block gator_pm_notifier = {
512 .notifier_call = gator_pm_notify,
513 };
515 static int gator_notifier_start(void)
516 {
517 int retval;
518 retval = register_hotcpu_notifier(&gator_cpu_notifier);
519 if (retval == 0)
520 retval = register_pm_notifier(&gator_pm_notifier);
521 return retval;
522 }
524 static void gator_notifier_stop(void)
525 {
526 unregister_pm_notifier(&gator_pm_notifier);
527 unregister_hotcpu_notifier(&gator_cpu_notifier);
528 }
530 /******************************************************************************
531 * Main
532 ******************************************************************************/
533 int gator_events_install(struct gator_interface *interface)
534 {
535 list_add_tail(&interface->list, &gator_events);
537 return 0;
538 }
540 int gator_events_get_key(void)
541 {
542 static int key;
544 return key++;
545 }
547 static int gator_init(void)
548 {
549 int i;
551 if (gator_timer_init())
552 return -1;
553 if (gator_trace_sched_init())
554 return -1;
555 if (gator_annotate_init())
556 return -1;
558 // events sources (gator_events.h, generated by gator_events.sh)
559 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
560 if (gator_events_list[i])
561 gator_events_list[i]();
563 return 0;
564 }
566 static int gator_start(void)
567 {
568 struct gator_interface *gi;
570 // start all events
571 list_for_each_entry(gi, &gator_events, list) {
572 if (gi->start && gi->start() != 0) {
573 struct list_head *ptr = gi->list.prev;
575 while (ptr != &gator_events) {
576 gi = list_entry(ptr, struct gator_interface,
577 list);
579 if (gi->stop)
580 gi->stop();
582 ptr = ptr->prev;
583 }
584 goto events_failure;
585 }
586 }
588 // cookies shall be initialized before trace_sched_start() and gator_timer_online()
589 if (cookies_initialize())
590 goto cookies_failure;
591 if (gator_annotate_start())
592 goto annotate_failure;
593 if (gator_trace_sched_start())
594 goto sched_failure;
595 if (gator_event_sampling_start())
596 goto event_sampling_failure;
597 if (gator_timer_online(gator_timer_count))
598 goto timer_failure;
599 if (gator_notifier_start())
600 goto notifier_failure;
602 return 0;
604 notifier_failure:
605 gator_timer_offline();
606 timer_failure:
607 gator_event_sampling_stop();
608 event_sampling_failure:
609 gator_trace_sched_stop();
610 sched_failure:
611 gator_annotate_stop();
612 annotate_failure:
613 cookies_release();
614 cookies_failure:
615 // stop all events
616 list_for_each_entry(gi, &gator_events, list)
617 if (gi->stop)
618 gi->stop();
619 events_failure:
621 return -1;
622 }
624 static void gator_stop(void)
625 {
626 struct gator_interface *gi;
628 // stop all events
629 list_for_each_entry(gi, &gator_events, list)
630 if (gi->stop)
631 gi->stop();
633 gator_annotate_stop();
634 gator_trace_sched_stop();
635 gator_event_sampling_stop();
637 // stop all interrupt callback reads before tearing down other interfaces
638 gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
639 gator_timer_offline();
640 }
642 static void gator_exit(void)
643 {
644 gator_annotate_exit();
645 }
647 /******************************************************************************
648 * Filesystem
649 ******************************************************************************/
650 /* fopen("buffer") */
651 static int gator_op_setup(void)
652 {
653 int err = 0;
654 int cpu, i;
656 mutex_lock(&start_mutex);
658 gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
659 gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
661 // must be a power of 2
662 if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
663 err = -ENOEXEC;
664 goto setup_error;
665 }
667 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
668 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
670 gator_net_traffic = 0;
672 // Initialize per buffer variables
673 for (i = 0; i < NUM_GATOR_BUFS; i++) {
674 gator_commit_read[i] = gator_commit_write[i] = 0;
675 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
676 if (!gator_commit[i]) {
677 err = -ENOMEM;
678 goto setup_error;
679 }
681 // Initialize percpu per buffer variables
682 for_each_present_cpu(cpu) {
683 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
684 if (!per_cpu(gator_buffer, cpu)[i]) {
685 err = -ENOMEM;
686 goto setup_error;
687 }
689 per_cpu(gator_buffer_read, cpu)[i] = 0;
690 per_cpu(gator_buffer_write, cpu)[i] = 0;
691 gator_buffer_header(cpu, i);
692 }
693 }
695 setup_error:
696 mutex_unlock(&start_mutex);
697 return err;
698 }
700 /* Actually start profiling (echo 1>/dev/gator/enable) */
701 static int gator_op_start(void)
702 {
703 int err = 0;
705 mutex_lock(&start_mutex);
707 if (gator_started || gator_start())
708 err = -EINVAL;
709 else
710 gator_started = 1;
712 mutex_unlock(&start_mutex);
714 return err;
715 }
717 /* echo 0>/dev/gator/enable */
718 static void gator_op_stop(void)
719 {
720 mutex_lock(&start_mutex);
722 if (gator_started) {
723 gator_stop();
725 mutex_lock(&gator_buffer_mutex);
727 gator_started = 0;
728 cookies_release();
729 wake_up(&gator_buffer_wait);
731 mutex_unlock(&gator_buffer_mutex);
732 }
734 mutex_unlock(&start_mutex);
735 }
737 static void gator_shutdown(void)
738 {
739 int cpu, i;
741 mutex_lock(&start_mutex);
743 gator_annotate_shutdown();
745 for (i = 0; i < NUM_GATOR_BUFS; i++) {
746 vfree(gator_commit[i]);
747 gator_commit[i] = NULL;
748 }
750 for_each_present_cpu(cpu) {
751 mutex_lock(&gator_buffer_mutex);
752 for (i = 0; i < NUM_GATOR_BUFS; i++) {
753 vfree(per_cpu(gator_buffer, cpu)[i]);
754 per_cpu(gator_buffer, cpu)[i] = NULL;
755 per_cpu(gator_buffer_read, cpu)[i] = 0;
756 per_cpu(gator_buffer_write, cpu)[i] = 0;
757 }
758 mutex_unlock(&gator_buffer_mutex);
759 }
761 mutex_unlock(&start_mutex);
762 }
764 static int gator_set_backtrace(unsigned long val)
765 {
766 int err = 0;
768 mutex_lock(&start_mutex);
770 if (gator_started)
771 err = -EBUSY;
772 else
773 gator_backtrace_depth = val;
775 mutex_unlock(&start_mutex);
777 return err;
778 }
780 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
781 {
782 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
783 }
785 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
786 {
787 unsigned long val;
788 int retval;
790 if (*offset)
791 return -EINVAL;
793 retval = gatorfs_ulong_from_user(&val, buf, count);
794 if (retval)
795 return retval;
797 if (val)
798 retval = gator_op_start();
799 else
800 gator_op_stop();
802 if (retval)
803 return retval;
804 return count;
805 }
807 static const struct file_operations enable_fops = {
808 .read = enable_read,
809 .write = enable_write,
810 };
812 static int userspace_buffer_open(struct inode *inode, struct file *file)
813 {
814 int err = -EPERM;
816 if (!capable(CAP_SYS_ADMIN))
817 return -EPERM;
819 if (test_and_set_bit_lock(0, &gator_buffer_opened))
820 return -EBUSY;
822 if ((err = gator_op_setup()))
823 goto fail;
825 /* NB: the actual start happens from userspace
826 * echo 1 >/dev/gator/enable
827 */
829 return 0;
831 fail:
832 __clear_bit_unlock(0, &gator_buffer_opened);
833 return err;
834 }
836 static int userspace_buffer_release(struct inode *inode, struct file *file)
837 {
838 gator_op_stop();
839 gator_shutdown();
840 __clear_bit_unlock(0, &gator_buffer_opened);
841 return 0;
842 }
844 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
845 size_t count, loff_t *offset)
846 {
847 int retval = -EINVAL;
848 int commit, length1, length2, read;
849 char *buffer1;
850 char *buffer2 = NULL;
851 int cpu, i;
853 /* do not handle partial reads */
854 if (count != userspace_buffer_size || *offset)
855 return -EINVAL;
857 // sleep until the condition is true or a signal is received
858 // the condition is checked each time gator_buffer_wait is woken up
859 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
861 if (signal_pending(current))
862 return -EINTR;
864 length2 = 0;
865 retval = -EFAULT;
867 mutex_lock(&gator_buffer_mutex);
869 i = -1;
870 if (buffer_commit_ready(TIMER_BUF)) {
871 i = TIMER_BUF;
872 } else if (buffer_commit_ready(EVENT_BUF)) {
873 i = EVENT_BUF;
874 }
876 if (i != -1) {
877 buffer_commit_read(&cpu, i, &read, &commit);
879 /* May happen if the buffer is freed during pending reads. */
880 if (!per_cpu(gator_buffer, cpu)[i]) {
881 retval = -EFAULT;
882 goto out;
883 }
885 /* determine the size of two halves */
886 length1 = commit - read;
887 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
888 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
889 if (length1 < 0) {
890 length1 = gator_buffer_size[i] - read;
891 length2 = commit;
892 }
893 } else if (gator_annotate_ready()) {
894 length1 = gator_annotate_read(&buffer1);
895 if (!length1)
896 goto out;
897 } else {
898 retval = 0;
899 goto out;
900 }
902 /* start, middle or end */
903 if (length1 > 0) {
904 if (copy_to_user(&buf[0], buffer1, length1)) {
905 goto out;
906 }
907 }
909 /* possible wrap around */
910 if (length2 > 0) {
911 if (copy_to_user(&buf[length1], buffer2, length2)) {
912 goto out;
913 }
914 }
916 retval = length1 + length2;
918 /* kick just in case we've lost an SMP event */
919 wake_up(&gator_buffer_wait);
921 out:
922 // only adjust network stats if in streaming mode
923 if (gator_streaming)
924 gator_net_traffic += retval;
925 mutex_unlock(&gator_buffer_mutex);
926 return retval;
927 }
929 const struct file_operations gator_event_buffer_fops = {
930 .open = userspace_buffer_open,
931 .release = userspace_buffer_release,
932 .read = userspace_buffer_read,
933 };
935 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
936 {
937 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
938 offset);
939 }
941 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
942 {
943 unsigned long val;
944 int retval;
946 if (*offset)
947 return -EINVAL;
949 retval = gatorfs_ulong_from_user(&val, buf, count);
950 if (retval)
951 return retval;
953 retval = gator_set_backtrace(val);
955 if (retval)
956 return retval;
957 return count;
958 }
960 static const struct file_operations depth_fops = {
961 .read = depth_read,
962 .write = depth_write
963 };
965 void gator_op_create_files(struct super_block *sb, struct dentry *root)
966 {
967 struct dentry *dir;
968 struct gator_interface *gi;
969 int cpu;
971 /* reinitialize default values */
972 gator_cpu_cores = 0;
973 for_each_present_cpu(cpu) {
974 gator_cpu_cores++;
975 }
976 userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
977 gator_streaming = 1;
979 gatorfs_create_file(sb, root, "enable", &enable_fops);
980 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
981 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
982 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
983 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
984 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
985 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
986 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
988 // Annotate interface
989 gator_annotate_create_files(sb, root);
991 // Linux Events
992 dir = gatorfs_mkdir(sb, root, "events");
993 list_for_each_entry(gi, &gator_events, list)
994 if (gi->create_files)
995 gi->create_files(sb, dir);
996 }
998 /******************************************************************************
999 * Module
1000 ******************************************************************************/
1001 static int __init gator_module_init(void)
1002 {
1003 if (gatorfs_register()) {
1004 return -1;
1005 }
1007 if (gator_init()) {
1008 gatorfs_unregister();
1009 return -1;
1010 }
1012 return 0;
1013 }
1015 static void __exit gator_module_exit(void)
1016 {
1017 tracepoint_synchronize_unregister();
1018 gatorfs_unregister();
1019 gator_exit();
1020 }
1022 module_init(gator_module_init);
1023 module_exit(gator_module_exit);
1025 MODULE_LICENSE("GPL");
1026 MODULE_AUTHOR("ARM Ltd");
1027 MODULE_DESCRIPTION("Gator system profiler");