1 /**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 6;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
22 #include "gator.h"
23 #include "gator_events.h"
25 #ifndef CONFIG_GENERIC_TRACER
26 #ifndef CONFIG_TRACING
27 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
28 #endif
29 #endif
31 #ifndef CONFIG_PROFILING
32 #error gator requires the kernel to have CONFIG_PROFILING defined
33 #endif
35 #ifndef CONFIG_HIGH_RES_TIMERS
36 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
37 #endif
39 #if defined (__arm__)
40 #ifdef CONFIG_SMP
41 #ifndef CONFIG_LOCAL_TIMERS
42 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
43 #endif
44 #endif
45 #endif
47 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
48 #error kernels prior to 2.6.32 are not supported
49 #endif
51 /******************************************************************************
52 * DEFINES
53 ******************************************************************************/
54 #define TIMER_BUFFER_SIZE_DEFAULT (256*1024)
55 #define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
57 #define NO_COOKIE 0UL
58 #define INVALID_COOKIE ~0UL
60 #define FRAME_HRTIMER 1
61 #define FRAME_EVENT 2
62 #define FRAME_ANNOTATE 3
64 #define MESSAGE_COOKIE 1
65 #define MESSAGE_COUNTERS 3
66 #define MESSAGE_START_BACKTRACE 5
67 #define MESSAGE_END_BACKTRACE 7
68 #define MESSAGE_SCHEDULER_TRACE 9
69 #define MESSAGE_PID_NAME 11
71 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
73 #if defined(__arm__)
74 #define PC_REG regs->ARM_pc
75 #else
76 #define PC_REG regs->ip
77 #endif
79 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
81 /******************************************************************************
82 * Globals
83 ******************************************************************************/
84 static unsigned long gator_cpu_cores;
85 static unsigned long userspace_buffer_size;
86 static unsigned long gator_backtrace_depth;
88 static unsigned long gator_started;
89 static unsigned long gator_buffer_opened;
90 static unsigned long gator_timer_count;
91 static unsigned long gator_streaming;
92 static DEFINE_MUTEX(start_mutex);
93 static DEFINE_MUTEX(gator_buffer_mutex);
95 unsigned long gator_net_traffic;
96 bool event_based_sampling;
98 #define COMMIT_SIZE 128
99 #define COMMIT_MASK (COMMIT_SIZE-1)
100 static DEFINE_SPINLOCK(timer_commit_lock);
101 static int *gator_commit[NUM_GATOR_BUFS];
102 static int gator_commit_read[NUM_GATOR_BUFS];
103 static int gator_commit_write[NUM_GATOR_BUFS];
105 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
106 static DEFINE_PER_CPU(int, gator_first_time);
108 #if LINUX_PMU_SUPPORT
109 static void event_buffer_check(int cpu);
110 static DEFINE_SPINLOCK(event_commit_lock);
111 #endif
113 /******************************************************************************
114 * Prototypes
115 ******************************************************************************/
116 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
117 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
118 static void gator_buffer_write_string(int cpu, int buftype, char *x);
119 static int gator_write_packed_int(char *buffer, unsigned int x);
120 static int gator_write_packed_int64(char *buffer, unsigned long long x);
121 static void gator_add_trace(int cpu, int buftype, unsigned int address);
122 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
123 static uint64_t gator_get_time(void);
125 /******************************************************************************
126 * Application Includes
127 ******************************************************************************/
128 #include "gator_cookies.c"
129 #include "gator_trace_sched.c"
130 #include "gator_backtrace.c"
131 #include "gator_annotate.c"
132 #include "gator_fs.c"
133 #include "gator_ebs.c"
135 /******************************************************************************
136 * Misc
137 ******************************************************************************/
138 #if defined(__arm__)
139 u32 gator_cpuid(void)
140 {
141 u32 val;
142 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
143 return (val >> 4) & 0xfff;
144 }
145 #endif
147 /******************************************************************************
148 * Commit interface
149 ******************************************************************************/
150 static int buffer_commit_ready(int buftype)
151 {
152 return gator_commit_read[buftype] != gator_commit_write[buftype];
153 }
155 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
156 {
157 int read = gator_commit_read[buftype];
158 *cpu = gator_commit[buftype][read+0];
159 *readval = gator_commit[buftype][read+1];
160 *writeval = gator_commit[buftype][read+2];
161 gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
162 }
164 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
165 int write = gator_commit_write[buftype];
166 gator_commit[buftype][write+0] = cpu;
167 gator_commit[buftype][write+1] = readval;
168 gator_commit[buftype][write+2] = writeval;
169 gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
170 }
172 /******************************************************************************
173 * Buffer management
174 ******************************************************************************/
175 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
176 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
177 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
178 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
179 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
180 #include "gator_pack.c"
182 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
183 {
184 int i;
185 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
186 u32 mask = gator_buffer_mask[buftype];
187 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
189 for (i = 0; i < len; i++) {
190 buffer[write] = x[i];
191 write = (write + 1) & mask;
192 }
194 per_cpu(gator_buffer_write, cpu)[buftype] = write;
195 }
197 static void gator_buffer_write_string(int cpu, int buftype, char *x)
198 {
199 int len = strlen(x);
200 gator_buffer_write_packed_int(cpu, buftype, len);
201 gator_buffer_write_bytes(cpu, buftype, x, len);
202 }
204 static void gator_buffer_header(int cpu, int buftype)
205 {
206 int frame;
208 if (buftype == TIMER_BUF)
209 frame = FRAME_HRTIMER;
210 else if (buftype == EVENT_BUF)
211 frame = FRAME_EVENT;
212 else
213 frame = -1;
215 gator_buffer_write_packed_int(cpu, buftype, frame);
216 gator_buffer_write_packed_int(cpu, buftype, cpu);
217 }
219 static void gator_buffer_commit(int cpu, int buftype)
220 {
221 buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
222 per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
223 gator_buffer_header(cpu, buftype);
224 wake_up(&gator_buffer_wait);
225 }
227 static void timer_buffer_check(int cpu)
228 {
229 int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
230 if (available < 0) {
231 available += gator_buffer_size[TIMER_BUF];
232 }
233 if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
234 spin_lock(&timer_commit_lock);
235 gator_buffer_commit(cpu, TIMER_BUF);
236 spin_unlock(&timer_commit_lock);
237 }
238 }
240 #if LINUX_PMU_SUPPORT
241 static void event_buffer_check(int cpu)
242 {
243 int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
244 if (available < 0) {
245 available += gator_buffer_size[EVENT_BUF];
246 }
247 if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
248 spin_lock(&event_commit_lock);
249 gator_buffer_commit(cpu, EVENT_BUF);
250 spin_unlock(&event_commit_lock);
251 }
252 }
253 #endif
255 static void gator_add_trace(int cpu, int buftype, unsigned int address)
256 {
257 off_t offset = 0;
258 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
260 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
261 offset = address;
262 }
264 gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
265 gator_buffer_write_packed_int(cpu, buftype, cookie);
266 }
268 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
269 {
270 struct module *mod;
271 unsigned int addr, cookie = 0;
272 int inKernel = regs ? !user_mode(regs) : 1;
273 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
275 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
276 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
277 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
278 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
279 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
280 gator_buffer_write_packed_int(cpu, buftype, inKernel);
282 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
283 if (regs) {
284 if (inKernel) {
285 addr = PC_REG;
286 mod = __module_address(addr);
287 if (mod) {
288 cookie = get_cookie(cpu, buftype, current, NULL, mod, true);
289 addr = addr - (unsigned long)mod->module_core;
290 }
291 gator_buffer_write_packed_int(cpu, buftype, addr & ~1);
292 gator_buffer_write_packed_int(cpu, buftype, cookie);
293 } else {
294 // Cookie+PC
295 gator_add_trace(cpu, buftype, PC_REG);
297 // Backtrace
298 if (gator_backtrace_depth)
299 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
300 }
301 }
303 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
304 }
306 /******************************************************************************
307 * hrtimer interrupt processing
308 ******************************************************************************/
309 static LIST_HEAD(gator_events);
311 static void gator_timer_interrupt(void)
312 {
313 struct pt_regs * const regs = get_irq_regs();
314 int cpu = smp_processor_id();
315 int *buffer, len, i, buftype = TIMER_BUF;
316 long long *buffer64;
317 struct gator_interface *gi;
319 // check full backtrace has enough space, otherwise may
320 // have breaks between samples in the same callstack
321 if (per_cpu(gator_first_time, cpu)) {
322 per_cpu(gator_first_time, cpu) = 0;
324 list_for_each_entry(gi, &gator_events, list)
325 if (gi->read)
326 gi->read(NULL);
328 return;
329 }
331 // Output scheduler
332 len = gator_trace_sched_read(&buffer64);
333 if (len > 0) {
334 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
335 gator_buffer_write_packed_int(cpu, buftype, len);
336 for (i = 0; i < len; i++) {
337 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
338 }
339 }
341 // Output counters
342 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
343 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
344 list_for_each_entry(gi, &gator_events, list) {
345 if (gi->read) {
346 len = gi->read(&buffer);
347 if (len > 0) {
348 gator_buffer_write_packed_int(cpu, buftype, len);
349 for (i = 0; i < len; i++) {
350 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
351 }
352 }
353 } else if (gi->read64) {
354 len = gi->read64(&buffer64);
355 if (len > 0)
356 gator_buffer_write_packed_int(cpu, buftype, len);
357 for (i = 0; i < len; i++) {
358 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
359 }
360 }
361 }
362 gator_buffer_write_packed_int(cpu, buftype, 0);
364 // Output backtrace
365 if (!event_based_sampling) {
366 gator_add_sample(cpu, buftype, regs);
367 }
369 // Check and commit; generally, commit is set to occur once per second
370 timer_buffer_check(cpu);
371 }
373 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
374 DEFINE_PER_CPU(int, hrtimer_is_active);
375 static int hrtimer_running;
376 static ktime_t profiling_interval;
378 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
379 {
380 hrtimer_forward_now(hrtimer, profiling_interval);
381 gator_timer_interrupt();
382 return HRTIMER_RESTART;
383 }
385 static int gator_timer_init(void)
386 {
387 return 0;
388 }
390 static void __gator_timer_offline(void *unused)
391 {
392 int cpu = smp_processor_id();
393 if (per_cpu(hrtimer_is_active, cpu)) {
394 struct gator_interface *gi;
395 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
396 hrtimer_cancel(hrtimer);
397 per_cpu(hrtimer_is_active, cpu) = 0;
398 gator_buffer_commit(cpu, TIMER_BUF);
399 if (event_based_sampling)
400 gator_buffer_commit(cpu, EVENT_BUF);
402 // offline any events
403 list_for_each_entry(gi, &gator_events, list)
404 if (gi->offline)
405 gi->offline();
406 }
407 }
409 static void gator_timer_offline(void)
410 {
411 if (hrtimer_running) {
412 hrtimer_running = 0;
414 on_each_cpu(__gator_timer_offline, NULL, 1);
415 }
416 }
418 static void __gator_timer_online(void *unused)
419 {
420 int cpu = smp_processor_id();
421 if (!per_cpu(hrtimer_is_active, cpu)) {
422 struct gator_interface *gi;
423 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
424 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
425 hrtimer->function = gator_hrtimer_notify;
426 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
427 per_cpu(gator_first_time, cpu) = 1;
428 per_cpu(hrtimer_is_active, cpu) = 1;
430 // online any events
431 list_for_each_entry(gi, &gator_events, list)
432 if (gi->online)
433 gi->online();
434 }
435 }
437 int gator_timer_online(unsigned long setup)
438 {
439 if (!setup) {
440 pr_err("gator: cannot start due to a system tick value of zero\n");
441 return -1;
442 } else if (hrtimer_running) {
443 pr_notice("gator: high res timer already running\n");
444 return 0;
445 }
447 hrtimer_running = 1;
449 // calculate profiling interval
450 profiling_interval = ns_to_ktime(1000000000UL / setup);
452 // timer interrupt
453 on_each_cpu(__gator_timer_online, NULL, 1);
455 return 0;
456 }
458 static uint64_t gator_get_time(void)
459 {
460 struct timespec ts;
461 uint64_t timestamp;
463 ktime_get_ts(&ts);
464 timestamp = timespec_to_ns(&ts);
466 return timestamp;
467 }
469 /******************************************************************************
470 * cpu online notifier
471 ******************************************************************************/
472 static int __cpuinit gator_cpu_notify(struct notifier_block *self,
473 unsigned long action, void *hcpu)
474 {
475 long cpu = (long)hcpu;
477 switch (action) {
478 case CPU_ONLINE:
479 case CPU_ONLINE_FROZEN:
480 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
481 break;
482 case CPU_DOWN_PREPARE:
483 case CPU_DOWN_PREPARE_FROZEN:
484 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
485 break;
486 }
488 return NOTIFY_OK;
489 }
491 static struct notifier_block __refdata gator_cpu_notifier = {
492 .notifier_call = gator_cpu_notify,
493 };
495 static int gator_notifier_start(void)
496 {
497 return register_hotcpu_notifier(&gator_cpu_notifier);
498 }
500 static void gator_notifier_stop(void)
501 {
502 unregister_hotcpu_notifier(&gator_cpu_notifier);
503 }
505 /******************************************************************************
506 * Main
507 ******************************************************************************/
508 int gator_events_install(struct gator_interface *interface)
509 {
510 list_add_tail(&interface->list, &gator_events);
512 return 0;
513 }
515 int gator_events_get_key(void)
516 {
517 static int key;
519 return key++;
520 }
522 static int gator_init(void)
523 {
524 int i;
526 if (gator_timer_init())
527 return -1;
528 if (gator_trace_sched_init())
529 return -1;
530 if (gator_annotate_init())
531 return -1;
533 // events sources (gator_events.h, generated by gator_events.sh)
534 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
535 if (gator_events_list[i])
536 gator_events_list[i]();
538 return 0;
539 }
541 static int gator_start(void)
542 {
543 struct gator_interface *gi;
545 // start all events
546 list_for_each_entry(gi, &gator_events, list) {
547 if (gi->start && gi->start() != 0) {
548 struct list_head *ptr = gi->list.prev;
550 while (ptr != &gator_events) {
551 gi = list_entry(ptr, struct gator_interface,
552 list);
554 if (gi->stop)
555 gi->stop();
557 ptr = ptr->prev;
558 }
559 goto events_failure;
560 }
561 }
563 // cookies shall be initialized before trace_sched_start() and gator_timer_online()
564 if (cookies_initialize())
565 goto cookies_failure;
566 if (gator_annotate_start())
567 goto annotate_failure;
568 if (gator_trace_sched_start())
569 goto sched_failure;
570 if (gator_event_sampling_start())
571 goto event_sampling_failure;
572 if (gator_timer_online(gator_timer_count))
573 goto timer_failure;
574 if (gator_notifier_start())
575 goto notifier_failure;
577 return 0;
579 notifier_failure:
580 gator_timer_offline();
581 timer_failure:
582 gator_event_sampling_stop();
583 event_sampling_failure:
584 gator_trace_sched_stop();
585 sched_failure:
586 gator_annotate_stop();
587 annotate_failure:
588 cookies_release();
589 cookies_failure:
590 // stop all events
591 list_for_each_entry(gi, &gator_events, list)
592 if (gi->stop)
593 gi->stop();
594 events_failure:
596 return -1;
597 }
599 static void gator_stop(void)
600 {
601 struct gator_interface *gi;
603 // stop all events
604 list_for_each_entry(gi, &gator_events, list)
605 if (gi->stop)
606 gi->stop();
608 gator_annotate_stop();
609 gator_trace_sched_stop();
610 gator_event_sampling_stop();
612 // stop all interrupt callback reads before tearing down other interfaces
613 gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
614 gator_timer_offline();
615 }
617 static void gator_exit(void)
618 {
619 gator_annotate_exit();
620 }
622 /******************************************************************************
623 * Filesystem
624 ******************************************************************************/
625 /* fopen("buffer") */
626 static int gator_op_setup(void)
627 {
628 int err = 0;
629 int cpu, i;
631 mutex_lock(&start_mutex);
633 gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
634 gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
636 // must be a power of 2
637 if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
638 err = -ENOEXEC;
639 goto setup_error;
640 }
642 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
643 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
645 gator_net_traffic = 0;
647 // Initialize per buffer variables
648 for (i = 0; i < NUM_GATOR_BUFS; i++) {
649 gator_commit_read[i] = gator_commit_write[i] = 0;
650 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
651 if (!gator_commit[i]) {
652 err = -ENOMEM;
653 goto setup_error;
654 }
656 // Initialize percpu per buffer variables
657 for_each_present_cpu(cpu) {
658 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
659 if (!per_cpu(gator_buffer, cpu)[i]) {
660 err = -ENOMEM;
661 goto setup_error;
662 }
664 per_cpu(gator_buffer_read, cpu)[i] = 0;
665 per_cpu(gator_buffer_write, cpu)[i] = 0;
666 gator_buffer_header(cpu, i);
667 }
668 }
670 setup_error:
671 mutex_unlock(&start_mutex);
672 return err;
673 }
675 /* Actually start profiling (echo 1>/dev/gator/enable) */
676 static int gator_op_start(void)
677 {
678 int err = 0;
680 mutex_lock(&start_mutex);
682 if (gator_started || gator_start())
683 err = -EINVAL;
684 else
685 gator_started = 1;
687 mutex_unlock(&start_mutex);
689 return err;
690 }
692 /* echo 0>/dev/gator/enable */
693 static void gator_op_stop(void)
694 {
695 mutex_lock(&start_mutex);
697 if (gator_started) {
698 gator_stop();
700 mutex_lock(&gator_buffer_mutex);
702 gator_started = 0;
703 cookies_release();
704 wake_up(&gator_buffer_wait);
706 mutex_unlock(&gator_buffer_mutex);
707 }
709 mutex_unlock(&start_mutex);
710 }
712 static void gator_shutdown(void)
713 {
714 int cpu, i;
716 mutex_lock(&start_mutex);
718 gator_annotate_shutdown();
720 for (i = 0; i < NUM_GATOR_BUFS; i++) {
721 vfree(gator_commit[i]);
722 gator_commit[i] = NULL;
723 }
725 for_each_present_cpu(cpu) {
726 mutex_lock(&gator_buffer_mutex);
727 for (i = 0; i < NUM_GATOR_BUFS; i++) {
728 vfree(per_cpu(gator_buffer, cpu)[i]);
729 per_cpu(gator_buffer, cpu)[i] = NULL;
730 per_cpu(gator_buffer_read, cpu)[i] = 0;
731 per_cpu(gator_buffer_write, cpu)[i] = 0;
732 }
733 mutex_unlock(&gator_buffer_mutex);
734 }
736 mutex_unlock(&start_mutex);
737 }
739 static int gator_set_backtrace(unsigned long val)
740 {
741 int err = 0;
743 mutex_lock(&start_mutex);
745 if (gator_started)
746 err = -EBUSY;
747 else
748 gator_backtrace_depth = val;
750 mutex_unlock(&start_mutex);
752 return err;
753 }
755 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
756 {
757 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
758 }
760 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
761 {
762 unsigned long val;
763 int retval;
765 if (*offset)
766 return -EINVAL;
768 retval = gatorfs_ulong_from_user(&val, buf, count);
769 if (retval)
770 return retval;
772 if (val)
773 retval = gator_op_start();
774 else
775 gator_op_stop();
777 if (retval)
778 return retval;
779 return count;
780 }
782 static const struct file_operations enable_fops = {
783 .read = enable_read,
784 .write = enable_write,
785 };
787 static int userspace_buffer_open(struct inode *inode, struct file *file)
788 {
789 int err = -EPERM;
791 if (!capable(CAP_SYS_ADMIN))
792 return -EPERM;
794 if (test_and_set_bit_lock(0, &gator_buffer_opened))
795 return -EBUSY;
797 if ((err = gator_op_setup()))
798 goto fail;
800 /* NB: the actual start happens from userspace
801 * echo 1 >/dev/gator/enable
802 */
804 return 0;
806 fail:
807 __clear_bit_unlock(0, &gator_buffer_opened);
808 return err;
809 }
811 static int userspace_buffer_release(struct inode *inode, struct file *file)
812 {
813 gator_op_stop();
814 gator_shutdown();
815 __clear_bit_unlock(0, &gator_buffer_opened);
816 return 0;
817 }
819 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
820 size_t count, loff_t *offset)
821 {
822 int retval = -EINVAL;
823 int commit, length1, length2, read;
824 char *buffer1;
825 char *buffer2 = NULL;
826 int cpu, i;
828 /* do not handle partial reads */
829 if (count != userspace_buffer_size || *offset)
830 return -EINVAL;
832 // sleep until the condition is true or a signal is received
833 // the condition is checked each time gator_buffer_wait is woken up
834 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
836 if (signal_pending(current))
837 return -EINTR;
839 length2 = 0;
840 retval = -EFAULT;
842 mutex_lock(&gator_buffer_mutex);
844 i = -1;
845 if (buffer_commit_ready(TIMER_BUF)) {
846 i = TIMER_BUF;
847 } else if (buffer_commit_ready(EVENT_BUF)) {
848 i = EVENT_BUF;
849 }
851 if (i != -1) {
852 buffer_commit_read(&cpu, i, &read, &commit);
854 /* May happen if the buffer is freed during pending reads. */
855 if (!per_cpu(gator_buffer, cpu)[i]) {
856 retval = -EFAULT;
857 goto out;
858 }
860 /* determine the size of two halves */
861 length1 = commit - read;
862 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
863 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
864 if (length1 < 0) {
865 length1 = gator_buffer_size[i] - read;
866 length2 = commit;
867 }
868 } else if (gator_annotate_ready()) {
869 length1 = gator_annotate_read(&buffer1);
870 if (!length1)
871 goto out;
872 } else {
873 retval = 0;
874 goto out;
875 }
877 /* start, middle or end */
878 if (length1 > 0) {
879 if (copy_to_user(&buf[0], buffer1, length1)) {
880 goto out;
881 }
882 }
884 /* possible wrap around */
885 if (length2 > 0) {
886 if (copy_to_user(&buf[length1], buffer2, length2)) {
887 goto out;
888 }
889 }
891 retval = length1 + length2;
893 /* kick just in case we've lost an SMP event */
894 wake_up(&gator_buffer_wait);
896 out:
897 // only adjust network stats if in streaming mode
898 if (gator_streaming)
899 gator_net_traffic += retval;
900 mutex_unlock(&gator_buffer_mutex);
901 return retval;
902 }
904 const struct file_operations gator_event_buffer_fops = {
905 .open = userspace_buffer_open,
906 .release = userspace_buffer_release,
907 .read = userspace_buffer_read,
908 };
910 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
911 {
912 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
913 offset);
914 }
916 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
917 {
918 unsigned long val;
919 int retval;
921 if (*offset)
922 return -EINVAL;
924 retval = gatorfs_ulong_from_user(&val, buf, count);
925 if (retval)
926 return retval;
928 retval = gator_set_backtrace(val);
930 if (retval)
931 return retval;
932 return count;
933 }
935 static const struct file_operations depth_fops = {
936 .read = depth_read,
937 .write = depth_write
938 };
940 void gator_op_create_files(struct super_block *sb, struct dentry *root)
941 {
942 struct dentry *dir;
943 struct gator_interface *gi;
944 int cpu;
946 /* reinitialize default values */
947 gator_cpu_cores = 0;
948 for_each_present_cpu(cpu) {
949 gator_cpu_cores++;
950 }
951 userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
952 gator_streaming = 1;
954 gatorfs_create_file(sb, root, "enable", &enable_fops);
955 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
956 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
957 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
958 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
959 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
960 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
961 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
963 // Annotate interface
964 gator_annotate_create_files(sb, root);
966 // Linux Events
967 dir = gatorfs_mkdir(sb, root, "events");
968 list_for_each_entry(gi, &gator_events, list)
969 if (gi->create_files)
970 gi->create_files(sb, dir);
971 }
973 /******************************************************************************
974 * Module
975 ******************************************************************************/
976 static int __init gator_module_init(void)
977 {
978 if (gatorfs_register()) {
979 return -1;
980 }
982 if (gator_init()) {
983 gatorfs_unregister();
984 return -1;
985 }
987 return 0;
988 }
990 static void __exit gator_module_exit(void)
991 {
992 tracepoint_synchronize_unregister();
993 gatorfs_unregister();
994 gator_exit();
995 }
997 module_init(gator_module_init);
998 module_exit(gator_module_exit);
1000 MODULE_LICENSE("GPL");
1001 MODULE_AUTHOR("ARM Ltd");
1002 MODULE_DESCRIPTION("Gator system profiler");