1 /**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 5;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
22 #include "gator.h"
23 #include "gator_events.h"
25 #ifndef CONFIG_GENERIC_TRACER
26 #ifndef CONFIG_TRACING
27 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
28 #endif
29 #endif
31 #ifndef CONFIG_PROFILING
32 #error gator requires the kernel to have CONFIG_PROFILING defined
33 #endif
35 #ifndef CONFIG_HIGH_RES_TIMERS
36 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
37 #endif
39 #ifdef CONFIG_SMP
40 #ifndef CONFIG_LOCAL_TIMERS
41 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
42 #endif
43 #endif
45 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
46 #error kernels prior to 2.6.32 are not supported
47 #endif
49 /******************************************************************************
50 * DEFINES
51 ******************************************************************************/
52 #define BUFFER_SIZE_DEFAULT (256*1024)
53 #define SYNC_FREQ_DEFAULT 1000
55 #define NO_COOKIE 0UL
56 #define INVALID_COOKIE ~0UL
58 #define PROTOCOL_FRAME ~0
59 #define PROTOCOL_START_TICK 1
60 #define PROTOCOL_END_TICK 3
61 #define PROTOCOL_START_BACKTRACE 5
62 #define PROTOCOL_END_BACKTRACE 7
63 #define PROTOCOL_COOKIE 9
64 #define PROTOCOL_SCHEDULER_TRACE 11
65 #define PROTOCOL_COUNTERS 13
66 #define PROTOCOL_ANNOTATE 15
67 #define PROTOCOL_CPU_SYNC 17
69 #if defined(__arm__)
70 #define PC_REG regs->ARM_pc
71 #else
72 #define PC_REG regs->ip
73 #endif
75 /******************************************************************************
76 * PER CPU
77 ******************************************************************************/
78 static unsigned long gator_cpu_cores;
79 static unsigned long gator_buffer_size;
80 static unsigned long gator_backtrace_depth;
82 static unsigned long gator_started;
83 static unsigned long gator_buffer_opened;
84 static unsigned long gator_timer_count;
85 static unsigned long gator_streaming;
86 static int gator_master_tick;
87 static DEFINE_MUTEX(start_mutex);
88 static DEFINE_MUTEX(gator_buffer_mutex);
90 unsigned long gator_net_traffic;
92 #define COMMIT_SIZE 128
93 #define COMMIT_MASK (COMMIT_SIZE-1)
94 static DEFINE_SPINLOCK(gator_commit_lock);
95 static int *gator_commit;
96 static int gator_commit_read;
97 static int gator_commit_write;
99 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
100 static DEFINE_PER_CPU(int, gator_cpu_sync);
101 static DEFINE_PER_CPU(int, gator_cpu_tick);
102 static DEFINE_PER_CPU(int, gator_first_time);
104 /******************************************************************************
105 * Prototypes
106 ******************************************************************************/
107 static void gator_buffer_write_packed_int(int cpu, unsigned int x);
108 static void gator_buffer_write_string(int cpu, char *x);
109 static int gator_write_packed_int(char *buffer, unsigned int x);
110 static void gator_add_trace(int cpu, unsigned int address);
111 static uint64_t gator_get_time(void);
113 /******************************************************************************
114 * Application Includes
115 ******************************************************************************/
116 #include "gator_cookies.c"
117 #include "gator_trace_sched.c"
118 #include "gator_backtrace.c"
119 #include "gator_annotate.c"
120 #include "gator_fs.c"
122 /******************************************************************************
123 * Misc
124 ******************************************************************************/
125 #if defined(__arm__)
126 u32 gator_cpuid(void)
127 {
128 u32 val;
129 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
130 return (val >> 4) & 0xfff;
131 }
132 #endif
134 /******************************************************************************
135 * Commit interface
136 ******************************************************************************/
137 static int buffer_commit_ready(void)
138 {
139 return (gator_commit_read != gator_commit_write);
140 }
142 static void buffer_commit_read(int *cpu, int *readval, int *writeval)
143 {
144 int read = gator_commit_read;
145 *cpu = gator_commit[read+0];
146 *readval = gator_commit[read+1];
147 *writeval = gator_commit[read+2];
148 gator_commit_read = (read + 4) & COMMIT_MASK;
149 }
151 static void buffer_commit_write(int cpu, int readval, int writeval) {
152 int write = gator_commit_write;
153 gator_commit[write+0] = cpu;
154 gator_commit[write+1] = readval;
155 gator_commit[write+2] = writeval;
156 gator_commit_write = (write + 4) & COMMIT_MASK;
157 }
159 /******************************************************************************
160 * Buffer management
161 ******************************************************************************/
162 static uint32_t use_buffer_size;
163 static uint32_t use_buffer_mask;
164 static DEFINE_PER_CPU(int, use_buffer_seq);
165 static DEFINE_PER_CPU(int, use_buffer_read);
166 static DEFINE_PER_CPU(int, use_buffer_write);
167 static DEFINE_PER_CPU(char *, use_buffer);
169 static void gator_buffer_write_packed_int(int cpu, unsigned int x)
170 {
171 uint32_t write = per_cpu(use_buffer_write, cpu);
172 uint32_t mask = use_buffer_mask;
173 char *buffer = per_cpu(use_buffer, cpu);
174 int write0 = (write + 0) & mask;
175 int write1 = (write + 1) & mask;
176 int write2 = (write + 2) & mask;
177 int write3 = (write + 3) & mask;
178 int write4 = (write + 4) & mask;
179 int write5 = (write + 5) & mask;
181 if ((x & 0xffffff80) == 0) {
182 buffer[write0] = x & 0x7f;
183 per_cpu(use_buffer_write, cpu) = write1;
184 } else if ((x & 0xffffc000) == 0) {
185 buffer[write0] = x | 0x80;
186 buffer[write1] = (x>>7) & 0x7f;
187 per_cpu(use_buffer_write, cpu) = write2;
188 } else if ((x & 0xffe00000) == 0) {
189 buffer[write0] = x | 0x80;
190 buffer[write1] = (x>>7) | 0x80;
191 buffer[write2] = (x>>14) & 0x7f;
192 per_cpu(use_buffer_write, cpu) = write3;
193 } else if ((x & 0xf0000000) == 0) {
194 buffer[write0] = x | 0x80;
195 buffer[write1] = (x>>7) | 0x80;
196 buffer[write2] = (x>>14) | 0x80;
197 buffer[write3] = (x>>21) & 0x7f;
198 per_cpu(use_buffer_write, cpu) = write4;
199 } else {
200 buffer[write0] = x | 0x80;
201 buffer[write1] = (x>>7) | 0x80;
202 buffer[write2] = (x>>14) | 0x80;
203 buffer[write3] = (x>>21) | 0x80;
204 buffer[write4] = (x>>28) & 0x0f;
205 per_cpu(use_buffer_write, cpu) = write5;
206 }
207 }
209 static int gator_write_packed_int(char *buffer, unsigned int x)
210 {
211 if ((x & 0xffffff80) == 0) {
212 buffer[0] = x & 0x7f;
213 return 1;
214 } else if ((x & 0xffffc000) == 0) {
215 buffer[0] = x | 0x80;
216 buffer[1] = (x>>7) & 0x7f;
217 return 2;
218 } else if ((x & 0xffe00000) == 0) {
219 buffer[0] = x | 0x80;
220 buffer[1] = (x>>7) | 0x80;
221 buffer[2] = (x>>14) & 0x7f;
222 return 3;
223 } else if ((x & 0xf0000000) == 0) {
224 buffer[0] = x | 0x80;
225 buffer[1] = (x>>7) | 0x80;
226 buffer[2] = (x>>14) | 0x80;
227 buffer[3] = (x>>21) & 0x7f;
228 return 4;
229 } else {
230 buffer[0] = x | 0x80;
231 buffer[1] = (x>>7) | 0x80;
232 buffer[2] = (x>>14) | 0x80;
233 buffer[3] = (x>>21) | 0x80;
234 buffer[4] = (x>>28) & 0x0f;
235 return 5;
236 }
237 }
239 static void gator_buffer_write_bytes(int cpu, char *x, int len)
240 {
241 uint32_t write = per_cpu(use_buffer_write, cpu);
242 uint32_t mask = use_buffer_mask;
243 char *buffer = per_cpu(use_buffer, cpu);
244 int i;
246 for (i = 0; i < len; i++) {
247 buffer[write] = x[i];
248 write = (write + 1) & mask;
249 }
251 per_cpu(use_buffer_write, cpu) = write;
252 }
254 static void gator_buffer_write_string(int cpu, char *x)
255 {
256 int len = strlen(x);
257 gator_buffer_write_packed_int(cpu, len);
258 gator_buffer_write_bytes(cpu, x, len);
259 }
261 static void gator_buffer_header(int cpu)
262 {
263 gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME);
264 gator_buffer_write_packed_int(cpu, cpu);
265 gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu));
266 per_cpu(use_buffer_seq, cpu)++;
267 }
269 static void gator_buffer_commit(int cpu)
270 {
271 buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu));
272 per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu);
273 gator_buffer_header(cpu);
274 wake_up(&gator_buffer_wait);
275 }
277 static void gator_buffer_check(int cpu, int tick)
278 {
279 if (!(tick % gator_timer_count)) {
280 int c, sync;
281 spin_lock(&gator_commit_lock);
282 // synchronize, if all online cpus have the same tick waypoint
283 sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu);
284 for_each_online_cpu(c) {
285 if (sync != per_cpu(gator_cpu_sync, c)) {
286 sync = 0;
287 break;
288 }
289 }
290 if (sync) {
291 gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC);
292 }
293 gator_buffer_commit(cpu);
294 spin_unlock(&gator_commit_lock);
295 } else {
296 int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu);
297 if (available < 0) {
298 available += use_buffer_size;
299 }
300 if (available >= ((use_buffer_size * 3) / 4)) {
301 spin_lock(&gator_commit_lock);
302 gator_buffer_commit(cpu);
303 spin_unlock(&gator_commit_lock);
304 }
305 }
306 }
308 static void gator_add_trace(int cpu, unsigned int address)
309 {
310 off_t offset = 0;
311 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
313 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
314 offset = address;
315 }
317 gator_buffer_write_packed_int(cpu, offset & ~1);
318 gator_buffer_write_packed_int(cpu, cookie);
319 }
321 static void gator_add_sample(int cpu, struct pt_regs * const regs)
322 {
323 struct module *mod;
324 unsigned int addr, cookie = 0;
325 int inKernel = regs ? !user_mode(regs) : 1;
326 unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
328 gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE);
330 // TGID::PID::inKernel
331 gator_buffer_write_packed_int(cpu, exec_cookie);
332 gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid);
333 gator_buffer_write_packed_int(cpu, (unsigned int)current->pid);
334 gator_buffer_write_packed_int(cpu, inKernel);
336 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
337 if (regs) {
338 if (inKernel) {
339 addr = PC_REG;
340 mod = __module_address(addr);
341 if (mod) {
342 cookie = get_cookie(cpu, current, NULL, mod);
343 addr = addr - (unsigned long)mod->module_core;
344 }
345 gator_buffer_write_packed_int(cpu, addr & ~1);
346 gator_buffer_write_packed_int(cpu, cookie);
347 } else {
348 // Cookie+PC
349 gator_add_trace(cpu, PC_REG);
351 // Backtrace
352 if (gator_backtrace_depth)
353 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
354 }
355 }
357 gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE);
358 }
360 static void gator_write_packet(int cpu, int type, int len, int *buffer)
361 {
362 int i;
363 gator_buffer_write_packed_int(cpu, type);
364 gator_buffer_write_packed_int(cpu, len);
365 for (i = 0; i < len; i++) {
366 gator_buffer_write_packed_int(cpu, buffer[i]);
367 }
368 }
370 /******************************************************************************
371 * Interrupt Processing
372 ******************************************************************************/
373 static LIST_HEAD(gator_events);
375 static void gator_timer_interrupt(void)
376 {
377 struct pt_regs * const regs = get_irq_regs();
378 int cpu = smp_processor_id();
379 int *buffer, len, tick;
380 struct gator_interface *gi;
382 // check full backtrace has enough space, otherwise may
383 // have breaks between samples in the same callstack
384 if (per_cpu(gator_first_time, cpu)) {
385 per_cpu(gator_first_time, cpu) = 0;
387 list_for_each_entry(gi, &gator_events, list)
388 if (gi->read)
389 gi->read(NULL);
391 return;
392 }
394 // Header
395 gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape
397 // Output scheduler
398 len = gator_trace_sched_read(&buffer);
399 if (len > 0) {
400 gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer);
401 }
403 // Output counters
404 list_for_each_entry(gi, &gator_events, list) {
405 if (gi->read) {
406 len = gi->read(&buffer);
407 if (len > 0)
408 gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer);
409 }
410 }
412 // Output backtrace
413 gator_add_sample(cpu, regs);
415 // Timer Tick
416 tick = per_cpu(gator_cpu_tick, cpu);
417 if (tick == gator_master_tick) {
418 tick++;
419 per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick;
420 } else {
421 per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick;
422 }
423 gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick);
425 // Check and commit; generally, commit is set to occur once per second
426 gator_buffer_check(cpu, tick);
427 }
429 /******************************************************************************
430 * hrtimer
431 ******************************************************************************/
432 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
433 DEFINE_PER_CPU(int, hrtimer_is_active);
434 static int hrtimer_running;
435 static ktime_t profiling_interval;
437 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
438 {
439 hrtimer_forward_now(hrtimer, profiling_interval);
440 gator_timer_interrupt();
441 return HRTIMER_RESTART;
442 }
444 static int gator_timer_init(void)
445 {
446 return 0;
447 }
449 static void __gator_timer_offline(void *unused)
450 {
451 int cpu = smp_processor_id();
452 if (per_cpu(hrtimer_is_active, cpu)) {
453 struct gator_interface *gi;
454 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
455 hrtimer_cancel(hrtimer);
456 per_cpu(hrtimer_is_active, cpu) = 0;
457 gator_buffer_commit(cpu);
459 // offline any events
460 list_for_each_entry(gi, &gator_events, list)
461 if (gi->offline)
462 gi->offline();
463 }
464 }
466 static void gator_timer_offline(void)
467 {
468 if (hrtimer_running) {
469 hrtimer_running = 0;
471 on_each_cpu(__gator_timer_offline, NULL, 1);
473 // output a final sync point
474 gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC);
475 gator_buffer_commit(0);
476 }
477 }
479 static void __gator_timer_online(void *unused)
480 {
481 int cpu = smp_processor_id();
482 if (!per_cpu(hrtimer_is_active, cpu)) {
483 struct gator_interface *gi;
484 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
485 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
486 hrtimer->function = gator_hrtimer_notify;
487 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
488 per_cpu(gator_cpu_tick, cpu) = 0;
489 per_cpu(gator_first_time, cpu) = 1;
490 per_cpu(hrtimer_is_active, cpu) = 1;
492 // online any events
493 list_for_each_entry(gi, &gator_events, list)
494 if (gi->online)
495 gi->online();
496 }
497 }
499 int gator_timer_online(unsigned long setup)
500 {
501 if (!setup) {
502 pr_err("gator: cannot start due to a system tick value of zero");
503 return -1;
504 } else if (hrtimer_running) {
505 pr_notice("gator: high res timer already running");
506 return 0;
507 }
509 hrtimer_running = 1;
511 // calculate profiling interval
512 profiling_interval = ns_to_ktime(1000000000UL / setup);
514 // timer interrupt
515 gator_master_tick = 0;
516 on_each_cpu(__gator_timer_online, NULL, 1);
518 return 0;
519 }
521 static uint64_t gator_get_time(void)
522 {
523 struct timespec ts;
524 uint64_t timestamp;
526 ktime_get_ts(&ts);
527 timestamp = timespec_to_ns(&ts);
529 return timestamp;
530 }
532 /******************************************************************************
533 * cpu online notifier
534 ******************************************************************************/
535 static int __cpuinit gator_cpu_notify(struct notifier_block *self,
536 unsigned long action, void *hcpu)
537 {
538 long cpu = (long)hcpu;
540 switch (action) {
541 case CPU_ONLINE:
542 case CPU_ONLINE_FROZEN:
543 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
544 break;
545 case CPU_DOWN_PREPARE:
546 case CPU_DOWN_PREPARE_FROZEN:
547 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
548 break;
549 }
551 return NOTIFY_OK;
552 }
554 static struct notifier_block __refdata gator_cpu_notifier = {
555 .notifier_call = gator_cpu_notify,
556 };
558 static int gator_notifier_start(void)
559 {
560 return register_hotcpu_notifier(&gator_cpu_notifier);
561 }
563 static void gator_notifier_stop(void)
564 {
565 unregister_hotcpu_notifier(&gator_cpu_notifier);
566 }
568 /******************************************************************************
569 * Main
570 ******************************************************************************/
571 int gator_events_install(struct gator_interface *interface)
572 {
573 list_add_tail(&interface->list, &gator_events);
575 return 0;
576 }
578 int gator_events_get_key(void)
579 {
580 static int key;
582 return key++;
583 }
585 static int gator_init(void)
586 {
587 int i;
589 if (gator_timer_init())
590 return -1;
591 if (gator_trace_sched_init())
592 return -1;
593 if (gator_annotate_init())
594 return -1;
596 // events sources (gator_events.h, generated by gator_events.sh)
597 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
598 if (gator_events_list[i])
599 gator_events_list[i]();
601 return 0;
602 }
604 static int gator_start(void)
605 {
606 struct gator_interface *gi;
608 // start all events
609 list_for_each_entry(gi, &gator_events, list) {
610 if (gi->start && gi->start() != 0) {
611 struct list_head *ptr = gi->list.prev;
613 while (ptr != &gator_events) {
614 gi = list_entry(ptr, struct gator_interface,
615 list);
617 if (gi->stop)
618 gi->stop();
620 ptr = ptr->prev;
621 }
622 goto events_failure;
623 }
624 }
626 if (gator_annotate_start())
627 goto annotate_failure;
628 if (gator_trace_sched_start())
629 goto sched_failure;
630 if (gator_timer_online(gator_timer_count))
631 goto timer_failure;
632 if (gator_notifier_start())
633 goto notifier_failure;
635 return 0;
637 notifier_failure:
638 gator_timer_offline();
639 timer_failure:
640 gator_trace_sched_stop();
641 sched_failure:
642 gator_annotate_stop();
643 annotate_failure:
644 events_failure:
646 return -1;
647 }
649 static void gator_stop(void)
650 {
651 struct gator_interface *gi;
653 // stop all events
654 list_for_each_entry(gi, &gator_events, list)
655 if (gi->stop)
656 gi->stop();
658 gator_annotate_stop();
659 gator_trace_sched_stop();
661 // stop all interrupt callback reads before tearing down other interfaces
662 gator_timer_offline();
663 gator_notifier_stop();
664 }
666 static void gator_exit(void)
667 {
668 gator_annotate_exit();
669 }
671 /******************************************************************************
672 * Filesystem
673 ******************************************************************************/
674 /* fopen("buffer") */
675 static int gator_op_setup(void)
676 {
677 int err = 0;
678 int cpu;
680 mutex_lock(&start_mutex);
682 use_buffer_size = gator_buffer_size;
683 use_buffer_mask = use_buffer_size - 1;
685 // must be a power of 2
686 if (use_buffer_size & (use_buffer_size - 1)) {
687 err = -ENOEXEC;
688 goto setup_error;
689 }
691 gator_net_traffic = 0;
693 gator_commit_read = gator_commit_write = 0;
694 gator_commit = vmalloc(COMMIT_SIZE * sizeof(int));
695 if (!gator_commit) {
696 err = -ENOMEM;
697 goto setup_error;
698 }
700 for_each_present_cpu(cpu) {
701 per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size);
702 if (!per_cpu(use_buffer, cpu)) {
703 err = -ENOMEM;
704 goto setup_error;
705 }
707 per_cpu(gator_cpu_sync, cpu) = 0;
708 per_cpu(gator_cpu_tick, cpu) = 0;
710 per_cpu(use_buffer_seq, cpu) = 0;
711 per_cpu(use_buffer_read, cpu) = 0;
712 per_cpu(use_buffer_write, cpu) = 0;
713 gator_buffer_header(cpu);
714 }
716 setup_error:
717 mutex_unlock(&start_mutex);
718 return err;
719 }
721 /* Actually start profiling (echo 1>/dev/gator/enable) */
722 static int gator_op_start(void)
723 {
724 int err = 0;
726 mutex_lock(&start_mutex);
728 if (gator_started || gator_start() || cookies_initialize())
729 err = -EINVAL;
730 else
731 gator_started = 1;
733 mutex_unlock(&start_mutex);
735 return err;
736 }
738 /* echo 0>/dev/gator/enable */
739 static void gator_op_stop(void)
740 {
741 mutex_lock(&start_mutex);
743 if (gator_started) {
744 gator_stop();
746 mutex_lock(&gator_buffer_mutex);
748 gator_started = 0;
749 cookies_release();
750 wake_up(&gator_buffer_wait);
752 mutex_unlock(&gator_buffer_mutex);
753 }
755 mutex_unlock(&start_mutex);
756 }
758 static void gator_shutdown(void)
759 {
760 int cpu;
762 mutex_lock(&start_mutex);
764 vfree(gator_commit);
765 gator_commit = NULL;
767 for_each_present_cpu(cpu) {
768 mutex_lock(&gator_buffer_mutex);
769 vfree(per_cpu(use_buffer, cpu));
770 per_cpu(use_buffer, cpu) = NULL;
771 per_cpu(use_buffer_seq, cpu) = 0;
772 per_cpu(use_buffer_read, cpu) = 0;
773 per_cpu(use_buffer_write, cpu) = 0;
774 mutex_unlock(&gator_buffer_mutex);
775 }
777 mutex_unlock(&start_mutex);
778 }
780 static int gator_set_backtrace(unsigned long val)
781 {
782 int err = 0;
784 mutex_lock(&start_mutex);
786 if (gator_started)
787 err = -EBUSY;
788 else
789 gator_backtrace_depth = val;
791 mutex_unlock(&start_mutex);
793 return err;
794 }
796 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
797 {
798 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
799 }
801 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
802 {
803 unsigned long val;
804 int retval;
806 if (*offset)
807 return -EINVAL;
809 retval = gatorfs_ulong_from_user(&val, buf, count);
810 if (retval)
811 return retval;
813 if (val)
814 retval = gator_op_start();
815 else
816 gator_op_stop();
818 if (retval)
819 return retval;
820 return count;
821 }
823 static const struct file_operations enable_fops = {
824 .read = enable_read,
825 .write = enable_write,
826 };
828 static int event_buffer_open(struct inode *inode, struct file *file)
829 {
830 int err = -EPERM;
832 if (!capable(CAP_SYS_ADMIN))
833 return -EPERM;
835 if (test_and_set_bit_lock(0, &gator_buffer_opened))
836 return -EBUSY;
838 if ((err = gator_op_setup()))
839 goto fail;
841 /* NB: the actual start happens from userspace
842 * echo 1 >/dev/gator/enable
843 */
845 return 0;
847 fail:
848 __clear_bit_unlock(0, &gator_buffer_opened);
849 return err;
850 }
852 static int event_buffer_release(struct inode *inode, struct file *file)
853 {
854 gator_op_stop();
855 gator_shutdown();
856 __clear_bit_unlock(0, &gator_buffer_opened);
857 return 0;
858 }
860 static ssize_t event_buffer_read(struct file *file, char __user *buf,
861 size_t count, loff_t *offset)
862 {
863 int retval = -EINVAL;
864 int commit, length1, length2, read;
865 char *buffer1, *buffer2;
866 char annotate_header[6];
867 int cpu;
869 /* do not handle partial reads */
870 if (count != use_buffer_size || *offset)
871 return -EINVAL;
873 // sleep until the condition is true or a signal is received
874 // the condition is checked each time gator_buffer_wait is woken up
875 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started);
877 if (signal_pending(current))
878 return -EINTR;
880 retval = -EFAULT;
882 mutex_lock(&gator_buffer_mutex);
884 if (buffer_commit_ready()) {
885 buffer_commit_read(&cpu, &read, &commit);
887 /* May happen if the buffer is freed during pending reads. */
888 if (!per_cpu(use_buffer, cpu)) {
889 retval = -EFAULT;
890 goto out;
891 }
893 /* determine the size of two halves */
894 length1 = commit - read;
895 length2 = 0;
896 buffer1 = &(per_cpu(use_buffer, cpu)[read]);
897 buffer2 = &(per_cpu(use_buffer, cpu)[0]);
898 if (length1 < 0) {
899 length1 = use_buffer_size - read;
900 length2 = commit;
901 }
902 } else if (gator_annotate_ready()) {
903 length2 = gator_annotate_read(&buffer2);
904 if (!length2)
905 goto out;
906 annotate_header[0] = PROTOCOL_ANNOTATE;
907 length1 = gator_write_packed_int(&annotate_header[1], length2) + 1;
908 buffer1 = annotate_header;
909 } else {
910 retval = 0;
911 goto out;
912 }
914 /* start, middle or end */
915 if (length1 > 0) {
916 if (copy_to_user(&buf[0], buffer1, length1)) {
917 goto out;
918 }
919 }
921 /* possible wrap around */
922 if (length2 > 0) {
923 if (copy_to_user(&buf[length1], buffer2, length2)) {
924 goto out;
925 }
926 }
928 retval = length1 + length2;
930 /* kick just in case we've lost an SMP event */
931 wake_up(&gator_buffer_wait);
933 out:
934 // only adjust network stats if in streaming mode
935 if (gator_streaming)
936 gator_net_traffic += retval;
937 mutex_unlock(&gator_buffer_mutex);
938 return retval;
939 }
941 const struct file_operations gator_event_buffer_fops = {
942 .open = event_buffer_open,
943 .release = event_buffer_release,
944 .read = event_buffer_read,
945 };
947 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
948 {
949 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
950 offset);
951 }
953 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
954 {
955 unsigned long val;
956 int retval;
958 if (*offset)
959 return -EINVAL;
961 retval = gatorfs_ulong_from_user(&val, buf, count);
962 if (retval)
963 return retval;
965 retval = gator_set_backtrace(val);
967 if (retval)
968 return retval;
969 return count;
970 }
972 static const struct file_operations depth_fops = {
973 .read = depth_read,
974 .write = depth_write
975 };
977 static const char gator_cpu_type[] = "gator";
979 static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
980 {
981 return gatorfs_str_to_user(gator_cpu_type, buf, count, offset);
982 }
984 static const struct file_operations cpu_type_fops = {
985 .read = cpu_type_read,
986 };
988 void gator_op_create_files(struct super_block *sb, struct dentry *root)
989 {
990 struct dentry *dir;
991 struct gator_interface *gi;
992 int cpu;
994 /* reinitialize default values */
995 gator_cpu_cores = 0;
996 for_each_present_cpu(cpu) {
997 gator_cpu_cores++;
998 }
999 gator_buffer_size = BUFFER_SIZE_DEFAULT;
1000 gator_streaming = 1;
1002 gatorfs_create_file(sb, root, "enable", &enable_fops);
1003 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1004 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1005 gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops);
1006 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1007 gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size);
1008 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1009 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
1010 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1012 // Annotate interface
1013 gator_annotate_create_files(sb, root);
1015 // Linux Events
1016 dir = gatorfs_mkdir(sb, root, "events");
1017 list_for_each_entry(gi, &gator_events, list)
1018 if (gi->create_files)
1019 gi->create_files(sb, dir);
1020 }
1022 /******************************************************************************
1023 * Module
1024 ******************************************************************************/
1025 static int __init gator_module_init(void)
1026 {
1027 if (gatorfs_register()) {
1028 return -1;
1029 }
1031 if (gator_init()) {
1032 gatorfs_unregister();
1033 return -1;
1034 }
1036 #ifdef GATOR_DEBUG
1037 pr_err("gator_module_init");
1038 #endif
1039 return 0;
1040 }
1042 static void __exit gator_module_exit(void)
1043 {
1044 #ifdef GATOR_DEBUG
1045 pr_err("gator_module_exit");
1046 #endif
1047 tracepoint_synchronize_unregister();
1048 gatorfs_unregister();
1049 gator_exit();
1050 }
1052 module_init(gator_module_init);
1053 module_exit(gator_module_exit);
1055 MODULE_LICENSE("GPL");
1056 MODULE_AUTHOR("ARM Ltd");
1057 MODULE_DESCRIPTION("Gator system profiler");