1 /**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 static unsigned long gator_protocol_version = 4;
12 #include "gator.h"
13 #include <linux/slab.h>
14 #include <linux/cpu.h>
15 #include <linux/sched.h>
16 #include <linux/irq.h>
17 #include <linux/vmalloc.h>
18 #include <linux/hardirq.h>
19 #include <linux/highmem.h>
20 #include <linux/pagemap.h>
21 #include <asm/uaccess.h>
23 #ifndef CONFIG_GENERIC_TRACER
24 #ifndef CONFIG_TRACING
25 #warning gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
26 #endif
27 #endif
29 #ifndef CONFIG_PROFILING
30 #warning gator requires the kernel to have CONFIG_PROFILING defined
31 #endif
33 #ifndef CONFIG_HIGH_RES_TIMERS
34 #warning gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
35 #endif
37 #ifdef CONFIG_SMP
38 #ifndef CONFIG_LOCAL_TIMERS
39 #warning gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
40 #endif
41 #endif
43 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
44 #error kernels prior to 2.6.32 are not supported
45 #endif
47 /******************************************************************************
48 * DEFINES
49 ******************************************************************************/
50 #define BUFFER_SIZE_DEFAULT (256*1024)
51 #define SYNC_FREQ_DEFAULT 1000
53 #define NO_COOKIE 0UL
54 #define INVALID_COOKIE ~0UL
56 #define PROTOCOL_FRAME ~0
57 #define PROTOCOL_START_TICK 1
58 #define PROTOCOL_END_TICK 3
59 #define PROTOCOL_START_BACKTRACE 5
60 #define PROTOCOL_END_BACKTRACE 7
61 #define PROTOCOL_COOKIE 9
62 #define PROTOCOL_SCHEDULER_TRACE 11
63 #define PROTOCOL_COUNTERS 13
64 #define PROTOCOL_ANNOTATE 15
65 #define PROTOCOL_CPU_SYNC 17
67 #if defined(__arm__)
68 #define PC_REG regs->ARM_pc
69 #else
70 #define PC_REG regs->ip
71 #endif
73 /******************************************************************************
74 * PER CPU
75 ******************************************************************************/
76 static unsigned long gator_cpu_cores;
77 static unsigned long gator_buffer_size;
78 static unsigned long gator_backtrace_depth;
80 static unsigned long gator_started;
81 static unsigned long gator_buffer_opened;
82 static unsigned long gator_timer_count;
83 static unsigned long gator_sync_freq;
84 static int gator_master_tick;
85 static DEFINE_MUTEX(start_mutex);
86 static DEFINE_MUTEX(gator_buffer_mutex);
88 unsigned long gator_net_traffic;
90 #define COMMIT_SIZE 128
91 #define COMMIT_MASK (COMMIT_SIZE-1)
92 static DEFINE_SPINLOCK(gator_commit_lock);
93 static int *gator_commit;
94 static int gator_commit_read;
95 static int gator_commit_write;
97 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
98 static DEFINE_PER_CPU(int, gator_cpu_sync);
99 static DEFINE_PER_CPU(int, gator_cpu_tick);
100 static DEFINE_PER_CPU(int, gator_first_time);
102 /******************************************************************************
103 * Prototypes
104 ******************************************************************************/
105 static void gator_buffer_write_packed_int(int cpu, unsigned int x);
106 static void gator_buffer_write_string(int cpu, char *x);
107 static void gator_add_trace(int cpu, unsigned int address);
108 static uint64_t gator_get_time(void);
110 /******************************************************************************
111 * Application Includes
112 ******************************************************************************/
113 #include "gator_cookies.c"
114 #include "gator_trace_sched.c"
115 #include "gator_backtrace.c"
116 #include "gator_annotate.c"
117 #include "gator_events.c"
118 #include "gator_fs.c"
120 /******************************************************************************
121 * Misc
122 ******************************************************************************/
123 #if defined(__arm__)
124 u32 gator_cpuid(void)
125 {
126 u32 val;
127 asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
128 return (val >> 4) & 0xfff;
129 }
130 #endif
132 /******************************************************************************
133 * Commit interface
134 ******************************************************************************/
135 static int buffer_commit_ready(void)
136 {
137 return (gator_commit_read != gator_commit_write);
138 }
140 static void buffer_commit_read(int *cpu, int *readval, int *writeval)
141 {
142 int read = gator_commit_read;
143 *cpu = gator_commit[read+0];
144 *readval = gator_commit[read+1];
145 *writeval = gator_commit[read+2];
146 gator_commit_read = (read + 4) & COMMIT_MASK;
147 }
149 static void buffer_commit_write(int cpu, int readval, int writeval) {
150 int write = gator_commit_write;
151 gator_commit[write+0] = cpu;
152 gator_commit[write+1] = readval;
153 gator_commit[write+2] = writeval;
154 gator_commit_write = (write + 4) & COMMIT_MASK;
155 }
157 /******************************************************************************
158 * Buffer management
159 ******************************************************************************/
160 static uint32_t use_buffer_size;
161 static uint32_t use_buffer_mask;
162 static DEFINE_PER_CPU(int, use_buffer_seq);
163 static DEFINE_PER_CPU(int, use_buffer_read);
164 static DEFINE_PER_CPU(int, use_buffer_write);
165 static DEFINE_PER_CPU(char *, use_buffer);
167 static void gator_buffer_write_packed_int(int cpu, unsigned int x)
168 {
169 uint32_t write = per_cpu(use_buffer_write, cpu);
170 uint32_t mask = use_buffer_mask;
171 char *buffer = per_cpu(use_buffer, cpu);
172 int write0 = (write + 0) & mask;
173 int write1 = (write + 1) & mask;
174 int write2 = (write + 2) & mask;
175 int write3 = (write + 3) & mask;
176 int write4 = (write + 4) & mask;
177 int write5 = (write + 5) & mask;
179 if ((x & 0xffffff80) == 0) {
180 buffer[write0] = x & 0x7f;
181 per_cpu(use_buffer_write, cpu) = write1;
182 } else if ((x & 0xffffc000) == 0) {
183 buffer[write0] = x | 0x80;
184 buffer[write1] = (x>>7) & 0x7f;
185 per_cpu(use_buffer_write, cpu) = write2;
186 } else if ((x & 0xffe00000) == 0) {
187 buffer[write0] = x | 0x80;
188 buffer[write1] = (x>>7) | 0x80;
189 buffer[write2] = (x>>14) & 0x7f;
190 per_cpu(use_buffer_write, cpu) = write3;
191 } else if ((x & 0xf0000000) == 0) {
192 buffer[write0] = x | 0x80;
193 buffer[write1] = (x>>7) | 0x80;
194 buffer[write2] = (x>>14) | 0x80;
195 buffer[write3] = (x>>21) & 0x7f;
196 per_cpu(use_buffer_write, cpu) = write4;
197 } else {
198 buffer[write0] = x | 0x80;
199 buffer[write1] = (x>>7) | 0x80;
200 buffer[write2] = (x>>14) | 0x80;
201 buffer[write3] = (x>>21) | 0x80;
202 buffer[write4] = (x>>28) & 0x0f;
203 per_cpu(use_buffer_write, cpu) = write5;
204 }
205 }
207 static void gator_buffer_write_bytes(int cpu, char *x, int len)
208 {
209 uint32_t write = per_cpu(use_buffer_write, cpu);
210 uint32_t mask = use_buffer_mask;
211 char *buffer = per_cpu(use_buffer, cpu);
212 int i;
214 for (i = 0; i < len; i++) {
215 buffer[write] = x[i];
216 write = (write + 1) & mask;
217 }
219 per_cpu(use_buffer_write, cpu) = write;
220 }
222 static void gator_buffer_write_string(int cpu, char *x)
223 {
224 int len = strlen(x);
225 gator_buffer_write_packed_int(cpu, len);
226 gator_buffer_write_bytes(cpu, x, len);
227 }
229 static void gator_buffer_header(int cpu)
230 {
231 gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME);
232 gator_buffer_write_packed_int(cpu, cpu);
233 gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu));
234 per_cpu(use_buffer_seq, cpu)++;
235 }
237 static void gator_buffer_commit(int cpu)
238 {
239 buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu));
240 per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu);
241 gator_buffer_header(cpu);
242 wake_up(&gator_buffer_wait);
243 }
245 static void gator_buffer_check(int cpu, int tick)
246 {
247 if (gator_sync_freq && !(tick % gator_sync_freq)) {
248 int c, sync;
249 spin_lock(&gator_commit_lock);
250 // synchronize, if all online cpus have the same tick waypoint
251 sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu);
252 for_each_online_cpu(c) {
253 if (sync != per_cpu(gator_cpu_sync, c)) {
254 sync = 0;
255 break;
256 }
257 }
258 if (sync) {
259 gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC);
260 }
261 // commit the buffer
262 gator_buffer_commit(cpu);
263 spin_unlock(&gator_commit_lock);
264 } else {
265 int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu);
266 if (available < 0) {
267 available += use_buffer_size;
268 }
269 if (available >= ((use_buffer_size * 3) / 4)) {
270 spin_lock(&gator_commit_lock);
271 gator_buffer_commit(cpu);
272 spin_unlock(&gator_commit_lock);
273 }
274 }
275 }
277 static void gator_add_trace(int cpu, unsigned int address)
278 {
279 off_t offset = 0;
280 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
282 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
283 offset = address;
284 }
286 gator_buffer_write_packed_int(cpu, offset & ~1);
287 gator_buffer_write_packed_int(cpu, cookie);
288 }
290 static void gator_add_sample(int cpu, struct pt_regs * const regs)
291 {
292 struct module *mod;
293 unsigned int addr, cookie = 0;
294 int inKernel = regs ? !user_mode(regs) : 1;
295 unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
297 gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE);
299 // TGID::PID::inKernel
300 gator_buffer_write_packed_int(cpu, exec_cookie);
301 gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid);
302 gator_buffer_write_packed_int(cpu, (unsigned int)current->pid);
303 gator_buffer_write_packed_int(cpu, inKernel);
305 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
306 if (regs) {
307 if (inKernel) {
308 addr = PC_REG;
309 mod = __module_address(addr);
310 if (mod) {
311 cookie = get_cookie(cpu, current, NULL, mod);
312 addr = addr - (unsigned long)mod->module_core;
313 }
314 gator_buffer_write_packed_int(cpu, addr & ~1);
315 gator_buffer_write_packed_int(cpu, cookie);
316 } else {
317 // Cookie+PC
318 gator_add_trace(cpu, PC_REG);
320 // Backtrace
321 if (gator_backtrace_depth)
322 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
323 }
324 }
326 gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE);
327 }
329 static void gator_write_packet(int cpu, int type, int len, int *buffer)
330 {
331 int i;
332 gator_buffer_write_packed_int(cpu, type);
333 gator_buffer_write_packed_int(cpu, len);
334 for (i = 0; i < len; i++) {
335 gator_buffer_write_packed_int(cpu, buffer[i]);
336 }
337 }
339 static void gator_write_annotate(int cpu, int len, int *buffer)
340 {
341 int pos = 0;
343 while (pos < len) {
344 unsigned int tid = buffer[pos++];
345 unsigned int bytes = buffer[pos++];
346 unsigned int words = (bytes + 3) / 4;
347 char *ptr = (char *)&buffer[pos];
348 pos += words;
350 gator_buffer_write_packed_int(cpu, PROTOCOL_ANNOTATE);
351 gator_buffer_write_packed_int(cpu, tid);
352 gator_buffer_write_packed_int(cpu, bytes);
353 gator_buffer_write_bytes(cpu, ptr, bytes);
354 }
355 }
357 /******************************************************************************
358 * Interrupt Processing
359 ******************************************************************************/
360 static gator_interface *gi = NULL;
362 static void gator_timer_interrupt(void)
363 {
364 struct pt_regs * const regs = get_irq_regs();
365 int cpu = raw_smp_processor_id();
366 int *buffer, len, tick;
367 gator_interface *i;
369 // check full backtrace has enough space, otherwise may
370 // have breaks between samples in the same callstack
371 if (per_cpu(gator_first_time, cpu)) {
372 per_cpu(gator_first_time, cpu) = 0;
374 for (i = gi; i != NULL; i = i->next) {
375 if (i->read) {
376 i->read(NULL);
377 }
378 }
379 return;
380 }
382 // Header
383 gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape
385 // Output scheduler
386 len = gator_trace_sched_read(&buffer);
387 if (len > 0) {
388 gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer);
389 }
391 // Output annotate
392 len = gator_annotate_read(&buffer);
393 if (len > 0)
394 gator_write_annotate(cpu, len, buffer);
396 // Output counters
397 for (i = gi; i != NULL; i = i->next) {
398 if (i->read) {
399 len = i->read(&buffer);
400 if (len > 0) {
401 gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer);
402 }
403 }
404 }
406 // Output backtrace
407 gator_add_sample(cpu, regs);
409 // Timer Tick
410 tick = per_cpu(gator_cpu_tick, cpu);
411 if (tick == gator_master_tick) {
412 tick++;
413 per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick;
414 } else {
415 per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick;
416 }
417 gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick);
419 // Check and commit; generally, commit is set to occur once per second
420 gator_buffer_check(cpu, tick);
421 }
423 /******************************************************************************
424 * hrtimer
425 ******************************************************************************/
426 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
427 DEFINE_PER_CPU(int, hrtimer_is_active);
428 static int hrtimer_running;
429 static ktime_t profiling_interval;
431 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
432 {
433 hrtimer_forward_now(hrtimer, profiling_interval);
434 gator_timer_interrupt();
435 return HRTIMER_RESTART;
436 }
438 static int gator_timer_init(void)
439 {
440 return 0;
441 }
443 static void __gator_timer_offline(void *unused)
444 {
445 int cpu = smp_processor_id();
446 if (per_cpu(hrtimer_is_active, cpu)) {
447 gator_interface *i;
448 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
449 hrtimer_cancel(hrtimer);
450 per_cpu(hrtimer_is_active, cpu) = 0;
451 gator_buffer_commit(cpu);
453 // offline any events
454 for (i = gi; i != NULL; i = i->next) {
455 if (i->offline) {
456 i->offline();
457 }
458 }
459 }
460 }
462 static void gator_timer_offline(void)
463 {
464 if (hrtimer_running) {
465 hrtimer_running = 0;
467 on_each_cpu(__gator_timer_offline, NULL, 1);
469 // output a final sync point
470 gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC);
471 gator_buffer_commit(0);
472 }
473 }
475 static void __gator_timer_online(void *unused)
476 {
477 int cpu = smp_processor_id();
478 if (!per_cpu(hrtimer_is_active, cpu)) {
479 gator_interface *i;
480 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
481 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
482 hrtimer->function = gator_hrtimer_notify;
483 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
484 per_cpu(gator_cpu_tick, cpu) = 0;
485 per_cpu(gator_first_time, cpu) = 1;
486 per_cpu(hrtimer_is_active, cpu) = 1;
488 // online any events
489 for (i = gi; i != NULL; i = i->next) {
490 if (i->online) {
491 i->online();
492 }
493 }
494 }
495 }
497 int gator_timer_online(unsigned long setup)
498 {
499 if (!setup) {
500 pr_err("gator: cannot start due to a system tick value of zero");
501 return -1;
502 } else if (hrtimer_running) {
503 pr_notice("gator: high res timer already running");
504 return 0;
505 }
507 hrtimer_running = 1;
509 // calculate profiling interval
510 profiling_interval = ns_to_ktime(1000000000UL / setup);
512 // timer interrupt
513 gator_master_tick = 0;
514 on_each_cpu(__gator_timer_online, NULL, 1);
516 return 0;
517 }
519 static uint64_t gator_get_time(void)
520 {
521 struct timespec ts;
522 uint64_t timestamp;
524 ktime_get_ts(&ts);
525 timestamp = timespec_to_ns(&ts);
527 return timestamp;
528 }
530 /******************************************************************************
531 * cpu online notifier
532 ******************************************************************************/
533 static int __cpuinit gator_cpu_notify(struct notifier_block *self,
534 unsigned long action, void *hcpu)
535 {
536 long cpu = (long)hcpu;
538 switch (action) {
539 case CPU_ONLINE:
540 case CPU_ONLINE_FROZEN:
541 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
542 break;
543 case CPU_DOWN_PREPARE:
544 case CPU_DOWN_PREPARE_FROZEN:
545 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
546 break;
547 }
549 return NOTIFY_OK;
550 }
552 static struct notifier_block __refdata gator_cpu_notifier = {
553 .notifier_call = gator_cpu_notify,
554 };
556 static int gator_notifier_start(void)
557 {
558 return register_hotcpu_notifier(&gator_cpu_notifier);
559 }
561 static void gator_notifier_stop(void)
562 {
563 unregister_hotcpu_notifier(&gator_cpu_notifier);
564 }
566 /******************************************************************************
567 * Main
568 ******************************************************************************/
569 int gator_event_install(int (*event_install)(gator_interface *))
570 {
571 gator_interface *ni = (gator_interface*)kmalloc(sizeof(gator_interface), GFP_KERNEL);
572 if (ni == NULL) {
573 return -1;
574 }
576 ni->create_files = NULL;
577 ni->init = NULL;
578 ni->start = NULL;
579 ni->stop = NULL;
580 ni->online = NULL;
581 ni->offline = NULL;
582 ni->read = NULL;
583 ni->next = NULL;
585 // Initialize ni gator interface
586 if (!event_install(ni)) {
587 if (gi == NULL) {
588 // Set gi to point to the first gator interface
589 gi = ni;
590 } else {
591 // Link the gator interfaces
592 gator_interface *i = gi;
593 while (i->next) {
594 i = i->next;
595 }
596 i->next = ni;
597 }
598 } else {
599 kfree(ni);
600 }
602 return 0;
603 }
605 static int gator_init(void)
606 {
607 gator_interface *i;
608 int key = 0;
610 if (gator_timer_init())
611 return -1;
612 if (gator_trace_sched_init())
613 return -1;
614 if (gator_annotate_init())
615 return -1;
617 // set up gator interface linked list structure
618 if (gator_events_install())
619 return -1;
621 // initialize all events
622 for (i = gi; i != NULL; i = i->next) {
623 if (i->init) {
624 if (i->init(&key)) {
625 return -1;
626 }
627 }
628 }
630 return 0;
631 }
633 static int gator_start(void)
634 {
635 gator_interface *i, *f;
637 // start all events
638 for (i = gi; i != NULL; i = i->next) {
639 if (i->start) {
640 if (i->start()) {
641 goto events_failure;
642 }
643 }
644 }
646 if (gator_annotate_start())
647 goto annotate_failure;
648 if (gator_trace_sched_start())
649 goto sched_failure;
650 if (gator_timer_online(gator_timer_count))
651 goto timer_failure;
652 if (gator_notifier_start())
653 goto notifier_failure;
655 return 0;
657 notifier_failure:
658 gator_timer_offline();
659 timer_failure:
660 gator_trace_sched_stop();
661 sched_failure:
662 gator_annotate_stop();
663 annotate_failure:
664 events_failure:
665 for (f = gi; f != i; f = f->next) {
666 f->stop();
667 }
669 return -1;
670 }
672 static void gator_stop(void)
673 {
674 gator_interface *i;
676 // stop all events
677 for (i = gi; i != NULL; i = i->next) {
678 if (i->stop) {
679 i->stop();
680 }
681 }
683 gator_annotate_stop();
684 gator_trace_sched_stop();
686 // stop all interrupt callback reads before tearing down other interfaces
687 gator_timer_offline();
688 gator_notifier_stop();
689 }
691 static void gator_exit(void)
692 {
693 gator_interface *i = gi;
695 while (i) {
696 gator_interface *p = i;
697 i = i->next;
698 kfree(p);
699 }
700 }
702 /******************************************************************************
703 * Filesystem
704 ******************************************************************************/
705 /* fopen("buffer") */
706 static int gator_op_setup(void)
707 {
708 int err = 0;
709 int cpu;
711 mutex_lock(&start_mutex);
713 use_buffer_size = gator_buffer_size;
714 use_buffer_mask = use_buffer_size - 1;
716 // must be a power of 2
717 if (use_buffer_size & (use_buffer_size - 1)) {
718 err = -ENOEXEC;
719 goto setup_error;
720 }
722 gator_net_traffic = 0;
724 gator_commit_read = gator_commit_write = 0;
725 gator_commit = vmalloc(COMMIT_SIZE * sizeof(int));
726 if (!gator_commit) {
727 err = -ENOMEM;
728 goto setup_error;
729 }
731 for_each_present_cpu(cpu) {
732 per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size);
733 if (!per_cpu(use_buffer, cpu)) {
734 err = -ENOMEM;
735 goto setup_error;
736 }
738 per_cpu(gator_cpu_sync, cpu) = 0;
739 per_cpu(gator_cpu_tick, cpu) = 0;
741 per_cpu(use_buffer_seq, cpu) = 0;
742 per_cpu(use_buffer_read, cpu) = 0;
743 per_cpu(use_buffer_write, cpu) = 0;
744 gator_buffer_header(cpu);
745 }
747 setup_error:
748 mutex_unlock(&start_mutex);
749 return err;
750 }
752 /* Actually start profiling (echo 1>/dev/gator/enable) */
753 static int gator_op_start(void)
754 {
755 int err = 0;
757 mutex_lock(&start_mutex);
759 if (gator_started || gator_start() || cookies_initialize())
760 err = -EINVAL;
761 else
762 gator_started = 1;
764 mutex_unlock(&start_mutex);
766 return err;
767 }
769 /* echo 0>/dev/gator/enable */
770 static void gator_op_stop(void)
771 {
772 mutex_lock(&start_mutex);
774 if (gator_started) {
775 gator_stop();
777 mutex_lock(&gator_buffer_mutex);
779 gator_started = 0;
780 cookies_release();
781 wake_up(&gator_buffer_wait);
783 mutex_unlock(&gator_buffer_mutex);
784 }
786 mutex_unlock(&start_mutex);
787 }
789 static void gator_shutdown(void)
790 {
791 int cpu;
793 mutex_lock(&start_mutex);
795 vfree(gator_commit);
796 gator_commit = NULL;
798 for_each_present_cpu(cpu) {
799 mutex_lock(&gator_buffer_mutex);
800 vfree(per_cpu(use_buffer, cpu));
801 per_cpu(use_buffer, cpu) = NULL;
802 per_cpu(use_buffer_seq, cpu) = 0;
803 per_cpu(use_buffer_read, cpu) = 0;
804 per_cpu(use_buffer_write, cpu) = 0;
805 mutex_unlock(&gator_buffer_mutex);
806 }
808 mutex_unlock(&start_mutex);
809 }
811 static int gator_set_backtrace(unsigned long val)
812 {
813 int err = 0;
815 mutex_lock(&start_mutex);
817 if (gator_started)
818 err = -EBUSY;
819 else
820 gator_backtrace_depth = val;
822 mutex_unlock(&start_mutex);
824 return err;
825 }
827 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
828 {
829 return gatorfs_ulong_to_user(gator_started, buf, count, offset);
830 }
832 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
833 {
834 unsigned long val;
835 int retval;
837 if (*offset)
838 return -EINVAL;
840 retval = gatorfs_ulong_from_user(&val, buf, count);
841 if (retval)
842 return retval;
844 if (val)
845 retval = gator_op_start();
846 else
847 gator_op_stop();
849 if (retval)
850 return retval;
851 return count;
852 }
854 static const struct file_operations enable_fops = {
855 .read = enable_read,
856 .write = enable_write,
857 };
859 static int event_buffer_open(struct inode *inode, struct file *file)
860 {
861 int err = -EPERM;
863 if (!capable(CAP_SYS_ADMIN))
864 return -EPERM;
866 if (test_and_set_bit_lock(0, &gator_buffer_opened))
867 return -EBUSY;
869 if ((err = gator_op_setup()))
870 goto fail;
872 /* NB: the actual start happens from userspace
873 * echo 1 >/dev/gator/enable
874 */
876 return 0;
878 fail:
879 __clear_bit_unlock(0, &gator_buffer_opened);
880 return err;
881 }
883 static int event_buffer_release(struct inode *inode, struct file *file)
884 {
885 gator_op_stop();
886 gator_shutdown();
887 __clear_bit_unlock(0, &gator_buffer_opened);
888 return 0;
889 }
891 static ssize_t event_buffer_read(struct file *file, char __user *buf,
892 size_t count, loff_t *offset)
893 {
894 int retval = -EINVAL;
895 int commit, length1, length2, read;
896 char *buffer1, *buffer2;
897 int cpu;
899 /* do not handle partial reads */
900 if (count != use_buffer_size || *offset)
901 return -EINVAL;
903 // sleep until the condition is true or a signal is received
904 // the condition is checked each time gator_buffer_wait is woken up
905 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || !gator_started);
907 if (signal_pending(current))
908 return -EINTR;
910 if (!buffer_commit_ready())
911 return 0;
913 buffer_commit_read(&cpu, &read, &commit);
915 mutex_lock(&gator_buffer_mutex);
917 retval = -EFAULT;
919 /* May happen if the buffer is freed during pending reads. */
920 if (!per_cpu(use_buffer, cpu)) {
921 retval = -EFAULT;
922 goto out;
923 }
925 /* determine the size of two halves */
926 length1 = commit - read;
927 length2 = 0;
928 buffer1 = &(per_cpu(use_buffer, cpu)[read]);
929 buffer2 = &(per_cpu(use_buffer, cpu)[0]);
930 if (length1 < 0) {
931 length1 = use_buffer_size - read;
932 length2 = commit;
933 }
935 /* start, middle or end */
936 if (length1 > 0) {
937 if (copy_to_user(&buf[0], buffer1, length1)) {
938 goto out;
939 }
940 }
942 /* possible wrap around */
943 if (length2 > 0) {
944 if (copy_to_user(&buf[length1], buffer2, length2)) {
945 goto out;
946 }
947 }
949 retval = length1 + length2;
951 /* kick just in case we've lost an SMP event */
952 wake_up(&gator_buffer_wait);
954 out:
955 // do not adjust network stats if in non-streaming buffer mode
956 if (gator_sync_freq)
957 gator_net_traffic += retval;
958 mutex_unlock(&gator_buffer_mutex);
959 return retval;
960 }
962 const struct file_operations gator_event_buffer_fops = {
963 .open = event_buffer_open,
964 .release = event_buffer_release,
965 .read = event_buffer_read,
966 };
968 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
969 {
970 return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
971 offset);
972 }
974 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
975 {
976 unsigned long val;
977 int retval;
979 if (*offset)
980 return -EINVAL;
982 retval = gatorfs_ulong_from_user(&val, buf, count);
983 if (retval)
984 return retval;
986 retval = gator_set_backtrace(val);
988 if (retval)
989 return retval;
990 return count;
991 }
993 static const struct file_operations depth_fops = {
994 .read = depth_read,
995 .write = depth_write
996 };
998 static const char gator_cpu_type[] = "gator";
1000 static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1001 {
1002 return gatorfs_str_to_user(gator_cpu_type, buf, count, offset);
1003 }
1005 static const struct file_operations cpu_type_fops = {
1006 .read = cpu_type_read,
1007 };
1009 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1010 {
1011 struct dentry *dir;
1012 gator_interface *i;
1013 int cpu;
1015 /* reinitialize default values */
1016 gator_cpu_cores = 0;
1017 for_each_present_cpu(cpu) {
1018 gator_cpu_cores++;
1019 }
1020 gator_buffer_size = BUFFER_SIZE_DEFAULT;
1021 gator_sync_freq = SYNC_FREQ_DEFAULT;
1023 gatorfs_create_file(sb, root, "enable", &enable_fops);
1024 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1025 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1026 gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops);
1027 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1028 gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size);
1029 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1030 gatorfs_create_ulong(sb, root, "sync_freq", &gator_sync_freq);
1031 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1033 // Annotate interface
1034 gator_annotate_create_files(sb, root);
1036 // Linux Events
1037 dir = gatorfs_mkdir(sb, root, "events");
1038 for (i = gi; i != NULL; i = i->next) {
1039 if (i->create_files) {
1040 i->create_files(sb, dir);
1041 }
1042 }
1043 }
1045 /******************************************************************************
1046 * Module
1047 ******************************************************************************/
1048 static int gator_initialized;
1050 static int __init gator_module_init(void)
1051 {
1052 if (gatorfs_register()) {
1053 return -1;
1054 }
1056 if (gator_init()) {
1057 gatorfs_unregister();
1058 return -1;
1059 }
1061 gator_initialized = 1;
1062 #ifdef GATOR_DEBUG
1063 pr_err("gator_module_init");
1064 #endif
1065 return 0;
1066 }
1068 static void __exit gator_module_exit(void)
1069 {
1070 #ifdef GATOR_DEBUG
1071 pr_err("gator_module_exit");
1072 #endif
1073 tracepoint_synchronize_unregister();
1074 gatorfs_unregister();
1075 if (gator_initialized) {
1076 gator_initialized = 0;
1077 gator_exit();
1078 }
1079 }
1081 module_init(gator_module_init);
1082 module_exit(gator_module_exit);
1084 MODULE_LICENSE("GPL");
1085 MODULE_AUTHOR("ARM Ltd");
1086 MODULE_DESCRIPTION("Gator system profiler");