gator: ARM DS-5.6 Streamline gator driver
[android-sdk/arm-ds5-gator.git] / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 5;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <asm/uaccess.h>
22 #include "gator.h"
23 #include "gator_events.h"
25 #ifndef CONFIG_GENERIC_TRACER
26 #ifndef CONFIG_TRACING
27 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
28 #endif
29 #endif
31 #ifndef CONFIG_PROFILING
32 #error gator requires the kernel to have CONFIG_PROFILING defined
33 #endif
35 #ifndef CONFIG_HIGH_RES_TIMERS
36 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
37 #endif
39 #ifdef CONFIG_SMP
40 #ifndef CONFIG_LOCAL_TIMERS
41 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
42 #endif
43 #endif
45 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
46 #error kernels prior to 2.6.32 are not supported
47 #endif
49 /******************************************************************************
50  * DEFINES
51  ******************************************************************************/
52 #define BUFFER_SIZE_DEFAULT             (256*1024)
53 #define SYNC_FREQ_DEFAULT               1000
55 #define NO_COOKIE                               0UL
56 #define INVALID_COOKIE                  ~0UL
58 #define PROTOCOL_FRAME                          ~0
59 #define PROTOCOL_START_TICK                     1
60 #define PROTOCOL_END_TICK                       3
61 #define PROTOCOL_START_BACKTRACE        5
62 #define PROTOCOL_END_BACKTRACE          7
63 #define PROTOCOL_COOKIE                         9
64 #define PROTOCOL_SCHEDULER_TRACE        11
65 #define PROTOCOL_COUNTERS                       13
66 #define PROTOCOL_ANNOTATE                       15
67 #define PROTOCOL_CPU_SYNC                       17
69 #if defined(__arm__)
70 #define PC_REG regs->ARM_pc
71 #else
72 #define PC_REG regs->ip
73 #endif
75 /******************************************************************************
76  * PER CPU
77  ******************************************************************************/
78 static unsigned long gator_cpu_cores;
79 static unsigned long gator_buffer_size;
80 static unsigned long gator_backtrace_depth;
82 static unsigned long gator_started;
83 static unsigned long gator_buffer_opened;
84 static unsigned long gator_timer_count;
85 static unsigned long gator_streaming;
86 static int gator_master_tick;
87 static DEFINE_MUTEX(start_mutex);
88 static DEFINE_MUTEX(gator_buffer_mutex);
90 unsigned long gator_net_traffic;
92 #define COMMIT_SIZE             128
93 #define COMMIT_MASK             (COMMIT_SIZE-1)
94 static DEFINE_SPINLOCK(gator_commit_lock);
95 static int *gator_commit;
96 static int gator_commit_read;
97 static int gator_commit_write;
99 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
100 static DEFINE_PER_CPU(int, gator_cpu_sync);
101 static DEFINE_PER_CPU(int, gator_cpu_tick);
102 static DEFINE_PER_CPU(int, gator_first_time);
104 /******************************************************************************
105  * Prototypes
106  ******************************************************************************/
107 static void gator_buffer_write_packed_int(int cpu, unsigned int x);
108 static void gator_buffer_write_string(int cpu, char *x);
109 static int  gator_write_packed_int(char *buffer, unsigned int x);
110 static void gator_add_trace(int cpu, unsigned int address);
111 static uint64_t gator_get_time(void);
113 /******************************************************************************
114  * Application Includes
115  ******************************************************************************/
116 #include "gator_cookies.c"
117 #include "gator_trace_sched.c"
118 #include "gator_backtrace.c"
119 #include "gator_annotate.c"
120 #include "gator_fs.c"
122 /******************************************************************************
123  * Misc
124  ******************************************************************************/
125 #if defined(__arm__)
126 u32 gator_cpuid(void)
128         u32 val;
129         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
130         return (val >> 4) & 0xfff;
132 #endif
134 /******************************************************************************
135  * Commit interface
136  ******************************************************************************/
137 static int buffer_commit_ready(void)
139         return (gator_commit_read != gator_commit_write);
142 static void buffer_commit_read(int *cpu, int *readval, int *writeval)
144         int read = gator_commit_read;
145         *cpu      = gator_commit[read+0];
146         *readval  = gator_commit[read+1];
147         *writeval = gator_commit[read+2];
148         gator_commit_read = (read + 4) & COMMIT_MASK;
151 static void buffer_commit_write(int cpu, int readval, int writeval) {
152         int write = gator_commit_write;
153         gator_commit[write+0] = cpu;
154         gator_commit[write+1] = readval;
155         gator_commit[write+2] = writeval;
156         gator_commit_write = (write + 4) & COMMIT_MASK;
159 /******************************************************************************
160  * Buffer management
161  ******************************************************************************/
162 static uint32_t use_buffer_size;
163 static uint32_t use_buffer_mask;
164 static DEFINE_PER_CPU(int, use_buffer_seq);
165 static DEFINE_PER_CPU(int, use_buffer_read);
166 static DEFINE_PER_CPU(int, use_buffer_write);
167 static DEFINE_PER_CPU(char *, use_buffer);
169 static void gator_buffer_write_packed_int(int cpu, unsigned int x)
171         uint32_t write = per_cpu(use_buffer_write, cpu);
172         uint32_t mask = use_buffer_mask;
173         char *buffer = per_cpu(use_buffer, cpu);
174         int write0 = (write + 0) & mask;
175         int write1 = (write + 1) & mask;
176         int write2 = (write + 2) & mask;
177         int write3 = (write + 3) & mask;
178         int write4 = (write + 4) & mask;
179         int write5 = (write + 5) & mask;
181         if ((x & 0xffffff80) == 0) {
182                 buffer[write0] = x & 0x7f;
183                 per_cpu(use_buffer_write, cpu) = write1;
184         } else if ((x & 0xffffc000) == 0) {
185                 buffer[write0] = x | 0x80;
186                 buffer[write1] = (x>>7) & 0x7f;
187                 per_cpu(use_buffer_write, cpu) = write2;
188         } else if ((x & 0xffe00000) == 0) {
189                 buffer[write0] = x | 0x80;
190                 buffer[write1] = (x>>7) | 0x80;
191                 buffer[write2] = (x>>14) & 0x7f;
192                 per_cpu(use_buffer_write, cpu) = write3;
193         } else if ((x & 0xf0000000) == 0) {
194                 buffer[write0] = x | 0x80;
195                 buffer[write1] = (x>>7) | 0x80;
196                 buffer[write2] = (x>>14) | 0x80;
197                 buffer[write3] = (x>>21) & 0x7f;
198                 per_cpu(use_buffer_write, cpu) = write4;
199         } else {
200                 buffer[write0] = x | 0x80;
201                 buffer[write1] = (x>>7) | 0x80;
202                 buffer[write2] = (x>>14) | 0x80;
203                 buffer[write3] = (x>>21) | 0x80;
204                 buffer[write4] = (x>>28) & 0x0f;
205                 per_cpu(use_buffer_write, cpu) = write5;
206         }
209 static int gator_write_packed_int(char *buffer, unsigned int x)
211         if ((x & 0xffffff80) == 0) {
212                 buffer[0] = x & 0x7f;
213                 return 1;
214         } else if ((x & 0xffffc000) == 0) {
215                 buffer[0] = x | 0x80;
216                 buffer[1] = (x>>7) & 0x7f;
217                 return 2;
218         } else if ((x & 0xffe00000) == 0) {
219                 buffer[0] = x | 0x80;
220                 buffer[1] = (x>>7) | 0x80;
221                 buffer[2] = (x>>14) & 0x7f;
222                 return 3;
223         } else if ((x & 0xf0000000) == 0) {
224                 buffer[0] = x | 0x80;
225                 buffer[1] = (x>>7) | 0x80;
226                 buffer[2] = (x>>14) | 0x80;
227                 buffer[3] = (x>>21) & 0x7f;
228                 return 4;
229         } else {
230                 buffer[0] = x | 0x80;
231                 buffer[1] = (x>>7) | 0x80;
232                 buffer[2] = (x>>14) | 0x80;
233                 buffer[3] = (x>>21) | 0x80;
234                 buffer[4] = (x>>28) & 0x0f;
235                 return 5;
236         }
239 static void gator_buffer_write_bytes(int cpu, char *x, int len)
241         uint32_t write = per_cpu(use_buffer_write, cpu);
242         uint32_t mask = use_buffer_mask;
243         char *buffer = per_cpu(use_buffer, cpu);
244         int i;
246         for (i = 0; i < len; i++) {
247                 buffer[write] = x[i];
248                 write = (write + 1) & mask;
249         }
251         per_cpu(use_buffer_write, cpu) = write;
254 static void gator_buffer_write_string(int cpu, char *x)
256         int len = strlen(x);
257         gator_buffer_write_packed_int(cpu, len);
258         gator_buffer_write_bytes(cpu, x, len);
261 static void gator_buffer_header(int cpu)
263         gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME);
264         gator_buffer_write_packed_int(cpu, cpu);
265         gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu));
266         per_cpu(use_buffer_seq, cpu)++;
269 static void gator_buffer_commit(int cpu)
271         buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu));
272         per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu);
273         gator_buffer_header(cpu);
274         wake_up(&gator_buffer_wait);
277 static void gator_buffer_check(int cpu, int tick)
279         if (!(tick % gator_timer_count)) {
280                 int c, sync;
281                 spin_lock(&gator_commit_lock);
282                 // synchronize, if all online cpus have the same tick waypoint
283                 sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu);
284                 for_each_online_cpu(c) {
285                         if (sync != per_cpu(gator_cpu_sync, c)) {
286                                 sync = 0;
287                                 break;
288                         }
289                 }
290                 if (sync) {
291                         gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC);
292                 }
293                 gator_buffer_commit(cpu);
294                 spin_unlock(&gator_commit_lock);
295         } else {
296                 int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu);
297                 if (available < 0) {
298                         available += use_buffer_size;
299                 }
300                 if (available >= ((use_buffer_size * 3) / 4)) {
301                         spin_lock(&gator_commit_lock);
302                         gator_buffer_commit(cpu);
303                         spin_unlock(&gator_commit_lock);
304                 }
305         }
308 static void gator_add_trace(int cpu, unsigned int address)
310         off_t offset = 0;
311         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
313         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
314                 offset = address;
315         }
317         gator_buffer_write_packed_int(cpu, offset & ~1);
318         gator_buffer_write_packed_int(cpu, cookie);
321 static void gator_add_sample(int cpu, struct pt_regs * const regs)
323         struct module *mod;
324         unsigned int addr, cookie = 0;
325         int inKernel = regs ? !user_mode(regs) : 1;
326         unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
328         gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE);
330         // TGID::PID::inKernel
331         gator_buffer_write_packed_int(cpu, exec_cookie);
332         gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid);
333         gator_buffer_write_packed_int(cpu, (unsigned int)current->pid);
334         gator_buffer_write_packed_int(cpu, inKernel);
336         // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
337         if (regs) {
338                 if (inKernel) {
339                         addr = PC_REG;
340                         mod = __module_address(addr);
341                         if (mod) {
342                                 cookie = get_cookie(cpu, current, NULL, mod);
343                                 addr = addr - (unsigned long)mod->module_core;
344                         }
345                         gator_buffer_write_packed_int(cpu, addr & ~1);
346                         gator_buffer_write_packed_int(cpu, cookie);
347                 } else {
348                         // Cookie+PC
349                         gator_add_trace(cpu, PC_REG);
351                         // Backtrace
352                         if (gator_backtrace_depth)
353                                 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
354                 }
355         }
357         gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE);
360 static void gator_write_packet(int cpu, int type, int len, int *buffer)
362         int i;
363         gator_buffer_write_packed_int(cpu, type);
364         gator_buffer_write_packed_int(cpu, len);
365         for (i = 0; i < len; i++) {
366                 gator_buffer_write_packed_int(cpu, buffer[i]);
367         }
370 /******************************************************************************
371  * Interrupt Processing
372  ******************************************************************************/
373 static LIST_HEAD(gator_events);
375 static void gator_timer_interrupt(void)
377         struct pt_regs * const regs = get_irq_regs();
378         int cpu = smp_processor_id();
379         int *buffer, len, tick;
380         struct gator_interface *gi;
382         // check full backtrace has enough space, otherwise may
383         // have breaks between samples in the same callstack
384         if (per_cpu(gator_first_time, cpu)) {
385                 per_cpu(gator_first_time, cpu) = 0;
387                 list_for_each_entry(gi, &gator_events, list)
388                         if (gi->read)
389                                 gi->read(NULL);
391                 return;
392         }
394         // Header
395         gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK);                        // Escape
397         // Output scheduler
398         len = gator_trace_sched_read(&buffer);
399         if (len > 0) {
400                 gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer);
401         }
403         // Output counters
404         list_for_each_entry(gi, &gator_events, list) {
405                 if (gi->read) {
406                         len = gi->read(&buffer);
407                         if (len > 0)
408                                 gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer);
409                 }
410         }
412         // Output backtrace
413         gator_add_sample(cpu, regs);
415         // Timer Tick
416         tick = per_cpu(gator_cpu_tick, cpu);
417         if (tick == gator_master_tick) {
418                 tick++;
419                 per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick;
420         } else {
421                 per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick;
422         }
423         gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick);
425         // Check and commit; generally, commit is set to occur once per second
426         gator_buffer_check(cpu, tick);
429 /******************************************************************************
430  * hrtimer
431  ******************************************************************************/
432 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
433 DEFINE_PER_CPU(int, hrtimer_is_active);
434 static int hrtimer_running;
435 static ktime_t profiling_interval;
437 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
439         hrtimer_forward_now(hrtimer, profiling_interval);
440         gator_timer_interrupt();
441         return HRTIMER_RESTART;
444 static int gator_timer_init(void)
446         return 0;
449 static void __gator_timer_offline(void *unused)
451         int cpu = smp_processor_id();
452         if (per_cpu(hrtimer_is_active, cpu)) {
453                 struct gator_interface *gi;
454                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
455                 hrtimer_cancel(hrtimer);
456                 per_cpu(hrtimer_is_active, cpu) = 0;
457                 gator_buffer_commit(cpu);
459                 // offline any events
460                 list_for_each_entry(gi, &gator_events, list)
461                         if (gi->offline)
462                                 gi->offline();
463         }
466 static void gator_timer_offline(void)
468         if (hrtimer_running) {
469                 hrtimer_running = 0;
471                 on_each_cpu(__gator_timer_offline, NULL, 1);
473                 // output a final sync point
474                 gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC);
475                 gator_buffer_commit(0);
476         }
479 static void __gator_timer_online(void *unused)
481         int cpu = smp_processor_id();
482         if (!per_cpu(hrtimer_is_active, cpu)) {
483                 struct gator_interface *gi;
484                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
485                 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
486                 hrtimer->function = gator_hrtimer_notify;
487                 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
488                 per_cpu(gator_cpu_tick, cpu) = 0;
489                 per_cpu(gator_first_time, cpu) = 1;
490                 per_cpu(hrtimer_is_active, cpu) = 1;
492                 // online any events
493                 list_for_each_entry(gi, &gator_events, list)
494                         if (gi->online)
495                                 gi->online();
496         }
499 int gator_timer_online(unsigned long setup)
501         if (!setup) {
502                 pr_err("gator: cannot start due to a system tick value of zero");
503                 return -1;
504         } else if (hrtimer_running) {
505                 pr_notice("gator: high res timer already running");
506                 return 0;
507         }
509         hrtimer_running = 1;
511         // calculate profiling interval
512         profiling_interval = ns_to_ktime(1000000000UL / setup);
514         // timer interrupt
515         gator_master_tick = 0;
516         on_each_cpu(__gator_timer_online, NULL, 1);
518         return 0;
521 static uint64_t gator_get_time(void)
523         struct timespec ts;
524         uint64_t timestamp;
526         ktime_get_ts(&ts);
527         timestamp = timespec_to_ns(&ts);
529         return timestamp;
532 /******************************************************************************
533  * cpu online notifier
534  ******************************************************************************/
535 static int __cpuinit gator_cpu_notify(struct notifier_block *self,
536                                                                                         unsigned long action, void *hcpu)
538         long cpu = (long)hcpu;
540         switch (action) {
541                 case CPU_ONLINE:
542                 case CPU_ONLINE_FROZEN:
543                         smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
544                         break;
545                 case CPU_DOWN_PREPARE:
546                 case CPU_DOWN_PREPARE_FROZEN:
547                         smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
548                         break;
549         }
551         return NOTIFY_OK;
554 static struct notifier_block __refdata gator_cpu_notifier = {
555         .notifier_call = gator_cpu_notify,
556 };
558 static int gator_notifier_start(void)
560         return register_hotcpu_notifier(&gator_cpu_notifier);
563 static void gator_notifier_stop(void)
565         unregister_hotcpu_notifier(&gator_cpu_notifier);
568 /******************************************************************************
569  * Main
570  ******************************************************************************/
571 int gator_events_install(struct gator_interface *interface)
573         list_add_tail(&interface->list, &gator_events);
575         return 0;
578 int gator_events_get_key(void)
580         static int key;
582         return key++;
585 static int gator_init(void)
587         int i;
589         if (gator_timer_init())
590                 return -1;
591         if (gator_trace_sched_init())
592                 return -1;
593         if (gator_annotate_init())
594                 return -1;
596         // events sources (gator_events.h, generated by gator_events.sh)
597         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
598                 if (gator_events_list[i])
599                         gator_events_list[i]();
601         return 0;
604 static int gator_start(void)
606         struct gator_interface *gi;
608         // start all events
609         list_for_each_entry(gi, &gator_events, list) {
610                 if (gi->start && gi->start() != 0) {
611                         struct list_head *ptr = gi->list.prev;
613                         while (ptr != &gator_events) {
614                                 gi = list_entry(ptr, struct gator_interface,
615                                                 list);
617                                 if (gi->stop)
618                                         gi->stop();
620                                 ptr = ptr->prev;
621                         }
622                         goto events_failure;
623                 }
624         }
626         if (gator_annotate_start())
627                 goto annotate_failure;
628         if (gator_trace_sched_start())
629                 goto sched_failure;
630         if (gator_timer_online(gator_timer_count))
631                 goto timer_failure;
632         if (gator_notifier_start())
633                 goto notifier_failure;
635         return 0;
637 notifier_failure:
638         gator_timer_offline();
639 timer_failure:
640         gator_trace_sched_stop();
641 sched_failure:
642         gator_annotate_stop();
643 annotate_failure:
644 events_failure:
646         return -1;
649 static void gator_stop(void)
651         struct gator_interface *gi;
653         // stop all events
654         list_for_each_entry(gi, &gator_events, list)
655                 if (gi->stop)
656                         gi->stop();
658         gator_annotate_stop();
659         gator_trace_sched_stop();
661         // stop all interrupt callback reads before tearing down other interfaces
662         gator_timer_offline();
663         gator_notifier_stop();
666 static void gator_exit(void)
668         gator_annotate_exit();
671 /******************************************************************************
672  * Filesystem
673  ******************************************************************************/
674 /* fopen("buffer") */
675 static int gator_op_setup(void)
677         int err = 0;
678         int cpu;
680         mutex_lock(&start_mutex);
682         use_buffer_size = gator_buffer_size;
683         use_buffer_mask = use_buffer_size - 1;
685         // must be a power of 2
686         if (use_buffer_size & (use_buffer_size - 1)) {
687                 err = -ENOEXEC;
688                 goto setup_error;
689         }
691         gator_net_traffic = 0;
693         gator_commit_read = gator_commit_write = 0;
694         gator_commit = vmalloc(COMMIT_SIZE * sizeof(int));
695         if (!gator_commit) {
696                 err = -ENOMEM;
697                 goto setup_error;
698         }
700         for_each_present_cpu(cpu) {
701                 per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size);
702                 if (!per_cpu(use_buffer, cpu)) {
703                         err = -ENOMEM;
704                         goto setup_error;
705                 }
707                 per_cpu(gator_cpu_sync, cpu) = 0;
708                 per_cpu(gator_cpu_tick, cpu) = 0;
710                 per_cpu(use_buffer_seq, cpu) = 0;
711                 per_cpu(use_buffer_read, cpu) = 0;
712                 per_cpu(use_buffer_write, cpu) = 0;
713                 gator_buffer_header(cpu);
714         }
716 setup_error:
717         mutex_unlock(&start_mutex);
718         return err;
721 /* Actually start profiling (echo 1>/dev/gator/enable) */
722 static int gator_op_start(void)
724         int err = 0;
726         mutex_lock(&start_mutex);
728         if (gator_started || gator_start() || cookies_initialize())
729                 err = -EINVAL;
730         else
731                 gator_started = 1;
733         mutex_unlock(&start_mutex);
735         return err;
738 /* echo 0>/dev/gator/enable */
739 static void gator_op_stop(void)
741         mutex_lock(&start_mutex);
743         if (gator_started) {
744                 gator_stop();
746                 mutex_lock(&gator_buffer_mutex);
748                 gator_started = 0;
749                 cookies_release();
750                 wake_up(&gator_buffer_wait);
752                 mutex_unlock(&gator_buffer_mutex);
753         }
755         mutex_unlock(&start_mutex);
758 static void gator_shutdown(void)
760         int cpu;
762         mutex_lock(&start_mutex);
764         vfree(gator_commit);
765         gator_commit = NULL;
767         for_each_present_cpu(cpu) {
768                 mutex_lock(&gator_buffer_mutex);
769                 vfree(per_cpu(use_buffer, cpu));
770                 per_cpu(use_buffer, cpu) = NULL;
771                 per_cpu(use_buffer_seq, cpu) = 0;
772                 per_cpu(use_buffer_read, cpu) = 0;
773                 per_cpu(use_buffer_write, cpu) = 0;
774                 mutex_unlock(&gator_buffer_mutex);
775         }
777         mutex_unlock(&start_mutex);
780 static int gator_set_backtrace(unsigned long val)
782         int err = 0;
784         mutex_lock(&start_mutex);
786         if (gator_started)
787                 err = -EBUSY;
788         else
789                 gator_backtrace_depth = val;
791         mutex_unlock(&start_mutex);
793         return err;
796 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
798         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
801 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
803         unsigned long val;
804         int retval;
806         if (*offset)
807                 return -EINVAL;
809         retval = gatorfs_ulong_from_user(&val, buf, count);
810         if (retval)
811                 return retval;
813         if (val)
814                 retval = gator_op_start();
815         else
816                 gator_op_stop();
818         if (retval)
819                 return retval;
820         return count;
823 static const struct file_operations enable_fops = {
824         .read           = enable_read,
825         .write          = enable_write,
826 };
828 static int event_buffer_open(struct inode *inode, struct file *file)
830         int err = -EPERM;
832         if (!capable(CAP_SYS_ADMIN))
833                 return -EPERM;
835         if (test_and_set_bit_lock(0, &gator_buffer_opened))
836                 return -EBUSY;
838         if ((err = gator_op_setup()))
839                 goto fail;
841         /* NB: the actual start happens from userspace
842          * echo 1 >/dev/gator/enable
843          */
845         return 0;
847 fail:
848         __clear_bit_unlock(0, &gator_buffer_opened);
849         return err;
852 static int event_buffer_release(struct inode *inode, struct file *file)
854         gator_op_stop();
855         gator_shutdown();
856         __clear_bit_unlock(0, &gator_buffer_opened);
857         return 0;
860 static ssize_t event_buffer_read(struct file *file, char __user *buf,
861                                  size_t count, loff_t *offset)
863         int retval = -EINVAL;
864         int commit, length1, length2, read;
865         char *buffer1, *buffer2;
866         char annotate_header[6];
867         int cpu;
869         /* do not handle partial reads */
870         if (count != use_buffer_size || *offset)
871                 return -EINVAL;
873         // sleep until the condition is true or a signal is received
874         // the condition is checked each time gator_buffer_wait is woken up
875         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started);
877         if (signal_pending(current))
878                 return -EINTR;
880         retval = -EFAULT;
882         mutex_lock(&gator_buffer_mutex);
884         if (buffer_commit_ready()) {
885                 buffer_commit_read(&cpu, &read, &commit);
887                 /* May happen if the buffer is freed during pending reads. */
888                 if (!per_cpu(use_buffer, cpu)) {
889                         retval = -EFAULT;
890                         goto out;
891                 }
893                 /* determine the size of two halves */
894                 length1 = commit - read;
895                 length2 = 0;
896                 buffer1 = &(per_cpu(use_buffer, cpu)[read]);
897                 buffer2 = &(per_cpu(use_buffer, cpu)[0]);
898                 if (length1 < 0) {
899                         length1 = use_buffer_size - read;
900                         length2 = commit;
901                 }
902         } else if (gator_annotate_ready()) {
903                 length2 = gator_annotate_read(&buffer2);
904                 if (!length2)
905                         goto out;
906                 annotate_header[0] = PROTOCOL_ANNOTATE;
907                 length1 = gator_write_packed_int(&annotate_header[1], length2) + 1;
908                 buffer1 = annotate_header;
909         } else {
910                 retval = 0;
911                 goto out;
912         }
914         /* start, middle or end */
915         if (length1 > 0) {
916                 if (copy_to_user(&buf[0], buffer1, length1)) {
917                         goto out;
918                 }
919         }
921         /* possible wrap around */
922         if (length2 > 0) {
923                 if (copy_to_user(&buf[length1], buffer2, length2)) {
924                         goto out;
925                 }
926         }
928         retval = length1 + length2;
930         /* kick just in case we've lost an SMP event */
931         wake_up(&gator_buffer_wait);
933 out:
934         // only adjust network stats if in streaming mode
935         if (gator_streaming)
936                 gator_net_traffic += retval;
937         mutex_unlock(&gator_buffer_mutex);
938         return retval;
941 const struct file_operations gator_event_buffer_fops = {
942         .open           = event_buffer_open,
943         .release        = event_buffer_release,
944         .read           = event_buffer_read,
945 };
947 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
949         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
950                                         offset);
953 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
955         unsigned long val;
956         int retval;
958         if (*offset)
959                 return -EINVAL;
961         retval = gatorfs_ulong_from_user(&val, buf, count);
962         if (retval)
963                 return retval;
965         retval = gator_set_backtrace(val);
967         if (retval)
968                 return retval;
969         return count;
972 static const struct file_operations depth_fops = {
973         .read           = depth_read,
974         .write          = depth_write
975 };
977 static const char gator_cpu_type[] = "gator";
979 static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
981         return gatorfs_str_to_user(gator_cpu_type, buf, count, offset);
984 static const struct file_operations cpu_type_fops = {
985         .read           = cpu_type_read,
986 };
988 void gator_op_create_files(struct super_block *sb, struct dentry *root)
990         struct dentry *dir;
991         struct gator_interface *gi;
992         int cpu;
994         /* reinitialize default values */
995         gator_cpu_cores = 0;
996         for_each_present_cpu(cpu) {
997                 gator_cpu_cores++;
998         }
999         gator_buffer_size =     BUFFER_SIZE_DEFAULT;
1000         gator_streaming = 1;
1002         gatorfs_create_file(sb, root, "enable", &enable_fops);
1003         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1004         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1005         gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops);
1006         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1007         gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size);
1008         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1009         gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
1010         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1012         // Annotate interface
1013         gator_annotate_create_files(sb, root);
1015         // Linux Events
1016         dir = gatorfs_mkdir(sb, root, "events");
1017         list_for_each_entry(gi, &gator_events, list)
1018                 if (gi->create_files)
1019                         gi->create_files(sb, dir);
1022 /******************************************************************************
1023  * Module
1024  ******************************************************************************/
1025 static int __init gator_module_init(void)
1027         if (gatorfs_register()) {
1028                 return -1;
1029         }
1031         if (gator_init()) {
1032                 gatorfs_unregister();
1033                 return -1;
1034         }
1036 #ifdef GATOR_DEBUG
1037         pr_err("gator_module_init");
1038 #endif
1039         return 0;
1042 static void __exit gator_module_exit(void)
1044 #ifdef GATOR_DEBUG
1045         pr_err("gator_module_exit");
1046 #endif
1047         tracepoint_synchronize_unregister();
1048         gatorfs_unregister();
1049         gator_exit();
1052 module_init(gator_module_init);
1053 module_exit(gator_module_exit);
1055 MODULE_LICENSE("GPL");
1056 MODULE_AUTHOR("ARM Ltd");
1057 MODULE_DESCRIPTION("Gator system profiler");