Merge branch 'master' into android
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 7;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <asm/stacktrace.h>
23 #include <asm/uaccess.h>
25 #include "gator.h"
26 #include "gator_events.h"
28 #ifndef CONFIG_GENERIC_TRACER
29 #ifndef CONFIG_TRACING
30 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
31 #endif
32 #endif
34 #ifndef CONFIG_PROFILING
35 #error gator requires the kernel to have CONFIG_PROFILING defined
36 #endif
38 #ifndef CONFIG_HIGH_RES_TIMERS
39 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
40 #endif
42 #if defined (__arm__)
43 #ifdef CONFIG_SMP
44 #ifndef CONFIG_LOCAL_TIMERS
45 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
46 #endif
47 #endif
48 #endif
50 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
51 #error kernels prior to 2.6.32 are not supported
52 #endif
54 /******************************************************************************
55  * DEFINES
56  ******************************************************************************/
57 #define TIMER_BUFFER_SIZE_DEFAULT       (512*1024)
58 #define EVENT_BUFFER_SIZE_DEFAULT       (128*1024)
60 #define NO_COOKIE                               0UL
61 #define INVALID_COOKIE                  ~0UL
63 #define FRAME_HRTIMER           1
64 #define FRAME_EVENT                     2
65 #define FRAME_ANNOTATE          3
67 #define MESSAGE_COOKIE                          1
68 #define MESSAGE_COUNTERS                        3
69 #define MESSAGE_START_BACKTRACE         5
70 #define MESSAGE_END_BACKTRACE           7
71 #define MESSAGE_SCHEDULER_TRACE         9
72 #define MESSAGE_PID_NAME                        11
74 #if 0 /* Disabled because of changes in Linaro 3.1 kernel */
75 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
76 #else
77 #define LINUX_PMU_SUPPORT 0
78 #endif
80 #if defined(__arm__)
81 #define PC_REG regs->ARM_pc
82 #else
83 #define PC_REG regs->ip
84 #endif
86 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
88 /******************************************************************************
89  * Globals
90  ******************************************************************************/
91 static unsigned long gator_cpu_cores;
92 static unsigned long userspace_buffer_size;
93 static unsigned long gator_backtrace_depth;
95 static unsigned long gator_started;
96 static unsigned long gator_buffer_opened;
97 static unsigned long gator_timer_count;
98 static unsigned long gator_streaming;
99 static DEFINE_MUTEX(start_mutex);
100 static DEFINE_MUTEX(gator_buffer_mutex);
102 unsigned long gator_net_traffic;
103 bool event_based_sampling;
105 #define COMMIT_SIZE             128
106 #define COMMIT_MASK             (COMMIT_SIZE-1)
107 static DEFINE_SPINLOCK(timer_commit_lock);
108 static int *gator_commit[NUM_GATOR_BUFS];
109 static int gator_commit_read[NUM_GATOR_BUFS];
110 static int gator_commit_write[NUM_GATOR_BUFS];
112 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
113 static DEFINE_PER_CPU(int, gator_first_time);
115 #if LINUX_PMU_SUPPORT
116 static void event_buffer_check(int cpu);
117 static DEFINE_SPINLOCK(event_commit_lock);
118 #endif
120 /******************************************************************************
121  * Prototypes
122  ******************************************************************************/
123 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
124 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
125 static void gator_buffer_write_string(int cpu, int buftype, char *x);
126 static int  gator_write_packed_int(char *buffer, unsigned int x);
127 static int  gator_write_packed_int64(char *buffer, unsigned long long x);
128 static void gator_add_trace(int cpu, int buftype, unsigned int address);
129 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
130 static uint64_t gator_get_time(void);
132 /******************************************************************************
133  * Application Includes
134  ******************************************************************************/
135 #include "gator_cookies.c"
136 #include "gator_trace_sched.c"
137 #include "gator_backtrace.c"
138 #include "gator_annotate.c"
139 #include "gator_fs.c"
140 #include "gator_ebs.c"
142 /******************************************************************************
143  * Misc
144  ******************************************************************************/
145 #if defined(__arm__)
146 u32 gator_cpuid(void)
148         u32 val;
149         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
150         return (val >> 4) & 0xfff;
152 #endif
154 /******************************************************************************
155  * Commit interface
156  ******************************************************************************/
157 static int buffer_commit_ready(int buftype)
159         return gator_commit_read[buftype] != gator_commit_write[buftype];
162 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
164         int read = gator_commit_read[buftype];
165         *cpu      = gator_commit[buftype][read+0];
166         *readval  = gator_commit[buftype][read+1];
167         *writeval = gator_commit[buftype][read+2];
168         gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
171 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
172         int write = gator_commit_write[buftype];
173         gator_commit[buftype][write+0] = cpu;
174         gator_commit[buftype][write+1] = readval;
175         gator_commit[buftype][write+2] = writeval;
176         gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
179 /******************************************************************************
180  * Buffer management
181  ******************************************************************************/
182 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
183 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
184 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
185 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
186 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
187 #include "gator_pack.c"
189 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
191         int i;
192         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
193         u32 mask = gator_buffer_mask[buftype];
194         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
196         for (i = 0; i < len; i++) {
197                 buffer[write] = x[i];
198                 write = (write + 1) & mask;
199         }
201         per_cpu(gator_buffer_write, cpu)[buftype] = write;
204 static void gator_buffer_write_string(int cpu, int buftype, char *x)
206         int len = strlen(x);
207         gator_buffer_write_packed_int(cpu, buftype, len);
208         gator_buffer_write_bytes(cpu, buftype, x, len);
211 static void gator_buffer_header(int cpu, int buftype)
213         int frame;
215         if (buftype == TIMER_BUF)
216                 frame = FRAME_HRTIMER;
217         else if (buftype == EVENT_BUF)
218                 frame = FRAME_EVENT;
219         else
220                 frame = -1;
222         gator_buffer_write_packed_int(cpu, buftype, frame);
223         gator_buffer_write_packed_int(cpu, buftype, cpu);
226 static void gator_buffer_commit(int cpu, int buftype)
228         buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
229         per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
230         gator_buffer_header(cpu, buftype);
231         wake_up(&gator_buffer_wait);
234 static void timer_buffer_check(int cpu)
236         int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
237         if (available < 0) {
238                 available += gator_buffer_size[TIMER_BUF];
239         }
240         if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
241                 spin_lock(&timer_commit_lock);
242                 gator_buffer_commit(cpu, TIMER_BUF);
243                 spin_unlock(&timer_commit_lock);
244         }
247 #if LINUX_PMU_SUPPORT
248 static void event_buffer_check(int cpu)
250         int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
251         if (available < 0) {
252                 available += gator_buffer_size[EVENT_BUF];
253         }
254         if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
255                 spin_lock(&event_commit_lock);
256                 gator_buffer_commit(cpu, EVENT_BUF);
257                 spin_unlock(&event_commit_lock);
258         }
260 #endif
262 static void gator_add_trace(int cpu, int buftype, unsigned int address)
264         off_t offset = 0;
265         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
267         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
268                 offset = address;
269         }
271         gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
272         gator_buffer_write_packed_int(cpu, buftype, cookie);
275 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
277         int inKernel = regs ? !user_mode(regs) : 1;
278         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
280         if (!regs)
281                 return;
283         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
284         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
285         gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
286         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); 
287         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
288         gator_buffer_write_packed_int(cpu, buftype, inKernel);
290         if (inKernel) {
291                 kernel_backtrace(cpu, buftype, regs);
292         } else {
293                 // Cookie+PC
294                 gator_add_trace(cpu, buftype, PC_REG);
296                 // Backtrace
297                 if (gator_backtrace_depth)
298                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
299         }
301         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
304 /******************************************************************************
305  * hrtimer interrupt processing
306  ******************************************************************************/
307 static LIST_HEAD(gator_events);
309 static void gator_timer_interrupt(void)
311         struct pt_regs * const regs = get_irq_regs();
312         int cpu = smp_processor_id();
313         int *buffer, len, i, buftype = TIMER_BUF;
314         long long *buffer64;
315         struct gator_interface *gi;
317         // check full backtrace has enough space, otherwise may
318         // have breaks between samples in the same callstack
319         if (per_cpu(gator_first_time, cpu)) {
320                 per_cpu(gator_first_time, cpu) = 0;
322                 list_for_each_entry(gi, &gator_events, list)
323                         if (gi->read)
324                                 gi->read(NULL);
326                 return;
327         }
329         // Output scheduler
330         len = gator_trace_sched_read(&buffer64);
331         if (len > 0) {
332                 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
333                 gator_buffer_write_packed_int(cpu, buftype, len);
334                 for (i = 0; i < len; i++) {
335                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
336                 }
337         }
339         // Output counters
340         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
341         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
342         list_for_each_entry(gi, &gator_events, list) {
343                 if (gi->read) {
344                         len = gi->read(&buffer);
345                         if (len > 0) {
346                                 gator_buffer_write_packed_int(cpu, buftype, len);
347                                 for (i = 0; i < len; i++) {
348                                         gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
349                                 }
350                         }
351                 } else if (gi->read64) {
352                         len = gi->read64(&buffer64);
353                         if (len > 0) {
354                                 gator_buffer_write_packed_int(cpu, buftype, len);
355                                 for (i = 0; i < len; i++) {
356                                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
357                                 }
358                         }
359                 }
360         }
361         gator_buffer_write_packed_int(cpu, buftype, 0);
363         // Output backtrace
364         if (!event_based_sampling) {
365                 gator_add_sample(cpu, buftype, regs);
366         }
368         // Check and commit; generally, commit is set to occur once per second
369         timer_buffer_check(cpu);
372 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
373 DEFINE_PER_CPU(int, hrtimer_is_active);
374 static int hrtimer_running;
375 static ktime_t profiling_interval;
377 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
379         hrtimer_forward_now(hrtimer, profiling_interval);
380         gator_timer_interrupt();
381         return HRTIMER_RESTART;
384 static int gator_timer_init(void)
386         return 0;
389 static void __gator_timer_offline(void *unused)
391         int cpu = smp_processor_id();
392         if (per_cpu(hrtimer_is_active, cpu)) {
393                 struct gator_interface *gi;
394                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
395                 hrtimer_cancel(hrtimer);
396                 per_cpu(hrtimer_is_active, cpu) = 0;
397                 gator_buffer_commit(cpu, TIMER_BUF);
398                 if (event_based_sampling)
399                         gator_buffer_commit(cpu, EVENT_BUF);
401                 // offline any events
402                 list_for_each_entry(gi, &gator_events, list)
403                         if (gi->offline)
404                                 gi->offline();
405         }
408 static void gator_timer_offline(void)
410         if (hrtimer_running) {
411                 hrtimer_running = 0;
413                 on_each_cpu(__gator_timer_offline, NULL, 1);
414         }
417 static void __gator_timer_online(void *unused)
419         int cpu = smp_processor_id();
420         if (!per_cpu(hrtimer_is_active, cpu)) {
421                 struct gator_interface *gi;
422                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
423                 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
424                 hrtimer->function = gator_hrtimer_notify;
425                 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
426                 per_cpu(gator_first_time, cpu) = 1;
427                 per_cpu(hrtimer_is_active, cpu) = 1;
429                 // online any events
430                 list_for_each_entry(gi, &gator_events, list)
431                         if (gi->online)
432                                 gi->online();
433         }
436 int gator_timer_online(unsigned long setup)
438         if (!setup) {
439                 pr_err("gator: cannot start due to a system tick value of zero\n");
440                 return -1;
441         } else if (hrtimer_running) {
442                 pr_notice("gator: high res timer already running\n");
443                 return 0;
444         }
446         hrtimer_running = 1;
448         // calculate profiling interval
449         profiling_interval = ns_to_ktime(1000000000UL / setup);
451         // timer interrupt
452         on_each_cpu(__gator_timer_online, NULL, 1);
454         return 0;
457 static uint64_t gator_get_time(void)
459         struct timespec ts;
460         uint64_t timestamp;
462         getnstimeofday(&ts);
463         timestamp = timespec_to_ns(&ts);
465         return timestamp;
468 /******************************************************************************
469  * cpu online and pm notifiers
470  ******************************************************************************/
471 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
473         long cpu = (long)hcpu;
475         switch (action) {
476                 case CPU_DOWN_PREPARE:
477                 case CPU_DOWN_PREPARE_FROZEN:
478                         smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
479                         break;
480                 case CPU_ONLINE:
481                 case CPU_ONLINE_FROZEN:
482                         smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
483                         break;
484         }
486         return NOTIFY_OK;
489 static struct notifier_block __refdata gator_cpu_notifier = {
490         .notifier_call = gator_cpu_notify,
491 };
493 // n.b. calling "on_each_cpu" only runs on those that are online
494 // Registered linux events are not disabled, so their counters will continue to collect
495 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
497         switch (event) {
498                 case PM_HIBERNATION_PREPARE:
499                 case PM_SUSPEND_PREPARE:
500                         unregister_hotcpu_notifier(&gator_cpu_notifier);
501                         unregister_scheduler_tracepoints();
502                         on_each_cpu(trace_sched_insert_idle, NULL, 1);
503                         on_each_cpu(__gator_timer_offline, NULL, 1);
504                         break;
505                 case PM_POST_HIBERNATION:
506                 case PM_POST_SUSPEND:
507                         on_each_cpu(__gator_timer_online, NULL, 1);
508                         register_scheduler_tracepoints();
509                         register_hotcpu_notifier(&gator_cpu_notifier);
510                         break;
511         }
513         return NOTIFY_OK;
516 static struct notifier_block gator_pm_notifier = {
517         .notifier_call = gator_pm_notify,
518 };
520 static int gator_notifier_start(void)
522         int retval;
523         retval = register_hotcpu_notifier(&gator_cpu_notifier);
524         if (retval == 0)
525                 retval = register_pm_notifier(&gator_pm_notifier);
526         return retval;
529 static void gator_notifier_stop(void)
531         unregister_pm_notifier(&gator_pm_notifier);
532         unregister_hotcpu_notifier(&gator_cpu_notifier);
535 /******************************************************************************
536  * Main
537  ******************************************************************************/
538 int gator_events_install(struct gator_interface *interface)
540         list_add_tail(&interface->list, &gator_events);
542         return 0;
545 int gator_events_get_key(void)
547         static int key;
549         return key++;
552 static int gator_init(void)
554         int i;
556         if (gator_timer_init())
557                 return -1;
558         if (gator_trace_sched_init())
559                 return -1;
560         if (gator_annotate_init())
561                 return -1;
563         // events sources (gator_events.h, generated by gator_events.sh)
564         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
565                 if (gator_events_list[i])
566                         gator_events_list[i]();
568         return 0;
571 static int gator_start(void)
573         struct gator_interface *gi;
575         // start all events
576         list_for_each_entry(gi, &gator_events, list) {
577                 if (gi->start && gi->start() != 0) {
578                         struct list_head *ptr = gi->list.prev;
580                         while (ptr != &gator_events) {
581                                 gi = list_entry(ptr, struct gator_interface,
582                                                 list);
584                                 if (gi->stop)
585                                         gi->stop();
587                                 ptr = ptr->prev;
588                         }
589                         goto events_failure;
590                 }
591         }
593         // cookies shall be initialized before trace_sched_start() and gator_timer_online()
594         if (cookies_initialize())
595                 goto cookies_failure;
596         if (gator_annotate_start())
597                 goto annotate_failure;
598         if (gator_trace_sched_start())
599                 goto sched_failure;
600         if (gator_event_sampling_start())
601                 goto event_sampling_failure;
602         if (gator_timer_online(gator_timer_count))
603                 goto timer_failure;
604         if (gator_notifier_start())
605                 goto notifier_failure;
607         return 0;
609 notifier_failure:
610         gator_timer_offline();
611 timer_failure:
612         gator_event_sampling_stop();
613 event_sampling_failure:
614         gator_trace_sched_stop();
615 sched_failure:
616         gator_annotate_stop();
617 annotate_failure:
618         cookies_release();
619 cookies_failure:
620         // stop all events
621         list_for_each_entry(gi, &gator_events, list)
622                 if (gi->stop)
623                         gi->stop();
624 events_failure:
626         return -1;
629 static void gator_stop(void)
631         struct gator_interface *gi;
633         // stop all events
634         list_for_each_entry(gi, &gator_events, list)
635                 if (gi->stop)
636                         gi->stop();
638         gator_annotate_stop();
639         gator_trace_sched_stop();
640         gator_event_sampling_stop();
642         // stop all interrupt callback reads before tearing down other interfaces
643         gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
644         gator_timer_offline();
647 static void gator_exit(void)
649         gator_annotate_exit();
652 /******************************************************************************
653  * Filesystem
654  ******************************************************************************/
655 /* fopen("buffer") */
656 static int gator_op_setup(void)
658         int err = 0;
659         int cpu, i;
661         mutex_lock(&start_mutex);
663         gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
664         gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
666         // must be a power of 2
667         if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
668                 err = -ENOEXEC;
669                 goto setup_error;
670         }
672         gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
673         gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
675         gator_net_traffic = 0;
677         // Initialize per buffer variables
678         for (i = 0; i < NUM_GATOR_BUFS; i++) {
679                 gator_commit_read[i] = gator_commit_write[i] = 0;
680                 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
681                 if (!gator_commit[i]) {
682                         err = -ENOMEM;
683                         goto setup_error;
684                 }
686                 // Initialize percpu per buffer variables
687                 for_each_present_cpu(cpu) {
688                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
689                         if (!per_cpu(gator_buffer, cpu)[i]) {
690                                 err = -ENOMEM;
691                                 goto setup_error;
692                         }
694                         per_cpu(gator_buffer_read, cpu)[i] = 0;
695                         per_cpu(gator_buffer_write, cpu)[i] = 0;
696                         gator_buffer_header(cpu, i);
697                 }
698         }
700 setup_error:
701         mutex_unlock(&start_mutex);
702         return err;
705 /* Actually start profiling (echo 1>/dev/gator/enable) */
706 static int gator_op_start(void)
708         int err = 0;
710         mutex_lock(&start_mutex);
712         if (gator_started || gator_start())
713                 err = -EINVAL;
714         else
715                 gator_started = 1;
717         mutex_unlock(&start_mutex);
719         return err;
722 /* echo 0>/dev/gator/enable */
723 static void gator_op_stop(void)
725         mutex_lock(&start_mutex);
727         if (gator_started) {
728                 gator_stop();
730                 mutex_lock(&gator_buffer_mutex);
732                 gator_started = 0;
733                 cookies_release();
734                 wake_up(&gator_buffer_wait);
736                 mutex_unlock(&gator_buffer_mutex);
737         }
739         mutex_unlock(&start_mutex);
742 static void gator_shutdown(void)
744         int cpu, i;
746         mutex_lock(&start_mutex);
748         gator_annotate_shutdown();
750         for (i = 0; i < NUM_GATOR_BUFS; i++) {
751                 vfree(gator_commit[i]);
752                 gator_commit[i] = NULL;
753         }
755         for_each_present_cpu(cpu) {
756                 mutex_lock(&gator_buffer_mutex);
757                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
758                         vfree(per_cpu(gator_buffer, cpu)[i]);
759                         per_cpu(gator_buffer, cpu)[i] = NULL;
760                         per_cpu(gator_buffer_read, cpu)[i] = 0;
761                         per_cpu(gator_buffer_write, cpu)[i] = 0;
762                 }
763                 mutex_unlock(&gator_buffer_mutex);
764         }
766         mutex_unlock(&start_mutex);
769 static int gator_set_backtrace(unsigned long val)
771         int err = 0;
773         mutex_lock(&start_mutex);
775         if (gator_started)
776                 err = -EBUSY;
777         else
778                 gator_backtrace_depth = val;
780         mutex_unlock(&start_mutex);
782         return err;
785 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
787         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
790 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
792         unsigned long val;
793         int retval;
795         if (*offset)
796                 return -EINVAL;
798         retval = gatorfs_ulong_from_user(&val, buf, count);
799         if (retval)
800                 return retval;
802         if (val)
803                 retval = gator_op_start();
804         else
805                 gator_op_stop();
807         if (retval)
808                 return retval;
809         return count;
812 static const struct file_operations enable_fops = {
813         .read           = enable_read,
814         .write          = enable_write,
815 };
817 static int userspace_buffer_open(struct inode *inode, struct file *file)
819         int err = -EPERM;
821         if (!capable(CAP_SYS_ADMIN))
822                 return -EPERM;
824         if (test_and_set_bit_lock(0, &gator_buffer_opened))
825                 return -EBUSY;
827         if ((err = gator_op_setup()))
828                 goto fail;
830         /* NB: the actual start happens from userspace
831          * echo 1 >/dev/gator/enable
832          */
834         return 0;
836 fail:
837         __clear_bit_unlock(0, &gator_buffer_opened);
838         return err;
841 static int userspace_buffer_release(struct inode *inode, struct file *file)
843         gator_op_stop();
844         gator_shutdown();
845         __clear_bit_unlock(0, &gator_buffer_opened);
846         return 0;
849 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
850                                  size_t count, loff_t *offset)
852         int retval = -EINVAL;
853         int commit, length1, length2, read;
854         char *buffer1;
855         char *buffer2 = NULL;
856         int cpu, i;
858         /* do not handle partial reads */
859         if (count != userspace_buffer_size || *offset)
860                 return -EINVAL;
862         // sleep until the condition is true or a signal is received
863         // the condition is checked each time gator_buffer_wait is woken up
864         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
866         if (signal_pending(current))
867                 return -EINTR;
869         length2 = 0;
870         retval = -EFAULT;
872         mutex_lock(&gator_buffer_mutex);
874         i = -1;
875         if (buffer_commit_ready(TIMER_BUF)) {
876                 i = TIMER_BUF;
877         } else if (buffer_commit_ready(EVENT_BUF)) {
878                 i = EVENT_BUF;
879         }
881         if (i != -1) {
882                 buffer_commit_read(&cpu, i, &read, &commit);
884                 /* May happen if the buffer is freed during pending reads. */
885                 if (!per_cpu(gator_buffer, cpu)[i]) {
886                         retval = -EFAULT;
887                         goto out;
888                 }
890                 /* determine the size of two halves */
891                 length1 = commit - read;
892                 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
893                 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
894                 if (length1 < 0) {
895                         length1 = gator_buffer_size[i] - read;
896                         length2 = commit;
897                 }
898         } else if (gator_annotate_ready()) {
899                 length1 = gator_annotate_read(&buffer1);
900                 if (!length1)
901                         goto out;
902         } else {
903                 retval = 0;
904                 goto out;
905         }
907         /* start, middle or end */
908         if (length1 > 0) {
909                 if (copy_to_user(&buf[0], buffer1, length1)) {
910                         goto out;
911                 }
912         }
914         /* possible wrap around */
915         if (length2 > 0) {
916                 if (copy_to_user(&buf[length1], buffer2, length2)) {
917                         goto out;
918                 }
919         }
921         retval = length1 + length2;
923         /* kick just in case we've lost an SMP event */
924         wake_up(&gator_buffer_wait);
926 out:
927         // only adjust network stats if in streaming mode
928         if (gator_streaming)
929                 gator_net_traffic += retval;
930         mutex_unlock(&gator_buffer_mutex);
931         return retval;
934 const struct file_operations gator_event_buffer_fops = {
935         .open           = userspace_buffer_open,
936         .release        = userspace_buffer_release,
937         .read           = userspace_buffer_read,
938 };
940 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
942         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
943                                         offset);
946 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
948         unsigned long val;
949         int retval;
951         if (*offset)
952                 return -EINVAL;
954         retval = gatorfs_ulong_from_user(&val, buf, count);
955         if (retval)
956                 return retval;
958         retval = gator_set_backtrace(val);
960         if (retval)
961                 return retval;
962         return count;
965 static const struct file_operations depth_fops = {
966         .read           = depth_read,
967         .write          = depth_write
968 };
970 void gator_op_create_files(struct super_block *sb, struct dentry *root)
972         struct dentry *dir;
973         struct gator_interface *gi;
974         int cpu;
976         /* reinitialize default values */
977         gator_cpu_cores = 0;
978         for_each_present_cpu(cpu) {
979                 gator_cpu_cores++;
980         }
981         userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
982         gator_streaming = 1;
984         gatorfs_create_file(sb, root, "enable", &enable_fops);
985         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
986         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
987         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
988         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
989         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
990         gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
991         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
993         // Annotate interface
994         gator_annotate_create_files(sb, root);
996         // Linux Events
997         dir = gatorfs_mkdir(sb, root, "events");
998         list_for_each_entry(gi, &gator_events, list)
999                 if (gi->create_files)
1000                         gi->create_files(sb, dir);
1003 /******************************************************************************
1004  * Module
1005  ******************************************************************************/
1006 static int __init gator_module_init(void)
1008         if (gatorfs_register()) {
1009                 return -1;
1010         }
1012         if (gator_init()) {
1013                 gatorfs_unregister();
1014                 return -1;
1015         }
1017         return 0;
1020 static void __exit gator_module_exit(void)
1022         tracepoint_synchronize_unregister();
1023         gatorfs_unregister();
1024         gator_exit();
1027 module_init(gator_module_init);
1028 module_exit(gator_module_exit);
1030 MODULE_LICENSE("GPL");
1031 MODULE_AUTHOR("ARM Ltd");
1032 MODULE_DESCRIPTION("Gator system profiler");