gator-driver: Disable event-base sampling support
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 7;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <asm/stacktrace.h>
22 #include <asm/uaccess.h>
24 #include "gator.h"
25 #include "gator_events.h"
27 #ifndef CONFIG_GENERIC_TRACER
28 #ifndef CONFIG_TRACING
29 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
30 #endif
31 #endif
33 #ifndef CONFIG_PROFILING
34 #error gator requires the kernel to have CONFIG_PROFILING defined
35 #endif
37 #ifndef CONFIG_HIGH_RES_TIMERS
38 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
39 #endif
41 #if defined (__arm__)
42 #ifdef CONFIG_SMP
43 #ifndef CONFIG_LOCAL_TIMERS
44 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
45 #endif
46 #endif
47 #endif
49 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
50 #error kernels prior to 2.6.32 are not supported
51 #endif
53 /******************************************************************************
54  * DEFINES
55  ******************************************************************************/
56 #define TIMER_BUFFER_SIZE_DEFAULT       (512*1024)
57 #define EVENT_BUFFER_SIZE_DEFAULT       (128*1024)
59 #define NO_COOKIE                               0UL
60 #define INVALID_COOKIE                  ~0UL
62 #define FRAME_HRTIMER           1
63 #define FRAME_EVENT                     2
64 #define FRAME_ANNOTATE          3
66 #define MESSAGE_COOKIE                          1
67 #define MESSAGE_COUNTERS                        3
68 #define MESSAGE_START_BACKTRACE         5
69 #define MESSAGE_END_BACKTRACE           7
70 #define MESSAGE_SCHEDULER_TRACE         9
71 #define MESSAGE_PID_NAME                        11
73 #if 0 /* Disabled because of changes in Linaro 3.1 kernel */
74 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
75 #else
76 #define LINUX_PMU_SUPPORT 0
77 #endif
79 #if defined(__arm__)
80 #define PC_REG regs->ARM_pc
81 #else
82 #define PC_REG regs->ip
83 #endif
85 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
87 /******************************************************************************
88  * Globals
89  ******************************************************************************/
90 static unsigned long gator_cpu_cores;
91 static unsigned long userspace_buffer_size;
92 static unsigned long gator_backtrace_depth;
94 static unsigned long gator_started;
95 static unsigned long gator_buffer_opened;
96 static unsigned long gator_timer_count;
97 static unsigned long gator_streaming;
98 static DEFINE_MUTEX(start_mutex);
99 static DEFINE_MUTEX(gator_buffer_mutex);
101 unsigned long gator_net_traffic;
102 bool event_based_sampling;
104 #define COMMIT_SIZE             128
105 #define COMMIT_MASK             (COMMIT_SIZE-1)
106 static DEFINE_SPINLOCK(timer_commit_lock);
107 static int *gator_commit[NUM_GATOR_BUFS];
108 static int gator_commit_read[NUM_GATOR_BUFS];
109 static int gator_commit_write[NUM_GATOR_BUFS];
111 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
112 static DEFINE_PER_CPU(int, gator_first_time);
114 #if LINUX_PMU_SUPPORT
115 static void event_buffer_check(int cpu);
116 static DEFINE_SPINLOCK(event_commit_lock);
117 #endif
119 /******************************************************************************
120  * Prototypes
121  ******************************************************************************/
122 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
123 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
124 static void gator_buffer_write_string(int cpu, int buftype, char *x);
125 static int  gator_write_packed_int(char *buffer, unsigned int x);
126 static int  gator_write_packed_int64(char *buffer, unsigned long long x);
127 static void gator_add_trace(int cpu, int buftype, unsigned int address);
128 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
129 static uint64_t gator_get_time(void);
131 /******************************************************************************
132  * Application Includes
133  ******************************************************************************/
134 #include "gator_cookies.c"
135 #include "gator_trace_sched.c"
136 #include "gator_backtrace.c"
137 #include "gator_annotate.c"
138 #include "gator_fs.c"
139 #include "gator_ebs.c"
141 /******************************************************************************
142  * Misc
143  ******************************************************************************/
144 #if defined(__arm__)
145 u32 gator_cpuid(void)
147         u32 val;
148         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
149         return (val >> 4) & 0xfff;
151 #endif
153 /******************************************************************************
154  * Commit interface
155  ******************************************************************************/
156 static int buffer_commit_ready(int buftype)
158         return gator_commit_read[buftype] != gator_commit_write[buftype];
161 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
163         int read = gator_commit_read[buftype];
164         *cpu      = gator_commit[buftype][read+0];
165         *readval  = gator_commit[buftype][read+1];
166         *writeval = gator_commit[buftype][read+2];
167         gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
170 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
171         int write = gator_commit_write[buftype];
172         gator_commit[buftype][write+0] = cpu;
173         gator_commit[buftype][write+1] = readval;
174         gator_commit[buftype][write+2] = writeval;
175         gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
178 /******************************************************************************
179  * Buffer management
180  ******************************************************************************/
181 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
182 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
183 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
184 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
185 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
186 #include "gator_pack.c"
188 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
190         int i;
191         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
192         u32 mask = gator_buffer_mask[buftype];
193         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
195         for (i = 0; i < len; i++) {
196                 buffer[write] = x[i];
197                 write = (write + 1) & mask;
198         }
200         per_cpu(gator_buffer_write, cpu)[buftype] = write;
203 static void gator_buffer_write_string(int cpu, int buftype, char *x)
205         int len = strlen(x);
206         gator_buffer_write_packed_int(cpu, buftype, len);
207         gator_buffer_write_bytes(cpu, buftype, x, len);
210 static void gator_buffer_header(int cpu, int buftype)
212         int frame;
214         if (buftype == TIMER_BUF)
215                 frame = FRAME_HRTIMER;
216         else if (buftype == EVENT_BUF)
217                 frame = FRAME_EVENT;
218         else
219                 frame = -1;
221         gator_buffer_write_packed_int(cpu, buftype, frame);
222         gator_buffer_write_packed_int(cpu, buftype, cpu);
225 static void gator_buffer_commit(int cpu, int buftype)
227         buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
228         per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
229         gator_buffer_header(cpu, buftype);
230         wake_up(&gator_buffer_wait);
233 static void timer_buffer_check(int cpu)
235         int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
236         if (available < 0) {
237                 available += gator_buffer_size[TIMER_BUF];
238         }
239         if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
240                 spin_lock(&timer_commit_lock);
241                 gator_buffer_commit(cpu, TIMER_BUF);
242                 spin_unlock(&timer_commit_lock);
243         }
246 #if LINUX_PMU_SUPPORT
247 static void event_buffer_check(int cpu)
249         int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
250         if (available < 0) {
251                 available += gator_buffer_size[EVENT_BUF];
252         }
253         if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
254                 spin_lock(&event_commit_lock);
255                 gator_buffer_commit(cpu, EVENT_BUF);
256                 spin_unlock(&event_commit_lock);
257         }
259 #endif
261 static void gator_add_trace(int cpu, int buftype, unsigned int address)
263         off_t offset = 0;
264         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
266         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
267                 offset = address;
268         }
270         gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
271         gator_buffer_write_packed_int(cpu, buftype, cookie);
274 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
276         int inKernel = regs ? !user_mode(regs) : 1;
277         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
279         if (!regs)
280                 return;
282         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
283         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
284         gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
285         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); 
286         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
287         gator_buffer_write_packed_int(cpu, buftype, inKernel);
289         if (inKernel) {
290                 kernel_backtrace(cpu, buftype, regs);
291         } else {
292                 // Cookie+PC
293                 gator_add_trace(cpu, buftype, PC_REG);
295                 // Backtrace
296                 if (gator_backtrace_depth)
297                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
298         }
300         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
303 /******************************************************************************
304  * hrtimer interrupt processing
305  ******************************************************************************/
306 static LIST_HEAD(gator_events);
308 static void gator_timer_interrupt(void)
310         struct pt_regs * const regs = get_irq_regs();
311         int cpu = smp_processor_id();
312         int *buffer, len, i, buftype = TIMER_BUF;
313         long long *buffer64;
314         struct gator_interface *gi;
316         // check full backtrace has enough space, otherwise may
317         // have breaks between samples in the same callstack
318         if (per_cpu(gator_first_time, cpu)) {
319                 per_cpu(gator_first_time, cpu) = 0;
321                 list_for_each_entry(gi, &gator_events, list)
322                         if (gi->read)
323                                 gi->read(NULL);
325                 return;
326         }
328         // Output scheduler
329         len = gator_trace_sched_read(&buffer64);
330         if (len > 0) {
331                 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
332                 gator_buffer_write_packed_int(cpu, buftype, len);
333                 for (i = 0; i < len; i++) {
334                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
335                 }
336         }
338         // Output counters
339         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
340         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
341         list_for_each_entry(gi, &gator_events, list) {
342                 if (gi->read) {
343                         len = gi->read(&buffer);
344                         if (len > 0) {
345                                 gator_buffer_write_packed_int(cpu, buftype, len);
346                                 for (i = 0; i < len; i++) {
347                                         gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
348                                 }
349                         }
350                 } else if (gi->read64) {
351                         len = gi->read64(&buffer64);
352                         if (len > 0) {
353                                 gator_buffer_write_packed_int(cpu, buftype, len);
354                                 for (i = 0; i < len; i++) {
355                                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
356                                 }
357                         }
358                 }
359         }
360         gator_buffer_write_packed_int(cpu, buftype, 0);
362         // Output backtrace
363         if (!event_based_sampling) {
364                 gator_add_sample(cpu, buftype, regs);
365         }
367         // Check and commit; generally, commit is set to occur once per second
368         timer_buffer_check(cpu);
371 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
372 DEFINE_PER_CPU(int, hrtimer_is_active);
373 static int hrtimer_running;
374 static ktime_t profiling_interval;
376 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
378         hrtimer_forward_now(hrtimer, profiling_interval);
379         gator_timer_interrupt();
380         return HRTIMER_RESTART;
383 static int gator_timer_init(void)
385         return 0;
388 static void __gator_timer_offline(void *unused)
390         int cpu = smp_processor_id();
391         if (per_cpu(hrtimer_is_active, cpu)) {
392                 struct gator_interface *gi;
393                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
394                 hrtimer_cancel(hrtimer);
395                 per_cpu(hrtimer_is_active, cpu) = 0;
396                 gator_buffer_commit(cpu, TIMER_BUF);
397                 if (event_based_sampling)
398                         gator_buffer_commit(cpu, EVENT_BUF);
400                 // offline any events
401                 list_for_each_entry(gi, &gator_events, list)
402                         if (gi->offline)
403                                 gi->offline();
404         }
407 static void gator_timer_offline(void)
409         if (hrtimer_running) {
410                 hrtimer_running = 0;
412                 on_each_cpu(__gator_timer_offline, NULL, 1);
413         }
416 static void __gator_timer_online(void *unused)
418         int cpu = smp_processor_id();
419         if (!per_cpu(hrtimer_is_active, cpu)) {
420                 struct gator_interface *gi;
421                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
422                 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
423                 hrtimer->function = gator_hrtimer_notify;
424                 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
425                 per_cpu(gator_first_time, cpu) = 1;
426                 per_cpu(hrtimer_is_active, cpu) = 1;
428                 // online any events
429                 list_for_each_entry(gi, &gator_events, list)
430                         if (gi->online)
431                                 gi->online();
432         }
435 int gator_timer_online(unsigned long setup)
437         if (!setup) {
438                 pr_err("gator: cannot start due to a system tick value of zero\n");
439                 return -1;
440         } else if (hrtimer_running) {
441                 pr_notice("gator: high res timer already running\n");
442                 return 0;
443         }
445         hrtimer_running = 1;
447         // calculate profiling interval
448         profiling_interval = ns_to_ktime(1000000000UL / setup);
450         // timer interrupt
451         on_each_cpu(__gator_timer_online, NULL, 1);
453         return 0;
456 static uint64_t gator_get_time(void)
458         struct timespec ts;
459         uint64_t timestamp;
461         getnstimeofday(&ts);
462         timestamp = timespec_to_ns(&ts);
464         return timestamp;
467 /******************************************************************************
468  * cpu online and pm notifiers
469  ******************************************************************************/
470 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
472         long cpu = (long)hcpu;
474         switch (action) {
475                 case CPU_DOWN_PREPARE:
476                 case CPU_DOWN_PREPARE_FROZEN:
477                         smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
478                         break;
479                 case CPU_ONLINE:
480                 case CPU_ONLINE_FROZEN:
481                         smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
482                         break;
483         }
485         return NOTIFY_OK;
488 static struct notifier_block __refdata gator_cpu_notifier = {
489         .notifier_call = gator_cpu_notify,
490 };
492 // n.b. calling "on_each_cpu" only runs on those that are online
493 // Registered linux events are not disabled, so their counters will continue to collect
494 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
496         switch (event) {
497                 case PM_HIBERNATION_PREPARE:
498                 case PM_SUSPEND_PREPARE:
499                         unregister_hotcpu_notifier(&gator_cpu_notifier);
500                         unregister_scheduler_tracepoints();
501                         on_each_cpu(trace_sched_insert_idle, NULL, 1);
502                         on_each_cpu(__gator_timer_offline, NULL, 1);
503                         break;
504                 case PM_POST_HIBERNATION:
505                 case PM_POST_SUSPEND:
506                         on_each_cpu(__gator_timer_online, NULL, 1);
507                         register_scheduler_tracepoints();
508                         register_hotcpu_notifier(&gator_cpu_notifier);
509                         break;
510         }
512         return NOTIFY_OK;
515 static struct notifier_block gator_pm_notifier = {
516         .notifier_call = gator_pm_notify,
517 };
519 static int gator_notifier_start(void)
521         int retval;
522         retval = register_hotcpu_notifier(&gator_cpu_notifier);
523         if (retval == 0)
524                 retval = register_pm_notifier(&gator_pm_notifier);
525         return retval;
528 static void gator_notifier_stop(void)
530         unregister_pm_notifier(&gator_pm_notifier);
531         unregister_hotcpu_notifier(&gator_cpu_notifier);
534 /******************************************************************************
535  * Main
536  ******************************************************************************/
537 int gator_events_install(struct gator_interface *interface)
539         list_add_tail(&interface->list, &gator_events);
541         return 0;
544 int gator_events_get_key(void)
546         static int key;
548         return key++;
551 static int gator_init(void)
553         int i;
555         if (gator_timer_init())
556                 return -1;
557         if (gator_trace_sched_init())
558                 return -1;
559         if (gator_annotate_init())
560                 return -1;
562         // events sources (gator_events.h, generated by gator_events.sh)
563         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
564                 if (gator_events_list[i])
565                         gator_events_list[i]();
567         return 0;
570 static int gator_start(void)
572         struct gator_interface *gi;
574         // start all events
575         list_for_each_entry(gi, &gator_events, list) {
576                 if (gi->start && gi->start() != 0) {
577                         struct list_head *ptr = gi->list.prev;
579                         while (ptr != &gator_events) {
580                                 gi = list_entry(ptr, struct gator_interface,
581                                                 list);
583                                 if (gi->stop)
584                                         gi->stop();
586                                 ptr = ptr->prev;
587                         }
588                         goto events_failure;
589                 }
590         }
592         // cookies shall be initialized before trace_sched_start() and gator_timer_online()
593         if (cookies_initialize())
594                 goto cookies_failure;
595         if (gator_annotate_start())
596                 goto annotate_failure;
597         if (gator_trace_sched_start())
598                 goto sched_failure;
599         if (gator_event_sampling_start())
600                 goto event_sampling_failure;
601         if (gator_timer_online(gator_timer_count))
602                 goto timer_failure;
603         if (gator_notifier_start())
604                 goto notifier_failure;
606         return 0;
608 notifier_failure:
609         gator_timer_offline();
610 timer_failure:
611         gator_event_sampling_stop();
612 event_sampling_failure:
613         gator_trace_sched_stop();
614 sched_failure:
615         gator_annotate_stop();
616 annotate_failure:
617         cookies_release();
618 cookies_failure:
619         // stop all events
620         list_for_each_entry(gi, &gator_events, list)
621                 if (gi->stop)
622                         gi->stop();
623 events_failure:
625         return -1;
628 static void gator_stop(void)
630         struct gator_interface *gi;
632         // stop all events
633         list_for_each_entry(gi, &gator_events, list)
634                 if (gi->stop)
635                         gi->stop();
637         gator_annotate_stop();
638         gator_trace_sched_stop();
639         gator_event_sampling_stop();
641         // stop all interrupt callback reads before tearing down other interfaces
642         gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
643         gator_timer_offline();
646 static void gator_exit(void)
648         gator_annotate_exit();
651 /******************************************************************************
652  * Filesystem
653  ******************************************************************************/
654 /* fopen("buffer") */
655 static int gator_op_setup(void)
657         int err = 0;
658         int cpu, i;
660         mutex_lock(&start_mutex);
662         gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
663         gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
665         // must be a power of 2
666         if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
667                 err = -ENOEXEC;
668                 goto setup_error;
669         }
671         gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
672         gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
674         gator_net_traffic = 0;
676         // Initialize per buffer variables
677         for (i = 0; i < NUM_GATOR_BUFS; i++) {
678                 gator_commit_read[i] = gator_commit_write[i] = 0;
679                 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
680                 if (!gator_commit[i]) {
681                         err = -ENOMEM;
682                         goto setup_error;
683                 }
685                 // Initialize percpu per buffer variables
686                 for_each_present_cpu(cpu) {
687                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
688                         if (!per_cpu(gator_buffer, cpu)[i]) {
689                                 err = -ENOMEM;
690                                 goto setup_error;
691                         }
693                         per_cpu(gator_buffer_read, cpu)[i] = 0;
694                         per_cpu(gator_buffer_write, cpu)[i] = 0;
695                         gator_buffer_header(cpu, i);
696                 }
697         }
699 setup_error:
700         mutex_unlock(&start_mutex);
701         return err;
704 /* Actually start profiling (echo 1>/dev/gator/enable) */
705 static int gator_op_start(void)
707         int err = 0;
709         mutex_lock(&start_mutex);
711         if (gator_started || gator_start())
712                 err = -EINVAL;
713         else
714                 gator_started = 1;
716         mutex_unlock(&start_mutex);
718         return err;
721 /* echo 0>/dev/gator/enable */
722 static void gator_op_stop(void)
724         mutex_lock(&start_mutex);
726         if (gator_started) {
727                 gator_stop();
729                 mutex_lock(&gator_buffer_mutex);
731                 gator_started = 0;
732                 cookies_release();
733                 wake_up(&gator_buffer_wait);
735                 mutex_unlock(&gator_buffer_mutex);
736         }
738         mutex_unlock(&start_mutex);
741 static void gator_shutdown(void)
743         int cpu, i;
745         mutex_lock(&start_mutex);
747         gator_annotate_shutdown();
749         for (i = 0; i < NUM_GATOR_BUFS; i++) {
750                 vfree(gator_commit[i]);
751                 gator_commit[i] = NULL;
752         }
754         for_each_present_cpu(cpu) {
755                 mutex_lock(&gator_buffer_mutex);
756                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
757                         vfree(per_cpu(gator_buffer, cpu)[i]);
758                         per_cpu(gator_buffer, cpu)[i] = NULL;
759                         per_cpu(gator_buffer_read, cpu)[i] = 0;
760                         per_cpu(gator_buffer_write, cpu)[i] = 0;
761                 }
762                 mutex_unlock(&gator_buffer_mutex);
763         }
765         mutex_unlock(&start_mutex);
768 static int gator_set_backtrace(unsigned long val)
770         int err = 0;
772         mutex_lock(&start_mutex);
774         if (gator_started)
775                 err = -EBUSY;
776         else
777                 gator_backtrace_depth = val;
779         mutex_unlock(&start_mutex);
781         return err;
784 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
786         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
789 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
791         unsigned long val;
792         int retval;
794         if (*offset)
795                 return -EINVAL;
797         retval = gatorfs_ulong_from_user(&val, buf, count);
798         if (retval)
799                 return retval;
801         if (val)
802                 retval = gator_op_start();
803         else
804                 gator_op_stop();
806         if (retval)
807                 return retval;
808         return count;
811 static const struct file_operations enable_fops = {
812         .read           = enable_read,
813         .write          = enable_write,
814 };
816 static int userspace_buffer_open(struct inode *inode, struct file *file)
818         int err = -EPERM;
820         if (!capable(CAP_SYS_ADMIN))
821                 return -EPERM;
823         if (test_and_set_bit_lock(0, &gator_buffer_opened))
824                 return -EBUSY;
826         if ((err = gator_op_setup()))
827                 goto fail;
829         /* NB: the actual start happens from userspace
830          * echo 1 >/dev/gator/enable
831          */
833         return 0;
835 fail:
836         __clear_bit_unlock(0, &gator_buffer_opened);
837         return err;
840 static int userspace_buffer_release(struct inode *inode, struct file *file)
842         gator_op_stop();
843         gator_shutdown();
844         __clear_bit_unlock(0, &gator_buffer_opened);
845         return 0;
848 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
849                                  size_t count, loff_t *offset)
851         int retval = -EINVAL;
852         int commit, length1, length2, read;
853         char *buffer1;
854         char *buffer2 = NULL;
855         int cpu, i;
857         /* do not handle partial reads */
858         if (count != userspace_buffer_size || *offset)
859                 return -EINVAL;
861         // sleep until the condition is true or a signal is received
862         // the condition is checked each time gator_buffer_wait is woken up
863         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
865         if (signal_pending(current))
866                 return -EINTR;
868         length2 = 0;
869         retval = -EFAULT;
871         mutex_lock(&gator_buffer_mutex);
873         i = -1;
874         if (buffer_commit_ready(TIMER_BUF)) {
875                 i = TIMER_BUF;
876         } else if (buffer_commit_ready(EVENT_BUF)) {
877                 i = EVENT_BUF;
878         }
880         if (i != -1) {
881                 buffer_commit_read(&cpu, i, &read, &commit);
883                 /* May happen if the buffer is freed during pending reads. */
884                 if (!per_cpu(gator_buffer, cpu)[i]) {
885                         retval = -EFAULT;
886                         goto out;
887                 }
889                 /* determine the size of two halves */
890                 length1 = commit - read;
891                 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
892                 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
893                 if (length1 < 0) {
894                         length1 = gator_buffer_size[i] - read;
895                         length2 = commit;
896                 }
897         } else if (gator_annotate_ready()) {
898                 length1 = gator_annotate_read(&buffer1);
899                 if (!length1)
900                         goto out;
901         } else {
902                 retval = 0;
903                 goto out;
904         }
906         /* start, middle or end */
907         if (length1 > 0) {
908                 if (copy_to_user(&buf[0], buffer1, length1)) {
909                         goto out;
910                 }
911         }
913         /* possible wrap around */
914         if (length2 > 0) {
915                 if (copy_to_user(&buf[length1], buffer2, length2)) {
916                         goto out;
917                 }
918         }
920         retval = length1 + length2;
922         /* kick just in case we've lost an SMP event */
923         wake_up(&gator_buffer_wait);
925 out:
926         // only adjust network stats if in streaming mode
927         if (gator_streaming)
928                 gator_net_traffic += retval;
929         mutex_unlock(&gator_buffer_mutex);
930         return retval;
933 const struct file_operations gator_event_buffer_fops = {
934         .open           = userspace_buffer_open,
935         .release        = userspace_buffer_release,
936         .read           = userspace_buffer_read,
937 };
939 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
941         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
942                                         offset);
945 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
947         unsigned long val;
948         int retval;
950         if (*offset)
951                 return -EINVAL;
953         retval = gatorfs_ulong_from_user(&val, buf, count);
954         if (retval)
955                 return retval;
957         retval = gator_set_backtrace(val);
959         if (retval)
960                 return retval;
961         return count;
964 static const struct file_operations depth_fops = {
965         .read           = depth_read,
966         .write          = depth_write
967 };
969 void gator_op_create_files(struct super_block *sb, struct dentry *root)
971         struct dentry *dir;
972         struct gator_interface *gi;
973         int cpu;
975         /* reinitialize default values */
976         gator_cpu_cores = 0;
977         for_each_present_cpu(cpu) {
978                 gator_cpu_cores++;
979         }
980         userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
981         gator_streaming = 1;
983         gatorfs_create_file(sb, root, "enable", &enable_fops);
984         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
985         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
986         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
987         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
988         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
989         gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
990         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
992         // Annotate interface
993         gator_annotate_create_files(sb, root);
995         // Linux Events
996         dir = gatorfs_mkdir(sb, root, "events");
997         list_for_each_entry(gi, &gator_events, list)
998                 if (gi->create_files)
999                         gi->create_files(sb, dir);
1002 /******************************************************************************
1003  * Module
1004  ******************************************************************************/
1005 static int __init gator_module_init(void)
1007         if (gatorfs_register()) {
1008                 return -1;
1009         }
1011         if (gator_init()) {
1012                 gatorfs_unregister();
1013                 return -1;
1014         }
1016         return 0;
1019 static void __exit gator_module_exit(void)
1021         tracepoint_synchronize_unregister();
1022         gatorfs_unregister();
1023         gator_exit();
1026 module_init(gator_module_init);
1027 module_exit(gator_module_exit);
1029 MODULE_LICENSE("GPL");
1030 MODULE_AUTHOR("ARM Ltd");
1031 MODULE_DESCRIPTION("Gator system profiler");