36e951ba915817309bd8b7599f59a27d451def8c
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 7;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <asm/stacktrace.h>
22 #include <asm/uaccess.h>
24 #include "gator.h"
25 #include "gator_events.h"
27 #ifndef CONFIG_GENERIC_TRACER
28 #ifndef CONFIG_TRACING
29 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
30 #endif
31 #endif
33 #ifndef CONFIG_PROFILING
34 #error gator requires the kernel to have CONFIG_PROFILING defined
35 #endif
37 #ifndef CONFIG_HIGH_RES_TIMERS
38 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
39 #endif
41 #if defined (__arm__)
42 #ifdef CONFIG_SMP
43 #ifndef CONFIG_LOCAL_TIMERS
44 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
45 #endif
46 #endif
47 #endif
49 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
50 #error kernels prior to 2.6.32 are not supported
51 #endif
53 /******************************************************************************
54  * DEFINES
55  ******************************************************************************/
56 #define TIMER_BUFFER_SIZE_DEFAULT       (512*1024)
57 #define EVENT_BUFFER_SIZE_DEFAULT       (128*1024)
59 #define NO_COOKIE                               0UL
60 #define INVALID_COOKIE                  ~0UL
62 #define FRAME_HRTIMER           1
63 #define FRAME_EVENT                     2
64 #define FRAME_ANNOTATE          3
66 #define MESSAGE_COOKIE                          1
67 #define MESSAGE_COUNTERS                        3
68 #define MESSAGE_START_BACKTRACE         5
69 #define MESSAGE_END_BACKTRACE           7
70 #define MESSAGE_SCHEDULER_TRACE         9
71 #define MESSAGE_PID_NAME                        11
73 #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
75 #if defined(__arm__)
76 #define PC_REG regs->ARM_pc
77 #else
78 #define PC_REG regs->ip
79 #endif
81 enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
83 /******************************************************************************
84  * Globals
85  ******************************************************************************/
86 static unsigned long gator_cpu_cores;
87 static unsigned long userspace_buffer_size;
88 static unsigned long gator_backtrace_depth;
90 static unsigned long gator_started;
91 static unsigned long gator_buffer_opened;
92 static unsigned long gator_timer_count;
93 static unsigned long gator_streaming;
94 static DEFINE_MUTEX(start_mutex);
95 static DEFINE_MUTEX(gator_buffer_mutex);
97 unsigned long gator_net_traffic;
98 bool event_based_sampling;
100 #define COMMIT_SIZE             128
101 #define COMMIT_MASK             (COMMIT_SIZE-1)
102 static DEFINE_SPINLOCK(timer_commit_lock);
103 static int *gator_commit[NUM_GATOR_BUFS];
104 static int gator_commit_read[NUM_GATOR_BUFS];
105 static int gator_commit_write[NUM_GATOR_BUFS];
107 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
108 static DEFINE_PER_CPU(int, gator_first_time);
110 #if LINUX_PMU_SUPPORT
111 static void event_buffer_check(int cpu);
112 static DEFINE_SPINLOCK(event_commit_lock);
113 #endif
115 /******************************************************************************
116  * Prototypes
117  ******************************************************************************/
118 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
119 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
120 static void gator_buffer_write_string(int cpu, int buftype, char *x);
121 static int  gator_write_packed_int(char *buffer, unsigned int x);
122 static int  gator_write_packed_int64(char *buffer, unsigned long long x);
123 static void gator_add_trace(int cpu, int buftype, unsigned int address);
124 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
125 static uint64_t gator_get_time(void);
127 /******************************************************************************
128  * Application Includes
129  ******************************************************************************/
130 #include "gator_cookies.c"
131 #include "gator_trace_sched.c"
132 #include "gator_backtrace.c"
133 #include "gator_annotate.c"
134 #include "gator_fs.c"
135 #include "gator_ebs.c"
137 /******************************************************************************
138  * Misc
139  ******************************************************************************/
140 #if defined(__arm__)
141 u32 gator_cpuid(void)
143         u32 val;
144         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
145         return (val >> 4) & 0xfff;
147 #endif
149 /******************************************************************************
150  * Commit interface
151  ******************************************************************************/
152 static int buffer_commit_ready(int buftype)
154         return gator_commit_read[buftype] != gator_commit_write[buftype];
157 static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
159         int read = gator_commit_read[buftype];
160         *cpu      = gator_commit[buftype][read+0];
161         *readval  = gator_commit[buftype][read+1];
162         *writeval = gator_commit[buftype][read+2];
163         gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
166 static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
167         int write = gator_commit_write[buftype];
168         gator_commit[buftype][write+0] = cpu;
169         gator_commit[buftype][write+1] = readval;
170         gator_commit[buftype][write+2] = writeval;
171         gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
174 /******************************************************************************
175  * Buffer management
176  ******************************************************************************/
177 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
178 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
179 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
180 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
181 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
182 #include "gator_pack.c"
184 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
186         int i;
187         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
188         u32 mask = gator_buffer_mask[buftype];
189         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
191         for (i = 0; i < len; i++) {
192                 buffer[write] = x[i];
193                 write = (write + 1) & mask;
194         }
196         per_cpu(gator_buffer_write, cpu)[buftype] = write;
199 static void gator_buffer_write_string(int cpu, int buftype, char *x)
201         int len = strlen(x);
202         gator_buffer_write_packed_int(cpu, buftype, len);
203         gator_buffer_write_bytes(cpu, buftype, x, len);
206 static void gator_buffer_header(int cpu, int buftype)
208         int frame;
210         if (buftype == TIMER_BUF)
211                 frame = FRAME_HRTIMER;
212         else if (buftype == EVENT_BUF)
213                 frame = FRAME_EVENT;
214         else
215                 frame = -1;
217         gator_buffer_write_packed_int(cpu, buftype, frame);
218         gator_buffer_write_packed_int(cpu, buftype, cpu);
221 static void gator_buffer_commit(int cpu, int buftype)
223         buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
224         per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
225         gator_buffer_header(cpu, buftype);
226         wake_up(&gator_buffer_wait);
229 static void timer_buffer_check(int cpu)
231         int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
232         if (available < 0) {
233                 available += gator_buffer_size[TIMER_BUF];
234         }
235         if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
236                 spin_lock(&timer_commit_lock);
237                 gator_buffer_commit(cpu, TIMER_BUF);
238                 spin_unlock(&timer_commit_lock);
239         }
242 #if LINUX_PMU_SUPPORT
243 static void event_buffer_check(int cpu)
245         int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
246         if (available < 0) {
247                 available += gator_buffer_size[EVENT_BUF];
248         }
249         if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
250                 spin_lock(&event_commit_lock);
251                 gator_buffer_commit(cpu, EVENT_BUF);
252                 spin_unlock(&event_commit_lock);
253         }
255 #endif
257 static void gator_add_trace(int cpu, int buftype, unsigned int address)
259         off_t offset = 0;
260         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
262         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
263                 offset = address;
264         }
266         gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
267         gator_buffer_write_packed_int(cpu, buftype, cookie);
270 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
272         int inKernel = regs ? !user_mode(regs) : 1;
273         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
275         if (!regs)
276                 return;
278         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
279         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
280         gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
281         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); 
282         gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
283         gator_buffer_write_packed_int(cpu, buftype, inKernel);
285         if (inKernel) {
286                 kernel_backtrace(cpu, buftype, regs);
287         } else {
288                 // Cookie+PC
289                 gator_add_trace(cpu, buftype, PC_REG);
291                 // Backtrace
292                 if (gator_backtrace_depth)
293                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
294         }
296         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
299 /******************************************************************************
300  * hrtimer interrupt processing
301  ******************************************************************************/
302 static LIST_HEAD(gator_events);
304 static void gator_timer_interrupt(void)
306         struct pt_regs * const regs = get_irq_regs();
307         int cpu = smp_processor_id();
308         int *buffer, len, i, buftype = TIMER_BUF;
309         long long *buffer64;
310         struct gator_interface *gi;
312         // check full backtrace has enough space, otherwise may
313         // have breaks between samples in the same callstack
314         if (per_cpu(gator_first_time, cpu)) {
315                 per_cpu(gator_first_time, cpu) = 0;
317                 list_for_each_entry(gi, &gator_events, list)
318                         if (gi->read)
319                                 gi->read(NULL);
321                 return;
322         }
324         // Output scheduler
325         len = gator_trace_sched_read(&buffer64);
326         if (len > 0) {
327                 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
328                 gator_buffer_write_packed_int(cpu, buftype, len);
329                 for (i = 0; i < len; i++) {
330                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
331                 }
332         }
334         // Output counters
335         gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
336         gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
337         list_for_each_entry(gi, &gator_events, list) {
338                 if (gi->read) {
339                         len = gi->read(&buffer);
340                         if (len > 0) {
341                                 gator_buffer_write_packed_int(cpu, buftype, len);
342                                 for (i = 0; i < len; i++) {
343                                         gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
344                                 }
345                         }
346                 } else if (gi->read64) {
347                         len = gi->read64(&buffer64);
348                         if (len > 0) {
349                                 gator_buffer_write_packed_int(cpu, buftype, len);
350                                 for (i = 0; i < len; i++) {
351                                         gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
352                                 }
353                         }
354                 }
355         }
356         gator_buffer_write_packed_int(cpu, buftype, 0);
358         // Output backtrace
359         if (!event_based_sampling) {
360                 gator_add_sample(cpu, buftype, regs);
361         }
363         // Check and commit; generally, commit is set to occur once per second
364         timer_buffer_check(cpu);
367 DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
368 DEFINE_PER_CPU(int, hrtimer_is_active);
369 static int hrtimer_running;
370 static ktime_t profiling_interval;
372 static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
374         hrtimer_forward_now(hrtimer, profiling_interval);
375         gator_timer_interrupt();
376         return HRTIMER_RESTART;
379 static int gator_timer_init(void)
381         return 0;
384 static void __gator_timer_offline(void *unused)
386         int cpu = smp_processor_id();
387         if (per_cpu(hrtimer_is_active, cpu)) {
388                 struct gator_interface *gi;
389                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
390                 hrtimer_cancel(hrtimer);
391                 per_cpu(hrtimer_is_active, cpu) = 0;
392                 gator_buffer_commit(cpu, TIMER_BUF);
393                 if (event_based_sampling)
394                         gator_buffer_commit(cpu, EVENT_BUF);
396                 // offline any events
397                 list_for_each_entry(gi, &gator_events, list)
398                         if (gi->offline)
399                                 gi->offline();
400         }
403 static void gator_timer_offline(void)
405         if (hrtimer_running) {
406                 hrtimer_running = 0;
408                 on_each_cpu(__gator_timer_offline, NULL, 1);
409         }
412 static void __gator_timer_online(void *unused)
414         int cpu = smp_processor_id();
415         if (!per_cpu(hrtimer_is_active, cpu)) {
416                 struct gator_interface *gi;
417                 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
418                 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
419                 hrtimer->function = gator_hrtimer_notify;
420                 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
421                 per_cpu(gator_first_time, cpu) = 1;
422                 per_cpu(hrtimer_is_active, cpu) = 1;
424                 // online any events
425                 list_for_each_entry(gi, &gator_events, list)
426                         if (gi->online)
427                                 gi->online();
428         }
431 int gator_timer_online(unsigned long setup)
433         if (!setup) {
434                 pr_err("gator: cannot start due to a system tick value of zero\n");
435                 return -1;
436         } else if (hrtimer_running) {
437                 pr_notice("gator: high res timer already running\n");
438                 return 0;
439         }
441         hrtimer_running = 1;
443         // calculate profiling interval
444         profiling_interval = ns_to_ktime(1000000000UL / setup);
446         // timer interrupt
447         on_each_cpu(__gator_timer_online, NULL, 1);
449         return 0;
452 static uint64_t gator_get_time(void)
454         struct timespec ts;
455         uint64_t timestamp;
457         getnstimeofday(&ts);
458         timestamp = timespec_to_ns(&ts);
460         return timestamp;
463 /******************************************************************************
464  * cpu online and pm notifiers
465  ******************************************************************************/
466 static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
468         long cpu = (long)hcpu;
470         switch (action) {
471                 case CPU_DOWN_PREPARE:
472                 case CPU_DOWN_PREPARE_FROZEN:
473                         smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
474                         break;
475                 case CPU_ONLINE:
476                 case CPU_ONLINE_FROZEN:
477                         smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
478                         break;
479         }
481         return NOTIFY_OK;
484 static struct notifier_block __refdata gator_cpu_notifier = {
485         .notifier_call = gator_cpu_notify,
486 };
488 // n.b. calling "on_each_cpu" only runs on those that are online
489 // Registered linux events are not disabled, so their counters will continue to collect
490 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
492         switch (event) {
493                 case PM_HIBERNATION_PREPARE:
494                 case PM_SUSPEND_PREPARE:
495                         unregister_hotcpu_notifier(&gator_cpu_notifier);
496                         unregister_scheduler_tracepoints();
497                         on_each_cpu(trace_sched_insert_idle, NULL, 1);
498                         on_each_cpu(__gator_timer_offline, NULL, 1);
499                         break;
500                 case PM_POST_HIBERNATION:
501                 case PM_POST_SUSPEND:
502                         on_each_cpu(__gator_timer_online, NULL, 1);
503                         register_scheduler_tracepoints();
504                         register_hotcpu_notifier(&gator_cpu_notifier);
505                         break;
506         }
508         return NOTIFY_OK;
511 static struct notifier_block gator_pm_notifier = {
512         .notifier_call = gator_pm_notify,
513 };
515 static int gator_notifier_start(void)
517         int retval;
518         retval = register_hotcpu_notifier(&gator_cpu_notifier);
519         if (retval == 0)
520                 retval = register_pm_notifier(&gator_pm_notifier);
521         return retval;
524 static void gator_notifier_stop(void)
526         unregister_pm_notifier(&gator_pm_notifier);
527         unregister_hotcpu_notifier(&gator_cpu_notifier);
530 /******************************************************************************
531  * Main
532  ******************************************************************************/
533 int gator_events_install(struct gator_interface *interface)
535         list_add_tail(&interface->list, &gator_events);
537         return 0;
540 int gator_events_get_key(void)
542         static int key;
544         return key++;
547 static int gator_init(void)
549         int i;
551         if (gator_timer_init())
552                 return -1;
553         if (gator_trace_sched_init())
554                 return -1;
555         if (gator_annotate_init())
556                 return -1;
558         // events sources (gator_events.h, generated by gator_events.sh)
559         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
560                 if (gator_events_list[i])
561                         gator_events_list[i]();
563         return 0;
566 static int gator_start(void)
568         struct gator_interface *gi;
570         // start all events
571         list_for_each_entry(gi, &gator_events, list) {
572                 if (gi->start && gi->start() != 0) {
573                         struct list_head *ptr = gi->list.prev;
575                         while (ptr != &gator_events) {
576                                 gi = list_entry(ptr, struct gator_interface,
577                                                 list);
579                                 if (gi->stop)
580                                         gi->stop();
582                                 ptr = ptr->prev;
583                         }
584                         goto events_failure;
585                 }
586         }
588         // cookies shall be initialized before trace_sched_start() and gator_timer_online()
589         if (cookies_initialize())
590                 goto cookies_failure;
591         if (gator_annotate_start())
592                 goto annotate_failure;
593         if (gator_trace_sched_start())
594                 goto sched_failure;
595         if (gator_event_sampling_start())
596                 goto event_sampling_failure;
597         if (gator_timer_online(gator_timer_count))
598                 goto timer_failure;
599         if (gator_notifier_start())
600                 goto notifier_failure;
602         return 0;
604 notifier_failure:
605         gator_timer_offline();
606 timer_failure:
607         gator_event_sampling_stop();
608 event_sampling_failure:
609         gator_trace_sched_stop();
610 sched_failure:
611         gator_annotate_stop();
612 annotate_failure:
613         cookies_release();
614 cookies_failure:
615         // stop all events
616         list_for_each_entry(gi, &gator_events, list)
617                 if (gi->stop)
618                         gi->stop();
619 events_failure:
621         return -1;
624 static void gator_stop(void)
626         struct gator_interface *gi;
628         // stop all events
629         list_for_each_entry(gi, &gator_events, list)
630                 if (gi->stop)
631                         gi->stop();
633         gator_annotate_stop();
634         gator_trace_sched_stop();
635         gator_event_sampling_stop();
637         // stop all interrupt callback reads before tearing down other interfaces
638         gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
639         gator_timer_offline();
642 static void gator_exit(void)
644         gator_annotate_exit();
647 /******************************************************************************
648  * Filesystem
649  ******************************************************************************/
650 /* fopen("buffer") */
651 static int gator_op_setup(void)
653         int err = 0;
654         int cpu, i;
656         mutex_lock(&start_mutex);
658         gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
659         gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
661         // must be a power of 2
662         if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
663                 err = -ENOEXEC;
664                 goto setup_error;
665         }
667         gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
668         gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
670         gator_net_traffic = 0;
672         // Initialize per buffer variables
673         for (i = 0; i < NUM_GATOR_BUFS; i++) {
674                 gator_commit_read[i] = gator_commit_write[i] = 0;
675                 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
676                 if (!gator_commit[i]) {
677                         err = -ENOMEM;
678                         goto setup_error;
679                 }
681                 // Initialize percpu per buffer variables
682                 for_each_present_cpu(cpu) {
683                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
684                         if (!per_cpu(gator_buffer, cpu)[i]) {
685                                 err = -ENOMEM;
686                                 goto setup_error;
687                         }
689                         per_cpu(gator_buffer_read, cpu)[i] = 0;
690                         per_cpu(gator_buffer_write, cpu)[i] = 0;
691                         gator_buffer_header(cpu, i);
692                 }
693         }
695 setup_error:
696         mutex_unlock(&start_mutex);
697         return err;
700 /* Actually start profiling (echo 1>/dev/gator/enable) */
701 static int gator_op_start(void)
703         int err = 0;
705         mutex_lock(&start_mutex);
707         if (gator_started || gator_start())
708                 err = -EINVAL;
709         else
710                 gator_started = 1;
712         mutex_unlock(&start_mutex);
714         return err;
717 /* echo 0>/dev/gator/enable */
718 static void gator_op_stop(void)
720         mutex_lock(&start_mutex);
722         if (gator_started) {
723                 gator_stop();
725                 mutex_lock(&gator_buffer_mutex);
727                 gator_started = 0;
728                 cookies_release();
729                 wake_up(&gator_buffer_wait);
731                 mutex_unlock(&gator_buffer_mutex);
732         }
734         mutex_unlock(&start_mutex);
737 static void gator_shutdown(void)
739         int cpu, i;
741         mutex_lock(&start_mutex);
743         gator_annotate_shutdown();
745         for (i = 0; i < NUM_GATOR_BUFS; i++) {
746                 vfree(gator_commit[i]);
747                 gator_commit[i] = NULL;
748         }
750         for_each_present_cpu(cpu) {
751                 mutex_lock(&gator_buffer_mutex);
752                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
753                         vfree(per_cpu(gator_buffer, cpu)[i]);
754                         per_cpu(gator_buffer, cpu)[i] = NULL;
755                         per_cpu(gator_buffer_read, cpu)[i] = 0;
756                         per_cpu(gator_buffer_write, cpu)[i] = 0;
757                 }
758                 mutex_unlock(&gator_buffer_mutex);
759         }
761         mutex_unlock(&start_mutex);
764 static int gator_set_backtrace(unsigned long val)
766         int err = 0;
768         mutex_lock(&start_mutex);
770         if (gator_started)
771                 err = -EBUSY;
772         else
773                 gator_backtrace_depth = val;
775         mutex_unlock(&start_mutex);
777         return err;
780 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
782         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
785 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
787         unsigned long val;
788         int retval;
790         if (*offset)
791                 return -EINVAL;
793         retval = gatorfs_ulong_from_user(&val, buf, count);
794         if (retval)
795                 return retval;
797         if (val)
798                 retval = gator_op_start();
799         else
800                 gator_op_stop();
802         if (retval)
803                 return retval;
804         return count;
807 static const struct file_operations enable_fops = {
808         .read           = enable_read,
809         .write          = enable_write,
810 };
812 static int userspace_buffer_open(struct inode *inode, struct file *file)
814         int err = -EPERM;
816         if (!capable(CAP_SYS_ADMIN))
817                 return -EPERM;
819         if (test_and_set_bit_lock(0, &gator_buffer_opened))
820                 return -EBUSY;
822         if ((err = gator_op_setup()))
823                 goto fail;
825         /* NB: the actual start happens from userspace
826          * echo 1 >/dev/gator/enable
827          */
829         return 0;
831 fail:
832         __clear_bit_unlock(0, &gator_buffer_opened);
833         return err;
836 static int userspace_buffer_release(struct inode *inode, struct file *file)
838         gator_op_stop();
839         gator_shutdown();
840         __clear_bit_unlock(0, &gator_buffer_opened);
841         return 0;
844 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
845                                  size_t count, loff_t *offset)
847         int retval = -EINVAL;
848         int commit, length1, length2, read;
849         char *buffer1;
850         char *buffer2 = NULL;
851         int cpu, i;
853         /* do not handle partial reads */
854         if (count != userspace_buffer_size || *offset)
855                 return -EINVAL;
857         // sleep until the condition is true or a signal is received
858         // the condition is checked each time gator_buffer_wait is woken up
859         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
861         if (signal_pending(current))
862                 return -EINTR;
864         length2 = 0;
865         retval = -EFAULT;
867         mutex_lock(&gator_buffer_mutex);
869         i = -1;
870         if (buffer_commit_ready(TIMER_BUF)) {
871                 i = TIMER_BUF;
872         } else if (buffer_commit_ready(EVENT_BUF)) {
873                 i = EVENT_BUF;
874         }
876         if (i != -1) {
877                 buffer_commit_read(&cpu, i, &read, &commit);
879                 /* May happen if the buffer is freed during pending reads. */
880                 if (!per_cpu(gator_buffer, cpu)[i]) {
881                         retval = -EFAULT;
882                         goto out;
883                 }
885                 /* determine the size of two halves */
886                 length1 = commit - read;
887                 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
888                 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
889                 if (length1 < 0) {
890                         length1 = gator_buffer_size[i] - read;
891                         length2 = commit;
892                 }
893         } else if (gator_annotate_ready()) {
894                 length1 = gator_annotate_read(&buffer1);
895                 if (!length1)
896                         goto out;
897         } else {
898                 retval = 0;
899                 goto out;
900         }
902         /* start, middle or end */
903         if (length1 > 0) {
904                 if (copy_to_user(&buf[0], buffer1, length1)) {
905                         goto out;
906                 }
907         }
909         /* possible wrap around */
910         if (length2 > 0) {
911                 if (copy_to_user(&buf[length1], buffer2, length2)) {
912                         goto out;
913                 }
914         }
916         retval = length1 + length2;
918         /* kick just in case we've lost an SMP event */
919         wake_up(&gator_buffer_wait);
921 out:
922         // only adjust network stats if in streaming mode
923         if (gator_streaming)
924                 gator_net_traffic += retval;
925         mutex_unlock(&gator_buffer_mutex);
926         return retval;
929 const struct file_operations gator_event_buffer_fops = {
930         .open           = userspace_buffer_open,
931         .release        = userspace_buffer_release,
932         .read           = userspace_buffer_read,
933 };
935 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
937         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
938                                         offset);
941 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
943         unsigned long val;
944         int retval;
946         if (*offset)
947                 return -EINVAL;
949         retval = gatorfs_ulong_from_user(&val, buf, count);
950         if (retval)
951                 return retval;
953         retval = gator_set_backtrace(val);
955         if (retval)
956                 return retval;
957         return count;
960 static const struct file_operations depth_fops = {
961         .read           = depth_read,
962         .write          = depth_write
963 };
965 void gator_op_create_files(struct super_block *sb, struct dentry *root)
967         struct dentry *dir;
968         struct gator_interface *gi;
969         int cpu;
971         /* reinitialize default values */
972         gator_cpu_cores = 0;
973         for_each_present_cpu(cpu) {
974                 gator_cpu_cores++;
975         }
976         userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
977         gator_streaming = 1;
979         gatorfs_create_file(sb, root, "enable", &enable_fops);
980         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
981         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
982         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
983         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
984         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
985         gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
986         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
988         // Annotate interface
989         gator_annotate_create_files(sb, root);
991         // Linux Events
992         dir = gatorfs_mkdir(sb, root, "events");
993         list_for_each_entry(gi, &gator_events, list)
994                 if (gi->create_files)
995                         gi->create_files(sb, dir);
998 /******************************************************************************
999  * Module
1000  ******************************************************************************/
1001 static int __init gator_module_init(void)
1003         if (gatorfs_register()) {
1004                 return -1;
1005         }
1007         if (gator_init()) {
1008                 gatorfs_unregister();
1009                 return -1;
1010         }
1012         return 0;
1015 static void __exit gator_module_exit(void)
1017         tracepoint_synchronize_unregister();
1018         gatorfs_unregister();
1019         gator_exit();
1022 module_init(gator_module_init);
1023 module_exit(gator_module_exit);
1025 MODULE_LICENSE("GPL");
1026 MODULE_AUTHOR("ARM Ltd");
1027 MODULE_DESCRIPTION("Gator system profiler");