gator-driver: Revert #error about lack of CONFIG_PERF_EVENTS
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 10;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58  * DEFINES
59  ******************************************************************************/
60 #define BACKTRACE_BUFFER_SIZE    (128*1024)
61 #define COUNTER_BUFFER_SIZE      (128*1024)
62 #define ANNOTATE_BUFFER_SIZE     (64*1024) // annotate  counters have the core as part of the data and the core value in the frame header may be discarded
63 #define SCHED_TRACE_BUFFER_SIZE  (128*1024)
64 #define GPU_TRACE_BUFFER_SIZE    (64*1024)
65 #define COUNTER2_BUFFER_SIZE     (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
66 #define WFI_BUFFER_SIZE          (32*1024) // wfi       counters have the core as part of the data and the core value in the frame header may be discarded
68 #define NO_COOKIE                               0UL
69 #define INVALID_COOKIE                  ~0UL
71 #define FRAME_BACKTRACE     1
72 #define FRAME_COUNTER       2
73 #define FRAME_ANNOTATE      3
74 #define FRAME_SCHED_TRACE   4
75 #define FRAME_GPU_TRACE     5
76 #define FRAME_COUNTER2      6
77 #define FRAME_WFI           7
79 #define MESSAGE_COOKIE              1
80 #define MESSAGE_START_BACKTRACE     5
81 #define MESSAGE_END_BACKTRACE       7
82 #define MESSAGE_SUMMARY             9
83 #define MESSAGE_PID_NAME            11
85 #define MAXSIZE_PACK32          5
86 #define MAXSIZE_PACK64          9
88 #if defined(__arm__)
89 #define PC_REG regs->ARM_pc
90 #else
91 #define PC_REG regs->ip
92 #endif
94 enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
96 /******************************************************************************
97  * Globals
98  ******************************************************************************/
99 static unsigned long gator_cpu_cores;
100 static unsigned long userspace_buffer_size;
101 static unsigned long gator_backtrace_depth;
103 static unsigned long gator_started;
104 static unsigned long gator_buffer_opened;
105 static unsigned long gator_timer_count;
106 static unsigned long gator_response_type;
107 static DEFINE_MUTEX(start_mutex);
108 static DEFINE_MUTEX(gator_buffer_mutex);
110 bool event_based_sampling;
111 #if defined(__arm__) && (GATOR_PERF_PMU_SUPPORT)
112 DEFINE_PER_CPU(struct perf_event *, pevent_ebs);
113 #endif
115 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
116 static struct timer_list gator_buffer_wake_up_timer;
117 static LIST_HEAD(gator_events);
119 /******************************************************************************
120  * Prototypes
121  ******************************************************************************/
122 static void buffer_check(int cpu, int buftype);
123 static int buffer_bytes_available(int cpu, int buftype);
124 static bool buffer_check_space(int cpu, int buftype, int bytes);
125 static int contiguous_space_available(int cpu, int bufytpe);
126 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
127 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
128 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
129 static void gator_buffer_write_string(int cpu, int buftype, char *x);
130 static void gator_add_trace(int cpu, int buftype, unsigned int address);
131 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
132 static uint64_t gator_get_time(void);
134 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
135 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
136 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
137 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
138 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
139 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
140 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
142 /******************************************************************************
143  * Application Includes
144  ******************************************************************************/
145 #include "gator_marshaling.c"
146 #include "gator_hrtimer_perf.c"
147 #include "gator_hrtimer_gator.c"
148 #include "gator_cookies.c"
149 #include "gator_trace_sched.c"
150 #include "gator_trace_power.c"
151 #include "gator_trace_gpu.c"
152 #include "gator_backtrace.c"
153 #include "gator_annotate.c"
154 #include "gator_fs.c"
155 #include "gator_ebs.c"
156 #include "gator_pack.c"
158 /******************************************************************************
159  * Misc
160  ******************************************************************************/
161 #if defined(__arm__)
162 u32 gator_cpuid(void)
164         u32 val;
165         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
166         return (val >> 4) & 0xfff;
168 #endif
170 static void gator_buffer_wake_up(unsigned long data)
172         wake_up(&gator_buffer_wait);
175 /******************************************************************************
176  * Commit interface
177  ******************************************************************************/
178 static bool buffer_commit_ready(int* cpu, int* buftype)
180         int cpu_x, x;
181         for_each_present_cpu(cpu_x) {
182                 for (x = 0; x < NUM_GATOR_BUFS; x++)
183                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
184                                 *cpu = cpu_x;
185                                 *buftype = x;
186                                 return true;
187                         }
188         }
189         return false;
192 /******************************************************************************
193  * Buffer management
194  ******************************************************************************/
195 static int buffer_bytes_available(int cpu, int buftype)
197         int remaining, filled;
199         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
200         if (filled < 0) {
201                 filled += gator_buffer_size[buftype];
202         }
204         remaining = gator_buffer_size[buftype] - filled;
206         if (per_cpu(buffer_space_available, cpu)[buftype]) {
207                 // Give some extra room; also allows space to insert the overflow error packet
208                 remaining -= 200;
209         } else {
210                 // Hysteresis, prevents multiple overflow messages
211                 remaining -= 2000;
212         }
214         return remaining;
217 static int contiguous_space_available(int cpu, int buftype)
219         int remaining = buffer_bytes_available(cpu, buftype);
220         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
221         if (remaining < contiguous)
222                 return remaining;
223         else
224                 return contiguous;
227 static bool buffer_check_space(int cpu, int buftype, int bytes)
229         int remaining = buffer_bytes_available(cpu, buftype);
231         if (remaining < bytes) {
232                 per_cpu(buffer_space_available, cpu)[buftype] = false;
233         } else {
234                 per_cpu(buffer_space_available, cpu)[buftype] = true;
235         }
237         return per_cpu(buffer_space_available, cpu)[buftype];
240 static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
242         int i;
243         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
244         u32 mask = gator_buffer_mask[buftype];
245         char* buffer = per_cpu(gator_buffer, cpu)[buftype];
247         for (i = 0; i < len; i++) {
248                 buffer[write] = x[i];
249                 write = (write + 1) & mask;
250         }
252         per_cpu(gator_buffer_write, cpu)[buftype] = write;
255 static void gator_buffer_write_string(int cpu, int buftype, char *x)
257         int len = strlen(x);
258         gator_buffer_write_packed_int(cpu, buftype, len);
259         gator_buffer_write_bytes(cpu, buftype, x, len);
262 static void gator_buffer_header(int cpu, int buftype)
264         int frame;
266         if (buftype == BACKTRACE_BUF)
267                 frame = FRAME_BACKTRACE;
268         else if (buftype == COUNTER_BUF)
269                 frame = FRAME_COUNTER;
270         else if (buftype == ANNOTATE_BUF)
271                 frame = FRAME_ANNOTATE;
272         else if (buftype == SCHED_TRACE_BUF)
273                 frame = FRAME_SCHED_TRACE;
274         else if (buftype == GPU_TRACE_BUF)
275                 frame = FRAME_GPU_TRACE;
276         else if (buftype == COUNTER2_BUF)
277                 frame = FRAME_COUNTER2;
278         else if (buftype == WFI_BUF)
279                 frame = FRAME_WFI;
280         else
281                 frame = -1;
283         if (per_cpu(gator_buffer, cpu)[buftype]) {
284                 marshal_frame(cpu, buftype, frame);
285         }
288 static void gator_commit_buffer(int cpu, int buftype)
290         if (!per_cpu(gator_buffer, cpu)[buftype])
291                 return;
293         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
294         gator_buffer_header(cpu, buftype);
296         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
297         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
300 static void buffer_check(int cpu, int buftype)
302         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
303         if (filled < 0) {
304                 filled += gator_buffer_size[buftype];
305         }
306         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
307                 gator_commit_buffer(cpu, buftype);
308         }
311 static void gator_add_trace(int cpu, int buftype, unsigned int address)
313         off_t offset = 0;
314         unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
316         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
317                 offset = address;
318         }
320         marshal_backtrace(offset & ~1, cookie);
323 static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
325         int inKernel = regs ? !user_mode(regs) : 1;
326         unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
328         if (!regs)
329                 return;
331         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
332                 return;
334         if (inKernel) {
335                 kernel_backtrace(cpu, buftype, regs);
336         } else {
337                 // Cookie+PC
338                 gator_add_trace(cpu, buftype, PC_REG);
340                 // Backtrace
341                 if (gator_backtrace_depth)
342                         arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
343         }
345         marshal_backtrace_footer();
348 /******************************************************************************
349  * hrtimer interrupt processing
350  ******************************************************************************/
351 static void gator_timer_interrupt(void)
353         struct pt_regs * const regs = get_irq_regs();
354         int cpu = smp_processor_id();
356         // Output backtrace
357         gator_add_sample(cpu, BACKTRACE_BUF, regs);
359         // Collect counters
360         collect_counters();
363 static int gator_running;
365 // This function runs in interrupt context and on the appropriate core
366 static void gator_timer_offline(void* unused)
368         struct gator_interface *gi;
369         int i, len, cpu = smp_processor_id();
370         int* buffer;
372         gator_trace_sched_offline();
373         gator_trace_power_offline();
375         gator_hrtimer_offline(cpu);
377         // Offline any events and output counters
378         if (marshal_event_header()) {
379                 list_for_each_entry(gi, &gator_events, list) {
380                         if (gi->offline) {
381                                 len = gi->offline(&buffer);
382                                 marshal_event(len, buffer);
383                         }
384                 }
385         }
387         // Flush all buffers on this core
388         for (i = 0; i < NUM_GATOR_BUFS; i++)
389                 gator_commit_buffer(cpu, i);
392 // This function runs in process context and may be running on a core other than core 'cpu'
393 static void gator_timer_offline_dispatch(int cpu)
395         struct gator_interface *gi;
397         list_for_each_entry(gi, &gator_events, list)
398                 if (gi->offline_dispatch)
399                         gi->offline_dispatch(cpu);
401         gator_event_sampling_offline_dispatch(cpu);
404 static void gator_timer_stop(void)
406         int cpu;
408         if (gator_running) {
409                 on_each_cpu(gator_timer_offline, NULL, 1);
410                 for_each_online_cpu(cpu) {
411                         gator_timer_offline_dispatch(cpu);
412                 }
414                 gator_running = 0;
415                 gator_hrtimer_shutdown();
416         }
419 // This function runs in interrupt context and on the appropriate core
420 static void gator_timer_online(void* unused)
422         struct gator_interface *gi;
423         int len, cpu = smp_processor_id();
424         int* buffer;
426         gator_trace_power_online();
428         // online any events and output counters
429         if (marshal_event_header()) {
430                 list_for_each_entry(gi, &gator_events, list) {
431                         if (gi->online) {
432                                 len = gi->online(&buffer);
433                                 marshal_event(len, buffer);
434                         }
435                 }
436         }
438         gator_hrtimer_online(cpu);
441 // This function runs in interrupt context and may be running on a core other than core 'cpu'
442 static void gator_timer_online_dispatch(int cpu)
444         struct gator_interface *gi;
446         list_for_each_entry(gi, &gator_events, list)
447                 if (gi->online_dispatch)
448                         gi->online_dispatch(cpu);
450         gator_event_sampling_online_dispatch(cpu);
453 int gator_timer_start(unsigned long sample_rate)
455         int cpu;
457         if (gator_running) {
458                 pr_notice("gator: already running\n");
459                 return 0;
460         }
462         gator_running = 1;
464         // event based sampling trumps hr timer based sampling
465         if (event_based_sampling)
466                 sample_rate = 0;
468         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
469                 return -1;
471         for_each_online_cpu(cpu) {
472                 gator_timer_online_dispatch(cpu);
473         }
474         on_each_cpu(gator_timer_online, NULL, 1);
476         return 0;
479 static uint64_t gator_get_time(void)
481         struct timespec ts;
482         uint64_t timestamp;
484         getnstimeofday(&ts);
485         timestamp = timespec_to_ns(&ts);
487         return timestamp;
490 /******************************************************************************
491  * cpu hotplug and pm notifiers
492  ******************************************************************************/
493 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
495         long cpu = (long)hcpu;
497         switch (action) {
498                 case CPU_DOWN_PREPARE:
499                 case CPU_DOWN_PREPARE_FROZEN:
500                         smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
501                         gator_timer_offline_dispatch(cpu);
502                         break;
503                 case CPU_ONLINE:
504                 case CPU_ONLINE_FROZEN:
505                         gator_timer_online_dispatch(cpu);
506                         smp_call_function_single(cpu, gator_timer_online, NULL, 1);
507                         break;
508         }
510         return NOTIFY_OK;
513 static struct notifier_block __refdata gator_hotcpu_notifier = {
514         .notifier_call = gator_hotcpu_notify,
515 };
517 // n.b. calling "on_each_cpu" only runs on those that are online
518 // Registered linux events are not disabled, so their counters will continue to collect
519 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
521         int cpu;
523         switch (event) {
524                 case PM_HIBERNATION_PREPARE:
525                 case PM_SUSPEND_PREPARE:
526                         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
527                         unregister_scheduler_tracepoints();
528                         on_each_cpu(gator_timer_offline, NULL, 1);
529                         for_each_online_cpu(cpu) {
530                                 gator_timer_offline_dispatch(cpu);
531                         }
532                         break;
533                 case PM_POST_HIBERNATION:
534                 case PM_POST_SUSPEND:
535                         for_each_online_cpu(cpu) {
536                                 gator_timer_online_dispatch(cpu);
537                         }
538                         on_each_cpu(gator_timer_online, NULL, 1);
539                         register_scheduler_tracepoints();
540                         register_hotcpu_notifier(&gator_hotcpu_notifier);
541                         break;
542         }
544         return NOTIFY_OK;
547 static struct notifier_block gator_pm_notifier = {
548         .notifier_call = gator_pm_notify,
549 };
551 static int gator_notifier_start(void)
553         int retval;
554         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
555         if (retval == 0)
556                 retval = register_pm_notifier(&gator_pm_notifier);
557         return retval;
560 static void gator_notifier_stop(void)
562         unregister_pm_notifier(&gator_pm_notifier);
563         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
566 /******************************************************************************
567  * Main
568  ******************************************************************************/
569 static void gator_summary(void)
571         uint64_t timestamp, uptime = 0;
572         struct timespec uptime_ts;
573         void (*m2b)(struct timespec *ts);
575         timestamp = gator_get_time();
577         do_posix_clock_monotonic_gettime(&uptime_ts);
578         m2b = symbol_get(monotonic_to_bootbased);
579         if (m2b) {
580                 m2b(&uptime_ts);
581                 uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
582         }
584         marshal_summary(timestamp, uptime);
587 int gator_events_install(struct gator_interface *interface)
589         list_add_tail(&interface->list, &gator_events);
591         return 0;
594 int gator_events_get_key(void)
596         // key of zero is reserved as a timestamp
597         static int key = 1;
599         return key++;
602 static int gator_init(void)
604         int i;
606         // events sources (gator_events.h, generated by gator_events.sh)
607         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
608                 if (gator_events_list[i])
609                         gator_events_list[i]();
611         gator_trace_power_init();
613         return 0;
616 static int gator_start(void)
618         unsigned long cpu, i;
619         struct gator_interface *gi;
621         // Initialize the buffer with the frame type and core
622         for_each_present_cpu(cpu) {
623                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
624                         gator_buffer_header(cpu, i);
625                 }
626         }
628         // Capture the start time  
629         gator_summary();
631         // start all events
632         list_for_each_entry(gi, &gator_events, list) {
633                 if (gi->start && gi->start() != 0) {
634                         struct list_head *ptr = gi->list.prev;
636                         while (ptr != &gator_events) {
637                                 gi = list_entry(ptr, struct gator_interface, list);
639                                 if (gi->stop)
640                                         gi->stop();
642                                 ptr = ptr->prev;
643                         }
644                         goto events_failure;
645                 }
646         }
648         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
649         if (cookies_initialize())
650                 goto cookies_failure;
651         if (gator_annotate_start())
652                 goto annotate_failure;
653         if (gator_trace_sched_start())
654                 goto sched_failure;
655         if (gator_trace_power_start())
656                 goto power_failure;
657         if (gator_trace_gpu_start())
658                 goto gpu_failure;
659         if (gator_event_sampling_start())
660                 goto event_sampling_failure;
661         if (gator_timer_start(gator_timer_count))
662                 goto timer_failure;
663         if (gator_notifier_start())
664                 goto notifier_failure;
666         return 0;
668 notifier_failure:
669         gator_timer_stop();
670 timer_failure:
671         gator_event_sampling_stop();
672 event_sampling_failure:
673         gator_trace_gpu_stop();
674 gpu_failure:
675         gator_trace_power_stop();
676 power_failure:
677         gator_trace_sched_stop();
678 sched_failure:
679         gator_annotate_stop();
680 annotate_failure:
681         cookies_release();
682 cookies_failure:
683         // stop all events
684         list_for_each_entry(gi, &gator_events, list)
685                 if (gi->stop)
686                         gi->stop();
687 events_failure:
689         return -1;
692 static void gator_stop(void)
694         struct gator_interface *gi;
696         gator_annotate_stop();
697         gator_trace_sched_stop();
698         gator_trace_power_stop();
699         gator_trace_gpu_stop();
700         gator_event_sampling_stop();
702         // stop all interrupt callback reads before tearing down other interfaces
703         gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
704         gator_timer_stop();
706         // stop all events
707         list_for_each_entry(gi, &gator_events, list)
708                 if (gi->stop)
709                         gi->stop();
712 /******************************************************************************
713  * Filesystem
714  ******************************************************************************/
715 /* fopen("buffer") */
716 static int gator_op_setup(void)
718         int err = 0;
719         int cpu, i;
721         mutex_lock(&start_mutex);
723         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
724         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
726         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
727         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
729         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
730         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
732         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
733         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
735         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
736         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
738         gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
739         gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
741         gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
742         gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
744         // Initialize percpu per buffer variables
745         for (i = 0; i < NUM_GATOR_BUFS; i++) {
746                 // Verify buffers are a power of 2
747                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
748                         err = -ENOEXEC;
749                         goto setup_error;
750                 }
752                 for_each_present_cpu(cpu) {
753                         per_cpu(gator_buffer_read, cpu)[i] = 0;
754                         per_cpu(gator_buffer_write, cpu)[i] = 0;
755                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
756                         per_cpu(buffer_space_available, cpu)[i] = true;
758                         // Annotation is a special case that only uses a single buffer
759                         if (cpu > 0 && i == ANNOTATE_BUF) {
760                                 per_cpu(gator_buffer, cpu)[i] = NULL;
761                                 continue;
762                         }
764                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
765                         if (!per_cpu(gator_buffer, cpu)[i]) {
766                                 err = -ENOMEM;
767                                 goto setup_error;
768                         }
769                 }
770         }
772 setup_error:
773         mutex_unlock(&start_mutex);
774         return err;
777 /* Actually start profiling (echo 1>/dev/gator/enable) */
778 static int gator_op_start(void)
780         int err = 0;
782         mutex_lock(&start_mutex);
784         if (gator_started || gator_start())
785                 err = -EINVAL;
786         else
787                 gator_started = 1;
789         mutex_unlock(&start_mutex);
791         return err;
794 /* echo 0>/dev/gator/enable */
795 static void gator_op_stop(void)
797         mutex_lock(&start_mutex);
799         if (gator_started) {
800                 gator_stop();
802                 mutex_lock(&gator_buffer_mutex);
804                 gator_started = 0;
805                 cookies_release();
806                 wake_up(&gator_buffer_wait);
808                 mutex_unlock(&gator_buffer_mutex);
809         }
811         mutex_unlock(&start_mutex);
814 static void gator_shutdown(void)
816         int cpu, i;
818         mutex_lock(&start_mutex);
820         for_each_present_cpu(cpu) {
821                 mutex_lock(&gator_buffer_mutex);
822                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
823                         vfree(per_cpu(gator_buffer, cpu)[i]);
824                         per_cpu(gator_buffer, cpu)[i] = NULL;
825                         per_cpu(gator_buffer_read, cpu)[i] = 0;
826                         per_cpu(gator_buffer_write, cpu)[i] = 0;
827                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
828                         per_cpu(buffer_space_available, cpu)[i] = true;
829                 }
830                 mutex_unlock(&gator_buffer_mutex);
831         }
833         mutex_unlock(&start_mutex);
836 static int gator_set_backtrace(unsigned long val)
838         int err = 0;
840         mutex_lock(&start_mutex);
842         if (gator_started)
843                 err = -EBUSY;
844         else
845                 gator_backtrace_depth = val;
847         mutex_unlock(&start_mutex);
849         return err;
852 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
854         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
857 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
859         unsigned long val;
860         int retval;
862         if (*offset)
863                 return -EINVAL;
865         retval = gatorfs_ulong_from_user(&val, buf, count);
866         if (retval)
867                 return retval;
869         if (val)
870                 retval = gator_op_start();
871         else
872                 gator_op_stop();
874         if (retval)
875                 return retval;
876         return count;
879 static const struct file_operations enable_fops = {
880         .read           = enable_read,
881         .write          = enable_write,
882 };
884 static int userspace_buffer_open(struct inode *inode, struct file *file)
886         int err = -EPERM;
888         if (!capable(CAP_SYS_ADMIN))
889                 return -EPERM;
891         if (test_and_set_bit_lock(0, &gator_buffer_opened))
892                 return -EBUSY;
894         if ((err = gator_op_setup()))
895                 goto fail;
897         /* NB: the actual start happens from userspace
898          * echo 1 >/dev/gator/enable
899          */
901         return 0;
903 fail:
904         __clear_bit_unlock(0, &gator_buffer_opened);
905         return err;
908 static int userspace_buffer_release(struct inode *inode, struct file *file)
910         gator_op_stop();
911         gator_shutdown();
912         __clear_bit_unlock(0, &gator_buffer_opened);
913         return 0;
916 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
917                                  size_t count, loff_t *offset)
919         int retval = -EINVAL;
920         int commit = 0, length, length1, length2, read, byte, type_length;
921         char *buffer1;
922         char *buffer2 = NULL;
923         int cpu, buftype;
925         /* do not handle partial reads */
926         if (count != userspace_buffer_size || *offset)
927                 return -EINVAL;
929         // sleep until the condition is true or a signal is received
930         // the condition is checked each time gator_buffer_wait is woken up
931         buftype = cpu = -1;
932         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
934         if (signal_pending(current))
935                 return -EINTR;
937         length2 = 0;
938         retval = -EFAULT;
940         mutex_lock(&gator_buffer_mutex);
942         if (buftype == -1 || cpu == -1) {
943                 retval = 0;
944                 goto out;
945         }
947         read = per_cpu(gator_buffer_read, cpu)[buftype];
948         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
950         /* May happen if the buffer is freed during pending reads. */
951         if (!per_cpu(gator_buffer, cpu)[buftype]) {
952                 retval = -EFAULT;
953                 goto out;
954         }
956         /* determine the size of two halves */
957         length1 = commit - read;
958         buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
959         buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
960         if (length1 < 0) {
961                 length1 = gator_buffer_size[buftype] - read;
962                 length2 = commit;
963         }
965         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
966         type_length = gator_response_type ? 1 : 0;
967         length = length1 + length2 - type_length - sizeof(int);
968         for (byte = 0; byte < sizeof(int); byte++) {
969                 per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
970         }
972         /* start, middle or end */
973         if (length1 > 0) {
974                 if (copy_to_user(&buf[0], buffer1, length1)) {
975                         goto out;
976                 }
977         }
979         /* possible wrap around */
980         if (length2 > 0) {
981                 if (copy_to_user(&buf[length1], buffer2, length2)) {
982                         goto out;
983                 }
984         }
986         per_cpu(gator_buffer_read, cpu)[buftype] = commit;
987         retval = length1 + length2;
989         /* kick just in case we've lost an SMP event */
990         wake_up(&gator_buffer_wait);
992 out:
993         mutex_unlock(&gator_buffer_mutex);
994         return retval;
997 const struct file_operations gator_event_buffer_fops = {
998         .open           = userspace_buffer_open,
999         .release        = userspace_buffer_release,
1000         .read           = userspace_buffer_read,
1001 };
1003 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1005         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count,
1006                                         offset);
1009 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1011         unsigned long val;
1012         int retval;
1014         if (*offset)
1015                 return -EINVAL;
1017         retval = gatorfs_ulong_from_user(&val, buf, count);
1018         if (retval)
1019                 return retval;
1021         retval = gator_set_backtrace(val);
1023         if (retval)
1024                 return retval;
1025         return count;
1028 static const struct file_operations depth_fops = {
1029         .read           = depth_read,
1030         .write          = depth_write
1031 };
1033 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1035         struct dentry *dir;
1036         struct gator_interface *gi;
1037         int cpu;
1039         /* reinitialize default values */
1040         gator_cpu_cores = 0;
1041         for_each_present_cpu(cpu) {
1042                 gator_cpu_cores++;
1043         }
1044         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1045         gator_response_type = 1;
1047         gatorfs_create_file(sb, root, "enable", &enable_fops);
1048         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1049         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1050         gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1051         gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1052         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1053         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1054         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1056         // Annotate interface
1057         gator_annotate_create_files(sb, root);
1059         // Linux Events
1060         dir = gatorfs_mkdir(sb, root, "events");
1061         list_for_each_entry(gi, &gator_events, list)
1062                 if (gi->create_files)
1063                         gi->create_files(sb, dir);
1065         // Power interface
1066         gator_trace_power_create_files(sb, dir);
1069 /******************************************************************************
1070  * Module
1071  ******************************************************************************/
1072 static int __init gator_module_init(void)
1074         if (gatorfs_register()) {
1075                 return -1;
1076         }
1078         if (gator_init()) {
1079                 gatorfs_unregister();
1080                 return -1;
1081         }
1083         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1085         return 0;
1088 static void __exit gator_module_exit(void)
1090         del_timer_sync(&gator_buffer_wake_up_timer);
1091         tracepoint_synchronize_unregister();
1092         gatorfs_unregister();
1095 module_init(gator_module_init);
1096 module_exit(gator_module_exit);
1098 MODULE_LICENSE("GPL");
1099 MODULE_AUTHOR("ARM Ltd");
1100 MODULE_DESCRIPTION("Gator system profiler");