gator-driver: Revert #error about lack of CONFIG_PERF_EVENTS
[android-sdk/arm-ds5-gator.git] / driver / gator_main.c
1 /**
2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  */
10 static unsigned long gator_protocol_version = 12;
12 #include <linux/slab.h>
13 #include <linux/cpu.h>
14 #include <linux/sched.h>
15 #include <linux/irq.h>
16 #include <linux/vmalloc.h>
17 #include <linux/hardirq.h>
18 #include <linux/highmem.h>
19 #include <linux/pagemap.h>
20 #include <linux/suspend.h>
21 #include <linux/module.h>
22 #include <linux/perf_event.h>
23 #include <asm/stacktrace.h>
24 #include <asm/uaccess.h>
26 #include "gator.h"
27 #include "gator_events.h"
29 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
30 #error kernels prior to 2.6.32 are not supported
31 #endif
33 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
35 #endif
37 #ifndef CONFIG_PROFILING
38 #error gator requires the kernel to have CONFIG_PROFILING defined
39 #endif
41 #ifndef CONFIG_HIGH_RES_TIMERS
42 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
43 #endif
45 #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
46 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
47 #endif
49 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50 #ifndef CONFIG_PERF_EVENTS
51 #warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52 #elif !defined CONFIG_HW_PERF_EVENTS
53 #warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54 #endif
55 #endif
57 /******************************************************************************
58  * DEFINES
59  ******************************************************************************/
60 #define SUMMARY_BUFFER_SIZE       (1*1024)
61 #define BACKTRACE_BUFFER_SIZE     (128*1024)
62 #define NAME_BUFFER_SIZE          (64*1024)
63 #define COUNTER_BUFFER_SIZE       (64*1024)     // counters have the core as part of the data and the core value in the frame header may be discarded
64 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
65 #define ANNOTATE_BUFFER_SIZE      (64*1024)     // annotate counters have the core as part of the data and the core value in the frame header may be discarded
66 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
67 #define GPU_TRACE_BUFFER_SIZE     (64*1024)     // gpu trace counters have the core as part of the data and the core value in the frame header may be discarded
68 #define IDLE_BUFFER_SIZE          (32*1024)     // idle counters have the core as part of the data and the core value in the frame header may be discarded
70 #define NO_COOKIE      0U
71 #define INVALID_COOKIE ~0U
73 #define FRAME_SUMMARY       1
74 #define FRAME_BACKTRACE     2
75 #define FRAME_NAME          3
76 #define FRAME_COUNTER       4
77 #define FRAME_BLOCK_COUNTER 5
78 #define FRAME_ANNOTATE      6
79 #define FRAME_SCHED_TRACE   7
80 #define FRAME_GPU_TRACE     8
81 #define FRAME_IDLE          9
83 #define MESSAGE_END_BACKTRACE 1
85 #define MESSAGE_COOKIE      1
86 #define MESSAGE_THREAD_NAME 2
87 #define HRTIMER_CORE_NAME   3
89 #define MESSAGE_GPU_START 1
90 #define MESSAGE_GPU_STOP  2
92 #define MESSAGE_SCHED_SWITCH 1
93 #define MESSAGE_SCHED_EXIT   2
95 #define MAXSIZE_PACK32     5
96 #define MAXSIZE_PACK64    10
98 #if defined(__arm__)
99 #define PC_REG regs->ARM_pc
100 #elif defined(__aarch64__)
101 #define PC_REG regs->pc
102 #else
103 #define PC_REG regs->ip
104 #endif
106 enum {
107         SUMMARY_BUF,
108         BACKTRACE_BUF,
109         NAME_BUF,
110         COUNTER_BUF,
111         BLOCK_COUNTER_BUF,
112         ANNOTATE_BUF,
113         SCHED_TRACE_BUF,
114         GPU_TRACE_BUF,
115         IDLE_BUF,
116         NUM_GATOR_BUFS
117 };
119 /******************************************************************************
120  * Globals
121  ******************************************************************************/
122 static unsigned long gator_cpu_cores;
123 // Size of the largest buffer. Effectively constant, set in gator_op_create_files
124 static unsigned long userspace_buffer_size;
125 static unsigned long gator_backtrace_depth;
127 static unsigned long gator_started;
128 static uint64_t monotonic_started;
129 static unsigned long gator_buffer_opened;
130 static unsigned long gator_timer_count;
131 static unsigned long gator_response_type;
132 static DEFINE_MUTEX(start_mutex);
133 static DEFINE_MUTEX(gator_buffer_mutex);
135 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
136 static struct timer_list gator_buffer_wake_up_timer;
137 static LIST_HEAD(gator_events);
139 /******************************************************************************
140  * Prototypes
141  ******************************************************************************/
142 static void buffer_check(int cpu, int buftype);
143 static int buffer_bytes_available(int cpu, int buftype);
144 static bool buffer_check_space(int cpu, int buftype, int bytes);
145 static int contiguous_space_available(int cpu, int bufytpe);
146 static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
147 static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
148 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len);
149 static void gator_buffer_write_string(int cpu, int buftype, const char *x);
150 static void gator_add_trace(int cpu, unsigned long address);
151 static void gator_add_sample(int cpu, struct pt_regs *const regs);
152 static uint64_t gator_get_time(void);
154 // Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
155 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
156 // gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup.
157 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
158 // Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read
159 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
160 // Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer
161 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
162 // Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace
163 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
164 // If set to false, decreases the number of bytes returned by buffer_bytes_available. Set in buffer_check_space if no space is remaining. Initialized to true in gator_op_setup
165 // This means that if we run out of space, continue to report that no space is available until bytes are read by userspace
166 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
167 // The buffer. Allocated in gator_op_setup
168 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
170 /******************************************************************************
171  * Application Includes
172  ******************************************************************************/
173 #include "gator_marshaling.c"
174 #include "gator_hrtimer_perf.c"
175 #include "gator_hrtimer_gator.c"
176 #include "gator_cookies.c"
177 #include "gator_trace_sched.c"
178 #include "gator_trace_power.c"
179 #include "gator_trace_gpu.c"
180 #include "gator_backtrace.c"
181 #include "gator_annotate.c"
182 #include "gator_fs.c"
183 #include "gator_pack.c"
185 /******************************************************************************
186  * Misc
187  ******************************************************************************/
189 struct gator_cpu gator_cpus[] = {
190   {
191                 .cpuid = ARM1136,
192                 .core_name = "ARM1136",
193                 .pmnc_name = "ARM_ARM11",
194                 .pmnc_counters = 3,
195                 .ccnt = 2,
196         },
197   {
198                 .cpuid = ARM1156,
199                 .core_name = "ARM1156",
200                 .pmnc_name = "ARM_ARM11",
201                 .pmnc_counters = 3,
202                 .ccnt = 2,
203         },
204   {
205                 .cpuid = ARM1176,
206                 .core_name = "ARM1176",
207                 .pmnc_name = "ARM_ARM11",
208                 .pmnc_counters = 3,
209                 .ccnt = 2,
210         },
211   {
212                 .cpuid = ARM11MPCORE,
213                 .core_name = "ARM11MPCore",
214                 .pmnc_name = "ARM_ARM11MPCore",
215                 .pmnc_counters = 3,
216         },
217   {
218                 .cpuid = CORTEX_A5,
219                 .core_name = "Cortex-A5",
220                 .pmnc_name = "ARM_Cortex-A5",
221                 .pmnc_counters = 2,
222         },
223   {
224                 .cpuid = CORTEX_A7,
225                 .core_name = "Cortex-A7",
226                 .pmnc_name = "ARM_Cortex-A7",
227                 .pmnc_counters = 4,
228         },
229   {
230                 .cpuid = CORTEX_A8,
231                 .core_name = "Cortex-A8",
232                 .pmnc_name = "ARM_Cortex-A8",
233                 .pmnc_counters = 4,
234         },
235   {
236                 .cpuid = CORTEX_A9,
237                 .core_name = "Cortex-A9",
238                 .pmnc_name = "ARM_Cortex-A9",
239                 .pmnc_counters = 6,
240         },
241   {
242                 .cpuid = CORTEX_A15,
243                 .core_name = "Cortex-A15",
244                 .pmnc_name = "ARM_Cortex-A15",
245                 .pmnc_counters = 6,
246         },
247   {
248                 .cpuid = SCORPION,
249                 .core_name = "Scorpion",
250                 .pmnc_name = "Scorpion",
251                 .pmnc_counters = 4,
252         },
253   {
254                 .cpuid = SCORPIONMP,
255                 .core_name = "ScorpionMP",
256                 .pmnc_name = "ScorpionMP",
257                 .pmnc_counters = 4,
258         },
259   {
260                 .cpuid = KRAITSIM,
261                 .core_name = "KraitSIM",
262                 .pmnc_name = "Krait",
263                 .pmnc_counters = 4,
264         },
265   {
266                 .cpuid = KRAIT,
267                 .core_name = "Krait",
268                 .pmnc_name = "Krait",
269                 .pmnc_counters = 4,
270         },
271         {
272                 .cpuid = KRAIT_S4_PRO,
273                 .core_name = "Krait S4 Pro",
274                 .pmnc_name = "Krait",
275                 .pmnc_counters = 4,
276         },
277   {
278                 .cpuid = CORTEX_A53,
279                 .core_name = "Cortex-A53",
280                 .pmnc_name = "ARM_Cortex-A53",
281                 .pmnc_counters = 6,
282         },
283   {
284                 .cpuid = CORTEX_A57,
285                 .core_name = "Cortex-A57",
286                 .pmnc_name = "ARM_Cortex-A57",
287                 .pmnc_counters = 6,
288         },
289   {
290                 .cpuid = AARCH64,
291                 .core_name = "AArch64",
292                 .pmnc_name = "ARM_AArch64",
293                 .pmnc_counters = 6,
294         },
295         {
296                 .cpuid = OTHER,
297                 .core_name = "Other",
298                 .pmnc_name = "Other",
299                 .pmnc_counters = 6,
300         },
301   {}
302 };
304 u32 gator_cpuid(void)
306 #if defined(__arm__) || defined(__aarch64__)
307         u32 val;
308 #if !defined(__aarch64__)
309         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
310 #else
311         asm volatile("mrs %0, midr_el1" : "=r" (val));
312 #endif
313         return (val >> 4) & 0xfff;
314 #else
315         return OTHER;
316 #endif
319 static void gator_buffer_wake_up(unsigned long data)
321         wake_up(&gator_buffer_wait);
324 /******************************************************************************
325  * Commit interface
326  ******************************************************************************/
327 static bool buffer_commit_ready(int *cpu, int *buftype)
329         int cpu_x, x;
330         for_each_present_cpu(cpu_x) {
331                 for (x = 0; x < NUM_GATOR_BUFS; x++)
332                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
333                                 *cpu = cpu_x;
334                                 *buftype = x;
335                                 return true;
336                         }
337         }
338         return false;
341 /******************************************************************************
342  * Buffer management
343  ******************************************************************************/
344 static int buffer_bytes_available(int cpu, int buftype)
346         int remaining, filled;
348         filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
349         if (filled < 0) {
350                 filled += gator_buffer_size[buftype];
351         }
353         remaining = gator_buffer_size[buftype] - filled;
355         if (per_cpu(buffer_space_available, cpu)[buftype]) {
356                 // Give some extra room; also allows space to insert the overflow error packet
357                 remaining -= 200;
358         } else {
359                 // Hysteresis, prevents multiple overflow messages
360                 remaining -= 2000;
361         }
363         return remaining;
366 static int contiguous_space_available(int cpu, int buftype)
368         int remaining = buffer_bytes_available(cpu, buftype);
369         int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
370         if (remaining < contiguous)
371                 return remaining;
372         else
373                 return contiguous;
376 static bool buffer_check_space(int cpu, int buftype, int bytes)
378         int remaining = buffer_bytes_available(cpu, buftype);
380         if (remaining < bytes) {
381                 per_cpu(buffer_space_available, cpu)[buftype] = false;
382         } else {
383                 per_cpu(buffer_space_available, cpu)[buftype] = true;
384         }
386         return per_cpu(buffer_space_available, cpu)[buftype];
389 static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int len)
391         int i;
392         u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
393         u32 mask = gator_buffer_mask[buftype];
394         char *buffer = per_cpu(gator_buffer, cpu)[buftype];
396         for (i = 0; i < len; i++) {
397                 buffer[write] = x[i];
398                 write = (write + 1) & mask;
399         }
401         per_cpu(gator_buffer_write, cpu)[buftype] = write;
404 static void gator_buffer_write_string(int cpu, int buftype, const char *x)
406         int len = strlen(x);
407         gator_buffer_write_packed_int(cpu, buftype, len);
408         gator_buffer_write_bytes(cpu, buftype, x, len);
411 static void gator_commit_buffer(int cpu, int buftype)
413         int type_length, commit, length, byte;
415         if (!per_cpu(gator_buffer, cpu)[buftype])
416                 return;
418         // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
419         type_length = gator_response_type ? 1 : 0;
420         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
421         length = per_cpu(gator_buffer_write, cpu)[buftype] - commit;
422         if (length < 0) {
423                 length += gator_buffer_size[buftype];
424         }
425         length = length - type_length - sizeof(int);
426         for (byte = 0; byte < sizeof(int); byte++) {
427                 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
428         }
430         per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
431         marshal_frame(cpu, buftype);
433         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
434         mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
437 static void buffer_check(int cpu, int buftype)
439         int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
440         if (filled < 0) {
441                 filled += gator_buffer_size[buftype];
442         }
443         if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
444                 gator_commit_buffer(cpu, buftype);
445         }
448 static void gator_add_trace(int cpu, unsigned long address)
450         off_t offset = 0;
451         unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset);
453         if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
454                 offset = address;
455         }
457         marshal_backtrace(offset & ~1, cookie);
460 static void gator_add_sample(int cpu, struct pt_regs *const regs)
462         bool inKernel;
463         unsigned long exec_cookie;
465         if (!regs)
466                 return;
468         inKernel = !user_mode(regs);
469         exec_cookie = get_exec_cookie(cpu, current);
471         if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
472                 return;
474         if (inKernel) {
475                 kernel_backtrace(cpu, regs);
476         } else {
477                 // Cookie+PC
478                 gator_add_trace(cpu, PC_REG);
480                 // Backtrace
481                 if (gator_backtrace_depth)
482                         arm_backtrace_eabi(cpu, regs, gator_backtrace_depth);
483         }
485         marshal_backtrace_footer();
488 /******************************************************************************
489  * hrtimer interrupt processing
490  ******************************************************************************/
491 static void gator_timer_interrupt(void)
493         struct pt_regs *const regs = get_irq_regs();
494         gator_backtrace_handler(regs);
497 void gator_backtrace_handler(struct pt_regs *const regs)
499         int cpu = smp_processor_id();
501         // Output backtrace
502         gator_add_sample(cpu, regs);
504         // Collect counters
505         if (!per_cpu(collecting, cpu)) {
506                 collect_counters();
507         }
510 static int gator_running;
512 // This function runs in interrupt context and on the appropriate core
513 static void gator_timer_offline(void *unused)
515         struct gator_interface *gi;
516         int i, len, cpu = smp_processor_id();
517         int *buffer;
519         gator_trace_sched_offline();
520         gator_trace_power_offline();
522         gator_hrtimer_offline(cpu);
524         // Offline any events and output counters
525         if (marshal_event_header()) {
526                 list_for_each_entry(gi, &gator_events, list) {
527                         if (gi->offline) {
528                                 len = gi->offline(&buffer);
529                                 marshal_event(len, buffer);
530                         }
531                 }
532         }
534         // Flush all buffers on this core
535         for (i = 0; i < NUM_GATOR_BUFS; i++)
536                 gator_commit_buffer(cpu, i);
539 // This function runs in process context and may be running on a core other than core 'cpu'
540 static void gator_timer_offline_dispatch(int cpu)
542         struct gator_interface *gi;
544         list_for_each_entry(gi, &gator_events, list)
545             if (gi->offline_dispatch)
546                 gi->offline_dispatch(cpu);
549 static void gator_timer_stop(void)
551         int cpu;
553         if (gator_running) {
554                 on_each_cpu(gator_timer_offline, NULL, 1);
555                 for_each_online_cpu(cpu) {
556                         gator_timer_offline_dispatch(cpu);
557                 }
559                 gator_running = 0;
560                 gator_hrtimer_shutdown();
561         }
564 // This function runs in interrupt context and on the appropriate core
565 static void gator_timer_online(void *unused)
567         struct gator_interface *gi;
568         int len, cpu = smp_processor_id();
569         int *buffer;
571         gator_trace_power_online();
573         // online any events and output counters
574         if (marshal_event_header()) {
575                 list_for_each_entry(gi, &gator_events, list) {
576                         if (gi->online) {
577                                 len = gi->online(&buffer);
578                                 marshal_event(len, buffer);
579                         }
580                 }
581         }
583         gator_hrtimer_online(cpu);
584 #if defined(__arm__) || defined(__aarch64__)
585         {
586                 const char *core_name = "Unknown";
587                 const u32 cpuid = gator_cpuid();
588                 int i;
590                 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
591                         if (gator_cpus[i].cpuid == cpuid) {
592                                 core_name = gator_cpus[i].core_name;
593                                 break;
594                         }
595                 }
597                 marshal_core_name(core_name);
598         }
599 #endif
602 // This function runs in interrupt context and may be running on a core other than core 'cpu'
603 static void gator_timer_online_dispatch(int cpu)
605         struct gator_interface *gi;
607         list_for_each_entry(gi, &gator_events, list)
608             if (gi->online_dispatch)
609                 gi->online_dispatch(cpu);
612 int gator_timer_start(unsigned long sample_rate)
614         int cpu;
616         if (gator_running) {
617                 pr_notice("gator: already running\n");
618                 return 0;
619         }
621         gator_running = 1;
623         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
624                 return -1;
626         for_each_online_cpu(cpu) {
627                 gator_timer_online_dispatch(cpu);
628         }
629         on_each_cpu(gator_timer_online, NULL, 1);
631         return 0;
634 static uint64_t gator_get_time(void)
636         struct timespec ts;
637         uint64_t timestamp;
639         //getnstimeofday(&ts);
640         do_posix_clock_monotonic_gettime(&ts);
641         timestamp = timespec_to_ns(&ts) - monotonic_started;
643         return timestamp;
646 /******************************************************************************
647  * cpu hotplug and pm notifiers
648  ******************************************************************************/
649 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
651         long cpu = (long)hcpu;
653         switch (action) {
654         case CPU_DOWN_PREPARE:
655         case CPU_DOWN_PREPARE_FROZEN:
656                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
657                 gator_timer_offline_dispatch(cpu);
658                 break;
659         case CPU_ONLINE:
660         case CPU_ONLINE_FROZEN:
661                 gator_timer_online_dispatch(cpu);
662                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
663                 break;
664         }
666         return NOTIFY_OK;
669 static struct notifier_block __refdata gator_hotcpu_notifier = {
670         .notifier_call = gator_hotcpu_notify,
671 };
673 // n.b. calling "on_each_cpu" only runs on those that are online
674 // Registered linux events are not disabled, so their counters will continue to collect
675 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
677         int cpu;
679         switch (event) {
680         case PM_HIBERNATION_PREPARE:
681         case PM_SUSPEND_PREPARE:
682                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
683                 unregister_scheduler_tracepoints();
684                 on_each_cpu(gator_timer_offline, NULL, 1);
685                 for_each_online_cpu(cpu) {
686                         gator_timer_offline_dispatch(cpu);
687                 }
688                 break;
689         case PM_POST_HIBERNATION:
690         case PM_POST_SUSPEND:
691                 for_each_online_cpu(cpu) {
692                         gator_timer_online_dispatch(cpu);
693                 }
694                 on_each_cpu(gator_timer_online, NULL, 1);
695                 register_scheduler_tracepoints();
696                 register_hotcpu_notifier(&gator_hotcpu_notifier);
697                 break;
698         }
700         return NOTIFY_OK;
703 static struct notifier_block gator_pm_notifier = {
704         .notifier_call = gator_pm_notify,
705 };
707 static int gator_notifier_start(void)
709         int retval;
710         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
711         if (retval == 0)
712                 retval = register_pm_notifier(&gator_pm_notifier);
713         return retval;
716 static void gator_notifier_stop(void)
718         unregister_pm_notifier(&gator_pm_notifier);
719         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
722 /******************************************************************************
723  * Main
724  ******************************************************************************/
725 static void gator_summary(void)
727         uint64_t timestamp;
728         struct timespec uptime_ts;
730         timestamp = gator_get_time();
732         do_posix_clock_monotonic_gettime(&uptime_ts);
733         monotonic_started = timespec_to_ns(&uptime_ts);
735         marshal_summary(timestamp, monotonic_started);
738 int gator_events_install(struct gator_interface *interface)
740         list_add_tail(&interface->list, &gator_events);
742         return 0;
745 int gator_events_get_key(void)
747         // key of zero is reserved as a timestamp
748         static int key = 1;
750         return key++;
753 static int gator_init(void)
755         int i;
757         // events sources (gator_events.h, generated by gator_events.sh)
758         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
759                 if (gator_events_list[i])
760                         gator_events_list[i]();
762         gator_trace_power_init();
764         return 0;
767 static void gator_exit(void)
769         struct gator_interface *gi;
771         list_for_each_entry(gi, &gator_events, list)
772                 if (gi->shutdown)
773                         gi->shutdown();
776 static int gator_start(void)
778         unsigned long cpu, i;
779         struct gator_interface *gi;
781         // Initialize the buffer with the frame type and core
782         for_each_present_cpu(cpu) {
783                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
784                         marshal_frame(cpu, i);
785                 }
786         }
788         // Capture the start time  
789         gator_summary();
791         // start all events
792         list_for_each_entry(gi, &gator_events, list) {
793                 if (gi->start && gi->start() != 0) {
794                         struct list_head *ptr = gi->list.prev;
796                         while (ptr != &gator_events) {
797                                 gi = list_entry(ptr, struct gator_interface, list);
799                                 if (gi->stop)
800                                         gi->stop();
802                                 ptr = ptr->prev;
803                         }
804                         goto events_failure;
805                 }
806         }
808         // cookies shall be initialized before trace_sched_start() and gator_timer_start()
809         if (cookies_initialize())
810                 goto cookies_failure;
811         if (gator_annotate_start())
812                 goto annotate_failure;
813         if (gator_trace_sched_start())
814                 goto sched_failure;
815         if (gator_trace_power_start())
816                 goto power_failure;
817         if (gator_trace_gpu_start())
818                 goto gpu_failure;
819         if (gator_timer_start(gator_timer_count))
820                 goto timer_failure;
821         if (gator_notifier_start())
822                 goto notifier_failure;
824         return 0;
826 notifier_failure:
827         gator_timer_stop();
828 timer_failure:
829         gator_trace_gpu_stop();
830 gpu_failure:
831         gator_trace_power_stop();
832 power_failure:
833         gator_trace_sched_stop();
834 sched_failure:
835         gator_annotate_stop();
836 annotate_failure:
837         cookies_release();
838 cookies_failure:
839         // stop all events
840         list_for_each_entry(gi, &gator_events, list)
841             if (gi->stop)
842                 gi->stop();
843 events_failure:
845         return -1;
848 static void gator_stop(void)
850         struct gator_interface *gi;
852         gator_annotate_stop();
853         gator_trace_sched_stop();
854         gator_trace_power_stop();
855         gator_trace_gpu_stop();
857         // stop all interrupt callback reads before tearing down other interfaces
858         gator_notifier_stop();  // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
859         gator_timer_stop();
861         // stop all events
862         list_for_each_entry(gi, &gator_events, list)
863             if (gi->stop)
864                 gi->stop();
867 /******************************************************************************
868  * Filesystem
869  ******************************************************************************/
870 /* fopen("buffer") */
871 static int gator_op_setup(void)
873         int err = 0;
874         int cpu, i;
876         mutex_lock(&start_mutex);
878         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
879         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
881         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
882         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
884         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
885         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
887         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
888         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
890         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
891         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
893         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
894         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
896         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
897         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
899         gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
900         gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
902         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
903         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
905         // Initialize percpu per buffer variables
906         for (i = 0; i < NUM_GATOR_BUFS; i++) {
907                 // Verify buffers are a power of 2
908                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
909                         err = -ENOEXEC;
910                         goto setup_error;
911                 }
913                 for_each_present_cpu(cpu) {
914                         per_cpu(gator_buffer_read, cpu)[i] = 0;
915                         per_cpu(gator_buffer_write, cpu)[i] = 0;
916                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
917                         per_cpu(buffer_space_available, cpu)[i] = true;
919                         // Annotation is a special case that only uses a single buffer
920                         if (cpu > 0 && i == ANNOTATE_BUF) {
921                                 per_cpu(gator_buffer, cpu)[i] = NULL;
922                                 continue;
923                         }
925                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
926                         if (!per_cpu(gator_buffer, cpu)[i]) {
927                                 err = -ENOMEM;
928                                 goto setup_error;
929                         }
930                 }
931         }
933 setup_error:
934         mutex_unlock(&start_mutex);
935         return err;
938 /* Actually start profiling (echo 1>/dev/gator/enable) */
939 static int gator_op_start(void)
941         int err = 0;
943         mutex_lock(&start_mutex);
945         if (gator_started || gator_start())
946                 err = -EINVAL;
947         else
948                 gator_started = 1;
950         mutex_unlock(&start_mutex);
952         return err;
955 /* echo 0>/dev/gator/enable */
956 static void gator_op_stop(void)
958         mutex_lock(&start_mutex);
960         if (gator_started) {
961                 gator_stop();
963                 mutex_lock(&gator_buffer_mutex);
965                 gator_started = 0;
966                 cookies_release();
967                 wake_up(&gator_buffer_wait);
969                 mutex_unlock(&gator_buffer_mutex);
970         }
972         mutex_unlock(&start_mutex);
975 static void gator_shutdown(void)
977         int cpu, i;
979         mutex_lock(&start_mutex);
981         for_each_present_cpu(cpu) {
982                 mutex_lock(&gator_buffer_mutex);
983                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
984                         vfree(per_cpu(gator_buffer, cpu)[i]);
985                         per_cpu(gator_buffer, cpu)[i] = NULL;
986                         per_cpu(gator_buffer_read, cpu)[i] = 0;
987                         per_cpu(gator_buffer_write, cpu)[i] = 0;
988                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
989                         per_cpu(buffer_space_available, cpu)[i] = true;
990                 }
991                 mutex_unlock(&gator_buffer_mutex);
992         }
994         mutex_unlock(&start_mutex);
997 static int gator_set_backtrace(unsigned long val)
999         int err = 0;
1001         mutex_lock(&start_mutex);
1003         if (gator_started)
1004                 err = -EBUSY;
1005         else
1006                 gator_backtrace_depth = val;
1008         mutex_unlock(&start_mutex);
1010         return err;
1013 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1015         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1018 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1020         unsigned long val;
1021         int retval;
1023         if (*offset)
1024                 return -EINVAL;
1026         retval = gatorfs_ulong_from_user(&val, buf, count);
1027         if (retval)
1028                 return retval;
1030         if (val)
1031                 retval = gator_op_start();
1032         else
1033                 gator_op_stop();
1035         if (retval)
1036                 return retval;
1037         return count;
1040 static const struct file_operations enable_fops = {
1041         .read = enable_read,
1042         .write = enable_write,
1043 };
1045 static int userspace_buffer_open(struct inode *inode, struct file *file)
1047         int err = -EPERM;
1049         if (!capable(CAP_SYS_ADMIN))
1050                 return -EPERM;
1052         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1053                 return -EBUSY;
1055         if ((err = gator_op_setup()))
1056                 goto fail;
1058         /* NB: the actual start happens from userspace
1059          * echo 1 >/dev/gator/enable
1060          */
1062         return 0;
1064 fail:
1065         __clear_bit_unlock(0, &gator_buffer_opened);
1066         return err;
1069 static int userspace_buffer_release(struct inode *inode, struct file *file)
1071         gator_op_stop();
1072         gator_shutdown();
1073         __clear_bit_unlock(0, &gator_buffer_opened);
1074         return 0;
1077 static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
1078                                      size_t count, loff_t *offset)
1080         int retval = -EINVAL;
1081         int commit = 0, length1, length2, read;
1082         char *buffer1;
1083         char *buffer2 = NULL;
1084         int cpu, buftype;
1086         /* do not handle partial reads */
1087         if (count != userspace_buffer_size || *offset)
1088                 return -EINVAL;
1090         // sleep until the condition is true or a signal is received
1091         // the condition is checked each time gator_buffer_wait is woken up
1092         buftype = cpu = -1;
1093         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1095         if (signal_pending(current))
1096                 return -EINTR;
1098         length2 = 0;
1099         retval = -EFAULT;
1101         mutex_lock(&gator_buffer_mutex);
1103         if (buftype == -1 || cpu == -1) {
1104                 retval = 0;
1105                 goto out;
1106         }
1108         read = per_cpu(gator_buffer_read, cpu)[buftype];
1109         commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1111         /* May happen if the buffer is freed during pending reads. */
1112         if (!per_cpu(gator_buffer, cpu)[buftype]) {
1113                 retval = -EFAULT;
1114                 goto out;
1115         }
1117         /* determine the size of two halves */
1118         length1 = commit - read;
1119         buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1120         buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1121         if (length1 < 0) {
1122                 length1 = gator_buffer_size[buftype] - read;
1123                 length2 = commit;
1124         }
1126         /* start, middle or end */
1127         if (length1 > 0) {
1128                 if (copy_to_user(&buf[0], buffer1, length1)) {
1129                         goto out;
1130                 }
1131         }
1133         /* possible wrap around */
1134         if (length2 > 0) {
1135                 if (copy_to_user(&buf[length1], buffer2, length2)) {
1136                         goto out;
1137                 }
1138         }
1140         per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1141         retval = length1 + length2;
1143         /* kick just in case we've lost an SMP event */
1144         wake_up(&gator_buffer_wait);
1146 out:
1147         mutex_unlock(&gator_buffer_mutex);
1148         return retval;
1151 const struct file_operations gator_event_buffer_fops = {
1152         .open = userspace_buffer_open,
1153         .release = userspace_buffer_release,
1154         .read = userspace_buffer_read,
1155 };
1157 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1159         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1162 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1164         unsigned long val;
1165         int retval;
1167         if (*offset)
1168                 return -EINVAL;
1170         retval = gatorfs_ulong_from_user(&val, buf, count);
1171         if (retval)
1172                 return retval;
1174         retval = gator_set_backtrace(val);
1176         if (retval)
1177                 return retval;
1178         return count;
1181 static const struct file_operations depth_fops = {
1182         .read = depth_read,
1183         .write = depth_write
1184 };
1186 void gator_op_create_files(struct super_block *sb, struct dentry *root)
1188         struct dentry *dir;
1189         struct gator_interface *gi;
1190         int cpu;
1192         /* reinitialize default values */
1193         gator_cpu_cores = 0;
1194         for_each_present_cpu(cpu) {
1195                 gator_cpu_cores++;
1196         }
1197         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1198         gator_response_type = 1;
1200         gatorfs_create_file(sb, root, "enable", &enable_fops);
1201         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1202         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1203         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1204         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1205         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1206         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1207         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1209         // Annotate interface
1210         gator_annotate_create_files(sb, root);
1212         // Linux Events
1213         dir = gatorfs_mkdir(sb, root, "events");
1214         list_for_each_entry(gi, &gator_events, list)
1215             if (gi->create_files)
1216                 gi->create_files(sb, dir);
1218         // Power interface
1219         gator_trace_power_create_files(sb, dir);
1222 /******************************************************************************
1223  * Module
1224  ******************************************************************************/
1225 static int __init gator_module_init(void)
1227         if (gatorfs_register()) {
1228                 return -1;
1229         }
1231         if (gator_init()) {
1232                 gatorfs_unregister();
1233                 return -1;
1234         }
1236         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1238         return 0;
1241 static void __exit gator_module_exit(void)
1243         del_timer_sync(&gator_buffer_wake_up_timer);
1244         tracepoint_synchronize_unregister();
1245         gator_exit();
1246         gatorfs_unregister();
1249 module_init(gator_module_init);
1250 module_exit(gator_module_exit);
1252 MODULE_LICENSE("GPL");
1253 MODULE_AUTHOR("ARM Ltd");
1254 MODULE_DESCRIPTION("Gator system profiler");