summaryrefslogtreecommitdiffstats
path: root/driver
diff options
context:
space:
mode:
authorPawel Moll2011-09-30 05:53:40 -0500
committerPawel Moll2011-10-04 04:34:11 -0500
commit3f0b05d287e7df2160ea80f14c69444b73b74ed8 (patch)
tree9fc694ea7c8ba0697bb462bf9bb7f5efbc108e10 /driver
parent1d5b785dbabac2dfc45103e7c2d787952c93488c (diff)
downloadarm-ds5-gator-3f0b05d287e7df2160ea80f14c69444b73b74ed8.tar.gz
arm-ds5-gator-3f0b05d287e7df2160ea80f14c69444b73b74ed8.tar.xz
arm-ds5-gator-3f0b05d287e7df2160ea80f14c69444b73b74ed8.zip
gator-driver: ARM DS-5.7 Streamline gator driver sources
Signed-off-by: Pawel Moll <pawel.moll@arm.com>
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile5
-rw-r--r--driver/README_Streamline.txt26
-rw-r--r--driver/gator.h12
-rw-r--r--driver/gator_annotate.c28
-rw-r--r--driver/gator_backtrace.c6
-rw-r--r--driver/gator_cookies.c40
-rw-r--r--driver/gator_ebs.c160
-rw-r--r--driver/gator_events.c40
-rw-r--r--driver/gator_events.h19
-rw-r--r--driver/gator_events_armv6.c40
-rw-r--r--driver/gator_events_armv7.c186
-rw-r--r--driver/gator_events_armv7.h38
-rw-r--r--driver/gator_events_l2c-310.c182
-rw-r--r--driver/gator_events_meminfo.c11
-rw-r--r--driver/gator_events_pl310.c176
-rw-r--r--driver/gator_events_scorpion.c40
-rw-r--r--driver/gator_main.c529
-rw-r--r--driver/gator_pack.c262
-rw-r--r--driver/gator_trace.h26
-rw-r--r--driver/gator_trace_sched.c228
20 files changed, 1196 insertions, 858 deletions
diff --git a/driver/Makefile b/driver/Makefile
index b3680e1..de59957 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -17,7 +17,8 @@ endif
17 17
18gator-$(CONFIG_ARM) += gator_events_armv6.o \ 18gator-$(CONFIG_ARM) += gator_events_armv6.o \
19 gator_events_armv7.o \ 19 gator_events_armv7.o \
20 gator_events_pl310.o 20 gator_events_l2c-310.o \
21 gator_events_scorpion.o
21 22
22$(obj)/gator_main.o: gator_events.h 23$(obj)/gator_main.o: gator_events.h
23 24
@@ -40,6 +41,6 @@ all:
40 $(error) 41 $(error)
41 42
42clean: 43clean:
43 rm -f *.o modules.order Module.symvers gator.ko gator.mod.c 44 rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c
44 45
45endif 46endif
diff --git a/driver/README_Streamline.txt b/driver/README_Streamline.txt
index 4688474..9b64b8e 100644
--- a/driver/README_Streamline.txt
+++ b/driver/README_Streamline.txt
@@ -5,10 +5,21 @@ Instructions on setting up ARM Streamline on the target.
5The gator driver and gator daemon are required to run on the ARM linux target in order for ARM Streamline to operate. 5The gator driver and gator daemon are required to run on the ARM linux target in order for ARM Streamline to operate.
6The driver should be built as a module and the daemon must run with root permissions on the target. 6The driver should be built as a module and the daemon must run with root permissions on the target.
7 7
8*** Introduction ***
9
10A linux development environment with cross compiling tools is most likely required, depending on what is already created and provided.
11-For users, the ideal environment is to be given a BSP with gatord and gator.ko already running on a properly configured kernel. In such a scenario, a development environment is not needed, root permission may or may not be needed (gatord must be executed with root permissions but can be automatically started, see below), and the user can run Streamline and profile the system without any setup.
12-The ideal development environment has the kernel source code available to be rebuilt and executed on the target. This environment allows the greatest flexibility in configuring the kernel and building the gator driver module.
13-However, it is possible that a user/developer has a kernel but does not have the source code. In this scenario it may or may not be possible to obtain a valid profile.
14 -First, check if the kernel has the proper configuration options (see below). Profiling cannot occur using a kernel that is not configured properly, a new kernel must be created.
15 -Second, given a properly configured kernel, check if the filesystem contains the kernel source/headers, which can be used to re-create the gator driver.
16 -If the kernel is not properly configured or sources/headers are not available, the developer is on their own and kernel creation is beyond the scope of this document. Note: It is possible for a module to work when compiled against a similar kernel source code, though this is not guaranteed to work due to differences in kernel structures, exported symbols and incompatible configuration parameters.
17
8*** Preparing and building the kernel *** 18*** Preparing and building the kernel ***
9 19
10cd into the root source dir of the linux kernel 20cd into the root source dir of the linux kernel
11make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- <platform_defconfig> (choose the appropriate configuration for your board) 21if your target has never been configured, choose the appropriate configuration for your target
22 make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- <platform_defconfig>
12make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfig 23make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfig
13 24
14Required Kernel Changes (depending on the kernel version, the location of these configuration settings within menuconfig may be different) 25Required Kernel Changes (depending on the kernel version, the location of these configuration settings within menuconfig may be different)
@@ -19,6 +30,7 @@ Required Kernel Changes (depending on the kernel version, the location of these
19 - [*] Trace process context switches and events 30 - [*] Trace process context switches and events
20- Kernel Features 31- Kernel Features
21 - [*] High Resolution Timer Support 32 - [*] High Resolution Timer Support
33 - [*] Use local timer interrupts (only required for SMP)
22 34
23The "context switches and events" option will not be available if other trace configurations are enabled. Other trace configurations being enabled is sufficient to turn on context switches and events. 35The "context switches and events" option will not be available if other trace configurations are enabled. Other trace configurations being enabled is sufficient to turn on context switches and events.
24 36
@@ -29,10 +41,14 @@ Note: Configurations may not be supported on all targets
29 41
30make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage 42make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage
31 43
44*** Checking the gator requirements ***
45
46(optional) Use the hrtimer_module utility to validate the kernel High Resolution Timer requirement.
47
32*** Building the gator module *** 48*** Building the gator module ***
33 49
34To create the gator.ko module, 50To create the gator.ko module,
35 cd /ds-5-install-directory/arm/src 51 cd /ds-5-install-directory/arm/gator/driver-src
36 tar xzf gator-driver.tar.gz 52 tar xzf gator-driver.tar.gz
37 cd gator-driver 53 cd gator-driver
38 make -C <kernel_build_dir> M=`pwd` ARCH=arm CROSS_COMPILE=<...> modules 54 make -C <kernel_build_dir> M=`pwd` ARCH=arm CROSS_COMPILE=<...> modules
@@ -50,7 +66,7 @@ Recommended compiler settings:
50*** Running gator *** 66*** Running gator ***
51 67
52Load the kernel onto the target and copy gatord and gator.ko into the target's filesystem. 68Load the kernel onto the target and copy gatord and gator.ko into the target's filesystem.
53gatord is located in <installdir>/arm/armv5t/. 69gatord is located in <installdir>/arm/gator/linux or <installdir>/arm/gator/android or can be built from source.
54Ensure gatord has execute permissions 70Ensure gatord has execute permissions
55 chmod +x gatord 71 chmod +x gatord
56gator.ko must be located in the same directory as gatord on the target. 72gator.ko must be located in the same directory as gatord on the target.
@@ -65,7 +81,7 @@ make ARCH=arm CROSS_COMPILE=$(CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfi
65 81
66make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage 82make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage
67Use vmlinux as the image for debug symbols in Streamline. 83Use vmlinux as the image for debug symbols in Streamline.
68Drivers may be profiled using this method by statically linking the driver into the kernel image. 84Drivers may be profiled using this method by statically linking the driver into the kernel image or adding the module as an image.
69Note that the gator driver does not perform kernel call stack recording. 85Note that the gator driver does not perform kernel call stack recording.
70 86
71*** Automatically start gator on boot (optional) *** 87*** Automatically start gator on boot (optional) ***
@@ -78,4 +94,4 @@ update-rc.d rungator.sh defaults
78 94
79*** GPL License *** 95*** GPL License ***
80 96
81For license information, please see the file LICENSE. 97For license information, please see the file LICENSE after unzipping driver-src/gator-driver.tar.gz.
diff --git a/driver/gator.h b/driver/gator.h
index 9d802a3..724ae19 100644
--- a/driver/gator.h
+++ b/driver/gator.h
@@ -14,6 +14,16 @@
14#include <linux/mm.h> 14#include <linux/mm.h>
15#include <linux/list.h> 15#include <linux/list.h>
16 16
17// cpu ids
18#define ARM1136 0xb36
19#define ARM1156 0xb56
20#define ARM1176 0xb76
21#define ARM11MPCORE 0xb02
22#define CORTEX_A5 0xc05
23#define CORTEX_A8 0xc08
24#define CORTEX_A9 0xc09
25#define CORTEX_A15 0xc0f
26
17/****************************************************************************** 27/******************************************************************************
18 * Filesystem 28 * Filesystem
19 ******************************************************************************/ 29 ******************************************************************************/
@@ -62,6 +72,7 @@ struct gator_interface {
62 void (*online)(void); 72 void (*online)(void);
63 void (*offline)(void); 73 void (*offline)(void);
64 int (*read)(int **buffer); 74 int (*read)(int **buffer);
75 int (*read64)(long long **buffer);
65 struct list_head list; 76 struct list_head list;
66}; 77};
67 78
@@ -75,5 +86,4 @@ extern u32 gator_cpuid(void);
75 86
76extern unsigned long gator_net_traffic; 87extern unsigned long gator_net_traffic;
77 88
78
79#endif // GATOR_H_ 89#endif // GATOR_H_
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index d8d86af..e096d15 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -22,6 +22,7 @@ static char *annotateBuf0;
22static char *annotateBuf1; 22static char *annotateBuf1;
23static int annotatePos; 23static int annotatePos;
24static int annotateSel; 24static int annotateSel;
25static bool collect_annotations = false;
25 26
26static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) 27static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
27{ 28{
@@ -45,16 +46,13 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
45 if (retval == 0) { 46 if (retval == 0) {
46 // synchronize shared variables annotateBuf and annotatePos 47 // synchronize shared variables annotateBuf and annotatePos
47 spin_lock(&annotate_lock); 48 spin_lock(&annotate_lock);
48 if (annotateBuf) { 49 if (collect_annotations && annotateBuf) {
49 uint32_t tid = current->pid; 50 uint32_t tid = current->pid;
50 uint32_t tick = gator_master_tick;
51 uint64_t time = gator_get_time(); 51 uint64_t time = gator_get_time();
52 uint32_t cpuid = smp_processor_id(); 52 uint32_t cpuid = smp_processor_id();
53 int pos = annotatePos; 53 int pos = annotatePos;
54 pos += gator_write_packed_int(&annotateBuf[pos], tid); 54 pos += gator_write_packed_int(&annotateBuf[pos], tid);
55 pos += gator_write_packed_int(&annotateBuf[pos], tick); 55 pos += gator_write_packed_int64(&annotateBuf[pos], time);
56 pos += gator_write_packed_int(&annotateBuf[pos], time);
57 pos += gator_write_packed_int(&annotateBuf[pos], time >> 32);
58 pos += gator_write_packed_int(&annotateBuf[pos], cpuid); 56 pos += gator_write_packed_int(&annotateBuf[pos], cpuid);
59 pos += gator_write_packed_int(&annotateBuf[pos], size); 57 pos += gator_write_packed_int(&annotateBuf[pos], size);
60 memcpy(&annotateBuf[pos], tempBuffer, size); 58 memcpy(&annotateBuf[pos], tempBuffer, size);
@@ -81,12 +79,9 @@ static int annotate_release(struct inode *inode, struct file *file)
81 spin_lock(&annotate_lock); 79 spin_lock(&annotate_lock);
82 if (annotateBuf) { 80 if (annotateBuf) {
83 uint32_t tid = current->pid; 81 uint32_t tid = current->pid;
84 uint32_t tick = gator_master_tick;
85 int pos = annotatePos; 82 int pos = annotatePos;
86 pos += gator_write_packed_int(&annotateBuf[pos], tid); 83 pos += gator_write_packed_int(&annotateBuf[pos], tid);
87 pos += gator_write_packed_int(&annotateBuf[pos], tick); 84 pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time
88 pos += gator_write_packed_int(&annotateBuf[pos], 0); // time
89 pos += gator_write_packed_int(&annotateBuf[pos], 0); // time
90 pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid 85 pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid
91 pos += gator_write_packed_int(&annotateBuf[pos], 0); // size 86 pos += gator_write_packed_int(&annotateBuf[pos], 0); // size
92 annotatePos = pos; 87 annotatePos = pos;
@@ -118,13 +113,21 @@ static int gator_annotate_init(void)
118 113
119static int gator_annotate_start(void) 114static int gator_annotate_start(void)
120{ 115{
121 annotatePos = annotateSel = 0; 116 annotateSel = 0;
117 annotatePos = 1;
122 annotateBuf = annotateBuf0; 118 annotateBuf = annotateBuf0;
119 annotateBuf[0] = FRAME_ANNOTATE;
120 collect_annotations = true;
123 return 0; 121 return 0;
124} 122}
125 123
126static void gator_annotate_stop(void) 124static void gator_annotate_stop(void)
127{ 125{
126 collect_annotations = false;
127}
128
129static void gator_annotate_shutdown(void)
130{
128 spin_lock(&annotate_lock); 131 spin_lock(&annotate_lock);
129 annotateBuf = NULL; 132 annotateBuf = NULL;
130 spin_unlock(&annotate_lock); 133 spin_unlock(&annotate_lock);
@@ -141,7 +144,7 @@ static void gator_annotate_exit(void)
141 144
142static int gator_annotate_ready(void) 145static int gator_annotate_ready(void)
143{ 146{
144 return annotatePos && annotateBuf; 147 return annotatePos > 1 && annotateBuf;
145} 148}
146 149
147static int gator_annotate_read(char **buffer) 150static int gator_annotate_read(char **buffer)
@@ -158,8 +161,9 @@ static int gator_annotate_read(char **buffer)
158 161
159 spin_lock(&annotate_lock); 162 spin_lock(&annotate_lock);
160 len = annotatePos; 163 len = annotatePos;
161 annotatePos = 0;
162 annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; 164 annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0;
165 annotateBuf[0] = FRAME_ANNOTATE;
166 annotatePos = 1;
163 spin_unlock(&annotate_lock); 167 spin_unlock(&annotate_lock);
164 168
165 return len; 169 return len;
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index 628a18f..b5b0e63 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -15,7 +15,7 @@ struct frame_tail_eabi {
15 unsigned long lr; 15 unsigned long lr;
16}; 16};
17 17
18static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned int depth) 18static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs, unsigned int depth)
19{ 19{
20#if defined(__arm__) 20#if defined(__arm__)
21 struct frame_tail_eabi *tail; 21 struct frame_tail_eabi *tail;
@@ -31,7 +31,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in
31 } 31 }
32 32
33 /* entry preamble may not have executed */ 33 /* entry preamble may not have executed */
34 gator_add_trace(cpu, lr); 34 gator_add_trace(cpu, buftype, lr);
35 35
36 /* check tail is valid */ 36 /* check tail is valid */
37 if (fp == 0) { 37 if (fp == 0) {
@@ -49,7 +49,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in
49 ptrtail = &buftail; 49 ptrtail = &buftail;
50 50
51 lr = ptrtail[0].lr; 51 lr = ptrtail[0].lr;
52 gator_add_trace(cpu, lr); 52 gator_add_trace(cpu, buftype, lr);
53 53
54 /* frame pointers should progress back up the stack, towards higher addresses */ 54 /* frame pointers should progress back up the stack, towards higher addresses */
55 next = (struct frame_tail_eabi *)(lr - 4); 55 next = (struct frame_tail_eabi *)(lr - 4);
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index 64be841..a646fb2 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, translate_buffer_read);
22static DEFINE_PER_CPU(int, translate_buffer_write); 22static DEFINE_PER_CPU(int, translate_buffer_write);
23static DEFINE_PER_CPU(unsigned int *, translate_buffer); 23static DEFINE_PER_CPU(unsigned int *, translate_buffer);
24 24
25static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod); 25static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt);
26static void wq_cookie_handler(struct work_struct *unused); 26static void wq_cookie_handler(struct work_struct *unused);
27DECLARE_WORK(cookie_work, wq_cookie_handler); 27DECLARE_WORK(cookie_work, wq_cookie_handler);
28 28
@@ -96,7 +96,7 @@ static void cookiemap_add(uint64_t key, uint32_t value) {
96 96
97 for (x = MAX_COLLISIONS-1; x > 0; x--) { 97 for (x = MAX_COLLISIONS-1; x > 0; x--) {
98 keys[x] = keys[x-1]; 98 keys[x] = keys[x-1];
99 values[x] = keys[x-1]; 99 values[x] = values[x-1];
100 } 100 }
101 keys[0] = key; 101 keys[0] = key;
102 values[0] = value; 102 values[0] = value;
@@ -126,12 +126,12 @@ static void wq_cookie_handler(struct work_struct *unused)
126 while (per_cpu(translate_buffer_read, cpu) != commit) { 126 while (per_cpu(translate_buffer_read, cpu) != commit) {
127 task = (struct task_struct *)translate_buffer_read_int(cpu); 127 task = (struct task_struct *)translate_buffer_read_int(cpu);
128 vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); 128 vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
129 cookie = get_cookie(cpu, task, vma, NULL); 129 cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false);
130 } 130 }
131} 131}
132 132
133// Retrieve full name from proc/pid/cmdline for java processes on Android 133// Retrieve full name from proc/pid/cmdline for java processes on Android
134static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma) 134static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt)
135{ 135{
136 void *maddr; 136 void *maddr;
137 unsigned int len; 137 unsigned int len;
@@ -143,7 +143,9 @@ static int translate_app_process(char** text, int cpu, struct task_struct * task
143 char * buf = per_cpu(translate_text, cpu); 143 char * buf = per_cpu(translate_text, cpu);
144 144
145 // Push work into a work queue if in atomic context as the kernel functions below might sleep 145 // Push work into a work queue if in atomic context as the kernel functions below might sleep
146 if (in_irq()) { 146 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
147 // inconsistent during a context switch between android/linux versions
148 if (in_interrupt) {
147 // Check if already in buffer 149 // Check if already in buffer
148 ptr = per_cpu(translate_buffer_read, cpu); 150 ptr = per_cpu(translate_buffer_read, cpu);
149 while (ptr != per_cpu(translate_buffer_write, cpu)) { 151 while (ptr != per_cpu(translate_buffer_write, cpu)) {
@@ -205,7 +207,7 @@ out:
205 return retval; 207 return retval;
206} 208}
207 209
208static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod) 210static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt)
209{ 211{
210 unsigned long flags, cookie; 212 unsigned long flags, cookie;
211 struct path *path; 213 struct path *path;
@@ -235,26 +237,26 @@ static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_a
235 } 237 }
236 238
237 if (strcmp(text, "app_process") == 0 && !mod) { 239 if (strcmp(text, "app_process") == 0 && !mod) {
238 if (!translate_app_process(&text, cpu, task, vma)) 240 if (!translate_app_process(&text, cpu, task, vma, in_interrupt))
239 return INVALID_COOKIE; 241 return INVALID_COOKIE;
240 } 242 }
241 243
242 // Can be called from interrupt handler or from work queue 244 // Can be called from interrupt handler or from work queue or from scheduler trace
243 local_irq_save(flags); 245 local_irq_save(flags);
244 246
245 cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; 247 cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids;
246 cookiemap_add(key, cookie); 248 cookiemap_add(key, cookie);
247 249
248 gator_buffer_write_packed_int(cpu, PROTOCOL_COOKIE); 250 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE);
249 gator_buffer_write_packed_int(cpu, cookie); 251 gator_buffer_write_packed_int(cpu, buftype, cookie);
250 gator_buffer_write_string(cpu, text); 252 gator_buffer_write_string(cpu, buftype, text);
251 253
252 local_irq_restore(flags); 254 local_irq_restore(flags);
253 255
254 return cookie; 256 return cookie;
255} 257}
256 258
257static int get_exec_cookie(int cpu, struct task_struct *task) 259static int get_exec_cookie(int cpu, int buftype, struct task_struct *task)
258{ 260{
259 unsigned long cookie = NO_COOKIE; 261 unsigned long cookie = NO_COOKIE;
260 struct mm_struct *mm = task->mm; 262 struct mm_struct *mm = task->mm;
@@ -268,14 +270,14 @@ static int get_exec_cookie(int cpu, struct task_struct *task)
268 continue; 270 continue;
269 if (!(vma->vm_flags & VM_EXECUTABLE)) 271 if (!(vma->vm_flags & VM_EXECUTABLE))
270 continue; 272 continue;
271 cookie = get_cookie(cpu, task, vma, NULL); 273 cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
272 break; 274 break;
273 } 275 }
274 276
275 return cookie; 277 return cookie;
276} 278}
277 279
278static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset) 280static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset)
279{ 281{
280 unsigned long cookie = NO_COOKIE; 282 unsigned long cookie = NO_COOKIE;
281 struct mm_struct *mm = task->mm; 283 struct mm_struct *mm = task->mm;
@@ -289,7 +291,7 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig
289 continue; 291 continue;
290 292
291 if (vma->vm_file) { 293 if (vma->vm_file) {
292 cookie = get_cookie(cpu, task, vma, NULL); 294 cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
293 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; 295 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
294 } else { 296 } else {
295 /* must be an anonymous map */ 297 /* must be an anonymous map */
@@ -318,10 +320,18 @@ static int cookies_initialize(void)
318 320
319 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); 321 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
320 per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); 322 per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
323 if (!per_cpu(cookie_keys, cpu)) {
324 err = -ENOMEM;
325 goto cookie_setup_error;
326 }
321 memset(per_cpu(cookie_keys, cpu), 0, size); 327 memset(per_cpu(cookie_keys, cpu), 0, size);
322 328
323 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); 329 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
324 per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL); 330 per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
331 if (!per_cpu(cookie_values, cpu)) {
332 err = -ENOMEM;
333 goto cookie_setup_error;
334 }
325 memset(per_cpu(cookie_values, cpu), 0, size); 335 memset(per_cpu(cookie_values, cpu), 0, size);
326 336
327 per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL); 337 per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL);
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c
new file mode 100644
index 0000000..9b55347
--- /dev/null
+++ b/driver/gator_ebs.c
@@ -0,0 +1,160 @@
1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10/******************************************************************************
11 * event based sampling handling
12 ******************************************************************************/
13
14#if defined (__arm__)
15#include "gator_events_armv7.h"
16#include <linux/platform_device.h>
17#include <linux/interrupt.h>
18#include <linux/irq.h>
19
20#if LINUX_PMU_SUPPORT
21#include <asm/pmu.h>
22
23static struct platform_device *pmu_device;
24
25static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg)
26{
27 unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF;
28 struct pt_regs * const regs = get_irq_regs();
29 u32 flags;
30
31 // Stop irq generation
32 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
33
34 // Get and reset overflow status flags
35 flags = armv7_pmnc_reset_interrupt();
36
37 // Counters header
38 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
39 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
40
41 // Cycle counter
42 if (flags & (1 << 31)) {
43 int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun
44 gator_buffer_write_packed_int(cpu, buftype, 2); // length
45 gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key
46 gator_buffer_write_packed_int(cpu, buftype, value); // value
47 }
48
49 // PMNC counters
50 for (cnt = CNT0; cnt < CNTMAX; cnt++) {
51 if (flags & (1 << (cnt - CNT0))) {
52 int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun
53 gator_buffer_write_packed_int(cpu, buftype, 2); // length
54 gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key
55 gator_buffer_write_packed_int(cpu, buftype, value); // value
56 }
57 }
58
59 // End Counters, length of zero
60 gator_buffer_write_packed_int(cpu, buftype, 0);
61
62 // Output backtrace
63 gator_add_sample(cpu, buftype, regs);
64
65 // Check and commit; commit is set to occur once buffer is 3/4 full
66 event_buffer_check(cpu);
67
68 // Allow irq generation
69 armv7_pmnc_write(armv7_pmnc_read() | PMNC_E);
70
71 return IRQ_HANDLED;
72}
73#endif
74
75static int gator_event_sampling_start(void)
76{
77 int cnt;
78
79 event_based_sampling = false;
80 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
81 if (pmnc_count[cnt] > 0) {
82 event_based_sampling = true;
83 break;
84 }
85 }
86
87#if LINUX_PMU_SUPPORT
88 pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU);
89 if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) {
90 pr_err("gator: unable to reserve the pmu\n");
91 return -1;
92 }
93
94 if (event_based_sampling) {
95 int irq, i;
96
97 if (IS_ERR(pmu_device)) {
98 pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed");
99 return -1;
100 }
101
102 init_pmu(ARM_PMU_DEVICE_CPU);
103 if (pmu_device->num_resources == 0) {
104 pr_err("gator: no irqs for PMUs defined\n");
105 release_pmu(pmu_device);
106 pmu_device = NULL;
107 return -1;
108 }
109
110 for (i = 0; i < pmu_device->num_resources; ++i) {
111 irq = platform_get_irq(pmu_device, i);
112 if (irq < 0)
113 continue;
114
115 if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) {
116 pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq);
117
118 // clean up and exit
119 for (i = i - 1; i >= 0; --i) {
120 irq = platform_get_irq(pmu_device, i);
121 if (irq >= 0)
122 free_irq(irq, NULL);
123 }
124 release_pmu(pmu_device);
125 pmu_device = NULL;
126 return -1;
127 }
128 }
129 }
130#else
131 if (event_based_sampling) {
132 pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n");
133 return -1;
134 }
135#endif
136
137 return 0;
138}
139
140static void gator_event_sampling_stop(void)
141{
142#if LINUX_PMU_SUPPORT
143 if (event_based_sampling) {
144 int i, irq;
145 for (i = pmu_device->num_resources - 1; i >= 0; --i) {
146 irq = platform_get_irq(pmu_device, i);
147 if (irq >= 0)
148 free_irq(irq, NULL);
149 }
150 }
151 if (!IS_ERR(pmu_device))
152 release_pmu(pmu_device);
153 pmu_device = NULL;
154#endif
155}
156
157#else
158static int gator_event_sampling_start(void) {return 0;}
159static void gator_event_sampling_stop(void) {}
160#endif
diff --git a/driver/gator_events.c b/driver/gator_events.c
deleted file mode 100644
index d837257..0000000
--- a/driver/gator_events.c
+++ /dev/null
@@ -1,40 +0,0 @@
1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10/**
11 * This file is #included in gator_main.c
12 * Update this file and Makefile to add custom counters.
13 */
14
15extern int gator_events_armv6_install(gator_interface *gi);
16extern int gator_events_armv7_install(gator_interface *gi);
17extern int gator_events_irq_install(gator_interface *gi);
18extern int gator_events_sched_install(gator_interface *gi);
19extern int gator_events_block_install(gator_interface *gi);
20extern int gator_events_meminfo_install(gator_interface *gi);
21extern int gator_events_net_install(gator_interface *gi);
22
23static int gator_events_install(void)
24{
25 if (gator_event_install(gator_events_armv6_install))
26 return -1;
27 if (gator_event_install(gator_events_armv7_install))
28 return -1;
29 if (gator_event_install(gator_events_irq_install))
30 return -1;
31 if (gator_event_install(gator_events_sched_install))
32 return -1;
33 if (gator_event_install(gator_events_block_install))
34 return -1;
35 if (gator_event_install(gator_events_meminfo_install))
36 return -1;
37 if (gator_event_install(gator_events_net_install))
38 return -1;
39 return 0;
40}
diff --git a/driver/gator_events.h b/driver/gator_events.h
deleted file mode 100644
index 8237c1a..0000000
--- a/driver/gator_events.h
+++ /dev/null
@@ -1,19 +0,0 @@
1/**
2 * Copyright 2010 ARM, Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9
10struct __gator_interface {
11 int (*create_files)(struct super_block *sb, struct dentry *root);
12 int (*init)(int *key);
13 int (*start)(void);
14 void (*stop)(void);
15 int (*read)(int **buffer);
16 struct __gator_interface *next;
17};
18
19typedef struct __gator_interface gator_interface;
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c
index 7b1d875..170f066 100644
--- a/driver/gator_events_armv6.c
+++ b/driver/gator_events_armv6.c
@@ -8,10 +8,6 @@
8 8
9#include "gator.h" 9#include "gator.h"
10 10
11#define ARM1136 0xb36
12#define ARM1156 0xb56
13#define ARM1176 0xb76
14
15static const char *pmnc_name; 11static const char *pmnc_name;
16 12
17/* 13/*
@@ -29,9 +25,10 @@ static const char *pmnc_name;
29#define CCNT 2 25#define CCNT 2
30#define CNTMAX (CCNT+1) 26#define CNTMAX (CCNT+1)
31 27
32static int pmnc_count = 0; 28static int pmnc_counters = 0;
33static unsigned long pmnc_enabled[CNTMAX]; 29static unsigned long pmnc_enabled[CNTMAX];
34static unsigned long pmnc_event[CNTMAX]; 30static unsigned long pmnc_event[CNTMAX];
31static unsigned long pmnc_count[CNTMAX];
35static unsigned long pmnc_key[CNTMAX]; 32static unsigned long pmnc_key[CNTMAX];
36 33
37static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 34static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -72,7 +69,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
72 struct dentry *dir; 69 struct dentry *dir;
73 int i; 70 int i;
74 71
75 pmnc_count = 3; 72 pmnc_counters = 3;
76 73
77 for (i = PMN0; i <= CCNT; i++) { 74 for (i = PMN0; i <= CCNT; i++) {
78 char buf[40]; 75 char buf[40];
@@ -86,10 +83,11 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
86 return -1; 83 return -1;
87 } 84 }
88 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 85 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
86 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
87 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
89 if (i != CCNT) { 88 if (i != CCNT) {
90 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 89 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
91 } 90 }
92 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
93 } 91 }
94 92
95 return 0; 93 return 0;
@@ -119,18 +117,14 @@ static void gator_events_armv6_online(void)
119 117
120 event = pmnc_event[cnt] & 255; 118 event = pmnc_event[cnt] & 255;
121 119
122 /* 120 // Set event (if destined for PMNx counters)
123 * Set event (if destined for PMNx counters)
124 */
125 if (cnt == PMN0) { 121 if (cnt == PMN0) {
126 pmnc |= event << 20; 122 pmnc |= event << 20;
127 } else if (cnt == PMN1) { 123 } else if (cnt == PMN1) {
128 pmnc |= event << 12; 124 pmnc |= event << 12;
129 } 125 }
130 126
131 /* 127 // Reset counter
132 * Reset counter
133 */
134 armv6_pmnc_reset_counter(cnt); 128 armv6_pmnc_reset_counter(cnt);
135 } 129 }
136 armv6_pmnc_write(pmnc | PMCR_E); 130 armv6_pmnc_write(pmnc | PMCR_E);
@@ -146,6 +140,20 @@ static void gator_events_armv6_offline(void)
146 } 140 }
147} 141}
148 142
143static int gator_events_armv6_start(void)
144{
145 int cnt;
146
147 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
148 if (pmnc_count[cnt] > 0) {
149 pr_err("gator: event based sampling not supported on ARM v6 architectures\n");
150 return -1;
151 }
152 }
153
154 return 0;
155}
156
149static void gator_events_armv6_stop(void) 157static void gator_events_armv6_stop(void)
150{ 158{
151 unsigned int cnt; 159 unsigned int cnt;
@@ -153,6 +161,7 @@ static void gator_events_armv6_stop(void)
153 for (cnt = PMN0; cnt <= CCNT; cnt++) { 161 for (cnt = PMN0; cnt <= CCNT; cnt++) {
154 pmnc_enabled[cnt] = 0; 162 pmnc_enabled[cnt] = 0;
155 pmnc_event[cnt] = 0; 163 pmnc_event[cnt] = 0;
164 pmnc_count[cnt] = 0;
156 } 165 }
157} 166}
158 167
@@ -193,6 +202,7 @@ static int gator_events_armv6_read(int **buffer)
193 202
194static struct gator_interface gator_events_armv6_interface = { 203static struct gator_interface gator_events_armv6_interface = {
195 .create_files = gator_events_armv6_create_files, 204 .create_files = gator_events_armv6_create_files,
205 .start = gator_events_armv6_start,
196 .stop = gator_events_armv6_stop, 206 .stop = gator_events_armv6_stop,
197 .online = gator_events_armv6_online, 207 .online = gator_events_armv6_online,
198 .offline = gator_events_armv6_offline, 208 .offline = gator_events_armv6_offline,
@@ -209,6 +219,9 @@ int gator_events_armv6_init(void)
209 case ARM1176: 219 case ARM1176:
210 pmnc_name = "ARM11"; 220 pmnc_name = "ARM11";
211 break; 221 break;
222 case ARM11MPCORE:
223 pmnc_name = "ARM11MPCore";
224 break;
212 default: 225 default:
213 return -1; 226 return -1;
214 } 227 }
@@ -216,6 +229,7 @@ int gator_events_armv6_init(void)
216 for (cnt = PMN0; cnt <= CCNT; cnt++) { 229 for (cnt = PMN0; cnt <= CCNT; cnt++) {
217 pmnc_enabled[cnt] = 0; 230 pmnc_enabled[cnt] = 0;
218 pmnc_event[cnt] = 0; 231 pmnc_event[cnt] = 0;
232 pmnc_count[cnt] = 0;
219 pmnc_key[cnt] = gator_events_get_key(); 233 pmnc_key[cnt] = gator_events_get_key();
220 } 234 }
221 235
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index 0b84745..58855f8 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -6,129 +6,116 @@
6 * published by the Free Software Foundation. 6 * published by the Free Software Foundation.
7 */ 7 */
8 8
9#include "gator.h" 9/* Disabling interrupts
10 10 * Many of the functions below disable interrupts via local_irq_save(). This disabling of interrupts is done to prevent any race conditions
11#define CORTEX_A5 0xc05 11 * between multiple entities (e.g. hrtimer interrupts and event based interrupts) calling the same functions. As accessing the pmu involves
12#define CORTEX_A8 0xc08 12 * several steps (disable, select, read, enable), these steps must be performed atomically. Normal synchronization routines cannot be used
13#define CORTEX_A9 0xc09 13 * as these functions are being called from interrupt context.
14#define CORTEX_A15 0xc0f 14 */
15
16static const char *pmnc_name;
17static int pmnc_count;
18
19// Per-CPU PMNC: config reg
20#define PMNC_E (1 << 0) /* Enable all counters */
21#define PMNC_P (1 << 1) /* Reset all counters */
22#define PMNC_C (1 << 2) /* Cycle counter reset */
23#define PMNC_MASK 0x3f /* Mask for writable bits */
24 15
25// ccnt reg 16#include "gator.h"
26#define CCNT_REG (1 << 31) 17#include "gator_events_armv7.h"
27 18
28#define CCNT 0 19const char *pmnc_name;
29#define CNT0 1 20int pmnc_counters;
30#define CNTMAX (6+1)
31 21
32static unsigned long pmnc_enabled[CNTMAX]; 22unsigned long pmnc_enabled[CNTMAX];
33static unsigned long pmnc_event[CNTMAX]; 23unsigned long pmnc_event[CNTMAX];
34static unsigned long pmnc_key[CNTMAX]; 24unsigned long pmnc_count[CNTMAX];
25unsigned long pmnc_key[CNTMAX];
35 26
36static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 27static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
37static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); 28static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
38 29
39static inline void armv7_pmnc_write(u32 val) 30inline void armv7_pmnc_write(u32 val)
40{ 31{
41 val &= PMNC_MASK; 32 val &= PMNC_MASK;
42 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); 33 asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val));
43} 34}
44 35
45static inline u32 armv7_pmnc_read(void) 36inline u32 armv7_pmnc_read(void)
46{ 37{
47 u32 val; 38 u32 val;
48 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); 39 asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
49 return val; 40 return val;
50} 41}
51 42
52static inline u32 armv7_ccnt_read(void) 43inline u32 armv7_ccnt_read(u32 reset_value)
53{ 44{
54 u32 zero = 0; 45 unsigned long flags;
46 u32 newval = -reset_value;
55 u32 den = CCNT_REG; 47 u32 den = CCNT_REG;
56 u32 val; 48 u32 val;
57 49
50 local_irq_save(flags);
58 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable 51 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable
59 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read 52 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read
60 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (zero)); // zero 53 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (newval));// new value
61 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable 54 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable
55 local_irq_restore(flags);
62 56
63 return val; 57 return val;
64} 58}
65 59
66static inline u32 armv7_cntn_read(unsigned int cnt) 60inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value)
67{ 61{
68 u32 zero = 0; 62 unsigned long flags;
63 u32 newval = -reset_value;
69 u32 sel = (cnt - CNT0); 64 u32 sel = (cnt - CNT0);
70 u32 den = 1 << sel; 65 u32 den = 1 << sel;
71 u32 val; 66 u32 oldval;
72 67
68 local_irq_save(flags);
73 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable 69 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable
74 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select 70 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select
75 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); // read 71 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (oldval)); // read
76 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (zero)); // zero 72 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (newval));// new value
77 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable 73 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable
74 local_irq_restore(flags);
78 75
79 return val; 76 return oldval;
80} 77}
81 78
82static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) 79static inline void armv7_pmnc_enable_interrupt(unsigned int cnt)
83{ 80{
84 u32 val; 81 u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
82 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
83}
85 84
86 if (cnt >= CNTMAX) { 85static inline void armv7_pmnc_disable_interrupt(unsigned int cnt)
87 pr_err("gator: CPU%u enabling wrong PMNC counter %d\n", smp_processor_id(), cnt); 86{
88 return -1; 87 u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
89 } 88 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
89}
90 90
91 if (cnt == CCNT) 91inline u32 armv7_pmnc_reset_interrupt()
92 val = CCNT_REG; 92{
93 else 93 // Get and reset overflow status flags
94 val = (1 << (cnt - CNT0)); 94 u32 flags;
95 asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (flags));
96 flags &= 0x8000003f;
97 asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (flags));
98 return flags;
99}
95 100
101static inline u32 armv7_pmnc_enable_counter(unsigned int cnt)
102{
103 u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG;
96 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); 104 asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
97
98 return cnt; 105 return cnt;
99} 106}
100 107
101static inline u32 armv7_pmnc_disable_counter(unsigned int cnt) 108static inline u32 armv7_pmnc_disable_counter(unsigned int cnt)
102{ 109{
103 u32 val; 110 u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG;
104
105 if (cnt >= CNTMAX) {
106 pr_err("gator: CPU%u disabling wrong PMNC counter %d\n", smp_processor_id(), cnt);
107 return -1;
108 }
109
110 if (cnt == CCNT)
111 val = CCNT_REG;
112 else
113 val = (1 << (cnt - CNT0));
114
115 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); 111 asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
116
117 return cnt; 112 return cnt;
118} 113}
119 114
120static inline int armv7_pmnc_select_counter(unsigned int cnt) 115static inline int armv7_pmnc_select_counter(unsigned int cnt)
121{ 116{
122 u32 val; 117 u32 val = (cnt - CNT0);
123
124 if ((cnt == CCNT) || (cnt >= CNTMAX)) {
125 pr_err("gator: CPU%u selecting wrong PMNC counter %d\n", smp_processor_id(), cnt);
126 return -1;
127 }
128
129 val = (cnt - CNT0);
130 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); 118 asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
131
132 return cnt; 119 return cnt;
133} 120}
134 121
@@ -139,37 +126,12 @@ static inline void armv7_pmnc_write_evtsel(unsigned int cnt, u32 val)
139 } 126 }
140} 127}
141 128
142static void armv7_pmnc_reset_counter(unsigned int cnt)
143{
144 u32 val = 0;
145
146 if (cnt == CCNT) {
147 armv7_pmnc_disable_counter(cnt);
148
149 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (val));
150
151 if (pmnc_enabled[cnt] != 0)
152 armv7_pmnc_enable_counter(cnt);
153
154 } else if (cnt >= CNTMAX) {
155 pr_err("gator: CPU%u resetting wrong PMNC counter %d\n", smp_processor_id(), cnt);
156 } else {
157 armv7_pmnc_disable_counter(cnt);
158
159 if (armv7_pmnc_select_counter(cnt) == cnt)
160 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (val));
161
162 if (pmnc_enabled[cnt] != 0)
163 armv7_pmnc_enable_counter(cnt);
164 }
165}
166
167static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root) 129static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root)
168{ 130{
169 struct dentry *dir; 131 struct dentry *dir;
170 int i; 132 int i;
171 133
172 for (i = 0; i < pmnc_count; i++) { 134 for (i = 0; i < pmnc_counters; i++) {
173 char buf[40]; 135 char buf[40];
174 if (i == 0) { 136 if (i == 0) {
175 snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); 137 snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name);
@@ -181,10 +143,11 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry
181 return -1; 143 return -1;
182 } 144 }
183 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 145 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
146 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
147 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
184 if (i > 0) { 148 if (i > 0) {
185 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 149 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
186 } 150 }
187 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
188 } 151 }
189 152
190 return 0; 153 return 0;
@@ -202,6 +165,9 @@ static void gator_events_armv7_online(void)
202 // Initialize & Reset PMNC: C bit and P bit 165 // Initialize & Reset PMNC: C bit and P bit
203 armv7_pmnc_write(PMNC_P | PMNC_C); 166 armv7_pmnc_write(PMNC_P | PMNC_C);
204 167
168 // Reset overflow flags
169 armv7_pmnc_reset_interrupt();
170
205 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 171 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
206 unsigned long event; 172 unsigned long event;
207 173
@@ -219,10 +185,16 @@ static void gator_events_armv7_online(void)
219 if (cnt != CCNT) 185 if (cnt != CCNT)
220 armv7_pmnc_write_evtsel(cnt, event); 186 armv7_pmnc_write_evtsel(cnt, event);
221 187
188 // Enable/disable interrupt
189 if (pmnc_count[cnt] > 0)
190 armv7_pmnc_enable_interrupt(cnt);
191 else
192 armv7_pmnc_disable_interrupt(cnt);
193
222 // Reset counter 194 // Reset counter
223 armv7_pmnc_reset_counter(cnt); 195 cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]);
224 196
225 // Enable counter, but do not enable interrupt for this counter 197 // Enable counter
226 armv7_pmnc_enable_counter(cnt); 198 armv7_pmnc_enable_counter(cnt);
227 } 199 }
228 200
@@ -242,6 +214,7 @@ static void gator_events_armv7_stop(void)
242 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 214 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
243 pmnc_enabled[cnt] = 0; 215 pmnc_enabled[cnt] = 0;
244 pmnc_event[cnt] = 0; 216 pmnc_event[cnt] = 0;
217 pmnc_count[cnt] = 0;
245 } 218 }
246} 219}
247 220
@@ -250,16 +223,16 @@ static int gator_events_armv7_read(int **buffer)
250 int cnt, len = 0; 223 int cnt, len = 0;
251 int cpu = smp_processor_id(); 224 int cpu = smp_processor_id();
252 225
253 if (!pmnc_count) 226 if (!pmnc_counters)
254 return 0; 227 return 0;
255 228
256 for (cnt = 0; cnt < pmnc_count; cnt++) { 229 for (cnt = 0; cnt < pmnc_counters; cnt++) {
257 if (pmnc_enabled[cnt]) { 230 if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) {
258 int value; 231 int value;
259 if (cnt == CCNT) { 232 if (cnt == CCNT) {
260 value = armv7_ccnt_read(); 233 value = armv7_ccnt_read(0);
261 } else { 234 } else {
262 value = armv7_cntn_read(cnt); 235 value = armv7_cntn_read(cnt, 0);
263 } 236 }
264 if (value != per_cpu(perfPrev, cpu)[cnt]) { 237 if (value != per_cpu(perfPrev, cpu)[cnt]) {
265 per_cpu(perfPrev, cpu)[cnt] = value; 238 per_cpu(perfPrev, cpu)[cnt] = value;
@@ -291,29 +264,30 @@ int gator_events_armv7_init(void)
291 switch (gator_cpuid()) { 264 switch (gator_cpuid()) {
292 case CORTEX_A5: 265 case CORTEX_A5:
293 pmnc_name = "Cortex-A5"; 266 pmnc_name = "Cortex-A5";
294 pmnc_count = 2; 267 pmnc_counters = 2;
295 break; 268 break;
296 case CORTEX_A8: 269 case CORTEX_A8:
297 pmnc_name = "Cortex-A8"; 270 pmnc_name = "Cortex-A8";
298 pmnc_count = 4; 271 pmnc_counters = 4;
299 break; 272 break;
300 case CORTEX_A9: 273 case CORTEX_A9:
301 pmnc_name = "Cortex-A9"; 274 pmnc_name = "Cortex-A9";
302 pmnc_count = 6; 275 pmnc_counters = 6;
303 break; 276 break;
304 case CORTEX_A15: 277 case CORTEX_A15:
305 pmnc_name = "Cortex-A15"; 278 pmnc_name = "Cortex-A15";
306 pmnc_count = 6; 279 pmnc_counters = 6;
307 break; 280 break;
308 default: 281 default:
309 return -1; 282 return -1;
310 } 283 }
311 284
312 pmnc_count++; // CNT[n] + CCNT 285 pmnc_counters++; // CNT[n] + CCNT
313 286
314 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 287 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
315 pmnc_enabled[cnt] = 0; 288 pmnc_enabled[cnt] = 0;
316 pmnc_event[cnt] = 0; 289 pmnc_event[cnt] = 0;
290 pmnc_count[cnt] = 0;
317 pmnc_key[cnt] = gator_events_get_key(); 291 pmnc_key[cnt] = gator_events_get_key();
318 } 292 }
319 293
diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h
new file mode 100644
index 0000000..d5e8d6e
--- /dev/null
+++ b/driver/gator_events_armv7.h
@@ -0,0 +1,38 @@
1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#ifndef GATOR_EVENTS_ARMV7_H_
10#define GATOR_EVENTS_ARMV7_H_
11
12// Per-CPU PMNC: config reg
13#define PMNC_E (1 << 0) /* Enable all counters */
14#define PMNC_P (1 << 1) /* Reset all counters */
15#define PMNC_C (1 << 2) /* Cycle counter reset */
16#define PMNC_MASK 0x3f /* Mask for writable bits */
17
18// ccnt reg
19#define CCNT_REG (1 << 31)
20
21#define CCNT 0
22#define CNT0 1
23#define CNTMAX (6+1)
24
25// Function prototypes
26extern void armv7_pmnc_write(u32 val);
27extern u32 armv7_pmnc_read(void);
28extern u32 armv7_ccnt_read(u32 reset_value);
29extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value);
30extern u32 armv7_pmnc_reset_interrupt(void);
31
32// Externed variables
33extern unsigned long pmnc_enabled[CNTMAX];
34extern unsigned long pmnc_event[CNTMAX];
35extern unsigned long pmnc_count[CNTMAX];
36extern unsigned long pmnc_key[CNTMAX];
37
38#endif // GATOR_EVENTS_ARMV7_H_
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c
new file mode 100644
index 0000000..96683b3
--- /dev/null
+++ b/driver/gator_events_l2c-310.c
@@ -0,0 +1,182 @@
1/**
2 * l2c310 (L2 Cache Controller) event counters for gator
3 *
4 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/init.h>
12#include <linux/io.h>
13#include <asm/hardware/cache-l2x0.h>
14
15#include "gator.h"
16
17#define L2C310_COUNTERS_NUM 2
18
19static struct {
20 unsigned long enabled;
21 unsigned long event;
22 unsigned long key;
23} l2c310_counters[L2C310_COUNTERS_NUM];
24
25static int l2c310_buffer[L2C310_COUNTERS_NUM * 2];
26
27static void __iomem *l2c310_base;
28
29
30
31static void gator_events_l2c310_reset_counters(void)
32{
33 u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL);
34
35 val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1;
36
37 writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL);
38}
39
40
41static int gator_events_l2c310_create_files(struct super_block *sb,
42 struct dentry *root)
43{
44 int i;
45
46 for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
47 char buf[16];
48 struct dentry *dir;
49
50 snprintf(buf, sizeof(buf), "L2C-310_cnt%d", i);
51 dir = gatorfs_mkdir(sb, root, buf);
52 if (WARN_ON(!dir))
53 return -1;
54 gatorfs_create_ulong(sb, dir, "enabled",
55 &l2c310_counters[i].enabled);
56 gatorfs_create_ulong(sb, dir, "event",
57 &l2c310_counters[i].event);
58 gatorfs_create_ro_ulong(sb, dir, "key",
59 &l2c310_counters[i].key);
60 }
61
62 return 0;
63}
64
65static int gator_events_l2c310_start(void)
66{
67 static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = {
68 L2X0_EVENT_CNT0_CFG,
69 L2X0_EVENT_CNT1_CFG,
70 };
71 int i;
72
73 /* Counter event sources */
74 for (i = 0; i < L2C310_COUNTERS_NUM; i++)
75 writel((l2c310_counters[i].event & 0xf) << 2,
76 l2c310_base + l2x0_event_cntx_cfg[i]);
77
78 gator_events_l2c310_reset_counters();
79
80 /* Event counter enable */
81 writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL);
82
83 return 0;
84}
85
86static void gator_events_l2c310_stop(void)
87{
88 /* Event counter disable */
89 writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL);
90}
91
92static int gator_events_l2c310_read(int **buffer)
93{
94 static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = {
95 L2X0_EVENT_CNT0_VAL,
96 L2X0_EVENT_CNT1_VAL,
97 };
98 int i;
99 int len = 0;
100
101 if (smp_processor_id())
102 return 0;
103
104 for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
105 if (l2c310_counters[i].enabled) {
106 l2c310_buffer[len++] = l2c310_counters[i].key;
107 l2c310_buffer[len++] = readl(l2c310_base +
108 l2x0_event_cntx_val[i]);
109 }
110 }
111
112 /* l2c310 counters are saturating, not wrapping in case of overflow */
113 gator_events_l2c310_reset_counters();
114
115 if (buffer)
116 *buffer = l2c310_buffer;
117
118 return len;
119}
120
121static struct gator_interface gator_events_l2c310_interface = {
122 .create_files = gator_events_l2c310_create_files,
123 .start = gator_events_l2c310_start,
124 .stop = gator_events_l2c310_stop,
125 .read = gator_events_l2c310_read,
126};
127
128static void __maybe_unused gator_events_l2c310_probe(unsigned long phys)
129{
130 if (l2c310_base)
131 return;
132
133 l2c310_base = ioremap(phys, SZ_4K);
134 if (l2c310_base) {
135 u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID);
136
137 if ((cache_id & 0xff0003c0) != 0x410000c0) {
138 iounmap(l2c310_base);
139 l2c310_base = NULL;
140 }
141 }
142}
143
144int gator_events_l2c310_init(void)
145{
146 int i;
147
148 if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9)
149 return -1;
150
151#if defined(CONFIG_ARCH_EXYNOS4)
152 gator_events_l2c310_probe(0xfe600000);
153#endif
154#if defined(CONFIG_ARCH_S5PV310)
155 gator_events_l2c310_probe(0x10502000);
156#endif
157#if defined(CONFIG_ARCH_OMAP4)
158 gator_events_l2c310_probe(0x48242000);
159#endif
160#if defined(CONFIG_ARCH_TEGRA)
161 gator_events_l2c310_probe(0x50043000);
162#endif
163#if defined(CONFIG_ARCH_U8500)
164 gator_events_l2c310_probe(0xa0412000);
165#endif
166#if defined(CONFIG_ARCH_VEXPRESS)
167 // A9x4 core tile (HBI-0191)
168 gator_events_l2c310_probe(0x1e00a000);
169 // New memory map tiles
170 gator_events_l2c310_probe(0x2c0f0000);
171#endif
172 if (!l2c310_base)
173 return -1;
174
175 for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
176 l2c310_counters[i].enabled = 0;
177 l2c310_counters[i].key = gator_events_get_key();
178 }
179
180 return gator_events_install(&gator_events_l2c310_interface);
181}
182gator_events_init(gator_events_l2c310_init);
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c
index f1595bd..a1a2031 100644
--- a/driver/gator_events_meminfo.c
+++ b/driver/gator_events_meminfo.c
@@ -19,7 +19,7 @@
19static ulong meminfo_global_enabled; 19static ulong meminfo_global_enabled;
20static ulong meminfo_enabled[MEMINFO_TOTAL]; 20static ulong meminfo_enabled[MEMINFO_TOTAL];
21static ulong meminfo_key[MEMINFO_TOTAL]; 21static ulong meminfo_key[MEMINFO_TOTAL];
22static int meminfo_buffer[MEMINFO_TOTAL * 2]; 22static unsigned long long meminfo_buffer[MEMINFO_TOTAL * 2];
23static int meminfo_length = 0; 23static int meminfo_length = 0;
24static unsigned int mem_event = 0; 24static unsigned int mem_event = 0;
25static bool new_data_avail; 25static bool new_data_avail;
@@ -120,7 +120,8 @@ static void gator_events_meminfo_stop(void)
120static void wq_sched_handler(struct work_struct *wsptr) 120static void wq_sched_handler(struct work_struct *wsptr)
121{ 121{
122 struct sysinfo info; 122 struct sysinfo info;
123 int i, len, value; 123 int i, len;
124 unsigned long long value;
124 125
125 meminfo_length = len = 0; 126 meminfo_length = len = 0;
126 127
@@ -141,7 +142,7 @@ static void wq_sched_handler(struct work_struct *wsptr)
141 value = 0; 142 value = 0;
142 break; 143 break;
143 } 144 }
144 meminfo_buffer[len++] = meminfo_key[i]; 145 meminfo_buffer[len++] = (unsigned long long)meminfo_key[i];
145 meminfo_buffer[len++] = value; 146 meminfo_buffer[len++] = value;
146 } 147 }
147 } 148 }
@@ -150,7 +151,7 @@ static void wq_sched_handler(struct work_struct *wsptr)
150 new_data_avail = true; 151 new_data_avail = true;
151} 152}
152 153
153static int gator_events_meminfo_read(int **buffer) 154static int gator_events_meminfo_read(long long **buffer)
154{ 155{
155 static unsigned int last_mem_event = 0; 156 static unsigned int last_mem_event = 0;
156 157
@@ -177,7 +178,7 @@ static struct gator_interface gator_events_meminfo_interface = {
177 .create_files = gator_events_meminfo_create_files, 178 .create_files = gator_events_meminfo_create_files,
178 .start = gator_events_meminfo_start, 179 .start = gator_events_meminfo_start,
179 .stop = gator_events_meminfo_stop, 180 .stop = gator_events_meminfo_stop,
180 .read = gator_events_meminfo_read, 181 .read64 = gator_events_meminfo_read,
181}; 182};
182 183
183int gator_events_meminfo_init(void) 184int gator_events_meminfo_init(void)
diff --git a/driver/gator_events_pl310.c b/driver/gator_events_pl310.c
deleted file mode 100644
index 0ef0cf3..0000000
--- a/driver/gator_events_pl310.c
+++ /dev/null
@@ -1,176 +0,0 @@
1/**
2 * PL310 (L2 Cache Controller) event counters for gator
3 *
4 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/init.h>
12#include <linux/io.h>
13#include <asm/hardware/cache-l2x0.h>
14
15#include "gator.h"
16
17#define PL310_COUNTERS_NUM 2
18
19static struct {
20 unsigned long enabled;
21 unsigned long event;
22 unsigned long key;
23} pl310_counters[PL310_COUNTERS_NUM];
24
25static int pl310_buffer[PL310_COUNTERS_NUM * 2];
26
27static void __iomem *pl310_base;
28
29
30
31static void gator_events_pl310_reset_counters(void)
32{
33 u32 val = readl(pl310_base + L2X0_EVENT_CNT_CTRL);
34
35 val |= ((1 << PL310_COUNTERS_NUM) - 1) << 1;
36
37 writel(val, pl310_base + L2X0_EVENT_CNT_CTRL);
38}
39
40
41static int gator_events_pl310_create_files(struct super_block *sb,
42 struct dentry *root)
43{
44 int i;
45
46 for (i = 0; i < PL310_COUNTERS_NUM; i++) {
47 char buf[16];
48 struct dentry *dir;
49
50 snprintf(buf, sizeof(buf), "PL310_cnt%d", i);
51 dir = gatorfs_mkdir(sb, root, buf);
52 if (WARN_ON(!dir))
53 return -1;
54 gatorfs_create_ulong(sb, dir, "enabled",
55 &pl310_counters[i].enabled);
56 gatorfs_create_ulong(sb, dir, "event",
57 &pl310_counters[i].event);
58 gatorfs_create_ro_ulong(sb, dir, "key",
59 &pl310_counters[i].key);
60 }
61
62 return 0;
63}
64
65static int gator_events_pl310_start(void)
66{
67 static const unsigned long l2x0_event_cntx_cfg[PL310_COUNTERS_NUM] = {
68 L2X0_EVENT_CNT0_CFG,
69 L2X0_EVENT_CNT1_CFG,
70 };
71 int i;
72
73 /* Counter event sources */
74 for (i = 0; i < PL310_COUNTERS_NUM; i++)
75 writel((pl310_counters[i].event & 0xf) << 2,
76 pl310_base + l2x0_event_cntx_cfg[i]);
77
78 gator_events_pl310_reset_counters();
79
80 /* Event counter enable */
81 writel(1, pl310_base + L2X0_EVENT_CNT_CTRL);
82
83 return 0;
84}
85
86static void gator_events_pl310_stop(void)
87{
88 /* Event counter disable */
89 writel(0, pl310_base + L2X0_EVENT_CNT_CTRL);
90}
91
92static int gator_events_pl310_read(int **buffer)
93{
94 static const unsigned long l2x0_event_cntx_val[PL310_COUNTERS_NUM] = {
95 L2X0_EVENT_CNT0_VAL,
96 L2X0_EVENT_CNT1_VAL,
97 };
98 int i;
99 int len = 0;
100
101 if (smp_processor_id())
102 return 0;
103
104 for (i = 0; i < PL310_COUNTERS_NUM; i++) {
105 if (pl310_counters[i].enabled) {
106 pl310_buffer[len++] = pl310_counters[i].key;
107 pl310_buffer[len++] = readl(pl310_base +
108 l2x0_event_cntx_val[i]);
109 }
110 }
111
112 /* PL310 counters are saturating, not wrapping in case of overflow */
113 gator_events_pl310_reset_counters();
114
115 if (buffer)
116 *buffer = pl310_buffer;
117
118 return len;
119}
120
121static struct gator_interface gator_events_pl310_interface = {
122 .create_files = gator_events_pl310_create_files,
123 .start = gator_events_pl310_start,
124 .stop = gator_events_pl310_stop,
125 .read = gator_events_pl310_read,
126};
127
128static void __maybe_unused gator_events_pl310_probe(unsigned long phys)
129{
130 if (pl310_base)
131 return;
132
133 pl310_base = ioremap(phys, SZ_4K);
134 if (pl310_base) {
135 u32 cache_id = readl(pl310_base + L2X0_CACHE_ID);
136
137 if ((cache_id & 0xff0003c0) != 0x410000c0) {
138 iounmap(pl310_base);
139 pl310_base = NULL;
140 }
141 }
142}
143
144int gator_events_pl310_init(void)
145{
146 int i;
147
148#if defined(CONFIG_ARCH_EXYNOS4)
149 gator_events_pl310_probe(0xfe600000);
150#endif
151#if defined(CONFIG_ARCH_OMAP4)
152 gator_events_pl310_probe(0x48242000);
153#endif
154#if defined(CONFIG_ARCH_TEGRA)
155 gator_events_pl310_probe(0x50043000);
156#endif
157#if defined(CONFIG_ARCH_U8500)
158 gator_events_pl310_probe(0xa0412000);
159#endif
160#if defined(CONFIG_ARCH_VEXPRESS) && !defined(CONFIG_ARCH_VEXPRESS_CA15X4)
161 // A9x4 core tile (HBI-0191)
162 gator_events_pl310_probe(0x1e00a000);
163 // New memory map tiles
164 gator_events_pl310_probe(0x2c0f0000);
165#endif
166 if (!pl310_base)
167 return -1;
168
169 for (i = 0; i < PL310_COUNTERS_NUM; i++) {
170 pl310_counters[i].enabled = 0;
171 pl310_counters[i].key = gator_events_get_key();
172 }
173
174 return gator_events_install(&gator_events_pl310_interface);
175}
176gator_events_init(gator_events_pl310_init);
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c
index d831a50..f51e292 100644
--- a/driver/gator_events_scorpion.c
+++ b/driver/gator_events_scorpion.c
@@ -12,7 +12,7 @@
12#define SCORPIONMP 0x2d 12#define SCORPIONMP 0x2d
13 13
14static const char *pmnc_name; 14static const char *pmnc_name;
15static int pmnc_count; 15static int pmnc_counters;
16 16
17// Per-CPU PMNC: config reg 17// Per-CPU PMNC: config reg
18#define PMNC_E (1 << 0) /* Enable all counters */ 18#define PMNC_E (1 << 0) /* Enable all counters */
@@ -32,6 +32,7 @@ static int pmnc_count;
32 32
33static unsigned long pmnc_enabled[CNTMAX]; 33static unsigned long pmnc_enabled[CNTMAX];
34static unsigned long pmnc_event[CNTMAX]; 34static unsigned long pmnc_event[CNTMAX];
35static unsigned long pmnc_count[CNTMAX];
35static unsigned long pmnc_key[CNTMAX]; 36static unsigned long pmnc_key[CNTMAX];
36 37
37static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 38static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -209,6 +210,10 @@ static const struct scorp_evt sc_evt[] = {
209 {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d}, 210 {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d},
210 {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f}, 211 {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f},
211 212
213#ifdef CONFIG_ARCH_MSM_SCORPIONMP
214 {SCORPIONMP_NUM_BARRIERS, 0x80000e00, 3, 0x59},
215 {SCORPIONMP_BARRIER_CYCLES, 0x800e0000, 3, 0x5a},
216#else
212 {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58}, 217 {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58},
213 {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59}, 218 {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59},
214 {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a}, 219 {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a},
@@ -228,6 +233,7 @@ static const struct scorp_evt sc_evt[] = {
228 {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59}, 233 {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59},
229 {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a}, 234 {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a},
230 {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b}, 235 {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b},
236#endif
231}; 237};
232 238
233static inline void scorpion_pmnc_write(u32 val) 239static inline void scorpion_pmnc_write(u32 val)
@@ -498,7 +504,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
498 struct dentry *dir; 504 struct dentry *dir;
499 int i; 505 int i;
500 506
501 for (i = 0; i < pmnc_count; i++) { 507 for (i = 0; i < pmnc_counters; i++) {
502 char buf[40]; 508 char buf[40];
503 if (i == 0) { 509 if (i == 0) {
504 snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); 510 snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name);
@@ -510,10 +516,11 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
510 return -1; 516 return -1;
511 } 517 }
512 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 518 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
519 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
520 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
513 if (i > 0) { 521 if (i > 0) {
514 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 522 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
515 } 523 }
516 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
517 } 524 }
518 525
519 return 0; 526 return 0;
@@ -565,6 +572,20 @@ static void gator_events_scorpion_offline(void)
565 // investigate: need to do the clearpmu() here on each counter? 572 // investigate: need to do the clearpmu() here on each counter?
566} 573}
567 574
575static int gator_events_scorpion_start(void)
576{
577 int cnt;
578
579 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
580 if (pmnc_count[cnt] > 0) {
581 pr_err("gator: event based sampling not supported on Scorpion cores\n");
582 return -1;
583 }
584 }
585
586 return 0;
587}
588
568static void gator_events_scorpion_stop(void) 589static void gator_events_scorpion_stop(void)
569{ 590{
570 unsigned int cnt; 591 unsigned int cnt;
@@ -572,6 +593,7 @@ static void gator_events_scorpion_stop(void)
572 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 593 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
573 pmnc_enabled[cnt] = 0; 594 pmnc_enabled[cnt] = 0;
574 pmnc_event[cnt] = 0; 595 pmnc_event[cnt] = 0;
596 pmnc_count[cnt] = 0;
575 } 597 }
576} 598}
577 599
@@ -580,10 +602,10 @@ static int gator_events_scorpion_read(int **buffer)
580 int cnt, len = 0; 602 int cnt, len = 0;
581 int cpu = smp_processor_id(); 603 int cpu = smp_processor_id();
582 604
583 if (!pmnc_count) 605 if (!pmnc_counters)
584 return 0; 606 return 0;
585 607
586 for (cnt = 0; cnt < pmnc_count; cnt++) { 608 for (cnt = 0; cnt < pmnc_counters; cnt++) {
587 if (pmnc_enabled[cnt]) { 609 if (pmnc_enabled[cnt]) {
588 int value; 610 int value;
589 if (cnt == CCNT) { 611 if (cnt == CCNT) {
@@ -611,6 +633,7 @@ static int gator_events_scorpion_read(int **buffer)
611 633
612static struct gator_interface gator_events_scorpion_interface = { 634static struct gator_interface gator_events_scorpion_interface = {
613 .create_files = gator_events_scorpion_create_files, 635 .create_files = gator_events_scorpion_create_files,
636 .start = gator_events_scorpion_start,
614 .stop = gator_events_scorpion_stop, 637 .stop = gator_events_scorpion_stop,
615 .online = gator_events_scorpion_online, 638 .online = gator_events_scorpion_online,
616 .offline = gator_events_scorpion_offline, 639 .offline = gator_events_scorpion_offline,
@@ -636,21 +659,22 @@ int gator_events_scorpion_init(void)
636 switch (gator_cpuid()) { 659 switch (gator_cpuid()) {
637 case SCORPION: 660 case SCORPION:
638 pmnc_name = "Scorpion"; 661 pmnc_name = "Scorpion";
639 pmnc_count = 4; 662 pmnc_counters = 4;
640 break; 663 break;
641 case SCORPIONMP: 664 case SCORPIONMP:
642 pmnc_name = "ScorpionMP"; 665 pmnc_name = "ScorpionMP";
643 pmnc_count = 4; 666 pmnc_counters = 4;
644 break; 667 break;
645 default: 668 default:
646 return -1; 669 return -1;
647 } 670 }
648 671
649 pmnc_count++; // CNT[n] + CCNT 672 pmnc_counters++; // CNT[n] + CCNT
650 673
651 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 674 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
652 pmnc_enabled[cnt] = 0; 675 pmnc_enabled[cnt] = 0;
653 pmnc_event[cnt] = 0; 676 pmnc_event[cnt] = 0;
677 pmnc_count[cnt] = 0;
654 pmnc_key[cnt] = gator_events_get_key(); 678 pmnc_key[cnt] = gator_events_get_key();
655 } 679 }
656 680
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 340756e..91744ad 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -7,7 +7,7 @@
7 * 7 *
8 */ 8 */
9 9
10static unsigned long gator_protocol_version = 5; 10static unsigned long gator_protocol_version = 6;
11 11
12#include <linux/slab.h> 12#include <linux/slab.h>
13#include <linux/cpu.h> 13#include <linux/cpu.h>
@@ -36,11 +36,13 @@ static unsigned long gator_protocol_version = 5;
36#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined 36#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
37#endif 37#endif
38 38
39#if defined (__arm__)
39#ifdef CONFIG_SMP 40#ifdef CONFIG_SMP
40#ifndef CONFIG_LOCAL_TIMERS 41#ifndef CONFIG_LOCAL_TIMERS
41#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems 42#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
42#endif 43#endif
43#endif 44#endif
45#endif
44 46
45#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) 47#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
46#error kernels prior to 2.6.32 are not supported 48#error kernels prior to 2.6.32 are not supported
@@ -49,22 +51,24 @@ static unsigned long gator_protocol_version = 5;
49/****************************************************************************** 51/******************************************************************************
50 * DEFINES 52 * DEFINES
51 ******************************************************************************/ 53 ******************************************************************************/
52#define BUFFER_SIZE_DEFAULT (256*1024) 54#define TIMER_BUFFER_SIZE_DEFAULT (256*1024)
53#define SYNC_FREQ_DEFAULT 1000 55#define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
54 56
55#define NO_COOKIE 0UL 57#define NO_COOKIE 0UL
56#define INVALID_COOKIE ~0UL 58#define INVALID_COOKIE ~0UL
57 59
58#define PROTOCOL_FRAME ~0 60#define FRAME_HRTIMER 1
59#define PROTOCOL_START_TICK 1 61#define FRAME_EVENT 2
60#define PROTOCOL_END_TICK 3 62#define FRAME_ANNOTATE 3
61#define PROTOCOL_START_BACKTRACE 5 63
62#define PROTOCOL_END_BACKTRACE 7 64#define MESSAGE_COOKIE 1
63#define PROTOCOL_COOKIE 9 65#define MESSAGE_COUNTERS 3
64#define PROTOCOL_SCHEDULER_TRACE 11 66#define MESSAGE_START_BACKTRACE 5
65#define PROTOCOL_COUNTERS 13 67#define MESSAGE_END_BACKTRACE 7
66#define PROTOCOL_ANNOTATE 15 68#define MESSAGE_SCHEDULER_TRACE 9
67#define PROTOCOL_CPU_SYNC 17 69#define MESSAGE_PID_NAME 11
70
71#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU)
68 72
69#if defined(__arm__) 73#if defined(__arm__)
70#define PC_REG regs->ARM_pc 74#define PC_REG regs->ARM_pc
@@ -72,42 +76,50 @@ static unsigned long gator_protocol_version = 5;
72#define PC_REG regs->ip 76#define PC_REG regs->ip
73#endif 77#endif
74 78
79enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
80
75/****************************************************************************** 81/******************************************************************************
76 * PER CPU 82 * Globals
77 ******************************************************************************/ 83 ******************************************************************************/
78static unsigned long gator_cpu_cores; 84static unsigned long gator_cpu_cores;
79static unsigned long gator_buffer_size; 85static unsigned long userspace_buffer_size;
80static unsigned long gator_backtrace_depth; 86static unsigned long gator_backtrace_depth;
81 87
82static unsigned long gator_started; 88static unsigned long gator_started;
83static unsigned long gator_buffer_opened; 89static unsigned long gator_buffer_opened;
84static unsigned long gator_timer_count; 90static unsigned long gator_timer_count;
85static unsigned long gator_streaming; 91static unsigned long gator_streaming;
86static int gator_master_tick;
87static DEFINE_MUTEX(start_mutex); 92static DEFINE_MUTEX(start_mutex);
88static DEFINE_MUTEX(gator_buffer_mutex); 93static DEFINE_MUTEX(gator_buffer_mutex);
89 94
90unsigned long gator_net_traffic; 95unsigned long gator_net_traffic;
96bool event_based_sampling;
91 97
92#define COMMIT_SIZE 128 98#define COMMIT_SIZE 128
93#define COMMIT_MASK (COMMIT_SIZE-1) 99#define COMMIT_MASK (COMMIT_SIZE-1)
94static DEFINE_SPINLOCK(gator_commit_lock); 100static DEFINE_SPINLOCK(timer_commit_lock);
95static int *gator_commit; 101static int *gator_commit[NUM_GATOR_BUFS];
96static int gator_commit_read; 102static int gator_commit_read[NUM_GATOR_BUFS];
97static int gator_commit_write; 103static int gator_commit_write[NUM_GATOR_BUFS];
98 104
99static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); 105static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
100static DEFINE_PER_CPU(int, gator_cpu_sync);
101static DEFINE_PER_CPU(int, gator_cpu_tick);
102static DEFINE_PER_CPU(int, gator_first_time); 106static DEFINE_PER_CPU(int, gator_first_time);
103 107
108#if LINUX_PMU_SUPPORT
109static void event_buffer_check(int cpu);
110static DEFINE_SPINLOCK(event_commit_lock);
111#endif
112
104/****************************************************************************** 113/******************************************************************************
105 * Prototypes 114 * Prototypes
106 ******************************************************************************/ 115 ******************************************************************************/
107static void gator_buffer_write_packed_int(int cpu, unsigned int x); 116static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
108static void gator_buffer_write_string(int cpu, char *x); 117static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
118static void gator_buffer_write_string(int cpu, int buftype, char *x);
109static int gator_write_packed_int(char *buffer, unsigned int x); 119static int gator_write_packed_int(char *buffer, unsigned int x);
110static void gator_add_trace(int cpu, unsigned int address); 120static int gator_write_packed_int64(char *buffer, unsigned long long x);
121static void gator_add_trace(int cpu, int buftype, unsigned int address);
122static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
111static uint64_t gator_get_time(void); 123static uint64_t gator_get_time(void);
112 124
113/****************************************************************************** 125/******************************************************************************
@@ -118,6 +130,7 @@ static uint64_t gator_get_time(void);
118#include "gator_backtrace.c" 130#include "gator_backtrace.c"
119#include "gator_annotate.c" 131#include "gator_annotate.c"
120#include "gator_fs.c" 132#include "gator_fs.c"
133#include "gator_ebs.c"
121 134
122/****************************************************************************** 135/******************************************************************************
123 * Misc 136 * Misc
@@ -134,204 +147,137 @@ u32 gator_cpuid(void)
134/****************************************************************************** 147/******************************************************************************
135 * Commit interface 148 * Commit interface
136 ******************************************************************************/ 149 ******************************************************************************/
137static int buffer_commit_ready(void) 150static int buffer_commit_ready(int buftype)
138{ 151{
139 return (gator_commit_read != gator_commit_write); 152 return gator_commit_read[buftype] != gator_commit_write[buftype];
140} 153}
141 154
142static void buffer_commit_read(int *cpu, int *readval, int *writeval) 155static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval)
143{ 156{
144 int read = gator_commit_read; 157 int read = gator_commit_read[buftype];
145 *cpu = gator_commit[read+0]; 158 *cpu = gator_commit[buftype][read+0];
146 *readval = gator_commit[read+1]; 159 *readval = gator_commit[buftype][read+1];
147 *writeval = gator_commit[read+2]; 160 *writeval = gator_commit[buftype][read+2];
148 gator_commit_read = (read + 4) & COMMIT_MASK; 161 gator_commit_read[buftype] = (read + 4) & COMMIT_MASK;
149} 162}
150 163
151static void buffer_commit_write(int cpu, int readval, int writeval) { 164static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
152 int write = gator_commit_write; 165 int write = gator_commit_write[buftype];
153 gator_commit[write+0] = cpu; 166 gator_commit[buftype][write+0] = cpu;
154 gator_commit[write+1] = readval; 167 gator_commit[buftype][write+1] = readval;
155 gator_commit[write+2] = writeval; 168 gator_commit[buftype][write+2] = writeval;
156 gator_commit_write = (write + 4) & COMMIT_MASK; 169 gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
157} 170}
158 171
159/****************************************************************************** 172/******************************************************************************
160 * Buffer management 173 * Buffer management
161 ******************************************************************************/ 174 ******************************************************************************/
162static uint32_t use_buffer_size; 175static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
163static uint32_t use_buffer_mask; 176static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
164static DEFINE_PER_CPU(int, use_buffer_seq); 177static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
165static DEFINE_PER_CPU(int, use_buffer_read); 178static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
166static DEFINE_PER_CPU(int, use_buffer_write); 179static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
167static DEFINE_PER_CPU(char *, use_buffer); 180#include "gator_pack.c"
168
169static void gator_buffer_write_packed_int(int cpu, unsigned int x)
170{
171 uint32_t write = per_cpu(use_buffer_write, cpu);
172 uint32_t mask = use_buffer_mask;
173 char *buffer = per_cpu(use_buffer, cpu);
174 int write0 = (write + 0) & mask;
175 int write1 = (write + 1) & mask;
176 int write2 = (write + 2) & mask;
177 int write3 = (write + 3) & mask;
178 int write4 = (write + 4) & mask;
179 int write5 = (write + 5) & mask;
180
181 if ((x & 0xffffff80) == 0) {
182 buffer[write0] = x & 0x7f;
183 per_cpu(use_buffer_write, cpu) = write1;
184 } else if ((x & 0xffffc000) == 0) {
185 buffer[write0] = x | 0x80;
186 buffer[write1] = (x>>7) & 0x7f;
187 per_cpu(use_buffer_write, cpu) = write2;
188 } else if ((x & 0xffe00000) == 0) {
189 buffer[write0] = x | 0x80;
190 buffer[write1] = (x>>7) | 0x80;
191 buffer[write2] = (x>>14) & 0x7f;
192 per_cpu(use_buffer_write, cpu) = write3;
193 } else if ((x & 0xf0000000) == 0) {
194 buffer[write0] = x | 0x80;
195 buffer[write1] = (x>>7) | 0x80;
196 buffer[write2] = (x>>14) | 0x80;
197 buffer[write3] = (x>>21) & 0x7f;
198 per_cpu(use_buffer_write, cpu) = write4;
199 } else {
200 buffer[write0] = x | 0x80;
201 buffer[write1] = (x>>7) | 0x80;
202 buffer[write2] = (x>>14) | 0x80;
203 buffer[write3] = (x>>21) | 0x80;
204 buffer[write4] = (x>>28) & 0x0f;
205 per_cpu(use_buffer_write, cpu) = write5;
206 }
207}
208 181
209static int gator_write_packed_int(char *buffer, unsigned int x) 182static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
210{ 183{
211 if ((x & 0xffffff80) == 0) {
212 buffer[0] = x & 0x7f;
213 return 1;
214 } else if ((x & 0xffffc000) == 0) {
215 buffer[0] = x | 0x80;
216 buffer[1] = (x>>7) & 0x7f;
217 return 2;
218 } else if ((x & 0xffe00000) == 0) {
219 buffer[0] = x | 0x80;
220 buffer[1] = (x>>7) | 0x80;
221 buffer[2] = (x>>14) & 0x7f;
222 return 3;
223 } else if ((x & 0xf0000000) == 0) {
224 buffer[0] = x | 0x80;
225 buffer[1] = (x>>7) | 0x80;
226 buffer[2] = (x>>14) | 0x80;
227 buffer[3] = (x>>21) & 0x7f;
228 return 4;
229 } else {
230 buffer[0] = x | 0x80;
231 buffer[1] = (x>>7) | 0x80;
232 buffer[2] = (x>>14) | 0x80;
233 buffer[3] = (x>>21) | 0x80;
234 buffer[4] = (x>>28) & 0x0f;
235 return 5;
236 }
237}
238
239static void gator_buffer_write_bytes(int cpu, char *x, int len)
240{
241 uint32_t write = per_cpu(use_buffer_write, cpu);
242 uint32_t mask = use_buffer_mask;
243 char *buffer = per_cpu(use_buffer, cpu);
244 int i; 184 int i;
185 u32 write = per_cpu(gator_buffer_write, cpu)[buftype];
186 u32 mask = gator_buffer_mask[buftype];
187 char* buffer = per_cpu(gator_buffer, cpu)[buftype];
245 188
246 for (i = 0; i < len; i++) { 189 for (i = 0; i < len; i++) {
247 buffer[write] = x[i]; 190 buffer[write] = x[i];
248 write = (write + 1) & mask; 191 write = (write + 1) & mask;
249 } 192 }
250 193
251 per_cpu(use_buffer_write, cpu) = write; 194 per_cpu(gator_buffer_write, cpu)[buftype] = write;
252} 195}
253 196
254static void gator_buffer_write_string(int cpu, char *x) 197static void gator_buffer_write_string(int cpu, int buftype, char *x)
255{ 198{
256 int len = strlen(x); 199 int len = strlen(x);
257 gator_buffer_write_packed_int(cpu, len); 200 gator_buffer_write_packed_int(cpu, buftype, len);
258 gator_buffer_write_bytes(cpu, x, len); 201 gator_buffer_write_bytes(cpu, buftype, x, len);
259} 202}
260 203
261static void gator_buffer_header(int cpu) 204static void gator_buffer_header(int cpu, int buftype)
262{ 205{
263 gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME); 206 int frame;
264 gator_buffer_write_packed_int(cpu, cpu); 207
265 gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu)); 208 if (buftype == TIMER_BUF)
266 per_cpu(use_buffer_seq, cpu)++; 209 frame = FRAME_HRTIMER;
210 else if (buftype == EVENT_BUF)
211 frame = FRAME_EVENT;
212 else
213 frame = -1;
214
215 gator_buffer_write_packed_int(cpu, buftype, frame);
216 gator_buffer_write_packed_int(cpu, buftype, cpu);
267} 217}
268 218
269static void gator_buffer_commit(int cpu) 219static void gator_buffer_commit(int cpu, int buftype)
270{ 220{
271 buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu)); 221 buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]);
272 per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu); 222 per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
273 gator_buffer_header(cpu); 223 gator_buffer_header(cpu, buftype);
274 wake_up(&gator_buffer_wait); 224 wake_up(&gator_buffer_wait);
275} 225}
276 226
277static void gator_buffer_check(int cpu, int tick) 227static void timer_buffer_check(int cpu)
278{ 228{
279 if (!(tick % gator_timer_count)) { 229 int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF];
280 int c, sync; 230 if (available < 0) {
281 spin_lock(&gator_commit_lock); 231 available += gator_buffer_size[TIMER_BUF];
282 // synchronize, if all online cpus have the same tick waypoint 232 }
283 sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu); 233 if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) {
284 for_each_online_cpu(c) { 234 spin_lock(&timer_commit_lock);
285 if (sync != per_cpu(gator_cpu_sync, c)) { 235 gator_buffer_commit(cpu, TIMER_BUF);
286 sync = 0; 236 spin_unlock(&timer_commit_lock);
287 break; 237 }
288 } 238}
289 } 239
290 if (sync) { 240#if LINUX_PMU_SUPPORT
291 gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC); 241static void event_buffer_check(int cpu)
292 } 242{
293 gator_buffer_commit(cpu); 243 int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
294 spin_unlock(&gator_commit_lock); 244 if (available < 0) {
295 } else { 245 available += gator_buffer_size[EVENT_BUF];
296 int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu); 246 }
297 if (available < 0) { 247 if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
298 available += use_buffer_size; 248 spin_lock(&event_commit_lock);
299 } 249 gator_buffer_commit(cpu, EVENT_BUF);
300 if (available >= ((use_buffer_size * 3) / 4)) { 250 spin_unlock(&event_commit_lock);
301 spin_lock(&gator_commit_lock);
302 gator_buffer_commit(cpu);
303 spin_unlock(&gator_commit_lock);
304 }
305 } 251 }
306} 252}
253#endif
307 254
308static void gator_add_trace(int cpu, unsigned int address) 255static void gator_add_trace(int cpu, int buftype, unsigned int address)
309{ 256{
310 off_t offset = 0; 257 off_t offset = 0;
311 unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset); 258 unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset);
312 259
313 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) { 260 if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) {
314 offset = address; 261 offset = address;
315 } 262 }
316 263
317 gator_buffer_write_packed_int(cpu, offset & ~1); 264 gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
318 gator_buffer_write_packed_int(cpu, cookie); 265 gator_buffer_write_packed_int(cpu, buftype, cookie);
319} 266}
320 267
321static void gator_add_sample(int cpu, struct pt_regs * const regs) 268static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
322{ 269{
323 struct module *mod; 270 struct module *mod;
324 unsigned int addr, cookie = 0; 271 unsigned int addr, cookie = 0;
325 int inKernel = regs ? !user_mode(regs) : 1; 272 int inKernel = regs ? !user_mode(regs) : 1;
326 unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE; 273 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
327 274
328 gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE); 275 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
329 276 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
330 // TGID::PID::inKernel 277 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
331 gator_buffer_write_packed_int(cpu, exec_cookie); 278 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
332 gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid); 279 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
333 gator_buffer_write_packed_int(cpu, (unsigned int)current->pid); 280 gator_buffer_write_packed_int(cpu, buftype, inKernel);
334 gator_buffer_write_packed_int(cpu, inKernel);
335 281
336 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ) 282 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
337 if (regs) { 283 if (regs) {
@@ -339,36 +285,26 @@ static void gator_add_sample(int cpu, struct pt_regs * const regs)
339 addr = PC_REG; 285 addr = PC_REG;
340 mod = __module_address(addr); 286 mod = __module_address(addr);
341 if (mod) { 287 if (mod) {
342 cookie = get_cookie(cpu, current, NULL, mod); 288 cookie = get_cookie(cpu, buftype, current, NULL, mod, true);
343 addr = addr - (unsigned long)mod->module_core; 289 addr = addr - (unsigned long)mod->module_core;
344 } 290 }
345 gator_buffer_write_packed_int(cpu, addr & ~1); 291 gator_buffer_write_packed_int(cpu, buftype, addr & ~1);
346 gator_buffer_write_packed_int(cpu, cookie); 292 gator_buffer_write_packed_int(cpu, buftype, cookie);
347 } else { 293 } else {
348 // Cookie+PC 294 // Cookie+PC
349 gator_add_trace(cpu, PC_REG); 295 gator_add_trace(cpu, buftype, PC_REG);
350 296
351 // Backtrace 297 // Backtrace
352 if (gator_backtrace_depth) 298 if (gator_backtrace_depth)
353 arm_backtrace_eabi(cpu, regs, gator_backtrace_depth); 299 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
354 } 300 }
355 } 301 }
356 302
357 gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE); 303 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
358}
359
360static void gator_write_packet(int cpu, int type, int len, int *buffer)
361{
362 int i;
363 gator_buffer_write_packed_int(cpu, type);
364 gator_buffer_write_packed_int(cpu, len);
365 for (i = 0; i < len; i++) {
366 gator_buffer_write_packed_int(cpu, buffer[i]);
367 }
368} 304}
369 305
370/****************************************************************************** 306/******************************************************************************
371 * Interrupt Processing 307 * hrtimer interrupt processing
372 ******************************************************************************/ 308 ******************************************************************************/
373static LIST_HEAD(gator_events); 309static LIST_HEAD(gator_events);
374 310
@@ -376,7 +312,8 @@ static void gator_timer_interrupt(void)
376{ 312{
377 struct pt_regs * const regs = get_irq_regs(); 313 struct pt_regs * const regs = get_irq_regs();
378 int cpu = smp_processor_id(); 314 int cpu = smp_processor_id();
379 int *buffer, len, tick; 315 int *buffer, len, i, buftype = TIMER_BUF;
316 long long *buffer64;
380 struct gator_interface *gi; 317 struct gator_interface *gi;
381 318
382 // check full backtrace has enough space, otherwise may 319 // check full backtrace has enough space, otherwise may
@@ -391,44 +328,48 @@ static void gator_timer_interrupt(void)
391 return; 328 return;
392 } 329 }
393 330
394 // Header
395 gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape
396
397 // Output scheduler 331 // Output scheduler
398 len = gator_trace_sched_read(&buffer); 332 len = gator_trace_sched_read(&buffer64);
399 if (len > 0) { 333 if (len > 0) {
400 gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer); 334 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
335 gator_buffer_write_packed_int(cpu, buftype, len);
336 for (i = 0; i < len; i++) {
337 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
338 }
401 } 339 }
402 340
403 // Output counters 341 // Output counters
342 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
343 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
404 list_for_each_entry(gi, &gator_events, list) { 344 list_for_each_entry(gi, &gator_events, list) {
405 if (gi->read) { 345 if (gi->read) {
406 len = gi->read(&buffer); 346 len = gi->read(&buffer);
347 if (len > 0) {
348 gator_buffer_write_packed_int(cpu, buftype, len);
349 for (i = 0; i < len; i++) {
350 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
351 }
352 }
353 } else if (gi->read64) {
354 len = gi->read64(&buffer64);
407 if (len > 0) 355 if (len > 0)
408 gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer); 356 gator_buffer_write_packed_int(cpu, buftype, len);
357 for (i = 0; i < len; i++) {
358 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
359 }
409 } 360 }
410 } 361 }
362 gator_buffer_write_packed_int(cpu, buftype, 0);
411 363
412 // Output backtrace 364 // Output backtrace
413 gator_add_sample(cpu, regs); 365 if (!event_based_sampling) {
414 366 gator_add_sample(cpu, buftype, regs);
415 // Timer Tick
416 tick = per_cpu(gator_cpu_tick, cpu);
417 if (tick == gator_master_tick) {
418 tick++;
419 per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick;
420 } else {
421 per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick;
422 } 367 }
423 gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick);
424 368
425 // Check and commit; generally, commit is set to occur once per second 369 // Check and commit; generally, commit is set to occur once per second
426 gator_buffer_check(cpu, tick); 370 timer_buffer_check(cpu);
427} 371}
428 372
429/******************************************************************************
430 * hrtimer
431 ******************************************************************************/
432DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); 373DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
433DEFINE_PER_CPU(int, hrtimer_is_active); 374DEFINE_PER_CPU(int, hrtimer_is_active);
434static int hrtimer_running; 375static int hrtimer_running;
@@ -454,7 +395,9 @@ static void __gator_timer_offline(void *unused)
454 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); 395 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
455 hrtimer_cancel(hrtimer); 396 hrtimer_cancel(hrtimer);
456 per_cpu(hrtimer_is_active, cpu) = 0; 397 per_cpu(hrtimer_is_active, cpu) = 0;
457 gator_buffer_commit(cpu); 398 gator_buffer_commit(cpu, TIMER_BUF);
399 if (event_based_sampling)
400 gator_buffer_commit(cpu, EVENT_BUF);
458 401
459 // offline any events 402 // offline any events
460 list_for_each_entry(gi, &gator_events, list) 403 list_for_each_entry(gi, &gator_events, list)
@@ -469,10 +412,6 @@ static void gator_timer_offline(void)
469 hrtimer_running = 0; 412 hrtimer_running = 0;
470 413
471 on_each_cpu(__gator_timer_offline, NULL, 1); 414 on_each_cpu(__gator_timer_offline, NULL, 1);
472
473 // output a final sync point
474 gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC);
475 gator_buffer_commit(0);
476 } 415 }
477} 416}
478 417
@@ -485,7 +424,6 @@ static void __gator_timer_online(void *unused)
485 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 424 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
486 hrtimer->function = gator_hrtimer_notify; 425 hrtimer->function = gator_hrtimer_notify;
487 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); 426 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
488 per_cpu(gator_cpu_tick, cpu) = 0;
489 per_cpu(gator_first_time, cpu) = 1; 427 per_cpu(gator_first_time, cpu) = 1;
490 per_cpu(hrtimer_is_active, cpu) = 1; 428 per_cpu(hrtimer_is_active, cpu) = 1;
491 429
@@ -499,10 +437,10 @@ static void __gator_timer_online(void *unused)
499int gator_timer_online(unsigned long setup) 437int gator_timer_online(unsigned long setup)
500{ 438{
501 if (!setup) { 439 if (!setup) {
502 pr_err("gator: cannot start due to a system tick value of zero"); 440 pr_err("gator: cannot start due to a system tick value of zero\n");
503 return -1; 441 return -1;
504 } else if (hrtimer_running) { 442 } else if (hrtimer_running) {
505 pr_notice("gator: high res timer already running"); 443 pr_notice("gator: high res timer already running\n");
506 return 0; 444 return 0;
507 } 445 }
508 446
@@ -512,7 +450,6 @@ int gator_timer_online(unsigned long setup)
512 profiling_interval = ns_to_ktime(1000000000UL / setup); 450 profiling_interval = ns_to_ktime(1000000000UL / setup);
513 451
514 // timer interrupt 452 // timer interrupt
515 gator_master_tick = 0;
516 on_each_cpu(__gator_timer_online, NULL, 1); 453 on_each_cpu(__gator_timer_online, NULL, 1);
517 454
518 return 0; 455 return 0;
@@ -623,10 +560,15 @@ static int gator_start(void)
623 } 560 }
624 } 561 }
625 562
563 // cookies shall be initialized before trace_sched_start() and gator_timer_online()
564 if (cookies_initialize())
565 goto cookies_failure;
626 if (gator_annotate_start()) 566 if (gator_annotate_start())
627 goto annotate_failure; 567 goto annotate_failure;
628 if (gator_trace_sched_start()) 568 if (gator_trace_sched_start())
629 goto sched_failure; 569 goto sched_failure;
570 if (gator_event_sampling_start())
571 goto event_sampling_failure;
630 if (gator_timer_online(gator_timer_count)) 572 if (gator_timer_online(gator_timer_count))
631 goto timer_failure; 573 goto timer_failure;
632 if (gator_notifier_start()) 574 if (gator_notifier_start())
@@ -637,10 +579,18 @@ static int gator_start(void)
637notifier_failure: 579notifier_failure:
638 gator_timer_offline(); 580 gator_timer_offline();
639timer_failure: 581timer_failure:
582 gator_event_sampling_stop();
583event_sampling_failure:
640 gator_trace_sched_stop(); 584 gator_trace_sched_stop();
641sched_failure: 585sched_failure:
642 gator_annotate_stop(); 586 gator_annotate_stop();
643annotate_failure: 587annotate_failure:
588 cookies_release();
589cookies_failure:
590 // stop all events
591 list_for_each_entry(gi, &gator_events, list)
592 if (gi->stop)
593 gi->stop();
644events_failure: 594events_failure:
645 595
646 return -1; 596 return -1;
@@ -657,10 +607,11 @@ static void gator_stop(void)
657 607
658 gator_annotate_stop(); 608 gator_annotate_stop();
659 gator_trace_sched_stop(); 609 gator_trace_sched_stop();
610 gator_event_sampling_stop();
660 611
661 // stop all interrupt callback reads before tearing down other interfaces 612 // stop all interrupt callback reads before tearing down other interfaces
613 gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined
662 gator_timer_offline(); 614 gator_timer_offline();
663 gator_notifier_stop();
664} 615}
665 616
666static void gator_exit(void) 617static void gator_exit(void)
@@ -675,42 +626,45 @@ static void gator_exit(void)
675static int gator_op_setup(void) 626static int gator_op_setup(void)
676{ 627{
677 int err = 0; 628 int err = 0;
678 int cpu; 629 int cpu, i;
679 630
680 mutex_lock(&start_mutex); 631 mutex_lock(&start_mutex);
681 632
682 use_buffer_size = gator_buffer_size; 633 gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
683 use_buffer_mask = use_buffer_size - 1; 634 gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
684 635
685 // must be a power of 2 636 // must be a power of 2
686 if (use_buffer_size & (use_buffer_size - 1)) { 637 if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
687 err = -ENOEXEC; 638 err = -ENOEXEC;
688 goto setup_error; 639 goto setup_error;
689 } 640 }
690 641
691 gator_net_traffic = 0; 642 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
643 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
692 644
693 gator_commit_read = gator_commit_write = 0; 645 gator_net_traffic = 0;
694 gator_commit = vmalloc(COMMIT_SIZE * sizeof(int));
695 if (!gator_commit) {
696 err = -ENOMEM;
697 goto setup_error;
698 }
699 646
700 for_each_present_cpu(cpu) { 647 // Initialize per buffer variables
701 per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size); 648 for (i = 0; i < NUM_GATOR_BUFS; i++) {
702 if (!per_cpu(use_buffer, cpu)) { 649 gator_commit_read[i] = gator_commit_write[i] = 0;
650 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
651 if (!gator_commit[i]) {
703 err = -ENOMEM; 652 err = -ENOMEM;
704 goto setup_error; 653 goto setup_error;
705 } 654 }
706 655
707 per_cpu(gator_cpu_sync, cpu) = 0; 656 // Initialize percpu per buffer variables
708 per_cpu(gator_cpu_tick, cpu) = 0; 657 for_each_present_cpu(cpu) {
658 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
659 if (!per_cpu(gator_buffer, cpu)[i]) {
660 err = -ENOMEM;
661 goto setup_error;
662 }
709 663
710 per_cpu(use_buffer_seq, cpu) = 0; 664 per_cpu(gator_buffer_read, cpu)[i] = 0;
711 per_cpu(use_buffer_read, cpu) = 0; 665 per_cpu(gator_buffer_write, cpu)[i] = 0;
712 per_cpu(use_buffer_write, cpu) = 0; 666 gator_buffer_header(cpu, i);
713 gator_buffer_header(cpu); 667 }
714 } 668 }
715 669
716setup_error: 670setup_error:
@@ -725,7 +679,7 @@ static int gator_op_start(void)
725 679
726 mutex_lock(&start_mutex); 680 mutex_lock(&start_mutex);
727 681
728 if (gator_started || gator_start() || cookies_initialize()) 682 if (gator_started || gator_start())
729 err = -EINVAL; 683 err = -EINVAL;
730 else 684 else
731 gator_started = 1; 685 gator_started = 1;
@@ -757,20 +711,25 @@ static void gator_op_stop(void)
757 711
758static void gator_shutdown(void) 712static void gator_shutdown(void)
759{ 713{
760 int cpu; 714 int cpu, i;
761 715
762 mutex_lock(&start_mutex); 716 mutex_lock(&start_mutex);
763 717
764 vfree(gator_commit); 718 gator_annotate_shutdown();
765 gator_commit = NULL; 719
720 for (i = 0; i < NUM_GATOR_BUFS; i++) {
721 vfree(gator_commit[i]);
722 gator_commit[i] = NULL;
723 }
766 724
767 for_each_present_cpu(cpu) { 725 for_each_present_cpu(cpu) {
768 mutex_lock(&gator_buffer_mutex); 726 mutex_lock(&gator_buffer_mutex);
769 vfree(per_cpu(use_buffer, cpu)); 727 for (i = 0; i < NUM_GATOR_BUFS; i++) {
770 per_cpu(use_buffer, cpu) = NULL; 728 vfree(per_cpu(gator_buffer, cpu)[i]);
771 per_cpu(use_buffer_seq, cpu) = 0; 729 per_cpu(gator_buffer, cpu)[i] = NULL;
772 per_cpu(use_buffer_read, cpu) = 0; 730 per_cpu(gator_buffer_read, cpu)[i] = 0;
773 per_cpu(use_buffer_write, cpu) = 0; 731 per_cpu(gator_buffer_write, cpu)[i] = 0;
732 }
774 mutex_unlock(&gator_buffer_mutex); 733 mutex_unlock(&gator_buffer_mutex);
775 } 734 }
776 735
@@ -825,7 +784,7 @@ static const struct file_operations enable_fops = {
825 .write = enable_write, 784 .write = enable_write,
826}; 785};
827 786
828static int event_buffer_open(struct inode *inode, struct file *file) 787static int userspace_buffer_open(struct inode *inode, struct file *file)
829{ 788{
830 int err = -EPERM; 789 int err = -EPERM;
831 790
@@ -849,7 +808,7 @@ fail:
849 return err; 808 return err;
850} 809}
851 810
852static int event_buffer_release(struct inode *inode, struct file *file) 811static int userspace_buffer_release(struct inode *inode, struct file *file)
853{ 812{
854 gator_op_stop(); 813 gator_op_stop();
855 gator_shutdown(); 814 gator_shutdown();
@@ -857,55 +816,59 @@ static int event_buffer_release(struct inode *inode, struct file *file)
857 return 0; 816 return 0;
858} 817}
859 818
860static ssize_t event_buffer_read(struct file *file, char __user *buf, 819static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
861 size_t count, loff_t *offset) 820 size_t count, loff_t *offset)
862{ 821{
863 int retval = -EINVAL; 822 int retval = -EINVAL;
864 int commit, length1, length2, read; 823 int commit, length1, length2, read;
865 char *buffer1, *buffer2; 824 char *buffer1;
866 char annotate_header[6]; 825 char *buffer2 = NULL;
867 int cpu; 826 int cpu, i;
868 827
869 /* do not handle partial reads */ 828 /* do not handle partial reads */
870 if (count != use_buffer_size || *offset) 829 if (count != userspace_buffer_size || *offset)
871 return -EINVAL; 830 return -EINVAL;
872 831
873 // sleep until the condition is true or a signal is received 832 // sleep until the condition is true or a signal is received
874 // the condition is checked each time gator_buffer_wait is woken up 833 // the condition is checked each time gator_buffer_wait is woken up
875 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started); 834 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started);
876 835
877 if (signal_pending(current)) 836 if (signal_pending(current))
878 return -EINTR; 837 return -EINTR;
879 838
839 length2 = 0;
880 retval = -EFAULT; 840 retval = -EFAULT;
881 841
882 mutex_lock(&gator_buffer_mutex); 842 mutex_lock(&gator_buffer_mutex);
883 843
884 if (buffer_commit_ready()) { 844 i = -1;
885 buffer_commit_read(&cpu, &read, &commit); 845 if (buffer_commit_ready(TIMER_BUF)) {
846 i = TIMER_BUF;
847 } else if (buffer_commit_ready(EVENT_BUF)) {
848 i = EVENT_BUF;
849 }
850
851 if (i != -1) {
852 buffer_commit_read(&cpu, i, &read, &commit);
886 853
887 /* May happen if the buffer is freed during pending reads. */ 854 /* May happen if the buffer is freed during pending reads. */
888 if (!per_cpu(use_buffer, cpu)) { 855 if (!per_cpu(gator_buffer, cpu)[i]) {
889 retval = -EFAULT; 856 retval = -EFAULT;
890 goto out; 857 goto out;
891 } 858 }
892 859
893 /* determine the size of two halves */ 860 /* determine the size of two halves */
894 length1 = commit - read; 861 length1 = commit - read;
895 length2 = 0; 862 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]);
896 buffer1 = &(per_cpu(use_buffer, cpu)[read]); 863 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]);
897 buffer2 = &(per_cpu(use_buffer, cpu)[0]);
898 if (length1 < 0) { 864 if (length1 < 0) {
899 length1 = use_buffer_size - read; 865 length1 = gator_buffer_size[i] - read;
900 length2 = commit; 866 length2 = commit;
901 } 867 }
902 } else if (gator_annotate_ready()) { 868 } else if (gator_annotate_ready()) {
903 length2 = gator_annotate_read(&buffer2); 869 length1 = gator_annotate_read(&buffer1);
904 if (!length2) 870 if (!length1)
905 goto out; 871 goto out;
906 annotate_header[0] = PROTOCOL_ANNOTATE;
907 length1 = gator_write_packed_int(&annotate_header[1], length2) + 1;
908 buffer1 = annotate_header;
909 } else { 872 } else {
910 retval = 0; 873 retval = 0;
911 goto out; 874 goto out;
@@ -939,9 +902,9 @@ out:
939} 902}
940 903
941const struct file_operations gator_event_buffer_fops = { 904const struct file_operations gator_event_buffer_fops = {
942 .open = event_buffer_open, 905 .open = userspace_buffer_open,
943 .release = event_buffer_release, 906 .release = userspace_buffer_release,
944 .read = event_buffer_read, 907 .read = userspace_buffer_read,
945}; 908};
946 909
947static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) 910static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
@@ -974,17 +937,6 @@ static const struct file_operations depth_fops = {
974 .write = depth_write 937 .write = depth_write
975}; 938};
976 939
977static const char gator_cpu_type[] = "gator";
978
979static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
980{
981 return gatorfs_str_to_user(gator_cpu_type, buf, count, offset);
982}
983
984static const struct file_operations cpu_type_fops = {
985 .read = cpu_type_read,
986};
987
988void gator_op_create_files(struct super_block *sb, struct dentry *root) 940void gator_op_create_files(struct super_block *sb, struct dentry *root)
989{ 941{
990 struct dentry *dir; 942 struct dentry *dir;
@@ -996,15 +948,14 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
996 for_each_present_cpu(cpu) { 948 for_each_present_cpu(cpu) {
997 gator_cpu_cores++; 949 gator_cpu_cores++;
998 } 950 }
999 gator_buffer_size = BUFFER_SIZE_DEFAULT; 951 userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
1000 gator_streaming = 1; 952 gator_streaming = 1;
1001 953
1002 gatorfs_create_file(sb, root, "enable", &enable_fops); 954 gatorfs_create_file(sb, root, "enable", &enable_fops);
1003 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); 955 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1004 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); 956 gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1005 gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops);
1006 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); 957 gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1007 gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size); 958 gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1008 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); 959 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1009 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); 960 gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
1010 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); 961 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
@@ -1033,17 +984,11 @@ static int __init gator_module_init(void)
1033 return -1; 984 return -1;
1034 } 985 }
1035 986
1036#ifdef GATOR_DEBUG
1037 pr_err("gator_module_init");
1038#endif
1039 return 0; 987 return 0;
1040} 988}
1041 989
1042static void __exit gator_module_exit(void) 990static void __exit gator_module_exit(void)
1043{ 991{
1044#ifdef GATOR_DEBUG
1045 pr_err("gator_module_exit");
1046#endif
1047 tracepoint_synchronize_unregister(); 992 tracepoint_synchronize_unregister();
1048 gatorfs_unregister(); 993 gatorfs_unregister();
1049 gator_exit(); 994 gator_exit();
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
new file mode 100644
index 0000000..fbab220
--- /dev/null
+++ b/driver/gator_pack.c
@@ -0,0 +1,262 @@
1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x)
11{
12 uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
13 uint32_t mask = gator_buffer_mask[buftype];
14 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
15 int write0 = (write + 0) & mask;
16 int write1 = (write + 1) & mask;
17
18 if ((x & 0xffffff80) == 0) {
19 buffer[write0] = x & 0x7f;
20 per_cpu(gator_buffer_write, cpu)[buftype] = write1;
21 } else if ((x & 0xffffc000) == 0) {
22 int write2 = (write + 2) & mask;
23 buffer[write0] = x | 0x80;
24 buffer[write1] = (x>>7) & 0x7f;
25 per_cpu(gator_buffer_write, cpu)[buftype] = write2;
26 } else if ((x & 0xffe00000) == 0) {
27 int write2 = (write + 2) & mask;
28 int write3 = (write + 3) & mask;
29 buffer[write0] = x | 0x80;
30 buffer[write1] = (x>>7) | 0x80;
31 buffer[write2] = (x>>14) & 0x7f;
32 per_cpu(gator_buffer_write, cpu)[buftype] = write3;
33 } else if ((x & 0xf0000000) == 0) {
34 int write2 = (write + 2) & mask;
35 int write3 = (write + 3) & mask;
36 int write4 = (write + 4) & mask;
37 buffer[write0] = x | 0x80;
38 buffer[write1] = (x>>7) | 0x80;
39 buffer[write2] = (x>>14) | 0x80;
40 buffer[write3] = (x>>21) & 0x7f;
41 per_cpu(gator_buffer_write, cpu)[buftype] = write4;
42 } else {
43 int write2 = (write + 2) & mask;
44 int write3 = (write + 3) & mask;
45 int write4 = (write + 4) & mask;
46 int write5 = (write + 5) & mask;
47 buffer[write0] = x | 0x80;
48 buffer[write1] = (x>>7) | 0x80;
49 buffer[write2] = (x>>14) | 0x80;
50 buffer[write3] = (x>>21) | 0x80;
51 buffer[write4] = (x>>28) & 0x0f;
52 per_cpu(gator_buffer_write, cpu)[buftype] = write5;
53 }
54}
55
56static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x)
57{
58 uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype];
59 uint32_t mask = gator_buffer_mask[buftype];
60 char *buffer = per_cpu(gator_buffer, cpu)[buftype];
61 int write0 = (write + 0) & mask;
62 int write1 = (write + 1) & mask;
63
64 if ((x & 0xffffffffffffff80LL) == 0) {
65 buffer[write0] = x & 0x7f;
66 per_cpu(gator_buffer_write, cpu)[buftype] = write1;
67 } else if ((x & 0xffffffffffffc000LL) == 0) {
68 int write2 = (write + 2) & mask;
69 buffer[write0] = x | 0x80;
70 buffer[write1] = (x>>7) & 0x7f;
71 per_cpu(gator_buffer_write, cpu)[buftype] = write2;
72 } else if ((x & 0xffffffffffe00000LL) == 0) {
73 int write2 = (write + 2) & mask;
74 int write3 = (write + 3) & mask;
75 buffer[write0] = x | 0x80;
76 buffer[write1] = (x>>7) | 0x80;
77 buffer[write2] = (x>>14) & 0x7f;
78 per_cpu(gator_buffer_write, cpu)[buftype] = write3;
79 } else if ((x & 0xfffffffff0000000LL) == 0) {
80 int write2 = (write + 2) & mask;
81 int write3 = (write + 3) & mask;
82 int write4 = (write + 4) & mask;
83 buffer[write0] = x | 0x80;
84 buffer[write1] = (x>>7) | 0x80;
85 buffer[write2] = (x>>14) | 0x80;
86 buffer[write3] = (x>>21) & 0x7f;
87 per_cpu(gator_buffer_write, cpu)[buftype] = write4;
88 } else if ((x & 0xfffffff800000000LL) == 0) {
89 int write2 = (write + 2) & mask;
90 int write3 = (write + 3) & mask;
91 int write4 = (write + 4) & mask;
92 int write5 = (write + 5) & mask;
93 buffer[write0] = x | 0x80;
94 buffer[write1] = (x>>7) | 0x80;
95 buffer[write2] = (x>>14) | 0x80;
96 buffer[write3] = (x>>21) | 0x80;
97 buffer[write4] = (x>>28) & 0x7f;
98 per_cpu(gator_buffer_write, cpu)[buftype] = write5;
99 } else if ((x & 0xfffffc0000000000LL) == 0) {
100 int write2 = (write + 2) & mask;
101 int write3 = (write + 3) & mask;
102 int write4 = (write + 4) & mask;
103 int write5 = (write + 5) & mask;
104 int write6 = (write + 6) & mask;
105 buffer[write0] = x | 0x80;
106 buffer[write1] = (x>>7) | 0x80;
107 buffer[write2] = (x>>14) | 0x80;
108 buffer[write3] = (x>>21) | 0x80;
109 buffer[write4] = (x>>28) | 0x80;
110 buffer[write5] = (x>>35) & 0x7f;
111 per_cpu(gator_buffer_write, cpu)[buftype] = write6;
112 } else if ((x & 0xfffe000000000000LL) == 0) {
113 int write2 = (write + 2) & mask;
114 int write3 = (write + 3) & mask;
115 int write4 = (write + 4) & mask;
116 int write5 = (write + 5) & mask;
117 int write6 = (write + 6) & mask;
118 int write7 = (write + 7) & mask;
119 buffer[write0] = x | 0x80;
120 buffer[write1] = (x>>7) | 0x80;
121 buffer[write2] = (x>>14) | 0x80;
122 buffer[write3] = (x>>21) | 0x80;
123 buffer[write4] = (x>>28) | 0x80;
124 buffer[write5] = (x>>35) | 0x80;
125 buffer[write6] = (x>>42) & 0x7f;
126 per_cpu(gator_buffer_write, cpu)[buftype] = write7;
127 } else if ((x & 0xff00000000000000LL) == 0) {
128 int write2 = (write + 2) & mask;
129 int write3 = (write + 3) & mask;
130 int write4 = (write + 4) & mask;
131 int write5 = (write + 5) & mask;
132 int write6 = (write + 6) & mask;
133 int write7 = (write + 7) & mask;
134 int write8 = (write + 8) & mask;
135 buffer[write0] = x | 0x80;
136 buffer[write1] = (x>>7) | 0x80;
137 buffer[write2] = (x>>14) | 0x80;
138 buffer[write3] = (x>>21) | 0x80;
139 buffer[write4] = (x>>28) | 0x80;
140 buffer[write5] = (x>>35) | 0x80;
141 buffer[write6] = (x>>42) | 0x80;
142 buffer[write7] = (x>>49) & 0x7f;
143 per_cpu(gator_buffer_write, cpu)[buftype] = write8;
144 } else {
145 int write2 = (write + 2) & mask;
146 int write3 = (write + 3) & mask;
147 int write4 = (write + 4) & mask;
148 int write5 = (write + 5) & mask;
149 int write6 = (write + 6) & mask;
150 int write7 = (write + 7) & mask;
151 int write8 = (write + 8) & mask;
152 int write9 = (write + 9) & mask;
153 buffer[write0] = x | 0x80;
154 buffer[write1] = (x>>7) | 0x80;
155 buffer[write2] = (x>>14) | 0x80;
156 buffer[write3] = (x>>21) | 0x80;
157 buffer[write4] = (x>>28) | 0x80;
158 buffer[write5] = (x>>35) | 0x80;
159 buffer[write6] = (x>>42) | 0x80;
160 buffer[write7] = (x>>49) | 0x80;
161 buffer[write8] = (x>>56) & 0xff;
162 per_cpu(gator_buffer_write, cpu)[buftype] = write9;
163 }
164}
165
166static int gator_write_packed_int(char *buffer, unsigned int x)
167{
168 if ((x & 0xffffff80) == 0) {
169 buffer[0] = x & 0x7f;
170 return 1;
171 } else if ((x & 0xffffc000) == 0) {
172 buffer[0] = x | 0x80;
173 buffer[1] = (x>>7) & 0x7f;
174 return 2;
175 } else if ((x & 0xffe00000) == 0) {
176 buffer[0] = x | 0x80;
177 buffer[1] = (x>>7) | 0x80;
178 buffer[2] = (x>>14) & 0x7f;
179 return 3;
180 } else if ((x & 0xf0000000) == 0) {
181 buffer[0] = x | 0x80;
182 buffer[1] = (x>>7) | 0x80;
183 buffer[2] = (x>>14) | 0x80;
184 buffer[3] = (x>>21) & 0x7f;
185 return 4;
186 } else {
187 buffer[0] = x | 0x80;
188 buffer[1] = (x>>7) | 0x80;
189 buffer[2] = (x>>14) | 0x80;
190 buffer[3] = (x>>21) | 0x80;
191 buffer[4] = (x>>28) & 0x0f;
192 return 5;
193 }
194}
195
196static int gator_write_packed_int64(char *buffer, unsigned long long x)
197{
198 if ((x & 0xffffffffffffff80LL) == 0) {
199 buffer[0] = x & 0x7f;
200 return 1;
201 } else if ((x & 0xffffffffffffc000LL) == 0) {
202 buffer[0] = x | 0x80;
203 buffer[1] = (x>>7) & 0x7f;
204 return 2;
205 } else if ((x & 0xffffffffffe00000LL) == 0) {
206 buffer[0] = x | 0x80;
207 buffer[1] = (x>>7) | 0x80;
208 buffer[2] = (x>>14) & 0x7f;
209 return 3;
210 } else if ((x & 0xfffffffff0000000LL) == 0) {
211 buffer[0] = x | 0x80;
212 buffer[1] = (x>>7) | 0x80;
213 buffer[2] = (x>>14) | 0x80;
214 buffer[3] = (x>>21) & 0x7f;
215 return 4;
216 } else if ((x & 0xfffffff800000000LL) == 0) {
217 buffer[0] = x | 0x80;
218 buffer[1] = (x>>7) | 0x80;
219 buffer[2] = (x>>14) | 0x80;
220 buffer[3] = (x>>21) | 0x80;
221 buffer[4] = (x>>28) & 0x7f;
222 return 5;
223 } else if ((x & 0xfffffc0000000000LL) == 0) {
224 buffer[0] = x | 0x80;
225 buffer[1] = (x>>7) | 0x80;
226 buffer[2] = (x>>14) | 0x80;
227 buffer[3] = (x>>21) | 0x80;
228 buffer[4] = (x>>28) | 0x80;
229 buffer[5] = (x>>35) & 0x7f;
230 return 6;
231 } else if ((x & 0xfffe000000000000LL) == 0) {
232 buffer[0] = x | 0x80;
233 buffer[1] = (x>>7) | 0x80;
234 buffer[2] = (x>>14) | 0x80;
235 buffer[3] = (x>>21) | 0x80;
236 buffer[4] = (x>>28) | 0x80;
237 buffer[5] = (x>>35) | 0x80;
238 buffer[6] = (x>>42) & 0x7f;
239 return 7;
240 } else if ((x & 0xff00000000000000LL) == 0) {
241 buffer[0] = x | 0x80;
242 buffer[1] = (x>>7) | 0x80;
243 buffer[2] = (x>>14) | 0x80;
244 buffer[3] = (x>>21) | 0x80;
245 buffer[4] = (x>>28) | 0x80;
246 buffer[5] = (x>>35) | 0x80;
247 buffer[6] = (x>>42) | 0x80;
248 buffer[7] = (x>>49) & 0x7f;
249 return 8;
250 } else {
251 buffer[0] = x | 0x80;
252 buffer[1] = (x>>7) | 0x80;
253 buffer[2] = (x>>14) | 0x80;
254 buffer[3] = (x>>21) | 0x80;
255 buffer[4] = (x>>28) | 0x80;
256 buffer[5] = (x>>35) | 0x80;
257 buffer[6] = (x>>42) | 0x80;
258 buffer[7] = (x>>49) | 0x80;
259 buffer[8] = (x>>56) & 0xff;
260 return 9;
261 }
262}
diff --git a/driver/gator_trace.h b/driver/gator_trace.h
deleted file mode 100644
index 446d37b..0000000
--- a/driver/gator_trace.h
+++ /dev/null
@@ -1,26 +0,0 @@
1/**
2 * Copyright 2010 ARM, Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
11# error Kernels prior to 2.6.32 not supported
12#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
13# define GATOR_DEFINE_PROBE(probe_name, proto) \
14 static void probe_##probe_name(PARAMS(proto))
15# define GATOR_REGISTER_TRACE(probe_name) \
16 register_trace_##probe_name(probe_##probe_name)
17# define GATOR_UNREGISTER_TRACE(probe_name) \
18 unregister_trace_##probe_name(probe_##probe_name)
19#else
20# define GATOR_DEFINE_PROBE(probe_name, proto) \
21 static void probe_##probe_name(void *data, PARAMS(proto))
22# define GATOR_REGISTER_TRACE(probe_name) \
23 register_trace_##probe_name(probe_##probe_name, NULL)
24# define GATOR_UNREGISTER_TRACE(probe_name) \
25 unregister_trace_##probe_name(probe_##probe_name, NULL)
26#endif
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index 19d8d89..7c0bd47 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -10,36 +10,89 @@
10#include <trace/events/sched.h> 10#include <trace/events/sched.h>
11#include "gator.h" 11#include "gator.h"
12 12
13#define SCHED_TIMER_EVENT 0
14#define SCHED_WAIT_TASK 1
15#define SCHED_WAKEUP 2
16#define SCHED_WAKEUP_NEW 3
17#define SCHED_SWITCH 4
18#define SCHED_MIGRATE_TASK 5
19#define SCHED_PROCESS_FREE 6
20#define SCHED_PROCESS_EXIT 7
21#define SCHED_PROCESS_WAIT 8
22#define SCHED_PROCESS_FORK 9
23#define SCHED_OVERFLOW -1 13#define SCHED_OVERFLOW -1
14#define SCHED_SWITCH 1
15#define SCHED_PROCESS_FREE 2
24 16
25#define SCHEDSIZE (16*1024) 17#define FIELD_TYPE 0
18#define FIELD_TIME 1
19#define FIELD_PARAM1 2
20#define FIELD_PARAM2 3
21#define FIELD_PARAM3 4
22#define FIELDS_PER_SCHED 5
26 23
27static DEFINE_PER_CPU(int *[2], theSchedBuf); 24#define SCHEDSIZE (8*1024)
25#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
26#define TASK_MAX_COLLISIONS 2
27
28static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf);
28static DEFINE_PER_CPU(int, theSchedSel); 29static DEFINE_PER_CPU(int, theSchedSel);
29static DEFINE_PER_CPU(int, theSchedPos); 30static DEFINE_PER_CPU(int, theSchedPos);
30static DEFINE_PER_CPU(int, theSchedErr); 31static DEFINE_PER_CPU(int, theSchedErr);
32static DEFINE_PER_CPU(uint64_t *, taskname_keys);
33
34void emit_pid_name(uint64_t time, struct task_struct* task)
35{
36 bool found = false;
37 unsigned long flags;
38 char taskcomm[TASK_COMM_LEN + 3];
39 int x, cpu = smp_processor_id();
40 uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
41 uint64_t value;
42
43 value = gator_chksum_crc32(task->comm);
44 value = (value << 32) | (uint32_t)task->pid;
45
46 // determine if the thread name was emitted already
47 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
48 if (keys[x] == value) {
49 found = true;
50 break;
51 }
52 }
53
54 if (!found) {
55 // shift values, new value always in front
56 uint64_t oldv, newv = value;
57 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
58 oldv = keys[x];
59 keys[x] = newv;
60 newv = oldv;
61 }
62
63 // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
64 if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1)
65 // append ellipses if task->comm has length of TASK_COMM_LEN - 1
66 strcat(taskcomm, "...");
67
68 // disable interrupts to synchronize with hrtimer populating timer buf
69 local_irq_save(flags);
70 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME);
71 gator_buffer_write_packed_int64(cpu, TIMER_BUF, time);
72 gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid);
73 gator_buffer_write_string(cpu, TIMER_BUF, taskcomm);
74 local_irq_restore(flags);
75 }
76}
31 77
32static void probe_sched_write(int type, int param1, int param2, int param3) 78static void probe_sched_write(int type, int param1, int param2, int param3)
33{ 79{
34 unsigned long flags; 80 unsigned long flags;
35 int cpu = smp_processor_id(); 81 int cpu = smp_processor_id();
36 uint64_t time = gator_get_time(); 82 uint64_t time = gator_get_time();
37 int *schedBuf; 83 uint64_t *schedBuf;
38 int schedPos; 84 int schedPos, cookie = param3;
39 85
40 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) 86 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
41 return; 87 return;
42 88
89 if (param3) {
90 // do as much work as possible before disabling interrupts
91 struct task_struct *task = (struct task_struct *)param3;
92 cookie = get_exec_cookie(cpu, TIMER_BUF, task);
93 emit_pid_name(time, task);
94 }
95
43 // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used 96 // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used
44 local_irq_save(flags); 97 local_irq_save(flags);
45 98
@@ -48,88 +101,37 @@ static void probe_sched_write(int type, int param1, int param2, int param3)
48 101
49 if (schedPos < (SCHEDSIZE-100)) { 102 if (schedPos < (SCHEDSIZE-100)) {
50 // capture 103 // capture
51 schedBuf[schedPos+0] = type; 104 schedBuf[schedPos+FIELD_TYPE] = type;
52 schedBuf[schedPos+1] = (int)time; 105 schedBuf[schedPos+FIELD_TIME] = time;
53 schedBuf[schedPos+2] = (int)(time >> 32); 106 schedBuf[schedPos+FIELD_PARAM1] = param1;
54 schedBuf[schedPos+3] = param1; 107 schedBuf[schedPos+FIELD_PARAM2] = param2;
55 schedBuf[schedPos+4] = param2; 108 schedBuf[schedPos+FIELD_PARAM3] = cookie;
56 schedBuf[schedPos+5] = param3; 109 per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED;
57 per_cpu(theSchedPos, cpu) = schedPos + 6;
58 } else if (!per_cpu(theSchedErr, cpu)) { 110 } else if (!per_cpu(theSchedErr, cpu)) {
59 per_cpu(theSchedErr, cpu) = 1; 111 per_cpu(theSchedErr, cpu) = 1;
60 schedBuf[schedPos+0] = SCHED_OVERFLOW; 112 schedBuf[schedPos+FIELD_TYPE] = SCHED_OVERFLOW;
61 schedBuf[schedPos+1] = 0; 113 schedBuf[schedPos+FIELD_TIME] = time;
62 schedBuf[schedPos+2] = 0; 114 schedBuf[schedPos+FIELD_PARAM1] = 0;
63 schedBuf[schedPos+3] = 0; 115 schedBuf[schedPos+FIELD_PARAM2] = 0;
64 schedBuf[schedPos+4] = 0; 116 schedBuf[schedPos+FIELD_PARAM3] = 0;
65 schedBuf[schedPos+5] = 0; 117 per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED;
66 per_cpu(theSchedPos, cpu) = schedPos + 6;
67 pr_debug("gator: tracepoint overflow\n"); 118 pr_debug("gator: tracepoint overflow\n");
68 } 119 }
69 local_irq_restore(flags); 120 local_irq_restore(flags);
70} 121}
71 122
72#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) 123#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
73GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct rq *rq, struct task_struct *p))
74#else
75GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct task_struct *p))
76#endif
77{
78 probe_sched_write(SCHED_WAIT_TASK, 0, p->pid, 0);
79}
80
81#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
82GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct rq *rq, struct task_struct *p, int success))
83#else
84GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct task_struct *p, int success))
85#endif
86{
87 if (success)
88 probe_sched_write(SCHED_WAKEUP, 0, p->pid, 0);
89}
90
91#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
92GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct rq *rq, struct task_struct *p, int success))
93#else
94GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct task_struct *p, int success))
95#endif
96{
97 if (success)
98 probe_sched_write(SCHED_WAKEUP_NEW, 0, p->tgid, p->pid);
99}
100
101#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
102GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) 124GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next))
103#else 125#else
104GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) 126GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
105#endif 127#endif
106{ 128{
107 probe_sched_write(SCHED_SWITCH, (int)next, next->tgid, next->pid); 129 probe_sched_write(SCHED_SWITCH, next->pid, next->tgid, (int)next);
108}
109
110GATOR_DEFINE_PROBE(sched_migrate_task, TP_PROTO(struct task_struct *p, int dest_cpu))
111{
112 probe_sched_write(SCHED_MIGRATE_TASK, 0, dest_cpu, p->pid);
113} 130}
114 131
115GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) 132GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
116{ 133{
117 probe_sched_write(SCHED_PROCESS_FREE, 0, p->pid, 0); 134 probe_sched_write(SCHED_PROCESS_FREE, p->pid, 0, 0);
118}
119
120GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p))
121{
122 probe_sched_write(SCHED_PROCESS_EXIT, 0, p->pid, 0);
123}
124
125GATOR_DEFINE_PROBE(sched_process_wait, TP_PROTO(struct pid *pid))
126{
127 probe_sched_write(SCHED_PROCESS_WAIT, 0, pid_nr(pid), 0);
128}
129
130GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
131{
132 probe_sched_write(SCHED_PROCESS_FORK, (int)child, parent->pid, child->pid);
133} 135}
134 136
135int gator_trace_sched_init(void) 137int gator_trace_sched_init(void)
@@ -139,59 +141,37 @@ int gator_trace_sched_init(void)
139 141
140int gator_trace_sched_start(void) 142int gator_trace_sched_start(void)
141{ 143{
142 int cpu; 144 int cpu, size;
143 145
144 for_each_present_cpu(cpu) { 146 for_each_present_cpu(cpu) {
145 per_cpu(theSchedSel, cpu) = 0; 147 per_cpu(theSchedSel, cpu) = 0;
146 per_cpu(theSchedPos, cpu) = 0; 148 per_cpu(theSchedPos, cpu) = 0;
147 per_cpu(theSchedErr, cpu) = 0; 149 per_cpu(theSchedErr, cpu) = 0;
148 per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); 150 per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
149 per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); 151 per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
150 if (!per_cpu(theSchedBuf, cpu)) 152 if (!per_cpu(theSchedBuf, cpu))
151 return -1; 153 return -1;
154
155 size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
156 per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
157 if (!per_cpu(taskname_keys, cpu))
158 return -1;
159 memset(per_cpu(taskname_keys, cpu), 0, size);
152 } 160 }
153 161
154 // register tracepoints 162 // register tracepoints
155 if (GATOR_REGISTER_TRACE(sched_wait_task))
156 goto fail_sched_wait_task;
157 if (GATOR_REGISTER_TRACE(sched_wakeup))
158 goto fail_sched_wakeup;
159 if (GATOR_REGISTER_TRACE(sched_wakeup_new))
160 goto fail_sched_wakeup_new;
161 if (GATOR_REGISTER_TRACE(sched_switch)) 163 if (GATOR_REGISTER_TRACE(sched_switch))
162 goto fail_sched_switch; 164 goto fail_sched_switch;
163 if (GATOR_REGISTER_TRACE(sched_migrate_task))
164 goto fail_sched_migrate_task;
165 if (GATOR_REGISTER_TRACE(sched_process_free)) 165 if (GATOR_REGISTER_TRACE(sched_process_free))
166 goto fail_sched_process_free; 166 goto fail_sched_process_free;
167 if (GATOR_REGISTER_TRACE(sched_process_exit))
168 goto fail_sched_process_exit;
169 if (GATOR_REGISTER_TRACE(sched_process_wait))
170 goto fail_sched_process_wait;
171 if (GATOR_REGISTER_TRACE(sched_process_fork))
172 goto fail_sched_process_fork;
173 pr_debug("gator: registered tracepoints\n"); 167 pr_debug("gator: registered tracepoints\n");
174 168
175 return 0; 169 return 0;
176 170
177 // unregister tracepoints on error 171 // unregister tracepoints on error
178fail_sched_process_fork:
179 GATOR_UNREGISTER_TRACE(sched_process_wait);
180fail_sched_process_wait:
181 GATOR_UNREGISTER_TRACE(sched_process_exit);
182fail_sched_process_exit:
183 GATOR_UNREGISTER_TRACE(sched_process_free);
184fail_sched_process_free: 172fail_sched_process_free:
185 GATOR_UNREGISTER_TRACE(sched_migrate_task);
186fail_sched_migrate_task:
187 GATOR_UNREGISTER_TRACE(sched_switch); 173 GATOR_UNREGISTER_TRACE(sched_switch);
188fail_sched_switch: 174fail_sched_switch:
189 GATOR_UNREGISTER_TRACE(sched_wakeup_new);
190fail_sched_wakeup_new:
191 GATOR_UNREGISTER_TRACE(sched_wakeup);
192fail_sched_wakeup:
193 GATOR_UNREGISTER_TRACE(sched_wait_task);
194fail_sched_wait_task:
195 pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); 175 pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
196 176
197 return -1; 177 return -1;
@@ -200,15 +180,8 @@ fail_sched_wait_task:
200void gator_trace_sched_stop(void) 180void gator_trace_sched_stop(void)
201{ 181{
202 int cpu; 182 int cpu;
203 GATOR_UNREGISTER_TRACE(sched_wait_task);
204 GATOR_UNREGISTER_TRACE(sched_wakeup);
205 GATOR_UNREGISTER_TRACE(sched_wakeup_new);
206 GATOR_UNREGISTER_TRACE(sched_switch); 183 GATOR_UNREGISTER_TRACE(sched_switch);
207 GATOR_UNREGISTER_TRACE(sched_migrate_task);
208 GATOR_UNREGISTER_TRACE(sched_process_free); 184 GATOR_UNREGISTER_TRACE(sched_process_free);
209 GATOR_UNREGISTER_TRACE(sched_process_exit);
210 GATOR_UNREGISTER_TRACE(sched_process_wait);
211 GATOR_UNREGISTER_TRACE(sched_process_fork);
212 pr_debug("gator: unregistered tracepoints\n"); 185 pr_debug("gator: unregistered tracepoints\n");
213 186
214 for_each_present_cpu(cpu) { 187 for_each_present_cpu(cpu) {
@@ -216,17 +189,16 @@ void gator_trace_sched_stop(void)
216 kfree(per_cpu(theSchedBuf, cpu)[1]); 189 kfree(per_cpu(theSchedBuf, cpu)[1]);
217 per_cpu(theSchedBuf, cpu)[0] = NULL; 190 per_cpu(theSchedBuf, cpu)[0] = NULL;
218 per_cpu(theSchedBuf, cpu)[1] = NULL; 191 per_cpu(theSchedBuf, cpu)[1] = NULL;
192 kfree(per_cpu(taskname_keys, cpu));
219 } 193 }
220} 194}
221 195
222int gator_trace_sched_read(int **buffer) 196int gator_trace_sched_read(long long **buffer)
223{ 197{
224 uint64_t time = gator_get_time();
225 int cpu = smp_processor_id(); 198 int cpu = smp_processor_id();
226 unsigned long flags; 199 unsigned long flags;
227 int *schedBuf; 200 uint64_t *schedBuf;
228 int schedPos; 201 int schedPos;
229 int i;
230 202
231 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) 203 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
232 return 0; 204 return 0;
@@ -242,20 +214,6 @@ int gator_trace_sched_read(int **buffer)
242 214
243 local_irq_restore(flags); 215 local_irq_restore(flags);
244 216
245 // find mm and replace with cookies
246 for (i = 0; i < schedPos; i += 6) {
247 uint32_t cookie = schedBuf[i+3];
248 if (cookie) {
249 struct task_struct *task = (struct task_struct *)cookie;
250 schedBuf[i+3] = get_exec_cookie(cpu, task);
251 }
252 }
253
254 // timer/end event
255 schedBuf[schedPos++] = SCHED_TIMER_EVENT;
256 schedBuf[schedPos++] = (int)time;
257 schedBuf[schedPos++] = (int)(time >> 32);
258
259 if (buffer) 217 if (buffer)
260 *buffer = schedBuf; 218 *buffer = schedBuf;
261 219