diff options
Diffstat (limited to 'driver')
-rw-r--r-- | driver/Makefile | 5 | ||||
-rw-r--r-- | driver/README_Streamline.txt | 26 | ||||
-rw-r--r-- | driver/gator.h | 12 | ||||
-rw-r--r-- | driver/gator_annotate.c | 28 | ||||
-rw-r--r-- | driver/gator_backtrace.c | 6 | ||||
-rw-r--r-- | driver/gator_cookies.c | 40 | ||||
-rw-r--r-- | driver/gator_ebs.c | 160 | ||||
-rw-r--r-- | driver/gator_events.c | 40 | ||||
-rw-r--r-- | driver/gator_events.h | 19 | ||||
-rw-r--r-- | driver/gator_events_armv6.c | 40 | ||||
-rw-r--r-- | driver/gator_events_armv7.c | 186 | ||||
-rw-r--r-- | driver/gator_events_armv7.h | 38 | ||||
-rw-r--r-- | driver/gator_events_l2c-310.c | 182 | ||||
-rw-r--r-- | driver/gator_events_meminfo.c | 11 | ||||
-rw-r--r-- | driver/gator_events_pl310.c | 176 | ||||
-rw-r--r-- | driver/gator_events_scorpion.c | 40 | ||||
-rw-r--r-- | driver/gator_main.c | 529 | ||||
-rw-r--r-- | driver/gator_pack.c | 262 | ||||
-rw-r--r-- | driver/gator_trace.h | 26 | ||||
-rw-r--r-- | driver/gator_trace_sched.c | 228 |
20 files changed, 1196 insertions, 858 deletions
diff --git a/driver/Makefile b/driver/Makefile index b3680e1..de59957 100644 --- a/driver/Makefile +++ b/driver/Makefile | |||
@@ -17,7 +17,8 @@ endif | |||
17 | 17 | ||
18 | gator-$(CONFIG_ARM) += gator_events_armv6.o \ | 18 | gator-$(CONFIG_ARM) += gator_events_armv6.o \ |
19 | gator_events_armv7.o \ | 19 | gator_events_armv7.o \ |
20 | gator_events_pl310.o | 20 | gator_events_l2c-310.o \ |
21 | gator_events_scorpion.o | ||
21 | 22 | ||
22 | $(obj)/gator_main.o: gator_events.h | 23 | $(obj)/gator_main.o: gator_events.h |
23 | 24 | ||
@@ -40,6 +41,6 @@ all: | |||
40 | $(error) | 41 | $(error) |
41 | 42 | ||
42 | clean: | 43 | clean: |
43 | rm -f *.o modules.order Module.symvers gator.ko gator.mod.c | 44 | rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c |
44 | 45 | ||
45 | endif | 46 | endif |
diff --git a/driver/README_Streamline.txt b/driver/README_Streamline.txt index 4688474..9b64b8e 100644 --- a/driver/README_Streamline.txt +++ b/driver/README_Streamline.txt | |||
@@ -5,10 +5,21 @@ Instructions on setting up ARM Streamline on the target. | |||
5 | The gator driver and gator daemon are required to run on the ARM linux target in order for ARM Streamline to operate. | 5 | The gator driver and gator daemon are required to run on the ARM linux target in order for ARM Streamline to operate. |
6 | The driver should be built as a module and the daemon must run with root permissions on the target. | 6 | The driver should be built as a module and the daemon must run with root permissions on the target. |
7 | 7 | ||
8 | *** Introduction *** | ||
9 | |||
10 | A linux development environment with cross compiling tools is most likely required, depending on what is already created and provided. | ||
11 | -For users, the ideal environment is to be given a BSP with gatord and gator.ko already running on a properly configured kernel. In such a scenario, a development environment is not needed, root permission may or may not be needed (gatord must be executed with root permissions but can be automatically started, see below), and the user can run Streamline and profile the system without any setup. | ||
12 | -The ideal development environment has the kernel source code available to be rebuilt and executed on the target. This environment allows the greatest flexibility in configuring the kernel and building the gator driver module. | ||
13 | -However, it is possible that a user/developer has a kernel but does not have the source code. In this scenario it may or may not be possible to obtain a valid profile. | ||
14 | -First, check if the kernel has the proper configuration options (see below). Profiling cannot occur using a kernel that is not configured properly, a new kernel must be created. | ||
15 | -Second, given a properly configured kernel, check if the filesystem contains the kernel source/headers, which can be used to re-create the gator driver. | ||
16 | -If the kernel is not properly configured or sources/headers are not available, the developer is on their own and kernel creation is beyond the scope of this document. Note: It is possible for a module to work when compiled against a similar kernel source code, though this is not guaranteed to work due to differences in kernel structures, exported symbols and incompatible configuration parameters. | ||
17 | |||
8 | *** Preparing and building the kernel *** | 18 | *** Preparing and building the kernel *** |
9 | 19 | ||
10 | cd into the root source dir of the linux kernel | 20 | cd into the root source dir of the linux kernel |
11 | make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- <platform_defconfig> (choose the appropriate configuration for your board) | 21 | if your target has never been configured, choose the appropriate configuration for your target |
22 | make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- <platform_defconfig> | ||
12 | make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfig | 23 | make ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfig |
13 | 24 | ||
14 | Required Kernel Changes (depending on the kernel version, the location of these configuration settings within menuconfig may be different) | 25 | Required Kernel Changes (depending on the kernel version, the location of these configuration settings within menuconfig may be different) |
@@ -19,6 +30,7 @@ Required Kernel Changes (depending on the kernel version, the location of these | |||
19 | - [*] Trace process context switches and events | 30 | - [*] Trace process context switches and events |
20 | - Kernel Features | 31 | - Kernel Features |
21 | - [*] High Resolution Timer Support | 32 | - [*] High Resolution Timer Support |
33 | - [*] Use local timer interrupts (only required for SMP) | ||
22 | 34 | ||
23 | The "context switches and events" option will not be available if other trace configurations are enabled. Other trace configurations being enabled is sufficient to turn on context switches and events. | 35 | The "context switches and events" option will not be available if other trace configurations are enabled. Other trace configurations being enabled is sufficient to turn on context switches and events. |
24 | 36 | ||
@@ -29,10 +41,14 @@ Note: Configurations may not be supported on all targets | |||
29 | 41 | ||
30 | make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage | 42 | make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage |
31 | 43 | ||
44 | *** Checking the gator requirements *** | ||
45 | |||
46 | (optional) Use the hrtimer_module utility to validate the kernel High Resolution Timer requirement. | ||
47 | |||
32 | *** Building the gator module *** | 48 | *** Building the gator module *** |
33 | 49 | ||
34 | To create the gator.ko module, | 50 | To create the gator.ko module, |
35 | cd /ds-5-install-directory/arm/src | 51 | cd /ds-5-install-directory/arm/gator/driver-src |
36 | tar xzf gator-driver.tar.gz | 52 | tar xzf gator-driver.tar.gz |
37 | cd gator-driver | 53 | cd gator-driver |
38 | make -C <kernel_build_dir> M=`pwd` ARCH=arm CROSS_COMPILE=<...> modules | 54 | make -C <kernel_build_dir> M=`pwd` ARCH=arm CROSS_COMPILE=<...> modules |
@@ -50,7 +66,7 @@ Recommended compiler settings: | |||
50 | *** Running gator *** | 66 | *** Running gator *** |
51 | 67 | ||
52 | Load the kernel onto the target and copy gatord and gator.ko into the target's filesystem. | 68 | Load the kernel onto the target and copy gatord and gator.ko into the target's filesystem. |
53 | gatord is located in <installdir>/arm/armv5t/. | 69 | gatord is located in <installdir>/arm/gator/linux or <installdir>/arm/gator/android or can be built from source. |
54 | Ensure gatord has execute permissions | 70 | Ensure gatord has execute permissions |
55 | chmod +x gatord | 71 | chmod +x gatord |
56 | gator.ko must be located in the same directory as gatord on the target. | 72 | gator.ko must be located in the same directory as gatord on the target. |
@@ -65,7 +81,7 @@ make ARCH=arm CROSS_COMPILE=$(CROSS_TOOLS}/bin/arm-none-linux-gnueabi- menuconfi | |||
65 | 81 | ||
66 | make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage | 82 | make -j5 ARCH=arm CROSS_COMPILE=${CROSS_TOOLS}/bin/arm-none-linux-gnueabi- uImage |
67 | Use vmlinux as the image for debug symbols in Streamline. | 83 | Use vmlinux as the image for debug symbols in Streamline. |
68 | Drivers may be profiled using this method by statically linking the driver into the kernel image. | 84 | Drivers may be profiled using this method by statically linking the driver into the kernel image or adding the module as an image. |
69 | Note that the gator driver does not perform kernel call stack recording. | 85 | Note that the gator driver does not perform kernel call stack recording. |
70 | 86 | ||
71 | *** Automatically start gator on boot (optional) *** | 87 | *** Automatically start gator on boot (optional) *** |
@@ -78,4 +94,4 @@ update-rc.d rungator.sh defaults | |||
78 | 94 | ||
79 | *** GPL License *** | 95 | *** GPL License *** |
80 | 96 | ||
81 | For license information, please see the file LICENSE. | 97 | For license information, please see the file LICENSE after unzipping driver-src/gator-driver.tar.gz. |
diff --git a/driver/gator.h b/driver/gator.h index 9d802a3..724ae19 100644 --- a/driver/gator.h +++ b/driver/gator.h | |||
@@ -14,6 +14,16 @@ | |||
14 | #include <linux/mm.h> | 14 | #include <linux/mm.h> |
15 | #include <linux/list.h> | 15 | #include <linux/list.h> |
16 | 16 | ||
17 | // cpu ids | ||
18 | #define ARM1136 0xb36 | ||
19 | #define ARM1156 0xb56 | ||
20 | #define ARM1176 0xb76 | ||
21 | #define ARM11MPCORE 0xb02 | ||
22 | #define CORTEX_A5 0xc05 | ||
23 | #define CORTEX_A8 0xc08 | ||
24 | #define CORTEX_A9 0xc09 | ||
25 | #define CORTEX_A15 0xc0f | ||
26 | |||
17 | /****************************************************************************** | 27 | /****************************************************************************** |
18 | * Filesystem | 28 | * Filesystem |
19 | ******************************************************************************/ | 29 | ******************************************************************************/ |
@@ -62,6 +72,7 @@ struct gator_interface { | |||
62 | void (*online)(void); | 72 | void (*online)(void); |
63 | void (*offline)(void); | 73 | void (*offline)(void); |
64 | int (*read)(int **buffer); | 74 | int (*read)(int **buffer); |
75 | int (*read64)(long long **buffer); | ||
65 | struct list_head list; | 76 | struct list_head list; |
66 | }; | 77 | }; |
67 | 78 | ||
@@ -75,5 +86,4 @@ extern u32 gator_cpuid(void); | |||
75 | 86 | ||
76 | extern unsigned long gator_net_traffic; | 87 | extern unsigned long gator_net_traffic; |
77 | 88 | ||
78 | |||
79 | #endif // GATOR_H_ | 89 | #endif // GATOR_H_ |
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c index d8d86af..e096d15 100644 --- a/driver/gator_annotate.c +++ b/driver/gator_annotate.c | |||
@@ -22,6 +22,7 @@ static char *annotateBuf0; | |||
22 | static char *annotateBuf1; | 22 | static char *annotateBuf1; |
23 | static int annotatePos; | 23 | static int annotatePos; |
24 | static int annotateSel; | 24 | static int annotateSel; |
25 | static bool collect_annotations = false; | ||
25 | 26 | ||
26 | static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) | 27 | static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) |
27 | { | 28 | { |
@@ -45,16 +46,13 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t | |||
45 | if (retval == 0) { | 46 | if (retval == 0) { |
46 | // synchronize shared variables annotateBuf and annotatePos | 47 | // synchronize shared variables annotateBuf and annotatePos |
47 | spin_lock(&annotate_lock); | 48 | spin_lock(&annotate_lock); |
48 | if (annotateBuf) { | 49 | if (collect_annotations && annotateBuf) { |
49 | uint32_t tid = current->pid; | 50 | uint32_t tid = current->pid; |
50 | uint32_t tick = gator_master_tick; | ||
51 | uint64_t time = gator_get_time(); | 51 | uint64_t time = gator_get_time(); |
52 | uint32_t cpuid = smp_processor_id(); | 52 | uint32_t cpuid = smp_processor_id(); |
53 | int pos = annotatePos; | 53 | int pos = annotatePos; |
54 | pos += gator_write_packed_int(&annotateBuf[pos], tid); | 54 | pos += gator_write_packed_int(&annotateBuf[pos], tid); |
55 | pos += gator_write_packed_int(&annotateBuf[pos], tick); | 55 | pos += gator_write_packed_int64(&annotateBuf[pos], time); |
56 | pos += gator_write_packed_int(&annotateBuf[pos], time); | ||
57 | pos += gator_write_packed_int(&annotateBuf[pos], time >> 32); | ||
58 | pos += gator_write_packed_int(&annotateBuf[pos], cpuid); | 56 | pos += gator_write_packed_int(&annotateBuf[pos], cpuid); |
59 | pos += gator_write_packed_int(&annotateBuf[pos], size); | 57 | pos += gator_write_packed_int(&annotateBuf[pos], size); |
60 | memcpy(&annotateBuf[pos], tempBuffer, size); | 58 | memcpy(&annotateBuf[pos], tempBuffer, size); |
@@ -81,12 +79,9 @@ static int annotate_release(struct inode *inode, struct file *file) | |||
81 | spin_lock(&annotate_lock); | 79 | spin_lock(&annotate_lock); |
82 | if (annotateBuf) { | 80 | if (annotateBuf) { |
83 | uint32_t tid = current->pid; | 81 | uint32_t tid = current->pid; |
84 | uint32_t tick = gator_master_tick; | ||
85 | int pos = annotatePos; | 82 | int pos = annotatePos; |
86 | pos += gator_write_packed_int(&annotateBuf[pos], tid); | 83 | pos += gator_write_packed_int(&annotateBuf[pos], tid); |
87 | pos += gator_write_packed_int(&annotateBuf[pos], tick); | 84 | pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time |
88 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // time | ||
89 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // time | ||
90 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid | 85 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid |
91 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // size | 86 | pos += gator_write_packed_int(&annotateBuf[pos], 0); // size |
92 | annotatePos = pos; | 87 | annotatePos = pos; |
@@ -118,13 +113,21 @@ static int gator_annotate_init(void) | |||
118 | 113 | ||
119 | static int gator_annotate_start(void) | 114 | static int gator_annotate_start(void) |
120 | { | 115 | { |
121 | annotatePos = annotateSel = 0; | 116 | annotateSel = 0; |
117 | annotatePos = 1; | ||
122 | annotateBuf = annotateBuf0; | 118 | annotateBuf = annotateBuf0; |
119 | annotateBuf[0] = FRAME_ANNOTATE; | ||
120 | collect_annotations = true; | ||
123 | return 0; | 121 | return 0; |
124 | } | 122 | } |
125 | 123 | ||
126 | static void gator_annotate_stop(void) | 124 | static void gator_annotate_stop(void) |
127 | { | 125 | { |
126 | collect_annotations = false; | ||
127 | } | ||
128 | |||
129 | static void gator_annotate_shutdown(void) | ||
130 | { | ||
128 | spin_lock(&annotate_lock); | 131 | spin_lock(&annotate_lock); |
129 | annotateBuf = NULL; | 132 | annotateBuf = NULL; |
130 | spin_unlock(&annotate_lock); | 133 | spin_unlock(&annotate_lock); |
@@ -141,7 +144,7 @@ static void gator_annotate_exit(void) | |||
141 | 144 | ||
142 | static int gator_annotate_ready(void) | 145 | static int gator_annotate_ready(void) |
143 | { | 146 | { |
144 | return annotatePos && annotateBuf; | 147 | return annotatePos > 1 && annotateBuf; |
145 | } | 148 | } |
146 | 149 | ||
147 | static int gator_annotate_read(char **buffer) | 150 | static int gator_annotate_read(char **buffer) |
@@ -158,8 +161,9 @@ static int gator_annotate_read(char **buffer) | |||
158 | 161 | ||
159 | spin_lock(&annotate_lock); | 162 | spin_lock(&annotate_lock); |
160 | len = annotatePos; | 163 | len = annotatePos; |
161 | annotatePos = 0; | ||
162 | annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; | 164 | annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; |
165 | annotateBuf[0] = FRAME_ANNOTATE; | ||
166 | annotatePos = 1; | ||
163 | spin_unlock(&annotate_lock); | 167 | spin_unlock(&annotate_lock); |
164 | 168 | ||
165 | return len; | 169 | return len; |
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c index 628a18f..b5b0e63 100644 --- a/driver/gator_backtrace.c +++ b/driver/gator_backtrace.c | |||
@@ -15,7 +15,7 @@ struct frame_tail_eabi { | |||
15 | unsigned long lr; | 15 | unsigned long lr; |
16 | }; | 16 | }; |
17 | 17 | ||
18 | static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned int depth) | 18 | static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs, unsigned int depth) |
19 | { | 19 | { |
20 | #if defined(__arm__) | 20 | #if defined(__arm__) |
21 | struct frame_tail_eabi *tail; | 21 | struct frame_tail_eabi *tail; |
@@ -31,7 +31,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in | |||
31 | } | 31 | } |
32 | 32 | ||
33 | /* entry preamble may not have executed */ | 33 | /* entry preamble may not have executed */ |
34 | gator_add_trace(cpu, lr); | 34 | gator_add_trace(cpu, buftype, lr); |
35 | 35 | ||
36 | /* check tail is valid */ | 36 | /* check tail is valid */ |
37 | if (fp == 0) { | 37 | if (fp == 0) { |
@@ -49,7 +49,7 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs * const regs, unsigned in | |||
49 | ptrtail = &buftail; | 49 | ptrtail = &buftail; |
50 | 50 | ||
51 | lr = ptrtail[0].lr; | 51 | lr = ptrtail[0].lr; |
52 | gator_add_trace(cpu, lr); | 52 | gator_add_trace(cpu, buftype, lr); |
53 | 53 | ||
54 | /* frame pointers should progress back up the stack, towards higher addresses */ | 54 | /* frame pointers should progress back up the stack, towards higher addresses */ |
55 | next = (struct frame_tail_eabi *)(lr - 4); | 55 | next = (struct frame_tail_eabi *)(lr - 4); |
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c index 64be841..a646fb2 100644 --- a/driver/gator_cookies.c +++ b/driver/gator_cookies.c | |||
@@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, translate_buffer_read); | |||
22 | static DEFINE_PER_CPU(int, translate_buffer_write); | 22 | static DEFINE_PER_CPU(int, translate_buffer_write); |
23 | static DEFINE_PER_CPU(unsigned int *, translate_buffer); | 23 | static DEFINE_PER_CPU(unsigned int *, translate_buffer); |
24 | 24 | ||
25 | static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod); | 25 | static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt); |
26 | static void wq_cookie_handler(struct work_struct *unused); | 26 | static void wq_cookie_handler(struct work_struct *unused); |
27 | DECLARE_WORK(cookie_work, wq_cookie_handler); | 27 | DECLARE_WORK(cookie_work, wq_cookie_handler); |
28 | 28 | ||
@@ -96,7 +96,7 @@ static void cookiemap_add(uint64_t key, uint32_t value) { | |||
96 | 96 | ||
97 | for (x = MAX_COLLISIONS-1; x > 0; x--) { | 97 | for (x = MAX_COLLISIONS-1; x > 0; x--) { |
98 | keys[x] = keys[x-1]; | 98 | keys[x] = keys[x-1]; |
99 | values[x] = keys[x-1]; | 99 | values[x] = values[x-1]; |
100 | } | 100 | } |
101 | keys[0] = key; | 101 | keys[0] = key; |
102 | values[0] = value; | 102 | values[0] = value; |
@@ -126,12 +126,12 @@ static void wq_cookie_handler(struct work_struct *unused) | |||
126 | while (per_cpu(translate_buffer_read, cpu) != commit) { | 126 | while (per_cpu(translate_buffer_read, cpu) != commit) { |
127 | task = (struct task_struct *)translate_buffer_read_int(cpu); | 127 | task = (struct task_struct *)translate_buffer_read_int(cpu); |
128 | vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); | 128 | vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); |
129 | cookie = get_cookie(cpu, task, vma, NULL); | 129 | cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false); |
130 | } | 130 | } |
131 | } | 131 | } |
132 | 132 | ||
133 | // Retrieve full name from proc/pid/cmdline for java processes on Android | 133 | // Retrieve full name from proc/pid/cmdline for java processes on Android |
134 | static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma) | 134 | static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt) |
135 | { | 135 | { |
136 | void *maddr; | 136 | void *maddr; |
137 | unsigned int len; | 137 | unsigned int len; |
@@ -143,7 +143,9 @@ static int translate_app_process(char** text, int cpu, struct task_struct * task | |||
143 | char * buf = per_cpu(translate_text, cpu); | 143 | char * buf = per_cpu(translate_text, cpu); |
144 | 144 | ||
145 | // Push work into a work queue if in atomic context as the kernel functions below might sleep | 145 | // Push work into a work queue if in atomic context as the kernel functions below might sleep |
146 | if (in_irq()) { | 146 | // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems |
147 | // inconsistent during a context switch between android/linux versions | ||
148 | if (in_interrupt) { | ||
147 | // Check if already in buffer | 149 | // Check if already in buffer |
148 | ptr = per_cpu(translate_buffer_read, cpu); | 150 | ptr = per_cpu(translate_buffer_read, cpu); |
149 | while (ptr != per_cpu(translate_buffer_write, cpu)) { | 151 | while (ptr != per_cpu(translate_buffer_write, cpu)) { |
@@ -205,7 +207,7 @@ out: | |||
205 | return retval; | 207 | return retval; |
206 | } | 208 | } |
207 | 209 | ||
208 | static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod) | 210 | static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt) |
209 | { | 211 | { |
210 | unsigned long flags, cookie; | 212 | unsigned long flags, cookie; |
211 | struct path *path; | 213 | struct path *path; |
@@ -235,26 +237,26 @@ static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_a | |||
235 | } | 237 | } |
236 | 238 | ||
237 | if (strcmp(text, "app_process") == 0 && !mod) { | 239 | if (strcmp(text, "app_process") == 0 && !mod) { |
238 | if (!translate_app_process(&text, cpu, task, vma)) | 240 | if (!translate_app_process(&text, cpu, task, vma, in_interrupt)) |
239 | return INVALID_COOKIE; | 241 | return INVALID_COOKIE; |
240 | } | 242 | } |
241 | 243 | ||
242 | // Can be called from interrupt handler or from work queue | 244 | // Can be called from interrupt handler or from work queue or from scheduler trace |
243 | local_irq_save(flags); | 245 | local_irq_save(flags); |
244 | 246 | ||
245 | cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; | 247 | cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; |
246 | cookiemap_add(key, cookie); | 248 | cookiemap_add(key, cookie); |
247 | 249 | ||
248 | gator_buffer_write_packed_int(cpu, PROTOCOL_COOKIE); | 250 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); |
249 | gator_buffer_write_packed_int(cpu, cookie); | 251 | gator_buffer_write_packed_int(cpu, buftype, cookie); |
250 | gator_buffer_write_string(cpu, text); | 252 | gator_buffer_write_string(cpu, buftype, text); |
251 | 253 | ||
252 | local_irq_restore(flags); | 254 | local_irq_restore(flags); |
253 | 255 | ||
254 | return cookie; | 256 | return cookie; |
255 | } | 257 | } |
256 | 258 | ||
257 | static int get_exec_cookie(int cpu, struct task_struct *task) | 259 | static int get_exec_cookie(int cpu, int buftype, struct task_struct *task) |
258 | { | 260 | { |
259 | unsigned long cookie = NO_COOKIE; | 261 | unsigned long cookie = NO_COOKIE; |
260 | struct mm_struct *mm = task->mm; | 262 | struct mm_struct *mm = task->mm; |
@@ -268,14 +270,14 @@ static int get_exec_cookie(int cpu, struct task_struct *task) | |||
268 | continue; | 270 | continue; |
269 | if (!(vma->vm_flags & VM_EXECUTABLE)) | 271 | if (!(vma->vm_flags & VM_EXECUTABLE)) |
270 | continue; | 272 | continue; |
271 | cookie = get_cookie(cpu, task, vma, NULL); | 273 | cookie = get_cookie(cpu, buftype, task, vma, NULL, true); |
272 | break; | 274 | break; |
273 | } | 275 | } |
274 | 276 | ||
275 | return cookie; | 277 | return cookie; |
276 | } | 278 | } |
277 | 279 | ||
278 | static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset) | 280 | static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset) |
279 | { | 281 | { |
280 | unsigned long cookie = NO_COOKIE; | 282 | unsigned long cookie = NO_COOKIE; |
281 | struct mm_struct *mm = task->mm; | 283 | struct mm_struct *mm = task->mm; |
@@ -289,7 +291,7 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig | |||
289 | continue; | 291 | continue; |
290 | 292 | ||
291 | if (vma->vm_file) { | 293 | if (vma->vm_file) { |
292 | cookie = get_cookie(cpu, task, vma, NULL); | 294 | cookie = get_cookie(cpu, buftype, task, vma, NULL, true); |
293 | *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; | 295 | *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; |
294 | } else { | 296 | } else { |
295 | /* must be an anonymous map */ | 297 | /* must be an anonymous map */ |
@@ -318,10 +320,18 @@ static int cookies_initialize(void) | |||
318 | 320 | ||
319 | size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); | 321 | size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); |
320 | per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); | 322 | per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); |
323 | if (!per_cpu(cookie_keys, cpu)) { | ||
324 | err = -ENOMEM; | ||
325 | goto cookie_setup_error; | ||
326 | } | ||
321 | memset(per_cpu(cookie_keys, cpu), 0, size); | 327 | memset(per_cpu(cookie_keys, cpu), 0, size); |
322 | 328 | ||
323 | size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); | 329 | size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); |
324 | per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL); | 330 | per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL); |
331 | if (!per_cpu(cookie_values, cpu)) { | ||
332 | err = -ENOMEM; | ||
333 | goto cookie_setup_error; | ||
334 | } | ||
325 | memset(per_cpu(cookie_values, cpu), 0, size); | 335 | memset(per_cpu(cookie_values, cpu), 0, size); |
326 | 336 | ||
327 | per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL); | 337 | per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL); |
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c new file mode 100644 index 0000000..9b55347 --- /dev/null +++ b/driver/gator_ebs.c | |||
@@ -0,0 +1,160 @@ | |||
1 | /** | ||
2 | * Copyright (C) ARM Limited 2011. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | */ | ||
9 | |||
10 | /****************************************************************************** | ||
11 | * event based sampling handling | ||
12 | ******************************************************************************/ | ||
13 | |||
14 | #if defined (__arm__) | ||
15 | #include "gator_events_armv7.h" | ||
16 | #include <linux/platform_device.h> | ||
17 | #include <linux/interrupt.h> | ||
18 | #include <linux/irq.h> | ||
19 | |||
20 | #if LINUX_PMU_SUPPORT | ||
21 | #include <asm/pmu.h> | ||
22 | |||
23 | static struct platform_device *pmu_device; | ||
24 | |||
25 | static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg) | ||
26 | { | ||
27 | unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF; | ||
28 | struct pt_regs * const regs = get_irq_regs(); | ||
29 | u32 flags; | ||
30 | |||
31 | // Stop irq generation | ||
32 | armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); | ||
33 | |||
34 | // Get and reset overflow status flags | ||
35 | flags = armv7_pmnc_reset_interrupt(); | ||
36 | |||
37 | // Counters header | ||
38 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type | ||
39 | gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time | ||
40 | |||
41 | // Cycle counter | ||
42 | if (flags & (1 << 31)) { | ||
43 | int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun | ||
44 | gator_buffer_write_packed_int(cpu, buftype, 2); // length | ||
45 | gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key | ||
46 | gator_buffer_write_packed_int(cpu, buftype, value); // value | ||
47 | } | ||
48 | |||
49 | // PMNC counters | ||
50 | for (cnt = CNT0; cnt < CNTMAX; cnt++) { | ||
51 | if (flags & (1 << (cnt - CNT0))) { | ||
52 | int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun | ||
53 | gator_buffer_write_packed_int(cpu, buftype, 2); // length | ||
54 | gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key | ||
55 | gator_buffer_write_packed_int(cpu, buftype, value); // value | ||
56 | } | ||
57 | } | ||
58 | |||
59 | // End Counters, length of zero | ||
60 | gator_buffer_write_packed_int(cpu, buftype, 0); | ||
61 | |||
62 | // Output backtrace | ||
63 | gator_add_sample(cpu, buftype, regs); | ||
64 | |||
65 | // Check and commit; commit is set to occur once buffer is 3/4 full | ||
66 | event_buffer_check(cpu); | ||
67 | |||
68 | // Allow irq generation | ||
69 | armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); | ||
70 | |||
71 | return IRQ_HANDLED; | ||
72 | } | ||
73 | #endif | ||
74 | |||
75 | static int gator_event_sampling_start(void) | ||
76 | { | ||
77 | int cnt; | ||
78 | |||
79 | event_based_sampling = false; | ||
80 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | ||
81 | if (pmnc_count[cnt] > 0) { | ||
82 | event_based_sampling = true; | ||
83 | break; | ||
84 | } | ||
85 | } | ||
86 | |||
87 | #if LINUX_PMU_SUPPORT | ||
88 | pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU); | ||
89 | if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) { | ||
90 | pr_err("gator: unable to reserve the pmu\n"); | ||
91 | return -1; | ||
92 | } | ||
93 | |||
94 | if (event_based_sampling) { | ||
95 | int irq, i; | ||
96 | |||
97 | if (IS_ERR(pmu_device)) { | ||
98 | pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed"); | ||
99 | return -1; | ||
100 | } | ||
101 | |||
102 | init_pmu(ARM_PMU_DEVICE_CPU); | ||
103 | if (pmu_device->num_resources == 0) { | ||
104 | pr_err("gator: no irqs for PMUs defined\n"); | ||
105 | release_pmu(pmu_device); | ||
106 | pmu_device = NULL; | ||
107 | return -1; | ||
108 | } | ||
109 | |||
110 | for (i = 0; i < pmu_device->num_resources; ++i) { | ||
111 | irq = platform_get_irq(pmu_device, i); | ||
112 | if (irq < 0) | ||
113 | continue; | ||
114 | |||
115 | if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) { | ||
116 | pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq); | ||
117 | |||
118 | // clean up and exit | ||
119 | for (i = i - 1; i >= 0; --i) { | ||
120 | irq = platform_get_irq(pmu_device, i); | ||
121 | if (irq >= 0) | ||
122 | free_irq(irq, NULL); | ||
123 | } | ||
124 | release_pmu(pmu_device); | ||
125 | pmu_device = NULL; | ||
126 | return -1; | ||
127 | } | ||
128 | } | ||
129 | } | ||
130 | #else | ||
131 | if (event_based_sampling) { | ||
132 | pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n"); | ||
133 | return -1; | ||
134 | } | ||
135 | #endif | ||
136 | |||
137 | return 0; | ||
138 | } | ||
139 | |||
140 | static void gator_event_sampling_stop(void) | ||
141 | { | ||
142 | #if LINUX_PMU_SUPPORT | ||
143 | if (event_based_sampling) { | ||
144 | int i, irq; | ||
145 | for (i = pmu_device->num_resources - 1; i >= 0; --i) { | ||
146 | irq = platform_get_irq(pmu_device, i); | ||
147 | if (irq >= 0) | ||
148 | free_irq(irq, NULL); | ||
149 | } | ||
150 | } | ||
151 | if (!IS_ERR(pmu_device)) | ||
152 | release_pmu(pmu_device); | ||
153 | pmu_device = NULL; | ||
154 | #endif | ||
155 | } | ||
156 | |||
157 | #else | ||
158 | static int gator_event_sampling_start(void) {return 0;} | ||
159 | static void gator_event_sampling_stop(void) {} | ||
160 | #endif | ||
diff --git a/driver/gator_events.c b/driver/gator_events.c deleted file mode 100644 index d837257..0000000 --- a/driver/gator_events.c +++ /dev/null | |||
@@ -1,40 +0,0 @@ | |||
1 | /** | ||
2 | * Copyright (C) ARM Limited 2010-2011. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | */ | ||
9 | |||
10 | /** | ||
11 | * This file is #included in gator_main.c | ||
12 | * Update this file and Makefile to add custom counters. | ||
13 | */ | ||
14 | |||
15 | extern int gator_events_armv6_install(gator_interface *gi); | ||
16 | extern int gator_events_armv7_install(gator_interface *gi); | ||
17 | extern int gator_events_irq_install(gator_interface *gi); | ||
18 | extern int gator_events_sched_install(gator_interface *gi); | ||
19 | extern int gator_events_block_install(gator_interface *gi); | ||
20 | extern int gator_events_meminfo_install(gator_interface *gi); | ||
21 | extern int gator_events_net_install(gator_interface *gi); | ||
22 | |||
23 | static int gator_events_install(void) | ||
24 | { | ||
25 | if (gator_event_install(gator_events_armv6_install)) | ||
26 | return -1; | ||
27 | if (gator_event_install(gator_events_armv7_install)) | ||
28 | return -1; | ||
29 | if (gator_event_install(gator_events_irq_install)) | ||
30 | return -1; | ||
31 | if (gator_event_install(gator_events_sched_install)) | ||
32 | return -1; | ||
33 | if (gator_event_install(gator_events_block_install)) | ||
34 | return -1; | ||
35 | if (gator_event_install(gator_events_meminfo_install)) | ||
36 | return -1; | ||
37 | if (gator_event_install(gator_events_net_install)) | ||
38 | return -1; | ||
39 | return 0; | ||
40 | } | ||
diff --git a/driver/gator_events.h b/driver/gator_events.h deleted file mode 100644 index 8237c1a..0000000 --- a/driver/gator_events.h +++ /dev/null | |||
@@ -1,19 +0,0 @@ | |||
1 | /** | ||
2 | * Copyright 2010 ARM, Ltd. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | */ | ||
8 | |||
9 | |||
10 | struct __gator_interface { | ||
11 | int (*create_files)(struct super_block *sb, struct dentry *root); | ||
12 | int (*init)(int *key); | ||
13 | int (*start)(void); | ||
14 | void (*stop)(void); | ||
15 | int (*read)(int **buffer); | ||
16 | struct __gator_interface *next; | ||
17 | }; | ||
18 | |||
19 | typedef struct __gator_interface gator_interface; | ||
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c index 7b1d875..170f066 100644 --- a/driver/gator_events_armv6.c +++ b/driver/gator_events_armv6.c | |||
@@ -8,10 +8,6 @@ | |||
8 | 8 | ||
9 | #include "gator.h" | 9 | #include "gator.h" |
10 | 10 | ||
11 | #define ARM1136 0xb36 | ||
12 | #define ARM1156 0xb56 | ||
13 | #define ARM1176 0xb76 | ||
14 | |||
15 | static const char *pmnc_name; | 11 | static const char *pmnc_name; |
16 | 12 | ||
17 | /* | 13 | /* |
@@ -29,9 +25,10 @@ static const char *pmnc_name; | |||
29 | #define CCNT 2 | 25 | #define CCNT 2 |
30 | #define CNTMAX (CCNT+1) | 26 | #define CNTMAX (CCNT+1) |
31 | 27 | ||
32 | static int pmnc_count = 0; | 28 | static int pmnc_counters = 0; |
33 | static unsigned long pmnc_enabled[CNTMAX]; | 29 | static unsigned long pmnc_enabled[CNTMAX]; |
34 | static unsigned long pmnc_event[CNTMAX]; | 30 | static unsigned long pmnc_event[CNTMAX]; |
31 | static unsigned long pmnc_count[CNTMAX]; | ||
35 | static unsigned long pmnc_key[CNTMAX]; | 32 | static unsigned long pmnc_key[CNTMAX]; |
36 | 33 | ||
37 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); | 34 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); |
@@ -72,7 +69,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) | |||
72 | struct dentry *dir; | 69 | struct dentry *dir; |
73 | int i; | 70 | int i; |
74 | 71 | ||
75 | pmnc_count = 3; | 72 | pmnc_counters = 3; |
76 | 73 | ||
77 | for (i = PMN0; i <= CCNT; i++) { | 74 | for (i = PMN0; i <= CCNT; i++) { |
78 | char buf[40]; | 75 | char buf[40]; |
@@ -86,10 +83,11 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) | |||
86 | return -1; | 83 | return -1; |
87 | } | 84 | } |
88 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); | 85 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); |
86 | gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); | ||
87 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
89 | if (i != CCNT) { | 88 | if (i != CCNT) { |
90 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); | 89 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); |
91 | } | 90 | } |
92 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
93 | } | 91 | } |
94 | 92 | ||
95 | return 0; | 93 | return 0; |
@@ -119,18 +117,14 @@ static void gator_events_armv6_online(void) | |||
119 | 117 | ||
120 | event = pmnc_event[cnt] & 255; | 118 | event = pmnc_event[cnt] & 255; |
121 | 119 | ||
122 | /* | 120 | // Set event (if destined for PMNx counters) |
123 | * Set event (if destined for PMNx counters) | ||
124 | */ | ||
125 | if (cnt == PMN0) { | 121 | if (cnt == PMN0) { |
126 | pmnc |= event << 20; | 122 | pmnc |= event << 20; |
127 | } else if (cnt == PMN1) { | 123 | } else if (cnt == PMN1) { |
128 | pmnc |= event << 12; | 124 | pmnc |= event << 12; |
129 | } | 125 | } |
130 | 126 | ||
131 | /* | 127 | // Reset counter |
132 | * Reset counter | ||
133 | */ | ||
134 | armv6_pmnc_reset_counter(cnt); | 128 | armv6_pmnc_reset_counter(cnt); |
135 | } | 129 | } |
136 | armv6_pmnc_write(pmnc | PMCR_E); | 130 | armv6_pmnc_write(pmnc | PMCR_E); |
@@ -146,6 +140,20 @@ static void gator_events_armv6_offline(void) | |||
146 | } | 140 | } |
147 | } | 141 | } |
148 | 142 | ||
143 | static int gator_events_armv6_start(void) | ||
144 | { | ||
145 | int cnt; | ||
146 | |||
147 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | ||
148 | if (pmnc_count[cnt] > 0) { | ||
149 | pr_err("gator: event based sampling not supported on ARM v6 architectures\n"); | ||
150 | return -1; | ||
151 | } | ||
152 | } | ||
153 | |||
154 | return 0; | ||
155 | } | ||
156 | |||
149 | static void gator_events_armv6_stop(void) | 157 | static void gator_events_armv6_stop(void) |
150 | { | 158 | { |
151 | unsigned int cnt; | 159 | unsigned int cnt; |
@@ -153,6 +161,7 @@ static void gator_events_armv6_stop(void) | |||
153 | for (cnt = PMN0; cnt <= CCNT; cnt++) { | 161 | for (cnt = PMN0; cnt <= CCNT; cnt++) { |
154 | pmnc_enabled[cnt] = 0; | 162 | pmnc_enabled[cnt] = 0; |
155 | pmnc_event[cnt] = 0; | 163 | pmnc_event[cnt] = 0; |
164 | pmnc_count[cnt] = 0; | ||
156 | } | 165 | } |
157 | } | 166 | } |
158 | 167 | ||
@@ -193,6 +202,7 @@ static int gator_events_armv6_read(int **buffer) | |||
193 | 202 | ||
194 | static struct gator_interface gator_events_armv6_interface = { | 203 | static struct gator_interface gator_events_armv6_interface = { |
195 | .create_files = gator_events_armv6_create_files, | 204 | .create_files = gator_events_armv6_create_files, |
205 | .start = gator_events_armv6_start, | ||
196 | .stop = gator_events_armv6_stop, | 206 | .stop = gator_events_armv6_stop, |
197 | .online = gator_events_armv6_online, | 207 | .online = gator_events_armv6_online, |
198 | .offline = gator_events_armv6_offline, | 208 | .offline = gator_events_armv6_offline, |
@@ -209,6 +219,9 @@ int gator_events_armv6_init(void) | |||
209 | case ARM1176: | 219 | case ARM1176: |
210 | pmnc_name = "ARM11"; | 220 | pmnc_name = "ARM11"; |
211 | break; | 221 | break; |
222 | case ARM11MPCORE: | ||
223 | pmnc_name = "ARM11MPCore"; | ||
224 | break; | ||
212 | default: | 225 | default: |
213 | return -1; | 226 | return -1; |
214 | } | 227 | } |
@@ -216,6 +229,7 @@ int gator_events_armv6_init(void) | |||
216 | for (cnt = PMN0; cnt <= CCNT; cnt++) { | 229 | for (cnt = PMN0; cnt <= CCNT; cnt++) { |
217 | pmnc_enabled[cnt] = 0; | 230 | pmnc_enabled[cnt] = 0; |
218 | pmnc_event[cnt] = 0; | 231 | pmnc_event[cnt] = 0; |
232 | pmnc_count[cnt] = 0; | ||
219 | pmnc_key[cnt] = gator_events_get_key(); | 233 | pmnc_key[cnt] = gator_events_get_key(); |
220 | } | 234 | } |
221 | 235 | ||
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c index 0b84745..58855f8 100644 --- a/driver/gator_events_armv7.c +++ b/driver/gator_events_armv7.c | |||
@@ -6,129 +6,116 @@ | |||
6 | * published by the Free Software Foundation. | 6 | * published by the Free Software Foundation. |
7 | */ | 7 | */ |
8 | 8 | ||
9 | #include "gator.h" | 9 | /* Disabling interrupts |
10 | 10 | * Many of the functions below disable interrupts via local_irq_save(). This disabling of interrupts is done to prevent any race conditions | |
11 | #define CORTEX_A5 0xc05 | 11 | * between multiple entities (e.g. hrtimer interrupts and event based interrupts) calling the same functions. As accessing the pmu involves |
12 | #define CORTEX_A8 0xc08 | 12 | * several steps (disable, select, read, enable), these steps must be performed atomically. Normal synchronization routines cannot be used |
13 | #define CORTEX_A9 0xc09 | 13 | * as these functions are being called from interrupt context. |
14 | #define CORTEX_A15 0xc0f | 14 | */ |
15 | |||
16 | static const char *pmnc_name; | ||
17 | static int pmnc_count; | ||
18 | |||
19 | // Per-CPU PMNC: config reg | ||
20 | #define PMNC_E (1 << 0) /* Enable all counters */ | ||
21 | #define PMNC_P (1 << 1) /* Reset all counters */ | ||
22 | #define PMNC_C (1 << 2) /* Cycle counter reset */ | ||
23 | #define PMNC_MASK 0x3f /* Mask for writable bits */ | ||
24 | 15 | ||
25 | // ccnt reg | 16 | #include "gator.h" |
26 | #define CCNT_REG (1 << 31) | 17 | #include "gator_events_armv7.h" |
27 | 18 | ||
28 | #define CCNT 0 | 19 | const char *pmnc_name; |
29 | #define CNT0 1 | 20 | int pmnc_counters; |
30 | #define CNTMAX (6+1) | ||
31 | 21 | ||
32 | static unsigned long pmnc_enabled[CNTMAX]; | 22 | unsigned long pmnc_enabled[CNTMAX]; |
33 | static unsigned long pmnc_event[CNTMAX]; | 23 | unsigned long pmnc_event[CNTMAX]; |
34 | static unsigned long pmnc_key[CNTMAX]; | 24 | unsigned long pmnc_count[CNTMAX]; |
25 | unsigned long pmnc_key[CNTMAX]; | ||
35 | 26 | ||
36 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); | 27 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); |
37 | static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); | 28 | static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); |
38 | 29 | ||
39 | static inline void armv7_pmnc_write(u32 val) | 30 | inline void armv7_pmnc_write(u32 val) |
40 | { | 31 | { |
41 | val &= PMNC_MASK; | 32 | val &= PMNC_MASK; |
42 | asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); | 33 | asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); |
43 | } | 34 | } |
44 | 35 | ||
45 | static inline u32 armv7_pmnc_read(void) | 36 | inline u32 armv7_pmnc_read(void) |
46 | { | 37 | { |
47 | u32 val; | 38 | u32 val; |
48 | asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); | 39 | asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); |
49 | return val; | 40 | return val; |
50 | } | 41 | } |
51 | 42 | ||
52 | static inline u32 armv7_ccnt_read(void) | 43 | inline u32 armv7_ccnt_read(u32 reset_value) |
53 | { | 44 | { |
54 | u32 zero = 0; | 45 | unsigned long flags; |
46 | u32 newval = -reset_value; | ||
55 | u32 den = CCNT_REG; | 47 | u32 den = CCNT_REG; |
56 | u32 val; | 48 | u32 val; |
57 | 49 | ||
50 | local_irq_save(flags); | ||
58 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable | 51 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable |
59 | asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read | 52 | asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read |
60 | asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (zero)); // zero | 53 | asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (newval));// new value |
61 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable | 54 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable |
55 | local_irq_restore(flags); | ||
62 | 56 | ||
63 | return val; | 57 | return val; |
64 | } | 58 | } |
65 | 59 | ||
66 | static inline u32 armv7_cntn_read(unsigned int cnt) | 60 | inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value) |
67 | { | 61 | { |
68 | u32 zero = 0; | 62 | unsigned long flags; |
63 | u32 newval = -reset_value; | ||
69 | u32 sel = (cnt - CNT0); | 64 | u32 sel = (cnt - CNT0); |
70 | u32 den = 1 << sel; | 65 | u32 den = 1 << sel; |
71 | u32 val; | 66 | u32 oldval; |
72 | 67 | ||
68 | local_irq_save(flags); | ||
73 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable | 69 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable |
74 | asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select | 70 | asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select |
75 | asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); // read | 71 | asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (oldval)); // read |
76 | asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (zero)); // zero | 72 | asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (newval));// new value |
77 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable | 73 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable |
74 | local_irq_restore(flags); | ||
78 | 75 | ||
79 | return val; | 76 | return oldval; |
80 | } | 77 | } |
81 | 78 | ||
82 | static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) | 79 | static inline void armv7_pmnc_enable_interrupt(unsigned int cnt) |
83 | { | 80 | { |
84 | u32 val; | 81 | u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); |
82 | asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); | ||
83 | } | ||
85 | 84 | ||
86 | if (cnt >= CNTMAX) { | 85 | static inline void armv7_pmnc_disable_interrupt(unsigned int cnt) |
87 | pr_err("gator: CPU%u enabling wrong PMNC counter %d\n", smp_processor_id(), cnt); | 86 | { |
88 | return -1; | 87 | u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); |
89 | } | 88 | asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); |
89 | } | ||
90 | 90 | ||
91 | if (cnt == CCNT) | 91 | inline u32 armv7_pmnc_reset_interrupt() |
92 | val = CCNT_REG; | 92 | { |
93 | else | 93 | // Get and reset overflow status flags |
94 | val = (1 << (cnt - CNT0)); | 94 | u32 flags; |
95 | asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (flags)); | ||
96 | flags &= 0x8000003f; | ||
97 | asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (flags)); | ||
98 | return flags; | ||
99 | } | ||
95 | 100 | ||
101 | static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) | ||
102 | { | ||
103 | u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; | ||
96 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); | 104 | asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); |
97 | |||
98 | return cnt; | 105 | return cnt; |
99 | } | 106 | } |
100 | 107 | ||
101 | static inline u32 armv7_pmnc_disable_counter(unsigned int cnt) | 108 | static inline u32 armv7_pmnc_disable_counter(unsigned int cnt) |
102 | { | 109 | { |
103 | u32 val; | 110 | u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; |
104 | |||
105 | if (cnt >= CNTMAX) { | ||
106 | pr_err("gator: CPU%u disabling wrong PMNC counter %d\n", smp_processor_id(), cnt); | ||
107 | return -1; | ||
108 | } | ||
109 | |||
110 | if (cnt == CCNT) | ||
111 | val = CCNT_REG; | ||
112 | else | ||
113 | val = (1 << (cnt - CNT0)); | ||
114 | |||
115 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); | 111 | asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); |
116 | |||
117 | return cnt; | 112 | return cnt; |
118 | } | 113 | } |
119 | 114 | ||
120 | static inline int armv7_pmnc_select_counter(unsigned int cnt) | 115 | static inline int armv7_pmnc_select_counter(unsigned int cnt) |
121 | { | 116 | { |
122 | u32 val; | 117 | u32 val = (cnt - CNT0); |
123 | |||
124 | if ((cnt == CCNT) || (cnt >= CNTMAX)) { | ||
125 | pr_err("gator: CPU%u selecting wrong PMNC counter %d\n", smp_processor_id(), cnt); | ||
126 | return -1; | ||
127 | } | ||
128 | |||
129 | val = (cnt - CNT0); | ||
130 | asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); | 118 | asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); |
131 | |||
132 | return cnt; | 119 | return cnt; |
133 | } | 120 | } |
134 | 121 | ||
@@ -139,37 +126,12 @@ static inline void armv7_pmnc_write_evtsel(unsigned int cnt, u32 val) | |||
139 | } | 126 | } |
140 | } | 127 | } |
141 | 128 | ||
142 | static void armv7_pmnc_reset_counter(unsigned int cnt) | ||
143 | { | ||
144 | u32 val = 0; | ||
145 | |||
146 | if (cnt == CCNT) { | ||
147 | armv7_pmnc_disable_counter(cnt); | ||
148 | |||
149 | asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (val)); | ||
150 | |||
151 | if (pmnc_enabled[cnt] != 0) | ||
152 | armv7_pmnc_enable_counter(cnt); | ||
153 | |||
154 | } else if (cnt >= CNTMAX) { | ||
155 | pr_err("gator: CPU%u resetting wrong PMNC counter %d\n", smp_processor_id(), cnt); | ||
156 | } else { | ||
157 | armv7_pmnc_disable_counter(cnt); | ||
158 | |||
159 | if (armv7_pmnc_select_counter(cnt) == cnt) | ||
160 | asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (val)); | ||
161 | |||
162 | if (pmnc_enabled[cnt] != 0) | ||
163 | armv7_pmnc_enable_counter(cnt); | ||
164 | } | ||
165 | } | ||
166 | |||
167 | static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root) | 129 | static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root) |
168 | { | 130 | { |
169 | struct dentry *dir; | 131 | struct dentry *dir; |
170 | int i; | 132 | int i; |
171 | 133 | ||
172 | for (i = 0; i < pmnc_count; i++) { | 134 | for (i = 0; i < pmnc_counters; i++) { |
173 | char buf[40]; | 135 | char buf[40]; |
174 | if (i == 0) { | 136 | if (i == 0) { |
175 | snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); | 137 | snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); |
@@ -181,10 +143,11 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry | |||
181 | return -1; | 143 | return -1; |
182 | } | 144 | } |
183 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); | 145 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); |
146 | gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); | ||
147 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
184 | if (i > 0) { | 148 | if (i > 0) { |
185 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); | 149 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); |
186 | } | 150 | } |
187 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
188 | } | 151 | } |
189 | 152 | ||
190 | return 0; | 153 | return 0; |
@@ -202,6 +165,9 @@ static void gator_events_armv7_online(void) | |||
202 | // Initialize & Reset PMNC: C bit and P bit | 165 | // Initialize & Reset PMNC: C bit and P bit |
203 | armv7_pmnc_write(PMNC_P | PMNC_C); | 166 | armv7_pmnc_write(PMNC_P | PMNC_C); |
204 | 167 | ||
168 | // Reset overflow flags | ||
169 | armv7_pmnc_reset_interrupt(); | ||
170 | |||
205 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | 171 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { |
206 | unsigned long event; | 172 | unsigned long event; |
207 | 173 | ||
@@ -219,10 +185,16 @@ static void gator_events_armv7_online(void) | |||
219 | if (cnt != CCNT) | 185 | if (cnt != CCNT) |
220 | armv7_pmnc_write_evtsel(cnt, event); | 186 | armv7_pmnc_write_evtsel(cnt, event); |
221 | 187 | ||
188 | // Enable/disable interrupt | ||
189 | if (pmnc_count[cnt] > 0) | ||
190 | armv7_pmnc_enable_interrupt(cnt); | ||
191 | else | ||
192 | armv7_pmnc_disable_interrupt(cnt); | ||
193 | |||
222 | // Reset counter | 194 | // Reset counter |
223 | armv7_pmnc_reset_counter(cnt); | 195 | cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]); |
224 | 196 | ||
225 | // Enable counter, but do not enable interrupt for this counter | 197 | // Enable counter |
226 | armv7_pmnc_enable_counter(cnt); | 198 | armv7_pmnc_enable_counter(cnt); |
227 | } | 199 | } |
228 | 200 | ||
@@ -242,6 +214,7 @@ static void gator_events_armv7_stop(void) | |||
242 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | 214 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { |
243 | pmnc_enabled[cnt] = 0; | 215 | pmnc_enabled[cnt] = 0; |
244 | pmnc_event[cnt] = 0; | 216 | pmnc_event[cnt] = 0; |
217 | pmnc_count[cnt] = 0; | ||
245 | } | 218 | } |
246 | } | 219 | } |
247 | 220 | ||
@@ -250,16 +223,16 @@ static int gator_events_armv7_read(int **buffer) | |||
250 | int cnt, len = 0; | 223 | int cnt, len = 0; |
251 | int cpu = smp_processor_id(); | 224 | int cpu = smp_processor_id(); |
252 | 225 | ||
253 | if (!pmnc_count) | 226 | if (!pmnc_counters) |
254 | return 0; | 227 | return 0; |
255 | 228 | ||
256 | for (cnt = 0; cnt < pmnc_count; cnt++) { | 229 | for (cnt = 0; cnt < pmnc_counters; cnt++) { |
257 | if (pmnc_enabled[cnt]) { | 230 | if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) { |
258 | int value; | 231 | int value; |
259 | if (cnt == CCNT) { | 232 | if (cnt == CCNT) { |
260 | value = armv7_ccnt_read(); | 233 | value = armv7_ccnt_read(0); |
261 | } else { | 234 | } else { |
262 | value = armv7_cntn_read(cnt); | 235 | value = armv7_cntn_read(cnt, 0); |
263 | } | 236 | } |
264 | if (value != per_cpu(perfPrev, cpu)[cnt]) { | 237 | if (value != per_cpu(perfPrev, cpu)[cnt]) { |
265 | per_cpu(perfPrev, cpu)[cnt] = value; | 238 | per_cpu(perfPrev, cpu)[cnt] = value; |
@@ -291,29 +264,30 @@ int gator_events_armv7_init(void) | |||
291 | switch (gator_cpuid()) { | 264 | switch (gator_cpuid()) { |
292 | case CORTEX_A5: | 265 | case CORTEX_A5: |
293 | pmnc_name = "Cortex-A5"; | 266 | pmnc_name = "Cortex-A5"; |
294 | pmnc_count = 2; | 267 | pmnc_counters = 2; |
295 | break; | 268 | break; |
296 | case CORTEX_A8: | 269 | case CORTEX_A8: |
297 | pmnc_name = "Cortex-A8"; | 270 | pmnc_name = "Cortex-A8"; |
298 | pmnc_count = 4; | 271 | pmnc_counters = 4; |
299 | break; | 272 | break; |
300 | case CORTEX_A9: | 273 | case CORTEX_A9: |
301 | pmnc_name = "Cortex-A9"; | 274 | pmnc_name = "Cortex-A9"; |
302 | pmnc_count = 6; | 275 | pmnc_counters = 6; |
303 | break; | 276 | break; |
304 | case CORTEX_A15: | 277 | case CORTEX_A15: |
305 | pmnc_name = "Cortex-A15"; | 278 | pmnc_name = "Cortex-A15"; |
306 | pmnc_count = 6; | 279 | pmnc_counters = 6; |
307 | break; | 280 | break; |
308 | default: | 281 | default: |
309 | return -1; | 282 | return -1; |
310 | } | 283 | } |
311 | 284 | ||
312 | pmnc_count++; // CNT[n] + CCNT | 285 | pmnc_counters++; // CNT[n] + CCNT |
313 | 286 | ||
314 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | 287 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { |
315 | pmnc_enabled[cnt] = 0; | 288 | pmnc_enabled[cnt] = 0; |
316 | pmnc_event[cnt] = 0; | 289 | pmnc_event[cnt] = 0; |
290 | pmnc_count[cnt] = 0; | ||
317 | pmnc_key[cnt] = gator_events_get_key(); | 291 | pmnc_key[cnt] = gator_events_get_key(); |
318 | } | 292 | } |
319 | 293 | ||
diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h new file mode 100644 index 0000000..d5e8d6e --- /dev/null +++ b/driver/gator_events_armv7.h | |||
@@ -0,0 +1,38 @@ | |||
1 | /** | ||
2 | * Copyright (C) ARM Limited 2011. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | */ | ||
8 | |||
9 | #ifndef GATOR_EVENTS_ARMV7_H_ | ||
10 | #define GATOR_EVENTS_ARMV7_H_ | ||
11 | |||
12 | // Per-CPU PMNC: config reg | ||
13 | #define PMNC_E (1 << 0) /* Enable all counters */ | ||
14 | #define PMNC_P (1 << 1) /* Reset all counters */ | ||
15 | #define PMNC_C (1 << 2) /* Cycle counter reset */ | ||
16 | #define PMNC_MASK 0x3f /* Mask for writable bits */ | ||
17 | |||
18 | // ccnt reg | ||
19 | #define CCNT_REG (1 << 31) | ||
20 | |||
21 | #define CCNT 0 | ||
22 | #define CNT0 1 | ||
23 | #define CNTMAX (6+1) | ||
24 | |||
25 | // Function prototypes | ||
26 | extern void armv7_pmnc_write(u32 val); | ||
27 | extern u32 armv7_pmnc_read(void); | ||
28 | extern u32 armv7_ccnt_read(u32 reset_value); | ||
29 | extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value); | ||
30 | extern u32 armv7_pmnc_reset_interrupt(void); | ||
31 | |||
32 | // Externed variables | ||
33 | extern unsigned long pmnc_enabled[CNTMAX]; | ||
34 | extern unsigned long pmnc_event[CNTMAX]; | ||
35 | extern unsigned long pmnc_count[CNTMAX]; | ||
36 | extern unsigned long pmnc_key[CNTMAX]; | ||
37 | |||
38 | #endif // GATOR_EVENTS_ARMV7_H_ | ||
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c new file mode 100644 index 0000000..96683b3 --- /dev/null +++ b/driver/gator_events_l2c-310.c | |||
@@ -0,0 +1,182 @@ | |||
1 | /** | ||
2 | * l2c310 (L2 Cache Controller) event counters for gator | ||
3 | * | ||
4 | * Copyright (C) ARM Limited 2010-2011. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify | ||
7 | * it under the terms of the GNU General Public License version 2 as | ||
8 | * published by the Free Software Foundation. | ||
9 | */ | ||
10 | |||
11 | #include <linux/init.h> | ||
12 | #include <linux/io.h> | ||
13 | #include <asm/hardware/cache-l2x0.h> | ||
14 | |||
15 | #include "gator.h" | ||
16 | |||
17 | #define L2C310_COUNTERS_NUM 2 | ||
18 | |||
19 | static struct { | ||
20 | unsigned long enabled; | ||
21 | unsigned long event; | ||
22 | unsigned long key; | ||
23 | } l2c310_counters[L2C310_COUNTERS_NUM]; | ||
24 | |||
25 | static int l2c310_buffer[L2C310_COUNTERS_NUM * 2]; | ||
26 | |||
27 | static void __iomem *l2c310_base; | ||
28 | |||
29 | |||
30 | |||
31 | static void gator_events_l2c310_reset_counters(void) | ||
32 | { | ||
33 | u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL); | ||
34 | |||
35 | val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1; | ||
36 | |||
37 | writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL); | ||
38 | } | ||
39 | |||
40 | |||
41 | static int gator_events_l2c310_create_files(struct super_block *sb, | ||
42 | struct dentry *root) | ||
43 | { | ||
44 | int i; | ||
45 | |||
46 | for (i = 0; i < L2C310_COUNTERS_NUM; i++) { | ||
47 | char buf[16]; | ||
48 | struct dentry *dir; | ||
49 | |||
50 | snprintf(buf, sizeof(buf), "L2C-310_cnt%d", i); | ||
51 | dir = gatorfs_mkdir(sb, root, buf); | ||
52 | if (WARN_ON(!dir)) | ||
53 | return -1; | ||
54 | gatorfs_create_ulong(sb, dir, "enabled", | ||
55 | &l2c310_counters[i].enabled); | ||
56 | gatorfs_create_ulong(sb, dir, "event", | ||
57 | &l2c310_counters[i].event); | ||
58 | gatorfs_create_ro_ulong(sb, dir, "key", | ||
59 | &l2c310_counters[i].key); | ||
60 | } | ||
61 | |||
62 | return 0; | ||
63 | } | ||
64 | |||
65 | static int gator_events_l2c310_start(void) | ||
66 | { | ||
67 | static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = { | ||
68 | L2X0_EVENT_CNT0_CFG, | ||
69 | L2X0_EVENT_CNT1_CFG, | ||
70 | }; | ||
71 | int i; | ||
72 | |||
73 | /* Counter event sources */ | ||
74 | for (i = 0; i < L2C310_COUNTERS_NUM; i++) | ||
75 | writel((l2c310_counters[i].event & 0xf) << 2, | ||
76 | l2c310_base + l2x0_event_cntx_cfg[i]); | ||
77 | |||
78 | gator_events_l2c310_reset_counters(); | ||
79 | |||
80 | /* Event counter enable */ | ||
81 | writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL); | ||
82 | |||
83 | return 0; | ||
84 | } | ||
85 | |||
86 | static void gator_events_l2c310_stop(void) | ||
87 | { | ||
88 | /* Event counter disable */ | ||
89 | writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL); | ||
90 | } | ||
91 | |||
92 | static int gator_events_l2c310_read(int **buffer) | ||
93 | { | ||
94 | static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = { | ||
95 | L2X0_EVENT_CNT0_VAL, | ||
96 | L2X0_EVENT_CNT1_VAL, | ||
97 | }; | ||
98 | int i; | ||
99 | int len = 0; | ||
100 | |||
101 | if (smp_processor_id()) | ||
102 | return 0; | ||
103 | |||
104 | for (i = 0; i < L2C310_COUNTERS_NUM; i++) { | ||
105 | if (l2c310_counters[i].enabled) { | ||
106 | l2c310_buffer[len++] = l2c310_counters[i].key; | ||
107 | l2c310_buffer[len++] = readl(l2c310_base + | ||
108 | l2x0_event_cntx_val[i]); | ||
109 | } | ||
110 | } | ||
111 | |||
112 | /* l2c310 counters are saturating, not wrapping in case of overflow */ | ||
113 | gator_events_l2c310_reset_counters(); | ||
114 | |||
115 | if (buffer) | ||
116 | *buffer = l2c310_buffer; | ||
117 | |||
118 | return len; | ||
119 | } | ||
120 | |||
121 | static struct gator_interface gator_events_l2c310_interface = { | ||
122 | .create_files = gator_events_l2c310_create_files, | ||
123 | .start = gator_events_l2c310_start, | ||
124 | .stop = gator_events_l2c310_stop, | ||
125 | .read = gator_events_l2c310_read, | ||
126 | }; | ||
127 | |||
128 | static void __maybe_unused gator_events_l2c310_probe(unsigned long phys) | ||
129 | { | ||
130 | if (l2c310_base) | ||
131 | return; | ||
132 | |||
133 | l2c310_base = ioremap(phys, SZ_4K); | ||
134 | if (l2c310_base) { | ||
135 | u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID); | ||
136 | |||
137 | if ((cache_id & 0xff0003c0) != 0x410000c0) { | ||
138 | iounmap(l2c310_base); | ||
139 | l2c310_base = NULL; | ||
140 | } | ||
141 | } | ||
142 | } | ||
143 | |||
144 | int gator_events_l2c310_init(void) | ||
145 | { | ||
146 | int i; | ||
147 | |||
148 | if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) | ||
149 | return -1; | ||
150 | |||
151 | #if defined(CONFIG_ARCH_EXYNOS4) | ||
152 | gator_events_l2c310_probe(0xfe600000); | ||
153 | #endif | ||
154 | #if defined(CONFIG_ARCH_S5PV310) | ||
155 | gator_events_l2c310_probe(0x10502000); | ||
156 | #endif | ||
157 | #if defined(CONFIG_ARCH_OMAP4) | ||
158 | gator_events_l2c310_probe(0x48242000); | ||
159 | #endif | ||
160 | #if defined(CONFIG_ARCH_TEGRA) | ||
161 | gator_events_l2c310_probe(0x50043000); | ||
162 | #endif | ||
163 | #if defined(CONFIG_ARCH_U8500) | ||
164 | gator_events_l2c310_probe(0xa0412000); | ||
165 | #endif | ||
166 | #if defined(CONFIG_ARCH_VEXPRESS) | ||
167 | // A9x4 core tile (HBI-0191) | ||
168 | gator_events_l2c310_probe(0x1e00a000); | ||
169 | // New memory map tiles | ||
170 | gator_events_l2c310_probe(0x2c0f0000); | ||
171 | #endif | ||
172 | if (!l2c310_base) | ||
173 | return -1; | ||
174 | |||
175 | for (i = 0; i < L2C310_COUNTERS_NUM; i++) { | ||
176 | l2c310_counters[i].enabled = 0; | ||
177 | l2c310_counters[i].key = gator_events_get_key(); | ||
178 | } | ||
179 | |||
180 | return gator_events_install(&gator_events_l2c310_interface); | ||
181 | } | ||
182 | gator_events_init(gator_events_l2c310_init); | ||
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c index f1595bd..a1a2031 100644 --- a/driver/gator_events_meminfo.c +++ b/driver/gator_events_meminfo.c | |||
@@ -19,7 +19,7 @@ | |||
19 | static ulong meminfo_global_enabled; | 19 | static ulong meminfo_global_enabled; |
20 | static ulong meminfo_enabled[MEMINFO_TOTAL]; | 20 | static ulong meminfo_enabled[MEMINFO_TOTAL]; |
21 | static ulong meminfo_key[MEMINFO_TOTAL]; | 21 | static ulong meminfo_key[MEMINFO_TOTAL]; |
22 | static int meminfo_buffer[MEMINFO_TOTAL * 2]; | 22 | static unsigned long long meminfo_buffer[MEMINFO_TOTAL * 2]; |
23 | static int meminfo_length = 0; | 23 | static int meminfo_length = 0; |
24 | static unsigned int mem_event = 0; | 24 | static unsigned int mem_event = 0; |
25 | static bool new_data_avail; | 25 | static bool new_data_avail; |
@@ -120,7 +120,8 @@ static void gator_events_meminfo_stop(void) | |||
120 | static void wq_sched_handler(struct work_struct *wsptr) | 120 | static void wq_sched_handler(struct work_struct *wsptr) |
121 | { | 121 | { |
122 | struct sysinfo info; | 122 | struct sysinfo info; |
123 | int i, len, value; | 123 | int i, len; |
124 | unsigned long long value; | ||
124 | 125 | ||
125 | meminfo_length = len = 0; | 126 | meminfo_length = len = 0; |
126 | 127 | ||
@@ -141,7 +142,7 @@ static void wq_sched_handler(struct work_struct *wsptr) | |||
141 | value = 0; | 142 | value = 0; |
142 | break; | 143 | break; |
143 | } | 144 | } |
144 | meminfo_buffer[len++] = meminfo_key[i]; | 145 | meminfo_buffer[len++] = (unsigned long long)meminfo_key[i]; |
145 | meminfo_buffer[len++] = value; | 146 | meminfo_buffer[len++] = value; |
146 | } | 147 | } |
147 | } | 148 | } |
@@ -150,7 +151,7 @@ static void wq_sched_handler(struct work_struct *wsptr) | |||
150 | new_data_avail = true; | 151 | new_data_avail = true; |
151 | } | 152 | } |
152 | 153 | ||
153 | static int gator_events_meminfo_read(int **buffer) | 154 | static int gator_events_meminfo_read(long long **buffer) |
154 | { | 155 | { |
155 | static unsigned int last_mem_event = 0; | 156 | static unsigned int last_mem_event = 0; |
156 | 157 | ||
@@ -177,7 +178,7 @@ static struct gator_interface gator_events_meminfo_interface = { | |||
177 | .create_files = gator_events_meminfo_create_files, | 178 | .create_files = gator_events_meminfo_create_files, |
178 | .start = gator_events_meminfo_start, | 179 | .start = gator_events_meminfo_start, |
179 | .stop = gator_events_meminfo_stop, | 180 | .stop = gator_events_meminfo_stop, |
180 | .read = gator_events_meminfo_read, | 181 | .read64 = gator_events_meminfo_read, |
181 | }; | 182 | }; |
182 | 183 | ||
183 | int gator_events_meminfo_init(void) | 184 | int gator_events_meminfo_init(void) |
diff --git a/driver/gator_events_pl310.c b/driver/gator_events_pl310.c deleted file mode 100644 index 0ef0cf3..0000000 --- a/driver/gator_events_pl310.c +++ /dev/null | |||
@@ -1,176 +0,0 @@ | |||
1 | /** | ||
2 | * PL310 (L2 Cache Controller) event counters for gator | ||
3 | * | ||
4 | * Copyright (C) ARM Limited 2010-2011. All rights reserved. | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify | ||
7 | * it under the terms of the GNU General Public License version 2 as | ||
8 | * published by the Free Software Foundation. | ||
9 | */ | ||
10 | |||
11 | #include <linux/init.h> | ||
12 | #include <linux/io.h> | ||
13 | #include <asm/hardware/cache-l2x0.h> | ||
14 | |||
15 | #include "gator.h" | ||
16 | |||
17 | #define PL310_COUNTERS_NUM 2 | ||
18 | |||
19 | static struct { | ||
20 | unsigned long enabled; | ||
21 | unsigned long event; | ||
22 | unsigned long key; | ||
23 | } pl310_counters[PL310_COUNTERS_NUM]; | ||
24 | |||
25 | static int pl310_buffer[PL310_COUNTERS_NUM * 2]; | ||
26 | |||
27 | static void __iomem *pl310_base; | ||
28 | |||
29 | |||
30 | |||
31 | static void gator_events_pl310_reset_counters(void) | ||
32 | { | ||
33 | u32 val = readl(pl310_base + L2X0_EVENT_CNT_CTRL); | ||
34 | |||
35 | val |= ((1 << PL310_COUNTERS_NUM) - 1) << 1; | ||
36 | |||
37 | writel(val, pl310_base + L2X0_EVENT_CNT_CTRL); | ||
38 | } | ||
39 | |||
40 | |||
41 | static int gator_events_pl310_create_files(struct super_block *sb, | ||
42 | struct dentry *root) | ||
43 | { | ||
44 | int i; | ||
45 | |||
46 | for (i = 0; i < PL310_COUNTERS_NUM; i++) { | ||
47 | char buf[16]; | ||
48 | struct dentry *dir; | ||
49 | |||
50 | snprintf(buf, sizeof(buf), "PL310_cnt%d", i); | ||
51 | dir = gatorfs_mkdir(sb, root, buf); | ||
52 | if (WARN_ON(!dir)) | ||
53 | return -1; | ||
54 | gatorfs_create_ulong(sb, dir, "enabled", | ||
55 | &pl310_counters[i].enabled); | ||
56 | gatorfs_create_ulong(sb, dir, "event", | ||
57 | &pl310_counters[i].event); | ||
58 | gatorfs_create_ro_ulong(sb, dir, "key", | ||
59 | &pl310_counters[i].key); | ||
60 | } | ||
61 | |||
62 | return 0; | ||
63 | } | ||
64 | |||
65 | static int gator_events_pl310_start(void) | ||
66 | { | ||
67 | static const unsigned long l2x0_event_cntx_cfg[PL310_COUNTERS_NUM] = { | ||
68 | L2X0_EVENT_CNT0_CFG, | ||
69 | L2X0_EVENT_CNT1_CFG, | ||
70 | }; | ||
71 | int i; | ||
72 | |||
73 | /* Counter event sources */ | ||
74 | for (i = 0; i < PL310_COUNTERS_NUM; i++) | ||
75 | writel((pl310_counters[i].event & 0xf) << 2, | ||
76 | pl310_base + l2x0_event_cntx_cfg[i]); | ||
77 | |||
78 | gator_events_pl310_reset_counters(); | ||
79 | |||
80 | /* Event counter enable */ | ||
81 | writel(1, pl310_base + L2X0_EVENT_CNT_CTRL); | ||
82 | |||
83 | return 0; | ||
84 | } | ||
85 | |||
86 | static void gator_events_pl310_stop(void) | ||
87 | { | ||
88 | /* Event counter disable */ | ||
89 | writel(0, pl310_base + L2X0_EVENT_CNT_CTRL); | ||
90 | } | ||
91 | |||
92 | static int gator_events_pl310_read(int **buffer) | ||
93 | { | ||
94 | static const unsigned long l2x0_event_cntx_val[PL310_COUNTERS_NUM] = { | ||
95 | L2X0_EVENT_CNT0_VAL, | ||
96 | L2X0_EVENT_CNT1_VAL, | ||
97 | }; | ||
98 | int i; | ||
99 | int len = 0; | ||
100 | |||
101 | if (smp_processor_id()) | ||
102 | return 0; | ||
103 | |||
104 | for (i = 0; i < PL310_COUNTERS_NUM; i++) { | ||
105 | if (pl310_counters[i].enabled) { | ||
106 | pl310_buffer[len++] = pl310_counters[i].key; | ||
107 | pl310_buffer[len++] = readl(pl310_base + | ||
108 | l2x0_event_cntx_val[i]); | ||
109 | } | ||
110 | } | ||
111 | |||
112 | /* PL310 counters are saturating, not wrapping in case of overflow */ | ||
113 | gator_events_pl310_reset_counters(); | ||
114 | |||
115 | if (buffer) | ||
116 | *buffer = pl310_buffer; | ||
117 | |||
118 | return len; | ||
119 | } | ||
120 | |||
121 | static struct gator_interface gator_events_pl310_interface = { | ||
122 | .create_files = gator_events_pl310_create_files, | ||
123 | .start = gator_events_pl310_start, | ||
124 | .stop = gator_events_pl310_stop, | ||
125 | .read = gator_events_pl310_read, | ||
126 | }; | ||
127 | |||
128 | static void __maybe_unused gator_events_pl310_probe(unsigned long phys) | ||
129 | { | ||
130 | if (pl310_base) | ||
131 | return; | ||
132 | |||
133 | pl310_base = ioremap(phys, SZ_4K); | ||
134 | if (pl310_base) { | ||
135 | u32 cache_id = readl(pl310_base + L2X0_CACHE_ID); | ||
136 | |||
137 | if ((cache_id & 0xff0003c0) != 0x410000c0) { | ||
138 | iounmap(pl310_base); | ||
139 | pl310_base = NULL; | ||
140 | } | ||
141 | } | ||
142 | } | ||
143 | |||
144 | int gator_events_pl310_init(void) | ||
145 | { | ||
146 | int i; | ||
147 | |||
148 | #if defined(CONFIG_ARCH_EXYNOS4) | ||
149 | gator_events_pl310_probe(0xfe600000); | ||
150 | #endif | ||
151 | #if defined(CONFIG_ARCH_OMAP4) | ||
152 | gator_events_pl310_probe(0x48242000); | ||
153 | #endif | ||
154 | #if defined(CONFIG_ARCH_TEGRA) | ||
155 | gator_events_pl310_probe(0x50043000); | ||
156 | #endif | ||
157 | #if defined(CONFIG_ARCH_U8500) | ||
158 | gator_events_pl310_probe(0xa0412000); | ||
159 | #endif | ||
160 | #if defined(CONFIG_ARCH_VEXPRESS) && !defined(CONFIG_ARCH_VEXPRESS_CA15X4) | ||
161 | // A9x4 core tile (HBI-0191) | ||
162 | gator_events_pl310_probe(0x1e00a000); | ||
163 | // New memory map tiles | ||
164 | gator_events_pl310_probe(0x2c0f0000); | ||
165 | #endif | ||
166 | if (!pl310_base) | ||
167 | return -1; | ||
168 | |||
169 | for (i = 0; i < PL310_COUNTERS_NUM; i++) { | ||
170 | pl310_counters[i].enabled = 0; | ||
171 | pl310_counters[i].key = gator_events_get_key(); | ||
172 | } | ||
173 | |||
174 | return gator_events_install(&gator_events_pl310_interface); | ||
175 | } | ||
176 | gator_events_init(gator_events_pl310_init); | ||
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c index d831a50..f51e292 100644 --- a/driver/gator_events_scorpion.c +++ b/driver/gator_events_scorpion.c | |||
@@ -12,7 +12,7 @@ | |||
12 | #define SCORPIONMP 0x2d | 12 | #define SCORPIONMP 0x2d |
13 | 13 | ||
14 | static const char *pmnc_name; | 14 | static const char *pmnc_name; |
15 | static int pmnc_count; | 15 | static int pmnc_counters; |
16 | 16 | ||
17 | // Per-CPU PMNC: config reg | 17 | // Per-CPU PMNC: config reg |
18 | #define PMNC_E (1 << 0) /* Enable all counters */ | 18 | #define PMNC_E (1 << 0) /* Enable all counters */ |
@@ -32,6 +32,7 @@ static int pmnc_count; | |||
32 | 32 | ||
33 | static unsigned long pmnc_enabled[CNTMAX]; | 33 | static unsigned long pmnc_enabled[CNTMAX]; |
34 | static unsigned long pmnc_event[CNTMAX]; | 34 | static unsigned long pmnc_event[CNTMAX]; |
35 | static unsigned long pmnc_count[CNTMAX]; | ||
35 | static unsigned long pmnc_key[CNTMAX]; | 36 | static unsigned long pmnc_key[CNTMAX]; |
36 | 37 | ||
37 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); | 38 | static DEFINE_PER_CPU(int[CNTMAX], perfPrev); |
@@ -209,6 +210,10 @@ static const struct scorp_evt sc_evt[] = { | |||
209 | {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d}, | 210 | {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d}, |
210 | {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f}, | 211 | {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f}, |
211 | 212 | ||
213 | #ifdef CONFIG_ARCH_MSM_SCORPIONMP | ||
214 | {SCORPIONMP_NUM_BARRIERS, 0x80000e00, 3, 0x59}, | ||
215 | {SCORPIONMP_BARRIER_CYCLES, 0x800e0000, 3, 0x5a}, | ||
216 | #else | ||
212 | {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58}, | 217 | {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58}, |
213 | {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59}, | 218 | {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59}, |
214 | {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a}, | 219 | {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a}, |
@@ -228,6 +233,7 @@ static const struct scorp_evt sc_evt[] = { | |||
228 | {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59}, | 233 | {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59}, |
229 | {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a}, | 234 | {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a}, |
230 | {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b}, | 235 | {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b}, |
236 | #endif | ||
231 | }; | 237 | }; |
232 | 238 | ||
233 | static inline void scorpion_pmnc_write(u32 val) | 239 | static inline void scorpion_pmnc_write(u32 val) |
@@ -498,7 +504,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den | |||
498 | struct dentry *dir; | 504 | struct dentry *dir; |
499 | int i; | 505 | int i; |
500 | 506 | ||
501 | for (i = 0; i < pmnc_count; i++) { | 507 | for (i = 0; i < pmnc_counters; i++) { |
502 | char buf[40]; | 508 | char buf[40]; |
503 | if (i == 0) { | 509 | if (i == 0) { |
504 | snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); | 510 | snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); |
@@ -510,10 +516,11 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den | |||
510 | return -1; | 516 | return -1; |
511 | } | 517 | } |
512 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); | 518 | gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); |
519 | gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); | ||
520 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
513 | if (i > 0) { | 521 | if (i > 0) { |
514 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); | 522 | gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); |
515 | } | 523 | } |
516 | gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); | ||
517 | } | 524 | } |
518 | 525 | ||
519 | return 0; | 526 | return 0; |
@@ -565,6 +572,20 @@ static void gator_events_scorpion_offline(void) | |||
565 | // investigate: need to do the clearpmu() here on each counter? | 572 | // investigate: need to do the clearpmu() here on each counter? |
566 | } | 573 | } |
567 | 574 | ||
575 | static int gator_events_scorpion_start(void) | ||
576 | { | ||
577 | int cnt; | ||
578 | |||
579 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | ||
580 | if (pmnc_count[cnt] > 0) { | ||
581 | pr_err("gator: event based sampling not supported on Scorpion cores\n"); | ||
582 | return -1; | ||
583 | } | ||
584 | } | ||
585 | |||
586 | return 0; | ||
587 | } | ||
588 | |||
568 | static void gator_events_scorpion_stop(void) | 589 | static void gator_events_scorpion_stop(void) |
569 | { | 590 | { |
570 | unsigned int cnt; | 591 | unsigned int cnt; |
@@ -572,6 +593,7 @@ static void gator_events_scorpion_stop(void) | |||
572 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | 593 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { |
573 | pmnc_enabled[cnt] = 0; | 594 | pmnc_enabled[cnt] = 0; |
574 | pmnc_event[cnt] = 0; | 595 | pmnc_event[cnt] = 0; |
596 | pmnc_count[cnt] = 0; | ||
575 | } | 597 | } |
576 | } | 598 | } |
577 | 599 | ||
@@ -580,10 +602,10 @@ static int gator_events_scorpion_read(int **buffer) | |||
580 | int cnt, len = 0; | 602 | int cnt, len = 0; |
581 | int cpu = smp_processor_id(); | 603 | int cpu = smp_processor_id(); |
582 | 604 | ||
583 | if (!pmnc_count) | 605 | if (!pmnc_counters) |
584 | return 0; | 606 | return 0; |
585 | 607 | ||
586 | for (cnt = 0; cnt < pmnc_count; cnt++) { | 608 | for (cnt = 0; cnt < pmnc_counters; cnt++) { |
587 | if (pmnc_enabled[cnt]) { | 609 | if (pmnc_enabled[cnt]) { |
588 | int value; | 610 | int value; |
589 | if (cnt == CCNT) { | 611 | if (cnt == CCNT) { |
@@ -611,6 +633,7 @@ static int gator_events_scorpion_read(int **buffer) | |||
611 | 633 | ||
612 | static struct gator_interface gator_events_scorpion_interface = { | 634 | static struct gator_interface gator_events_scorpion_interface = { |
613 | .create_files = gator_events_scorpion_create_files, | 635 | .create_files = gator_events_scorpion_create_files, |
636 | .start = gator_events_scorpion_start, | ||
614 | .stop = gator_events_scorpion_stop, | 637 | .stop = gator_events_scorpion_stop, |
615 | .online = gator_events_scorpion_online, | 638 | .online = gator_events_scorpion_online, |
616 | .offline = gator_events_scorpion_offline, | 639 | .offline = gator_events_scorpion_offline, |
@@ -636,21 +659,22 @@ int gator_events_scorpion_init(void) | |||
636 | switch (gator_cpuid()) { | 659 | switch (gator_cpuid()) { |
637 | case SCORPION: | 660 | case SCORPION: |
638 | pmnc_name = "Scorpion"; | 661 | pmnc_name = "Scorpion"; |
639 | pmnc_count = 4; | 662 | pmnc_counters = 4; |
640 | break; | 663 | break; |
641 | case SCORPIONMP: | 664 | case SCORPIONMP: |
642 | pmnc_name = "ScorpionMP"; | 665 | pmnc_name = "ScorpionMP"; |
643 | pmnc_count = 4; | 666 | pmnc_counters = 4; |
644 | break; | 667 | break; |
645 | default: | 668 | default: |
646 | return -1; | 669 | return -1; |
647 | } | 670 | } |
648 | 671 | ||
649 | pmnc_count++; // CNT[n] + CCNT | 672 | pmnc_counters++; // CNT[n] + CCNT |
650 | 673 | ||
651 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { | 674 | for (cnt = CCNT; cnt < CNTMAX; cnt++) { |
652 | pmnc_enabled[cnt] = 0; | 675 | pmnc_enabled[cnt] = 0; |
653 | pmnc_event[cnt] = 0; | 676 | pmnc_event[cnt] = 0; |
677 | pmnc_count[cnt] = 0; | ||
654 | pmnc_key[cnt] = gator_events_get_key(); | 678 | pmnc_key[cnt] = gator_events_get_key(); |
655 | } | 679 | } |
656 | 680 | ||
diff --git a/driver/gator_main.c b/driver/gator_main.c index 340756e..91744ad 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c | |||
@@ -7,7 +7,7 @@ | |||
7 | * | 7 | * |
8 | */ | 8 | */ |
9 | 9 | ||
10 | static unsigned long gator_protocol_version = 5; | 10 | static unsigned long gator_protocol_version = 6; |
11 | 11 | ||
12 | #include <linux/slab.h> | 12 | #include <linux/slab.h> |
13 | #include <linux/cpu.h> | 13 | #include <linux/cpu.h> |
@@ -36,11 +36,13 @@ static unsigned long gator_protocol_version = 5; | |||
36 | #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined | 36 | #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined |
37 | #endif | 37 | #endif |
38 | 38 | ||
39 | #if defined (__arm__) | ||
39 | #ifdef CONFIG_SMP | 40 | #ifdef CONFIG_SMP |
40 | #ifndef CONFIG_LOCAL_TIMERS | 41 | #ifndef CONFIG_LOCAL_TIMERS |
41 | #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems | 42 | #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems |
42 | #endif | 43 | #endif |
43 | #endif | 44 | #endif |
45 | #endif | ||
44 | 46 | ||
45 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) | 47 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) |
46 | #error kernels prior to 2.6.32 are not supported | 48 | #error kernels prior to 2.6.32 are not supported |
@@ -49,22 +51,24 @@ static unsigned long gator_protocol_version = 5; | |||
49 | /****************************************************************************** | 51 | /****************************************************************************** |
50 | * DEFINES | 52 | * DEFINES |
51 | ******************************************************************************/ | 53 | ******************************************************************************/ |
52 | #define BUFFER_SIZE_DEFAULT (256*1024) | 54 | #define TIMER_BUFFER_SIZE_DEFAULT (256*1024) |
53 | #define SYNC_FREQ_DEFAULT 1000 | 55 | #define EVENT_BUFFER_SIZE_DEFAULT (128*1024) |
54 | 56 | ||
55 | #define NO_COOKIE 0UL | 57 | #define NO_COOKIE 0UL |
56 | #define INVALID_COOKIE ~0UL | 58 | #define INVALID_COOKIE ~0UL |
57 | 59 | ||
58 | #define PROTOCOL_FRAME ~0 | 60 | #define FRAME_HRTIMER 1 |
59 | #define PROTOCOL_START_TICK 1 | 61 | #define FRAME_EVENT 2 |
60 | #define PROTOCOL_END_TICK 3 | 62 | #define FRAME_ANNOTATE 3 |
61 | #define PROTOCOL_START_BACKTRACE 5 | 63 | |
62 | #define PROTOCOL_END_BACKTRACE 7 | 64 | #define MESSAGE_COOKIE 1 |
63 | #define PROTOCOL_COOKIE 9 | 65 | #define MESSAGE_COUNTERS 3 |
64 | #define PROTOCOL_SCHEDULER_TRACE 11 | 66 | #define MESSAGE_START_BACKTRACE 5 |
65 | #define PROTOCOL_COUNTERS 13 | 67 | #define MESSAGE_END_BACKTRACE 7 |
66 | #define PROTOCOL_ANNOTATE 15 | 68 | #define MESSAGE_SCHEDULER_TRACE 9 |
67 | #define PROTOCOL_CPU_SYNC 17 | 69 | #define MESSAGE_PID_NAME 11 |
70 | |||
71 | #define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU) | ||
68 | 72 | ||
69 | #if defined(__arm__) | 73 | #if defined(__arm__) |
70 | #define PC_REG regs->ARM_pc | 74 | #define PC_REG regs->ARM_pc |
@@ -72,42 +76,50 @@ static unsigned long gator_protocol_version = 5; | |||
72 | #define PC_REG regs->ip | 76 | #define PC_REG regs->ip |
73 | #endif | 77 | #endif |
74 | 78 | ||
79 | enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS}; | ||
80 | |||
75 | /****************************************************************************** | 81 | /****************************************************************************** |
76 | * PER CPU | 82 | * Globals |
77 | ******************************************************************************/ | 83 | ******************************************************************************/ |
78 | static unsigned long gator_cpu_cores; | 84 | static unsigned long gator_cpu_cores; |
79 | static unsigned long gator_buffer_size; | 85 | static unsigned long userspace_buffer_size; |
80 | static unsigned long gator_backtrace_depth; | 86 | static unsigned long gator_backtrace_depth; |
81 | 87 | ||
82 | static unsigned long gator_started; | 88 | static unsigned long gator_started; |
83 | static unsigned long gator_buffer_opened; | 89 | static unsigned long gator_buffer_opened; |
84 | static unsigned long gator_timer_count; | 90 | static unsigned long gator_timer_count; |
85 | static unsigned long gator_streaming; | 91 | static unsigned long gator_streaming; |
86 | static int gator_master_tick; | ||
87 | static DEFINE_MUTEX(start_mutex); | 92 | static DEFINE_MUTEX(start_mutex); |
88 | static DEFINE_MUTEX(gator_buffer_mutex); | 93 | static DEFINE_MUTEX(gator_buffer_mutex); |
89 | 94 | ||
90 | unsigned long gator_net_traffic; | 95 | unsigned long gator_net_traffic; |
96 | bool event_based_sampling; | ||
91 | 97 | ||
92 | #define COMMIT_SIZE 128 | 98 | #define COMMIT_SIZE 128 |
93 | #define COMMIT_MASK (COMMIT_SIZE-1) | 99 | #define COMMIT_MASK (COMMIT_SIZE-1) |
94 | static DEFINE_SPINLOCK(gator_commit_lock); | 100 | static DEFINE_SPINLOCK(timer_commit_lock); |
95 | static int *gator_commit; | 101 | static int *gator_commit[NUM_GATOR_BUFS]; |
96 | static int gator_commit_read; | 102 | static int gator_commit_read[NUM_GATOR_BUFS]; |
97 | static int gator_commit_write; | 103 | static int gator_commit_write[NUM_GATOR_BUFS]; |
98 | 104 | ||
99 | static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); | 105 | static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); |
100 | static DEFINE_PER_CPU(int, gator_cpu_sync); | ||
101 | static DEFINE_PER_CPU(int, gator_cpu_tick); | ||
102 | static DEFINE_PER_CPU(int, gator_first_time); | 106 | static DEFINE_PER_CPU(int, gator_first_time); |
103 | 107 | ||
108 | #if LINUX_PMU_SUPPORT | ||
109 | static void event_buffer_check(int cpu); | ||
110 | static DEFINE_SPINLOCK(event_commit_lock); | ||
111 | #endif | ||
112 | |||
104 | /****************************************************************************** | 113 | /****************************************************************************** |
105 | * Prototypes | 114 | * Prototypes |
106 | ******************************************************************************/ | 115 | ******************************************************************************/ |
107 | static void gator_buffer_write_packed_int(int cpu, unsigned int x); | 116 | static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); |
108 | static void gator_buffer_write_string(int cpu, char *x); | 117 | static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); |
118 | static void gator_buffer_write_string(int cpu, int buftype, char *x); | ||
109 | static int gator_write_packed_int(char *buffer, unsigned int x); | 119 | static int gator_write_packed_int(char *buffer, unsigned int x); |
110 | static void gator_add_trace(int cpu, unsigned int address); | 120 | static int gator_write_packed_int64(char *buffer, unsigned long long x); |
121 | static void gator_add_trace(int cpu, int buftype, unsigned int address); | ||
122 | static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); | ||
111 | static uint64_t gator_get_time(void); | 123 | static uint64_t gator_get_time(void); |
112 | 124 | ||
113 | /****************************************************************************** | 125 | /****************************************************************************** |
@@ -118,6 +130,7 @@ static uint64_t gator_get_time(void); | |||
118 | #include "gator_backtrace.c" | 130 | #include "gator_backtrace.c" |
119 | #include "gator_annotate.c" | 131 | #include "gator_annotate.c" |
120 | #include "gator_fs.c" | 132 | #include "gator_fs.c" |
133 | #include "gator_ebs.c" | ||
121 | 134 | ||
122 | /****************************************************************************** | 135 | /****************************************************************************** |
123 | * Misc | 136 | * Misc |
@@ -134,204 +147,137 @@ u32 gator_cpuid(void) | |||
134 | /****************************************************************************** | 147 | /****************************************************************************** |
135 | * Commit interface | 148 | * Commit interface |
136 | ******************************************************************************/ | 149 | ******************************************************************************/ |
137 | static int buffer_commit_ready(void) | 150 | static int buffer_commit_ready(int buftype) |
138 | { | 151 | { |
139 | return (gator_commit_read != gator_commit_write); | 152 | return gator_commit_read[buftype] != gator_commit_write[buftype]; |
140 | } | 153 | } |
141 | 154 | ||
142 | static void buffer_commit_read(int *cpu, int *readval, int *writeval) | 155 | static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval) |
143 | { | 156 | { |
144 | int read = gator_commit_read; | 157 | int read = gator_commit_read[buftype]; |
145 | *cpu = gator_commit[read+0]; | 158 | *cpu = gator_commit[buftype][read+0]; |
146 | *readval = gator_commit[read+1]; | 159 | *readval = gator_commit[buftype][read+1]; |
147 | *writeval = gator_commit[read+2]; | 160 | *writeval = gator_commit[buftype][read+2]; |
148 | gator_commit_read = (read + 4) & COMMIT_MASK; | 161 | gator_commit_read[buftype] = (read + 4) & COMMIT_MASK; |
149 | } | 162 | } |
150 | 163 | ||
151 | static void buffer_commit_write(int cpu, int readval, int writeval) { | 164 | static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) { |
152 | int write = gator_commit_write; | 165 | int write = gator_commit_write[buftype]; |
153 | gator_commit[write+0] = cpu; | 166 | gator_commit[buftype][write+0] = cpu; |
154 | gator_commit[write+1] = readval; | 167 | gator_commit[buftype][write+1] = readval; |
155 | gator_commit[write+2] = writeval; | 168 | gator_commit[buftype][write+2] = writeval; |
156 | gator_commit_write = (write + 4) & COMMIT_MASK; | 169 | gator_commit_write[buftype] = (write + 4) & COMMIT_MASK; |
157 | } | 170 | } |
158 | 171 | ||
159 | /****************************************************************************** | 172 | /****************************************************************************** |
160 | * Buffer management | 173 | * Buffer management |
161 | ******************************************************************************/ | 174 | ******************************************************************************/ |
162 | static uint32_t use_buffer_size; | 175 | static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; |
163 | static uint32_t use_buffer_mask; | 176 | static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; |
164 | static DEFINE_PER_CPU(int, use_buffer_seq); | 177 | static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); |
165 | static DEFINE_PER_CPU(int, use_buffer_read); | 178 | static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); |
166 | static DEFINE_PER_CPU(int, use_buffer_write); | 179 | static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); |
167 | static DEFINE_PER_CPU(char *, use_buffer); | 180 | #include "gator_pack.c" |
168 | |||
169 | static void gator_buffer_write_packed_int(int cpu, unsigned int x) | ||
170 | { | ||
171 | uint32_t write = per_cpu(use_buffer_write, cpu); | ||
172 | uint32_t mask = use_buffer_mask; | ||
173 | char *buffer = per_cpu(use_buffer, cpu); | ||
174 | int write0 = (write + 0) & mask; | ||
175 | int write1 = (write + 1) & mask; | ||
176 | int write2 = (write + 2) & mask; | ||
177 | int write3 = (write + 3) & mask; | ||
178 | int write4 = (write + 4) & mask; | ||
179 | int write5 = (write + 5) & mask; | ||
180 | |||
181 | if ((x & 0xffffff80) == 0) { | ||
182 | buffer[write0] = x & 0x7f; | ||
183 | per_cpu(use_buffer_write, cpu) = write1; | ||
184 | } else if ((x & 0xffffc000) == 0) { | ||
185 | buffer[write0] = x | 0x80; | ||
186 | buffer[write1] = (x>>7) & 0x7f; | ||
187 | per_cpu(use_buffer_write, cpu) = write2; | ||
188 | } else if ((x & 0xffe00000) == 0) { | ||
189 | buffer[write0] = x | 0x80; | ||
190 | buffer[write1] = (x>>7) | 0x80; | ||
191 | buffer[write2] = (x>>14) & 0x7f; | ||
192 | per_cpu(use_buffer_write, cpu) = write3; | ||
193 | } else if ((x & 0xf0000000) == 0) { | ||
194 | buffer[write0] = x | 0x80; | ||
195 | buffer[write1] = (x>>7) | 0x80; | ||
196 | buffer[write2] = (x>>14) | 0x80; | ||
197 | buffer[write3] = (x>>21) & 0x7f; | ||
198 | per_cpu(use_buffer_write, cpu) = write4; | ||
199 | } else { | ||
200 | buffer[write0] = x | 0x80; | ||
201 | buffer[write1] = (x>>7) | 0x80; | ||
202 | buffer[write2] = (x>>14) | 0x80; | ||
203 | buffer[write3] = (x>>21) | 0x80; | ||
204 | buffer[write4] = (x>>28) & 0x0f; | ||
205 | per_cpu(use_buffer_write, cpu) = write5; | ||
206 | } | ||
207 | } | ||
208 | 181 | ||
209 | static int gator_write_packed_int(char *buffer, unsigned int x) | 182 | static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) |
210 | { | 183 | { |
211 | if ((x & 0xffffff80) == 0) { | ||
212 | buffer[0] = x & 0x7f; | ||
213 | return 1; | ||
214 | } else if ((x & 0xffffc000) == 0) { | ||
215 | buffer[0] = x | 0x80; | ||
216 | buffer[1] = (x>>7) & 0x7f; | ||
217 | return 2; | ||
218 | } else if ((x & 0xffe00000) == 0) { | ||
219 | buffer[0] = x | 0x80; | ||
220 | buffer[1] = (x>>7) | 0x80; | ||
221 | buffer[2] = (x>>14) & 0x7f; | ||
222 | return 3; | ||
223 | } else if ((x & 0xf0000000) == 0) { | ||
224 | buffer[0] = x | 0x80; | ||
225 | buffer[1] = (x>>7) | 0x80; | ||
226 | buffer[2] = (x>>14) | 0x80; | ||
227 | buffer[3] = (x>>21) & 0x7f; | ||
228 | return 4; | ||
229 | } else { | ||
230 | buffer[0] = x | 0x80; | ||
231 | buffer[1] = (x>>7) | 0x80; | ||
232 | buffer[2] = (x>>14) | 0x80; | ||
233 | buffer[3] = (x>>21) | 0x80; | ||
234 | buffer[4] = (x>>28) & 0x0f; | ||
235 | return 5; | ||
236 | } | ||
237 | } | ||
238 | |||
239 | static void gator_buffer_write_bytes(int cpu, char *x, int len) | ||
240 | { | ||
241 | uint32_t write = per_cpu(use_buffer_write, cpu); | ||
242 | uint32_t mask = use_buffer_mask; | ||
243 | char *buffer = per_cpu(use_buffer, cpu); | ||
244 | int i; | 184 | int i; |
185 | u32 write = per_cpu(gator_buffer_write, cpu)[buftype]; | ||
186 | u32 mask = gator_buffer_mask[buftype]; | ||
187 | char* buffer = per_cpu(gator_buffer, cpu)[buftype]; | ||
245 | 188 | ||
246 | for (i = 0; i < len; i++) { | 189 | for (i = 0; i < len; i++) { |
247 | buffer[write] = x[i]; | 190 | buffer[write] = x[i]; |
248 | write = (write + 1) & mask; | 191 | write = (write + 1) & mask; |
249 | } | 192 | } |
250 | 193 | ||
251 | per_cpu(use_buffer_write, cpu) = write; | 194 | per_cpu(gator_buffer_write, cpu)[buftype] = write; |
252 | } | 195 | } |
253 | 196 | ||
254 | static void gator_buffer_write_string(int cpu, char *x) | 197 | static void gator_buffer_write_string(int cpu, int buftype, char *x) |
255 | { | 198 | { |
256 | int len = strlen(x); | 199 | int len = strlen(x); |
257 | gator_buffer_write_packed_int(cpu, len); | 200 | gator_buffer_write_packed_int(cpu, buftype, len); |
258 | gator_buffer_write_bytes(cpu, x, len); | 201 | gator_buffer_write_bytes(cpu, buftype, x, len); |
259 | } | 202 | } |
260 | 203 | ||
261 | static void gator_buffer_header(int cpu) | 204 | static void gator_buffer_header(int cpu, int buftype) |
262 | { | 205 | { |
263 | gator_buffer_write_packed_int(cpu, PROTOCOL_FRAME); | 206 | int frame; |
264 | gator_buffer_write_packed_int(cpu, cpu); | 207 | |
265 | gator_buffer_write_packed_int(cpu, per_cpu(use_buffer_seq, cpu)); | 208 | if (buftype == TIMER_BUF) |
266 | per_cpu(use_buffer_seq, cpu)++; | 209 | frame = FRAME_HRTIMER; |
210 | else if (buftype == EVENT_BUF) | ||
211 | frame = FRAME_EVENT; | ||
212 | else | ||
213 | frame = -1; | ||
214 | |||
215 | gator_buffer_write_packed_int(cpu, buftype, frame); | ||
216 | gator_buffer_write_packed_int(cpu, buftype, cpu); | ||
267 | } | 217 | } |
268 | 218 | ||
269 | static void gator_buffer_commit(int cpu) | 219 | static void gator_buffer_commit(int cpu, int buftype) |
270 | { | 220 | { |
271 | buffer_commit_write(cpu, per_cpu(use_buffer_read, cpu), per_cpu(use_buffer_write, cpu)); | 221 | buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]); |
272 | per_cpu(use_buffer_read, cpu) = per_cpu(use_buffer_write, cpu); | 222 | per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; |
273 | gator_buffer_header(cpu); | 223 | gator_buffer_header(cpu, buftype); |
274 | wake_up(&gator_buffer_wait); | 224 | wake_up(&gator_buffer_wait); |
275 | } | 225 | } |
276 | 226 | ||
277 | static void gator_buffer_check(int cpu, int tick) | 227 | static void timer_buffer_check(int cpu) |
278 | { | 228 | { |
279 | if (!(tick % gator_timer_count)) { | 229 | int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF]; |
280 | int c, sync; | 230 | if (available < 0) { |
281 | spin_lock(&gator_commit_lock); | 231 | available += gator_buffer_size[TIMER_BUF]; |
282 | // synchronize, if all online cpus have the same tick waypoint | 232 | } |
283 | sync = per_cpu(gator_cpu_sync, cpu) = per_cpu(gator_cpu_tick, cpu); | 233 | if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) { |
284 | for_each_online_cpu(c) { | 234 | spin_lock(&timer_commit_lock); |
285 | if (sync != per_cpu(gator_cpu_sync, c)) { | 235 | gator_buffer_commit(cpu, TIMER_BUF); |
286 | sync = 0; | 236 | spin_unlock(&timer_commit_lock); |
287 | break; | 237 | } |
288 | } | 238 | } |
289 | } | 239 | |
290 | if (sync) { | 240 | #if LINUX_PMU_SUPPORT |
291 | gator_buffer_write_packed_int(cpu, PROTOCOL_CPU_SYNC); | 241 | static void event_buffer_check(int cpu) |
292 | } | 242 | { |
293 | gator_buffer_commit(cpu); | 243 | int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF]; |
294 | spin_unlock(&gator_commit_lock); | 244 | if (available < 0) { |
295 | } else { | 245 | available += gator_buffer_size[EVENT_BUF]; |
296 | int available = per_cpu(use_buffer_write, cpu) - per_cpu(use_buffer_read, cpu); | 246 | } |
297 | if (available < 0) { | 247 | if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) { |
298 | available += use_buffer_size; | 248 | spin_lock(&event_commit_lock); |
299 | } | 249 | gator_buffer_commit(cpu, EVENT_BUF); |
300 | if (available >= ((use_buffer_size * 3) / 4)) { | 250 | spin_unlock(&event_commit_lock); |
301 | spin_lock(&gator_commit_lock); | ||
302 | gator_buffer_commit(cpu); | ||
303 | spin_unlock(&gator_commit_lock); | ||
304 | } | ||
305 | } | 251 | } |
306 | } | 252 | } |
253 | #endif | ||
307 | 254 | ||
308 | static void gator_add_trace(int cpu, unsigned int address) | 255 | static void gator_add_trace(int cpu, int buftype, unsigned int address) |
309 | { | 256 | { |
310 | off_t offset = 0; | 257 | off_t offset = 0; |
311 | unsigned long cookie = get_address_cookie(cpu, current, address & ~1, &offset); | 258 | unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset); |
312 | 259 | ||
313 | if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) { | 260 | if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) { |
314 | offset = address; | 261 | offset = address; |
315 | } | 262 | } |
316 | 263 | ||
317 | gator_buffer_write_packed_int(cpu, offset & ~1); | 264 | gator_buffer_write_packed_int(cpu, buftype, offset & ~1); |
318 | gator_buffer_write_packed_int(cpu, cookie); | 265 | gator_buffer_write_packed_int(cpu, buftype, cookie); |
319 | } | 266 | } |
320 | 267 | ||
321 | static void gator_add_sample(int cpu, struct pt_regs * const regs) | 268 | static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) |
322 | { | 269 | { |
323 | struct module *mod; | 270 | struct module *mod; |
324 | unsigned int addr, cookie = 0; | 271 | unsigned int addr, cookie = 0; |
325 | int inKernel = regs ? !user_mode(regs) : 1; | 272 | int inKernel = regs ? !user_mode(regs) : 1; |
326 | unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE; | 273 | unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current); |
327 | 274 | ||
328 | gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE); | 275 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE); |
329 | 276 | gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); | |
330 | // TGID::PID::inKernel | 277 | gator_buffer_write_packed_int(cpu, buftype, exec_cookie); |
331 | gator_buffer_write_packed_int(cpu, exec_cookie); | 278 | gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); |
332 | gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid); | 279 | gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid); |
333 | gator_buffer_write_packed_int(cpu, (unsigned int)current->pid); | 280 | gator_buffer_write_packed_int(cpu, buftype, inKernel); |
334 | gator_buffer_write_packed_int(cpu, inKernel); | ||
335 | 281 | ||
336 | // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ) | 282 | // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ) |
337 | if (regs) { | 283 | if (regs) { |
@@ -339,36 +285,26 @@ static void gator_add_sample(int cpu, struct pt_regs * const regs) | |||
339 | addr = PC_REG; | 285 | addr = PC_REG; |
340 | mod = __module_address(addr); | 286 | mod = __module_address(addr); |
341 | if (mod) { | 287 | if (mod) { |
342 | cookie = get_cookie(cpu, current, NULL, mod); | 288 | cookie = get_cookie(cpu, buftype, current, NULL, mod, true); |
343 | addr = addr - (unsigned long)mod->module_core; | 289 | addr = addr - (unsigned long)mod->module_core; |
344 | } | 290 | } |
345 | gator_buffer_write_packed_int(cpu, addr & ~1); | 291 | gator_buffer_write_packed_int(cpu, buftype, addr & ~1); |
346 | gator_buffer_write_packed_int(cpu, cookie); | 292 | gator_buffer_write_packed_int(cpu, buftype, cookie); |
347 | } else { | 293 | } else { |
348 | // Cookie+PC | 294 | // Cookie+PC |
349 | gator_add_trace(cpu, PC_REG); | 295 | gator_add_trace(cpu, buftype, PC_REG); |
350 | 296 | ||
351 | // Backtrace | 297 | // Backtrace |
352 | if (gator_backtrace_depth) | 298 | if (gator_backtrace_depth) |
353 | arm_backtrace_eabi(cpu, regs, gator_backtrace_depth); | 299 | arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth); |
354 | } | 300 | } |
355 | } | 301 | } |
356 | 302 | ||
357 | gator_buffer_write_packed_int(cpu, PROTOCOL_END_BACKTRACE); | 303 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE); |
358 | } | ||
359 | |||
360 | static void gator_write_packet(int cpu, int type, int len, int *buffer) | ||
361 | { | ||
362 | int i; | ||
363 | gator_buffer_write_packed_int(cpu, type); | ||
364 | gator_buffer_write_packed_int(cpu, len); | ||
365 | for (i = 0; i < len; i++) { | ||
366 | gator_buffer_write_packed_int(cpu, buffer[i]); | ||
367 | } | ||
368 | } | 304 | } |
369 | 305 | ||
370 | /****************************************************************************** | 306 | /****************************************************************************** |
371 | * Interrupt Processing | 307 | * hrtimer interrupt processing |
372 | ******************************************************************************/ | 308 | ******************************************************************************/ |
373 | static LIST_HEAD(gator_events); | 309 | static LIST_HEAD(gator_events); |
374 | 310 | ||
@@ -376,7 +312,8 @@ static void gator_timer_interrupt(void) | |||
376 | { | 312 | { |
377 | struct pt_regs * const regs = get_irq_regs(); | 313 | struct pt_regs * const regs = get_irq_regs(); |
378 | int cpu = smp_processor_id(); | 314 | int cpu = smp_processor_id(); |
379 | int *buffer, len, tick; | 315 | int *buffer, len, i, buftype = TIMER_BUF; |
316 | long long *buffer64; | ||
380 | struct gator_interface *gi; | 317 | struct gator_interface *gi; |
381 | 318 | ||
382 | // check full backtrace has enough space, otherwise may | 319 | // check full backtrace has enough space, otherwise may |
@@ -391,44 +328,48 @@ static void gator_timer_interrupt(void) | |||
391 | return; | 328 | return; |
392 | } | 329 | } |
393 | 330 | ||
394 | // Header | ||
395 | gator_buffer_write_packed_int(cpu, PROTOCOL_START_TICK); // Escape | ||
396 | |||
397 | // Output scheduler | 331 | // Output scheduler |
398 | len = gator_trace_sched_read(&buffer); | 332 | len = gator_trace_sched_read(&buffer64); |
399 | if (len > 0) { | 333 | if (len > 0) { |
400 | gator_write_packet(cpu, PROTOCOL_SCHEDULER_TRACE, len, buffer); | 334 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); |
335 | gator_buffer_write_packed_int(cpu, buftype, len); | ||
336 | for (i = 0; i < len; i++) { | ||
337 | gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); | ||
338 | } | ||
401 | } | 339 | } |
402 | 340 | ||
403 | // Output counters | 341 | // Output counters |
342 | gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); | ||
343 | gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); | ||
404 | list_for_each_entry(gi, &gator_events, list) { | 344 | list_for_each_entry(gi, &gator_events, list) { |
405 | if (gi->read) { | 345 | if (gi->read) { |
406 | len = gi->read(&buffer); | 346 | len = gi->read(&buffer); |
347 | if (len > 0) { | ||
348 | gator_buffer_write_packed_int(cpu, buftype, len); | ||
349 | for (i = 0; i < len; i++) { | ||
350 | gator_buffer_write_packed_int(cpu, buftype, buffer[i]); | ||
351 | } | ||
352 | } | ||
353 | } else if (gi->read64) { | ||
354 | len = gi->read64(&buffer64); | ||
407 | if (len > 0) | 355 | if (len > 0) |
408 | gator_write_packet(cpu, PROTOCOL_COUNTERS, len, buffer); | 356 | gator_buffer_write_packed_int(cpu, buftype, len); |
357 | for (i = 0; i < len; i++) { | ||
358 | gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); | ||
359 | } | ||
409 | } | 360 | } |
410 | } | 361 | } |
362 | gator_buffer_write_packed_int(cpu, buftype, 0); | ||
411 | 363 | ||
412 | // Output backtrace | 364 | // Output backtrace |
413 | gator_add_sample(cpu, regs); | 365 | if (!event_based_sampling) { |
414 | 366 | gator_add_sample(cpu, buftype, regs); | |
415 | // Timer Tick | ||
416 | tick = per_cpu(gator_cpu_tick, cpu); | ||
417 | if (tick == gator_master_tick) { | ||
418 | tick++; | ||
419 | per_cpu(gator_cpu_tick, cpu) = gator_master_tick = tick; | ||
420 | } else { | ||
421 | per_cpu(gator_cpu_tick, cpu) = tick = gator_master_tick; | ||
422 | } | 367 | } |
423 | gator_write_packet(cpu, PROTOCOL_END_TICK, 1, &tick); | ||
424 | 368 | ||
425 | // Check and commit; generally, commit is set to occur once per second | 369 | // Check and commit; generally, commit is set to occur once per second |
426 | gator_buffer_check(cpu, tick); | 370 | timer_buffer_check(cpu); |
427 | } | 371 | } |
428 | 372 | ||
429 | /****************************************************************************** | ||
430 | * hrtimer | ||
431 | ******************************************************************************/ | ||
432 | DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); | 373 | DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); |
433 | DEFINE_PER_CPU(int, hrtimer_is_active); | 374 | DEFINE_PER_CPU(int, hrtimer_is_active); |
434 | static int hrtimer_running; | 375 | static int hrtimer_running; |
@@ -454,7 +395,9 @@ static void __gator_timer_offline(void *unused) | |||
454 | struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); | 395 | struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); |
455 | hrtimer_cancel(hrtimer); | 396 | hrtimer_cancel(hrtimer); |
456 | per_cpu(hrtimer_is_active, cpu) = 0; | 397 | per_cpu(hrtimer_is_active, cpu) = 0; |
457 | gator_buffer_commit(cpu); | 398 | gator_buffer_commit(cpu, TIMER_BUF); |
399 | if (event_based_sampling) | ||
400 | gator_buffer_commit(cpu, EVENT_BUF); | ||
458 | 401 | ||
459 | // offline any events | 402 | // offline any events |
460 | list_for_each_entry(gi, &gator_events, list) | 403 | list_for_each_entry(gi, &gator_events, list) |
@@ -469,10 +412,6 @@ static void gator_timer_offline(void) | |||
469 | hrtimer_running = 0; | 412 | hrtimer_running = 0; |
470 | 413 | ||
471 | on_each_cpu(__gator_timer_offline, NULL, 1); | 414 | on_each_cpu(__gator_timer_offline, NULL, 1); |
472 | |||
473 | // output a final sync point | ||
474 | gator_buffer_write_packed_int(0, PROTOCOL_CPU_SYNC); | ||
475 | gator_buffer_commit(0); | ||
476 | } | 415 | } |
477 | } | 416 | } |
478 | 417 | ||
@@ -485,7 +424,6 @@ static void __gator_timer_online(void *unused) | |||
485 | hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 424 | hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
486 | hrtimer->function = gator_hrtimer_notify; | 425 | hrtimer->function = gator_hrtimer_notify; |
487 | hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); | 426 | hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); |
488 | per_cpu(gator_cpu_tick, cpu) = 0; | ||
489 | per_cpu(gator_first_time, cpu) = 1; | 427 | per_cpu(gator_first_time, cpu) = 1; |
490 | per_cpu(hrtimer_is_active, cpu) = 1; | 428 | per_cpu(hrtimer_is_active, cpu) = 1; |
491 | 429 | ||
@@ -499,10 +437,10 @@ static void __gator_timer_online(void *unused) | |||
499 | int gator_timer_online(unsigned long setup) | 437 | int gator_timer_online(unsigned long setup) |
500 | { | 438 | { |
501 | if (!setup) { | 439 | if (!setup) { |
502 | pr_err("gator: cannot start due to a system tick value of zero"); | 440 | pr_err("gator: cannot start due to a system tick value of zero\n"); |
503 | return -1; | 441 | return -1; |
504 | } else if (hrtimer_running) { | 442 | } else if (hrtimer_running) { |
505 | pr_notice("gator: high res timer already running"); | 443 | pr_notice("gator: high res timer already running\n"); |
506 | return 0; | 444 | return 0; |
507 | } | 445 | } |
508 | 446 | ||
@@ -512,7 +450,6 @@ int gator_timer_online(unsigned long setup) | |||
512 | profiling_interval = ns_to_ktime(1000000000UL / setup); | 450 | profiling_interval = ns_to_ktime(1000000000UL / setup); |
513 | 451 | ||
514 | // timer interrupt | 452 | // timer interrupt |
515 | gator_master_tick = 0; | ||
516 | on_each_cpu(__gator_timer_online, NULL, 1); | 453 | on_each_cpu(__gator_timer_online, NULL, 1); |
517 | 454 | ||
518 | return 0; | 455 | return 0; |
@@ -623,10 +560,15 @@ static int gator_start(void) | |||
623 | } | 560 | } |
624 | } | 561 | } |
625 | 562 | ||
563 | // cookies shall be initialized before trace_sched_start() and gator_timer_online() | ||
564 | if (cookies_initialize()) | ||
565 | goto cookies_failure; | ||
626 | if (gator_annotate_start()) | 566 | if (gator_annotate_start()) |
627 | goto annotate_failure; | 567 | goto annotate_failure; |
628 | if (gator_trace_sched_start()) | 568 | if (gator_trace_sched_start()) |
629 | goto sched_failure; | 569 | goto sched_failure; |
570 | if (gator_event_sampling_start()) | ||
571 | goto event_sampling_failure; | ||
630 | if (gator_timer_online(gator_timer_count)) | 572 | if (gator_timer_online(gator_timer_count)) |
631 | goto timer_failure; | 573 | goto timer_failure; |
632 | if (gator_notifier_start()) | 574 | if (gator_notifier_start()) |
@@ -637,10 +579,18 @@ static int gator_start(void) | |||
637 | notifier_failure: | 579 | notifier_failure: |
638 | gator_timer_offline(); | 580 | gator_timer_offline(); |
639 | timer_failure: | 581 | timer_failure: |
582 | gator_event_sampling_stop(); | ||
583 | event_sampling_failure: | ||
640 | gator_trace_sched_stop(); | 584 | gator_trace_sched_stop(); |
641 | sched_failure: | 585 | sched_failure: |
642 | gator_annotate_stop(); | 586 | gator_annotate_stop(); |
643 | annotate_failure: | 587 | annotate_failure: |
588 | cookies_release(); | ||
589 | cookies_failure: | ||
590 | // stop all events | ||
591 | list_for_each_entry(gi, &gator_events, list) | ||
592 | if (gi->stop) | ||
593 | gi->stop(); | ||
644 | events_failure: | 594 | events_failure: |
645 | 595 | ||
646 | return -1; | 596 | return -1; |
@@ -657,10 +607,11 @@ static void gator_stop(void) | |||
657 | 607 | ||
658 | gator_annotate_stop(); | 608 | gator_annotate_stop(); |
659 | gator_trace_sched_stop(); | 609 | gator_trace_sched_stop(); |
610 | gator_event_sampling_stop(); | ||
660 | 611 | ||
661 | // stop all interrupt callback reads before tearing down other interfaces | 612 | // stop all interrupt callback reads before tearing down other interfaces |
613 | gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined | ||
662 | gator_timer_offline(); | 614 | gator_timer_offline(); |
663 | gator_notifier_stop(); | ||
664 | } | 615 | } |
665 | 616 | ||
666 | static void gator_exit(void) | 617 | static void gator_exit(void) |
@@ -675,42 +626,45 @@ static void gator_exit(void) | |||
675 | static int gator_op_setup(void) | 626 | static int gator_op_setup(void) |
676 | { | 627 | { |
677 | int err = 0; | 628 | int err = 0; |
678 | int cpu; | 629 | int cpu, i; |
679 | 630 | ||
680 | mutex_lock(&start_mutex); | 631 | mutex_lock(&start_mutex); |
681 | 632 | ||
682 | use_buffer_size = gator_buffer_size; | 633 | gator_buffer_size[TIMER_BUF] = userspace_buffer_size; |
683 | use_buffer_mask = use_buffer_size - 1; | 634 | gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1; |
684 | 635 | ||
685 | // must be a power of 2 | 636 | // must be a power of 2 |
686 | if (use_buffer_size & (use_buffer_size - 1)) { | 637 | if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) { |
687 | err = -ENOEXEC; | 638 | err = -ENOEXEC; |
688 | goto setup_error; | 639 | goto setup_error; |
689 | } | 640 | } |
690 | 641 | ||
691 | gator_net_traffic = 0; | 642 | gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; |
643 | gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; | ||
692 | 644 | ||
693 | gator_commit_read = gator_commit_write = 0; | 645 | gator_net_traffic = 0; |
694 | gator_commit = vmalloc(COMMIT_SIZE * sizeof(int)); | ||
695 | if (!gator_commit) { | ||
696 | err = -ENOMEM; | ||
697 | goto setup_error; | ||
698 | } | ||
699 | 646 | ||
700 | for_each_present_cpu(cpu) { | 647 | // Initialize per buffer variables |
701 | per_cpu(use_buffer, cpu) = vmalloc(use_buffer_size); | 648 | for (i = 0; i < NUM_GATOR_BUFS; i++) { |
702 | if (!per_cpu(use_buffer, cpu)) { | 649 | gator_commit_read[i] = gator_commit_write[i] = 0; |
650 | gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int)); | ||
651 | if (!gator_commit[i]) { | ||
703 | err = -ENOMEM; | 652 | err = -ENOMEM; |
704 | goto setup_error; | 653 | goto setup_error; |
705 | } | 654 | } |
706 | 655 | ||
707 | per_cpu(gator_cpu_sync, cpu) = 0; | 656 | // Initialize percpu per buffer variables |
708 | per_cpu(gator_cpu_tick, cpu) = 0; | 657 | for_each_present_cpu(cpu) { |
658 | per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); | ||
659 | if (!per_cpu(gator_buffer, cpu)[i]) { | ||
660 | err = -ENOMEM; | ||
661 | goto setup_error; | ||
662 | } | ||
709 | 663 | ||
710 | per_cpu(use_buffer_seq, cpu) = 0; | 664 | per_cpu(gator_buffer_read, cpu)[i] = 0; |
711 | per_cpu(use_buffer_read, cpu) = 0; | 665 | per_cpu(gator_buffer_write, cpu)[i] = 0; |
712 | per_cpu(use_buffer_write, cpu) = 0; | 666 | gator_buffer_header(cpu, i); |
713 | gator_buffer_header(cpu); | 667 | } |
714 | } | 668 | } |
715 | 669 | ||
716 | setup_error: | 670 | setup_error: |
@@ -725,7 +679,7 @@ static int gator_op_start(void) | |||
725 | 679 | ||
726 | mutex_lock(&start_mutex); | 680 | mutex_lock(&start_mutex); |
727 | 681 | ||
728 | if (gator_started || gator_start() || cookies_initialize()) | 682 | if (gator_started || gator_start()) |
729 | err = -EINVAL; | 683 | err = -EINVAL; |
730 | else | 684 | else |
731 | gator_started = 1; | 685 | gator_started = 1; |
@@ -757,20 +711,25 @@ static void gator_op_stop(void) | |||
757 | 711 | ||
758 | static void gator_shutdown(void) | 712 | static void gator_shutdown(void) |
759 | { | 713 | { |
760 | int cpu; | 714 | int cpu, i; |
761 | 715 | ||
762 | mutex_lock(&start_mutex); | 716 | mutex_lock(&start_mutex); |
763 | 717 | ||
764 | vfree(gator_commit); | 718 | gator_annotate_shutdown(); |
765 | gator_commit = NULL; | 719 | |
720 | for (i = 0; i < NUM_GATOR_BUFS; i++) { | ||
721 | vfree(gator_commit[i]); | ||
722 | gator_commit[i] = NULL; | ||
723 | } | ||
766 | 724 | ||
767 | for_each_present_cpu(cpu) { | 725 | for_each_present_cpu(cpu) { |
768 | mutex_lock(&gator_buffer_mutex); | 726 | mutex_lock(&gator_buffer_mutex); |
769 | vfree(per_cpu(use_buffer, cpu)); | 727 | for (i = 0; i < NUM_GATOR_BUFS; i++) { |
770 | per_cpu(use_buffer, cpu) = NULL; | 728 | vfree(per_cpu(gator_buffer, cpu)[i]); |
771 | per_cpu(use_buffer_seq, cpu) = 0; | 729 | per_cpu(gator_buffer, cpu)[i] = NULL; |
772 | per_cpu(use_buffer_read, cpu) = 0; | 730 | per_cpu(gator_buffer_read, cpu)[i] = 0; |
773 | per_cpu(use_buffer_write, cpu) = 0; | 731 | per_cpu(gator_buffer_write, cpu)[i] = 0; |
732 | } | ||
774 | mutex_unlock(&gator_buffer_mutex); | 733 | mutex_unlock(&gator_buffer_mutex); |
775 | } | 734 | } |
776 | 735 | ||
@@ -825,7 +784,7 @@ static const struct file_operations enable_fops = { | |||
825 | .write = enable_write, | 784 | .write = enable_write, |
826 | }; | 785 | }; |
827 | 786 | ||
828 | static int event_buffer_open(struct inode *inode, struct file *file) | 787 | static int userspace_buffer_open(struct inode *inode, struct file *file) |
829 | { | 788 | { |
830 | int err = -EPERM; | 789 | int err = -EPERM; |
831 | 790 | ||
@@ -849,7 +808,7 @@ fail: | |||
849 | return err; | 808 | return err; |
850 | } | 809 | } |
851 | 810 | ||
852 | static int event_buffer_release(struct inode *inode, struct file *file) | 811 | static int userspace_buffer_release(struct inode *inode, struct file *file) |
853 | { | 812 | { |
854 | gator_op_stop(); | 813 | gator_op_stop(); |
855 | gator_shutdown(); | 814 | gator_shutdown(); |
@@ -857,55 +816,59 @@ static int event_buffer_release(struct inode *inode, struct file *file) | |||
857 | return 0; | 816 | return 0; |
858 | } | 817 | } |
859 | 818 | ||
860 | static ssize_t event_buffer_read(struct file *file, char __user *buf, | 819 | static ssize_t userspace_buffer_read(struct file *file, char __user *buf, |
861 | size_t count, loff_t *offset) | 820 | size_t count, loff_t *offset) |
862 | { | 821 | { |
863 | int retval = -EINVAL; | 822 | int retval = -EINVAL; |
864 | int commit, length1, length2, read; | 823 | int commit, length1, length2, read; |
865 | char *buffer1, *buffer2; | 824 | char *buffer1; |
866 | char annotate_header[6]; | 825 | char *buffer2 = NULL; |
867 | int cpu; | 826 | int cpu, i; |
868 | 827 | ||
869 | /* do not handle partial reads */ | 828 | /* do not handle partial reads */ |
870 | if (count != use_buffer_size || *offset) | 829 | if (count != userspace_buffer_size || *offset) |
871 | return -EINVAL; | 830 | return -EINVAL; |
872 | 831 | ||
873 | // sleep until the condition is true or a signal is received | 832 | // sleep until the condition is true or a signal is received |
874 | // the condition is checked each time gator_buffer_wait is woken up | 833 | // the condition is checked each time gator_buffer_wait is woken up |
875 | wait_event_interruptible(gator_buffer_wait, buffer_commit_ready() || gator_annotate_ready() || !gator_started); | 834 | wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started); |
876 | 835 | ||
877 | if (signal_pending(current)) | 836 | if (signal_pending(current)) |
878 | return -EINTR; | 837 | return -EINTR; |
879 | 838 | ||
839 | length2 = 0; | ||
880 | retval = -EFAULT; | 840 | retval = -EFAULT; |
881 | 841 | ||
882 | mutex_lock(&gator_buffer_mutex); | 842 | mutex_lock(&gator_buffer_mutex); |
883 | 843 | ||
884 | if (buffer_commit_ready()) { | 844 | i = -1; |
885 | buffer_commit_read(&cpu, &read, &commit); | 845 | if (buffer_commit_ready(TIMER_BUF)) { |
846 | i = TIMER_BUF; | ||
847 | } else if (buffer_commit_ready(EVENT_BUF)) { | ||
848 | i = EVENT_BUF; | ||
849 | } | ||
850 | |||
851 | if (i != -1) { | ||
852 | buffer_commit_read(&cpu, i, &read, &commit); | ||
886 | 853 | ||
887 | /* May happen if the buffer is freed during pending reads. */ | 854 | /* May happen if the buffer is freed during pending reads. */ |
888 | if (!per_cpu(use_buffer, cpu)) { | 855 | if (!per_cpu(gator_buffer, cpu)[i]) { |
889 | retval = -EFAULT; | 856 | retval = -EFAULT; |
890 | goto out; | 857 | goto out; |
891 | } | 858 | } |
892 | 859 | ||
893 | /* determine the size of two halves */ | 860 | /* determine the size of two halves */ |
894 | length1 = commit - read; | 861 | length1 = commit - read; |
895 | length2 = 0; | 862 | buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]); |
896 | buffer1 = &(per_cpu(use_buffer, cpu)[read]); | 863 | buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]); |
897 | buffer2 = &(per_cpu(use_buffer, cpu)[0]); | ||
898 | if (length1 < 0) { | 864 | if (length1 < 0) { |
899 | length1 = use_buffer_size - read; | 865 | length1 = gator_buffer_size[i] - read; |
900 | length2 = commit; | 866 | length2 = commit; |
901 | } | 867 | } |
902 | } else if (gator_annotate_ready()) { | 868 | } else if (gator_annotate_ready()) { |
903 | length2 = gator_annotate_read(&buffer2); | 869 | length1 = gator_annotate_read(&buffer1); |
904 | if (!length2) | 870 | if (!length1) |
905 | goto out; | 871 | goto out; |
906 | annotate_header[0] = PROTOCOL_ANNOTATE; | ||
907 | length1 = gator_write_packed_int(&annotate_header[1], length2) + 1; | ||
908 | buffer1 = annotate_header; | ||
909 | } else { | 872 | } else { |
910 | retval = 0; | 873 | retval = 0; |
911 | goto out; | 874 | goto out; |
@@ -939,9 +902,9 @@ out: | |||
939 | } | 902 | } |
940 | 903 | ||
941 | const struct file_operations gator_event_buffer_fops = { | 904 | const struct file_operations gator_event_buffer_fops = { |
942 | .open = event_buffer_open, | 905 | .open = userspace_buffer_open, |
943 | .release = event_buffer_release, | 906 | .release = userspace_buffer_release, |
944 | .read = event_buffer_read, | 907 | .read = userspace_buffer_read, |
945 | }; | 908 | }; |
946 | 909 | ||
947 | static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) | 910 | static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) |
@@ -974,17 +937,6 @@ static const struct file_operations depth_fops = { | |||
974 | .write = depth_write | 937 | .write = depth_write |
975 | }; | 938 | }; |
976 | 939 | ||
977 | static const char gator_cpu_type[] = "gator"; | ||
978 | |||
979 | static ssize_t cpu_type_read(struct file *file, char __user *buf, size_t count, loff_t *offset) | ||
980 | { | ||
981 | return gatorfs_str_to_user(gator_cpu_type, buf, count, offset); | ||
982 | } | ||
983 | |||
984 | static const struct file_operations cpu_type_fops = { | ||
985 | .read = cpu_type_read, | ||
986 | }; | ||
987 | |||
988 | void gator_op_create_files(struct super_block *sb, struct dentry *root) | 940 | void gator_op_create_files(struct super_block *sb, struct dentry *root) |
989 | { | 941 | { |
990 | struct dentry *dir; | 942 | struct dentry *dir; |
@@ -996,15 +948,14 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) | |||
996 | for_each_present_cpu(cpu) { | 948 | for_each_present_cpu(cpu) { |
997 | gator_cpu_cores++; | 949 | gator_cpu_cores++; |
998 | } | 950 | } |
999 | gator_buffer_size = BUFFER_SIZE_DEFAULT; | 951 | userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT; |
1000 | gator_streaming = 1; | 952 | gator_streaming = 1; |
1001 | 953 | ||
1002 | gatorfs_create_file(sb, root, "enable", &enable_fops); | 954 | gatorfs_create_file(sb, root, "enable", &enable_fops); |
1003 | gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); | 955 | gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); |
1004 | gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); | 956 | gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); |
1005 | gatorfs_create_file(sb, root, "cpu_type", &cpu_type_fops); | ||
1006 | gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); | 957 | gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); |
1007 | gatorfs_create_ulong(sb, root, "buffer_size", &gator_buffer_size); | 958 | gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size); |
1008 | gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); | 959 | gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); |
1009 | gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); | 960 | gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); |
1010 | gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); | 961 | gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); |
@@ -1033,17 +984,11 @@ static int __init gator_module_init(void) | |||
1033 | return -1; | 984 | return -1; |
1034 | } | 985 | } |
1035 | 986 | ||
1036 | #ifdef GATOR_DEBUG | ||
1037 | pr_err("gator_module_init"); | ||
1038 | #endif | ||
1039 | return 0; | 987 | return 0; |
1040 | } | 988 | } |
1041 | 989 | ||
1042 | static void __exit gator_module_exit(void) | 990 | static void __exit gator_module_exit(void) |
1043 | { | 991 | { |
1044 | #ifdef GATOR_DEBUG | ||
1045 | pr_err("gator_module_exit"); | ||
1046 | #endif | ||
1047 | tracepoint_synchronize_unregister(); | 992 | tracepoint_synchronize_unregister(); |
1048 | gatorfs_unregister(); | 993 | gatorfs_unregister(); |
1049 | gator_exit(); | 994 | gator_exit(); |
diff --git a/driver/gator_pack.c b/driver/gator_pack.c new file mode 100644 index 0000000..fbab220 --- /dev/null +++ b/driver/gator_pack.c | |||
@@ -0,0 +1,262 @@ | |||
1 | /** | ||
2 | * Copyright (C) ARM Limited 2010-2011. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | */ | ||
9 | |||
10 | static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x) | ||
11 | { | ||
12 | uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; | ||
13 | uint32_t mask = gator_buffer_mask[buftype]; | ||
14 | char *buffer = per_cpu(gator_buffer, cpu)[buftype]; | ||
15 | int write0 = (write + 0) & mask; | ||
16 | int write1 = (write + 1) & mask; | ||
17 | |||
18 | if ((x & 0xffffff80) == 0) { | ||
19 | buffer[write0] = x & 0x7f; | ||
20 | per_cpu(gator_buffer_write, cpu)[buftype] = write1; | ||
21 | } else if ((x & 0xffffc000) == 0) { | ||
22 | int write2 = (write + 2) & mask; | ||
23 | buffer[write0] = x | 0x80; | ||
24 | buffer[write1] = (x>>7) & 0x7f; | ||
25 | per_cpu(gator_buffer_write, cpu)[buftype] = write2; | ||
26 | } else if ((x & 0xffe00000) == 0) { | ||
27 | int write2 = (write + 2) & mask; | ||
28 | int write3 = (write + 3) & mask; | ||
29 | buffer[write0] = x | 0x80; | ||
30 | buffer[write1] = (x>>7) | 0x80; | ||
31 | buffer[write2] = (x>>14) & 0x7f; | ||
32 | per_cpu(gator_buffer_write, cpu)[buftype] = write3; | ||
33 | } else if ((x & 0xf0000000) == 0) { | ||
34 | int write2 = (write + 2) & mask; | ||
35 | int write3 = (write + 3) & mask; | ||
36 | int write4 = (write + 4) & mask; | ||
37 | buffer[write0] = x | 0x80; | ||
38 | buffer[write1] = (x>>7) | 0x80; | ||
39 | buffer[write2] = (x>>14) | 0x80; | ||
40 | buffer[write3] = (x>>21) & 0x7f; | ||
41 | per_cpu(gator_buffer_write, cpu)[buftype] = write4; | ||
42 | } else { | ||
43 | int write2 = (write + 2) & mask; | ||
44 | int write3 = (write + 3) & mask; | ||
45 | int write4 = (write + 4) & mask; | ||
46 | int write5 = (write + 5) & mask; | ||
47 | buffer[write0] = x | 0x80; | ||
48 | buffer[write1] = (x>>7) | 0x80; | ||
49 | buffer[write2] = (x>>14) | 0x80; | ||
50 | buffer[write3] = (x>>21) | 0x80; | ||
51 | buffer[write4] = (x>>28) & 0x0f; | ||
52 | per_cpu(gator_buffer_write, cpu)[buftype] = write5; | ||
53 | } | ||
54 | } | ||
55 | |||
56 | static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x) | ||
57 | { | ||
58 | uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; | ||
59 | uint32_t mask = gator_buffer_mask[buftype]; | ||
60 | char *buffer = per_cpu(gator_buffer, cpu)[buftype]; | ||
61 | int write0 = (write + 0) & mask; | ||
62 | int write1 = (write + 1) & mask; | ||
63 | |||
64 | if ((x & 0xffffffffffffff80LL) == 0) { | ||
65 | buffer[write0] = x & 0x7f; | ||
66 | per_cpu(gator_buffer_write, cpu)[buftype] = write1; | ||
67 | } else if ((x & 0xffffffffffffc000LL) == 0) { | ||
68 | int write2 = (write + 2) & mask; | ||
69 | buffer[write0] = x | 0x80; | ||
70 | buffer[write1] = (x>>7) & 0x7f; | ||
71 | per_cpu(gator_buffer_write, cpu)[buftype] = write2; | ||
72 | } else if ((x & 0xffffffffffe00000LL) == 0) { | ||
73 | int write2 = (write + 2) & mask; | ||
74 | int write3 = (write + 3) & mask; | ||
75 | buffer[write0] = x | 0x80; | ||
76 | buffer[write1] = (x>>7) | 0x80; | ||
77 | buffer[write2] = (x>>14) & 0x7f; | ||
78 | per_cpu(gator_buffer_write, cpu)[buftype] = write3; | ||
79 | } else if ((x & 0xfffffffff0000000LL) == 0) { | ||
80 | int write2 = (write + 2) & mask; | ||
81 | int write3 = (write + 3) & mask; | ||
82 | int write4 = (write + 4) & mask; | ||
83 | buffer[write0] = x | 0x80; | ||
84 | buffer[write1] = (x>>7) | 0x80; | ||
85 | buffer[write2] = (x>>14) | 0x80; | ||
86 | buffer[write3] = (x>>21) & 0x7f; | ||
87 | per_cpu(gator_buffer_write, cpu)[buftype] = write4; | ||
88 | } else if ((x & 0xfffffff800000000LL) == 0) { | ||
89 | int write2 = (write + 2) & mask; | ||
90 | int write3 = (write + 3) & mask; | ||
91 | int write4 = (write + 4) & mask; | ||
92 | int write5 = (write + 5) & mask; | ||
93 | buffer[write0] = x | 0x80; | ||
94 | buffer[write1] = (x>>7) | 0x80; | ||
95 | buffer[write2] = (x>>14) | 0x80; | ||
96 | buffer[write3] = (x>>21) | 0x80; | ||
97 | buffer[write4] = (x>>28) & 0x7f; | ||
98 | per_cpu(gator_buffer_write, cpu)[buftype] = write5; | ||
99 | } else if ((x & 0xfffffc0000000000LL) == 0) { | ||
100 | int write2 = (write + 2) & mask; | ||
101 | int write3 = (write + 3) & mask; | ||
102 | int write4 = (write + 4) & mask; | ||
103 | int write5 = (write + 5) & mask; | ||
104 | int write6 = (write + 6) & mask; | ||
105 | buffer[write0] = x | 0x80; | ||
106 | buffer[write1] = (x>>7) | 0x80; | ||
107 | buffer[write2] = (x>>14) | 0x80; | ||
108 | buffer[write3] = (x>>21) | 0x80; | ||
109 | buffer[write4] = (x>>28) | 0x80; | ||
110 | buffer[write5] = (x>>35) & 0x7f; | ||
111 | per_cpu(gator_buffer_write, cpu)[buftype] = write6; | ||
112 | } else if ((x & 0xfffe000000000000LL) == 0) { | ||
113 | int write2 = (write + 2) & mask; | ||
114 | int write3 = (write + 3) & mask; | ||
115 | int write4 = (write + 4) & mask; | ||
116 | int write5 = (write + 5) & mask; | ||
117 | int write6 = (write + 6) & mask; | ||
118 | int write7 = (write + 7) & mask; | ||
119 | buffer[write0] = x | 0x80; | ||
120 | buffer[write1] = (x>>7) | 0x80; | ||
121 | buffer[write2] = (x>>14) | 0x80; | ||
122 | buffer[write3] = (x>>21) | 0x80; | ||
123 | buffer[write4] = (x>>28) | 0x80; | ||
124 | buffer[write5] = (x>>35) | 0x80; | ||
125 | buffer[write6] = (x>>42) & 0x7f; | ||
126 | per_cpu(gator_buffer_write, cpu)[buftype] = write7; | ||
127 | } else if ((x & 0xff00000000000000LL) == 0) { | ||
128 | int write2 = (write + 2) & mask; | ||
129 | int write3 = (write + 3) & mask; | ||
130 | int write4 = (write + 4) & mask; | ||
131 | int write5 = (write + 5) & mask; | ||
132 | int write6 = (write + 6) & mask; | ||
133 | int write7 = (write + 7) & mask; | ||
134 | int write8 = (write + 8) & mask; | ||
135 | buffer[write0] = x | 0x80; | ||
136 | buffer[write1] = (x>>7) | 0x80; | ||
137 | buffer[write2] = (x>>14) | 0x80; | ||
138 | buffer[write3] = (x>>21) | 0x80; | ||
139 | buffer[write4] = (x>>28) | 0x80; | ||
140 | buffer[write5] = (x>>35) | 0x80; | ||
141 | buffer[write6] = (x>>42) | 0x80; | ||
142 | buffer[write7] = (x>>49) & 0x7f; | ||
143 | per_cpu(gator_buffer_write, cpu)[buftype] = write8; | ||
144 | } else { | ||
145 | int write2 = (write + 2) & mask; | ||
146 | int write3 = (write + 3) & mask; | ||
147 | int write4 = (write + 4) & mask; | ||
148 | int write5 = (write + 5) & mask; | ||
149 | int write6 = (write + 6) & mask; | ||
150 | int write7 = (write + 7) & mask; | ||
151 | int write8 = (write + 8) & mask; | ||
152 | int write9 = (write + 9) & mask; | ||
153 | buffer[write0] = x | 0x80; | ||
154 | buffer[write1] = (x>>7) | 0x80; | ||
155 | buffer[write2] = (x>>14) | 0x80; | ||
156 | buffer[write3] = (x>>21) | 0x80; | ||
157 | buffer[write4] = (x>>28) | 0x80; | ||
158 | buffer[write5] = (x>>35) | 0x80; | ||
159 | buffer[write6] = (x>>42) | 0x80; | ||
160 | buffer[write7] = (x>>49) | 0x80; | ||
161 | buffer[write8] = (x>>56) & 0xff; | ||
162 | per_cpu(gator_buffer_write, cpu)[buftype] = write9; | ||
163 | } | ||
164 | } | ||
165 | |||
166 | static int gator_write_packed_int(char *buffer, unsigned int x) | ||
167 | { | ||
168 | if ((x & 0xffffff80) == 0) { | ||
169 | buffer[0] = x & 0x7f; | ||
170 | return 1; | ||
171 | } else if ((x & 0xffffc000) == 0) { | ||
172 | buffer[0] = x | 0x80; | ||
173 | buffer[1] = (x>>7) & 0x7f; | ||
174 | return 2; | ||
175 | } else if ((x & 0xffe00000) == 0) { | ||
176 | buffer[0] = x | 0x80; | ||
177 | buffer[1] = (x>>7) | 0x80; | ||
178 | buffer[2] = (x>>14) & 0x7f; | ||
179 | return 3; | ||
180 | } else if ((x & 0xf0000000) == 0) { | ||
181 | buffer[0] = x | 0x80; | ||
182 | buffer[1] = (x>>7) | 0x80; | ||
183 | buffer[2] = (x>>14) | 0x80; | ||
184 | buffer[3] = (x>>21) & 0x7f; | ||
185 | return 4; | ||
186 | } else { | ||
187 | buffer[0] = x | 0x80; | ||
188 | buffer[1] = (x>>7) | 0x80; | ||
189 | buffer[2] = (x>>14) | 0x80; | ||
190 | buffer[3] = (x>>21) | 0x80; | ||
191 | buffer[4] = (x>>28) & 0x0f; | ||
192 | return 5; | ||
193 | } | ||
194 | } | ||
195 | |||
196 | static int gator_write_packed_int64(char *buffer, unsigned long long x) | ||
197 | { | ||
198 | if ((x & 0xffffffffffffff80LL) == 0) { | ||
199 | buffer[0] = x & 0x7f; | ||
200 | return 1; | ||
201 | } else if ((x & 0xffffffffffffc000LL) == 0) { | ||
202 | buffer[0] = x | 0x80; | ||
203 | buffer[1] = (x>>7) & 0x7f; | ||
204 | return 2; | ||
205 | } else if ((x & 0xffffffffffe00000LL) == 0) { | ||
206 | buffer[0] = x | 0x80; | ||
207 | buffer[1] = (x>>7) | 0x80; | ||
208 | buffer[2] = (x>>14) & 0x7f; | ||
209 | return 3; | ||
210 | } else if ((x & 0xfffffffff0000000LL) == 0) { | ||
211 | buffer[0] = x | 0x80; | ||
212 | buffer[1] = (x>>7) | 0x80; | ||
213 | buffer[2] = (x>>14) | 0x80; | ||
214 | buffer[3] = (x>>21) & 0x7f; | ||
215 | return 4; | ||
216 | } else if ((x & 0xfffffff800000000LL) == 0) { | ||
217 | buffer[0] = x | 0x80; | ||
218 | buffer[1] = (x>>7) | 0x80; | ||
219 | buffer[2] = (x>>14) | 0x80; | ||
220 | buffer[3] = (x>>21) | 0x80; | ||
221 | buffer[4] = (x>>28) & 0x7f; | ||
222 | return 5; | ||
223 | } else if ((x & 0xfffffc0000000000LL) == 0) { | ||
224 | buffer[0] = x | 0x80; | ||
225 | buffer[1] = (x>>7) | 0x80; | ||
226 | buffer[2] = (x>>14) | 0x80; | ||
227 | buffer[3] = (x>>21) | 0x80; | ||
228 | buffer[4] = (x>>28) | 0x80; | ||
229 | buffer[5] = (x>>35) & 0x7f; | ||
230 | return 6; | ||
231 | } else if ((x & 0xfffe000000000000LL) == 0) { | ||
232 | buffer[0] = x | 0x80; | ||
233 | buffer[1] = (x>>7) | 0x80; | ||
234 | buffer[2] = (x>>14) | 0x80; | ||
235 | buffer[3] = (x>>21) | 0x80; | ||
236 | buffer[4] = (x>>28) | 0x80; | ||
237 | buffer[5] = (x>>35) | 0x80; | ||
238 | buffer[6] = (x>>42) & 0x7f; | ||
239 | return 7; | ||
240 | } else if ((x & 0xff00000000000000LL) == 0) { | ||
241 | buffer[0] = x | 0x80; | ||
242 | buffer[1] = (x>>7) | 0x80; | ||
243 | buffer[2] = (x>>14) | 0x80; | ||
244 | buffer[3] = (x>>21) | 0x80; | ||
245 | buffer[4] = (x>>28) | 0x80; | ||
246 | buffer[5] = (x>>35) | 0x80; | ||
247 | buffer[6] = (x>>42) | 0x80; | ||
248 | buffer[7] = (x>>49) & 0x7f; | ||
249 | return 8; | ||
250 | } else { | ||
251 | buffer[0] = x | 0x80; | ||
252 | buffer[1] = (x>>7) | 0x80; | ||
253 | buffer[2] = (x>>14) | 0x80; | ||
254 | buffer[3] = (x>>21) | 0x80; | ||
255 | buffer[4] = (x>>28) | 0x80; | ||
256 | buffer[5] = (x>>35) | 0x80; | ||
257 | buffer[6] = (x>>42) | 0x80; | ||
258 | buffer[7] = (x>>49) | 0x80; | ||
259 | buffer[8] = (x>>56) & 0xff; | ||
260 | return 9; | ||
261 | } | ||
262 | } | ||
diff --git a/driver/gator_trace.h b/driver/gator_trace.h deleted file mode 100644 index 446d37b..0000000 --- a/driver/gator_trace.h +++ /dev/null | |||
@@ -1,26 +0,0 @@ | |||
1 | /** | ||
2 | * Copyright 2010 ARM, Ltd. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License version 2 as | ||
6 | * published by the Free Software Foundation. | ||
7 | * | ||
8 | */ | ||
9 | |||
10 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) | ||
11 | # error Kernels prior to 2.6.32 not supported | ||
12 | #elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) | ||
13 | # define GATOR_DEFINE_PROBE(probe_name, proto) \ | ||
14 | static void probe_##probe_name(PARAMS(proto)) | ||
15 | # define GATOR_REGISTER_TRACE(probe_name) \ | ||
16 | register_trace_##probe_name(probe_##probe_name) | ||
17 | # define GATOR_UNREGISTER_TRACE(probe_name) \ | ||
18 | unregister_trace_##probe_name(probe_##probe_name) | ||
19 | #else | ||
20 | # define GATOR_DEFINE_PROBE(probe_name, proto) \ | ||
21 | static void probe_##probe_name(void *data, PARAMS(proto)) | ||
22 | # define GATOR_REGISTER_TRACE(probe_name) \ | ||
23 | register_trace_##probe_name(probe_##probe_name, NULL) | ||
24 | # define GATOR_UNREGISTER_TRACE(probe_name) \ | ||
25 | unregister_trace_##probe_name(probe_##probe_name, NULL) | ||
26 | #endif | ||
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c index 19d8d89..7c0bd47 100644 --- a/driver/gator_trace_sched.c +++ b/driver/gator_trace_sched.c | |||
@@ -10,36 +10,89 @@ | |||
10 | #include <trace/events/sched.h> | 10 | #include <trace/events/sched.h> |
11 | #include "gator.h" | 11 | #include "gator.h" |
12 | 12 | ||
13 | #define SCHED_TIMER_EVENT 0 | ||
14 | #define SCHED_WAIT_TASK 1 | ||
15 | #define SCHED_WAKEUP 2 | ||
16 | #define SCHED_WAKEUP_NEW 3 | ||
17 | #define SCHED_SWITCH 4 | ||
18 | #define SCHED_MIGRATE_TASK 5 | ||
19 | #define SCHED_PROCESS_FREE 6 | ||
20 | #define SCHED_PROCESS_EXIT 7 | ||
21 | #define SCHED_PROCESS_WAIT 8 | ||
22 | #define SCHED_PROCESS_FORK 9 | ||
23 | #define SCHED_OVERFLOW -1 | 13 | #define SCHED_OVERFLOW -1 |
14 | #define SCHED_SWITCH 1 | ||
15 | #define SCHED_PROCESS_FREE 2 | ||
24 | 16 | ||
25 | #define SCHEDSIZE (16*1024) | 17 | #define FIELD_TYPE 0 |
18 | #define FIELD_TIME 1 | ||
19 | #define FIELD_PARAM1 2 | ||
20 | #define FIELD_PARAM2 3 | ||
21 | #define FIELD_PARAM3 4 | ||
22 | #define FIELDS_PER_SCHED 5 | ||
26 | 23 | ||
27 | static DEFINE_PER_CPU(int *[2], theSchedBuf); | 24 | #define SCHEDSIZE (8*1024) |
25 | #define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ | ||
26 | #define TASK_MAX_COLLISIONS 2 | ||
27 | |||
28 | static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf); | ||
28 | static DEFINE_PER_CPU(int, theSchedSel); | 29 | static DEFINE_PER_CPU(int, theSchedSel); |
29 | static DEFINE_PER_CPU(int, theSchedPos); | 30 | static DEFINE_PER_CPU(int, theSchedPos); |
30 | static DEFINE_PER_CPU(int, theSchedErr); | 31 | static DEFINE_PER_CPU(int, theSchedErr); |
32 | static DEFINE_PER_CPU(uint64_t *, taskname_keys); | ||
33 | |||
34 | void emit_pid_name(uint64_t time, struct task_struct* task) | ||
35 | { | ||
36 | bool found = false; | ||
37 | unsigned long flags; | ||
38 | char taskcomm[TASK_COMM_LEN + 3]; | ||
39 | int x, cpu = smp_processor_id(); | ||
40 | uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); | ||
41 | uint64_t value; | ||
42 | |||
43 | value = gator_chksum_crc32(task->comm); | ||
44 | value = (value << 32) | (uint32_t)task->pid; | ||
45 | |||
46 | // determine if the thread name was emitted already | ||
47 | for (x = 0; x < TASK_MAX_COLLISIONS; x++) { | ||
48 | if (keys[x] == value) { | ||
49 | found = true; | ||
50 | break; | ||
51 | } | ||
52 | } | ||
53 | |||
54 | if (!found) { | ||
55 | // shift values, new value always in front | ||
56 | uint64_t oldv, newv = value; | ||
57 | for (x = 0; x < TASK_MAX_COLLISIONS; x++) { | ||
58 | oldv = keys[x]; | ||
59 | keys[x] = newv; | ||
60 | newv = oldv; | ||
61 | } | ||
62 | |||
63 | // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions | ||
64 | if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) | ||
65 | // append ellipses if task->comm has length of TASK_COMM_LEN - 1 | ||
66 | strcat(taskcomm, "..."); | ||
67 | |||
68 | // disable interrupts to synchronize with hrtimer populating timer buf | ||
69 | local_irq_save(flags); | ||
70 | gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME); | ||
71 | gator_buffer_write_packed_int64(cpu, TIMER_BUF, time); | ||
72 | gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid); | ||
73 | gator_buffer_write_string(cpu, TIMER_BUF, taskcomm); | ||
74 | local_irq_restore(flags); | ||
75 | } | ||
76 | } | ||
31 | 77 | ||
32 | static void probe_sched_write(int type, int param1, int param2, int param3) | 78 | static void probe_sched_write(int type, int param1, int param2, int param3) |
33 | { | 79 | { |
34 | unsigned long flags; | 80 | unsigned long flags; |
35 | int cpu = smp_processor_id(); | 81 | int cpu = smp_processor_id(); |
36 | uint64_t time = gator_get_time(); | 82 | uint64_t time = gator_get_time(); |
37 | int *schedBuf; | 83 | uint64_t *schedBuf; |
38 | int schedPos; | 84 | int schedPos, cookie = param3; |
39 | 85 | ||
40 | if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) | 86 | if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) |
41 | return; | 87 | return; |
42 | 88 | ||
89 | if (param3) { | ||
90 | // do as much work as possible before disabling interrupts | ||
91 | struct task_struct *task = (struct task_struct *)param3; | ||
92 | cookie = get_exec_cookie(cpu, TIMER_BUF, task); | ||
93 | emit_pid_name(time, task); | ||
94 | } | ||
95 | |||
43 | // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used | 96 | // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used |
44 | local_irq_save(flags); | 97 | local_irq_save(flags); |
45 | 98 | ||
@@ -48,88 +101,37 @@ static void probe_sched_write(int type, int param1, int param2, int param3) | |||
48 | 101 | ||
49 | if (schedPos < (SCHEDSIZE-100)) { | 102 | if (schedPos < (SCHEDSIZE-100)) { |
50 | // capture | 103 | // capture |
51 | schedBuf[schedPos+0] = type; | 104 | schedBuf[schedPos+FIELD_TYPE] = type; |
52 | schedBuf[schedPos+1] = (int)time; | 105 | schedBuf[schedPos+FIELD_TIME] = time; |
53 | schedBuf[schedPos+2] = (int)(time >> 32); | 106 | schedBuf[schedPos+FIELD_PARAM1] = param1; |
54 | schedBuf[schedPos+3] = param1; | 107 | schedBuf[schedPos+FIELD_PARAM2] = param2; |
55 | schedBuf[schedPos+4] = param2; | 108 | schedBuf[schedPos+FIELD_PARAM3] = cookie; |
56 | schedBuf[schedPos+5] = param3; | 109 | per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; |
57 | per_cpu(theSchedPos, cpu) = schedPos + 6; | ||
58 | } else if (!per_cpu(theSchedErr, cpu)) { | 110 | } else if (!per_cpu(theSchedErr, cpu)) { |
59 | per_cpu(theSchedErr, cpu) = 1; | 111 | per_cpu(theSchedErr, cpu) = 1; |
60 | schedBuf[schedPos+0] = SCHED_OVERFLOW; | 112 | schedBuf[schedPos+FIELD_TYPE] = SCHED_OVERFLOW; |
61 | schedBuf[schedPos+1] = 0; | 113 | schedBuf[schedPos+FIELD_TIME] = time; |
62 | schedBuf[schedPos+2] = 0; | 114 | schedBuf[schedPos+FIELD_PARAM1] = 0; |
63 | schedBuf[schedPos+3] = 0; | 115 | schedBuf[schedPos+FIELD_PARAM2] = 0; |
64 | schedBuf[schedPos+4] = 0; | 116 | schedBuf[schedPos+FIELD_PARAM3] = 0; |
65 | schedBuf[schedPos+5] = 0; | 117 | per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; |
66 | per_cpu(theSchedPos, cpu) = schedPos + 6; | ||
67 | pr_debug("gator: tracepoint overflow\n"); | 118 | pr_debug("gator: tracepoint overflow\n"); |
68 | } | 119 | } |
69 | local_irq_restore(flags); | 120 | local_irq_restore(flags); |
70 | } | 121 | } |
71 | 122 | ||
72 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) | 123 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) |
73 | GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct rq *rq, struct task_struct *p)) | ||
74 | #else | ||
75 | GATOR_DEFINE_PROBE(sched_wait_task, TP_PROTO(struct task_struct *p)) | ||
76 | #endif | ||
77 | { | ||
78 | probe_sched_write(SCHED_WAIT_TASK, 0, p->pid, 0); | ||
79 | } | ||
80 | |||
81 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) | ||
82 | GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct rq *rq, struct task_struct *p, int success)) | ||
83 | #else | ||
84 | GATOR_DEFINE_PROBE(sched_wakeup, TP_PROTO(struct task_struct *p, int success)) | ||
85 | #endif | ||
86 | { | ||
87 | if (success) | ||
88 | probe_sched_write(SCHED_WAKEUP, 0, p->pid, 0); | ||
89 | } | ||
90 | |||
91 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) | ||
92 | GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct rq *rq, struct task_struct *p, int success)) | ||
93 | #else | ||
94 | GATOR_DEFINE_PROBE(sched_wakeup_new, TP_PROTO(struct task_struct *p, int success)) | ||
95 | #endif | ||
96 | { | ||
97 | if (success) | ||
98 | probe_sched_write(SCHED_WAKEUP_NEW, 0, p->tgid, p->pid); | ||
99 | } | ||
100 | |||
101 | #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) | ||
102 | GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) | 124 | GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) |
103 | #else | 125 | #else |
104 | GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) | 126 | GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) |
105 | #endif | 127 | #endif |
106 | { | 128 | { |
107 | probe_sched_write(SCHED_SWITCH, (int)next, next->tgid, next->pid); | 129 | probe_sched_write(SCHED_SWITCH, next->pid, next->tgid, (int)next); |
108 | } | ||
109 | |||
110 | GATOR_DEFINE_PROBE(sched_migrate_task, TP_PROTO(struct task_struct *p, int dest_cpu)) | ||
111 | { | ||
112 | probe_sched_write(SCHED_MIGRATE_TASK, 0, dest_cpu, p->pid); | ||
113 | } | 130 | } |
114 | 131 | ||
115 | GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) | 132 | GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) |
116 | { | 133 | { |
117 | probe_sched_write(SCHED_PROCESS_FREE, 0, p->pid, 0); | 134 | probe_sched_write(SCHED_PROCESS_FREE, p->pid, 0, 0); |
118 | } | ||
119 | |||
120 | GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p)) | ||
121 | { | ||
122 | probe_sched_write(SCHED_PROCESS_EXIT, 0, p->pid, 0); | ||
123 | } | ||
124 | |||
125 | GATOR_DEFINE_PROBE(sched_process_wait, TP_PROTO(struct pid *pid)) | ||
126 | { | ||
127 | probe_sched_write(SCHED_PROCESS_WAIT, 0, pid_nr(pid), 0); | ||
128 | } | ||
129 | |||
130 | GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child)) | ||
131 | { | ||
132 | probe_sched_write(SCHED_PROCESS_FORK, (int)child, parent->pid, child->pid); | ||
133 | } | 135 | } |
134 | 136 | ||
135 | int gator_trace_sched_init(void) | 137 | int gator_trace_sched_init(void) |
@@ -139,59 +141,37 @@ int gator_trace_sched_init(void) | |||
139 | 141 | ||
140 | int gator_trace_sched_start(void) | 142 | int gator_trace_sched_start(void) |
141 | { | 143 | { |
142 | int cpu; | 144 | int cpu, size; |
143 | 145 | ||
144 | for_each_present_cpu(cpu) { | 146 | for_each_present_cpu(cpu) { |
145 | per_cpu(theSchedSel, cpu) = 0; | 147 | per_cpu(theSchedSel, cpu) = 0; |
146 | per_cpu(theSchedPos, cpu) = 0; | 148 | per_cpu(theSchedPos, cpu) = 0; |
147 | per_cpu(theSchedErr, cpu) = 0; | 149 | per_cpu(theSchedErr, cpu) = 0; |
148 | per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); | 150 | per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); |
149 | per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(int), GFP_KERNEL); | 151 | per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); |
150 | if (!per_cpu(theSchedBuf, cpu)) | 152 | if (!per_cpu(theSchedBuf, cpu)) |
151 | return -1; | 153 | return -1; |
154 | |||
155 | size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t); | ||
156 | per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); | ||
157 | if (!per_cpu(taskname_keys, cpu)) | ||
158 | return -1; | ||
159 | memset(per_cpu(taskname_keys, cpu), 0, size); | ||
152 | } | 160 | } |
153 | 161 | ||
154 | // register tracepoints | 162 | // register tracepoints |
155 | if (GATOR_REGISTER_TRACE(sched_wait_task)) | ||
156 | goto fail_sched_wait_task; | ||
157 | if (GATOR_REGISTER_TRACE(sched_wakeup)) | ||
158 | goto fail_sched_wakeup; | ||
159 | if (GATOR_REGISTER_TRACE(sched_wakeup_new)) | ||
160 | goto fail_sched_wakeup_new; | ||
161 | if (GATOR_REGISTER_TRACE(sched_switch)) | 163 | if (GATOR_REGISTER_TRACE(sched_switch)) |
162 | goto fail_sched_switch; | 164 | goto fail_sched_switch; |
163 | if (GATOR_REGISTER_TRACE(sched_migrate_task)) | ||
164 | goto fail_sched_migrate_task; | ||
165 | if (GATOR_REGISTER_TRACE(sched_process_free)) | 165 | if (GATOR_REGISTER_TRACE(sched_process_free)) |
166 | goto fail_sched_process_free; | 166 | goto fail_sched_process_free; |
167 | if (GATOR_REGISTER_TRACE(sched_process_exit)) | ||
168 | goto fail_sched_process_exit; | ||
169 | if (GATOR_REGISTER_TRACE(sched_process_wait)) | ||
170 | goto fail_sched_process_wait; | ||
171 | if (GATOR_REGISTER_TRACE(sched_process_fork)) | ||
172 | goto fail_sched_process_fork; | ||
173 | pr_debug("gator: registered tracepoints\n"); | 167 | pr_debug("gator: registered tracepoints\n"); |
174 | 168 | ||
175 | return 0; | 169 | return 0; |
176 | 170 | ||
177 | // unregister tracepoints on error | 171 | // unregister tracepoints on error |
178 | fail_sched_process_fork: | ||
179 | GATOR_UNREGISTER_TRACE(sched_process_wait); | ||
180 | fail_sched_process_wait: | ||
181 | GATOR_UNREGISTER_TRACE(sched_process_exit); | ||
182 | fail_sched_process_exit: | ||
183 | GATOR_UNREGISTER_TRACE(sched_process_free); | ||
184 | fail_sched_process_free: | 172 | fail_sched_process_free: |
185 | GATOR_UNREGISTER_TRACE(sched_migrate_task); | ||
186 | fail_sched_migrate_task: | ||
187 | GATOR_UNREGISTER_TRACE(sched_switch); | 173 | GATOR_UNREGISTER_TRACE(sched_switch); |
188 | fail_sched_switch: | 174 | fail_sched_switch: |
189 | GATOR_UNREGISTER_TRACE(sched_wakeup_new); | ||
190 | fail_sched_wakeup_new: | ||
191 | GATOR_UNREGISTER_TRACE(sched_wakeup); | ||
192 | fail_sched_wakeup: | ||
193 | GATOR_UNREGISTER_TRACE(sched_wait_task); | ||
194 | fail_sched_wait_task: | ||
195 | pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); | 175 | pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); |
196 | 176 | ||
197 | return -1; | 177 | return -1; |
@@ -200,15 +180,8 @@ fail_sched_wait_task: | |||
200 | void gator_trace_sched_stop(void) | 180 | void gator_trace_sched_stop(void) |
201 | { | 181 | { |
202 | int cpu; | 182 | int cpu; |
203 | GATOR_UNREGISTER_TRACE(sched_wait_task); | ||
204 | GATOR_UNREGISTER_TRACE(sched_wakeup); | ||
205 | GATOR_UNREGISTER_TRACE(sched_wakeup_new); | ||
206 | GATOR_UNREGISTER_TRACE(sched_switch); | 183 | GATOR_UNREGISTER_TRACE(sched_switch); |
207 | GATOR_UNREGISTER_TRACE(sched_migrate_task); | ||
208 | GATOR_UNREGISTER_TRACE(sched_process_free); | 184 | GATOR_UNREGISTER_TRACE(sched_process_free); |
209 | GATOR_UNREGISTER_TRACE(sched_process_exit); | ||
210 | GATOR_UNREGISTER_TRACE(sched_process_wait); | ||
211 | GATOR_UNREGISTER_TRACE(sched_process_fork); | ||
212 | pr_debug("gator: unregistered tracepoints\n"); | 185 | pr_debug("gator: unregistered tracepoints\n"); |
213 | 186 | ||
214 | for_each_present_cpu(cpu) { | 187 | for_each_present_cpu(cpu) { |
@@ -216,17 +189,16 @@ void gator_trace_sched_stop(void) | |||
216 | kfree(per_cpu(theSchedBuf, cpu)[1]); | 189 | kfree(per_cpu(theSchedBuf, cpu)[1]); |
217 | per_cpu(theSchedBuf, cpu)[0] = NULL; | 190 | per_cpu(theSchedBuf, cpu)[0] = NULL; |
218 | per_cpu(theSchedBuf, cpu)[1] = NULL; | 191 | per_cpu(theSchedBuf, cpu)[1] = NULL; |
192 | kfree(per_cpu(taskname_keys, cpu)); | ||
219 | } | 193 | } |
220 | } | 194 | } |
221 | 195 | ||
222 | int gator_trace_sched_read(int **buffer) | 196 | int gator_trace_sched_read(long long **buffer) |
223 | { | 197 | { |
224 | uint64_t time = gator_get_time(); | ||
225 | int cpu = smp_processor_id(); | 198 | int cpu = smp_processor_id(); |
226 | unsigned long flags; | 199 | unsigned long flags; |
227 | int *schedBuf; | 200 | uint64_t *schedBuf; |
228 | int schedPos; | 201 | int schedPos; |
229 | int i; | ||
230 | 202 | ||
231 | if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) | 203 | if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) |
232 | return 0; | 204 | return 0; |
@@ -242,20 +214,6 @@ int gator_trace_sched_read(int **buffer) | |||
242 | 214 | ||
243 | local_irq_restore(flags); | 215 | local_irq_restore(flags); |
244 | 216 | ||
245 | // find mm and replace with cookies | ||
246 | for (i = 0; i < schedPos; i += 6) { | ||
247 | uint32_t cookie = schedBuf[i+3]; | ||
248 | if (cookie) { | ||
249 | struct task_struct *task = (struct task_struct *)cookie; | ||
250 | schedBuf[i+3] = get_exec_cookie(cpu, task); | ||
251 | } | ||
252 | } | ||
253 | |||
254 | // timer/end event | ||
255 | schedBuf[schedPos++] = SCHED_TIMER_EVENT; | ||
256 | schedBuf[schedPos++] = (int)time; | ||
257 | schedBuf[schedPos++] = (int)(time >> 32); | ||
258 | |||
259 | if (buffer) | 217 | if (buffer) |
260 | *buffer = schedBuf; | 218 | *buffer = schedBuf; |
261 | 219 | ||