summaryrefslogtreecommitdiffstats
path: root/driver
diff options
context:
space:
mode:
authorJon Medhurst2012-02-20 06:57:24 -0600
committerJon Medhurst2012-02-20 06:57:24 -0600
commit7442caca986378aa639d960ddb726abfd680b7c7 (patch)
treedd369e445dd37bcfcc976950c44ea99509e0bd34 /driver
parent6d51ed1aa2f14b8ee097e940a4d747bf04020195 (diff)
downloadarm-ds5-gator-7442caca986378aa639d960ddb726abfd680b7c7.tar.gz
arm-ds5-gator-7442caca986378aa639d960ddb726abfd680b7c7.tar.xz
arm-ds5-gator-7442caca986378aa639d960ddb726abfd680b7c7.zip
gator-driver: ARM DS-5.9 Streamline gator driver (RC2)
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile4
-rw-r--r--driver/gator.h20
-rw-r--r--driver/gator_annotate.c2
-rw-r--r--driver/gator_annotate_kernel.c3
-rw-r--r--driver/gator_backtrace.c2
-rw-r--r--driver/gator_cookies.c15
-rw-r--r--driver/gator_ebs.c255
-rw-r--r--driver/gator_events_armv6.c52
-rw-r--r--driver/gator_events_armv7.c89
-rw-r--r--driver/gator_events_armv7.h38
-rw-r--r--driver/gator_events_block.c2
-rw-r--r--driver/gator_events_irq.c38
-rw-r--r--driver/gator_events_l2c-310.c7
-rw-r--r--driver/gator_events_mali.c611
-rw-r--r--driver/gator_events_meminfo.c2
-rw-r--r--driver/gator_events_mmaped.c6
-rw-r--r--driver/gator_events_net.c47
-rw-r--r--driver/gator_events_perf_pmu.c296
-rw-r--r--driver/gator_events_power.c178
-rw-r--r--driver/gator_events_sched.c2
-rw-r--r--driver/gator_events_scorpion.c79
-rw-r--r--driver/gator_hrtimer_gator.c76
-rw-r--r--driver/gator_hrtimer_perf.c113
-rw-r--r--driver/gator_main.c492
-rw-r--r--driver/gator_pack.c2
-rw-r--r--driver/gator_trace_gpu.c250
-rw-r--r--driver/gator_trace_gpu.h79
-rw-r--r--driver/gator_trace_sched.c19
28 files changed, 2235 insertions, 544 deletions
diff --git a/driver/Makefile b/driver/Makefile
index b3981ff..e521b99 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -10,7 +10,9 @@ gator-y := gator_main.o \
10 gator_events_sched.o \ 10 gator_events_sched.o \
11 gator_events_net.o \ 11 gator_events_net.o \
12 gator_events_block.o \ 12 gator_events_block.o \
13 gator_events_meminfo.o 13 gator_events_meminfo.o \
14 gator_events_power.o \
15 gator_events_perf_pmu.o
14 16
15gator-y += gator_events_mmaped.o 17gator-y += gator_events_mmaped.o
16 18
diff --git a/driver/gator.h b/driver/gator.h
index 724ae19..a7a323c 100644
--- a/driver/gator.h
+++ b/driver/gator.h
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -14,15 +14,25 @@
14#include <linux/mm.h> 14#include <linux/mm.h>
15#include <linux/list.h> 15#include <linux/list.h>
16 16
17#define GATOR_PERF_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
18#define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && defined(CONFIG_HW_PERF_EVENTS)
19#define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT))
20#define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ)
21
17// cpu ids 22// cpu ids
18#define ARM1136 0xb36 23#define ARM1136 0xb36
19#define ARM1156 0xb56 24#define ARM1156 0xb56
20#define ARM1176 0xb76 25#define ARM1176 0xb76
21#define ARM11MPCORE 0xb02 26#define ARM11MPCORE 0xb02
22#define CORTEX_A5 0xc05 27#define CORTEX_A5 0xc05
28#define CORTEX_A7 0xc07
23#define CORTEX_A8 0xc08 29#define CORTEX_A8 0xc08
24#define CORTEX_A9 0xc09 30#define CORTEX_A9 0xc09
25#define CORTEX_A15 0xc0f 31#define CORTEX_A15 0xc0f
32#define SCORPION 0x00f
33#define SCORPIONMP 0x02d
34#define KRAITSIM 0x049
35#define KRAIT 0x04d
26 36
27/****************************************************************************** 37/******************************************************************************
28 * Filesystem 38 * Filesystem
@@ -69,8 +79,10 @@ struct gator_interface {
69 int (*create_files)(struct super_block *sb, struct dentry *root); 79 int (*create_files)(struct super_block *sb, struct dentry *root);
70 int (*start)(void); 80 int (*start)(void);
71 void (*stop)(void); 81 void (*stop)(void);
72 void (*online)(void); 82 int (*online)(int** buffer);
73 void (*offline)(void); 83 int (*offline)(int** buffer);
84 void (*online_dispatch)(int cpu); // called in process context but may not be running on core 'cpu'
85 void (*offline_dispatch)(int cpu); // called in process context but may not be running on core 'cpu'
74 int (*read)(int **buffer); 86 int (*read)(int **buffer);
75 int (*read64)(long long **buffer); 87 int (*read64)(long long **buffer);
76 struct list_head list; 88 struct list_head list;
@@ -84,6 +96,4 @@ int gator_events_install(struct gator_interface *interface);
84int gator_events_get_key(void); 96int gator_events_get_key(void);
85extern u32 gator_cpuid(void); 97extern u32 gator_cpuid(void);
86 98
87extern unsigned long gator_net_traffic;
88
89#endif // GATOR_H_ 99#endif // GATOR_H_
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index ee5a160..36a921c 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_annotate_kernel.c b/driver/gator_annotate_kernel.c
index 4f690e8..ffab087 100644
--- a/driver/gator_annotate_kernel.c
+++ b/driver/gator_annotate_kernel.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved. 2 * Copyright (C) ARM Limited 2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -25,7 +25,6 @@ static void kannotate_write(char* ptr, unsigned int size)
25// String annotation 25// String annotation
26void gator_annotate(char* string) 26void gator_annotate(char* string)
27{ 27{
28 printk(KERN_ERR "module: %s\n", string);
29 kannotate_write(string, strlen(string) + 1); 28 kannotate_write(string, strlen(string) + 1);
30} 29}
31EXPORT_SYMBOL(gator_annotate); 30EXPORT_SYMBOL(gator_annotate);
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index fc81233..26503ef 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index c0c4b9c..1beb34f 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -250,12 +250,15 @@ static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task
250 // Can be called from interrupt handler or from work queue or from scheduler trace 250 // Can be called from interrupt handler or from work queue or from scheduler trace
251 local_irq_save(flags); 251 local_irq_save(flags);
252 252
253 cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids; 253 cookie = INVALID_COOKIE;
254 cookiemap_add(key, cookie); 254 if (buffer_check_space(cpu, buftype, strlen(text) + 2 * MAXSIZE_PACK32)) {
255 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
256 cookiemap_add(key, cookie);
255 257
256 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); 258 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE);
257 gator_buffer_write_packed_int(cpu, buftype, cookie); 259 gator_buffer_write_packed_int(cpu, buftype, cookie);
258 gator_buffer_write_string(cpu, buftype, text); 260 gator_buffer_write_string(cpu, buftype, text);
261 }
259 262
260 local_irq_restore(flags); 263 local_irq_restore(flags);
261 264
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c
index 8b4b5ff..8c2997c 100644
--- a/driver/gator_ebs.c
+++ b/driver/gator_ebs.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved. 2 * Copyright (C) ARM Limited 2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -7,171 +7,186 @@
7 * 7 *
8 */ 8 */
9 9
10/****************************************************************************** 10#if defined(__arm__) && (GATOR_PERF_PMU_SUPPORT)
11 * event based sampling handling
12 ******************************************************************************/
13
14#if defined (__arm__)
15#include "gator_events_armv7.h"
16#include <linux/platform_device.h> 11#include <linux/platform_device.h>
17#include <linux/interrupt.h> 12#include <linux/interrupt.h>
18#include <linux/irq.h> 13#include <linux/irq.h>
19 14
20#if LINUX_PMU_SUPPORT
21#include <asm/pmu.h> 15#include <asm/pmu.h>
22 16
23static struct platform_device *pmu_device; 17extern int pmnc_counters;
18extern int ccnt;
19extern unsigned long pmnc_enabled[];
20extern unsigned long pmnc_event[];
21extern unsigned long pmnc_count[];
22extern unsigned long pmnc_key[];
23
24static DEFINE_PER_CPU(struct perf_event *, pevent);
25static DEFINE_PER_CPU(struct perf_event_attr *, pevent_attr);
26static DEFINE_PER_CPU(int, key);
27static DEFINE_PER_CPU(unsigned int, prev_value);
24 28
25static irqreturn_t armv7_pmnc_interrupt(int irq, void *arg) 29#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
30static void ebs_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
31#else
32static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
33#endif
26{ 34{
27 unsigned int cnt, cpu = smp_processor_id(), buftype = EVENT_BUF; 35 unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF;
28 struct pt_regs * const regs = get_irq_regs();
29 u32 flags;
30
31 // Stop irq generation
32 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
33
34 // Get and reset overflow status flags
35 flags = armv7_pmnc_reset_interrupt();
36
37 // Counters header
38 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
39 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
40
41 // Cycle counter
42 if (flags & (1 << 31)) {
43 int value = armv7_ccnt_read(pmnc_count[CCNT]); // overrun
44 gator_buffer_write_packed_int(cpu, buftype, 2); // length
45 gator_buffer_write_packed_int(cpu, buftype, pmnc_key[CCNT]); // key
46 gator_buffer_write_packed_int(cpu, buftype, value); // value
47 }
48 36
49 // PMNC counters 37 if (event != per_cpu(pevent, cpu))
50 for (cnt = CNT0; cnt < CNTMAX; cnt++) { 38 return;
51 if (flags & (1 << (cnt - CNT0))) {
52 int value = armv7_cntn_read(cnt, pmnc_count[cnt]); // overrun
53 gator_buffer_write_packed_int(cpu, buftype, 2); // length
54 gator_buffer_write_packed_int(cpu, buftype, pmnc_key[cnt]); // key
55 gator_buffer_write_packed_int(cpu, buftype, value); // value
56 }
57 }
58 39
59 // End Counters, length of zero 40 if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
60 gator_buffer_write_packed_int(cpu, buftype, 0); 41 value = local64_read(&event->count);
42 delta = value - per_cpu(prev_value, cpu);
43 per_cpu(prev_value, cpu) = value;
44
45 // Counters header
46 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
47 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
48
49 // Output counter
50 gator_buffer_write_packed_int(cpu, buftype, 2); // length
51 gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
52 gator_buffer_write_packed_int(cpu, buftype, delta); // delta
53
54 // End Counters, length of zero
55 gator_buffer_write_packed_int(cpu, buftype, 0);
56 }
61 57
62 // Output backtrace 58 // Output backtrace
63 gator_add_sample(cpu, buftype, regs); 59 if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
60 gator_add_sample(cpu, buftype, regs);
64 61
65 // Check and commit; commit is set to occur once buffer is 3/4 full 62 // Check and commit; commit is set to occur once buffer is 3/4 full
66 event_buffer_check(cpu); 63 buffer_check(cpu, buftype);
64}
65
66static void gator_event_sampling_online(void)
67{
68 int cpu = smp_processor_id(), buftype = EVENT_BUF;
67 69
68 // Allow irq generation 70 // read the counter and toss the invalid data, return zero instead
69 armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); 71 struct perf_event * ev = per_cpu(pevent, cpu);
72 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
73 ev->pmu->read(ev);
74 per_cpu(prev_value, cpu) = local64_read(&ev->count);
70 75
71 return IRQ_HANDLED; 76 // Counters header
77 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
78 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
79
80 // Output counter
81 gator_buffer_write_packed_int(cpu, buftype, 2); // length
82 gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
83 gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization
84
85 // End Counters, length of zero
86 gator_buffer_write_packed_int(cpu, buftype, 0);
87 }
72} 88}
89
90static void gator_event_sampling_online_dispatch(int cpu)
91{
92 struct perf_event * ev;
93
94 if (!event_based_sampling)
95 return;
96
97#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
98 ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler);
99#else
100 ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler, 0);
73#endif 101#endif
74 102
103 if (IS_ERR(ev)) {
104 pr_err("gator: unable to start event-based-sampling");
105 return;
106 }
107
108 if (ev->state != PERF_EVENT_STATE_ACTIVE) {
109 pr_err("gator: unable to start event-based-sampling");
110 perf_event_release_kernel(ev);
111 return;
112 }
113
114 ev->pmu->read(ev);
115 per_cpu(prev_value, cpu) = local64_read(&ev->count);
116}
117
118static void gator_event_sampling_offline_dispatch(int cpu)
119{
120 if (per_cpu(pevent, cpu)) {
121 perf_event_release_kernel(per_cpu(pevent, cpu));
122 per_cpu(pevent, cpu) = NULL;
123 }
124}
125
75static int gator_event_sampling_start(void) 126static int gator_event_sampling_start(void)
76{ 127{
77 int cnt; 128 int cnt, event = 0, count = 0, ebs_key = 0, cpu;
129
130 for_each_present_cpu(cpu) {
131 per_cpu(pevent, cpu) = NULL;
132 per_cpu(pevent_attr, cpu) = NULL;
133 }
78 134
79 event_based_sampling = false; 135 event_based_sampling = false;
80 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 136 for (cnt = 0; cnt < pmnc_counters; cnt++) {
81 if (pmnc_count[cnt] > 0) { 137 if (pmnc_count[cnt] > 0) {
82 event_based_sampling = true; 138 event_based_sampling = true;
139 event = pmnc_event[cnt];
140 count = pmnc_count[cnt];
141 ebs_key = pmnc_key[cnt];
83 break; 142 break;
84 } 143 }
85 } 144 }
86 145
87#if LINUX_PMU_SUPPORT 146 if (!event_based_sampling)
88 pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU); 147 return 0;
89 if (IS_ERR(pmu_device) && (unsigned int)pmu_device != -ENODEV) {
90 pr_err("gator: unable to reserve the pmu\n");
91 return -1;
92 }
93
94 if (event_based_sampling) {
95 int irq, i;
96 148
97 if (IS_ERR(pmu_device)) { 149 for_each_present_cpu(cpu) {
98 pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed\n"); 150 u32 size = sizeof(struct perf_event_attr);
151 per_cpu(pevent_attr, cpu) = kmalloc(size, GFP_KERNEL);
152 if (!per_cpu(pevent_attr, cpu))
99 return -1; 153 return -1;
100 }
101 154
102 // init_pmu sets the irq affinity, therefore we do not care if it fails for single core 155 memset(per_cpu(pevent_attr, cpu), 0, size);
103 if (init_pmu(ARM_PMU_DEVICE_CPU) != 0 && gator_cpu_cores > 1) { 156 per_cpu(pevent_attr, cpu)->type = PERF_TYPE_RAW;
104 pr_err("gator: unable to initialize the pmu\n"); 157 per_cpu(pevent_attr, cpu)->size = size;
105 goto out_ebs_start; 158 per_cpu(pevent_attr, cpu)->config = event;
159 per_cpu(pevent_attr, cpu)->sample_period = count;
160 per_cpu(pevent_attr, cpu)->pinned = 1;
161
162 // handle special case for ccnt
163 if (cnt == ccnt) {
164 per_cpu(pevent_attr, cpu)->type = PERF_TYPE_HARDWARE;
165 per_cpu(pevent_attr, cpu)->config = PERF_COUNT_HW_CPU_CYCLES;
106 } 166 }
107 167
108 if (pmu_device->num_resources == 0) { 168 per_cpu(key, cpu) = ebs_key;
109 pr_err("gator: no irqs for PMUs defined\n");
110 goto out_ebs_start;
111 }
112
113 for (i = 0; i < pmu_device->num_resources; ++i) {
114 irq = platform_get_irq(pmu_device, i);
115 if (irq < 0)
116 continue;
117
118 if (request_irq(irq, armv7_pmnc_interrupt, IRQF_DISABLED | IRQF_NOBALANCING, "armpmu", NULL)) {
119 pr_err("gator: unable to request IRQ%d for ARM perf counters\n", irq);
120
121 // clean up and exit
122 for (i = i - 1; i >= 0; --i) {
123 irq = platform_get_irq(pmu_device, i);
124 if (irq >= 0)
125 free_irq(irq, NULL);
126 }
127 goto out_ebs_start;
128 }
129 }
130 } 169 }
131#else
132 if (event_based_sampling) {
133 pr_err("gator: event based sampling only supported in kernel versions 2.6.35 and higher and CONFIG_CPU_HAS_PMU=y\n");
134 return -1;
135 }
136#endif
137 170
138 return 0; 171 return 0;
139
140#if LINUX_PMU_SUPPORT
141out_ebs_start:
142#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
143 release_pmu(pmu_device);
144#else
145 release_pmu(ARM_PMU_DEVICE_CPU);
146#endif
147 pmu_device = NULL;
148 return -1;
149#endif
150} 172}
151 173
152static void gator_event_sampling_stop(void) 174static void gator_event_sampling_stop(void)
153{ 175{
154#if LINUX_PMU_SUPPORT 176 int cpu;
155 if (event_based_sampling) { 177
156 int i, irq; 178 for_each_present_cpu(cpu) {
157 for (i = pmu_device->num_resources - 1; i >= 0; --i) { 179 if (per_cpu(pevent_attr, cpu)) {
158 irq = platform_get_irq(pmu_device, i); 180 kfree(per_cpu(pevent_attr, cpu));
159 if (irq >= 0) 181 per_cpu(pevent_attr, cpu) = NULL;
160 free_irq(irq, NULL);
161 } 182 }
162 } 183 }
163 if (!IS_ERR(pmu_device)) {
164#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
165 release_pmu(pmu_device);
166#else
167 release_pmu(ARM_PMU_DEVICE_CPU);
168#endif
169 }
170 pmu_device = NULL;
171#endif
172} 184}
173 185
174#else 186#else
187static void gator_event_sampling_online(void) {}
188static void gator_event_sampling_online_dispatch(int cpu) {}
189static void gator_event_sampling_offline_dispatch(int cpu) {}
175static int gator_event_sampling_start(void) {return 0;} 190static int gator_event_sampling_start(void) {return 0;}
176static void gator_event_sampling_stop(void) {} 191static void gator_event_sampling_stop(void) {}
177#endif 192#endif
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c
index 170f066..ef51898 100644
--- a/driver/gator_events_armv6.c
+++ b/driver/gator_events_armv6.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -8,6 +8,9 @@
8 8
9#include "gator.h" 9#include "gator.h"
10 10
11// gator_events_perf_pmu.c is used if perf is supported
12#if GATOR_NO_PERF_SUPPORT
13
11static const char *pmnc_name; 14static const char *pmnc_name;
12 15
13/* 16/*
@@ -28,7 +31,6 @@ static const char *pmnc_name;
28static int pmnc_counters = 0; 31static int pmnc_counters = 0;
29static unsigned long pmnc_enabled[CNTMAX]; 32static unsigned long pmnc_enabled[CNTMAX];
30static unsigned long pmnc_event[CNTMAX]; 33static unsigned long pmnc_event[CNTMAX];
31static unsigned long pmnc_count[CNTMAX];
32static unsigned long pmnc_key[CNTMAX]; 34static unsigned long pmnc_key[CNTMAX];
33 35
34static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 36static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -83,7 +85,6 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
83 return -1; 85 return -1;
84 } 86 }
85 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 87 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
86 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
87 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); 88 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
88 if (i != CCNT) { 89 if (i != CCNT) {
89 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 90 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
@@ -93,9 +94,9 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
93 return 0; 94 return 0;
94} 95}
95 96
96static void gator_events_armv6_online(void) 97static int gator_events_armv6_online(int** buffer)
97{ 98{
98 unsigned int cnt; 99 unsigned int cnt, len = 0, cpu = smp_processor_id();
99 u32 pmnc; 100 u32 pmnc;
100 101
101 if (armv6_pmnc_read() & PMCR_E) { 102 if (armv6_pmnc_read() & PMCR_E) {
@@ -110,7 +111,7 @@ static void gator_events_armv6_online(void)
110 for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) { 111 for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) {
111 unsigned long event; 112 unsigned long event;
112 113
113 per_cpu(perfPrev, smp_processor_id())[cnt] = 0; 114 per_cpu(perfPrev, cpu)[cnt] = 0;
114 115
115 if (!pmnc_enabled[cnt]) 116 if (!pmnc_enabled[cnt])
116 continue; 117 continue;
@@ -128,9 +129,22 @@ static void gator_events_armv6_online(void)
128 armv6_pmnc_reset_counter(cnt); 129 armv6_pmnc_reset_counter(cnt);
129 } 130 }
130 armv6_pmnc_write(pmnc | PMCR_E); 131 armv6_pmnc_write(pmnc | PMCR_E);
132
133 // return zero values, no need to read as the counters were just reset
134 for (cnt = PMN0; cnt <= CCNT; cnt++) {
135 if (pmnc_enabled[cnt]) {
136 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
137 per_cpu(perfCnt, cpu)[len++] = 0;
138 }
139 }
140
141 if (buffer)
142 *buffer = per_cpu(perfCnt, cpu);
143
144 return len;
131} 145}
132 146
133static void gator_events_armv6_offline(void) 147static int gator_events_armv6_offline(int** buffer)
134{ 148{
135 unsigned int cnt; 149 unsigned int cnt;
136 150
@@ -138,18 +152,6 @@ static void gator_events_armv6_offline(void)
138 for (cnt = PMN0; cnt <= CCNT; cnt++) { 152 for (cnt = PMN0; cnt <= CCNT; cnt++) {
139 armv6_pmnc_reset_counter(cnt); 153 armv6_pmnc_reset_counter(cnt);
140 } 154 }
141}
142
143static int gator_events_armv6_start(void)
144{
145 int cnt;
146
147 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
148 if (pmnc_count[cnt] > 0) {
149 pr_err("gator: event based sampling not supported on ARM v6 architectures\n");
150 return -1;
151 }
152 }
153 155
154 return 0; 156 return 0;
155} 157}
@@ -161,7 +163,6 @@ static void gator_events_armv6_stop(void)
161 for (cnt = PMN0; cnt <= CCNT; cnt++) { 163 for (cnt = PMN0; cnt <= CCNT; cnt++) {
162 pmnc_enabled[cnt] = 0; 164 pmnc_enabled[cnt] = 0;
163 pmnc_event[cnt] = 0; 165 pmnc_event[cnt] = 0;
164 pmnc_count[cnt] = 0;
165 } 166 }
166} 167}
167 168
@@ -193,7 +194,6 @@ static int gator_events_armv6_read(int **buffer)
193 } 194 }
194 } 195 }
195 196
196 // update or discard
197 if (buffer) 197 if (buffer)
198 *buffer = per_cpu(perfCnt, cpu); 198 *buffer = per_cpu(perfCnt, cpu);
199 199
@@ -202,7 +202,6 @@ static int gator_events_armv6_read(int **buffer)
202 202
203static struct gator_interface gator_events_armv6_interface = { 203static struct gator_interface gator_events_armv6_interface = {
204 .create_files = gator_events_armv6_create_files, 204 .create_files = gator_events_armv6_create_files,
205 .start = gator_events_armv6_start,
206 .stop = gator_events_armv6_stop, 205 .stop = gator_events_armv6_stop,
207 .online = gator_events_armv6_online, 206 .online = gator_events_armv6_online,
208 .offline = gator_events_armv6_offline, 207 .offline = gator_events_armv6_offline,
@@ -229,10 +228,17 @@ int gator_events_armv6_init(void)
229 for (cnt = PMN0; cnt <= CCNT; cnt++) { 228 for (cnt = PMN0; cnt <= CCNT; cnt++) {
230 pmnc_enabled[cnt] = 0; 229 pmnc_enabled[cnt] = 0;
231 pmnc_event[cnt] = 0; 230 pmnc_event[cnt] = 0;
232 pmnc_count[cnt] = 0;
233 pmnc_key[cnt] = gator_events_get_key(); 231 pmnc_key[cnt] = gator_events_get_key();
234 } 232 }
235 233
236 return gator_events_install(&gator_events_armv6_interface); 234 return gator_events_install(&gator_events_armv6_interface);
237} 235}
236
238gator_events_init(gator_events_armv6_init); 237gator_events_init(gator_events_armv6_init);
238
239#else
240int gator_events_armv6_init(void)
241{
242 return -1;
243}
244#endif
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index e1434e2..cdf450f 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -14,15 +14,29 @@
14 */ 14 */
15 15
16#include "gator.h" 16#include "gator.h"
17#include "gator_events_armv7.h"
18 17
19const char *pmnc_name; 18// gator_events_perf_pmu.c is used if perf is supported
20int pmnc_counters; 19#if GATOR_NO_PERF_SUPPORT
21 20
22unsigned long pmnc_enabled[CNTMAX]; 21// Per-CPU PMNC: config reg
23unsigned long pmnc_event[CNTMAX]; 22#define PMNC_E (1 << 0) /* Enable all counters */
24unsigned long pmnc_count[CNTMAX]; 23#define PMNC_P (1 << 1) /* Reset all counters */
25unsigned long pmnc_key[CNTMAX]; 24#define PMNC_C (1 << 2) /* Cycle counter reset */
25#define PMNC_MASK 0x3f /* Mask for writable bits */
26
27// ccnt reg
28#define CCNT_REG (1 << 31)
29
30#define CCNT 0
31#define CNT0 1
32#define CNTMAX (6+1)
33
34static const char *pmnc_name;
35static int pmnc_counters;
36
37static unsigned long pmnc_enabled[CNTMAX];
38static unsigned long pmnc_event[CNTMAX];
39static unsigned long pmnc_key[CNTMAX];
26 40
27static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 41static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
28static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); 42static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
@@ -76,19 +90,13 @@ inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value)
76 return oldval; 90 return oldval;
77} 91}
78 92
79static inline void armv7_pmnc_enable_interrupt(unsigned int cnt)
80{
81 u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
82 asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
83}
84
85static inline void armv7_pmnc_disable_interrupt(unsigned int cnt) 93static inline void armv7_pmnc_disable_interrupt(unsigned int cnt)
86{ 94{
87 u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); 95 u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31);
88 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); 96 asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
89} 97}
90 98
91inline u32 armv7_pmnc_reset_interrupt() 99inline u32 armv7_pmnc_reset_interrupt(void)
92{ 100{
93 // Get and reset overflow status flags 101 // Get and reset overflow status flags
94 u32 flags; 102 u32 flags;
@@ -143,7 +151,6 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry
143 return -1; 151 return -1;
144 } 152 }
145 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 153 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
146 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
147 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); 154 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
148 if (i > 0) { 155 if (i > 0) {
149 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 156 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
@@ -153,10 +160,9 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry
153 return 0; 160 return 0;
154} 161}
155 162
156static void gator_events_armv7_online(void) 163static int gator_events_armv7_online(int** buffer)
157{ 164{
158 unsigned int cnt; 165 unsigned int cnt, len = 0, cpu = smp_processor_id();
159 int cpu = smp_processor_id();
160 166
161 if (armv7_pmnc_read() & PMNC_E) { 167 if (armv7_pmnc_read() & PMNC_E) {
162 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); 168 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
@@ -185,14 +191,10 @@ static void gator_events_armv7_online(void)
185 if (cnt != CCNT) 191 if (cnt != CCNT)
186 armv7_pmnc_write_evtsel(cnt, event); 192 armv7_pmnc_write_evtsel(cnt, event);
187 193
188 // Enable/disable interrupt 194 armv7_pmnc_disable_interrupt(cnt);
189 if (pmnc_count[cnt] > 0)
190 armv7_pmnc_enable_interrupt(cnt);
191 else
192 armv7_pmnc_disable_interrupt(cnt);
193 195
194 // Reset counter 196 // Reset counter
195 cnt ? armv7_cntn_read(cnt, pmnc_count[cnt]) : armv7_ccnt_read(pmnc_count[cnt]); 197 cnt ? armv7_cntn_read(cnt, 0) : armv7_ccnt_read(0);
196 198
197 // Enable counter 199 // Enable counter
198 armv7_pmnc_enable_counter(cnt); 200 armv7_pmnc_enable_counter(cnt);
@@ -200,12 +202,27 @@ static void gator_events_armv7_online(void)
200 202
201 // enable 203 // enable
202 armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); 204 armv7_pmnc_write(armv7_pmnc_read() | PMNC_E);
205
206 // return zero values, no need to read as the counters were just reset
207 for (cnt = 0; cnt < pmnc_counters; cnt++) {
208 if (pmnc_enabled[cnt]) {
209 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
210 per_cpu(perfCnt, cpu)[len++] = 0;
211 }
212 }
213
214 if (buffer)
215 *buffer = per_cpu(perfCnt, cpu);
216
217 return len;
203} 218}
204 219
205static void gator_events_armv7_offline(void) 220static int gator_events_armv7_offline(int** buffer)
206{ 221{
207 // disbale all counters, including PMCCNTR; overflow IRQs will not be signaled 222 // disbale all counters, including PMCCNTR; overflow IRQs will not be signaled
208 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); 223 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
224
225 return 0;
209} 226}
210 227
211static void gator_events_armv7_stop(void) 228static void gator_events_armv7_stop(void)
@@ -215,7 +232,6 @@ static void gator_events_armv7_stop(void)
215 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 232 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
216 pmnc_enabled[cnt] = 0; 233 pmnc_enabled[cnt] = 0;
217 pmnc_event[cnt] = 0; 234 pmnc_event[cnt] = 0;
218 pmnc_count[cnt] = 0;
219 } 235 }
220} 236}
221 237
@@ -224,11 +240,8 @@ static int gator_events_armv7_read(int **buffer)
224 int cnt, len = 0; 240 int cnt, len = 0;
225 int cpu = smp_processor_id(); 241 int cpu = smp_processor_id();
226 242
227 if (!pmnc_counters)
228 return 0;
229
230 for (cnt = 0; cnt < pmnc_counters; cnt++) { 243 for (cnt = 0; cnt < pmnc_counters; cnt++) {
231 if (pmnc_enabled[cnt] && pmnc_count[cnt] == 0) { 244 if (pmnc_enabled[cnt]) {
232 int value; 245 int value;
233 if (cnt == CCNT) { 246 if (cnt == CCNT) {
234 value = armv7_ccnt_read(0); 247 value = armv7_ccnt_read(0);
@@ -243,7 +256,6 @@ static int gator_events_armv7_read(int **buffer)
243 } 256 }
244 } 257 }
245 258
246 // update or discard
247 if (buffer) 259 if (buffer)
248 *buffer = per_cpu(perfCnt, cpu); 260 *buffer = per_cpu(perfCnt, cpu);
249 261
@@ -267,6 +279,10 @@ int gator_events_armv7_init(void)
267 pmnc_name = "Cortex-A5"; 279 pmnc_name = "Cortex-A5";
268 pmnc_counters = 2; 280 pmnc_counters = 2;
269 break; 281 break;
282 case CORTEX_A7:
283 pmnc_name = "Cortex-A7";
284 pmnc_counters = 4;
285 break;
270 case CORTEX_A8: 286 case CORTEX_A8:
271 pmnc_name = "Cortex-A8"; 287 pmnc_name = "Cortex-A8";
272 pmnc_counters = 4; 288 pmnc_counters = 4;
@@ -288,10 +304,17 @@ int gator_events_armv7_init(void)
288 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 304 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
289 pmnc_enabled[cnt] = 0; 305 pmnc_enabled[cnt] = 0;
290 pmnc_event[cnt] = 0; 306 pmnc_event[cnt] = 0;
291 pmnc_count[cnt] = 0;
292 pmnc_key[cnt] = gator_events_get_key(); 307 pmnc_key[cnt] = gator_events_get_key();
293 } 308 }
294 309
295 return gator_events_install(&gator_events_armv7_interface); 310 return gator_events_install(&gator_events_armv7_interface);
296} 311}
312
297gator_events_init(gator_events_armv7_init); 313gator_events_init(gator_events_armv7_init);
314
315#else
316int gator_events_armv7_init(void)
317{
318 return -1;
319}
320#endif
diff --git a/driver/gator_events_armv7.h b/driver/gator_events_armv7.h
deleted file mode 100644
index d5e8d6e..0000000
--- a/driver/gator_events_armv7.h
+++ /dev/null
@@ -1,38 +0,0 @@
1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#ifndef GATOR_EVENTS_ARMV7_H_
10#define GATOR_EVENTS_ARMV7_H_
11
12// Per-CPU PMNC: config reg
13#define PMNC_E (1 << 0) /* Enable all counters */
14#define PMNC_P (1 << 1) /* Reset all counters */
15#define PMNC_C (1 << 2) /* Cycle counter reset */
16#define PMNC_MASK 0x3f /* Mask for writable bits */
17
18// ccnt reg
19#define CCNT_REG (1 << 31)
20
21#define CCNT 0
22#define CNT0 1
23#define CNTMAX (6+1)
24
25// Function prototypes
26extern void armv7_pmnc_write(u32 val);
27extern u32 armv7_pmnc_read(void);
28extern u32 armv7_ccnt_read(u32 reset_value);
29extern u32 armv7_cntn_read(unsigned int cnt, u32 reset_value);
30extern u32 armv7_pmnc_reset_interrupt(void);
31
32// Externed variables
33extern unsigned long pmnc_enabled[CNTMAX];
34extern unsigned long pmnc_event[CNTMAX];
35extern unsigned long pmnc_count[CNTMAX];
36extern unsigned long pmnc_key[CNTMAX];
37
38#endif // GATOR_EVENTS_ARMV7_H_
diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c
index cbfed86..f1bbbc8 100644
--- a/driver/gator_events_block.c
+++ b/driver/gator_events_block.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c
index 36a6589..59461b9 100644
--- a/driver/gator_events_irq.c
+++ b/driver/gator_events_irq.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -72,15 +72,38 @@ static int gator_events_irq_create_files(struct super_block *sb, struct dentry *
72 return 0; 72 return 0;
73} 73}
74 74
75static int gator_events_irq_start(void) 75static int gator_events_irq_online(int** buffer)
76{ 76{
77 int cpu, i; 77 int len = 0, cpu = smp_processor_id();
78 unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt
79
80 // synchronization with the irq_exit functions is not necessary as the values are being reset
81 if (hardirq_enabled) {
82 local_irq_save(flags);
83 per_cpu(irqCnt, cpu)[HARDIRQ] = 0;
84 local_irq_restore(flags);
85 per_cpu(irqPrev, cpu)[HARDIRQ] = 0;
86 per_cpu(irqGet, cpu)[len++] = hardirq_key;
87 per_cpu(irqGet, cpu)[len++] = 0;
88 }
78 89
79 for_each_present_cpu(cpu) { 90 if (softirq_enabled) {
80 for (i = 0; i < TOTALIRQ; i++) 91 local_irq_save(flags);
81 per_cpu(irqPrev, cpu)[i] = 0; 92 per_cpu(irqCnt, cpu)[SOFTIRQ] = 0;
93 local_irq_restore(flags);
94 per_cpu(irqPrev, cpu)[SOFTIRQ] = 0;
95 per_cpu(irqGet, cpu)[len++] = softirq_key;
96 per_cpu(irqGet, cpu)[len++] = 0;
82 } 97 }
83 98
99 if (buffer)
100 *buffer = per_cpu(irqGet, cpu);
101
102 return len;
103}
104
105static int gator_events_irq_start(void)
106{
84 // register tracepoints 107 // register tracepoints
85 if (hardirq_enabled) 108 if (hardirq_enabled)
86 if (GATOR_REGISTER_TRACE(irq_handler_exit)) 109 if (GATOR_REGISTER_TRACE(irq_handler_exit))
@@ -116,7 +139,7 @@ static void gator_events_irq_stop(void)
116 139
117static int gator_events_irq_read(int **buffer) 140static int gator_events_irq_read(int **buffer)
118{ 141{
119 unsigned long flags; 142 unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt
120 int len, value; 143 int len, value;
121 int cpu = smp_processor_id(); 144 int cpu = smp_processor_id();
122 145
@@ -153,6 +176,7 @@ static int gator_events_irq_read(int **buffer)
153 176
154static struct gator_interface gator_events_irq_interface = { 177static struct gator_interface gator_events_irq_interface = {
155 .create_files = gator_events_irq_create_files, 178 .create_files = gator_events_irq_create_files,
179 .online = gator_events_irq_online,
156 .start = gator_events_irq_start, 180 .start = gator_events_irq_start,
157 .stop = gator_events_irq_stop, 181 .stop = gator_events_irq_stop,
158 .read = gator_events_irq_read, 182 .read = gator_events_irq_read,
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c
index 96683b3..bd1c48a 100644
--- a/driver/gator_events_l2c-310.c
+++ b/driver/gator_events_l2c-310.c
@@ -1,7 +1,7 @@
1/** 1/**
2 * l2c310 (L2 Cache Controller) event counters for gator 2 * l2c310 (L2 Cache Controller) event counters for gator
3 * 3 *
4 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 4 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -148,10 +148,7 @@ int gator_events_l2c310_init(void)
148 if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) 148 if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9)
149 return -1; 149 return -1;
150 150
151#if defined(CONFIG_ARCH_EXYNOS4) 151#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310)
152 gator_events_l2c310_probe(0xfe600000);
153#endif
154#if defined(CONFIG_ARCH_S5PV310)
155 gator_events_l2c310_probe(0x10502000); 152 gator_events_l2c310_probe(0x10502000);
156#endif 153#endif
157#if defined(CONFIG_ARCH_OMAP4) 154#if defined(CONFIG_ARCH_OMAP4)
diff --git a/driver/gator_events_mali.c b/driver/gator_events_mali.c
new file mode 100644
index 0000000..31e8f0d
--- /dev/null
+++ b/driver/gator_events_mali.c
@@ -0,0 +1,611 @@
1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include "gator.h"
10
11#include <linux/module.h>
12#include <linux/time.h>
13#include <linux/math64.h>
14
15#include "linux/mali_linux_trace.h"
16
17#define ACTIVITY_START 1
18#define ACTIVITY_STOP 2
19
20#ifndef MALI_SUPPORT
21#error MALI_SUPPORT not defined!
22#endif
23
24#define MALI_200 0x0a07
25#define MALI_300 0x0b06 //This is not actually true; Mali-300 is also 0x0b07
26#define MALI_400 0x0b07
27#define MALI_T6xx 0x0056
28
29static const char *mali_name;
30
31enum counters {
32 /* Timeline activity */
33 ACTIVITY_VP = 0,
34 ACTIVITY_FP0,
35 ACTIVITY_FP1,
36 ACTIVITY_FP2,
37 ACTIVITY_FP3,
38
39 /* L2 cache counters */
40 COUNTER_L2_C0,
41 COUNTER_L2_C1,
42
43 /* Vertex processor counters */
44 COUNTER_VP_C0,
45 COUNTER_VP_C1,
46
47 /* Fragment processor counters */
48 COUNTER_FP0_C0,
49 COUNTER_FP0_C1,
50 COUNTER_FP1_C0,
51 COUNTER_FP1_C1,
52 COUNTER_FP2_C0,
53 COUNTER_FP2_C1,
54 COUNTER_FP3_C0,
55 COUNTER_FP3_C1,
56
57 /* EGL Software Counters */
58 COUNTER_EGL_BLIT_TIME,
59
60 /* GLES Software Counters */
61 COUNTER_GLES_DRAW_ELEMENTS_CALLS,
62 COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES,
63 COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED,
64 COUNTER_GLES_DRAW_ARRAYS_CALLS,
65 COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED,
66 COUNTER_GLES_DRAW_POINTS,
67 COUNTER_GLES_DRAW_LINES,
68 COUNTER_GLES_DRAW_LINE_LOOP,
69 COUNTER_GLES_DRAW_LINE_STRIP,
70 COUNTER_GLES_DRAW_TRIANGLES,
71 COUNTER_GLES_DRAW_TRIANGLE_STRIP,
72 COUNTER_GLES_DRAW_TRIANGLE_FAN,
73 COUNTER_GLES_NON_VBO_DATA_COPY_TIME,
74 COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI,
75 COUNTER_GLES_UPLOAD_TEXTURE_TIME,
76 COUNTER_GLES_UPLOAD_VBO_TIME,
77 COUNTER_GLES_NUM_FLUSHES,
78 COUNTER_GLES_NUM_VSHADERS_GENERATED,
79 COUNTER_GLES_NUM_FSHADERS_GENERATED,
80 COUNTER_GLES_VSHADER_GEN_TIME,
81 COUNTER_GLES_FSHADER_GEN_TIME,
82 COUNTER_GLES_INPUT_TRIANGLES,
83 COUNTER_GLES_VXCACHE_HIT,
84 COUNTER_GLES_VXCACHE_MISS,
85 COUNTER_GLES_VXCACHE_COLLISION,
86 COUNTER_GLES_CULLED_TRIANGLES,
87 COUNTER_GLES_CULLED_LINES,
88 COUNTER_GLES_BACKFACE_TRIANGLES,
89 COUNTER_GLES_GBCLIP_TRIANGLES,
90 COUNTER_GLES_GBCLIP_LINES,
91 COUNTER_GLES_TRIANGLES_DRAWN,
92 COUNTER_GLES_DRAWCALL_TIME,
93 COUNTER_GLES_TRIANGLES_COUNT,
94 COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT,
95 COUNTER_GLES_STRIP_TRIANGLES_COUNT,
96 COUNTER_GLES_FAN_TRIANGLES_COUNT,
97 COUNTER_GLES_LINES_COUNT,
98 COUNTER_GLES_INDEPENDENT_LINES_COUNT,
99 COUNTER_GLES_STRIP_LINES_COUNT,
100 COUNTER_GLES_LOOP_LINES_COUNT,
101
102 COUNTER_FILMSTRIP,
103
104 NUMBER_OF_EVENTS
105};
106
107#define FIRST_ACTIVITY_EVENT ACTIVITY_VP
108#define LAST_ACTIVITY_EVENT ACTIVITY_FP3
109
110#define FIRST_HW_COUNTER COUNTER_L2_C0
111#define LAST_HW_COUNTER COUNTER_FP3_C1
112
113#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME
114#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT
115
116#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP
117#define LAST_SPECIAL_COUNTER COUNTER_FILMSTRIP
118
119/* gatorfs variables for counter enable state,
120 * the event the counter should count and the
121 * 'key' (a unique id set by gatord and returned
122 * by gator.ko)
123 */
124static unsigned long counter_enabled[NUMBER_OF_EVENTS];
125static unsigned long counter_event[NUMBER_OF_EVENTS];
126static unsigned long counter_key[NUMBER_OF_EVENTS];
127
128/* The data we have recorded */
129static u32 counter_data[NUMBER_OF_EVENTS];
130/* The address to sample (or 0 if samples are sent to us) */
131static u32* counter_address[NUMBER_OF_EVENTS];
132
133/* An array used to return the data we recorded
134 * as key,value pairs hence the *2
135 */
136static unsigned long counter_dump[NUMBER_OF_EVENTS * 2];
137static unsigned long counter_prev[NUMBER_OF_EVENTS];
138
139/* Note whether tracepoints have been registered */
140static int trace_registered;
141
142/**
143 * Calculate the difference and handle the overflow.
144 */
145static u32 get_difference(u32 start, u32 end)
146{
147 if (start - end >= 0)
148 {
149 return start - end;
150 }
151
152 // Mali counters are unsigned 32 bit values that wrap.
153 return (4294967295u - end) + start;
154}
155
156/**
157 * Returns non-zero if the given counter ID is an activity counter.
158 */
159static inline int is_activity_counter(unsigned int event_id)
160{
161 return (event_id >= FIRST_ACTIVITY_EVENT &&
162 event_id <= LAST_ACTIVITY_EVENT);
163}
164
165/**
166 * Returns non-zero if the given counter ID is a hardware counter.
167 */
168static inline int is_hw_counter(unsigned int event_id)
169{
170 return (event_id >= FIRST_HW_COUNTER && event_id <= LAST_HW_COUNTER);
171}
172
173/**
174 * Returns non-zero if the given counter ID is a software counter.
175 */
176static inline int is_sw_counter(unsigned int event_id)
177{
178 return (event_id >= FIRST_SW_COUNTER && event_id <= LAST_SW_COUNTER);
179}
180
181/*
182 * The Mali DDK uses s64 types to contain software counter values, but gator
183 * can only use a maximum of 32 bits. This function scales a software counter
184 * to an appopriate range.
185 */
186static u32 scale_sw_counter_value(unsigned int event_id, signed long long value)
187{
188 u32 scaled_value;
189
190 switch (event_id) {
191 case COUNTER_GLES_UPLOAD_TEXTURE_TIME:
192 case COUNTER_GLES_UPLOAD_VBO_TIME:
193 scaled_value = (u32)div_s64(value, 1000000);
194 break;
195 default:
196 scaled_value = (u32)value;
197 break;
198 }
199
200 return scaled_value;
201}
202
203/* Probe for continuously sampled counter */
204#if 0 //WE_DONT_CURRENTLY_USE_THIS_SO_SUPPRESS_WARNING
205GATOR_DEFINE_PROBE(mali_sample_address, TP_PROTO(unsigned int event_id, u32* addr))
206{
207 /* Turning on too many pr_debug statements in frequently called functions
208 * can cause stability and/or performance problems
209 */
210 //pr_debug("gator: mali_sample_address %d %d\n", event_id, addr);
211 if (event_id >= ACTIVITY_VP && event_id <= COUNTER_FP3_C1) {
212 counter_address[event_id] = addr;
213 }
214}
215#endif
216
217/* Probe for hardware counter events */
218GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int value))
219{
220 /* Turning on too many pr_debug statements in frequently called functions
221 * can cause stability and/or performance problems
222 */
223 //pr_debug("gator: mali_hw_counter %d %d\n", event_id, value);
224 if (is_hw_counter(event_id)) {
225 counter_data[event_id] = value;
226 }
227}
228
229GATOR_DEFINE_PROBE(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value))
230{
231 if (is_sw_counter(event_id)) {
232 counter_data[event_id] = scale_sw_counter_value(event_id, value);
233 }
234}
235
236//TODO need to work out how many fp units we have
237u32 gator_mali_get_n_fp(void) {
238 return 4;
239}
240
241//TODO need to work out what kind of Mali we are looking at
242u32 gator_mali_get_id(void) {
243 return MALI_SUPPORT;
244}
245
246int gator_events_mali_create_files(struct super_block *sb, struct dentry *root) {
247 struct dentry *dir;
248 int event;
249 int n_fp = gator_mali_get_n_fp();
250
251 /*
252 * Create the filesystem entries for vertex processor, fragement processor
253 * and L2 cache timeline and hardware counters. Software counters get
254 * special handling after this block.
255 */
256 for (event = FIRST_ACTIVITY_EVENT; event <= LAST_HW_COUNTER; event++)
257 {
258 char buf[40];
259
260 /*
261 * We can skip this event if it's for a non-existent fragment
262 * processor.
263 */
264 if (((event - ACTIVITY_FP0 >= n_fp) && (event < COUNTER_L2_C0)) ||
265 (((event - COUNTER_FP0_C0)/2 >= n_fp)))
266 {
267 continue;
268 }
269
270 /* Otherwise, set up the filesystem entry for this event. */
271 switch (event) {
272 case ACTIVITY_VP:
273 snprintf(buf, sizeof buf, "ARM_%s_VP_active", mali_name);
274 break;
275 case ACTIVITY_FP0:
276 case ACTIVITY_FP1:
277 case ACTIVITY_FP2:
278 case ACTIVITY_FP3:
279 snprintf(buf, sizeof buf, "ARM_%s_FP%d_active",
280 mali_name, event - ACTIVITY_FP0);
281 break;
282 case COUNTER_L2_C0:
283 case COUNTER_L2_C1:
284 snprintf(buf, sizeof buf, "ARM_%s_L2_cnt%d",
285 mali_name, event - COUNTER_L2_C0);
286 break;
287 case COUNTER_VP_C0:
288 case COUNTER_VP_C1:
289 snprintf(buf, sizeof buf, "ARM_%s_VP_cnt%d",
290 mali_name, event - COUNTER_VP_C0);
291 break;
292 case COUNTER_FP0_C0:
293 case COUNTER_FP0_C1:
294 case COUNTER_FP1_C0:
295 case COUNTER_FP1_C1:
296 case COUNTER_FP2_C0:
297 case COUNTER_FP2_C1:
298 case COUNTER_FP3_C0:
299 case COUNTER_FP3_C1:
300 snprintf(buf, sizeof buf, "ARM_%s_FP%d_cnt%d", mali_name,
301 (event - COUNTER_FP0_C0) / 2, (event - COUNTER_FP0_C0) % 2);
302 break;
303 default:
304 printk("gator: trying to create file for non-existent counter (%d)\n", event);
305 continue;
306 }
307
308 dir = gatorfs_mkdir(sb, root, buf);
309
310 if (!dir) {
311 return -1;
312 }
313
314 gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]);
315
316 /* Only create an event node for counters that can change what they count */
317 if (event >= COUNTER_L2_C0) {
318 gatorfs_create_ulong(sb, dir, "event", &counter_event[event]);
319 }
320
321 gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]);
322 }
323
324 /* Now set up the software counter entries */
325 for (event = FIRST_SW_COUNTER; event <= LAST_SW_COUNTER; event++)
326 {
327 char buf[40];
328
329 snprintf(buf, sizeof(buf), "ARM_%s_SW_%d", mali_name, event);
330
331 dir = gatorfs_mkdir(sb, root, buf);
332
333 if (!dir) {
334 return -1;
335 }
336
337 gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]);
338 gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]);
339 }
340
341 /* Now set up the special counter entries */
342 for (event = FIRST_SPECIAL_COUNTER; event <= LAST_SPECIAL_COUNTER; event++)
343 {
344 char buf[40];
345
346 snprintf(buf, sizeof(buf), "ARM_%s_Filmstrip", mali_name);
347
348 dir = gatorfs_mkdir(sb, root, buf);
349
350 if (!dir) {
351 return -1;
352 }
353
354 gatorfs_create_ulong(sb, dir, "event", &counter_event[event]);
355 gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]);
356 gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]);
357 }
358
359
360 return 0;
361}
362
363//TODO
364void _mali_profiling_set_event(unsigned int, unsigned int);
365void _mali_osk_fb_control_set(unsigned int, unsigned int);
366
367void _mali_profiling_get_counters(unsigned int*, unsigned int*, unsigned int*, unsigned int*);
368void (*_mali_profiling_get_counters_function_pointer)(unsigned int*, unsigned int*, unsigned int*, unsigned int*);
369
370static void mali_counter_initialize(void)
371{
372 /* If a Mali driver is present and exporting the appropriate symbol
373 * then we can request the HW counters (of which there are only 2)
374 * be configured to count the desired events
375 */
376 void (*set_hw_event)(unsigned int, unsigned int);
377 void (*set_fb_event)(unsigned int, unsigned int);
378
379 set_hw_event = symbol_get(_mali_profiling_set_event);
380
381 if (set_hw_event) {
382 int i;
383
384 pr_debug("gator: mali online _mali_profiling_set_event symbol @ %p\n",set_hw_event);
385
386 for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) {
387 if (counter_enabled[i]) {
388 set_hw_event(i, counter_event[i]);
389 } else {
390 set_hw_event(i, 0xFFFFFFFF);
391 }
392 }
393
394 symbol_put(_mali_profiling_set_event);
395 } else {
396 printk("gator: mali online _mali_profiling_set_event symbol not found\n");
397 }
398
399 set_fb_event = symbol_get(_mali_osk_fb_control_set);
400
401 if (set_fb_event) {
402 pr_debug("gator: mali online _mali_osk_fb_control_set symbol @ %p\n", set_fb_event);
403
404 set_fb_event(0,(counter_enabled[COUNTER_FILMSTRIP]?1:0));
405
406 symbol_put(_mali_osk_fb_control_set);
407 } else {
408 printk("gator: mali online _mali_osk_fb_control_set symbol not found\n");
409 }
410
411 _mali_profiling_get_counters_function_pointer = symbol_get(_mali_profiling_get_counters);
412 if (_mali_profiling_get_counters_function_pointer){
413 pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", _mali_profiling_get_counters_function_pointer);
414 counter_prev[COUNTER_L2_C0] = 0;
415 counter_prev[COUNTER_L2_C1] = 0;
416 }
417 else{
418 pr_debug("gator WARNING: mali _mali_profiling_get_counters symbol not defined");
419 }
420
421}
422
423static void mali_counter_deinitialize(void)
424{
425 void (*set_hw_event)(unsigned int, unsigned int);
426 void (*set_fb_event)(unsigned int, unsigned int);
427
428 set_hw_event = symbol_get(_mali_profiling_set_event);
429
430 if (set_hw_event) {
431 int i;
432
433 pr_debug("gator: mali offline _mali_profiling_set_event symbol @ %p\n",set_hw_event);
434 for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) {
435 set_hw_event(i, 0xFFFFFFFF);
436 }
437
438 symbol_put(_mali_profiling_set_event);
439 } else {
440 printk("gator: mali offline _mali_profiling_set_event symbol not found\n");
441 }
442
443 set_fb_event = symbol_get(_mali_osk_fb_control_set);
444
445 if (set_fb_event) {
446 pr_debug("gator: mali offline _mali_osk_fb_control_set symbol @ %p\n", set_fb_event);
447
448 set_fb_event(0,0);
449
450 symbol_put(_mali_osk_fb_control_set);
451 } else {
452 printk("gator: mali offline _mali_osk_fb_control_set symbol not found\n");
453 }
454
455 if (_mali_profiling_get_counters_function_pointer){
456 symbol_put(_mali_profiling_get_counters);
457 }
458
459}
460
461static int gator_events_mali_start(void) {
462 // register tracepoints
463 if (GATOR_REGISTER_TRACE(mali_hw_counter)) {
464 printk("gator: mali_hw_counter tracepoint failed to activate\n");
465 return -1;
466 }
467
468 if (GATOR_REGISTER_TRACE(mali_sw_counter)) {
469 printk("gator: mali_sw_counter tracepoint failed to activate\n");
470 return -1;
471 }
472
473 trace_registered = 1;
474
475 mali_counter_initialize();
476 return 0;
477}
478
479static void gator_events_mali_stop(void) {
480 unsigned int cnt;
481
482 pr_debug("gator: mali stop\n");
483
484 if (trace_registered) {
485 GATOR_UNREGISTER_TRACE(mali_hw_counter);
486 GATOR_UNREGISTER_TRACE(mali_sw_counter);
487
488 pr_debug("gator: mali timeline tracepoint deactivated\n");
489
490 trace_registered = 0;
491 }
492
493 for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) {
494 counter_enabled[cnt] = 0;
495 counter_event[cnt] = 0;
496 counter_address[cnt] = NULL;
497 }
498
499 mali_counter_deinitialize();
500}
501
502static int gator_events_mali_read(int **buffer) {
503 int cnt, len = 0;
504
505 if (smp_processor_id()) return 0;
506
507 // Read the L2 C0 and C1 here.
508 if (counter_enabled[COUNTER_L2_C0] || counter_enabled[COUNTER_L2_C1] ) {
509 u32 src0 = 0;
510 u32 val0 = 0;
511 u32 src1 = 0;
512 u32 val1 = 0;
513
514 // Poke the driver to get the counter values
515 if (_mali_profiling_get_counters_function_pointer){
516 _mali_profiling_get_counters_function_pointer(&src0, &val0, &src1, &val1);
517 }
518
519 if (counter_enabled[COUNTER_L2_C0])
520 {
521 // Calculate and save src0's counter val0
522 counter_dump[len++] = counter_key[COUNTER_L2_C0];
523 counter_dump[len++] = get_difference(val0, counter_prev[COUNTER_L2_C0]);
524 }
525
526 if (counter_enabled[COUNTER_L2_C1])
527 {
528 // Calculate and save src1's counter val1
529 counter_dump[len++] = counter_key[COUNTER_L2_C1];
530 counter_dump[len++] = get_difference(val1, counter_prev[COUNTER_L2_C1]);
531 }
532
533 // Save the previous values for the counters.
534 counter_prev[COUNTER_L2_C0] = val0;
535 counter_prev[COUNTER_L2_C1] = val1;
536 }
537
538 // Process other (non-timeline) counters.
539 for (cnt = COUNTER_VP_C0; cnt <= LAST_SW_COUNTER; cnt++) {
540 if (counter_enabled[cnt]) {
541 u32 value = 0;
542
543 // Determine the current value of the counter.
544 if( counter_address[cnt] != NULL && 0 ) { // Never true!
545 value = *counter_address[cnt];
546 } else if (counter_data[cnt]!=0) {
547 value = counter_data[cnt];
548 counter_data[cnt] = 0;
549 }
550
551 // Send the counter value only if it differs from last time.
552 if (value != counter_prev[cnt]) {
553 counter_prev[cnt] = value;
554 counter_dump[len++] = counter_key[cnt];
555 counter_dump[len++] = value;
556 }
557 }
558 }
559
560 if (buffer) {
561 *buffer = (int*) counter_dump;
562 }
563
564 return len;
565}
566
567static struct gator_interface gator_events_mali_interface = {
568 .create_files = gator_events_mali_create_files,
569 .start = gator_events_mali_start,
570 .stop = gator_events_mali_stop,
571 .read = gator_events_mali_read,
572};
573
574int gator_events_mali_init(void)
575{
576 unsigned int cnt;
577 u32 id = gator_mali_get_id();
578
579 switch (id) {
580 case MALI_T6xx:
581 mali_name = "Mali-T6xx";
582 break;
583 case MALI_400:
584 mali_name = "Mali-400";
585 break;
586 case MALI_300:
587 mali_name = "Mali-300";
588 break;
589 case MALI_200:
590 mali_name = "Mali-200";
591 break;
592 default:
593 printk("Unknown Mali ID (%d)\n", id);
594 return -1;
595 }
596
597 pr_debug("gator: mali init\n");
598
599 for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) {
600 counter_enabled[cnt] = 0;
601 counter_event[cnt] = 0;
602 counter_key[cnt] = gator_events_get_key();
603 counter_address[cnt] = NULL;
604 counter_data[cnt] = 0;
605 }
606
607 trace_registered = 0;
608
609 return gator_events_install(&gator_events_mali_interface);
610}
611gator_events_init(gator_events_mali_init);
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c
index a1a2031..8af9cfc 100644
--- a/driver/gator_events_meminfo.c
+++ b/driver/gator_events_meminfo.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c
index 6884684..cbb22b1 100644
--- a/driver/gator_events_mmaped.c
+++ b/driver/gator_events_mmaped.c
@@ -1,13 +1,15 @@
1/* 1/*
2 * Example events provider 2 * Example events provider
3 * 3 *
4 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 4 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation. 8 * published by the Free Software Foundation.
9 * 9 *
10 * Similar entires must be present in events.xml file: 10 * Similar entries to those below must be present in the events.xml file.
11 * To add them to the events.xml, create an events-mmap.xml with the
12 * following contents and rebuild gatord:
11 * 13 *
12 * <counter_set name="mmaped_cntX"> 14 * <counter_set name="mmaped_cntX">
13 * <counter name="mmaped_cnt0"/> 15 * <counter name="mmaped_cnt0"/>
diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c
index a921586..ef1623b 100644
--- a/driver/gator_events_net.c
+++ b/driver/gator_events_net.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -12,13 +12,10 @@
12 12
13#define NETRX 0 13#define NETRX 0
14#define NETTX 1 14#define NETTX 1
15#define NETDRV 2 15#define TOTALNET 2
16#define TOTALNET (NETDRV+1)
17 16
18static ulong netdrv_enabled;
19static ulong netrx_enabled; 17static ulong netrx_enabled;
20static ulong nettx_enabled; 18static ulong nettx_enabled;
21static ulong netdrv_key;
22static ulong netrx_key; 19static ulong netrx_key;
23static ulong nettx_key; 20static ulong nettx_key;
24static int rx_total, tx_total; 21static int rx_total, tx_total;
@@ -44,19 +41,9 @@ static void get_network_stats(struct work_struct *wsptr) {
44} 41}
45DECLARE_WORK(wq_get_stats, get_network_stats); 42DECLARE_WORK(wq_get_stats, get_network_stats);
46 43
47static void calculate_delta(int *drv, int *rx, int *tx) 44static void calculate_delta(int *rx, int *tx)
48{ 45{
49 int drv_calc, rx_calc, tx_calc; 46 int rx_calc, tx_calc;
50
51 drv_calc = gator_net_traffic - netPrev[NETDRV];
52 if (drv_calc > 0) {
53 netPrev[NETDRV] += drv_calc;
54 netPrev[NETTX] += drv_calc;
55 // remove tcp/ip header overhead
56 // approximation based on empirical measurement
57 netPrev[NETRX] += drv_calc / 42;
58 netPrev[NETTX] += drv_calc / 18;
59 }
60 47
61 rx_calc = (int)(rx_total - netPrev[NETRX]); 48 rx_calc = (int)(rx_total - netPrev[NETRX]);
62 if (rx_calc < 0) 49 if (rx_calc < 0)
@@ -68,7 +55,6 @@ static void calculate_delta(int *drv, int *rx, int *tx)
68 tx_calc = 0; 55 tx_calc = 0;
69 netPrev[NETTX] += tx_calc; 56 netPrev[NETTX] += tx_calc;
70 57
71 *drv = drv_calc;
72 *rx = rx_calc; 58 *rx = rx_calc;
73 *tx = tx_calc; 59 *tx = tx_calc;
74} 60}
@@ -77,13 +63,6 @@ static int gator_events_net_create_files(struct super_block *sb, struct dentry *
77{ 63{
78 struct dentry *dir; 64 struct dentry *dir;
79 65
80 dir = gatorfs_mkdir(sb, root, "Linux_net_drv");
81 if (!dir) {
82 return -1;
83 }
84 gatorfs_create_ulong(sb, dir, "enabled", &netdrv_enabled);
85 gatorfs_create_ro_ulong(sb, dir, "key", &netdrv_key);
86
87 dir = gatorfs_mkdir(sb, root, "Linux_net_rx"); 66 dir = gatorfs_mkdir(sb, root, "Linux_net_rx");
88 if (!dir) { 67 if (!dir) {
89 return -1; 68 return -1;
@@ -104,7 +83,6 @@ static int gator_events_net_create_files(struct super_block *sb, struct dentry *
104static int gator_events_net_start(void) 83static int gator_events_net_start(void)
105{ 84{
106 get_network_stats(NULL); 85 get_network_stats(NULL);
107 netPrev[NETDRV] = 0;
108 netPrev[NETRX] = rx_total; 86 netPrev[NETRX] = rx_total;
109 netPrev[NETTX] = tx_total; 87 netPrev[NETTX] = tx_total;
110 return 0; 88 return 0;
@@ -112,29 +90,22 @@ static int gator_events_net_start(void)
112 90
113static void gator_events_net_stop(void) 91static void gator_events_net_stop(void)
114{ 92{
115 netdrv_enabled = 0;
116 netrx_enabled = 0; 93 netrx_enabled = 0;
117 nettx_enabled = 0; 94 nettx_enabled = 0;
118} 95}
119 96
120static int gator_events_net_read(int **buffer) 97static int gator_events_net_read(int **buffer)
121{ 98{
122 int len, drv_delta, rx_delta, tx_delta; 99 int len, rx_delta, tx_delta;
123 static int last_drv_delta = 0, last_rx_delta = 0, last_tx_delta = 0; 100 static int last_rx_delta = 0, last_tx_delta = 0;
124 101
125 if (smp_processor_id() != 0) 102 if (smp_processor_id() != 0)
126 return 0; 103 return 0;
127 104
128 schedule_work(&wq_get_stats); 105 schedule_work(&wq_get_stats);
129 calculate_delta(&drv_delta, &rx_delta, &tx_delta); 106 calculate_delta(&rx_delta, &tx_delta);
130 107
131 len = 0; 108 len = 0;
132 if (netdrv_enabled && last_drv_delta != drv_delta) {
133 last_drv_delta = drv_delta;
134 netGet[len++] = netdrv_key;
135 netGet[len++] = drv_delta;
136 }
137
138 if (netrx_enabled && last_rx_delta != rx_delta) { 109 if (netrx_enabled && last_rx_delta != rx_delta) {
139 last_rx_delta = rx_delta; 110 last_rx_delta = rx_delta;
140 netGet[len++] = netrx_key; 111 netGet[len++] = netrx_key;
@@ -162,13 +133,9 @@ static struct gator_interface gator_events_net_interface = {
162 133
163int gator_events_net_init(void) 134int gator_events_net_init(void)
164{ 135{
165 gator_net_traffic++;
166
167 netdrv_key = gator_events_get_key();
168 netrx_key = gator_events_get_key(); 136 netrx_key = gator_events_get_key();
169 nettx_key = gator_events_get_key(); 137 nettx_key = gator_events_get_key();
170 138
171 netdrv_enabled = 0;
172 netrx_enabled = 0; 139 netrx_enabled = 0;
173 nettx_enabled = 0; 140 nettx_enabled = 0;
174 141
diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c
new file mode 100644
index 0000000..322ebc4
--- /dev/null
+++ b/driver/gator_events_perf_pmu.c
@@ -0,0 +1,296 @@
1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include <linux/slab.h>
10#include <linux/perf_event.h>
11#include "gator.h"
12
13// gator_events_armvX.c is used for Linux 2.6.x
14#if GATOR_PERF_PMU_SUPPORT
15
16static const char *pmnc_name;
17int pmnc_counters;
18int ccnt = 0;
19
20#define CNTMAX (6+1)
21
22unsigned long pmnc_enabled[CNTMAX];
23unsigned long pmnc_event[CNTMAX];
24unsigned long pmnc_count[CNTMAX];
25unsigned long pmnc_key[CNTMAX];
26
27static DEFINE_PER_CPU(int[CNTMAX], perfCurr);
28static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
29static DEFINE_PER_CPU(int[CNTMAX], perfPrevDelta);
30static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
31static DEFINE_PER_CPU(struct perf_event *[CNTMAX], pevent);
32static DEFINE_PER_CPU(struct perf_event_attr *[CNTMAX], pevent_attr);
33
34static void gator_events_perf_pmu_stop(void);
35
36static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root)
37{
38 struct dentry *dir;
39 int i;
40
41 for (i = 0; i < pmnc_counters; i++) {
42 char buf[40];
43 if (i == 0) {
44 snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name);
45 } else {
46 snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i-1);
47 }
48 dir = gatorfs_mkdir(sb, root, buf);
49 if (!dir) {
50 return -1;
51 }
52 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
53 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
54 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
55 if (i > 0) {
56 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
57 }
58 }
59
60 return 0;
61}
62
63#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
64static void dummy_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
65#else
66static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
67#endif
68{
69// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll
70}
71
72static int gator_events_perf_pmu_online(int** buffer)
73{
74 int cnt, len = 0, cpu = smp_processor_id();
75
76 // read the counters and toss the invalid data, return zero instead
77 for (cnt = 0; cnt < pmnc_counters; cnt++) {
78 struct perf_event * ev = per_cpu(pevent, cpu)[cnt];
79 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
80 ev->pmu->read(ev);
81 per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count);
82 per_cpu(perfPrevDelta, cpu)[cnt] = 0;
83 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
84 per_cpu(perfCnt, cpu)[len++] = 0;
85 }
86 }
87
88 if (buffer)
89 *buffer = per_cpu(perfCnt, cpu);
90
91 return len;
92}
93
94static void gator_events_perf_pmu_online_dispatch(int cpu)
95{
96 int cnt;
97
98 for (cnt = 0; cnt < pmnc_counters; cnt++) {
99 if (per_cpu(pevent, cpu)[cnt] != NULL || per_cpu(pevent_attr, cpu)[cnt] == 0)
100 continue;
101
102#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
103 per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler);
104#else
105 per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler, 0);
106#endif
107 if (IS_ERR(per_cpu(pevent, cpu)[cnt])) {
108 pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
109 per_cpu(pevent, cpu)[cnt] = NULL;
110 continue;
111 }
112
113 if (per_cpu(pevent, cpu)[cnt]->state != PERF_EVENT_STATE_ACTIVE) {
114 pr_debug("gator: inactive counter on cpu %d\n", cpu);
115 perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]);
116 per_cpu(pevent, cpu)[cnt] = NULL;
117 continue;
118 }
119 }
120}
121
122static void gator_events_perf_pmu_offline_dispatch(int cpu)
123{
124 int cnt;
125
126 for (cnt = 0; cnt < pmnc_counters; cnt++) {
127 if (per_cpu(pevent, cpu)[cnt] != NULL) {
128 perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]);
129 per_cpu(pevent, cpu)[cnt] = NULL;
130 }
131 }
132}
133
134static int gator_events_perf_pmu_start(void)
135{
136 int cnt, cpu;
137 u32 size = sizeof(struct perf_event_attr);
138
139 for_each_present_cpu(cpu) {
140 for (cnt = 0; cnt < pmnc_counters; cnt++) {
141 per_cpu(pevent, cpu)[cnt] = NULL;
142 if (!pmnc_enabled[cnt] || pmnc_count[cnt] > 0) // Skip disabled counters and EBS counters
143 continue;
144
145 per_cpu(perfPrev, cpu)[cnt] = 0;
146 per_cpu(perfCurr, cpu)[cnt] = 0;
147 per_cpu(perfPrevDelta, cpu)[cnt] = 0;
148 per_cpu(pevent_attr, cpu)[cnt] = kmalloc(size, GFP_KERNEL);
149 if (!per_cpu(pevent_attr, cpu)[cnt]) {
150 gator_events_perf_pmu_stop();
151 return -1;
152 }
153
154 memset(per_cpu(pevent_attr, cpu)[cnt], 0, size);
155 per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_RAW;
156 per_cpu(pevent_attr, cpu)[cnt]->size = size;
157 per_cpu(pevent_attr, cpu)[cnt]->config = pmnc_event[cnt];
158 per_cpu(pevent_attr, cpu)[cnt]->sample_period = 0;
159 per_cpu(pevent_attr, cpu)[cnt]->pinned = 1;
160
161 // handle special case for ccnt
162 if (cnt == ccnt) {
163 per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_HARDWARE;
164 per_cpu(pevent_attr, cpu)[cnt]->config = PERF_COUNT_HW_CPU_CYCLES;
165 }
166 }
167 }
168
169 return 0;
170}
171
172static void gator_events_perf_pmu_stop(void)
173{
174 unsigned int cnt, cpu;
175
176 for_each_present_cpu(cpu) {
177 for (cnt = 0; cnt < pmnc_counters; cnt++) {
178 if (per_cpu(pevent_attr, cpu)[cnt]) {
179 kfree(per_cpu(pevent_attr, cpu)[cnt]);
180 per_cpu(pevent_attr, cpu)[cnt] = NULL;
181 }
182 }
183 }
184
185 for (cnt = 0; cnt < pmnc_counters; cnt++) {
186 pmnc_enabled[cnt] = 0;
187 pmnc_event[cnt] = 0;
188 pmnc_count[cnt] = 0;
189 }
190}
191
192static int gator_events_perf_pmu_read(int **buffer)
193{
194 int cnt, delta, len = 0;
195 int cpu = smp_processor_id();
196
197 for (cnt = 0; cnt < pmnc_counters; cnt++) {
198 struct perf_event * ev = per_cpu(pevent, cpu)[cnt];
199 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
200 ev->pmu->read(ev);
201 per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count);
202 delta = per_cpu(perfCurr, cpu)[cnt] - per_cpu(perfPrev, cpu)[cnt];
203 if (delta != per_cpu(perfPrevDelta, cpu)[cnt]) {
204 per_cpu(perfPrevDelta, cpu)[cnt] = delta;
205 per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt];
206 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
207 if (delta < 0)
208 delta *= -1;
209 per_cpu(perfCnt, cpu)[len++] = delta;
210 }
211 }
212 }
213
214 if (buffer)
215 *buffer = per_cpu(perfCnt, cpu);
216
217 return len;
218}
219
220static struct gator_interface gator_events_perf_pmu_interface = {
221 .create_files = gator_events_perf_pmu_create_files,
222 .start = gator_events_perf_pmu_start,
223 .stop = gator_events_perf_pmu_stop,
224 .online = gator_events_perf_pmu_online,
225 .online_dispatch = gator_events_perf_pmu_online_dispatch,
226 .offline_dispatch = gator_events_perf_pmu_offline_dispatch,
227 .read = gator_events_perf_pmu_read,
228};
229
230int gator_events_perf_pmu_init(void)
231{
232 unsigned int cnt;
233
234 switch (gator_cpuid()) {
235 case ARM1136:
236 case ARM1156:
237 case ARM1176:
238 pmnc_name = "ARM_ARM11";
239 pmnc_counters = 3;
240 ccnt = 2;
241 break;
242 case ARM11MPCORE:
243 pmnc_name = "ARM_ARM11MPCore";
244 pmnc_counters = 3;
245 break;
246 case CORTEX_A5:
247 pmnc_name = "ARM_Cortex-A5";
248 pmnc_counters = 2;
249 break;
250 case CORTEX_A7:
251 pmnc_name = "ARM_Cortex-A7";
252 pmnc_counters = 4;
253 break;
254 case CORTEX_A8:
255 pmnc_name = "ARM_Cortex-A8";
256 pmnc_counters = 4;
257 break;
258 case CORTEX_A9:
259 pmnc_name = "ARM_Cortex-A9";
260 pmnc_counters = 6;
261 break;
262 case CORTEX_A15:
263 pmnc_name = "ARM_Cortex-A15";
264 pmnc_counters = 6;
265 break;
266 case SCORPION:
267 pmnc_name = "Scorpion";
268 pmnc_counters = 4;
269 break;
270 case SCORPIONMP:
271 pmnc_name = "ScorpionMP";
272 pmnc_counters = 4;
273 break;
274 case KRAITSIM:
275 case KRAIT:
276 pmnc_name = "Krait";
277 pmnc_counters = 4;
278 break;
279 default:
280 return -1;
281 }
282
283 pmnc_counters++; // CNT[n] + CCNT
284
285 for (cnt = 0; cnt < CNTMAX; cnt++) {
286 pmnc_enabled[cnt] = 0;
287 pmnc_event[cnt] = 0;
288 pmnc_count[cnt] = 0;
289 pmnc_key[cnt] = gator_events_get_key();
290 }
291
292 return gator_events_install(&gator_events_perf_pmu_interface);
293}
294
295gator_events_init(gator_events_perf_pmu_init);
296#endif
diff --git a/driver/gator_events_power.c b/driver/gator_events_power.c
new file mode 100644
index 0000000..a0ae684
--- /dev/null
+++ b/driver/gator_events_power.c
@@ -0,0 +1,178 @@
1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#include "gator.h"
11#include <linux/slab.h>
12#include <linux/cpufreq.h>
13#include <trace/events/power.h>
14
15// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38
16// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86
17// CPU Idle is currently disabled in the .xml
18#if GATOR_CPU_FREQ_SUPPORT
19enum {
20 POWER_CPU_FREQ,
21 POWER_CPU_IDLE,
22 POWER_TOTAL
23};
24
25static ulong power_cpu_enabled[POWER_TOTAL];
26static ulong power_cpu_key[POWER_TOTAL];
27static DEFINE_PER_CPU(ulong[POWER_TOTAL], power);
28static DEFINE_PER_CPU(ulong[POWER_TOTAL], prev);
29static DEFINE_PER_CPU(int *, powerGet);
30
31GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu))
32{
33 per_cpu(power, cpu)[POWER_CPU_FREQ] = frequency * 1000;
34}
35
36GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
37{
38 per_cpu(power, cpu)[POWER_CPU_IDLE] = state;
39}
40
41static int gator_events_power_create_files(struct super_block *sb, struct dentry *root)
42{
43 struct dentry *dir;
44
45 // cpu_frequency
46 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq");
47 if (!dir) {
48 return -1;
49 }
50 gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]);
51 gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]);
52
53 // cpu_idle
54 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle");
55 if (!dir) {
56 return -1;
57 }
58 gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]);
59 gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]);
60
61 return 0;
62}
63
64static int gator_events_power_populate(int cpu, int** buffer)
65{
66 int i, len = 0;
67
68 for (i = 0; i < POWER_TOTAL; i++) {
69 if (power_cpu_enabled[i]) {
70 if (per_cpu(power, cpu)[i] != per_cpu(prev, cpu)[i]) {
71 per_cpu(prev, cpu)[i] = per_cpu(power, cpu)[i];
72 per_cpu(powerGet, cpu)[len++] = power_cpu_key[i];
73 per_cpu(powerGet, cpu)[len++] = per_cpu(power, cpu)[i];
74 }
75 }
76 }
77
78 if (buffer)
79 *buffer = per_cpu(powerGet, cpu);
80
81 return len;
82}
83
84static int gator_events_power_online(int** buffer)
85{
86 int i, cpu = smp_processor_id();
87 for (i = 0; i < POWER_TOTAL; i++)
88 per_cpu(prev, cpu)[i] = -1;
89 per_cpu(power, cpu)[POWER_CPU_FREQ] = cpufreq_quick_get(cpu) * 1000;
90 return gator_events_power_populate(cpu, buffer);
91}
92
93static int gator_events_power_offline(int** buffer)
94{
95 int cpu = smp_processor_id();
96 // Set frequency to zero on an offline
97 per_cpu(power, cpu)[POWER_CPU_FREQ] = 0;
98 return gator_events_power_populate(cpu, buffer);
99}
100
101static int gator_events_power_start(void)
102{
103 int cpu;
104
105 for_each_present_cpu(cpu) {
106 per_cpu(powerGet, cpu) = kmalloc(POWER_TOTAL * 2, GFP_KERNEL);
107 if (!per_cpu(powerGet, cpu))
108 return -1;
109 }
110
111 // register tracepoints
112 if (power_cpu_enabled[POWER_CPU_FREQ])
113 if (GATOR_REGISTER_TRACE(cpu_frequency))
114 goto fail_cpu_frequency_exit;
115 if (power_cpu_enabled[POWER_CPU_IDLE])
116 if (GATOR_REGISTER_TRACE(cpu_idle))
117 goto fail_cpu_idle_exit;
118 pr_debug("gator: registered power event tracepoints\n");
119
120 return 0;
121
122 // unregister tracepoints on error
123fail_cpu_idle_exit:
124 GATOR_UNREGISTER_TRACE(cpu_frequency);
125fail_cpu_frequency_exit:
126 pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
127
128 return -1;
129}
130
131static void gator_events_power_stop(void)
132{
133 int i, cpu;
134 if (power_cpu_enabled[POWER_CPU_FREQ])
135 GATOR_UNREGISTER_TRACE(cpu_frequency);
136 if (power_cpu_enabled[POWER_CPU_IDLE])
137 GATOR_UNREGISTER_TRACE(cpu_idle);
138 pr_debug("gator: unregistered power event tracepoints\n");
139
140 for (i = 0; i < POWER_TOTAL; i++) {
141 power_cpu_enabled[i] = 0;
142 }
143
144 for_each_present_cpu(cpu) {
145 kfree(per_cpu(powerGet, cpu));
146 }
147}
148
149static int gator_events_power_read(int **buffer)
150{
151 return gator_events_power_populate(smp_processor_id(), buffer);
152}
153
154static struct gator_interface gator_events_power_interface = {
155 .create_files = gator_events_power_create_files,
156 .online = gator_events_power_online,
157 .offline = gator_events_power_offline,
158 .start = gator_events_power_start,
159 .stop = gator_events_power_stop,
160 .read = gator_events_power_read,
161};
162#endif
163
164int gator_events_power_init(void)
165{
166#if (GATOR_CPU_FREQ_SUPPORT)
167 int i;
168 for (i = 0; i < POWER_TOTAL; i++) {
169 power_cpu_enabled[i] = 0;
170 power_cpu_key[i] = gator_events_get_key();
171 }
172
173 return gator_events_install(&gator_events_power_interface);
174#else
175 return -1;
176#endif
177}
178gator_events_init(gator_events_power_init);
diff --git a/driver/gator_events_sched.c b/driver/gator_events_sched.c
index 7e9db60..9bed364 100644
--- a/driver/gator_events_sched.c
+++ b/driver/gator_events_sched.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c
index f51e292..477e7c9 100644
--- a/driver/gator_events_scorpion.c
+++ b/driver/gator_events_scorpion.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -8,8 +8,8 @@
8 8
9#include "gator.h" 9#include "gator.h"
10 10
11#define SCORPION 0xf 11// gator_events_perf_pmu.c is used if perf is supported
12#define SCORPIONMP 0x2d 12#if GATOR_NO_PERF_SUPPORT
13 13
14static const char *pmnc_name; 14static const char *pmnc_name;
15static int pmnc_counters; 15static int pmnc_counters;
@@ -32,7 +32,6 @@ static int pmnc_counters;
32 32
33static unsigned long pmnc_enabled[CNTMAX]; 33static unsigned long pmnc_enabled[CNTMAX];
34static unsigned long pmnc_event[CNTMAX]; 34static unsigned long pmnc_event[CNTMAX];
35static unsigned long pmnc_count[CNTMAX];
36static unsigned long pmnc_key[CNTMAX]; 35static unsigned long pmnc_key[CNTMAX];
37 36
38static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 37static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
@@ -516,7 +515,6 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
516 return -1; 515 return -1;
517 } 516 }
518 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 517 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]);
519 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]);
520 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); 518 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]);
521 if (i > 0) { 519 if (i > 0) {
522 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 520 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]);
@@ -526,9 +524,9 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
526 return 0; 524 return 0;
527} 525}
528 526
529static void gator_events_scorpion_online(void) 527static int gator_events_scorpion_online(int** buffer)
530{ 528{
531 unsigned int cnt; 529 unsigned int cnt, len = 0, cpu = smp_processor_id();
532 530
533 if (scorpion_pmnc_read() & PMNC_E) { 531 if (scorpion_pmnc_read() & PMNC_E) {
534 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); 532 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E);
@@ -563,26 +561,34 @@ static void gator_events_scorpion_online(void)
563 561
564 // enable 562 // enable
565 scorpion_pmnc_write(scorpion_pmnc_read() | PMNC_E); 563 scorpion_pmnc_write(scorpion_pmnc_read() | PMNC_E);
566}
567 564
568static void gator_events_scorpion_offline(void) 565 // read the counters and toss the invalid data, return zero instead
569{ 566 for (cnt = 0; cnt < pmnc_counters; cnt++) {
570 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); 567 if (pmnc_enabled[cnt]) {
568 int value;
569 if (cnt == CCNT) {
570 value = scorpion_ccnt_read();
571 } else if (scorpion_pmnc_select_counter(cnt) == cnt) {
572 value = scorpion_cntn_read();
573 } else {
574 value = 0;
575 }
576 scorpion_pmnc_reset_counter(cnt);
577 per_cpu(perfPrev, cpu)[cnt] = 0;
578 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
579 per_cpu(perfCnt, cpu)[len++] = 0;
580 }
581 }
571 582
572 // investigate: need to do the clearpmu() here on each counter? 583 if (buffer)
584 *buffer = per_cpu(perfCnt, cpu);
585
586 return len;
573} 587}
574 588
575static int gator_events_scorpion_start(void) 589static int gator_events_scorpion_offline(int** buffer)
576{ 590{
577 int cnt; 591 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E);
578
579 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
580 if (pmnc_count[cnt] > 0) {
581 pr_err("gator: event based sampling not supported on Scorpion cores\n");
582 return -1;
583 }
584 }
585
586 return 0; 592 return 0;
587} 593}
588 594
@@ -593,7 +599,6 @@ static void gator_events_scorpion_stop(void)
593 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 599 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
594 pmnc_enabled[cnt] = 0; 600 pmnc_enabled[cnt] = 0;
595 pmnc_event[cnt] = 0; 601 pmnc_event[cnt] = 0;
596 pmnc_count[cnt] = 0;
597 } 602 }
598} 603}
599 604
@@ -602,9 +607,6 @@ static int gator_events_scorpion_read(int **buffer)
602 int cnt, len = 0; 607 int cnt, len = 0;
603 int cpu = smp_processor_id(); 608 int cpu = smp_processor_id();
604 609
605 if (!pmnc_counters)
606 return 0;
607
608 for (cnt = 0; cnt < pmnc_counters; cnt++) { 610 for (cnt = 0; cnt < pmnc_counters; cnt++) {
609 if (pmnc_enabled[cnt]) { 611 if (pmnc_enabled[cnt]) {
610 int value; 612 int value;
@@ -624,7 +626,6 @@ static int gator_events_scorpion_read(int **buffer)
624 } 626 }
625 } 627 }
626 628
627 // update or discard
628 if (buffer) 629 if (buffer)
629 *buffer = per_cpu(perfCnt, cpu); 630 *buffer = per_cpu(perfCnt, cpu);
630 631
@@ -633,25 +634,12 @@ static int gator_events_scorpion_read(int **buffer)
633 634
634static struct gator_interface gator_events_scorpion_interface = { 635static struct gator_interface gator_events_scorpion_interface = {
635 .create_files = gator_events_scorpion_create_files, 636 .create_files = gator_events_scorpion_create_files,
636 .start = gator_events_scorpion_start,
637 .stop = gator_events_scorpion_stop, 637 .stop = gator_events_scorpion_stop,
638 .online = gator_events_scorpion_online, 638 .online = gator_events_scorpion_online,
639 .offline = gator_events_scorpion_offline, 639 .offline = gator_events_scorpion_offline,
640 .read = gator_events_scorpion_read, 640 .read = gator_events_scorpion_read,
641}; 641};
642 642
643
644static void scorpion_clear_pmuregs(void)
645{
646 scorpion_write_lpm0(0);
647 scorpion_write_lpm1(0);
648 scorpion_write_lpm2(0);
649 scorpion_write_l2lpm(0);
650 scorpion_pre_vlpm();
651 scorpion_write_vlpm(0);
652 scorpion_post_vlpm();
653}
654
655int gator_events_scorpion_init(void) 643int gator_events_scorpion_init(void)
656{ 644{
657 unsigned int cnt; 645 unsigned int cnt;
@@ -674,12 +662,17 @@ int gator_events_scorpion_init(void)
674 for (cnt = CCNT; cnt < CNTMAX; cnt++) { 662 for (cnt = CCNT; cnt < CNTMAX; cnt++) {
675 pmnc_enabled[cnt] = 0; 663 pmnc_enabled[cnt] = 0;
676 pmnc_event[cnt] = 0; 664 pmnc_event[cnt] = 0;
677 pmnc_count[cnt] = 0;
678 pmnc_key[cnt] = gator_events_get_key(); 665 pmnc_key[cnt] = gator_events_get_key();
679 } 666 }
680 667
681 scorpion_clear_pmuregs();
682
683 return gator_events_install(&gator_events_scorpion_interface); 668 return gator_events_install(&gator_events_scorpion_interface);
684} 669}
670
685gator_events_init(gator_events_scorpion_init); 671gator_events_init(gator_events_scorpion_init);
672
673#else
674int gator_events_scorpion_init(void)
675{
676 return -1;
677}
678#endif
diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c
new file mode 100644
index 0000000..5896b3c
--- /dev/null
+++ b/driver/gator_hrtimer_gator.c
@@ -0,0 +1,76 @@
1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10// gator_hrtimer_perf.c is used if perf is supported
11// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers
12#if 1
13
14void (*callback)(void);
15DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
16static ktime_t profiling_interval;
17static void gator_hrtimer_online(int cpu);
18static void gator_hrtimer_offline(int cpu);
19
20static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
21{
22 hrtimer_forward_now(hrtimer, profiling_interval);
23 (*callback)();
24 return HRTIMER_RESTART;
25}
26
27static void gator_hrtimer_switch_cpus_online(void *unused)
28{
29 gator_hrtimer_online(smp_processor_id());
30}
31
32static void gator_hrtimer_online(int cpu)
33{
34 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
35 if (cpu != smp_processor_id()) {
36 smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1);
37 return;
38 }
39
40 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
41 hrtimer->function = gator_hrtimer_notify;
42 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
43}
44
45static void gator_hrtimer_switch_cpus_offline(void *unused)
46{
47 gator_hrtimer_offline(smp_processor_id());
48}
49
50static void gator_hrtimer_offline(int cpu)
51{
52 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
53 if (cpu != smp_processor_id()) {
54 smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1);
55 return;
56 }
57
58 hrtimer_cancel(hrtimer);
59}
60
61static int gator_hrtimer_init(int interval, void (*func)(void))
62{
63 (callback) = (func);
64
65 // calculate profiling interval
66 profiling_interval = ns_to_ktime(1000000000UL / interval);
67
68 return 0;
69}
70
71static void gator_hrtimer_shutdown(void)
72{
73 /* empty */
74}
75
76#endif
diff --git a/driver/gator_hrtimer_perf.c b/driver/gator_hrtimer_perf.c
new file mode 100644
index 0000000..7c0333f
--- /dev/null
+++ b/driver/gator_hrtimer_perf.c
@@ -0,0 +1,113 @@
1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10// gator_hrtimer_gator.c is used if perf is not supported
11// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers
12#if 0
13
14// Note: perf Cortex support added in 2.6.35 and PERF_COUNT_SW_CPU_CLOCK/hrtimer broken on 2.6.35 and 2.6.36
15// not relevant as this code is not active until 3.0.0, but wanted to document the issue
16
17void (*callback)(void);
18static int profiling_interval;
19static DEFINE_PER_CPU(struct perf_event *, perf_hrtimer);
20static DEFINE_PER_CPU(struct perf_event_attr *, perf_hrtimer_attr);
21
22static void gator_hrtimer_shutdown(void);
23
24#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
25static void hrtimer_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs)
26#else
27static void hrtimer_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
28#endif
29{
30 (*callback)();
31}
32
33static int gator_online_single_hrtimer(int cpu)
34{
35 if (per_cpu(perf_hrtimer, cpu) != 0 || per_cpu(perf_hrtimer_attr, cpu) == 0)
36 return 0;
37
38#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
39 per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler);
40#else
41 per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler, 0);
42#endif
43 if (IS_ERR(per_cpu(perf_hrtimer, cpu))) {
44 per_cpu(perf_hrtimer, cpu) = NULL;
45 return -1;
46 }
47
48 if (per_cpu(perf_hrtimer, cpu)->state != PERF_EVENT_STATE_ACTIVE) {
49 perf_event_release_kernel(per_cpu(perf_hrtimer, cpu));
50 per_cpu(perf_hrtimer, cpu) = NULL;
51 return -1;
52 }
53
54 return 0;
55}
56
57static void gator_hrtimer_online(int cpu)
58{
59 if (gator_online_single_hrtimer(cpu) < 0) {
60 pr_debug("gator: unable to online the hrtimer on cpu%d\n", cpu);
61 }
62}
63
64static void gator_hrtimer_offline(int cpu)
65{
66 if (per_cpu(perf_hrtimer, cpu)) {
67 perf_event_release_kernel(per_cpu(perf_hrtimer, cpu));
68 per_cpu(perf_hrtimer, cpu) = NULL;
69 }
70}
71
72static int gator_hrtimer_init(int interval, void (*func)(void))
73{
74 u32 size = sizeof(struct perf_event_attr);
75 int cpu;
76
77 callback = func;
78
79 // calculate profiling interval
80 profiling_interval = 1000000000 / interval;
81
82 for_each_present_cpu(cpu) {
83 per_cpu(perf_hrtimer, cpu) = 0;
84 per_cpu(perf_hrtimer_attr, cpu) = kmalloc(size, GFP_KERNEL);
85 if (per_cpu(perf_hrtimer_attr, cpu) == 0) {
86 gator_hrtimer_shutdown();
87 return -1;
88 }
89
90 memset(per_cpu(perf_hrtimer_attr, cpu), 0, size);
91 per_cpu(perf_hrtimer_attr, cpu)->type = PERF_TYPE_SOFTWARE;
92 per_cpu(perf_hrtimer_attr, cpu)->size = size;
93 per_cpu(perf_hrtimer_attr, cpu)->config = PERF_COUNT_SW_CPU_CLOCK;
94 per_cpu(perf_hrtimer_attr, cpu)->sample_period = profiling_interval;
95 per_cpu(perf_hrtimer_attr, cpu)->pinned = 1;
96 }
97
98 return 0;
99}
100
101static void gator_hrtimer_shutdown(void)
102{
103 int cpu;
104
105 for_each_present_cpu(cpu) {
106 if (per_cpu(perf_hrtimer_attr, cpu)) {
107 kfree(per_cpu(perf_hrtimer_attr, cpu));
108 per_cpu(perf_hrtimer_attr, cpu) = NULL;
109 }
110 }
111}
112
113#endif
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 36e951b..fff2d19 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -7,7 +7,7 @@
7 * 7 *
8 */ 8 */
9 9
10static unsigned long gator_protocol_version = 7; 10static unsigned long gator_protocol_version = 8;
11 11
12#include <linux/slab.h> 12#include <linux/slab.h>
13#include <linux/cpu.h> 13#include <linux/cpu.h>
@@ -18,16 +18,20 @@ static unsigned long gator_protocol_version = 7;
18#include <linux/highmem.h> 18#include <linux/highmem.h>
19#include <linux/pagemap.h> 19#include <linux/pagemap.h>
20#include <linux/suspend.h> 20#include <linux/suspend.h>
21#include <linux/module.h>
22#include <linux/perf_event.h>
21#include <asm/stacktrace.h> 23#include <asm/stacktrace.h>
22#include <asm/uaccess.h> 24#include <asm/uaccess.h>
23 25
24#include "gator.h" 26#include "gator.h"
25#include "gator_events.h" 27#include "gator_events.h"
26 28
27#ifndef CONFIG_GENERIC_TRACER 29#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
28#ifndef CONFIG_TRACING 30#error kernels prior to 2.6.32 are not supported
29#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
30#endif 31#endif
32
33#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
34#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
31#endif 35#endif
32 36
33#ifndef CONFIG_PROFILING 37#ifndef CONFIG_PROFILING
@@ -38,16 +42,20 @@ static unsigned long gator_protocol_version = 7;
38#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined 42#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
39#endif 43#endif
40 44
41#if defined (__arm__) 45#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
42#ifdef CONFIG_SMP
43#ifndef CONFIG_LOCAL_TIMERS
44#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems 46#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
45#endif 47#endif
48
49#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50#ifndef CONFIG_PERF_EVENTS
51#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52#elif !defined CONFIG_HW_PERF_EVENTS
53#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
46#endif 54#endif
47#endif 55#endif
48 56
49#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) 57#if (!(GATOR_CPU_FREQ_SUPPORT))
50#error kernels prior to 2.6.32 are not supported 58#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart
51#endif 59#endif
52 60
53/****************************************************************************** 61/******************************************************************************
@@ -69,8 +77,11 @@ static unsigned long gator_protocol_version = 7;
69#define MESSAGE_END_BACKTRACE 7 77#define MESSAGE_END_BACKTRACE 7
70#define MESSAGE_SCHEDULER_TRACE 9 78#define MESSAGE_SCHEDULER_TRACE 9
71#define MESSAGE_PID_NAME 11 79#define MESSAGE_PID_NAME 11
80#define MESSAGE_GPU_TRACE 13
81#define MESSAGE_OVERFLOW 127
72 82
73#define LINUX_PMU_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 35) && defined(CONFIG_CPU_HAS_PMU) 83#define MAXSIZE_PACK32 5
84#define MAXSIZE_PACK64 9
74 85
75#if defined(__arm__) 86#if defined(__arm__)
76#define PC_REG regs->ARM_pc 87#define PC_REG regs->ARM_pc
@@ -94,27 +105,16 @@ static unsigned long gator_streaming;
94static DEFINE_MUTEX(start_mutex); 105static DEFINE_MUTEX(start_mutex);
95static DEFINE_MUTEX(gator_buffer_mutex); 106static DEFINE_MUTEX(gator_buffer_mutex);
96 107
97unsigned long gator_net_traffic;
98bool event_based_sampling; 108bool event_based_sampling;
99 109
100#define COMMIT_SIZE 128
101#define COMMIT_MASK (COMMIT_SIZE-1)
102static DEFINE_SPINLOCK(timer_commit_lock);
103static int *gator_commit[NUM_GATOR_BUFS];
104static int gator_commit_read[NUM_GATOR_BUFS];
105static int gator_commit_write[NUM_GATOR_BUFS];
106
107static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); 110static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
108static DEFINE_PER_CPU(int, gator_first_time);
109 111
110#if LINUX_PMU_SUPPORT 112static void buffer_check(int cpu, int buftype);
111static void event_buffer_check(int cpu);
112static DEFINE_SPINLOCK(event_commit_lock);
113#endif
114 113
115/****************************************************************************** 114/******************************************************************************
116 * Prototypes 115 * Prototypes
117 ******************************************************************************/ 116 ******************************************************************************/
117static bool buffer_check_space(int cpu, int buftype, int bytes);
118static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); 118static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
119static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); 119static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
120static void gator_buffer_write_string(int cpu, int buftype, char *x); 120static void gator_buffer_write_string(int cpu, int buftype, char *x);
@@ -124,15 +124,28 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address);
124static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); 124static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
125static uint64_t gator_get_time(void); 125static uint64_t gator_get_time(void);
126 126
127static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
128static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
129static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
130static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
131static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
132static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
133static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
134static DEFINE_PER_CPU(uint64_t, emit_overflow);
135
127/****************************************************************************** 136/******************************************************************************
128 * Application Includes 137 * Application Includes
129 ******************************************************************************/ 138 ******************************************************************************/
139#include "gator_hrtimer_perf.c"
140#include "gator_hrtimer_gator.c"
130#include "gator_cookies.c" 141#include "gator_cookies.c"
131#include "gator_trace_sched.c" 142#include "gator_trace_sched.c"
143#include "gator_trace_gpu.c"
132#include "gator_backtrace.c" 144#include "gator_backtrace.c"
133#include "gator_annotate.c" 145#include "gator_annotate.c"
134#include "gator_fs.c" 146#include "gator_fs.c"
135#include "gator_ebs.c" 147#include "gator_ebs.c"
148#include "gator_pack.c"
136 149
137/****************************************************************************** 150/******************************************************************************
138 * Misc 151 * Misc
@@ -149,37 +162,55 @@ u32 gator_cpuid(void)
149/****************************************************************************** 162/******************************************************************************
150 * Commit interface 163 * Commit interface
151 ******************************************************************************/ 164 ******************************************************************************/
152static int buffer_commit_ready(int buftype) 165static bool buffer_commit_ready(int* cpu, int* buftype)
153{ 166{
154 return gator_commit_read[buftype] != gator_commit_write[buftype]; 167 int cpu_x, x;
155} 168 for_each_present_cpu(cpu_x) {
156 169 for (x = 0; x < NUM_GATOR_BUFS; x++)
157static void buffer_commit_read(int *cpu, int buftype, int *readval, int *writeval) 170 if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
158{ 171 *cpu = cpu_x;
159 int read = gator_commit_read[buftype]; 172 *buftype = x;
160 *cpu = gator_commit[buftype][read+0]; 173 return true;
161 *readval = gator_commit[buftype][read+1]; 174 }
162 *writeval = gator_commit[buftype][read+2]; 175 }
163 gator_commit_read[buftype] = (read + 4) & COMMIT_MASK; 176 return false;
164}
165
166static void buffer_commit_write(int cpu, int buftype, int readval, int writeval) {
167 int write = gator_commit_write[buftype];
168 gator_commit[buftype][write+0] = cpu;
169 gator_commit[buftype][write+1] = readval;
170 gator_commit[buftype][write+2] = writeval;
171 gator_commit_write[buftype] = (write + 4) & COMMIT_MASK;
172} 177}
173 178
174/****************************************************************************** 179/******************************************************************************
175 * Buffer management 180 * Buffer management
176 ******************************************************************************/ 181 ******************************************************************************/
177static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; 182static bool buffer_check_space(int cpu, int buftype, int bytes)
178static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; 183{
179static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); 184 int remaining, filled;
180static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); 185
181static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); 186 filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
182#include "gator_pack.c" 187 if (filled < 0) {
188 filled += gator_buffer_size[buftype];
189 }
190
191 remaining = gator_buffer_size[buftype] - filled;
192
193 if (per_cpu(buffer_space_available, cpu)[buftype]) {
194 // Give some extra room; also allows space to insert the overflow error packet
195 remaining -= 200;
196 } else {
197 // Hysteresis, prevents multiple overflow messages
198 remaining -= 2000;
199 }
200
201 if (remaining < bytes) {
202 if (per_cpu(buffer_space_available, cpu)[buftype] == true) {
203 // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters
204 per_cpu(emit_overflow, cpu) = gator_get_time();
205 pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled);
206 }
207 per_cpu(buffer_space_available, cpu)[buftype] = false;
208 } else {
209 per_cpu(buffer_space_available, cpu)[buftype] = true;
210 }
211
212 return per_cpu(buffer_space_available, cpu)[buftype];
213}
183 214
184static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) 215static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len)
185{ 216{
@@ -218,42 +249,24 @@ static void gator_buffer_header(int cpu, int buftype)
218 gator_buffer_write_packed_int(cpu, buftype, cpu); 249 gator_buffer_write_packed_int(cpu, buftype, cpu);
219} 250}
220 251
221static void gator_buffer_commit(int cpu, int buftype) 252static void gator_commit_buffer(int cpu, int buftype)
222{ 253{
223 buffer_commit_write(cpu, buftype, per_cpu(gator_buffer_read, cpu)[buftype], per_cpu(gator_buffer_write, cpu)[buftype]); 254 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
224 per_cpu(gator_buffer_read, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
225 gator_buffer_header(cpu, buftype); 255 gator_buffer_header(cpu, buftype);
226 wake_up(&gator_buffer_wait); 256 wake_up(&gator_buffer_wait);
227} 257}
228 258
229static void timer_buffer_check(int cpu) 259static void buffer_check(int cpu, int buftype)
230{ 260{
231 int available = per_cpu(gator_buffer_write, cpu)[TIMER_BUF] - per_cpu(gator_buffer_read, cpu)[TIMER_BUF]; 261 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
232 if (available < 0) { 262 if (filled < 0) {
233 available += gator_buffer_size[TIMER_BUF]; 263 filled += gator_buffer_size[buftype];
234 } 264 }
235 if (available >= ((gator_buffer_size[TIMER_BUF] * 3) / 4)) { 265 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
236 spin_lock(&timer_commit_lock); 266 gator_commit_buffer(cpu, buftype);
237 gator_buffer_commit(cpu, TIMER_BUF);
238 spin_unlock(&timer_commit_lock);
239 } 267 }
240} 268}
241 269
242#if LINUX_PMU_SUPPORT
243static void event_buffer_check(int cpu)
244{
245 int available = per_cpu(gator_buffer_write, cpu)[EVENT_BUF] - per_cpu(gator_buffer_read, cpu)[EVENT_BUF];
246 if (available < 0) {
247 available += gator_buffer_size[EVENT_BUF];
248 }
249 if (available >= ((gator_buffer_size[EVENT_BUF] * 3) / 4)) {
250 spin_lock(&event_commit_lock);
251 gator_buffer_commit(cpu, EVENT_BUF);
252 spin_unlock(&event_commit_lock);
253 }
254}
255#endif
256
257static void gator_add_trace(int cpu, int buftype, unsigned int address) 270static void gator_add_trace(int cpu, int buftype, unsigned int address)
258{ 271{
259 off_t offset = 0; 272 off_t offset = 0;
@@ -309,21 +322,9 @@ static void gator_timer_interrupt(void)
309 long long *buffer64; 322 long long *buffer64;
310 struct gator_interface *gi; 323 struct gator_interface *gi;
311 324
312 // check full backtrace has enough space, otherwise may 325 // Output scheduler trace
313 // have breaks between samples in the same callstack
314 if (per_cpu(gator_first_time, cpu)) {
315 per_cpu(gator_first_time, cpu) = 0;
316
317 list_for_each_entry(gi, &gator_events, list)
318 if (gi->read)
319 gi->read(NULL);
320
321 return;
322 }
323
324 // Output scheduler
325 len = gator_trace_sched_read(&buffer64); 326 len = gator_trace_sched_read(&buffer64);
326 if (len > 0) { 327 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
327 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); 328 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
328 gator_buffer_write_packed_int(cpu, buftype, len); 329 gator_buffer_write_packed_int(cpu, buftype, len);
329 for (i = 0; i < len; i++) { 330 for (i = 0; i < len; i++) {
@@ -331,105 +332,189 @@ static void gator_timer_interrupt(void)
331 } 332 }
332 } 333 }
333 334
335 // Output GPU trace
336 len = gator_trace_gpu_read(&buffer64);
337 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
338 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE);
339 gator_buffer_write_packed_int(cpu, buftype, len);
340 for (i = 0; i < len; i++) {
341 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
342 }
343 }
344
334 // Output counters 345 // Output counters
335 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); 346 if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) {
336 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); 347 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
337 list_for_each_entry(gi, &gator_events, list) { 348 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
338 if (gi->read) { 349 list_for_each_entry(gi, &gator_events, list) {
339 len = gi->read(&buffer); 350 if (gi->read) {
340 if (len > 0) { 351 len = gi->read(&buffer);
341 gator_buffer_write_packed_int(cpu, buftype, len); 352 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
342 for (i = 0; i < len; i++) { 353 gator_buffer_write_packed_int(cpu, buftype, len);
343 gator_buffer_write_packed_int(cpu, buftype, buffer[i]); 354 for (i = 0; i < len; i++) {
355 gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
356 }
344 } 357 }
345 } 358 } else if (gi->read64) {
346 } else if (gi->read64) { 359 len = gi->read64(&buffer64);
347 len = gi->read64(&buffer64); 360 if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) {
348 if (len > 0) { 361 gator_buffer_write_packed_int(cpu, buftype, len);
349 gator_buffer_write_packed_int(cpu, buftype, len); 362 for (i = 0; i < len; i++) {
350 for (i = 0; i < len; i++) { 363 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
351 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); 364 }
352 } 365 }
353 } 366 }
354 } 367 }
368 gator_buffer_write_packed_int(cpu, buftype, 0);
355 } 369 }
356 gator_buffer_write_packed_int(cpu, buftype, 0);
357 370
358 // Output backtrace 371 // Output backtrace
359 if (!event_based_sampling) { 372 if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
360 gator_add_sample(cpu, buftype, regs); 373 gator_add_sample(cpu, buftype, regs);
374
375 // Overflow message
376 if (per_cpu(emit_overflow, cpu)) {
377 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW);
378 gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu));
379 per_cpu(emit_overflow, cpu) = 0;
361 } 380 }
362 381
363 // Check and commit; generally, commit is set to occur once per second 382 // Check and commit; generally, commit is set to occur once per second
364 timer_buffer_check(cpu); 383 buffer_check(cpu, buftype);
365} 384}
366 385
367DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
368DEFINE_PER_CPU(int, hrtimer_is_active); 386DEFINE_PER_CPU(int, hrtimer_is_active);
369static int hrtimer_running; 387static int hrtimer_running;
370static ktime_t profiling_interval;
371 388
372static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) 389// This function runs in interrupt context and on the appropriate core
390static void gator_timer_offline(void* unused)
373{ 391{
374 hrtimer_forward_now(hrtimer, profiling_interval); 392 int i, len, cpu = smp_processor_id();
375 gator_timer_interrupt(); 393 int* buffer;
376 return HRTIMER_RESTART; 394 long long* buffer64;
377}
378 395
379static int gator_timer_init(void)
380{
381 return 0;
382}
383
384static void __gator_timer_offline(void *unused)
385{
386 int cpu = smp_processor_id();
387 if (per_cpu(hrtimer_is_active, cpu)) { 396 if (per_cpu(hrtimer_is_active, cpu)) {
388 struct gator_interface *gi; 397 struct gator_interface *gi;
389 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); 398 gator_hrtimer_offline(cpu);
390 hrtimer_cancel(hrtimer);
391 per_cpu(hrtimer_is_active, cpu) = 0; 399 per_cpu(hrtimer_is_active, cpu) = 0;
392 gator_buffer_commit(cpu, TIMER_BUF); 400
393 if (event_based_sampling) 401 // Output scheduler trace
394 gator_buffer_commit(cpu, EVENT_BUF); 402 len = gator_trace_sched_offline(&buffer64);
395 403 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
396 // offline any events 404 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE);
397 list_for_each_entry(gi, &gator_events, list) 405 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
398 if (gi->offline) 406 for (i = 0; i < len; i++) {
399 gi->offline(); 407 gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
408 }
409 }
410
411 // Output GPU trace
412 len = gator_trace_gpu_offline(&buffer64);
413 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
414 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE);
415 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
416 for (i = 0; i < len; i++) {
417 gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
418 }
419 }
420
421 // offline any events and output counters
422 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
423 gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
424 list_for_each_entry(gi, &gator_events, list) {
425 if (gi->offline) {
426 len = gi->offline(&buffer);
427 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
428 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
429 for (i = 0; i < len; i++)
430 gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
431 }
432 }
433 }
434 gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
435
436 gator_commit_buffer(cpu, TIMER_BUF);
437 }
438
439 if (event_based_sampling) {
440 gator_commit_buffer(cpu, EVENT_BUF);
400 } 441 }
401} 442}
402 443
403static void gator_timer_offline(void) 444// This function runs in interrupt context and may be running on a core other than core 'cpu'
445static void gator_timer_offline_dispatch(int cpu)
404{ 446{
447 struct gator_interface *gi;
448
449 list_for_each_entry(gi, &gator_events, list)
450 if (gi->offline_dispatch)
451 gi->offline_dispatch(cpu);
452
453 gator_event_sampling_offline_dispatch(cpu);
454}
455
456static void gator_timer_stop(void)
457{
458 int cpu;
459
405 if (hrtimer_running) { 460 if (hrtimer_running) {
406 hrtimer_running = 0; 461 on_each_cpu(gator_timer_offline, NULL, 1);
462 for_each_online_cpu(cpu) {
463 gator_timer_offline_dispatch(cpu);
464 }
407 465
408 on_each_cpu(__gator_timer_offline, NULL, 1); 466 hrtimer_running = 0;
467 gator_hrtimer_shutdown();
409 } 468 }
410} 469}
411 470
412static void __gator_timer_online(void *unused) 471// This function runs in interrupt context and on the appropriate core
472static void gator_timer_online(void* unused)
413{ 473{
414 int cpu = smp_processor_id(); 474 int i, len, cpu = smp_processor_id();
475 int* buffer;
476
415 if (!per_cpu(hrtimer_is_active, cpu)) { 477 if (!per_cpu(hrtimer_is_active, cpu)) {
416 struct gator_interface *gi; 478 struct gator_interface *gi;
417 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
418 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
419 hrtimer->function = gator_hrtimer_notify;
420 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
421 per_cpu(gator_first_time, cpu) = 1;
422 per_cpu(hrtimer_is_active, cpu) = 1;
423 479
424 // online any events 480 // online any events and output counters
425 list_for_each_entry(gi, &gator_events, list) 481 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
426 if (gi->online) 482 gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
427 gi->online(); 483 list_for_each_entry(gi, &gator_events, list) {
484 if (gi->online) {
485 len = gi->online(&buffer);
486 if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
487 gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
488 for (i = 0; i < len; i++)
489 gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
490 }
491 }
492 }
493 gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
494
495 gator_event_sampling_online();
496
497 gator_hrtimer_online(cpu);
498 per_cpu(hrtimer_is_active, cpu) = 1;
428 } 499 }
429} 500}
430 501
431int gator_timer_online(unsigned long setup) 502// This function runs in interrupt context and may be running on a core other than core 'cpu'
503static void gator_timer_online_dispatch(int cpu)
504{
505 struct gator_interface *gi;
506
507 list_for_each_entry(gi, &gator_events, list)
508 if (gi->online_dispatch)
509 gi->online_dispatch(cpu);
510
511 gator_event_sampling_online_dispatch(cpu);
512}
513
514int gator_timer_start(unsigned long setup)
432{ 515{
516 int cpu;
517
433 if (!setup) { 518 if (!setup) {
434 pr_err("gator: cannot start due to a system tick value of zero\n"); 519 pr_err("gator: cannot start due to a system tick value of zero\n");
435 return -1; 520 return -1;
@@ -440,11 +525,13 @@ int gator_timer_online(unsigned long setup)
440 525
441 hrtimer_running = 1; 526 hrtimer_running = 1;
442 527
443 // calculate profiling interval 528 if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1)
444 profiling_interval = ns_to_ktime(1000000000UL / setup); 529 return -1;
445 530
446 // timer interrupt 531 for_each_online_cpu(cpu) {
447 on_each_cpu(__gator_timer_online, NULL, 1); 532 gator_timer_online_dispatch(cpu);
533 }
534 on_each_cpu(gator_timer_online, NULL, 1);
448 535
449 return 0; 536 return 0;
450} 537}
@@ -461,7 +548,7 @@ static uint64_t gator_get_time(void)
461} 548}
462 549
463/****************************************************************************** 550/******************************************************************************
464 * cpu online and pm notifiers 551 * cpu hotplug and pm notifiers
465 ******************************************************************************/ 552 ******************************************************************************/
466static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 553static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
467{ 554{
@@ -470,11 +557,13 @@ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long
470 switch (action) { 557 switch (action) {
471 case CPU_DOWN_PREPARE: 558 case CPU_DOWN_PREPARE:
472 case CPU_DOWN_PREPARE_FROZEN: 559 case CPU_DOWN_PREPARE_FROZEN:
473 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1); 560 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
561 gator_timer_offline_dispatch(cpu);
474 break; 562 break;
475 case CPU_ONLINE: 563 case CPU_ONLINE:
476 case CPU_ONLINE_FROZEN: 564 case CPU_ONLINE_FROZEN:
477 smp_call_function_single(cpu, __gator_timer_online, NULL, 1); 565 gator_timer_online_dispatch(cpu);
566 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
478 break; 567 break;
479 } 568 }
480 569
@@ -489,17 +578,24 @@ static struct notifier_block __refdata gator_cpu_notifier = {
489// Registered linux events are not disabled, so their counters will continue to collect 578// Registered linux events are not disabled, so their counters will continue to collect
490static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy) 579static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
491{ 580{
581 int cpu;
582
492 switch (event) { 583 switch (event) {
493 case PM_HIBERNATION_PREPARE: 584 case PM_HIBERNATION_PREPARE:
494 case PM_SUSPEND_PREPARE: 585 case PM_SUSPEND_PREPARE:
495 unregister_hotcpu_notifier(&gator_cpu_notifier); 586 unregister_hotcpu_notifier(&gator_cpu_notifier);
496 unregister_scheduler_tracepoints(); 587 unregister_scheduler_tracepoints();
497 on_each_cpu(trace_sched_insert_idle, NULL, 1); 588 on_each_cpu(gator_timer_offline, NULL, 1);
498 on_each_cpu(__gator_timer_offline, NULL, 1); 589 for_each_online_cpu(cpu) {
590 gator_timer_offline_dispatch(cpu);
591 }
499 break; 592 break;
500 case PM_POST_HIBERNATION: 593 case PM_POST_HIBERNATION:
501 case PM_POST_SUSPEND: 594 case PM_POST_SUSPEND:
502 on_each_cpu(__gator_timer_online, NULL, 1); 595 for_each_online_cpu(cpu) {
596 gator_timer_online_dispatch(cpu);
597 }
598 on_each_cpu(gator_timer_online, NULL, 1);
503 register_scheduler_tracepoints(); 599 register_scheduler_tracepoints();
504 register_hotcpu_notifier(&gator_cpu_notifier); 600 register_hotcpu_notifier(&gator_cpu_notifier);
505 break; 601 break;
@@ -548,10 +644,6 @@ static int gator_init(void)
548{ 644{
549 int i; 645 int i;
550 646
551 if (gator_timer_init())
552 return -1;
553 if (gator_trace_sched_init())
554 return -1;
555 if (gator_annotate_init()) 647 if (gator_annotate_init())
556 return -1; 648 return -1;
557 649
@@ -573,8 +665,7 @@ static int gator_start(void)
573 struct list_head *ptr = gi->list.prev; 665 struct list_head *ptr = gi->list.prev;
574 666
575 while (ptr != &gator_events) { 667 while (ptr != &gator_events) {
576 gi = list_entry(ptr, struct gator_interface, 668 gi = list_entry(ptr, struct gator_interface, list);
577 list);
578 669
579 if (gi->stop) 670 if (gi->stop)
580 gi->stop(); 671 gi->stop();
@@ -585,16 +676,18 @@ static int gator_start(void)
585 } 676 }
586 } 677 }
587 678
588 // cookies shall be initialized before trace_sched_start() and gator_timer_online() 679 // cookies shall be initialized before trace_sched_start() and gator_timer_start()
589 if (cookies_initialize()) 680 if (cookies_initialize())
590 goto cookies_failure; 681 goto cookies_failure;
591 if (gator_annotate_start()) 682 if (gator_annotate_start())
592 goto annotate_failure; 683 goto annotate_failure;
593 if (gator_trace_sched_start()) 684 if (gator_trace_sched_start())
594 goto sched_failure; 685 goto sched_failure;
686 if (gator_trace_gpu_start())
687 goto gpu_failure;
595 if (gator_event_sampling_start()) 688 if (gator_event_sampling_start())
596 goto event_sampling_failure; 689 goto event_sampling_failure;
597 if (gator_timer_online(gator_timer_count)) 690 if (gator_timer_start(gator_timer_count))
598 goto timer_failure; 691 goto timer_failure;
599 if (gator_notifier_start()) 692 if (gator_notifier_start())
600 goto notifier_failure; 693 goto notifier_failure;
@@ -602,10 +695,12 @@ static int gator_start(void)
602 return 0; 695 return 0;
603 696
604notifier_failure: 697notifier_failure:
605 gator_timer_offline(); 698 gator_timer_stop();
606timer_failure: 699timer_failure:
607 gator_event_sampling_stop(); 700 gator_event_sampling_stop();
608event_sampling_failure: 701event_sampling_failure:
702 gator_trace_gpu_stop();
703gpu_failure:
609 gator_trace_sched_stop(); 704 gator_trace_sched_stop();
610sched_failure: 705sched_failure:
611 gator_annotate_stop(); 706 gator_annotate_stop();
@@ -632,11 +727,12 @@ static void gator_stop(void)
632 727
633 gator_annotate_stop(); 728 gator_annotate_stop();
634 gator_trace_sched_stop(); 729 gator_trace_sched_stop();
730 gator_trace_gpu_stop();
635 gator_event_sampling_stop(); 731 gator_event_sampling_stop();
636 732
637 // stop all interrupt callback reads before tearing down other interfaces 733 // stop all interrupt callback reads before tearing down other interfaces
638 gator_notifier_stop(); // should be called before gator_timer_offline to avoid re-enabling the hrtimer after it has been offlined 734 gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined
639 gator_timer_offline(); 735 gator_timer_stop();
640} 736}
641 737
642static void gator_exit(void) 738static void gator_exit(void)
@@ -667,18 +763,8 @@ static int gator_op_setup(void)
667 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; 763 gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
668 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; 764 gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
669 765
670 gator_net_traffic = 0; 766 // Initialize percpu per buffer variables
671
672 // Initialize per buffer variables
673 for (i = 0; i < NUM_GATOR_BUFS; i++) { 767 for (i = 0; i < NUM_GATOR_BUFS; i++) {
674 gator_commit_read[i] = gator_commit_write[i] = 0;
675 gator_commit[i] = vmalloc(COMMIT_SIZE * sizeof(int));
676 if (!gator_commit[i]) {
677 err = -ENOMEM;
678 goto setup_error;
679 }
680
681 // Initialize percpu per buffer variables
682 for_each_present_cpu(cpu) { 768 for_each_present_cpu(cpu) {
683 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); 769 per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
684 if (!per_cpu(gator_buffer, cpu)[i]) { 770 if (!per_cpu(gator_buffer, cpu)[i]) {
@@ -688,6 +774,9 @@ static int gator_op_setup(void)
688 774
689 per_cpu(gator_buffer_read, cpu)[i] = 0; 775 per_cpu(gator_buffer_read, cpu)[i] = 0;
690 per_cpu(gator_buffer_write, cpu)[i] = 0; 776 per_cpu(gator_buffer_write, cpu)[i] = 0;
777 per_cpu(gator_buffer_commit, cpu)[i] = 0;
778 per_cpu(buffer_space_available, cpu)[i] = true;
779 per_cpu(emit_overflow, cpu) = 0;
691 gator_buffer_header(cpu, i); 780 gator_buffer_header(cpu, i);
692 } 781 }
693 } 782 }
@@ -742,11 +831,6 @@ static void gator_shutdown(void)
742 831
743 gator_annotate_shutdown(); 832 gator_annotate_shutdown();
744 833
745 for (i = 0; i < NUM_GATOR_BUFS; i++) {
746 vfree(gator_commit[i]);
747 gator_commit[i] = NULL;
748 }
749
750 for_each_present_cpu(cpu) { 834 for_each_present_cpu(cpu) {
751 mutex_lock(&gator_buffer_mutex); 835 mutex_lock(&gator_buffer_mutex);
752 for (i = 0; i < NUM_GATOR_BUFS; i++) { 836 for (i = 0; i < NUM_GATOR_BUFS; i++) {
@@ -754,6 +838,9 @@ static void gator_shutdown(void)
754 per_cpu(gator_buffer, cpu)[i] = NULL; 838 per_cpu(gator_buffer, cpu)[i] = NULL;
755 per_cpu(gator_buffer_read, cpu)[i] = 0; 839 per_cpu(gator_buffer_read, cpu)[i] = 0;
756 per_cpu(gator_buffer_write, cpu)[i] = 0; 840 per_cpu(gator_buffer_write, cpu)[i] = 0;
841 per_cpu(gator_buffer_commit, cpu)[i] = 0;
842 per_cpu(buffer_space_available, cpu)[i] = true;
843 per_cpu(emit_overflow, cpu) = 0;
757 } 844 }
758 mutex_unlock(&gator_buffer_mutex); 845 mutex_unlock(&gator_buffer_mutex);
759 } 846 }
@@ -845,10 +932,10 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
845 size_t count, loff_t *offset) 932 size_t count, loff_t *offset)
846{ 933{
847 int retval = -EINVAL; 934 int retval = -EINVAL;
848 int commit, length1, length2, read; 935 int commit = 0, length1, length2, read;
849 char *buffer1; 936 char *buffer1;
850 char *buffer2 = NULL; 937 char *buffer2 = NULL;
851 int cpu, i; 938 int cpu, buftype;
852 939
853 /* do not handle partial reads */ 940 /* do not handle partial reads */
854 if (count != userspace_buffer_size || *offset) 941 if (count != userspace_buffer_size || *offset)
@@ -856,7 +943,8 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
856 943
857 // sleep until the condition is true or a signal is received 944 // sleep until the condition is true or a signal is received
858 // the condition is checked each time gator_buffer_wait is woken up 945 // the condition is checked each time gator_buffer_wait is woken up
859 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(TIMER_BUF) || buffer_commit_ready(EVENT_BUF) || gator_annotate_ready() || !gator_started); 946 buftype = cpu = -1;
947 wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started);
860 948
861 if (signal_pending(current)) 949 if (signal_pending(current))
862 return -EINTR; 950 return -EINTR;
@@ -866,28 +954,22 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
866 954
867 mutex_lock(&gator_buffer_mutex); 955 mutex_lock(&gator_buffer_mutex);
868 956
869 i = -1; 957 if (buftype != -1 && cpu != -1) {
870 if (buffer_commit_ready(TIMER_BUF)) { 958 read = per_cpu(gator_buffer_read, cpu)[buftype];
871 i = TIMER_BUF; 959 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
872 } else if (buffer_commit_ready(EVENT_BUF)) {
873 i = EVENT_BUF;
874 }
875
876 if (i != -1) {
877 buffer_commit_read(&cpu, i, &read, &commit);
878 960
879 /* May happen if the buffer is freed during pending reads. */ 961 /* May happen if the buffer is freed during pending reads. */
880 if (!per_cpu(gator_buffer, cpu)[i]) { 962 if (!per_cpu(gator_buffer, cpu)[buftype]) {
881 retval = -EFAULT; 963 retval = -EFAULT;
882 goto out; 964 goto out;
883 } 965 }
884 966
885 /* determine the size of two halves */ 967 /* determine the size of two halves */
886 length1 = commit - read; 968 length1 = commit - read;
887 buffer1 = &(per_cpu(gator_buffer, cpu)[i][read]); 969 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
888 buffer2 = &(per_cpu(gator_buffer, cpu)[i][0]); 970 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
889 if (length1 < 0) { 971 if (length1 < 0) {
890 length1 = gator_buffer_size[i] - read; 972 length1 = gator_buffer_size[buftype] - read;
891 length2 = commit; 973 length2 = commit;
892 } 974 }
893 } else if (gator_annotate_ready()) { 975 } else if (gator_annotate_ready()) {
@@ -913,15 +995,15 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
913 } 995 }
914 } 996 }
915 997
998 if (buftype != -1 && cpu != -1)
999 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1000
916 retval = length1 + length2; 1001 retval = length1 + length2;
917 1002
918 /* kick just in case we've lost an SMP event */ 1003 /* kick just in case we've lost an SMP event */
919 wake_up(&gator_buffer_wait); 1004 wake_up(&gator_buffer_wait);
920 1005
921out: 1006out:
922 // only adjust network stats if in streaming mode
923 if (gator_streaming)
924 gator_net_traffic += retval;
925 mutex_unlock(&gator_buffer_mutex); 1007 mutex_unlock(&gator_buffer_mutex);
926 return retval; 1008 return retval;
927} 1009}
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
index fbab220..985e960 100644
--- a/driver/gator_pack.c
+++ b/driver/gator_pack.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2011. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c
new file mode 100644
index 0000000..bc63995
--- /dev/null
+++ b/driver/gator_trace_gpu.c
@@ -0,0 +1,250 @@
1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#include "gator.h"
10
11#include <linux/slab.h>
12#include <linux/module.h>
13#include <linux/time.h>
14#include <linux/math64.h>
15
16#ifdef MALI_SUPPORT
17#include "linux/mali_linux_trace.h"
18#endif
19#include "gator_trace_gpu.h"
20
21#define ACTIVITY_START 1
22#define ACTIVITY_STOP 2
23
24/* Note whether tracepoints have been registered */
25static int mali_trace_registered;
26static int gpu_trace_registered;
27
28#define GPU_OVERFLOW -1
29#define GPU_START 1
30#define GPU_STOP 2
31
32#define GPU_UNIT_VP 1
33#define GPU_UNIT_FP 2
34
35#define TRACESIZE (8*1024)
36
37static DEFINE_PER_CPU(uint64_t *[2], theGpuTraceBuf);
38static DEFINE_PER_CPU(int, theGpuTraceSel);
39static DEFINE_PER_CPU(int, theGpuTracePos);
40static DEFINE_PER_CPU(int, theGpuTraceErr);
41
42int gator_trace_gpu_read(long long **buffer);
43
44static void probe_gpu_write(int type, int unit, int core, struct task_struct* task)
45{
46 int tracePos;
47 unsigned long flags;
48 uint64_t *traceBuf, time;
49 int pid, tgid;
50 int cpu = smp_processor_id();
51
52 if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)])
53 return;
54
55 if (task) {
56 tgid = (int)task->tgid;
57 pid = (int)task->pid;
58 } else {
59 tgid = pid = 0;
60 }
61
62 // disable interrupts to synchronize with gator_trace_gpu_read(); spinlocks not needed since percpu buffers are used
63 local_irq_save(flags);
64
65 time = gator_get_time();
66 tracePos = per_cpu(theGpuTracePos, cpu);
67 traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)];
68
69 if (tracePos < (TRACESIZE - 100)) {
70 // capture
71 traceBuf[tracePos++] = type;
72 traceBuf[tracePos++] = time;
73 traceBuf[tracePos++] = unit;
74 traceBuf[tracePos++] = core;
75 traceBuf[tracePos++] = tgid;
76 traceBuf[tracePos++] = pid;
77 } else if (!per_cpu(theGpuTraceErr, cpu)) {
78 per_cpu(theGpuTraceErr, cpu) = 1;
79 traceBuf[tracePos++] = GPU_OVERFLOW;
80 traceBuf[tracePos++] = time;
81 traceBuf[tracePos++] = 0;
82 traceBuf[tracePos++] = 0;
83 traceBuf[tracePos++] = 0;
84 traceBuf[tracePos++] = 0;
85 pr_debug("gator: gpu trace overflow\n");
86 }
87 per_cpu(theGpuTracePos, cpu) = tracePos;
88 local_irq_restore(flags);
89}
90
91#ifdef MALI_SUPPORT
92
93enum components {
94 COMPONENT_VP0 = 1,
95 COMPONENT_FP0 = 5,
96 COMPONENT_FP1,
97 COMPONENT_FP2,
98 COMPONENT_FP3,
99 COMPONENT_FP4,
100 COMPONENT_FP5,
101 COMPONENT_FP6,
102 COMPONENT_FP7,
103};
104
105GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4))
106{
107 unsigned int component, state;
108
109 // do as much work as possible before disabling interrupts
110 component = (event_id >> 16) & 0xF;
111 state = (event_id >> 24) & 0xF;
112
113 if ((component == COMPONENT_VP0) || (component >= COMPONENT_FP0 && component <= COMPONENT_FP7)) {
114 if (state == ACTIVITY_START || state == ACTIVITY_STOP) {
115 unsigned int type = (state == ACTIVITY_START) ? GPU_START : GPU_STOP;
116 unsigned int unit = (component < COMPONENT_FP0) ? GPU_UNIT_VP : GPU_UNIT_FP;
117 unsigned int core = (component < COMPONENT_FP0) ? component - COMPONENT_VP0 : component - COMPONENT_FP0;
118 struct task_struct* task = (state == ACTIVITY_START) ? (struct task_struct*)d2 : NULL;
119
120 probe_gpu_write(type, unit, core, task);
121 }
122 }
123}
124#endif
125
126GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p))
127{
128 probe_gpu_write(GPU_START, gpu_unit, gpu_core, p);
129}
130
131GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core))
132{
133 probe_gpu_write(GPU_STOP, gpu_unit, gpu_core, NULL);
134}