summaryrefslogtreecommitdiffstats
path: root/driver
diff options
context:
space:
mode:
authorJon Medhurst2013-04-03 05:59:40 -0500
committerJon Medhurst2013-04-03 06:54:15 -0500
commit7ef1b3596e3625d6335fd03904dfdcc75f365919 (patch)
tree4b309e83ab767e561a2235668b3ad6cd830bb40c /driver
parent06ebd1eab0a782377611efee820bb57f09692cbf (diff)
downloadarm-ds5-gator-7ef1b3596e3625d6335fd03904dfdcc75f365919.tar.gz
arm-ds5-gator-7ef1b3596e3625d6335fd03904dfdcc75f365919.tar.xz
arm-ds5-gator-7ef1b3596e3625d6335fd03904dfdcc75f365919.zip
gator: Version 5.14
Signed-off-by: Jon Medhurst <tixy@linaro.org>
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile1
-rw-r--r--driver/gator.h43
-rw-r--r--driver/gator_annotate.c23
-rw-r--r--driver/gator_annotate_kernel.c2
-rw-r--r--driver/gator_backtrace.c39
-rw-r--r--driver/gator_cookies.c8
-rwxr-xr-x[-rw-r--r--]driver/gator_events.sh0
-rw-r--r--driver/gator_events_armv6.c6
-rw-r--r--driver/gator_events_armv7.c6
-rw-r--r--driver/gator_events_block.c9
-rw-r--r--driver/gator_events_irq.c48
-rw-r--r--driver/gator_events_l2c-310.c74
-rw-r--r--driver/gator_events_mali_400.c12
-rw-r--r--driver/gator_events_mali_400.h2
-rw-r--r--driver/gator_events_mali_common.c2
-rw-r--r--driver/gator_events_mali_common.h2
-rw-r--r--driver/gator_events_mali_t6xx.c4
-rw-r--r--driver/gator_events_mali_t6xx_hw.c4
-rw-r--r--driver/gator_events_mali_t6xx_hw_test.c2
-rw-r--r--driver/gator_events_meminfo.c4
-rw-r--r--driver/gator_events_mmaped.c6
-rw-r--r--driver/gator_events_net.c4
-rw-r--r--driver/gator_events_perf_pmu.c520
-rw-r--r--driver/gator_events_sched.c6
-rw-r--r--driver/gator_events_scorpion.c6
-rw-r--r--driver/gator_fs.c87
-rw-r--r--driver/gator_hrtimer_gator.c35
-rw-r--r--driver/gator_hrtimer_perf.c2
-rw-r--r--driver/gator_iks.c144
-rw-r--r--driver/gator_main.c335
-rw-r--r--driver/gator_marshaling.c119
-rw-r--r--driver/gator_pack.c2
-rw-r--r--driver/gator_trace_gpu.c86
-rw-r--r--driver/gator_trace_gpu.h2
-rw-r--r--driver/gator_trace_power.c43
-rw-r--r--driver/gator_trace_sched.c28
36 files changed, 1212 insertions, 504 deletions
diff --git a/driver/Makefile b/driver/Makefile
index d22d29d..3af8b8d 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -58,5 +58,6 @@ all:
58 58
59clean: 59clean:
60 rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c 60 rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c
61 rm -rf .tmp_versions
61 62
62endif 63endif
diff --git a/driver/gator.h b/driver/gator.h
index 9a4617b..205cbcd 100644
--- a/driver/gator.h
+++ b/driver/gator.h
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -18,6 +18,9 @@
18#define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && (!(defined(__arm__) || defined(__aarch64__)) || defined(CONFIG_HW_PERF_EVENTS)) 18#define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && (!(defined(__arm__) || defined(__aarch64__)) || defined(CONFIG_HW_PERF_EVENTS))
19#define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT)) 19#define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT))
20#define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ) 20#define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ)
21#define GATOR_IKS_SUPPORT defined(CONFIG_BL_SWITCHER)
22
23#define GATOR_LIVE 1
21 24
22// cpu ids 25// cpu ids
23#define ARM1136 0xb36 26#define ARM1136 0xb36
@@ -42,14 +45,15 @@
42#define MAXSIZE_CORE_NAME 32 45#define MAXSIZE_CORE_NAME 32
43 46
44struct gator_cpu { 47struct gator_cpu {
45 const int cpuid; 48 const int cpuid;
46 const char core_name[MAXSIZE_CORE_NAME]; 49 const char core_name[MAXSIZE_CORE_NAME];
47 const char * const pmnc_name; 50 const char * const pmu_name;
48 const int pmnc_counters; 51 const char * const pmnc_name;
49 const int ccnt; 52 const int pmnc_counters;
50}; 53};
51 54
52extern struct gator_cpu gator_cpus[]; 55const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid);
56const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name);
53 57
54/****************************************************************************** 58/******************************************************************************
55 * Filesystem 59 * Filesystem
@@ -98,10 +102,10 @@ struct gator_interface {
98 int (*create_files)(struct super_block *sb, struct dentry *root); 102 int (*create_files)(struct super_block *sb, struct dentry *root);
99 int (*start)(void); 103 int (*start)(void);
100 void (*stop)(void); // Complementary function to start 104 void (*stop)(void); // Complementary function to start
101 int (*online)(int **buffer); 105 int (*online)(int **buffer, bool migrate);
102 int (*offline)(int **buffer); 106 int (*offline)(int **buffer, bool migrate);
103 void (*online_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' 107 void (*online_dispatch)(int cpu, bool migrate); // called in process context but may not be running on core 'cpu'
104 void (*offline_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' 108 void (*offline_dispatch)(int cpu, bool migrate); // called in process context but may not be running on core 'cpu'
105 int (*read)(int **buffer); 109 int (*read)(int **buffer);
106 int (*read64)(long long **buffer); 110 int (*read64)(long long **buffer);
107 struct list_head list; 111 struct list_head list;
@@ -118,4 +122,21 @@ u32 gator_cpuid(void);
118 122
119void gator_backtrace_handler(struct pt_regs *const regs); 123void gator_backtrace_handler(struct pt_regs *const regs);
120 124
125#if !GATOR_IKS_SUPPORT
126
127#define get_physical_cpu() smp_processor_id()
128#define lcpu_to_pcpu(lcpu) lcpu
129#define pcpu_to_lcpu(pcpu) pcpu
130
131#else
132
133#define get_physical_cpu() lcpu_to_pcpu(get_logical_cpu())
134int lcpu_to_pcpu(const int lcpu);
135int pcpu_to_lcpu(const int pcpu);
136
137#endif
138
139#define get_logical_cpu() smp_processor_id()
140#define on_primary_core() (get_logical_cpu() == 0)
141
121#endif // GATOR_H_ 142#endif // GATOR_H_
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index 42f9951..ad9f309 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -50,6 +50,7 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
50 return -EINVAL; 50 return -EINVAL;
51 } 51 }
52 52
53 retry:
53 // synchronize between cores and with collect_annotations 54 // synchronize between cores and with collect_annotations
54 spin_lock(&annotate_lock); 55 spin_lock(&annotate_lock);
55 56
@@ -74,17 +75,18 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
74 size = count < available ? count : available; 75 size = count < available ? count : available;
75 76
76 if (size <= 0) { 77 if (size <= 0) {
77 // Buffer is full but don't return an error. Instead return 0 so the 78 // Buffer is full, wait until space is available
78 // caller knows nothing was written and they can try again. 79 spin_unlock(&annotate_lock);
79 size = 0; 80 wait_event_interruptible(gator_annotate_wait, buffer_bytes_available(cpu, ANNOTATE_BUF) > header_size || !collect_annotations);
80 goto annotate_write_out; 81 goto retry;
81 } 82 }
82 83
83 // synchronize shared variables annotateBuf and annotatePos 84 // synchronize shared variables annotateBuf and annotatePos
84 if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) { 85 if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) {
85 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id()); 86 u64 time = gator_get_time();
87 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
86 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); 88 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
87 gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, gator_get_time()); 89 gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, time);
88 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size); 90 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size);
89 91
90 // determine the sizes to capture, length1 + length2 will equal size 92 // determine the sizes to capture, length1 + length2 will equal size
@@ -108,7 +110,7 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
108 } 110 }
109 111
110 // Check and commit; commit is set to occur once buffer is 3/4 full 112 // Check and commit; commit is set to occur once buffer is 3/4 full
111 buffer_check(cpu, ANNOTATE_BUF); 113 buffer_check(cpu, ANNOTATE_BUF, time);
112 } 114 }
113 115
114annotate_write_out: 116annotate_write_out:
@@ -129,14 +131,14 @@ static int annotate_release(struct inode *inode, struct file *file)
129 131
130 if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { 132 if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
131 uint32_t pid = current->pid; 133 uint32_t pid = current->pid;
132 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id()); 134 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, get_physical_cpu());
133 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid); 135 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, pid);
134 gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); // time 136 gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); // time
135 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); // size 137 gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); // size
136 } 138 }
137 139
138 // Check and commit; commit is set to occur once buffer is 3/4 full 140 // Check and commit; commit is set to occur once buffer is 3/4 full
139 buffer_check(cpu, ANNOTATE_BUF); 141 buffer_check(cpu, ANNOTATE_BUF, gator_get_time());
140 142
141 spin_unlock(&annotate_lock); 143 spin_unlock(&annotate_lock);
142 144
@@ -164,5 +166,6 @@ static void gator_annotate_stop(void)
164 // the spinlock here will ensure that when this function exits, we are not in the middle of an annotation 166 // the spinlock here will ensure that when this function exits, we are not in the middle of an annotation
165 spin_lock(&annotate_lock); 167 spin_lock(&annotate_lock);
166 collect_annotations = false; 168 collect_annotations = false;
169 wake_up(&gator_annotate_wait);
167 spin_unlock(&annotate_lock); 170 spin_unlock(&annotate_lock);
168} 171}
diff --git a/driver/gator_annotate_kernel.c b/driver/gator_annotate_kernel.c
index 67d2d6c..4715f64 100644
--- a/driver/gator_annotate_kernel.c
+++ b/driver/gator_annotate_kernel.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index e6125b3..94f01e6 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -11,8 +11,17 @@
11 * EABI backtrace stores {fp,lr} on the stack. 11 * EABI backtrace stores {fp,lr} on the stack.
12 */ 12 */
13struct frame_tail_eabi { 13struct frame_tail_eabi {
14 unsigned long fp; // points to prev_lr 14 union {
15 unsigned long lr; 15 struct {
16 unsigned long fp; // points to prev_lr
17 unsigned long lr;
18 };
19 // Used to read 32 bit fp/lr from a 64 bit kernel
20 struct {
21 u32 fp_32;
22 u32 lr_32;
23 };
24 };
16}; 25};
17 26
18static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth) 27static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int depth)
@@ -20,18 +29,20 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int
20#if defined(__arm__) || defined(__aarch64__) 29#if defined(__arm__) || defined(__aarch64__)
21 struct frame_tail_eabi *tail; 30 struct frame_tail_eabi *tail;
22 struct frame_tail_eabi *next; 31 struct frame_tail_eabi *next;
23 struct frame_tail_eabi *ptrtail;
24 struct frame_tail_eabi buftail; 32 struct frame_tail_eabi buftail;
25#if defined(__arm__) 33#if defined(__arm__)
34 const bool is_compat = false;
26 unsigned long fp = regs->ARM_fp; 35 unsigned long fp = regs->ARM_fp;
27 unsigned long sp = regs->ARM_sp; 36 unsigned long sp = regs->ARM_sp;
28 unsigned long lr = regs->ARM_lr; 37 unsigned long lr = regs->ARM_lr;
29 const int frame_offset = 4; 38 const int frame_offset = 4;
30#else 39#else
31 unsigned long fp = regs->regs[29]; 40 // Is userspace aarch32 (32 bit)
32 unsigned long sp = regs->sp; 41 const bool is_compat = compat_user_mode(regs);
33 unsigned long lr = regs->regs[30]; 42 unsigned long fp = (is_compat ? regs->regs[11] : regs->regs[29]);
34 const int frame_offset = 0; 43 unsigned long sp = (is_compat ? regs->compat_sp : regs->sp);
44 unsigned long lr = (is_compat ? regs->compat_lr : regs->regs[30]);
45 const int frame_offset = (is_compat ? 4 : 0);
35#endif 46#endif
36 int is_user_mode = user_mode(regs); 47 int is_user_mode = user_mode(regs);
37 48
@@ -55,15 +66,14 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int
55 return; 66 return;
56 if (__copy_from_user_inatomic(&buftail, tail, sizeof(struct frame_tail_eabi))) 67 if (__copy_from_user_inatomic(&buftail, tail, sizeof(struct frame_tail_eabi)))
57 return; 68 return;
58 ptrtail = &buftail;
59 69
60 lr = ptrtail[0].lr; 70 lr = (is_compat ? buftail.lr_32 : buftail.lr);
61 gator_add_trace(cpu, lr); 71 gator_add_trace(cpu, lr);
62 72
63 /* frame pointers should progress back up the stack, towards higher addresses */ 73 /* frame pointers should progress back up the stack, towards higher addresses */
64 next = (struct frame_tail_eabi *)(lr - frame_offset); 74 next = (struct frame_tail_eabi *)(lr - frame_offset);
65 if (tail >= next || lr == 0) { 75 if (tail >= next || lr == 0) {
66 fp = ptrtail[0].fp; 76 fp = (is_compat ? buftail.fp_32 : buftail.fp);
67 next = (struct frame_tail_eabi *)(fp - frame_offset); 77 next = (struct frame_tail_eabi *)(fp - frame_offset);
68 /* check tail is valid */ 78 /* check tail is valid */
69 if (tail >= next || fp == 0) { 79 if (tail >= next || fp == 0) {
@@ -79,16 +89,17 @@ static void arm_backtrace_eabi(int cpu, struct pt_regs *const regs, unsigned int
79#if defined(__arm__) || defined(__aarch64__) 89#if defined(__arm__) || defined(__aarch64__)
80static int report_trace(struct stackframe *frame, void *d) 90static int report_trace(struct stackframe *frame, void *d)
81{ 91{
82 struct module *mod; 92 unsigned int *depth = d, cookie = NO_COOKIE, cpu = get_physical_cpu();
83 unsigned int *depth = d, cookie = NO_COOKIE, cpu = smp_processor_id();
84 unsigned long addr = frame->pc; 93 unsigned long addr = frame->pc;
85 94
86 if (*depth) { 95 if (*depth) {
87 mod = __module_address(addr); 96#if defined(MODULE)
97 struct module *mod = __module_address(addr);
88 if (mod) { 98 if (mod) {
89 cookie = get_cookie(cpu, current, mod->name, false); 99 cookie = get_cookie(cpu, current, mod->name, false);
90 addr = addr - (unsigned long)mod->module_core; 100 addr = addr - (unsigned long)mod->module_core;
91 } 101 }
102#endif
92 marshal_backtrace(addr & ~1, cookie); 103 marshal_backtrace(addr & ~1, cookie);
93 (*depth)--; 104 (*depth)--;
94 } 105 }
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index bb401bb..c332187 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -61,7 +61,7 @@ static uint32_t gator_chksum_crc32(const char *data)
61static uint32_t cookiemap_exists(uint64_t key) 61static uint32_t cookiemap_exists(uint64_t key)
62{ 62{
63 unsigned long x, flags, retval = 0; 63 unsigned long x, flags, retval = 0;
64 int cpu = smp_processor_id(); 64 int cpu = get_physical_cpu();
65 uint32_t cookiecode = cookiemap_code(key); 65 uint32_t cookiecode = cookiemap_code(key);
66 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); 66 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
67 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); 67 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
@@ -93,7 +93,7 @@ static uint32_t cookiemap_exists(uint64_t key)
93 */ 93 */
94static void cookiemap_add(uint64_t key, uint32_t value) 94static void cookiemap_add(uint64_t key, uint32_t value)
95{ 95{
96 int cpu = smp_processor_id(); 96 int cpu = get_physical_cpu();
97 int cookiecode = cookiemap_code(key); 97 int cookiecode = cookiemap_code(key);
98 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); 98 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
99 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); 99 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
@@ -124,7 +124,7 @@ static void wq_cookie_handler(struct work_struct *unused)
124{ 124{
125 struct task_struct *task; 125 struct task_struct *task;
126 char *text; 126 char *text;
127 int cpu = smp_processor_id(); 127 int cpu = get_physical_cpu();
128 unsigned int commit; 128 unsigned int commit;
129 129
130 mutex_lock(&start_mutex); 130 mutex_lock(&start_mutex);
diff --git a/driver/gator_events.sh b/driver/gator_events.sh
index 5467dd6..5467dd6 100644..100755
--- a/driver/gator_events.sh
+++ b/driver/gator_events.sh
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c
index ee36dd0..4f1bca6 100644
--- a/driver/gator_events_armv6.c
+++ b/driver/gator_events_armv6.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -93,7 +93,7 @@ int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root)
93 return 0; 93 return 0;
94} 94}
95 95
96static int gator_events_armv6_online(int **buffer) 96static int gator_events_armv6_online(int **buffer, bool migrate)
97{ 97{
98 unsigned int cnt, len = 0, cpu = smp_processor_id(); 98 unsigned int cnt, len = 0, cpu = smp_processor_id();
99 u32 pmnc; 99 u32 pmnc;
@@ -141,7 +141,7 @@ static int gator_events_armv6_online(int **buffer)
141 return len; 141 return len;
142} 142}
143 143
144static int gator_events_armv6_offline(int **buffer) 144static int gator_events_armv6_offline(int **buffer, bool migrate)
145{ 145{
146 unsigned int cnt; 146 unsigned int cnt;
147 147
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index 212b17b..58f2956 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -159,7 +159,7 @@ static int gator_events_armv7_create_files(struct super_block *sb, struct dentry
159 return 0; 159 return 0;
160} 160}
161 161
162static int gator_events_armv7_online(int **buffer) 162static int gator_events_armv7_online(int **buffer, bool migrate)
163{ 163{
164 unsigned int cnt, len = 0, cpu = smp_processor_id(); 164 unsigned int cnt, len = 0, cpu = smp_processor_id();
165 165
@@ -214,7 +214,7 @@ static int gator_events_armv7_online(int **buffer)
214 return len; 214 return len;
215} 215}
216 216
217static int gator_events_armv7_offline(int **buffer) 217static int gator_events_armv7_offline(int **buffer, bool migrate)
218{ 218{
219 // disable all counters, including PMCCNTR; overflow IRQs will not be signaled 219 // disable all counters, including PMCCNTR; overflow IRQs will not be signaled
220 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); 220 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c
index f512b13..56c6a67 100644
--- a/driver/gator_events_block.c
+++ b/driver/gator_events_block.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -30,7 +30,6 @@ static int blockGet[BLOCK_TOTAL * 4];
30 30
31GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq)) 31GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq))
32{ 32{
33 unsigned long flags;
34 int write, size; 33 int write, size;
35 34
36 if (!rq) 35 if (!rq)
@@ -42,9 +41,6 @@ GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct r
42 if (!size) 41 if (!size)
43 return; 42 return;
44 43
45 // disable interrupts to synchronize with gator_events_block_read()
46 // spinlocks not needed since percpu buffers are used
47 local_irq_save(flags);
48 if (write) { 44 if (write) {
49 if (block_rq_wr_enabled) { 45 if (block_rq_wr_enabled) {
50 atomic_add(size, &blockCnt[BLOCK_RQ_WR]); 46 atomic_add(size, &blockCnt[BLOCK_RQ_WR]);
@@ -54,7 +50,6 @@ GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct r
54 atomic_add(size, &blockCnt[BLOCK_RQ_RD]); 50 atomic_add(size, &blockCnt[BLOCK_RQ_RD]);
55 } 51 }
56 } 52 }
57 local_irq_restore(flags);
58} 53}
59 54
60static int gator_events_block_create_files(struct super_block *sb, struct dentry *root) 55static int gator_events_block_create_files(struct super_block *sb, struct dentry *root)
@@ -111,7 +106,7 @@ static int gator_events_block_read(int **buffer)
111{ 106{
112 int len, value, data = 0; 107 int len, value, data = 0;
113 108
114 if (smp_processor_id() != 0) { 109 if (!on_primary_core()) {
115 return 0; 110 return 0;
116 } 111 }
117 112
diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c
index 1221372..b4df7fa 100644
--- a/driver/gator_events_irq.c
+++ b/driver/gator_events_irq.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -18,19 +18,13 @@ static ulong hardirq_enabled;
18static ulong softirq_enabled; 18static ulong softirq_enabled;
19static ulong hardirq_key; 19static ulong hardirq_key;
20static ulong softirq_key; 20static ulong softirq_key;
21static DEFINE_PER_CPU(int[TOTALIRQ], irqCnt); 21static DEFINE_PER_CPU(atomic_t[TOTALIRQ], irqCnt);
22static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet); 22static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet);
23 23
24GATOR_DEFINE_PROBE(irq_handler_exit, 24GATOR_DEFINE_PROBE(irq_handler_exit,
25 TP_PROTO(int irq, struct irqaction *action, int ret)) 25 TP_PROTO(int irq, struct irqaction *action, int ret))
26{ 26{
27 unsigned long flags; 27 atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[HARDIRQ]);
28
29 // disable interrupts to synchronize with gator_events_irq_read()
30 // spinlocks not needed since percpu buffers are used
31 local_irq_save(flags);
32 per_cpu(irqCnt, smp_processor_id())[HARDIRQ]++;
33 local_irq_restore(flags);
34} 28}
35 29
36#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37) 30#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
@@ -39,13 +33,7 @@ GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(struct softirq_action *h, struct softi
39GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr)) 33GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr))
40#endif 34#endif
41{ 35{
42 unsigned long flags; 36 atomic_inc(&per_cpu(irqCnt, get_physical_cpu())[SOFTIRQ]);
43
44 // disable interrupts to synchronize with gator_events_irq_read()
45 // spinlocks not needed since percpu buffers are used
46 local_irq_save(flags);
47 per_cpu(irqCnt, smp_processor_id())[SOFTIRQ]++;
48 local_irq_restore(flags);
49} 37}
50 38
51static int gator_events_irq_create_files(struct super_block *sb, struct dentry *root) 39static int gator_events_irq_create_files(struct super_block *sb, struct dentry *root)
@@ -71,24 +59,19 @@ static int gator_events_irq_create_files(struct super_block *sb, struct dentry *
71 return 0; 59 return 0;
72} 60}
73 61
74static int gator_events_irq_online(int **buffer) 62static int gator_events_irq_online(int **buffer, bool migrate)
75{ 63{
76 int len = 0, cpu = smp_processor_id(); 64 int len = 0, cpu = get_physical_cpu();
77 unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt
78 65
79 // synchronization with the irq_exit functions is not necessary as the values are being reset 66 // synchronization with the irq_exit functions is not necessary as the values are being reset
80 if (hardirq_enabled) { 67 if (hardirq_enabled) {
81 local_irq_save(flags); 68 atomic_set(&per_cpu(irqCnt, cpu)[HARDIRQ], 0);
82 per_cpu(irqCnt, cpu)[HARDIRQ] = 0;
83 local_irq_restore(flags);
84 per_cpu(irqGet, cpu)[len++] = hardirq_key; 69 per_cpu(irqGet, cpu)[len++] = hardirq_key;
85 per_cpu(irqGet, cpu)[len++] = 0; 70 per_cpu(irqGet, cpu)[len++] = 0;
86 } 71 }
87 72
88 if (softirq_enabled) { 73 if (softirq_enabled) {
89 local_irq_save(flags); 74 atomic_set(&per_cpu(irqCnt, cpu)[SOFTIRQ], 0);
90 per_cpu(irqCnt, cpu)[SOFTIRQ] = 0;
91 local_irq_restore(flags);
92 per_cpu(irqGet, cpu)[len++] = softirq_key; 75 per_cpu(irqGet, cpu)[len++] = softirq_key;
93 per_cpu(irqGet, cpu)[len++] = 0; 76 per_cpu(irqGet, cpu)[len++] = 0;
94 } 77 }
@@ -136,26 +119,21 @@ static void gator_events_irq_stop(void)
136 119
137static int gator_events_irq_read(int **buffer) 120static int gator_events_irq_read(int **buffer)
138{ 121{
139 unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt
140 int len, value; 122 int len, value;
141 int cpu = smp_processor_id(); 123 int cpu = get_physical_cpu();
142 124
143 len = 0; 125 len = 0;
144 if (hardirq_enabled) { 126 if (hardirq_enabled) {
145 local_irq_save(flags); 127 value = atomic_read(&per_cpu(irqCnt, cpu)[HARDIRQ]);
146 value = per_cpu(irqCnt, cpu)[HARDIRQ]; 128 atomic_sub(value, &per_cpu(irqCnt, cpu)[HARDIRQ]);
147 per_cpu(irqCnt, cpu)[HARDIRQ] = 0;
148 local_irq_restore(flags);
149 129
150 per_cpu(irqGet, cpu)[len++] = hardirq_key; 130 per_cpu(irqGet, cpu)[len++] = hardirq_key;
151 per_cpu(irqGet, cpu)[len++] = value; 131 per_cpu(irqGet, cpu)[len++] = value;
152 } 132 }
153 133
154 if (softirq_enabled) { 134 if (softirq_enabled) {
155 local_irq_save(flags); 135 value = atomic_read(&per_cpu(irqCnt, cpu)[SOFTIRQ]);
156 value = per_cpu(irqCnt, cpu)[SOFTIRQ]; 136 atomic_sub(value, &per_cpu(irqCnt, cpu)[SOFTIRQ]);
157 per_cpu(irqCnt, cpu)[SOFTIRQ] = 0;
158 local_irq_restore(flags);
159 137
160 per_cpu(irqGet, cpu)[len++] = softirq_key; 138 per_cpu(irqGet, cpu)[len++] = softirq_key;
161 per_cpu(irqGet, cpu)[len++] = value; 139 per_cpu(irqGet, cpu)[len++] = value;
diff --git a/driver/gator_events_l2c-310.c b/driver/gator_events_l2c-310.c
index 197af04..52472c7 100644
--- a/driver/gator_events_l2c-310.c
+++ b/driver/gator_events_l2c-310.c
@@ -1,7 +1,7 @@
1/** 1/**
2 * l2c310 (L2 Cache Controller) event counters for gator 2 * l2c310 (L2 Cache Controller) event counters for gator
3 * 3 *
4 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 4 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -10,6 +10,7 @@
10 10
11#include <linux/init.h> 11#include <linux/init.h>
12#include <linux/io.h> 12#include <linux/io.h>
13#include <linux/module.h>
13#include <asm/hardware/cache-l2x0.h> 14#include <asm/hardware/cache-l2x0.h>
14 15
15#include "gator.h" 16#include "gator.h"
@@ -95,7 +96,7 @@ static int gator_events_l2c310_read(int **buffer)
95 int i; 96 int i;
96 int len = 0; 97 int len = 0;
97 98
98 if (smp_processor_id()) 99 if (!on_primary_core())
99 return 0; 100 return 0;
100 101
101 for (i = 0; i < L2C310_COUNTERS_NUM; i++) { 102 for (i = 0; i < L2C310_COUNTERS_NUM; i++) {
@@ -122,20 +123,48 @@ static struct gator_interface gator_events_l2c310_interface = {
122 .read = gator_events_l2c310_read, 123 .read = gator_events_l2c310_read,
123}; 124};
124 125
125static void __maybe_unused gator_events_l2c310_probe(unsigned long phys) 126#define L2C310_ADDR_PROBE (~0)
127
128MODULE_PARM_DESC(l2c310_addr, "L2C310 physical base address (0 to disable)");
129static unsigned long l2c310_addr = L2C310_ADDR_PROBE;
130module_param(l2c310_addr, ulong, 0444);
131
132static void __iomem *gator_events_l2c310_probe(void)
126{ 133{
127 if (l2c310_base) 134 phys_addr_t variants[] = {
128 return; 135#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310)
136 0x10502000,
137#endif
138#if defined(CONFIG_ARCH_OMAP4)
139 0x48242000,
140#endif
141#if defined(CONFIG_ARCH_TEGRA)
142 0x50043000,
143#endif
144#if defined(CONFIG_ARCH_U8500)
145 0xa0412000,
146#endif
147#if defined(CONFIG_ARCH_VEXPRESS)
148 0x1e00a000, // A9x4 core tile (HBI-0191)
149 0x2c0f0000, // New memory map tiles
150#endif
151 };
152 int i;
129 153
130 l2c310_base = ioremap(phys, SZ_4K); 154 for (i = 0; i < ARRAY_SIZE(variants); i++) {
131 if (l2c310_base) { 155 void __iomem *base = ioremap(variants[i], SZ_4K);
132 u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID);
133 156
134 if ((cache_id & 0xff0003c0) != 0x410000c0) { 157 if (base) {
135 iounmap(l2c310_base); 158 u32 cache_id = readl(base + L2X0_CACHE_ID);
136 l2c310_base = NULL; 159
160 if ((cache_id & 0xff0003c0) == 0x410000c0)
161 return base;
162
163 iounmap(base);
137 } 164 }
138 } 165 }
166
167 return NULL;
139} 168}
140 169
141int gator_events_l2c310_init(void) 170int gator_events_l2c310_init(void)
@@ -145,24 +174,11 @@ int gator_events_l2c310_init(void)
145 if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) 174 if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9)
146 return -1; 175 return -1;
147 176
148#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) 177 if (l2c310_addr == L2C310_ADDR_PROBE)
149 gator_events_l2c310_probe(0x10502000); 178 l2c310_base = gator_events_l2c310_probe();
150#endif 179 else if (l2c310_addr)
151#if defined(CONFIG_ARCH_OMAP4) 180 l2c310_base = ioremap(l2c310_addr, SZ_4K);
152 gator_events_l2c310_probe(0x48242000); 181
153#endif
154#if defined(CONFIG_ARCH_TEGRA)
155 gator_events_l2c310_probe(0x50043000);
156#endif
157#if defined(CONFIG_ARCH_U8500)
158 gator_events_l2c310_probe(0xa0412000);
159#endif
160#if defined(CONFIG_ARCH_VEXPRESS)
161 // A9x4 core tile (HBI-0191)
162 gator_events_l2c310_probe(0x1e00a000);
163 // New memory map tiles
164 gator_events_l2c310_probe(0x2c0f0000);
165#endif
166 if (!l2c310_base) 182 if (!l2c310_base)
167 return -1; 183 return -1;
168 184
diff --git a/driver/gator_events_mali_400.c b/driver/gator_events_mali_400.c
index 4888b54..38c97d1 100644
--- a/driver/gator_events_mali_400.c
+++ b/driver/gator_events_mali_400.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -17,14 +17,6 @@
17#include "gator_events_mali_common.h" 17#include "gator_events_mali_common.h"
18#include "gator_events_mali_400.h" 18#include "gator_events_mali_400.h"
19 19
20#if !defined(GATOR_MALI_INTERFACE_STYLE)
21/*
22 * At the moment, we only have users with the old style interface, so
23 * make our life easier by making it the default...
24 */
25#define GATOR_MALI_INTERFACE_STYLE (2)
26#endif
27
28/* 20/*
29 * There are (currently) three different variants of the comms between gator and Mali: 21 * There are (currently) three different variants of the comms between gator and Mali:
30 * 1 (deprecated): No software counter support 22 * 1 (deprecated): No software counter support
@@ -638,7 +630,7 @@ static int read(int **buffer)
638{ 630{
639 int cnt, len = 0; 631 int cnt, len = 0;
640 632
641 if (smp_processor_id()) 633 if (!on_primary_core())
642 return 0; 634 return 0;
643 635
644 // Read the L2 C0 and C1 here. 636 // Read the L2 C0 and C1 here.
diff --git a/driver/gator_events_mali_400.h b/driver/gator_events_mali_400.h
index a09757e..43aec49 100644
--- a/driver/gator_events_mali_400.h
+++ b/driver/gator_events_mali_400.h
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_mali_common.c b/driver/gator_events_mali_common.c
index 2186eee..22a517d 100644
--- a/driver/gator_events_mali_common.c
+++ b/driver/gator_events_mali_common.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_mali_common.h b/driver/gator_events_mali_common.h
index 8e33edf..27eaacc 100644
--- a/driver/gator_events_mali_common.h
+++ b/driver/gator_events_mali_common.h
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_mali_t6xx.c b/driver/gator_events_mali_t6xx.c
index 1b3a53d..2576a99 100644
--- a/driver/gator_events_mali_t6xx.c
+++ b/driver/gator_events_mali_t6xx.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -411,7 +411,7 @@ static int read(int **buffer)
411 long sample_interval_us = 0; 411 long sample_interval_us = 0;
412 struct timespec read_timestamp; 412 struct timespec read_timestamp;
413 413
414 if (smp_processor_id() != 0) { 414 if (!on_primary_core()) {
415 return 0; 415 return 0;
416 } 416 }
417 417
diff --git a/driver/gator_events_mali_t6xx_hw.c b/driver/gator_events_mali_t6xx_hw.c
index 72498c8..fb2e15c 100644
--- a/driver/gator_events_mali_t6xx_hw.c
+++ b/driver/gator_events_mali_t6xx_hw.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -608,7 +608,7 @@ static int read(int **buffer)
608 static u32 prev_time_s = 0; 608 static u32 prev_time_s = 0;
609 static s32 next_read_time_ns = 0; 609 static s32 next_read_time_ns = 0;
610 610
611 if (smp_processor_id() != 0) { 611 if (!on_primary_core()) {
612 return 0; 612 return 0;
613 } 613 }
614 614
diff --git a/driver/gator_events_mali_t6xx_hw_test.c b/driver/gator_events_mali_t6xx_hw_test.c
index eb77110..efb32dd 100644
--- a/driver/gator_events_mali_t6xx_hw_test.c
+++ b/driver/gator_events_mali_t6xx_hw_test.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c
index fd063b2..c1e360d 100644
--- a/driver/gator_events_meminfo.c
+++ b/driver/gator_events_meminfo.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -198,7 +198,7 @@ static int gator_events_meminfo_read(long long **buffer)
198{ 198{
199 static unsigned int last_mem_event = 0; 199 static unsigned int last_mem_event = 0;
200 200
201 if (smp_processor_id() || !meminfo_global_enabled) 201 if (!on_primary_core() || !meminfo_global_enabled)
202 return 0; 202 return 0;
203 203
204 if (last_mem_event != mem_event) { 204 if (last_mem_event != mem_event) {
diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c
index c4cb44f..0027564 100644
--- a/driver/gator_events_mmaped.c
+++ b/driver/gator_events_mmaped.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * Example events provider 2 * Example events provider
3 * 3 *
4 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 4 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as 7 * it under the terms of the GNU General Public License version 2 as
@@ -141,7 +141,7 @@ static int mmaped_simulate(int counter, int delta_in_us)
141 break; 141 break;
142 case 2: /* PWM signal */ 142 case 2: /* PWM signal */
143 { 143 {
144 static int t, dc, x; 144 static int dc, x, t = 0;
145 145
146 t += delta_in_us; 146 t += delta_in_us;
147 if (t > 1000000) 147 if (t > 1000000)
@@ -170,7 +170,7 @@ static int gator_events_mmaped_read(int **buffer)
170#endif 170#endif
171 171
172 /* System wide counters - read from one core only */ 172 /* System wide counters - read from one core only */
173 if (smp_processor_id()) 173 if (!on_primary_core())
174 return 0; 174 return 0;
175 175
176#ifndef TODO 176#ifndef TODO
diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c
index b6ce06a..80cdee4 100644
--- a/driver/gator_events_net.c
+++ b/driver/gator_events_net.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -117,7 +117,7 @@ static int gator_events_net_read(int **buffer)
117 int len, rx_delta, tx_delta; 117 int len, rx_delta, tx_delta;
118 static int last_rx_delta = 0, last_tx_delta = 0; 118 static int last_rx_delta = 0, last_tx_delta = 0;
119 119
120 if (smp_processor_id() != 0) 120 if (!on_primary_core())
121 return 0; 121 return 0;
122 122
123 if (!netrx_enabled && !nettx_enabled) 123 if (!netrx_enabled && !nettx_enabled)
diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c
index ce3a40f..34a6bc7 100644
--- a/driver/gator_events_perf_pmu.c
+++ b/driver/gator_events_perf_pmu.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -13,49 +13,74 @@
13// gator_events_armvX.c is used for Linux 2.6.x 13// gator_events_armvX.c is used for Linux 2.6.x
14#if GATOR_PERF_PMU_SUPPORT 14#if GATOR_PERF_PMU_SUPPORT
15 15
16static const char *pmnc_name; 16extern bool event_based_sampling;
17int pmnc_counters;
18int ccnt = 0;
19 17
20#define CNTMAX (6+1) 18#define CNTMAX 16
19#define CCI_400 4
20// + 1 for the cci-400 cycles counter
21#define UCCNT (CCI_400 + 1)
21 22
22static DEFINE_MUTEX(perf_mutex); 23struct gator_attr {
24 char name[40];
25 unsigned long enabled;
26 unsigned long type;
27 unsigned long event;
28 unsigned long count;
29 unsigned long key;
30};
23 31
24unsigned long pmnc_enabled[CNTMAX]; 32static struct gator_attr attrs[CNTMAX];
25unsigned long pmnc_event[CNTMAX]; 33static int attr_count;
26unsigned long pmnc_count[CNTMAX]; 34static struct gator_attr uc_attrs[UCCNT];
27unsigned long pmnc_key[CNTMAX]; 35static int uc_attr_count;
36
37struct gator_event {
38 int curr;
39 int prev;
40 int prev_delta;
41 bool zero;
42 struct perf_event *pevent;
43 struct perf_event_attr *pevent_attr;
44};
28 45
29static DEFINE_PER_CPU(int[CNTMAX], perfCurr); 46static DEFINE_PER_CPU(struct gator_event[CNTMAX], events);
30static DEFINE_PER_CPU(int[CNTMAX], perfPrev); 47static struct gator_event uc_events[UCCNT];
31static DEFINE_PER_CPU(int[CNTMAX], perfPrevDelta); 48static DEFINE_PER_CPU(int[(CNTMAX + UCCNT)*2], perf_cnt);
32static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
33static DEFINE_PER_CPU(struct perf_event *[CNTMAX], pevent);
34static DEFINE_PER_CPU(struct perf_event_attr *[CNTMAX], pevent_attr);
35 49
36static void gator_events_perf_pmu_stop(void); 50static void gator_events_perf_pmu_stop(void);
37 51
38static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) 52static int __create_files(struct super_block *sb, struct dentry *root, struct gator_attr *const attr)
39{ 53{
40 struct dentry *dir; 54 struct dentry *dir;
41 int i;
42 55
43 for (i = 0; i < pmnc_counters; i++) { 56 if (attr->name[0] == '\0') {
44 char buf[40]; 57 return 0;
45 if (i == 0) { 58 }
46 snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); 59 dir = gatorfs_mkdir(sb, root, attr->name);
47 } else { 60 if (!dir) {
48 snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i - 1); 61 return -1;
49 } 62 }
50 dir = gatorfs_mkdir(sb, root, buf); 63 gatorfs_create_ulong(sb, dir, "enabled", &attr->enabled);
51 if (!dir) { 64 gatorfs_create_ulong(sb, dir, "count", &attr->count);
65 gatorfs_create_ro_ulong(sb, dir, "key", &attr->key);
66 gatorfs_create_ulong(sb, dir, "event", &attr->event);
67
68 return 0;
69}
70
71static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root)
72{
73 int cnt;
74
75 for (cnt = 0; cnt < attr_count; cnt++) {
76 if (__create_files(sb, root, &attrs[cnt]) != 0) {
52 return -1; 77 return -1;
53 } 78 }
54 gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); 79 }
55 gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); 80
56 gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); 81 for (cnt = 0; cnt < uc_attr_count; cnt++) {
57 if (i > 0) { 82 if (__create_files(sb, root, &uc_attrs[cnt]) != 0) {
58 gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); 83 return -1;
59 } 84 }
60 } 85 }
61 86
@@ -80,177 +105,268 @@ static void dummy_handler(struct perf_event *event, struct perf_sample_data *dat
80// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll 105// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll
81} 106}
82 107
83static int gator_events_perf_pmu_online(int **buffer) 108static int gator_events_perf_pmu_read(int **buffer);
109
110static int gator_events_perf_pmu_online(int **buffer, bool migrate)
84{ 111{
85 int cnt, len = 0, cpu = smp_processor_id(); 112 return gator_events_perf_pmu_read(buffer);
113}
86 114
87 // read the counters and toss the invalid data, return zero instead 115static void __online_dispatch(int cpu, bool migrate, struct gator_attr *const attr, struct gator_event *const event)
88 for (cnt = 0; cnt < pmnc_counters; cnt++) { 116{
89 struct perf_event *ev = per_cpu(pevent, cpu)[cnt]; 117 perf_overflow_handler_t handler;
90 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { 118
91 ev->pmu->read(ev); 119 event->zero = true;
92 per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); 120
93 per_cpu(perfPrevDelta, cpu)[cnt] = 0; 121 if (event->pevent != NULL || event->pevent_attr == 0 || migrate) {
94 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; 122 return;
95 per_cpu(perfCnt, cpu)[len++] = 0;
96 }
97 } 123 }
98 124
99 if (buffer) 125 if (attr->count > 0) {
100 *buffer = per_cpu(perfCnt, cpu); 126 handler = ebs_overflow_handler;
127 } else {
128 handler = dummy_handler;
129 }
101 130
102 return len; 131#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
132 event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler);
133#else
134 event->pevent = perf_event_create_kernel_counter(event->pevent_attr, cpu, 0, handler, 0);
135#endif
136 if (IS_ERR(event->pevent)) {
137 pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
138 event->pevent = NULL;
139 return;
140 }
141
142 if (event->pevent->state != PERF_EVENT_STATE_ACTIVE) {
143 pr_debug("gator: inactive counter on cpu %d\n", cpu);
144 perf_event_release_kernel(event->pevent);
145 event->pevent = NULL;
146 return;
147 }
103} 148}
104 149
105static void gator_events_perf_pmu_online_dispatch(int cpu) 150static void gator_events_perf_pmu_online_dispatch(int cpu, bool migrate)
106{ 151{
107 int cnt; 152 int cnt;
108 perf_overflow_handler_t handler;
109 153
110 for (cnt = 0; cnt < pmnc_counters; cnt++) { 154 cpu = pcpu_to_lcpu(cpu);
111 if (per_cpu(pevent, cpu)[cnt] != NULL || per_cpu(pevent_attr, cpu)[cnt] == 0)
112 continue;
113 155
114 if (pmnc_count[cnt] > 0) { 156 for (cnt = 0; cnt < attr_count; cnt++) {
115 handler = ebs_overflow_handler; 157 __online_dispatch(cpu, migrate, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
116 } else { 158 }
117 handler = dummy_handler;
118 }
119 159
120#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) 160 if (cpu == 0) {
121 per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, handler); 161 for (cnt = 0; cnt < uc_attr_count; cnt++) {
122#else 162 __online_dispatch(cpu, migrate, &uc_attrs[cnt], &uc_events[cnt]);
123 per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, handler, 0);
124#endif
125 if (IS_ERR(per_cpu(pevent, cpu)[cnt])) {
126 pr_debug("gator: unable to online a counter on cpu %d\n", cpu);
127 per_cpu(pevent, cpu)[cnt] = NULL;
128 continue;
129 } 163 }
164 }
165}
130 166
131 if (per_cpu(pevent, cpu)[cnt]->state != PERF_EVENT_STATE_ACTIVE) { 167static void __offline_dispatch(int cpu, struct gator_event *const event)
132 pr_debug("gator: inactive counter on cpu %d\n", cpu); 168{
133 perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); 169 struct perf_event *pe = NULL;
134 per_cpu(pevent, cpu)[cnt] = NULL; 170
135 continue; 171 if (event->pevent) {
136 } 172 pe = event->pevent;
173 event->pevent = NULL;
174 }
175
176 if (pe) {
177 perf_event_release_kernel(pe);
137 } 178 }
138} 179}
139 180
140static void gator_events_perf_pmu_offline_dispatch(int cpu) 181static void gator_events_perf_pmu_offline_dispatch(int cpu, bool migrate)
141{ 182{
142 int cnt; 183 int cnt;
143 struct perf_event *pe;
144 184
145 for (cnt = 0; cnt < pmnc_counters; cnt++) { 185 if (migrate) {
146 pe = NULL; 186 return;
147 mutex_lock(&perf_mutex); 187 }
148 if (per_cpu(pevent, cpu)[cnt]) { 188 cpu = pcpu_to_lcpu(cpu);
149 pe = per_cpu(pevent, cpu)[cnt]; 189
150 per_cpu(pevent, cpu)[cnt] = NULL; 190 for (cnt = 0; cnt < attr_count; cnt++) {
191 __offline_dispatch(cpu, &per_cpu(events, cpu)[cnt]);
192 }
193
194 if (cpu == 0) {
195 for (cnt = 0; cnt < uc_attr_count; cnt++) {
196 __offline_dispatch(cpu, &uc_events[cnt]);
151 } 197 }
152 mutex_unlock(&perf_mutex); 198 }
199}
153 200
154 if (pe) { 201static int __check_ebs(struct gator_attr *const attr)
155 perf_event_release_kernel(pe); 202{
203 if (attr->count > 0) {
204 if (!event_based_sampling) {
205 event_based_sampling = true;
206 } else {
207 printk(KERN_WARNING "gator: Only one ebs counter is allowed\n");
208 return -1;
156 } 209 }
157 } 210 }
211
212 return 0;
213}
214
215static int __start(struct gator_attr *const attr, struct gator_event *const event)
216{
217 u32 size = sizeof(struct perf_event_attr);
218
219 event->pevent = NULL;
220 if (!attr->enabled) { // Skip disabled counters
221 return 0;
222 }
223
224 event->prev = 0;
225 event->curr = 0;
226 event->prev_delta = 0;
227 event->pevent_attr = kmalloc(size, GFP_KERNEL);
228 if (!event->pevent_attr) {
229 gator_events_perf_pmu_stop();
230 return -1;
231 }
232
233 memset(event->pevent_attr, 0, size);
234 event->pevent_attr->type = attr->type;
235 event->pevent_attr->size = size;
236 event->pevent_attr->config = attr->event;
237 event->pevent_attr->sample_period = attr->count;
238 event->pevent_attr->pinned = 1;
239
240 return 0;
158} 241}
159 242
160static int gator_events_perf_pmu_start(void) 243static int gator_events_perf_pmu_start(void)
161{ 244{
162 int cnt, cpu; 245 int cnt, cpu;
163 u32 size = sizeof(struct perf_event_attr); 246
164 int found_ebs = false; 247 event_based_sampling = false;
165 248 for (cnt = 0; cnt < attr_count; cnt++) {
166 for (cnt = 0; cnt < pmnc_counters; cnt++) { 249 if (__check_ebs(&attrs[cnt]) != 0) {
167 if (pmnc_count[cnt] > 0) { 250 return -1;
168 if (!found_ebs) { 251 }
169 found_ebs = true; 252 }
170 } else { 253
171 // Only one ebs counter is allowed 254 for (cnt = 0; cnt < uc_attr_count; cnt++) {
172 return -1; 255 if (__check_ebs(&uc_attrs[cnt]) != 0) {
173 } 256 return -1;
174 } 257 }
175 } 258 }
176 259
177 for_each_present_cpu(cpu) { 260 for_each_present_cpu(cpu) {
178 for (cnt = 0; cnt < pmnc_counters; cnt++) { 261 for (cnt = 0; cnt < attr_count; cnt++) {
179 per_cpu(pevent, cpu)[cnt] = NULL; 262 if (__start(&attrs[cnt], &per_cpu(events, cpu)[cnt]) != 0) {
180 if (!pmnc_enabled[cnt]) // Skip disabled counters
181 continue;
182
183 per_cpu(perfPrev, cpu)[cnt] = 0;
184 per_cpu(perfCurr, cpu)[cnt] = 0;
185 per_cpu(perfPrevDelta, cpu)[cnt] = 0;
186 per_cpu(pevent_attr, cpu)[cnt] = kmalloc(size, GFP_KERNEL);
187 if (!per_cpu(pevent_attr, cpu)[cnt]) {
188 gator_events_perf_pmu_stop();
189 return -1; 263 return -1;
190 } 264 }
265 }
266 }
191 267
192 memset(per_cpu(pevent_attr, cpu)[cnt], 0, size); 268 for (cnt = 0; cnt < uc_attr_count; cnt++) {
193 per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_RAW; 269 if (__start(&uc_attrs[cnt], &uc_events[cnt]) != 0) {
194 per_cpu(pevent_attr, cpu)[cnt]->size = size; 270 return -1;
195 per_cpu(pevent_attr, cpu)[cnt]->config = pmnc_event[cnt];
196 per_cpu(pevent_attr, cpu)[cnt]->sample_period = pmnc_count[cnt];
197 per_cpu(pevent_attr, cpu)[cnt]->pinned = 1;
198
199 // handle special case for ccnt
200 if (cnt == ccnt) {
201 per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_HARDWARE;
202 per_cpu(pevent_attr, cpu)[cnt]->config = PERF_COUNT_HW_CPU_CYCLES;
203 }
204 } 271 }
205 } 272 }
206 273
207 return 0; 274 return 0;
208} 275}
209 276
277static void __event_stop(struct gator_event *const event)
278{
279 if (event->pevent_attr) {
280 kfree(event->pevent_attr);
281 event->pevent_attr = NULL;
282 }
283}
284
285static void __attr_stop(struct gator_attr *const attr)
286{
287 attr->enabled = 0;
288 attr->event = 0;
289 attr->count = 0;
290}
291
210static void gator_events_perf_pmu_stop(void) 292static void gator_events_perf_pmu_stop(void)
211{ 293{
212 unsigned int cnt, cpu; 294 unsigned int cnt, cpu;
213 295
214 for_each_present_cpu(cpu) { 296 for_each_present_cpu(cpu) {
215 for (cnt = 0; cnt < pmnc_counters; cnt++) { 297 for (cnt = 0; cnt < attr_count; cnt++) {
216 if (per_cpu(pevent_attr, cpu)[cnt]) { 298 __event_stop(&per_cpu(events, cpu)[cnt]);
217 kfree(per_cpu(pevent_attr, cpu)[cnt]);
218 per_cpu(pevent_attr, cpu)[cnt] = NULL;
219 }
220 } 299 }
221 } 300 }
222 301
223 for (cnt = 0; cnt < pmnc_counters; cnt++) { 302 for (cnt = 0; cnt < uc_attr_count; cnt++) {
224 pmnc_enabled[cnt] = 0; 303 __event_stop(&uc_events[cnt]);
225 pmnc_event[cnt] = 0; 304 }
226 pmnc_count[cnt] = 0; 305
306 for (cnt = 0; cnt < attr_count; cnt++) {
307 __attr_stop(&attrs[cnt]);
308 }
309
310 for (cnt = 0; cnt < uc_attr_count; cnt++) {
311 __attr_stop(&uc_attrs[cnt]);
227 } 312 }
228} 313}
229 314
230static int gator_events_perf_pmu_read(int **buffer) 315static void __read(int *const len, int cpu, struct gator_attr *const attr, struct gator_event *const event)
231{ 316{
232 int cnt, delta, len = 0; 317 int delta;
233 int cpu = smp_processor_id(); 318
234 319 struct perf_event *const ev = event->pevent;
235 for (cnt = 0; cnt < pmnc_counters; cnt++) { 320 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
236 struct perf_event *ev = per_cpu(pevent, cpu)[cnt]; 321 /* After creating the perf counter in __online_dispatch, there
237 if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { 322 * is a race condition between gator_events_perf_pmu_online and
323 * gator_events_perf_pmu_read. So have
324 * gator_events_perf_pmu_online call gator_events_perf_pmu_read
325 * and in __read check to see if it's the first call after
326 * __online_dispatch and if so, run the online code.
327 */
328 if (event->zero) {
329 ev->pmu->read(ev);
330 event->prev = event->curr = local64_read(&ev->count);
331 event->prev_delta = 0;
332 per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
333 per_cpu(perf_cnt, cpu)[(*len)++] = 0;
334 event->zero = false;
335 } else {
238 ev->pmu->read(ev); 336 ev->pmu->read(ev);
239 per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); 337 event->curr = local64_read(&ev->count);
240 delta = per_cpu(perfCurr, cpu)[cnt] - per_cpu(perfPrev, cpu)[cnt]; 338 delta = event->curr - event->prev;
241 if (delta != 0 || delta != per_cpu(perfPrevDelta, cpu)[cnt]) { 339 if (delta != 0 || delta != event->prev_delta) {
242 per_cpu(perfPrevDelta, cpu)[cnt] = delta; 340 event->prev_delta = delta;
243 per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt]; 341 event->prev = event->curr;
244 per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; 342 per_cpu(perf_cnt, cpu)[(*len)++] = attr->key;
245 if (delta < 0) 343 if (delta < 0) {
246 delta *= -1; 344 delta *= -1;
247 per_cpu(perfCnt, cpu)[len++] = delta; 345 }
346 per_cpu(perf_cnt, cpu)[(*len)++] = delta;
248 } 347 }
249 } 348 }
250 } 349 }
350}
251 351
252 if (buffer) 352static int gator_events_perf_pmu_read(int **buffer)
253 *buffer = per_cpu(perfCnt, cpu); 353{
354 int cnt, len = 0;
355 const int cpu = get_logical_cpu();
356
357 for (cnt = 0; cnt < attr_count; cnt++) {
358 __read(&len, cpu, &attrs[cnt], &per_cpu(events, cpu)[cnt]);
359 }
360
361 if (cpu == 0) {
362 for (cnt = 0; cnt < uc_attr_count; cnt++) {
363 __read(&len, cpu, &uc_attrs[cnt], &uc_events[cnt]);
364 }
365 }
366
367 if (buffer) {
368 *buffer = per_cpu(perf_cnt, cpu);
369 }
254 370
255 return len; 371 return len;
256} 372}
@@ -265,30 +381,116 @@ static struct gator_interface gator_events_perf_pmu_interface = {
265 .read = gator_events_perf_pmu_read, 381 .read = gator_events_perf_pmu_read,
266}; 382};
267 383
384static void __attr_init(struct gator_attr *const attr)
385{
386 attr->name[0] = '\0';
387 attr->enabled = 0;
388 attr->type = 0;
389 attr->event = 0;
390 attr->count = 0;
391 attr->key = gator_events_get_key();
392}
393
394static void gator_events_perf_pmu_cci_init(const int type)
395{
396 int cnt;
397
398 strncpy(uc_attrs[uc_attr_count].name, "cci-400_ccnt", sizeof(uc_attrs[uc_attr_count].name));
399 uc_attrs[uc_attr_count].type = type;
400 ++uc_attr_count;
401
402 for (cnt = 0; cnt < CCI_400; ++cnt, ++uc_attr_count) {
403 struct gator_attr *const attr = &uc_attrs[uc_attr_count];
404 snprintf(attr->name, sizeof(attr->name), "cci-400_cnt%d", cnt);
405 attr->type = type;
406 }
407}
408
409static void gator_events_perf_pmu_cpu_init(const struct gator_cpu *const gator_cpu, const int type)
410{
411 int cnt;
412
413 snprintf(attrs[attr_count].name, sizeof(attrs[attr_count].name), "%s_ccnt", gator_cpu->pmnc_name);
414 attrs[attr_count].type = type;
415 ++attr_count;
416
417 for (cnt = 0; cnt < gator_cpu->pmnc_counters; ++cnt, ++attr_count) {
418 struct gator_attr *const attr = &attrs[attr_count];
419 snprintf(attr->name, sizeof(attr->name), "%s_cnt%d", gator_cpu->pmnc_name, cnt);
420 attr->type = type;
421 }
422}
423
268int gator_events_perf_pmu_init(void) 424int gator_events_perf_pmu_init(void)
269{ 425{
270 unsigned int cnt; 426 struct perf_event_attr pea;
271 const u32 cpuid = gator_cpuid(); 427 struct perf_event *pe;
272 428 const struct gator_cpu *gator_cpu;
273 for (cnt = 0; gator_cpus[cnt].cpuid != 0; ++cnt) { 429 int type;
274 if (gator_cpus[cnt].cpuid == cpuid) { 430 int cpu;
275 pmnc_name = gator_cpus[cnt].pmnc_name; 431 int cnt;
276 pmnc_counters = gator_cpus[cnt].pmnc_counters; 432 bool found_cpu = false;
277 ccnt = gator_cpus[cnt].ccnt; 433
434 for (cnt = 0; cnt < CNTMAX; cnt++) {
435 __attr_init(&attrs[cnt]);
436 }
437 for (cnt = 0; cnt < UCCNT; cnt++) {
438 __attr_init(&uc_attrs[cnt]);
439 }
440
441 memset(&pea, 0, sizeof(pea));
442 pea.size = sizeof(pea);
443 pea.config = 0xFF;
444 attr_count = 0;
445 uc_attr_count = 0;
446 for (type = PERF_TYPE_MAX; type < 0x20; ++type) {
447 pea.type = type;
448
449 // A particular PMU may work on some but not all cores, so try on each core
450 pe = NULL;
451 for_each_present_cpu(cpu) {
452#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
453 pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler);
454#else
455 pe = perf_event_create_kernel_counter(&pea, cpu, 0, dummy_handler, 0);
456#endif
457 if (!IS_ERR(pe)) {
458 break;
459 }
460 }
461 // Assume that valid PMUs are contigious
462 if (IS_ERR(pe)) {
278 break; 463 break;
279 } 464 }
465
466 if (pe->pmu != NULL && type == pe->pmu->type) {
467 if (strcmp("CCI", pe->pmu->name) == 0) {
468 gator_events_perf_pmu_cci_init(type);
469 } else if ((gator_cpu = gator_find_cpu_by_pmu_name(pe->pmu->name)) != NULL) {
470 found_cpu = true;
471 gator_events_perf_pmu_cpu_init(gator_cpu, type);
472 }
473 }
474
475 perf_event_release_kernel(pe);
280 } 476 }
281 if (gator_cpus[cnt].cpuid == 0) { 477
282 return -1; 478 if (!found_cpu) {
479 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(gator_cpuid());
480 if (gator_cpu == NULL) {
481 return -1;
482 }
483 gator_events_perf_pmu_cpu_init(gator_cpu, PERF_TYPE_RAW);
283 } 484 }
284 485
285 pmnc_counters++; // CNT[n] + CCNT 486 if (attr_count > CNTMAX) {
487 printk(KERN_ERR "gator: Too many perf counters\n");
488 return -1;
489 }
286 490
287 for (cnt = 0; cnt < CNTMAX; cnt++) { 491 if (uc_attr_count > UCCNT) {
288 pmnc_enabled[cnt] = 0; 492 printk(KERN_ERR "gator: Too many perf uncore counters\n");
289 pmnc_event[cnt] = 0; 493 return -1;
290 pmnc_count[cnt] = 0;
291 pmnc_key[cnt] = gator_events_get_key();
292 } 494 }
293 495
294 return gator_events_install(&gator_events_perf_pmu_interface); 496 return gator_events_install(&gator_events_perf_pmu_interface);
diff --git a/driver/gator_events_sched.c b/driver/gator_events_sched.c
index ba6744d..461a051 100644
--- a/driver/gator_events_sched.c
+++ b/driver/gator_events_sched.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -29,7 +29,7 @@ GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_
29 // disable interrupts to synchronize with gator_events_sched_read() 29 // disable interrupts to synchronize with gator_events_sched_read()
30 // spinlocks not needed since percpu buffers are used 30 // spinlocks not needed since percpu buffers are used
31 local_irq_save(flags); 31 local_irq_save(flags);
32 per_cpu(schedCnt, smp_processor_id())[SCHED_SWITCH]++; 32 per_cpu(schedCnt, get_physical_cpu())[SCHED_SWITCH]++;
33 local_irq_restore(flags); 33 local_irq_restore(flags);
34} 34}
35 35
@@ -78,7 +78,7 @@ static int gator_events_sched_read(int **buffer)
78{ 78{
79 unsigned long flags; 79 unsigned long flags;
80 int len, value; 80 int len, value;
81 int cpu = smp_processor_id(); 81 int cpu = get_physical_cpu();
82 82
83 len = 0; 83 len = 0;
84 if (sched_switch_enabled) { 84 if (sched_switch_enabled) {
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c
index 5ffc63a..aaf306a 100644
--- a/driver/gator_events_scorpion.c
+++ b/driver/gator_events_scorpion.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -524,7 +524,7 @@ static int gator_events_scorpion_create_files(struct super_block *sb, struct den
524 return 0; 524 return 0;
525} 525}
526 526
527static int gator_events_scorpion_online(int **buffer) 527static int gator_events_scorpion_online(int **buffer, bool migrate)
528{ 528{
529 unsigned int cnt, len = 0, cpu = smp_processor_id(); 529 unsigned int cnt, len = 0, cpu = smp_processor_id();
530 530
@@ -581,7 +581,7 @@ static int gator_events_scorpion_online(int **buffer)
581 return len; 581 return len;
582} 582}
583 583
584static int gator_events_scorpion_offline(int **buffer) 584static int gator_events_scorpion_offline(int **buffer, bool migrate)
585{ 585{
586 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); 586 scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E);
587 return 0; 587 return 0;
diff --git a/driver/gator_fs.c b/driver/gator_fs.c
index 9ff118b..fe6f83d 100644
--- a/driver/gator_fs.c
+++ b/driver/gator_fs.c
@@ -53,6 +53,15 @@ ssize_t gatorfs_ulong_to_user(unsigned long val, char __user *buf, size_t count,
53 return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); 53 return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen);
54} 54}
55 55
56ssize_t gatorfs_u64_to_user(u64 val, char __user *buf, size_t count, loff_t *offset)
57{
58 char tmpbuf[TMPBUFSIZE];
59 size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%llu\n", val);
60 if (maxlen > TMPBUFSIZE)
61 maxlen = TMPBUFSIZE;
62 return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen);
63}
64
56int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count) 65int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count)
57{ 66{
58 char tmpbuf[TMPBUFSIZE]; 67 char tmpbuf[TMPBUFSIZE];
@@ -75,12 +84,40 @@ int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t c
75 return 0; 84 return 0;
76} 85}
77 86
87int gatorfs_u64_from_user(u64 *val, char const __user *buf, size_t count)
88{
89 char tmpbuf[TMPBUFSIZE];
90 unsigned long flags;
91
92 if (!count)
93 return 0;
94
95 if (count > TMPBUFSIZE - 1)
96 return -EINVAL;
97
98 memset(tmpbuf, 0x0, TMPBUFSIZE);
99
100 if (copy_from_user(tmpbuf, buf, count))
101 return -EFAULT;
102
103 spin_lock_irqsave(&gatorfs_lock, flags);
104 *val = simple_strtoull(tmpbuf, NULL, 0);
105 spin_unlock_irqrestore(&gatorfs_lock, flags);
106 return 0;
107}
108
78static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) 109static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset)
79{ 110{
80 unsigned long *val = file->private_data; 111 unsigned long *val = file->private_data;
81 return gatorfs_ulong_to_user(*val, buf, count, offset); 112 return gatorfs_ulong_to_user(*val, buf, count, offset);
82} 113}
83 114
115static ssize_t u64_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset)
116{
117 u64 *val = file->private_data;
118 return gatorfs_u64_to_user(*val, buf, count, offset);
119}
120
84static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) 121static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset)
85{ 122{
86 unsigned long *value = file->private_data; 123 unsigned long *value = file->private_data;
@@ -96,6 +133,21 @@ static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_
96 return count; 133 return count;
97} 134}
98 135
136static ssize_t u64_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset)
137{
138 u64 *value = file->private_data;
139 int retval;
140
141 if (*offset)
142 return -EINVAL;
143
144 retval = gatorfs_u64_from_user(value, buf, count);
145
146 if (retval)
147 return retval;
148 return count;
149}
150
99static int default_open(struct inode *inode, struct file *filp) 151static int default_open(struct inode *inode, struct file *filp)
100{ 152{
101 if (inode->i_private) 153 if (inode->i_private)
@@ -109,11 +161,22 @@ static const struct file_operations ulong_fops = {
109 .open = default_open, 161 .open = default_open,
110}; 162};
111 163
164static const struct file_operations u64_fops = {
165 .read = u64_read_file,
166 .write = u64_write_file,
167 .open = default_open,
168};
169
112static const struct file_operations ulong_ro_fops = { 170static const struct file_operations ulong_ro_fops = {
113 .read = ulong_read_file, 171 .read = ulong_read_file,
114 .open = default_open, 172 .open = default_open,
115}; 173};
116 174
175static const struct file_operations u64_ro_fops = {
176 .read = u64_read_file,
177 .open = default_open,
178};
179
117static struct dentry *__gatorfs_create_file(struct super_block *sb, 180static struct dentry *__gatorfs_create_file(struct super_block *sb,
118 struct dentry *root, 181 struct dentry *root,
119 char const *name, 182 char const *name,
@@ -148,6 +211,18 @@ int gatorfs_create_ulong(struct super_block *sb, struct dentry *root,
148 return 0; 211 return 0;
149} 212}
150 213
214int gatorfs_create_u64(struct super_block *sb, struct dentry *root,
215 char const *name, u64 *val)
216{
217 struct dentry *d = __gatorfs_create_file(sb, root, name,
218 &u64_fops, 0644);
219 if (!d)
220 return -EFAULT;
221
222 d->d_inode->i_private = val;
223 return 0;
224}
225
151int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, 226int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root,
152 char const *name, unsigned long *val) 227 char const *name, unsigned long *val)
153{ 228{
@@ -160,6 +235,18 @@ int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root,
160 return 0; 235 return 0;
161} 236}
162 237
238int gatorfs_create_ro_u64(struct super_block *sb, struct dentry *root,
239 char const *name, u64 * val)
240{
241 struct dentry *d =
242 __gatorfs_create_file(sb, root, name, &u64_ro_fops, 0444);
243 if (!d)
244 return -EFAULT;
245
246 d->d_inode->i_private = val;
247 return 0;
248}
249
163static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) 250static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset)
164{ 251{
165 atomic_t *val = file->private_data; 252 atomic_t *val = file->private_data;
diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c
index 846fba4..8c35d49 100644
--- a/driver/gator_hrtimer_gator.c
+++ b/driver/gator_hrtimer_gator.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -15,8 +15,8 @@ void (*callback)(void);
15DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); 15DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
16DEFINE_PER_CPU(int, hrtimer_is_active); 16DEFINE_PER_CPU(int, hrtimer_is_active);
17static ktime_t profiling_interval; 17static ktime_t profiling_interval;
18static void gator_hrtimer_online(int cpu); 18static void gator_hrtimer_online(void);
19static void gator_hrtimer_offline(int cpu); 19static void gator_hrtimer_offline(void);
20 20
21static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) 21static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
22{ 22{
@@ -25,43 +25,28 @@ static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer)
25 return HRTIMER_RESTART; 25 return HRTIMER_RESTART;
26} 26}
27 27
28static void gator_hrtimer_switch_cpus_online(void *unused) 28static void gator_hrtimer_online(void)
29{
30 gator_hrtimer_online(smp_processor_id());
31}
32
33static void gator_hrtimer_online(int cpu)
34{ 29{
30 int cpu = get_logical_cpu();
35 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); 31 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
36 32
37 if (cpu != smp_processor_id()) {
38 smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1);
39 return;
40 }
41
42 if (per_cpu(hrtimer_is_active, cpu) || profiling_interval.tv64 == 0) 33 if (per_cpu(hrtimer_is_active, cpu) || profiling_interval.tv64 == 0)
43 return; 34 return;
44 35
45 per_cpu(hrtimer_is_active, cpu) = 1; 36 per_cpu(hrtimer_is_active, cpu) = 1;
46 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 37 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
47 hrtimer->function = gator_hrtimer_notify; 38 hrtimer->function = gator_hrtimer_notify;
39#ifdef CONFIG_PREEMPT_RT_BASE
40 hrtimer->irqsafe = 1;
41#endif
48 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); 42 hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
49} 43}
50 44
51static void gator_hrtimer_switch_cpus_offline(void *unused) 45static void gator_hrtimer_offline(void)
52{
53 gator_hrtimer_offline(smp_processor_id());
54}
55
56static void gator_hrtimer_offline(int cpu)
57{ 46{
47 int cpu = get_logical_cpu();
58 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); 48 struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
59 49
60 if (cpu != smp_processor_id()) {
61 smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1);
62 return;
63 }
64
65 if (!per_cpu(hrtimer_is_active, cpu)) 50 if (!per_cpu(hrtimer_is_active, cpu))
66 return; 51 return;
67 52
diff --git a/driver/gator_hrtimer_perf.c b/driver/gator_hrtimer_perf.c
index 7c0333f..7b95399 100644
--- a/driver/gator_hrtimer_perf.c
+++ b/driver/gator_hrtimer_perf.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_iks.c b/driver/gator_iks.c
new file mode 100644
index 0000000..6f45c54
--- /dev/null
+++ b/driver/gator_iks.c
@@ -0,0 +1,144 @@
1/**
2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10#if GATOR_IKS_SUPPORT
11
12#include <linux/of.h>
13#include <asm/bL_switcher.h>
14#include <asm/smp_plat.h>
15#include <trace/events/power_cpu_migrate.h>
16
17static int mpidr_cpuids[NR_CPUS];
18static int __lcpu_to_pcpu[NR_CPUS];
19
20static void calc_first_cluster_size(void)
21{
22 int len;
23 const u32 *val;
24 struct device_node *cn = NULL;
25 int mpidr_cpuids_count = 0;
26
27 // Zero is a valid cpuid, so initialize the array to 0xff's
28 memset(&mpidr_cpuids, 0xff, sizeof(mpidr_cpuids));
29
30 while ((cn = of_find_node_by_type(cn, "cpu"))) {
31 BUG_ON(mpidr_cpuids_count >= NR_CPUS);
32
33 val = of_get_property(cn, "reg", &len);
34 if (!val || len != 4) {
35 pr_err("%s missing reg property\n", cn->full_name);
36 continue;
37 }
38
39 mpidr_cpuids[mpidr_cpuids_count] = be32_to_cpup(val);
40 ++mpidr_cpuids_count;
41 }
42
43 BUG_ON(mpidr_cpuids_count != nr_cpu_ids);
44}
45
46static int linearize_mpidr(int mpidr)
47{
48 int i;
49 for (i = 0; i < nr_cpu_ids; ++i) {
50 if (mpidr_cpuids[i] == mpidr) {
51 return i;
52 }
53 }
54
55 BUG();
56}
57
58int lcpu_to_pcpu(const int lcpu)
59{
60 int pcpu;
61 BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0);
62 pcpu = __lcpu_to_pcpu[lcpu];
63 BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0);
64 return pcpu;
65}
66
67int pcpu_to_lcpu(const int pcpu)
68{
69 int lcpu;
70 BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0);
71 for (lcpu = 0; lcpu < nr_cpu_ids; ++lcpu) {
72 if (__lcpu_to_pcpu[lcpu] == pcpu) {
73 BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0);
74 return lcpu;
75 }
76 }
77 BUG();
78}
79
80static void gator_update_cpu_mapping(u32 cpu_hwid)
81{
82 int lcpu = smp_processor_id();
83 int pcpu = linearize_mpidr(cpu_hwid & MPIDR_HWID_BITMASK);
84 BUG_ON(lcpu >= nr_cpu_ids || lcpu < 0);
85 BUG_ON(pcpu >= nr_cpu_ids || pcpu < 0);
86 __lcpu_to_pcpu[lcpu] = pcpu;
87}
88
89GATOR_DEFINE_PROBE(cpu_migrate_begin, TP_PROTO(u64 timestamp, u32 cpu_hwid))
90{
91 const int cpu = get_physical_cpu();
92
93 gator_timer_offline((void *)1);
94 gator_timer_offline_dispatch(cpu, true);
95}
96
97GATOR_DEFINE_PROBE(cpu_migrate_finish, TP_PROTO(u64 timestamp, u32 cpu_hwid))
98{
99 int cpu;
100
101 gator_update_cpu_mapping(cpu_hwid);
102
103 // get_physical_cpu must be called after gator_update_cpu_mapping
104 cpu = get_physical_cpu();
105 gator_timer_online_dispatch(cpu, true);
106 gator_timer_online((void *)1);
107}
108
109GATOR_DEFINE_PROBE(cpu_migrate_current, TP_PROTO(u64 timestamp, u32 cpu_hwid))
110{
111 gator_update_cpu_mapping(cpu_hwid);
112}
113
114static int gator_migrate_start(void)
115{
116 int retval = 0;
117 if (retval == 0)
118 retval = GATOR_REGISTER_TRACE(cpu_migrate_begin);
119 if (retval == 0)
120 retval = GATOR_REGISTER_TRACE(cpu_migrate_finish);
121 if (retval == 0)
122 retval = GATOR_REGISTER_TRACE(cpu_migrate_current);
123 if (retval == 0) {
124 // Initialize the logical to physical cpu mapping
125 memset(&__lcpu_to_pcpu, 0xff, sizeof(__lcpu_to_pcpu));
126 bL_switcher_trace_trigger();
127 }
128 return retval;
129}
130
131static void gator_migrate_stop(void)
132{
133 GATOR_UNREGISTER_TRACE(cpu_migrate_current);
134 GATOR_UNREGISTER_TRACE(cpu_migrate_finish);
135 GATOR_UNREGISTER_TRACE(cpu_migrate_begin);
136}
137
138#else
139
140#define calc_first_cluster_size()
141#define gator_migrate_start() 0
142#define gator_migrate_stop()
143
144#endif
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 8c35caa..3e62b59 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -7,7 +7,8 @@
7 * 7 *
8 */ 8 */
9 9
10static unsigned long gator_protocol_version = 12; 10// This version must match the gator daemon version
11static unsigned long gator_protocol_version = 13;
11 12
12#include <linux/slab.h> 13#include <linux/slab.h>
13#include <linux/cpu.h> 14#include <linux/cpu.h>
@@ -20,6 +21,7 @@ static unsigned long gator_protocol_version = 12;
20#include <linux/suspend.h> 21#include <linux/suspend.h>
21#include <linux/module.h> 22#include <linux/module.h>
22#include <linux/perf_event.h> 23#include <linux/perf_event.h>
24#include <linux/utsname.h>
23#include <asm/stacktrace.h> 25#include <asm/stacktrace.h>
24#include <asm/uaccess.h> 26#include <asm/uaccess.h>
25 27
@@ -48,9 +50,9 @@ static unsigned long gator_protocol_version = 12;
48 50
49#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT)) 51#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
50#ifndef CONFIG_PERF_EVENTS 52#ifndef CONFIG_PERF_EVENTS
51#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters 53#error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
52#elif !defined CONFIG_HW_PERF_EVENTS 54#elif !defined CONFIG_HW_PERF_EVENTS
53#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters 55#error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
54#endif 56#endif
55#endif 57#endif
56 58
@@ -92,9 +94,14 @@ static unsigned long gator_protocol_version = 12;
92#define MESSAGE_SCHED_SWITCH 1 94#define MESSAGE_SCHED_SWITCH 1
93#define MESSAGE_SCHED_EXIT 2 95#define MESSAGE_SCHED_EXIT 2
94 96
97#define MESSAGE_IDLE_ENTER 1
98#define MESSAGE_IDLE_EXIT 2
99
95#define MAXSIZE_PACK32 5 100#define MAXSIZE_PACK32 5
96#define MAXSIZE_PACK64 10 101#define MAXSIZE_PACK64 10
97 102
103#define FRAME_HEADER_SIZE 3
104
98#if defined(__arm__) 105#if defined(__arm__)
99#define PC_REG regs->ARM_pc 106#define PC_REG regs->ARM_pc
100#elif defined(__aarch64__) 107#elif defined(__aarch64__)
@@ -123,23 +130,35 @@ static unsigned long gator_cpu_cores;
123// Size of the largest buffer. Effectively constant, set in gator_op_create_files 130// Size of the largest buffer. Effectively constant, set in gator_op_create_files
124static unsigned long userspace_buffer_size; 131static unsigned long userspace_buffer_size;
125static unsigned long gator_backtrace_depth; 132static unsigned long gator_backtrace_depth;
133// How often to commit the buffers for live in nanoseconds
134static u64 gator_live_rate;
126 135
127static unsigned long gator_started; 136static unsigned long gator_started;
128static uint64_t monotonic_started; 137static u64 gator_monotonic_started;
129static unsigned long gator_buffer_opened; 138static unsigned long gator_buffer_opened;
130static unsigned long gator_timer_count; 139static unsigned long gator_timer_count;
131static unsigned long gator_response_type; 140static unsigned long gator_response_type;
132static DEFINE_MUTEX(start_mutex); 141static DEFINE_MUTEX(start_mutex);
133static DEFINE_MUTEX(gator_buffer_mutex); 142static DEFINE_MUTEX(gator_buffer_mutex);
134 143
144bool event_based_sampling;
145
135static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); 146static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
147static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
136static struct timer_list gator_buffer_wake_up_timer; 148static struct timer_list gator_buffer_wake_up_timer;
137static LIST_HEAD(gator_events); 149static LIST_HEAD(gator_events);
138 150
151static DEFINE_PER_CPU(u64, last_timestamp);
152
153static bool printed_monotonic_warning;
154
155static bool sent_core_name[NR_CPUS];
156
139/****************************************************************************** 157/******************************************************************************
140 * Prototypes 158 * Prototypes
141 ******************************************************************************/ 159 ******************************************************************************/
142static void buffer_check(int cpu, int buftype); 160static void buffer_check(int cpu, int buftype, u64 time);
161static void gator_commit_buffer(int cpu, int buftype, u64 time);
143static int buffer_bytes_available(int cpu, int buftype); 162static int buffer_bytes_available(int cpu, int buftype);
144static bool buffer_check_space(int cpu, int buftype, int bytes); 163static bool buffer_check_space(int cpu, int buftype, int bytes);
145static int contiguous_space_available(int cpu, int bufytpe); 164static int contiguous_space_available(int cpu, int bufytpe);
@@ -149,7 +168,7 @@ static void gator_buffer_write_bytes(int cpu, int buftype, const char *x, int le
149static void gator_buffer_write_string(int cpu, int buftype, const char *x); 168static void gator_buffer_write_string(int cpu, int buftype, const char *x);
150static void gator_add_trace(int cpu, unsigned long address); 169static void gator_add_trace(int cpu, unsigned long address);
151static void gator_add_sample(int cpu, struct pt_regs *const regs); 170static void gator_add_sample(int cpu, struct pt_regs *const regs);
152static uint64_t gator_get_time(void); 171static u64 gator_get_time(void);
153 172
154// Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup. 173// Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup.
155static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; 174static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
@@ -167,6 +186,11 @@ static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
167// The buffer. Allocated in gator_op_setup 186// The buffer. Allocated in gator_op_setup
168static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); 187static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
169 188
189#if GATOR_LIVE
190// The time after which the buffer should be committed for live display
191static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
192#endif
193
170/****************************************************************************** 194/******************************************************************************
171 * Application Includes 195 * Application Includes
172 ******************************************************************************/ 196 ******************************************************************************/
@@ -186,83 +210,85 @@ static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
186 * Misc 210 * Misc
187 ******************************************************************************/ 211 ******************************************************************************/
188 212
189struct gator_cpu gator_cpus[] = { 213const struct gator_cpu gator_cpus[] = {
190 { 214 {
191 .cpuid = ARM1136, 215 .cpuid = ARM1136,
192 .core_name = "ARM1136", 216 .core_name = "ARM1136",
193 .pmnc_name = "ARM_ARM11", 217 .pmnc_name = "ARM_ARM11",
194 .pmnc_counters = 3, 218 .pmnc_counters = 3,
195 .ccnt = 2,
196 }, 219 },
197 { 220 {
198 .cpuid = ARM1156, 221 .cpuid = ARM1156,
199 .core_name = "ARM1156", 222 .core_name = "ARM1156",
200 .pmnc_name = "ARM_ARM11", 223 .pmnc_name = "ARM_ARM11",
201 .pmnc_counters = 3, 224 .pmnc_counters = 3,
202 .ccnt = 2,
203 }, 225 },
204 { 226 {
205 .cpuid = ARM1176, 227 .cpuid = ARM1176,
206 .core_name = "ARM1176", 228 .core_name = "ARM1176",
207 .pmnc_name = "ARM_ARM11", 229 .pmnc_name = "ARM_ARM11",
208 .pmnc_counters = 3, 230 .pmnc_counters = 3,
209 .ccnt = 2,
210 }, 231 },
211 { 232 {
212 .cpuid = ARM11MPCORE, 233 .cpuid = ARM11MPCORE,
213 .core_name = "ARM11MPCore", 234 .core_name = "ARM11MPCore",
214 .pmnc_name = "ARM_ARM11MPCore", 235 .pmnc_name = "ARM_ARM11MPCore",
215 .pmnc_counters = 3, 236 .pmnc_counters = 3,
216 }, 237 },
217 { 238 {
218 .cpuid = CORTEX_A5, 239 .cpuid = CORTEX_A5,
219 .core_name = "Cortex-A5", 240 .core_name = "Cortex-A5",
241 .pmu_name = "ARMv7_Cortex_A5",
220 .pmnc_name = "ARM_Cortex-A5", 242 .pmnc_name = "ARM_Cortex-A5",
221 .pmnc_counters = 2, 243 .pmnc_counters = 2,
222 }, 244 },
223 { 245 {
224 .cpuid = CORTEX_A7, 246 .cpuid = CORTEX_A7,
225 .core_name = "Cortex-A7", 247 .core_name = "Cortex-A7",
248 .pmu_name = "ARMv7_Cortex_A7",
226 .pmnc_name = "ARM_Cortex-A7", 249 .pmnc_name = "ARM_Cortex-A7",
227 .pmnc_counters = 4, 250 .pmnc_counters = 4,
228 }, 251 },
229 { 252 {
230 .cpuid = CORTEX_A8, 253 .cpuid = CORTEX_A8,
231 .core_name = "Cortex-A8", 254 .core_name = "Cortex-A8",
255 .pmu_name = "ARMv7_Cortex_A8",
232 .pmnc_name = "ARM_Cortex-A8", 256 .pmnc_name = "ARM_Cortex-A8",
233 .pmnc_counters = 4, 257 .pmnc_counters = 4,
234 }, 258 },
235 { 259 {
236 .cpuid = CORTEX_A9, 260 .cpuid = CORTEX_A9,
237 .core_name = "Cortex-A9", 261 .core_name = "Cortex-A9",
262 .pmu_name = "ARMv7_Cortex_A9",
238 .pmnc_name = "ARM_Cortex-A9", 263 .pmnc_name = "ARM_Cortex-A9",
239 .pmnc_counters = 6, 264 .pmnc_counters = 6,
240 }, 265 },
241 { 266 {
242 .cpuid = CORTEX_A15, 267 .cpuid = CORTEX_A15,
243 .core_name = "Cortex-A15", 268 .core_name = "Cortex-A15",
269 .pmu_name = "ARMv7_Cortex_A15",
244 .pmnc_name = "ARM_Cortex-A15", 270 .pmnc_name = "ARM_Cortex-A15",
245 .pmnc_counters = 6, 271 .pmnc_counters = 6,
246 }, 272 },
247 { 273 {
248 .cpuid = SCORPION, 274 .cpuid = SCORPION,
249 .core_name = "Scorpion", 275 .core_name = "Scorpion",
250 .pmnc_name = "Scorpion", 276 .pmnc_name = "Scorpion",
251 .pmnc_counters = 4, 277 .pmnc_counters = 4,
252 }, 278 },
253 { 279 {
254 .cpuid = SCORPIONMP, 280 .cpuid = SCORPIONMP,
255 .core_name = "ScorpionMP", 281 .core_name = "ScorpionMP",
256 .pmnc_name = "ScorpionMP", 282 .pmnc_name = "ScorpionMP",
257 .pmnc_counters = 4, 283 .pmnc_counters = 4,
258 }, 284 },
259 { 285 {
260 .cpuid = KRAITSIM, 286 .cpuid = KRAITSIM,
261 .core_name = "KraitSIM", 287 .core_name = "KraitSIM",
262 .pmnc_name = "Krait", 288 .pmnc_name = "Krait",
263 .pmnc_counters = 4, 289 .pmnc_counters = 4,
264 }, 290 },
265 { 291 {
266 .cpuid = KRAIT, 292 .cpuid = KRAIT,
267 .core_name = "Krait", 293 .core_name = "Krait",
268 .pmnc_name = "Krait", 294 .pmnc_name = "Krait",
@@ -274,19 +300,19 @@ struct gator_cpu gator_cpus[] = {
274 .pmnc_name = "Krait", 300 .pmnc_name = "Krait",
275 .pmnc_counters = 4, 301 .pmnc_counters = 4,
276 }, 302 },
277 { 303 {
278 .cpuid = CORTEX_A53, 304 .cpuid = CORTEX_A53,
279 .core_name = "Cortex-A53", 305 .core_name = "Cortex-A53",
280 .pmnc_name = "ARM_Cortex-A53", 306 .pmnc_name = "ARM_Cortex-A53",
281 .pmnc_counters = 6, 307 .pmnc_counters = 6,
282 }, 308 },
283 { 309 {
284 .cpuid = CORTEX_A57, 310 .cpuid = CORTEX_A57,
285 .core_name = "Cortex-A57", 311 .core_name = "Cortex-A57",
286 .pmnc_name = "ARM_Cortex-A57", 312 .pmnc_name = "ARM_Cortex-A57",
287 .pmnc_counters = 6, 313 .pmnc_counters = 6,
288 }, 314 },
289 { 315 {
290 .cpuid = AARCH64, 316 .cpuid = AARCH64,
291 .core_name = "AArch64", 317 .core_name = "AArch64",
292 .pmnc_name = "ARM_AArch64", 318 .pmnc_name = "ARM_AArch64",
@@ -298,9 +324,37 @@ struct gator_cpu gator_cpus[] = {
298 .pmnc_name = "Other", 324 .pmnc_name = "Other",
299 .pmnc_counters = 6, 325 .pmnc_counters = 6,
300 }, 326 },
301 {} 327 {}
302}; 328};
303 329
330const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
331{
332 int i;
333
334 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
335 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
336 if (gator_cpu->cpuid == cpuid) {
337 return gator_cpu;
338 }
339 }
340
341 return NULL;
342}
343
344const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
345{
346 int i;
347
348 for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
349 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
350 if (gator_cpu->pmu_name != NULL && strcmp(gator_cpu->pmu_name, name) == 0) {
351 return gator_cpu;
352 }
353 }
354
355 return NULL;
356}
357
304u32 gator_cpuid(void) 358u32 gator_cpuid(void)
305{ 359{
306#if defined(__arm__) || defined(__aarch64__) 360#if defined(__arm__) || defined(__aarch64__)
@@ -408,7 +462,7 @@ static void gator_buffer_write_string(int cpu, int buftype, const char *x)
408 gator_buffer_write_bytes(cpu, buftype, x, len); 462 gator_buffer_write_bytes(cpu, buftype, x, len);
409} 463}
410 464
411static void gator_commit_buffer(int cpu, int buftype) 465static void gator_commit_buffer(int cpu, int buftype, u64 time)
412{ 466{
413 int type_length, commit, length, byte; 467 int type_length, commit, length, byte;
414 468
@@ -423,25 +477,40 @@ static void gator_commit_buffer(int cpu, int buftype)
423 length += gator_buffer_size[buftype]; 477 length += gator_buffer_size[buftype];
424 } 478 }
425 length = length - type_length - sizeof(int); 479 length = length - type_length - sizeof(int);
480
481 if (length <= FRAME_HEADER_SIZE) {
482 // Nothing to write, only the frame header is present
483 return;
484 }
485
426 for (byte = 0; byte < sizeof(int); byte++) { 486 for (byte = 0; byte < sizeof(int); byte++) {
427 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF; 487 per_cpu(gator_buffer, cpu)[buftype][(commit + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
428 } 488 }
429 489
430 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; 490 per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
491
492#if GATOR_LIVE
493 if (gator_live_rate > 0) {
494 while (time > per_cpu(gator_buffer_commit_time, cpu)) {
495 per_cpu(gator_buffer_commit_time, cpu) += gator_live_rate;
496 }
497 }
498#endif
499
431 marshal_frame(cpu, buftype); 500 marshal_frame(cpu, buftype);
432 501
433 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater 502 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
434 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1); 503 mod_timer(&gator_buffer_wake_up_timer, jiffies + 1);
435} 504}
436 505
437static void buffer_check(int cpu, int buftype) 506static void buffer_check(int cpu, int buftype, u64 time)
438{ 507{
439 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; 508 int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype];
440 if (filled < 0) { 509 if (filled < 0) {
441 filled += gator_buffer_size[buftype]; 510 filled += gator_buffer_size[buftype];
442 } 511 }
443 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) { 512 if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) {
444 gator_commit_buffer(cpu, buftype); 513 gator_commit_buffer(cpu, buftype, time);
445 } 514 }
446} 515}
447 516
@@ -496,7 +565,7 @@ static void gator_timer_interrupt(void)
496 565
497void gator_backtrace_handler(struct pt_regs *const regs) 566void gator_backtrace_handler(struct pt_regs *const regs)
498{ 567{
499 int cpu = smp_processor_id(); 568 int cpu = get_physical_cpu();
500 569
501 // Output backtrace 570 // Output backtrace
502 gator_add_sample(cpu, regs); 571 gator_add_sample(cpu, regs);
@@ -510,40 +579,48 @@ void gator_backtrace_handler(struct pt_regs *const regs)
510static int gator_running; 579static int gator_running;
511 580
512// This function runs in interrupt context and on the appropriate core 581// This function runs in interrupt context and on the appropriate core
513static void gator_timer_offline(void *unused) 582static void gator_timer_offline(void *migrate)
514{ 583{
515 struct gator_interface *gi; 584 struct gator_interface *gi;
516 int i, len, cpu = smp_processor_id(); 585 int i, len, cpu = get_physical_cpu();
517 int *buffer; 586 int *buffer;
587 u64 time;
518 588
519 gator_trace_sched_offline(); 589 gator_trace_sched_offline();
520 gator_trace_power_offline(); 590 gator_trace_power_offline();
521 591
522 gator_hrtimer_offline(cpu); 592 if (!migrate) {
593 gator_hrtimer_offline();
594 }
523 595
524 // Offline any events and output counters 596 // Offline any events and output counters
597 time = gator_get_time();
525 if (marshal_event_header()) { 598 if (marshal_event_header()) {
526 list_for_each_entry(gi, &gator_events, list) { 599 list_for_each_entry(gi, &gator_events, list) {
527 if (gi->offline) { 600 if (gi->offline) {
528 len = gi->offline(&buffer); 601 len = gi->offline(&buffer, migrate);
529 marshal_event(len, buffer); 602 marshal_event(len, buffer);
530 } 603 }
531 } 604 }
605 // Only check after writing all counters so that time and corresponding counters appear in the same frame
606 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
532 } 607 }
533 608
534 // Flush all buffers on this core 609 // Flush all buffers on this core
535 for (i = 0; i < NUM_GATOR_BUFS; i++) 610 for (i = 0; i < NUM_GATOR_BUFS; i++)
536 gator_commit_buffer(cpu, i); 611 gator_commit_buffer(cpu, i, time);
537} 612}
538 613
539// This function runs in process context and may be running on a core other than core 'cpu' 614// This function runs in interrupt context and may be running on a core other than core 'cpu'
540static void gator_timer_offline_dispatch(int cpu) 615static void gator_timer_offline_dispatch(int cpu, bool migrate)
541{ 616{
542 struct gator_interface *gi; 617 struct gator_interface *gi;
543 618
544 list_for_each_entry(gi, &gator_events, list) 619 list_for_each_entry(gi, &gator_events, list) {
545 if (gi->offline_dispatch) 620 if (gi->offline_dispatch) {
546 gi->offline_dispatch(cpu); 621 gi->offline_dispatch(cpu, migrate);
622 }
623 }
547} 624}
548 625
549static void gator_timer_stop(void) 626static void gator_timer_stop(void)
@@ -553,7 +630,7 @@ static void gator_timer_stop(void)
553 if (gator_running) { 630 if (gator_running) {
554 on_each_cpu(gator_timer_offline, NULL, 1); 631 on_each_cpu(gator_timer_offline, NULL, 1);
555 for_each_online_cpu(cpu) { 632 for_each_online_cpu(cpu) {
556 gator_timer_offline_dispatch(cpu); 633 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
557 } 634 }
558 635
559 gator_running = 0; 636 gator_running = 0;
@@ -562,10 +639,10 @@ static void gator_timer_stop(void)
562} 639}
563 640
564// This function runs in interrupt context and on the appropriate core 641// This function runs in interrupt context and on the appropriate core
565static void gator_timer_online(void *unused) 642static void gator_timer_online(void *migrate)
566{ 643{
567 struct gator_interface *gi; 644 struct gator_interface *gi;
568 int len, cpu = smp_processor_id(); 645 int len, cpu = get_physical_cpu();
569 int *buffer; 646 int *buffer;
570 647
571 gator_trace_power_online(); 648 gator_trace_power_online();
@@ -574,39 +651,48 @@ static void gator_timer_online(void *unused)
574 if (marshal_event_header()) { 651 if (marshal_event_header()) {
575 list_for_each_entry(gi, &gator_events, list) { 652 list_for_each_entry(gi, &gator_events, list) {
576 if (gi->online) { 653 if (gi->online) {
577 len = gi->online(&buffer); 654 len = gi->online(&buffer, migrate);
578 marshal_event(len, buffer); 655 marshal_event(len, buffer);
579 } 656 }
580 } 657 }
658 // Only check after writing all counters so that time and corresponding counters appear in the same frame
659 buffer_check(cpu, BLOCK_COUNTER_BUF, gator_get_time());
660 }
661
662 if (!migrate) {
663 gator_hrtimer_online();
581 } 664 }
582 665
583 gator_hrtimer_online(cpu);
584#if defined(__arm__) || defined(__aarch64__) 666#if defined(__arm__) || defined(__aarch64__)
585 { 667 if (!sent_core_name[cpu]) {
586 const char *core_name = "Unknown"; 668 const char *core_name = NULL;
587 const u32 cpuid = gator_cpuid(); 669 const u32 cpuid = gator_cpuid();
588 int i; 670 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
589 671 char core_name_buf[32];
590 for (i = 0; gator_cpus[i].cpuid != 0; ++i) { 672
591 if (gator_cpus[i].cpuid == cpuid) { 673 if (gator_cpu != NULL) {
592 core_name = gator_cpus[i].core_name; 674 core_name = gator_cpu->core_name;
593 break; 675 } else {
594 } 676 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
677 core_name = core_name_buf;
595 } 678 }
596 679
597 marshal_core_name(core_name); 680 marshal_core_name(cpuid, core_name);
681 sent_core_name[cpu] = true;
598 } 682 }
599#endif 683#endif
600} 684}
601 685
602// This function runs in interrupt context and may be running on a core other than core 'cpu' 686// This function runs in interrupt context and may be running on a core other than core 'cpu'
603static void gator_timer_online_dispatch(int cpu) 687static void gator_timer_online_dispatch(int cpu, bool migrate)
604{ 688{
605 struct gator_interface *gi; 689 struct gator_interface *gi;
606 690
607 list_for_each_entry(gi, &gator_events, list) 691 list_for_each_entry(gi, &gator_events, list) {
608 if (gi->online_dispatch) 692 if (gi->online_dispatch) {
609 gi->online_dispatch(cpu); 693 gi->online_dispatch(cpu, migrate);
694 }
695 }
610} 696}
611 697
612int gator_timer_start(unsigned long sample_rate) 698int gator_timer_start(unsigned long sample_rate)
@@ -620,45 +706,72 @@ int gator_timer_start(unsigned long sample_rate)
620 706
621 gator_running = 1; 707 gator_running = 1;
622 708
709 // event based sampling trumps hr timer based sampling
710 if (event_based_sampling) {
711 sample_rate = 0;
712 }
713
623 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1) 714 if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
624 return -1; 715 return -1;
625 716
626 for_each_online_cpu(cpu) { 717 for_each_online_cpu(cpu) {
627 gator_timer_online_dispatch(cpu); 718 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
628 } 719 }
629 on_each_cpu(gator_timer_online, NULL, 1); 720 on_each_cpu(gator_timer_online, NULL, 1);
630 721
631 return 0; 722 return 0;
632} 723}
633 724
634static uint64_t gator_get_time(void) 725static u64 gator_get_time(void)
635{ 726{
636 struct timespec ts; 727 struct timespec ts;
637 uint64_t timestamp; 728 u64 timestamp;
729 u64 prev_timestamp;
730 u64 delta;
731 int cpu = smp_processor_id();
638 732
639 //getnstimeofday(&ts); 733 // Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace
640 do_posix_clock_monotonic_gettime(&ts); 734 getrawmonotonic(&ts);
641 timestamp = timespec_to_ns(&ts) - monotonic_started; 735 timestamp = timespec_to_ns(&ts);
642 736
643 return timestamp; 737 // getrawmonotonic is not monotonic on all systems. Detect and attempt to correct these cases.
738 // up to 0.5ms delta has been seen on some systems, which can skew Streamline data when viewing at high resolution.
739 prev_timestamp = per_cpu(last_timestamp, cpu);
740 if (prev_timestamp <= timestamp) {
741 per_cpu(last_timestamp, cpu) = timestamp;
742 } else {
743 delta = prev_timestamp - timestamp;
744 // Log the error once
745 if (!printed_monotonic_warning && delta > 500000) {
746 printk(KERN_ERR "%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __FUNCTION__, cpu, delta);
747 printed_monotonic_warning = true;
748 } else {
749 pr_debug("%s: getrawmonotonic is not monotonic cpu: %i delta: %lli\n", __FUNCTION__, cpu, delta);
750 }
751 timestamp = prev_timestamp;
752 }
753
754 return timestamp - gator_monotonic_started;
644} 755}
645 756
646/****************************************************************************** 757/******************************************************************************
647 * cpu hotplug and pm notifiers 758 * cpu hotplug and pm notifiers
648 ******************************************************************************/ 759 ******************************************************************************/
760#include "gator_iks.c"
761
649static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 762static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
650{ 763{
651 long cpu = (long)hcpu; 764 int cpu = lcpu_to_pcpu((long)hcpu);
652 765
653 switch (action) { 766 switch (action) {
654 case CPU_DOWN_PREPARE: 767 case CPU_DOWN_PREPARE:
655 case CPU_DOWN_PREPARE_FROZEN: 768 case CPU_DOWN_PREPARE_FROZEN:
656 smp_call_function_single(cpu, gator_timer_offline, NULL, 1); 769 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
657 gator_timer_offline_dispatch(cpu); 770 gator_timer_offline_dispatch(cpu, false);
658 break; 771 break;
659 case CPU_ONLINE: 772 case CPU_ONLINE:
660 case CPU_ONLINE_FROZEN: 773 case CPU_ONLINE_FROZEN:
661 gator_timer_online_dispatch(cpu); 774 gator_timer_online_dispatch(cpu, false);
662 smp_call_function_single(cpu, gator_timer_online, NULL, 1); 775 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
663 break; 776 break;
664 } 777 }
@@ -683,13 +796,13 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void
683 unregister_scheduler_tracepoints(); 796 unregister_scheduler_tracepoints();
684 on_each_cpu(gator_timer_offline, NULL, 1); 797 on_each_cpu(gator_timer_offline, NULL, 1);
685 for_each_online_cpu(cpu) { 798 for_each_online_cpu(cpu) {
686 gator_timer_offline_dispatch(cpu); 799 gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
687 } 800 }
688 break; 801 break;
689 case PM_POST_HIBERNATION: 802 case PM_POST_HIBERNATION:
690 case PM_POST_SUSPEND: 803 case PM_POST_SUSPEND:
691 for_each_online_cpu(cpu) { 804 for_each_online_cpu(cpu) {
692 gator_timer_online_dispatch(cpu); 805 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
693 } 806 }
694 on_each_cpu(gator_timer_online, NULL, 1); 807 on_each_cpu(gator_timer_online, NULL, 1);
695 register_scheduler_tracepoints(); 808 register_scheduler_tracepoints();
@@ -724,15 +837,33 @@ static void gator_notifier_stop(void)
724 ******************************************************************************/ 837 ******************************************************************************/
725static void gator_summary(void) 838static void gator_summary(void)
726{ 839{
727 uint64_t timestamp; 840 u64 timestamp, uptime;
728 struct timespec uptime_ts; 841 struct timespec ts;
842 char uname_buf[512];
843 void (*m2b)(struct timespec *ts);
844 unsigned long flags;
729 845
730 timestamp = gator_get_time(); 846 snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
731 847
732 do_posix_clock_monotonic_gettime(&uptime_ts); 848 getnstimeofday(&ts);
733 monotonic_started = timespec_to_ns(&uptime_ts); 849 timestamp = timespec_to_ns(&ts);
734 850
735 marshal_summary(timestamp, monotonic_started); 851 do_posix_clock_monotonic_gettime(&ts);
852 // monotonic_to_bootbased is not defined for some versions of Android
853 m2b = symbol_get(monotonic_to_bootbased);
854 if (m2b) {
855 m2b(&ts);
856 }
857 uptime = timespec_to_ns(&ts);
858
859 // Disable interrupts as gator_get_time calls smp_processor_id to verify time is monotonic
860 local_irq_save(flags);
861 // Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started
862 gator_monotonic_started = 0;
863 gator_monotonic_started = gator_get_time();
864 local_irq_restore(flags);
865
866 marshal_summary(timestamp, uptime, uname_buf);
736} 867}
737 868
738int gator_events_install(struct gator_interface *interface) 869int gator_events_install(struct gator_interface *interface)
@@ -747,13 +878,17 @@ int gator_events_get_key(void)
747 // key of zero is reserved as a timestamp 878 // key of zero is reserved as a timestamp
748 static int key = 1; 879 static int key = 1;
749 880
750 return key++; 881 const int ret = key;
882 key += 2;
883 return ret;
751} 884}
752 885
753static int gator_init(void) 886static int gator_init(void)
754{ 887{
755 int i; 888 int i;
756 889
890 calc_first_cluster_size();
891
757 // events sources (gator_events.h, generated by gator_events.sh) 892 // events sources (gator_events.h, generated by gator_events.sh)
758 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++) 893 for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
759 if (gator_events_list[i]) 894 if (gator_events_list[i])
@@ -778,14 +913,19 @@ static int gator_start(void)
778 unsigned long cpu, i; 913 unsigned long cpu, i;
779 struct gator_interface *gi; 914 struct gator_interface *gi;
780 915
916 if (gator_migrate_start())
917 goto migrate_failure;
918
781 // Initialize the buffer with the frame type and core 919 // Initialize the buffer with the frame type and core
782 for_each_present_cpu(cpu) { 920 for_each_present_cpu(cpu) {
783 for (i = 0; i < NUM_GATOR_BUFS; i++) { 921 for (i = 0; i < NUM_GATOR_BUFS; i++) {
784 marshal_frame(cpu, i); 922 marshal_frame(cpu, i);
785 } 923 }
924 per_cpu(last_timestamp, cpu) = 0;
786 } 925 }
926 printed_monotonic_warning = false;
787 927
788 // Capture the start time 928 // Capture the start time
789 gator_summary(); 929 gator_summary();
790 930
791 // start all events 931 // start all events
@@ -838,9 +978,11 @@ annotate_failure:
838cookies_failure: 978cookies_failure:
839 // stop all events 979 // stop all events
840 list_for_each_entry(gi, &gator_events, list) 980 list_for_each_entry(gi, &gator_events, list)
841 if (gi->stop) 981 if (gi->stop)
842 gi->stop(); 982 gi->stop();
843events_failure: 983events_failure:
984 gator_migrate_stop();
985migrate_failure:
844 986
845 return -1; 987 return -1;
846} 988}
@@ -860,8 +1002,10 @@ static void gator_stop(void)
860 1002
861 // stop all events 1003 // stop all events
862 list_for_each_entry(gi, &gator_events, list) 1004 list_for_each_entry(gi, &gator_events, list)
863 if (gi->stop) 1005 if (gi->stop)
864 gi->stop(); 1006 gi->stop();
1007
1008 gator_migrate_stop();
865} 1009}
866 1010
867/****************************************************************************** 1011/******************************************************************************
@@ -915,6 +1059,9 @@ static int gator_op_setup(void)
915 per_cpu(gator_buffer_write, cpu)[i] = 0; 1059 per_cpu(gator_buffer_write, cpu)[i] = 0;
916 per_cpu(gator_buffer_commit, cpu)[i] = 0; 1060 per_cpu(gator_buffer_commit, cpu)[i] = 0;
917 per_cpu(buffer_space_available, cpu)[i] = true; 1061 per_cpu(buffer_space_available, cpu)[i] = true;
1062#if GATOR_LIVE
1063 per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1064#endif
918 1065
919 // Annotation is a special case that only uses a single buffer 1066 // Annotation is a special case that only uses a single buffer
920 if (cpu > 0 && i == ANNOTATE_BUF) { 1067 if (cpu > 0 && i == ANNOTATE_BUF) {
@@ -963,6 +1110,7 @@ static void gator_op_stop(void)
963 mutex_lock(&gator_buffer_mutex); 1110 mutex_lock(&gator_buffer_mutex);
964 1111
965 gator_started = 0; 1112 gator_started = 0;
1113 gator_monotonic_started = 0;
966 cookies_release(); 1114 cookies_release();
967 wake_up(&gator_buffer_wait); 1115 wake_up(&gator_buffer_wait);
968 1116
@@ -987,10 +1135,15 @@ static void gator_shutdown(void)
987 per_cpu(gator_buffer_write, cpu)[i] = 0; 1135 per_cpu(gator_buffer_write, cpu)[i] = 0;
988 per_cpu(gator_buffer_commit, cpu)[i] = 0; 1136 per_cpu(gator_buffer_commit, cpu)[i] = 0;
989 per_cpu(buffer_space_available, cpu)[i] = true; 1137 per_cpu(buffer_space_available, cpu)[i] = true;
1138#if GATOR_LIVE
1139 per_cpu(gator_buffer_commit_time, cpu) = 0;
1140#endif
990 } 1141 }
991 mutex_unlock(&gator_buffer_mutex); 1142 mutex_unlock(&gator_buffer_mutex);
992 } 1143 }
993 1144
1145 memset(&sent_core_name, 0, sizeof(sent_core_name));
1146
994 mutex_unlock(&start_mutex); 1147 mutex_unlock(&start_mutex);
995} 1148}
996 1149
@@ -1143,6 +1296,11 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
1143 /* kick just in case we've lost an SMP event */ 1296 /* kick just in case we've lost an SMP event */
1144 wake_up(&gator_buffer_wait); 1297 wake_up(&gator_buffer_wait);
1145 1298
1299 // Wake up annotate_write if more space is available
1300 if (buftype == ANNOTATE_BUF) {
1301 wake_up(&gator_annotate_wait);
1302 }
1303
1146out: 1304out:
1147 mutex_unlock(&gator_buffer_mutex); 1305 mutex_unlock(&gator_buffer_mutex);
1148 return retval; 1306 return retval;
@@ -1196,6 +1354,7 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
1196 } 1354 }
1197 userspace_buffer_size = BACKTRACE_BUFFER_SIZE; 1355 userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1198 gator_response_type = 1; 1356 gator_response_type = 1;
1357 gator_live_rate = 0;
1199 1358
1200 gatorfs_create_file(sb, root, "enable", &enable_fops); 1359 gatorfs_create_file(sb, root, "enable", &enable_fops);
1201 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); 1360 gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
@@ -1205,6 +1364,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
1205 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); 1364 gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1206 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type); 1365 gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1207 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); 1366 gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1367 gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1368 gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1208 1369
1209 // Annotate interface 1370 // Annotate interface
1210 gator_annotate_create_files(sb, root); 1371 gator_annotate_create_files(sb, root);
@@ -1212,8 +1373,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
1212 // Linux Events 1373 // Linux Events
1213 dir = gatorfs_mkdir(sb, root, "events"); 1374 dir = gatorfs_mkdir(sb, root, "events");
1214 list_for_each_entry(gi, &gator_events, list) 1375 list_for_each_entry(gi, &gator_events, list)
1215 if (gi->create_files) 1376 if (gi->create_files)
1216 gi->create_files(sb, dir); 1377 gi->create_files(sb, dir);
1217 1378
1218 // Power interface 1379 // Power interface
1219 gator_trace_power_create_files(sb, dir); 1380 gator_trace_power_create_files(sb, dir);
diff --git a/driver/gator_marshaling.c b/driver/gator_marshaling.c
index b2efdd2..627b441 100644
--- a/driver/gator_marshaling.c
+++ b/driver/gator_marshaling.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2012. All rights reserved. 2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -7,50 +7,72 @@
7 * 7 *
8 */ 8 */
9 9
10static void marshal_summary(long long timestamp, long long uptime) 10#define NEWLINE_CANARY \
11 /* Unix */ \
12 "1\n" \
13 /* Windows */ \
14 "2\r\n" \
15 /* Mac OS */ \
16 "3\r" \
17 /* RISC OS */ \
18 "4\n\r" \
19 /* Add another character so the length isn't 0x0a bytes */ \
20 "5"
21
22static void marshal_summary(long long timestamp, long long uptime, const char * uname)
11{ 23{
24 unsigned long flags;
12 int cpu = 0; 25 int cpu = 0;
26
27 local_irq_save(flags);
28 gator_buffer_write_string(cpu, SUMMARY_BUF, NEWLINE_CANARY);
13 gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp); 29 gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, timestamp);
14 gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime); 30 gator_buffer_write_packed_int64(cpu, SUMMARY_BUF, uptime);
15 buffer_check(cpu, SUMMARY_BUF); 31 gator_buffer_write_string(cpu, SUMMARY_BUF, uname);
32 // Commit the buffer now so it can be one of the first frames read by Streamline
33 gator_commit_buffer(cpu, SUMMARY_BUF, gator_get_time());
34 local_irq_restore(flags);
16} 35}
17 36
18static bool marshal_cookie_header(const char *text) 37static bool marshal_cookie_header(const char *text)
19{ 38{
20 int cpu = smp_processor_id(); 39 int cpu = get_physical_cpu();
21 return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32); 40 return buffer_check_space(cpu, NAME_BUF, strlen(text) + 3 * MAXSIZE_PACK32);
22} 41}
23 42
24static void marshal_cookie(int cookie, const char *text) 43static void marshal_cookie(int cookie, const char *text)
25{ 44{
26 int cpu = smp_processor_id(); 45 int cpu = get_physical_cpu();
27 // buffer_check_space already called by marshal_cookie_header 46 // buffer_check_space already called by marshal_cookie_header
28 gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE); 47 gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_COOKIE);
29 gator_buffer_write_packed_int(cpu, NAME_BUF, cookie); 48 gator_buffer_write_packed_int(cpu, NAME_BUF, cookie);
30 gator_buffer_write_string(cpu, NAME_BUF, text); 49 gator_buffer_write_string(cpu, NAME_BUF, text);
31 buffer_check(cpu, NAME_BUF); 50 buffer_check(cpu, NAME_BUF, gator_get_time());
32} 51}
33 52
34static void marshal_thread_name(int pid, char *name) 53static void marshal_thread_name(int pid, char *name)
35{ 54{
36 unsigned long flags, cpu; 55 unsigned long flags, cpu;
56 u64 time;
37 local_irq_save(flags); 57 local_irq_save(flags);
38 cpu = smp_processor_id(); 58 cpu = get_physical_cpu();
59 time = gator_get_time();
39 if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { 60 if (buffer_check_space(cpu, NAME_BUF, TASK_COMM_LEN + 3 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
40 gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME); 61 gator_buffer_write_packed_int(cpu, NAME_BUF, MESSAGE_THREAD_NAME);
41 gator_buffer_write_packed_int64(cpu, NAME_BUF, gator_get_time()); 62 gator_buffer_write_packed_int64(cpu, NAME_BUF, time);
42 gator_buffer_write_packed_int(cpu, NAME_BUF, pid); 63 gator_buffer_write_packed_int(cpu, NAME_BUF, pid);
43 gator_buffer_write_string(cpu, NAME_BUF, name); 64 gator_buffer_write_string(cpu, NAME_BUF, name);
44 } 65 }
45 buffer_check(cpu, NAME_BUF); 66 buffer_check(cpu, NAME_BUF, time);
46 local_irq_restore(flags); 67 local_irq_restore(flags);
47} 68}
48 69
49static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inKernel) 70static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inKernel)
50{ 71{
51 int cpu = smp_processor_id(); 72 int cpu = get_physical_cpu();
73 u64 time = gator_get_time();
52 if (buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) { 74 if (buffer_check_space(cpu, BACKTRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32 + gator_backtrace_depth * 2 * MAXSIZE_PACK32)) {
53 gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time()); 75 gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, time);
54 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie); 76 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie);
55 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid); 77 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid);
56 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid); 78 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid);
@@ -59,30 +81,30 @@ static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inK
59 } 81 }
60 82
61 // Check and commit; commit is set to occur once buffer is 3/4 full 83 // Check and commit; commit is set to occur once buffer is 3/4 full
62 buffer_check(cpu, BACKTRACE_BUF); 84 buffer_check(cpu, BACKTRACE_BUF, time);
63 85
64 return false; 86 return false;
65} 87}
66 88
67static void marshal_backtrace(unsigned long address, int cookie) 89static void marshal_backtrace(unsigned long address, int cookie)
68{ 90{
69 int cpu = smp_processor_id(); 91 int cpu = get_physical_cpu();
70 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie); 92 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie);
71 gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, address); 93 gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, address);
72} 94}
73 95
74static void marshal_backtrace_footer(void) 96static void marshal_backtrace_footer(void)
75{ 97{
76 int cpu = smp_processor_id(); 98 int cpu = get_physical_cpu();
77 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE); 99 gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE);
78 100
79 // Check and commit; commit is set to occur once buffer is 3/4 full 101 // Check and commit; commit is set to occur once buffer is 3/4 full
80 buffer_check(cpu, BACKTRACE_BUF); 102 buffer_check(cpu, BACKTRACE_BUF, gator_get_time());
81} 103}
82 104
83static bool marshal_event_header(void) 105static bool marshal_event_header(void)
84{ 106{
85 unsigned long flags, cpu = smp_processor_id(); 107 unsigned long flags, cpu = get_physical_cpu();
86 bool retval = false; 108 bool retval = false;
87 109
88 local_irq_save(flags); 110 local_irq_save(flags);
@@ -91,8 +113,6 @@ static bool marshal_event_header(void)
91 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, gator_get_time()); 113 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, gator_get_time());
92 retval = true; 114 retval = true;
93 } 115 }
94 // Check and commit; commit is set to occur once buffer is 3/4 full
95 buffer_check(cpu, BLOCK_COUNTER_BUF);
96 local_irq_restore(flags); 116 local_irq_restore(flags);
97 117
98 return retval; 118 return retval;
@@ -100,7 +120,7 @@ static bool marshal_event_header(void)
100 120
101static void marshal_event(int len, int *buffer) 121static void marshal_event(int len, int *buffer)
102{ 122{
103 unsigned long i, flags, cpu = smp_processor_id(); 123 unsigned long i, flags, cpu = get_physical_cpu();
104 124
105 if (len <= 0) 125 if (len <= 0)
106 return; 126 return;
@@ -120,14 +140,12 @@ static void marshal_event(int len, int *buffer)
120 gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i]); 140 gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i]);
121 gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i + 1]); 141 gator_buffer_write_packed_int(cpu, BLOCK_COUNTER_BUF, buffer[i + 1]);
122 } 142 }
123 // Check and commit; commit is set to occur once buffer is 3/4 full
124 buffer_check(cpu, BLOCK_COUNTER_BUF);
125 local_irq_restore(flags); 143 local_irq_restore(flags);
126} 144}
127 145
128static void marshal_event64(int len, long long *buffer64) 146static void marshal_event64(int len, long long *buffer64)
129{ 147{
130 unsigned long i, flags, cpu = smp_processor_id(); 148 unsigned long i, flags, cpu = get_physical_cpu();
131 149
132 if (len <= 0) 150 if (len <= 0)
133 return; 151 return;
@@ -147,8 +165,6 @@ static void marshal_event64(int len, long long *buffer64)
147 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]); 165 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i]);
148 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]); 166 gator_buffer_write_packed_int64(cpu, BLOCK_COUNTER_BUF, buffer64[i + 1]);
149 } 167 }
150 // Check and commit; commit is set to occur once buffer is 3/4 full
151 buffer_check(cpu, BLOCK_COUNTER_BUF);
152 local_irq_restore(flags); 168 local_irq_restore(flags);
153} 169}
154 170
@@ -156,97 +172,107 @@ static void marshal_event64(int len, long long *buffer64)
156static void marshal_event_single(int core, int key, int value) 172static void marshal_event_single(int core, int key, int value)
157{ 173{
158 unsigned long flags, cpu; 174 unsigned long flags, cpu;
175 u64 time;
159 176
160 local_irq_save(flags); 177 local_irq_save(flags);
161 cpu = smp_processor_id(); 178 cpu = get_physical_cpu();
179 time = gator_get_time();
162 if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { 180 if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
163 gator_buffer_write_packed_int64(cpu, COUNTER_BUF, gator_get_time()); 181 gator_buffer_write_packed_int64(cpu, COUNTER_BUF, time);
164 gator_buffer_write_packed_int(cpu, COUNTER_BUF, core); 182 gator_buffer_write_packed_int(cpu, COUNTER_BUF, core);
165 gator_buffer_write_packed_int(cpu, COUNTER_BUF, key); 183 gator_buffer_write_packed_int(cpu, COUNTER_BUF, key);
166 gator_buffer_write_packed_int(cpu, COUNTER_BUF, value); 184 gator_buffer_write_packed_int(cpu, COUNTER_BUF, value);
167 } 185 }
168 // Check and commit; commit is set to occur once buffer is 3/4 full 186 // Check and commit; commit is set to occur once buffer is 3/4 full
169 buffer_check(cpu, COUNTER_BUF); 187 buffer_check(cpu, COUNTER_BUF, time);
170 local_irq_restore(flags); 188 local_irq_restore(flags);
171} 189}
172#endif 190#endif
173 191
174static void marshal_sched_gpu_start(int unit, int core, int tgid, int pid) 192static void marshal_sched_gpu_start(int unit, int core, int tgid, int pid)
175{ 193{
176 unsigned long cpu = smp_processor_id(), flags; 194 unsigned long cpu = get_physical_cpu(), flags;
195 u64 time;
177 196
178 if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF]) 197 if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF])
179 return; 198 return;
180 199
181 local_irq_save(flags); 200 local_irq_save(flags);
201 time = gator_get_time();
182 if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { 202 if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
183 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_START); 203 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_START);
184 gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time()); 204 gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, time);
185 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit); 205 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit);
186 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core); 206 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core);
187 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, tgid); 207 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, tgid);
188 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, pid); 208 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, pid);
189 } 209 }
190 // Check and commit; commit is set to occur once buffer is 3/4 full 210 // Check and commit; commit is set to occur once buffer is 3/4 full
191 buffer_check(cpu, GPU_TRACE_BUF); 211 buffer_check(cpu, GPU_TRACE_BUF, time);
192 local_irq_restore(flags); 212 local_irq_restore(flags);
193} 213}
194 214
195static void marshal_sched_gpu_stop(int unit, int core) 215static void marshal_sched_gpu_stop(int unit, int core)
196{ 216{
197 unsigned long cpu = smp_processor_id(), flags; 217 unsigned long cpu = get_physical_cpu(), flags;
218 u64 time;
198 219
199 if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF]) 220 if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF])
200 return; 221 return;
201 222
202 local_irq_save(flags); 223 local_irq_save(flags);
224 time = gator_get_time();
203 if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) { 225 if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 3 * MAXSIZE_PACK32)) {
204 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_STOP); 226 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, MESSAGE_GPU_STOP);
205 gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time()); 227 gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, time);
206 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit); 228 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit);
207 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core); 229 gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core);
208 } 230 }
209 // Check and commit; commit is set to occur once buffer is 3/4 full 231 // Check and commit; commit is set to occur once buffer is 3/4 full
210 buffer_check(cpu, GPU_TRACE_BUF); 232 buffer_check(cpu, GPU_TRACE_BUF, time);
211 local_irq_restore(flags); 233 local_irq_restore(flags);
212} 234}
213 235
214static void marshal_sched_trace_switch(int tgid, int pid, int cookie, int state) 236static void marshal_sched_trace_switch(int tgid, int pid, int cookie, int state)
215{ 237{
216 unsigned long cpu = smp_processor_id(), flags; 238 unsigned long cpu = get_physical_cpu(), flags;
239 u64 time;
217 240
218 if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) 241 if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
219 return; 242 return;
220 243
221 local_irq_save(flags); 244 local_irq_save(flags);
245 time = gator_get_time();
222 if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { 246 if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
223 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_SWITCH); 247 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_SWITCH);
224 gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time()); 248 gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time);
225 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, tgid); 249 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, tgid);
226 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); 250 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
227 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, cookie); 251 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, cookie);
228 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state); 252 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state);
229 } 253 }
230 // Check and commit; commit is set to occur once buffer is 3/4 full 254 // Check and commit; commit is set to occur once buffer is 3/4 full
231 buffer_check(cpu, SCHED_TRACE_BUF); 255 buffer_check(cpu, SCHED_TRACE_BUF, time);
232 local_irq_restore(flags); 256 local_irq_restore(flags);
233} 257}
234 258
235static void marshal_sched_trace_exit(int tgid, int pid) 259static void marshal_sched_trace_exit(int tgid, int pid)
236{ 260{
237 unsigned long cpu = smp_processor_id(), flags; 261 unsigned long cpu = get_physical_cpu(), flags;
262 u64 time;
238 263
239 if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) 264 if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
240 return; 265 return;
241 266
242 local_irq_save(flags); 267 local_irq_save(flags);
268 time = gator_get_time();
243 if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { 269 if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
244 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT); 270 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, MESSAGE_SCHED_EXIT);
245 gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time()); 271 gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, time);
246 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); 272 gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
247 } 273 }
248 // Check and commit; commit is set to occur once buffer is 3/4 full 274 // Check and commit; commit is set to occur once buffer is 3/4 full
249 buffer_check(cpu, SCHED_TRACE_BUF); 275 buffer_check(cpu, SCHED_TRACE_BUF, time);
250 local_irq_restore(flags); 276 local_irq_restore(flags);
251} 277}
252 278
@@ -254,16 +280,18 @@ static void marshal_sched_trace_exit(int tgid, int pid)
254static void marshal_idle(int core, int state) 280static void marshal_idle(int core, int state)
255{ 281{
256 unsigned long flags, cpu; 282 unsigned long flags, cpu;
283 u64 time;
257 284
258 local_irq_save(flags); 285 local_irq_save(flags);
259 cpu = smp_processor_id(); 286 cpu = get_physical_cpu();
287 time = gator_get_time();
260 if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { 288 if (buffer_check_space(cpu, IDLE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
261 gator_buffer_write_packed_int(cpu, IDLE_BUF, state); 289 gator_buffer_write_packed_int(cpu, IDLE_BUF, state);
262 gator_buffer_write_packed_int64(cpu, IDLE_BUF, gator_get_time()); 290 gator_buffer_write_packed_int64(cpu, IDLE_BUF, time);
263 gator_buffer_write_packed_int(cpu, IDLE_BUF, core); 291 gator_buffer_write_packed_int(cpu, IDLE_BUF, core);
264 } 292 }
265 // Check and commit; commit is set to occur once buffer is 3/4 full 293 // Check and commit; commit is set to occur once buffer is 3/4 full
266 buffer_check(cpu, IDLE_BUF); 294 buffer_check(cpu, IDLE_BUF, time);
267 local_irq_restore(flags); 295 local_irq_restore(flags);
268} 296}
269#endif 297#endif
@@ -323,16 +351,17 @@ static void marshal_frame(int cpu, int buftype)
323} 351}
324 352
325#if defined(__arm__) || defined(__aarch64__) 353#if defined(__arm__) || defined(__aarch64__)
326static void marshal_core_name(const char *name) 354static void marshal_core_name(const int cpuid, const char *name)
327{ 355{
328 int cpu = smp_processor_id(); 356 int cpu = get_physical_cpu();
329 unsigned long flags; 357 unsigned long flags;
330 local_irq_save(flags); 358 local_irq_save(flags);
331 if (buffer_check_space(cpu, NAME_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) { 359 if (buffer_check_space(cpu, NAME_BUF, MAXSIZE_PACK32 + MAXSIZE_CORE_NAME)) {
332 gator_buffer_write_packed_int(cpu, NAME_BUF, HRTIMER_CORE_NAME); 360 gator_buffer_write_packed_int(cpu, NAME_BUF, HRTIMER_CORE_NAME);
361 gator_buffer_write_packed_int(cpu, NAME_BUF, cpuid);
333 gator_buffer_write_string(cpu, NAME_BUF, name); 362 gator_buffer_write_string(cpu, NAME_BUF, name);
334 } 363 }
335 buffer_check(cpu, NAME_BUF); 364 buffer_check(cpu, NAME_BUF, gator_get_time());
336 local_irq_restore(flags); 365 local_irq_restore(flags);
337} 366}
338#endif 367#endif
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
index 119746b..2bddcbe 100644
--- a/driver/gator_pack.c
+++ b/driver/gator_pack.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c
index 9fc488b..c94f6a0 100644
--- a/driver/gator_trace_gpu.c
+++ b/driver/gator_trace_gpu.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -32,14 +32,72 @@ static int mali_timeline_trace_registered;
32static int mali_job_slots_trace_registered; 32static int mali_job_slots_trace_registered;
33static int gpu_trace_registered; 33static int gpu_trace_registered;
34 34
35#define GPU_UNIT_NONE 0 35enum {
36#define GPU_UNIT_VP 1 36 GPU_UNIT_NONE = 0,
37#define GPU_UNIT_FP 2 37 GPU_UNIT_VP,
38#define GPU_UNIT_CL 3 38 GPU_UNIT_FP,
39 GPU_UNIT_CL,
40 NUMBER_OF_GPU_UNITS
41};
39 42
40#define MALI_400 (0x0b07) 43#define MALI_400 (0x0b07)
41#define MALI_T6xx (0x0056) 44#define MALI_T6xx (0x0056)
42 45
46struct mali_gpu_job {
47 int count;
48 int last_core;
49 int last_tgid;
50 int last_pid;
51};
52
53#define NUMBER_OF_GPU_CORES 16
54static struct mali_gpu_job mali_gpu_jobs[NUMBER_OF_GPU_UNITS][NUMBER_OF_GPU_CORES];
55static DEFINE_SPINLOCK(mali_gpu_jobs_lock);
56
57static void mali_gpu_enqueue(int unit, int core, int tgid, int pid)
58{
59 int count;
60
61 spin_lock(&mali_gpu_jobs_lock);
62 count = mali_gpu_jobs[unit][core].count;
63 BUG_ON(count < 0);
64 ++mali_gpu_jobs[unit][core].count;
65 if (count) {
66 mali_gpu_jobs[unit][core].last_core = core;
67 mali_gpu_jobs[unit][core].last_tgid = tgid;
68 mali_gpu_jobs[unit][core].last_pid = pid;
69 }
70 spin_unlock(&mali_gpu_jobs_lock);
71
72 if (!count) {
73 marshal_sched_gpu_start(unit, core, tgid, pid);
74 }
75}
76
77static void mali_gpu_stop(int unit, int core)
78{
79 int count;
80 int last_core = 0;
81 int last_tgid = 0;
82 int last_pid = 0;
83
84 spin_lock(&mali_gpu_jobs_lock);
85 --mali_gpu_jobs[unit][core].count;
86 count = mali_gpu_jobs[unit][core].count;
87 BUG_ON(count < 0);
88 if (count) {
89 last_core = mali_gpu_jobs[unit][core].last_core;
90 last_tgid = mali_gpu_jobs[unit][core].last_tgid;
91 last_pid = mali_gpu_jobs[unit][core].last_pid;
92 }
93 spin_unlock(&mali_gpu_jobs_lock);
94
95 marshal_sched_gpu_stop(unit, core);
96 if (count) {
97 marshal_sched_gpu_start(unit, last_core, last_tgid, last_pid);
98 }
99}
100
43#if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_T6xx) 101#if defined(MALI_SUPPORT) && (MALI_SUPPORT != MALI_T6xx)
44#include "gator_events_mali_400.h" 102#include "gator_events_mali_400.h"
45 103
@@ -80,18 +138,18 @@ GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned
80 case EVENT_TYPE_START: 138 case EVENT_TYPE_START:
81 if (component == EVENT_CHANNEL_VP0) { 139 if (component == EVENT_CHANNEL_VP0) {
82 /* tgid = d0; pid = d1; */ 140 /* tgid = d0; pid = d1; */
83 marshal_sched_gpu_start(GPU_UNIT_VP, 0, d0, d1); 141 mali_gpu_enqueue(GPU_UNIT_VP, 0, d0, d1);
84 } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { 142 } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) {
85 /* tgid = d0; pid = d1; */ 143 /* tgid = d0; pid = d1; */
86 marshal_sched_gpu_start(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0, d0, d1); 144 mali_gpu_enqueue(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0, d0, d1);
87 } 145 }
88 break; 146 break;
89 147
90 case EVENT_TYPE_STOP: 148 case EVENT_TYPE_STOP:
91 if (component == EVENT_CHANNEL_VP0) { 149 if (component == EVENT_CHANNEL_VP0) {
92 marshal_sched_gpu_stop(GPU_UNIT_VP, 0); 150 mali_gpu_stop(GPU_UNIT_VP, 0);
93 } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) { 151 } else if (component >= EVENT_CHANNEL_FP0 && component <= EVENT_CHANNEL_FP7) {
94 marshal_sched_gpu_stop(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0); 152 mali_gpu_stop(GPU_UNIT_FP, component - EVENT_CHANNEL_FP0);
95 } 153 }
96 break; 154 break;
97 155
@@ -136,16 +194,16 @@ GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigne
136 if (unit != GPU_UNIT_NONE) { 194 if (unit != GPU_UNIT_NONE) {
137 switch (state) { 195 switch (state) {
138 case EVENT_TYPE_START: 196 case EVENT_TYPE_START:
139 marshal_sched_gpu_start(unit, 0, tgid, (pid != 0 ? pid : tgid)); 197 mali_gpu_enqueue(unit, 0, tgid, (pid != 0 ? pid : tgid));
140 break; 198 break;
141 case EVENT_TYPE_STOP: 199 case EVENT_TYPE_STOP:
142 marshal_sched_gpu_stop(unit, 0); 200 mali_gpu_stop(unit, 0);
143 break; 201 break;
144 default: 202 default:
145 /* 203 /*
146 * Some jobs can be soft-stopped, so ensure that this terminates the activity trace. 204 * Some jobs can be soft-stopped, so ensure that this terminates the activity trace.
147 */ 205 */
148 marshal_sched_gpu_stop(unit, 0); 206 mali_gpu_stop(unit, 0);
149 } 207 }
150 } 208 }
151} 209}
@@ -153,12 +211,12 @@ GATOR_DEFINE_PROBE(mali_job_slots_event, TP_PROTO(unsigned int event_id, unsigne
153 211
154GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p)) 212GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p))
155{ 213{
156 marshal_sched_gpu_start(gpu_unit, gpu_core, (int)p->tgid, (int)p->pid); 214 mali_gpu_enqueue(gpu_unit, gpu_core, (int)p->tgid, (int)p->pid);
157} 215}
158 216
159GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core)) 217GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core))
160{ 218{
161 marshal_sched_gpu_stop(gpu_unit, gpu_core); 219 mali_gpu_stop(gpu_unit, gpu_core);
162} 220}
163 221
164int gator_trace_gpu_start(void) 222int gator_trace_gpu_start(void)
diff --git a/driver/gator_trace_gpu.h b/driver/gator_trace_gpu.h
index efb47c6..bb0f42d 100644
--- a/driver/gator_trace_gpu.h
+++ b/driver/gator_trace_gpu.h
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
diff --git a/driver/gator_trace_power.c b/driver/gator_trace_power.c
index 79fa13c..272e056 100644
--- a/driver/gator_trace_power.c
+++ b/driver/gator_trace_power.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2011-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2011-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -38,14 +38,27 @@ static ulong power_cpu_key[POWER_TOTAL];
38static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) 38static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root)
39{ 39{
40 struct dentry *dir; 40 struct dentry *dir;
41 int cpu;
42 bool found_nonzero_freq = false;
43
44 // Even if CONFIG_CPU_FREQ is defined, it still may not be used. Check
45 // for non-zero values from cpufreq_quick_get
46 for_each_online_cpu(cpu) {
47 if (cpufreq_quick_get(cpu) > 0) {
48 found_nonzero_freq = true;
49 break;
50 }
51 }
41 52
42 // cpu_frequency 53 if (found_nonzero_freq) {
43 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); 54 // cpu_frequency
44 if (!dir) { 55 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq");
45 return -1; 56 if (!dir) {
57 return -1;
58 }
59 gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]);
60 gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]);
46 } 61 }
47 gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]);
48 gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]);
49 62
50 // cpu_idle 63 // cpu_idle
51 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); 64 dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle");
@@ -61,13 +74,14 @@ static int gator_trace_power_create_files(struct super_block *sb, struct dentry
61// 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change 74// 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change
62GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) 75GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu))
63{ 76{
77 cpu = lcpu_to_pcpu(cpu);
64 marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], frequency * 1000); 78 marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], frequency * 1000);
65} 79}
66 80
67#define WFI_EXIT 2
68#define WFI_ENTER 1
69GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) 81GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
70{ 82{
83 cpu = lcpu_to_pcpu(cpu);
84
71 if (state == per_cpu(idle_prev_state, cpu)) { 85 if (state == per_cpu(idle_prev_state, cpu)) {
72 return; 86 return;
73 } 87 }
@@ -75,10 +89,10 @@ GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
75 if (implements_wfi()) { 89 if (implements_wfi()) {
76 if (state == PWR_EVENT_EXIT) { 90 if (state == PWR_EVENT_EXIT) {
77 // transition from wfi to non-wfi 91 // transition from wfi to non-wfi
78 marshal_idle(cpu, WFI_EXIT); 92 marshal_idle(cpu, MESSAGE_IDLE_EXIT);
79 } else { 93 } else {
80 // transition from non-wfi to wfi 94 // transition from non-wfi to wfi
81 marshal_idle(cpu, WFI_ENTER); 95 marshal_idle(cpu, MESSAGE_IDLE_ENTER);
82 } 96 }
83 } 97 }
84 98
@@ -92,16 +106,17 @@ GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
92 106
93static void gator_trace_power_online(void) 107static void gator_trace_power_online(void)
94{ 108{
95 int cpu = smp_processor_id(); 109 int pcpu = get_physical_cpu();
110 int lcpu = get_logical_cpu();
96 if (power_cpu_enabled[POWER_CPU_FREQ]) { 111 if (power_cpu_enabled[POWER_CPU_FREQ]) {
97 marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(cpu) * 1000); 112 marshal_event_single(pcpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(lcpu) * 1000);
98 } 113 }
99} 114}
100 115
101static void gator_trace_power_offline(void) 116static void gator_trace_power_offline(void)
102{ 117{
103 // Set frequency to zero on an offline 118 // Set frequency to zero on an offline
104 int cpu = smp_processor_id(); 119 int cpu = get_physical_cpu();
105 if (power_cpu_enabled[POWER_CPU_FREQ]) { 120 if (power_cpu_enabled[POWER_CPU_FREQ]) {
106 marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], 0); 121 marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], 0);
107 } 122 }
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index d0336f9..eb989b5 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -1,5 +1,5 @@
1/** 1/**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved. 2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
@@ -23,14 +23,13 @@ enum {
23 STATE_WAIT_ON_OTHER = 0, 23 STATE_WAIT_ON_OTHER = 0,
24 STATE_CONTENTION, 24 STATE_CONTENTION,
25 STATE_WAIT_ON_IO, 25 STATE_WAIT_ON_IO,
26 STATE_WAIT_ON_MUTEX,
27}; 26};
28 27
29void emit_pid_name(struct task_struct *task) 28void emit_pid_name(struct task_struct *task)
30{ 29{
31 bool found = false;