summaryrefslogtreecommitdiffstats
path: root/driver
diff options
context:
space:
mode:
authorPawel Moll2011-11-17 12:14:03 -0600
committerPawel Moll2011-11-17 12:14:03 -0600
commit6d51ed1aa2f14b8ee097e940a4d747bf04020195 (patch)
tree5fb49aff4be9956cfdb58dfe9e4129a86130d292 /driver
parent3081bf8f00d4b17641988db512efd8828f59eb05 (diff)
downloadarm-ds5-gator-6d51ed1aa2f14b8ee097e940a4d747bf04020195.tar.gz
arm-ds5-gator-6d51ed1aa2f14b8ee097e940a4d747bf04020195.tar.xz
arm-ds5-gator-6d51ed1aa2f14b8ee097e940a4d747bf04020195.zip
gator-driver: ARM DS-5.8 Streamline gator driver (RC1)
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile8
-rw-r--r--driver/gator_annotate.c49
-rw-r--r--driver/gator_annotate_kernel.c91
-rw-r--r--driver/gator_backtrace.c50
-rw-r--r--driver/gator_cookies.c16
-rw-r--r--driver/gator_ebs.c35
-rw-r--r--driver/gator_events_armv7.c1
-rw-r--r--driver/gator_main.c91
-rw-r--r--driver/gator_trace_sched.c141
9 files changed, 367 insertions, 115 deletions
diff --git a/driver/Makefile b/driver/Makefile
index de59957..b3981ff 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -1,5 +1,8 @@
1ifneq ($(KERNELRELEASE),) 1ifneq ($(KERNELRELEASE),)
2 2
3# Uncomment the following line to enable kernel stack unwinding within gator, or update gator_backtrace.c
4# EXTRA_CFLAGS += -DGATOR_KERNEL_STACK_UNWINDING
5
3obj-m := gator.o 6obj-m := gator.o
4 7
5gator-y := gator_main.o \ 8gator-y := gator_main.o \
@@ -12,8 +15,13 @@ gator-y := gator_main.o \
12gator-y += gator_events_mmaped.o 15gator-y += gator_events_mmaped.o
13 16
14ifneq ($(GATOR_WITH_MALI_SUPPORT),) 17ifneq ($(GATOR_WITH_MALI_SUPPORT),)
18ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_T6xx)
19gator-y += gator_events_mali_t6xx.o
20else
15gator-y += gator_events_mali.o 21gator-y += gator_events_mali.o
16endif 22endif
23EXTRA_CFLAGS += -DMALI_SUPPORT=$(GATOR_WITH_MALI_SUPPORT)
24endif
17 25
18gator-$(CONFIG_ARM) += gator_events_armv6.o \ 26gator-$(CONFIG_ARM) += gator_events_armv6.o \
19 gator_events_armv7.o \ 27 gator_events_armv7.o \
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index e096d15..ee5a160 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -27,27 +27,35 @@ static bool collect_annotations = false;
27static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) 27static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
28{ 28{
29 char tempBuffer[512]; 29 char tempBuffer[512];
30 int retval, remaining, size; 30 int remaining, size;
31 uint32_t tid;
31 32
32 if (*offset) 33 if (*offset)
33 return -EINVAL; 34 return -EINVAL;
34 35
35 // determine size to capture 36 // determine size to capture
36 remaining = ANNOTATE_SIZE - annotatePos - 256; // pad for headers and release
37 size = count < sizeof(tempBuffer) ? count : sizeof(tempBuffer); 37 size = count < sizeof(tempBuffer) ? count : sizeof(tempBuffer);
38 size = size < remaining ? size : remaining; 38
39 if (size <= 0) { 39 // note: copy may be for naught if remaining is zero, but better to do the copy outside of the spinlock
40 wake_up(&gator_buffer_wait); 40 if (file == NULL) {
41 return 0; 41 // copy from kernel
42 memcpy(tempBuffer, buf, size);
43
44 // set the thread id to the kernel thread, not the current thread
45 tid = -1;
46 } else {
47 // copy from user space
48 if (copy_from_user(tempBuffer, buf, size) != 0)
49 return -EINVAL;
50 tid = current->pid;
42 } 51 }
43 52
44 // copy from user space 53 // synchronize shared variables annotateBuf and annotatePos
45 retval = copy_from_user(tempBuffer, buf, size); 54 spin_lock(&annotate_lock);
46 if (retval == 0) { 55 if (collect_annotations && annotateBuf) {
47 // synchronize shared variables annotateBuf and annotatePos 56 remaining = ANNOTATE_SIZE - annotatePos - 256; // pad for headers and release
48 spin_lock(&annotate_lock); 57 size = size < remaining ? size : remaining;
49 if (collect_annotations && annotateBuf) { 58 if (size > 0) {
50 uint32_t tid = current->pid;
51 uint64_t time = gator_get_time(); 59 uint64_t time = gator_get_time();
52 uint32_t cpuid = smp_processor_id(); 60 uint32_t cpuid = smp_processor_id();
53 int pos = annotatePos; 61 int pos = annotatePos;
@@ -58,17 +66,20 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
58 memcpy(&annotateBuf[pos], tempBuffer, size); 66 memcpy(&annotateBuf[pos], tempBuffer, size);
59 annotatePos = pos + size; 67 annotatePos = pos + size;
60 } 68 }
61 spin_unlock(&annotate_lock); 69 }
70 spin_unlock(&annotate_lock);
62 71
63 // return the number of bytes written 72 if (size <= 0) {
64 retval = size; 73 wake_up(&gator_buffer_wait);
65 } else { 74 return 0;
66 retval = -EINVAL;
67 } 75 }
68 76
69 return retval; 77 // return the number of bytes written
78 return size;
70} 79}
71 80
81#include "gator_annotate_kernel.c"
82
72static int annotate_release(struct inode *inode, struct file *file) 83static int annotate_release(struct inode *inode, struct file *file)
73{ 84{
74 int remaining = ANNOTATE_SIZE - annotatePos; 85 int remaining = ANNOTATE_SIZE - annotatePos;
diff --git a/driver/gator_annotate_kernel.c b/driver/gator_annotate_kernel.c
new file mode 100644
index 0000000..4f690e8
--- /dev/null
+++ b/driver/gator_annotate_kernel.c
@@ -0,0 +1,91 @@
1/**
2 * Copyright (C) ARM Limited 2011. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
9
10static void kannotate_write(char* ptr, unsigned int size)
11{
12 int retval;
13 int pos = 0;
14 loff_t offset = 0;
15 while (pos < size) {
16 retval = annotate_write(NULL, &ptr[pos], size - pos, &offset);
17 if (retval < 0) {
18 printk(KERN_WARNING "gator: kannotate_write failed with return value %d\n", retval);
19 return;
20 }
21 pos += retval;
22 }
23}
24
25// String annotation
26void gator_annotate(char* string)
27{
28 printk(KERN_ERR "module: %s\n", string);
29 kannotate_write(string, strlen(string) + 1);
30}
31EXPORT_SYMBOL(gator_annotate);
32
33// String annotation with color
34void gator_annotate_color(int color, char* string)
35{
36 kannotate_write((char*)&color, sizeof(color));
37 kannotate_write(string, strlen(string) + 1);
38}
39EXPORT_SYMBOL(gator_annotate_color);
40
41// Terminate an annotation
42void gator_annotate_end(void)
43{
44 char nul = 0;
45 kannotate_write(&nul, sizeof(nul));
46}
47EXPORT_SYMBOL(gator_annotate_end);
48
49// Image annotation with optional string
50void gator_annotate_visual(char* data, unsigned int length, char* string)
51{
52 long long visual_annotation = 0x011c | (strlen(string) << 16) | ((long long)length << 32);
53 kannotate_write((char*)&visual_annotation, 8);
54 kannotate_write(string, strlen(string));
55 kannotate_write(data, length);
56}
57EXPORT_SYMBOL(gator_annotate_visual);
58
59// Marker annotation
60void gator_annotate_marker(void)
61{
62 int marker_annotation = 0x00021c;
63 kannotate_write((char*)&marker_annotation, 3);
64}
65EXPORT_SYMBOL(gator_annotate_marker);
66
67// Marker annotation with a string
68void gator_annotate_marker_str(char* string)
69{
70 int marker_annotation = 0x021c;
71 kannotate_write((char*)&marker_annotation, 2);
72 kannotate_write(string, strlen(string) + 1);
73}
74EXPORT_SYMBOL(gator_annotate_marker_str);
75
76// Marker annotation with a color
77void gator_annotate_marker_color(int color)
78{
79 long long marker_annotation = (0x021c | ((long long)color << 16)) & 0x0000ffffffffffffLL;
80 kannotate_write((char*)&marker_annotation, 7);
81}
82EXPORT_SYMBOL(gator_annotate_marker_color);
83
84// Marker annotationw ith a string and color
85void gator_annotate_marker_color_str(int color, char* string)
86{
87 long long marker_annotation = 0x021c | ((long long)color << 16);
88 kannotate_write((char*)&marker_annotation, 6);
89 kannotate_write(string, strlen(string) + 1);
90}
91EXPORT_SYMBOL(gator_annotate_marker_color_str);
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index b5b0e63..fc81233 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -23,6 +23,7 @@ static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs
23 struct frame_tail_eabi *ptrtail; 23 struct frame_tail_eabi *ptrtail;
24 struct frame_tail_eabi buftail; 24 struct frame_tail_eabi buftail;
25 unsigned long fp = regs->ARM_fp; 25 unsigned long fp = regs->ARM_fp;
26 unsigned long sp = regs->ARM_sp;
26 unsigned long lr = regs->ARM_lr; 27 unsigned long lr = regs->ARM_lr;
27 int is_user_mode = user_mode(regs); 28 int is_user_mode = user_mode(regs);
28 29
@@ -34,7 +35,7 @@ static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs
34 gator_add_trace(cpu, buftype, lr); 35 gator_add_trace(cpu, buftype, lr);
35 36
36 /* check tail is valid */ 37 /* check tail is valid */
37 if (fp == 0) { 38 if (fp == 0 || fp < sp) {
38 return; 39 return;
39 } 40 }
40 41
@@ -66,3 +67,50 @@ static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs
66 } 67 }
67#endif 68#endif
68} 69}
70
71#if defined(__arm__)
72static DEFINE_PER_CPU(int, backtrace_buffer);
73static int report_trace(struct stackframe *frame, void *d)
74{
75 struct module *mod;
76 unsigned int *depth = d, addr = frame->pc, cookie = NO_COOKIE, cpu = smp_processor_id();
77
78 if (*depth) {
79 mod = __module_address(addr);
80 if (mod) {
81 cookie = get_cookie(cpu, per_cpu(backtrace_buffer, cpu), current, NULL, mod, true);
82 addr = addr - (unsigned long)mod->module_core;
83 }
84 gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), addr & ~1);
85 gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), cookie);
86 (*depth)--;
87 }
88
89 return *depth == 0;
90}
91#endif
92
93// Uncomment the following line to enable kernel stack unwinding within gator, note it can also be defined from the Makefile
94// #define GATOR_KERNEL_STACK_UNWINDING
95static void kernel_backtrace(int cpu, int buftype, struct pt_regs * const regs)
96{
97#if defined(__arm__)
98#ifdef GATOR_KERNEL_STACK_UNWINDING
99 int depth = gator_backtrace_depth;
100#else
101 int depth = 1;
102#endif
103 struct stackframe frame;
104 if (depth == 0)
105 depth = 1;
106 frame.fp = regs->ARM_fp;
107 frame.sp = regs->ARM_sp;
108 frame.lr = regs->ARM_lr;
109 frame.pc = regs->ARM_pc;
110 per_cpu(backtrace_buffer, cpu) = buftype;
111 walk_stackframe(&frame, report_trace, &depth);
112#else
113 gator_buffer_write_packed_int(cpu, buftype, PC_REG & ~1);
114 gator_buffer_write_packed_int(cpu, buftype, NO_COOKIE);
115#endif
116}
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index a646fb2..c0c4b9c 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -122,12 +122,18 @@ static void wq_cookie_handler(struct work_struct *unused)
122 int cpu = smp_processor_id(); 122 int cpu = smp_processor_id();
123 unsigned int cookie, commit; 123 unsigned int cookie, commit;
124 124
125 commit = per_cpu(translate_buffer_write, cpu); 125 mutex_lock(&start_mutex);
126 while (per_cpu(translate_buffer_read, cpu) != commit) { 126
127 task = (struct task_struct *)translate_buffer_read_int(cpu); 127 if (gator_started != 0) {
128 vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); 128 commit = per_cpu(translate_buffer_write, cpu);
129 cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false); 129 while (per_cpu(translate_buffer_read, cpu) != commit) {
130 task = (struct task_struct *)translate_buffer_read_int(cpu);
131 vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
132 cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false);
133 }
130 } 134 }
135
136 mutex_unlock(&start_mutex);
131} 137}
132 138
133// Retrieve full name from proc/pid/cmdline for java processes on Android 139// Retrieve full name from proc/pid/cmdline for java processes on Android
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c
index 9b55347..8b4b5ff 100644
--- a/driver/gator_ebs.c
+++ b/driver/gator_ebs.c
@@ -95,16 +95,19 @@ static int gator_event_sampling_start(void)
95 int irq, i; 95 int irq, i;
96 96
97 if (IS_ERR(pmu_device)) { 97 if (IS_ERR(pmu_device)) {
98 pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed"); 98 pr_err("gator: event based sampling is not supported as the kernel function reserve_pmu() failed\n");
99 return -1; 99 return -1;
100 } 100 }
101 101
102 init_pmu(ARM_PMU_DEVICE_CPU); 102 // init_pmu sets the irq affinity, therefore we do not care if it fails for single core
103 if (init_pmu(ARM_PMU_DEVICE_CPU) != 0 && gator_cpu_cores > 1) {
104 pr_err("gator: unable to initialize the pmu\n");
105 goto out_ebs_start;
106 }
107
103 if (pmu_device->num_resources == 0) { 108 if (pmu_device->num_resources == 0) {
104 pr_err("gator: no irqs for PMUs defined\n"); 109 pr_err("gator: no irqs for PMUs defined\n");
105 release_pmu(pmu_device); 110 goto out_ebs_start;
106 pmu_device = NULL;
107 return -1;
108 } 111 }
109 112
110 for (i = 0; i < pmu_device->num_resources; ++i) { 113 for (i = 0; i < pmu_device->num_resources; ++i) {
@@ -121,9 +124,7 @@ static int gator_event_sampling_start(void)
121 if (irq >= 0) 124 if (irq >= 0)
122 free_irq(irq, NULL); 125 free_irq(irq, NULL);
123 } 126 }
124 release_pmu(pmu_device); 127 goto out_ebs_start;
125 pmu_device = NULL;
126 return -1;
127 } 128 }
128 } 129 }
129 } 130 }
@@ -135,6 +136,17 @@ static int gator_event_sampling_start(void)
135#endif 136#endif
136 137
137 return 0; 138 return 0;
139
140#if LINUX_PMU_SUPPORT
141out_ebs_start:
142#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
143 release_pmu(pmu_device);
144#else
145 release_pmu(ARM_PMU_DEVICE_CPU);
146#endif
147 pmu_device = NULL;
148 return -1;
149#endif
138} 150}
139 151
140static void gator_event_sampling_stop(void) 152static void gator_event_sampling_stop(void)
@@ -148,8 +160,13 @@ static void gator_event_sampling_stop(void)
148 free_irq(irq, NULL); 160 free_irq(irq, NULL);
149 } 161 }
150 } 162 }
151 if (!IS_ERR(pmu_device)) 163 if (!IS_ERR(pmu_device)) {
164#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0)
152 release_pmu(pmu_device); 165 release_pmu(pmu_device);
166#else
167 release_pmu(ARM_PMU_DEVICE_CPU);
168#endif
169 }
153 pmu_device = NULL; 170 pmu_device = NULL;
154#endif 171#endif
155} 172}
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index 58855f8..e1434e2 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -204,6 +204,7 @@ static void gator_events_armv7_online(void)
204 204
205static void gator_events_armv7_offline(void) 205static void gator_events_armv7_offline(void)
206{ 206{
207 // disbale all counters, including PMCCNTR; overflow IRQs will not be signaled
207 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); 208 armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E);
208} 209}
209 210
diff --git a/driver/gator_main.c b/driver/gator_main.c
index 91744ad..36e951b 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -7,7 +7,7 @@
7 * 7 *
8 */ 8 */
9 9
10static unsigned long gator_protocol_version = 6; 10static unsigned long gator_protocol_version = 7;
11 11
12#include <linux/slab.h> 12#include <linux/slab.h>
13#include <linux/cpu.h> 13#include <linux/cpu.h>
@@ -17,6 +17,8 @@ static unsigned long gator_protocol_version = 6;
17#include <linux/hardirq.h> 17#include <linux/hardirq.h>
18#include <linux/highmem.h> 18#include <linux/highmem.h>
19#include <linux/pagemap.h> 19#include <linux/pagemap.h>
20#include <linux/suspend.h>
21#include <asm/stacktrace.h>
20#include <asm/uaccess.h> 22#include <asm/uaccess.h>
21 23
22#include "gator.h" 24#include "gator.h"
@@ -51,7 +53,7 @@ static unsigned long gator_protocol_version = 6;
51/****************************************************************************** 53/******************************************************************************
52 * DEFINES 54 * DEFINES
53 ******************************************************************************/ 55 ******************************************************************************/
54#define TIMER_BUFFER_SIZE_DEFAULT (256*1024) 56#define TIMER_BUFFER_SIZE_DEFAULT (512*1024)
55#define EVENT_BUFFER_SIZE_DEFAULT (128*1024) 57#define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
56 58
57#define NO_COOKIE 0UL 59#define NO_COOKIE 0UL
@@ -267,11 +269,12 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address)
267 269
268static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) 270static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
269{ 271{
270 struct module *mod;
271 unsigned int addr, cookie = 0;
272 int inKernel = regs ? !user_mode(regs) : 1; 272 int inKernel = regs ? !user_mode(regs) : 1;
273 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current); 273 unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current);
274 274
275 if (!regs)
276 return;
277
275 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE); 278 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
276 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); 279 gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
277 gator_buffer_write_packed_int(cpu, buftype, exec_cookie); 280 gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
@@ -279,25 +282,15 @@ static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
279 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid); 282 gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
280 gator_buffer_write_packed_int(cpu, buftype, inKernel); 283 gator_buffer_write_packed_int(cpu, buftype, inKernel);
281 284
282 // get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ) 285 if (inKernel) {
283 if (regs) { 286 kernel_backtrace(cpu, buftype, regs);
284 if (inKernel) { 287 } else {
285 addr = PC_REG; 288 // Cookie+PC
286 mod = __module_address(addr); 289 gator_add_trace(cpu, buftype, PC_REG);
287 if (mod) { 290
288 cookie = get_cookie(cpu, buftype, current, NULL, mod, true); 291 // Backtrace
289 addr = addr - (unsigned long)mod->module_core; 292 if (gator_backtrace_depth)
290 } 293 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
291 gator_buffer_write_packed_int(cpu, buftype, addr & ~1);
292 gator_buffer_write_packed_int(cpu, buftype, cookie);
293 } else {
294 // Cookie+PC
295 gator_add_trace(cpu, buftype, PC_REG);
296
297 // Backtrace
298 if (gator_backtrace_depth)
299 arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
300 }
301 } 294 }
302 295
303 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE); 296 gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
@@ -352,11 +345,12 @@ static void gator_timer_interrupt(void)
352 } 345 }
353 } else if (gi->read64) { 346 } else if (gi->read64) {
354 len = gi->read64(&buffer64); 347 len = gi->read64(&buffer64);
355 if (len > 0) 348 if (len > 0) {
356 gator_buffer_write_packed_int(cpu, buftype, len); 349 gator_buffer_write_packed_int(cpu, buftype, len);
357 for (i = 0; i < len; i++) { 350 for (i = 0; i < len; i++) {
358 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); 351 gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
359 } 352 }
353 }
360 } 354 }
361 } 355 }
362 gator_buffer_write_packed_int(cpu, buftype, 0); 356 gator_buffer_write_packed_int(cpu, buftype, 0);
@@ -460,29 +454,28 @@ static uint64_t gator_get_time(void)
460 struct timespec ts; 454 struct timespec ts;
461 uint64_t timestamp; 455 uint64_t timestamp;
462 456
463 ktime_get_ts(&ts); 457 getnstimeofday(&ts);
464 timestamp = timespec_to_ns(&ts); 458 timestamp = timespec_to_ns(&ts);
465 459
466 return timestamp; 460 return timestamp;
467} 461}
468 462
469/****************************************************************************** 463/******************************************************************************
470 * cpu online notifier 464 * cpu online and pm notifiers
471 ******************************************************************************/ 465 ******************************************************************************/
472static int __cpuinit gator_cpu_notify(struct notifier_block *self, 466static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
473 unsigned long action, void *hcpu)
474{ 467{
475 long cpu = (long)hcpu; 468 long cpu = (long)hcpu;
476 469
477 switch (action) { 470 switch (action) {
478 case CPU_ONLINE:
479 case CPU_ONLINE_FROZEN:
480 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
481 break;
482 case CPU_DOWN_PREPARE: 471 case CPU_DOWN_PREPARE:
483 case CPU_DOWN_PREPARE_FROZEN: 472 case CPU_DOWN_PREPARE_FROZEN:
484 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1); 473 smp_call_function_single(cpu, __gator_timer_offline, NULL, 1);
485 break; 474 break;
475 case CPU_ONLINE:
476 case CPU_ONLINE_FROZEN:
477 smp_call_function_single(cpu, __gator_timer_online, NULL, 1);
478 break;
486 } 479 }
487 480
488 return NOTIFY_OK; 481 return NOTIFY_OK;
@@ -492,13 +485,45 @@ static struct notifier_block __refdata gator_cpu_notifier = {
492 .notifier_call = gator_cpu_notify, 485 .notifier_call = gator_cpu_notify,
493}; 486};
494 487
488// n.b. calling "on_each_cpu" only runs on those that are online
489// Registered linux events are not disabled, so their counters will continue to collect
490static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
491{
492 switch (event) {
493 case PM_HIBERNATION_PREPARE:
494 case PM_SUSPEND_PREPARE:
495 unregister_hotcpu_notifier(&gator_cpu_notifier);
496 unregister_scheduler_tracepoints();
497 on_each_cpu(trace_sched_insert_idle, NULL, 1);
498 on_each_cpu(__gator_timer_offline, NULL, 1);
499 break;
500 case PM_POST_HIBERNATION:
501 case PM_POST_SUSPEND:
502 on_each_cpu(__gator_timer_online, NULL, 1);
503 register_scheduler_tracepoints();
504 register_hotcpu_notifier(&gator_cpu_notifier);
505 break;
506 }
507
508 return NOTIFY_OK;
509}
510
511static struct notifier_block gator_pm_notifier = {
512 .notifier_call = gator_pm_notify,
513};
514
495static int gator_notifier_start(void) 515static int gator_notifier_start(void)
496{ 516{
497 return register_hotcpu_notifier(&gator_cpu_notifier); 517 int retval;
518 retval = register_hotcpu_notifier(&gator_cpu_notifier);
519 if (retval == 0)
520 retval = register_pm_notifier(&gator_pm_notifier);
521 return retval;
498} 522}
499 523
500static void gator_notifier_stop(void) 524static void gator_notifier_stop(void)
501{ 525{
526 unregister_pm_notifier(&gator_pm_notifier);
502 unregister_hotcpu_notifier(&gator_cpu_notifier); 527 unregister_hotcpu_notifier(&gator_cpu_notifier);
503} 528}
504 529
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index 7c0bd47..0225bfb 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -14,13 +14,6 @@
14#define SCHED_SWITCH 1 14#define SCHED_SWITCH 1
15#define SCHED_PROCESS_FREE 2 15#define SCHED_PROCESS_FREE 2
16 16
17#define FIELD_TYPE 0
18#define FIELD_TIME 1
19#define FIELD_PARAM1 2
20#define FIELD_PARAM2 3
21#define FIELD_PARAM3 4
22#define FIELDS_PER_SCHED 5
23
24#define SCHEDSIZE (8*1024) 17#define SCHEDSIZE (8*1024)
25#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ 18#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
26#define TASK_MAX_COLLISIONS 2 19#define TASK_MAX_COLLISIONS 2
@@ -31,7 +24,13 @@ static DEFINE_PER_CPU(int, theSchedPos);
31static DEFINE_PER_CPU(int, theSchedErr); 24static DEFINE_PER_CPU(int, theSchedErr);
32static DEFINE_PER_CPU(uint64_t *, taskname_keys); 25static DEFINE_PER_CPU(uint64_t *, taskname_keys);
33 26
34void emit_pid_name(uint64_t time, struct task_struct* task) 27enum {
28 STATE_CONTENTION = 0,
29 STATE_WAIT_ON_IO,
30 STATE_WAIT_ON_OTHER
31};
32
33void emit_pid_name(struct task_struct* task)
35{ 34{
36 bool found = false; 35 bool found = false;
37 unsigned long flags; 36 unsigned long flags;
@@ -68,55 +67,92 @@ void emit_pid_name(uint64_t time, struct task_struct* task)
68 // disable interrupts to synchronize with hrtimer populating timer buf 67 // disable interrupts to synchronize with hrtimer populating timer buf
69 local_irq_save(flags); 68 local_irq_save(flags);
70 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME); 69 gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME);
71 gator_buffer_write_packed_int64(cpu, TIMER_BUF, time); 70 gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
72 gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid); 71 gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid);
73 gator_buffer_write_string(cpu, TIMER_BUF, taskcomm); 72 gator_buffer_write_string(cpu, TIMER_BUF, taskcomm);
74 local_irq_restore(flags); 73 local_irq_restore(flags);
75 } 74 }
76} 75}
77 76
78static void probe_sched_write(int type, int param1, int param2, int param3) 77static void probe_sched_write(int type, struct task_struct* task, struct task_struct* old_task)
79{ 78{
79 int schedPos, cookie = 0, state = 0;
80 unsigned long flags; 80 unsigned long flags;
81 uint64_t *schedBuf, time;
81 int cpu = smp_processor_id(); 82 int cpu = smp_processor_id();
82 uint64_t time = gator_get_time(); 83 int pid = task->pid;
83 uint64_t *schedBuf; 84 int tgid = task->tgid;
84 int schedPos, cookie = param3;
85 85
86 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) 86 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
87 return; 87 return;
88 88
89 if (param3) { 89 if (type == SCHED_SWITCH) {
90 // do as much work as possible before disabling interrupts 90 // do as much work as possible before disabling interrupts
91 struct task_struct *task = (struct task_struct *)param3;
92 cookie = get_exec_cookie(cpu, TIMER_BUF, task); 91 cookie = get_exec_cookie(cpu, TIMER_BUF, task);
93 emit_pid_name(time, task); 92 emit_pid_name(task);
93 if (old_task->state == 0)
94 state = STATE_CONTENTION;
95 else if (old_task->in_iowait)
96 state = STATE_WAIT_ON_IO;
97 else
98 state = STATE_WAIT_ON_OTHER;
94 } 99 }
95 100
96 // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used 101 // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used
97 local_irq_save(flags); 102 local_irq_save(flags);
98 103
104 time = gator_get_time();
99 schedPos = per_cpu(theSchedPos, cpu); 105 schedPos = per_cpu(theSchedPos, cpu);
100 schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; 106 schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)];
101 107
102 if (schedPos < (SCHEDSIZE-100)) { 108 if (schedPos < (SCHEDSIZE - 100)) {
103 // capture 109 // capture
104 schedBuf[schedPos+FIELD_TYPE] = type; 110 schedBuf[schedPos++] = type;
105 schedBuf[schedPos+FIELD_TIME] = time; 111 schedBuf[schedPos++] = time;
106 schedBuf[schedPos+FIELD_PARAM1] = param1; 112 schedBuf[schedPos++] = pid;
107 schedBuf[schedPos+FIELD_PARAM2] = param2; 113 schedBuf[schedPos++] = tgid;
108 schedBuf[schedPos+FIELD_PARAM3] = cookie; 114 schedBuf[schedPos++] = cookie;
109 per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; 115 schedBuf[schedPos++] = state;
110 } else if (!per_cpu(theSchedErr, cpu)) { 116 } else if (!per_cpu(theSchedErr, cpu)) {
111 per_cpu(theSchedErr, cpu) = 1; 117 per_cpu(theSchedErr, cpu) = 1;
112 schedBuf[schedPos+FIELD_TYPE] = SCHED_OVERFLOW; 118 schedBuf[schedPos++] = SCHED_OVERFLOW;
113 schedBuf[schedPos+FIELD_TIME] = time; 119 schedBuf[schedPos++] = time;
114 schedBuf[schedPos+FIELD_PARAM1] = 0; 120 schedBuf[schedPos++] = 0;
115 schedBuf[schedPos+FIELD_PARAM2] = 0; 121 schedBuf[schedPos++] = 0;
116 schedBuf[schedPos+FIELD_PARAM3] = 0; 122 schedBuf[schedPos++] = 0;
117 per_cpu(theSchedPos, cpu) = schedPos + FIELDS_PER_SCHED; 123 schedBuf[schedPos++] = 0;
118 pr_debug("gator: tracepoint overflow\n"); 124 pr_debug("gator: tracepoint overflow\n");
119 } 125 }
126 per_cpu(theSchedPos, cpu) = schedPos;
127 local_irq_restore(flags);
128}
129
130// special case used during a suspend of the system
131static void trace_sched_insert_idle(void* unused)
132{
133 unsigned long flags;
134 uint64_t *schedBuf;
135 int schedPos, cpu = smp_processor_id();
136
137 if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
138 return;
139
140 local_irq_save(flags);
141
142 schedPos = per_cpu(theSchedPos, cpu);
143 schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)];
144
145 if (schedPos < (SCHEDSIZE - (6 * 8))) {
146 // capture
147 schedBuf[schedPos++] = SCHED_SWITCH;
148 schedBuf[schedPos++] = gator_get_time();
149 schedBuf[schedPos++] = 0; // idle pid is zero
150 schedBuf[schedPos++] = 0; // idle tid is zero
151 schedBuf[schedPos++] = 0; // idle cookie is zero
152 schedBuf[schedPos++] = STATE_WAIT_ON_OTHER;
153 }
154
155 per_cpu(theSchedPos, cpu) = schedPos;
120 local_irq_restore(flags); 156 local_irq_restore(flags);
121} 157}
122 158
@@ -126,12 +162,12 @@ GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *pre
126GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) 162GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
127#endif 163#endif
128{ 164{
129 probe_sched_write(SCHED_SWITCH, next->pid, next->tgid, (int)next); 165 probe_sched_write(SCHED_SWITCH, next, prev);
130} 166}
131 167
132GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) 168GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
133{ 169{
134 probe_sched_write(SCHED_PROCESS_FREE, p->pid, 0, 0); 170 probe_sched_write(SCHED_PROCESS_FREE, p, 0);
135} 171}
136 172
137int gator_trace_sched_init(void) 173int gator_trace_sched_init(void)
@@ -139,6 +175,25 @@ int gator_trace_sched_init(void)
139 return 0; 175 return 0;
140} 176}
141 177
178static int register_scheduler_tracepoints(void) {
179 // register tracepoints
180 if (GATOR_REGISTER_TRACE(sched_switch))
181 goto fail_sched_switch;
182 if (GATOR_REGISTER_TRACE(sched_process_free))
183 goto fail_sched_process_free;
184 pr_debug("gator: registered tracepoints\n");
185
186 return 0;
187
188 // unregister tracepoints on error
189fail_sched_process_free:
190 GATOR_UNREGISTER_TRACE(sched_switch);
191fail_sched_switch:
192 pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
193
194 return -1;
195}
196
142int gator_trace_sched_start(void) 197int gator_trace_sched_start(void)
143{ 198{
144 int cpu, size; 199 int cpu, size;
@@ -159,30 +214,20 @@ int gator_trace_sched_start(void)
159 memset(per_cpu(taskname_keys, cpu), 0, size); 214 memset(per_cpu(taskname_keys, cpu), 0, size);
160 } 215 }
161 216
162 // register tracepoints 217 return register_scheduler_tracepoints();
163 if (GATOR_REGISTER_TRACE(sched_switch)) 218}
164 goto fail_sched_switch;
165 if (GATOR_REGISTER_TRACE(sched_process_free))
166 goto fail_sched_process_free;
167 pr_debug("gator: registered tracepoints\n");
168
169 return 0;
170 219
171 // unregister tracepoints on error 220static void unregister_scheduler_tracepoints(void)
172fail_sched_process_free: 221{
173 GATOR_UNREGISTER_TRACE(sched_switch); 222 GATOR_UNREGISTER_TRACE(sched_switch);
174fail_sched_switch: 223 GATOR_UNREGISTER_TRACE(sched_process_free);
175 pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); 224 pr_debug("gator: unregistered tracepoints\n");
176
177 return -1;
178} 225}
179 226
180void gator_trace_sched_stop(void) 227void gator_trace_sched_stop(void)
181{ 228{
182 int cpu; 229 int cpu;
183 GATOR_UNREGISTER_TRACE(sched_switch); 230 unregister_scheduler_tracepoints();
184 GATOR_UNREGISTER_TRACE(sched_process_free);
185 pr_debug("gator: unregistered tracepoints\n");
186 231
187 for_each_present_cpu(cpu) { 232 for_each_present_cpu(cpu) {
188 kfree(per_cpu(theSchedBuf, cpu)[0]); 233 kfree(per_cpu(theSchedBuf, cpu)[0]);