driver/gator_trace_sched.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include <trace/events/sched.h>
  11 #include "gator.h"
  12
  13 #define SCHED_SWITCH                    1
  14 #define SCHED_PROCESS_EXIT              2
  15
  16 #define TASK_MAP_ENTRIES                1024    /* must be power of 2 */
  17 #define TASK_MAX_COLLISIONS             2
  18
  19 enum {
  20         STATE_WAIT_ON_OTHER = 0,
  21         STATE_CONTENTION,
  22         STATE_WAIT_ON_IO,
  23         CPU_WAIT_TOTAL
  24 };
  25
  26 static DEFINE_PER_CPU(uint64_t *, taskname_keys);
  27 static DEFINE_PER_CPU(int, collecting);
  28
  29 // this array is never read as the cpu wait charts are derived counters
  30 // the files are needed, nonetheless, to show that these counters are available
  31 static ulong cpu_wait_enabled[CPU_WAIT_TOTAL];
  32 static ulong sched_cpu_key[CPU_WAIT_TOTAL];
  33
  34 static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
  35 {
  36         struct dentry *dir;
  37
  38         // CPU Wait - Contention
  39         dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_contention");
  40         if (!dir) {
  41                 return -1;
  42         }
  43         gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_CONTENTION]);
  44         gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_CONTENTION]);
  45
  46         // CPU Wait - I/O
  47         dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_io");
  48         if (!dir) {
  49                 return -1;
  50         }
  51         gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_WAIT_ON_IO]);
  52         gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_WAIT_ON_IO]);
  53
  54         return 0;
  55 }
  56
  57 void emit_pid_name(struct task_struct *task)
  58 {
  59         bool found = false;
  60         char taskcomm[TASK_COMM_LEN + 3];
  61         unsigned long x, cpu = get_physical_cpu();
  62         uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
  63         uint64_t value;
  64
  65         value = gator_chksum_crc32(task->comm);
  66         value = (value << 32) | (uint32_t)task->pid;
  67
  68         // determine if the thread name was emitted already
  69         for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
  70                 if (keys[x] == value) {
  71                         found = true;
  72                         break;
  73                 }
  74         }
  75
  76         if (!found) {
  77                 // shift values, new value always in front
  78                 uint64_t oldv, newv = value;
  79                 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
  80                         oldv = keys[x];
  81                         keys[x] = newv;
  82                         newv = oldv;
  83                 }
  84
  85                 // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
  86                 if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
  87                         // append ellipses if task->comm has length of TASK_COMM_LEN - 1
  88                         strcat(taskcomm, "...");
  89                 }
  90
  91                 marshal_thread_name(task->pid, taskcomm);
  92         }
  93 }
  94
  95 static void collect_counters(void)
  96 {
  97         int *buffer, len, cpu = get_physical_cpu();
  98         long long *buffer64;
  99         struct gator_interface *gi;
 100         u64 time;
 101
 102         time = gator_get_time();
 103         if (marshal_event_header(time)) {
 104                 list_for_each_entry(gi, &gator_events, list) {
 105                         if (gi->read) {
 106                                 len = gi->read(&buffer);
 107                                 marshal_event(len, buffer);
 108                         } else if (gi->read64) {
 109                                 len = gi->read64(&buffer64);
 110                                 marshal_event64(len, buffer64);
 111                         }
 112                 }
 113                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 114                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 115
 116 #if GATOR_LIVE
 117                 // Commit buffers on timeout
 118                 if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
 119                         static const int buftypes[] = { COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF };
 120                         int i;
 121                         for (i = 0; i < sizeof(buftypes)/sizeof(buftypes[0]); ++i) {
 122                                 gator_commit_buffer(cpu, buftypes[i], time);
 123                         }
 124                 }
 125 #endif
 126         }
 127 }
 128
 129 static void probe_sched_write(int type, struct task_struct *task, struct task_struct *old_task)
 130 {
 131         int cookie = 0, state = 0;
 132         int cpu = get_physical_cpu();
 133         int tgid = task->tgid;
 134         int pid = task->pid;
 135
 136         if (type == SCHED_SWITCH) {
 137                 // do as much work as possible before disabling interrupts
 138                 cookie = get_exec_cookie(cpu, task);
 139                 emit_pid_name(task);
 140                 if (old_task->state == TASK_RUNNING) {
 141                         state = STATE_CONTENTION;
 142                 } else if (old_task->in_iowait) {
 143                         state = STATE_WAIT_ON_IO;
 144                 } else {
 145                         state = STATE_WAIT_ON_OTHER;
 146                 }
 147
 148                 per_cpu(collecting, cpu) = 1;
 149                 collect_counters();
 150                 per_cpu(collecting, cpu) = 0;
 151         }
 152
 153         // marshal_sched_trace() disables interrupts as the free may trigger while switch is writing to the buffer; disabling preemption is not sufficient
 154         // is disable interrupts necessary now that exit is used instead of free?
 155         if (type == SCHED_SWITCH) {
 156                 marshal_sched_trace_switch(tgid, pid, cookie, state);
 157         } else {
 158                 marshal_sched_trace_exit(tgid, pid);
 159         }
 160 }
 161
 162 // special case used during a suspend of the system
 163 static void trace_sched_insert_idle(void)
 164 {
 165         marshal_sched_trace_switch(0, 0, 0, 0);
 166 }
 167
 168 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
 169 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next))
 170 #else
 171 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
 172 #endif
 173 {
 174         probe_sched_write(SCHED_SWITCH, next, prev);
 175 }
 176
 177 GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p))
 178 {
 179         probe_sched_write(SCHED_PROCESS_EXIT, p, 0);
 180 }
 181
 182 static void do_nothing(void *info)
 183 {
 184         // Intentionally do nothing
 185         (void)info;
 186 }
 187
 188 static int register_scheduler_tracepoints(void)
 189 {
 190         // register tracepoints
 191         if (GATOR_REGISTER_TRACE(sched_switch))
 192                 goto fail_sched_switch;
 193         if (GATOR_REGISTER_TRACE(sched_process_exit))
 194                 goto fail_sched_process_exit;
 195         pr_debug("gator: registered tracepoints\n");
 196
 197         // Now that the scheduler tracepoint is registered, force a context switch
 198         // on all cpus to capture what is currently running.
 199         on_each_cpu(do_nothing, NULL, 0);
 200
 201         return 0;
 202
 203         // unregister tracepoints on error
 204 fail_sched_process_exit:
 205         GATOR_UNREGISTER_TRACE(sched_switch);
 206 fail_sched_switch:
 207         pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
 208
 209         return -1;
 210 }
 211
 212 int gator_trace_sched_start(void)
 213 {
 214         int cpu, size;
 215
 216         for_each_present_cpu(cpu) {
 217                 size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
 218                 per_cpu(taskname_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
 219                 if (!per_cpu(taskname_keys, cpu))
 220                         return -1;
 221                 memset(per_cpu(taskname_keys, cpu), 0, size);
 222         }
 223
 224         return register_scheduler_tracepoints();
 225 }
 226
 227 void gator_trace_sched_offline(void)
 228 {
 229         trace_sched_insert_idle();
 230 }
 231
 232 static void unregister_scheduler_tracepoints(void)
 233 {
 234         GATOR_UNREGISTER_TRACE(sched_switch);
 235         GATOR_UNREGISTER_TRACE(sched_process_exit);
 236         pr_debug("gator: unregistered tracepoints\n");
 237 }
 238
 239 void gator_trace_sched_stop(void)
 240 {
 241         int cpu;
 242         unregister_scheduler_tracepoints();
 243
 244         for_each_present_cpu(cpu) {
 245                 kfree(per_cpu(taskname_keys, cpu));
 246         }
 247 }
 248
 249 void gator_trace_sched_init(void)
 250 {
 251         int i;
 252         for (i = 0; i < CPU_WAIT_TOTAL; i++) {
 253                 cpu_wait_enabled[i] = 0;
 254                 sched_cpu_key[i] = gator_events_get_key();
 255         }
 256 }