driver/gator_cookies.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #define COOKIEMAP_ENTRIES       1024            /* must be power of 2 */
  11 #define TRANSLATE_SIZE          256
  12 #define MAX_COLLISIONS          2
  13
  14 static uint32_t *gator_crc32_table;
  15 static uint32_t translate_buffer_mask;
  16
  17 static DEFINE_PER_CPU(char *, translate_text);
  18 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
  19 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
  20 static DEFINE_PER_CPU(uint32_t *, cookie_values);
  21 static DEFINE_PER_CPU(int, translate_buffer_read);
  22 static DEFINE_PER_CPU(int, translate_buffer_write);
  23 static DEFINE_PER_CPU(unsigned int *, translate_buffer);
  24
  25 static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt);
  26 static void wq_cookie_handler(struct work_struct *unused);
  27 DECLARE_WORK(cookie_work, wq_cookie_handler);
  28
  29 static uint32_t cookiemap_code(uint64_t value64) {
  30         uint32_t value = (uint32_t)((value64 >> 32) + value64);
  31         uint32_t cookiecode = (value >> 24) & 0xff;
  32         cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
  33         cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
  34         cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
  35         cookiecode &= (COOKIEMAP_ENTRIES-1);
  36         return cookiecode * MAX_COLLISIONS;
  37 }
  38
  39 static uint32_t gator_chksum_crc32(char *data)
  40 {
  41    register unsigned long crc;
  42    unsigned char *block = data;
  43    int i, length = strlen(data);
  44
  45    crc = 0xFFFFFFFF;
  46    for (i = 0; i < length; i++) {
  47       crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
  48    }
  49
  50    return (crc ^ 0xFFFFFFFF);
  51 }
  52
  53 /*
  54  * Exists
  55  *  Pre:  [0][1][v][3]..[n-1]
  56  *  Post: [v][0][1][3]..[n-1]
  57  */
  58 static uint32_t cookiemap_exists(uint64_t key) {
  59         unsigned long x, flags, retval = 0;
  60         int cpu = smp_processor_id();
  61         uint32_t cookiecode = cookiemap_code(key);
  62         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  63         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  64
  65         // Can be called from interrupt handler or from work queue
  66         local_irq_save(flags);
  67         for (x = 0; x < MAX_COLLISIONS; x++) {
  68                 if (keys[x] == key) {
  69                         uint32_t value = values[x];
  70                         for (; x > 0; x--) {
  71                                 keys[x] = keys[x-1];
  72                                 values[x] = values[x-1];
  73                         }
  74                         keys[0] = key;
  75                         values[0] = value;
  76                         retval = value;
  77                         break;
  78                 }
  79         }
  80         local_irq_restore(flags);
  81
  82         return retval;
  83 }
  84
  85 /*
  86  * Add
  87  *  Pre:  [0][1][2][3]..[n-1]
  88  *  Post: [v][0][1][2]..[n-2]
  89  */
  90 static void cookiemap_add(uint64_t key, uint32_t value) {
  91         int cpu = smp_processor_id();
  92         int cookiecode = cookiemap_code(key);
  93         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  94         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  95         int x;
  96
  97         for (x = MAX_COLLISIONS-1; x > 0; x--) {
  98                 keys[x] = keys[x-1];
  99                 values[x] = values[x-1];
 100         }
 101         keys[0] = key;
 102         values[0] = value;
 103 }
 104
 105 static void translate_buffer_write_int(int cpu, unsigned int x)
 106 {
 107         per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
 108         per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
 109 }
 110
 111 static unsigned int translate_buffer_read_int(int cpu)
 112 {
 113         unsigned int value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
 114         per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
 115         return value;
 116 }
 117
 118 static void wq_cookie_handler(struct work_struct *unused)
 119 {
 120         struct task_struct *task;
 121         struct vm_area_struct *vma;
 122         int cpu = smp_processor_id();
 123         unsigned int cookie, commit;
 124
 125         mutex_lock(&start_mutex);
 126
 127         if (gator_started != 0) {
 128                 commit = per_cpu(translate_buffer_write, cpu);
 129                 while (per_cpu(translate_buffer_read, cpu) != commit) {
 130                         task = (struct task_struct *)translate_buffer_read_int(cpu);
 131                         vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
 132                         cookie = get_cookie(cpu, BACKTRACE_BUF, task, vma, NULL, false);
 133                 }
 134         }
 135
 136         mutex_unlock(&start_mutex);
 137 }
 138
 139 // Retrieve full name from proc/pid/cmdline for java processes on Android
 140 static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt)
 141 {
 142         void *maddr;
 143         unsigned int len;
 144         unsigned long addr;
 145         struct mm_struct *mm;
 146         struct page *page = NULL;
 147         struct vm_area_struct *page_vma;
 148         int bytes, offset, retval = 0, ptr;
 149         char * buf = per_cpu(translate_text, cpu);
 150
 151         // Push work into a work queue if in atomic context as the kernel functions below might sleep
 152         // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
 153         //   inconsistent during a context switch between android/linux versions
 154         if (in_interrupt) {
 155                 // Check if already in buffer
 156                 ptr = per_cpu(translate_buffer_read, cpu);
 157                 while (ptr != per_cpu(translate_buffer_write, cpu)) {
 158                         if (per_cpu(translate_buffer, cpu)[ptr] == (int)task)
 159                                 goto out;
 160                         ptr = (ptr + 2) & translate_buffer_mask;
 161                 }
 162
 163                 translate_buffer_write_int(cpu, (unsigned int)task);
 164                 translate_buffer_write_int(cpu, (unsigned int)vma);
 165                 schedule_work(&cookie_work);
 166                 goto out;
 167         }
 168
 169         mm = get_task_mm(task);
 170         if (!mm)
 171                 goto out;
 172         if (!mm->arg_end)
 173                 goto outmm;
 174         addr = mm->arg_start;
 175         len = mm->arg_end - mm->arg_start;
 176
 177         if (len > TRANSLATE_SIZE)
 178                 len = TRANSLATE_SIZE;
 179
 180         down_read(&mm->mmap_sem);
 181         while (len) {
 182                 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
 183                         goto outsem;
 184
 185                 maddr = kmap(page);
 186                 offset = addr & (PAGE_SIZE-1);
 187                 bytes = len;
 188                 if (bytes > PAGE_SIZE - offset)
 189                         bytes = PAGE_SIZE - offset;
 190
 191                 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
 192
 193                 kunmap(page);   // release page allocated by get_user_pages()
 194                 page_cache_release(page);
 195
 196                 len -= bytes;
 197                 buf += bytes;
 198                 addr += bytes;
 199
 200                 *text = per_cpu(translate_text, cpu);
 201                 retval = 1;
 202         }
 203
 204         // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
 205         if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
 206                 retval = 0;
 207
 208 outsem:
 209         up_read(&mm->mmap_sem);
 210 outmm:
 211         mmput(mm);
 212 out:
 213         return retval;
 214 }
 215
 216 static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt)
 217 {
 218         unsigned long flags, cookie;
 219         struct path *path;
 220         uint64_t key;
 221         char *text;
 222
 223         if (mod) {
 224                 text = mod->name;
 225         } else {
 226                 if (!vma || !vma->vm_file) {
 227                         return INVALID_COOKIE;
 228                 }
 229                 path = &vma->vm_file->f_path;
 230                 if (!path || !path->dentry) {
 231                         return INVALID_COOKIE;
 232                 }
 233
 234                 text = (char*)path->dentry->d_name.name;
 235         }
 236
 237         key = gator_chksum_crc32(text);
 238         key = (key << 32) | (uint32_t)task->tgid;
 239
 240         cookie = cookiemap_exists(key);
 241         if (cookie) {
 242                 return cookie;
 243         }
 244
 245         if (strcmp(text, "app_process") == 0 && !mod) {
 246                 if (!translate_app_process(&text, cpu, task, vma, in_interrupt))
 247                         return INVALID_COOKIE;
 248         }
 249
 250         // Can be called from interrupt handler or from work queue or from scheduler trace
 251         local_irq_save(flags);
 252
 253         cookie = INVALID_COOKIE;
 254         if (marshal_cookie_header(text)) {
 255                 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
 256                 cookiemap_add(key, cookie);
 257                 marshal_cookie(cookie, text);
 258         }
 259
 260         local_irq_restore(flags);
 261
 262         return cookie;
 263 }
 264
 265 static int get_exec_cookie(int cpu, int buftype, struct task_struct *task)
 266 {
 267         unsigned long cookie = NO_COOKIE;
 268         struct mm_struct *mm = task->mm;
 269         struct vm_area_struct *vma;
 270
 271         // kernel threads have no address space
 272         if (!mm)
 273                 return cookie;
 274
 275         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 276                 if (!vma->vm_file)
 277                         continue;
 278                 if (!(vma->vm_flags & VM_EXECUTABLE))
 279                         continue;
 280                 cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
 281                 break;
 282         }
 283
 284         return cookie;
 285 }
 286
 287 static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset)
 288 {
 289         unsigned long cookie = NO_COOKIE;
 290         struct mm_struct *mm = task->mm;
 291         struct vm_area_struct *vma;
 292
 293         if (!mm)
 294                 return cookie;
 295
 296         for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 297                 if (addr < vma->vm_start || addr >= vma->vm_end)
 298                         continue;
 299
 300                 if (vma->vm_file) {
 301                         cookie = get_cookie(cpu, buftype, task, vma, NULL, true);
 302                         *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 303                 } else {
 304                         /* must be an anonymous map */
 305                         *offset = addr;
 306                 }
 307
 308                 break;
 309         }
 310
 311         if (!vma)
 312                 cookie = INVALID_COOKIE;
 313
 314         return cookie;
 315 }
 316
 317 static int cookies_initialize(void)
 318 {
 319         uint32_t crc, poly;
 320         int i, j, cpu, size, err = 0;
 321
 322         int translate_buffer_size = 512; // must be a power of 2
 323         translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 324
 325         for_each_present_cpu(cpu) {
 326                 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
 327
 328                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
 329                 per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
 330                 if (!per_cpu(cookie_keys, cpu)) {
 331                         err = -ENOMEM;
 332                         goto cookie_setup_error;
 333                 }
 334                 memset(per_cpu(cookie_keys, cpu), 0, size);
 335
 336                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 337                 per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
 338                 if (!per_cpu(cookie_values, cpu)) {
 339                         err = -ENOMEM;
 340                         goto cookie_setup_error;
 341                 }
 342                 memset(per_cpu(cookie_values, cpu), 0, size);
 343
 344                 per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL);
 345                 if (!per_cpu(translate_buffer, cpu)) {
 346                         err = -ENOMEM;
 347                         goto cookie_setup_error;
 348                 }
 349
 350                 per_cpu(translate_buffer_write, cpu) = 0;
 351                 per_cpu(translate_buffer_read, cpu) = 0;
 352
 353                 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL);
 354                 if (!per_cpu(translate_text, cpu)) {
 355                         err = -ENOMEM;
 356                         goto cookie_setup_error;
 357                 }
 358         }
 359
 360         // build CRC32 table
 361         poly = 0x04c11db7;
 362         gator_crc32_table = (uint32_t*)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
 363         for (i = 0; i < 256; i++) {
 364                 crc = i;
 365                 for (j = 8; j > 0; j--) {
 366                         if (crc & 1) {
 367                                 crc = (crc >> 1) ^ poly;
 368                         } else {
 369                                 crc >>= 1;
 370                         }
 371                 }
 372                 gator_crc32_table[i] = crc;
 373         }
 374
 375 cookie_setup_error:
 376         return err;
 377 }
 378
 379 static void cookies_release(void)
 380 {
 381         int cpu;
 382
 383         for_each_present_cpu(cpu) {
 384                 kfree(per_cpu(cookie_keys, cpu));
 385                 per_cpu(cookie_keys, cpu) = NULL;
 386
 387                 kfree(per_cpu(cookie_values, cpu));
 388                 per_cpu(cookie_values, cpu) = NULL;
 389
 390                 kfree(per_cpu(translate_buffer, cpu));
 391                 per_cpu(translate_buffer, cpu) = NULL;
 392                 per_cpu(translate_buffer_read, cpu) = 0;
 393                 per_cpu(translate_buffer_write, cpu) = 0;
 394
 395                 kfree(per_cpu(translate_text, cpu));
 396                 per_cpu(translate_text, cpu) = NULL;
 397         }
 398
 399         kfree(gator_crc32_table);
 400         gator_crc32_table = NULL;
 401 }