driver/gator_cookies.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2012. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #define COOKIEMAP_ENTRIES       1024            /* must be power of 2 */
  11 #define TRANSLATE_SIZE          256
  12 #define MAX_COLLISIONS          2
  13
  14 static uint32_t *gator_crc32_table;
  15 static unsigned int translate_buffer_mask;
  16
  17 static DEFINE_PER_CPU(char *, translate_text);
  18 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
  19 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
  20 static DEFINE_PER_CPU(uint32_t *, cookie_values);
  21 static DEFINE_PER_CPU(int, translate_buffer_read);
  22 static DEFINE_PER_CPU(int, translate_buffer_write);
  23 static DEFINE_PER_CPU(void * *, translate_buffer);
  24
  25 static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt);
  26 static void wq_cookie_handler(struct work_struct *unused);
  27 DECLARE_WORK(cookie_work, wq_cookie_handler);
  28 static struct timer_list app_process_wake_up_timer;
  29 static void app_process_wake_up_handler(unsigned long unused_data);
  30
  31 static uint32_t cookiemap_code(uint64_t value64) {
  32         uint32_t value = (uint32_t)((value64 >> 32) + value64);
  33         uint32_t cookiecode = (value >> 24) & 0xff;
  34         cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
  35         cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
  36         cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
  37         cookiecode &= (COOKIEMAP_ENTRIES-1);
  38         return cookiecode * MAX_COLLISIONS;
  39 }
  40
  41 static uint32_t gator_chksum_crc32(char *data)
  42 {
  43         register unsigned long crc;
  44         unsigned char *block = data;
  45         int i, length = strlen(data);
  46
  47         crc = 0xFFFFFFFF;
  48         for (i = 0; i < length; i++) {
  49                 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
  50         }
  51
  52         return (crc ^ 0xFFFFFFFF);
  53 }
  54
  55 /*
  56  * Exists
  57  *  Pre:  [0][1][v][3]..[n-1]
  58  *  Post: [v][0][1][3]..[n-1]
  59  */
  60 static uint32_t cookiemap_exists(uint64_t key) {
  61         unsigned long x, flags, retval = 0;
  62         int cpu = smp_processor_id();
  63         uint32_t cookiecode = cookiemap_code(key);
  64         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  65         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  66
  67         // Can be called from interrupt handler or from work queue
  68         local_irq_save(flags);
  69         for (x = 0; x < MAX_COLLISIONS; x++) {
  70                 if (keys[x] == key) {
  71                         uint32_t value = values[x];
  72                         for (; x > 0; x--) {
  73                                 keys[x] = keys[x-1];
  74                                 values[x] = values[x-1];
  75                         }
  76                         keys[0] = key;
  77                         values[0] = value;
  78                         retval = value;
  79                         break;
  80                 }
  81         }
  82         local_irq_restore(flags);
  83
  84         return retval;
  85 }
  86
  87 /*
  88  * Add
  89  *  Pre:  [0][1][2][3]..[n-1]
  90  *  Post: [v][0][1][2]..[n-2]
  91  */
  92 static void cookiemap_add(uint64_t key, uint32_t value) {
  93         int cpu = smp_processor_id();
  94         int cookiecode = cookiemap_code(key);
  95         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  96         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  97         int x;
  98
  99         for (x = MAX_COLLISIONS-1; x > 0; x--) {
 100                 keys[x] = keys[x-1];
 101                 values[x] = values[x-1];
 102         }
 103         keys[0] = key;
 104         values[0] = value;
 105 }
 106
 107 static void translate_buffer_write_ptr(int cpu, void * x)
 108 {
 109         per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
 110         per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
 111 }
 112
 113 static void * translate_buffer_read_ptr(int cpu)
 114 {
 115         void * value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
 116         per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
 117         return value;
 118 }
 119
 120 static void wq_cookie_handler(struct work_struct *unused)
 121 {
 122         struct task_struct *task;
 123         struct vm_area_struct *vma;
 124         int cpu = smp_processor_id();
 125         unsigned int cookie, commit;
 126
 127         mutex_lock(&start_mutex);
 128
 129         if (gator_started != 0) {
 130                 commit = per_cpu(translate_buffer_write, cpu);
 131                 while (per_cpu(translate_buffer_read, cpu) != commit) {
 132                         task = (struct task_struct *)translate_buffer_read_ptr(cpu);
 133                         vma = (struct vm_area_struct *)translate_buffer_read_ptr(cpu);
 134                         cookie = get_cookie(cpu, task, vma, NULL, false);
 135                 }
 136         }
 137
 138         mutex_unlock(&start_mutex);
 139 }
 140
 141 static void app_process_wake_up_handler(unsigned long unused_data)
 142 {
 143         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 144         schedule_work(&cookie_work);
 145 }
 146
 147 // Retrieve full name from proc/pid/cmdline for java processes on Android
 148 static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt)
 149 {
 150         void *maddr;
 151         unsigned int len;
 152         unsigned long addr;
 153         struct mm_struct *mm;
 154         struct page *page = NULL;
 155         struct vm_area_struct *page_vma;
 156         int bytes, offset, retval = 0, ptr;
 157         char * buf = per_cpu(translate_text, cpu);
 158
 159         // Push work into a work queue if in atomic context as the kernel functions below might sleep
 160         // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
 161         //   inconsistent during a context switch between android/linux versions
 162         if (in_interrupt) {
 163                 // Check if already in buffer
 164                 ptr = per_cpu(translate_buffer_read, cpu);
 165                 while (ptr != per_cpu(translate_buffer_write, cpu)) {
 166                         if (per_cpu(translate_buffer, cpu)[ptr] == (void *)task)
 167                                 goto out;
 168                         ptr = (ptr + 2) & translate_buffer_mask;
 169                 }
 170
 171                 translate_buffer_write_ptr(cpu, (void *)task);
 172                 translate_buffer_write_ptr(cpu, (void *)vma);
 173
 174                 mod_timer(&app_process_wake_up_timer, jiffies + 1);
 175                 goto out;
 176         }
 177
 178         mm = get_task_mm(task);
 179         if (!mm)
 180                 goto out;
 181         if (!mm->arg_end)
 182                 goto outmm;
 183         addr = mm->arg_start;
 184         len = mm->arg_end - mm->arg_start;
 185
 186         if (len > TRANSLATE_SIZE)
 187                 len = TRANSLATE_SIZE;
 188
 189         down_read(&mm->mmap_sem);
 190         while (len) {
 191                 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
 192                         goto outsem;
 193
 194                 maddr = kmap(page);
 195                 offset = addr & (PAGE_SIZE-1);
 196                 bytes = len;
 197                 if (bytes > PAGE_SIZE - offset)
 198                         bytes = PAGE_SIZE - offset;
 199
 200                 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
 201
 202                 kunmap(page);   // release page allocated by get_user_pages()
 203                 page_cache_release(page);
 204
 205                 len -= bytes;
 206                 buf += bytes;
 207                 addr += bytes;
 208
 209                 *text = per_cpu(translate_text, cpu);
 210                 retval = 1;
 211         }
 212
 213         // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
 214         if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
 215                 retval = 0;
 216
 217 outsem:
 218         up_read(&mm->mmap_sem);
 219 outmm:
 220         mmput(mm);
 221 out:
 222         return retval;
 223 }
 224
 225 static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt)
 226 {
 227         unsigned long flags, cookie;
 228         struct path *path;
 229         uint64_t key;
 230         char *text;
 231
 232         if (mod) {
 233                 text = mod->name;
 234         } else {
 235                 if (!vma || !vma->vm_file) {
 236                         return INVALID_COOKIE;
 237                 }
 238                 path = &vma->vm_file->f_path;
 239                 if (!path || !path->dentry) {
 240                         return INVALID_COOKIE;
 241                 }
 242
 243                 text = (char*)path->dentry->d_name.name;
 244         }
 245
 246         key = gator_chksum_crc32(text);
 247         key = (key << 32) | (uint32_t)task->tgid;
 248
 249         cookie = cookiemap_exists(key);
 250         if (cookie) {
 251                 return cookie;
 252         }
 253
 254         if (strcmp(text, "app_process") == 0 && !mod) {
 255                 if (!translate_app_process(&text, cpu, task, vma, in_interrupt))
 256                         return INVALID_COOKIE;
 257         }
 258
 259         // Can be called from interrupt handler or from work queue or from scheduler trace
 260         local_irq_save(flags);
 261
 262         cookie = INVALID_COOKIE;
 263         if (marshal_cookie_header(text)) {
 264                 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
 265                 cookiemap_add(key, cookie);
 266                 marshal_cookie(cookie, text);
 267         }
 268
 269         local_irq_restore(flags);
 270
 271         return cookie;
 272 }
 273
 274 static int get_exec_cookie(int cpu, struct task_struct *task)
 275 {
 276         unsigned long cookie = NO_COOKIE;
 277         struct mm_struct *mm = task->mm;
 278         struct vm_area_struct *vma;
 279
 280         // kernel threads have no address space
 281         if (!mm)
 282                 return cookie;
 283
 284         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 285                 if (!vma->vm_file)
 286                         continue;
 287                 if (!(vma->vm_flags & VM_EXECUTABLE))
 288                         continue;
 289                 cookie = get_cookie(cpu, task, vma, NULL, true);
 290                 break;
 291         }
 292
 293         return cookie;
 294 }
 295
 296 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
 297 {
 298         unsigned long cookie = NO_COOKIE;
 299         struct mm_struct *mm = task->mm;
 300         struct vm_area_struct *vma;
 301
 302         if (!mm)
 303                 return cookie;
 304
 305         for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 306                 if (addr < vma->vm_start || addr >= vma->vm_end)
 307                         continue;
 308
 309                 if (vma->vm_file) {
 310                         cookie = get_cookie(cpu, task, vma, NULL, true);
 311                         *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 312                 } else {
 313                         /* must be an anonymous map */
 314                         *offset = addr;
 315                 }
 316
 317                 break;
 318         }
 319
 320         if (!vma)
 321                 cookie = INVALID_COOKIE;
 322
 323         return cookie;
 324 }
 325
 326 static int cookies_initialize(void)
 327 {
 328         uint32_t crc, poly;
 329         int i, j, cpu, size, err = 0;
 330
 331         int translate_buffer_size = 512; // must be a power of 2
 332         translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 333
 334         for_each_present_cpu(cpu) {
 335                 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
 336
 337                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
 338                 per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
 339                 if (!per_cpu(cookie_keys, cpu)) {
 340                         err = -ENOMEM;
 341                         goto cookie_setup_error;
 342                 }
 343                 memset(per_cpu(cookie_keys, cpu), 0, size);
 344
 345                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 346                 per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
 347                 if (!per_cpu(cookie_values, cpu)) {
 348                         err = -ENOMEM;
 349                         goto cookie_setup_error;
 350                 }
 351                 memset(per_cpu(cookie_values, cpu), 0, size);
 352
 353                 per_cpu(translate_buffer, cpu) = (void * *)kmalloc(translate_buffer_size, GFP_KERNEL);
 354                 if (!per_cpu(translate_buffer, cpu)) {
 355                         err = -ENOMEM;
 356                         goto cookie_setup_error;
 357                 }
 358
 359                 per_cpu(translate_buffer_write, cpu) = 0;
 360                 per_cpu(translate_buffer_read, cpu) = 0;
 361
 362                 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL);
 363                 if (!per_cpu(translate_text, cpu)) {
 364                         err = -ENOMEM;
 365                         goto cookie_setup_error;
 366                 }
 367         }
 368
 369         // build CRC32 table
 370         poly = 0x04c11db7;
 371         gator_crc32_table = (uint32_t*)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
 372         for (i = 0; i < 256; i++) {
 373                 crc = i;
 374                 for (j = 8; j > 0; j--) {
 375                         if (crc & 1) {
 376                                 crc = (crc >> 1) ^ poly;
 377                         } else {
 378                                 crc >>= 1;
 379                         }
 380                 }
 381                 gator_crc32_table[i] = crc;
 382         }
 383
 384         setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
 385
 386 cookie_setup_error:
 387         return err;
 388 }
 389
 390 static void cookies_release(void)
 391 {
 392         int cpu;
 393
 394         for_each_present_cpu(cpu) {
 395                 kfree(per_cpu(cookie_keys, cpu));
 396                 per_cpu(cookie_keys, cpu) = NULL;
 397
 398                 kfree(per_cpu(cookie_values, cpu));
 399                 per_cpu(cookie_values, cpu) = NULL;
 400
 401                 kfree(per_cpu(translate_buffer, cpu));
 402                 per_cpu(translate_buffer, cpu) = NULL;
 403                 per_cpu(translate_buffer_read, cpu) = 0;
 404                 per_cpu(translate_buffer_write, cpu) = 0;
 405
 406                 kfree(per_cpu(translate_text, cpu));
 407                 per_cpu(translate_text, cpu) = NULL;
 408         }
 409
 410         del_timer_sync(&app_process_wake_up_timer);
 411         kfree(gator_crc32_table);
 412         gator_crc32_table = NULL;
 413 }