1 /**
2 * Copyright (C) ARM Limited 2010-2012. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 */
10 #define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */
11 #define TRANSLATE_SIZE 256
12 #define MAX_COLLISIONS 2
14 static uint32_t *gator_crc32_table;
15 static unsigned int translate_buffer_mask;
17 static DEFINE_PER_CPU(char *, translate_text);
18 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
19 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
20 static DEFINE_PER_CPU(uint32_t *, cookie_values);
21 static DEFINE_PER_CPU(int, translate_buffer_read);
22 static DEFINE_PER_CPU(int, translate_buffer_write);
23 static DEFINE_PER_CPU(void * *, translate_buffer);
25 static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt);
26 static void wq_cookie_handler(struct work_struct *unused);
27 DECLARE_WORK(cookie_work, wq_cookie_handler);
28 static struct timer_list app_process_wake_up_timer;
29 static void app_process_wake_up_handler(unsigned long unused_data);
31 static uint32_t cookiemap_code(uint64_t value64) {
32 uint32_t value = (uint32_t)((value64 >> 32) + value64);
33 uint32_t cookiecode = (value >> 24) & 0xff;
34 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
35 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
36 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
37 cookiecode &= (COOKIEMAP_ENTRIES-1);
38 return cookiecode * MAX_COLLISIONS;
39 }
41 static uint32_t gator_chksum_crc32(char *data)
42 {
43 register unsigned long crc;
44 unsigned char *block = data;
45 int i, length = strlen(data);
47 crc = 0xFFFFFFFF;
48 for (i = 0; i < length; i++) {
49 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
50 }
52 return (crc ^ 0xFFFFFFFF);
53 }
55 /*
56 * Exists
57 * Pre: [0][1][v][3]..[n-1]
58 * Post: [v][0][1][3]..[n-1]
59 */
60 static uint32_t cookiemap_exists(uint64_t key) {
61 unsigned long x, flags, retval = 0;
62 int cpu = smp_processor_id();
63 uint32_t cookiecode = cookiemap_code(key);
64 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
65 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
67 // Can be called from interrupt handler or from work queue
68 local_irq_save(flags);
69 for (x = 0; x < MAX_COLLISIONS; x++) {
70 if (keys[x] == key) {
71 uint32_t value = values[x];
72 for (; x > 0; x--) {
73 keys[x] = keys[x-1];
74 values[x] = values[x-1];
75 }
76 keys[0] = key;
77 values[0] = value;
78 retval = value;
79 break;
80 }
81 }
82 local_irq_restore(flags);
84 return retval;
85 }
87 /*
88 * Add
89 * Pre: [0][1][2][3]..[n-1]
90 * Post: [v][0][1][2]..[n-2]
91 */
92 static void cookiemap_add(uint64_t key, uint32_t value) {
93 int cpu = smp_processor_id();
94 int cookiecode = cookiemap_code(key);
95 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
96 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
97 int x;
99 for (x = MAX_COLLISIONS-1; x > 0; x--) {
100 keys[x] = keys[x-1];
101 values[x] = values[x-1];
102 }
103 keys[0] = key;
104 values[0] = value;
105 }
107 static void translate_buffer_write_ptr(int cpu, void * x)
108 {
109 per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
110 per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
111 }
113 static void * translate_buffer_read_ptr(int cpu)
114 {
115 void * value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
116 per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
117 return value;
118 }
120 static void wq_cookie_handler(struct work_struct *unused)
121 {
122 struct task_struct *task;
123 struct vm_area_struct *vma;
124 int cpu = smp_processor_id();
125 unsigned int cookie, commit;
127 mutex_lock(&start_mutex);
129 if (gator_started != 0) {
130 commit = per_cpu(translate_buffer_write, cpu);
131 while (per_cpu(translate_buffer_read, cpu) != commit) {
132 task = (struct task_struct *)translate_buffer_read_ptr(cpu);
133 vma = (struct vm_area_struct *)translate_buffer_read_ptr(cpu);
134 cookie = get_cookie(cpu, task, vma, NULL, false);
135 }
136 }
138 mutex_unlock(&start_mutex);
139 }
141 static void app_process_wake_up_handler(unsigned long unused_data)
142 {
143 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
144 schedule_work(&cookie_work);
145 }
147 // Retrieve full name from proc/pid/cmdline for java processes on Android
148 static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt)
149 {
150 void *maddr;
151 unsigned int len;
152 unsigned long addr;
153 struct mm_struct *mm;
154 struct page *page = NULL;
155 struct vm_area_struct *page_vma;
156 int bytes, offset, retval = 0, ptr;
157 char * buf = per_cpu(translate_text, cpu);
159 // Push work into a work queue if in atomic context as the kernel functions below might sleep
160 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
161 // inconsistent during a context switch between android/linux versions
162 if (in_interrupt) {
163 // Check if already in buffer
164 ptr = per_cpu(translate_buffer_read, cpu);
165 while (ptr != per_cpu(translate_buffer_write, cpu)) {
166 if (per_cpu(translate_buffer, cpu)[ptr] == (void *)task)
167 goto out;
168 ptr = (ptr + 2) & translate_buffer_mask;
169 }
171 translate_buffer_write_ptr(cpu, (void *)task);
172 translate_buffer_write_ptr(cpu, (void *)vma);
174 mod_timer(&app_process_wake_up_timer, jiffies + 1);
175 goto out;
176 }
178 mm = get_task_mm(task);
179 if (!mm)
180 goto out;
181 if (!mm->arg_end)
182 goto outmm;
183 addr = mm->arg_start;
184 len = mm->arg_end - mm->arg_start;
186 if (len > TRANSLATE_SIZE)
187 len = TRANSLATE_SIZE;
189 down_read(&mm->mmap_sem);
190 while (len) {
191 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
192 goto outsem;
194 maddr = kmap(page);
195 offset = addr & (PAGE_SIZE-1);
196 bytes = len;
197 if (bytes > PAGE_SIZE - offset)
198 bytes = PAGE_SIZE - offset;
200 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
202 kunmap(page); // release page allocated by get_user_pages()
203 page_cache_release(page);
205 len -= bytes;
206 buf += bytes;
207 addr += bytes;
209 *text = per_cpu(translate_text, cpu);
210 retval = 1;
211 }
213 // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
214 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
215 retval = 0;
217 outsem:
218 up_read(&mm->mmap_sem);
219 outmm:
220 mmput(mm);
221 out:
222 return retval;
223 }
225 static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt)
226 {
227 unsigned long flags, cookie;
228 struct path *path;
229 uint64_t key;
230 char *text;
232 if (mod) {
233 text = mod->name;
234 } else {
235 if (!vma || !vma->vm_file) {
236 return INVALID_COOKIE;
237 }
238 path = &vma->vm_file->f_path;
239 if (!path || !path->dentry) {
240 return INVALID_COOKIE;
241 }
243 text = (char*)path->dentry->d_name.name;
244 }
246 key = gator_chksum_crc32(text);
247 key = (key << 32) | (uint32_t)task->tgid;
249 cookie = cookiemap_exists(key);
250 if (cookie) {
251 return cookie;
252 }
254 if (strcmp(text, "app_process") == 0 && !mod) {
255 if (!translate_app_process(&text, cpu, task, vma, in_interrupt))
256 return INVALID_COOKIE;
257 }
259 // Can be called from interrupt handler or from work queue or from scheduler trace
260 local_irq_save(flags);
262 cookie = INVALID_COOKIE;
263 if (marshal_cookie_header(text)) {
264 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
265 cookiemap_add(key, cookie);
266 marshal_cookie(cookie, text);
267 }
269 local_irq_restore(flags);
271 return cookie;
272 }
274 static int get_exec_cookie(int cpu, struct task_struct *task)
275 {
276 unsigned long cookie = NO_COOKIE;
277 struct mm_struct *mm = task->mm;
278 struct vm_area_struct *vma;
280 // kernel threads have no address space
281 if (!mm)
282 return cookie;
284 for (vma = mm->mmap; vma; vma = vma->vm_next) {
285 if (!vma->vm_file)
286 continue;
287 if (!(vma->vm_flags & VM_EXECUTABLE))
288 continue;
289 cookie = get_cookie(cpu, task, vma, NULL, true);
290 break;
291 }
293 return cookie;
294 }
296 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
297 {
298 unsigned long cookie = NO_COOKIE;
299 struct mm_struct *mm = task->mm;
300 struct vm_area_struct *vma;
302 if (!mm)
303 return cookie;
305 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
306 if (addr < vma->vm_start || addr >= vma->vm_end)
307 continue;
309 if (vma->vm_file) {
310 cookie = get_cookie(cpu, task, vma, NULL, true);
311 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
312 } else {
313 /* must be an anonymous map */
314 *offset = addr;
315 }
317 break;
318 }
320 if (!vma)
321 cookie = INVALID_COOKIE;
323 return cookie;
324 }
326 static int cookies_initialize(void)
327 {
328 uint32_t crc, poly;
329 int i, j, cpu, size, err = 0;
331 int translate_buffer_size = 512; // must be a power of 2
332 translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
334 for_each_present_cpu(cpu) {
335 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
337 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
338 per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
339 if (!per_cpu(cookie_keys, cpu)) {
340 err = -ENOMEM;
341 goto cookie_setup_error;
342 }
343 memset(per_cpu(cookie_keys, cpu), 0, size);
345 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
346 per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
347 if (!per_cpu(cookie_values, cpu)) {
348 err = -ENOMEM;
349 goto cookie_setup_error;
350 }
351 memset(per_cpu(cookie_values, cpu), 0, size);
353 per_cpu(translate_buffer, cpu) = (void * *)kmalloc(translate_buffer_size, GFP_KERNEL);
354 if (!per_cpu(translate_buffer, cpu)) {
355 err = -ENOMEM;
356 goto cookie_setup_error;
357 }
359 per_cpu(translate_buffer_write, cpu) = 0;
360 per_cpu(translate_buffer_read, cpu) = 0;
362 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL);
363 if (!per_cpu(translate_text, cpu)) {
364 err = -ENOMEM;
365 goto cookie_setup_error;
366 }
367 }
369 // build CRC32 table
370 poly = 0x04c11db7;
371 gator_crc32_table = (uint32_t*)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
372 for (i = 0; i < 256; i++) {
373 crc = i;
374 for (j = 8; j > 0; j--) {
375 if (crc & 1) {
376 crc = (crc >> 1) ^ poly;
377 } else {
378 crc >>= 1;
379 }
380 }
381 gator_crc32_table[i] = crc;
382 }
384 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
386 cookie_setup_error:
387 return err;
388 }
390 static void cookies_release(void)
391 {
392 int cpu;
394 for_each_present_cpu(cpu) {
395 kfree(per_cpu(cookie_keys, cpu));
396 per_cpu(cookie_keys, cpu) = NULL;
398 kfree(per_cpu(cookie_values, cpu));
399 per_cpu(cookie_values, cpu) = NULL;
401 kfree(per_cpu(translate_buffer, cpu));
402 per_cpu(translate_buffer, cpu) = NULL;
403 per_cpu(translate_buffer_read, cpu) = 0;
404 per_cpu(translate_buffer_write, cpu) = 0;
406 kfree(per_cpu(translate_text, cpu));
407 per_cpu(translate_text, cpu) = NULL;
408 }
410 del_timer_sync(&app_process_wake_up_timer);
411 kfree(gator_crc32_table);
412 gator_crc32_table = NULL;
413 }