2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */
11 #define TRANSLATE_SIZE 256
12 #define MAX_COLLISIONS 2
14 static uint32_t *gator_crc32_table;
15 static unsigned int translate_buffer_mask;
17 static DEFINE_PER_CPU(char *, translate_text);
18 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
19 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
20 static DEFINE_PER_CPU(uint32_t *, cookie_values);
21 static DEFINE_PER_CPU(int, translate_buffer_read);
22 static DEFINE_PER_CPU(int, translate_buffer_write);
23 static DEFINE_PER_CPU(void **, translate_buffer);
25 static inline uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
26 static void wq_cookie_handler(struct work_struct *unused);
27 DECLARE_WORK(cookie_work, wq_cookie_handler);
28 static struct timer_list app_process_wake_up_timer;
29 static void app_process_wake_up_handler(unsigned long unused_data);
31 static uint32_t cookiemap_code(uint64_t value64)
33 uint32_t value = (uint32_t)((value64 >> 32) + value64);
34 uint32_t cookiecode = (value >> 24) & 0xff;
35 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
36 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
37 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
38 cookiecode &= (COOKIEMAP_ENTRIES - 1);
39 return cookiecode * MAX_COLLISIONS;
42 static uint32_t gator_chksum_crc32(const char *data)
44 register unsigned long crc;
45 const unsigned char *block = data;
46 int i, length = strlen(data);
49 for (i = 0; i < length; i++) {
50 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
53 return (crc ^ 0xFFFFFFFF);
58 * Pre: [0][1][v][3]..[n-1]
59 * Post: [v][0][1][3]..[n-1]
61 static uint32_t cookiemap_exists(uint64_t key)
63 unsigned long x, flags, retval = 0;
64 int cpu = get_physical_cpu();
65 uint32_t cookiecode = cookiemap_code(key);
66 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
67 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
69 // Can be called from interrupt handler or from work queue
70 local_irq_save(flags);
71 for (x = 0; x < MAX_COLLISIONS; x++) {
73 uint32_t value = values[x];
75 keys[x] = keys[x - 1];
76 values[x] = values[x - 1];
84 local_irq_restore(flags);
91 * Pre: [0][1][2][3]..[n-1]
92 * Post: [v][0][1][2]..[n-2]
94 static void cookiemap_add(uint64_t key, uint32_t value)
96 int cpu = get_physical_cpu();
97 int cookiecode = cookiemap_code(key);
98 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
99 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
102 for (x = MAX_COLLISIONS - 1; x > 0; x--) {
103 keys[x] = keys[x - 1];
104 values[x] = values[x - 1];
110 static void translate_buffer_write_ptr(int cpu, void *x)
112 per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
113 per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
116 static void *translate_buffer_read_ptr(int cpu)
118 void *value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
119 per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
123 static void wq_cookie_handler(struct work_struct *unused)
125 struct task_struct *task;
127 int cpu = get_physical_cpu();
130 mutex_lock(&start_mutex);
132 if (gator_started != 0) {
133 commit = per_cpu(translate_buffer_write, cpu);
134 while (per_cpu(translate_buffer_read, cpu) != commit) {
135 task = (struct task_struct *)translate_buffer_read_ptr(cpu);
136 text = (char *)translate_buffer_read_ptr(cpu);
137 get_cookie(cpu, task, text, true);
141 mutex_unlock(&start_mutex);
144 static void app_process_wake_up_handler(unsigned long unused_data)
146 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
147 schedule_work(&cookie_work);
150 // Retrieve full name from proc/pid/cmdline for java processes on Android
151 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
156 struct mm_struct *mm;
157 struct page *page = NULL;
158 struct vm_area_struct *page_vma;
159 int bytes, offset, retval = 0, ptr;
160 char *buf = per_cpu(translate_text, cpu);
162 // Push work into a work queue if in atomic context as the kernel functions below might sleep
163 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
164 // inconsistent during a context switch between android/linux versions
166 // Check if already in buffer
167 ptr = per_cpu(translate_buffer_read, cpu);
168 while (ptr != per_cpu(translate_buffer_write, cpu)) {
169 if (per_cpu(translate_buffer, cpu)[ptr] == (void *)task)
171 ptr = (ptr + 2) & translate_buffer_mask;
174 translate_buffer_write_ptr(cpu, (void *)task);
175 translate_buffer_write_ptr(cpu, (void *)*text);
177 mod_timer(&app_process_wake_up_timer, jiffies + 1);
181 mm = get_task_mm(task);
186 addr = mm->arg_start;
187 len = mm->arg_end - mm->arg_start;
189 if (len > TRANSLATE_SIZE)
190 len = TRANSLATE_SIZE;
192 down_read(&mm->mmap_sem);
194 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
198 offset = addr & (PAGE_SIZE - 1);
200 if (bytes > PAGE_SIZE - offset)
201 bytes = PAGE_SIZE - offset;
203 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
205 kunmap(page); // release page allocated by get_user_pages()
206 page_cache_release(page);
212 *text = per_cpu(translate_text, cpu);
216 // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
217 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
221 up_read(&mm->mmap_sem);
228 static inline uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
230 unsigned long flags, cookie;
233 key = gator_chksum_crc32(text);
234 key = (key << 32) | (uint32_t)task->tgid;
236 cookie = cookiemap_exists(key);
241 if (strcmp(text, "app_process") == 0) {
242 if (!translate_app_process(&text, cpu, task, from_wq))
243 return UNRESOLVED_COOKIE;
246 // Can be called from interrupt handler or from work queue or from scheduler trace
247 local_irq_save(flags);
249 cookie = UNRESOLVED_COOKIE;
250 if (marshal_cookie_header(text)) {
251 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
252 cookiemap_add(key, cookie);
253 marshal_cookie(cookie, text);
256 local_irq_restore(flags);
261 static int get_exec_cookie(int cpu, struct task_struct *task)
263 struct mm_struct *mm = task->mm;
266 // kernel threads have no address space
270 if (task && task->mm && task->mm->exe_file) {
271 text = task->mm->exe_file->f_path.dentry->d_name.name;
272 return get_cookie(cpu, task, text, false);
275 return UNRESOLVED_COOKIE;
278 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
280 unsigned long cookie = NO_COOKIE;
281 struct mm_struct *mm = task->mm;
282 struct vm_area_struct *vma;
288 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
289 if (addr < vma->vm_start || addr >= vma->vm_end)
293 text = vma->vm_file->f_path.dentry->d_name.name;
294 cookie = get_cookie(cpu, task, text, false);
295 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
297 /* must be an anonymous map */
305 cookie = UNRESOLVED_COOKIE;
310 static int cookies_initialize(void)
313 int i, j, cpu, size, err = 0;
315 int translate_buffer_size = 512; // must be a power of 2
316 translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
318 for_each_present_cpu(cpu) {
319 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
321 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
322 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
323 if (!per_cpu(cookie_keys, cpu)) {
325 goto cookie_setup_error;
327 memset(per_cpu(cookie_keys, cpu), 0, size);
329 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
330 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
331 if (!per_cpu(cookie_values, cpu)) {
333 goto cookie_setup_error;
335 memset(per_cpu(cookie_values, cpu), 0, size);
337 per_cpu(translate_buffer, cpu) = (void **)kmalloc(translate_buffer_size, GFP_KERNEL);
338 if (!per_cpu(translate_buffer, cpu)) {
340 goto cookie_setup_error;
343 per_cpu(translate_buffer_write, cpu) = 0;
344 per_cpu(translate_buffer_read, cpu) = 0;
346 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL);
347 if (!per_cpu(translate_text, cpu)) {
349 goto cookie_setup_error;
355 gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
356 for (i = 0; i < 256; i++) {
358 for (j = 8; j > 0; j--) {
360 crc = (crc >> 1) ^ poly;
365 gator_crc32_table[i] = crc;
368 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
374 static void cookies_release(void)
378 for_each_present_cpu(cpu) {
379 kfree(per_cpu(cookie_keys, cpu));
380 per_cpu(cookie_keys, cpu) = NULL;
382 kfree(per_cpu(cookie_values, cpu));
383 per_cpu(cookie_values, cpu) = NULL;
385 kfree(per_cpu(translate_buffer, cpu));
386 per_cpu(translate_buffer, cpu) = NULL;
387 per_cpu(translate_buffer_read, cpu) = 0;
388 per_cpu(translate_buffer_write, cpu) = 0;
390 kfree(per_cpu(translate_text, cpu));
391 per_cpu(translate_text, cpu) = NULL;
394 del_timer_sync(&app_process_wake_up_timer);
395 kfree(gator_crc32_table);
396 gator_crc32_table = NULL;