2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */
11 #define TRANSLATE_BUFFER_SIZE 512 // must be a power of 2 - 512/4 = 128 entries
12 #define TRANSLATE_TEXT_SIZE 256
13 #define MAX_COLLISIONS 2
15 static uint32_t *gator_crc32_table;
16 static unsigned int translate_buffer_mask;
19 struct task_struct *task;
23 static DEFINE_PER_CPU(char *, translate_text);
24 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
25 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
26 static DEFINE_PER_CPU(uint32_t *, cookie_values);
27 static DEFINE_PER_CPU(int, translate_buffer_read);
28 static DEFINE_PER_CPU(int, translate_buffer_write);
29 static DEFINE_PER_CPU(struct cookie_args *, translate_buffer);
31 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
32 static void wq_cookie_handler(struct work_struct *unused);
33 DECLARE_WORK(cookie_work, wq_cookie_handler);
34 static struct timer_list app_process_wake_up_timer;
35 static void app_process_wake_up_handler(unsigned long unused_data);
37 static uint32_t cookiemap_code(uint64_t value64)
39 uint32_t value = (uint32_t)((value64 >> 32) + value64);
40 uint32_t cookiecode = (value >> 24) & 0xff;
41 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
42 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
43 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
44 cookiecode &= (COOKIEMAP_ENTRIES - 1);
45 return cookiecode * MAX_COLLISIONS;
48 static uint32_t gator_chksum_crc32(const char *data)
50 register unsigned long crc;
51 const unsigned char *block = data;
52 int i, length = strlen(data);
55 for (i = 0; i < length; i++) {
56 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
59 return (crc ^ 0xFFFFFFFF);
64 * Pre: [0][1][v][3]..[n-1]
65 * Post: [v][0][1][3]..[n-1]
67 static uint32_t cookiemap_exists(uint64_t key)
69 unsigned long x, flags, retval = 0;
70 int cpu = get_physical_cpu();
71 uint32_t cookiecode = cookiemap_code(key);
72 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
73 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
75 // Can be called from interrupt handler or from work queue
76 local_irq_save(flags);
77 for (x = 0; x < MAX_COLLISIONS; x++) {
79 uint32_t value = values[x];
81 keys[x] = keys[x - 1];
82 values[x] = values[x - 1];
90 local_irq_restore(flags);
97 * Pre: [0][1][2][3]..[n-1]
98 * Post: [v][0][1][2]..[n-2]
100 static void cookiemap_add(uint64_t key, uint32_t value)
102 int cpu = get_physical_cpu();
103 int cookiecode = cookiemap_code(key);
104 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
105 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
108 for (x = MAX_COLLISIONS - 1; x > 0; x--) {
109 keys[x] = keys[x - 1];
110 values[x] = values[x - 1];
116 #ifndef CONFIG_PREEMPT_RT_FULL
117 static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text)
122 struct cookie_args *args;
124 local_irq_save(flags);
126 write = per_cpu(translate_buffer_write, cpu);
127 next_write = (write + 1) & translate_buffer_mask;
129 // At least one entry must always remain available as when read == write, the queue is empty not full
130 if (next_write != per_cpu(translate_buffer_read, cpu)) {
131 args = &per_cpu(translate_buffer, cpu)[write];
134 get_task_struct(task);
135 per_cpu(translate_buffer_write, cpu) = next_write;
138 local_irq_restore(flags);
142 static void translate_buffer_read_args(int cpu, struct cookie_args *args)
147 local_irq_save(flags);
149 read = per_cpu(translate_buffer_read, cpu);
150 *args = per_cpu(translate_buffer, cpu)[read];
151 per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask;
153 local_irq_restore(flags);
156 static void wq_cookie_handler(struct work_struct *unused)
158 struct cookie_args args;
159 int cpu = get_physical_cpu(), cookie;
161 mutex_lock(&start_mutex);
163 if (gator_started != 0) {
164 while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) {
165 translate_buffer_read_args(cpu, &args);
166 cookie = get_cookie(cpu, args.task, args.text, true);
167 marshal_link(cookie, args.task->tgid, args.task->pid);
168 put_task_struct(args.task);
172 mutex_unlock(&start_mutex);
175 static void app_process_wake_up_handler(unsigned long unused_data)
177 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
178 schedule_work(&cookie_work);
181 // Retrieve full name from proc/pid/cmdline for java processes on Android
182 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
187 struct mm_struct *mm;
188 struct page *page = NULL;
189 struct vm_area_struct *page_vma;
190 int bytes, offset, retval = 0;
191 char *buf = per_cpu(translate_text, cpu);
193 #ifndef CONFIG_PREEMPT_RT_FULL
194 // Push work into a work queue if in atomic context as the kernel functions below might sleep
195 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
196 // inconsistent during a context switch between android/linux versions
198 // Check if already in buffer
199 int pos = per_cpu(translate_buffer_read, cpu);
200 while (pos != per_cpu(translate_buffer_write, cpu)) {
201 if (per_cpu(translate_buffer, cpu)[pos].task == task)
203 pos = (pos + 1) & translate_buffer_mask;
206 translate_buffer_write_args(cpu, task, *text);
208 // Not safe to call in RT-Preempt full in schedule switch context
209 mod_timer(&app_process_wake_up_timer, jiffies + 1);
214 mm = get_task_mm(task);
219 addr = mm->arg_start;
220 len = mm->arg_end - mm->arg_start;
222 if (len > TRANSLATE_TEXT_SIZE)
223 len = TRANSLATE_TEXT_SIZE;
225 down_read(&mm->mmap_sem);
227 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
231 offset = addr & (PAGE_SIZE - 1);
233 if (bytes > PAGE_SIZE - offset)
234 bytes = PAGE_SIZE - offset;
236 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
238 kunmap(page); // release page allocated by get_user_pages()
239 page_cache_release(page);
245 *text = per_cpu(translate_text, cpu);
249 // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
250 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
254 up_read(&mm->mmap_sem);
261 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
263 unsigned long flags, cookie;
266 key = gator_chksum_crc32(text);
267 key = (key << 32) | (uint32_t)task->tgid;
269 cookie = cookiemap_exists(key);
274 if (strcmp(text, "app_process") == 0) {
275 if (!translate_app_process(&text, cpu, task, from_wq))
276 return UNRESOLVED_COOKIE;
279 // Can be called from interrupt handler or from work queue or from scheduler trace
280 local_irq_save(flags);
282 cookie = UNRESOLVED_COOKIE;
283 if (marshal_cookie_header(text)) {
284 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
285 cookiemap_add(key, cookie);
286 marshal_cookie(cookie, text);
289 local_irq_restore(flags);
294 static int get_exec_cookie(int cpu, struct task_struct *task)
296 struct mm_struct *mm = task->mm;
299 // kernel threads have no address space
303 if (task && task->mm && task->mm->exe_file) {
304 text = task->mm->exe_file->f_path.dentry->d_name.name;
305 return get_cookie(cpu, task, text, false);
308 return UNRESOLVED_COOKIE;
311 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
313 unsigned long cookie = NO_COOKIE;
314 struct mm_struct *mm = task->mm;
315 struct vm_area_struct *vma;
321 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
322 if (addr < vma->vm_start || addr >= vma->vm_end)
326 text = vma->vm_file->f_path.dentry->d_name.name;
327 cookie = get_cookie(cpu, task, text, false);
328 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
330 /* must be an anonymous map */
338 cookie = UNRESOLVED_COOKIE;
343 static int cookies_initialize(void)
346 int i, j, cpu, size, err = 0;
348 translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
350 for_each_present_cpu(cpu) {
351 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
353 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
354 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
355 if (!per_cpu(cookie_keys, cpu)) {
357 goto cookie_setup_error;
359 memset(per_cpu(cookie_keys, cpu), 0, size);
361 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
362 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
363 if (!per_cpu(cookie_values, cpu)) {
365 goto cookie_setup_error;
367 memset(per_cpu(cookie_values, cpu), 0, size);
369 per_cpu(translate_buffer, cpu) = (struct cookie_args *)kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
370 if (!per_cpu(translate_buffer, cpu)) {
372 goto cookie_setup_error;
375 per_cpu(translate_buffer_write, cpu) = 0;
376 per_cpu(translate_buffer_read, cpu) = 0;
378 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
379 if (!per_cpu(translate_text, cpu)) {
381 goto cookie_setup_error;
387 gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
388 if (!gator_crc32_table) {
390 goto cookie_setup_error;
392 for (i = 0; i < 256; i++) {
394 for (j = 8; j > 0; j--) {
396 crc = (crc >> 1) ^ poly;
401 gator_crc32_table[i] = crc;
404 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
410 static void cookies_release(void)
414 for_each_present_cpu(cpu) {
415 kfree(per_cpu(cookie_keys, cpu));
416 per_cpu(cookie_keys, cpu) = NULL;
418 kfree(per_cpu(cookie_values, cpu));
419 per_cpu(cookie_values, cpu) = NULL;
421 kfree(per_cpu(translate_buffer, cpu));
422 per_cpu(translate_buffer, cpu) = NULL;
423 per_cpu(translate_buffer_read, cpu) = 0;
424 per_cpu(translate_buffer_write, cpu) = 0;
426 kfree(per_cpu(translate_text, cpu));
427 per_cpu(translate_text, cpu) = NULL;
430 del_timer_sync(&app_process_wake_up_timer);
431 kfree(gator_crc32_table);
432 gator_crc32_table = NULL;