2 * Copyright (C) ARM Limited 2010-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */
11 #define TRANSLATE_BUFFER_SIZE 512 // must be a power of 2 - 512/4 = 128 entries
12 #define TRANSLATE_TEXT_SIZE 256
13 #define MAX_COLLISIONS 2
15 static uint32_t *gator_crc32_table;
16 static unsigned int translate_buffer_mask;
18 static DEFINE_PER_CPU(char *, translate_text);
19 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
20 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
21 static DEFINE_PER_CPU(uint32_t *, cookie_values);
22 static DEFINE_PER_CPU(int, translate_buffer_read);
23 static DEFINE_PER_CPU(int, translate_buffer_write);
24 static DEFINE_PER_CPU(void **, translate_buffer);
26 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
27 static void wq_cookie_handler(struct work_struct *unused);
28 DECLARE_WORK(cookie_work, wq_cookie_handler);
29 static struct timer_list app_process_wake_up_timer;
30 static void app_process_wake_up_handler(unsigned long unused_data);
32 static uint32_t cookiemap_code(uint64_t value64)
34 uint32_t value = (uint32_t)((value64 >> 32) + value64);
35 uint32_t cookiecode = (value >> 24) & 0xff;
36 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
37 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
38 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
39 cookiecode &= (COOKIEMAP_ENTRIES - 1);
40 return cookiecode * MAX_COLLISIONS;
43 static uint32_t gator_chksum_crc32(const char *data)
45 register unsigned long crc;
46 const unsigned char *block = data;
47 int i, length = strlen(data);
50 for (i = 0; i < length; i++) {
51 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
54 return (crc ^ 0xFFFFFFFF);
59 * Pre: [0][1][v][3]..[n-1]
60 * Post: [v][0][1][3]..[n-1]
62 static uint32_t cookiemap_exists(uint64_t key)
64 unsigned long x, flags, retval = 0;
65 int cpu = get_physical_cpu();
66 uint32_t cookiecode = cookiemap_code(key);
67 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
68 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
70 // Can be called from interrupt handler or from work queue
71 local_irq_save(flags);
72 for (x = 0; x < MAX_COLLISIONS; x++) {
74 uint32_t value = values[x];
76 keys[x] = keys[x - 1];
77 values[x] = values[x - 1];
85 local_irq_restore(flags);
92 * Pre: [0][1][2][3]..[n-1]
93 * Post: [v][0][1][2]..[n-2]
95 static void cookiemap_add(uint64_t key, uint32_t value)
97 int cpu = get_physical_cpu();
98 int cookiecode = cookiemap_code(key);
99 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
100 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
103 for (x = MAX_COLLISIONS - 1; x > 0; x--) {
104 keys[x] = keys[x - 1];
105 values[x] = values[x - 1];
111 #ifndef CONFIG_PREEMPT_RT_FULL
112 static void translate_buffer_write_ptr(int cpu, void *x)
114 per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
115 per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
119 static void *translate_buffer_read_ptr(int cpu)
121 void *value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
122 per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
126 static void wq_cookie_handler(struct work_struct *unused)
128 struct task_struct *task;
130 int cpu = get_physical_cpu(), cookie;
133 mutex_lock(&start_mutex);
135 if (gator_started != 0) {
136 commit = per_cpu(translate_buffer_write, cpu);
137 while (per_cpu(translate_buffer_read, cpu) != commit) {
138 task = (struct task_struct *)translate_buffer_read_ptr(cpu);
139 text = (char *)translate_buffer_read_ptr(cpu);
140 cookie = get_cookie(cpu, task, text, true);
141 marshal_link(cookie, task->tgid, task->pid);
145 mutex_unlock(&start_mutex);
148 static void app_process_wake_up_handler(unsigned long unused_data)
150 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
151 schedule_work(&cookie_work);
154 // Retrieve full name from proc/pid/cmdline for java processes on Android
155 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
160 struct mm_struct *mm;
161 struct page *page = NULL;
162 struct vm_area_struct *page_vma;
163 int bytes, offset, retval = 0;
164 char *buf = per_cpu(translate_text, cpu);
166 #ifndef CONFIG_PREEMPT_RT_FULL
167 // Push work into a work queue if in atomic context as the kernel functions below might sleep
168 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
169 // inconsistent during a context switch between android/linux versions
171 // Check if already in buffer
172 int ptr = per_cpu(translate_buffer_read, cpu);
173 while (ptr != per_cpu(translate_buffer_write, cpu)) {
174 if (per_cpu(translate_buffer, cpu)[ptr] == (void *)task)
176 ptr = (ptr + 2) & translate_buffer_mask;
179 translate_buffer_write_ptr(cpu, (void *)task);
180 translate_buffer_write_ptr(cpu, (void *)*text);
182 // Not safe to call in RT-Preempt full in schedule switch context
183 mod_timer(&app_process_wake_up_timer, jiffies + 1);
188 mm = get_task_mm(task);
193 addr = mm->arg_start;
194 len = mm->arg_end - mm->arg_start;
196 if (len > TRANSLATE_TEXT_SIZE)
197 len = TRANSLATE_TEXT_SIZE;
199 down_read(&mm->mmap_sem);
201 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
205 offset = addr & (PAGE_SIZE - 1);
207 if (bytes > PAGE_SIZE - offset)
208 bytes = PAGE_SIZE - offset;
210 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
212 kunmap(page); // release page allocated by get_user_pages()
213 page_cache_release(page);
219 *text = per_cpu(translate_text, cpu);
223 // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
224 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
228 up_read(&mm->mmap_sem);
235 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
237 unsigned long flags, cookie;
240 key = gator_chksum_crc32(text);
241 key = (key << 32) | (uint32_t)task->tgid;
243 cookie = cookiemap_exists(key);
248 if (strcmp(text, "app_process") == 0) {
249 if (!translate_app_process(&text, cpu, task, from_wq))
250 return UNRESOLVED_COOKIE;
253 // Can be called from interrupt handler or from work queue or from scheduler trace
254 local_irq_save(flags);
256 cookie = UNRESOLVED_COOKIE;
257 if (marshal_cookie_header(text)) {
258 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
259 cookiemap_add(key, cookie);
260 marshal_cookie(cookie, text);
263 local_irq_restore(flags);
268 static int get_exec_cookie(int cpu, struct task_struct *task)
270 struct mm_struct *mm = task->mm;
273 // kernel threads have no address space
277 if (task && task->mm && task->mm->exe_file) {
278 text = task->mm->exe_file->f_path.dentry->d_name.name;
279 return get_cookie(cpu, task, text, false);
282 return UNRESOLVED_COOKIE;
285 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
287 unsigned long cookie = NO_COOKIE;
288 struct mm_struct *mm = task->mm;
289 struct vm_area_struct *vma;
295 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
296 if (addr < vma->vm_start || addr >= vma->vm_end)
300 text = vma->vm_file->f_path.dentry->d_name.name;
301 cookie = get_cookie(cpu, task, text, false);
302 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
304 /* must be an anonymous map */
312 cookie = UNRESOLVED_COOKIE;
317 static int cookies_initialize(void)
320 int i, j, cpu, size, err = 0;
322 translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
324 for_each_present_cpu(cpu) {
325 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
327 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
328 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
329 if (!per_cpu(cookie_keys, cpu)) {
331 goto cookie_setup_error;
333 memset(per_cpu(cookie_keys, cpu), 0, size);
335 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
336 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
337 if (!per_cpu(cookie_values, cpu)) {
339 goto cookie_setup_error;
341 memset(per_cpu(cookie_values, cpu), 0, size);
343 per_cpu(translate_buffer, cpu) = (void **)kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
344 if (!per_cpu(translate_buffer, cpu)) {
346 goto cookie_setup_error;
349 per_cpu(translate_buffer_write, cpu) = 0;
350 per_cpu(translate_buffer_read, cpu) = 0;
352 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
353 if (!per_cpu(translate_text, cpu)) {
355 goto cookie_setup_error;
361 gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
362 if (!gator_crc32_table) {
364 goto cookie_setup_error;
366 for (i = 0; i < 256; i++) {
368 for (j = 8; j > 0; j--) {
370 crc = (crc >> 1) ^ poly;
375 gator_crc32_table[i] = crc;
378 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
384 static void cookies_release(void)
388 for_each_present_cpu(cpu) {
389 kfree(per_cpu(cookie_keys, cpu));
390 per_cpu(cookie_keys, cpu) = NULL;
392 kfree(per_cpu(cookie_values, cpu));
393 per_cpu(cookie_values, cpu) = NULL;
395 kfree(per_cpu(translate_buffer, cpu));
396 per_cpu(translate_buffer, cpu) = NULL;
397 per_cpu(translate_buffer_read, cpu) = 0;
398 per_cpu(translate_buffer_write, cpu) = 0;
400 kfree(per_cpu(translate_text, cpu));
401 per_cpu(translate_text, cpu) = NULL;
404 del_timer_sync(&app_process_wake_up_timer);
405 kfree(gator_crc32_table);
406 gator_crc32_table = NULL;