2 * linux/arch/arm/mm/context.c
4 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
5 * Copyright (C) 2012 ARM Limited
7 * Author: Will Deacon <will.deacon@arm.com>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
13 #include <linux/init.h>
14 #include <linux/sched.h>
16 #include <linux/smp.h>
17 #include <linux/percpu.h>
19 #include <asm/mmu_context.h>
20 #include <asm/smp_plat.h>
21 #include <asm/thread_notify.h>
22 #include <asm/tlbflush.h>
25 * On ARMv6, we have the following structure in the Context ID:
28 * +-------------------------+-----------+
29 * | process ID | ASID |
30 * +-------------------------+-----------+
32 * +-------------------------------------+
34 * The ASID is used to tag entries in the CPU caches and TLBs.
35 * The context ID is used by debuggers and trace logic, and
36 * should be unique within all running processes.
38 * In big endian operation, the two 32 bit words are swapped if accesed by
39 * non 64-bit operations.
41 #define ASID_FIRST_VERSION (1ULL << ASID_BITS)
42 #define NUM_USER_ASIDS ASID_FIRST_VERSION
44 static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
45 static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
46 static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
48 static DEFINE_PER_CPU(atomic64_t, active_asids);
49 static DEFINE_PER_CPU(u64, reserved_asids);
50 static cpumask_t tlb_flush_pending;
52 #ifdef CONFIG_ARM_ERRATA_798181
53 void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
60 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
61 context_id = mm->context.id.counter;
62 for_each_online_cpu(cpu) {
66 * We only need to send an IPI if the other CPUs are
67 * running the same ASID as the one being invalidated.
69 asid = per_cpu(active_asids, cpu).counter;
71 asid = per_cpu(reserved_asids, cpu);
72 if (context_id == asid)
73 cpumask_set_cpu(cpu, mask);
75 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
79 #ifdef CONFIG_ARM_LPAE
80 static void cpu_set_reserved_ttbr0(void)
82 unsigned long ttbl = __pa(swapper_pg_dir);
83 unsigned long ttbh = 0;
86 * Set TTBR0 to swapper_pg_dir which contains only global entries. The
90 " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n"
92 : "r" (ttbl), "r" (ttbh));
96 static void cpu_set_reserved_ttbr0(void)
99 /* Copy TTBR1 into TTBR0 */
101 " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n"
102 " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n"
108 #ifdef CONFIG_PID_IN_CONTEXTIDR
109 static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
114 struct thread_info *thread = t;
116 if (cmd != THREAD_NOTIFY_SWITCH)
119 pid = task_pid_nr(thread->task) << ASID_BITS;
121 " mrc p15, 0, %0, c13, c0, 1\n"
124 " mcr p15, 0, %0, c13, c0, 1\n"
125 : "=r" (contextidr), "+r" (pid)
132 static struct notifier_block contextidr_notifier_block = {
133 .notifier_call = contextidr_notifier,
136 static int __init contextidr_notifier_init(void)
138 return thread_register_notifier(&contextidr_notifier_block);
140 arch_initcall(contextidr_notifier_init);
143 static void flush_context(unsigned int cpu)
148 /* Update the list of reserved ASIDs and the ASID bitmap. */
149 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
150 for_each_possible_cpu(i) {
154 asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
156 * If this CPU has already been through a
157 * rollover, but hasn't run another task in
158 * the meantime, we must preserve its reserved
159 * ASID, as this is the only trace we have of
160 * the process it is still running.
163 asid = per_cpu(reserved_asids, i);
164 __set_bit(asid & ~ASID_MASK, asid_map);
166 per_cpu(reserved_asids, i) = asid;
169 /* Queue a TLB invalidate and flush the I-cache if necessary. */
170 if (!tlb_ops_need_broadcast())
171 cpumask_set_cpu(cpu, &tlb_flush_pending);
173 cpumask_setall(&tlb_flush_pending);
175 if (icache_is_vivt_asid_tagged())
176 __flush_icache_all();
179 static int is_reserved_asid(u64 asid)
182 for_each_possible_cpu(cpu)
183 if (per_cpu(reserved_asids, cpu) == asid)
188 static u64 new_context(struct mm_struct *mm, unsigned int cpu)
190 u64 asid = atomic64_read(&mm->context.id);
191 u64 generation = atomic64_read(&asid_generation);
193 if (asid != 0 && is_reserved_asid(asid)) {
195 * Our current ASID was active during a rollover, we can
196 * continue to use it and this was just a false alarm.
198 asid = generation | (asid & ~ASID_MASK);
201 * Allocate a free ASID. If we can't find one, take a
202 * note of the currently active ASIDs and mark the TLBs
203 * as requiring flushes. We always count from ASID #1,
204 * as we reserve ASID #0 to switch via TTBR0 and indicate
207 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
208 if (asid == NUM_USER_ASIDS) {
209 generation = atomic64_add_return(ASID_FIRST_VERSION,
212 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
214 __set_bit(asid, asid_map);
216 cpumask_clear(mm_cpumask(mm));
222 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
225 unsigned int cpu = smp_processor_id();
228 if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
229 __check_vmalloc_seq(mm);
232 * Required during context switch to avoid speculative page table
233 * walking with the wrong TTBR.
235 cpu_set_reserved_ttbr0();
237 asid = atomic64_read(&mm->context.id);
238 if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
239 && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
240 goto switch_mm_fastpath;
242 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
243 /* Check that our ASID belongs to the current generation. */
244 asid = atomic64_read(&mm->context.id);
245 if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
246 asid = new_context(mm, cpu);
247 atomic64_set(&mm->context.id, asid);
250 if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
251 local_flush_bp_all();
252 local_flush_tlb_all();
253 dummy_flush_tlb_a15_erratum();
256 atomic64_set(&per_cpu(active_asids, cpu), asid);
257 cpumask_set_cpu(cpu, mm_cpumask(mm));
258 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
261 cpu_switch_mm(mm->pgd, mm);