Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
d84b4711 | 2 | * linux/arch/arm/mm/context.c |
1da177e4 LT |
3 | * |
4 | * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. | |
b5466f87 WD |
5 | * Copyright (C) 2012 ARM Limited |
6 | * | |
7 | * Author: Will Deacon <will.deacon@arm.com> | |
1da177e4 LT |
8 | * |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of the GNU General Public License version 2 as | |
11 | * published by the Free Software Foundation. | |
12 | */ | |
13 | #include <linux/init.h> | |
14 | #include <linux/sched.h> | |
15 | #include <linux/mm.h> | |
11805bcf CM |
16 | #include <linux/smp.h> |
17 | #include <linux/percpu.h> | |
1da177e4 LT |
18 | |
19 | #include <asm/mmu_context.h> | |
b5466f87 | 20 | #include <asm/smp_plat.h> |
575320d6 | 21 | #include <asm/thread_notify.h> |
1da177e4 LT |
22 | #include <asm/tlbflush.h> |
23 | ||
b5466f87 WD |
24 | /* |
25 | * On ARMv6, we have the following structure in the Context ID: | |
26 | * | |
27 | * 31 7 0 | |
28 | * +-------------------------+-----------+ | |
29 | * | process ID | ASID | | |
30 | * +-------------------------+-----------+ | |
31 | * | context ID | | |
32 | * +-------------------------------------+ | |
33 | * | |
34 | * The ASID is used to tag entries in the CPU caches and TLBs. | |
35 | * The context ID is used by debuggers and trace logic, and | |
36 | * should be unique within all running processes. | |
9520a5be BD |
37 | * |
38 | * In big endian operation, the two 32 bit words are swapped if accesed by | |
39 | * non 64-bit operations. | |
b5466f87 WD |
40 | */ |
41 | #define ASID_FIRST_VERSION (1ULL << ASID_BITS) | |
bf51bb82 WD |
42 | #define NUM_USER_ASIDS (ASID_FIRST_VERSION - 1) |
43 | ||
44 | #define ASID_TO_IDX(asid) ((asid & ~ASID_MASK) - 1) | |
45 | #define IDX_TO_ASID(idx) ((idx + 1) & ~ASID_MASK) | |
b5466f87 | 46 | |
bd31b859 | 47 | static DEFINE_RAW_SPINLOCK(cpu_asid_lock); |
bf51bb82 WD |
48 | static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION); |
49 | static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS); | |
b5466f87 | 50 | |
4b883160 | 51 | static DEFINE_PER_CPU(atomic64_t, active_asids); |
b5466f87 WD |
52 | static DEFINE_PER_CPU(u64, reserved_asids); |
53 | static cpumask_t tlb_flush_pending; | |
1da177e4 | 54 | |
14d8c951 | 55 | #ifdef CONFIG_ARM_LPAE |
b5466f87 | 56 | static void cpu_set_reserved_ttbr0(void) |
3c5f7e7b WD |
57 | { |
58 | unsigned long ttbl = __pa(swapper_pg_dir); | |
59 | unsigned long ttbh = 0; | |
60 | ||
61 | /* | |
62 | * Set TTBR0 to swapper_pg_dir which contains only global entries. The | |
63 | * ASID is set to 0. | |
64 | */ | |
65 | asm volatile( | |
66 | " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" | |
67 | : | |
68 | : "r" (ttbl), "r" (ttbh)); | |
69 | isb(); | |
14d8c951 CM |
70 | } |
71 | #else | |
b5466f87 | 72 | static void cpu_set_reserved_ttbr0(void) |
3c5f7e7b WD |
73 | { |
74 | u32 ttb; | |
75 | /* Copy TTBR1 into TTBR0 */ | |
76 | asm volatile( | |
77 | " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n" | |
78 | " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n" | |
79 | : "=r" (ttb)); | |
80 | isb(); | |
81 | } | |
14d8c951 CM |
82 | #endif |
83 | ||
575320d6 WD |
84 | #ifdef CONFIG_PID_IN_CONTEXTIDR |
85 | static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd, | |
86 | void *t) | |
87 | { | |
88 | u32 contextidr; | |
89 | pid_t pid; | |
90 | struct thread_info *thread = t; | |
91 | ||
92 | if (cmd != THREAD_NOTIFY_SWITCH) | |
93 | return NOTIFY_DONE; | |
94 | ||
95 | pid = task_pid_nr(thread->task) << ASID_BITS; | |
96 | asm volatile( | |
97 | " mrc p15, 0, %0, c13, c0, 1\n" | |
ae3790b8 WD |
98 | " and %0, %0, %2\n" |
99 | " orr %0, %0, %1\n" | |
100 | " mcr p15, 0, %0, c13, c0, 1\n" | |
575320d6 | 101 | : "=r" (contextidr), "+r" (pid) |
ae3790b8 | 102 | : "I" (~ASID_MASK)); |
575320d6 WD |
103 | isb(); |
104 | ||
105 | return NOTIFY_OK; | |
106 | } | |
107 | ||
108 | static struct notifier_block contextidr_notifier_block = { | |
109 | .notifier_call = contextidr_notifier, | |
110 | }; | |
111 | ||
112 | static int __init contextidr_notifier_init(void) | |
113 | { | |
114 | return thread_register_notifier(&contextidr_notifier_block); | |
115 | } | |
116 | arch_initcall(contextidr_notifier_init); | |
117 | #endif | |
118 | ||
b5466f87 | 119 | static void flush_context(unsigned int cpu) |
1da177e4 | 120 | { |
b5466f87 | 121 | int i; |
bf51bb82 WD |
122 | u64 asid; |
123 | ||
124 | /* Update the list of reserved ASIDs and the ASID bitmap. */ | |
125 | bitmap_clear(asid_map, 0, NUM_USER_ASIDS); | |
126 | for_each_possible_cpu(i) { | |
127 | if (i == cpu) { | |
128 | asid = 0; | |
129 | } else { | |
130 | asid = atomic64_xchg(&per_cpu(active_asids, i), 0); | |
131 | __set_bit(ASID_TO_IDX(asid), asid_map); | |
132 | } | |
133 | per_cpu(reserved_asids, i) = asid; | |
134 | } | |
b5466f87 WD |
135 | |
136 | /* Queue a TLB invalidate and flush the I-cache if necessary. */ | |
137 | if (!tlb_ops_need_broadcast()) | |
138 | cpumask_set_cpu(cpu, &tlb_flush_pending); | |
139 | else | |
140 | cpumask_setall(&tlb_flush_pending); | |
141 | ||
142 | if (icache_is_vivt_asid_tagged()) | |
11805bcf | 143 | __flush_icache_all(); |
11805bcf CM |
144 | } |
145 | ||
bf51bb82 | 146 | static int is_reserved_asid(u64 asid) |
b5466f87 WD |
147 | { |
148 | int cpu; | |
149 | for_each_possible_cpu(cpu) | |
bf51bb82 | 150 | if (per_cpu(reserved_asids, cpu) == asid) |
b5466f87 WD |
151 | return 1; |
152 | return 0; | |
153 | } | |
11805bcf | 154 | |
8a4e3a9e | 155 | static u64 new_context(struct mm_struct *mm, unsigned int cpu) |
11805bcf | 156 | { |
8a4e3a9e | 157 | u64 asid = atomic64_read(&mm->context.id); |
bf51bb82 | 158 | u64 generation = atomic64_read(&asid_generation); |
11805bcf | 159 | |
bf51bb82 | 160 | if (asid != 0 && is_reserved_asid(asid)) { |
11805bcf | 161 | /* |
b5466f87 WD |
162 | * Our current ASID was active during a rollover, we can |
163 | * continue to use it and this was just a false alarm. | |
11805bcf | 164 | */ |
bf51bb82 | 165 | asid = generation | (asid & ~ASID_MASK); |
b5466f87 WD |
166 | } else { |
167 | /* | |
168 | * Allocate a free ASID. If we can't find one, take a | |
169 | * note of the currently active ASIDs and mark the TLBs | |
170 | * as requiring flushes. | |
171 | */ | |
bf51bb82 WD |
172 | asid = find_first_zero_bit(asid_map, NUM_USER_ASIDS); |
173 | if (asid == NUM_USER_ASIDS) { | |
174 | generation = atomic64_add_return(ASID_FIRST_VERSION, | |
175 | &asid_generation); | |
176 | flush_context(cpu); | |
177 | asid = find_first_zero_bit(asid_map, NUM_USER_ASIDS); | |
178 | } | |
179 | __set_bit(asid, asid_map); | |
180 | asid = generation | IDX_TO_ASID(asid); | |
11805bcf CM |
181 | cpumask_clear(mm_cpumask(mm)); |
182 | } | |
11805bcf | 183 | |
8a4e3a9e | 184 | return asid; |
11805bcf CM |
185 | } |
186 | ||
b5466f87 | 187 | void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) |
11805bcf | 188 | { |
b5466f87 | 189 | unsigned long flags; |
11805bcf | 190 | unsigned int cpu = smp_processor_id(); |
8a4e3a9e | 191 | u64 asid; |
11805bcf | 192 | |
3e99675a NP |
193 | if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) |
194 | __check_vmalloc_seq(mm); | |
11805bcf | 195 | |
11805bcf | 196 | /* |
b5466f87 WD |
197 | * Required during context switch to avoid speculative page table |
198 | * walking with the wrong TTBR. | |
11805bcf | 199 | */ |
b5466f87 | 200 | cpu_set_reserved_ttbr0(); |
1da177e4 | 201 | |
8a4e3a9e WD |
202 | asid = atomic64_read(&mm->context.id); |
203 | if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) | |
204 | && atomic64_xchg(&per_cpu(active_asids, cpu), asid)) | |
4b883160 WD |
205 | goto switch_mm_fastpath; |
206 | ||
b5466f87 WD |
207 | raw_spin_lock_irqsave(&cpu_asid_lock, flags); |
208 | /* Check that our ASID belongs to the current generation. */ | |
8a4e3a9e WD |
209 | asid = atomic64_read(&mm->context.id); |
210 | if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) { | |
211 | asid = new_context(mm, cpu); | |
212 | atomic64_set(&mm->context.id, asid); | |
213 | } | |
1da177e4 | 214 | |
b5466f87 WD |
215 | if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) |
216 | local_flush_tlb_all(); | |
37f47e3d | 217 | |
8a4e3a9e | 218 | atomic64_set(&per_cpu(active_asids, cpu), asid); |
37f47e3d | 219 | cpumask_set_cpu(cpu, mm_cpumask(mm)); |
b5466f87 WD |
220 | raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); |
221 | ||
4b883160 | 222 | switch_mm_fastpath: |
b5466f87 | 223 | cpu_switch_mm(mm->pgd, mm); |
1da177e4 | 224 | } |