Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* arch/sparc64/mm/tlb.c |
2 | * | |
3 | * Copyright (C) 2004 David S. Miller <davem@redhat.com> | |
4 | */ | |
5 | ||
6 | #include <linux/kernel.h> | |
1da177e4 LT |
7 | #include <linux/percpu.h> |
8 | #include <linux/mm.h> | |
9 | #include <linux/swap.h> | |
c9f2946f | 10 | #include <linux/preempt.h> |
1da177e4 LT |
11 | |
12 | #include <asm/pgtable.h> | |
13 | #include <asm/pgalloc.h> | |
14 | #include <asm/tlbflush.h> | |
15 | #include <asm/cacheflush.h> | |
16 | #include <asm/mmu_context.h> | |
17 | #include <asm/tlb.h> | |
18 | ||
19 | /* Heavily inspired by the ppc64 code. */ | |
20 | ||
90f08e39 | 21 | static DEFINE_PER_CPU(struct tlb_batch, tlb_batch); |
1da177e4 LT |
22 | |
23 | void flush_tlb_pending(void) | |
24 | { | |
90f08e39 | 25 | struct tlb_batch *tb = &get_cpu_var(tlb_batch); |
f36391d2 | 26 | struct mm_struct *mm = tb->mm; |
1da177e4 | 27 | |
f36391d2 DM |
28 | if (!tb->tlb_nr) |
29 | goto out; | |
74bf4312 | 30 | |
f36391d2 DM |
31 | flush_tsb_user(tb); |
32 | ||
33 | if (CTX_VALID(mm->context)) { | |
34 | if (tb->tlb_nr == 1) { | |
35 | global_flush_tlb_page(mm, tb->vaddrs[0]); | |
36 | } else { | |
1da177e4 | 37 | #ifdef CONFIG_SMP |
90f08e39 PZ |
38 | smp_flush_tlb_pending(tb->mm, tb->tlb_nr, |
39 | &tb->vaddrs[0]); | |
1da177e4 | 40 | #else |
90f08e39 PZ |
41 | __flush_tlb_pending(CTX_HWBITS(tb->mm->context), |
42 | tb->tlb_nr, &tb->vaddrs[0]); | |
1da177e4 LT |
43 | #endif |
44 | } | |
1da177e4 | 45 | } |
c9f2946f | 46 | |
f36391d2 DM |
47 | tb->tlb_nr = 0; |
48 | ||
49 | out: | |
90f08e39 | 50 | put_cpu_var(tlb_batch); |
1da177e4 LT |
51 | } |
52 | ||
f36391d2 DM |
53 | void arch_enter_lazy_mmu_mode(void) |
54 | { | |
494fc421 | 55 | struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); |
f36391d2 DM |
56 | |
57 | tb->active = 1; | |
58 | } | |
59 | ||
60 | void arch_leave_lazy_mmu_mode(void) | |
61 | { | |
494fc421 | 62 | struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); |
f36391d2 DM |
63 | |
64 | if (tb->tlb_nr) | |
65 | flush_tlb_pending(); | |
66 | tb->active = 0; | |
67 | } | |
68 | ||
9e695d2e DM |
69 | static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, |
70 | bool exec) | |
1da177e4 | 71 | { |
90f08e39 | 72 | struct tlb_batch *tb = &get_cpu_var(tlb_batch); |
1da177e4 LT |
73 | unsigned long nr; |
74 | ||
75 | vaddr &= PAGE_MASK; | |
9e695d2e | 76 | if (exec) |
1da177e4 LT |
77 | vaddr |= 0x1UL; |
78 | ||
9e695d2e DM |
79 | nr = tb->tlb_nr; |
80 | ||
81 | if (unlikely(nr != 0 && mm != tb->mm)) { | |
82 | flush_tlb_pending(); | |
83 | nr = 0; | |
84 | } | |
85 | ||
f36391d2 | 86 | if (!tb->active) { |
f36391d2 | 87 | flush_tsb_user_page(mm, vaddr); |
23a01138 | 88 | global_flush_tlb_page(mm, vaddr); |
f0af9707 | 89 | goto out; |
f36391d2 DM |
90 | } |
91 | ||
9e695d2e DM |
92 | if (nr == 0) |
93 | tb->mm = mm; | |
94 | ||
95 | tb->vaddrs[nr] = vaddr; | |
96 | tb->tlb_nr = ++nr; | |
97 | if (nr >= TLB_BATCH_NR) | |
98 | flush_tlb_pending(); | |
99 | ||
f0af9707 | 100 | out: |
9e695d2e DM |
101 | put_cpu_var(tlb_batch); |
102 | } | |
103 | ||
104 | void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, | |
105 | pte_t *ptep, pte_t orig, int fullmm) | |
106 | { | |
7a591cfe DM |
107 | if (tlb_type != hypervisor && |
108 | pte_dirty(orig)) { | |
1da177e4 LT |
109 | unsigned long paddr, pfn = pte_pfn(orig); |
110 | struct address_space *mapping; | |
111 | struct page *page; | |
112 | ||
113 | if (!pfn_valid(pfn)) | |
114 | goto no_cache_flush; | |
115 | ||
116 | page = pfn_to_page(pfn); | |
117 | if (PageReserved(page)) | |
118 | goto no_cache_flush; | |
119 | ||
120 | /* A real file page? */ | |
121 | mapping = page_mapping(page); | |
122 | if (!mapping) | |
123 | goto no_cache_flush; | |
124 | ||
125 | paddr = (unsigned long) page_address(page); | |
126 | if ((paddr ^ vaddr) & (1 << 13)) | |
127 | flush_dcache_page_all(mm, page); | |
128 | } | |
129 | ||
130 | no_cache_flush: | |
9e695d2e DM |
131 | if (!fullmm) |
132 | tlb_batch_add_one(mm, vaddr, pte_exec(orig)); | |
133 | } | |
134 | ||
135 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | |
136 | static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, | |
5b1e94fa | 137 | pmd_t pmd) |
9e695d2e DM |
138 | { |
139 | unsigned long end; | |
140 | pte_t *pte; | |
141 | ||
142 | pte = pte_offset_map(&pmd, vaddr); | |
143 | end = vaddr + HPAGE_SIZE; | |
144 | while (vaddr < end) { | |
5b1e94fa DM |
145 | if (pte_val(*pte) & _PAGE_VALID) { |
146 | bool exec = pte_exec(*pte); | |
147 | ||
9e695d2e | 148 | tlb_batch_add_one(mm, vaddr, exec); |
5b1e94fa | 149 | } |
9e695d2e DM |
150 | pte++; |
151 | vaddr += PAGE_SIZE; | |
152 | } | |
153 | pte_unmap(pte); | |
154 | } | |
1da177e4 | 155 | |
9e695d2e DM |
156 | void set_pmd_at(struct mm_struct *mm, unsigned long addr, |
157 | pmd_t *pmdp, pmd_t pmd) | |
158 | { | |
159 | pmd_t orig = *pmdp; | |
160 | ||
161 | *pmdp = pmd; | |
162 | ||
163 | if (mm == &init_mm) | |
1da177e4 | 164 | return; |
9e695d2e | 165 | |
a7b9403f DM |
166 | if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { |
167 | if (pmd_val(pmd) & _PAGE_PMD_HUGE) | |
9e695d2e DM |
168 | mm->context.huge_pte_count++; |
169 | else | |
170 | mm->context.huge_pte_count--; | |
0fbebed6 DM |
171 | |
172 | /* Do not try to allocate the TSB hash table if we | |
173 | * don't have one already. We have various locks held | |
174 | * and thus we'll end up doing a GFP_KERNEL allocation | |
175 | * in an atomic context. | |
176 | * | |
177 | * Instead, we let the first TLB miss on a hugepage | |
178 | * take care of this. | |
179 | */ | |
90f08e39 | 180 | } |
1da177e4 | 181 | |
9e695d2e | 182 | if (!pmd_none(orig)) { |
9e695d2e | 183 | addr &= HPAGE_MASK; |
a7b9403f | 184 | if (pmd_trans_huge(orig)) { |
5b1e94fa DM |
185 | pte_t orig_pte = __pte(pmd_val(orig)); |
186 | bool exec = pte_exec(orig_pte); | |
187 | ||
9e695d2e | 188 | tlb_batch_add_one(mm, addr, exec); |
37b3a8ff DM |
189 | tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); |
190 | } else { | |
5b1e94fa | 191 | tlb_batch_pmd_scan(mm, addr, orig); |
37b3a8ff | 192 | } |
1da177e4 | 193 | } |
9e695d2e | 194 | } |
1da177e4 | 195 | |
51e5ef1b DM |
196 | void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, |
197 | pmd_t *pmdp) | |
198 | { | |
199 | pmd_t entry = *pmdp; | |
200 | ||
201 | pmd_val(entry) &= ~_PAGE_VALID; | |
202 | ||
203 | set_pmd_at(vma->vm_mm, address, pmdp, entry); | |
204 | flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); | |
205 | } | |
206 | ||
6b0b50b0 AK |
207 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
208 | pgtable_t pgtable) | |
9e695d2e DM |
209 | { |
210 | struct list_head *lh = (struct list_head *) pgtable; | |
1da177e4 | 211 | |
9e695d2e | 212 | assert_spin_locked(&mm->page_table_lock); |
90f08e39 | 213 | |
9e695d2e | 214 | /* FIFO */ |
c389a250 | 215 | if (!pmd_huge_pte(mm, pmdp)) |
9e695d2e DM |
216 | INIT_LIST_HEAD(lh); |
217 | else | |
c389a250 KS |
218 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
219 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9e695d2e DM |
220 | } |
221 | ||
6b0b50b0 | 222 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9e695d2e DM |
223 | { |
224 | struct list_head *lh; | |
225 | pgtable_t pgtable; | |
226 | ||
227 | assert_spin_locked(&mm->page_table_lock); | |
228 | ||
229 | /* FIFO */ | |
c389a250 | 230 | pgtable = pmd_huge_pte(mm, pmdp); |
9e695d2e DM |
231 | lh = (struct list_head *) pgtable; |
232 | if (list_empty(lh)) | |
c389a250 | 233 | pmd_huge_pte(mm, pmdp) = NULL; |
9e695d2e | 234 | else { |
c389a250 | 235 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9e695d2e DM |
236 | list_del(lh); |
237 | } | |
238 | pte_val(pgtable[0]) = 0; | |
239 | pte_val(pgtable[1]) = 0; | |
240 | ||
241 | return pgtable; | |
1da177e4 | 242 | } |
9e695d2e | 243 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |