Commit | Line | Data |
---|---|---|
58d0ba57 CM |
1 | /* |
2 | * Based on arch/arm/include/asm/tlb.h | |
3 | * | |
4 | * Copyright (C) 2002 Russell King | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | #ifndef __ASM_TLB_H | |
20 | #define __ASM_TLB_H | |
21 | ||
22 | #include <linux/pagemap.h> | |
23 | #include <linux/swap.h> | |
24 | ||
25 | #include <asm/pgalloc.h> | |
26 | #include <asm/tlbflush.h> | |
27 | ||
28 | #define MMU_GATHER_BUNDLE 8 | |
29 | ||
30 | /* | |
31 | * TLB handling. This allows us to remove pages from the page | |
32 | * tables, and efficiently handle the TLB issues. | |
33 | */ | |
34 | struct mmu_gather { | |
35 | struct mm_struct *mm; | |
36 | unsigned int fullmm; | |
37 | struct vm_area_struct *vma; | |
2b047252 | 38 | unsigned long start, end; |
58d0ba57 CM |
39 | unsigned long range_start; |
40 | unsigned long range_end; | |
41 | unsigned int nr; | |
42 | unsigned int max; | |
43 | struct page **pages; | |
44 | struct page *local[MMU_GATHER_BUNDLE]; | |
45 | }; | |
46 | ||
47 | /* | |
48 | * This is unnecessarily complex. There's three ways the TLB shootdown | |
49 | * code is used: | |
50 | * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). | |
51 | * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. | |
52 | * tlb->vma will be non-NULL. | |
53 | * 2. Unmapping all vmas. See exit_mmap(). | |
54 | * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. | |
55 | * tlb->vma will be non-NULL. Additionally, page tables will be freed. | |
56 | * 3. Unmapping argument pages. See shift_arg_pages(). | |
57 | * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. | |
58 | * tlb->vma will be NULL. | |
59 | */ | |
60 | static inline void tlb_flush(struct mmu_gather *tlb) | |
61 | { | |
62 | if (tlb->fullmm || !tlb->vma) | |
63 | flush_tlb_mm(tlb->mm); | |
64 | else if (tlb->range_end > 0) { | |
65 | flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); | |
66 | tlb->range_start = TASK_SIZE; | |
67 | tlb->range_end = 0; | |
68 | } | |
69 | } | |
70 | ||
71 | static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) | |
72 | { | |
73 | if (!tlb->fullmm) { | |
74 | if (addr < tlb->range_start) | |
75 | tlb->range_start = addr; | |
76 | if (addr + PAGE_SIZE > tlb->range_end) | |
77 | tlb->range_end = addr + PAGE_SIZE; | |
78 | } | |
79 | } | |
80 | ||
81 | static inline void __tlb_alloc_page(struct mmu_gather *tlb) | |
82 | { | |
83 | unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); | |
84 | ||
85 | if (addr) { | |
86 | tlb->pages = (void *)addr; | |
87 | tlb->max = PAGE_SIZE / sizeof(struct page *); | |
88 | } | |
89 | } | |
90 | ||
91 | static inline void tlb_flush_mmu(struct mmu_gather *tlb) | |
92 | { | |
93 | tlb_flush(tlb); | |
94 | free_pages_and_swap_cache(tlb->pages, tlb->nr); | |
95 | tlb->nr = 0; | |
96 | if (tlb->pages == tlb->local) | |
97 | __tlb_alloc_page(tlb); | |
98 | } | |
99 | ||
100 | static inline void | |
2b047252 | 101 | tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end) |
58d0ba57 CM |
102 | { |
103 | tlb->mm = mm; | |
2b047252 LT |
104 | tlb->fullmm = !(start | (end+1)); |
105 | tlb->start = start; | |
106 | tlb->end = end; | |
58d0ba57 CM |
107 | tlb->vma = NULL; |
108 | tlb->max = ARRAY_SIZE(tlb->local); | |
109 | tlb->pages = tlb->local; | |
110 | tlb->nr = 0; | |
111 | __tlb_alloc_page(tlb); | |
112 | } | |
113 | ||
114 | static inline void | |
115 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | |
116 | { | |
117 | tlb_flush_mmu(tlb); | |
118 | ||
119 | /* keep the page table cache within bounds */ | |
120 | check_pgt_cache(); | |
121 | ||
122 | if (tlb->pages != tlb->local) | |
123 | free_pages((unsigned long)tlb->pages, 0); | |
124 | } | |
125 | ||
126 | /* | |
127 | * Memorize the range for the TLB flush. | |
128 | */ | |
129 | static inline void | |
130 | tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) | |
131 | { | |
132 | tlb_add_flush(tlb, addr); | |
133 | } | |
134 | ||
135 | /* | |
136 | * In the case of tlb vma handling, we can optimise these away in the | |
137 | * case where we're doing a full MM flush. When we're doing a munmap, | |
138 | * the vmas are adjusted to only cover the region to be torn down. | |
139 | */ | |
140 | static inline void | |
141 | tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) | |
142 | { | |
143 | if (!tlb->fullmm) { | |
144 | tlb->vma = vma; | |
145 | tlb->range_start = TASK_SIZE; | |
146 | tlb->range_end = 0; | |
147 | } | |
148 | } | |
149 | ||
150 | static inline void | |
151 | tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) | |
152 | { | |
153 | if (!tlb->fullmm) | |
154 | tlb_flush(tlb); | |
155 | } | |
156 | ||
157 | static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) | |
158 | { | |
159 | tlb->pages[tlb->nr++] = page; | |
160 | VM_BUG_ON(tlb->nr > tlb->max); | |
161 | return tlb->max - tlb->nr; | |
162 | } | |
163 | ||
164 | static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) | |
165 | { | |
166 | if (!__tlb_remove_page(tlb, page)) | |
167 | tlb_flush_mmu(tlb); | |
168 | } | |
169 | ||
170 | static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, | |
171 | unsigned long addr) | |
172 | { | |
173 | pgtable_page_dtor(pte); | |
174 | tlb_add_flush(tlb, addr); | |
175 | tlb_remove_page(tlb, pte); | |
176 | } | |
177 | ||
178 | #ifndef CONFIG_ARM64_64K_PAGES | |
179 | static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, | |
180 | unsigned long addr) | |
181 | { | |
182 | tlb_add_flush(tlb, addr); | |
183 | tlb_remove_page(tlb, virt_to_page(pmdp)); | |
184 | } | |
185 | #endif | |
186 | ||
187 | #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) | |
188 | #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) | |
189 | #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) | |
190 | ||
191 | #define tlb_migrate_finish(mm) do { } while (0) | |
192 | ||
af074848 SC |
193 | static inline void |
194 | tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) | |
195 | { | |
196 | tlb_add_flush(tlb, addr); | |
197 | } | |
198 | ||
58d0ba57 | 199 | #endif |