Commit | Line | Data |
---|---|---|
58d0ba57 CM |
1 | /* |
2 | * Based on arch/arm/include/asm/tlb.h | |
3 | * | |
4 | * Copyright (C) 2002 Russell King | |
5 | * Copyright (C) 2012 ARM Ltd. | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | #ifndef __ASM_TLB_H | |
20 | #define __ASM_TLB_H | |
21 | ||
22 | #include <linux/pagemap.h> | |
23 | #include <linux/swap.h> | |
24 | ||
25 | #include <asm/pgalloc.h> | |
26 | #include <asm/tlbflush.h> | |
27 | ||
28 | #define MMU_GATHER_BUNDLE 8 | |
29 | ||
30 | /* | |
31 | * TLB handling. This allows us to remove pages from the page | |
32 | * tables, and efficiently handle the TLB issues. | |
33 | */ | |
34 | struct mmu_gather { | |
35 | struct mm_struct *mm; | |
36 | unsigned int fullmm; | |
37 | struct vm_area_struct *vma; | |
38 | unsigned long range_start; | |
39 | unsigned long range_end; | |
40 | unsigned int nr; | |
41 | unsigned int max; | |
42 | struct page **pages; | |
43 | struct page *local[MMU_GATHER_BUNDLE]; | |
44 | }; | |
45 | ||
46 | /* | |
47 | * This is unnecessarily complex. There's three ways the TLB shootdown | |
48 | * code is used: | |
49 | * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). | |
50 | * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. | |
51 | * tlb->vma will be non-NULL. | |
52 | * 2. Unmapping all vmas. See exit_mmap(). | |
53 | * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. | |
54 | * tlb->vma will be non-NULL. Additionally, page tables will be freed. | |
55 | * 3. Unmapping argument pages. See shift_arg_pages(). | |
56 | * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. | |
57 | * tlb->vma will be NULL. | |
58 | */ | |
59 | static inline void tlb_flush(struct mmu_gather *tlb) | |
60 | { | |
61 | if (tlb->fullmm || !tlb->vma) | |
62 | flush_tlb_mm(tlb->mm); | |
63 | else if (tlb->range_end > 0) { | |
64 | flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end); | |
65 | tlb->range_start = TASK_SIZE; | |
66 | tlb->range_end = 0; | |
67 | } | |
68 | } | |
69 | ||
70 | static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) | |
71 | { | |
72 | if (!tlb->fullmm) { | |
73 | if (addr < tlb->range_start) | |
74 | tlb->range_start = addr; | |
75 | if (addr + PAGE_SIZE > tlb->range_end) | |
76 | tlb->range_end = addr + PAGE_SIZE; | |
77 | } | |
78 | } | |
79 | ||
80 | static inline void __tlb_alloc_page(struct mmu_gather *tlb) | |
81 | { | |
82 | unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0); | |
83 | ||
84 | if (addr) { | |
85 | tlb->pages = (void *)addr; | |
86 | tlb->max = PAGE_SIZE / sizeof(struct page *); | |
87 | } | |
88 | } | |
89 | ||
90 | static inline void tlb_flush_mmu(struct mmu_gather *tlb) | |
91 | { | |
92 | tlb_flush(tlb); | |
93 | free_pages_and_swap_cache(tlb->pages, tlb->nr); | |
94 | tlb->nr = 0; | |
95 | if (tlb->pages == tlb->local) | |
96 | __tlb_alloc_page(tlb); | |
97 | } | |
98 | ||
99 | static inline void | |
100 | tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int fullmm) | |
101 | { | |
102 | tlb->mm = mm; | |
103 | tlb->fullmm = fullmm; | |
104 | tlb->vma = NULL; | |
105 | tlb->max = ARRAY_SIZE(tlb->local); | |
106 | tlb->pages = tlb->local; | |
107 | tlb->nr = 0; | |
108 | __tlb_alloc_page(tlb); | |
109 | } | |
110 | ||
111 | static inline void | |
112 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | |
113 | { | |
114 | tlb_flush_mmu(tlb); | |
115 | ||
116 | /* keep the page table cache within bounds */ | |
117 | check_pgt_cache(); | |
118 | ||
119 | if (tlb->pages != tlb->local) | |
120 | free_pages((unsigned long)tlb->pages, 0); | |
121 | } | |
122 | ||
123 | /* | |
124 | * Memorize the range for the TLB flush. | |
125 | */ | |
126 | static inline void | |
127 | tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr) | |
128 | { | |
129 | tlb_add_flush(tlb, addr); | |
130 | } | |
131 | ||
132 | /* | |
133 | * In the case of tlb vma handling, we can optimise these away in the | |
134 | * case where we're doing a full MM flush. When we're doing a munmap, | |
135 | * the vmas are adjusted to only cover the region to be torn down. | |
136 | */ | |
137 | static inline void | |
138 | tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) | |
139 | { | |
140 | if (!tlb->fullmm) { | |
141 | tlb->vma = vma; | |
142 | tlb->range_start = TASK_SIZE; | |
143 | tlb->range_end = 0; | |
144 | } | |
145 | } | |
146 | ||
147 | static inline void | |
148 | tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma) | |
149 | { | |
150 | if (!tlb->fullmm) | |
151 | tlb_flush(tlb); | |
152 | } | |
153 | ||
154 | static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) | |
155 | { | |
156 | tlb->pages[tlb->nr++] = page; | |
157 | VM_BUG_ON(tlb->nr > tlb->max); | |
158 | return tlb->max - tlb->nr; | |
159 | } | |
160 | ||
161 | static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) | |
162 | { | |
163 | if (!__tlb_remove_page(tlb, page)) | |
164 | tlb_flush_mmu(tlb); | |
165 | } | |
166 | ||
167 | static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, | |
168 | unsigned long addr) | |
169 | { | |
170 | pgtable_page_dtor(pte); | |
171 | tlb_add_flush(tlb, addr); | |
172 | tlb_remove_page(tlb, pte); | |
173 | } | |
174 | ||
175 | #ifndef CONFIG_ARM64_64K_PAGES | |
176 | static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, | |
177 | unsigned long addr) | |
178 | { | |
179 | tlb_add_flush(tlb, addr); | |
180 | tlb_remove_page(tlb, virt_to_page(pmdp)); | |
181 | } | |
182 | #endif | |
183 | ||
184 | #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) | |
185 | #define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) | |
186 | #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) | |
187 | ||
188 | #define tlb_migrate_finish(mm) do { } while (0) | |
189 | ||
af074848 SC |
190 | static inline void |
191 | tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) | |
192 | { | |
193 | tlb_add_flush(tlb, addr); | |
194 | } | |
195 | ||
58d0ba57 | 196 | #endif |