Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _S390_TLB_H |
2 | #define _S390_TLB_H | |
3 | ||
4 | /* | |
ba8a9229 MS |
5 | * TLB flushing on s390 is complicated. The following requirement |
6 | * from the principles of operation is the most arduous: | |
7 | * | |
8 | * "A valid table entry must not be changed while it is attached | |
9 | * to any CPU and may be used for translation by that CPU except to | |
10 | * (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY, | |
11 | * or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page | |
12 | * table entry, or (3) make a change by means of a COMPARE AND SWAP | |
13 | * AND PURGE instruction that purges the TLB." | |
14 | * | |
15 | * The modification of a pte of an active mm struct therefore is | |
16 | * a two step process: i) invalidate the pte, ii) store the new pte. | |
17 | * This is true for the page protection bit as well. | |
18 | * The only possible optimization is to flush at the beginning of | |
19 | * a tlb_gather_mmu cycle if the mm_struct is currently not in use. | |
20 | * | |
21 | * Pages used for the page tables is a different story. FIXME: more | |
1da177e4 | 22 | */ |
ba8a9229 MS |
23 | |
24 | #include <linux/mm.h> | |
c84ca008 | 25 | #include <linux/pagemap.h> |
ba8a9229 MS |
26 | #include <linux/swap.h> |
27 | #include <asm/processor.h> | |
28 | #include <asm/pgalloc.h> | |
29 | #include <asm/smp.h> | |
30 | #include <asm/tlbflush.h> | |
31 | ||
32 | #ifndef CONFIG_SMP | |
33 | #define TLB_NR_PTRS 1 | |
34 | #else | |
35 | #define TLB_NR_PTRS 508 | |
36 | #endif | |
37 | ||
38 | struct mmu_gather { | |
39 | struct mm_struct *mm; | |
40 | unsigned int fullmm; | |
41 | unsigned int nr_ptes; | |
5a216a20 | 42 | unsigned int nr_pxds; |
ba8a9229 MS |
43 | void *array[TLB_NR_PTRS]; |
44 | }; | |
45 | ||
46 | DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); | |
47 | ||
48 | static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, | |
49 | unsigned int full_mm_flush) | |
50 | { | |
51 | struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); | |
52 | ||
53 | tlb->mm = mm; | |
050eef36 | 54 | tlb->fullmm = full_mm_flush; |
ba8a9229 | 55 | tlb->nr_ptes = 0; |
5a216a20 | 56 | tlb->nr_pxds = TLB_NR_PTRS; |
ba8a9229 MS |
57 | if (tlb->fullmm) |
58 | __tlb_flush_mm(mm); | |
59 | return tlb; | |
60 | } | |
61 | ||
62 | static inline void tlb_flush_mmu(struct mmu_gather *tlb, | |
63 | unsigned long start, unsigned long end) | |
64 | { | |
5a216a20 | 65 | if (!tlb->fullmm && (tlb->nr_ptes > 0 || tlb->nr_pxds < TLB_NR_PTRS)) |
ba8a9229 MS |
66 | __tlb_flush_mm(tlb->mm); |
67 | while (tlb->nr_ptes > 0) | |
80217147 | 68 | page_table_free_rcu(tlb->mm, tlb->array[--tlb->nr_ptes]); |
5a216a20 | 69 | while (tlb->nr_pxds < TLB_NR_PTRS) |
80217147 | 70 | crst_table_free_rcu(tlb->mm, tlb->array[tlb->nr_pxds++]); |
ba8a9229 MS |
71 | } |
72 | ||
73 | static inline void tlb_finish_mmu(struct mmu_gather *tlb, | |
74 | unsigned long start, unsigned long end) | |
75 | { | |
76 | tlb_flush_mmu(tlb, start, end); | |
77 | ||
80217147 MS |
78 | rcu_table_freelist_finish(); |
79 | ||
ba8a9229 MS |
80 | /* keep the page table cache within bounds */ |
81 | check_pgt_cache(); | |
82 | ||
83 | put_cpu_var(mmu_gathers); | |
84 | } | |
1da177e4 LT |
85 | |
86 | /* | |
ba8a9229 MS |
87 | * Release the page cache reference for a pte removed by |
88 | * tlb_ptep_clear_flush. In both flush modes the tlb fo a page cache page | |
89 | * has already been freed, so just do free_page_and_swap_cache. | |
1da177e4 | 90 | */ |
ba8a9229 MS |
91 | static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) |
92 | { | |
93 | free_page_and_swap_cache(page); | |
94 | } | |
1da177e4 | 95 | |
ba8a9229 MS |
96 | /* |
97 | * pte_free_tlb frees a pte table and clears the CRSTE for the | |
98 | * page table from the tlb. | |
99 | */ | |
9e1b32ca BH |
100 | static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, |
101 | unsigned long address) | |
ba8a9229 MS |
102 | { |
103 | if (!tlb->fullmm) { | |
146e4b3c | 104 | tlb->array[tlb->nr_ptes++] = pte; |
5a216a20 | 105 | if (tlb->nr_ptes >= tlb->nr_pxds) |
ba8a9229 MS |
106 | tlb_flush_mmu(tlb, 0, 0); |
107 | } else | |
80217147 | 108 | page_table_free(tlb->mm, (unsigned long *) pte); |
ba8a9229 | 109 | } |
1da177e4 | 110 | |
ba8a9229 MS |
111 | /* |
112 | * pmd_free_tlb frees a pmd table and clears the CRSTE for the | |
113 | * segment table entry from the tlb. | |
6252d702 MS |
114 | * If the mm uses a two level page table the single pmd is freed |
115 | * as the pgd. pmd_free_tlb checks the asce_limit against 2GB | |
116 | * to avoid the double free of the pmd in this case. | |
ba8a9229 | 117 | */ |
9e1b32ca BH |
118 | static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, |
119 | unsigned long address) | |
ba8a9229 MS |
120 | { |
121 | #ifdef __s390x__ | |
6252d702 MS |
122 | if (tlb->mm->context.asce_limit <= (1UL << 31)) |
123 | return; | |
ba8a9229 | 124 | if (!tlb->fullmm) { |
5a216a20 MS |
125 | tlb->array[--tlb->nr_pxds] = pmd; |
126 | if (tlb->nr_ptes >= tlb->nr_pxds) | |
ba8a9229 MS |
127 | tlb_flush_mmu(tlb, 0, 0); |
128 | } else | |
80217147 | 129 | crst_table_free(tlb->mm, (unsigned long *) pmd); |
1da177e4 | 130 | #endif |
ba8a9229 MS |
131 | } |
132 | ||
5a216a20 MS |
133 | /* |
134 | * pud_free_tlb frees a pud table and clears the CRSTE for the | |
135 | * region third table entry from the tlb. | |
6252d702 MS |
136 | * If the mm uses a three level page table the single pud is freed |
137 | * as the pgd. pud_free_tlb checks the asce_limit against 4TB | |
138 | * to avoid the double free of the pud in this case. | |
5a216a20 | 139 | */ |
9e1b32ca BH |
140 | static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, |
141 | unsigned long address) | |
5a216a20 MS |
142 | { |
143 | #ifdef __s390x__ | |
6252d702 MS |
144 | if (tlb->mm->context.asce_limit <= (1UL << 42)) |
145 | return; | |
5a216a20 MS |
146 | if (!tlb->fullmm) { |
147 | tlb->array[--tlb->nr_pxds] = pud; | |
148 | if (tlb->nr_ptes >= tlb->nr_pxds) | |
149 | tlb_flush_mmu(tlb, 0, 0); | |
150 | } else | |
80217147 | 151 | crst_table_free(tlb->mm, (unsigned long *) pud); |
5a216a20 MS |
152 | #endif |
153 | } | |
190a1d72 | 154 | |
ba8a9229 MS |
155 | #define tlb_start_vma(tlb, vma) do { } while (0) |
156 | #define tlb_end_vma(tlb, vma) do { } while (0) | |
157 | #define tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0) | |
158 | #define tlb_migrate_finish(mm) do { } while (0) | |
159 | ||
160 | #endif /* _S390_TLB_H */ |