2 * arch/s390/mm/pgtable.c
4 * Copyright IBM Corp. 2007
5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
8 #include <linux/sched.h>
9 #include <linux/kernel.h>
10 #include <linux/errno.h>
12 #include <linux/swap.h>
13 #include <linux/smp.h>
14 #include <linux/highmem.h>
15 #include <linux/slab.h>
16 #include <linux/pagemap.h>
17 #include <linux/spinlock.h>
18 #include <linux/module.h>
19 #include <linux/quicklist.h>
21 #include <asm/system.h>
22 #include <asm/pgtable.h>
23 #include <asm/pgalloc.h>
25 #include <asm/tlbflush.h>
26 #include <asm/mmu_context.h>
30 #define TABLES_PER_PAGE 4
31 #define FRAG_MASK 15UL
32 #define SECOND_HALVES 10UL
34 void clear_table_pgstes(unsigned long *table
)
36 clear_table(table
, _PAGE_TYPE_EMPTY
, PAGE_SIZE
/4);
37 memset(table
+ 256, 0, PAGE_SIZE
/4);
38 clear_table(table
+ 512, _PAGE_TYPE_EMPTY
, PAGE_SIZE
/4);
39 memset(table
+ 768, 0, PAGE_SIZE
/4);
44 #define TABLES_PER_PAGE 2
46 #define SECOND_HALVES 2UL
48 void clear_table_pgstes(unsigned long *table
)
50 clear_table(table
, _PAGE_TYPE_EMPTY
, PAGE_SIZE
/2);
51 memset(table
+ 256, 0, PAGE_SIZE
/2);
56 unsigned long *crst_table_alloc(struct mm_struct
*mm
, int noexec
)
58 struct page
*page
= alloc_pages(GFP_KERNEL
, ALLOC_ORDER
);
64 struct page
*shadow
= alloc_pages(GFP_KERNEL
, ALLOC_ORDER
);
66 __free_pages(page
, ALLOC_ORDER
);
69 page
->index
= page_to_phys(shadow
);
71 spin_lock(&mm
->page_table_lock
);
72 list_add(&page
->lru
, &mm
->context
.crst_list
);
73 spin_unlock(&mm
->page_table_lock
);
74 return (unsigned long *) page_to_phys(page
);
77 void crst_table_free(struct mm_struct
*mm
, unsigned long *table
)
79 unsigned long *shadow
= get_shadow_table(table
);
80 struct page
*page
= virt_to_page(table
);
82 spin_lock(&mm
->page_table_lock
);
84 spin_unlock(&mm
->page_table_lock
);
86 free_pages((unsigned long) shadow
, ALLOC_ORDER
);
87 free_pages((unsigned long) table
, ALLOC_ORDER
);
91 int crst_table_upgrade(struct mm_struct
*mm
, unsigned long limit
)
93 unsigned long *table
, *pgd
;
96 BUG_ON(limit
> (1UL << 53));
98 table
= crst_table_alloc(mm
, mm
->context
.noexec
);
101 spin_lock(&mm
->page_table_lock
);
102 if (mm
->context
.asce_limit
< limit
) {
103 pgd
= (unsigned long *) mm
->pgd
;
104 if (mm
->context
.asce_limit
<= (1UL << 31)) {
105 entry
= _REGION3_ENTRY_EMPTY
;
106 mm
->context
.asce_limit
= 1UL << 42;
107 mm
->context
.asce_bits
= _ASCE_TABLE_LENGTH
|
111 entry
= _REGION2_ENTRY_EMPTY
;
112 mm
->context
.asce_limit
= 1UL << 53;
113 mm
->context
.asce_bits
= _ASCE_TABLE_LENGTH
|
117 crst_table_init(table
, entry
);
118 pgd_populate(mm
, (pgd_t
*) table
, (pud_t
*) pgd
);
119 mm
->pgd
= (pgd_t
*) table
;
120 mm
->task_size
= mm
->context
.asce_limit
;
123 spin_unlock(&mm
->page_table_lock
);
125 crst_table_free(mm
, table
);
126 if (mm
->context
.asce_limit
< limit
)
128 update_mm(mm
, current
);
132 void crst_table_downgrade(struct mm_struct
*mm
, unsigned long limit
)
136 if (mm
->context
.asce_limit
<= limit
)
139 while (mm
->context
.asce_limit
> limit
) {
141 switch (pgd_val(*pgd
) & _REGION_ENTRY_TYPE_MASK
) {
142 case _REGION_ENTRY_TYPE_R2
:
143 mm
->context
.asce_limit
= 1UL << 42;
144 mm
->context
.asce_bits
= _ASCE_TABLE_LENGTH
|
148 case _REGION_ENTRY_TYPE_R3
:
149 mm
->context
.asce_limit
= 1UL << 31;
150 mm
->context
.asce_bits
= _ASCE_TABLE_LENGTH
|
157 mm
->pgd
= (pgd_t
*) (pgd_val(*pgd
) & _REGION_ENTRY_ORIGIN
);
158 mm
->task_size
= mm
->context
.asce_limit
;
159 crst_table_free(mm
, (unsigned long *) pgd
);
161 update_mm(mm
, current
);
166 * page table entry allocation/free routines.
168 unsigned long *page_table_alloc(struct mm_struct
*mm
)
171 unsigned long *table
;
174 bits
= (mm
->context
.noexec
|| mm
->context
.has_pgste
) ? 3UL : 1UL;
175 spin_lock(&mm
->page_table_lock
);
177 if (!list_empty(&mm
->context
.pgtable_list
)) {
178 page
= list_first_entry(&mm
->context
.pgtable_list
,
180 if ((page
->flags
& FRAG_MASK
) == ((1UL << TABLES_PER_PAGE
) - 1))
184 spin_unlock(&mm
->page_table_lock
);
185 page
= alloc_page(GFP_KERNEL
|__GFP_REPEAT
);
188 pgtable_page_ctor(page
);
189 page
->flags
&= ~FRAG_MASK
;
190 table
= (unsigned long *) page_to_phys(page
);
191 if (mm
->context
.has_pgste
)
192 clear_table_pgstes(table
);
194 clear_table(table
, _PAGE_TYPE_EMPTY
, PAGE_SIZE
);
195 spin_lock(&mm
->page_table_lock
);
196 list_add(&page
->lru
, &mm
->context
.pgtable_list
);
198 table
= (unsigned long *) page_to_phys(page
);
199 while (page
->flags
& bits
) {
204 if ((page
->flags
& FRAG_MASK
) == ((1UL << TABLES_PER_PAGE
) - 1))
205 list_move_tail(&page
->lru
, &mm
->context
.pgtable_list
);
206 spin_unlock(&mm
->page_table_lock
);
210 void page_table_free(struct mm_struct
*mm
, unsigned long *table
)
215 bits
= (mm
->context
.noexec
|| mm
->context
.has_pgste
) ? 3UL : 1UL;
216 bits
<<= (__pa(table
) & (PAGE_SIZE
- 1)) / 256 / sizeof(unsigned long);
217 page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
218 spin_lock(&mm
->page_table_lock
);
220 if (page
->flags
& FRAG_MASK
) {
221 /* Page now has some free pgtable fragments. */
222 list_move(&page
->lru
, &mm
->context
.pgtable_list
);
225 /* All fragments of the 4K page have been freed. */
226 list_del(&page
->lru
);
227 spin_unlock(&mm
->page_table_lock
);
229 pgtable_page_dtor(page
);
234 void disable_noexec(struct mm_struct
*mm
, struct task_struct
*tsk
)
238 spin_lock(&mm
->page_table_lock
);
239 /* Free shadow region and segment tables. */
240 list_for_each_entry(page
, &mm
->context
.crst_list
, lru
)
242 free_pages((unsigned long) page
->index
, ALLOC_ORDER
);
245 /* "Free" second halves of page tables. */
246 list_for_each_entry(page
, &mm
->context
.pgtable_list
, lru
)
247 page
->flags
&= ~SECOND_HALVES
;
248 spin_unlock(&mm
->page_table_lock
);
249 mm
->context
.noexec
= 0;
254 * switch on pgstes for its userspace process (for kvm)
256 int s390_enable_sie(void)
258 struct task_struct
*tsk
= current
;
259 struct mm_struct
*mm
, *old_mm
;
261 /* Do we have switched amode? If no, we cannot do sie */
265 /* Do we have pgstes? if yes, we are done */
266 if (tsk
->mm
->context
.has_pgste
)
269 /* lets check if we are allowed to replace the mm */
271 if (!tsk
->mm
|| atomic_read(&tsk
->mm
->mm_users
) > 1 ||
272 tsk
->mm
!= tsk
->active_mm
|| !hlist_empty(&tsk
->mm
->ioctx_list
)) {
278 /* we copy the mm and let dup_mm create the page tables with_pgstes */
279 tsk
->mm
->context
.alloc_pgste
= 1;
281 tsk
->mm
->context
.alloc_pgste
= 0;
285 /* Now lets check again if something happened */
287 if (!tsk
->mm
|| atomic_read(&tsk
->mm
->mm_users
) > 1 ||
288 tsk
->mm
!= tsk
->active_mm
|| !hlist_empty(&tsk
->mm
->ioctx_list
)) {
294 /* ok, we are alone. No ptrace, no threads, etc. */
296 tsk
->mm
= tsk
->active_mm
= mm
;
299 cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm
));
305 EXPORT_SYMBOL_GPL(s390_enable_sie
);
This page took 0.042333 seconds and 5 git commands to generate.