4 * (C) Copyright 1996 Linus Torvalds
6 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
7 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
11 #include <linux/hugetlb.h>
12 #include <linux/shm.h>
13 #include <linux/ksm.h>
14 #include <linux/mman.h>
15 #include <linux/swap.h>
16 #include <linux/capability.h>
18 #include <linux/highmem.h>
19 #include <linux/security.h>
20 #include <linux/syscalls.h>
21 #include <linux/mmu_notifier.h>
23 #include <asm/uaccess.h>
24 #include <asm/cacheflush.h>
25 #include <asm/tlbflush.h>
29 static pmd_t
*get_old_pmd(struct mm_struct
*mm
, unsigned long addr
)
35 pgd
= pgd_offset(mm
, addr
);
36 if (pgd_none_or_clear_bad(pgd
))
39 pud
= pud_offset(pgd
, addr
);
40 if (pud_none_or_clear_bad(pud
))
43 pmd
= pmd_offset(pud
, addr
);
44 split_huge_page_pmd(mm
, pmd
);
45 if (pmd_none_or_clear_bad(pmd
))
51 static pmd_t
*alloc_new_pmd(struct mm_struct
*mm
, struct vm_area_struct
*vma
,
58 pgd
= pgd_offset(mm
, addr
);
59 pud
= pud_alloc(mm
, pgd
, addr
);
63 pmd
= pmd_alloc(mm
, pud
, addr
);
67 VM_BUG_ON(pmd_trans_huge(*pmd
));
68 if (pmd_none(*pmd
) && __pte_alloc(mm
, vma
, pmd
, addr
))
74 static void move_ptes(struct vm_area_struct
*vma
, pmd_t
*old_pmd
,
75 unsigned long old_addr
, unsigned long old_end
,
76 struct vm_area_struct
*new_vma
, pmd_t
*new_pmd
,
77 unsigned long new_addr
)
79 struct address_space
*mapping
= NULL
;
80 struct mm_struct
*mm
= vma
->vm_mm
;
81 pte_t
*old_pte
, *new_pte
, pte
;
82 spinlock_t
*old_ptl
, *new_ptl
;
83 unsigned long old_start
;
86 mmu_notifier_invalidate_range_start(vma
->vm_mm
,
90 * Subtle point from Rajesh Venkatasubramanian: before
91 * moving file-based ptes, we must lock truncate_pagecache
92 * out, since it might clean the dst vma before the src vma,
93 * and we propagate stale pages into the dst afterward.
95 mapping
= vma
->vm_file
->f_mapping
;
96 spin_lock(&mapping
->i_mmap_lock
);
97 new_vma
->vm_truncate_count
= 0;
101 * We don't have to worry about the ordering of src and dst
102 * pte locks because exclusive mmap_sem prevents deadlock.
104 old_pte
= pte_offset_map_lock(mm
, old_pmd
, old_addr
, &old_ptl
);
105 new_pte
= pte_offset_map(new_pmd
, new_addr
);
106 new_ptl
= pte_lockptr(mm
, new_pmd
);
107 if (new_ptl
!= old_ptl
)
108 spin_lock_nested(new_ptl
, SINGLE_DEPTH_NESTING
);
109 arch_enter_lazy_mmu_mode();
111 for (; old_addr
< old_end
; old_pte
++, old_addr
+= PAGE_SIZE
,
112 new_pte
++, new_addr
+= PAGE_SIZE
) {
113 if (pte_none(*old_pte
))
115 pte
= ptep_clear_flush(vma
, old_addr
, old_pte
);
116 pte
= move_pte(pte
, new_vma
->vm_page_prot
, old_addr
, new_addr
);
117 set_pte_at(mm
, new_addr
, new_pte
, pte
);
120 arch_leave_lazy_mmu_mode();
121 if (new_ptl
!= old_ptl
)
122 spin_unlock(new_ptl
);
123 pte_unmap(new_pte
- 1);
124 pte_unmap_unlock(old_pte
- 1, old_ptl
);
126 spin_unlock(&mapping
->i_mmap_lock
);
127 mmu_notifier_invalidate_range_end(vma
->vm_mm
, old_start
, old_end
);
130 #define LATENCY_LIMIT (64 * PAGE_SIZE)
132 unsigned long move_page_tables(struct vm_area_struct
*vma
,
133 unsigned long old_addr
, struct vm_area_struct
*new_vma
,
134 unsigned long new_addr
, unsigned long len
)
136 unsigned long extent
, next
, old_end
;
137 pmd_t
*old_pmd
, *new_pmd
;
139 old_end
= old_addr
+ len
;
140 flush_cache_range(vma
, old_addr
, old_end
);
142 for (; old_addr
< old_end
; old_addr
+= extent
, new_addr
+= extent
) {
144 next
= (old_addr
+ PMD_SIZE
) & PMD_MASK
;
145 if (next
- 1 > old_end
)
147 extent
= next
- old_addr
;
148 old_pmd
= get_old_pmd(vma
->vm_mm
, old_addr
);
151 new_pmd
= alloc_new_pmd(vma
->vm_mm
, vma
, new_addr
);
154 next
= (new_addr
+ PMD_SIZE
) & PMD_MASK
;
155 if (extent
> next
- new_addr
)
156 extent
= next
- new_addr
;
157 if (extent
> LATENCY_LIMIT
)
158 extent
= LATENCY_LIMIT
;
159 move_ptes(vma
, old_pmd
, old_addr
, old_addr
+ extent
,
160 new_vma
, new_pmd
, new_addr
);
163 return len
+ old_addr
- old_end
; /* how much done */
166 static unsigned long move_vma(struct vm_area_struct
*vma
,
167 unsigned long old_addr
, unsigned long old_len
,
168 unsigned long new_len
, unsigned long new_addr
)
170 struct mm_struct
*mm
= vma
->vm_mm
;
171 struct vm_area_struct
*new_vma
;
172 unsigned long vm_flags
= vma
->vm_flags
;
173 unsigned long new_pgoff
;
174 unsigned long moved_len
;
175 unsigned long excess
= 0;
176 unsigned long hiwater_vm
;
181 * We'd prefer to avoid failure later on in do_munmap:
182 * which may split one vma into three before unmapping.
184 if (mm
->map_count
>= sysctl_max_map_count
- 3)
188 * Advise KSM to break any KSM pages in the area to be moved:
189 * it would be confusing if they were to turn up at the new
190 * location, where they happen to coincide with different KSM
191 * pages recently unmapped. But leave vma->vm_flags as it was,
192 * so KSM can come around to merge on vma and new_vma afterwards.
194 err
= ksm_madvise(vma
, old_addr
, old_addr
+ old_len
,
195 MADV_UNMERGEABLE
, &vm_flags
);
199 new_pgoff
= vma
->vm_pgoff
+ ((old_addr
- vma
->vm_start
) >> PAGE_SHIFT
);
200 new_vma
= copy_vma(&vma
, new_addr
, new_len
, new_pgoff
);
204 moved_len
= move_page_tables(vma
, old_addr
, new_vma
, new_addr
, old_len
);
205 if (moved_len
< old_len
) {
207 * On error, move entries back from new area to old,
208 * which will succeed since page tables still there,
209 * and then proceed to unmap new area instead of old.
211 move_page_tables(new_vma
, new_addr
, vma
, old_addr
, moved_len
);
218 /* Conceal VM_ACCOUNT so old reservation is not undone */
219 if (vm_flags
& VM_ACCOUNT
) {
220 vma
->vm_flags
&= ~VM_ACCOUNT
;
221 excess
= vma
->vm_end
- vma
->vm_start
- old_len
;
222 if (old_addr
> vma
->vm_start
&&
223 old_addr
+ old_len
< vma
->vm_end
)
228 * If we failed to move page tables we still do total_vm increment
229 * since do_munmap() will decrement it by old_len == new_len.
231 * Since total_vm is about to be raised artificially high for a
232 * moment, we need to restore high watermark afterwards: if stats
233 * are taken meanwhile, total_vm and hiwater_vm appear too high.
234 * If this were a serious issue, we'd add a flag to do_munmap().
236 hiwater_vm
= mm
->hiwater_vm
;
237 mm
->total_vm
+= new_len
>> PAGE_SHIFT
;
238 vm_stat_account(mm
, vma
->vm_flags
, vma
->vm_file
, new_len
>>PAGE_SHIFT
);
240 if (do_munmap(mm
, old_addr
, old_len
) < 0) {
241 /* OOM: unable to split vma, just get accounts right */
242 vm_unacct_memory(excess
>> PAGE_SHIFT
);
245 mm
->hiwater_vm
= hiwater_vm
;
247 /* Restore VM_ACCOUNT if one or two pieces of vma left */
249 vma
->vm_flags
|= VM_ACCOUNT
;
251 vma
->vm_next
->vm_flags
|= VM_ACCOUNT
;
254 if (vm_flags
& VM_LOCKED
) {
255 mm
->locked_vm
+= new_len
>> PAGE_SHIFT
;
256 if (new_len
> old_len
)
257 mlock_vma_pages_range(new_vma
, new_addr
+ old_len
,
264 static struct vm_area_struct
*vma_to_resize(unsigned long addr
,
265 unsigned long old_len
, unsigned long new_len
, unsigned long *p
)
267 struct mm_struct
*mm
= current
->mm
;
268 struct vm_area_struct
*vma
= find_vma(mm
, addr
);
270 if (!vma
|| vma
->vm_start
> addr
)
273 if (is_vm_hugetlb_page(vma
))
276 /* We can't remap across vm area boundaries */
277 if (old_len
> vma
->vm_end
- addr
)
280 if (vma
->vm_flags
& (VM_DONTEXPAND
| VM_PFNMAP
)) {
281 if (new_len
> old_len
)
285 if (vma
->vm_flags
& VM_LOCKED
) {
286 unsigned long locked
, lock_limit
;
287 locked
= mm
->locked_vm
<< PAGE_SHIFT
;
288 lock_limit
= rlimit(RLIMIT_MEMLOCK
);
289 locked
+= new_len
- old_len
;
290 if (locked
> lock_limit
&& !capable(CAP_IPC_LOCK
))
294 if (!may_expand_vm(mm
, (new_len
- old_len
) >> PAGE_SHIFT
))
297 if (vma
->vm_flags
& VM_ACCOUNT
) {
298 unsigned long charged
= (new_len
- old_len
) >> PAGE_SHIFT
;
299 if (security_vm_enough_memory(charged
))
306 Efault
: /* very odd choice for most of the cases, but... */
307 return ERR_PTR(-EFAULT
);
309 return ERR_PTR(-EINVAL
);
311 return ERR_PTR(-ENOMEM
);
313 return ERR_PTR(-EAGAIN
);
316 static unsigned long mremap_to(unsigned long addr
,
317 unsigned long old_len
, unsigned long new_addr
,
318 unsigned long new_len
)
320 struct mm_struct
*mm
= current
->mm
;
321 struct vm_area_struct
*vma
;
322 unsigned long ret
= -EINVAL
;
323 unsigned long charged
= 0;
324 unsigned long map_flags
;
326 if (new_addr
& ~PAGE_MASK
)
329 if (new_len
> TASK_SIZE
|| new_addr
> TASK_SIZE
- new_len
)
332 /* Check if the location we're moving into overlaps the
333 * old location at all, and fail if it does.
335 if ((new_addr
<= addr
) && (new_addr
+new_len
) > addr
)
338 if ((addr
<= new_addr
) && (addr
+old_len
) > new_addr
)
341 ret
= security_file_mmap(NULL
, 0, 0, 0, new_addr
, 1);
345 ret
= do_munmap(mm
, new_addr
, new_len
);
349 if (old_len
>= new_len
) {
350 ret
= do_munmap(mm
, addr
+new_len
, old_len
- new_len
);
351 if (ret
&& old_len
!= new_len
)
356 vma
= vma_to_resize(addr
, old_len
, new_len
, &charged
);
362 map_flags
= MAP_FIXED
;
363 if (vma
->vm_flags
& VM_MAYSHARE
)
364 map_flags
|= MAP_SHARED
;
366 ret
= get_unmapped_area(vma
->vm_file
, new_addr
, new_len
, vma
->vm_pgoff
+
367 ((addr
- vma
->vm_start
) >> PAGE_SHIFT
),
369 if (ret
& ~PAGE_MASK
)
372 ret
= move_vma(vma
, addr
, old_len
, new_len
, new_addr
);
373 if (!(ret
& ~PAGE_MASK
))
376 vm_unacct_memory(charged
);
382 static int vma_expandable(struct vm_area_struct
*vma
, unsigned long delta
)
384 unsigned long end
= vma
->vm_end
+ delta
;
385 if (end
< vma
->vm_end
) /* overflow */
387 if (vma
->vm_next
&& vma
->vm_next
->vm_start
< end
) /* intersection */
389 if (get_unmapped_area(NULL
, vma
->vm_start
, end
- vma
->vm_start
,
390 0, MAP_FIXED
) & ~PAGE_MASK
)
396 * Expand (or shrink) an existing mapping, potentially moving it at the
397 * same time (controlled by the MREMAP_MAYMOVE flag and available VM space)
399 * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
400 * This option implies MREMAP_MAYMOVE.
402 unsigned long do_mremap(unsigned long addr
,
403 unsigned long old_len
, unsigned long new_len
,
404 unsigned long flags
, unsigned long new_addr
)
406 struct mm_struct
*mm
= current
->mm
;
407 struct vm_area_struct
*vma
;
408 unsigned long ret
= -EINVAL
;
409 unsigned long charged
= 0;
411 if (flags
& ~(MREMAP_FIXED
| MREMAP_MAYMOVE
))
414 if (addr
& ~PAGE_MASK
)
417 old_len
= PAGE_ALIGN(old_len
);
418 new_len
= PAGE_ALIGN(new_len
);
421 * We allow a zero old-len as a special case
422 * for DOS-emu "duplicate shm area" thing. But
423 * a zero new-len is nonsensical.
428 if (flags
& MREMAP_FIXED
) {
429 if (flags
& MREMAP_MAYMOVE
)
430 ret
= mremap_to(addr
, old_len
, new_addr
, new_len
);
435 * Always allow a shrinking remap: that just unmaps
436 * the unnecessary pages..
437 * do_munmap does all the needed commit accounting
439 if (old_len
>= new_len
) {
440 ret
= do_munmap(mm
, addr
+new_len
, old_len
- new_len
);
441 if (ret
&& old_len
!= new_len
)
448 * Ok, we need to grow..
450 vma
= vma_to_resize(addr
, old_len
, new_len
, &charged
);
456 /* old_len exactly to the end of the area..
458 if (old_len
== vma
->vm_end
- addr
) {
459 /* can we just expand the current mapping? */
460 if (vma_expandable(vma
, new_len
- old_len
)) {
461 int pages
= (new_len
- old_len
) >> PAGE_SHIFT
;
463 if (vma_adjust(vma
, vma
->vm_start
, addr
+ new_len
,
464 vma
->vm_pgoff
, NULL
)) {
469 mm
->total_vm
+= pages
;
470 vm_stat_account(mm
, vma
->vm_flags
, vma
->vm_file
, pages
);
471 if (vma
->vm_flags
& VM_LOCKED
) {
472 mm
->locked_vm
+= pages
;
473 mlock_vma_pages_range(vma
, addr
+ old_len
,
482 * We weren't able to just expand or shrink the area,
483 * we need to create a new one and move it..
486 if (flags
& MREMAP_MAYMOVE
) {
487 unsigned long map_flags
= 0;
488 if (vma
->vm_flags
& VM_MAYSHARE
)
489 map_flags
|= MAP_SHARED
;
491 new_addr
= get_unmapped_area(vma
->vm_file
, 0, new_len
,
493 ((addr
- vma
->vm_start
) >> PAGE_SHIFT
),
495 if (new_addr
& ~PAGE_MASK
) {
500 ret
= security_file_mmap(NULL
, 0, 0, 0, new_addr
, 1);
503 ret
= move_vma(vma
, addr
, old_len
, new_len
, new_addr
);
506 if (ret
& ~PAGE_MASK
)
507 vm_unacct_memory(charged
);
511 SYSCALL_DEFINE5(mremap
, unsigned long, addr
, unsigned long, old_len
,
512 unsigned long, new_len
, unsigned long, flags
,
513 unsigned long, new_addr
)
517 down_write(¤t
->mm
->mmap_sem
);
518 ret
= do_mremap(addr
, old_len
, new_len
, flags
, new_addr
);
519 up_write(¤t
->mm
->mmap_sem
);
This page took 0.041073 seconds and 5 git commands to generate.