2 #include <linux/highmem.h>
3 #include <linux/sched.h>
4 #include <linux/hugetlb.h>
6 static int walk_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
12 pte
= pte_offset_map(pmd
, addr
);
14 err
= walk
->pte_entry(pte
, addr
, addr
+ PAGE_SIZE
, walk
);
27 static int walk_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
34 pmd
= pmd_offset(pud
, addr
);
37 next
= pmd_addr_end(addr
, end
);
38 if (pmd_none(*pmd
) || !walk
->vma
) {
40 err
= walk
->pte_hole(addr
, next
, walk
);
46 * This implies that each ->pmd_entry() handler
47 * needs to know about pmd_trans_huge() pmds
50 err
= walk
->pmd_entry(pmd
, addr
, next
, walk
);
55 * Check this here so we only break down trans_huge
56 * pages when we _need_ to
61 split_huge_page_pmd_mm(walk
->mm
, addr
, pmd
);
62 if (pmd_trans_unstable(pmd
))
64 err
= walk_pte_range(pmd
, addr
, next
, walk
);
67 } while (pmd
++, addr
= next
, addr
!= end
);
72 static int walk_pud_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
79 pud
= pud_offset(pgd
, addr
);
81 next
= pud_addr_end(addr
, end
);
82 if (pud_none_or_clear_bad(pud
)) {
84 err
= walk
->pte_hole(addr
, next
, walk
);
89 if (walk
->pmd_entry
|| walk
->pte_entry
)
90 err
= walk_pmd_range(pud
, addr
, next
, walk
);
93 } while (pud
++, addr
= next
, addr
!= end
);
98 static int walk_pgd_range(unsigned long addr
, unsigned long end
,
105 pgd
= pgd_offset(walk
->mm
, addr
);
107 next
= pgd_addr_end(addr
, end
);
108 if (pgd_none_or_clear_bad(pgd
)) {
110 err
= walk
->pte_hole(addr
, next
, walk
);
115 if (walk
->pmd_entry
|| walk
->pte_entry
)
116 err
= walk_pud_range(pgd
, addr
, next
, walk
);
119 } while (pgd
++, addr
= next
, addr
!= end
);
124 #ifdef CONFIG_HUGETLB_PAGE
125 static unsigned long hugetlb_entry_end(struct hstate
*h
, unsigned long addr
,
128 unsigned long boundary
= (addr
& huge_page_mask(h
)) + huge_page_size(h
);
129 return boundary
< end
? boundary
: end
;
132 static int walk_hugetlb_range(unsigned long addr
, unsigned long end
,
133 struct mm_walk
*walk
)
135 struct vm_area_struct
*vma
= walk
->vma
;
136 struct hstate
*h
= hstate_vma(vma
);
138 unsigned long hmask
= huge_page_mask(h
);
143 next
= hugetlb_entry_end(h
, addr
, end
);
144 pte
= huge_pte_offset(walk
->mm
, addr
& hmask
);
145 if (pte
&& walk
->hugetlb_entry
)
146 err
= walk
->hugetlb_entry(pte
, hmask
, addr
, next
, walk
);
149 } while (addr
= next
, addr
!= end
);
154 #else /* CONFIG_HUGETLB_PAGE */
155 static int walk_hugetlb_range(unsigned long addr
, unsigned long end
,
156 struct mm_walk
*walk
)
161 #endif /* CONFIG_HUGETLB_PAGE */
164 * Decide whether we really walk over the current vma on [@start, @end)
165 * or skip it via the returned value. Return 0 if we do walk over the
166 * current vma, and return 1 if we skip the vma. Negative values means
167 * error, where we abort the current walk.
169 static int walk_page_test(unsigned long start
, unsigned long end
,
170 struct mm_walk
*walk
)
172 struct vm_area_struct
*vma
= walk
->vma
;
175 return walk
->test_walk(start
, end
, walk
);
178 * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP
179 * range, so we don't walk over it as we do for normal vmas. However,
180 * Some callers are interested in handling hole range and they don't
181 * want to just ignore any single address range. Such users certainly
182 * define their ->pte_hole() callbacks, so let's delegate them to handle
185 if (vma
->vm_flags
& VM_PFNMAP
) {
188 err
= walk
->pte_hole(start
, end
, walk
);
189 return err
? err
: 1;
194 static int __walk_page_range(unsigned long start
, unsigned long end
,
195 struct mm_walk
*walk
)
198 struct vm_area_struct
*vma
= walk
->vma
;
200 if (vma
&& is_vm_hugetlb_page(vma
)) {
201 if (walk
->hugetlb_entry
)
202 err
= walk_hugetlb_range(start
, end
, walk
);
204 err
= walk_pgd_range(start
, end
, walk
);
210 * walk_page_range - walk page table with caller specific callbacks
212 * Recursively walk the page table tree of the process represented by @walk->mm
213 * within the virtual address range [@start, @end). During walking, we can do
214 * some caller-specific works for each entry, by setting up pmd_entry(),
215 * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these
216 * callbacks, the associated entries/pages are just ignored.
217 * The return values of these callbacks are commonly defined like below:
218 * - 0 : succeeded to handle the current entry, and if you don't reach the
219 * end address yet, continue to walk.
220 * - >0 : succeeded to handle the current entry, and return to the caller
221 * with caller specific value.
222 * - <0 : failed to handle the current entry, and return to the caller
225 * Before starting to walk page table, some callers want to check whether
226 * they really want to walk over the current vma, typically by checking
227 * its vm_flags. walk_page_test() and @walk->test_walk() are used for this
230 * struct mm_walk keeps current values of some common data like vma and pmd,
231 * which are useful for the access from callbacks. If you want to pass some
232 * caller-specific data to callbacks, @walk->private should be helpful.
235 * Callers of walk_page_range() and walk_page_vma() should hold
236 * @walk->mm->mmap_sem, because these function traverse vma list and/or
237 * access to vma's data.
239 int walk_page_range(unsigned long start
, unsigned long end
,
240 struct mm_walk
*walk
)
244 struct vm_area_struct
*vma
;
252 VM_BUG_ON_MM(!rwsem_is_locked(&walk
->mm
->mmap_sem
), walk
->mm
);
254 vma
= find_vma(walk
->mm
, start
);
256 if (!vma
) { /* after the last vma */
259 } else if (start
< vma
->vm_start
) { /* outside vma */
261 next
= min(end
, vma
->vm_start
);
262 } else { /* inside vma */
264 next
= min(end
, vma
->vm_end
);
267 err
= walk_page_test(start
, next
, walk
);
270 * positive return values are purely for
271 * controlling the pagewalk, so should never
272 * be passed to the callers.
280 if (walk
->vma
|| walk
->pte_hole
)
281 err
= __walk_page_range(start
, next
, walk
);
284 } while (start
= next
, start
< end
);
288 int walk_page_vma(struct vm_area_struct
*vma
, struct mm_walk
*walk
)
295 VM_BUG_ON(!rwsem_is_locked(&walk
->mm
->mmap_sem
));
298 err
= walk_page_test(vma
->vm_start
, vma
->vm_end
, walk
);
303 return __walk_page_range(vma
->vm_start
, vma
->vm_end
, walk
);
This page took 0.035719 seconds and 5 git commands to generate.