2 #include <linux/highmem.h>
3 #include <linux/sched.h>
4 #include <linux/hugetlb.h>
6 static int walk_pte_range(pmd_t
*pmd
, unsigned long addr
, unsigned long end
,
12 pte
= pte_offset_map(pmd
, addr
);
14 err
= walk
->pte_entry(pte
, addr
, addr
+ PAGE_SIZE
, walk
);
27 static int walk_pmd_range(pud_t
*pud
, unsigned long addr
, unsigned long end
,
34 pmd
= pmd_offset(pud
, addr
);
37 next
= pmd_addr_end(addr
, end
);
40 err
= walk
->pte_hole(addr
, next
, walk
);
46 * This implies that each ->pmd_entry() handler
47 * needs to know about pmd_trans_huge() pmds
50 err
= walk
->pmd_entry(pmd
, addr
, next
, walk
);
55 * Check this here so we only break down trans_huge
56 * pages when we _need_ to
61 split_huge_page_pmd(walk
->mm
, pmd
);
62 if (pmd_none_or_clear_bad(pmd
))
64 err
= walk_pte_range(pmd
, addr
, next
, walk
);
67 } while (pmd
++, addr
= next
, addr
!= end
);
72 static int walk_pud_range(pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
79 pud
= pud_offset(pgd
, addr
);
81 next
= pud_addr_end(addr
, end
);
82 if (pud_none_or_clear_bad(pud
)) {
84 err
= walk
->pte_hole(addr
, next
, walk
);
90 err
= walk
->pud_entry(pud
, addr
, next
, walk
);
91 if (!err
&& (walk
->pmd_entry
|| walk
->pte_entry
))
92 err
= walk_pmd_range(pud
, addr
, next
, walk
);
95 } while (pud
++, addr
= next
, addr
!= end
);
100 #ifdef CONFIG_HUGETLB_PAGE
101 static unsigned long hugetlb_entry_end(struct hstate
*h
, unsigned long addr
,
104 unsigned long boundary
= (addr
& huge_page_mask(h
)) + huge_page_size(h
);
105 return boundary
< end
? boundary
: end
;
108 static int walk_hugetlb_range(struct vm_area_struct
*vma
,
109 unsigned long addr
, unsigned long end
,
110 struct mm_walk
*walk
)
112 struct hstate
*h
= hstate_vma(vma
);
114 unsigned long hmask
= huge_page_mask(h
);
119 next
= hugetlb_entry_end(h
, addr
, end
);
120 pte
= huge_pte_offset(walk
->mm
, addr
& hmask
);
121 if (pte
&& walk
->hugetlb_entry
)
122 err
= walk
->hugetlb_entry(pte
, hmask
, addr
, next
, walk
);
125 } while (addr
= next
, addr
!= end
);
132 * walk_page_range - walk a memory map's page tables with a callback
133 * @mm: memory map to walk
134 * @addr: starting address
135 * @end: ending address
136 * @walk: set of callbacks to invoke for each level of the tree
138 * Recursively walk the page table for the memory area in a VMA,
139 * calling supplied callbacks. Callbacks are called in-order (first
140 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
141 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
143 * Each callback receives an entry pointer and the start and end of the
144 * associated range, and a copy of the original mm_walk for access to
145 * the ->private or ->mm fields.
147 * No locks are taken, but the bottom level iterator will map PTE
148 * directories from highmem if necessary.
150 * If any callback returns a non-zero value, the walk is aborted and
151 * the return value is propagated back to the caller. Otherwise 0 is returned.
153 int walk_page_range(unsigned long addr
, unsigned long end
,
154 struct mm_walk
*walk
)
166 pgd
= pgd_offset(walk
->mm
, addr
);
168 struct vm_area_struct
*uninitialized_var(vma
);
170 next
= pgd_addr_end(addr
, end
);
172 #ifdef CONFIG_HUGETLB_PAGE
174 * handle hugetlb vma individually because pagetable walk for
175 * the hugetlb page is dependent on the architecture and
176 * we can't handled it in the same manner as non-huge pages.
178 vma
= find_vma(walk
->mm
, addr
);
179 if (vma
&& is_vm_hugetlb_page(vma
)) {
180 if (vma
->vm_end
< next
)
183 * Hugepage is very tightly coupled with vma, so
184 * walk through hugetlb entries within a given vma.
186 err
= walk_hugetlb_range(vma
, addr
, next
, walk
);
189 pgd
= pgd_offset(walk
->mm
, next
);
193 if (pgd_none_or_clear_bad(pgd
)) {
195 err
= walk
->pte_hole(addr
, next
, walk
);
202 err
= walk
->pgd_entry(pgd
, addr
, next
, walk
);
204 (walk
->pud_entry
|| walk
->pmd_entry
|| walk
->pte_entry
))
205 err
= walk_pud_range(pgd
, addr
, next
, walk
);
209 } while (addr
= next
, addr
!= end
);
This page took 0.034709 seconds and 5 git commands to generate.