| | 1 | /* |
| | 2 | * Copyright (C) 2012 ARM Ltd. |
| | 3 | * |
| | 4 | * This program is free software; you can redistribute it and/or modify |
| | 5 | * it under the terms of the GNU General Public License version 2 as |
| | 6 | * published by the Free Software Foundation. |
| | 7 | * |
| | 8 | * This program is distributed in the hope that it will be useful, |
| | 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| | 11 | * GNU General Public License for more details. |
| | 12 | * |
| | 13 | * You should have received a copy of the GNU General Public License |
| | 14 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| | 15 | */ |
| | 16 | #ifndef __ASM_PGTABLE_H |
| | 17 | #define __ASM_PGTABLE_H |
| | 18 | |
| | 19 | #include <asm/bug.h> |
| | 20 | #include <asm/proc-fns.h> |
| | 21 | |
| | 22 | #include <asm/memory.h> |
| | 23 | #include <asm/pgtable-hwdef.h> |
| | 24 | #include <asm/pgtable-prot.h> |
| | 25 | |
| | 26 | /* |
| | 27 | * VMALLOC range. |
| | 28 | * |
| | 29 | * VMALLOC_START: beginning of the kernel vmalloc space |
| | 30 | * VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space |
| | 31 | * and fixed mappings |
| | 32 | */ |
| | 33 | #define VMALLOC_START (MODULES_END) |
| | 34 | #define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K) |
| | 35 | |
| | 36 | #define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT)) |
| | 37 | |
| | 38 | #define FIRST_USER_ADDRESS 0UL |
| | 39 | |
| | 40 | #ifndef __ASSEMBLY__ |
| | 41 | |
| | 42 | #include <asm/fixmap.h> |
| | 43 | #include <linux/mmdebug.h> |
| | 44 | |
| | 45 | extern void __pte_error(const char *file, int line, unsigned long val); |
| | 46 | extern void __pmd_error(const char *file, int line, unsigned long val); |
| | 47 | extern void __pud_error(const char *file, int line, unsigned long val); |
| | 48 | extern void __pgd_error(const char *file, int line, unsigned long val); |
| | 49 | |
| | 50 | /* |
| | 51 | * ZERO_PAGE is a global shared page that is always zero: used |
| | 52 | * for zero-mapped memory areas etc.. |
| | 53 | */ |
| | 54 | extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; |
| | 55 | #define ZERO_PAGE(vaddr) pfn_to_page(PHYS_PFN(__pa(empty_zero_page))) |
| | 56 | |
| | 57 | #define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte)) |
| | 58 | |
| | 59 | #define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT) |
| | 60 | |
| | 61 | #define pfn_pte(pfn,prot) (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) |
| | 62 | |
| | 63 | #define pte_none(pte) (!pte_val(pte)) |
| | 64 | #define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0)) |
| | 65 | #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) |
| | 66 | |
| | 67 | /* |
| | 68 | * The following only work if pte_present(). Undefined behaviour otherwise. |
| | 69 | */ |
| | 70 | #define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE))) |
| | 71 | #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) |
| | 72 | #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) |
| | 73 | #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) |
| | 74 | #define pte_exec(pte) (!(pte_val(pte) & PTE_UXN)) |
| | 75 | #define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT)) |
| | 76 | #define pte_user(pte) (!!(pte_val(pte) & PTE_USER)) |
| | 77 | |
| | 78 | #ifdef CONFIG_ARM64_HW_AFDBM |
| | 79 | #define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY)) |
| | 80 | #else |
| | 81 | #define pte_hw_dirty(pte) (0) |
| | 82 | #endif |
| | 83 | #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) |
| | 84 | #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) |
| | 85 | |
| | 86 | #define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) |
| | 87 | #define pte_valid_not_user(pte) \ |
| | 88 | ((pte_val(pte) & (PTE_VALID | PTE_USER)) == PTE_VALID) |
| | 89 | #define pte_valid_young(pte) \ |
| | 90 | ((pte_val(pte) & (PTE_VALID | PTE_AF)) == (PTE_VALID | PTE_AF)) |
| | 91 | |
| | 92 | /* |
| | 93 | * Could the pte be present in the TLB? We must check mm_tlb_flush_pending |
| | 94 | * so that we don't erroneously return false for pages that have been |
| | 95 | * remapped as PROT_NONE but are yet to be flushed from the TLB. |
| | 96 | */ |
| | 97 | #define pte_accessible(mm, pte) \ |
| | 98 | (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid_young(pte)) |
| | 99 | |
| | 100 | static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot) |
| | 101 | { |
| | 102 | pte_val(pte) &= ~pgprot_val(prot); |
| | 103 | return pte; |
| | 104 | } |
| | 105 | |
| | 106 | static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot) |
| | 107 | { |
| | 108 | pte_val(pte) |= pgprot_val(prot); |
| | 109 | return pte; |
| | 110 | } |
| | 111 | |
| | 112 | static inline pte_t pte_wrprotect(pte_t pte) |
| | 113 | { |
| | 114 | return clear_pte_bit(pte, __pgprot(PTE_WRITE)); |
| | 115 | } |
| | 116 | |
| | 117 | static inline pte_t pte_mkwrite(pte_t pte) |
| | 118 | { |
| | 119 | return set_pte_bit(pte, __pgprot(PTE_WRITE)); |
| | 120 | } |
| | 121 | |
| | 122 | static inline pte_t pte_mkclean(pte_t pte) |
| | 123 | { |
| | 124 | return clear_pte_bit(pte, __pgprot(PTE_DIRTY)); |
| | 125 | } |
| | 126 | |
| | 127 | static inline pte_t pte_mkdirty(pte_t pte) |
| | 128 | { |
| | 129 | return set_pte_bit(pte, __pgprot(PTE_DIRTY)); |
| | 130 | } |
| | 131 | |
| | 132 | static inline pte_t pte_mkold(pte_t pte) |
| | 133 | { |
| | 134 | return clear_pte_bit(pte, __pgprot(PTE_AF)); |
| | 135 | } |
| | 136 | |
| | 137 | static inline pte_t pte_mkyoung(pte_t pte) |
| | 138 | { |
| | 139 | return set_pte_bit(pte, __pgprot(PTE_AF)); |
| | 140 | } |
| | 141 | |
| | 142 | static inline pte_t pte_mkspecial(pte_t pte) |
| | 143 | { |
| | 144 | return set_pte_bit(pte, __pgprot(PTE_SPECIAL)); |
| | 145 | } |
| | 146 | |
| | 147 | static inline pte_t pte_mkcont(pte_t pte) |
| | 148 | { |
| | 149 | pte = set_pte_bit(pte, __pgprot(PTE_CONT)); |
| | 150 | return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE)); |
| | 151 | } |
| | 152 | |
| | 153 | static inline pte_t pte_mknoncont(pte_t pte) |
| | 154 | { |
| | 155 | return clear_pte_bit(pte, __pgprot(PTE_CONT)); |
| | 156 | } |
| | 157 | |
| | 158 | static inline pmd_t pmd_mkcont(pmd_t pmd) |
| | 159 | { |
| | 160 | return __pmd(pmd_val(pmd) | PMD_SECT_CONT); |
| | 161 | } |
| | 162 | |
| | 163 | static inline void set_pte(pte_t *ptep, pte_t pte) |
| | 164 | { |
| | 165 | *ptep = pte; |
| | 166 | |
| | 167 | /* |
| | 168 | * Only if the new pte is valid and kernel, otherwise TLB maintenance |
| | 169 | * or update_mmu_cache() have the necessary barriers. |
| | 170 | */ |
| | 171 | if (pte_valid_not_user(pte)) { |
| | 172 | dsb(ishst); |
| | 173 | isb(); |
| | 174 | } |
| | 175 | } |
| | 176 | |
| | 177 | struct mm_struct; |
| | 178 | struct vm_area_struct; |
| | 179 | |
| | 180 | extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); |
| | 181 | |
| | 182 | /* |
| | 183 | * PTE bits configuration in the presence of hardware Dirty Bit Management |
| | 184 | * (PTE_WRITE == PTE_DBM): |
| | 185 | * |
| | 186 | * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) |
| | 187 | * 0 0 | 1 0 0 |
| | 188 | * 0 1 | 1 1 0 |
| | 189 | * 1 0 | 1 0 1 |
| | 190 | * 1 1 | 0 1 x |
| | 191 | * |
| | 192 | * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via |
| | 193 | * the page fault mechanism. Checking the dirty status of a pte becomes: |
| | 194 | * |
| | 195 | * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY) |
| | 196 | */ |
| | 197 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
| | 198 | pte_t *ptep, pte_t pte) |
| | 199 | { |
| | 200 | if (pte_present(pte)) { |
| | 201 | if (pte_sw_dirty(pte) && pte_write(pte)) |
| | 202 | pte_val(pte) &= ~PTE_RDONLY; |
| | 203 | else |
| | 204 | pte_val(pte) |= PTE_RDONLY; |
| | 205 | if (pte_user(pte) && pte_exec(pte) && !pte_special(pte)) |
| | 206 | __sync_icache_dcache(pte, addr); |
| | 207 | } |
| | 208 | |
| | 209 | /* |
| | 210 | * If the existing pte is valid, check for potential race with |
| | 211 | * hardware updates of the pte (ptep_set_access_flags safely changes |
| | 212 | * valid ptes without going through an invalid entry). |
| | 213 | */ |
| | 214 | if (IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && |
| | 215 | pte_valid(*ptep) && pte_valid(pte)) { |
| | 216 | VM_WARN_ONCE(!pte_young(pte), |
| | 217 | "%s: racy access flag clearing: 0x%016llx -> 0x%016llx", |
| | 218 | __func__, pte_val(*ptep), pte_val(pte)); |
| | 219 | VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(pte), |
| | 220 | "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx", |
| | 221 | __func__, pte_val(*ptep), pte_val(pte)); |
| | 222 | } |
| | 223 | |
| | 224 | set_pte(ptep, pte); |
| | 225 | } |
| | 226 | |
| | 227 | /* |
| | 228 | * Huge pte definitions. |
| | 229 | */ |
| | 230 | #define pte_huge(pte) (!(pte_val(pte) & PTE_TABLE_BIT)) |
| | 231 | #define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT)) |
| | 232 | |
| | 233 | /* |
| | 234 | * Hugetlb definitions. |
| | 235 | */ |
| | 236 | #define HUGE_MAX_HSTATE 4 |
| | 237 | #define HPAGE_SHIFT PMD_SHIFT |
| | 238 | #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) |
| | 239 | #define HPAGE_MASK (~(HPAGE_SIZE - 1)) |
| | 240 | #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) |
| | 241 | |
| | 242 | #define __HAVE_ARCH_PTE_SPECIAL |
| | 243 | |
| | 244 | static inline pte_t pud_pte(pud_t pud) |
| | 245 | { |
| | 246 | return __pte(pud_val(pud)); |
| | 247 | } |
| | 248 | |
| | 249 | static inline pmd_t pud_pmd(pud_t pud) |
| | 250 | { |
| | 251 | return __pmd(pud_val(pud)); |
| | 252 | } |
| | 253 | |
| | 254 | static inline pte_t pmd_pte(pmd_t pmd) |
| | 255 | { |
| | 256 | return __pte(pmd_val(pmd)); |
| | 257 | } |
| | 258 | |
| | 259 | static inline pmd_t pte_pmd(pte_t pte) |
| | 260 | { |
| | 261 | return __pmd(pte_val(pte)); |
| | 262 | } |
| | 263 | |
| | 264 | static inline pgprot_t mk_sect_prot(pgprot_t prot) |
| | 265 | { |
| | 266 | return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT); |
| | 267 | } |
| | 268 | |
| | 269 | #ifdef CONFIG_NUMA_BALANCING |
| | 270 | /* |
| | 271 | * See the comment in include/asm-generic/pgtable.h |
| | 272 | */ |
| | 273 | static inline int pte_protnone(pte_t pte) |
| | 274 | { |
| | 275 | return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE; |
| | 276 | } |
| | 277 | |
| | 278 | static inline int pmd_protnone(pmd_t pmd) |
| | 279 | { |
| | 280 | return pte_protnone(pmd_pte(pmd)); |
| | 281 | } |
| | 282 | #endif |
| | 283 | |
| | 284 | /* |
| | 285 | * THP definitions. |
| | 286 | */ |
| | 287 | |
| | 288 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| | 289 | #define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT)) |
| | 290 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| | 291 | |
| | 292 | #define pmd_present(pmd) pte_present(pmd_pte(pmd)) |
| | 293 | #define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) |
| | 294 | #define pmd_young(pmd) pte_young(pmd_pte(pmd)) |
| | 295 | #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) |
| | 296 | #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd))) |
| | 297 | #define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd))) |
| | 298 | #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) |
| | 299 | #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) |
| | 300 | #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) |
| | 301 | #define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID)) |
| | 302 | |
| | 303 | #define __HAVE_ARCH_PMD_WRITE |
| | 304 | #define pmd_write(pmd) pte_write(pmd_pte(pmd)) |
| | 305 | |
| | 306 | #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT)) |
| | 307 | |
| | 308 | #define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT) |
| | 309 | #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) |
| | 310 | #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) |
| | 311 | |
| | 312 | #define pud_write(pud) pte_write(pud_pte(pud)) |
| | 313 | #define pud_pfn(pud) (((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT) |
| | 314 | |
| | 315 | #define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd)) |
| | 316 | |
| | 317 | static inline int has_transparent_hugepage(void) |
| | 318 | { |
| | 319 | return 1; |
| | 320 | } |
| | 321 | |
| | 322 | #define __pgprot_modify(prot,mask,bits) \ |
| | 323 | __pgprot((pgprot_val(prot) & ~(mask)) | (bits)) |
| | 324 | |
| | 325 | /* |
| | 326 | * Mark the prot value as uncacheable and unbufferable. |
| | 327 | */ |
| | 328 | #define pgprot_noncached(prot) \ |
| | 329 | __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN) |
| | 330 | #define pgprot_writecombine(prot) \ |
| | 331 | __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN) |
| | 332 | #define pgprot_device(prot) \ |
| | 333 | __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN) |
| | 334 | #define __HAVE_PHYS_MEM_ACCESS_PROT |
| | 335 | struct file; |
| | 336 | extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
| | 337 | unsigned long size, pgprot_t vma_prot); |
| | 338 | |
| | 339 | #define pmd_none(pmd) (!pmd_val(pmd)) |
| | 340 | |
| | 341 | #define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT)) |
| | 342 | |
| | 343 | #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ |
| | 344 | PMD_TYPE_TABLE) |
| | 345 | #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ |
| | 346 | PMD_TYPE_SECT) |
| | 347 | |
| | 348 | #if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3 |
| | 349 | #define pud_sect(pud) (0) |
| | 350 | #define pud_table(pud) (1) |
| | 351 | #else |
| | 352 | #define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ |
| | 353 | PUD_TYPE_SECT) |
| | 354 | #define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ |
| | 355 | PUD_TYPE_TABLE) |
| | 356 | #endif |
| | 357 | |
| | 358 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) |
| | 359 | { |
| | 360 | *pmdp = pmd; |
| | 361 | dsb(ishst); |
| | 362 | isb(); |
| | 363 | } |
| | 364 | |
| | 365 | static inline void pmd_clear(pmd_t *pmdp) |
| | 366 | { |
| | 367 | set_pmd(pmdp, __pmd(0)); |
| | 368 | } |
| | 369 | |
| | 370 | static inline phys_addr_t pmd_page_paddr(pmd_t pmd) |
| | 371 | { |
| | 372 | return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK; |
| | 373 | } |
| | 374 | |
| | 375 | /* Find an entry in the third-level page table. */ |
| | 376 | #define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) |
| | 377 | |
| | 378 | #define pte_offset_phys(dir,addr) (pmd_page_paddr(*(dir)) + pte_index(addr) * sizeof(pte_t)) |
| | 379 | #define pte_offset_kernel(dir,addr) ((pte_t *)__va(pte_offset_phys((dir), (addr)))) |
| | 380 | |
| | 381 | #define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) |
| | 382 | #define pte_offset_map_nested(dir,addr) pte_offset_kernel((dir), (addr)) |
| | 383 | #define pte_unmap(pte) do { } while (0) |
| | 384 | #define pte_unmap_nested(pte) do { } while (0) |
| | 385 | |
| | 386 | #define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) |
| | 387 | #define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) |
| | 388 | #define pte_clear_fixmap() clear_fixmap(FIX_PTE) |
| | 389 | |
| | 390 | #define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) |
| | 391 | |
| | 392 | /* use ONLY for statically allocated translation tables */ |
| | 393 | #define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr)))) |
| | 394 | |
| | 395 | /* |
| | 396 | * Conversion functions: convert a page and protection to a page entry, |
| | 397 | * and a page entry and page directory to the page they refer to. |
| | 398 | */ |
| | 399 | #define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot) |
| | 400 | |
| | 401 | #if CONFIG_PGTABLE_LEVELS > 2 |
| | 402 | |
| | 403 | #define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd)) |
| | 404 | |
| | 405 | #define pud_none(pud) (!pud_val(pud)) |
| | 406 | #define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) |
| | 407 | #define pud_present(pud) (pud_val(pud)) |
| | 408 | |
| | 409 | static inline void set_pud(pud_t *pudp, pud_t pud) |
| | 410 | { |
| | 411 | *pudp = pud; |
| | 412 | dsb(ishst); |
| | 413 | isb(); |
| | 414 | } |
| | 415 | |
| | 416 | static inline void pud_clear(pud_t *pudp) |
| | 417 | { |
| | 418 | set_pud(pudp, __pud(0)); |
| | 419 | } |
| | 420 | |
| | 421 | static inline phys_addr_t pud_page_paddr(pud_t pud) |
| | 422 | { |
| | 423 | return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK; |
| | 424 | } |
| | 425 | |
| | 426 | /* Find an entry in the second-level page table. */ |
| | 427 | #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) |
| | 428 | |
| | 429 | #define pmd_offset_phys(dir, addr) (pud_page_paddr(*(dir)) + pmd_index(addr) * sizeof(pmd_t)) |
| | 430 | #define pmd_offset(dir, addr) ((pmd_t *)__va(pmd_offset_phys((dir), (addr)))) |
| | 431 | |
| | 432 | #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) |
| | 433 | #define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) |
| | 434 | #define pmd_clear_fixmap() clear_fixmap(FIX_PMD) |
| | 435 | |
| | 436 | #define pud_page(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) |
| | 437 | |
| | 438 | /* use ONLY for statically allocated translation tables */ |
| | 439 | #define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr)))) |
| | 440 | |
| | 441 | #else |
| | 442 | |
| | 443 | #define pud_page_paddr(pud) ({ BUILD_BUG(); 0; }) |
| | 444 | |
| | 445 | /* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */ |
| | 446 | #define pmd_set_fixmap(addr) NULL |
| | 447 | #define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp) |
| | 448 | #define pmd_clear_fixmap() |
| | 449 | |
| | 450 | #define pmd_offset_kimg(dir,addr) ((pmd_t *)dir) |
| | 451 | |
| | 452 | #endif /* CONFIG_PGTABLE_LEVELS > 2 */ |
| | 453 | |
| | 454 | #if CONFIG_PGTABLE_LEVELS > 3 |
| | 455 | |
| | 456 | #define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud)) |
| | 457 | |
| | 458 | #define pgd_none(pgd) (!pgd_val(pgd)) |
| | 459 | #define pgd_bad(pgd) (!(pgd_val(pgd) & 2)) |
| | 460 | #define pgd_present(pgd) (pgd_val(pgd)) |
| | 461 | |
| | 462 | static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) |
| | 463 | { |
| | 464 | *pgdp = pgd; |
| | 465 | dsb(ishst); |
| | 466 | } |
| | 467 | |
| | 468 | static inline void pgd_clear(pgd_t *pgdp) |
| | 469 | { |
| | 470 | set_pgd(pgdp, __pgd(0)); |
| | 471 | } |
| | 472 | |
| | 473 | static inline phys_addr_t pgd_page_paddr(pgd_t pgd) |
| | 474 | { |
| | 475 | return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK; |
| | 476 | } |
| | 477 | |
| | 478 | /* Find an entry in the frst-level page table. */ |
| | 479 | #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) |
| | 480 | |
| | 481 | #define pud_offset_phys(dir, addr) (pgd_page_paddr(*(dir)) + pud_index(addr) * sizeof(pud_t)) |
| | 482 | #define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr)))) |
| | 483 | |
| | 484 | #define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) |
| | 485 | #define pud_set_fixmap_offset(pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr)) |
| | 486 | #define pud_clear_fixmap() clear_fixmap(FIX_PUD) |
| | 487 | |
| | 488 | #define pgd_page(pgd) pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK)) |
| | 489 | |
| | 490 | /* use ONLY for statically allocated translation tables */ |
| | 491 | #define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr)))) |
| | 492 | |
| | 493 | #else |
| | 494 | |
| | 495 | #define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0;}) |
| | 496 | |
| | 497 | /* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */ |
| | 498 | #define pud_set_fixmap(addr) NULL |
| | 499 | #define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp) |
| | 500 | #define pud_clear_fixmap() |
| | 501 | |
| | 502 | #define pud_offset_kimg(dir,addr) ((pud_t *)dir) |
| | 503 | |
| | 504 | #endif /* CONFIG_PGTABLE_LEVELS > 3 */ |
| | 505 | |
| | 506 | #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) |
| | 507 | |
| | 508 | /* to find an entry in a page-table-directory */ |
| | 509 | #define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) |
| | 510 | |
| | 511 | #define pgd_offset_raw(pgd, addr) ((pgd) + pgd_index(addr)) |
| | 512 | |
| | 513 | #define pgd_offset(mm, addr) (pgd_offset_raw((mm)->pgd, (addr))) |
| | 514 | |
| | 515 | /* to find an entry in a kernel page-table-directory */ |
| | 516 | #define pgd_offset_k(addr) pgd_offset(&init_mm, addr) |
| | 517 | |
| | 518 | #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) |
| | 519 | #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) |
| | 520 | |
| | 521 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) |
| | 522 | { |
| | 523 | const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | |
| | 524 | PTE_PROT_NONE | PTE_VALID | PTE_WRITE; |
| | 525 | /* preserve the hardware dirty information */ |
| | 526 | if (pte_hw_dirty(pte)) |
| | 527 | pte = pte_mkdirty(pte); |
| | 528 | pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); |
| | 529 | return pte; |
| | 530 | } |
| | 531 | |
| | 532 | static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) |
| | 533 | { |
| | 534 | return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); |
| | 535 | } |
| | 536 | |
| | 537 | #ifdef CONFIG_ARM64_HW_AFDBM |
| | 538 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
| | 539 | extern int ptep_set_access_flags(struct vm_area_struct *vma, |
| | 540 | unsigned long address, pte_t *ptep, |
| | 541 | pte_t entry, int dirty); |
| | 542 | |
| | 543 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| | 544 | #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS |
| | 545 | static inline int pmdp_set_access_flags(struct vm_area_struct *vma, |
| | 546 | unsigned long address, pmd_t *pmdp, |
| | 547 | pmd_t entry, int dirty) |
| | 548 | { |
| | 549 | return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty); |
| | 550 | } |
| | 551 | #endif |
| | 552 | |
| | 553 | /* |
| | 554 | * Atomic pte/pmd modifications. |
| | 555 | */ |
| | 556 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG |
| | 557 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, |
| | 558 | unsigned long address, |
| | 559 | pte_t *ptep) |
| | 560 | { |
| | 561 | pteval_t pteval; |
| | 562 | unsigned int tmp, res; |
| | 563 | |
| | 564 | asm volatile("// ptep_test_and_clear_young\n" |
| | 565 | " prfm pstl1strm, %2\n" |
| | 566 | "1: ldxr %0, %2\n" |
| | 567 | " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n" |
| | 568 | " and %0, %0, %4 // clear PTE_AF\n" |
| | 569 | " stxr %w1, %0, %2\n" |
| | 570 | " cbnz %w1, 1b\n" |
| | 571 | : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res) |
| | 572 | : "L" (~PTE_AF), "I" (ilog2(PTE_AF))); |
| | 573 | |
| | 574 | return res; |
| | 575 | } |
| | 576 | |
| | 577 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| | 578 | #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG |
| | 579 | static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, |
| | 580 | unsigned long address, |
| | 581 | pmd_t *pmdp) |
| | 582 | { |
| | 583 | return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); |
| | 584 | } |
| | 585 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| | 586 | |
| | 587 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR |
| | 588 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, |
| | 589 | unsigned long address, pte_t *ptep) |
| | 590 | { |
| | 591 | pteval_t old_pteval; |
| | 592 | unsigned int tmp; |
| | 593 | |
| | 594 | asm volatile("// ptep_get_and_clear\n" |
| | 595 | " prfm pstl1strm, %2\n" |
| | 596 | "1: ldxr %0, %2\n" |
| | 597 | " stxr %w1, xzr, %2\n" |
| | 598 | " cbnz %w1, 1b\n" |
| | 599 | : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))); |
| | 600 | |
| | 601 | return __pte(old_pteval); |
| | 602 | } |
| | 603 | |
| | 604 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| | 605 | #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR |
| | 606 | static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, |
| | 607 | unsigned long address, pmd_t *pmdp) |
| | 608 | { |
| | 609 | return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp)); |
| | 610 | } |
| | 611 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| | 612 | |
| | 613 | /* |
| | 614 | * ptep_set_wrprotect - mark read-only while trasferring potential hardware |
| | 615 | * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. |
| | 616 | */ |
| | 617 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
| | 618 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) |
| | 619 | { |
| | 620 | pteval_t pteval; |
| | 621 | unsigned long tmp; |
| | 622 | |
| | 623 | asm volatile("// ptep_set_wrprotect\n" |
| | 624 | " prfm pstl1strm, %2\n" |
| | 625 | "1: ldxr %0, %2\n" |
| | 626 | " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n" |
| | 627 | " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n" |
| | 628 | " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n" |
| | 629 | " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n" |
| | 630 | " stxr %w1, %0, %2\n" |
| | 631 | " cbnz %w1, 1b\n" |
| | 632 | : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) |
| | 633 | : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE) |
| | 634 | : "cc"); |
| | 635 | } |
| | 636 | |
| | 637 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| | 638 | #define __HAVE_ARCH_PMDP_SET_WRPROTECT |
| | 639 | static inline void pmdp_set_wrprotect(struct mm_struct *mm, |
| | 640 | unsigned long address, pmd_t *pmdp) |
| | 641 | { |
| | 642 | ptep_set_wrprotect(mm, address, (pte_t *)pmdp); |
| | 643 | } |
| | 644 | #endif |
| | 645 | #endif /* CONFIG_ARM64_HW_AFDBM */ |
| | 646 | |
| | 647 | extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; |
| | 648 | extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; |
| | 649 | |
| | 650 | /* |
| | 651 | * Encode and decode a swap entry: |
| | 652 | * bits 0-1: present (must be zero) |
| | 653 | * bits 2-7: swap type |
| | 654 | * bits 8-57: swap offset |
| | 655 | * bit 58: PTE_PROT_NONE (must be zero) |
| | 656 | */ |
| | 657 | #define __SWP_TYPE_SHIFT 2 |
| | 658 | #define __SWP_TYPE_BITS 6 |
| | 659 | #define __SWP_OFFSET_BITS 50 |
| | 660 | #define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) |
| | 661 | #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) |
| | 662 | #define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) |
| | 663 | |
| | 664 | #define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) |
| | 665 | #define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK) |
| | 666 | #define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) }) |
| | 667 | |
| | 668 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) |
| | 669 | #define __swp_entry_to_pte(swp) ((pte_t) { (swp).val }) |
| | 670 | |
| | 671 | /* |
| | 672 | * Ensure that there are not more swap files than can be encoded in the kernel |
| | 673 | * PTEs. |
| | 674 | */ |
| | 675 | #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) |
| | 676 | |
| | 677 | extern int kern_addr_valid(unsigned long addr); |
| | 678 | |
| | 679 | #include <asm-generic/pgtable.h> |
| | 680 | |
| | 681 | void pgd_cache_init(void); |
| | 682 | #define pgtable_cache_init pgd_cache_init |
| | 683 | |
| | 684 | /* |
| | 685 | * On AArch64, the cache coherency is handled via the set_pte_at() function. |
| | 686 | */ |
| | 687 | static inline void update_mmu_cache(struct vm_area_struct *vma, |
| | 688 | unsigned long addr, pte_t *ptep) |
| | 689 | { |
| | 690 | /* |
| | 691 | * We don't do anything here, so there's a very small chance of |
| | 692 | * us retaking a user fault which we just fixed up. The alternative |
| | 693 | * is doing a dsb(ishst), but that penalises the fastpath. |
| | 694 | */ |
| | 695 | } |
| | 696 | |
| | 697 | #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) |
| | 698 | |
| | 699 | #define kc_vaddr_to_offset(v) ((v) & ~VA_START) |
| | 700 | #define kc_offset_to_vaddr(o) ((o) | VA_START) |
| | 701 | |
| | 702 | #endif /* !__ASSEMBLY__ */ |
| | 703 | |
| | 704 | #endif /* __ASM_PGTABLE_H */ |
projects / deliverable / linux.git / blame_incremental