4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns.
8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable.
13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn.
24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it.
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
41 #include <linux/sched.h>
42 #include <linux/highmem.h>
43 #include <linux/debugfs.h>
44 #include <linux/bug.h>
45 #include <linux/vmalloc.h>
46 #include <linux/module.h>
47 #include <linux/gfp.h>
48 #include <linux/memblock.h>
49 #include <linux/seq_file.h>
50 #include <linux/crash_dump.h>
52 #include <trace/events/xen.h>
54 #include <asm/pgtable.h>
55 #include <asm/tlbflush.h>
56 #include <asm/fixmap.h>
57 #include <asm/mmu_context.h>
58 #include <asm/setup.h>
59 #include <asm/paravirt.h>
61 #include <asm/linkage.h>
67 #include <asm/xen/hypercall.h>
68 #include <asm/xen/hypervisor.h>
72 #include <xen/interface/xen.h>
73 #include <xen/interface/hvm/hvm_op.h>
74 #include <xen/interface/version.h>
75 #include <xen/interface/memory.h>
76 #include <xen/hvc-console.h>
78 #include "multicalls.h"
83 * Protects atomic reservation decrease/increase against concurrent increases.
84 * Also protects non-atomic updates of current_pages and balloon lists.
86 DEFINE_SPINLOCK(xen_reservation_lock
);
90 * Identity map, in addition to plain kernel map. This needs to be
91 * large enough to allocate page table pages to allocate the rest.
92 * Each page can map 2MB.
94 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
95 static RESERVE_BRK_ARRAY(pte_t
, level1_ident_pgt
, LEVEL1_IDENT_ENTRIES
);
98 /* l3 pud for userspace vsyscall mapping */
99 static pud_t level3_user_vsyscall
[PTRS_PER_PUD
] __page_aligned_bss
;
100 #endif /* CONFIG_X86_64 */
103 * Note about cr3 (pagetable base) values:
105 * xen_cr3 contains the current logical cr3 value; it contains the
106 * last set cr3. This may not be the current effective cr3, because
107 * its update may be being lazily deferred. However, a vcpu looking
108 * at its own cr3 can use this value knowing that it everything will
109 * be self-consistent.
111 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
112 * hypercall to set the vcpu cr3 is complete (so it may be a little
113 * out of date, but it will never be set early). If one vcpu is
114 * looking at another vcpu's cr3 value, it should use this variable.
116 DEFINE_PER_CPU(unsigned long, xen_cr3
); /* cr3 stored as physaddr */
117 DEFINE_PER_CPU(unsigned long, xen_current_cr3
); /* actual vcpu cr3 */
121 * Just beyond the highest usermode address. STACK_TOP_MAX has a
122 * redzone above it, so round it up to a PGD boundary.
124 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
126 unsigned long arbitrary_virt_to_mfn(void *vaddr
)
128 xmaddr_t maddr
= arbitrary_virt_to_machine(vaddr
);
130 return PFN_DOWN(maddr
.maddr
);
133 xmaddr_t
arbitrary_virt_to_machine(void *vaddr
)
135 unsigned long address
= (unsigned long)vaddr
;
141 * if the PFN is in the linear mapped vaddr range, we can just use
142 * the (quick) virt_to_machine() p2m lookup
144 if (virt_addr_valid(vaddr
))
145 return virt_to_machine(vaddr
);
147 /* otherwise we have to do a (slower) full page-table walk */
149 pte
= lookup_address(address
, &level
);
151 offset
= address
& ~PAGE_MASK
;
152 return XMADDR(((phys_addr_t
)pte_mfn(*pte
) << PAGE_SHIFT
) + offset
);
154 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine
);
156 void make_lowmem_page_readonly(void *vaddr
)
159 unsigned long address
= (unsigned long)vaddr
;
162 pte
= lookup_address(address
, &level
);
164 return; /* vaddr missing */
166 ptev
= pte_wrprotect(*pte
);
168 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
172 void make_lowmem_page_readwrite(void *vaddr
)
175 unsigned long address
= (unsigned long)vaddr
;
178 pte
= lookup_address(address
, &level
);
180 return; /* vaddr missing */
182 ptev
= pte_mkwrite(*pte
);
184 if (HYPERVISOR_update_va_mapping(address
, ptev
, 0))
189 static bool xen_page_pinned(void *ptr
)
191 struct page
*page
= virt_to_page(ptr
);
193 return PagePinned(page
);
196 void xen_set_domain_pte(pte_t
*ptep
, pte_t pteval
, unsigned domid
)
198 struct multicall_space mcs
;
199 struct mmu_update
*u
;
201 trace_xen_mmu_set_domain_pte(ptep
, pteval
, domid
);
203 mcs
= xen_mc_entry(sizeof(*u
));
206 /* ptep might be kmapped when using 32-bit HIGHPTE */
207 u
->ptr
= virt_to_machine(ptep
).maddr
;
208 u
->val
= pte_val_ma(pteval
);
210 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, domid
);
212 xen_mc_issue(PARAVIRT_LAZY_MMU
);
214 EXPORT_SYMBOL_GPL(xen_set_domain_pte
);
216 static void xen_extend_mmu_update(const struct mmu_update
*update
)
218 struct multicall_space mcs
;
219 struct mmu_update
*u
;
221 mcs
= xen_mc_extend_args(__HYPERVISOR_mmu_update
, sizeof(*u
));
223 if (mcs
.mc
!= NULL
) {
226 mcs
= __xen_mc_entry(sizeof(*u
));
227 MULTI_mmu_update(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
234 static void xen_extend_mmuext_op(const struct mmuext_op
*op
)
236 struct multicall_space mcs
;
239 mcs
= xen_mc_extend_args(__HYPERVISOR_mmuext_op
, sizeof(*u
));
241 if (mcs
.mc
!= NULL
) {
244 mcs
= __xen_mc_entry(sizeof(*u
));
245 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
252 static void xen_set_pmd_hyper(pmd_t
*ptr
, pmd_t val
)
260 /* ptr may be ioremapped for 64-bit pagetable setup */
261 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
262 u
.val
= pmd_val_ma(val
);
263 xen_extend_mmu_update(&u
);
265 xen_mc_issue(PARAVIRT_LAZY_MMU
);
270 static void xen_set_pmd(pmd_t
*ptr
, pmd_t val
)
272 trace_xen_mmu_set_pmd(ptr
, val
);
274 /* If page is not pinned, we can just update the entry
276 if (!xen_page_pinned(ptr
)) {
281 xen_set_pmd_hyper(ptr
, val
);
285 * Associate a virtual page frame with a given physical page frame
286 * and protection flags for that frame.
288 void set_pte_mfn(unsigned long vaddr
, unsigned long mfn
, pgprot_t flags
)
290 set_pte_vaddr(vaddr
, mfn_pte(mfn
, flags
));
293 static bool xen_batched_set_pte(pte_t
*ptep
, pte_t pteval
)
297 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU
)
302 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
303 u
.val
= pte_val_ma(pteval
);
304 xen_extend_mmu_update(&u
);
306 xen_mc_issue(PARAVIRT_LAZY_MMU
);
311 static inline void __xen_set_pte(pte_t
*ptep
, pte_t pteval
)
313 if (!xen_batched_set_pte(ptep
, pteval
)) {
315 * Could call native_set_pte() here and trap and
316 * emulate the PTE write but with 32-bit guests this
317 * needs two traps (one for each of the two 32-bit
318 * words in the PTE) so do one hypercall directly
323 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_NORMAL_PT_UPDATE
;
324 u
.val
= pte_val_ma(pteval
);
325 HYPERVISOR_mmu_update(&u
, 1, NULL
, DOMID_SELF
);
329 static void xen_set_pte(pte_t
*ptep
, pte_t pteval
)
331 trace_xen_mmu_set_pte(ptep
, pteval
);
332 __xen_set_pte(ptep
, pteval
);
335 static void xen_set_pte_at(struct mm_struct
*mm
, unsigned long addr
,
336 pte_t
*ptep
, pte_t pteval
)
338 trace_xen_mmu_set_pte_at(mm
, addr
, ptep
, pteval
);
339 __xen_set_pte(ptep
, pteval
);
342 pte_t
xen_ptep_modify_prot_start(struct mm_struct
*mm
,
343 unsigned long addr
, pte_t
*ptep
)
345 /* Just return the pte as-is. We preserve the bits on commit */
346 trace_xen_mmu_ptep_modify_prot_start(mm
, addr
, ptep
, *ptep
);
350 void xen_ptep_modify_prot_commit(struct mm_struct
*mm
, unsigned long addr
,
351 pte_t
*ptep
, pte_t pte
)
355 trace_xen_mmu_ptep_modify_prot_commit(mm
, addr
, ptep
, pte
);
358 u
.ptr
= virt_to_machine(ptep
).maddr
| MMU_PT_UPDATE_PRESERVE_AD
;
359 u
.val
= pte_val_ma(pte
);
360 xen_extend_mmu_update(&u
);
362 xen_mc_issue(PARAVIRT_LAZY_MMU
);
365 /* Assume pteval_t is equivalent to all the other *val_t types. */
366 static pteval_t
pte_mfn_to_pfn(pteval_t val
)
368 if (val
& _PAGE_PRESENT
) {
369 unsigned long mfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
370 unsigned long pfn
= mfn_to_pfn(mfn
);
372 pteval_t flags
= val
& PTE_FLAGS_MASK
;
373 if (unlikely(pfn
== ~0))
374 val
= flags
& ~_PAGE_PRESENT
;
376 val
= ((pteval_t
)pfn
<< PAGE_SHIFT
) | flags
;
382 static pteval_t
pte_pfn_to_mfn(pteval_t val
)
384 if (val
& _PAGE_PRESENT
) {
385 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
386 pteval_t flags
= val
& PTE_FLAGS_MASK
;
389 if (!xen_feature(XENFEAT_auto_translated_physmap
))
390 mfn
= get_phys_to_machine(pfn
);
394 * If there's no mfn for the pfn, then just create an
395 * empty non-present pte. Unfortunately this loses
396 * information about the original pfn, so
397 * pte_mfn_to_pfn is asymmetric.
399 if (unlikely(mfn
== INVALID_P2M_ENTRY
)) {
404 * Paramount to do this test _after_ the
405 * INVALID_P2M_ENTRY as INVALID_P2M_ENTRY &
406 * IDENTITY_FRAME_BIT resolves to true.
408 mfn
&= ~FOREIGN_FRAME_BIT
;
409 if (mfn
& IDENTITY_FRAME_BIT
) {
410 mfn
&= ~IDENTITY_FRAME_BIT
;
411 flags
|= _PAGE_IOMAP
;
414 val
= ((pteval_t
)mfn
<< PAGE_SHIFT
) | flags
;
420 static pteval_t
iomap_pte(pteval_t val
)
422 if (val
& _PAGE_PRESENT
) {
423 unsigned long pfn
= (val
& PTE_PFN_MASK
) >> PAGE_SHIFT
;
424 pteval_t flags
= val
& PTE_FLAGS_MASK
;
426 /* We assume the pte frame number is a MFN, so
427 just use it as-is. */
428 val
= ((pteval_t
)pfn
<< PAGE_SHIFT
) | flags
;
434 static pteval_t
xen_pte_val(pte_t pte
)
436 pteval_t pteval
= pte
.pte
;
438 /* If this is a WC pte, convert back from Xen WC to Linux WC */
439 if ((pteval
& (_PAGE_PAT
| _PAGE_PCD
| _PAGE_PWT
)) == _PAGE_PAT
) {
440 WARN_ON(!pat_enabled
);
441 pteval
= (pteval
& ~_PAGE_PAT
) | _PAGE_PWT
;
444 if (xen_initial_domain() && (pteval
& _PAGE_IOMAP
))
447 return pte_mfn_to_pfn(pteval
);
449 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val
);
451 static pgdval_t
xen_pgd_val(pgd_t pgd
)
453 return pte_mfn_to_pfn(pgd
.pgd
);
455 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val
);
458 * Xen's PAT setup is part of its ABI, though I assume entries 6 & 7
459 * are reserved for now, to correspond to the Intel-reserved PAT
462 * We expect Linux's PAT set as follows:
464 * Idx PTE flags Linux Xen Default
471 * 6 PAT PCD UC- UC UC-
472 * 7 PAT PCD PWT UC UC UC
475 void xen_set_pat(u64 pat
)
477 /* We expect Linux to use a PAT setting of
478 * UC UC- WC WB (ignoring the PAT flag) */
479 WARN_ON(pat
!= 0x0007010600070106ull
);
482 static pte_t
xen_make_pte(pteval_t pte
)
484 phys_addr_t addr
= (pte
& PTE_PFN_MASK
);
486 /* If Linux is trying to set a WC pte, then map to the Xen WC.
487 * If _PAGE_PAT is set, then it probably means it is really
488 * _PAGE_PSE, so avoid fiddling with the PAT mapping and hope
489 * things work out OK...
491 * (We should never see kernel mappings with _PAGE_PSE set,
492 * but we could see hugetlbfs mappings, I think.).
494 if (pat_enabled
&& !WARN_ON(pte
& _PAGE_PAT
)) {
495 if ((pte
& (_PAGE_PCD
| _PAGE_PWT
)) == _PAGE_PWT
)
496 pte
= (pte
& ~(_PAGE_PCD
| _PAGE_PWT
)) | _PAGE_PAT
;
500 * Unprivileged domains are allowed to do IOMAPpings for
501 * PCI passthrough, but not map ISA space. The ISA
502 * mappings are just dummy local mappings to keep other
503 * parts of the kernel happy.
505 if (unlikely(pte
& _PAGE_IOMAP
) &&
506 (xen_initial_domain() || addr
>= ISA_END_ADDRESS
)) {
507 pte
= iomap_pte(pte
);
510 pte
= pte_pfn_to_mfn(pte
);
513 return native_make_pte(pte
);
515 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte
);
517 static pgd_t
xen_make_pgd(pgdval_t pgd
)
519 pgd
= pte_pfn_to_mfn(pgd
);
520 return native_make_pgd(pgd
);
522 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd
);
524 static pmdval_t
xen_pmd_val(pmd_t pmd
)
526 return pte_mfn_to_pfn(pmd
.pmd
);
528 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val
);
530 static void xen_set_pud_hyper(pud_t
*ptr
, pud_t val
)
538 /* ptr may be ioremapped for 64-bit pagetable setup */
539 u
.ptr
= arbitrary_virt_to_machine(ptr
).maddr
;
540 u
.val
= pud_val_ma(val
);
541 xen_extend_mmu_update(&u
);
543 xen_mc_issue(PARAVIRT_LAZY_MMU
);
548 static void xen_set_pud(pud_t
*ptr
, pud_t val
)
550 trace_xen_mmu_set_pud(ptr
, val
);
552 /* If page is not pinned, we can just update the entry
554 if (!xen_page_pinned(ptr
)) {
559 xen_set_pud_hyper(ptr
, val
);
562 #ifdef CONFIG_X86_PAE
563 static void xen_set_pte_atomic(pte_t
*ptep
, pte_t pte
)
565 trace_xen_mmu_set_pte_atomic(ptep
, pte
);
566 set_64bit((u64
*)ptep
, native_pte_val(pte
));
569 static void xen_pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
571 trace_xen_mmu_pte_clear(mm
, addr
, ptep
);
572 if (!xen_batched_set_pte(ptep
, native_make_pte(0)))
573 native_pte_clear(mm
, addr
, ptep
);
576 static void xen_pmd_clear(pmd_t
*pmdp
)
578 trace_xen_mmu_pmd_clear(pmdp
);
579 set_pmd(pmdp
, __pmd(0));
581 #endif /* CONFIG_X86_PAE */
583 static pmd_t
xen_make_pmd(pmdval_t pmd
)
585 pmd
= pte_pfn_to_mfn(pmd
);
586 return native_make_pmd(pmd
);
588 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd
);
590 #if PAGETABLE_LEVELS == 4
591 static pudval_t
xen_pud_val(pud_t pud
)
593 return pte_mfn_to_pfn(pud
.pud
);
595 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val
);
597 static pud_t
xen_make_pud(pudval_t pud
)
599 pud
= pte_pfn_to_mfn(pud
);
601 return native_make_pud(pud
);
603 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud
);
605 static pgd_t
*xen_get_user_pgd(pgd_t
*pgd
)
607 pgd_t
*pgd_page
= (pgd_t
*)(((unsigned long)pgd
) & PAGE_MASK
);
608 unsigned offset
= pgd
- pgd_page
;
609 pgd_t
*user_ptr
= NULL
;
611 if (offset
< pgd_index(USER_LIMIT
)) {
612 struct page
*page
= virt_to_page(pgd_page
);
613 user_ptr
= (pgd_t
*)page
->private;
621 static void __xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
625 u
.ptr
= virt_to_machine(ptr
).maddr
;
626 u
.val
= pgd_val_ma(val
);
627 xen_extend_mmu_update(&u
);
631 * Raw hypercall-based set_pgd, intended for in early boot before
632 * there's a page structure. This implies:
633 * 1. The only existing pagetable is the kernel's
634 * 2. It is always pinned
635 * 3. It has no user pagetable attached to it
637 static void __init
xen_set_pgd_hyper(pgd_t
*ptr
, pgd_t val
)
643 __xen_set_pgd_hyper(ptr
, val
);
645 xen_mc_issue(PARAVIRT_LAZY_MMU
);
650 static void xen_set_pgd(pgd_t
*ptr
, pgd_t val
)
652 pgd_t
*user_ptr
= xen_get_user_pgd(ptr
);
654 trace_xen_mmu_set_pgd(ptr
, user_ptr
, val
);
656 /* If page is not pinned, we can just update the entry
658 if (!xen_page_pinned(ptr
)) {
661 WARN_ON(xen_page_pinned(user_ptr
));
667 /* If it's pinned, then we can at least batch the kernel and
668 user updates together. */
671 __xen_set_pgd_hyper(ptr
, val
);
673 __xen_set_pgd_hyper(user_ptr
, val
);
675 xen_mc_issue(PARAVIRT_LAZY_MMU
);
677 #endif /* PAGETABLE_LEVELS == 4 */
680 * (Yet another) pagetable walker. This one is intended for pinning a
681 * pagetable. This means that it walks a pagetable and calls the
682 * callback function on each page it finds making up the page table,
683 * at every level. It walks the entire pagetable, but it only bothers
684 * pinning pte pages which are below limit. In the normal case this
685 * will be STACK_TOP_MAX, but at boot we need to pin up to
688 * For 32-bit the important bit is that we don't pin beyond there,
689 * because then we start getting into Xen's ptes.
691 * For 64-bit, we must skip the Xen hole in the middle of the address
692 * space, just after the big x86-64 virtual hole.
694 static int __xen_pgd_walk(struct mm_struct
*mm
, pgd_t
*pgd
,
695 int (*func
)(struct mm_struct
*mm
, struct page
*,
700 unsigned hole_low
, hole_high
;
701 unsigned pgdidx_limit
, pudidx_limit
, pmdidx_limit
;
702 unsigned pgdidx
, pudidx
, pmdidx
;
704 /* The limit is the last byte to be touched */
706 BUG_ON(limit
>= FIXADDR_TOP
);
708 if (xen_feature(XENFEAT_auto_translated_physmap
))
712 * 64-bit has a great big hole in the middle of the address
713 * space, which contains the Xen mappings. On 32-bit these
714 * will end up making a zero-sized hole and so is a no-op.
716 hole_low
= pgd_index(USER_LIMIT
);
717 hole_high
= pgd_index(PAGE_OFFSET
);
719 pgdidx_limit
= pgd_index(limit
);
721 pudidx_limit
= pud_index(limit
);
726 pmdidx_limit
= pmd_index(limit
);
731 for (pgdidx
= 0; pgdidx
<= pgdidx_limit
; pgdidx
++) {
734 if (pgdidx
>= hole_low
&& pgdidx
< hole_high
)
737 if (!pgd_val(pgd
[pgdidx
]))
740 pud
= pud_offset(&pgd
[pgdidx
], 0);
742 if (PTRS_PER_PUD
> 1) /* not folded */
743 flush
|= (*func
)(mm
, virt_to_page(pud
), PT_PUD
);
745 for (pudidx
= 0; pudidx
< PTRS_PER_PUD
; pudidx
++) {
748 if (pgdidx
== pgdidx_limit
&&
749 pudidx
> pudidx_limit
)
752 if (pud_none(pud
[pudidx
]))
755 pmd
= pmd_offset(&pud
[pudidx
], 0);
757 if (PTRS_PER_PMD
> 1) /* not folded */
758 flush
|= (*func
)(mm
, virt_to_page(pmd
), PT_PMD
);
760 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
; pmdidx
++) {
763 if (pgdidx
== pgdidx_limit
&&
764 pudidx
== pudidx_limit
&&
765 pmdidx
> pmdidx_limit
)
768 if (pmd_none(pmd
[pmdidx
]))
771 pte
= pmd_page(pmd
[pmdidx
]);
772 flush
|= (*func
)(mm
, pte
, PT_PTE
);
778 /* Do the top level last, so that the callbacks can use it as
779 a cue to do final things like tlb flushes. */
780 flush
|= (*func
)(mm
, virt_to_page(pgd
), PT_PGD
);
785 static int xen_pgd_walk(struct mm_struct
*mm
,
786 int (*func
)(struct mm_struct
*mm
, struct page
*,
790 return __xen_pgd_walk(mm
, mm
->pgd
, func
, limit
);
793 /* If we're using split pte locks, then take the page's lock and
794 return a pointer to it. Otherwise return NULL. */
795 static spinlock_t
*xen_pte_lock(struct page
*page
, struct mm_struct
*mm
)
797 spinlock_t
*ptl
= NULL
;
799 #if USE_SPLIT_PTLOCKS
800 ptl
= __pte_lockptr(page
);
801 spin_lock_nest_lock(ptl
, &mm
->page_table_lock
);
807 static void xen_pte_unlock(void *v
)
813 static void xen_do_pin(unsigned level
, unsigned long pfn
)
818 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
820 xen_extend_mmuext_op(&op
);
823 static int xen_pin_page(struct mm_struct
*mm
, struct page
*page
,
826 unsigned pgfl
= TestSetPagePinned(page
);
830 flush
= 0; /* already pinned */
831 else if (PageHighMem(page
))
832 /* kmaps need flushing if we found an unpinned
836 void *pt
= lowmem_page_address(page
);
837 unsigned long pfn
= page_to_pfn(page
);
838 struct multicall_space mcs
= __xen_mc_entry(0);
844 * We need to hold the pagetable lock between the time
845 * we make the pagetable RO and when we actually pin
846 * it. If we don't, then other users may come in and
847 * attempt to update the pagetable by writing it,
848 * which will fail because the memory is RO but not
849 * pinned, so Xen won't do the trap'n'emulate.
851 * If we're using split pte locks, we can't hold the
852 * entire pagetable's worth of locks during the
853 * traverse, because we may wrap the preempt count (8
854 * bits). The solution is to mark RO and pin each PTE
855 * page while holding the lock. This means the number
856 * of locks we end up holding is never more than a
857 * batch size (~32 entries, at present).
859 * If we're not using split pte locks, we needn't pin
860 * the PTE pages independently, because we're
861 * protected by the overall pagetable lock.
865 ptl
= xen_pte_lock(page
, mm
);
867 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
868 pfn_pte(pfn
, PAGE_KERNEL_RO
),
869 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
872 xen_do_pin(MMUEXT_PIN_L1_TABLE
, pfn
);
874 /* Queue a deferred unlock for when this batch
876 xen_mc_callback(xen_pte_unlock
, ptl
);
883 /* This is called just after a mm has been created, but it has not
884 been used yet. We need to make sure that its pagetable is all
885 read-only, and can be pinned. */
886 static void __xen_pgd_pin(struct mm_struct
*mm
, pgd_t
*pgd
)
888 trace_xen_mmu_pgd_pin(mm
, pgd
);
892 if (__xen_pgd_walk(mm
, pgd
, xen_pin_page
, USER_LIMIT
)) {
893 /* re-enable interrupts for flushing */
903 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
905 xen_do_pin(MMUEXT_PIN_L4_TABLE
, PFN_DOWN(__pa(pgd
)));
908 xen_pin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
909 xen_do_pin(MMUEXT_PIN_L4_TABLE
,
910 PFN_DOWN(__pa(user_pgd
)));
913 #else /* CONFIG_X86_32 */
914 #ifdef CONFIG_X86_PAE
915 /* Need to make sure unshared kernel PMD is pinnable */
916 xen_pin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
919 xen_do_pin(MMUEXT_PIN_L3_TABLE
, PFN_DOWN(__pa(pgd
)));
920 #endif /* CONFIG_X86_64 */
924 static void xen_pgd_pin(struct mm_struct
*mm
)
926 __xen_pgd_pin(mm
, mm
->pgd
);
930 * On save, we need to pin all pagetables to make sure they get their
931 * mfns turned into pfns. Search the list for any unpinned pgds and pin
932 * them (unpinned pgds are not currently in use, probably because the
933 * process is under construction or destruction).
935 * Expected to be called in stop_machine() ("equivalent to taking
936 * every spinlock in the system"), so the locking doesn't really
937 * matter all that much.
939 void xen_mm_pin_all(void)
943 spin_lock(&pgd_lock
);
945 list_for_each_entry(page
, &pgd_list
, lru
) {
946 if (!PagePinned(page
)) {
947 __xen_pgd_pin(&init_mm
, (pgd_t
*)page_address(page
));
948 SetPageSavePinned(page
);
952 spin_unlock(&pgd_lock
);
956 * The init_mm pagetable is really pinned as soon as its created, but
957 * that's before we have page structures to store the bits. So do all
958 * the book-keeping now.
960 static int __init
xen_mark_pinned(struct mm_struct
*mm
, struct page
*page
,
967 static void __init
xen_mark_init_mm_pinned(void)
969 xen_pgd_walk(&init_mm
, xen_mark_pinned
, FIXADDR_TOP
);
972 static int xen_unpin_page(struct mm_struct
*mm
, struct page
*page
,
975 unsigned pgfl
= TestClearPagePinned(page
);
977 if (pgfl
&& !PageHighMem(page
)) {
978 void *pt
= lowmem_page_address(page
);
979 unsigned long pfn
= page_to_pfn(page
);
980 spinlock_t
*ptl
= NULL
;
981 struct multicall_space mcs
;
984 * Do the converse to pin_page. If we're using split
985 * pte locks, we must be holding the lock for while
986 * the pte page is unpinned but still RO to prevent
987 * concurrent updates from seeing it in this
988 * partially-pinned state.
990 if (level
== PT_PTE
) {
991 ptl
= xen_pte_lock(page
, mm
);
994 xen_do_pin(MMUEXT_UNPIN_TABLE
, pfn
);
997 mcs
= __xen_mc_entry(0);
999 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)pt
,
1000 pfn_pte(pfn
, PAGE_KERNEL
),
1001 level
== PT_PGD
? UVMF_TLB_FLUSH
: 0);
1004 /* unlock when batch completed */
1005 xen_mc_callback(xen_pte_unlock
, ptl
);
1009 return 0; /* never need to flush on unpin */
1012 /* Release a pagetables pages back as normal RW */
1013 static void __xen_pgd_unpin(struct mm_struct
*mm
, pgd_t
*pgd
)
1015 trace_xen_mmu_pgd_unpin(mm
, pgd
);
1019 xen_do_pin(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1021 #ifdef CONFIG_X86_64
1023 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1026 xen_do_pin(MMUEXT_UNPIN_TABLE
,
1027 PFN_DOWN(__pa(user_pgd
)));
1028 xen_unpin_page(mm
, virt_to_page(user_pgd
), PT_PGD
);
1033 #ifdef CONFIG_X86_PAE
1034 /* Need to make sure unshared kernel PMD is unpinned */
1035 xen_unpin_page(mm
, pgd_page(pgd
[pgd_index(TASK_SIZE
)]),
1039 __xen_pgd_walk(mm
, pgd
, xen_unpin_page
, USER_LIMIT
);
1044 static void xen_pgd_unpin(struct mm_struct
*mm
)
1046 __xen_pgd_unpin(mm
, mm
->pgd
);
1050 * On resume, undo any pinning done at save, so that the rest of the
1051 * kernel doesn't see any unexpected pinned pagetables.
1053 void xen_mm_unpin_all(void)
1057 spin_lock(&pgd_lock
);
1059 list_for_each_entry(page
, &pgd_list
, lru
) {
1060 if (PageSavePinned(page
)) {
1061 BUG_ON(!PagePinned(page
));
1062 __xen_pgd_unpin(&init_mm
, (pgd_t
*)page_address(page
));
1063 ClearPageSavePinned(page
);
1067 spin_unlock(&pgd_lock
);
1070 static void xen_activate_mm(struct mm_struct
*prev
, struct mm_struct
*next
)
1072 spin_lock(&next
->page_table_lock
);
1074 spin_unlock(&next
->page_table_lock
);
1077 static void xen_dup_mmap(struct mm_struct
*oldmm
, struct mm_struct
*mm
)
1079 spin_lock(&mm
->page_table_lock
);
1081 spin_unlock(&mm
->page_table_lock
);
1086 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1087 we need to repoint it somewhere else before we can unpin it. */
1088 static void drop_other_mm_ref(void *info
)
1090 struct mm_struct
*mm
= info
;
1091 struct mm_struct
*active_mm
;
1093 active_mm
= this_cpu_read(cpu_tlbstate
.active_mm
);
1095 if (active_mm
== mm
&& this_cpu_read(cpu_tlbstate
.state
) != TLBSTATE_OK
)
1096 leave_mm(smp_processor_id());
1098 /* If this cpu still has a stale cr3 reference, then make sure
1099 it has been flushed. */
1100 if (this_cpu_read(xen_current_cr3
) == __pa(mm
->pgd
))
1101 load_cr3(swapper_pg_dir
);
1104 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1109 if (current
->active_mm
== mm
) {
1110 if (current
->mm
== mm
)
1111 load_cr3(swapper_pg_dir
);
1113 leave_mm(smp_processor_id());
1116 /* Get the "official" set of cpus referring to our pagetable. */
1117 if (!alloc_cpumask_var(&mask
, GFP_ATOMIC
)) {
1118 for_each_online_cpu(cpu
) {
1119 if (!cpumask_test_cpu(cpu
, mm_cpumask(mm
))
1120 && per_cpu(xen_current_cr3
, cpu
) != __pa(mm
->pgd
))
1122 smp_call_function_single(cpu
, drop_other_mm_ref
, mm
, 1);
1126 cpumask_copy(mask
, mm_cpumask(mm
));
1128 /* It's possible that a vcpu may have a stale reference to our
1129 cr3, because its in lazy mode, and it hasn't yet flushed
1130 its set of pending hypercalls yet. In this case, we can
1131 look at its actual current cr3 value, and force it to flush
1133 for_each_online_cpu(cpu
) {
1134 if (per_cpu(xen_current_cr3
, cpu
) == __pa(mm
->pgd
))
1135 cpumask_set_cpu(cpu
, mask
);
1138 if (!cpumask_empty(mask
))
1139 smp_call_function_many(mask
, drop_other_mm_ref
, mm
, 1);
1140 free_cpumask_var(mask
);
1143 static void xen_drop_mm_ref(struct mm_struct
*mm
)
1145 if (current
->active_mm
== mm
)
1146 load_cr3(swapper_pg_dir
);
1151 * While a process runs, Xen pins its pagetables, which means that the
1152 * hypervisor forces it to be read-only, and it controls all updates
1153 * to it. This means that all pagetable updates have to go via the
1154 * hypervisor, which is moderately expensive.
1156 * Since we're pulling the pagetable down, we switch to use init_mm,
1157 * unpin old process pagetable and mark it all read-write, which
1158 * allows further operations on it to be simple memory accesses.
1160 * The only subtle point is that another CPU may be still using the
1161 * pagetable because of lazy tlb flushing. This means we need need to
1162 * switch all CPUs off this pagetable before we can unpin it.
1164 static void xen_exit_mmap(struct mm_struct
*mm
)
1166 get_cpu(); /* make sure we don't move around */
1167 xen_drop_mm_ref(mm
);
1170 spin_lock(&mm
->page_table_lock
);
1172 /* pgd may not be pinned in the error exit path of execve */
1173 if (xen_page_pinned(mm
->pgd
))
1176 spin_unlock(&mm
->page_table_lock
);
1179 static void xen_post_allocator_init(void);
1181 static __init
void xen_mapping_pagetable_reserve(u64 start
, u64 end
)
1183 /* reserve the range used */
1184 native_pagetable_reserve(start
, end
);
1186 /* set as RW the rest */
1187 printk(KERN_DEBUG
"xen: setting RW the range %llx - %llx\n", end
,
1188 PFN_PHYS(pgt_buf_top
));
1189 while (end
< PFN_PHYS(pgt_buf_top
)) {
1190 make_lowmem_page_readwrite(__va(end
));
1195 #ifdef CONFIG_X86_64
1196 static void __init
xen_cleanhighmap(unsigned long vaddr
,
1197 unsigned long vaddr_end
)
1199 unsigned long kernel_end
= roundup((unsigned long)_brk_end
, PMD_SIZE
) - 1;
1200 pmd_t
*pmd
= level2_kernel_pgt
+ pmd_index(vaddr
);
1202 /* NOTE: The loop is more greedy than the cleanup_highmap variant.
1203 * We include the PMD passed in on _both_ boundaries. */
1204 for (; vaddr
<= vaddr_end
&& (pmd
< (level2_kernel_pgt
+ PAGE_SIZE
));
1205 pmd
++, vaddr
+= PMD_SIZE
) {
1208 if (vaddr
< (unsigned long) _text
|| vaddr
> kernel_end
)
1209 set_pmd(pmd
, __pmd(0));
1211 /* In case we did something silly, we should crash in this function
1212 * instead of somewhere later and be confusing. */
1216 static void __init
xen_pagetable_init(void)
1218 #ifdef CONFIG_X86_64
1223 xen_setup_shared_info();
1224 #ifdef CONFIG_X86_64
1225 if (!xen_feature(XENFEAT_auto_translated_physmap
)) {
1226 unsigned long new_mfn_list
;
1228 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
1230 /* On 32-bit, we get zero so this never gets executed. */
1231 new_mfn_list
= xen_revector_p2m_tree();
1232 if (new_mfn_list
&& new_mfn_list
!= xen_start_info
->mfn_list
) {
1233 /* using __ka address and sticking INVALID_P2M_ENTRY! */
1234 memset((void *)xen_start_info
->mfn_list
, 0xff, size
);
1236 /* We should be in __ka space. */
1237 BUG_ON(xen_start_info
->mfn_list
< __START_KERNEL_map
);
1238 addr
= xen_start_info
->mfn_list
;
1239 /* We roundup to the PMD, which means that if anybody at this stage is
1240 * using the __ka address of xen_start_info or xen_start_info->shared_info
1241 * they are in going to crash. Fortunatly we have already revectored
1242 * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */
1243 size
= roundup(size
, PMD_SIZE
);
1244 xen_cleanhighmap(addr
, addr
+ size
);
1246 size
= PAGE_ALIGN(xen_start_info
->nr_pages
* sizeof(unsigned long));
1247 memblock_free(__pa(xen_start_info
->mfn_list
), size
);
1248 /* And revector! Bye bye old array */
1249 xen_start_info
->mfn_list
= new_mfn_list
;
1253 /* At this stage, cleanup_highmap has already cleaned __ka space
1254 * from _brk_limit way up to the max_pfn_mapped (which is the end of
1255 * the ramdisk). We continue on, erasing PMD entries that point to page
1256 * tables - do note that they are accessible at this stage via __va.
1257 * For good measure we also round up to the PMD - which means that if
1258 * anybody is using __ka address to the initial boot-stack - and try
1259 * to use it - they are going to crash. The xen_start_info has been
1260 * taken care of already in xen_setup_kernel_pagetable. */
1261 addr
= xen_start_info
->pt_base
;
1262 size
= roundup(xen_start_info
->nr_pt_frames
* PAGE_SIZE
, PMD_SIZE
);
1264 xen_cleanhighmap(addr
, addr
+ size
);
1265 xen_start_info
->pt_base
= (unsigned long)__va(__pa(xen_start_info
->pt_base
));
1267 /* This is superflous and is not neccessary, but you know what
1268 * lets do it. The MODULES_VADDR -> MODULES_END should be clear of
1269 * anything at this stage. */
1270 xen_cleanhighmap(MODULES_VADDR
, roundup(MODULES_VADDR
, PUD_SIZE
) - 1);
1274 xen_post_allocator_init();
1276 static void xen_write_cr2(unsigned long cr2
)
1278 this_cpu_read(xen_vcpu
)->arch
.cr2
= cr2
;
1281 static unsigned long xen_read_cr2(void)
1283 return this_cpu_read(xen_vcpu
)->arch
.cr2
;
1286 unsigned long xen_read_cr2_direct(void)
1288 return this_cpu_read(xen_vcpu_info
.arch
.cr2
);
1291 static void xen_flush_tlb(void)
1293 struct mmuext_op
*op
;
1294 struct multicall_space mcs
;
1296 trace_xen_mmu_flush_tlb(0);
1300 mcs
= xen_mc_entry(sizeof(*op
));
1303 op
->cmd
= MMUEXT_TLB_FLUSH_LOCAL
;
1304 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1306 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1311 static void xen_flush_tlb_single(unsigned long addr
)
1313 struct mmuext_op
*op
;
1314 struct multicall_space mcs
;
1316 trace_xen_mmu_flush_tlb_single(addr
);
1320 mcs
= xen_mc_entry(sizeof(*op
));
1322 op
->cmd
= MMUEXT_INVLPG_LOCAL
;
1323 op
->arg1
.linear_addr
= addr
& PAGE_MASK
;
1324 MULTI_mmuext_op(mcs
.mc
, op
, 1, NULL
, DOMID_SELF
);
1326 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1331 static void xen_flush_tlb_others(const struct cpumask
*cpus
,
1332 struct mm_struct
*mm
, unsigned long start
,
1336 struct mmuext_op op
;
1338 DECLARE_BITMAP(mask
, num_processors
);
1340 DECLARE_BITMAP(mask
, NR_CPUS
);
1343 struct multicall_space mcs
;
1345 trace_xen_mmu_flush_tlb_others(cpus
, mm
, start
, end
);
1347 if (cpumask_empty(cpus
))
1348 return; /* nothing to do */
1350 mcs
= xen_mc_entry(sizeof(*args
));
1352 args
->op
.arg2
.vcpumask
= to_cpumask(args
->mask
);
1354 /* Remove us, and any offline CPUS. */
1355 cpumask_and(to_cpumask(args
->mask
), cpus
, cpu_online_mask
);
1356 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args
->mask
));
1358 args
->op
.cmd
= MMUEXT_TLB_FLUSH_MULTI
;
1359 if (end
!= TLB_FLUSH_ALL
&& (end
- start
) <= PAGE_SIZE
) {
1360 args
->op
.cmd
= MMUEXT_INVLPG_MULTI
;
1361 args
->op
.arg1
.linear_addr
= start
;
1364 MULTI_mmuext_op(mcs
.mc
, &args
->op
, 1, NULL
, DOMID_SELF
);
1366 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1369 static unsigned long xen_read_cr3(void)
1371 return this_cpu_read(xen_cr3
);
1374 static void set_current_cr3(void *v
)
1376 this_cpu_write(xen_current_cr3
, (unsigned long)v
);
1379 static void __xen_write_cr3(bool kernel
, unsigned long cr3
)
1381 struct mmuext_op op
;
1384 trace_xen_mmu_write_cr3(kernel
, cr3
);
1387 mfn
= pfn_to_mfn(PFN_DOWN(cr3
));
1391 WARN_ON(mfn
== 0 && kernel
);
1393 op
.cmd
= kernel
? MMUEXT_NEW_BASEPTR
: MMUEXT_NEW_USER_BASEPTR
;
1396 xen_extend_mmuext_op(&op
);
1399 this_cpu_write(xen_cr3
, cr3
);
1401 /* Update xen_current_cr3 once the batch has actually
1403 xen_mc_callback(set_current_cr3
, (void *)cr3
);
1407 static void xen_write_cr3(unsigned long cr3
)
1409 BUG_ON(preemptible());
1411 xen_mc_batch(); /* disables interrupts */
1413 /* Update while interrupts are disabled, so its atomic with
1415 this_cpu_write(xen_cr3
, cr3
);
1417 __xen_write_cr3(true, cr3
);
1419 #ifdef CONFIG_X86_64
1421 pgd_t
*user_pgd
= xen_get_user_pgd(__va(cr3
));
1423 __xen_write_cr3(false, __pa(user_pgd
));
1425 __xen_write_cr3(false, 0);
1429 xen_mc_issue(PARAVIRT_LAZY_CPU
); /* interrupts restored */
1432 static int xen_pgd_alloc(struct mm_struct
*mm
)
1434 pgd_t
*pgd
= mm
->pgd
;
1437 BUG_ON(PagePinned(virt_to_page(pgd
)));
1439 #ifdef CONFIG_X86_64
1441 struct page
*page
= virt_to_page(pgd
);
1444 BUG_ON(page
->private != 0);
1448 user_pgd
= (pgd_t
*)__get_free_page(GFP_KERNEL
| __GFP_ZERO
);
1449 page
->private = (unsigned long)user_pgd
;
1451 if (user_pgd
!= NULL
) {
1452 user_pgd
[pgd_index(VSYSCALL_START
)] =
1453 __pgd(__pa(level3_user_vsyscall
) | _PAGE_TABLE
);
1457 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd
))));
1464 static void xen_pgd_free(struct mm_struct
*mm
, pgd_t
*pgd
)
1466 #ifdef CONFIG_X86_64
1467 pgd_t
*user_pgd
= xen_get_user_pgd(pgd
);
1470 free_page((unsigned long)user_pgd
);
1474 #ifdef CONFIG_X86_32
1475 static pte_t __init
mask_rw_pte(pte_t
*ptep
, pte_t pte
)
1477 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1478 if (pte_val_ma(*ptep
) & _PAGE_PRESENT
)
1479 pte
= __pte_ma(((pte_val_ma(*ptep
) & _PAGE_RW
) | ~_PAGE_RW
) &
1484 #else /* CONFIG_X86_64 */
1485 static pte_t __init
mask_rw_pte(pte_t
*ptep
, pte_t pte
)
1487 unsigned long pfn
= pte_pfn(pte
);
1490 * If the new pfn is within the range of the newly allocated
1491 * kernel pagetable, and it isn't being mapped into an
1492 * early_ioremap fixmap slot as a freshly allocated page, make sure
1495 if (((!is_early_ioremap_ptep(ptep
) &&
1496 pfn
>= pgt_buf_start
&& pfn
< pgt_buf_top
)) ||
1497 (is_early_ioremap_ptep(ptep
) && pfn
!= (pgt_buf_end
- 1)))
1498 pte
= pte_wrprotect(pte
);
1502 #endif /* CONFIG_X86_64 */
1505 * Init-time set_pte while constructing initial pagetables, which
1506 * doesn't allow RO page table pages to be remapped RW.
1508 * If there is no MFN for this PFN then this page is initially
1509 * ballooned out so clear the PTE (as in decrease_reservation() in
1510 * drivers/xen/balloon.c).
1512 * Many of these PTE updates are done on unpinned and writable pages
1513 * and doing a hypercall for these is unnecessary and expensive. At
1514 * this point it is not possible to tell if a page is pinned or not,
1515 * so always write the PTE directly and rely on Xen trapping and
1516 * emulating any updates as necessary.
1518 static void __init
xen_set_pte_init(pte_t
*ptep
, pte_t pte
)
1520 if (pte_mfn(pte
) != INVALID_P2M_ENTRY
)
1521 pte
= mask_rw_pte(ptep
, pte
);
1525 native_set_pte(ptep
, pte
);
1528 static void pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1530 struct mmuext_op op
;
1532 op
.arg1
.mfn
= pfn_to_mfn(pfn
);
1533 if (HYPERVISOR_mmuext_op(&op
, 1, NULL
, DOMID_SELF
))
1537 /* Early in boot, while setting up the initial pagetable, assume
1538 everything is pinned. */
1539 static void __init
xen_alloc_pte_init(struct mm_struct
*mm
, unsigned long pfn
)
1541 #ifdef CONFIG_FLATMEM
1542 BUG_ON(mem_map
); /* should only be used early */
1544 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1545 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1548 /* Used for pmd and pud */
1549 static void __init
xen_alloc_pmd_init(struct mm_struct
*mm
, unsigned long pfn
)
1551 #ifdef CONFIG_FLATMEM
1552 BUG_ON(mem_map
); /* should only be used early */
1554 make_lowmem_page_readonly(__va(PFN_PHYS(pfn
)));
1557 /* Early release_pte assumes that all pts are pinned, since there's
1558 only init_mm and anything attached to that is pinned. */
1559 static void __init
xen_release_pte_init(unsigned long pfn
)
1561 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1562 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1565 static void __init
xen_release_pmd_init(unsigned long pfn
)
1567 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn
)));
1570 static inline void __pin_pagetable_pfn(unsigned cmd
, unsigned long pfn
)
1572 struct multicall_space mcs
;
1573 struct mmuext_op
*op
;
1575 mcs
= __xen_mc_entry(sizeof(*op
));
1578 op
->arg1
.mfn
= pfn_to_mfn(pfn
);
1580 MULTI_mmuext_op(mcs
.mc
, mcs
.args
, 1, NULL
, DOMID_SELF
);
1583 static inline void __set_pfn_prot(unsigned long pfn
, pgprot_t prot
)
1585 struct multicall_space mcs
;
1586 unsigned long addr
= (unsigned long)__va(pfn
<< PAGE_SHIFT
);
1588 mcs
= __xen_mc_entry(0);
1589 MULTI_update_va_mapping(mcs
.mc
, (unsigned long)addr
,
1590 pfn_pte(pfn
, prot
), 0);
1593 /* This needs to make sure the new pte page is pinned iff its being
1594 attached to a pinned pagetable. */
1595 static inline void xen_alloc_ptpage(struct mm_struct
*mm
, unsigned long pfn
,
1598 bool pinned
= PagePinned(virt_to_page(mm
->pgd
));
1600 trace_xen_mmu_alloc_ptpage(mm
, pfn
, level
, pinned
);
1603 struct page
*page
= pfn_to_page(pfn
);
1605 SetPagePinned(page
);
1607 if (!PageHighMem(page
)) {
1610 __set_pfn_prot(pfn
, PAGE_KERNEL_RO
);
1612 if (level
== PT_PTE
&& USE_SPLIT_PTLOCKS
)
1613 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE
, pfn
);
1615 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1617 /* make sure there are no stray mappings of
1619 kmap_flush_unused();
1624 static void xen_alloc_pte(struct mm_struct
*mm
, unsigned long pfn
)
1626 xen_alloc_ptpage(mm
, pfn
, PT_PTE
);
1629 static void xen_alloc_pmd(struct mm_struct
*mm
, unsigned long pfn
)
1631 xen_alloc_ptpage(mm
, pfn
, PT_PMD
);
1634 /* This should never happen until we're OK to use struct page */
1635 static inline void xen_release_ptpage(unsigned long pfn
, unsigned level
)
1637 struct page
*page
= pfn_to_page(pfn
);
1638 bool pinned
= PagePinned(page
);
1640 trace_xen_mmu_release_ptpage(pfn
, level
, pinned
);
1643 if (!PageHighMem(page
)) {
1646 if (level
== PT_PTE
&& USE_SPLIT_PTLOCKS
)
1647 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, pfn
);
1649 __set_pfn_prot(pfn
, PAGE_KERNEL
);
1651 xen_mc_issue(PARAVIRT_LAZY_MMU
);
1653 ClearPagePinned(page
);
1657 static void xen_release_pte(unsigned long pfn
)
1659 xen_release_ptpage(pfn
, PT_PTE
);
1662 static void xen_release_pmd(unsigned long pfn
)
1664 xen_release_ptpage(pfn
, PT_PMD
);
1667 #if PAGETABLE_LEVELS == 4
1668 static void xen_alloc_pud(struct mm_struct
*mm
, unsigned long pfn
)
1670 xen_alloc_ptpage(mm
, pfn
, PT_PUD
);
1673 static void xen_release_pud(unsigned long pfn
)
1675 xen_release_ptpage(pfn
, PT_PUD
);
1679 void __init
xen_reserve_top(void)
1681 #ifdef CONFIG_X86_32
1682 unsigned long top
= HYPERVISOR_VIRT_START
;
1683 struct xen_platform_parameters pp
;
1685 if (HYPERVISOR_xen_version(XENVER_platform_parameters
, &pp
) == 0)
1686 top
= pp
.virt_start
;
1688 reserve_top_address(-top
);
1689 #endif /* CONFIG_X86_32 */
1693 * Like __va(), but returns address in the kernel mapping (which is
1694 * all we have until the physical memory mapping has been set up.
1696 static void *__ka(phys_addr_t paddr
)
1698 #ifdef CONFIG_X86_64
1699 return (void *)(paddr
+ __START_KERNEL_map
);
1705 /* Convert a machine address to physical address */
1706 static unsigned long m2p(phys_addr_t maddr
)
1710 maddr
&= PTE_PFN_MASK
;
1711 paddr
= mfn_to_pfn(maddr
>> PAGE_SHIFT
) << PAGE_SHIFT
;
1716 /* Convert a machine address to kernel virtual */
1717 static void *m2v(phys_addr_t maddr
)
1719 return __ka(m2p(maddr
));
1722 /* Set the page permissions on an identity-mapped pages */
1723 static void set_page_prot(void *addr
, pgprot_t prot
)
1725 unsigned long pfn
= __pa(addr
) >> PAGE_SHIFT
;
1726 pte_t pte
= pfn_pte(pfn
, prot
);
1728 if (HYPERVISOR_update_va_mapping((unsigned long)addr
, pte
, 0))
1731 #ifdef CONFIG_X86_32
1732 static void __init
xen_map_identity_early(pmd_t
*pmd
, unsigned long max_pfn
)
1734 unsigned pmdidx
, pteidx
;
1738 level1_ident_pgt
= extend_brk(sizeof(pte_t
) * LEVEL1_IDENT_ENTRIES
,
1743 for (pmdidx
= 0; pmdidx
< PTRS_PER_PMD
&& pfn
< max_pfn
; pmdidx
++) {
1746 /* Reuse or allocate a page of ptes */
1747 if (pmd_present(pmd
[pmdidx
]))
1748 pte_page
= m2v(pmd
[pmdidx
].pmd
);
1750 /* Check for free pte pages */
1751 if (ident_pte
== LEVEL1_IDENT_ENTRIES
)
1754 pte_page
= &level1_ident_pgt
[ident_pte
];
1755 ident_pte
+= PTRS_PER_PTE
;
1757 pmd
[pmdidx
] = __pmd(__pa(pte_page
) | _PAGE_TABLE
);
1760 /* Install mappings */
1761 for (pteidx
= 0; pteidx
< PTRS_PER_PTE
; pteidx
++, pfn
++) {
1764 #ifdef CONFIG_X86_32
1765 if (pfn
> max_pfn_mapped
)
1766 max_pfn_mapped
= pfn
;
1769 if (!pte_none(pte_page
[pteidx
]))
1772 pte
= pfn_pte(pfn
, PAGE_KERNEL_EXEC
);
1773 pte_page
[pteidx
] = pte
;
1777 for (pteidx
= 0; pteidx
< ident_pte
; pteidx
+= PTRS_PER_PTE
)
1778 set_page_prot(&level1_ident_pgt
[pteidx
], PAGE_KERNEL_RO
);
1780 set_page_prot(pmd
, PAGE_KERNEL_RO
);
1783 void __init
xen_setup_machphys_mapping(void)
1785 struct xen_machphys_mapping mapping
;
1787 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping
, &mapping
) == 0) {
1788 machine_to_phys_mapping
= (unsigned long *)mapping
.v_start
;
1789 machine_to_phys_nr
= mapping
.max_mfn
+ 1;
1791 machine_to_phys_nr
= MACH2PHYS_NR_ENTRIES
;
1793 #ifdef CONFIG_X86_32
1794 WARN_ON((machine_to_phys_mapping
+ (machine_to_phys_nr
- 1))
1795 < machine_to_phys_mapping
);
1799 #ifdef CONFIG_X86_64
1800 static void convert_pfn_mfn(void *v
)
1805 /* All levels are converted the same way, so just treat them
1807 for (i
= 0; i
< PTRS_PER_PTE
; i
++)
1808 pte
[i
] = xen_make_pte(pte
[i
].pte
);
1810 static void __init
check_pt_base(unsigned long *pt_base
, unsigned long *pt_end
,
1813 if (*pt_base
== PFN_DOWN(__pa(addr
))) {
1814 set_page_prot((void *)addr
, PAGE_KERNEL
);
1815 clear_page((void *)addr
);
1818 if (*pt_end
== PFN_DOWN(__pa(addr
))) {
1819 set_page_prot((void *)addr
, PAGE_KERNEL
);
1820 clear_page((void *)addr
);
1825 * Set up the initial kernel pagetable.
1827 * We can construct this by grafting the Xen provided pagetable into
1828 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1829 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
1830 * means that only the kernel has a physical mapping to start with -
1831 * but that's enough to get __va working. We need to fill in the rest
1832 * of the physical mapping once some sort of allocator has been set
1835 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
1839 unsigned long addr
[3];
1840 unsigned long pt_base
, pt_end
;
1843 /* max_pfn_mapped is the last pfn mapped in the initial memory
1844 * mappings. Considering that on Xen after the kernel mappings we
1845 * have the mappings of some pages that don't exist in pfn space, we
1846 * set max_pfn_mapped to the last real pfn mapped. */
1847 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->mfn_list
));
1849 pt_base
= PFN_DOWN(__pa(xen_start_info
->pt_base
));
1850 pt_end
= pt_base
+ xen_start_info
->nr_pt_frames
;
1852 /* Zap identity mapping */
1853 init_level4_pgt
[0] = __pgd(0);
1855 /* Pre-constructed entries are in pfn, so convert to mfn */
1856 /* L4[272] -> level3_ident_pgt
1857 * L4[511] -> level3_kernel_pgt */
1858 convert_pfn_mfn(init_level4_pgt
);
1860 /* L3_i[0] -> level2_ident_pgt */
1861 convert_pfn_mfn(level3_ident_pgt
);
1862 /* L3_k[510] -> level2_kernel_pgt
1863 * L3_i[511] -> level2_fixmap_pgt */
1864 convert_pfn_mfn(level3_kernel_pgt
);
1866 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
1867 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
)].pgd
);
1868 l2
= m2v(l3
[pud_index(__START_KERNEL_map
)].pud
);
1870 addr
[0] = (unsigned long)pgd
;
1871 addr
[1] = (unsigned long)l3
;
1872 addr
[2] = (unsigned long)l2
;
1873 /* Graft it onto L4[272][0]. Note that we creating an aliasing problem:
1874 * Both L4[272][0] and L4[511][511] have entries that point to the same
1875 * L2 (PMD) tables. Meaning that if you modify it in __va space
1876 * it will be also modified in the __ka space! (But if you just
1877 * modify the PMD table to point to other PTE's or none, then you
1878 * are OK - which is what cleanup_highmap does) */
1879 copy_page(level2_ident_pgt
, l2
);
1880 /* Graft it onto L4[511][511] */
1881 copy_page(level2_kernel_pgt
, l2
);
1883 /* Get [511][510] and graft that in level2_fixmap_pgt */
1884 l3
= m2v(pgd
[pgd_index(__START_KERNEL_map
+ PMD_SIZE
)].pgd
);
1885 l2
= m2v(l3
[pud_index(__START_KERNEL_map
+ PMD_SIZE
)].pud
);
1886 copy_page(level2_fixmap_pgt
, l2
);
1887 /* Note that we don't do anything with level1_fixmap_pgt which
1890 /* Make pagetable pieces RO */
1891 set_page_prot(init_level4_pgt
, PAGE_KERNEL_RO
);
1892 set_page_prot(level3_ident_pgt
, PAGE_KERNEL_RO
);
1893 set_page_prot(level3_kernel_pgt
, PAGE_KERNEL_RO
);
1894 set_page_prot(level3_user_vsyscall
, PAGE_KERNEL_RO
);
1895 set_page_prot(level2_ident_pgt
, PAGE_KERNEL_RO
);
1896 set_page_prot(level2_kernel_pgt
, PAGE_KERNEL_RO
);
1897 set_page_prot(level2_fixmap_pgt
, PAGE_KERNEL_RO
);
1899 /* Pin down new L4 */
1900 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE
,
1901 PFN_DOWN(__pa_symbol(init_level4_pgt
)));
1903 /* Unpin Xen-provided one */
1904 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1907 * At this stage there can be no user pgd, and no page
1908 * structure to attach it to, so make sure we just set kernel
1912 __xen_write_cr3(true, __pa(init_level4_pgt
));
1913 xen_mc_issue(PARAVIRT_LAZY_CPU
);
1915 /* We can't that easily rip out L3 and L2, as the Xen pagetables are
1916 * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for
1917 * the initial domain. For guests using the toolstack, they are in:
1918 * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only
1919 * rip out the [L4] (pgd), but for guests we shave off three pages.
1921 for (i
= 0; i
< ARRAY_SIZE(addr
); i
++)
1922 check_pt_base(&pt_base
, &pt_end
, addr
[i
]);
1924 /* Our (by three pages) smaller Xen pagetable that we are using */
1925 memblock_reserve(PFN_PHYS(pt_base
), (pt_end
- pt_base
) * PAGE_SIZE
);
1926 /* Revector the xen_start_info */
1927 xen_start_info
= (struct start_info
*)__va(__pa(xen_start_info
));
1929 #else /* !CONFIG_X86_64 */
1930 static RESERVE_BRK_ARRAY(pmd_t
, initial_kernel_pmd
, PTRS_PER_PMD
);
1931 static RESERVE_BRK_ARRAY(pmd_t
, swapper_kernel_pmd
, PTRS_PER_PMD
);
1933 static void __init
xen_write_cr3_init(unsigned long cr3
)
1935 unsigned long pfn
= PFN_DOWN(__pa(swapper_pg_dir
));
1937 BUG_ON(read_cr3() != __pa(initial_page_table
));
1938 BUG_ON(cr3
!= __pa(swapper_pg_dir
));
1941 * We are switching to swapper_pg_dir for the first time (from
1942 * initial_page_table) and therefore need to mark that page
1943 * read-only and then pin it.
1945 * Xen disallows sharing of kernel PMDs for PAE
1946 * guests. Therefore we must copy the kernel PMD from
1947 * initial_page_table into a new kernel PMD to be used in
1950 swapper_kernel_pmd
=
1951 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
1952 copy_page(swapper_kernel_pmd
, initial_kernel_pmd
);
1953 swapper_pg_dir
[KERNEL_PGD_BOUNDARY
] =
1954 __pgd(__pa(swapper_kernel_pmd
) | _PAGE_PRESENT
);
1955 set_page_prot(swapper_kernel_pmd
, PAGE_KERNEL_RO
);
1957 set_page_prot(swapper_pg_dir
, PAGE_KERNEL_RO
);
1959 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
, pfn
);
1961 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
,
1962 PFN_DOWN(__pa(initial_page_table
)));
1963 set_page_prot(initial_page_table
, PAGE_KERNEL
);
1964 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL
);
1966 pv_mmu_ops
.write_cr3
= &xen_write_cr3
;
1969 void __init
xen_setup_kernel_pagetable(pgd_t
*pgd
, unsigned long max_pfn
)
1973 initial_kernel_pmd
=
1974 extend_brk(sizeof(pmd_t
) * PTRS_PER_PMD
, PAGE_SIZE
);
1976 max_pfn_mapped
= PFN_DOWN(__pa(xen_start_info
->pt_base
) +
1977 xen_start_info
->nr_pt_frames
* PAGE_SIZE
+
1980 kernel_pmd
= m2v(pgd
[KERNEL_PGD_BOUNDARY
].pgd
);
1981 copy_page(initial_kernel_pmd
, kernel_pmd
);
1983 xen_map_identity_early(initial_kernel_pmd
, max_pfn
);
1985 copy_page(initial_page_table
, pgd
);
1986 initial_page_table
[KERNEL_PGD_BOUNDARY
] =
1987 __pgd(__pa(initial_kernel_pmd
) | _PAGE_PRESENT
);
1989 set_page_prot(initial_kernel_pmd
, PAGE_KERNEL_RO
);
1990 set_page_prot(initial_page_table
, PAGE_KERNEL_RO
);
1991 set_page_prot(empty_zero_page
, PAGE_KERNEL_RO
);
1993 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE
, PFN_DOWN(__pa(pgd
)));
1995 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE
,
1996 PFN_DOWN(__pa(initial_page_table
)));
1997 xen_write_cr3(__pa(initial_page_table
));
1999 memblock_reserve(__pa(xen_start_info
->pt_base
),
2000 xen_start_info
->nr_pt_frames
* PAGE_SIZE
);
2002 #endif /* CONFIG_X86_64 */
2004 static unsigned char dummy_mapping
[PAGE_SIZE
] __page_aligned_bss
;
2006 static void xen_set_fixmap(unsigned idx
, phys_addr_t phys
, pgprot_t prot
)
2010 phys
>>= PAGE_SHIFT
;
2013 case FIX_BTMAP_END
... FIX_BTMAP_BEGIN
:
2014 #ifdef CONFIG_X86_F00F_BUG
2017 #ifdef CONFIG_X86_32
2020 # ifdef CONFIG_HIGHMEM
2021 case FIX_KMAP_BEGIN
... FIX_KMAP_END
:
2024 case VSYSCALL_LAST_PAGE
... VSYSCALL_FIRST_PAGE
:
2027 case FIX_TEXT_POKE0
:
2028 case FIX_TEXT_POKE1
:
2029 /* All local page mappings */
2030 pte
= pfn_pte(phys
, prot
);
2033 #ifdef CONFIG_X86_LOCAL_APIC
2034 case FIX_APIC_BASE
: /* maps dummy local APIC */
2035 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2039 #ifdef CONFIG_X86_IO_APIC
2040 case FIX_IO_APIC_BASE_0
... FIX_IO_APIC_BASE_END
:
2042 * We just don't map the IO APIC - all access is via
2043 * hypercalls. Keep the address in the pte for reference.
2045 pte
= pfn_pte(PFN_DOWN(__pa(dummy_mapping
)), PAGE_KERNEL
);
2049 case FIX_PARAVIRT_BOOTMAP
:
2050 /* This is an MFN, but it isn't an IO mapping from the
2052 pte
= mfn_pte(phys
, prot
);
2056 /* By default, set_fixmap is used for hardware mappings */
2057 pte
= mfn_pte(phys
, __pgprot(pgprot_val(prot
) | _PAGE_IOMAP
));
2061 __native_set_fixmap(idx
, pte
);
2063 #ifdef CONFIG_X86_64
2064 /* Replicate changes to map the vsyscall page into the user
2065 pagetable vsyscall mapping. */
2066 if ((idx
>= VSYSCALL_LAST_PAGE
&& idx
<= VSYSCALL_FIRST_PAGE
) ||
2068 unsigned long vaddr
= __fix_to_virt(idx
);
2069 set_pte_vaddr_pud(level3_user_vsyscall
, vaddr
, pte
);
2074 static void __init
xen_post_allocator_init(void)
2076 pv_mmu_ops
.set_pte
= xen_set_pte
;
2077 pv_mmu_ops
.set_pmd
= xen_set_pmd
;
2078 pv_mmu_ops
.set_pud
= xen_set_pud
;
2079 #if PAGETABLE_LEVELS == 4
2080 pv_mmu_ops
.set_pgd
= xen_set_pgd
;
2083 /* This will work as long as patching hasn't happened yet
2084 (which it hasn't) */
2085 pv_mmu_ops
.alloc_pte
= xen_alloc_pte
;
2086 pv_mmu_ops
.alloc_pmd
= xen_alloc_pmd
;
2087 pv_mmu_ops
.release_pte
= xen_release_pte
;
2088 pv_mmu_ops
.release_pmd
= xen_release_pmd
;
2089 #if PAGETABLE_LEVELS == 4
2090 pv_mmu_ops
.alloc_pud
= xen_alloc_pud
;
2091 pv_mmu_ops
.release_pud
= xen_release_pud
;
2094 #ifdef CONFIG_X86_64
2095 SetPagePinned(virt_to_page(level3_user_vsyscall
));
2097 xen_mark_init_mm_pinned();
2100 static void xen_leave_lazy_mmu(void)
2104 paravirt_leave_lazy_mmu();
2108 static const struct pv_mmu_ops xen_mmu_ops __initconst
= {
2109 .read_cr2
= xen_read_cr2
,
2110 .write_cr2
= xen_write_cr2
,
2112 .read_cr3
= xen_read_cr3
,
2113 #ifdef CONFIG_X86_32
2114 .write_cr3
= xen_write_cr3_init
,
2116 .write_cr3
= xen_write_cr3
,
2119 .flush_tlb_user
= xen_flush_tlb
,
2120 .flush_tlb_kernel
= xen_flush_tlb
,
2121 .flush_tlb_single
= xen_flush_tlb_single
,
2122 .flush_tlb_others
= xen_flush_tlb_others
,
2124 .pte_update
= paravirt_nop
,
2125 .pte_update_defer
= paravirt_nop
,
2127 .pgd_alloc
= xen_pgd_alloc
,
2128 .pgd_free
= xen_pgd_free
,
2130 .alloc_pte
= xen_alloc_pte_init
,
2131 .release_pte
= xen_release_pte_init
,
2132 .alloc_pmd
= xen_alloc_pmd_init
,
2133 .release_pmd
= xen_release_pmd_init
,
2135 .set_pte
= xen_set_pte_init
,
2136 .set_pte_at
= xen_set_pte_at
,
2137 .set_pmd
= xen_set_pmd_hyper
,
2139 .ptep_modify_prot_start
= __ptep_modify_prot_start
,
2140 .ptep_modify_prot_commit
= __ptep_modify_prot_commit
,
2142 .pte_val
= PV_CALLEE_SAVE(xen_pte_val
),
2143 .pgd_val
= PV_CALLEE_SAVE(xen_pgd_val
),
2145 .make_pte
= PV_CALLEE_SAVE(xen_make_pte
),
2146 .make_pgd
= PV_CALLEE_SAVE(xen_make_pgd
),
2148 #ifdef CONFIG_X86_PAE
2149 .set_pte_atomic
= xen_set_pte_atomic
,
2150 .pte_clear
= xen_pte_clear
,
2151 .pmd_clear
= xen_pmd_clear
,
2152 #endif /* CONFIG_X86_PAE */
2153 .set_pud
= xen_set_pud_hyper
,
2155 .make_pmd
= PV_CALLEE_SAVE(xen_make_pmd
),
2156 .pmd_val
= PV_CALLEE_SAVE(xen_pmd_val
),
2158 #if PAGETABLE_LEVELS == 4
2159 .pud_val
= PV_CALLEE_SAVE(xen_pud_val
),
2160 .make_pud
= PV_CALLEE_SAVE(xen_make_pud
),
2161 .set_pgd
= xen_set_pgd_hyper
,
2163 .alloc_pud
= xen_alloc_pmd_init
,
2164 .release_pud
= xen_release_pmd_init
,
2165 #endif /* PAGETABLE_LEVELS == 4 */
2167 .activate_mm
= xen_activate_mm
,
2168 .dup_mmap
= xen_dup_mmap
,
2169 .exit_mmap
= xen_exit_mmap
,
2172 .enter
= paravirt_enter_lazy_mmu
,
2173 .leave
= xen_leave_lazy_mmu
,
2176 .set_fixmap
= xen_set_fixmap
,
2179 void __init
xen_init_mmu_ops(void)
2181 x86_init
.mapping
.pagetable_reserve
= xen_mapping_pagetable_reserve
;
2182 x86_init
.paging
.pagetable_init
= xen_pagetable_init
;
2183 pv_mmu_ops
= xen_mmu_ops
;
2185 memset(dummy_mapping
, 0xff, PAGE_SIZE
);
2188 /* Protected by xen_reservation_lock. */
2189 #define MAX_CONTIG_ORDER 9 /* 2MB */
2190 static unsigned long discontig_frames
[1<<MAX_CONTIG_ORDER
];
2192 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2193 static void xen_zap_pfn_range(unsigned long vaddr
, unsigned int order
,
2194 unsigned long *in_frames
,
2195 unsigned long *out_frames
)
2198 struct multicall_space mcs
;
2201 for (i
= 0; i
< (1UL<<order
); i
++, vaddr
+= PAGE_SIZE
) {
2202 mcs
= __xen_mc_entry(0);
2205 in_frames
[i
] = virt_to_mfn(vaddr
);
2207 MULTI_update_va_mapping(mcs
.mc
, vaddr
, VOID_PTE
, 0);
2208 __set_phys_to_machine(virt_to_pfn(vaddr
), INVALID_P2M_ENTRY
);
2211 out_frames
[i
] = virt_to_pfn(vaddr
);
2217 * Update the pfn-to-mfn mappings for a virtual address range, either to
2218 * point to an array of mfns, or contiguously from a single starting
2221 static void xen_remap_exchanged_ptes(unsigned long vaddr
, int order
,
2222 unsigned long *mfns
,
2223 unsigned long first_mfn
)
2230 limit
= 1u << order
;
2231 for (i
= 0; i
< limit
; i
++, vaddr
+= PAGE_SIZE
) {
2232 struct multicall_space mcs
;
2235 mcs
= __xen_mc_entry(0);
2239 mfn
= first_mfn
+ i
;
2241 if (i
< (limit
- 1))
2245 flags
= UVMF_INVLPG
| UVMF_ALL
;
2247 flags
= UVMF_TLB_FLUSH
| UVMF_ALL
;
2250 MULTI_update_va_mapping(mcs
.mc
, vaddr
,
2251 mfn_pte(mfn
, PAGE_KERNEL
), flags
);
2253 set_phys_to_machine(virt_to_pfn(vaddr
), mfn
);
2260 * Perform the hypercall to exchange a region of our pfns to point to
2261 * memory with the required contiguous alignment. Takes the pfns as
2262 * input, and populates mfns as output.
2264 * Returns a success code indicating whether the hypervisor was able to
2265 * satisfy the request or not.
2267 static int xen_exchange_memory(unsigned long extents_in
, unsigned int order_in
,
2268 unsigned long *pfns_in
,
2269 unsigned long extents_out
,
2270 unsigned int order_out
,
2271 unsigned long *mfns_out
,
2272 unsigned int address_bits
)
2277 struct xen_memory_exchange exchange
= {
2279 .nr_extents
= extents_in
,
2280 .extent_order
= order_in
,
2281 .extent_start
= pfns_in
,
2285 .nr_extents
= extents_out
,
2286 .extent_order
= order_out
,
2287 .extent_start
= mfns_out
,
2288 .address_bits
= address_bits
,
2293 BUG_ON(extents_in
<< order_in
!= extents_out
<< order_out
);
2295 rc
= HYPERVISOR_memory_op(XENMEM_exchange
, &exchange
);
2296 success
= (exchange
.nr_exchanged
== extents_in
);
2298 BUG_ON(!success
&& ((exchange
.nr_exchanged
!= 0) || (rc
== 0)));
2299 BUG_ON(success
&& (rc
!= 0));
2304 int xen_create_contiguous_region(unsigned long vstart
, unsigned int order
,
2305 unsigned int address_bits
)
2307 unsigned long *in_frames
= discontig_frames
, out_frame
;
2308 unsigned long flags
;
2312 * Currently an auto-translated guest will not perform I/O, nor will
2313 * it require PAE page directories below 4GB. Therefore any calls to
2314 * this function are redundant and can be ignored.
2317 if (xen_feature(XENFEAT_auto_translated_physmap
))
2320 if (unlikely(order
> MAX_CONTIG_ORDER
))
2323 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2325 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2327 /* 1. Zap current PTEs, remembering MFNs. */
2328 xen_zap_pfn_range(vstart
, order
, in_frames
, NULL
);
2330 /* 2. Get a new contiguous memory extent. */
2331 out_frame
= virt_to_pfn(vstart
);
2332 success
= xen_exchange_memory(1UL << order
, 0, in_frames
,
2333 1, order
, &out_frame
,
2336 /* 3. Map the new extent in place of old pages. */
2338 xen_remap_exchanged_ptes(vstart
, order
, NULL
, out_frame
);
2340 xen_remap_exchanged_ptes(vstart
, order
, in_frames
, 0);
2342 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2344 return success
? 0 : -ENOMEM
;
2346 EXPORT_SYMBOL_GPL(xen_create_contiguous_region
);
2348 void xen_destroy_contiguous_region(unsigned long vstart
, unsigned int order
)
2350 unsigned long *out_frames
= discontig_frames
, in_frame
;
2351 unsigned long flags
;
2354 if (xen_feature(XENFEAT_auto_translated_physmap
))
2357 if (unlikely(order
> MAX_CONTIG_ORDER
))
2360 memset((void *) vstart
, 0, PAGE_SIZE
<< order
);
2362 spin_lock_irqsave(&xen_reservation_lock
, flags
);
2364 /* 1. Find start MFN of contiguous extent. */
2365 in_frame
= virt_to_mfn(vstart
);
2367 /* 2. Zap current PTEs. */
2368 xen_zap_pfn_range(vstart
, order
, NULL
, out_frames
);
2370 /* 3. Do the exchange for non-contiguous MFNs. */
2371 success
= xen_exchange_memory(1, order
, &in_frame
, 1UL << order
,
2374 /* 4. Map new pages in place of old pages. */
2376 xen_remap_exchanged_ptes(vstart
, order
, out_frames
, 0);
2378 xen_remap_exchanged_ptes(vstart
, order
, NULL
, in_frame
);
2380 spin_unlock_irqrestore(&xen_reservation_lock
, flags
);
2382 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region
);
2384 #ifdef CONFIG_XEN_PVHVM
2385 #ifdef CONFIG_PROC_VMCORE
2387 * This function is used in two contexts:
2388 * - the kdump kernel has to check whether a pfn of the crashed kernel
2389 * was a ballooned page. vmcore is using this function to decide
2390 * whether to access a pfn of the crashed kernel.
2391 * - the kexec kernel has to check whether a pfn was ballooned by the
2392 * previous kernel. If the pfn is ballooned, handle it properly.
2393 * Returns 0 if the pfn is not backed by a RAM page, the caller may
2394 * handle the pfn special in this case.
2396 static int xen_oldmem_pfn_is_ram(unsigned long pfn
)
2398 struct xen_hvm_get_mem_type a
= {
2399 .domid
= DOMID_SELF
,
2404 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type
, &a
))
2407 switch (a
.mem_type
) {
2408 case HVMMEM_mmio_dm
:
2422 static void xen_hvm_exit_mmap(struct mm_struct
*mm
)
2424 struct xen_hvm_pagetable_dying a
;
2427 a
.domid
= DOMID_SELF
;
2428 a
.gpa
= __pa(mm
->pgd
);
2429 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2430 WARN_ON_ONCE(rc
< 0);
2433 static int is_pagetable_dying_supported(void)
2435 struct xen_hvm_pagetable_dying a
;
2438 a
.domid
= DOMID_SELF
;
2440 rc
= HYPERVISOR_hvm_op(HVMOP_pagetable_dying
, &a
);
2442 printk(KERN_DEBUG
"HVMOP_pagetable_dying not supported\n");
2448 void __init
xen_hvm_init_mmu_ops(void)
2450 if (is_pagetable_dying_supported())
2451 pv_mmu_ops
.exit_mmap
= xen_hvm_exit_mmap
;
2452 #ifdef CONFIG_PROC_VMCORE
2453 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram
);
2458 #define REMAP_BATCH_SIZE 16
2463 struct mmu_update
*mmu_update
;
2466 static int remap_area_mfn_pte_fn(pte_t
*ptep
, pgtable_t token
,
2467 unsigned long addr
, void *data
)
2469 struct remap_data
*rmd
= data
;
2470 pte_t pte
= pte_mkspecial(pfn_pte(rmd
->mfn
++, rmd
->prot
));
2472 rmd
->mmu_update
->ptr
= virt_to_machine(ptep
).maddr
;
2473 rmd
->mmu_update
->val
= pte_val_ma(pte
);
2479 int xen_remap_domain_mfn_range(struct vm_area_struct
*vma
,
2481 xen_pfn_t mfn
, int nr
,
2482 pgprot_t prot
, unsigned domid
,
2483 struct page
**pages
)
2486 struct remap_data rmd
;
2487 struct mmu_update mmu_update
[REMAP_BATCH_SIZE
];
2489 unsigned long range
;
2492 if (xen_feature(XENFEAT_auto_translated_physmap
))
2495 prot
= __pgprot(pgprot_val(prot
) | _PAGE_IOMAP
);
2497 BUG_ON(!((vma
->vm_flags
& (VM_PFNMAP
| VM_IO
)) == (VM_PFNMAP
| VM_IO
)));
2503 batch
= min(REMAP_BATCH_SIZE
, nr
);
2504 range
= (unsigned long)batch
<< PAGE_SHIFT
;
2506 rmd
.mmu_update
= mmu_update
;
2507 err
= apply_to_page_range(vma
->vm_mm
, addr
, range
,
2508 remap_area_mfn_pte_fn
, &rmd
);
2512 err
= HYPERVISOR_mmu_update(mmu_update
, batch
, NULL
, domid
);
2527 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range
);
2529 /* Returns: 0 success */
2530 int xen_unmap_domain_mfn_range(struct vm_area_struct
*vma
,
2531 int numpgs
, struct page
**pages
)
2533 if (!pages
|| !xen_feature(XENFEAT_auto_translated_physmap
))
2538 EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range
);