projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: Remove unused function
[deliverable/linux.git] / drivers / kvm / mmu.c
CommitLineData
6aa8b732
AK		 1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19#include <linux/types.h>
20#include <linux/string.h>
21#include <asm/page.h>
22#include <linux/mm.h>
23#include <linux/highmem.h>
24#include <linux/module.h>
25
26#include "vmx.h"
27#include "kvm.h"
28
37a7d8b0
AK		 29#undef MMU_DEBUG
30
31#undef AUDIT
32
33#ifdef AUDIT
34static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
35#else
36static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
37#endif
38
39#ifdef MMU_DEBUG
40
41#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
42#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
43
44#else
45
46#define pgprintk(x...) do { } while (0)
47#define rmap_printk(x...) do { } while (0)
48
49#endif
50
51#if defined(MMU_DEBUG) || defined(AUDIT)
52static int dbg = 1;
53#endif
6aa8b732
AK		 54
55#define ASSERT(x) \
56 if (!(x)) { \
57 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
58 __FILE__, __LINE__, #x); \
59 }
60
cea0f0e7
AK		 61#define PT64_PT_BITS 9
62#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
63#define PT32_PT_BITS 10
64#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
6aa8b732
AK		 65
66#define PT_WRITABLE_SHIFT 1
67
68#define PT_PRESENT_MASK (1ULL << 0)
69#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
70#define PT_USER_MASK (1ULL << 2)
71#define PT_PWT_MASK (1ULL << 3)
72#define PT_PCD_MASK (1ULL << 4)
73#define PT_ACCESSED_MASK (1ULL << 5)
74#define PT_DIRTY_MASK (1ULL << 6)
75#define PT_PAGE_SIZE_MASK (1ULL << 7)
76#define PT_PAT_MASK (1ULL << 7)
77#define PT_GLOBAL_MASK (1ULL << 8)
78#define PT64_NX_MASK (1ULL << 63)
79
80#define PT_PAT_SHIFT 7
81#define PT_DIR_PAT_SHIFT 12
82#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
83
84#define PT32_DIR_PSE36_SIZE 4
85#define PT32_DIR_PSE36_SHIFT 13
86#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
87
88
89#define PT32_PTE_COPY_MASK \
8c7bb723 90 (PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_GLOBAL_MASK)
6aa8b732 91
8c7bb723 92#define PT64_PTE_COPY_MASK (PT64_NX_MASK | PT32_PTE_COPY_MASK)
6aa8b732
AK		 93
94#define PT_FIRST_AVAIL_BITS_SHIFT 9
95#define PT64_SECOND_AVAIL_BITS_SHIFT 52
96
97#define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
98#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
99
100#define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
101#define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
102
103#define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
104#define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
105
106#define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
107
108#define VALID_PAGE(x) ((x) != INVALID_PAGE)
109
110#define PT64_LEVEL_BITS 9
111
112#define PT64_LEVEL_SHIFT(level) \
113 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
114
115#define PT64_LEVEL_MASK(level) \
116 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
117
118#define PT64_INDEX(address, level)\
119 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
120
121
122#define PT32_LEVEL_BITS 10
123
124#define PT32_LEVEL_SHIFT(level) \
125 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
126
127#define PT32_LEVEL_MASK(level) \
128 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
129
130#define PT32_INDEX(address, level)\
131 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
132
133
27aba766 134#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
6aa8b732
AK		 135#define PT64_DIR_BASE_ADDR_MASK \
136 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
137
138#define PT32_BASE_ADDR_MASK PAGE_MASK
139#define PT32_DIR_BASE_ADDR_MASK \
140 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
141
142
143#define PFERR_PRESENT_MASK (1U << 0)
144#define PFERR_WRITE_MASK (1U << 1)
145#define PFERR_USER_MASK (1U << 2)
73b1087e 146#define PFERR_FETCH_MASK (1U << 4)
6aa8b732
AK		 147
148#define PT64_ROOT_LEVEL 4
149#define PT32_ROOT_LEVEL 2
150#define PT32E_ROOT_LEVEL 3
151
152#define PT_DIRECTORY_LEVEL 2
153#define PT_PAGE_TABLE_LEVEL 1
154
cd4a4e53
AK		 155#define RMAP_EXT 4
156
157struct kvm_rmap_desc {
158 u64 *shadow_ptes[RMAP_EXT];
159 struct kvm_rmap_desc *more;
160};
161
6aa8b732
AK		 162static int is_write_protection(struct kvm_vcpu *vcpu)
163{
164 return vcpu->cr0 & CR0_WP_MASK;
165}
166
167static int is_cpuid_PSE36(void)
168{
169 return 1;
170}
171
73b1087e
AK		 172static int is_nx(struct kvm_vcpu *vcpu)
173{
174 return vcpu->shadow_efer & EFER_NX;
175}
176
6aa8b732
AK		 177static int is_present_pte(unsigned long pte)
178{
179 return pte & PT_PRESENT_MASK;
180}
181
182static int is_writeble_pte(unsigned long pte)
183{
184 return pte & PT_WRITABLE_MASK;
185}
186
187static int is_io_pte(unsigned long pte)
188{
189 return pte & PT_SHADOW_IO_MARK;
190}
191
cd4a4e53
AK		 192static int is_rmap_pte(u64 pte)
193{
194 return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
195 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
196}
197
e2dec939
AK		 198static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
199 size_t objsize, int min)
714b93da
AK		 200{
201 void *obj;
202
203 if (cache->nobjs >= min)
e2dec939 204 return 0;
714b93da
AK		 205 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
206 obj = kzalloc(objsize, GFP_NOWAIT);
207 if (!obj)
e2dec939 208 return -ENOMEM;
714b93da
AK		 209 cache->objects[cache->nobjs++] = obj;
210 }
e2dec939 211 return 0;
714b93da
AK		 212}
213
214static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
215{
216 while (mc->nobjs)
217 kfree(mc->objects[--mc->nobjs]);
218}
219
e2dec939 220static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
714b93da 221{
e2dec939
AK		 222 int r;
223
224 r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
225 sizeof(struct kvm_pte_chain), 4);
226 if (r)
227 goto out;
228 r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
229 sizeof(struct kvm_rmap_desc), 1);
230out:
231 return r;
714b93da
AK		 232}
233
234static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
235{
236 mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
237 mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
238}
239
240static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
241 size_t size)
242{
243 void *p;
244
245 BUG_ON(!mc->nobjs);
246 p = mc->objects[--mc->nobjs];
247 memset(p, 0, size);
248 return p;
249}
250
251static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj)
252{
253 if (mc->nobjs < KVM_NR_MEM_OBJS)
254 mc->objects[mc->nobjs++] = obj;
255 else
256 kfree(obj);
257}
258
259static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
260{
261 return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
262 sizeof(struct kvm_pte_chain));
263}
264
265static void mmu_free_pte_chain(struct kvm_vcpu *vcpu,
266 struct kvm_pte_chain *pc)
267{
268 mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc);
269}
270
271static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
272{
273 return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
274 sizeof(struct kvm_rmap_desc));
275}
276
277static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu,
278 struct kvm_rmap_desc *rd)
279{
280 mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd);
281}
282
cd4a4e53
AK		 283/*
284 * Reverse mapping data structures:
285 *
286 * If page->private bit zero is zero, then page->private points to the
287 * shadow page table entry that points to page_address(page).
288 *
289 * If page->private bit zero is one, (then page->private & ~1) points
290 * to a struct kvm_rmap_desc containing more mappings.
291 */
714b93da 292static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
cd4a4e53
AK		 293{
294 struct page *page;
295 struct kvm_rmap_desc *desc;
296 int i;
297
298 if (!is_rmap_pte(*spte))
299 return;
300 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
5972e953 301 if (!page_private(page)) {
cd4a4e53 302 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
5972e953
MR		 303 set_page_private(page,(unsigned long)spte);
304 } else if (!(page_private(page) & 1)) {
cd4a4e53 305 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
714b93da 306 desc = mmu_alloc_rmap_desc(vcpu);
5972e953 307 desc->shadow_ptes[0] = (u64 *)page_private(page);
cd4a4e53 308 desc->shadow_ptes[1] = spte;
5972e953 309 set_page_private(page,(unsigned long)desc | 1);
cd4a4e53
AK		 310 } else {
311 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
5972e953 312 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
cd4a4e53
AK		 313 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
314 desc = desc->more;
315 if (desc->shadow_ptes[RMAP_EXT-1]) {
714b93da 316 desc->more = mmu_alloc_rmap_desc(vcpu);
cd4a4e53
AK		 317 desc = desc->more;
318 }
319 for (i = 0; desc->shadow_ptes[i]; ++i)
320 ;
321 desc->shadow_ptes[i] = spte;
322 }
323}
324
714b93da
AK		 325static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu,
326 struct page *page,
cd4a4e53
AK		 327 struct kvm_rmap_desc *desc,
328 int i,
329 struct kvm_rmap_desc *prev_desc)
330{
331 int j;
332
333 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
334 ;
335 desc->shadow_ptes[i] = desc->shadow_ptes[j];
11718b4d 336 desc->shadow_ptes[j] = NULL;
cd4a4e53
AK		 337 if (j != 0)
338 return;
339 if (!prev_desc && !desc->more)
5972e953 340 set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
cd4a4e53
AK		 341 else
342 if (prev_desc)
343 prev_desc->more = desc->more;
344 else
5972e953 345 set_page_private(page,(unsigned long)desc->more | 1);
714b93da 346 mmu_free_rmap_desc(vcpu, desc);
cd4a4e53
AK		 347}
348
714b93da 349static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte)
cd4a4e53
AK		 350{
351 struct page *page;
352 struct kvm_rmap_desc *desc;
353 struct kvm_rmap_desc *prev_desc;
354 int i;
355
356 if (!is_rmap_pte(*spte))
357 return;
358 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
5972e953 359 if (!page_private(page)) {
cd4a4e53
AK		 360 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
361 BUG();
5972e953 362 } else if (!(page_private(page) & 1)) {
cd4a4e53 363 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
5972e953 364 if ((u64 *)page_private(page) != spte) {
cd4a4e53
AK		 365 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
366 spte, *spte);
367 BUG();
368 }
5972e953 369 set_page_private(page,0);
cd4a4e53
AK		 370 } else {
371 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
5972e953 372 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
cd4a4e53
AK		 373 prev_desc = NULL;
374 while (desc) {
375 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
376 if (desc->shadow_ptes[i] == spte) {
714b93da
AK		 377 rmap_desc_remove_entry(vcpu, page,
378 desc, i,
cd4a4e53
AK		 379 prev_desc);
380 return;
381 }
382 prev_desc = desc;
383 desc = desc->more;
384 }
385 BUG();
386 }
387}
388
714b93da 389static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
374cbac0 390{
714b93da 391 struct kvm *kvm = vcpu->kvm;
374cbac0
AK		 392 struct page *page;
393 struct kvm_memory_slot *slot;
394 struct kvm_rmap_desc *desc;
395 u64 *spte;
396
397 slot = gfn_to_memslot(kvm, gfn);
398 BUG_ON(!slot);
399 page = gfn_to_page(slot, gfn);
400
5972e953
MR		 401 while (page_private(page)) {
402 if (!(page_private(page) & 1))
403 spte = (u64 *)page_private(page);
374cbac0 404 else {
5972e953 405 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
374cbac0
AK		 406 spte = desc->shadow_ptes[0];
407 }
408 BUG_ON(!spte);
27aba766
AK		 409 BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
410 != page_to_pfn(page));
374cbac0
AK		 411 BUG_ON(!(*spte & PT_PRESENT_MASK));
412 BUG_ON(!(*spte & PT_WRITABLE_MASK));
413 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
714b93da 414 rmap_remove(vcpu, spte);
40907d57 415 kvm_arch_ops->tlb_flush(vcpu);
374cbac0
AK		 416 *spte &= ~(u64)PT_WRITABLE_MASK;
417 }
418}
419
6aa8b732
AK		 420static int is_empty_shadow_page(hpa_t page_hpa)
421{
139bdb2d
AK		 422 u64 *pos;
423 u64 *end;
424
425 for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u64);
6aa8b732 426 pos != end; pos++)
139bdb2d
AK		 427 if (*pos != 0) {
428 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
429 pos, *pos);
6aa8b732 430 return 0;
139bdb2d 431 }
6aa8b732
AK		 432 return 1;
433}
434
260746c0
AK		 435static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa)
436{
437 struct kvm_mmu_page *page_head = page_header(page_hpa);
438
5f1e0b6a 439 ASSERT(is_empty_shadow_page(page_hpa));
260746c0
AK		 440 list_del(&page_head->link);
441 page_head->page_hpa = page_hpa;
442 list_add(&page_head->link, &vcpu->free_pages);
443 ++vcpu->kvm->n_free_mmu_pages;
444}
445
cea0f0e7
AK		 446static unsigned kvm_page_table_hashfn(gfn_t gfn)
447{
448 return gfn;
449}
450
25c0de2c
AK		 451static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
452 u64 *parent_pte)
6aa8b732
AK		 453{
454 struct kvm_mmu_page *page;
455
456 if (list_empty(&vcpu->free_pages))
25c0de2c 457 return NULL;
6aa8b732
AK		 458
459 page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link);
460 list_del(&page->link);
461 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
462 ASSERT(is_empty_shadow_page(page->page_hpa));
463 page->slot_bitmap = 0;
cea0f0e7 464 page->multimapped = 0;
6aa8b732 465 page->parent_pte = parent_pte;
ebeace86 466 --vcpu->kvm->n_free_mmu_pages;
25c0de2c 467 return page;
6aa8b732
AK		 468}
469
714b93da
AK		 470static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
471 struct kvm_mmu_page *page, u64 *parent_pte)
cea0f0e7
AK		 472{
473 struct kvm_pte_chain *pte_chain;
474 struct hlist_node *node;
475 int i;
476
477 if (!parent_pte)
478 return;
479 if (!page->multimapped) {
480 u64 *old = page->parent_pte;
481
482 if (!old) {
483 page->parent_pte = parent_pte;
484 return;
485 }
486 page->multimapped = 1;
714b93da 487 pte_chain = mmu_alloc_pte_chain(vcpu);
cea0f0e7
AK		 488 INIT_HLIST_HEAD(&page->parent_ptes);
489 hlist_add_head(&pte_chain->link, &page->parent_ptes);
490 pte_chain->parent_ptes[0] = old;
491 }
492 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
493 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
494 continue;
495 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
496 if (!pte_chain->parent_ptes[i]) {
497 pte_chain->parent_ptes[i] = parent_pte;
498 return;
499 }
500 }
714b93da 501 pte_chain = mmu_alloc_pte_chain(vcpu);
cea0f0e7
AK		 502 BUG_ON(!pte_chain);
503 hlist_add_head(&pte_chain->link, &page->parent_ptes);
504 pte_chain->parent_ptes[0] = parent_pte;
505}
506
714b93da
AK		 507static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu,
508 struct kvm_mmu_page *page,
cea0f0e7
AK		 509 u64 *parent_pte)
510{
511 struct kvm_pte_chain *pte_chain;
512 struct hlist_node *node;
513 int i;
514
515 if (!page->multimapped) {
516 BUG_ON(page->parent_pte != parent_pte);
517 page->parent_pte = NULL;
518 return;
519 }
520 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
521 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
522 if (!pte_chain->parent_ptes[i])
523 break;
524 if (pte_chain->parent_ptes[i] != parent_pte)
525 continue;
697fe2e2
AK		 526 while (i + 1 < NR_PTE_CHAIN_ENTRIES
527 && pte_chain->parent_ptes[i + 1]) {
cea0f0e7
AK		 528 pte_chain->parent_ptes[i]
529 = pte_chain->parent_ptes[i + 1];
530 ++i;
531 }
532 pte_chain->parent_ptes[i] = NULL;
697fe2e2
AK		 533 if (i == 0) {
534 hlist_del(&pte_chain->link);
714b93da 535 mmu_free_pte_chain(vcpu, pte_chain);
697fe2e2
AK		 536 if (hlist_empty(&page->parent_ptes)) {
537 page->multimapped = 0;
538 page->parent_pte = NULL;
539 }
540 }
cea0f0e7
AK		 541 return;
542 }
543 BUG();
544}
545
546static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
547 gfn_t gfn)
548{
549 unsigned index;
550 struct hlist_head *bucket;
551 struct kvm_mmu_page *page;
552 struct hlist_node *node;
553
554 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
555 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
556 bucket = &vcpu->kvm->mmu_page_hash[index];
557 hlist_for_each_entry(page, node, bucket, hash_link)
558 if (page->gfn == gfn && !page->role.metaphysical) {
559 pgprintk("%s: found role %x\n",
560 __FUNCTION__, page->role.word);
561 return page;
562 }
563 return NULL;
564}
565
566static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
567 gfn_t gfn,
568 gva_t gaddr,
569 unsigned level,
570 int metaphysical,
d28c6cfb 571 unsigned hugepage_access,
cea0f0e7
AK		 572 u64 *parent_pte)
573{
574 union kvm_mmu_page_role role;
575 unsigned index;
576 unsigned quadrant;
577 struct hlist_head *bucket;
578 struct kvm_mmu_page *page;
579 struct hlist_node *node;
580
581 role.word = 0;
582 role.glevels = vcpu->mmu.root_level;
583 role.level = level;
584 role.metaphysical = metaphysical;
d28c6cfb 585 role.hugepage_access = hugepage_access;
cea0f0e7
AK		 586 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
587 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
588 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
589 role.quadrant = quadrant;
590 }
591 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
592 gfn, role.word);
593 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
594 bucket = &vcpu->kvm->mmu_page_hash[index];
595 hlist_for_each_entry(page, node, bucket, hash_link)
596 if (page->gfn == gfn && page->role.word == role.word) {
714b93da 597 mmu_page_add_parent_pte(vcpu, page, parent_pte);
cea0f0e7
AK		 598 pgprintk("%s: found\n", __FUNCTION__);
599 return page;
600 }
601 page = kvm_mmu_alloc_page(vcpu, parent_pte);
602 if (!page)
603 return page;
604 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
605 page->gfn = gfn;
606 page->role = role;
607 hlist_add_head(&page->hash_link, bucket);
374cbac0 608 if (!metaphysical)
714b93da 609 rmap_write_protect(vcpu, gfn);
cea0f0e7
AK		 610 return page;
611}
612
a436036b
AK		 613static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
614 struct kvm_mmu_page *page)
615{
697fe2e2
AK		 616 unsigned i;
617 u64 *pt;
618 u64 ent;
619
620 pt = __va(page->page_hpa);
621
622 if (page->role.level == PT_PAGE_TABLE_LEVEL) {
623 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
624 if (pt[i] & PT_PRESENT_MASK)
714b93da 625 rmap_remove(vcpu, &pt[i]);
697fe2e2
AK		 626 pt[i] = 0;
627 }
40907d57 628 kvm_arch_ops->tlb_flush(vcpu);
697fe2e2
AK		 629 return;
630 }
631
632 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
633 ent = pt[i];
634
635 pt[i] = 0;
636 if (!(ent & PT_PRESENT_MASK))
637 continue;
638 ent &= PT64_BASE_ADDR_MASK;
714b93da 639 mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
697fe2e2 640 }
a436036b
AK		 641}
642
cea0f0e7
AK		 643static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
644 struct kvm_mmu_page *page,
645 u64 *parent_pte)
646{
714b93da 647 mmu_page_remove_parent_pte(vcpu, page, parent_pte);
a436036b
AK		 648}
649
650static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
651 struct kvm_mmu_page *page)
652{
653 u64 *parent_pte;
654
655 while (page->multimapped || page->parent_pte) {
656 if (!page->multimapped)
657 parent_pte = page->parent_pte;
658 else {
659 struct kvm_pte_chain *chain;
660
661 chain = container_of(page->parent_ptes.first,
662 struct kvm_pte_chain, link);
663 parent_pte = chain->parent_ptes[0];
664 }
697fe2e2 665 BUG_ON(!parent_pte);
a436036b
AK		 666 kvm_mmu_put_page(vcpu, page, parent_pte);
667 *parent_pte = 0;
668 }
cc4529ef 669 kvm_mmu_page_unlink_children(vcpu, page);
3bb65a22
AK		 670 if (!page->root_count) {
671 hlist_del(&page->hash_link);
672 kvm_mmu_free_page(vcpu, page->page_hpa);
673 } else {
674 list_del(&page->link);
675 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
676 }
a436036b
AK		 677}
678
679static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
680{
681 unsigned index;
682 struct hlist_head *bucket;
683 struct kvm_mmu_page *page;
684 struct hlist_node *node, *n;
685 int r;
686
687 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
688 r = 0;
689 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
690 bucket = &vcpu->kvm->mmu_page_hash[index];
691 hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
692 if (page->gfn == gfn && !page->role.metaphysical) {
697fe2e2
AK		 693 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
694 page->role.word);
a436036b
AK		 695 kvm_mmu_zap_page(vcpu, page);
696 r = 1;
697 }
698 return r;
cea0f0e7
AK		 699}
700
6aa8b732
AK		 701static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
702{
703 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
704 struct kvm_mmu_page *page_head = page_header(__pa(pte));
705
706 __set_bit(slot, &page_head->slot_bitmap);
707}
708
709hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
710{
711 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
712
713 return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
714}
715
716hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
717{
718 struct kvm_memory_slot *slot;
719 struct page *page;
720
721 ASSERT((gpa & HPA_ERR_MASK) == 0);
722 slot = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
723 if (!slot)
724 return gpa | HPA_ERR_MASK;
725 page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
726 return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
727 | (gpa & (PAGE_SIZE-1));
728}
729
730hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
731{
732 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
733
734 if (gpa == UNMAPPED_GVA)
735 return UNMAPPED_GVA;
736 return gpa_to_hpa(vcpu, gpa);
737}
738
039576c0
AK		 739struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
740{
741 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
742
743 if (gpa == UNMAPPED_GVA)
744 return NULL;
745 return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
746}
747
6aa8b732
AK		 748static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
749{
750}
751
752static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
753{
754 int level = PT32E_ROOT_LEVEL;
755 hpa_t table_addr = vcpu->mmu.root_hpa;
756
757 for (; ; level--) {
758 u32 index = PT64_INDEX(v, level);
759 u64 *table;
cea0f0e7 760 u64 pte;
6aa8b732
AK		 761
762 ASSERT(VALID_PAGE(table_addr));
763 table = __va(table_addr);
764
765 if (level == 1) {
cea0f0e7
AK		 766 pte = table[index];
767 if (is_present_pte(pte) && is_writeble_pte(pte))
768 return 0;
6aa8b732
AK		 769 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
770 page_header_update_slot(vcpu->kvm, table, v);
771 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
772 PT_USER_MASK;
714b93da 773 rmap_add(vcpu, &table[index]);
6aa8b732
AK		 774 return 0;
775 }
776
777 if (table[index] == 0) {
25c0de2c 778 struct kvm_mmu_page *new_table;
cea0f0e7 779 gfn_t pseudo_gfn;
6aa8b732 780
cea0f0e7
AK		 781 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
782 >> PAGE_SHIFT;
783 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
784 v, level - 1,
d28c6cfb 785 1, 0, &table[index]);
25c0de2c 786 if (!new_table) {
6aa8b732
AK		 787 pgprintk("nonpaging_map: ENOMEM\n");
788 return -ENOMEM;
789 }
790
25c0de2c
AK		 791 table[index] = new_table->page_hpa | PT_PRESENT_MASK
792 | PT_WRITABLE_MASK | PT_USER_MASK;
6aa8b732
AK		 793 }
794 table_addr = table[index] & PT64_BASE_ADDR_MASK;
795 }
796}
797
17ac10ad
AK		 798static void mmu_free_roots(struct kvm_vcpu *vcpu)
799{
800 int i;
3bb65a22 801 struct kvm_mmu_page *page;
17ac10ad
AK		 802
803#ifdef CONFIG_X86_64
804 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
805 hpa_t root = vcpu->mmu.root_hpa;
806
807 ASSERT(VALID_PAGE(root));
3bb65a22
AK		 808 page = page_header(root);
809 --page->root_count;
17ac10ad
AK		 810 vcpu->mmu.root_hpa = INVALID_PAGE;
811 return;
812 }
813#endif
814 for (i = 0; i < 4; ++i) {
815 hpa_t root = vcpu->mmu.pae_root[i];
816
817 ASSERT(VALID_PAGE(root));
818 root &= PT64_BASE_ADDR_MASK;
3bb65a22
AK		 819 page = page_header(root);
820 --page->root_count;
17ac10ad
AK		 821 vcpu->mmu.pae_root[i] = INVALID_PAGE;
822 }
823 vcpu->mmu.root_hpa = INVALID_PAGE;
824}
825
826static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
827{
828 int i;
cea0f0e7 829 gfn_t root_gfn;
3bb65a22
AK		 830 struct kvm_mmu_page *page;
831
cea0f0e7 832 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
17ac10ad
AK		 833
834#ifdef CONFIG_X86_64
835 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
836 hpa_t root = vcpu->mmu.root_hpa;
837
838 ASSERT(!VALID_PAGE(root));
68a99f6d 839 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
d28c6cfb 840 PT64_ROOT_LEVEL, 0, 0, NULL);
68a99f6d 841 root = page->page_hpa;
3bb65a22 842 ++page->root_count;
17ac10ad
AK		 843 vcpu->mmu.root_hpa = root;
844 return;
845 }
846#endif
847 for (i = 0; i < 4; ++i) {
848 hpa_t root = vcpu->mmu.pae_root[i];
849
850 ASSERT(!VALID_PAGE(root));
cea0f0e7
AK		 851 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL)
852 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
853 else if (vcpu->mmu.root_level == 0)
854 root_gfn = 0;
68a99f6d 855 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
cea0f0e7 856 PT32_ROOT_LEVEL, !is_paging(vcpu),
d28c6cfb 857 0, NULL);
68a99f6d 858 root = page->page_hpa;
3bb65a22 859 ++page->root_count;
17ac10ad
AK		 860 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
861 }
862 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
863}
864
6aa8b732
AK		 865static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
866{
867 return vaddr;
868}
869
870static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
871 u32 error_code)
872{
6aa8b732 873 gpa_t addr = gva;
ebeace86 874 hpa_t paddr;
e2dec939 875 int r;
6aa8b732 876
e2dec939
AK		 877 r = mmu_topup_memory_caches(vcpu);
878 if (r)
879 return r;
714b93da 880
6aa8b732
AK		 881 ASSERT(vcpu);
882 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
883
6aa8b732 884
ebeace86 885 paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
6aa8b732 886
ebeace86
AK		 887 if (is_error_hpa(paddr))
888 return 1;
6aa8b732 889
ebeace86 890 return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
6aa8b732
AK		 891}
892
6aa8b732
AK		 893static void nonpaging_free(struct kvm_vcpu *vcpu)
894{
17ac10ad 895 mmu_free_roots(vcpu);
6aa8b732
AK		 896}
897
898static int nonpaging_init_context(struct kvm_vcpu *vcpu)
899{
900 struct kvm_mmu *context = &vcpu->mmu;
901
902 context->new_cr3 = nonpaging_new_cr3;
903 context->page_fault = nonpaging_page_fault;
6aa8b732
AK		 904 context->gva_to_gpa = nonpaging_gva_to_gpa;
905 context->free = nonpaging_free;
cea0f0e7 906 context->root_level = 0;
6aa8b732 907 context->shadow_root_level = PT32E_ROOT_LEVEL;
17ac10ad 908 mmu_alloc_roots(vcpu);
6aa8b732
AK		 909 ASSERT(VALID_PAGE(context->root_hpa));
910 kvm_arch_ops->set_cr3(vcpu, context->root_hpa);
911 return 0;
912}
913
6aa8b732
AK		 914static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
915{
6aa8b732
AK		 916 ++kvm_stat.tlb_flush;
917 kvm_arch_ops->tlb_flush(vcpu);
918}
919
920static void paging_new_cr3(struct kvm_vcpu *vcpu)
921{
374cbac0 922 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
cea0f0e7 923 mmu_free_roots(vcpu);
7f7417d6
IM		 924 if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES))
925 kvm_mmu_free_some_pages(vcpu);
cea0f0e7 926 mmu_alloc_roots(vcpu);
6aa8b732 927 kvm_mmu_flush_tlb(vcpu);
cea0f0e7 928 kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
6aa8b732
AK		 929}
930
6aa8b732
AK		 931static inline void set_pte_common(struct kvm_vcpu *vcpu,
932 u64 *shadow_pte,
933 gpa_t gaddr,
934 int dirty,
815af8d4
AK		 935 u64 access_bits,
936 gfn_t gfn)
6aa8b732
AK		 937{
938 hpa_t paddr;
939
940 *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
941 if (!dirty)
942 access_bits &= ~PT_WRITABLE_MASK;
cea0f0e7 943
374cbac0 944 paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
6aa8b732
AK		 945
946 *shadow_pte |= access_bits;
947
6aa8b732
AK		 948 if (is_error_hpa(paddr)) {
949 *shadow_pte |= gaddr;
950 *shadow_pte |= PT_SHADOW_IO_MARK;
951 *shadow_pte &= ~PT_PRESENT_MASK;
374cbac0 952 return;
6aa8b732 953 }
374cbac0
AK		 954
955 *shadow_pte |= paddr;
956
957 if (access_bits & PT_WRITABLE_MASK) {
958 struct kvm_mmu_page *shadow;
959
815af8d4 960 shadow = kvm_mmu_lookup_page(vcpu, gfn);
374cbac0
AK		 961 if (shadow) {
962 pgprintk("%s: found shadow page for %lx, marking ro\n",
815af8d4 963 __FUNCTION__, gfn);
374cbac0 964 access_bits &= ~PT_WRITABLE_MASK;
40907d57
AK		 965 if (is_writeble_pte(*shadow_pte)) {
966 *shadow_pte &= ~PT_WRITABLE_MASK;
967 kvm_arch_ops->tlb_flush(vcpu);
968 }
374cbac0
AK		 969 }
970 }
971
972 if (access_bits & PT_WRITABLE_MASK)
973 mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT);
974
975 page_header_update_slot(vcpu->kvm, shadow_pte, gaddr);
714b93da 976 rmap_add(vcpu, shadow_pte);
6aa8b732
AK		 977}
978
979static void inject_page_fault(struct kvm_vcpu *vcpu,
980 u64 addr,
981 u32 err_code)
982{
983 kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
984}
985
986static inline int fix_read_pf(u64 *shadow_ent)
987{
988 if ((*shadow_ent & PT_SHADOW_USER_MASK) &&
989 !(*shadow_ent & PT_USER_MASK)) {
990 /*
991 * If supervisor write protect is disabled, we shadow kernel
992 * pages as user pages so we can trap the write access.
993 */
994 *shadow_ent |= PT_USER_MASK;
995 *shadow_ent &= ~PT_WRITABLE_MASK;
996
997 return 1;
998
999 }
1000 return 0;
1001}
1002
6aa8b732
AK		 1003static void paging_free(struct kvm_vcpu *vcpu)
1004{
1005 nonpaging_free(vcpu);
1006}
1007
1008#define PTTYPE 64
1009#include "paging_tmpl.h"
1010#undef PTTYPE
1011
1012#define PTTYPE 32
1013#include "paging_tmpl.h"
1014#undef PTTYPE
1015
17ac10ad 1016static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
6aa8b732
AK		 1017{
1018 struct kvm_mmu *context = &vcpu->mmu;
1019
1020 ASSERT(is_pae(vcpu));
1021 context->new_cr3 = paging_new_cr3;
1022 context->page_fault = paging64_page_fault;
6aa8b732
AK		 1023 context->gva_to_gpa = paging64_gva_to_gpa;
1024 context->free = paging_free;
17ac10ad
AK		 1025 context->root_level = level;
1026 context->shadow_root_level = level;
1027 mmu_alloc_roots(vcpu);
6aa8b732
AK		 1028 ASSERT(VALID_PAGE(context->root_hpa));
1029 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
1030 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
1031 return 0;
1032}
1033
17ac10ad
AK		 1034static int paging64_init_context(struct kvm_vcpu *vcpu)
1035{
1036 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1037}
1038
6aa8b732
AK		 1039static int paging32_init_context(struct kvm_vcpu *vcpu)
1040{
1041 struct kvm_mmu *context = &vcpu->mmu;
1042
1043 context->new_cr3 = paging_new_cr3;
1044 context->page_fault = paging32_page_fault;
6aa8b732
AK		 1045 context->gva_to_gpa = paging32_gva_to_gpa;
1046 context->free = paging_free;
1047 context->root_level = PT32_ROOT_LEVEL;
1048 context->shadow_root_level = PT32E_ROOT_LEVEL;
17ac10ad 1049 mmu_alloc_roots(vcpu);
6aa8b732
AK		 1050 ASSERT(VALID_PAGE(context->root_hpa));
1051 kvm_arch_ops->set_cr3(vcpu, context->root_hpa |
1052 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
1053 return 0;
1054}
1055
1056static int paging32E_init_context(struct kvm_vcpu *vcpu)
1057{
17ac10ad 1058 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
6aa8b732
AK		 1059}
1060
1061static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1062{
1063 ASSERT(vcpu);
1064 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1065
1066 if (!is_paging(vcpu))
1067 return nonpaging_init_context(vcpu);
a9058ecd 1068 else if (is_long_mode(vcpu))
6aa8b732
AK		 1069 return paging64_init_context(vcpu);
1070 else if (is_pae(vcpu))
1071 return paging32E_init_context(vcpu);
1072 else
1073 return paging32_init_context(vcpu);
1074}
1075
1076static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1077{
1078 ASSERT(vcpu);
1079 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
1080 vcpu->mmu.free(vcpu);
1081 vcpu->mmu.root_hpa = INVALID_PAGE;
1082 }
1083}
1084
1085int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1086{
714b93da
AK		 1087 int r;
1088
6aa8b732 1089 destroy_kvm_mmu(vcpu);
714b93da
AK		 1090 r = init_kvm_mmu(vcpu);
1091 if (r < 0)
1092 goto out;
e2dec939 1093 r = mmu_topup_memory_caches(vcpu);
714b93da
AK		 1094out:
1095 return r;
6aa8b732
AK		 1096}
1097
ac1b714e
AK		 1098static void mmu_pre_write_zap_pte(struct kvm_vcpu *vcpu,
1099 struct kvm_mmu_page *page,
1100 u64 *spte)
1101{
1102 u64 pte;
1103 struct kvm_mmu_page *child;
1104
1105 pte = *spte;
1106 if (is_present_pte(pte)) {
1107 if (page->role.level == PT_PAGE_TABLE_LEVEL)
1108 rmap_remove(vcpu, spte);
1109 else {
1110 child = page_header(pte & PT64_BASE_ADDR_MASK);
1111 mmu_page_remove_parent_pte(vcpu, child, spte);
1112 }
1113 }
1114 *spte = 0;
1115}
1116
da4a00f0
AK		 1117void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
1118{
9b7a0325
AK		 1119 gfn_t gfn = gpa >> PAGE_SHIFT;
1120 struct kvm_mmu_page *page;
0e7bc4b9 1121 struct hlist_node *node, *n;
9b7a0325
AK		 1122 struct hlist_head *bucket;
1123 unsigned index;
1124 u64 *spte;
9b7a0325 1125 unsigned offset = offset_in_page(gpa);
0e7bc4b9 1126 unsigned pte_size;
9b7a0325 1127 unsigned page_offset;
0e7bc4b9 1128 unsigned misaligned;
9b7a0325 1129 int level;
86a5ba02 1130 int flooded = 0;
ac1b714e 1131 int npte;
9b7a0325 1132
da4a00f0 1133 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
86a5ba02
AK		 1134 if (gfn == vcpu->last_pt_write_gfn) {
1135 ++vcpu->last_pt_write_count;
1136 if (vcpu->last_pt_write_count >= 3)
1137 flooded = 1;
1138 } else {
1139 vcpu->last_pt_write_gfn = gfn;
1140 vcpu->last_pt_write_count = 1;
1141 }
9b7a0325
AK		 1142 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1143 bucket = &vcpu->kvm->mmu_page_hash[index];
0e7bc4b9 1144 hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
9b7a0325
AK		 1145 if (page->gfn != gfn || page->role.metaphysical)
1146 continue;
0e7bc4b9
AK		 1147 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1148 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
86a5ba02 1149 if (misaligned || flooded) {
0e7bc4b9
AK		 1150 /*
1151 * Misaligned accesses are too much trouble to fix
1152 * up; also, they usually indicate a page is not used
1153 * as a page table.
86a5ba02
AK		 1154 *
1155 * If we're seeing too many writes to a page,
1156 * it may no longer be a page table, or we may be
1157 * forking, in which case it is better to unmap the
1158 * page.
0e7bc4b9
AK		 1159 */
1160 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1161 gpa, bytes, page->role.word);
1162 kvm_mmu_zap_page(vcpu, page);
1163 continue;
1164 }
9b7a0325
AK		 1165 page_offset = offset;
1166 level = page->role.level;
ac1b714e 1167 npte = 1;
9b7a0325 1168 if (page->role.glevels == PT32_ROOT_LEVEL) {
ac1b714e
AK		 1169 page_offset <<= 1; /* 32->64 */
1170 /*
1171 * A 32-bit pde maps 4MB while the shadow pdes map
1172 * only 2MB. So we need to double the offset again
1173 * and zap two pdes instead of one.
1174 */
1175 if (level == PT32_ROOT_LEVEL) {
6b8d0f9b 1176 page_offset &= ~7; /* kill rounding error */
ac1b714e
AK		 1177 page_offset <<= 1;
1178 npte = 2;
1179 }
9b7a0325
AK		 1180 page_offset &= ~PAGE_MASK;
1181 }
1182 spte = __va(page->page_hpa);
1183 spte += page_offset / sizeof(*spte);
ac1b714e
AK		 1184 while (npte--) {
1185 mmu_pre_write_zap_pte(vcpu, page, spte);
1186 ++spte;
9b7a0325 1187 }
9b7a0325 1188 }
da4a00f0
AK		 1189}
1190
1191void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes)
1192{
1193}
1194
a436036b
AK		 1195int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1196{
1197 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
1198
1199 return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
1200}
1201
ebeace86
AK		 1202void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1203{
1204 while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
1205 struct kvm_mmu_page *page;
1206
1207 page = container_of(vcpu->kvm->active_mmu_pages.prev,
1208 struct kvm_mmu_page, link);
1209 kvm_mmu_zap_page(vcpu, page);
1210 }
1211}
1212EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);
1213
6aa8b732
AK		 1214static void free_mmu_pages(struct kvm_vcpu *vcpu)
1215{
f51234c2 1216 struct kvm_mmu_page *page;
6aa8b732 1217
f51234c2
AK		 1218 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1219 page = container_of(vcpu->kvm->active_mmu_pages.next,
1220 struct kvm_mmu_page, link);
1221 kvm_mmu_zap_page(vcpu, page);
1222 }
1223 while (!list_empty(&vcpu->free_pages)) {
6aa8b732
AK		 1224 page = list_entry(vcpu->free_pages.next,
1225 struct kvm_mmu_page, link);
1226 list_del(&page->link);
1227 __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT));
1228 page->page_hpa = INVALID_PAGE;
1229 }
17ac10ad 1230 free_page((unsigned long)vcpu->mmu.pae_root);
6aa8b732
AK		 1231}
1232
1233static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1234{
17ac10ad 1235 struct page *page;
6aa8b732
AK		 1236 int i;
1237
1238 ASSERT(vcpu);
1239
1240 for (i = 0; i < KVM_NUM_MMU_PAGES; i++) {
6aa8b732
AK		 1241 struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i];
1242
1243 INIT_LIST_HEAD(&page_header->link);
17ac10ad 1244 if ((page = alloc_page(GFP_KERNEL)) == NULL)
6aa8b732 1245 goto error_1;
5972e953 1246 set_page_private(page, (unsigned long)page_header);
6aa8b732
AK		 1247 page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT;
1248 memset(__va(page_header->page_hpa), 0, PAGE_SIZE);
1249 list_add(&page_header->link, &vcpu->free_pages);
ebeace86 1250 ++vcpu->kvm->n_free_mmu_pages;
6aa8b732 1251 }
17ac10ad
AK		 1252
1253 /*
1254 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1255 * Therefore we need to allocate shadow page tables in the first
1256 * 4GB of memory, which happens to fit the DMA32 zone.
1257 */
1258 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1259 if (!page)
1260 goto error_1;
1261 vcpu->mmu.pae_root = page_address(page);
1262 for (i = 0; i < 4; ++i)
1263 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1264
6aa8b732
AK		 1265 return 0;
1266
1267error_1:
1268 free_mmu_pages(vcpu);
1269 return -ENOMEM;
1270}
1271
8018c27b 1272int kvm_mmu_create(struct kvm_vcpu *vcpu)
6aa8b732 1273{
6aa8b732
AK		 1274 ASSERT(vcpu);
1275 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1276 ASSERT(list_empty(&vcpu->free_pages));
1277
8018c27b
IM		 1278 return alloc_mmu_pages(vcpu);
1279}
6aa8b732 1280
8018c27b
IM		 1281int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1282{
1283 ASSERT(vcpu);
1284 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1285 ASSERT(!list_empty(&vcpu->free_pages));
2c264957 1286
8018c27b 1287 return init_kvm_mmu(vcpu);
6aa8b732
AK		 1288}
1289
1290void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1291{
1292 ASSERT(vcpu);
1293
1294 destroy_kvm_mmu(vcpu);
1295 free_mmu_pages(vcpu);
714b93da 1296 mmu_free_memory_caches(vcpu);
6aa8b732
AK		 1297}
1298
714b93da 1299void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
6aa8b732 1300{
714b93da 1301 struct kvm *kvm = vcpu->kvm;
6aa8b732
AK		 1302 struct kvm_mmu_page *page;
1303
1304 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1305 int i;
1306 u64 *pt;
1307
1308 if (!test_bit(slot, &page->slot_bitmap))
1309 continue;
1310
1311 pt = __va(page->page_hpa);
1312 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1313 /* avoid RMW */
cd4a4e53 1314 if (pt[i] & PT_WRITABLE_MASK) {
714b93da 1315 rmap_remove(vcpu, &pt[i]);
6aa8b732 1316 pt[i] &= ~PT_WRITABLE_MASK;
cd4a4e53 1317 }
6aa8b732
AK		 1318 }
1319}
37a7d8b0
AK		 1320
1321#ifdef AUDIT
1322
1323static const char *audit_msg;
1324
1325static gva_t canonicalize(gva_t gva)
1326{
1327#ifdef CONFIG_X86_64
1328 gva = (long long)(gva << 16) >> 16;
1329#endif
1330 return gva;
1331}
1332
1333static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1334 gva_t va, int level)
1335{
1336 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1337 int i;
1338 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1339
1340 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1341 u64 ent = pt[i];
1342
1343 if (!ent & PT_PRESENT_MASK)
1344 continue;
1345
1346 va = canonicalize(va);
1347 if (level > 1)
1348 audit_mappings_page(vcpu, ent, va, level - 1);
1349 else {
1350 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
1351 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
1352
1353 if ((ent & PT_PRESENT_MASK)
1354 && (ent & PT64_BASE_ADDR_MASK) != hpa)
1355 printk(KERN_ERR "audit error: (%s) levels %d"
1356 " gva %lx gpa %llx hpa %llx ent %llx\n",
1357 audit_msg, vcpu->mmu.root_level,
1358 va, gpa, hpa, ent);
1359 }
1360 }
1361}
1362
1363static void audit_mappings(struct kvm_vcpu *vcpu)
1364{
1ea252af 1365 unsigned i;
37a7d8b0
AK		 1366
1367 if (vcpu->mmu.root_level == 4)
1368 audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
1369 else
1370 for (i = 0; i < 4; ++i)
1371 if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
1372 audit_mappings_page(vcpu,
1373 vcpu->mmu.pae_root[i],
1374 i << 30,
1375 2);
1376}
1377
1378static int count_rmaps(struct kvm_vcpu *vcpu)
1379{
1380 int nmaps = 0;
1381 int i, j, k;
1382
1383 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1384 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1385 struct kvm_rmap_desc *d;
1386
1387 for (j = 0; j < m->npages; ++j) {
1388 struct page *page = m->phys_mem[j];
1389
1390 if (!page->private)
1391 continue;
1392 if (!(page->private & 1)) {
1393 ++nmaps;
1394 continue;
1395 }
1396 d = (struct kvm_rmap_desc *)(page->private & ~1ul);
1397 while (d) {
1398 for (k = 0; k < RMAP_EXT; ++k)
1399 if (d->shadow_ptes[k])
1400 ++nmaps;
1401 else
1402 break;
1403 d = d->more;
1404 }
1405 }
1406 }
1407 return nmaps;
1408}
1409
1410static int count_writable_mappings(struct kvm_vcpu *vcpu)
1411{
1412 int nmaps = 0;
1413 struct kvm_mmu_page *page;
1414 int i;
1415
1416 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1417 u64 *pt = __va(page->page_hpa);
1418
1419 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1420 continue;
1421
1422 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1423 u64 ent = pt[i];
1424
1425 if (!(ent & PT_PRESENT_MASK))
1426 continue;
1427 if (!(ent & PT_WRITABLE_MASK))
1428 continue;
1429 ++nmaps;
1430 }
1431 }
1432 return nmaps;
1433}
1434
1435static void audit_rmap(struct kvm_vcpu *vcpu)
1436{
1437 int n_rmap = count_rmaps(vcpu);
1438 int n_actual = count_writable_mappings(vcpu);
1439
1440 if (n_rmap != n_actual)
1441 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1442 __FUNCTION__, audit_msg, n_rmap, n_actual);
1443}
1444
1445static void audit_write_protection(struct kvm_vcpu *vcpu)
1446{
1447 struct kvm_mmu_page *page;
1448
1449 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1450 hfn_t hfn;
1451 struct page *pg;
1452
1453 if (page->role.metaphysical)
1454 continue;
1455
1456 hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
1457 >> PAGE_SHIFT;
1458 pg = pfn_to_page(hfn);
1459 if (pg->private)
1460 printk(KERN_ERR "%s: (%s) shadow page has writable"
1461 " mappings: gfn %lx role %x\n",
1462 __FUNCTION__, audit_msg, page->gfn,
1463 page->role.word);
1464 }
1465}
1466
1467static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1468{
1469 int olddbg = dbg;
1470
1471 dbg = 0;
1472 audit_msg = msg;
1473 audit_rmap(vcpu);
1474 audit_write_protection(vcpu);
1475 audit_mappings(vcpu);
1476 dbg = olddbg;
1477}
1478
1479#endif
Linux kernel - Deliverables
This page took 0.12776 seconds and 5 git commands to generate.