projects / deliverable / linux.git / blob
? search:


652d56c081f7fb0a3fbd015e180dbbdd3478a77d
[deliverable/linux.git] / arch / x86 / kvm / paging_tmpl.h
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 *
12 * Authors:
13 * Yaniv Kamay <yaniv@qumranet.com>
14 * Avi Kivity <avi@qumranet.com>
15 *
16 * This work is licensed under the terms of the GNU GPL, version 2. See
17 * the COPYING file in the top-level directory.
18 *
19 */
20
21 /*
22 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
23 * so the code in this file is compiled twice, once per pte size.
24 */
25
26 #if PTTYPE == 64
27 #define pt_element_t u64
28 #define guest_walker guest_walker64
29 #define FNAME(name) paging##64_##name
30 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
31 #define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
32 #define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
33 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
34 #define PT_LEVEL_BITS PT64_LEVEL_BITS
35 #ifdef CONFIG_X86_64
36 #define PT_MAX_FULL_LEVELS 4
37 #define CMPXCHG cmpxchg
38 #else
39 #define CMPXCHG cmpxchg64
40 #define PT_MAX_FULL_LEVELS 2
41 #endif
42 #elif PTTYPE == 32
43 #define pt_element_t u32
44 #define guest_walker guest_walker32
45 #define FNAME(name) paging##32_##name
46 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
47 #define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
48 #define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
49 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
50 #define PT_LEVEL_BITS PT32_LEVEL_BITS
51 #define PT_MAX_FULL_LEVELS 2
52 #define CMPXCHG cmpxchg
53 #else
54 #error Invalid PTTYPE value
55 #endif
56
57 #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
58 #define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
59
60 /*
61 * The guest_walker structure emulates the behavior of the hardware page
62 * table walker.
63 */
64 struct guest_walker {
65 int level;
66 gfn_t table_gfn[PT_MAX_FULL_LEVELS];
67 pt_element_t ptes[PT_MAX_FULL_LEVELS];
68 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
69 gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
70 unsigned pt_access;
71 unsigned pte_access;
72 gfn_t gfn;
73 struct x86_exception fault;
74 };
75
76 static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
77 {
78 return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
79 }
80
81 static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
82 gfn_t table_gfn, unsigned index,
83 pt_element_t orig_pte, pt_element_t new_pte)
84 {
85 pt_element_t ret;
86 pt_element_t *table;
87 struct page *page;
88 gpa_t gpa;
89
90 gpa = mmu->translate_gpa(vcpu, table_gfn << PAGE_SHIFT,
91 PFERR_USER_MASK|PFERR_WRITE_MASK);
92 if (gpa == UNMAPPED_GVA)
93 return -EFAULT;
94
95 page = gfn_to_page(vcpu->kvm, gpa_to_gfn(gpa));
96
97 table = kmap_atomic(page, KM_USER0);
98 ret = CMPXCHG(&table[index], orig_pte, new_pte);
99 kunmap_atomic(table, KM_USER0);
100
101 kvm_release_page_dirty(page);
102
103 return (ret != orig_pte);
104 }
105
106 static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
107 {
108 unsigned access;
109
110 access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
111 #if PTTYPE == 64
112 if (vcpu->arch.mmu.nx)
113 access &= ~(gpte >> PT64_NX_SHIFT);
114 #endif
115 return access;
116 }
117
118 /*
119 * Fetch a guest pte for a guest virtual address
120 */
121 static int FNAME(walk_addr_generic)(struct guest_walker *walker,
122 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
123 gva_t addr, u32 access)
124 {
125 pt_element_t pte;
126 pt_element_t __user *ptep_user;
127 gfn_t table_gfn;
128 unsigned index, pt_access, uninitialized_var(pte_access);
129 gpa_t pte_gpa;
130 bool eperm, present, rsvd_fault;
131 int offset, write_fault, user_fault, fetch_fault;
132
133 write_fault = access & PFERR_WRITE_MASK;
134 user_fault = access & PFERR_USER_MASK;
135 fetch_fault = access & PFERR_FETCH_MASK;
136
137 trace_kvm_mmu_pagetable_walk(addr, write_fault, user_fault,
138 fetch_fault);
139 walk:
140 present = true;
141 eperm = rsvd_fault = false;
142 walker->level = mmu->root_level;
143 pte = mmu->get_cr3(vcpu);
144
145 #if PTTYPE == 64
146 if (walker->level == PT32E_ROOT_LEVEL) {
147 pte = kvm_pdptr_read_mmu(vcpu, mmu, (addr >> 30) & 3);
148 trace_kvm_mmu_paging_element(pte, walker->level);
149 if (!is_present_gpte(pte)) {
150 present = false;
151 goto error;
152 }
153 --walker->level;
154 }
155 #endif
156 ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
157 (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0);
158
159 pt_access = ACC_ALL;
160
161 for (;;) {
162 gfn_t real_gfn;
163 unsigned long host_addr;
164
165 index = PT_INDEX(addr, walker->level);
166
167 table_gfn = gpte_to_gfn(pte);
168 offset = index * sizeof(pt_element_t);
169 pte_gpa = gfn_to_gpa(table_gfn) + offset;
170 walker->table_gfn[walker->level - 1] = table_gfn;
171 walker->pte_gpa[walker->level - 1] = pte_gpa;
172
173 real_gfn = mmu->translate_gpa(vcpu, gfn_to_gpa(table_gfn),
174 PFERR_USER_MASK|PFERR_WRITE_MASK);
175 if (unlikely(real_gfn == UNMAPPED_GVA)) {
176 present = false;
177 break;
178 }
179 real_gfn = gpa_to_gfn(real_gfn);
180
181 host_addr = gfn_to_hva(vcpu->kvm, real_gfn);
182 if (unlikely(kvm_is_error_hva(host_addr))) {
183 present = false;
184 break;
185 }
186
187 ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
188 if (unlikely(get_user(pte, ptep_user))) {
189 present = false;
190 break;
191 }
192
193 trace_kvm_mmu_paging_element(pte, walker->level);
194
195 if (unlikely(!is_present_gpte(pte))) {
196 present = false;
197 break;
198 }
199
200 if (unlikely(is_rsvd_bits_set(&vcpu->arch.mmu, pte,
201 walker->level))) {
202 rsvd_fault = true;
203 break;
204 }
205
206 if (unlikely(write_fault && !is_writable_pte(pte)
207 && (user_fault || is_write_protection(vcpu))))
208 eperm = true;
209
210 if (unlikely(user_fault && !(pte & PT_USER_MASK)))
211 eperm = true;
212
213 #if PTTYPE == 64
214 if (unlikely(fetch_fault && (pte & PT64_NX_MASK)))
215 eperm = true;
216 #endif
217
218 if (!eperm && !rsvd_fault
219 && unlikely(!(pte & PT_ACCESSED_MASK))) {
220 int ret;
221 trace_kvm_mmu_set_accessed_bit(table_gfn, index,
222 sizeof(pte));
223 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, table_gfn,
224 index, pte, pte|PT_ACCESSED_MASK);
225 if (ret < 0) {
226 present = false;
227 break;
228 } else if (ret)
229 goto walk;
230
231 mark_page_dirty(vcpu->kvm, table_gfn);
232 pte |= PT_ACCESSED_MASK;
233 }
234
235 pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
236
237 walker->ptes[walker->level - 1] = pte;
238
239 if ((walker->level == PT_PAGE_TABLE_LEVEL) ||
240 ((walker->level == PT_DIRECTORY_LEVEL) &&
241 is_large_pte(pte) &&
242 (PTTYPE == 64 || is_pse(vcpu))) ||
243 ((walker->level == PT_PDPE_LEVEL) &&
244 is_large_pte(pte) &&
245 mmu->root_level == PT64_ROOT_LEVEL)) {
246 int lvl = walker->level;
247 gpa_t real_gpa;
248 gfn_t gfn;
249 u32 ac;
250
251 gfn = gpte_to_gfn_lvl(pte, lvl);
252 gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT;
253
254 if (PTTYPE == 32 &&
255 walker->level == PT_DIRECTORY_LEVEL &&
256 is_cpuid_PSE36())
257 gfn += pse36_gfn_delta(pte);
258
259 ac = write_fault | fetch_fault | user_fault;
260
261 real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn),
262 ac);
263 if (real_gpa == UNMAPPED_GVA)
264 return 0;
265
266 walker->gfn = real_gpa >> PAGE_SHIFT;
267
268 break;
269 }
270
271 pt_access = pte_access;
272 --walker->level;
273 }
274
275 if (unlikely(!present || eperm || rsvd_fault))
276 goto error;
277
278 if (write_fault && unlikely(!is_dirty_gpte(pte))) {
279 int ret;
280
281 trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
282 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, table_gfn, index, pte,
283 pte|PT_DIRTY_MASK);
284 if (ret < 0) {
285 present = false;
286 goto error;
287 } else if (ret)
288 goto walk;
289
290 mark_page_dirty(vcpu->kvm, table_gfn);
291 pte |= PT_DIRTY_MASK;
292 walker->ptes[walker->level - 1] = pte;
293 }
294
295 walker->pt_access = pt_access;
296 walker->pte_access = pte_access;
297 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
298 __func__, (u64)pte, pte_access, pt_access);
299 return 1;
300
301 error:
302 walker->fault.vector = PF_VECTOR;
303 walker->fault.error_code_valid = true;
304 walker->fault.error_code = 0;
305 if (present)
306 walker->fault.error_code |= PFERR_PRESENT_MASK;
307
308 walker->fault.error_code |= write_fault | user_fault;
309
310 if (fetch_fault && mmu->nx)
311 walker->fault.error_code |= PFERR_FETCH_MASK;
312 if (rsvd_fault)
313 walker->fault.error_code |= PFERR_RSVD_MASK;
314
315 walker->fault.address = addr;
316 walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
317
318 trace_kvm_mmu_walker_error(walker->fault.error_code);
319 return 0;
320 }
321
322 static int FNAME(walk_addr)(struct guest_walker *walker,
323 struct kvm_vcpu *vcpu, gva_t addr, u32 access)
324 {
325 return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.mmu, addr,
326 access);
327 }
328
329 static int FNAME(walk_addr_nested)(struct guest_walker *walker,
330 struct kvm_vcpu *vcpu, gva_t addr,
331 u32 access)
332 {
333 return FNAME(walk_addr_generic)(walker, vcpu, &vcpu->arch.nested_mmu,
334 addr, access);
335 }
336
337 static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
338 struct kvm_mmu_page *sp, u64 *spte,
339 pt_element_t gpte)
340 {
341 u64 nonpresent = shadow_trap_nonpresent_pte;
342
343 if (is_rsvd_bits_set(&vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL))
344 goto no_present;
345
346 if (!is_present_gpte(gpte)) {
347 if (!sp->unsync)
348 nonpresent = shadow_notrap_nonpresent_pte;
349 goto no_present;
350 }
351
352 if (!(gpte & PT_ACCESSED_MASK))
353 goto no_present;
354
355 return false;
356
357 no_present:
358 drop_spte(vcpu->kvm, spte, nonpresent);
359 return true;
360 }
361
362 static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
363 u64 *spte, const void *pte)
364 {
365 pt_element_t gpte;
366 unsigned pte_access;
367 pfn_t pfn;
368
369 gpte = *(const pt_element_t *)pte;
370 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
371 return;
372
373 pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
374 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
375 pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
376 if (is_error_pfn(pfn)) {
377 kvm_release_pfn_clean(pfn);
378 return;
379 }
380
381 /*
382 * we call mmu_set_spte() with host_writable = true because that
383 * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
384 */
385 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
386 is_dirty_gpte(gpte), NULL, PT_PAGE_TABLE_LEVEL,
387 gpte_to_gfn(gpte), pfn, true, true);
388 }
389
390 static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu,
391 struct guest_walker *gw, int level)
392 {
393 pt_element_t curr_pte;
394 gpa_t base_gpa, pte_gpa = gw->pte_gpa[level - 1];
395 u64 mask;
396 int r, index;
397
398 if (level == PT_PAGE_TABLE_LEVEL) {
399 mask = PTE_PREFETCH_NUM * sizeof(pt_element_t) - 1;
400 base_gpa = pte_gpa & ~mask;
401 index = (pte_gpa - base_gpa) / sizeof(pt_element_t);
402
403 r = kvm_read_guest_atomic(vcpu->kvm, base_gpa,
404 gw->prefetch_ptes, sizeof(gw->prefetch_ptes));
405 curr_pte = gw->prefetch_ptes[index];
406 } else
407 r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa,
408 &curr_pte, sizeof(curr_pte));
409
410 return r || curr_pte != gw->ptes[level - 1];
411 }
412
413 static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
414 u64 *sptep)
415 {
416 struct kvm_mmu_page *sp;
417 pt_element_t *gptep = gw->prefetch_ptes;
418 u64 *spte;
419 int i;
420
421 sp = page_header(__pa(sptep));
422
423 if (sp->role.level > PT_PAGE_TABLE_LEVEL)
424 return;
425
426 if (sp->role.direct)
427 return __direct_pte_prefetch(vcpu, sp, sptep);
428
429 i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
430 spte = sp->spt + i;
431
432 for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
433 pt_element_t gpte;
434 unsigned pte_access;
435 gfn_t gfn;
436 pfn_t pfn;
437 bool dirty;
438
439 if (spte == sptep)
440 continue;
441
442 if (*spte != shadow_trap_nonpresent_pte)
443 continue;
444
445 gpte = gptep[i];
446
447 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
448 continue;
449
450 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
451 gfn = gpte_to_gfn(gpte);
452 dirty = is_dirty_gpte(gpte);
453 pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
454 (pte_access & ACC_WRITE_MASK) && dirty);
455 if (is_error_pfn(pfn)) {
456 kvm_release_pfn_clean(pfn);
457 break;
458 }
459
460 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
461 dirty, NULL, PT_PAGE_TABLE_LEVEL, gfn,
462 pfn, true, true);
463 }
464 }
465
466 /*
467 * Fetch a shadow pte for a specific level in the paging hierarchy.
468 */
469 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
470 struct guest_walker *gw,
471 int user_fault, int write_fault, int hlevel,
472 int *ptwrite, pfn_t pfn, bool map_writable,
473 bool prefault)
474 {
475 unsigned access = gw->pt_access;
476 struct kvm_mmu_page *sp = NULL;
477 bool dirty = is_dirty_gpte(gw->ptes[gw->level - 1]);
478 int top_level;
479 unsigned direct_access;
480 struct kvm_shadow_walk_iterator it;
481
482 if (!is_present_gpte(gw->ptes[gw->level - 1]))
483 return NULL;
484
485 direct_access = gw->pt_access & gw->pte_access;
486 if (!dirty)
487 direct_access &= ~ACC_WRITE_MASK;
488
489 top_level = vcpu->arch.mmu.root_level;
490 if (top_level == PT32E_ROOT_LEVEL)
491 top_level = PT32_ROOT_LEVEL;
492 /*
493 * Verify that the top-level gpte is still there. Since the page
494 * is a root page, it is either write protected (and cannot be
495 * changed from now on) or it is invalid (in which case, we don't
496 * really care if it changes underneath us after this point).
497 */
498 if (FNAME(gpte_changed)(vcpu, gw, top_level))
499 goto out_gpte_changed;
500
501 for (shadow_walk_init(&it, vcpu, addr);
502 shadow_walk_okay(&it) && it.level > gw->level;
503 shadow_walk_next(&it)) {
504 gfn_t table_gfn;
505
506 drop_large_spte(vcpu, it.sptep);
507
508 sp = NULL;
509 if (!is_shadow_present_pte(*it.sptep)) {
510 table_gfn = gw->table_gfn[it.level - 2];
511 sp = kvm_mmu_get_page(vcpu, table_gfn, addr, it.level-1,
512 false, access, it.sptep);
513 }
514
515 /*
516 * Verify that the gpte in the page we've just write
517 * protected is still there.
518 */
519 if (FNAME(gpte_changed)(vcpu, gw, it.level - 1))
520 goto out_gpte_changed;
521
522 if (sp)
523 link_shadow_page(it.sptep, sp);
524 }
525
526 for (;
527 shadow_walk_okay(&it) && it.level > hlevel;
528 shadow_walk_next(&it)) {
529 gfn_t direct_gfn;
530
531 validate_direct_spte(vcpu, it.sptep, direct_access);
532
533 drop_large_spte(vcpu, it.sptep);
534
535 if (is_shadow_present_pte(*it.sptep))
536 continue;
537
538 direct_gfn = gw->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
539
540 sp = kvm_mmu_get_page(vcpu, direct_gfn, addr, it.level-1,
541 true, direct_access, it.sptep);
542 link_shadow_page(it.sptep, sp);
543 }
544
545 mmu_set_spte(vcpu, it.sptep, access, gw->pte_access & access,
546 user_fault, write_fault, dirty, ptwrite, it.level,
547 gw->gfn, pfn, prefault, map_writable);
548 FNAME(pte_prefetch)(vcpu, gw, it.sptep);
549
550 return it.sptep;
551
552 out_gpte_changed:
553 if (sp)
554 kvm_mmu_put_page(sp, it.sptep);
555 kvm_release_pfn_clean(pfn);
556 return NULL;
557 }
558
559 /*
560 * Page fault handler. There are several causes for a page fault:
561 * - there is no shadow pte for the guest pte
562 * - write access through a shadow pte marked read only so that we can set
563 * the dirty bit
564 * - write access to a shadow pte marked read only so we can update the page
565 * dirty bitmap, when userspace requests it
566 * - mmio access; in this case we will never install a present shadow pte
567 * - normal guest page fault due to the guest pte marked not present, not
568 * writable, or not executable
569 *
570 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
571 * a negative value on error.
572 */
573 static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code,
574 bool prefault)
575 {
576 int write_fault = error_code & PFERR_WRITE_MASK;
577 int user_fault = error_code & PFERR_USER_MASK;
578 struct guest_walker walker;
579 u64 *sptep;
580 int write_pt = 0;
581 int r;
582 pfn_t pfn;
583 int level = PT_PAGE_TABLE_LEVEL;
584 int force_pt_level;
585 unsigned long mmu_seq;
586 bool map_writable;
587
588 pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
589
590 r = mmu_topup_memory_caches(vcpu);
591 if (r)
592 return r;
593
594 /*
595 * Look up the guest pte for the faulting address.
596 */
597 r = FNAME(walk_addr)(&walker, vcpu, addr, error_code);
598
599 /*
600 * The page is not mapped by the guest. Let the guest handle it.
601 */
602 if (!r) {
603 pgprintk("%s: guest page fault\n", __func__);
604 if (!prefault) {
605 inject_page_fault(vcpu, &walker.fault);
606 /* reset fork detector */
607 vcpu->arch.last_pt_write_count = 0;
608 }
609 return 0;
610 }
611
612 if (walker.level >= PT_DIRECTORY_LEVEL)
613 force_pt_level = mapping_level_dirty_bitmap(vcpu, walker.gfn);
614 else
615 force_pt_level = 1;
616 if (!force_pt_level) {
617 level = min(walker.level, mapping_level(vcpu, walker.gfn));
618 walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
619 }
620
621 mmu_seq = vcpu->kvm->mmu_notifier_seq;
622 smp_rmb();
623
624 if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault,
625 &map_writable))
626 return 0;
627
628 /* mmio */
629 if (is_error_pfn(pfn))
630 return kvm_handle_bad_page(vcpu->kvm, walker.gfn, pfn);
631
632 spin_lock(&vcpu->kvm->mmu_lock);
633 if (mmu_notifier_retry(vcpu, mmu_seq))
634 goto out_unlock;
635
636 trace_kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT);
637 kvm_mmu_free_some_pages(vcpu);
638 if (!force_pt_level)
639 transparent_hugepage_adjust(vcpu, &walker.gfn, &pfn, &level);
640 sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
641 level, &write_pt, pfn, map_writable, prefault);
642 (void)sptep;
643 pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__,
644 sptep, *sptep, write_pt);
645
646 if (!write_pt)
647 vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
648
649 ++vcpu->stat.pf_fixed;
650 trace_kvm_mmu_audit(vcpu, AUDIT_POST_PAGE_FAULT);
651 spin_unlock(&vcpu->kvm->mmu_lock);
652
653 return write_pt;
654
655 out_unlock:
656 spin_unlock(&vcpu->kvm->mmu_lock);
657 kvm_release_pfn_clean(pfn);
658 return 0;
659 }
660
661 static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
662 {
663 struct kvm_shadow_walk_iterator iterator;
664 struct kvm_mmu_page *sp;
665 gpa_t pte_gpa = -1;
666 int level;
667 u64 *sptep;
668 int need_flush = 0;
669
670 spin_lock(&vcpu->kvm->mmu_lock);
671
672 for_each_shadow_entry(vcpu, gva, iterator) {
673 level = iterator.level;
674 sptep = iterator.sptep;
675
676 sp = page_header(__pa(sptep));
677 if (is_last_spte(*sptep, level)) {
678 int offset, shift;
679
680 if (!sp->unsync)
681 break;
682
683 shift = PAGE_SHIFT -
684 (PT_LEVEL_BITS - PT64_LEVEL_BITS) * level;
685 offset = sp->role.quadrant << shift;
686
687 pte_gpa = (sp->gfn << PAGE_SHIFT) + offset;
688 pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t);
689
690 if (is_shadow_present_pte(*sptep)) {
691 if (is_large_pte(*sptep))
692 --vcpu->kvm->stat.lpages;
693 drop_spte(vcpu->kvm, sptep,
694 shadow_trap_nonpresent_pte);
695 need_flush = 1;
696 } else
697 __set_spte(sptep, shadow_trap_nonpresent_pte);
698 break;
699 }
700
701 if (!is_shadow_present_pte(*sptep) || !sp->unsync_children)
702 break;
703 }
704
705 if (need_flush)
706 kvm_flush_remote_tlbs(vcpu->kvm);
707
708 atomic_inc(&vcpu->kvm->arch.invlpg_counter);
709
710 spin_unlock(&vcpu->kvm->mmu_lock);
711
712 if (pte_gpa == -1)
713 return;
714
715 if (mmu_topup_memory_caches(vcpu))
716 return;
717 kvm_mmu_pte_write(vcpu, pte_gpa, NULL, sizeof(pt_element_t), 0);
718 }
719
720 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
721 struct x86_exception *exception)
722 {
723 struct guest_walker walker;
724 gpa_t gpa = UNMAPPED_GVA;
725 int r;
726
727 r = FNAME(walk_addr)(&walker, vcpu, vaddr, access);
728
729 if (r) {
730 gpa = gfn_to_gpa(walker.gfn);
731 gpa |= vaddr & ~PAGE_MASK;
732 } else if (exception)
733 *exception = walker.fault;
734
735 return gpa;
736 }
737
738 static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gva_t vaddr,
739 u32 access,
740 struct x86_exception *exception)
741 {
742 struct guest_walker walker;
743 gpa_t gpa = UNMAPPED_GVA;
744 int r;
745
746 r = FNAME(walk_addr_nested)(&walker, vcpu, vaddr, access);
747
748 if (r) {
749 gpa = gfn_to_gpa(walker.gfn);
750 gpa |= vaddr & ~PAGE_MASK;
751 } else if (exception)
752 *exception = walker.fault;
753
754 return gpa;
755 }
756
757 static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
758 struct kvm_mmu_page *sp)
759 {
760 int i, j, offset, r;
761 pt_element_t pt[256 / sizeof(pt_element_t)];
762 gpa_t pte_gpa;
763
764 if (sp->role.direct
765 || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) {
766 nonpaging_prefetch_page(vcpu, sp);
767 return;
768 }
769
770 pte_gpa = gfn_to_gpa(sp->gfn);
771 if (PTTYPE == 32) {
772 offset = sp->role.quadrant << PT64_LEVEL_BITS;
773 pte_gpa += offset * sizeof(pt_element_t);
774 }
775
776 for (i = 0; i < PT64_ENT_PER_PAGE; i += ARRAY_SIZE(pt)) {
777 r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa, pt, sizeof pt);
778 pte_gpa += ARRAY_SIZE(pt) * sizeof(pt_element_t);
779 for (j = 0; j < ARRAY_SIZE(pt); ++j)
780 if (r || is_present_gpte(pt[j]))
781 sp->spt[i+j] = shadow_trap_nonpresent_pte;
782 else
783 sp->spt[i+j] = shadow_notrap_nonpresent_pte;
784 }
785 }
786
787 /*
788 * Using the cached information from sp->gfns is safe because:
789 * - The spte has a reference to the struct page, so the pfn for a given gfn
790 * can't change unless all sptes pointing to it are nuked first.
791 *
792 * Note:
793 * We should flush all tlbs if spte is dropped even though guest is
794 * responsible for it. Since if we don't, kvm_mmu_notifier_invalidate_page
795 * and kvm_mmu_notifier_invalidate_range_start detect the mapping page isn't
796 * used by guest then tlbs are not flushed, so guest is allowed to access the
797 * freed pages.
798 * And we increase kvm->tlbs_dirty to delay tlbs flush in this case.
799 */
800 static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
801 {
802 int i, offset, nr_present;
803 bool host_writable;
804 gpa_t first_pte_gpa;
805
806 offset = nr_present = 0;
807
808 /* direct kvm_mmu_page can not be unsync. */
809 BUG_ON(sp->role.direct);
810
811 if (PTTYPE == 32)
812 offset = sp->role.quadrant << PT64_LEVEL_BITS;
813
814 first_pte_gpa = gfn_to_gpa(sp->gfn) + offset * sizeof(pt_element_t);
815
816 for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
817 unsigned pte_access;
818 pt_element_t gpte;
819 gpa_t pte_gpa;
820 gfn_t gfn;
821
822 if (!is_shadow_present_pte(sp->spt[i]))
823 continue;
824
825 pte_gpa = first_pte_gpa + i * sizeof(pt_element_t);
826
827 if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte,
828 sizeof(pt_element_t)))
829 return -EINVAL;
830
831 gfn = gpte_to_gfn(gpte);
832
833 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, &sp->spt[i], gpte)) {
834 vcpu->kvm->tlbs_dirty++;
835 continue;
836 }
837
838 if (gfn != sp->gfns[i]) {
839 drop_spte(vcpu->kvm, &sp->spt[i],
840 shadow_trap_nonpresent_pte);
841 vcpu->kvm->tlbs_dirty++;
842 continue;
843 }
844
845 nr_present++;
846 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
847 host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE;
848
849 set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
850 is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn,
851 spte_to_pfn(sp->spt[i]), true, false,
852 host_writable);
853 }
854
855 return !nr_present;
856 }
857
858 #undef pt_element_t
859 #undef guest_walker
860 #undef FNAME
861 #undef PT_BASE_ADDR_MASK
862 #undef PT_INDEX
863 #undef PT_LVL_ADDR_MASK
864 #undef PT_LVL_OFFSET_MASK
865 #undef PT_LEVEL_BITS
866 #undef PT_MAX_FULL_LEVELS
867 #undef gpte_to_gfn
868 #undef gpte_to_gfn_lvl
869 #undef CMPXCHG
Linux kernel - Deliverables
This page took 0.070486 seconds and 5 git commands to generate.