diff options
-rw-r--r-- | hw/hypercall.c | 90 | ||||
-rw-r--r-- | hw/usb-uhci.c | 54 | ||||
-rw-r--r-- | kvm/Makefile | 2 | ||||
-rw-r--r-- | kvm/kernel/Kbuild | 6 | ||||
-rw-r--r-- | kvm/kernel/external-module-compat.h | 21 | ||||
-rw-r--r-- | kvm/kernel/kvm.h | 102 | ||||
-rw-r--r-- | kvm/kernel/kvm_main.c | 136 | ||||
-rw-r--r-- | kvm/kernel/mmu.c | 1114 | ||||
-rw-r--r-- | kvm/kernel/paging_tmpl.h | 260 | ||||
-rw-r--r-- | kvm/kernel/svm.c | 23 | ||||
-rw-r--r-- | kvm/kernel/vmx-debug.c (renamed from kvm/kernel/debug.c) | 40 | ||||
-rw-r--r-- | kvm/kernel/vmx.c | 55 | ||||
-rw-r--r-- | kvm/kernel/x86_emulate.c | 2 | ||||
-rwxr-xr-x | kvm/kvm | 43 | ||||
-rw-r--r-- | kvm/kvm.spec | 2 | ||||
-rw-r--r-- | kvm/scripts/65-kvm.rules | 1 | ||||
-rw-r--r-- | kvm/user/main.c | 10 | ||||
-rw-r--r-- | monitor.c | 12 | ||||
-rw-r--r-- | vl.c | 59 | ||||
-rw-r--r-- | vnc.c | 15 |
20 files changed, 1641 insertions, 406 deletions
diff --git a/hw/hypercall.c b/hw/hypercall.c index d4de16a28..afa953e2e 100644 --- a/hw/hypercall.c +++ b/hw/hypercall.c @@ -25,22 +25,23 @@ #include "vl.h" int use_hypercall_dev = 0; +static CharDriverState *vmchannel_hd; typedef struct HypercallState { - uint8_t cmd; - uint32_t start; - uint32_t stop; + int irq; + PCIDevice *pci_dev; } HypercallState; static void hp_ioport_write(void *opaque, uint32_t addr, uint32_t val) { - //printf("hp_ioport_write, val=0x%x\n", val); + //printf("hp_ioport_write, val=0x%x\n", val); + qemu_chr_write(vmchannel_hd, (const uint8_t*)&val, 1); } static uint32_t hp_ioport_read(void *opaque, uint32_t addr) { - //printf("hp_ioport_read\n"); - return 0; + //printf("hp_ioport_read\n"); + return 0; } /***********************************************************/ @@ -64,40 +65,41 @@ static void hp_map(PCIDevice *pci_dev, int region_num, void pci_hypercall_init(PCIBus *bus) { - PCIHypercallState *d; - uint8_t *pci_conf; - - - // If the vmchannel wasn't initialized, we don't want the Hypercall device in the guest - if (use_hypercall_dev == 0) { - return; - } - - d = (PCIHypercallState *)pci_register_device(bus, - "HPNAME", sizeof(PCIHypercallState), - -1, - NULL, NULL); - - pci_conf = d->dev.config; - pci_conf[0x00] = 0x02; // Qumranet vendor ID 0x5002 - pci_conf[0x01] = 0x50; - pci_conf[0x02] = 0x58; // Qumranet DeviceID 0x2258 - pci_conf[0x03] = 0x22; - - pci_conf[0x09] = 0x00; // ProgIf - pci_conf[0x0a] = 0x00; // SubClass - pci_conf[0x0b] = 0x05; // BaseClass - - pci_conf[0x0e] = 0x00; // header_type - pci_conf[0x3d] = 0x00; // interrupt pin 0 - - pci_register_io_region(&d->dev, 0, 0x100, - PCI_ADDRESS_SPACE_IO, hp_map); + PCIHypercallState *d; + HypercallState *s; + uint8_t *pci_conf; + + // If the vmchannel wasn't initialized, we don't want the Hypercall device in the guest + if (use_hypercall_dev == 0) { + return; + } + + d = (PCIHypercallState *)pci_register_device(bus, + "Hypercall", sizeof(PCIHypercallState), + -1, + NULL, NULL); + + pci_conf = d->dev.config; + pci_conf[0x00] = 0x02; // Qumranet vendor ID 0x5002 + pci_conf[0x01] = 0x50; + pci_conf[0x02] = 0x58; // Qumranet DeviceID 0x2258 + pci_conf[0x03] = 0x22; + + pci_conf[0x09] = 0x00; // ProgIf + pci_conf[0x0a] = 0x00; // SubClass + pci_conf[0x0b] = 0x05; // BaseClass + + pci_conf[0x0e] = 0x00; // header_type + pci_conf[0x3d] = 1; // interrupt pin 0 + + pci_register_io_region(&d->dev, 0, 0x100, + PCI_ADDRESS_SPACE_IO, hp_map); + s = &d->hp; + s->irq = 16; /* PCI interrupt */ + s->pci_dev = (PCIDevice *)d; } -static CharDriverState *vmchannel_hd; - static int vmchannel_can_read(void *opaque) { return 128; @@ -107,20 +109,18 @@ static void vmchannel_read(void *opaque, const uint8_t *buf, int size) { int i; - //printf("vmchannel_read buf:%p, size:%d\n", buf, size); + //printf("vmchannel_read buf:%p, size:%d\n", buf, size); for(i = 0; i < size; i++) { - printf("buf[i]=%c\n",buf[i]); readline_handle_byte(buf[i]); - } + } } void vmchannel_init(CharDriverState *hd) { - vmchannel_hd = hd; - - use_hypercall_dev = 1; + vmchannel_hd = hd; - qemu_chr_add_read_handler(hd, vmchannel_can_read, vmchannel_read, NULL); - //vmchannel_start_input(); + use_hypercall_dev = 1; + qemu_chr_add_read_handler(vmchannel_hd, vmchannel_can_read, vmchannel_read, NULL); + //vmchannel_start_input(); } diff --git a/hw/usb-uhci.c b/hw/usb-uhci.c index 1a6e0130f..b23659073 100644 --- a/hw/usb-uhci.c +++ b/hw/usb-uhci.c @@ -132,6 +132,58 @@ static void uhci_reset(UHCIState *s) } } +static void uhci_save(QEMUFile *f, void *opaque) +{ + UHCIState *s = opaque; + uint8_t num_ports = NB_PORTS; + int i; + + generic_pci_save(f, &s->dev); + + qemu_put_8s(f, &num_ports); + for (i = 0; i < num_ports; ++i) + qemu_put_be16s(f, &s->ports[i].ctrl); + qemu_put_be16s(f, &s->cmd); + qemu_put_be16s(f, &s->status); + qemu_put_be16s(f, &s->intr); + qemu_put_be16s(f, &s->frnum); + qemu_put_be32s(f, &s->fl_base_addr); + qemu_put_8s(f, &s->sof_timing); + qemu_put_8s(f, &s->status2); + qemu_put_timer(f, s->frame_timer); +} + +static int uhci_load(QEMUFile* f,void* opaque,int version_id) +{ + UHCIState *s = opaque; + uint8_t num_ports; + int i, ret; + + if (version_id > 1) + return -EINVAL; + + ret = generic_pci_load(f, &s->dev, 1); + if (ret < 0) + return ret; + + qemu_get_8s(f, &num_ports); + if (num_ports != NB_PORTS) + return -EINVAL; + + for (i = 0; i < num_ports; ++i) + qemu_get_be16s(f, &s->ports[i].ctrl); + qemu_get_be16s(f, &s->cmd); + qemu_get_be16s(f, &s->status); + qemu_get_be16s(f, &s->intr); + qemu_get_be16s(f, &s->frnum); + qemu_get_be32s(f, &s->fl_base_addr); + qemu_get_8s(f, &s->sof_timing); + qemu_get_8s(f, &s->status2); + qemu_get_timer(f, s->frame_timer); + + return 0; +} + static void uhci_ioport_writeb(void *opaque, uint32_t addr, uint32_t val) { UHCIState *s = opaque; @@ -671,4 +723,6 @@ void usb_uhci_init(PCIBus *bus, int devfn) to rely on this. */ pci_register_io_region(&s->dev, 4, 0x20, PCI_ADDRESS_SPACE_IO, uhci_map); + + register_savevm("uhci", 0, 1, uhci_save, uhci_load, s); } diff --git a/kvm/Makefile b/kvm/Makefile index 07d555099..48dda0264 100644 --- a/kvm/Makefile +++ b/kvm/Makefile @@ -27,9 +27,11 @@ install-rpm: mkdir -p $(DESTDIR)/$(confdir) mkdir -p $(DESTDIR)/$(initdir) mkdir -p $(DESTDIR)/$(utilsdir) + mkdir -p $(DESTDIR)/etc/udev/rules.d cp qemu/x86_64-softmmu/qemu-system-x86_64 $(DESTDIR)/$(bin) cp scripts/kvm $(DESTDIR)/$(initdir)/kvm cp scripts/qemu-ifup $(DESTDIR)/$(confdir)/qemu-ifup + install -t $(DESTDIR)/etc/udev/rules.d scripts/*kvm*.rules cp kvm $(DESTDIR)/$(utilsdir)/kvm install: diff --git a/kvm/kernel/Kbuild b/kvm/kernel/Kbuild index d6781aff2..e9bcda78d 100644 --- a/kvm/kernel/Kbuild +++ b/kvm/kernel/Kbuild @@ -1,5 +1,5 @@ -EXTRA_CFLAGS := -I$(src)/include +EXTRA_CFLAGS := -I$(src)/include -include $(src)/external-module-compat.h obj-m := kvm.o kvm-intel.o kvm-amd.o -kvm-objs := kvm_main.o mmu.o x86_emulate.o debug.o -kvm-intel-objs := vmx.o +kvm-objs := kvm_main.o mmu.o x86_emulate.o +kvm-intel-objs := vmx.o vmx-debug.o kvm-amd-objs := svm.o diff --git a/kvm/kernel/external-module-compat.h b/kvm/kernel/external-module-compat.h new file mode 100644 index 000000000..3c445124d --- /dev/null +++ b/kvm/kernel/external-module-compat.h @@ -0,0 +1,21 @@ + +/* + * Compatibility header for building as an external module. + */ + +/* + * Avoid picking up the kernel's kvm.h in case we have a newer one. + */ + +#include <linux/compiler.h> +#include "include/linux/kvm.h" + +/* + * 2.6.16 does not have GFP_NOWAIT + */ + +#include <linux/gfp.h> + +#ifndef GFP_NOWAIT +#define GFP_NOWAIT (GFP_ATOMIC & ~__GFP_HIGH) +#endif diff --git a/kvm/kernel/kvm.h b/kvm/kernel/kvm.h index 32023d1ac..91e0c75ac 100644 --- a/kvm/kernel/kvm.h +++ b/kvm/kernel/kvm.h @@ -52,6 +52,8 @@ #define KVM_MAX_VCPUS 1 #define KVM_MEMORY_SLOTS 4 #define KVM_NUM_MMU_PAGES 256 +#define KVM_MIN_FREE_MMU_PAGES 5 +#define KVM_REFILL_PAGES 25 #define FX_IMAGE_SIZE 512 #define FX_IMAGE_ALIGN 16 @@ -89,14 +91,54 @@ typedef unsigned long hva_t; typedef u64 hpa_t; typedef unsigned long hfn_t; +#define NR_PTE_CHAIN_ENTRIES 5 + +struct kvm_pte_chain { + u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; + struct hlist_node link; +}; + +/* + * kvm_mmu_page_role, below, is defined as: + * + * bits 0:3 - total guest paging levels (2-4, or zero for real mode) + * bits 4:7 - page table level for this shadow (1-4) + * bits 8:9 - page table quadrant for 2-level guests + * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) + */ +union kvm_mmu_page_role { + unsigned word; + struct { + unsigned glevels : 4; + unsigned level : 4; + unsigned quadrant : 2; + unsigned pad_for_nice_hex_output : 6; + unsigned metaphysical : 1; + }; +}; + struct kvm_mmu_page { struct list_head link; + struct hlist_node hash_link; + + /* + * The following two entries are used to key the shadow page in the + * hash table. + */ + gfn_t gfn; + union kvm_mmu_page_role role; + hpa_t page_hpa; unsigned long slot_bitmap; /* One bit set per slot which has memory * in this shadow page. */ int global; /* Set if all ptes in this page are global */ - u64 *parent_pte; + int multimapped; /* More than one parent_pte? */ + int root_count; /* Currently serving as active root */ + union { + u64 *parent_pte; /* !multimapped */ + struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ + }; }; struct vmcs { @@ -117,14 +159,26 @@ struct kvm_vcpu; struct kvm_mmu { void (*new_cr3)(struct kvm_vcpu *vcpu); int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*inval_page)(struct kvm_vcpu *vcpu, gva_t gva); void (*free)(struct kvm_vcpu *vcpu); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); hpa_t root_hpa; int root_level; int shadow_root_level; + + u64 *pae_root; +}; + +#define KVM_NR_MEM_OBJS 20 + +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; }; +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ struct kvm_guest_debug { int enabled; unsigned long bp[4]; @@ -185,6 +239,7 @@ struct kvm_vcpu { unsigned long cr3; unsigned long cr4; unsigned long cr8; + u64 pdptrs[4]; /* pae */ u64 shadow_efer; u64 apic_base; int nmsrs; @@ -195,6 +250,12 @@ struct kvm_vcpu { struct kvm_mmu_page page_header_buf[KVM_NUM_MMU_PAGES]; struct kvm_mmu mmu; + struct kvm_mmu_memory_cache mmu_pte_chain_cache; + struct kvm_mmu_memory_cache mmu_rmap_desc_cache; + + gfn_t last_pt_write_gfn; + int last_pt_write_count; + struct kvm_guest_debug guest_debug; char fx_buf[FX_BUF_SIZE]; @@ -232,10 +293,16 @@ struct kvm { spinlock_t lock; /* protects everything except vcpus */ int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS]; + /* + * Hash table of struct kvm_mmu_page. + */ struct list_head active_mmu_pages; + int n_free_mmu_pages; + struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu vcpus[KVM_MAX_VCPUS]; int memory_config_version; int busy; + unsigned long rmap_overflow; }; struct kvm_stat { @@ -283,6 +350,7 @@ struct kvm_arch_ops { void (*set_segment)(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); + void (*decache_cr0_cr4_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); void (*set_cr0_no_modeswitch)(struct kvm_vcpu *vcpu, unsigned long cr0); @@ -327,7 +395,7 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu); int kvm_mmu_setup(struct kvm_vcpu *vcpu); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); +void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot); hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) @@ -400,6 +468,19 @@ int kvm_write_guest(struct kvm_vcpu *vcpu, unsigned long segment_base(u16 selector); +void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes); +void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes); +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); +void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); + +static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, + u32 error_code) +{ + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + kvm_mmu_free_some_pages(vcpu); + return vcpu->mmu.page_fault(vcpu, gva, error_code); +} + static inline struct page *_gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); @@ -545,19 +626,4 @@ static inline u32 get_rdx_init_val(void) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) -#ifdef CONFIG_X86_64 - -/* - * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. Therefore - * we need to allocate shadow page tables in the first 4GB of memory, which - * happens to fit the DMA32 zone. - */ -#define GFP_KVM_MMU (GFP_KERNEL | __GFP_DMA32) - -#else - -#define GFP_KVM_MMU GFP_KERNEL - -#endif - #endif diff --git a/kvm/kernel/kvm_main.c b/kvm/kernel/kvm_main.c index aca14139a..67c115496 100644 --- a/kvm/kernel/kvm_main.c +++ b/kvm/kernel/kvm_main.c @@ -230,6 +230,7 @@ static int kvm_dev_open(struct inode *inode, struct file *filp) struct kvm_vcpu *vcpu = &kvm->vcpus[i]; mutex_init(&vcpu->mutex); + vcpu->kvm = kvm; vcpu->mmu.root_hpa = INVALID_PAGE; INIT_LIST_HEAD(&vcpu->free_pages); } @@ -271,8 +272,8 @@ static void kvm_free_physmem(struct kvm *kvm) static void kvm_free_vcpu(struct kvm_vcpu *vcpu) { - kvm_arch_ops->vcpu_free(vcpu); kvm_mmu_destroy(vcpu); + kvm_arch_ops->vcpu_free(vcpu); } static void kvm_free_vcpus(struct kvm *kvm) @@ -298,14 +299,17 @@ static void inject_gp(struct kvm_vcpu *vcpu) kvm_arch_ops->inject_gp(vcpu, 0); } -static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu, - unsigned long cr3) +/* + * Load the pae pdptrs. Return true is they are all valid. + */ +static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) { gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - unsigned offset = (cr3 & (PAGE_SIZE-1)) >> 5; + unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; int i; u64 pdpte; u64 *pdpt; + int ret; struct kvm_memory_slot *memslot; spin_lock(&vcpu->kvm->lock); @@ -313,16 +317,23 @@ static int pdptrs_have_reserved_bits_set(struct kvm_vcpu *vcpu, /* FIXME: !memslot - emulate? 0xff? */ pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0); + ret = 1; for (i = 0; i < 4; ++i) { pdpte = pdpt[offset + i]; - if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) - break; + if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) { + ret = 0; + goto out; + } } + for (i = 0; i < 4; ++i) + vcpu->pdptrs[i] = pdpt[offset + i]; + +out: kunmap_atomic(pdpt, KM_USER0); spin_unlock(&vcpu->kvm->lock); - return i != 4; + return ret; } void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) @@ -368,8 +379,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } } else #endif - if (is_pae(vcpu) && - pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) { + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr0: #GP, pdptrs " "reserved bits\n"); inject_gp(vcpu); @@ -390,6 +400,7 @@ EXPORT_SYMBOL_GPL(set_cr0); void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) { + kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu); set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); } EXPORT_SYMBOL_GPL(lmsw); @@ -410,7 +421,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return; } } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK) - && pdptrs_have_reserved_bits_set(vcpu, vcpu->cr3)) { + && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); inject_gp(vcpu); } @@ -442,7 +453,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) return; } if (is_paging(vcpu) && is_pae(vcpu) && - pdptrs_have_reserved_bits_set(vcpu, cr3)) { + !load_pdptrs(vcpu, cr3)) { printk(KERN_DEBUG "set_cr3: #GP, pdptrs " "reserved bits\n"); inject_gp(vcpu); @@ -452,7 +463,19 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) vcpu->cr3 = cr3; spin_lock(&vcpu->kvm->lock); - vcpu->mmu.new_cr3(vcpu); + /* + * Does the new cr3 value map to physical memory? (Note, we + * catch an invalid cr3 even in real-mode, because it would + * cause trouble later on when we turn on paging anyway.) + * + * A real CPU would silently accept an invalid cr3 and would + * attempt to use it - with largely undefined (and often hard + * to debug) behavior on the guest side. + */ + if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) + inject_gp(vcpu); + else + vcpu->mmu.new_cr3(vcpu); spin_unlock(&vcpu->kvm->lock); } EXPORT_SYMBOL_GPL(set_cr3); @@ -520,7 +543,6 @@ static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n) vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE; vcpu->cpu = -1; /* First load will set up TR */ - vcpu->kvm = kvm; r = kvm_arch_ops->vcpu_create(vcpu); if (r < 0) goto out_free_vcpus; @@ -637,6 +659,7 @@ raced: | __GFP_ZERO); if (!new.phys_mem[i]) goto out_free; + new.phys_mem[i]->private = 0; } } @@ -691,6 +714,13 @@ out: return r; } +static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot) +{ + spin_lock(&vcpu->kvm->lock); + kvm_mmu_slot_remove_write_access(vcpu, slot); + spin_unlock(&vcpu->kvm->lock); +} + /* * Get (and clear) the dirty memory log for a memory slot. */ @@ -700,6 +730,7 @@ static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot; int r, i; int n; + int cleared; unsigned long any = 0; spin_lock(&kvm->lock); @@ -730,15 +761,17 @@ static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm, if (any) { - spin_lock(&kvm->lock); - kvm_mmu_slot_remove_write_access(kvm, log->slot); - spin_unlock(&kvm->lock); - memset(memslot->dirty_bitmap, 0, n); + cleared = 0; for (i = 0; i < KVM_MAX_VCPUS; ++i) { struct kvm_vcpu *vcpu = vcpu_load(kvm, i); if (!vcpu) continue; + if (!cleared) { + do_remove_write_access(vcpu, log->slot); + memset(memslot->dirty_bitmap, 0, n); + cleared = 1; + } kvm_arch_ops->tlb_flush(vcpu); vcpu_put(vcpu); } @@ -866,6 +899,27 @@ static int emulator_read_emulated(unsigned long addr, } } +static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long val, int bytes) +{ + struct kvm_memory_slot *m; + struct page *page; + void *virt; + + if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT)) + return 0; + m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT); + if (!m) + return 0; + page = gfn_to_page(m, gpa >> PAGE_SHIFT); + kvm_mmu_pre_write(vcpu, gpa, bytes); + virt = kmap_atomic(page, KM_USER0); + memcpy(virt + offset_in_page(gpa), &val, bytes); + kunmap_atomic(virt, KM_USER0); + kvm_mmu_post_write(vcpu, gpa, bytes); + return 1; +} + static int emulator_write_emulated(unsigned long addr, unsigned long val, unsigned int bytes, @@ -877,6 +931,9 @@ static int emulator_write_emulated(unsigned long addr, if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; + if (emulator_write_phys(vcpu, gpa, val, bytes)) + return X86EMUL_CONTINUE; + vcpu->mmio_needed = 1; vcpu->mmio_phys_addr = gpa; vcpu->mmio_size = bytes; @@ -901,6 +958,30 @@ static int emulator_cmpxchg_emulated(unsigned long addr, return emulator_write_emulated(addr, new, bytes, ctxt); } +#ifdef CONFIG_X86_32 + +static int emulator_cmpxchg8b_emulated(unsigned long addr, + unsigned long old_lo, + unsigned long old_hi, + unsigned long new_lo, + unsigned long new_hi, + struct x86_emulate_ctxt *ctxt) +{ + static int reported; + int r; + + if (!reported) { + reported = 1; + printk(KERN_WARNING "kvm: emulating exchange8b as write\n"); + } + r = emulator_write_emulated(addr, new_lo, 4, ctxt); + if (r != X86EMUL_CONTINUE) + return r; + return emulator_write_emulated(addr+4, new_hi, 4, ctxt); +} + +#endif + static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) { return kvm_arch_ops->get_segment_base(vcpu, seg); @@ -908,18 +989,15 @@ static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) { - spin_lock(&vcpu->kvm->lock); - vcpu->mmu.inval_page(vcpu, address); - spin_unlock(&vcpu->kvm->lock); - kvm_arch_ops->invlpg(vcpu, address); return X86EMUL_CONTINUE; } int emulate_clts(struct kvm_vcpu *vcpu) { - unsigned long cr0 = vcpu->cr0; + unsigned long cr0; - cr0 &= ~CR0_TS_MASK; + kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu); + cr0 = vcpu->cr0 & ~CR0_TS_MASK; kvm_arch_ops->set_cr0(vcpu, cr0); return X86EMUL_CONTINUE; } @@ -978,6 +1056,9 @@ struct x86_emulate_ops emulate_ops = { .read_emulated = emulator_read_emulated, .write_emulated = emulator_write_emulated, .cmpxchg_emulated = emulator_cmpxchg_emulated, +#ifdef CONFIG_X86_32 + .cmpxchg8b_emulated = emulator_cmpxchg8b_emulated, +#endif }; int emulate_instruction(struct kvm_vcpu *vcpu, @@ -1027,6 +1108,8 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; if (!vcpu->mmio_needed) { report_emulation_failure(&emulate_ctxt); return EMULATE_FAIL; @@ -1072,6 +1155,7 @@ void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) { + kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu); switch (cr) { case 0: return vcpu->cr0; @@ -1406,6 +1490,7 @@ static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs) sregs->gdt.limit = dt.limit; sregs->gdt.base = dt.base; + kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu); sregs->cr0 = vcpu->cr0; sregs->cr2 = vcpu->cr2; sregs->cr3 = vcpu->cr3; @@ -1470,11 +1555,15 @@ static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs) #endif vcpu->apic_base = sregs->apic_base; + kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu); + mmu_reset_needed |= vcpu->cr0 != sregs->cr0; kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0); mmu_reset_needed |= vcpu->cr4 != sregs->cr4; kvm_arch_ops->set_cr4(vcpu, sregs->cr4); + if (!is_long_mode(vcpu) && is_pae(vcpu)) + load_pdptrs(vcpu, vcpu->cr3); if (mmu_reset_needed) kvm_mmu_reset_context(vcpu); @@ -1845,6 +1934,7 @@ static long kvm_dev_ioctl(struct file *filp, num_msrs_to_save * sizeof(u32))) goto out; r = 0; + break; } default: ; diff --git a/kvm/kernel/mmu.c b/kvm/kernel/mmu.c index 790423c5f..c6f972914 100644 --- a/kvm/kernel/mmu.c +++ b/kvm/kernel/mmu.c @@ -26,7 +26,31 @@ #include "vmx.h" #include "kvm.h" +#undef MMU_DEBUG + +#undef AUDIT + +#ifdef AUDIT +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg); +#else +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {} +#endif + +#ifdef MMU_DEBUG + +#define pgprintk(x...) do { if (dbg) printk(x); } while (0) +#define rmap_printk(x...) do { if (dbg) printk(x); } while (0) + +#else + #define pgprintk(x...) do { } while (0) +#define rmap_printk(x...) do { } while (0) + +#endif + +#if defined(MMU_DEBUG) || defined(AUDIT) +static int dbg = 1; +#endif #define ASSERT(x) \ if (!(x)) { \ @@ -34,8 +58,10 @@ __FILE__, __LINE__, #x); \ } -#define PT64_ENT_PER_PAGE 512 -#define PT32_ENT_PER_PAGE 1024 +#define PT64_PT_BITS 9 +#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS) +#define PT32_PT_BITS 10 +#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS) #define PT_WRITABLE_SHIFT 1 @@ -125,6 +151,13 @@ #define PT_DIRECTORY_LEVEL 2 #define PT_PAGE_TABLE_LEVEL 1 +#define RMAP_EXT 4 + +struct kvm_rmap_desc { + u64 *shadow_ptes[RMAP_EXT]; + struct kvm_rmap_desc *more; +}; + static int is_write_protection(struct kvm_vcpu *vcpu) { return vcpu->cr0 & CR0_WP_MASK; @@ -150,32 +183,272 @@ static int is_io_pte(unsigned long pte) return pte & PT_SHADOW_IO_MARK; } +static int is_rmap_pte(u64 pte) +{ + return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK)) + == (PT_WRITABLE_MASK | PT_PRESENT_MASK); +} + +static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, + size_t objsize, int min) +{ + void *obj; + + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < ARRAY_SIZE(cache->objects)) { + obj = kzalloc(objsize, GFP_NOWAIT); + if (!obj) + return -ENOMEM; + cache->objects[cache->nobjs++] = obj; + } + return 0; +} + +static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + kfree(mc->objects[--mc->nobjs]); +} + +static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) +{ + int r; + + r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache, + sizeof(struct kvm_pte_chain), 4); + if (r) + goto out; + r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache, + sizeof(struct kvm_rmap_desc), 1); +out: + return r; +} + +static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache); + mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache); +} + +static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, + size_t size) +{ + void *p; + + BUG_ON(!mc->nobjs); + p = mc->objects[--mc->nobjs]; + memset(p, 0, size); + return p; +} + +static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj) +{ + if (mc->nobjs < KVM_NR_MEM_OBJS) + mc->objects[mc->nobjs++] = obj; + else + kfree(obj); +} + +static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu) +{ + return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache, + sizeof(struct kvm_pte_chain)); +} + +static void mmu_free_pte_chain(struct kvm_vcpu *vcpu, + struct kvm_pte_chain *pc) +{ + mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc); +} + +static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu) +{ + return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache, + sizeof(struct kvm_rmap_desc)); +} + +static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu, + struct kvm_rmap_desc *rd) +{ + mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd); +} + +/* + * Reverse mapping data structures: + * + * If page->private bit zero is zero, then page->private points to the + * shadow page table entry that points to page_address(page). + * + * If page->private bit zero is one, (then page->private & ~1) points + * to a struct kvm_rmap_desc containing more mappings. + */ +static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte) +{ + struct page *page; + struct kvm_rmap_desc *desc; + int i; + + if (!is_rmap_pte(*spte)) + return; + page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (!page->private) { + rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); + page->private = (unsigned long)spte; + } else if (!(page->private & 1)) { + rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte); + desc = mmu_alloc_rmap_desc(vcpu); + desc->shadow_ptes[0] = (u64 *)page->private; + desc->shadow_ptes[1] = spte; + page->private = (unsigned long)desc | 1; + } else { + rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte); + desc = (struct kvm_rmap_desc *)(page->private & ~1ul); + while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) + desc = desc->more; + if (desc->shadow_ptes[RMAP_EXT-1]) { + desc->more = mmu_alloc_rmap_desc(vcpu); + desc = desc->more; + } + for (i = 0; desc->shadow_ptes[i]; ++i) + ; + desc->shadow_ptes[i] = spte; + } +} + +static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu, + struct page *page, + struct kvm_rmap_desc *desc, + int i, + struct kvm_rmap_desc *prev_desc) +{ + int j; + + for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j) + ; + desc->shadow_ptes[i] = desc->shadow_ptes[j]; + desc->shadow_ptes[j] = 0; + if (j != 0) + return; + if (!prev_desc && !desc->more) + page->private = (unsigned long)desc->shadow_ptes[0]; + else + if (prev_desc) + prev_desc->more = desc->more; + else + page->private = (unsigned long)desc->more | 1; + mmu_free_rmap_desc(vcpu, desc); +} + +static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte) +{ + struct page *page; + struct kvm_rmap_desc *desc; + struct kvm_rmap_desc *prev_desc; + int i; + + if (!is_rmap_pte(*spte)) + return; + page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (!page->private) { + printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); + BUG(); + } else if (!(page->private & 1)) { + rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte); + if ((u64 *)page->private != spte) { + printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n", + spte, *spte); + BUG(); + } + page->private = 0; + } else { + rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte); + desc = (struct kvm_rmap_desc *)(page->private & ~1ul); + prev_desc = NULL; + while (desc) { + for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) + if (desc->shadow_ptes[i] == spte) { + rmap_desc_remove_entry(vcpu, page, + desc, i, + prev_desc); + return; + } + prev_desc = desc; + desc = desc->more; + } + BUG(); + } +} + +static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) +{ + struct kvm *kvm = vcpu->kvm; + struct page *page; + struct kvm_memory_slot *slot; + struct kvm_rmap_desc *desc; + u64 *spte; + + slot = gfn_to_memslot(kvm, gfn); + BUG_ON(!slot); + page = gfn_to_page(slot, gfn); + + while (page->private) { + if (!(page->private & 1)) + spte = (u64 *)page->private; + else { + desc = (struct kvm_rmap_desc *)(page->private & ~1ul); + spte = desc->shadow_ptes[0]; + } + BUG_ON(!spte); + BUG_ON((*spte & PT64_BASE_ADDR_MASK) != + page_to_pfn(page) << PAGE_SHIFT); + BUG_ON(!(*spte & PT_PRESENT_MASK)); + BUG_ON(!(*spte & PT_WRITABLE_MASK)); + rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); + rmap_remove(vcpu, spte); + kvm_arch_ops->tlb_flush(vcpu); + *spte &= ~(u64)PT_WRITABLE_MASK; + } +} + +static int is_empty_shadow_page(hpa_t page_hpa) +{ + u64 *pos; + u64 *end; + + for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u64); + pos != end; pos++) + if (*pos != 0) { + printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__, + pos, *pos); + return 0; + } + return 1; +} + static void kvm_mmu_free_page(struct kvm_vcpu *vcpu, hpa_t page_hpa) { struct kvm_mmu_page *page_head = page_header(page_hpa); + ASSERT(is_empty_shadow_page(page_hpa)); list_del(&page_head->link); page_head->page_hpa = page_hpa; list_add(&page_head->link, &vcpu->free_pages); + ++vcpu->kvm->n_free_mmu_pages; } -static int is_empty_shadow_page(hpa_t page_hpa) +static unsigned kvm_page_table_hashfn(gfn_t gfn) { - u32 *pos; - u32 *end; - for (pos = __va(page_hpa), end = pos + PAGE_SIZE / sizeof(u32); - pos != end; pos++) - if (*pos != 0) - return 0; - return 1; + return gfn; } -static hpa_t kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte) +static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, + u64 *parent_pte) { struct kvm_mmu_page *page; if (list_empty(&vcpu->free_pages)) - return INVALID_PAGE; + return NULL; page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link); list_del(&page->link); @@ -183,8 +456,239 @@ static hpa_t kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte) ASSERT(is_empty_shadow_page(page->page_hpa)); page->slot_bitmap = 0; page->global = 1; + page->multimapped = 0; page->parent_pte = parent_pte; - return page->page_hpa; + --vcpu->kvm->n_free_mmu_pages; + return page; +} + +static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page, u64 *parent_pte) +{ + struct kvm_pte_chain *pte_chain; + struct hlist_node *node; + int i; + + if (!parent_pte) + return; + if (!page->multimapped) { + u64 *old = page->parent_pte; + + if (!old) { + page->parent_pte = parent_pte; + return; + } + page->multimapped = 1; + pte_chain = mmu_alloc_pte_chain(vcpu); + INIT_HLIST_HEAD(&page->parent_ptes); + hlist_add_head(&pte_chain->link, &page->parent_ptes); + pte_chain->parent_ptes[0] = old; + } + hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) { + if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1]) + continue; + for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) + if (!pte_chain->parent_ptes[i]) { + pte_chain->parent_ptes[i] = parent_pte; + return; + } + } + pte_chain = mmu_alloc_pte_chain(vcpu); + BUG_ON(!pte_chain); + hlist_add_head(&pte_chain->link, &page->parent_ptes); + pte_chain->parent_ptes[0] = parent_pte; +} + +static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page, + u64 *parent_pte) +{ + struct kvm_pte_chain *pte_chain; + struct hlist_node *node; + int i; + + if (!page->multimapped) { + BUG_ON(page->parent_pte != parent_pte); + page->parent_pte = NULL; + return; + } + hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) + for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) { + if (!pte_chain->parent_ptes[i]) + break; + if (pte_chain->parent_ptes[i] != parent_pte) + continue; + while (i + 1 < NR_PTE_CHAIN_ENTRIES + && pte_chain->parent_ptes[i + 1]) { + pte_chain->parent_ptes[i] + = pte_chain->parent_ptes[i + 1]; + ++i; + } + pte_chain->parent_ptes[i] = NULL; + if (i == 0) { + hlist_del(&pte_chain->link); + mmu_free_pte_chain(vcpu, pte_chain); + if (hlist_empty(&page->parent_ptes)) { + page->multimapped = 0; + page->parent_pte = NULL; + } + } + return; + } + BUG(); +} + +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu, + gfn_t gfn) +{ + unsigned index; + struct hlist_head *bucket; + struct kvm_mmu_page *page; + struct hlist_node *node; + + pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->mmu_page_hash[index]; + hlist_for_each_entry(page, node, bucket, hash_link) + if (page->gfn == gfn && !page->role.metaphysical) { + pgprintk("%s: found role %x\n", + __FUNCTION__, page->role.word); + return page; + } + return NULL; +} + +static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, + gfn_t gfn, + gva_t gaddr, + unsigned level, + int metaphysical, + u64 *parent_pte) +{ + union kvm_mmu_page_role role; + unsigned index; + unsigned quadrant; + struct hlist_head *bucket; + struct kvm_mmu_page *page; + struct hlist_node *node; + + role.word = 0; + role.glevels = vcpu->mmu.root_level; + role.level = level; + role.metaphysical = metaphysical; + if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) { + quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); + quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; + role.quadrant = quadrant; + } + pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__, + gfn, role.word); + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->mmu_page_hash[index]; + hlist_for_each_entry(page, node, bucket, hash_link) + if (page->gfn == gfn && page->role.word == role.word) { + mmu_page_add_parent_pte(vcpu, page, parent_pte); + pgprintk("%s: found\n", __FUNCTION__); + return page; + } + page = kvm_mmu_alloc_page(vcpu, parent_pte); + if (!page) + return page; + pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word); + page->gfn = gfn; + page->role = role; + hlist_add_head(&page->hash_link, bucket); + if (!metaphysical) + rmap_write_protect(vcpu, gfn); + return page; +} + +static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page) +{ + unsigned i; + u64 *pt; + u64 ent; + + pt = __va(page->page_hpa); + + if (page->role.level == PT_PAGE_TABLE_LEVEL) { + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + if (pt[i] & PT_PRESENT_MASK) + rmap_remove(vcpu, &pt[i]); + pt[i] = 0; + } + kvm_arch_ops->tlb_flush(vcpu); + return; + } + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + ent = pt[i]; + + pt[i] = 0; + if (!(ent & PT_PRESENT_MASK)) + continue; + ent &= PT64_BASE_ADDR_MASK; + mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]); + } +} + +static void kvm_mmu_put_page(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page, + u64 *parent_pte) +{ + mmu_page_remove_parent_pte(vcpu, page, parent_pte); +} + +static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page) +{ + u64 *parent_pte; + + while (page->multimapped || page->parent_pte) { + if (!page->multimapped) + parent_pte = page->parent_pte; + else { + struct kvm_pte_chain *chain; + + chain = container_of(page->parent_ptes.first, + struct kvm_pte_chain, link); + parent_pte = chain->parent_ptes[0]; + } + BUG_ON(!parent_pte); + kvm_mmu_put_page(vcpu, page, parent_pte); + *parent_pte = 0; + } + kvm_mmu_page_unlink_children(vcpu, page); + if (!page->root_count) { + hlist_del(&page->hash_link); + kvm_mmu_free_page(vcpu, page->page_hpa); + } else { + list_del(&page->link); + list_add(&page->link, &vcpu->kvm->active_mmu_pages); + } +} + +static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) +{ + unsigned index; + struct hlist_head *bucket; + struct kvm_mmu_page *page; + struct hlist_node *node, *n; + int r; + + pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); + r = 0; + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->mmu_page_hash[index]; + hlist_for_each_entry_safe(page, node, n, bucket, hash_link) + if (page->gfn == gfn && !page->role.metaphysical) { + pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, + page->role.word); + kvm_mmu_zap_page(vcpu, page); + r = 1; + } + return r; } static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) @@ -225,35 +729,6 @@ hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) return gpa_to_hpa(vcpu, gpa); } - -static void release_pt_page_64(struct kvm_vcpu *vcpu, hpa_t page_hpa, - int level) -{ - ASSERT(vcpu); - ASSERT(VALID_PAGE(page_hpa)); - ASSERT(level <= PT64_ROOT_LEVEL && level > 0); - - if (level == 1) - memset(__va(page_hpa), 0, PAGE_SIZE); - else { - u64 *pos; - u64 *end; - - for (pos = __va(page_hpa), end = pos + PT64_ENT_PER_PAGE; - pos != end; pos++) { - u64 current_ent = *pos; - - *pos = 0; - if (is_present_pte(current_ent)) - release_pt_page_64(vcpu, - current_ent & - PT64_BASE_ADDR_MASK, - level - 1); - } - } - kvm_mmu_free_page(vcpu, page_hpa); -} - static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } @@ -266,52 +741,109 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; + u64 pte; ASSERT(VALID_PAGE(table_addr)); table = __va(table_addr); if (level == 1) { + pte = table[index]; + if (is_present_pte(pte) && is_writeble_pte(pte)) + return 0; mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; + rmap_add(vcpu, &table[index]); return 0; } if (table[index] == 0) { - hpa_t new_table = kvm_mmu_alloc_page(vcpu, - &table[index]); - - if (!VALID_PAGE(new_table)) { + struct kvm_mmu_page *new_table; + gfn_t pseudo_gfn; + + pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK) + >> PAGE_SHIFT; + new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, + v, level - 1, + 1, &table[index]); + if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); return -ENOMEM; } - if (level == PT32E_ROOT_LEVEL) - table[index] = new_table | PT_PRESENT_MASK; - else - table[index] = new_table | PT_PRESENT_MASK | - PT_WRITABLE_MASK | PT_USER_MASK; + table[index] = new_table->page_hpa | PT_PRESENT_MASK + | PT_WRITABLE_MASK | PT_USER_MASK; } table_addr = table[index] & PT64_BASE_ADDR_MASK; } } -static void nonpaging_flush(struct kvm_vcpu *vcpu) +static void mmu_free_roots(struct kvm_vcpu *vcpu) { - hpa_t root = vcpu->mmu.root_hpa; + int i; + struct kvm_mmu_page *page; - ++kvm_stat.tlb_flush; - pgprintk("nonpaging_flush\n"); - ASSERT(VALID_PAGE(root)); - release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level); - root = kvm_mmu_alloc_page(vcpu, NULL); - ASSERT(VALID_PAGE(root)); - vcpu->mmu.root_hpa = root; - if (is_paging(vcpu)) - root |= (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)); - kvm_arch_ops->set_cr3(vcpu, root); - kvm_arch_ops->tlb_flush(vcpu); +#ifdef CONFIG_X86_64 + if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->mmu.root_hpa; + + ASSERT(VALID_PAGE(root)); + page = page_header(root); + --page->root_count; + vcpu->mmu.root_hpa = INVALID_PAGE; + return; + } +#endif + for (i = 0; i < 4; ++i) { + hpa_t root = vcpu->mmu.pae_root[i]; + + ASSERT(VALID_PAGE(root)); + root &= PT64_BASE_ADDR_MASK; + page = page_header(root); + --page->root_count; + vcpu->mmu.pae_root[i] = INVALID_PAGE; + } + vcpu->mmu.root_hpa = INVALID_PAGE; +} + +static void mmu_alloc_roots(struct kvm_vcpu *vcpu) +{ + int i; + gfn_t root_gfn; + struct kvm_mmu_page *page; + + root_gfn = vcpu->cr3 >> PAGE_SHIFT; + +#ifdef CONFIG_X86_64 + if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->mmu.root_hpa; + + ASSERT(!VALID_PAGE(root)); + page = kvm_mmu_get_page(vcpu, root_gfn, 0, + PT64_ROOT_LEVEL, 0, NULL); + root = page->page_hpa; + ++page->root_count; + vcpu->mmu.root_hpa = root; + return; + } +#endif + for (i = 0; i < 4; ++i) { + hpa_t root = vcpu->mmu.pae_root[i]; + + ASSERT(!VALID_PAGE(root)); + if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) + root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT; + else if (vcpu->mmu.root_level == 0) + root_gfn = 0; + page = kvm_mmu_get_page(vcpu, root_gfn, i << 30, + PT32_ROOT_LEVEL, !is_paging(vcpu), + NULL); + root = page->page_hpa; + ++page->root_count; + vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; + } + vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root); } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) @@ -322,43 +854,29 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code) { - int ret; gpa_t addr = gva; + hpa_t paddr; + int r; + + r = mmu_topup_memory_caches(vcpu); + if (r) + return r; ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); - for (;;) { - hpa_t paddr; - - paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK); - if (is_error_hpa(paddr)) - return 1; + paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK); - ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr); - if (ret) { - nonpaging_flush(vcpu); - continue; - } - break; - } - return ret; -} + if (is_error_hpa(paddr)) + return 1; -static void nonpaging_inval_page(struct kvm_vcpu *vcpu, gva_t addr) -{ + return nonpaging_map(vcpu, addr & PAGE_MASK, paddr); } static void nonpaging_free(struct kvm_vcpu *vcpu) { - hpa_t root; - - ASSERT(vcpu); - root = vcpu->mmu.root_hpa; - if (VALID_PAGE(root)) - release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level); - vcpu->mmu.root_hpa = INVALID_PAGE; + mmu_free_roots(vcpu); } static int nonpaging_init_context(struct kvm_vcpu *vcpu) @@ -367,40 +885,31 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu) context->new_cr3 = nonpaging_new_cr3; context->page_fault = nonpaging_page_fault; - context->inval_page = nonpaging_inval_page; context->gva_to_gpa = nonpaging_gva_to_gpa; context->free = nonpaging_free; - context->root_level = PT32E_ROOT_LEVEL; + context->root_level = 0; context->shadow_root_level = PT32E_ROOT_LEVEL; - context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL); + mmu_alloc_roots(vcpu); ASSERT(VALID_PAGE(context->root_hpa)); kvm_arch_ops->set_cr3(vcpu, context->root_hpa); return 0; } - static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *page, *npage; - - list_for_each_entry_safe(page, npage, &vcpu->kvm->active_mmu_pages, - link) { - if (page->global) - continue; - - if (!page->parent_pte) - continue; - - *page->parent_pte = 0; - release_pt_page_64(vcpu, page->page_hpa, 1); - } ++kvm_stat.tlb_flush; kvm_arch_ops->tlb_flush(vcpu); } static void paging_new_cr3(struct kvm_vcpu *vcpu) { + pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3); + mmu_free_roots(vcpu); + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + kvm_mmu_free_some_pages(vcpu); + mmu_alloc_roots(vcpu); kvm_mmu_flush_tlb(vcpu); + kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa); } static void mark_pagetable_nonglobal(void *shadow_pte) @@ -412,7 +921,8 @@ static inline void set_pte_common(struct kvm_vcpu *vcpu, u64 *shadow_pte, gpa_t gaddr, int dirty, - u64 access_bits) + u64 access_bits, + gfn_t gfn) { hpa_t paddr; @@ -420,13 +930,10 @@ static inline void set_pte_common(struct kvm_vcpu *vcpu, if (!dirty) access_bits &= ~PT_WRITABLE_MASK; - if (access_bits & PT_WRITABLE_MASK) - mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK); *shadow_pte |= access_bits; - paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK); - if (!(*shadow_pte & PT_GLOBAL_MASK)) mark_pagetable_nonglobal(shadow_pte); @@ -434,10 +941,31 @@ static inline void set_pte_common(struct kvm_vcpu *vcpu, *shadow_pte |= gaddr; *shadow_pte |= PT_SHADOW_IO_MARK; *shadow_pte &= ~PT_PRESENT_MASK; - } else { - *shadow_pte |= paddr; - page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); + return; + } + + *shadow_pte |= paddr; + + if (access_bits & PT_WRITABLE_MASK) { + struct kvm_mmu_page *shadow; + + shadow = kvm_mmu_lookup_page(vcpu, gfn); + if (shadow) { + pgprintk("%s: found shadow page for %lx, marking ro\n", + __FUNCTION__, gfn); + access_bits &= ~PT_WRITABLE_MASK; + if (is_writeble_pte(*shadow_pte)) { + *shadow_pte &= ~PT_WRITABLE_MASK; + kvm_arch_ops->tlb_flush(vcpu); + } + } } + + if (access_bits & PT_WRITABLE_MASK) + mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + + page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); + rmap_add(vcpu, shadow_pte); } static void inject_page_fault(struct kvm_vcpu *vcpu, @@ -474,41 +1002,6 @@ static int may_access(u64 pte, int write, int user) return 1; } -/* - * Remove a shadow pte. - */ -static void paging_inval_page(struct kvm_vcpu *vcpu, gva_t addr) -{ - hpa_t page_addr = vcpu->mmu.root_hpa; - int level = vcpu->mmu.shadow_root_level; - - ++kvm_stat.invlpg; - - for (; ; level--) { - u32 index = PT64_INDEX(addr, level); - u64 *table = __va(page_addr); - - if (level == PT_PAGE_TABLE_LEVEL ) { - table[index] = 0; - return; - } - - if (!is_present_pte(table[index])) - return; - - page_addr = table[index] & PT64_BASE_ADDR_MASK; - - if (level == PT_DIRECTORY_LEVEL && - (table[index] & PT_SHADOW_PS_MARK)) { - table[index] = 0; - release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL); - - kvm_arch_ops->tlb_flush(vcpu); - return; - } - } -} - static void paging_free(struct kvm_vcpu *vcpu) { nonpaging_free(vcpu); @@ -522,37 +1015,40 @@ static void paging_free(struct kvm_vcpu *vcpu) #include "paging_tmpl.h" #undef PTTYPE -static int paging64_init_context(struct kvm_vcpu *vcpu) +static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) { struct kvm_mmu *context = &vcpu->mmu; ASSERT(is_pae(vcpu)); context->new_cr3 = paging_new_cr3; context->page_fault = paging64_page_fault; - context->inval_page = paging_inval_page; context->gva_to_gpa = paging64_gva_to_gpa; context->free = paging_free; - context->root_level = PT64_ROOT_LEVEL; - context->shadow_root_level = PT64_ROOT_LEVEL; - context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL); + context->root_level = level; + context->shadow_root_level = level; + mmu_alloc_roots(vcpu); ASSERT(VALID_PAGE(context->root_hpa)); kvm_arch_ops->set_cr3(vcpu, context->root_hpa | (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK))); return 0; } +static int paging64_init_context(struct kvm_vcpu *vcpu) +{ + return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL); +} + static int paging32_init_context(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = &vcpu->mmu; context->new_cr3 = paging_new_cr3; context->page_fault = paging32_page_fault; - context->inval_page = paging_inval_page; context->gva_to_gpa = paging32_gva_to_gpa; context->free = paging_free; context->root_level = PT32_ROOT_LEVEL; context->shadow_root_level = PT32E_ROOT_LEVEL; - context->root_hpa = kvm_mmu_alloc_page(vcpu, NULL); + mmu_alloc_roots(vcpu); ASSERT(VALID_PAGE(context->root_hpa)); kvm_arch_ops->set_cr3(vcpu, context->root_hpa | (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK))); @@ -561,14 +1057,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu) static int paging32E_init_context(struct kvm_vcpu *vcpu) { - int ret; - - if ((ret = paging64_init_context(vcpu))) - return ret; - - vcpu->mmu.root_level = PT32E_ROOT_LEVEL; - vcpu->mmu.shadow_root_level = PT32E_ROOT_LEVEL; - return 0; + return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL); } static int init_kvm_mmu(struct kvm_vcpu *vcpu) @@ -597,41 +1086,161 @@ static void destroy_kvm_mmu(struct kvm_vcpu *vcpu) int kvm_mmu_reset_context(struct kvm_vcpu *vcpu) { + int r; + destroy_kvm_mmu(vcpu); - return init_kvm_mmu(vcpu); + r = init_kvm_mmu(vcpu); + if (r < 0) + goto out; + r = mmu_topup_memory_caches(vcpu); +out: + return r; } -static void free_mmu_pages(struct kvm_vcpu *vcpu) +void kvm_mmu_pre_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes) { - while (!list_empty(&vcpu->free_pages)) { + gfn_t gfn = gpa >> PAGE_SHIFT; + struct kvm_mmu_page *page; + struct kvm_mmu_page *child; + struct hlist_node *node, *n; + struct hlist_head *bucket; + unsigned index; + u64 *spte; + u64 pte; + unsigned offset = offset_in_page(gpa); + unsigned pte_size; + unsigned page_offset; + unsigned misaligned; + int level; + int flooded = 0; + + pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + if (gfn == vcpu->last_pt_write_gfn) { + ++vcpu->last_pt_write_count; + if (vcpu->last_pt_write_count >= 3) + flooded = 1; + } else { + vcpu->last_pt_write_gfn = gfn; + vcpu->last_pt_write_count = 1; + } + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->mmu_page_hash[index]; + hlist_for_each_entry_safe(page, node, n, bucket, hash_link) { + if (page->gfn != gfn || page->role.metaphysical) + continue; + pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; + misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); + if (misaligned || flooded) { + /* + * Misaligned accesses are too much trouble to fix + * up; also, they usually indicate a page is not used + * as a page table. + * + * If we're seeing too many writes to a page, + * it may no longer be a page table, or we may be + * forking, in which case it is better to unmap the + * page. + */ + pgprintk("misaligned: gpa %llx bytes %d role %x\n", + gpa, bytes, page->role.word); + kvm_mmu_zap_page(vcpu, page); + continue; + } + page_offset = offset; + level = page->role.level; + if (page->role.glevels == PT32_ROOT_LEVEL) { + page_offset <<= 1; /* 32->64 */ + page_offset &= ~PAGE_MASK; + } + spte = __va(page->page_hpa); + spte += page_offset / sizeof(*spte); + pte = *spte; + if (is_present_pte(pte)) { + if (level == PT_PAGE_TABLE_LEVEL) + rmap_remove(vcpu, spte); + else { + child = page_header(pte & PT64_BASE_ADDR_MASK); + mmu_page_remove_parent_pte(vcpu, child, spte); + } + } + *spte = 0; + } +} + +void kvm_mmu_post_write(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes) +{ +} + +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) +{ + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); + + return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT); +} + +void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *page; + page = container_of(vcpu->kvm->active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu, page); + } +} +EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages); + +static void free_mmu_pages(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu_page *page; + + while (!list_empty(&vcpu->kvm->active_mmu_pages)) { + page = container_of(vcpu->kvm->active_mmu_pages.next, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu, page); + } + while (!list_empty(&vcpu->free_pages)) { page = list_entry(vcpu->free_pages.next, struct kvm_mmu_page, link); list_del(&page->link); __free_page(pfn_to_page(page->page_hpa >> PAGE_SHIFT)); page->page_hpa = INVALID_PAGE; } + free_page((unsigned long)vcpu->mmu.pae_root); } static int alloc_mmu_pages(struct kvm_vcpu *vcpu) { + struct page *page; int i; ASSERT(vcpu); for (i = 0; i < KVM_NUM_MMU_PAGES; i++) { - struct page *page; struct kvm_mmu_page *page_header = &vcpu->page_header_buf[i]; INIT_LIST_HEAD(&page_header->link); - if ((page = alloc_page(GFP_KVM_MMU)) == NULL) + if ((page = alloc_page(GFP_KERNEL)) == NULL) goto error_1; page->private = (unsigned long)page_header; page_header->page_hpa = (hpa_t)page_to_pfn(page) << PAGE_SHIFT; memset(__va(page_header->page_hpa), 0, PAGE_SIZE); list_add(&page_header->link, &vcpu->free_pages); + ++vcpu->kvm->n_free_mmu_pages; } + + /* + * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. + * Therefore we need to allocate shadow page tables in the first + * 4GB of memory, which happens to fit the DMA32 zone. + */ + page = alloc_page(GFP_KERNEL | __GFP_DMA32); + if (!page) + goto error_1; + vcpu->mmu.pae_root = page_address(page); + for (i = 0; i < 4; ++i) + vcpu->mmu.pae_root[i] = INVALID_PAGE; + return 0; error_1: @@ -663,10 +1272,12 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu) destroy_kvm_mmu(vcpu); free_mmu_pages(vcpu); + mmu_free_memory_caches(vcpu); } -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) +void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot) { + struct kvm *kvm = vcpu->kvm; struct kvm_mmu_page *page; list_for_each_entry(page, &kvm->active_mmu_pages, link) { @@ -679,8 +1290,169 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) pt = __va(page->page_hpa); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ - if (pt[i] & PT_WRITABLE_MASK) + if (pt[i] & PT_WRITABLE_MASK) { + rmap_remove(vcpu, &pt[i]); pt[i] &= ~PT_WRITABLE_MASK; + } + } +} + +#ifdef AUDIT + +static const char *audit_msg; + +static gva_t canonicalize(gva_t gva) +{ +#ifdef CONFIG_X86_64 + gva = (long long)(gva << 16) >> 16; +#endif + return gva; +} +static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, + gva_t va, int level) +{ + u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK); + int i; + gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1)); + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) { + u64 ent = pt[i]; + + if (!ent & PT_PRESENT_MASK) + continue; + + va = canonicalize(va); + if (level > 1) + audit_mappings_page(vcpu, ent, va, level - 1); + else { + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); + hpa_t hpa = gpa_to_hpa(vcpu, gpa); + + if ((ent & PT_PRESENT_MASK) + && (ent & PT64_BASE_ADDR_MASK) != hpa) + printk(KERN_ERR "audit error: (%s) levels %d" + " gva %lx gpa %llx hpa %llx ent %llx\n", + audit_msg, vcpu->mmu.root_level, + va, gpa, hpa, ent); + } } } + +static void audit_mappings(struct kvm_vcpu *vcpu) +{ + int i; + + if (vcpu->mmu.root_level == 4) + audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4); + else + for (i = 0; i < 4; ++i) + if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK) + audit_mappings_page(vcpu, + vcpu->mmu.pae_root[i], + i << 30, + 2); +} + +static int count_rmaps(struct kvm_vcpu *vcpu) +{ + int nmaps = 0; + int i, j, k; + + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *m = &vcpu->kvm->memslots[i]; + struct kvm_rmap_desc *d; + + for (j = 0; j < m->npages; ++j) { + struct page *page = m->phys_mem[j]; + + if (!page->private) + continue; + if (!(page->private & 1)) { + ++nmaps; + continue; + } + d = (struct kvm_rmap_desc *)(page->private & ~1ul); + while (d) { + for (k = 0; k < RMAP_EXT; ++k) + if (d->shadow_ptes[k]) + ++nmaps; + else + break; + d = d->more; + } + } + } + return nmaps; +} + +static int count_writable_mappings(struct kvm_vcpu *vcpu) +{ + int nmaps = 0; + struct kvm_mmu_page *page; + int i; + + list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { + u64 *pt = __va(page->page_hpa); + + if (page->role.level != PT_PAGE_TABLE_LEVEL) + continue; + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + u64 ent = pt[i]; + + if (!(ent & PT_PRESENT_MASK)) + continue; + if (!(ent & PT_WRITABLE_MASK)) + continue; + ++nmaps; + } + } + return nmaps; +} + +static void audit_rmap(struct kvm_vcpu *vcpu) +{ + int n_rmap = count_rmaps(vcpu); + int n_actual = count_writable_mappings(vcpu); + + if (n_rmap != n_actual) + printk(KERN_ERR "%s: (%s) rmap %d actual %d\n", + __FUNCTION__, audit_msg, n_rmap, n_actual); +} + +static void audit_write_protection(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu_page *page; + + list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { + hfn_t hfn; + struct page *pg; + + if (page->role.metaphysical) + continue; + + hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT) + >> PAGE_SHIFT; + pg = pfn_to_page(hfn); + if (pg->private) + printk(KERN_ERR "%s: (%s) shadow page has writable" + " mappings: gfn %lx role %x\n", + __FUNCTION__, audit_msg, page->gfn, + page->role.word); + } +} + +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) +{ + int olddbg = dbg; + + dbg = 0; + audit_msg = msg; + audit_rmap(vcpu); + audit_write_protection(vcpu); + audit_mappings(vcpu); + dbg = olddbg; +} + +#endif diff --git a/kvm/kernel/paging_tmpl.h b/kvm/kernel/paging_tmpl.h index 09bb9b4ed..2dbf4307e 100644 --- a/kvm/kernel/paging_tmpl.h +++ b/kvm/kernel/paging_tmpl.h @@ -32,6 +32,11 @@ #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level) #define PT_PTE_COPY_MASK PT64_PTE_COPY_MASK + #ifdef CONFIG_X86_64 + #define PT_MAX_FULL_LEVELS 4 + #else + #define PT_MAX_FULL_LEVELS 2 + #endif #elif PTTYPE == 32 #define pt_element_t u32 #define guest_walker guest_walker32 @@ -42,6 +47,7 @@ #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) #define PT_PTE_COPY_MASK PT32_PTE_COPY_MASK + #define PT_MAX_FULL_LEVELS 2 #else #error Invalid PTTYPE value #endif @@ -52,93 +58,126 @@ */ struct guest_walker { int level; + gfn_t table_gfn[PT_MAX_FULL_LEVELS]; pt_element_t *table; + pt_element_t *ptep; pt_element_t inherited_ar; + gfn_t gfn; }; -static void FNAME(init_walker)(struct guest_walker *walker, - struct kvm_vcpu *vcpu) +/* + * Fetch a guest pte for a guest virtual address + */ +static void FNAME(walk_addr)(struct guest_walker *walker, + struct kvm_vcpu *vcpu, gva_t addr) { hpa_t hpa; struct kvm_memory_slot *slot; + pt_element_t *ptep; + pt_element_t root; + gfn_t table_gfn; + pgprintk("%s: addr %lx\n", __FUNCTION__, addr); walker->level = vcpu->mmu.root_level; - slot = gfn_to_memslot(vcpu->kvm, - (vcpu->cr3 & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - hpa = safe_gpa_to_hpa(vcpu, vcpu->cr3 & PT64_BASE_ADDR_MASK); + walker->table = NULL; + root = vcpu->cr3; +#if PTTYPE == 64 + if (!is_long_mode(vcpu)) { + walker->ptep = &vcpu->pdptrs[(addr >> 30) & 3]; + root = *walker->ptep; + if (!(root & PT_PRESENT_MASK)) + return; + --walker->level; + } +#endif + table_gfn = (root & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + walker->table_gfn[walker->level - 1] = table_gfn; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); + slot = gfn_to_memslot(vcpu->kvm, table_gfn); + hpa = safe_gpa_to_hpa(vcpu, root & PT64_BASE_ADDR_MASK); walker->table = kmap_atomic(pfn_to_page(hpa >> PAGE_SHIFT), KM_USER0); ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & ~(PAGE_MASK | CR3_FLAGS_MASK)) == 0); - walker->table = (pt_element_t *)( (unsigned long)walker->table | - (unsigned long)(vcpu->cr3 & ~(PAGE_MASK | CR3_FLAGS_MASK)) ); walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; + + for (;;) { + int index = PT_INDEX(addr, walker->level); + hpa_t paddr; + + ptep = &walker->table[index]; + ASSERT(((unsigned long)walker->table & PAGE_MASK) == + ((unsigned long)ptep & PAGE_MASK)); + + if (is_present_pte(*ptep) && !(*ptep & PT_ACCESSED_MASK)) + *ptep |= PT_ACCESSED_MASK; + + if (!is_present_pte(*ptep)) + break; + + if (walker->level == PT_PAGE_TABLE_LEVEL) { + walker->gfn = (*ptep & PT_BASE_ADDR_MASK) + >> PAGE_SHIFT; + break; + } + + if (walker->level == PT_DIRECTORY_LEVEL + && (*ptep & PT_PAGE_SIZE_MASK) + && (PTTYPE == 64 || is_pse(vcpu))) { + walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) + >> PAGE_SHIFT; + walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + break; + } + + if (walker->level != 3 || is_long_mode(vcpu)) + walker->inherited_ar &= walker->table[index]; + table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; + paddr = safe_gpa_to_hpa(vcpu, *ptep & PT_BASE_ADDR_MASK); + kunmap_atomic(walker->table, KM_USER0); + walker->table = kmap_atomic(pfn_to_page(paddr >> PAGE_SHIFT), + KM_USER0); + --walker->level; + walker->table_gfn[walker->level - 1 ] = table_gfn; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); + } + walker->ptep = ptep; + pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)*ptep); } static void FNAME(release_walker)(struct guest_walker *walker) { - kunmap_atomic(walker->table, KM_USER0); + if (walker->table) + kunmap_atomic(walker->table, KM_USER0); } static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 guest_pte, - u64 *shadow_pte, u64 access_bits) + u64 *shadow_pte, u64 access_bits, gfn_t gfn) { ASSERT(*shadow_pte == 0); access_bits &= guest_pte; *shadow_pte = (guest_pte & PT_PTE_COPY_MASK); set_pte_common(vcpu, shadow_pte, guest_pte & PT_BASE_ADDR_MASK, - guest_pte & PT_DIRTY_MASK, access_bits); + guest_pte & PT_DIRTY_MASK, access_bits, gfn); } static void FNAME(set_pde)(struct kvm_vcpu *vcpu, u64 guest_pde, - u64 *shadow_pte, u64 access_bits, - int index) + u64 *shadow_pte, u64 access_bits, gfn_t gfn) { gpa_t gaddr; ASSERT(*shadow_pte == 0); access_bits &= guest_pde; - gaddr = (guest_pde & PT_DIR_BASE_ADDR_MASK) + PAGE_SIZE * index; + gaddr = (gpa_t)gfn << PAGE_SHIFT; if (PTTYPE == 32 && is_cpuid_PSE36()) gaddr |= (guest_pde & PT32_DIR_PSE36_MASK) << (32 - PT32_DIR_PSE36_SHIFT); *shadow_pte = guest_pde & PT_PTE_COPY_MASK; set_pte_common(vcpu, shadow_pte, gaddr, - guest_pde & PT_DIRTY_MASK, access_bits); -} - -/* - * Fetch a guest pte from a specific level in the paging hierarchy. - */ -static pt_element_t *FNAME(fetch_guest)(struct kvm_vcpu *vcpu, - struct guest_walker *walker, - int level, - gva_t addr) -{ - - ASSERT(level > 0 && level <= walker->level); - - for (;;) { - int index = PT_INDEX(addr, walker->level); - hpa_t paddr; - - ASSERT(((unsigned long)walker->table & PAGE_MASK) == - ((unsigned long)&walker->table[index] & PAGE_MASK)); - if (level == walker->level || - !is_present_pte(walker->table[index]) || - (walker->level == PT_DIRECTORY_LEVEL && - (walker->table[index] & PT_PAGE_SIZE_MASK) && - (PTTYPE == 64 || is_pse(vcpu)))) - return &walker->table[index]; - if (walker->level != 3 || is_long_mode(vcpu)) - walker->inherited_ar &= walker->table[index]; - paddr = safe_gpa_to_hpa(vcpu, walker->table[index] & PT_BASE_ADDR_MASK); - kunmap_atomic(walker->table, KM_USER0); - walker->table = kmap_atomic(pfn_to_page(paddr >> PAGE_SHIFT), - KM_USER0); - --walker->level; - } + guest_pde & PT_DIRTY_MASK, access_bits, gfn); } /* @@ -150,15 +189,26 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *prev_shadow_ent = NULL; + pt_element_t *guest_ent = walker->ptep; + + if (!is_present_pte(*guest_ent)) + return NULL; shadow_addr = vcpu->mmu.root_hpa; level = vcpu->mmu.shadow_root_level; + if (level == PT32E_ROOT_LEVEL) { + shadow_addr = vcpu->mmu.pae_root[(addr >> 30) & 3]; + shadow_addr &= PT64_BASE_ADDR_MASK; + --level; + } for (; ; level--) { u32 index = SHADOW_PT_INDEX(addr, level); u64 *shadow_ent = ((u64 *)__va(shadow_addr)) + index; - pt_element_t *guest_ent; + struct kvm_mmu_page *shadow_page; u64 shadow_pte; + int metaphysical; + gfn_t table_gfn; if (is_present_pte(*shadow_ent) || is_io_pte(*shadow_ent)) { if (level == PT_PAGE_TABLE_LEVEL) @@ -168,21 +218,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, continue; } - if (PTTYPE == 32 && level > PT32_ROOT_LEVEL) { - ASSERT(level == PT32E_ROOT_LEVEL); - guest_ent = FNAME(fetch_guest)(vcpu, walker, - PT32_ROOT_LEVEL, addr); - } else - guest_ent = FNAME(fetch_guest)(vcpu, walker, - level, addr); - - if (!is_present_pte(*guest_ent)) - return NULL; - - /* Don't set accessed bit on PAE PDPTRs */ - if (vcpu->mmu.root_level != 3 || walker->level != 3) - *guest_ent |= PT_ACCESSED_MASK; - if (level == PT_PAGE_TABLE_LEVEL) { if (walker->level == PT_DIRECTORY_LEVEL) { @@ -190,21 +225,30 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, *prev_shadow_ent |= PT_SHADOW_PS_MARK; FNAME(set_pde)(vcpu, *guest_ent, shadow_ent, walker->inherited_ar, - PT_INDEX(addr, PT_PAGE_TABLE_LEVEL)); + walker->gfn); } else { ASSERT(walker->level == PT_PAGE_TABLE_LEVEL); - FNAME(set_pte)(vcpu, *guest_ent, shadow_ent, walker->inherited_ar); + FNAME(set_pte)(vcpu, *guest_ent, shadow_ent, + walker->inherited_ar, + walker->gfn); } return shadow_ent; } - shadow_addr = kvm_mmu_alloc_page(vcpu, shadow_ent); - if (!VALID_PAGE(shadow_addr)) - return ERR_PTR(-ENOMEM); - shadow_pte = shadow_addr | PT_PRESENT_MASK; - if (vcpu->mmu.root_level > 3 || level != 3) - shadow_pte |= PT_ACCESSED_MASK - | PT_WRITABLE_MASK | PT_USER_MASK; + if (level - 1 == PT_PAGE_TABLE_LEVEL + && walker->level == PT_DIRECTORY_LEVEL) { + metaphysical = 1; + table_gfn = (*guest_ent & PT_BASE_ADDR_MASK) + >> PAGE_SHIFT; + } else { + metaphysical = 0; + table_gfn = walker->table_gfn[level - 2]; + } + shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, + metaphysical, shadow_ent); + shadow_addr = shadow_page->page_hpa; + shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK + | PT_WRITABLE_MASK | PT_USER_MASK; *shadow_ent = shadow_pte; prev_shadow_ent = shadow_ent; } @@ -221,11 +265,13 @@ static int FNAME(fix_write_pf)(struct kvm_vcpu *vcpu, u64 *shadow_ent, struct guest_walker *walker, gva_t addr, - int user) + int user, + int *write_pt) { pt_element_t *guest_ent; int writable_shadow; gfn_t gfn; + struct kvm_mmu_page *page; if (is_writeble_pte(*shadow_ent)) return 0; @@ -250,17 +296,35 @@ static int FNAME(fix_write_pf)(struct kvm_vcpu *vcpu, *shadow_ent &= ~PT_USER_MASK; } - guest_ent = FNAME(fetch_guest)(vcpu, walker, PT_PAGE_TABLE_LEVEL, addr); + guest_ent = walker->ptep; if (!is_present_pte(*guest_ent)) { *shadow_ent = 0; return 0; } - gfn = (*guest_ent & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + gfn = walker->gfn; + + if (user) { + /* + * Usermode page faults won't be for page table updates. + */ + while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) { + pgprintk("%s: zap %lx %x\n", + __FUNCTION__, gfn, page->role.word); + kvm_mmu_zap_page(vcpu, page); + } + } else if (kvm_mmu_lookup_page(vcpu, gfn)) { + pgprintk("%s: found shadow page for %lx, marking ro\n", + __FUNCTION__, gfn); + *guest_ent |= PT_DIRTY_MASK; + *write_pt = 1; + return 0; + } mark_page_dirty(vcpu->kvm, gfn); *shadow_ent |= PT_WRITABLE_MASK; *guest_ent |= PT_DIRTY_MASK; + rmap_add(vcpu, shadow_ent); return 1; } @@ -276,7 +340,8 @@ static int FNAME(fix_write_pf)(struct kvm_vcpu *vcpu, * - normal guest page fault due to the guest pte marked not present, not * writable, or not executable * - * Returns: 1 if we need to emulate the instruction, 0 otherwise + * Returns: 1 if we need to emulate the instruction, 0 otherwise, or + * a negative value on error. */ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, u32 error_code) @@ -287,39 +352,47 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, struct guest_walker walker; u64 *shadow_pte; int fixed; + int write_pt = 0; + int r; + + pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code); + kvm_mmu_audit(vcpu, "pre page fault"); + + r = mmu_topup_memory_caches(vcpu); + if (r) + return r; /* * Look up the shadow pte for the faulting address. */ - for (;;) { - FNAME(init_walker)(&walker, vcpu); - shadow_pte = FNAME(fetch)(vcpu, addr, &walker); - if (IS_ERR(shadow_pte)) { /* must be -ENOMEM */ - nonpaging_flush(vcpu); - FNAME(release_walker)(&walker); - continue; - } - break; - } + FNAME(walk_addr)(&walker, vcpu, addr); + shadow_pte = FNAME(fetch)(vcpu, addr, &walker); /* * The page is not mapped by the guest. Let the guest handle it. */ if (!shadow_pte) { + pgprintk("%s: not mapped\n", __FUNCTION__); inject_page_fault(vcpu, addr, error_code); FNAME(release_walker)(&walker); return 0; } + pgprintk("%s: shadow pte %p %llx\n", __FUNCTION__, + shadow_pte, *shadow_pte); + /* * Update the shadow pte. */ if (write_fault) fixed = FNAME(fix_write_pf)(vcpu, shadow_pte, &walker, addr, - user_fault); + user_fault, &write_pt); else fixed = fix_read_pf(shadow_pte); + pgprintk("%s: updated shadow pte %p %llx\n", __FUNCTION__, + shadow_pte, *shadow_pte); + FNAME(release_walker)(&walker); /* @@ -331,20 +404,23 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, pgprintk("%s: io work, no access\n", __FUNCTION__); inject_page_fault(vcpu, addr, error_code | PFERR_PRESENT_MASK); + kvm_mmu_audit(vcpu, "post page fault (io)"); return 0; } /* * pte not present, guest page fault. */ - if (pte_present && !fixed) { + if (pte_present && !fixed && !write_pt) { inject_page_fault(vcpu, addr, error_code); + kvm_mmu_audit(vcpu, "post page fault (guest)"); return 0; } ++kvm_stat.pf_fixed; + kvm_mmu_audit(vcpu, "post page fault (fixed)"); - return 0; + return write_pt; } static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) @@ -353,9 +429,8 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) pt_element_t guest_pte; gpa_t gpa; - FNAME(init_walker)(&walker, vcpu); - guest_pte = *FNAME(fetch_guest)(vcpu, &walker, PT_PAGE_TABLE_LEVEL, - vaddr); + FNAME(walk_addr)(&walker, vcpu, vaddr); + guest_pte = *walker.ptep; FNAME(release_walker)(&walker); if (!is_present_pte(guest_pte)) @@ -389,3 +464,4 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) #undef PT_PTE_COPY_MASK #undef PT_NON_PTE_COPY_MASK #undef PT_DIR_BASE_ADDR_MASK +#undef PT_MAX_FULL_LEVELS diff --git a/kvm/kernel/svm.c b/kvm/kernel/svm.c index 855207a9b..ccc06b1b9 100644 --- a/kvm/kernel/svm.c +++ b/kvm/kernel/svm.c @@ -497,7 +497,6 @@ static void init_vmcb(struct vmcb *vmcb) /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */ (1ULL << INTERCEPT_CPUID) | (1ULL << INTERCEPT_HLT) | - (1ULL << INTERCEPT_INVLPG) | (1ULL << INTERCEPT_INVLPGA) | (1ULL << INTERCEPT_IOIO_PROT) | (1ULL << INTERCEPT_MSR_PROT) | @@ -702,6 +701,10 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) vcpu->svm->vmcb->save.gdtr.base = dt->base ; } +static void svm_decache_cr0_cr4_guest_bits(struct kvm_vcpu *vcpu) +{ +} + static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { #ifdef CONFIG_X86_64 @@ -849,6 +852,7 @@ static int pf_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) u64 fault_address; u32 error_code; enum emulation_result er; + int r; if (is_external_interrupt(exit_int_info)) push_irq(vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK); @@ -857,7 +861,12 @@ static int pf_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) fault_address = vcpu->svm->vmcb->control.exit_info_2; error_code = vcpu->svm->vmcb->control.exit_info_1; - if (!vcpu->mmu.page_fault(vcpu, fault_address, error_code)) { + r = kvm_mmu_page_fault(vcpu, fault_address, error_code); + if (r < 0) { + spin_unlock(&vcpu->kvm->lock); + return r; + } + if (!r) { spin_unlock(&vcpu->kvm->lock); return 1; } @@ -1197,8 +1206,7 @@ static int interrupt_window_interception(struct kvm_vcpu *vcpu, * possible */ if (kvm_run->request_interrupt_window && - !vcpu->irq_summary && - (vcpu->svm->vmcb->save.rflags & X86_EFLAGS_IF)) { + !vcpu->irq_summary) { ++kvm_stat.irq_window_exits; kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; return 0; @@ -1395,6 +1403,7 @@ static int svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) u16 fs_selector; u16 gs_selector; u16 ldt_selector; + int r; again: do_interrupt_requests(vcpu, kvm_run); @@ -1562,7 +1571,8 @@ again: return 0; } - if (handle_exit(vcpu, kvm_run)) { + r = handle_exit(vcpu, kvm_run); + if (r > 0) { if (signal_pending(current)) { ++kvm_stat.signal_exits; post_kvm_run_save(vcpu, kvm_run); @@ -1578,7 +1588,7 @@ again: goto again; } post_kvm_run_save(vcpu, kvm_run); - return 0; + return r; } static void svm_flush_tlb(struct kvm_vcpu *vcpu) @@ -1645,6 +1655,7 @@ static struct kvm_arch_ops svm_arch_ops = { .get_segment = svm_get_segment, .set_segment = svm_set_segment, .get_cs_db_l_bits = svm_get_cs_db_l_bits, + .decache_cr0_cr4_guest_bits = svm_decache_cr0_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr0_no_modeswitch = svm_set_cr0, .set_cr3 = svm_set_cr3, diff --git a/kvm/kernel/debug.c b/kvm/kernel/vmx-debug.c index 5b297fc36..cf9458c64 100644 --- a/kvm/kernel/debug.c +++ b/kvm/kernel/vmx-debug.c @@ -28,6 +28,34 @@ static const char *vmx_msr_name[] = { #define NR_VMX_MSR (sizeof(vmx_msr_name) / sizeof(char*)) +static unsigned long vmcs_readl(unsigned long field) +{ + unsigned long value; + + asm volatile (ASM_VMX_VMREAD_RDX_RAX + : "=a"(value) : "d"(field) : "cc"); + return value; +} + +static u16 vmcs_read16(unsigned long field) +{ + return vmcs_readl(field); +} + +static u32 vmcs_read32(unsigned long field) +{ + return vmcs_readl(field); +} + +static u64 vmcs_read64(unsigned long field) +{ +#ifdef CONFIG_X86_64 + return vmcs_readl(field); +#else + return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32); +#endif +} + void show_msrs(struct kvm_vcpu *vcpu) { int i; @@ -47,7 +75,7 @@ void show_code(struct kvm_vcpu *vcpu) char buf[30 + 3 * sizeof code]; int i; - if (!is_long_mode()) + if (!is_long_mode(vcpu)) rip += vmcs_readl(GUEST_CS_BASE); kvm_read_guest(vcpu, rip, sizeof code, code); @@ -71,10 +99,10 @@ void show_irq(struct kvm_vcpu *vcpu, int irq) unsigned long idt_limit = vmcs_readl(GUEST_IDTR_LIMIT); struct gate_struct gate; - if (!is_long_mode()) + if (!is_long_mode(vcpu)) vcpu_printf(vcpu, "%s: not in long mode\n", __FUNCTION__); - if (!is_long_mode() || idt_limit < irq * sizeof(gate)) { + if (!is_long_mode(vcpu) || idt_limit < irq * sizeof(gate)) { vcpu_printf(vcpu, "%s: 0x%x read_guest err\n", __FUNCTION__, irq); @@ -235,7 +263,7 @@ int vm_entry_test_guest(struct kvm_vcpu *vcpu) return 0; } - long_mode = is_long_mode(); + long_mode = is_long_mode(vcpu); if (long_mode) { } @@ -1017,10 +1045,10 @@ void regs_dump(struct kvm_vcpu *vcpu) void sregs_dump(struct kvm_vcpu *vcpu) { vcpu_printf(vcpu, "************************ sregs_dump ************************\n"); - vcpu_printf(vcpu, "cr0 = 0x%lx\n", guest_cr0()); + vcpu_printf(vcpu, "cr0 = 0x%lx\n", vcpu->cr0); vcpu_printf(vcpu, "cr2 = 0x%lx\n", vcpu->cr2); vcpu_printf(vcpu, "cr3 = 0x%lx\n", vcpu->cr3); - vcpu_printf(vcpu, "cr4 = 0x%lx\n", guest_cr4()); + vcpu_printf(vcpu, "cr4 = 0x%lx\n", vcpu->cr4); vcpu_printf(vcpu, "cr8 = 0x%lx\n", vcpu->cr8); vcpu_printf(vcpu, "shadow_efer = 0x%llx\n", vcpu->shadow_efer); vcpu_printf(vcpu, "***********************************************************\n"); diff --git a/kvm/kernel/vmx.c b/kvm/kernel/vmx.c index c55635ddf..d4701cb4c 100644 --- a/kvm/kernel/vmx.c +++ b/kvm/kernel/vmx.c @@ -152,15 +152,21 @@ static u64 vmcs_read64(unsigned long field) #endif } +static noinline void vmwrite_error(unsigned long field, unsigned long value) +{ + printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n", + field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); + dump_stack(); +} + static void vmcs_writel(unsigned long field, unsigned long value) { u8 error; asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0" : "=q"(error) : "a"(value), "d"(field) : "cc" ); - if (error) - printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n", - field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); + if (unlikely(error)) + vmwrite_error(field, value); } static void vmcs_write16(unsigned long field, u16 value) @@ -737,6 +743,15 @@ static void exit_lmode(struct kvm_vcpu *vcpu) #endif +static void vmx_decache_cr0_cr4_guest_bits(struct kvm_vcpu *vcpu) +{ + vcpu->cr0 &= KVM_GUEST_CR0_MASK; + vcpu->cr0 |= vmcs_readl(GUEST_CR0) & ~KVM_GUEST_CR0_MASK; + + vcpu->cr4 &= KVM_GUEST_CR4_MASK; + vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; +} + static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { if (vcpu->rmode.active && (cr0 & CR0_PE_MASK)) @@ -1012,8 +1027,6 @@ static int vmx_vcpu_setup(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_RIP, 0xfff0); vmcs_writel(GUEST_RSP, 0); - vmcs_writel(GUEST_CR3, 0); - //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 vmcs_writel(GUEST_DR7, 0x400); @@ -1050,7 +1063,6 @@ static int vmx_vcpu_setup(struct kvm_vcpu *vcpu) | CPU_BASED_CR8_LOAD_EXITING /* 20.6.2 */ | CPU_BASED_CR8_STORE_EXITING /* 20.6.2 */ | CPU_BASED_UNCOND_IO_EXITING /* 20.6.2 */ - | CPU_BASED_INVDPG_EXITING | CPU_BASED_MOV_DR_EXITING | CPU_BASED_USE_TSC_OFFSETING /* 21.3 */ ); @@ -1281,6 +1293,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) unsigned long cr2, rip; u32 vect_info; enum emulation_result er; + int r; vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); intr_info = vmcs_read32(VM_EXIT_INTR_INFO); @@ -1309,7 +1322,12 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) cr2 = vmcs_readl(EXIT_QUALIFICATION); spin_lock(&vcpu->kvm->lock); - if (!vcpu->mmu.page_fault(vcpu, cr2, error_code)) { + r = kvm_mmu_page_fault(vcpu, cr2, error_code); + if (r < 0) { + spin_unlock(&vcpu->kvm->lock); + return r; + } + if (!r) { spin_unlock(&vcpu->kvm->lock); return 1; } @@ -1429,17 +1447,6 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 0; } -static int handle_invlpg(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - u64 address = vmcs_read64(EXIT_QUALIFICATION); - int instruction_length = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); - spin_lock(&vcpu->kvm->lock); - vcpu->mmu.inval_page(vcpu, address); - spin_unlock(&vcpu->kvm->lock); - vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) + instruction_length); - return 1; -} - static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { u64 exit_qualification; @@ -1597,8 +1604,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu, * possible */ if (kvm_run->request_interrupt_window && - !vcpu->irq_summary && - (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF)) { + !vcpu->irq_summary) { kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; ++kvm_stat.irq_window_exits; return 0; @@ -1627,7 +1633,6 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, [EXIT_REASON_EXCEPTION_NMI] = handle_exception, [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, [EXIT_REASON_IO_INSTRUCTION] = handle_io, - [EXIT_REASON_INVLPG] = handle_invlpg, [EXIT_REASON_CR_ACCESS] = handle_cr, [EXIT_REASON_DR_ACCESS] = handle_dr, [EXIT_REASON_CPUID] = handle_cpuid, @@ -1684,6 +1689,7 @@ static int vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) u8 fail; u16 fs_sel, gs_sel, ldt_sel; int fs_gs_ldt_reload_needed; + int r; again: /* @@ -1857,6 +1863,7 @@ again: if (fail) { kvm_run->exit_type = KVM_EXIT_TYPE_FAIL_ENTRY; kvm_run->exit_reason = vmcs_read32(VM_INSTRUCTION_ERROR); + r = 0; } else { if (fs_gs_ldt_reload_needed) { load_ldt(ldt_sel); @@ -1876,7 +1883,8 @@ again: } vcpu->launched = 1; kvm_run->exit_type = KVM_EXIT_TYPE_VM_EXIT; - if (kvm_handle_exit(kvm_run, vcpu)) { + r = kvm_handle_exit(kvm_run, vcpu); + if (r > 0) { /* Give scheduler a change to reschedule. */ if (signal_pending(current)) { ++kvm_stat.signal_exits; @@ -1896,7 +1904,7 @@ again: } post_kvm_run_save(vcpu, kvm_run); - return 0; + return r; } static void vmx_flush_tlb(struct kvm_vcpu *vcpu) @@ -2002,6 +2010,7 @@ static struct kvm_arch_ops vmx_arch_ops = { .get_segment = vmx_get_segment, .set_segment = vmx_set_segment, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, + .decache_cr0_cr4_guest_bits = vmx_decache_cr0_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr0_no_modeswitch = vmx_set_cr0_no_modeswitch, .set_cr3 = vmx_set_cr3, diff --git a/kvm/kernel/x86_emulate.c b/kvm/kernel/x86_emulate.c index 1bff3e925..be70795b4 100644 --- a/kvm/kernel/x86_emulate.c +++ b/kvm/kernel/x86_emulate.c @@ -1323,7 +1323,7 @@ twobyte_special_insn: ctxt)) != 0)) goto done; if ((old_lo != _regs[VCPU_REGS_RAX]) - || (old_hi != _regs[VCPU_REGS_RDI])) { + || (old_hi != _regs[VCPU_REGS_RDX])) { _regs[VCPU_REGS_RAX] = old_lo; _regs[VCPU_REGS_RDX] = old_hi; _eflags &= ~EFLG_ZF; @@ -56,9 +56,29 @@ optparser.add_option('--cdrom', optparser.add_option('--loadvm', help = 'select saved vm-image', dest = 'saved_image', - default = '', + default = None, + ) + +optparser.add_option('--monitor', + help = 'redirect monitor (currently only stdio or tcp)', + dest = 'monitor', + default = None, ) +optparser.add_option('--stopped', + help = 'start image in stopped mode', + action = 'store_true', + default = False, + ) + +optparser.add_option('-n', '--dry-run', + help = "just print the qemu command line; don't run it", + action = 'store_true', + dest = 'dry_run', + default = False, + ) + + (options, args) = optparser.parse_args() if len(args) > 0: @@ -160,13 +180,26 @@ if not options.notap: qemu_args += ('-net', 'nic,macaddr=%s' % (mac,), '-net', 'tap,script=/etc/kvm/qemu-ifup',) -if options.vnc is not None: +if options.vnc: qemu_args += ('-vnc', str(options.vnc)) -if options.saved_image != '': +if options.saved_image: qemu_args += ('-loadvm' , options.saved_image, ) -def concat_func(x,y): return x + ' ' + y -print 'Running %s' % reduce(concat_func, qemu_args) +if options.monitor: + if options.monitor == 'stdio': + qemu_args += ('-monitor' , 'stdio', ) + elif options.monitor == 'tcp': + qemu_args += ('-monitor' , 'tcp:0:5555,server,nowait', ) + else: + raise Exception('illegal monitor option %s' % option.monitor) + +if options.stopped: + qemu_args += ('-S',) + +if options.dry_run: + def concat_func(x,y): return x + ' ' + y + print reduce(concat_func, qemu_args) + sys.exit(0) os.execvp(cmd, qemu_args) diff --git a/kvm/kvm.spec b/kvm/kvm.spec index fcb9a42c8..2d6116448 100644 --- a/kvm/kvm.spec +++ b/kvm/kvm.spec @@ -78,6 +78,7 @@ make DESTDIR=%{buildroot} install-rpm depmod %{kverrel} /sbin/chkconfig --level 2345 kvm on /sbin/chkconfig --level 16 kvm off +/usr/sbin/groupadd -fg 444 kvm %postun @@ -91,4 +92,5 @@ depmod %{kverrel} %{confdir}/qemu-ifup %{initdir}/kvm %{utilsdir}/kvm +/etc/udev/rules.d/*kvm*.rules %changelog diff --git a/kvm/scripts/65-kvm.rules b/kvm/scripts/65-kvm.rules new file mode 100644 index 000000000..481cfcf4a --- /dev/null +++ b/kvm/scripts/65-kvm.rules @@ -0,0 +1 @@ +KERNEL=="kvm", NAME="%k", GROUP="kvm", MODE="0660" diff --git a/kvm/user/main.c b/kvm/user/main.c index 3d6d73234..0b43d32dd 100644 --- a/kvm/user/main.c +++ b/kvm/user/main.c @@ -99,6 +99,14 @@ static int test_io_window(void *opaque) return 0; } +static int test_try_push_interrupts(void *opaque) +{ +} + +static void test_post_kvm_run(void *opaque, struct kvm_run *kvm_run) +{ +} + static struct kvm_callbacks test_callbacks = { .cpuid = test_cpuid, .inb = test_inb, @@ -110,6 +118,8 @@ static struct kvm_callbacks test_callbacks = { .debug = test_debug, .halt = test_halt, .io_window = test_io_window, + .try_push_interrupts = test_try_push_interrupts, + .post_kvm_run = test_post_kvm_run, }; @@ -25,6 +25,9 @@ #include "disas.h" #include <dirent.h> #include "migration.h" +#if USE_KVM +#include "qemu-kvm.h" +#endif //#define DEBUG //#define DEBUG_COMPLETION @@ -2231,6 +2234,15 @@ static void monitor_handle_command(const term_cmd_t *cmds, const char *cmdline) goto fail; } +#ifdef USE_KVM + if(1) + { + CPUState *env=mon_get_cpu(); + if (kvm_allowed) + kvm_save_registers(env); + } +#endif + switch(nb_args) { case 0: cmd->handler(); @@ -38,6 +38,7 @@ #include <sys/mman.h> #include <sys/ioctl.h> #include <sys/socket.h> +#include <sys/un.h> #include <netinet/in.h> #include <dirent.h> #include <netdb.h> @@ -2190,6 +2191,9 @@ static void udp_chr_add_read_handler(CharDriverState *chr, } int parse_host_port(struct sockaddr_in *saddr, const char *str); +int parse_host_unix_path(struct sockaddr_un *uaddr, + const char *str, + int sep); int parse_host_src_port(struct sockaddr_in *haddr, struct sockaddr_in *saddr, const char *str); @@ -2430,8 +2434,9 @@ static void tcp_chr_close(CharDriverState *chr) qemu_free(s); } -static CharDriverState *qemu_chr_open_tcp(const char *host_str, - int is_telnet) +static CharDriverState *qemu_chr_open_stream(const char *host_str, + int is_telnet, + int is_unix_domain) { CharDriverState *chr = NULL; TCPCharDriver *s = NULL; @@ -2439,9 +2444,27 @@ static CharDriverState *qemu_chr_open_tcp(const char *host_str, int is_listen = 0; int is_waitconnect = 1; const char *ptr; + struct sockaddr* addr; struct sockaddr_in saddr; + struct sockaddr_un uaddr; + int addrlen; - if (parse_host_port(&saddr, host_str) < 0) +#ifndef _WIN32 + if (is_unix_domain) { + parse_host_unix_path(&uaddr, host_str, ','); + addr = (struct sockaddr *)&uaddr; + addrlen = sizeof(uaddr); + fd = socket(PF_UNIX, SOCK_STREAM, 0); + } else +#endif + { + if (parse_host_port(&saddr, host_str) < 0) + goto fail; + addr = (struct sockaddr *)&saddr; + addrlen = sizeof(saddr); + fd = socket(PF_INET, SOCK_STREAM, 0); + } + if (fd < 0) goto fail; ptr = host_str; @@ -2465,10 +2488,6 @@ static CharDriverState *qemu_chr_open_tcp(const char *host_str, s = qemu_mallocz(sizeof(TCPCharDriver)); if (!s) goto fail; - - fd = socket(PF_INET, SOCK_STREAM, 0); - if (fd < 0) - goto fail; if (!is_waitconnect) socket_set_nonblock(fd); @@ -2480,8 +2499,7 @@ static CharDriverState *qemu_chr_open_tcp(const char *host_str, /* allow fast reuse */ val = 1; setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char *)&val, sizeof(val)); - - ret = bind(fd, (struct sockaddr *)&saddr, sizeof(saddr)); + ret = bind(fd, (struct sockaddr *)addr, addrlen); if (ret < 0) goto fail; ret = listen(fd, 0); @@ -2493,7 +2511,7 @@ static CharDriverState *qemu_chr_open_tcp(const char *host_str, s->do_telnetopt = 1; } else { for(;;) { - ret = connect(fd, (struct sockaddr *)&saddr, sizeof(saddr)); + ret = connect(fd, (struct sockaddr *)addr, addrlen); if (ret < 0) { err = socket_error(); if (err == EINTR || err == EWOULDBLOCK) { @@ -2543,10 +2561,10 @@ CharDriverState *qemu_chr_open(const char *filename) return qemu_chr_open_null(); } else if (strstart(filename, "tcp:", &p)) { - return qemu_chr_open_tcp(p, 0); + return qemu_chr_open_stream(p, 0, 0); } else if (strstart(filename, "telnet:", &p)) { - return qemu_chr_open_tcp(p, 1); + return qemu_chr_open_stream(p, 1, 0); } else if (strstart(filename, "udp:", &p)) { return qemu_chr_open_udp(p); @@ -2560,6 +2578,8 @@ CharDriverState *qemu_chr_open(const char *filename) return qemu_chr_open_pty(); } else if (!strcmp(filename, "stdio")) { return qemu_chr_open_stdio(); + } else if (strstart(filename, "unix:", &p)) { + return qemu_chr_open_stream(p, 0, 1); } else #endif #if defined(__linux__) @@ -2727,6 +2747,20 @@ int parse_host_port(struct sockaddr_in *saddr, const char *str) return 0; } +int parse_host_unix_path(struct sockaddr_un *uaddr, const char *str, int sep) +{ + const char *p; + + p = strchrnul(str, sep); + uaddr->sun_family = AF_UNIX; + memcpy(uaddr->sun_path, str, p - str); + uaddr->sun_path[p-str] = '\0'; + unlink(uaddr->sun_path); + + return 0; +} + + /* find or alloc a new VLAN */ VLANState *qemu_find_vlan(int id) { @@ -5023,6 +5057,7 @@ void vm_start(void) cpu_enable_ticks(); vm_running = 1; vm_state_notify(1); + console_select(0); /* focus on guest's display */ } } @@ -81,6 +81,8 @@ struct VncState VncReadEvent *read_handler; size_t read_handler_expect; + + int size; }; /* TODO @@ -169,7 +171,7 @@ static void vnc_dpy_resize(DisplayState *ds, int w, int h) ds->data = realloc(ds->data, w * h * vs->depth); vs->old_data = realloc(vs->old_data, w * h * vs->depth); - + vs->size = w * h * vs->depth; if (ds->data == NULL || vs->old_data == NULL) { fprintf(stderr, "vnc: memory allocation failed\n"); exit(1); @@ -732,6 +734,15 @@ static void framebuffer_update_request(VncState *vs, int incremental, for (i = 0; i < h; i++) { vnc_set_bits(vs->dirty_row[y_position + i], (vs->ds->width / 16), VNC_DIRTY_WORDS); + if (old_row + vs->ds->width*vs->depth > vs->old_data + vs->size) { + if (0) /* FIXME: */ + printf("vnc: BUG: %s: x=%d y=%d w=%d h=%d i=%d: " + "old_row(%p) + w*d(0x%x) > old_data(%p) + size(0x%x)\n", + __FUNCTION__, x_position, y_position, w, h, i, + old_row,vs->ds->width * vs->depth, + vs->old_data, vs->size); + break; + } memset(old_row, 42, vs->ds->width * vs->depth); old_row += vs->ds->linesize; } @@ -1032,6 +1043,8 @@ void vnc_display_init(DisplayState *ds, int display) vs->csock = -1; vs->depth = 4; + vs->size = 0; + vs->ds = ds; if (!keyboard_layout) |