void
GetGuestState()
{
PHYSICAL_ADDRESS HighestAcceptableAddress;
HighestAcceptableAddress.QuadPart = 0xFFFFFFFF00000000;
g_GuestState.CR0 = __readcr0();
g_GuestState.CR3 = __readcr3();
g_GuestState.CR4 = __readcr4() | CR4_VMXE;
g_GuestState.RFLAGS = __readeflags();
g_GuestState.Cs = __readcs();
g_GuestState.Ds = __readds();
g_GuestState.Es = __reades();
g_GuestState.Ss = __readss();
g_GuestState.Fs = __readfs();
g_GuestState.Gs = __readgs();
g_GuestState.Ldtr = __sldt();
g_GuestState.Tr = __str();
__sgdt(&(g_GuestState.Gdtr));
__sidt(&(g_GuestState.Idtr));
g_GuestState.S_CS = __readmsr(IA32_SYSENTER_CS);
g_GuestState.SEIP = __readmsr(IA64_SYSENTER_EIP);
g_GuestState.SESP = __readmsr(IA32_SYSENTER_ESP);
g_GuestState.VMXON = MmAllocateNonCachedMemory(PAGE_SIZE);
RtlZeroMemory(g_GuestState.VMXON, PAGE_SIZE);
g_GuestState.VMCS = MmAllocateNonCachedMemory(PAGE_SIZE);
RtlZeroMemory(g_GuestState.VMCS, PAGE_SIZE);
g_GuestState.hvStack = // 分配的是非页面内存, 且保证在物理内存中是连续的, MmFreeContiguousMemory
MmAllocateContiguousMemory(PAGE_SIZE * 2, HighestAcceptableAddress);
RtlZeroMemory(g_GuestState.hvStack, PAGE_SIZE * 2);
}
// See: VMM SETUP & TEAR DOWN
_Use_decl_annotations_ static bool VmpEnterVmxMode(
ProcessorData *processor_data) {
// Apply FIXED bits
const Cr0 cr0_fixed0 = {UtilReadMsr(Msr::kIa32VmxCr0Fixed0)};
const Cr0 cr0_fixed1 = {UtilReadMsr(Msr::kIa32VmxCr0Fixed1)};
Cr0 cr0 = {__readcr0()};
cr0.all &= cr0_fixed1.all;
cr0.all |= cr0_fixed0.all;
__writecr0(cr0.all);
const Cr4 cr4_fixed0 = {UtilReadMsr(Msr::kIa32VmxCr4Fixed0)};
const Cr4 cr4_fixed1 = {UtilReadMsr(Msr::kIa32VmxCr4Fixed1)};
Cr4 cr4 = {__readcr4()};
cr4.all &= cr4_fixed1.all;
cr4.all |= cr4_fixed0.all;
__writecr4(cr4.all);
// Write a VMCS revision identifier
const Ia32VmxBasicMsr vmx_basic_msr = {UtilReadMsr64(Msr::kIa32VmxBasic)};
processor_data->vmxon_region->revision_identifier =
vmx_basic_msr.fields.revision_identifier;
auto vmxon_region_pa = UtilPaFromVa(processor_data->vmxon_region);
if (__vmx_on(&vmxon_region_pa)) {
return false;
}
UtilInveptAll();
return true;
}
static inline bool enter_vmx(struct vmcs *vmxon)
{
/* If we're running nested on a hypervisor that does not
* support VT-x, this will cause #GP. */
u64 cr0 = __readcr0();
cr0 &= __readmsr(MSR_IA32_VMX_CR0_FIXED1);
cr0 |= __readmsr(MSR_IA32_VMX_CR0_FIXED0);
__writecr0(cr0);
u64 cr4 = __readcr4();
cr4 &= __readmsr(MSR_IA32_VMX_CR4_FIXED1);
cr4 |= __readmsr(MSR_IA32_VMX_CR4_FIXED0);
__writecr4(cr4);
u64 vmx = __readmsr(MSR_IA32_VMX_BASIC);
vmxon->revision_id = (u32)vmx;
/* Enter VMX root operation */
uintptr_t pa = __pa(vmxon);
if (__vmx_on(&pa))
return false;
/* This is necessary here or just before we exit the VM,
* we do it here as it's easier. */
__invept_all();
return true;
}
_Use_decl_annotations_ EXTERN_C static bool VminitpEnterVmxMode(
PER_PROCESSOR_DATA *ProcessorData) {
// Apply FIXED bits
const CR0_REG cr0Fixed0 = {__readmsr(IA32_VMX_CR0_FIXED0)};
const CR0_REG cr0Fixed1 = {__readmsr(IA32_VMX_CR0_FIXED1)};
CR0_REG cr0 = {__readcr0()};
cr0.All &= cr0Fixed1.All;
cr0.All |= cr0Fixed0.All;
__writecr0(cr0.All);
const CR4_REG cr4Fixed0 = {__readmsr(IA32_VMX_CR4_FIXED0)};
const CR4_REG cr4Fixed1 = {__readmsr(IA32_VMX_CR4_FIXED1)};
CR4_REG cr4 = {__readcr4()};
cr4.All &= cr4Fixed1.All;
cr4.All |= cr4Fixed0.All;
__writecr4(cr4.All);
// Write a VMCS revision identifier
IA32_VMX_BASIC_MSR vmxBasicMsr = {__readmsr(IA32_VMX_BASIC)};
ProcessorData->VmxonRegion->RevisionIdentifier =
vmxBasicMsr.Fields.RevisionIdentifier;
auto vmxonRegionPA = MmGetPhysicalAddress(ProcessorData->VmxonRegion);
if (__vmx_on(
reinterpret_cast<unsigned long long *>(&vmxonRegionPA.QuadPart))) {
return false;
}
return true;
}
// ----------------------------------------------------------------------------
void SetupCR0()
{
dword cr0 = __readcr0();
SetBit(cr0, 31); // Set PG 1 (Enable Paging)
ClearBit(cr0, 30); // Set CD 0 (Enable Cache)
ClearBit(cr0, 29); // Set NW 0 (Enable Writethrough)
SetBit(cr0, 16); // Set WP 1 (Enable Write Protect)
SetBit(cr0, 5); // Set NE 1 (Enable Internal FP Mode)
__writecr0(cr0);
}
// See: VMM SETUP & TEAR DOWN
_Use_decl_annotations_ static bool VmpEnterVmxMode(
ProcessorData *processor_data) {
PAGED_CODE();
// Apply FIXED bits
// See: VMX-FIXED BITS IN CR0
// IA32_VMX_CRx_FIXED0 IA32_VMX_CRx_FIXED1 Meaning
// Values 1 1 bit of CRx is fixed to 1
// Values 0 1 bit of CRx is flexible
// Values 0 0 bit of CRx is fixed to 0
const Cr0 cr0_fixed0 = {UtilReadMsr(Msr::kIa32VmxCr0Fixed0)};
const Cr0 cr0_fixed1 = {UtilReadMsr(Msr::kIa32VmxCr0Fixed1)};
Cr0 cr0 = {__readcr0()};
Cr0 cr0_original = cr0;
cr0.all &= cr0_fixed1.all;
cr0.all |= cr0_fixed0.all;
__writecr0(cr0.all);
HYPERPLATFORM_LOG_DEBUG("IA32_VMX_CR0_FIXED0 = %08x", cr0_fixed0.all);
HYPERPLATFORM_LOG_DEBUG("IA32_VMX_CR0_FIXED1 = %08x", cr0_fixed1.all);
HYPERPLATFORM_LOG_DEBUG("Original CR0 = %08x", cr0_original.all);
HYPERPLATFORM_LOG_DEBUG("Fixed CR0 = %08x", cr0.all);
// See: VMX-FIXED BITS IN CR4
const Cr4 cr4_fixed0 = {UtilReadMsr(Msr::kIa32VmxCr4Fixed0)};
const Cr4 cr4_fixed1 = {UtilReadMsr(Msr::kIa32VmxCr4Fixed1)};
Cr4 cr4 = {__readcr4()};
Cr4 cr4_original = cr4;
cr4.all &= cr4_fixed1.all;
cr4.all |= cr4_fixed0.all;
__writecr4(cr4.all);
HYPERPLATFORM_LOG_DEBUG("IA32_VMX_CR4_FIXED0 = %08x", cr4_fixed0.all);
HYPERPLATFORM_LOG_DEBUG("IA32_VMX_CR4_FIXED1 = %08x", cr4_fixed1.all);
HYPERPLATFORM_LOG_DEBUG("Original CR4 = %08x", cr4_original.all);
HYPERPLATFORM_LOG_DEBUG("Fixed CR4 = %08x", cr4.all);
// Write a VMCS revision identifier
const Ia32VmxBasicMsr vmx_basic_msr = {UtilReadMsr64(Msr::kIa32VmxBasic)};
processor_data->vmxon_region->revision_identifier =
vmx_basic_msr.fields.revision_identifier;
auto vmxon_region_pa = UtilPaFromVa(processor_data->vmxon_region);
if (__vmx_on(&vmxon_region_pa)) {
return false;
}
// See: Guidelines for Use of the INVVPID Instruction, and Guidelines for Use
// of the INVEPT Instruction
UtilInveptGlobal();
UtilInvvpidAllContext();
return true;
}
void save_cpu_state(mon_guest_cpu_startup_state_t *s)
{
ia32_gdtr_t gdtr;
ia32_idtr_t idtr;
ia32_selector_t sel;
ia32_segment_descriptor_t *desc;
s->size_of_this_struct = sizeof(mon_guest_cpu_startup_state_t);
s->version_of_this_struct = MON_GUEST_CPU_STARTUP_STATE_VERSION;
__readgdtr(&gdtr);
__sidt(&idtr);
s->control.gdtr.base = (uint64_t)gdtr.base;
s->control.gdtr.limit = (uint32_t)gdtr.limit;
s->control.idtr.base = (uint64_t)idtr.base;
s->control.idtr.limit = (uint32_t)idtr.limit;
s->control.cr[IA32_CTRL_CR0] = __readcr0();
s->control.cr[IA32_CTRL_CR2] = __readcr2();
s->control.cr[IA32_CTRL_CR3] = __readcr3();
s->control.cr[IA32_CTRL_CR4] = __readcr4();
s->msr.msr_sysenter_cs = (uint32_t)__readmsr(IA32_MSR_SYSENTER_CS);
s->msr.msr_sysenter_eip = __readmsr(IA32_MSR_SYSENTER_EIP);
s->msr.msr_sysenter_esp = __readmsr(IA32_MSR_SYSENTER_ESP);
s->msr.msr_efer = __readmsr(IA32_MSR_EFER);
s->msr.msr_pat = __readmsr(IA32_MSR_PAT);
s->msr.msr_debugctl = __readmsr(IA32_MSR_DEBUGCTL);
s->msr.pending_exceptions = 0;
s->msr.interruptibility_state = 0;
s->msr.activity_state = 0;
s->msr.smbase = 0;
sel.sel16 = __readldtr();
if (sel.bits.index != 0) {
return;
}
s->seg.segment[IA32_SEG_LDTR].attributes = 0x00010000;
s->seg.segment[IA32_SEG_TR].attributes = 0x0000808b;
s->seg.segment[IA32_SEG_TR].limit = 0xffffffff;
save_segment_data((uint16_t)__readcs(), &s->seg.segment[IA32_SEG_CS]);
save_segment_data((uint16_t)__readds(), &s->seg.segment[IA32_SEG_DS]);
save_segment_data((uint16_t)__reades(), &s->seg.segment[IA32_SEG_ES]);
save_segment_data((uint16_t)__readfs(), &s->seg.segment[IA32_SEG_FS]);
save_segment_data((uint16_t)__readgs(), &s->seg.segment[IA32_SEG_GS]);
save_segment_data((uint16_t)__readss(), &s->seg.segment[IA32_SEG_SS]);
return;
}
void WPOFF()
{
#ifndef _WIN64
_asm
{
cli
mov eax,cr0
and eax,not 10000h
mov cr0,eax
};
#else
UINT64 cr0=__readcr0();
cr0 &= 0xfffffffffffeffff;
__writecr0(cr0);
_disable();
#endif //_WIN64
}
VOID WPON()
{
#ifndef _WIN64
_asm
{
mov eax,cr0
or eax,10000h
mov cr0,eax
sti
};
#else
UINT64 cr0=__readcr0();
cr0 &= 0xfffffffffffeffff;
__writecr0(cr0);
_disable();
#endif //_WIN64
}
VOID
NTAPI
HalpBiosCall()
{
/* Must be volatile so it doesn't get optimized away! */
volatile KTRAP_FRAME V86TrapFrame;
ULONG_PTR StackOffset, CodeOffset;
/* Save the context, check for return */
if (_setjmp(HalpSavedContext))
{
/* Returned from v86 */
return;
}
/* Kill alignment faults */
__writecr0(__readcr0() & ~CR0_AM);
/* Set new stack address */
KeGetPcr()->TSS->Esp0 = (ULONG)&V86TrapFrame - 0x20 - sizeof(FX_SAVE_AREA);
/* Compute segmented IP and SP offsets */
StackOffset = (ULONG_PTR)&HalpRealModeEnd - 4 - (ULONG_PTR)HalpRealModeStart;
CodeOffset = (ULONG_PTR)HalpRealModeStart & 0xFFF;
/* Now build the V86 trap frame */
V86TrapFrame.V86Es = 0;
V86TrapFrame.V86Ds = 0;
V86TrapFrame.V86Gs = 0;
V86TrapFrame.V86Fs = 0;
V86TrapFrame.HardwareSegSs = 0x2000;
V86TrapFrame.HardwareEsp = StackOffset + CodeOffset;
V86TrapFrame.EFlags = __readeflags() | EFLAGS_V86_MASK | EFLAGS_IOPL;
V86TrapFrame.SegCs = 0x2000;
V86TrapFrame.Eip = CodeOffset;
/* Exit to V86 mode */
HalpExitToV86((PKTRAP_FRAME)&V86TrapFrame);
}
// See: PREPARATION AND LAUNCHING A VIRTUAL MACHINE
_Use_decl_annotations_ static bool VmpSetupVMCS(
const ProcessorData *processor_data, ULONG_PTR guest_stack_pointer,
ULONG_PTR guest_instruction_pointer, ULONG_PTR vmm_stack_pointer) {
Gdtr gdtr = {};
__sgdt(&gdtr);
Idtr idtr = {};
__sidt(&idtr);
// See: Algorithms for Determining VMX Capabilities
const auto use_true_msrs = Ia32VmxBasicMsr{
UtilReadMsr64(
Msr::kIa32VmxBasic)}.fields.vmx_capability_hint;
VmxVmEntryControls vm_entryctl_requested = {};
vm_entryctl_requested.fields.ia32e_mode_guest = IsX64();
VmxVmEntryControls vm_entryctl = {VmpAdjustControlValue(
(use_true_msrs) ? Msr::kIa32VmxTrueEntryCtls : Msr::kIa32VmxEntryCtls,
vm_entryctl_requested.all)};
VmxVmExitControls vm_exitctl_requested = {};
vm_exitctl_requested.fields.acknowledge_interrupt_on_exit = true;
vm_exitctl_requested.fields.host_address_space_size = IsX64();
VmxVmExitControls vm_exitctl = {VmpAdjustControlValue(
(use_true_msrs) ? Msr::kIa32VmxTrueExitCtls : Msr::kIa32VmxExitCtls,
vm_exitctl_requested.all)};
VmxPinBasedControls vm_pinctl_requested = {};
VmxPinBasedControls vm_pinctl = {
VmpAdjustControlValue((use_true_msrs) ? Msr::kIa32VmxTruePinbasedCtls
: Msr::kIa32VmxPinbasedCtls,
vm_pinctl_requested.all)};
VmxProcessorBasedControls vm_procctl_requested = {};
vm_procctl_requested.fields.invlpg_exiting = false;
vm_procctl_requested.fields.rdtsc_exiting = false;
vm_procctl_requested.fields.cr3_load_exiting = true;
vm_procctl_requested.fields.cr8_load_exiting = false; // NB: very frequent
vm_procctl_requested.fields.mov_dr_exiting = true;
vm_procctl_requested.fields.use_msr_bitmaps = true;
vm_procctl_requested.fields.activate_secondary_control = true;
VmxProcessorBasedControls vm_procctl = {
VmpAdjustControlValue((use_true_msrs) ? Msr::kIa32VmxTrueProcBasedCtls
: Msr::kIa32VmxProcBasedCtls,
vm_procctl_requested.all)};
VmxSecondaryProcessorBasedControls vm_procctl2_requested = {};
vm_procctl2_requested.fields.enable_ept = true;
vm_procctl2_requested.fields.enable_rdtscp = true; // required for Win10
vm_procctl2_requested.fields.descriptor_table_exiting = true;
// required for Win10
vm_procctl2_requested.fields.enable_xsaves_xstors = true;
VmxSecondaryProcessorBasedControls vm_procctl2 = {VmpAdjustControlValue(
Msr::kIa32VmxProcBasedCtls2, vm_procctl2_requested.all)};
// Set up CR0 and CR4 bitmaps
// - Where a bit is masked, the shadow bit appears
// - Where a bit is not masked, the actual bit appears
// VM-exit occurs when a guest modifies any of those fields
Cr0 cr0_mask = {};
Cr4 cr4_mask = {};
// See: PDPTE Registers
// If PAE paging would be in use following an execution of MOV to CR0 or MOV
// to CR4 (see Section 4.1.1) and the instruction is modifying any of CR0.CD,
// CR0.NW, CR0.PG, CR4.PAE, CR4.PGE, CR4.PSE, or CR4.SMEP; then the PDPTEs are
// loaded from the address in CR3.
if (UtilIsX86Pae()) {
cr0_mask.fields.pg = true;
cr0_mask.fields.cd = true;
cr0_mask.fields.nw = true;
cr4_mask.fields.pae = true;
cr4_mask.fields.pge = true;
cr4_mask.fields.pse = true;
cr4_mask.fields.smep = true;
}
const auto exception_bitmap =
// 1 << InterruptionVector::kBreakpointException |
// 1 << InterruptionVector::kGeneralProtectionException |
// 1 << InterruptionVector::kPageFaultException |
0;
// clang-format off
/* 16-Bit Control Field */
/* 16-Bit Guest-State Fields */
auto error = VmxStatus::kOk;
error |= UtilVmWrite(VmcsField::kGuestEsSelector, AsmReadES());
error |= UtilVmWrite(VmcsField::kGuestCsSelector, AsmReadCS());
error |= UtilVmWrite(VmcsField::kGuestSsSelector, AsmReadSS());
error |= UtilVmWrite(VmcsField::kGuestDsSelector, AsmReadDS());
error |= UtilVmWrite(VmcsField::kGuestFsSelector, AsmReadFS());
error |= UtilVmWrite(VmcsField::kGuestGsSelector, AsmReadGS());
error |= UtilVmWrite(VmcsField::kGuestLdtrSelector, AsmReadLDTR());
error |= UtilVmWrite(VmcsField::kGuestTrSelector, AsmReadTR());
/* 16-Bit Host-State Fields */
// RPL and TI have to be 0
error |= UtilVmWrite(VmcsField::kHostEsSelector, AsmReadES() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostCsSelector, AsmReadCS() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostSsSelector, AsmReadSS() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostDsSelector, AsmReadDS() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostFsSelector, AsmReadFS() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostGsSelector, AsmReadGS() & 0xf8);
error |= UtilVmWrite(VmcsField::kHostTrSelector, AsmReadTR() & 0xf8);
/* 64-Bit Control Fields */
error |= UtilVmWrite64(VmcsField::kIoBitmapA, 0);
error |= UtilVmWrite64(VmcsField::kIoBitmapB, 0);
error |= UtilVmWrite64(VmcsField::kMsrBitmap, UtilPaFromVa(processor_data->shared_data->msr_bitmap));
error |= UtilVmWrite64(VmcsField::kEptPointer, EptGetEptPointer(processor_data->ept_data));
/* 64-Bit Guest-State Fields */
error |= UtilVmWrite64(VmcsField::kVmcsLinkPointer, MAXULONG64);
error |= UtilVmWrite64(VmcsField::kGuestIa32Debugctl, UtilReadMsr64(Msr::kIa32Debugctl));
if (UtilIsX86Pae()) {
UtilLoadPdptes(__readcr3());
}
/* 32-Bit Control Fields */
error |= UtilVmWrite(VmcsField::kPinBasedVmExecControl, vm_pinctl.all);
error |= UtilVmWrite(VmcsField::kCpuBasedVmExecControl, vm_procctl.all);
error |= UtilVmWrite(VmcsField::kExceptionBitmap, exception_bitmap);
error |= UtilVmWrite(VmcsField::kPageFaultErrorCodeMask, 0);
error |= UtilVmWrite(VmcsField::kPageFaultErrorCodeMatch, 0);
error |= UtilVmWrite(VmcsField::kCr3TargetCount, 0);
error |= UtilVmWrite(VmcsField::kVmExitControls, vm_exitctl.all);
error |= UtilVmWrite(VmcsField::kVmExitMsrStoreCount, 0);
error |= UtilVmWrite(VmcsField::kVmExitMsrLoadCount, 0);
error |= UtilVmWrite(VmcsField::kVmEntryControls, vm_entryctl.all);
error |= UtilVmWrite(VmcsField::kVmEntryMsrLoadCount, 0);
error |= UtilVmWrite(VmcsField::kVmEntryIntrInfoField, 0);
error |= UtilVmWrite(VmcsField::kSecondaryVmExecControl, vm_procctl2.all);
/* 32-Bit Guest-State Fields */
error |= UtilVmWrite(VmcsField::kGuestEsLimit, GetSegmentLimit(AsmReadES()));
error |= UtilVmWrite(VmcsField::kGuestCsLimit, GetSegmentLimit(AsmReadCS()));
error |= UtilVmWrite(VmcsField::kGuestSsLimit, GetSegmentLimit(AsmReadSS()));
error |= UtilVmWrite(VmcsField::kGuestDsLimit, GetSegmentLimit(AsmReadDS()));
error |= UtilVmWrite(VmcsField::kGuestFsLimit, GetSegmentLimit(AsmReadFS()));
error |= UtilVmWrite(VmcsField::kGuestGsLimit, GetSegmentLimit(AsmReadGS()));
error |= UtilVmWrite(VmcsField::kGuestLdtrLimit, GetSegmentLimit(AsmReadLDTR()));
error |= UtilVmWrite(VmcsField::kGuestTrLimit, GetSegmentLimit(AsmReadTR()));
error |= UtilVmWrite(VmcsField::kGuestGdtrLimit, gdtr.limit);
error |= UtilVmWrite(VmcsField::kGuestIdtrLimit, idtr.limit);
error |= UtilVmWrite(VmcsField::kGuestEsArBytes, VmpGetSegmentAccessRight(AsmReadES()));
error |= UtilVmWrite(VmcsField::kGuestCsArBytes, VmpGetSegmentAccessRight(AsmReadCS()));
error |= UtilVmWrite(VmcsField::kGuestSsArBytes, VmpGetSegmentAccessRight(AsmReadSS()));
error |= UtilVmWrite(VmcsField::kGuestDsArBytes, VmpGetSegmentAccessRight(AsmReadDS()));
error |= UtilVmWrite(VmcsField::kGuestFsArBytes, VmpGetSegmentAccessRight(AsmReadFS()));
error |= UtilVmWrite(VmcsField::kGuestGsArBytes, VmpGetSegmentAccessRight(AsmReadGS()));
error |= UtilVmWrite(VmcsField::kGuestLdtrArBytes, VmpGetSegmentAccessRight(AsmReadLDTR()));
error |= UtilVmWrite(VmcsField::kGuestTrArBytes, VmpGetSegmentAccessRight(AsmReadTR()));
error |= UtilVmWrite(VmcsField::kGuestInterruptibilityInfo, 0);
error |= UtilVmWrite(VmcsField::kGuestActivityState, 0);
error |= UtilVmWrite(VmcsField::kGuestSysenterCs, UtilReadMsr(Msr::kIa32SysenterCs));
/* 32-Bit Host-State Field */
error |= UtilVmWrite(VmcsField::kHostIa32SysenterCs, UtilReadMsr(Msr::kIa32SysenterCs));
/* Natural-Width Control Fields */
error |= UtilVmWrite(VmcsField::kCr0GuestHostMask, cr0_mask.all);
error |= UtilVmWrite(VmcsField::kCr4GuestHostMask, cr4_mask.all);
error |= UtilVmWrite(VmcsField::kCr0ReadShadow, __readcr0());
error |= UtilVmWrite(VmcsField::kCr4ReadShadow, __readcr4());
/* Natural-Width Guest-State Fields */
error |= UtilVmWrite(VmcsField::kGuestCr0, __readcr0());
error |= UtilVmWrite(VmcsField::kGuestCr3, __readcr3());
error |= UtilVmWrite(VmcsField::kGuestCr4, __readcr4());
#if defined(_AMD64_)
error |= UtilVmWrite(VmcsField::kGuestEsBase, 0);
error |= UtilVmWrite(VmcsField::kGuestCsBase, 0);
error |= UtilVmWrite(VmcsField::kGuestSsBase, 0);
error |= UtilVmWrite(VmcsField::kGuestDsBase, 0);
error |= UtilVmWrite(VmcsField::kGuestFsBase, UtilReadMsr(Msr::kIa32FsBase));
error |= UtilVmWrite(VmcsField::kGuestGsBase, UtilReadMsr(Msr::kIa32GsBase));
#else
error |= UtilVmWrite(VmcsField::kGuestEsBase, VmpGetSegmentBase(gdtr.base, AsmReadES()));
error |= UtilVmWrite(VmcsField::kGuestCsBase, VmpGetSegmentBase(gdtr.base, AsmReadCS()));
error |= UtilVmWrite(VmcsField::kGuestSsBase, VmpGetSegmentBase(gdtr.base, AsmReadSS()));
error |= UtilVmWrite(VmcsField::kGuestDsBase, VmpGetSegmentBase(gdtr.base, AsmReadDS()));
error |= UtilVmWrite(VmcsField::kGuestFsBase, VmpGetSegmentBase(gdtr.base, AsmReadFS()));
error |= UtilVmWrite(VmcsField::kGuestGsBase, VmpGetSegmentBase(gdtr.base, AsmReadGS()));
#endif
error |= UtilVmWrite(VmcsField::kGuestLdtrBase, VmpGetSegmentBase(gdtr.base, AsmReadLDTR()));
error |= UtilVmWrite(VmcsField::kGuestTrBase, VmpGetSegmentBase(gdtr.base, AsmReadTR()));
error |= UtilVmWrite(VmcsField::kGuestGdtrBase, gdtr.base);
error |= UtilVmWrite(VmcsField::kGuestIdtrBase, idtr.base);
error |= UtilVmWrite(VmcsField::kGuestDr7, __readdr(7));
error |= UtilVmWrite(VmcsField::kGuestRsp, guest_stack_pointer);
error |= UtilVmWrite(VmcsField::kGuestRip, guest_instruction_pointer);
error |= UtilVmWrite(VmcsField::kGuestRflags, __readeflags());
error |= UtilVmWrite(VmcsField::kGuestSysenterEsp, UtilReadMsr(Msr::kIa32SysenterEsp));
error |= UtilVmWrite(VmcsField::kGuestSysenterEip, UtilReadMsr(Msr::kIa32SysenterEip));
/* Natural-Width Host-State Fields */
error |= UtilVmWrite(VmcsField::kHostCr0, __readcr0());
error |= UtilVmWrite(VmcsField::kHostCr3, __readcr3());
error |= UtilVmWrite(VmcsField::kHostCr4, __readcr4());
#if defined(_AMD64_)
error |= UtilVmWrite(VmcsField::kHostFsBase, UtilReadMsr(Msr::kIa32FsBase));
error |= UtilVmWrite(VmcsField::kHostGsBase, UtilReadMsr(Msr::kIa32GsBase));
#else
error |= UtilVmWrite(VmcsField::kHostFsBase, VmpGetSegmentBase(gdtr.base, AsmReadFS()));
error |= UtilVmWrite(VmcsField::kHostGsBase, VmpGetSegmentBase(gdtr.base, AsmReadGS()));
#endif
error |= UtilVmWrite(VmcsField::kHostTrBase, VmpGetSegmentBase(gdtr.base, AsmReadTR()));
error |= UtilVmWrite(VmcsField::kHostGdtrBase, gdtr.base);
error |= UtilVmWrite(VmcsField::kHostIdtrBase, idtr.base);
error |= UtilVmWrite(VmcsField::kHostIa32SysenterEsp, UtilReadMsr(Msr::kIa32SysenterEsp));
error |= UtilVmWrite(VmcsField::kHostIa32SysenterEip, UtilReadMsr(Msr::kIa32SysenterEip));
error |= UtilVmWrite(VmcsField::kHostRsp, vmm_stack_pointer);
error |= UtilVmWrite(VmcsField::kHostRip, reinterpret_cast<ULONG_PTR>(AsmVmmEntryPoint));
// clang-format on
const auto vmx_status = static_cast<VmxStatus>(error);
return vmx_status == VmxStatus::kOk;
}
DECLSPEC_NORETURN
VOID
FASTCALL
KiTrap07Handler(IN PKTRAP_FRAME TrapFrame)
{
PKTHREAD Thread, NpxThread;
PFX_SAVE_AREA SaveArea, NpxSaveArea;
ULONG Cr0;
/* Save trap frame */
KiEnterTrap(TrapFrame);
/* Try to handle NPX delay load */
while (TRUE)
{
/* Get the current thread */
Thread = KeGetCurrentThread();
/* Get the NPX frame */
SaveArea = KiGetThreadNpxArea(Thread);
/* Check if emulation is enabled */
if (SaveArea->Cr0NpxState & CR0_EM)
{
/* Not implemented */
UNIMPLEMENTED;
while (TRUE);
}
/* Save CR0 and check NPX state */
Cr0 = __readcr0();
if (Thread->NpxState != NPX_STATE_LOADED)
{
/* Update CR0 */
Cr0 &= ~(CR0_MP | CR0_EM | CR0_TS);
__writecr0(Cr0);
/* Get the NPX thread */
NpxThread = KeGetCurrentPrcb()->NpxThread;
if (NpxThread)
{
/* Get the NPX frame */
NpxSaveArea = KiGetThreadNpxArea(NpxThread);
/* Save FPU state */
DPRINT("FIXME: Save FPU state: %p\n", NpxSaveArea);
//Ke386SaveFpuState(NpxSaveArea);
/* Update NPX state */
Thread->NpxState = NPX_STATE_NOT_LOADED;
}
/* Load FPU state */
//Ke386LoadFpuState(SaveArea);
/* Update NPX state */
Thread->NpxState = NPX_STATE_LOADED;
KeGetCurrentPrcb()->NpxThread = Thread;
/* Enable interrupts */
_enable();
/* Check if CR0 needs to be reloaded due to context switch */
if (!SaveArea->Cr0NpxState) KiEoiHelper(TrapFrame);
/* Otherwise, we need to reload CR0, disable interrupts */
_disable();
/* Reload CR0 */
Cr0 = __readcr0();
Cr0 |= SaveArea->Cr0NpxState;
__writecr0(Cr0);
/* Now restore interrupts and check for TS */
_enable();
if (Cr0 & CR0_TS) KiEoiHelper(TrapFrame);
/* We're still here -- clear TS and try again */
__writecr0(__readcr0() &~ CR0_TS);
_disable();
}
else
{
/* This is an actual fault, not a lack of FPU state */
break;
}
}
/* TS should not be set */
if (Cr0 & CR0_TS)
{
/*
* If it's incorrectly set, then maybe the state is actually still valid
* but we could've lock track of that due to a BIOS call.
* Make sure MP is still set, which should verify the theory.
*/
if (Cr0 & CR0_MP)
{
/* Indeed, the state is actually still valid, so clear TS */
__writecr0(__readcr0() &~ CR0_TS);
KiEoiHelper(TrapFrame);
}
/* Otherwise, something strange is going on */
KeBugCheckWithTf(TRAP_CAUSE_UNKNOWN, 2, Cr0, 0, 0, TrapFrame);
}
/* It's not a delayed load, so process this trap as an NPX fault */
KiNpxHandler(TrapFrame, Thread, SaveArea);
}
DECLSPEC_NORETURN
VOID
FASTCALL
KiNpxHandler(IN PKTRAP_FRAME TrapFrame,
IN PKTHREAD Thread,
IN PFX_SAVE_AREA SaveArea)
{
ULONG Cr0, Mask, Error, ErrorOffset, DataOffset;
/* Check for VDM trap */
ASSERT((KiVdmTrap(TrapFrame)) == FALSE);
/* Check for kernel trap */
if (!KiUserTrap(TrapFrame))
{
/* Kernel might've tripped a delayed error */
SaveArea->Cr0NpxState |= CR0_TS;
/* Only valid if it happened during a restore */
//if ((PVOID)TrapFrame->Eip == FrRestore)
{
/* It did, so just skip the instruction */
//TrapFrame->Eip += 3; /* sizeof(FRSTOR) */
//KiEoiHelper(TrapFrame);
}
}
/* User or kernel trap -- get ready to issue an exception */
//if (Thread->NpxState == NPX_STATE_NOT_LOADED)
{
/* Update CR0 */
Cr0 = __readcr0();
Cr0 &= ~(CR0_MP | CR0_EM | CR0_TS);
__writecr0(Cr0);
/* Save FPU state */
Ke386SaveFpuState(SaveArea);
/* Mark CR0 state dirty */
Cr0 |= NPX_STATE_NOT_LOADED;
Cr0 |= SaveArea->Cr0NpxState;
__writecr0(Cr0);
/* Update NPX state */
Thread->NpxState = NPX_STATE_NOT_LOADED;
KeGetCurrentPrcb()->NpxThread = NULL;
}
/* Clear the TS bit and re-enable interrupts */
SaveArea->Cr0NpxState &= ~CR0_TS;
_enable();
/* Check if we should get the FN or FX error */
if (KeI386FxsrPresent)
{
/* Get it from FX */
Mask = SaveArea->U.FxArea.ControlWord;
Error = SaveArea->U.FxArea.StatusWord;
/* Get the FPU exception address too */
ErrorOffset = SaveArea->U.FxArea.ErrorOffset;
DataOffset = SaveArea->U.FxArea.DataOffset;
}
else
{
/* Get it from FN */
Mask = SaveArea->U.FnArea.ControlWord;
Error = SaveArea->U.FnArea.StatusWord;
/* Get the FPU exception address too */
ErrorOffset = SaveArea->U.FnArea.ErrorOffset;
DataOffset = SaveArea->U.FnArea.DataOffset;
}
/* Get legal exceptions that software should handle */
Error &= (FSW_INVALID_OPERATION |
FSW_DENORMAL |
FSW_ZERO_DIVIDE |
FSW_OVERFLOW |
FSW_UNDERFLOW |
FSW_PRECISION);
Error &= ~Mask;
if (Error & FSW_STACK_FAULT)
{
/* Issue stack check fault */
KiDispatchException2Args(STATUS_FLOAT_STACK_CHECK,
ErrorOffset,
0,
DataOffset,
TrapFrame);
}
/* Check for invalid operation */
if (Error & FSW_INVALID_OPERATION)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_INVALID_OPERATION,
ErrorOffset,
0,
TrapFrame);
}
/* Check for divide by zero */
if (Error & FSW_ZERO_DIVIDE)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_DIVIDE_BY_ZERO,
ErrorOffset,
0,
TrapFrame);
}
/* Check for denormal */
if (Error & FSW_DENORMAL)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_INVALID_OPERATION,
ErrorOffset,
0,
TrapFrame);
}
/* Check for overflow */
if (Error & FSW_OVERFLOW)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_OVERFLOW,
ErrorOffset,
0,
TrapFrame);
}
/* Check for underflow */
if (Error & FSW_UNDERFLOW)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_UNDERFLOW,
ErrorOffset,
0,
TrapFrame);
}
/* Check for precision fault */
if (Error & FSW_PRECISION)
{
/* Issue fault */
KiDispatchException1Args(STATUS_FLOAT_INEXACT_RESULT,
ErrorOffset,
0,
TrapFrame);
}
/* Unknown FPU fault */
KeBugCheckWithTf(TRAP_CAUSE_UNKNOWN, 1, Error, 0, 0, TrapFrame);
}
_Use_decl_annotations_ EXTERN_C static bool VminitpSetupVMCS(
const PER_PROCESSOR_DATA *ProcessorData, ULONG_PTR GuestStackPointer,
ULONG_PTR GuestInstructionPointer, ULONG_PTR VmmStackPointer) {
unsigned char error = 0;
GDTR gdtr = {};
__sgdt(&gdtr);
IDTR idtr = {};
__sidt(&idtr);
VMX_VM_ENTER_CONTROLS vmEnterCtlRequested = {};
vmEnterCtlRequested.Fields.IA32eModeGuest = true;
VMX_VM_ENTER_CONTROLS vmEnterCtl = {
VminitpAdjustControlValue(IA32_VMX_ENTRY_CTLS, vmEnterCtlRequested.All)};
VMX_VM_EXIT_CONTROLS vmExitCtlRequested = {};
vmExitCtlRequested.Fields.AcknowledgeInterruptOnExit = true;
vmExitCtlRequested.Fields.HostAddressSpaceSize = true;
VMX_VM_EXIT_CONTROLS vmExitCtl = {
VminitpAdjustControlValue(IA32_VMX_EXIT_CTLS, vmExitCtlRequested.All)};
VMX_PIN_BASED_CONTROLS vmPinCtlRequested = {};
VMX_PIN_BASED_CONTROLS vmPinCtl = {
VminitpAdjustControlValue(IA32_VMX_PINBASED_CTLS, vmPinCtlRequested.All)};
VMX_CPU_BASED_CONTROLS vmCpuCtlRequested = {};
vmCpuCtlRequested.Fields.RDTSCExiting = true;
vmCpuCtlRequested.Fields.CR3LoadExiting = true; // MOV to CR3
vmCpuCtlRequested.Fields.CR8LoadExiting = true; // MOV to CR8
vmCpuCtlRequested.Fields.MovDRExiting = true;
vmCpuCtlRequested.Fields.UseMSRBitmaps = true;
vmCpuCtlRequested.Fields.ActivateSecondaryControl = true;
VMX_CPU_BASED_CONTROLS vmCpuCtl = {VminitpAdjustControlValue(
IA32_VMX_PROCBASED_CTLS, vmCpuCtlRequested.All)};
VMX_SECONDARY_CPU_BASED_CONTROLS vmCpuCtl2Requested = {};
vmCpuCtl2Requested.Fields.EnableRDTSCP = true;
vmCpuCtl2Requested.Fields.DescriptorTableExiting = true;
VMX_CPU_BASED_CONTROLS vmCpuCtl2 = {VminitpAdjustControlValue(
IA32_VMX_PROCBASED_CTLS2, vmCpuCtl2Requested.All)};
// Set up the MSR bitmap
// Activate VM-exit for RDMSR against all MSRs
const auto bitMapReadLow =
reinterpret_cast<UCHAR *>(ProcessorData->MsrBitmap);
const auto bitMapReadHigh = bitMapReadLow + 1024;
RtlFillMemory(bitMapReadLow, 1024, 0xff); // read 0 - 1fff
RtlFillMemory(bitMapReadHigh, 1024, 0xff); // read c0000000 - c0001fff
// But ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8)
RTL_BITMAP bitMapReadLowHeader = {};
RtlInitializeBitMap(&bitMapReadLowHeader,
reinterpret_cast<PULONG>(bitMapReadLow), 1024 * 8);
RtlClearBits(&bitMapReadLowHeader, 0xe7, 2);
// But ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102)
RTL_BITMAP bitMapReadHighHeader = {};
RtlInitializeBitMap(&bitMapReadHighHeader,
reinterpret_cast<PULONG>(bitMapReadHigh), 1024 * 8);
RtlClearBits(&bitMapReadHighHeader, 0x101, 2);
const auto msrBitmapPA = MmGetPhysicalAddress(ProcessorData->MsrBitmap);
// Set up CR0 and CR4 bitmaps
// Where a bit is masked, the shadow bit appears
// Where a bit is not masked, the actual bit appears
CR0_REG cr0mask = {};
cr0mask.Fields.WP = true;
CR4_REG cr4mask = {};
cr4mask.Fields.PGE = true;
// clang-format off
/* 16-Bit Control Field */
/* 16-Bit Guest-State Fields */
error |= __vmx_vmwrite(GUEST_ES_SELECTOR, AsmReadES());
error |= __vmx_vmwrite(GUEST_CS_SELECTOR, AsmReadCS());
error |= __vmx_vmwrite(GUEST_SS_SELECTOR, AsmReadSS());
error |= __vmx_vmwrite(GUEST_DS_SELECTOR, AsmReadDS());
error |= __vmx_vmwrite(GUEST_FS_SELECTOR, AsmReadFS());
error |= __vmx_vmwrite(GUEST_GS_SELECTOR, AsmReadGS());
error |= __vmx_vmwrite(GUEST_LDTR_SELECTOR, AsmReadLDTR());
error |= __vmx_vmwrite(GUEST_TR_SELECTOR, AsmReadTR());
/* 16-Bit Host-State Fields */
error |= __vmx_vmwrite(HOST_ES_SELECTOR, AsmReadES() & 0xf8); // RPL and TI
error |= __vmx_vmwrite(HOST_CS_SELECTOR, AsmReadCS() & 0xf8); // have to be 0
error |= __vmx_vmwrite(HOST_SS_SELECTOR, AsmReadSS() & 0xf8);
error |= __vmx_vmwrite(HOST_DS_SELECTOR, AsmReadDS() & 0xf8);
error |= __vmx_vmwrite(HOST_FS_SELECTOR, AsmReadFS() & 0xf8);
error |= __vmx_vmwrite(HOST_GS_SELECTOR, AsmReadGS() & 0xf8);
error |= __vmx_vmwrite(HOST_TR_SELECTOR, AsmReadTR() & 0xf8);
/* 64-Bit Control Fields */
error |= __vmx_vmwrite(IO_BITMAP_A, 0);
error |= __vmx_vmwrite(IO_BITMAP_B, 0);
error |= __vmx_vmwrite(MSR_BITMAP, msrBitmapPA.QuadPart);
error |= __vmx_vmwrite(TSC_OFFSET, 0);
/* 64-Bit Guest-State Fields */
error |= __vmx_vmwrite(VMCS_LINK_POINTER, 0xffffffffffffffff);
error |= __vmx_vmwrite(GUEST_IA32_DEBUGCTL, __readmsr(IA32_DEBUGCTL));
/* 32-Bit Control Fields */
error |= __vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmPinCtl.All);
error |= __vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmCpuCtl.All);
error |= __vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, vmCpuCtl2.All);
error |= __vmx_vmwrite(EXCEPTION_BITMAP, 0);
error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MASK, 0);
error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MATCH, 0);
error |= __vmx_vmwrite(CR3_TARGET_COUNT, 0);
error |= __vmx_vmwrite(VM_EXIT_CONTROLS, vmExitCtl.All);
error |= __vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0);
error |= __vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0);
error |= __vmx_vmwrite(VM_ENTRY_CONTROLS, vmEnterCtl.All);
error |= __vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0);
error |= __vmx_vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0);
/* 32-Bit Guest-State Fields */
error |= __vmx_vmwrite(GUEST_ES_LIMIT, GetSegmentLimit(AsmReadES()));
error |= __vmx_vmwrite(GUEST_CS_LIMIT, GetSegmentLimit(AsmReadCS()));
error |= __vmx_vmwrite(GUEST_SS_LIMIT, GetSegmentLimit(AsmReadSS()));
error |= __vmx_vmwrite(GUEST_DS_LIMIT, GetSegmentLimit(AsmReadDS()));
error |= __vmx_vmwrite(GUEST_FS_LIMIT, GetSegmentLimit(AsmReadFS()));
error |= __vmx_vmwrite(GUEST_GS_LIMIT, GetSegmentLimit(AsmReadGS()));
error |= __vmx_vmwrite(GUEST_LDTR_LIMIT, GetSegmentLimit(AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_LIMIT, GetSegmentLimit(AsmReadTR()));
error |= __vmx_vmwrite(GUEST_GDTR_LIMIT, gdtr.Limit);
error |= __vmx_vmwrite(GUEST_IDTR_LIMIT, idtr.Limit);
error |= __vmx_vmwrite(GUEST_ES_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadES()));
error |= __vmx_vmwrite(GUEST_CS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadCS()));
error |= __vmx_vmwrite(GUEST_SS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadSS()));
error |= __vmx_vmwrite(GUEST_DS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadDS()));
error |= __vmx_vmwrite(GUEST_FS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadFS()));
error |= __vmx_vmwrite(GUEST_GS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadGS()));
error |= __vmx_vmwrite(GUEST_LDTR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadTR()));
error |= __vmx_vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
error |= __vmx_vmwrite(GUEST_ACTIVITY_STATE, 0);
error |= __vmx_vmwrite(GUEST_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS));
/* 32-Bit Host-State Field */
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS));
/* Natural-Width Control Fields */
error |= __vmx_vmwrite(CR0_GUEST_HOST_MASK, cr0mask.All);
error |= __vmx_vmwrite(CR4_GUEST_HOST_MASK, cr4mask.All);
error |= __vmx_vmwrite(CR0_READ_SHADOW, __readcr0());
error |= __vmx_vmwrite(CR4_READ_SHADOW, __readcr4());
error |= __vmx_vmwrite(CR3_TARGET_VALUE0, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE1, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE2, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE3, 0);
/* Natural-Width Guest-State Fields */
error |= __vmx_vmwrite(GUEST_CR0, __readcr0());
error |= __vmx_vmwrite(GUEST_CR3, __readcr3());
error |= __vmx_vmwrite(GUEST_CR4, __readcr4());
error |= __vmx_vmwrite(GUEST_ES_BASE, 0);
error |= __vmx_vmwrite(GUEST_CS_BASE, 0);
error |= __vmx_vmwrite(GUEST_SS_BASE, 0);
error |= __vmx_vmwrite(GUEST_DS_BASE, 0);
error |= __vmx_vmwrite(GUEST_FS_BASE, __readmsr(IA32_FS_BASE));
error |= __vmx_vmwrite(GUEST_GS_BASE, __readmsr(IA32_GS_BASE));
error |= __vmx_vmwrite(GUEST_LDTR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR()));
error |= __vmx_vmwrite(GUEST_GDTR_BASE, gdtr.Address);
error |= __vmx_vmwrite(GUEST_IDTR_BASE, idtr.Address);
error |= __vmx_vmwrite(GUEST_DR7, __readdr(7));
error |= __vmx_vmwrite(GUEST_RSP, GuestStackPointer);
error |= __vmx_vmwrite(GUEST_RIP, GuestInstructionPointer);
error |= __vmx_vmwrite(GUEST_RFLAGS, __readeflags());
error |= __vmx_vmwrite(GUEST_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP));
error |= __vmx_vmwrite(GUEST_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP));
/* Natural-Width Host-State Fields */
error |= __vmx_vmwrite(HOST_CR0, __readcr0());
error |= __vmx_vmwrite(HOST_CR3, __readcr3());
error |= __vmx_vmwrite(HOST_CR4, __readcr4());
error |= __vmx_vmwrite(HOST_FS_BASE, __readmsr(IA32_FS_BASE));
error |= __vmx_vmwrite(HOST_GS_BASE, __readmsr(IA32_GS_BASE));
error |= __vmx_vmwrite(HOST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR()));
error |= __vmx_vmwrite(HOST_GDTR_BASE, gdtr.Address);
error |= __vmx_vmwrite(HOST_IDTR_BASE, idtr.Address);
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP));
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP));
error |= __vmx_vmwrite(HOST_RSP, VmmStackPointer);
error |= __vmx_vmwrite(HOST_RIP, reinterpret_cast<size_t>(AsmVmmEntryPoint));
// clang-format on
const auto vmxStatus = static_cast<VMX_STATUS>(error);
return vmxStatus == VMX_OK;
}
static bool setup_vmcs(struct vcpu *vcpu, uintptr_t sp, uintptr_t ip, uintptr_t stack_base)
{
struct gdtr gdtr;
__sgdt(&gdtr);
struct gdtr idtr;
__sidt(&idtr);
/* Get this CPU's EPT */
struct ept *ept = &vcpu->ept;
u64 cr0 = __readcr0();
u64 cr4 = __readcr4();
u64 err = 0;
u16 es = __reades();
u16 cs = __readcs();
u16 ss = __readss();
u16 ds = __readds();
u16 fs = __readfs();
u16 gs = __readgs();
u16 ldt = __sldt();
u16 tr = __str();
vcpu->g_idt.base = idtr.base;
vcpu->g_idt.limit = idtr.limit;
struct kidt_entry64 *current = (struct kidt_entry64 *)idtr.base;
struct kidt_entry64 *shadow = (struct kidt_entry64 *)vcpu->idt.base;
unsigned count = idtr.limit / sizeof(*shadow);
for (unsigned n = 0; n < count; ++n)
memcpy(&shadow[n], ¤t[n], sizeof(*shadow));
vcpu_put_idt(vcpu, cs, X86_TRAP_VE, __ept_violation);
u8 msr_off = 0;
if (__readmsr(MSR_IA32_VMX_BASIC) & VMX_BASIC_TRUE_CTLS)
msr_off = 0xC;
u64 vm_entry = VM_ENTRY_IA32E_MODE;// | VM_ENTRY_LOAD_IA32_PAT;
adjust_ctl_val(MSR_IA32_VMX_ENTRY_CTLS + msr_off, &vm_entry);
u64 vm_exit = VM_EXIT_ACK_INTR_ON_EXIT | VM_EXIT_HOST_ADDR_SPACE_SIZE;
adjust_ctl_val(MSR_IA32_VMX_EXIT_CTLS + msr_off, &vm_exit);
u64 vm_pinctl = 0;
adjust_ctl_val(MSR_IA32_VMX_PINBASED_CTLS + msr_off, &vm_pinctl);
u64 vm_cpuctl = CPU_BASED_ACTIVATE_SECONDARY_CONTROLS | CPU_BASED_USE_MSR_BITMAPS |
CPU_BASED_MOV_DR_EXITING | CPU_BASED_USE_TSC_OFFSETING;
adjust_ctl_val(MSR_IA32_VMX_PROCBASED_CTLS + msr_off, &vm_cpuctl);
u64 vm_2ndctl = SECONDARY_EXEC_ENABLE_EPT | SECONDARY_EXEC_TSC_SCALING |
SECONDARY_EXEC_DESC_TABLE_EXITING | SECONDARY_EXEC_XSAVES | SECONDARY_EXEC_RDTSCP |
SECONDARY_EXEC_ENABLE_VMFUNC | SECONDARY_EXEC_ENABLE_VE;
adjust_ctl_val(MSR_IA32_VMX_PROCBASED_CTLS2, &vm_2ndctl);
/* Processor control fields */
err |= __vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, vm_pinctl);
err |= __vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, vm_cpuctl);
err |= __vmx_vmwrite(EXCEPTION_BITMAP, __EXCEPTION_BITMAP);
err |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MASK, 0);
err |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MATCH, 0);
err |= __vmx_vmwrite(CR3_TARGET_COUNT, 0);
err |= __vmx_vmwrite(VM_EXIT_CONTROLS, vm_exit);
err |= __vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0);
err |= __vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0);
err |= __vmx_vmwrite(VM_ENTRY_CONTROLS, vm_entry);
err |= __vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0);
err |= __vmx_vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0);
err |= __vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, vm_2ndctl);
/* Control Fields */
err |= __vmx_vmwrite(IO_BITMAP_A, 0);
err |= __vmx_vmwrite(IO_BITMAP_B, 0);
err |= __vmx_vmwrite(MSR_BITMAP, __pa(ksm.msr_bitmap));
err |= __vmx_vmwrite(EPT_POINTER, EPTP(ept, EPTP_DEFAULT));
err |= __vmx_vmwrite(VM_FUNCTION_CTRL, VM_FUNCTION_CTL_EPTP_SWITCHING);
err |= __vmx_vmwrite(EPTP_INDEX, EPTP_DEFAULT);
err |= __vmx_vmwrite(EPTP_LIST_ADDRESS, __pa(ept->ptr_list));
err |= __vmx_vmwrite(VE_INFO_ADDRESS, __pa(vcpu->ve));
err |= __vmx_vmwrite(CR0_GUEST_HOST_MASK, __CR0_GUEST_HOST_MASK);
err |= __vmx_vmwrite(CR4_GUEST_HOST_MASK, __CR4_GUEST_HOST_MASK);
err |= __vmx_vmwrite(CR0_READ_SHADOW, cr0);
err |= __vmx_vmwrite(CR4_READ_SHADOW, cr4);
err |= __vmx_vmwrite(VMCS_LINK_POINTER, -1ULL);
/* Guest */
err |= __vmx_vmwrite(GUEST_ES_SELECTOR, es);
err |= __vmx_vmwrite(GUEST_CS_SELECTOR, cs);
err |= __vmx_vmwrite(GUEST_SS_SELECTOR, ss);
err |= __vmx_vmwrite(GUEST_DS_SELECTOR, ds);
err |= __vmx_vmwrite(GUEST_FS_SELECTOR, fs);
err |= __vmx_vmwrite(GUEST_GS_SELECTOR, gs);
err |= __vmx_vmwrite(GUEST_LDTR_SELECTOR, ldt);
err |= __vmx_vmwrite(GUEST_TR_SELECTOR, tr);
err |= __vmx_vmwrite(GUEST_ES_LIMIT, __segmentlimit(es));
err |= __vmx_vmwrite(GUEST_CS_LIMIT, __segmentlimit(cs));
err |= __vmx_vmwrite(GUEST_SS_LIMIT, __segmentlimit(ss));
err |= __vmx_vmwrite(GUEST_DS_LIMIT, __segmentlimit(ds));
err |= __vmx_vmwrite(GUEST_FS_LIMIT, __segmentlimit(fs));
err |= __vmx_vmwrite(GUEST_GS_LIMIT, __segmentlimit(gs));
err |= __vmx_vmwrite(GUEST_LDTR_LIMIT, __segmentlimit(ldt));
err |= __vmx_vmwrite(GUEST_TR_LIMIT, __segmentlimit(tr));
err |= __vmx_vmwrite(GUEST_GDTR_LIMIT, gdtr.limit);
err |= __vmx_vmwrite(GUEST_IDTR_LIMIT, idtr.limit);
err |= __vmx_vmwrite(GUEST_ES_AR_BYTES, __accessright(es));
err |= __vmx_vmwrite(GUEST_CS_AR_BYTES, __accessright(cs));
err |= __vmx_vmwrite(GUEST_SS_AR_BYTES, __accessright(ss));
err |= __vmx_vmwrite(GUEST_DS_AR_BYTES, __accessright(ds));
err |= __vmx_vmwrite(GUEST_FS_AR_BYTES, __accessright(fs));
err |= __vmx_vmwrite(GUEST_GS_AR_BYTES, __accessright(gs));
err |= __vmx_vmwrite(GUEST_LDTR_AR_BYTES, __accessright(ldt));
err |= __vmx_vmwrite(GUEST_TR_AR_BYTES, __accessright(tr));
err |= __vmx_vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
err |= __vmx_vmwrite(GUEST_ACTIVITY_STATE, 0);
err |= __vmx_vmwrite(GUEST_IA32_DEBUGCTL, __readmsr(MSR_IA32_DEBUGCTLMSR));
err |= __vmx_vmwrite(GUEST_SYSENTER_CS, __readmsr(MSR_IA32_SYSENTER_CS));
err |= __vmx_vmwrite(GUEST_CR0, cr0);
err |= __vmx_vmwrite(GUEST_CR3, ksm.origin_cr3);
err |= __vmx_vmwrite(GUEST_CR4, cr4);
err |= __vmx_vmwrite(GUEST_ES_BASE, 0);
err |= __vmx_vmwrite(GUEST_CS_BASE, 0);
err |= __vmx_vmwrite(GUEST_SS_BASE, 0);
err |= __vmx_vmwrite(GUEST_DS_BASE, 0);
err |= __vmx_vmwrite(GUEST_FS_BASE, __readmsr(MSR_IA32_FS_BASE));
err |= __vmx_vmwrite(GUEST_GS_BASE, __readmsr(MSR_IA32_GS_BASE));
err |= __vmx_vmwrite(GUEST_LDTR_BASE, __segmentbase(gdtr.base, ldt));
err |= __vmx_vmwrite(GUEST_TR_BASE, __segmentbase(gdtr.base, tr));
err |= __vmx_vmwrite(GUEST_GDTR_BASE, gdtr.base);
err |= __vmx_vmwrite(GUEST_IDTR_BASE, vcpu->idt.base);
err |= __vmx_vmwrite(GUEST_DR7, __readdr(7));
err |= __vmx_vmwrite(GUEST_RSP, sp);
err |= __vmx_vmwrite(GUEST_RIP, ip);
err |= __vmx_vmwrite(GUEST_RFLAGS, __readeflags());
err |= __vmx_vmwrite(GUEST_SYSENTER_ESP, __readmsr(MSR_IA32_SYSENTER_ESP));
err |= __vmx_vmwrite(GUEST_SYSENTER_EIP, __readmsr(MSR_IA32_SYSENTER_EIP));
/* Host */
err |= __vmx_vmwrite(HOST_ES_SELECTOR, es & 0xf8);
err |= __vmx_vmwrite(HOST_CS_SELECTOR, cs & 0xf8);
err |= __vmx_vmwrite(HOST_SS_SELECTOR, ss & 0xf8);
err |= __vmx_vmwrite(HOST_DS_SELECTOR, ds & 0xf8);
err |= __vmx_vmwrite(HOST_FS_SELECTOR, fs & 0xf8);
err |= __vmx_vmwrite(HOST_GS_SELECTOR, gs & 0xf8);
err |= __vmx_vmwrite(HOST_TR_SELECTOR, tr & 0xf8);
err |= __vmx_vmwrite(HOST_CR0, cr0);
err |= __vmx_vmwrite(HOST_CR3, ksm.kernel_cr3);
err |= __vmx_vmwrite(HOST_CR4, cr4);
err |= __vmx_vmwrite(HOST_FS_BASE, __readmsr(MSR_IA32_FS_BASE));
err |= __vmx_vmwrite(HOST_GS_BASE, __readmsr(MSR_IA32_GS_BASE));
err |= __vmx_vmwrite(HOST_TR_BASE, __segmentbase(gdtr.base, tr));
err |= __vmx_vmwrite(HOST_GDTR_BASE, gdtr.base);
err |= __vmx_vmwrite(HOST_IDTR_BASE, idtr.base);
err |= __vmx_vmwrite(HOST_IA32_SYSENTER_CS, __readmsr(MSR_IA32_SYSENTER_CS));
err |= __vmx_vmwrite(HOST_IA32_SYSENTER_ESP, __readmsr(MSR_IA32_SYSENTER_ESP));
err |= __vmx_vmwrite(HOST_IA32_SYSENTER_EIP, __readmsr(MSR_IA32_SYSENTER_EIP));
err |= __vmx_vmwrite(HOST_RSP, stack_base);
err |= __vmx_vmwrite(HOST_RIP, (uintptr_t)__vmx_entrypoint);
return err == 0;
}
NTSTATUS VTxSoftwareStatus()
{
//
// Check the feature control bit MSR
//
IA32_FEATURE_CONTROL_MSR msr;
TO_ULL(msr) = __readmsr(MSR_IA32_FEATURE_CONTROL);
if (msr.Lock == 1)
{
// If the MSR is locked, it can't be modified
// If 'EnableVmxon' is unset, virtualization is not possible
if (msr.EnableVmxon == 0)
{
DbgLog("VMX is disabled in bios: MSR_IA32_FEATURE_CONTROL is 0x%llx\n", msr);
return STATUS_NOT_SUPPORTED;
}
}
else
{
// Force the lock to be on and enable VMXON
msr.Lock = 1;
msr.VmxonInSmx = 1;
msr.EnableVmxon = 1;
__writemsr(MSR_IA32_FEATURE_CONTROL, TO_ULL(msr));
}
//
// Setup CR0 correctly (Protected mode and paging must be enabled)
//
CR0_REG cr0;
TO_ULL(cr0) = __readcr0();
if (cr0.PE == 0 || cr0.PG == 0)
{
DbgLog("Error: Protected mode or paging is not set in CR0\n");
return STATUS_NOT_SUPPORTED;
}
else
{
// Required by first processors that supported VMX
cr0.NE = 1;
}
__writecr0(TO_ULL(cr0));
//
// Virtual Machine eXtensions Enable in CR4
// BIT #13 VMXE
//
__try
{
__writecr4(__readcr4() | (1 << 13));
}
__except (EXCEPTION_EXECUTE_HANDLER)
{
// Possible 'Privileged Instruction Exception' with CR4 bits
return GetExceptionCode();
}
return STATUS_SUCCESS;
}
VOID
NTAPI
KiInitializeCpu(PKIPCR Pcr)
{
ULONG64 Pat;
ULONG FeatureBits;
/* Initialize gs */
KiInitializeSegments();
/* Set GS base */
__writemsr(MSR_GS_BASE, (ULONG64)Pcr);
__writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
/* Detect and set the CPU Type */
KiSetProcessorType();
/* Get the processor features for this CPU */
FeatureBits = KiGetFeatureBits();
/* Check if we support all needed features */
if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
{
/* If not, bugcheck system */
FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
FeatureBits, REQUIRED_FEATURE_BITS);
KeBugCheck(0);
}
/* Set DEP to always on */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
FeatureBits |= KF_NX_ENABLED;
/* Save feature bits */
Pcr->Prcb.FeatureBits = FeatureBits;
/* Enable fx save restore support */
__writecr4(__readcr4() | CR4_FXSR);
/* Enable XMMI exceptions */
__writecr4(__readcr4() | CR4_XMMEXCPT);
/* Enable Write-Protection */
__writecr0(__readcr0() | CR0_WP);
/* Disable fpu monitoring */
__writecr0(__readcr0() & ~CR0_MP);
/* Disable x87 fpu exceptions */
__writecr0(__readcr0() & ~CR0_NE);
/* LDT is unused */
__lldt(0);
/* Set the systemcall entry points */
__writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
__writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
__writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
/* Set the flags to be cleared when doing a syscall */
__writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
/* Enable syscall instruction and no-execute support */
__writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
/* Initialize the PAT */
Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
(PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
__writemsr(MSR_PAT, Pat);
}
