// Perform IO instruction according with parameters
_Use_decl_annotations_ static void VmmpIoWrapper(bool to_memory, bool is_string,
SIZE_T size_of_access,
unsigned short port,
void *address,
unsigned long count) {
NT_ASSERT(size_of_access == 1 || size_of_access == 2 || size_of_access == 4);
// Update CR3 with that of the guest since below code is going to access
// memory.
const auto guest_cr3 = UtilVmRead(VmcsField::kGuestCr3);
const auto vmm_cr3 = __readcr3();
__writecr3(guest_cr3);
// clang-format off
if (to_memory) {
if (is_string) {
// IN
switch (size_of_access) {
case 1: *reinterpret_cast<UCHAR*>(address) = __inbyte(port); break;
case 2: *reinterpret_cast<USHORT*>(address) = __inword(port); break;
case 4: *reinterpret_cast<ULONG*>(address) = __indword(port); break;
}
} else {
// INS
switch (size_of_access) {
case 1: __inbytestring(port, reinterpret_cast<UCHAR*>(address), count); break;
case 2: __inwordstring(port, reinterpret_cast<USHORT*>(address), count); break;
case 4: __indwordstring(port, reinterpret_cast<ULONG*>(address), count); break;
}
}
} else {
if (is_string) {
// OUT
switch (size_of_access) {
case 1: __outbyte(port, *reinterpret_cast<UCHAR*>(address)); break;
case 2: __outword(port, *reinterpret_cast<USHORT*>(address)); break;
case 4: __outdword(port, *reinterpret_cast<ULONG*>(address)); break;
}
} else {
// OUTS
switch (size_of_access) {
case 1: __outbytestring(port, reinterpret_cast<UCHAR*>(address), count); break;
case 2: __outwordstring(port, reinterpret_cast<USHORT*>(address), count); break;
case 4: __outdwordstring(port, reinterpret_cast<ULONG*>(address), count); break;
}
}
}
// clang-format on
__writecr3(vmm_cr3);
}
VOID
NTAPI
KiSwapProcess(IN PKPROCESS NewProcess,
IN PKPROCESS OldProcess)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
#ifdef CONFIG_SMP
LONG SetMember;
/* Update active processor mask */
SetMember = (LONG)Pcr->SetMember;
InterlockedXor((PLONG)&NewProcess->ActiveProcessors, SetMember);
InterlockedXor((PLONG)&OldProcess->ActiveProcessors, SetMember);
#endif
/* Check for new LDT */
if (NewProcess->LdtDescriptor.LimitLow != OldProcess->LdtDescriptor.LimitLow)
{
/* Not handled yet */
UNIMPLEMENTED_DBGBREAK();
return;
}
/* Update CR3 */
__writecr3(NewProcess->DirectoryTableBase[0]);
/* Clear GS */
Ke386SetGs(0);
/* Update IOPM offset */
Pcr->TSS->IoMapBase = NewProcess->IopmOffset;
}
// __asm__ blocks are only checked for inline functions that end up being
// emitted, so call functions with __asm__ blocks to make sure their inline
// assembly parses.
void f() {
__movsb(0, 0, 0);
__movsd(0, 0, 0);
__movsw(0, 0, 0);
__stosd(0, 0, 0);
__stosw(0, 0, 0);
#ifdef _M_X64
__movsq(0, 0, 0);
__stosq(0, 0, 0);
#endif
int info[4];
__cpuid(info, 0);
__cpuidex(info, 0, 0);
_xgetbv(0);
__halt();
__nop();
__readmsr(0);
// FIXME: Call these in 64-bit too once the intrinsics have been fixed to
// work there, PR19301
#ifndef _M_X64
__readcr3();
__writecr3(0);
#endif
#ifdef _M_ARM
__dmb(_ARM_BARRIER_ISHST);
#endif
}
static
KDSTATUS
handle_gdb_read_mem(
_Out_ DBGKD_MANIPULATE_STATE64* State,
_Out_ PSTRING MessageData,
_Out_ PULONG MessageLength,
_Inout_ PKD_CONTEXT KdContext)
{
State->ApiNumber = DbgKdReadVirtualMemoryApi;
State->ReturnStatus = STATUS_SUCCESS; /* ? */
State->Processor = CurrentStateChange.Processor;
State->ProcessorLevel = CurrentStateChange.ProcessorLevel;
if (MessageData)
MessageData->Length = 0;
*MessageLength = 0;
#if !MONOPROCESS
/* Set the TLB according to the process being read. Pid 0 means any process. */
if ((gdb_dbg_pid != 0) && gdb_pid_to_handle(gdb_dbg_pid) != PsGetCurrentProcessId())
{
PEPROCESS AttachedProcess = find_process(gdb_dbg_pid);
if (AttachedProcess == NULL)
{
KDDBGPRINT("The current GDB debug thread is invalid!");
send_gdb_packet("E03");
return gdb_receive_and_interpret_packet(State, MessageData, MessageLength, KdContext);
}
__writecr3(AttachedProcess->Pcb.DirectoryTableBase[0]);
}
#endif
State->u.ReadMemory.TargetBaseAddress = hex_to_address(&gdb_input[1]);
State->u.ReadMemory.TransferCount = hex_to_address(strstr(&gdb_input[1], ",") + 1);
/* KD will reply with KdSendPacket. Catch it */
KdpSendPacketHandler = ReadMemorySendHandler;
return KdPacketReceived;
}
static
void
ReadMemorySendHandler(
_In_ ULONG PacketType,
_In_ PSTRING MessageHeader,
_In_ PSTRING MessageData)
{
DBGKD_MANIPULATE_STATE64* State = (DBGKD_MANIPULATE_STATE64*)MessageHeader->Buffer;
if (PacketType != PACKET_TYPE_KD_STATE_MANIPULATE)
{
// KdAssert
KDDBGPRINT("Wrong packet type (%lu) received after DbgKdReadVirtualMemoryApi request.\n", PacketType);
while (1);
}
if (State->ApiNumber != DbgKdReadVirtualMemoryApi)
{
KDDBGPRINT("Wrong API number (%lu) after DbgKdReadVirtualMemoryApi request.\n", State->ApiNumber);
}
/* Check status */
if (!NT_SUCCESS(State->ReturnStatus))
send_gdb_ntstatus(State->ReturnStatus);
else
send_gdb_memory(MessageData->Buffer, MessageData->Length);
KdpSendPacketHandler = NULL;
KdpManipulateStateHandler = NULL;
#if !MONOPROCESS
/* Reset the TLB */
if ((gdb_dbg_pid != 0) && gdb_pid_to_handle(gdb_dbg_pid) != PsGetCurrentProcessId())
{
__writecr3(PsGetCurrentProcess()->Pcb.DirectoryTableBase[0]);
}
#endif
}
// ----------------------------------------------------------------------------
void Entry()
{
// Move cursor off-screen
_outpw(0x3D4, 0x070E);
_outpw(0x3D4, 0xD00F);
CKernelHeader* KH = (CKernelHeader*)CMemMap::c_KernelImageBase;
dword* BootInfo = (dword*)(*(dword*)(CMemMap::c_KernelImageBase + 0x100));
RawOutString("Checking Kernel...", 0, 0, 0xA);
if (IsKernelOK(*KH, BootInfo))
{
RawOutString("OK", 18, 0, 0xA);
}
else
{
RawOutString("Fail", 18, 0, 0xC);
ErrIf(true);
}
dword BootType = BootInfo[1];
dword DriversCount = BootInfo[2] - 1;
CDriverInfo DriverInfos[12];
for (dword i = 0; i < DriversCount; i++)
{
DriverInfos[i].m_BytesSize = BootInfo[3 + (i + 1) * 3];
DriverInfos[i].m_LoadPage = BootInfo[4 + (i + 1) * 3];
char* Name = PC(BootInfo[5 + (i + 1) * 3]);
for (int j = 0;; j++)
{
DriverInfos[i].m_Name[j] = Name[j];
if (Name[j] == 0)
break;
}
}
InitPIT();
CPhysMemManager PMM;
PMM.AllocBlockAt(KH->GetKernelCodeBase(), KH->m_KernelCodePageCount, false);
PMM.AllocBlockAt(KH->GetKernelRDataBase(), KH->m_KernelRDataPageCount, false);
PMM.AllocBlockAt(KH->GetKernelDataBase(), KH->m_KernelDataPageCount, true);
PMM.AllocBlockAt((byte*)CMemMap::c_VideoRamTextBase, 8, true);
__writecr3(CMemMap::c_PmmPageDirectory);
SetupCR0();
for (dword i = 0; i < DriversCount; i++)
{
PMM.AllocBlockAt(
DriverInfos[i].GetImageBase(),
DriverInfos[i].GetImagePageCount(), false);
}
CGDT GDT(PMM);
GDT.CreateNewDescriptor(0, 0xFFFFF, 0x92, 1);
GDT.CreateNewDescriptor(0, 0xFFFFF, 0x98, 1);
GDT.CreateNewDescriptor(0, 0xFFFFF, 0xF2, 1);
GDT.CreateNewDescriptor(0, 0xFFFFF, 0xF8, 1);
static const dword HeapPageCount = 32;
void* HeapBlock = PMM.AllocBlock(HeapPageCount);
CHeap SysHeap(PB(HeapBlock), HeapPageCount * 4096);
g_SysHeap = &SysHeap;
CTask KernelTask(GDT, true, 0, 0, 0, 0, CMemMap::c_PmmPageDirectory);
KernelTask._setActive();
CIntManager IM(PMM, KernelTask.GetTSS().GetSelector());
CKernel K(KernelTask, PMM, IM, GDT, IM.GetIDT(),
BootType, DriverInfos, DriversCount);
}
// LLDT, LTR, SLDT, and STR
_Use_decl_annotations_ static void VmmpHandleLdtrOrTrAccess(
GuestContext *guest_context) {
HYPERPLATFORM_PERFORMANCE_MEASURE_THIS_SCOPE();
const LdtrOrTrAccessQualification exit_qualification = {
static_cast<ULONG32>(UtilVmRead(VmcsField::kVmxInstructionInfo))};
// Calculate an address or a register to be used for the instruction
const auto displacement = UtilVmRead(VmcsField::kExitQualification);
ULONG_PTR operation_address = 0;
if (exit_qualification.fields.register_access) {
// Register
const auto register_used =
VmmpSelectRegister(exit_qualification.fields.register1, guest_context);
operation_address = reinterpret_cast<ULONG_PTR>(register_used);
} else {
// Base
ULONG_PTR base_value = 0;
if (!exit_qualification.fields.base_register_invalid) {
const auto register_used = VmmpSelectRegister(
exit_qualification.fields.base_register, guest_context);
base_value = *register_used;
}
// Index
ULONG_PTR index_value = 0;
if (!exit_qualification.fields.index_register_invalid) {
const auto register_used = VmmpSelectRegister(
exit_qualification.fields.index_register, guest_context);
index_value = *register_used;
switch (
static_cast<GdtrOrIdtrScaling>(exit_qualification.fields.scalling)) {
case GdtrOrIdtrScaling::kNoScaling:
index_value = index_value;
break;
case GdtrOrIdtrScaling::kScaleBy2:
index_value = index_value * 2;
break;
case GdtrOrIdtrScaling::kScaleBy4:
index_value = index_value * 4;
break;
case GdtrOrIdtrScaling::kScaleBy8:
index_value = index_value * 8;
break;
default:
break;
}
}
operation_address = base_value + index_value + displacement;
if (static_cast<GdtrOrIdtrAaddressSize>(
exit_qualification.fields.address_size) ==
GdtrOrIdtrAaddressSize::k32bit) {
operation_address &= MAXULONG;
}
}
// Update CR3 with that of the guest since below code is going to access
// memory.
const auto guest_cr3 = UtilVmRead(VmcsField::kGuestCr3);
const auto vmm_cr3 = __readcr3();
__writecr3(guest_cr3);
// Emulate the instruction
auto selector = reinterpret_cast<USHORT *>(operation_address);
switch (static_cast<LdtrOrTrInstructionIdentity>(
exit_qualification.fields.instruction_identity)) {
case LdtrOrTrInstructionIdentity::kSldt:
*selector =
static_cast<USHORT>(UtilVmRead(VmcsField::kGuestLdtrSelector));
break;
case LdtrOrTrInstructionIdentity::kStr:
*selector = static_cast<USHORT>(UtilVmRead(VmcsField::kGuestTrSelector));
break;
case LdtrOrTrInstructionIdentity::kLldt:
UtilVmWrite(VmcsField::kGuestLdtrSelector, *selector);
break;
case LdtrOrTrInstructionIdentity::kLtr:
UtilVmWrite(VmcsField::kGuestTrSelector, *selector);
break;
}
__writecr3(vmm_cr3);
VmmpAdjustGuestInstructionPointer(guest_context->ip);
}
// LIDT, SIDT, LGDT and SGDT
_Use_decl_annotations_ static void VmmpHandleGdtrOrIdtrAccess(
GuestContext *guest_context) {
HYPERPLATFORM_PERFORMANCE_MEASURE_THIS_SCOPE();
const GdtrOrIdtrAccessQualification exit_qualification = {
static_cast<ULONG32>(UtilVmRead(VmcsField::kVmxInstructionInfo))};
// Calculate an address to be used for the instruction
const auto displacement = UtilVmRead(VmcsField::kExitQualification);
// Base
ULONG_PTR base_value = 0;
if (!exit_qualification.fields.base_register_invalid) {
const auto register_used = VmmpSelectRegister(
exit_qualification.fields.base_register, guest_context);
base_value = *register_used;
}
// Index
ULONG_PTR index_value = 0;
if (!exit_qualification.fields.index_register_invalid) {
const auto register_used = VmmpSelectRegister(
exit_qualification.fields.index_register, guest_context);
index_value = *register_used;
switch (
static_cast<GdtrOrIdtrScaling>(exit_qualification.fields.scalling)) {
case GdtrOrIdtrScaling::kNoScaling:
index_value = index_value;
break;
case GdtrOrIdtrScaling::kScaleBy2:
index_value = index_value * 2;
break;
case GdtrOrIdtrScaling::kScaleBy4:
index_value = index_value * 4;
break;
case GdtrOrIdtrScaling::kScaleBy8:
index_value = index_value * 8;
break;
default:
break;
}
}
auto operation_address = base_value + index_value + displacement;
if (static_cast<GdtrOrIdtrAaddressSize>(
exit_qualification.fields.address_size) ==
GdtrOrIdtrAaddressSize::k32bit) {
operation_address &= MAXULONG;
}
// Update CR3 with that of the guest since below code is going to access
// memory.
const auto guest_cr3 = UtilVmRead(VmcsField::kGuestCr3);
const auto vmm_cr3 = __readcr3();
__writecr3(guest_cr3);
// Emulate the instruction
auto descriptor_table_reg = reinterpret_cast<Idtr *>(operation_address);
switch (static_cast<GdtrOrIdtrInstructionIdentity>(
exit_qualification.fields.instruction_identity)) {
case GdtrOrIdtrInstructionIdentity::kSgdt:
descriptor_table_reg->base = UtilVmRead(VmcsField::kGuestGdtrBase);
descriptor_table_reg->limit =
static_cast<unsigned short>(UtilVmRead(VmcsField::kGuestGdtrLimit));
break;
case GdtrOrIdtrInstructionIdentity::kSidt:
descriptor_table_reg->base = UtilVmRead(VmcsField::kGuestIdtrBase);
descriptor_table_reg->limit =
static_cast<unsigned short>(UtilVmRead(VmcsField::kGuestIdtrLimit));
break;
case GdtrOrIdtrInstructionIdentity::kLgdt:
UtilVmWrite(VmcsField::kGuestGdtrBase, descriptor_table_reg->base);
UtilVmWrite(VmcsField::kGuestGdtrLimit, descriptor_table_reg->limit);
break;
case GdtrOrIdtrInstructionIdentity::kLidt:
UtilVmWrite(VmcsField::kGuestIdtrBase, descriptor_table_reg->base);
UtilVmWrite(VmcsField::kGuestIdtrLimit, descriptor_table_reg->limit);
break;
}
__writecr3(vmm_cr3);
VmmpAdjustGuestInstructionPointer(guest_context->ip);
}
// Handles #BP. Determinas if the #BP is caused by a shadow breakpoint, and if
// so, runs its handler, switchs a page view to read/write shadow page and sets
// the monitor trap flag to execute only one instruction where is located on the
// read/write shadow page. Then saves the breakpoint info as the last event.
_Use_decl_annotations_ bool SbpHandleBreakpoint(EptData* ept_data,
void* guest_ip,
GpRegisters* gp_regs) {
if (!SbppIsSbpActive()) {
return false;
}
const auto info = SbppFindPatchInfoByAddress(guest_ip);
if (!info) {
return false;
}
if (!SbppIsShadowBreakpoint(*info)) {
return false;
}
// DdiMon is unable to handle it
if (KeGetCurrentIrql() > DISPATCH_LEVEL) {
HYPERPLATFORM_COMMON_BUG_CHECK(HyperPlatformBugCheck::kUnspecified, 0, 0,
0);
}
// VMM has to change the current CR3 to a guest's CR3 in order to access
// memory address because VMM runs with System's CR3 saved in and restored
// from
// VmcsField::kHostCr3, while a guest's CR3 is depends on thread contexts.
// Without using guest's CR3, it is likely that any use-address space is
// inaccessible from a VMM ending up with a bug check.
const auto guest_cr3 = UtilVmRead(VmcsField::kGuestCr3);
const auto vmm_cr3 = __readcr3();
if (info->type == BreakpointType::kPre) {
// Pre breakpoint
__writecr3(guest_cr3);
info->handler(*info, ept_data, gp_regs, UtilVmRead(VmcsField::kGuestRsp));
__writecr3(vmm_cr3);
SbppEnablePageShadowingForRW(*info, ept_data);
SbppSetMonitorTrapFlag(true);
SbppSaveLastPatchInfo(*info);
} else {
// Post breakpoint
if (info->target_tid == PsGetCurrentThreadId()) {
// It is a target thread. Execute the post handler and let it continue
// subsequence instructions.
__writecr3(guest_cr3);
info->handler(*info, ept_data, gp_regs, UtilVmRead(VmcsField::kGuestRsp));
__writecr3(vmm_cr3);
SbppDisablePageShadowing(*info, ept_data);
SbppDeleteBreakpointFromList(*info);
} else {
// It is not. Let it allow to run one instruction without breakpoint
SbppEnablePageShadowingForRW(*info, ept_data);
SbppSetMonitorTrapFlag(true);
SbppSaveLastPatchInfo(*info);
}
}
// Yes, it was caused by shadow breakpoint. Do not deliver the #BP to a guest.
return true;
}
DECLSPEC_NORETURN
EXTERN_C
VOID
ShvVmxEntryHandler (
_In_ PCONTEXT Context
)
{
SHV_VP_STATE guestContext;
PSHV_VP_DATA vpData;
//
// Because we run with interrupts disabled during the entire hypervisor's
// exit handling, raise the IRQL to HIGH_LEVEL which matches the reality of
// the situation. This will block IPIs and the clock interrupt timer, which
// means that it's critical to spend as little time here as possible. You
// can expect CLOCK_WATCHDOG_TIMEOUT bugchecks to happen otherwise. If you
// chose to enable interrupts note that this will result in further crashes
// as we are not on a correct OS stack, and you will be hitting crashes if
// RtlpCheckStackLimits is ever called, or if PatchGuard validates the RSP
// value.
//
KeRaiseIrql(HIGH_LEVEL, &guestContext.GuestIrql);
//
// Because we had to use RCX when calling RtlCaptureContext, its true value
// was actually pushed on the stack right before the call. Go dig into the
// stack to find it, and overwrite the bogus value that's there now.
//
Context->Rcx = *(PULONG64)((ULONG_PTR)Context - sizeof(Context->Rcx));
//
// Get the per-VP data for this processor.
//
vpData = &ShvGlobalData->VpData[KeGetCurrentProcessorNumberEx(NULL)];
//
// Build a little stack context to make it easier to keep track of certain
// guest state, such as the RIP/RSP/RFLAGS, and the exit reason. The rest
// of the general purpose registers come from the context structure that we
// captured on our own with RtlCaptureContext in the assembly entrypoint.
//
guestContext.GuestEFlags = ShvVmxRead(GUEST_RFLAGS);
guestContext.GuestRip = ShvVmxRead(GUEST_RIP);
guestContext.GuestRsp = ShvVmxRead(GUEST_RSP);
guestContext.ExitReason = ShvVmxRead(VM_EXIT_REASON) & 0xFFFF;
guestContext.VpRegs = Context;
guestContext.ExitVm = FALSE;
//
// Call the generic handler
//
ShvVmxHandleExit(&guestContext);
//
// Did we hit the magic exit sequence, or should we resume back to the VM
// context?
//
if (guestContext.ExitVm)
{
//
// When running in VMX root mode, the processor will set limits of the
// GDT and IDT to 0xFFFF (notice that there are no Host VMCS fields to
// set these values). This causes problems with PatchGuard, which will
// believe that the GDTR and IDTR have been modified by malware, and
// eventually crash the system. Since we know what the original state
// of the GDTR and IDTR was, simply restore it now.
//
__lgdt(&vpData->HostState.SpecialRegisters.Gdtr.Limit);
__lidt(&vpData->HostState.SpecialRegisters.Idtr.Limit);
//
// Our DPC routine may have interrupted an arbitrary user process, and
// not an idle or system thread as usually happens on an idle system.
// Therefore if we return back to the original caller after turning off
// VMX, it will keep our current "host" CR3 value which we set on entry
// to the PML4 of the SYSTEM process. We want to return back with the
// correct value of the "guest" CR3, so that the currently executing
// process continues to run with its expected address space mappings.
//
__writecr3(ShvVmxRead(GUEST_CR3));
//
// Finally, set the stack and instruction pointer to whatever location
// had the instruction causing our VM-Exit, such as ShvVpUninitialize.
// This will effectively act as a longjmp back to that location.
//
Context->Rsp = guestContext.GuestRsp;
Context->Rip = (ULONG64)guestContext.GuestRip;
//
// Turn off VMX root mode on this logical processor. We're done here.
//
__vmx_off();
}
else
{
//
// Because we won't be returning back into assembly code, nothing will
// ever know about the "pop rcx" that must technically be done (or more
// accurately "add rsp, 4" as rcx will already be correct thanks to the
// fixup earlier. In order to keep the stack sane, do that adjustment
// here.
//
Context->Rsp += sizeof(Context->Rcx);
//
// Return into a VMXRESUME intrinsic, which we broke out as its own
// function, in order to allow this to work. No assembly code will be
// needed as RtlRestoreContext will fix all the GPRs, and what we just
// did to RSP will take care of the rest.
//
Context->Rip = (ULONG64)ShvVmxResume;
}
//
// Restore the IRQL back to the original level
//
KeLowerIrql(guestContext.GuestIrql);
//
// Restore the context to either ShvVmxResume, in which case the CPU's VMX
// facility will do the "true" return back to the VM (but without restoring
// GPRs, which is why we must do it here), or to the original guest's RIP,
// which we use in case an exit was requested. In this case VMX must now be
// off, and this will look like a longjmp to the original stack and RIP.
//
RtlRestoreContext(Context, NULL);
}
