// Initialize shared processor data
_Use_decl_annotations_ static SharedProcessorData *VmpInitializeSharedData() {
PAGED_CODE();
const auto shared_data = reinterpret_cast<SharedProcessorData *>(
ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(SharedProcessorData),
kHyperPlatformCommonPoolTag));
if (!shared_data) {
return nullptr;
}
RtlZeroMemory(shared_data, sizeof(SharedProcessorData));
HYPERPLATFORM_LOG_DEBUG("SharedData= %p", shared_data);
// Set up the MSR bitmap
const auto msr_bitmap = ExAllocatePoolWithTag(NonPagedPoolNx, PAGE_SIZE,
kHyperPlatformCommonPoolTag);
if (!msr_bitmap) {
ExFreePoolWithTag(shared_data, kHyperPlatformCommonPoolTag);
return nullptr;
}
RtlZeroMemory(msr_bitmap, PAGE_SIZE);
shared_data->msr_bitmap = msr_bitmap;
// Checks MSRs causing #GP and should not cause VM-exit from 0 to 0xfff.
bool unsafe_msr_map[0x1000] = {};
for (auto msr = 0ul; msr < RTL_NUMBER_OF(unsafe_msr_map); ++msr) {
__try {
UtilReadMsr(static_cast<Msr>(msr));
} __except (EXCEPTION_EXECUTE_HANDLER) {
unsafe_msr_map[msr] = true;
}
}
// Activate VM-exit for RDMSR against all MSRs
const auto bitmap_read_low = reinterpret_cast<UCHAR *>(msr_bitmap);
const auto bitmap_read_high = bitmap_read_low + 1024;
RtlFillMemory(bitmap_read_low, 1024, 0xff); // read 0 - 1fff
RtlFillMemory(bitmap_read_high, 1024, 0xff); // read c0000000 - c0001fff
// But ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8)
RTL_BITMAP bitmap_read_low_header = {};
RtlInitializeBitMap(&bitmap_read_low_header,
reinterpret_cast<PULONG>(bitmap_read_low), 1024 * 8);
RtlClearBits(&bitmap_read_low_header, 0xe7, 2);
// Also ignore the unsage MSRs
for (auto msr = 0ul; msr < RTL_NUMBER_OF(unsafe_msr_map); ++msr) {
const auto ignore = unsafe_msr_map[msr];
if (ignore) {
RtlClearBits(&bitmap_read_low_header, msr, 1);
}
}
// But ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102)
RTL_BITMAP bitmap_read_high_header = {};
RtlInitializeBitMap(&bitmap_read_high_header,
reinterpret_cast<PULONG>(bitmap_read_high), 1024 * 8);
RtlClearBits(&bitmap_read_high_header, 0x101, 2);
return shared_data;
}
static NTSTATUS init_msr_bitmap(struct ksm *k)
{
void *msr_bitmap = ExAllocatePool(NonPagedPoolNx, PAGE_SIZE);
if (!msr_bitmap)
return STATUS_NO_MEMORY;
k->msr_bitmap = msr_bitmap;
RtlZeroMemory(msr_bitmap, PAGE_SIZE);
/* For all MSRs... */
u8 *bitmap_read_lo = (u8 *)msr_bitmap;
u8 *bitmap_read_hi = bitmap_read_lo + 1024;
memset(bitmap_read_lo, 0xff, 1024); // 0 -> 1fff
memset(bitmap_read_hi, 0xff, 1024); // c0000000 - c0001fff
/* ... ignore MSR_IA32_MPERF and MSR_IA32_APERF */
RTL_BITMAP bitmap_read_lo_hdr;
RtlInitializeBitMap(&bitmap_read_lo_hdr, (PULONG)bitmap_read_lo, 1024 * CHAR_BIT);
RtlClearBits(&bitmap_read_lo_hdr, MSR_IA32_MPERF, 2);
for (u32 msr = 0; msr < PAGE_SIZE; ++msr) {
__try {
__readmsr(msr);
} __except (EXCEPTION_EXECUTE_HANDLER)
{
RtlClearBits(&bitmap_read_lo_hdr, msr, 1);
}
}
/* ... and ignore MSR_IA32_GS_BASE and MSR_IA32_KERNEL_GS_BASE */
RTL_BITMAP bitmap_read_hi_hdr;
RtlInitializeBitMap(&bitmap_read_hi_hdr, (PULONG)bitmap_read_hi, 1024 * CHAR_BIT);
RtlClearBits(&bitmap_read_hi_hdr, 0x101, 2);
return STATUS_SUCCESS;
}
void
Test_RtlClearBits(void)
{
RTL_BITMAP BitMapHeader;
ULONG *Buffer;
ULONG BufferSize = 2 * sizeof(*Buffer);
Buffer = AllocateGuarded(BufferSize);
RtlInitializeBitMap(&BitMapHeader, Buffer, 19);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 0, 0);
ok_hex(Buffer[0], 0xffffffff);
ok_hex(Buffer[1], 0xffffffff);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 0, 1);
ok_hex(Buffer[0], 0xfffffffe);
ok_hex(Buffer[1], 0xffffffff);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 21, 1);
ok_hex(Buffer[0], 0xffdfffff);
ok_hex(Buffer[1], 0xffffffff);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 7, 9);
ok_hex(Buffer[0], 0xffff007f);
ok_hex(Buffer[1], 0xffffffff);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 13, 22);
ok_hex(Buffer[0], 0x00001fff);
ok_hex(Buffer[1], 0xfffffff8);
memset(Buffer, 0xff, BufferSize);
RtlClearBits(&BitMapHeader, 63, 1);
ok_hex(Buffer[0], 0xffffffff);
ok_hex(Buffer[1], 0x7fffffff);
memset(Buffer, 0xcc, BufferSize);
RtlClearBits(&BitMapHeader, 3, 6);
RtlClearBits(&BitMapHeader, 11, 5);
RtlClearBits(&BitMapHeader, 21, 7);
RtlClearBits(&BitMapHeader, 37, 4);
ok_hex(Buffer[0], 0xc00c0404);
ok_hex(Buffer[1], 0xcccccc0c);
FreeGuarded(Buffer);
}
/*
* @implemented
*/
BOOL
WINAPI
TlsFree(DWORD Index)
{
BOOL BitSet;
if (Index >= TLS_EXPANSION_SLOTS + TLS_MINIMUM_AVAILABLE)
{
SetLastErrorByStatus(STATUS_INVALID_PARAMETER);
return FALSE;
}
RtlAcquirePebLock();
if (Index >= TLS_MINIMUM_AVAILABLE)
{
BitSet = RtlAreBitsSet(NtCurrentPeb()->TlsExpansionBitmap,
Index - TLS_MINIMUM_AVAILABLE,
1);
if (BitSet)
RtlClearBits(NtCurrentPeb()->TlsExpansionBitmap,
Index - TLS_MINIMUM_AVAILABLE,
1);
}
else
{
BitSet = RtlAreBitsSet(NtCurrentPeb()->TlsBitmap, Index, 1);
if (BitSet)
RtlClearBits(NtCurrentPeb()->TlsBitmap, Index, 1);
}
if (BitSet)
{
/* Clear the TLS cells (slots) in all threads of the current process. */
NtSetInformationThread(NtCurrentThread(),
ThreadZeroTlsCell,
&Index,
sizeof(DWORD));
}
else
{
SetLastError(ERROR_INVALID_PARAMETER);
}
RtlReleasePebLock();
return BitSet;
}
static UCHAR EmsFree(USHORT Handle)
{
PLIST_ENTRY Entry;
PEMS_HANDLE HandleEntry = GetHandleRecord(Handle);
if (!ValidateHandle(HandleEntry))
return EMS_STATUS_INVALID_HANDLE;
for (Entry = HandleEntry->PageList.Flink;
Entry != &HandleEntry->PageList;
Entry = Entry->Flink)
{
PEMS_PAGE PageEntry = (PEMS_PAGE)CONTAINING_RECORD(Entry, EMS_PAGE, Entry);
ULONG PageNumber = ARRAY_INDEX(PageEntry, PageTable);
/* Free the page */
RtlClearBits(&AllocBitmap, PageNumber, 1);
}
InitializeListHead(&HandleEntry->PageList);
if (Handle != EMS_SYSTEM_HANDLE)
FreeHandle(HandleEntry);
return EMS_STATUS_SUCCESS;
}
/******************************************************************
* macho_fill_sect_is_code
*
* Callback for macho_enum_load_commands. Determines which segments
* of a Mach-O file contain code. All commands are expected to be
* of LC_SEGMENT type.
*/
static int macho_fill_sect_is_code(struct macho_file_map* fmap,
const struct load_command* lc, void* user)
{
const struct segment_command* sc = (const struct segment_command*)lc;
const struct section* sections;
int* cursect = user;
int i;
TRACE("(%p/%d, %p, %p/%d) scanning %u sections\n", fmap, fmap->fd, lc,
cursect, *cursect, sc->nsects);
sections = (const struct section*)(sc + 1);
for (i = 0; i < sc->nsects; i++)
{
if (*cursect > MAX_SECT) return -1;
(*cursect)++;
if (!(sections[i].flags & SECTION_TYPE) &&
(sections[i].flags & (S_ATTR_PURE_INSTRUCTIONS|S_ATTR_SOME_INSTRUCTIONS)))
RtlSetBits(&fmap->sect_is_code, *cursect, 1);
else
RtlClearBits(&fmap->sect_is_code, *cursect, 1);
TRACE("Section %d (%d of this segment) is%s code\n", *cursect, i,
(RtlAreBitsSet(&fmap->sect_is_code, *cursect, 1) ? "" : " not"));
}
return 0;
}
VOID DeallocatePort( PPORT_SET PortSet, ULONG Port ) {
KIRQL OldIrql;
Port = htons(Port);
ASSERT(Port >= PortSet->StartingPort);
ASSERT(Port < PortSet->StartingPort + PortSet->PortsToOversee);
KeAcquireSpinLock( &PortSet->Lock, &OldIrql );
RtlClearBits( &PortSet->ProtoBitmap, Port - PortSet->StartingPort, 1 );
KeReleaseSpinLock( &PortSet->Lock, OldIrql );
}
// Build MSR bitmap
_Use_decl_annotations_ static void *VmpBuildMsrBitmap() {
PAGED_CODE();
const auto msr_bitmap = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE,
kHyperPlatformCommonPoolTag);
if (!msr_bitmap) {
return nullptr;
}
RtlZeroMemory(msr_bitmap, PAGE_SIZE);
// Activate VM-exit for RDMSR against all MSRs
const auto bitmap_read_low = reinterpret_cast<UCHAR *>(msr_bitmap);
const auto bitmap_read_high = bitmap_read_low + 1024;
RtlFillMemory(bitmap_read_low, 1024, 0xff); // read 0 - 1fff
RtlFillMemory(bitmap_read_high, 1024, 0xff); // read c0000000 - c0001fff
// Ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8)
RTL_BITMAP bitmap_read_low_header = {};
RtlInitializeBitMap(&bitmap_read_low_header,
reinterpret_cast<PULONG>(bitmap_read_low), 1024 * 8);
RtlClearBits(&bitmap_read_low_header, 0xe7, 2);
// Checks MSRs that cause #GP from 0 to 0xfff, and ignore all of them
for (auto msr = 0ul; msr < 0x1000; ++msr) {
__try {
UtilReadMsr(static_cast<Msr>(msr));
} __except (EXCEPTION_EXECUTE_HANDLER) {
RtlClearBits(&bitmap_read_low_header, msr, 1);
}
}
// Ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102)
RTL_BITMAP bitmap_read_high_header = {};
RtlInitializeBitMap(&bitmap_read_high_header,
reinterpret_cast<PULONG>(bitmap_read_high),
1024 * CHAR_BIT);
RtlClearBits(&bitmap_read_high_header, 0x101, 2);
return msr_bitmap;
}
/*
* @implemented
*/
DWORD
WINAPI
TlsAlloc(VOID)
{
ULONG Index;
RtlAcquirePebLock();
/* Try to get regular TEB slot. */
Index = RtlFindClearBitsAndSet(NtCurrentPeb()->TlsBitmap, 1, 0);
if (Index == ~0U)
{
/* If it fails, try to find expansion TEB slot. */
Index = RtlFindClearBitsAndSet(NtCurrentPeb()->TlsExpansionBitmap, 1, 0);
if (Index != ~0U)
{
if (NtCurrentTeb()->TlsExpansionSlots == NULL)
{
NtCurrentTeb()->TlsExpansionSlots = HeapAlloc(RtlGetProcessHeap(),
HEAP_ZERO_MEMORY,
TLS_EXPANSION_SLOTS *
sizeof(PVOID));
}
if (NtCurrentTeb()->TlsExpansionSlots == NULL)
{
RtlClearBits(NtCurrentPeb()->TlsExpansionBitmap, Index, 1);
Index = ~0;
SetLastError(ERROR_NOT_ENOUGH_MEMORY);
}
else
{
/* Clear the value. */
NtCurrentTeb()->TlsExpansionSlots[Index] = 0;
Index += TLS_MINIMUM_AVAILABLE;
}
}
else
{
SetLastError(ERROR_NO_MORE_ITEMS);
}
}
else
{
/* Clear the value. */
NtCurrentTeb()->TlsSlots[Index] = 0;
}
RtlReleasePebLock();
return Index;
}
static UCHAR XmsFree(WORD Handle)
{
DWORD BlockNumber;
PXMS_HANDLE HandleEntry = GetHandleRecord(Handle);
if (!ValidateHandle(HandleEntry))
return XMS_STATUS_INVALID_HANDLE;
if (HandleEntry->LockCount)
return XMS_STATUS_LOCKED;
BlockNumber = (HandleEntry->Address - XMS_ADDRESS) / XMS_BLOCK_SIZE;
RtlClearBits(&AllocBitmap, BlockNumber, HandleEntry->Size);
HandleEntry->Handle = 0;
FreeBlocks += HandleEntry->Size;
return XMS_STATUS_SUCCESS;
}
VOID
XenevtchnReleaseIoMemory(PVOID va, ULONG nr_bytes)
{
KIRQL old_irql;
ULONG page_nr;
XM_ASSERT(((ULONG_PTR)va & (PAGE_SIZE - 1)) == 0);
nr_bytes = (nr_bytes + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
page_nr =
(ULONG)(((ULONG_PTR)va - (ULONG_PTR)io_hole_va_start) / PAGE_SIZE);
old_irql = acquire_irqsafe_lock(&io_hole_lock);
XM_ASSERT(RtlAreBitsSet(&io_hole_in_use,
page_nr,
nr_bytes / PAGE_SIZE));
RtlClearBits(&io_hole_in_use, page_nr, nr_bytes / PAGE_SIZE);
release_irqsafe_lock(&io_hole_lock, old_irql);
}
_Use_decl_annotations_ EXTERN_C static bool VminitpSetupVMCS(
const PER_PROCESSOR_DATA *ProcessorData, ULONG_PTR GuestStackPointer,
ULONG_PTR GuestInstructionPointer, ULONG_PTR VmmStackPointer) {
unsigned char error = 0;
GDTR gdtr = {};
__sgdt(&gdtr);
IDTR idtr = {};
__sidt(&idtr);
VMX_VM_ENTER_CONTROLS vmEnterCtlRequested = {};
vmEnterCtlRequested.Fields.IA32eModeGuest = true;
VMX_VM_ENTER_CONTROLS vmEnterCtl = {
VminitpAdjustControlValue(IA32_VMX_ENTRY_CTLS, vmEnterCtlRequested.All)};
VMX_VM_EXIT_CONTROLS vmExitCtlRequested = {};
vmExitCtlRequested.Fields.AcknowledgeInterruptOnExit = true;
vmExitCtlRequested.Fields.HostAddressSpaceSize = true;
VMX_VM_EXIT_CONTROLS vmExitCtl = {
VminitpAdjustControlValue(IA32_VMX_EXIT_CTLS, vmExitCtlRequested.All)};
VMX_PIN_BASED_CONTROLS vmPinCtlRequested = {};
VMX_PIN_BASED_CONTROLS vmPinCtl = {
VminitpAdjustControlValue(IA32_VMX_PINBASED_CTLS, vmPinCtlRequested.All)};
VMX_CPU_BASED_CONTROLS vmCpuCtlRequested = {};
vmCpuCtlRequested.Fields.RDTSCExiting = true;
vmCpuCtlRequested.Fields.CR3LoadExiting = true; // MOV to CR3
vmCpuCtlRequested.Fields.CR8LoadExiting = true; // MOV to CR8
vmCpuCtlRequested.Fields.MovDRExiting = true;
vmCpuCtlRequested.Fields.UseMSRBitmaps = true;
vmCpuCtlRequested.Fields.ActivateSecondaryControl = true;
VMX_CPU_BASED_CONTROLS vmCpuCtl = {VminitpAdjustControlValue(
IA32_VMX_PROCBASED_CTLS, vmCpuCtlRequested.All)};
VMX_SECONDARY_CPU_BASED_CONTROLS vmCpuCtl2Requested = {};
vmCpuCtl2Requested.Fields.EnableRDTSCP = true;
vmCpuCtl2Requested.Fields.DescriptorTableExiting = true;
VMX_CPU_BASED_CONTROLS vmCpuCtl2 = {VminitpAdjustControlValue(
IA32_VMX_PROCBASED_CTLS2, vmCpuCtl2Requested.All)};
// Set up the MSR bitmap
// Activate VM-exit for RDMSR against all MSRs
const auto bitMapReadLow =
reinterpret_cast<UCHAR *>(ProcessorData->MsrBitmap);
const auto bitMapReadHigh = bitMapReadLow + 1024;
RtlFillMemory(bitMapReadLow, 1024, 0xff); // read 0 - 1fff
RtlFillMemory(bitMapReadHigh, 1024, 0xff); // read c0000000 - c0001fff
// But ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8)
RTL_BITMAP bitMapReadLowHeader = {};
RtlInitializeBitMap(&bitMapReadLowHeader,
reinterpret_cast<PULONG>(bitMapReadLow), 1024 * 8);
RtlClearBits(&bitMapReadLowHeader, 0xe7, 2);
// But ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102)
RTL_BITMAP bitMapReadHighHeader = {};
RtlInitializeBitMap(&bitMapReadHighHeader,
reinterpret_cast<PULONG>(bitMapReadHigh), 1024 * 8);
RtlClearBits(&bitMapReadHighHeader, 0x101, 2);
const auto msrBitmapPA = MmGetPhysicalAddress(ProcessorData->MsrBitmap);
// Set up CR0 and CR4 bitmaps
// Where a bit is masked, the shadow bit appears
// Where a bit is not masked, the actual bit appears
CR0_REG cr0mask = {};
cr0mask.Fields.WP = true;
CR4_REG cr4mask = {};
cr4mask.Fields.PGE = true;
// clang-format off
/* 16-Bit Control Field */
/* 16-Bit Guest-State Fields */
error |= __vmx_vmwrite(GUEST_ES_SELECTOR, AsmReadES());
error |= __vmx_vmwrite(GUEST_CS_SELECTOR, AsmReadCS());
error |= __vmx_vmwrite(GUEST_SS_SELECTOR, AsmReadSS());
error |= __vmx_vmwrite(GUEST_DS_SELECTOR, AsmReadDS());
error |= __vmx_vmwrite(GUEST_FS_SELECTOR, AsmReadFS());
error |= __vmx_vmwrite(GUEST_GS_SELECTOR, AsmReadGS());
error |= __vmx_vmwrite(GUEST_LDTR_SELECTOR, AsmReadLDTR());
error |= __vmx_vmwrite(GUEST_TR_SELECTOR, AsmReadTR());
/* 16-Bit Host-State Fields */
error |= __vmx_vmwrite(HOST_ES_SELECTOR, AsmReadES() & 0xf8); // RPL and TI
error |= __vmx_vmwrite(HOST_CS_SELECTOR, AsmReadCS() & 0xf8); // have to be 0
error |= __vmx_vmwrite(HOST_SS_SELECTOR, AsmReadSS() & 0xf8);
error |= __vmx_vmwrite(HOST_DS_SELECTOR, AsmReadDS() & 0xf8);
error |= __vmx_vmwrite(HOST_FS_SELECTOR, AsmReadFS() & 0xf8);
error |= __vmx_vmwrite(HOST_GS_SELECTOR, AsmReadGS() & 0xf8);
error |= __vmx_vmwrite(HOST_TR_SELECTOR, AsmReadTR() & 0xf8);
/* 64-Bit Control Fields */
error |= __vmx_vmwrite(IO_BITMAP_A, 0);
error |= __vmx_vmwrite(IO_BITMAP_B, 0);
error |= __vmx_vmwrite(MSR_BITMAP, msrBitmapPA.QuadPart);
error |= __vmx_vmwrite(TSC_OFFSET, 0);
/* 64-Bit Guest-State Fields */
error |= __vmx_vmwrite(VMCS_LINK_POINTER, 0xffffffffffffffff);
error |= __vmx_vmwrite(GUEST_IA32_DEBUGCTL, __readmsr(IA32_DEBUGCTL));
/* 32-Bit Control Fields */
error |= __vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmPinCtl.All);
error |= __vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmCpuCtl.All);
error |= __vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, vmCpuCtl2.All);
error |= __vmx_vmwrite(EXCEPTION_BITMAP, 0);
error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MASK, 0);
error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MATCH, 0);
error |= __vmx_vmwrite(CR3_TARGET_COUNT, 0);
error |= __vmx_vmwrite(VM_EXIT_CONTROLS, vmExitCtl.All);
error |= __vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0);
error |= __vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0);
error |= __vmx_vmwrite(VM_ENTRY_CONTROLS, vmEnterCtl.All);
error |= __vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0);
error |= __vmx_vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0);
/* 32-Bit Guest-State Fields */
error |= __vmx_vmwrite(GUEST_ES_LIMIT, GetSegmentLimit(AsmReadES()));
error |= __vmx_vmwrite(GUEST_CS_LIMIT, GetSegmentLimit(AsmReadCS()));
error |= __vmx_vmwrite(GUEST_SS_LIMIT, GetSegmentLimit(AsmReadSS()));
error |= __vmx_vmwrite(GUEST_DS_LIMIT, GetSegmentLimit(AsmReadDS()));
error |= __vmx_vmwrite(GUEST_FS_LIMIT, GetSegmentLimit(AsmReadFS()));
error |= __vmx_vmwrite(GUEST_GS_LIMIT, GetSegmentLimit(AsmReadGS()));
error |= __vmx_vmwrite(GUEST_LDTR_LIMIT, GetSegmentLimit(AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_LIMIT, GetSegmentLimit(AsmReadTR()));
error |= __vmx_vmwrite(GUEST_GDTR_LIMIT, gdtr.Limit);
error |= __vmx_vmwrite(GUEST_IDTR_LIMIT, idtr.Limit);
error |= __vmx_vmwrite(GUEST_ES_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadES()));
error |= __vmx_vmwrite(GUEST_CS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadCS()));
error |= __vmx_vmwrite(GUEST_SS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadSS()));
error |= __vmx_vmwrite(GUEST_DS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadDS()));
error |= __vmx_vmwrite(GUEST_FS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadFS()));
error |= __vmx_vmwrite(GUEST_GS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadGS()));
error |= __vmx_vmwrite(GUEST_LDTR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadTR()));
error |= __vmx_vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0);
error |= __vmx_vmwrite(GUEST_ACTIVITY_STATE, 0);
error |= __vmx_vmwrite(GUEST_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS));
/* 32-Bit Host-State Field */
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS));
/* Natural-Width Control Fields */
error |= __vmx_vmwrite(CR0_GUEST_HOST_MASK, cr0mask.All);
error |= __vmx_vmwrite(CR4_GUEST_HOST_MASK, cr4mask.All);
error |= __vmx_vmwrite(CR0_READ_SHADOW, __readcr0());
error |= __vmx_vmwrite(CR4_READ_SHADOW, __readcr4());
error |= __vmx_vmwrite(CR3_TARGET_VALUE0, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE1, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE2, 0);
error |= __vmx_vmwrite(CR3_TARGET_VALUE3, 0);
/* Natural-Width Guest-State Fields */
error |= __vmx_vmwrite(GUEST_CR0, __readcr0());
error |= __vmx_vmwrite(GUEST_CR3, __readcr3());
error |= __vmx_vmwrite(GUEST_CR4, __readcr4());
error |= __vmx_vmwrite(GUEST_ES_BASE, 0);
error |= __vmx_vmwrite(GUEST_CS_BASE, 0);
error |= __vmx_vmwrite(GUEST_SS_BASE, 0);
error |= __vmx_vmwrite(GUEST_DS_BASE, 0);
error |= __vmx_vmwrite(GUEST_FS_BASE, __readmsr(IA32_FS_BASE));
error |= __vmx_vmwrite(GUEST_GS_BASE, __readmsr(IA32_GS_BASE));
error |= __vmx_vmwrite(GUEST_LDTR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadLDTR()));
error |= __vmx_vmwrite(GUEST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR()));
error |= __vmx_vmwrite(GUEST_GDTR_BASE, gdtr.Address);
error |= __vmx_vmwrite(GUEST_IDTR_BASE, idtr.Address);
error |= __vmx_vmwrite(GUEST_DR7, __readdr(7));
error |= __vmx_vmwrite(GUEST_RSP, GuestStackPointer);
error |= __vmx_vmwrite(GUEST_RIP, GuestInstructionPointer);
error |= __vmx_vmwrite(GUEST_RFLAGS, __readeflags());
error |= __vmx_vmwrite(GUEST_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP));
error |= __vmx_vmwrite(GUEST_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP));
/* Natural-Width Host-State Fields */
error |= __vmx_vmwrite(HOST_CR0, __readcr0());
error |= __vmx_vmwrite(HOST_CR3, __readcr3());
error |= __vmx_vmwrite(HOST_CR4, __readcr4());
error |= __vmx_vmwrite(HOST_FS_BASE, __readmsr(IA32_FS_BASE));
error |= __vmx_vmwrite(HOST_GS_BASE, __readmsr(IA32_GS_BASE));
error |= __vmx_vmwrite(HOST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR()));
error |= __vmx_vmwrite(HOST_GDTR_BASE, gdtr.Address);
error |= __vmx_vmwrite(HOST_IDTR_BASE, idtr.Address);
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP));
error |= __vmx_vmwrite(HOST_IA32_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP));
error |= __vmx_vmwrite(HOST_RSP, VmmStackPointer);
error |= __vmx_vmwrite(HOST_RIP, reinterpret_cast<size_t>(AsmVmmEntryPoint));
// clang-format on
const auto vmxStatus = static_cast<VMX_STATUS>(error);
return vmxStatus == VMX_OK;
}
ULONG
NTAPI
MiFreePoolPages(IN PVOID StartingVa)
{
PMMPTE PointerPte, StartPte;
PMMPFN Pfn1, StartPfn;
PFN_NUMBER FreePages, NumberOfPages;
KIRQL OldIrql;
PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry;
ULONG i, End;
//
// Handle paged pool
//
if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd))
{
//
// Calculate the offset from the beginning of paged pool, and convert it
// into pages
//
i = ((ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart) >> PAGE_SHIFT;
End = i;
//
// Now use the end bitmap to scan until we find a set bit, meaning that
// this allocation finishes here
//
while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++;
//
// Now calculate the total number of pages this allocation spans
//
NumberOfPages = End - i + 1;
/* Delete the actual pages */
PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i;
FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL);
ASSERT(FreePages == NumberOfPages);
//
// Acquire the paged pool lock
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Clear the allocation and free bits
//
RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End);
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages);
//
// Update the hint if we need to
//
if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i;
//
// Release the lock protecting the bitmaps
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// And finally return the number of pages freed
//
return NumberOfPages;
}
VOID
IoFreeMapRegisters(
PADAPTER_OBJECT AdapterObject,
PVOID MapRegisterBase,
ULONG NumberOfMapRegisters
)
/*++
Routine Description:
This routine deallocates the map registers for the adapter. If there are
any queued adapter waiting for an attempt is made to allocate the next
entry.
Arguments:
AdapterObject - The adapter object to where the map register should be
returned.
MapRegisterBase - The map register base of the registers to be deallocated.
NumberOfMapRegisters - The number of registers to be deallocated.
Return Value:
None
--+*/
{
PADAPTER_OBJECT MasterAdapter;
LONG MapRegisterNumber;
PWAIT_CONTEXT_BLOCK Wcb;
PLIST_ENTRY Packet;
IO_ALLOCATION_ACTION Action;
KIRQL Irql;
//
// Begin by getting the address of the master adapter.
//
if (AdapterObject->MasterAdapter != NULL && MapRegisterBase != NULL) {
MasterAdapter = AdapterObject->MasterAdapter;
} else {
//
// There are no map registers to return.
//
return;
}
//
// Strip no scatter/gather flag.
//
MapRegisterBase = (PVOID) ((ULONG) MapRegisterBase & ~NO_SCATTER_GATHER);
MapRegisterNumber = (PTRANSLATION_ENTRY) MapRegisterBase -
(PTRANSLATION_ENTRY) MasterAdapter->MapRegisterBase;
//
// Acquire the master adapter spinlock which locks the adapter queue and the
// bit map for the map registers.
//
Irql = KfAcquireSpinLock(&MasterAdapter->SpinLock);
//
// Return the registers to the bit map.
//
RtlClearBits( MasterAdapter->MapRegisters,
MapRegisterNumber,
NumberOfMapRegisters
);
//
// Process any requests waiting for map registers in the adapter queue.
// Requests are processed until a request cannot be satisfied or until
// there are no more requests in the queue.
//
while(TRUE) {
if ( IsListEmpty(&MasterAdapter->AdapterQueue) ){
break;
}
Packet = RemoveHeadList( &MasterAdapter->AdapterQueue );
AdapterObject = CONTAINING_RECORD( Packet,
ADAPTER_OBJECT,
AdapterQueue
);
Wcb = AdapterObject->CurrentWcb;
//
// Attempt to allocate map registers for this request. Use the previous
// register base as a hint.
//
MapRegisterNumber = RtlFindClearBitsAndSet( MasterAdapter->MapRegisters,
AdapterObject->NumberOfMapRegisters,
MasterAdapter->NumberOfMapRegisters
);
if (MapRegisterNumber == -1) {
//
// There were not enough free map registers. Put this request back on
// the adapter queue where is came from.
//
InsertHeadList( &MasterAdapter->AdapterQueue,
&AdapterObject->AdapterQueue
);
break;
}
KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql );
AdapterObject->MapRegisterBase = (PVOID) ((PTRANSLATION_ENTRY)
MasterAdapter->MapRegisterBase + MapRegisterNumber);
//
// Set the no scatter/gather flag if scatter/gather not
// supported.
//
if (!AdapterObject->ScatterGather) {
AdapterObject->MapRegisterBase = (PVOID)
((ULONG) AdapterObject->MapRegisterBase | NO_SCATTER_GATHER);
}
//
// Invoke the driver's execution routine now.
//
Action = Wcb->DeviceRoutine( Wcb->DeviceObject,
Wcb->CurrentIrp,
AdapterObject->MapRegisterBase,
Wcb->DeviceContext );
//
// If the driver wishes to keep the map registers then set the number
// allocated to zero and set the action to deallocate object.
//
if (Action == DeallocateObjectKeepRegisters) {
AdapterObject->NumberOfMapRegisters = 0;
Action = DeallocateObject;
}
//
// If the driver would like to have the adapter deallocated,
// then deallocate any map registers allocated and then release
// the adapter object.
//
if (Action == DeallocateObject) {
//
// The map registers registers are deallocated here rather than in
// IoFreeAdapterChannel. This limits the number of times
// this routine can be called recursively possibly overflowing
// the stack. The worst case occurs if there is a pending
// request for the adapter that uses map registers and whos
// excution routine decallocates the adapter. In that case if there
// are no requests in the master adapter queue, then IoFreeMapRegisters
// will get called again.
//
if (AdapterObject->NumberOfMapRegisters != 0) {
//
// Deallocate the map registers and clear the count so that
// IoFreeAdapterChannel will not deallocate them again.
//
Irql = KfAcquireSpinLock( &MasterAdapter->SpinLock );
RtlClearBits( MasterAdapter->MapRegisters,
MapRegisterNumber,
AdapterObject->NumberOfMapRegisters
);
AdapterObject->NumberOfMapRegisters = 0;
KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql );
}
IoFreeAdapterChannel( AdapterObject );
}
Irql = KfAcquireSpinLock( &MasterAdapter->SpinLock );
}
KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql );
}
VOID
HvFreeHivePartial(
PHHIVE Hive,
HCELL_INDEX Start,
HSTORAGE_TYPE Type
)
/*++
Routine Description:
Free the memory and associated maps for the end of a hive
starting at Start. The baseblock, hive, etc will not be touched.
Arguments:
Hive - supplies a pointer to hive control structure for hive to
partially free.
Start - HCELL_INDEX of first bin to free, will free from this
bin (inclusive) to the end of the hives stable storage.
Type - Type of storage (Stable or Volatile) to be freed.
Return Value:
NONE.
--*/
{
PHMAP_DIRECTORY Dir;
PHMAP_ENTRY Me;
HCELL_INDEX Address;
ULONG StartTable;
ULONG Length;
PHBIN Bin;
ULONG Tables;
ULONG FirstBit;
ULONG LastBit;
PFREE_HBIN FreeBin;
ASSERT(Hive->Flat == FALSE);
ASSERT(Hive->ReadOnly == FALSE);
Address = Start;
Length = Hive->Storage[Type].Length;
ASSERT(Address <= Length);
if (Address == Length) {
return;
}
//
// Sweep through bin set
//
do {
Me = HvpGetCellMap(Hive, Address + (Type*HCELL_TYPE_MASK));
VALIDATE_CELL_MAP(__LINE__,Me,Hive,Address + (Type*HCELL_TYPE_MASK));
if (Me->BinAddress & HMAP_DISCARDABLE) {
FreeBin = (PFREE_HBIN)Me->BlockAddress;
if (FreeBin->Flags & FREE_HBIN_DISCARDABLE) {
CmpFree((PVOID)HBIN_BASE(Me->BinAddress), FreeBin->Size);
} else {
//
// The bin has been freed, but quota is still charged.
// Since the file will now shrink, the quota must be
// returned here.
//
if( Me->BinAddress & HMAP_INPAGEDPOOL) {
//
// we charge quota only for bins in paged-pool
//
CmpReleaseGlobalQuota(FreeBin->Size);
}
}
RemoveEntryList(&FreeBin->ListEntry);
Address += FreeBin->Size;
CmpFree(FreeBin, sizeof(FREE_HBIN));
} else {
Bin = (PHBIN)HBIN_BASE(Me->BinAddress);
Address += HvpGetBinMemAlloc(Hive,Bin,Type);
if( Me->BinAddress & HMAP_INPAGEDPOOL && HvpGetBinMemAlloc(Hive,Bin,Type) ) {
//
// free the bin only if it is allocated from paged pool
//
CmpFree(Bin, HvpGetBinMemAlloc(Hive,Bin,Type));
}
}
} while (Address < Length);
//
// Free map table storage
//
Tables = (((Hive->Storage[Type].Length) / HBLOCK_SIZE) - 1) / HTABLE_SLOTS;
Dir = Hive->Storage[Type].Map;
if (Start > 0) {
StartTable = ((Start-1) / HBLOCK_SIZE) / HTABLE_SLOTS;
} else {
StartTable = (ULONG)-1;
}
HvpFreeMap(Hive, Dir, StartTable+1, Tables);
//
// update hysteresis (eventually queue work item)
//
if( Type == Stable) {
CmpUpdateSystemHiveHysteresis(Hive,(Start&(~HCELL_TYPE_MASK)),Hive->Storage[Type].Length);
}
Hive->Storage[Type].Length = (Start&(~HCELL_TYPE_MASK));
if (Type==Stable) {
//
// Clear dirty vector for data past Hive->Storage[Stable].Length
//
FirstBit = Start / HSECTOR_SIZE;
LastBit = Hive->DirtyVector.SizeOfBitMap;
ASSERT(Hive->DirtyCount == RtlNumberOfSetBits(&Hive->DirtyVector));
RtlClearBits(&Hive->DirtyVector, FirstBit, LastBit-FirstBit);
Hive->DirtyCount = RtlNumberOfSetBits(&Hive->DirtyVector);
}
HvpAdjustHiveFreeDisplay(Hive,Hive->Storage[Type].Length,Type);
return;
}
ULONG
NTAPI
MiFreePoolPages(IN PVOID StartingVa)
{
PMMPTE PointerPte, StartPte;
PMMPFN Pfn1, StartPfn;
PFN_COUNT FreePages, NumberOfPages;
KIRQL OldIrql;
PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry;
ULONG i, End;
ULONG_PTR Offset;
//
// Handle paged pool
//
if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd))
{
//
// Calculate the offset from the beginning of paged pool, and convert it
// into pages
//
Offset = (ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart;
i = (ULONG)(Offset >> PAGE_SHIFT);
End = i;
//
// Now use the end bitmap to scan until we find a set bit, meaning that
// this allocation finishes here
//
while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++;
//
// Now calculate the total number of pages this allocation spans. If it's
// only one page, add it to the S-LIST instead of freeing it
//
NumberOfPages = End - i + 1;
if ((NumberOfPages == 1) &&
(ExQueryDepthSList(&MiPagedPoolSListHead) < MiPagedPoolSListMaximum))
{
InterlockedPushEntrySList(&MiPagedPoolSListHead, StartingVa);
return 1;
}
/* Delete the actual pages */
PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i;
FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL);
ASSERT(FreePages == NumberOfPages);
//
// Acquire the paged pool lock
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Clear the allocation and free bits
//
RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End);
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages);
//
// Update the hint if we need to
//
if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i;
//
// Release the lock protecting the bitmaps
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// And finally return the number of pages freed
//
return NumberOfPages;
}
ULONG
FASTCALL
MiReleasePageFileSpace (
IN MMPTE PteContents
)
/*++
Routine Description:
This routine frees the paging file allocated to the specified PTE
and adjusts the necessary quotas.
Arguments:
PteContents - Supplies the PTE which is in page file format.
Return Value:
Returns TRUE if any paging file space was deallocated.
Environment:
Kernel mode, APCs disabled, PFN lock held.
--*/
{
ULONG FreeBit;
ULONG PageFileNumber;
MM_PFN_LOCK_ASSERT();
if (PteContents.u.Soft.Prototype == 1) {
//
// Not in page file format.
//
return FALSE;
}
FreeBit = GET_PAGING_FILE_OFFSET (PteContents);
if ((FreeBit == 0) || (FreeBit == 0xFFFFF)) {
//
// Page is not in a paging file, just return.
//
return FALSE;
}
PageFileNumber = GET_PAGING_FILE_NUMBER (PteContents);
ASSERT (RtlCheckBit( MmPagingFile[PageFileNumber]->Bitmap, FreeBit) == 1);
#if DBG
if ((FreeBit < 8192) && (PageFileNumber == 0)) {
ASSERT ((MmPagingFileDebug[FreeBit] & 1) != 0);
MmPagingFileDebug[FreeBit] &= 0xfffffffe;
}
#endif //DBG
RtlClearBits ( MmPagingFile[PageFileNumber]->Bitmap, FreeBit, 1);
MmPagingFile[PageFileNumber]->FreeSpace += 1;
MmPagingFile[PageFileNumber]->CurrentUsage -= 1;
//
// Check to see if we should move some MDL entries for the
// modified page writer now that more free space is available.
//
if ((MmNumberOfActiveMdlEntries == 0) ||
(MmPagingFile[PageFileNumber]->FreeSpace == MM_USABLE_PAGES_FREE)) {
MiUpdateModifiedWriterMdls (PageFileNumber);
}
return TRUE;
}
NTSTATUS
FatSetFsLabelInfo (
IN PIRP_CONTEXT IrpContext,
IN PVCB Vcb,
IN PFILE_FS_LABEL_INFORMATION Buffer
)
/*++
Routine Description:
This routine implements the set volume label call
Arguments:
Vcb - Supplies the Vcb being queried
Buffer - Supplies the input where the information is stored.
Return Value:
NTSTATUS - Returns the status for the operation
--*/
{
NTSTATUS Status;
PDIRENT Dirent;
PBCB DirentBcb = NULL;
ULONG ByteOffset;
WCHAR TmpBuffer[11];
UCHAR OemBuffer[11];
OEM_STRING OemLabel;
UNICODE_STRING UnicodeString;
UNICODE_STRING UpcasedLabel;
DebugTrace(+1, Dbg, "FatSetFsLabelInfo...\n", 0);
//
// Setup our local variable
//
UnicodeString.Length = (USHORT)Buffer->VolumeLabelLength;
UnicodeString.MaximumLength = UnicodeString.Length;
UnicodeString.Buffer = (PWSTR) &Buffer->VolumeLabel[0];
//
// Make sure the name can fit into the stack buffer
//
if ( UnicodeString.Length > 11*sizeof(WCHAR) ) {
return STATUS_INVALID_VOLUME_LABEL;
}
//
// Upcase the name and convert it to the Oem code page.
//
OemLabel.Buffer = &OemBuffer[0];
OemLabel.Length = 0;
OemLabel.MaximumLength = 11;
Status = RtlUpcaseUnicodeStringToCountedOemString( &OemLabel,
&UnicodeString,
FALSE );
//
// Volume label that fits in 11 unicode character length limit
// is not necessary within 11 characters in OEM character set.
//
if (!NT_SUCCESS( Status )) {
DebugTrace(-1, Dbg, "FatSetFsLabelInfo: Label must be too long. %08lx\n", Status );
return STATUS_INVALID_VOLUME_LABEL;
}
//
// Strip spaces off of the label.
//
if (OemLabel.Length > 0) {
USHORT i;
USHORT LastSpaceIndex = MAXUSHORT;
//
// Check the label for illegal characters
//
for ( i = 0; i < (ULONG)OemLabel.Length; i += 1 ) {
if ( FsRtlIsLeadDbcsCharacter( OemLabel.Buffer[i] ) ) {
LastSpaceIndex = MAXUSHORT;
i += 1;
continue;
}
if (!FsRtlIsAnsiCharacterLegalFat(OemLabel.Buffer[i], FALSE) ||
(OemLabel.Buffer[i] == '.')) {
return STATUS_INVALID_VOLUME_LABEL;
}
//
// Watch for the last run of spaces, so we can strip them.
//
if (OemLabel.Buffer[i] == ' ' &&
LastSpaceIndex == MAXUSHORT) {
LastSpaceIndex = i;
} else {
LastSpaceIndex = MAXUSHORT;
}
}
if (LastSpaceIndex != MAXUSHORT) {
OemLabel.Length = LastSpaceIndex;
}
}
//
// Get the Unicode upcased string to store in the VPB.
//
UpcasedLabel.Length = UnicodeString.Length;
UpcasedLabel.MaximumLength = 11*sizeof(WCHAR);
UpcasedLabel.Buffer = &TmpBuffer[0];
Status = RtlOemStringToCountedUnicodeString( &UpcasedLabel,
&OemLabel,
FALSE );
if (!NT_SUCCESS( Status )) {
DebugTrace(-1, Dbg, "FatSetFsLabelInfo: Label must be too long. %08lx\n", Status );
return STATUS_INVALID_VOLUME_LABEL;
}
DirentBcb = NULL;
//
// Make this look like a write through to disk. This is important to
// avoid a unpleasant window where it looks like we have the wrong volume.
//
SetFlag( IrpContext->Flags, IRP_CONTEXT_FLAG_WRITE_THROUGH );
try {
//
// Are we setting or removing the label? Note that shaving spaces could
// make this different than wondering if the input buffer is non-zero length.
//
if (OemLabel.Length > 0) {
//
// Locate the volume label if there already is one
//
FatLocateVolumeLabel( IrpContext,
Vcb,
&Dirent,
&DirentBcb,
&ByteOffset );
//
// Check that we really got one, if not then we need to create
// a new one. The procedure we call will raise an appropriate
// status if we are not able to allocate a new dirent
//
if (Dirent == NULL) {
ByteOffset = FatCreateNewDirent( IrpContext,
Vcb->RootDcb,
1 );
FatPrepareWriteDirectoryFile( IrpContext,
Vcb->RootDcb,
ByteOffset,
sizeof(DIRENT),
&DirentBcb,
&Dirent,
FALSE,
TRUE,
&Status );
ASSERT( NT_SUCCESS( Status ));
} else {
//
// Just mark this guy dirty now.
//
FatSetDirtyBcb( IrpContext, DirentBcb, Vcb, TRUE );
}
//
// Now reconstruct the volume label dirent.
//
FatConstructLabelDirent( IrpContext,
Dirent,
&OemLabel );
//
// Unpin the Bcb here so that we will get any IO errors
// here before changing the VPB label.
//
FatUnpinBcb( IrpContext, DirentBcb );
FatUnpinRepinnedBcbs( IrpContext );
//
// Now set the upcased label in the VPB
//
RtlCopyMemory( &Vcb->Vpb->VolumeLabel[0],
&UpcasedLabel.Buffer[0],
UpcasedLabel.Length );
Vcb->Vpb->VolumeLabelLength = UpcasedLabel.Length;
} else {
//
// Otherwise we're trying to delete the label
// Locate the current volume label if there already is one
//
FatLocateVolumeLabel( IrpContext,
Vcb,
&Dirent,
&DirentBcb,
&ByteOffset );
//
// Check that we really got one
//
if (Dirent == NULL) {
try_return( Status = STATUS_SUCCESS );
}
//
// Now delete the current label.
//
Dirent->FileName[0] = FAT_DIRENT_DELETED;
ASSERT( (Vcb->RootDcb->Specific.Dcb.UnusedDirentVbo == 0xffffffff) ||
RtlAreBitsSet( &Vcb->RootDcb->Specific.Dcb.FreeDirentBitmap,
ByteOffset / sizeof(DIRENT),
1 ) );
RtlClearBits( &Vcb->RootDcb->Specific.Dcb.FreeDirentBitmap,
ByteOffset / sizeof(DIRENT),
1 );
FatSetDirtyBcb( IrpContext, DirentBcb, Vcb, TRUE );
//
// Unpin the Bcb here so that we will get any IO errors
// here before changing the VPB label.
//
FatUnpinBcb( IrpContext, DirentBcb );
FatUnpinRepinnedBcbs( IrpContext );
//
// Now set the label in the VPB
//
Vcb->Vpb->VolumeLabelLength = 0;
}
Status = STATUS_SUCCESS;
FatSortDirectory(IrpContext, Vcb->RootDcb);
try_exit: NOTHING;
} finally {
DebugUnwind( FatSetFsALabelInfo );
FatUnpinBcb( IrpContext, DirentBcb );
DebugTrace(-1, Dbg, "FatSetFsALabelInfo -> STATUS_SUCCESS\n", 0);
}
return Status;
}
PVOID
NTAPI
MiAllocatePoolPages(IN POOL_TYPE PoolType,
IN SIZE_T SizeInBytes)
{
PFN_NUMBER PageFrameNumber;
PFN_COUNT SizeInPages, PageTableCount;
ULONG i;
KIRQL OldIrql;
PLIST_ENTRY NextEntry, NextHead, LastHead;
PMMPTE PointerPte, StartPte;
PMMPDE PointerPde;
ULONG EndAllocation;
MMPTE TempPte;
MMPDE TempPde;
PMMPFN Pfn1;
PVOID BaseVa, BaseVaStart;
PMMFREE_POOL_ENTRY FreeEntry;
PKSPIN_LOCK_QUEUE LockQueue;
//
// Figure out how big the allocation is in pages
//
SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes);
//
// Check for overflow
//
if (SizeInPages == 0)
{
//
// Fail
//
return NULL;
}
//
// Handle paged pool
//
if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool)
{
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Lock the paged pool mutex
//
KeAcquireGuardedMutex(&MmPagedPoolMutex);
//
// Find some empty allocation space
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
MmPagedPoolInfo.PagedPoolHint);
if (i == 0xFFFFFFFF)
{
//
// Get the page bit count
//
i = ((SizeInPages - 1) / PTE_COUNT) + 1;
DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages);
//
// Check if there is enougn paged pool expansion space left
//
if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
//
// Check if we'll have to expand past the last PTE we have available
//
if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) >
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool))
{
//
// We can only support this much then
//
PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool);
PageTableCount = (PFN_COUNT)(PointerPde + 1 -
MmPagedPoolInfo.NextPdeForPagedPoolExpansion);
ASSERT(PageTableCount < i);
i = PageTableCount;
}
else
{
//
// Otherwise, there is plenty of space left for this expansion
//
PageTableCount = i;
}
//
// Get the template PDE we'll use to expand
//
TempPde = ValidKernelPde;
//
// Get the first PTE in expansion space
//
PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion;
BaseVa = MiPdeToPte(PointerPde);
BaseVaStart = BaseVa;
//
// Lock the PFN database and loop pages
//
OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
do
{
//
// It should not already be valid
//
ASSERT(PointerPde->u.Hard.Valid == 0);
/* Request a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
TempPde.u.Hard.PageFrameNumber = PageFrameNumber;
#if (_MI_PAGING_LEVELS >= 3)
/* On PAE/x64 systems, there's no double-buffering */
ASSERT(FALSE);
#else
//
// Save it into our double-buffered system page directory
//
MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde;
/* Initialize the PFN */
MiInitializePfnForOtherProcess(PageFrameNumber,
(PMMPTE)PointerPde,
MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]);
/* Write the actual PDE now */
// MI_WRITE_VALID_PDE(PointerPde, TempPde);
#endif
//
// Move on to the next expansion address
//
PointerPde++;
BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE);
i--;
} while (i > 0);
//
// Release the PFN database lock
//
KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
//
// These pages are now available, clear their availablity bits
//
EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion -
(PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) *
PTE_COUNT;
RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap,
EndAllocation,
PageTableCount * PTE_COUNT);
//
// Update the next expansion location
//
MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount;
//
// Zero out the newly available memory
//
RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE);
//
// Now try consuming the pages again
//
i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap,
SizeInPages,
0);
if (i == 0xFFFFFFFF)
{
//
// Out of memory!
//
DPRINT1("OUT OF PAGED POOL!!!\n");
KeReleaseGuardedMutex(&MmPagedPoolMutex);
return NULL;
}
}
//
// Update the pool hint if the request was just one page
//
if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1;
//
// Update the end bitmap so we know the bounds of this allocation when
// the time comes to free it
//
EndAllocation = i + SizeInPages - 1;
RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation);
//
// Now we can release the lock (it mainly protects the bitmap)
//
KeReleaseGuardedMutex(&MmPagedPoolMutex);
//
// Now figure out where this allocation starts
//
BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT));
//
// Flush the TLB
//
KeFlushEntireTb(TRUE, TRUE);
/* Setup a demand-zero writable PTE */
MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
//
// Find the first and last PTE, then loop them all
//
PointerPte = MiAddressToPte(BaseVa);
StartPte = PointerPte + SizeInPages;
do
{
//
// Write the demand zero PTE and keep going
//
MI_WRITE_INVALID_PTE(PointerPte, TempPte);
} while (++PointerPte < StartPte);
//
// Return the allocation address to the caller
//
return BaseVa;
}
//
// If only one page is being requested, try to grab it from the S-LIST
//
if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead)))
{
BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead);
if (BaseVa) return BaseVa;
}
//
// Allocations of less than 4 pages go into their individual buckets
//
i = SizeInPages - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
//
// Loop through all the free page lists based on the page index
//
NextHead = &MmNonPagedPoolFreeListHead[i];
LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS];
//
// Acquire the nonpaged pool lock
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
do
{
//
// Now loop through all the free page entries in this given list
//
NextEntry = NextHead->Flink;
while (NextEntry != NextHead)
{
/* Is freed non paged pool enabled */
if (MmProtectFreedNonPagedPool)
{
/* We need to be able to touch this page, unprotect it */
MiUnProtectFreeNonPagedPool(NextEntry, 0);
}
//
// Grab the entry and see if it can handle our allocation
//
FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List);
ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE);
if (FreeEntry->Size >= SizeInPages)
{
//
// It does, so consume the pages from here
//
FreeEntry->Size -= SizeInPages;
//
// The allocation will begin in this free page area
//
BaseVa = (PVOID)((ULONG_PTR)FreeEntry +
(FreeEntry->Size << PAGE_SHIFT));
/* Remove the item from the list, depending if pool is protected */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolRemoveEntryList(&FreeEntry->List);
else
RemoveEntryList(&FreeEntry->List);
//
// However, check if its' still got space left
//
if (FreeEntry->Size != 0)
{
/* Check which list to insert this entry into */
i = FreeEntry->Size - 1;
if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1;
/* Insert the entry into the free list head, check for prot. pool */
if (MmProtectFreedNonPagedPool)
MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE);
else
InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List);
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
//
// Grab the PTE for this allocation
//
PointerPte = MiAddressToPte(BaseVa);
ASSERT(PointerPte->u.Hard.Valid == 1);
//
// Grab the PFN NextEntry and index
//
Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
//
// Now mark it as the beginning of an allocation
//
ASSERT(Pfn1->u3.e1.StartOfAllocation == 0);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as special pool if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK)
{
Pfn1->u4.VerifierAllocation = 1;
}
//
// Check if the allocation is larger than one page
//
if (SizeInPages != 1)
{
//
// Navigate to the last PFN entry and PTE
//
PointerPte += SizeInPages - 1;
ASSERT(PointerPte->u.Hard.Valid == 1);
Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber);
}
//
// Mark this PFN as the last (might be the same as the first)
//
ASSERT(Pfn1->u3.e1.EndOfAllocation == 0);
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Release the nonpaged pool lock, and return the allocation
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
return BaseVa;
}
//
// Try the next free page entry
//
NextEntry = FreeEntry->List.Flink;
/* Is freed non paged pool protected? */
if (MmProtectFreedNonPagedPool)
{
/* Protect the freed pool! */
MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size);
}
}
} while (++NextHead < LastHead);
//
// If we got here, we're out of space.
// Start by releasing the lock
//
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Allocate some system PTEs
//
StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion);
PointerPte = StartPte;
if (StartPte == NULL)
{
//
// Ran out of memory
//
DPRINT1("Out of NP Expansion Pool\n");
return NULL;
}
//
// Acquire the pool lock now
//
OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock);
//
// Lock the PFN database too
//
LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock];
KeAcquireQueuedSpinLockAtDpcLevel(LockQueue);
//
// Loop the pages
//
TempPte = ValidKernelPte;
do
{
/* Allocate a page */
MI_SET_USAGE(MI_USAGE_PAGED_POOL);
MI_SET_PROCESS2("Kernel");
PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
/* Get the PFN entry for it and fill it out */
Pfn1 = MiGetPfnEntry(PageFrameNumber);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
Pfn1->PteAddress = PointerPte;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
Pfn1->u4.VerifierAllocation = 0;
/* Write the PTE for it */
TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
MI_WRITE_VALID_PTE(PointerPte++, TempPte);
} while (--SizeInPages > 0);
//
// This is the last page
//
Pfn1->u3.e1.EndOfAllocation = 1;
//
// Get the first page and mark it as such
//
Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber);
Pfn1->u3.e1.StartOfAllocation = 1;
/* Mark it as a verifier allocation if needed */
ASSERT(Pfn1->u4.VerifierAllocation == 0);
if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1;
//
// Release the PFN and nonpaged pool lock
//
KeReleaseQueuedSpinLockFromDpcLevel(LockQueue);
KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql);
//
// Return the address
//
return MiPteToAddress(StartPte);
}
VOID NtdllBitmap::ClearBits( ULONG starting_index, ULONG number_to_clear )
{
assert(RtlClearBits != NULL);
RtlClearBits(this, starting_index, number_to_clear);
}
static UCHAR XmsRealloc(WORD Handle, WORD NewSize)
{
DWORD BlockNumber;
PXMS_HANDLE HandleEntry = GetHandleRecord(Handle);
DWORD CurrentIndex = 0;
ULONG RunStart;
ULONG RunSize;
if (!ValidateHandle(HandleEntry))
return XMS_STATUS_INVALID_HANDLE;
if (HandleEntry->LockCount)
return XMS_STATUS_LOCKED;
/* Get the block number */
BlockNumber = (HandleEntry->Address - XMS_ADDRESS) / XMS_BLOCK_SIZE;
if (NewSize < HandleEntry->Size)
{
/* Just reduce the size of this block */
RtlClearBits(&AllocBitmap, BlockNumber + NewSize, HandleEntry->Size - NewSize);
FreeBlocks += HandleEntry->Size - NewSize;
HandleEntry->Size = NewSize;
}
else if (NewSize > HandleEntry->Size)
{
/* Check if we can expand in-place */
if (RtlAreBitsClear(&AllocBitmap,
BlockNumber + HandleEntry->Size,
NewSize - HandleEntry->Size))
{
/* Just increase the size of this block */
RtlSetBits(&AllocBitmap,
BlockNumber + HandleEntry->Size,
NewSize - HandleEntry->Size);
FreeBlocks -= NewSize - HandleEntry->Size;
HandleEntry->Size = NewSize;
/* We're done */
return XMS_STATUS_SUCCESS;
}
/* Deallocate the current block range */
RtlClearBits(&AllocBitmap, BlockNumber, HandleEntry->Size);
/* Find a new place for this block */
while (CurrentIndex < XMS_BLOCKS)
{
RunSize = RtlFindNextForwardRunClear(&AllocBitmap, CurrentIndex, &RunStart);
if (RunSize == 0) break;
if (RunSize >= NewSize)
{
/* Allocate the new range */
RtlSetBits(&AllocBitmap, RunStart, NewSize);
/* Move the data to the new location */
RtlMoveMemory((PVOID)REAL_TO_PHYS(XMS_ADDRESS + RunStart * XMS_BLOCK_SIZE),
(PVOID)REAL_TO_PHYS(HandleEntry->Address),
HandleEntry->Size * XMS_BLOCK_SIZE);
/* Update the handle entry */
HandleEntry->Address = XMS_ADDRESS + RunStart * XMS_BLOCK_SIZE;
HandleEntry->Size = NewSize;
/* Update the free block counter */
FreeBlocks -= NewSize - HandleEntry->Size;
return XMS_STATUS_SUCCESS;
}
/* Keep searching */
CurrentIndex = RunStart + RunSize;
}
/* Restore the old block range */
RtlSetBits(&AllocBitmap, BlockNumber, HandleEntry->Size);
return XMS_STATUS_OUT_OF_MEMORY;
}
return XMS_STATUS_SUCCESS;
}
VOID
MiZeroPageFile (
IN PVOID Context
)
/*++
Routine Description:
This routine zeroes all inactive pagefile blocks in the specified paging
file.
Arguments:
Context - Supplies the information on which pagefile to zero and a zeroed
page to use for the I/O.
Return Value:
Returns TRUE on success, FALSE on failure.
Environment:
Kernel mode, the caller must lock down PAGELK.
--*/
{
PFN_NUMBER MaxPagesToWrite;
PMMPFN Pfn1;
PPFN_NUMBER Page;
PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER];
PMDL Mdl;
NTSTATUS Status;
KEVENT IoEvent;
IO_STATUS_BLOCK IoStatus;
KIRQL OldIrql;
LARGE_INTEGER StartingOffset;
ULONG count;
ULONG i;
PFN_NUMBER first;
ULONG write;
PKEVENT AllDone;
SIZE_T NumberOfBytes;
PMMPAGING_FILE PagingFile;
PFN_NUMBER ZeroedPageFrame;
PMM_ZERO_PAGEFILE_CONTEXT ZeroContext;
ZeroContext = (PMM_ZERO_PAGEFILE_CONTEXT) Context;
PagingFile = ZeroContext->PagingFile;
ZeroedPageFrame = ZeroContext->ZeroedPageFrame;
AllDone = ZeroContext->AllDone;
ExFreePool (Context);
NumberOfBytes = MmModifiedWriteClusterSize << PAGE_SHIFT;
MaxPagesToWrite = NumberOfBytes >> PAGE_SHIFT;
Mdl = (PMDL) MdlHack;
Page = (PPFN_NUMBER)(Mdl + 1);
KeInitializeEvent (&IoEvent, NotificationEvent, FALSE);
MmInitializeMdl (Mdl, NULL, PAGE_SIZE);
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
Mdl->StartVa = NULL;
i = 0;
Page = (PPFN_NUMBER)(Mdl + 1);
for (i = 0; i < MaxPagesToWrite; i += 1) {
*Page = ZeroedPageFrame;
Page += 1;
}
count = 0;
write = FALSE;
SATISFY_OVERZEALOUS_COMPILER (first = 0);
LOCK_PFN (OldIrql);
for (i = 1; i < PagingFile->Size; i += 1) {
if (RtlCheckBit (PagingFile->Bitmap, (ULONG) i) == 0) {
//
// Claim the pagefile location as the modified writer
// may already be scanning.
//
RtlSetBit (PagingFile->Bitmap, (ULONG) i);
if (count == 0) {
first = i;
}
count += 1;
if ((count == MaxPagesToWrite) || (i == PagingFile->Size - 1)) {
write = TRUE;
}
}
else {
if (count != 0) {
//
// Issue a write.
//
write = TRUE;
}
}
if (write) {
UNLOCK_PFN (OldIrql);
StartingOffset.QuadPart = (LONGLONG)first << PAGE_SHIFT;
Mdl->ByteCount = count << PAGE_SHIFT;
KeClearEvent (&IoEvent);
Status = IoSynchronousPageWrite (PagingFile->File,
Mdl,
&StartingOffset,
&IoEvent,
&IoStatus);
//
// Ignore all I/O failures - there is nothing that can
// be done at this point.
//
if (!NT_SUCCESS (Status)) {
KeSetEvent (&IoEvent, 0, FALSE);
}
Status = KeWaitForSingleObject (&IoEvent,
WrPageOut,
KernelMode,
FALSE,
(PLARGE_INTEGER)&MmTwentySeconds);
if (Status == STATUS_TIMEOUT) {
//
// The write did not complete in 20 seconds, assume
// that the file systems are hung and return an error.
//
// Note the zero page (and any MDL system virtual address a
// driver may have created) is leaked because we don't know
// what the filesystem or storage stack might (still) be
// doing to them.
//
Pfn1 = MI_PFN_ELEMENT (ZeroedPageFrame);
LOCK_PFN (OldIrql);
//
// Increment the reference count on the zeroed page to ensure
// it is never freed.
//
InterlockedIncrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
break;
}
if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) {
MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl);
}
write = FALSE;
LOCK_PFN (OldIrql);
RtlClearBits (PagingFile->Bitmap, (ULONG) first, count);
count = 0;
}
}
UNLOCK_PFN (OldIrql);
KeSetEvent (AllDone, 0, FALSE);
return;
}
