// Initialize shared processor data _Use_decl_annotations_ static SharedProcessorData *VmpInitializeSharedData() { PAGED_CODE(); const auto shared_data = reinterpret_cast<SharedProcessorData *>( ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(SharedProcessorData), kHyperPlatformCommonPoolTag)); if (!shared_data) { return nullptr; } RtlZeroMemory(shared_data, sizeof(SharedProcessorData)); HYPERPLATFORM_LOG_DEBUG("SharedData= %p", shared_data); // Set up the MSR bitmap const auto msr_bitmap = ExAllocatePoolWithTag(NonPagedPoolNx, PAGE_SIZE, kHyperPlatformCommonPoolTag); if (!msr_bitmap) { ExFreePoolWithTag(shared_data, kHyperPlatformCommonPoolTag); return nullptr; } RtlZeroMemory(msr_bitmap, PAGE_SIZE); shared_data->msr_bitmap = msr_bitmap; // Checks MSRs causing #GP and should not cause VM-exit from 0 to 0xfff. bool unsafe_msr_map[0x1000] = {}; for (auto msr = 0ul; msr < RTL_NUMBER_OF(unsafe_msr_map); ++msr) { __try { UtilReadMsr(static_cast<Msr>(msr)); } __except (EXCEPTION_EXECUTE_HANDLER) { unsafe_msr_map[msr] = true; } } // Activate VM-exit for RDMSR against all MSRs const auto bitmap_read_low = reinterpret_cast<UCHAR *>(msr_bitmap); const auto bitmap_read_high = bitmap_read_low + 1024; RtlFillMemory(bitmap_read_low, 1024, 0xff); // read 0 - 1fff RtlFillMemory(bitmap_read_high, 1024, 0xff); // read c0000000 - c0001fff // But ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8) RTL_BITMAP bitmap_read_low_header = {}; RtlInitializeBitMap(&bitmap_read_low_header, reinterpret_cast<PULONG>(bitmap_read_low), 1024 * 8); RtlClearBits(&bitmap_read_low_header, 0xe7, 2); // Also ignore the unsage MSRs for (auto msr = 0ul; msr < RTL_NUMBER_OF(unsafe_msr_map); ++msr) { const auto ignore = unsafe_msr_map[msr]; if (ignore) { RtlClearBits(&bitmap_read_low_header, msr, 1); } } // But ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102) RTL_BITMAP bitmap_read_high_header = {}; RtlInitializeBitMap(&bitmap_read_high_header, reinterpret_cast<PULONG>(bitmap_read_high), 1024 * 8); RtlClearBits(&bitmap_read_high_header, 0x101, 2); return shared_data; }
static NTSTATUS init_msr_bitmap(struct ksm *k) { void *msr_bitmap = ExAllocatePool(NonPagedPoolNx, PAGE_SIZE); if (!msr_bitmap) return STATUS_NO_MEMORY; k->msr_bitmap = msr_bitmap; RtlZeroMemory(msr_bitmap, PAGE_SIZE); /* For all MSRs... */ u8 *bitmap_read_lo = (u8 *)msr_bitmap; u8 *bitmap_read_hi = bitmap_read_lo + 1024; memset(bitmap_read_lo, 0xff, 1024); // 0 -> 1fff memset(bitmap_read_hi, 0xff, 1024); // c0000000 - c0001fff /* ... ignore MSR_IA32_MPERF and MSR_IA32_APERF */ RTL_BITMAP bitmap_read_lo_hdr; RtlInitializeBitMap(&bitmap_read_lo_hdr, (PULONG)bitmap_read_lo, 1024 * CHAR_BIT); RtlClearBits(&bitmap_read_lo_hdr, MSR_IA32_MPERF, 2); for (u32 msr = 0; msr < PAGE_SIZE; ++msr) { __try { __readmsr(msr); } __except (EXCEPTION_EXECUTE_HANDLER) { RtlClearBits(&bitmap_read_lo_hdr, msr, 1); } } /* ... and ignore MSR_IA32_GS_BASE and MSR_IA32_KERNEL_GS_BASE */ RTL_BITMAP bitmap_read_hi_hdr; RtlInitializeBitMap(&bitmap_read_hi_hdr, (PULONG)bitmap_read_hi, 1024 * CHAR_BIT); RtlClearBits(&bitmap_read_hi_hdr, 0x101, 2); return STATUS_SUCCESS; }
void Test_RtlClearBits(void) { RTL_BITMAP BitMapHeader; ULONG *Buffer; ULONG BufferSize = 2 * sizeof(*Buffer); Buffer = AllocateGuarded(BufferSize); RtlInitializeBitMap(&BitMapHeader, Buffer, 19); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 0, 0); ok_hex(Buffer[0], 0xffffffff); ok_hex(Buffer[1], 0xffffffff); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 0, 1); ok_hex(Buffer[0], 0xfffffffe); ok_hex(Buffer[1], 0xffffffff); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 21, 1); ok_hex(Buffer[0], 0xffdfffff); ok_hex(Buffer[1], 0xffffffff); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 7, 9); ok_hex(Buffer[0], 0xffff007f); ok_hex(Buffer[1], 0xffffffff); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 13, 22); ok_hex(Buffer[0], 0x00001fff); ok_hex(Buffer[1], 0xfffffff8); memset(Buffer, 0xff, BufferSize); RtlClearBits(&BitMapHeader, 63, 1); ok_hex(Buffer[0], 0xffffffff); ok_hex(Buffer[1], 0x7fffffff); memset(Buffer, 0xcc, BufferSize); RtlClearBits(&BitMapHeader, 3, 6); RtlClearBits(&BitMapHeader, 11, 5); RtlClearBits(&BitMapHeader, 21, 7); RtlClearBits(&BitMapHeader, 37, 4); ok_hex(Buffer[0], 0xc00c0404); ok_hex(Buffer[1], 0xcccccc0c); FreeGuarded(Buffer); }
/* * @implemented */ BOOL WINAPI TlsFree(DWORD Index) { BOOL BitSet; if (Index >= TLS_EXPANSION_SLOTS + TLS_MINIMUM_AVAILABLE) { SetLastErrorByStatus(STATUS_INVALID_PARAMETER); return FALSE; } RtlAcquirePebLock(); if (Index >= TLS_MINIMUM_AVAILABLE) { BitSet = RtlAreBitsSet(NtCurrentPeb()->TlsExpansionBitmap, Index - TLS_MINIMUM_AVAILABLE, 1); if (BitSet) RtlClearBits(NtCurrentPeb()->TlsExpansionBitmap, Index - TLS_MINIMUM_AVAILABLE, 1); } else { BitSet = RtlAreBitsSet(NtCurrentPeb()->TlsBitmap, Index, 1); if (BitSet) RtlClearBits(NtCurrentPeb()->TlsBitmap, Index, 1); } if (BitSet) { /* Clear the TLS cells (slots) in all threads of the current process. */ NtSetInformationThread(NtCurrentThread(), ThreadZeroTlsCell, &Index, sizeof(DWORD)); } else { SetLastError(ERROR_INVALID_PARAMETER); } RtlReleasePebLock(); return BitSet; }
static UCHAR EmsFree(USHORT Handle) { PLIST_ENTRY Entry; PEMS_HANDLE HandleEntry = GetHandleRecord(Handle); if (!ValidateHandle(HandleEntry)) return EMS_STATUS_INVALID_HANDLE; for (Entry = HandleEntry->PageList.Flink; Entry != &HandleEntry->PageList; Entry = Entry->Flink) { PEMS_PAGE PageEntry = (PEMS_PAGE)CONTAINING_RECORD(Entry, EMS_PAGE, Entry); ULONG PageNumber = ARRAY_INDEX(PageEntry, PageTable); /* Free the page */ RtlClearBits(&AllocBitmap, PageNumber, 1); } InitializeListHead(&HandleEntry->PageList); if (Handle != EMS_SYSTEM_HANDLE) FreeHandle(HandleEntry); return EMS_STATUS_SUCCESS; }
/****************************************************************** * macho_fill_sect_is_code * * Callback for macho_enum_load_commands. Determines which segments * of a Mach-O file contain code. All commands are expected to be * of LC_SEGMENT type. */ static int macho_fill_sect_is_code(struct macho_file_map* fmap, const struct load_command* lc, void* user) { const struct segment_command* sc = (const struct segment_command*)lc; const struct section* sections; int* cursect = user; int i; TRACE("(%p/%d, %p, %p/%d) scanning %u sections\n", fmap, fmap->fd, lc, cursect, *cursect, sc->nsects); sections = (const struct section*)(sc + 1); for (i = 0; i < sc->nsects; i++) { if (*cursect > MAX_SECT) return -1; (*cursect)++; if (!(sections[i].flags & SECTION_TYPE) && (sections[i].flags & (S_ATTR_PURE_INSTRUCTIONS|S_ATTR_SOME_INSTRUCTIONS))) RtlSetBits(&fmap->sect_is_code, *cursect, 1); else RtlClearBits(&fmap->sect_is_code, *cursect, 1); TRACE("Section %d (%d of this segment) is%s code\n", *cursect, i, (RtlAreBitsSet(&fmap->sect_is_code, *cursect, 1) ? "" : " not")); } return 0; }
VOID DeallocatePort( PPORT_SET PortSet, ULONG Port ) { KIRQL OldIrql; Port = htons(Port); ASSERT(Port >= PortSet->StartingPort); ASSERT(Port < PortSet->StartingPort + PortSet->PortsToOversee); KeAcquireSpinLock( &PortSet->Lock, &OldIrql ); RtlClearBits( &PortSet->ProtoBitmap, Port - PortSet->StartingPort, 1 ); KeReleaseSpinLock( &PortSet->Lock, OldIrql ); }
// Build MSR bitmap _Use_decl_annotations_ static void *VmpBuildMsrBitmap() { PAGED_CODE(); const auto msr_bitmap = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, kHyperPlatformCommonPoolTag); if (!msr_bitmap) { return nullptr; } RtlZeroMemory(msr_bitmap, PAGE_SIZE); // Activate VM-exit for RDMSR against all MSRs const auto bitmap_read_low = reinterpret_cast<UCHAR *>(msr_bitmap); const auto bitmap_read_high = bitmap_read_low + 1024; RtlFillMemory(bitmap_read_low, 1024, 0xff); // read 0 - 1fff RtlFillMemory(bitmap_read_high, 1024, 0xff); // read c0000000 - c0001fff // Ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8) RTL_BITMAP bitmap_read_low_header = {}; RtlInitializeBitMap(&bitmap_read_low_header, reinterpret_cast<PULONG>(bitmap_read_low), 1024 * 8); RtlClearBits(&bitmap_read_low_header, 0xe7, 2); // Checks MSRs that cause #GP from 0 to 0xfff, and ignore all of them for (auto msr = 0ul; msr < 0x1000; ++msr) { __try { UtilReadMsr(static_cast<Msr>(msr)); } __except (EXCEPTION_EXECUTE_HANDLER) { RtlClearBits(&bitmap_read_low_header, msr, 1); } } // Ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102) RTL_BITMAP bitmap_read_high_header = {}; RtlInitializeBitMap(&bitmap_read_high_header, reinterpret_cast<PULONG>(bitmap_read_high), 1024 * CHAR_BIT); RtlClearBits(&bitmap_read_high_header, 0x101, 2); return msr_bitmap; }
/* * @implemented */ DWORD WINAPI TlsAlloc(VOID) { ULONG Index; RtlAcquirePebLock(); /* Try to get regular TEB slot. */ Index = RtlFindClearBitsAndSet(NtCurrentPeb()->TlsBitmap, 1, 0); if (Index == ~0U) { /* If it fails, try to find expansion TEB slot. */ Index = RtlFindClearBitsAndSet(NtCurrentPeb()->TlsExpansionBitmap, 1, 0); if (Index != ~0U) { if (NtCurrentTeb()->TlsExpansionSlots == NULL) { NtCurrentTeb()->TlsExpansionSlots = HeapAlloc(RtlGetProcessHeap(), HEAP_ZERO_MEMORY, TLS_EXPANSION_SLOTS * sizeof(PVOID)); } if (NtCurrentTeb()->TlsExpansionSlots == NULL) { RtlClearBits(NtCurrentPeb()->TlsExpansionBitmap, Index, 1); Index = ~0; SetLastError(ERROR_NOT_ENOUGH_MEMORY); } else { /* Clear the value. */ NtCurrentTeb()->TlsExpansionSlots[Index] = 0; Index += TLS_MINIMUM_AVAILABLE; } } else { SetLastError(ERROR_NO_MORE_ITEMS); } } else { /* Clear the value. */ NtCurrentTeb()->TlsSlots[Index] = 0; } RtlReleasePebLock(); return Index; }
static UCHAR XmsFree(WORD Handle) { DWORD BlockNumber; PXMS_HANDLE HandleEntry = GetHandleRecord(Handle); if (!ValidateHandle(HandleEntry)) return XMS_STATUS_INVALID_HANDLE; if (HandleEntry->LockCount) return XMS_STATUS_LOCKED; BlockNumber = (HandleEntry->Address - XMS_ADDRESS) / XMS_BLOCK_SIZE; RtlClearBits(&AllocBitmap, BlockNumber, HandleEntry->Size); HandleEntry->Handle = 0; FreeBlocks += HandleEntry->Size; return XMS_STATUS_SUCCESS; }
VOID XenevtchnReleaseIoMemory(PVOID va, ULONG nr_bytes) { KIRQL old_irql; ULONG page_nr; XM_ASSERT(((ULONG_PTR)va & (PAGE_SIZE - 1)) == 0); nr_bytes = (nr_bytes + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); page_nr = (ULONG)(((ULONG_PTR)va - (ULONG_PTR)io_hole_va_start) / PAGE_SIZE); old_irql = acquire_irqsafe_lock(&io_hole_lock); XM_ASSERT(RtlAreBitsSet(&io_hole_in_use, page_nr, nr_bytes / PAGE_SIZE)); RtlClearBits(&io_hole_in_use, page_nr, nr_bytes / PAGE_SIZE); release_irqsafe_lock(&io_hole_lock, old_irql); }
_Use_decl_annotations_ EXTERN_C static bool VminitpSetupVMCS( const PER_PROCESSOR_DATA *ProcessorData, ULONG_PTR GuestStackPointer, ULONG_PTR GuestInstructionPointer, ULONG_PTR VmmStackPointer) { unsigned char error = 0; GDTR gdtr = {}; __sgdt(&gdtr); IDTR idtr = {}; __sidt(&idtr); VMX_VM_ENTER_CONTROLS vmEnterCtlRequested = {}; vmEnterCtlRequested.Fields.IA32eModeGuest = true; VMX_VM_ENTER_CONTROLS vmEnterCtl = { VminitpAdjustControlValue(IA32_VMX_ENTRY_CTLS, vmEnterCtlRequested.All)}; VMX_VM_EXIT_CONTROLS vmExitCtlRequested = {}; vmExitCtlRequested.Fields.AcknowledgeInterruptOnExit = true; vmExitCtlRequested.Fields.HostAddressSpaceSize = true; VMX_VM_EXIT_CONTROLS vmExitCtl = { VminitpAdjustControlValue(IA32_VMX_EXIT_CTLS, vmExitCtlRequested.All)}; VMX_PIN_BASED_CONTROLS vmPinCtlRequested = {}; VMX_PIN_BASED_CONTROLS vmPinCtl = { VminitpAdjustControlValue(IA32_VMX_PINBASED_CTLS, vmPinCtlRequested.All)}; VMX_CPU_BASED_CONTROLS vmCpuCtlRequested = {}; vmCpuCtlRequested.Fields.RDTSCExiting = true; vmCpuCtlRequested.Fields.CR3LoadExiting = true; // MOV to CR3 vmCpuCtlRequested.Fields.CR8LoadExiting = true; // MOV to CR8 vmCpuCtlRequested.Fields.MovDRExiting = true; vmCpuCtlRequested.Fields.UseMSRBitmaps = true; vmCpuCtlRequested.Fields.ActivateSecondaryControl = true; VMX_CPU_BASED_CONTROLS vmCpuCtl = {VminitpAdjustControlValue( IA32_VMX_PROCBASED_CTLS, vmCpuCtlRequested.All)}; VMX_SECONDARY_CPU_BASED_CONTROLS vmCpuCtl2Requested = {}; vmCpuCtl2Requested.Fields.EnableRDTSCP = true; vmCpuCtl2Requested.Fields.DescriptorTableExiting = true; VMX_CPU_BASED_CONTROLS vmCpuCtl2 = {VminitpAdjustControlValue( IA32_VMX_PROCBASED_CTLS2, vmCpuCtl2Requested.All)}; // Set up the MSR bitmap // Activate VM-exit for RDMSR against all MSRs const auto bitMapReadLow = reinterpret_cast<UCHAR *>(ProcessorData->MsrBitmap); const auto bitMapReadHigh = bitMapReadLow + 1024; RtlFillMemory(bitMapReadLow, 1024, 0xff); // read 0 - 1fff RtlFillMemory(bitMapReadHigh, 1024, 0xff); // read c0000000 - c0001fff // But ignore IA32_MPERF (000000e7) and IA32_APERF (000000e8) RTL_BITMAP bitMapReadLowHeader = {}; RtlInitializeBitMap(&bitMapReadLowHeader, reinterpret_cast<PULONG>(bitMapReadLow), 1024 * 8); RtlClearBits(&bitMapReadLowHeader, 0xe7, 2); // But ignore IA32_GS_BASE (c0000101) and IA32_KERNEL_GS_BASE (c0000102) RTL_BITMAP bitMapReadHighHeader = {}; RtlInitializeBitMap(&bitMapReadHighHeader, reinterpret_cast<PULONG>(bitMapReadHigh), 1024 * 8); RtlClearBits(&bitMapReadHighHeader, 0x101, 2); const auto msrBitmapPA = MmGetPhysicalAddress(ProcessorData->MsrBitmap); // Set up CR0 and CR4 bitmaps // Where a bit is masked, the shadow bit appears // Where a bit is not masked, the actual bit appears CR0_REG cr0mask = {}; cr0mask.Fields.WP = true; CR4_REG cr4mask = {}; cr4mask.Fields.PGE = true; // clang-format off /* 16-Bit Control Field */ /* 16-Bit Guest-State Fields */ error |= __vmx_vmwrite(GUEST_ES_SELECTOR, AsmReadES()); error |= __vmx_vmwrite(GUEST_CS_SELECTOR, AsmReadCS()); error |= __vmx_vmwrite(GUEST_SS_SELECTOR, AsmReadSS()); error |= __vmx_vmwrite(GUEST_DS_SELECTOR, AsmReadDS()); error |= __vmx_vmwrite(GUEST_FS_SELECTOR, AsmReadFS()); error |= __vmx_vmwrite(GUEST_GS_SELECTOR, AsmReadGS()); error |= __vmx_vmwrite(GUEST_LDTR_SELECTOR, AsmReadLDTR()); error |= __vmx_vmwrite(GUEST_TR_SELECTOR, AsmReadTR()); /* 16-Bit Host-State Fields */ error |= __vmx_vmwrite(HOST_ES_SELECTOR, AsmReadES() & 0xf8); // RPL and TI error |= __vmx_vmwrite(HOST_CS_SELECTOR, AsmReadCS() & 0xf8); // have to be 0 error |= __vmx_vmwrite(HOST_SS_SELECTOR, AsmReadSS() & 0xf8); error |= __vmx_vmwrite(HOST_DS_SELECTOR, AsmReadDS() & 0xf8); error |= __vmx_vmwrite(HOST_FS_SELECTOR, AsmReadFS() & 0xf8); error |= __vmx_vmwrite(HOST_GS_SELECTOR, AsmReadGS() & 0xf8); error |= __vmx_vmwrite(HOST_TR_SELECTOR, AsmReadTR() & 0xf8); /* 64-Bit Control Fields */ error |= __vmx_vmwrite(IO_BITMAP_A, 0); error |= __vmx_vmwrite(IO_BITMAP_B, 0); error |= __vmx_vmwrite(MSR_BITMAP, msrBitmapPA.QuadPart); error |= __vmx_vmwrite(TSC_OFFSET, 0); /* 64-Bit Guest-State Fields */ error |= __vmx_vmwrite(VMCS_LINK_POINTER, 0xffffffffffffffff); error |= __vmx_vmwrite(GUEST_IA32_DEBUGCTL, __readmsr(IA32_DEBUGCTL)); /* 32-Bit Control Fields */ error |= __vmx_vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmPinCtl.All); error |= __vmx_vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmCpuCtl.All); error |= __vmx_vmwrite(SECONDARY_VM_EXEC_CONTROL, vmCpuCtl2.All); error |= __vmx_vmwrite(EXCEPTION_BITMAP, 0); error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MASK, 0); error |= __vmx_vmwrite(PAGE_FAULT_ERROR_CODE_MATCH, 0); error |= __vmx_vmwrite(CR3_TARGET_COUNT, 0); error |= __vmx_vmwrite(VM_EXIT_CONTROLS, vmExitCtl.All); error |= __vmx_vmwrite(VM_EXIT_MSR_STORE_COUNT, 0); error |= __vmx_vmwrite(VM_EXIT_MSR_LOAD_COUNT, 0); error |= __vmx_vmwrite(VM_ENTRY_CONTROLS, vmEnterCtl.All); error |= __vmx_vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0); error |= __vmx_vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0); /* 32-Bit Guest-State Fields */ error |= __vmx_vmwrite(GUEST_ES_LIMIT, GetSegmentLimit(AsmReadES())); error |= __vmx_vmwrite(GUEST_CS_LIMIT, GetSegmentLimit(AsmReadCS())); error |= __vmx_vmwrite(GUEST_SS_LIMIT, GetSegmentLimit(AsmReadSS())); error |= __vmx_vmwrite(GUEST_DS_LIMIT, GetSegmentLimit(AsmReadDS())); error |= __vmx_vmwrite(GUEST_FS_LIMIT, GetSegmentLimit(AsmReadFS())); error |= __vmx_vmwrite(GUEST_GS_LIMIT, GetSegmentLimit(AsmReadGS())); error |= __vmx_vmwrite(GUEST_LDTR_LIMIT, GetSegmentLimit(AsmReadLDTR())); error |= __vmx_vmwrite(GUEST_TR_LIMIT, GetSegmentLimit(AsmReadTR())); error |= __vmx_vmwrite(GUEST_GDTR_LIMIT, gdtr.Limit); error |= __vmx_vmwrite(GUEST_IDTR_LIMIT, idtr.Limit); error |= __vmx_vmwrite(GUEST_ES_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadES())); error |= __vmx_vmwrite(GUEST_CS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadCS())); error |= __vmx_vmwrite(GUEST_SS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadSS())); error |= __vmx_vmwrite(GUEST_DS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadDS())); error |= __vmx_vmwrite(GUEST_FS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadFS())); error |= __vmx_vmwrite(GUEST_GS_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadGS())); error |= __vmx_vmwrite(GUEST_LDTR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadLDTR())); error |= __vmx_vmwrite(GUEST_TR_AR_BYTES, VminitpGetSegmentAccessRight(AsmReadTR())); error |= __vmx_vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0); error |= __vmx_vmwrite(GUEST_ACTIVITY_STATE, 0); error |= __vmx_vmwrite(GUEST_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS)); /* 32-Bit Host-State Field */ error |= __vmx_vmwrite(HOST_IA32_SYSENTER_CS, __readmsr(IA32_SYSENTER_CS)); /* Natural-Width Control Fields */ error |= __vmx_vmwrite(CR0_GUEST_HOST_MASK, cr0mask.All); error |= __vmx_vmwrite(CR4_GUEST_HOST_MASK, cr4mask.All); error |= __vmx_vmwrite(CR0_READ_SHADOW, __readcr0()); error |= __vmx_vmwrite(CR4_READ_SHADOW, __readcr4()); error |= __vmx_vmwrite(CR3_TARGET_VALUE0, 0); error |= __vmx_vmwrite(CR3_TARGET_VALUE1, 0); error |= __vmx_vmwrite(CR3_TARGET_VALUE2, 0); error |= __vmx_vmwrite(CR3_TARGET_VALUE3, 0); /* Natural-Width Guest-State Fields */ error |= __vmx_vmwrite(GUEST_CR0, __readcr0()); error |= __vmx_vmwrite(GUEST_CR3, __readcr3()); error |= __vmx_vmwrite(GUEST_CR4, __readcr4()); error |= __vmx_vmwrite(GUEST_ES_BASE, 0); error |= __vmx_vmwrite(GUEST_CS_BASE, 0); error |= __vmx_vmwrite(GUEST_SS_BASE, 0); error |= __vmx_vmwrite(GUEST_DS_BASE, 0); error |= __vmx_vmwrite(GUEST_FS_BASE, __readmsr(IA32_FS_BASE)); error |= __vmx_vmwrite(GUEST_GS_BASE, __readmsr(IA32_GS_BASE)); error |= __vmx_vmwrite(GUEST_LDTR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadLDTR())); error |= __vmx_vmwrite(GUEST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR())); error |= __vmx_vmwrite(GUEST_GDTR_BASE, gdtr.Address); error |= __vmx_vmwrite(GUEST_IDTR_BASE, idtr.Address); error |= __vmx_vmwrite(GUEST_DR7, __readdr(7)); error |= __vmx_vmwrite(GUEST_RSP, GuestStackPointer); error |= __vmx_vmwrite(GUEST_RIP, GuestInstructionPointer); error |= __vmx_vmwrite(GUEST_RFLAGS, __readeflags()); error |= __vmx_vmwrite(GUEST_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP)); error |= __vmx_vmwrite(GUEST_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP)); /* Natural-Width Host-State Fields */ error |= __vmx_vmwrite(HOST_CR0, __readcr0()); error |= __vmx_vmwrite(HOST_CR3, __readcr3()); error |= __vmx_vmwrite(HOST_CR4, __readcr4()); error |= __vmx_vmwrite(HOST_FS_BASE, __readmsr(IA32_FS_BASE)); error |= __vmx_vmwrite(HOST_GS_BASE, __readmsr(IA32_GS_BASE)); error |= __vmx_vmwrite(HOST_TR_BASE, VminitpGetSegmentBase(gdtr.Address, AsmReadTR())); error |= __vmx_vmwrite(HOST_GDTR_BASE, gdtr.Address); error |= __vmx_vmwrite(HOST_IDTR_BASE, idtr.Address); error |= __vmx_vmwrite(HOST_IA32_SYSENTER_ESP, __readmsr(IA32_SYSENTER_ESP)); error |= __vmx_vmwrite(HOST_IA32_SYSENTER_EIP, __readmsr(IA32_SYSENTER_EIP)); error |= __vmx_vmwrite(HOST_RSP, VmmStackPointer); error |= __vmx_vmwrite(HOST_RIP, reinterpret_cast<size_t>(AsmVmmEntryPoint)); // clang-format on const auto vmxStatus = static_cast<VMX_STATUS>(error); return vmxStatus == VMX_OK; }
ULONG NTAPI MiFreePoolPages(IN PVOID StartingVa) { PMMPTE PointerPte, StartPte; PMMPFN Pfn1, StartPfn; PFN_NUMBER FreePages, NumberOfPages; KIRQL OldIrql; PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry; ULONG i, End; // // Handle paged pool // if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd)) { // // Calculate the offset from the beginning of paged pool, and convert it // into pages // i = ((ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart) >> PAGE_SHIFT; End = i; // // Now use the end bitmap to scan until we find a set bit, meaning that // this allocation finishes here // while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++; // // Now calculate the total number of pages this allocation spans // NumberOfPages = End - i + 1; /* Delete the actual pages */ PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i; FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL); ASSERT(FreePages == NumberOfPages); // // Acquire the paged pool lock // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Clear the allocation and free bits // RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End); RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages); // // Update the hint if we need to // if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i; // // Release the lock protecting the bitmaps // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // And finally return the number of pages freed // return NumberOfPages; }
VOID IoFreeMapRegisters( PADAPTER_OBJECT AdapterObject, PVOID MapRegisterBase, ULONG NumberOfMapRegisters ) /*++ Routine Description: This routine deallocates the map registers for the adapter. If there are any queued adapter waiting for an attempt is made to allocate the next entry. Arguments: AdapterObject - The adapter object to where the map register should be returned. MapRegisterBase - The map register base of the registers to be deallocated. NumberOfMapRegisters - The number of registers to be deallocated. Return Value: None --+*/ { PADAPTER_OBJECT MasterAdapter; LONG MapRegisterNumber; PWAIT_CONTEXT_BLOCK Wcb; PLIST_ENTRY Packet; IO_ALLOCATION_ACTION Action; KIRQL Irql; // // Begin by getting the address of the master adapter. // if (AdapterObject->MasterAdapter != NULL && MapRegisterBase != NULL) { MasterAdapter = AdapterObject->MasterAdapter; } else { // // There are no map registers to return. // return; } // // Strip no scatter/gather flag. // MapRegisterBase = (PVOID) ((ULONG) MapRegisterBase & ~NO_SCATTER_GATHER); MapRegisterNumber = (PTRANSLATION_ENTRY) MapRegisterBase - (PTRANSLATION_ENTRY) MasterAdapter->MapRegisterBase; // // Acquire the master adapter spinlock which locks the adapter queue and the // bit map for the map registers. // Irql = KfAcquireSpinLock(&MasterAdapter->SpinLock); // // Return the registers to the bit map. // RtlClearBits( MasterAdapter->MapRegisters, MapRegisterNumber, NumberOfMapRegisters ); // // Process any requests waiting for map registers in the adapter queue. // Requests are processed until a request cannot be satisfied or until // there are no more requests in the queue. // while(TRUE) { if ( IsListEmpty(&MasterAdapter->AdapterQueue) ){ break; } Packet = RemoveHeadList( &MasterAdapter->AdapterQueue ); AdapterObject = CONTAINING_RECORD( Packet, ADAPTER_OBJECT, AdapterQueue ); Wcb = AdapterObject->CurrentWcb; // // Attempt to allocate map registers for this request. Use the previous // register base as a hint. // MapRegisterNumber = RtlFindClearBitsAndSet( MasterAdapter->MapRegisters, AdapterObject->NumberOfMapRegisters, MasterAdapter->NumberOfMapRegisters ); if (MapRegisterNumber == -1) { // // There were not enough free map registers. Put this request back on // the adapter queue where is came from. // InsertHeadList( &MasterAdapter->AdapterQueue, &AdapterObject->AdapterQueue ); break; } KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql ); AdapterObject->MapRegisterBase = (PVOID) ((PTRANSLATION_ENTRY) MasterAdapter->MapRegisterBase + MapRegisterNumber); // // Set the no scatter/gather flag if scatter/gather not // supported. // if (!AdapterObject->ScatterGather) { AdapterObject->MapRegisterBase = (PVOID) ((ULONG) AdapterObject->MapRegisterBase | NO_SCATTER_GATHER); } // // Invoke the driver's execution routine now. // Action = Wcb->DeviceRoutine( Wcb->DeviceObject, Wcb->CurrentIrp, AdapterObject->MapRegisterBase, Wcb->DeviceContext ); // // If the driver wishes to keep the map registers then set the number // allocated to zero and set the action to deallocate object. // if (Action == DeallocateObjectKeepRegisters) { AdapterObject->NumberOfMapRegisters = 0; Action = DeallocateObject; } // // If the driver would like to have the adapter deallocated, // then deallocate any map registers allocated and then release // the adapter object. // if (Action == DeallocateObject) { // // The map registers registers are deallocated here rather than in // IoFreeAdapterChannel. This limits the number of times // this routine can be called recursively possibly overflowing // the stack. The worst case occurs if there is a pending // request for the adapter that uses map registers and whos // excution routine decallocates the adapter. In that case if there // are no requests in the master adapter queue, then IoFreeMapRegisters // will get called again. // if (AdapterObject->NumberOfMapRegisters != 0) { // // Deallocate the map registers and clear the count so that // IoFreeAdapterChannel will not deallocate them again. // Irql = KfAcquireSpinLock( &MasterAdapter->SpinLock ); RtlClearBits( MasterAdapter->MapRegisters, MapRegisterNumber, AdapterObject->NumberOfMapRegisters ); AdapterObject->NumberOfMapRegisters = 0; KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql ); } IoFreeAdapterChannel( AdapterObject ); } Irql = KfAcquireSpinLock( &MasterAdapter->SpinLock ); } KfReleaseSpinLock( &MasterAdapter->SpinLock, Irql ); }
VOID HvFreeHivePartial( PHHIVE Hive, HCELL_INDEX Start, HSTORAGE_TYPE Type ) /*++ Routine Description: Free the memory and associated maps for the end of a hive starting at Start. The baseblock, hive, etc will not be touched. Arguments: Hive - supplies a pointer to hive control structure for hive to partially free. Start - HCELL_INDEX of first bin to free, will free from this bin (inclusive) to the end of the hives stable storage. Type - Type of storage (Stable or Volatile) to be freed. Return Value: NONE. --*/ { PHMAP_DIRECTORY Dir; PHMAP_ENTRY Me; HCELL_INDEX Address; ULONG StartTable; ULONG Length; PHBIN Bin; ULONG Tables; ULONG FirstBit; ULONG LastBit; PFREE_HBIN FreeBin; ASSERT(Hive->Flat == FALSE); ASSERT(Hive->ReadOnly == FALSE); Address = Start; Length = Hive->Storage[Type].Length; ASSERT(Address <= Length); if (Address == Length) { return; } // // Sweep through bin set // do { Me = HvpGetCellMap(Hive, Address + (Type*HCELL_TYPE_MASK)); VALIDATE_CELL_MAP(__LINE__,Me,Hive,Address + (Type*HCELL_TYPE_MASK)); if (Me->BinAddress & HMAP_DISCARDABLE) { FreeBin = (PFREE_HBIN)Me->BlockAddress; if (FreeBin->Flags & FREE_HBIN_DISCARDABLE) { CmpFree((PVOID)HBIN_BASE(Me->BinAddress), FreeBin->Size); } else { // // The bin has been freed, but quota is still charged. // Since the file will now shrink, the quota must be // returned here. // if( Me->BinAddress & HMAP_INPAGEDPOOL) { // // we charge quota only for bins in paged-pool // CmpReleaseGlobalQuota(FreeBin->Size); } } RemoveEntryList(&FreeBin->ListEntry); Address += FreeBin->Size; CmpFree(FreeBin, sizeof(FREE_HBIN)); } else { Bin = (PHBIN)HBIN_BASE(Me->BinAddress); Address += HvpGetBinMemAlloc(Hive,Bin,Type); if( Me->BinAddress & HMAP_INPAGEDPOOL && HvpGetBinMemAlloc(Hive,Bin,Type) ) { // // free the bin only if it is allocated from paged pool // CmpFree(Bin, HvpGetBinMemAlloc(Hive,Bin,Type)); } } } while (Address < Length); // // Free map table storage // Tables = (((Hive->Storage[Type].Length) / HBLOCK_SIZE) - 1) / HTABLE_SLOTS; Dir = Hive->Storage[Type].Map; if (Start > 0) { StartTable = ((Start-1) / HBLOCK_SIZE) / HTABLE_SLOTS; } else { StartTable = (ULONG)-1; } HvpFreeMap(Hive, Dir, StartTable+1, Tables); // // update hysteresis (eventually queue work item) // if( Type == Stable) { CmpUpdateSystemHiveHysteresis(Hive,(Start&(~HCELL_TYPE_MASK)),Hive->Storage[Type].Length); } Hive->Storage[Type].Length = (Start&(~HCELL_TYPE_MASK)); if (Type==Stable) { // // Clear dirty vector for data past Hive->Storage[Stable].Length // FirstBit = Start / HSECTOR_SIZE; LastBit = Hive->DirtyVector.SizeOfBitMap; ASSERT(Hive->DirtyCount == RtlNumberOfSetBits(&Hive->DirtyVector)); RtlClearBits(&Hive->DirtyVector, FirstBit, LastBit-FirstBit); Hive->DirtyCount = RtlNumberOfSetBits(&Hive->DirtyVector); } HvpAdjustHiveFreeDisplay(Hive,Hive->Storage[Type].Length,Type); return; }
ULONG NTAPI MiFreePoolPages(IN PVOID StartingVa) { PMMPTE PointerPte, StartPte; PMMPFN Pfn1, StartPfn; PFN_COUNT FreePages, NumberOfPages; KIRQL OldIrql; PMMFREE_POOL_ENTRY FreeEntry, NextEntry, LastEntry; ULONG i, End; ULONG_PTR Offset; // // Handle paged pool // if ((StartingVa >= MmPagedPoolStart) && (StartingVa <= MmPagedPoolEnd)) { // // Calculate the offset from the beginning of paged pool, and convert it // into pages // Offset = (ULONG_PTR)StartingVa - (ULONG_PTR)MmPagedPoolStart; i = (ULONG)(Offset >> PAGE_SHIFT); End = i; // // Now use the end bitmap to scan until we find a set bit, meaning that // this allocation finishes here // while (!RtlTestBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End)) End++; // // Now calculate the total number of pages this allocation spans. If it's // only one page, add it to the S-LIST instead of freeing it // NumberOfPages = End - i + 1; if ((NumberOfPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead) < MiPagedPoolSListMaximum)) { InterlockedPushEntrySList(&MiPagedPoolSListHead, StartingVa); return 1; } /* Delete the actual pages */ PointerPte = MmPagedPoolInfo.FirstPteForPagedPool + i; FreePages = MiDeleteSystemPageableVm(PointerPte, NumberOfPages, 0, NULL); ASSERT(FreePages == NumberOfPages); // // Acquire the paged pool lock // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Clear the allocation and free bits // RtlClearBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, End); RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, i, NumberOfPages); // // Update the hint if we need to // if (i < MmPagedPoolInfo.PagedPoolHint) MmPagedPoolInfo.PagedPoolHint = i; // // Release the lock protecting the bitmaps // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // And finally return the number of pages freed // return NumberOfPages; }
ULONG FASTCALL MiReleasePageFileSpace ( IN MMPTE PteContents ) /*++ Routine Description: This routine frees the paging file allocated to the specified PTE and adjusts the necessary quotas. Arguments: PteContents - Supplies the PTE which is in page file format. Return Value: Returns TRUE if any paging file space was deallocated. Environment: Kernel mode, APCs disabled, PFN lock held. --*/ { ULONG FreeBit; ULONG PageFileNumber; MM_PFN_LOCK_ASSERT(); if (PteContents.u.Soft.Prototype == 1) { // // Not in page file format. // return FALSE; } FreeBit = GET_PAGING_FILE_OFFSET (PteContents); if ((FreeBit == 0) || (FreeBit == 0xFFFFF)) { // // Page is not in a paging file, just return. // return FALSE; } PageFileNumber = GET_PAGING_FILE_NUMBER (PteContents); ASSERT (RtlCheckBit( MmPagingFile[PageFileNumber]->Bitmap, FreeBit) == 1); #if DBG if ((FreeBit < 8192) && (PageFileNumber == 0)) { ASSERT ((MmPagingFileDebug[FreeBit] & 1) != 0); MmPagingFileDebug[FreeBit] &= 0xfffffffe; } #endif //DBG RtlClearBits ( MmPagingFile[PageFileNumber]->Bitmap, FreeBit, 1); MmPagingFile[PageFileNumber]->FreeSpace += 1; MmPagingFile[PageFileNumber]->CurrentUsage -= 1; // // Check to see if we should move some MDL entries for the // modified page writer now that more free space is available. // if ((MmNumberOfActiveMdlEntries == 0) || (MmPagingFile[PageFileNumber]->FreeSpace == MM_USABLE_PAGES_FREE)) { MiUpdateModifiedWriterMdls (PageFileNumber); } return TRUE; }
NTSTATUS FatSetFsLabelInfo ( IN PIRP_CONTEXT IrpContext, IN PVCB Vcb, IN PFILE_FS_LABEL_INFORMATION Buffer ) /*++ Routine Description: This routine implements the set volume label call Arguments: Vcb - Supplies the Vcb being queried Buffer - Supplies the input where the information is stored. Return Value: NTSTATUS - Returns the status for the operation --*/ { NTSTATUS Status; PDIRENT Dirent; PBCB DirentBcb = NULL; ULONG ByteOffset; WCHAR TmpBuffer[11]; UCHAR OemBuffer[11]; OEM_STRING OemLabel; UNICODE_STRING UnicodeString; UNICODE_STRING UpcasedLabel; DebugTrace(+1, Dbg, "FatSetFsLabelInfo...\n", 0); // // Setup our local variable // UnicodeString.Length = (USHORT)Buffer->VolumeLabelLength; UnicodeString.MaximumLength = UnicodeString.Length; UnicodeString.Buffer = (PWSTR) &Buffer->VolumeLabel[0]; // // Make sure the name can fit into the stack buffer // if ( UnicodeString.Length > 11*sizeof(WCHAR) ) { return STATUS_INVALID_VOLUME_LABEL; } // // Upcase the name and convert it to the Oem code page. // OemLabel.Buffer = &OemBuffer[0]; OemLabel.Length = 0; OemLabel.MaximumLength = 11; Status = RtlUpcaseUnicodeStringToCountedOemString( &OemLabel, &UnicodeString, FALSE ); // // Volume label that fits in 11 unicode character length limit // is not necessary within 11 characters in OEM character set. // if (!NT_SUCCESS( Status )) { DebugTrace(-1, Dbg, "FatSetFsLabelInfo: Label must be too long. %08lx\n", Status ); return STATUS_INVALID_VOLUME_LABEL; } // // Strip spaces off of the label. // if (OemLabel.Length > 0) { USHORT i; USHORT LastSpaceIndex = MAXUSHORT; // // Check the label for illegal characters // for ( i = 0; i < (ULONG)OemLabel.Length; i += 1 ) { if ( FsRtlIsLeadDbcsCharacter( OemLabel.Buffer[i] ) ) { LastSpaceIndex = MAXUSHORT; i += 1; continue; } if (!FsRtlIsAnsiCharacterLegalFat(OemLabel.Buffer[i], FALSE) || (OemLabel.Buffer[i] == '.')) { return STATUS_INVALID_VOLUME_LABEL; } // // Watch for the last run of spaces, so we can strip them. // if (OemLabel.Buffer[i] == ' ' && LastSpaceIndex == MAXUSHORT) { LastSpaceIndex = i; } else { LastSpaceIndex = MAXUSHORT; } } if (LastSpaceIndex != MAXUSHORT) { OemLabel.Length = LastSpaceIndex; } } // // Get the Unicode upcased string to store in the VPB. // UpcasedLabel.Length = UnicodeString.Length; UpcasedLabel.MaximumLength = 11*sizeof(WCHAR); UpcasedLabel.Buffer = &TmpBuffer[0]; Status = RtlOemStringToCountedUnicodeString( &UpcasedLabel, &OemLabel, FALSE ); if (!NT_SUCCESS( Status )) { DebugTrace(-1, Dbg, "FatSetFsLabelInfo: Label must be too long. %08lx\n", Status ); return STATUS_INVALID_VOLUME_LABEL; } DirentBcb = NULL; // // Make this look like a write through to disk. This is important to // avoid a unpleasant window where it looks like we have the wrong volume. // SetFlag( IrpContext->Flags, IRP_CONTEXT_FLAG_WRITE_THROUGH ); try { // // Are we setting or removing the label? Note that shaving spaces could // make this different than wondering if the input buffer is non-zero length. // if (OemLabel.Length > 0) { // // Locate the volume label if there already is one // FatLocateVolumeLabel( IrpContext, Vcb, &Dirent, &DirentBcb, &ByteOffset ); // // Check that we really got one, if not then we need to create // a new one. The procedure we call will raise an appropriate // status if we are not able to allocate a new dirent // if (Dirent == NULL) { ByteOffset = FatCreateNewDirent( IrpContext, Vcb->RootDcb, 1 ); FatPrepareWriteDirectoryFile( IrpContext, Vcb->RootDcb, ByteOffset, sizeof(DIRENT), &DirentBcb, &Dirent, FALSE, TRUE, &Status ); ASSERT( NT_SUCCESS( Status )); } else { // // Just mark this guy dirty now. // FatSetDirtyBcb( IrpContext, DirentBcb, Vcb, TRUE ); } // // Now reconstruct the volume label dirent. // FatConstructLabelDirent( IrpContext, Dirent, &OemLabel ); // // Unpin the Bcb here so that we will get any IO errors // here before changing the VPB label. // FatUnpinBcb( IrpContext, DirentBcb ); FatUnpinRepinnedBcbs( IrpContext ); // // Now set the upcased label in the VPB // RtlCopyMemory( &Vcb->Vpb->VolumeLabel[0], &UpcasedLabel.Buffer[0], UpcasedLabel.Length ); Vcb->Vpb->VolumeLabelLength = UpcasedLabel.Length; } else { // // Otherwise we're trying to delete the label // Locate the current volume label if there already is one // FatLocateVolumeLabel( IrpContext, Vcb, &Dirent, &DirentBcb, &ByteOffset ); // // Check that we really got one // if (Dirent == NULL) { try_return( Status = STATUS_SUCCESS ); } // // Now delete the current label. // Dirent->FileName[0] = FAT_DIRENT_DELETED; ASSERT( (Vcb->RootDcb->Specific.Dcb.UnusedDirentVbo == 0xffffffff) || RtlAreBitsSet( &Vcb->RootDcb->Specific.Dcb.FreeDirentBitmap, ByteOffset / sizeof(DIRENT), 1 ) ); RtlClearBits( &Vcb->RootDcb->Specific.Dcb.FreeDirentBitmap, ByteOffset / sizeof(DIRENT), 1 ); FatSetDirtyBcb( IrpContext, DirentBcb, Vcb, TRUE ); // // Unpin the Bcb here so that we will get any IO errors // here before changing the VPB label. // FatUnpinBcb( IrpContext, DirentBcb ); FatUnpinRepinnedBcbs( IrpContext ); // // Now set the label in the VPB // Vcb->Vpb->VolumeLabelLength = 0; } Status = STATUS_SUCCESS; FatSortDirectory(IrpContext, Vcb->RootDcb); try_exit: NOTHING; } finally { DebugUnwind( FatSetFsALabelInfo ); FatUnpinBcb( IrpContext, DirentBcb ); DebugTrace(-1, Dbg, "FatSetFsALabelInfo -> STATUS_SUCCESS\n", 0); } return Status; }
PVOID NTAPI MiAllocatePoolPages(IN POOL_TYPE PoolType, IN SIZE_T SizeInBytes) { PFN_NUMBER PageFrameNumber; PFN_COUNT SizeInPages, PageTableCount; ULONG i; KIRQL OldIrql; PLIST_ENTRY NextEntry, NextHead, LastHead; PMMPTE PointerPte, StartPte; PMMPDE PointerPde; ULONG EndAllocation; MMPTE TempPte; MMPDE TempPde; PMMPFN Pfn1; PVOID BaseVa, BaseVaStart; PMMFREE_POOL_ENTRY FreeEntry; PKSPIN_LOCK_QUEUE LockQueue; // // Figure out how big the allocation is in pages // SizeInPages = (PFN_COUNT)BYTES_TO_PAGES(SizeInBytes); // // Check for overflow // if (SizeInPages == 0) { // // Fail // return NULL; } // // Handle paged pool // if ((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) { // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Lock the paged pool mutex // KeAcquireGuardedMutex(&MmPagedPoolMutex); // // Find some empty allocation space // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, MmPagedPoolInfo.PagedPoolHint); if (i == 0xFFFFFFFF) { // // Get the page bit count // i = ((SizeInPages - 1) / PTE_COUNT) + 1; DPRINT("Paged pool expansion: %lu %x\n", i, SizeInPages); // // Check if there is enougn paged pool expansion space left // if (MmPagedPoolInfo.NextPdeForPagedPoolExpansion > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } // // Check if we'll have to expand past the last PTE we have available // if (((i - 1) + MmPagedPoolInfo.NextPdeForPagedPoolExpansion) > (PMMPDE)MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool)) { // // We can only support this much then // PointerPde = MiAddressToPte(MmPagedPoolInfo.LastPteForPagedPool); PageTableCount = (PFN_COUNT)(PointerPde + 1 - MmPagedPoolInfo.NextPdeForPagedPoolExpansion); ASSERT(PageTableCount < i); i = PageTableCount; } else { // // Otherwise, there is plenty of space left for this expansion // PageTableCount = i; } // // Get the template PDE we'll use to expand // TempPde = ValidKernelPde; // // Get the first PTE in expansion space // PointerPde = MmPagedPoolInfo.NextPdeForPagedPoolExpansion; BaseVa = MiPdeToPte(PointerPde); BaseVaStart = BaseVa; // // Lock the PFN database and loop pages // OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); do { // // It should not already be valid // ASSERT(PointerPde->u.Hard.Valid == 0); /* Request a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPde.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) /* On PAE/x64 systems, there's no double-buffering */ ASSERT(FALSE); #else // // Save it into our double-buffered system page directory // MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)] = TempPde; /* Initialize the PFN */ MiInitializePfnForOtherProcess(PageFrameNumber, (PMMPTE)PointerPde, MmSystemPageDirectory[(PointerPde - MiAddressToPde(NULL)) / PDE_COUNT]); /* Write the actual PDE now */ // MI_WRITE_VALID_PDE(PointerPde, TempPde); #endif // // Move on to the next expansion address // PointerPde++; BaseVa = (PVOID)((ULONG_PTR)BaseVa + PAGE_SIZE); i--; } while (i > 0); // // Release the PFN database lock // KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); // // These pages are now available, clear their availablity bits // EndAllocation = (ULONG)(MmPagedPoolInfo.NextPdeForPagedPoolExpansion - (PMMPDE)MiAddressToPte(MmPagedPoolInfo.FirstPteForPagedPool)) * PTE_COUNT; RtlClearBits(MmPagedPoolInfo.PagedPoolAllocationMap, EndAllocation, PageTableCount * PTE_COUNT); // // Update the next expansion location // MmPagedPoolInfo.NextPdeForPagedPoolExpansion += PageTableCount; // // Zero out the newly available memory // RtlZeroMemory(BaseVaStart, PageTableCount * PAGE_SIZE); // // Now try consuming the pages again // i = RtlFindClearBitsAndSet(MmPagedPoolInfo.PagedPoolAllocationMap, SizeInPages, 0); if (i == 0xFFFFFFFF) { // // Out of memory! // DPRINT1("OUT OF PAGED POOL!!!\n"); KeReleaseGuardedMutex(&MmPagedPoolMutex); return NULL; } } // // Update the pool hint if the request was just one page // if (SizeInPages == 1) MmPagedPoolInfo.PagedPoolHint = i + 1; // // Update the end bitmap so we know the bounds of this allocation when // the time comes to free it // EndAllocation = i + SizeInPages - 1; RtlSetBit(MmPagedPoolInfo.EndOfPagedPoolBitmap, EndAllocation); // // Now we can release the lock (it mainly protects the bitmap) // KeReleaseGuardedMutex(&MmPagedPoolMutex); // // Now figure out where this allocation starts // BaseVa = (PVOID)((ULONG_PTR)MmPagedPoolStart + (i << PAGE_SHIFT)); // // Flush the TLB // KeFlushEntireTb(TRUE, TRUE); /* Setup a demand-zero writable PTE */ MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE); // // Find the first and last PTE, then loop them all // PointerPte = MiAddressToPte(BaseVa); StartPte = PointerPte + SizeInPages; do { // // Write the demand zero PTE and keep going // MI_WRITE_INVALID_PTE(PointerPte, TempPte); } while (++PointerPte < StartPte); // // Return the allocation address to the caller // return BaseVa; } // // If only one page is being requested, try to grab it from the S-LIST // if ((SizeInPages == 1) && (ExQueryDepthSList(&MiNonPagedPoolSListHead))) { BaseVa = InterlockedPopEntrySList(&MiNonPagedPoolSListHead); if (BaseVa) return BaseVa; } // // Allocations of less than 4 pages go into their individual buckets // i = SizeInPages - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; // // Loop through all the free page lists based on the page index // NextHead = &MmNonPagedPoolFreeListHead[i]; LastHead = &MmNonPagedPoolFreeListHead[MI_MAX_FREE_PAGE_LISTS]; // // Acquire the nonpaged pool lock // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); do { // // Now loop through all the free page entries in this given list // NextEntry = NextHead->Flink; while (NextEntry != NextHead) { /* Is freed non paged pool enabled */ if (MmProtectFreedNonPagedPool) { /* We need to be able to touch this page, unprotect it */ MiUnProtectFreeNonPagedPool(NextEntry, 0); } // // Grab the entry and see if it can handle our allocation // FreeEntry = CONTAINING_RECORD(NextEntry, MMFREE_POOL_ENTRY, List); ASSERT(FreeEntry->Signature == MM_FREE_POOL_SIGNATURE); if (FreeEntry->Size >= SizeInPages) { // // It does, so consume the pages from here // FreeEntry->Size -= SizeInPages; // // The allocation will begin in this free page area // BaseVa = (PVOID)((ULONG_PTR)FreeEntry + (FreeEntry->Size << PAGE_SHIFT)); /* Remove the item from the list, depending if pool is protected */ if (MmProtectFreedNonPagedPool) MiProtectedPoolRemoveEntryList(&FreeEntry->List); else RemoveEntryList(&FreeEntry->List); // // However, check if its' still got space left // if (FreeEntry->Size != 0) { /* Check which list to insert this entry into */ i = FreeEntry->Size - 1; if (i >= MI_MAX_FREE_PAGE_LISTS) i = MI_MAX_FREE_PAGE_LISTS - 1; /* Insert the entry into the free list head, check for prot. pool */ if (MmProtectFreedNonPagedPool) MiProtectedPoolInsertList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List, TRUE); else InsertTailList(&MmNonPagedPoolFreeListHead[i], &FreeEntry->List); /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } // // Grab the PTE for this allocation // PointerPte = MiAddressToPte(BaseVa); ASSERT(PointerPte->u.Hard.Valid == 1); // // Grab the PFN NextEntry and index // Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); // // Now mark it as the beginning of an allocation // ASSERT(Pfn1->u3.e1.StartOfAllocation == 0); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as special pool if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) { Pfn1->u4.VerifierAllocation = 1; } // // Check if the allocation is larger than one page // if (SizeInPages != 1) { // // Navigate to the last PFN entry and PTE // PointerPte += SizeInPages - 1; ASSERT(PointerPte->u.Hard.Valid == 1); Pfn1 = MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber); } // // Mark this PFN as the last (might be the same as the first) // ASSERT(Pfn1->u3.e1.EndOfAllocation == 0); Pfn1->u3.e1.EndOfAllocation = 1; // // Release the nonpaged pool lock, and return the allocation // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); return BaseVa; } // // Try the next free page entry // NextEntry = FreeEntry->List.Flink; /* Is freed non paged pool protected? */ if (MmProtectFreedNonPagedPool) { /* Protect the freed pool! */ MiProtectFreeNonPagedPool(FreeEntry, FreeEntry->Size); } } } while (++NextHead < LastHead); // // If we got here, we're out of space. // Start by releasing the lock // KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Allocate some system PTEs // StartPte = MiReserveSystemPtes(SizeInPages, NonPagedPoolExpansion); PointerPte = StartPte; if (StartPte == NULL) { // // Ran out of memory // DPRINT1("Out of NP Expansion Pool\n"); return NULL; } // // Acquire the pool lock now // OldIrql = KeAcquireQueuedSpinLock(LockQueueMmNonPagedPoolLock); // // Lock the PFN database too // LockQueue = &KeGetCurrentPrcb()->LockQueue[LockQueuePfnLock]; KeAcquireQueuedSpinLockAtDpcLevel(LockQueue); // // Loop the pages // TempPte = ValidKernelPte; do { /* Allocate a page */ MI_SET_USAGE(MI_USAGE_PAGED_POOL); MI_SET_PROCESS2("Kernel"); PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); Pfn1->u3.e2.ReferenceCount = 1; Pfn1->u2.ShareCount = 1; Pfn1->PteAddress = PointerPte; Pfn1->u3.e1.PageLocation = ActiveAndValid; Pfn1->u4.VerifierAllocation = 0; /* Write the PTE for it */ TempPte.u.Hard.PageFrameNumber = PageFrameNumber; MI_WRITE_VALID_PTE(PointerPte++, TempPte); } while (--SizeInPages > 0); // // This is the last page // Pfn1->u3.e1.EndOfAllocation = 1; // // Get the first page and mark it as such // Pfn1 = MiGetPfnEntry(StartPte->u.Hard.PageFrameNumber); Pfn1->u3.e1.StartOfAllocation = 1; /* Mark it as a verifier allocation if needed */ ASSERT(Pfn1->u4.VerifierAllocation == 0); if (PoolType & VERIFIER_POOL_MASK) Pfn1->u4.VerifierAllocation = 1; // // Release the PFN and nonpaged pool lock // KeReleaseQueuedSpinLockFromDpcLevel(LockQueue); KeReleaseQueuedSpinLock(LockQueueMmNonPagedPoolLock, OldIrql); // // Return the address // return MiPteToAddress(StartPte); }
VOID NtdllBitmap::ClearBits( ULONG starting_index, ULONG number_to_clear ) { assert(RtlClearBits != NULL); RtlClearBits(this, starting_index, number_to_clear); }
static UCHAR XmsRealloc(WORD Handle, WORD NewSize) { DWORD BlockNumber; PXMS_HANDLE HandleEntry = GetHandleRecord(Handle); DWORD CurrentIndex = 0; ULONG RunStart; ULONG RunSize; if (!ValidateHandle(HandleEntry)) return XMS_STATUS_INVALID_HANDLE; if (HandleEntry->LockCount) return XMS_STATUS_LOCKED; /* Get the block number */ BlockNumber = (HandleEntry->Address - XMS_ADDRESS) / XMS_BLOCK_SIZE; if (NewSize < HandleEntry->Size) { /* Just reduce the size of this block */ RtlClearBits(&AllocBitmap, BlockNumber + NewSize, HandleEntry->Size - NewSize); FreeBlocks += HandleEntry->Size - NewSize; HandleEntry->Size = NewSize; } else if (NewSize > HandleEntry->Size) { /* Check if we can expand in-place */ if (RtlAreBitsClear(&AllocBitmap, BlockNumber + HandleEntry->Size, NewSize - HandleEntry->Size)) { /* Just increase the size of this block */ RtlSetBits(&AllocBitmap, BlockNumber + HandleEntry->Size, NewSize - HandleEntry->Size); FreeBlocks -= NewSize - HandleEntry->Size; HandleEntry->Size = NewSize; /* We're done */ return XMS_STATUS_SUCCESS; } /* Deallocate the current block range */ RtlClearBits(&AllocBitmap, BlockNumber, HandleEntry->Size); /* Find a new place for this block */ while (CurrentIndex < XMS_BLOCKS) { RunSize = RtlFindNextForwardRunClear(&AllocBitmap, CurrentIndex, &RunStart); if (RunSize == 0) break; if (RunSize >= NewSize) { /* Allocate the new range */ RtlSetBits(&AllocBitmap, RunStart, NewSize); /* Move the data to the new location */ RtlMoveMemory((PVOID)REAL_TO_PHYS(XMS_ADDRESS + RunStart * XMS_BLOCK_SIZE), (PVOID)REAL_TO_PHYS(HandleEntry->Address), HandleEntry->Size * XMS_BLOCK_SIZE); /* Update the handle entry */ HandleEntry->Address = XMS_ADDRESS + RunStart * XMS_BLOCK_SIZE; HandleEntry->Size = NewSize; /* Update the free block counter */ FreeBlocks -= NewSize - HandleEntry->Size; return XMS_STATUS_SUCCESS; } /* Keep searching */ CurrentIndex = RunStart + RunSize; } /* Restore the old block range */ RtlSetBits(&AllocBitmap, BlockNumber, HandleEntry->Size); return XMS_STATUS_OUT_OF_MEMORY; } return XMS_STATUS_SUCCESS; }
VOID MiZeroPageFile ( IN PVOID Context ) /*++ Routine Description: This routine zeroes all inactive pagefile blocks in the specified paging file. Arguments: Context - Supplies the information on which pagefile to zero and a zeroed page to use for the I/O. Return Value: Returns TRUE on success, FALSE on failure. Environment: Kernel mode, the caller must lock down PAGELK. --*/ { PFN_NUMBER MaxPagesToWrite; PMMPFN Pfn1; PPFN_NUMBER Page; PFN_NUMBER MdlHack[(sizeof(MDL)/sizeof(PFN_NUMBER)) + MM_MAXIMUM_WRITE_CLUSTER]; PMDL Mdl; NTSTATUS Status; KEVENT IoEvent; IO_STATUS_BLOCK IoStatus; KIRQL OldIrql; LARGE_INTEGER StartingOffset; ULONG count; ULONG i; PFN_NUMBER first; ULONG write; PKEVENT AllDone; SIZE_T NumberOfBytes; PMMPAGING_FILE PagingFile; PFN_NUMBER ZeroedPageFrame; PMM_ZERO_PAGEFILE_CONTEXT ZeroContext; ZeroContext = (PMM_ZERO_PAGEFILE_CONTEXT) Context; PagingFile = ZeroContext->PagingFile; ZeroedPageFrame = ZeroContext->ZeroedPageFrame; AllDone = ZeroContext->AllDone; ExFreePool (Context); NumberOfBytes = MmModifiedWriteClusterSize << PAGE_SHIFT; MaxPagesToWrite = NumberOfBytes >> PAGE_SHIFT; Mdl = (PMDL) MdlHack; Page = (PPFN_NUMBER)(Mdl + 1); KeInitializeEvent (&IoEvent, NotificationEvent, FALSE); MmInitializeMdl (Mdl, NULL, PAGE_SIZE); Mdl->MdlFlags |= MDL_PAGES_LOCKED; Mdl->StartVa = NULL; i = 0; Page = (PPFN_NUMBER)(Mdl + 1); for (i = 0; i < MaxPagesToWrite; i += 1) { *Page = ZeroedPageFrame; Page += 1; } count = 0; write = FALSE; SATISFY_OVERZEALOUS_COMPILER (first = 0); LOCK_PFN (OldIrql); for (i = 1; i < PagingFile->Size; i += 1) { if (RtlCheckBit (PagingFile->Bitmap, (ULONG) i) == 0) { // // Claim the pagefile location as the modified writer // may already be scanning. // RtlSetBit (PagingFile->Bitmap, (ULONG) i); if (count == 0) { first = i; } count += 1; if ((count == MaxPagesToWrite) || (i == PagingFile->Size - 1)) { write = TRUE; } } else { if (count != 0) { // // Issue a write. // write = TRUE; } } if (write) { UNLOCK_PFN (OldIrql); StartingOffset.QuadPart = (LONGLONG)first << PAGE_SHIFT; Mdl->ByteCount = count << PAGE_SHIFT; KeClearEvent (&IoEvent); Status = IoSynchronousPageWrite (PagingFile->File, Mdl, &StartingOffset, &IoEvent, &IoStatus); // // Ignore all I/O failures - there is nothing that can // be done at this point. // if (!NT_SUCCESS (Status)) { KeSetEvent (&IoEvent, 0, FALSE); } Status = KeWaitForSingleObject (&IoEvent, WrPageOut, KernelMode, FALSE, (PLARGE_INTEGER)&MmTwentySeconds); if (Status == STATUS_TIMEOUT) { // // The write did not complete in 20 seconds, assume // that the file systems are hung and return an error. // // Note the zero page (and any MDL system virtual address a // driver may have created) is leaked because we don't know // what the filesystem or storage stack might (still) be // doing to them. // Pfn1 = MI_PFN_ELEMENT (ZeroedPageFrame); LOCK_PFN (OldIrql); // // Increment the reference count on the zeroed page to ensure // it is never freed. // InterlockedIncrementPfn ((PSHORT)&Pfn1->u3.e2.ReferenceCount); RtlClearBits (PagingFile->Bitmap, (ULONG) first, count); break; } if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } write = FALSE; LOCK_PFN (OldIrql); RtlClearBits (PagingFile->Bitmap, (ULONG) first, count); count = 0; } } UNLOCK_PFN (OldIrql); KeSetEvent (AllDone, 0, FALSE); return; }