/// <summary>
/// Get EPT PTE entry for guest physical address
/// </summary>
/// <param name="PML4">PTE PML4 pointer</param>
/// <param name="phys">Guest physical address</param>
/// <param name="pEntry">Found entry or NULL</param>
/// <returns>Status code</returns>
NTSTATUS EptGetPTEForPhysical( IN PEPT_PML4_ENTRY PML4, IN PHYSICAL_ADDRESS phys, OUT PEPT_PTE_ENTRY* pEntry )
{
NT_ASSERT( pEntry != NULL && PML4 != NULL );
if (pEntry == NULL|| PML4 == NULL)
return STATUS_INVALID_PARAMETER;
ULONG64 offset = EptpTableOffset( PFN( phys.QuadPart ), 3 );
ULONG64 pfn = PML4[offset].Fields.PhysAddr;
if (pfn != 0)
{
for (CHAR i = 2; i >= 0; i--)
{
PHYSICAL_ADDRESS addr = { 0 };
addr.QuadPart = pfn << PAGE_SHIFT;
PEPT_MMPTE ptr = MmGetVirtualForPhysical( addr );
if (ptr == NULL)
break;
offset = EptpTableOffset( PFN( phys.QuadPart ), i );
if (i == 0)
{
*pEntry = (PEPT_PTE_ENTRY)&ptr[offset];
return STATUS_SUCCESS;
}
pfn = ptr[offset].Fields.PhysAddr;
}
}
return STATUS_NOT_FOUND;
}
/*
* @to: index of the smallest element which is >= then @pfn.
*
* Return the index of the pfn if found, otherwise negative value.
*/
static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)
{
u64 this_pfn;
int min = 0, max = ca->n;
while (min < max) {
int tmp = (max + min) >> 1;
this_pfn = PFN(ca->array[tmp]);
if (this_pfn < pfn)
min = tmp + 1;
else if (this_pfn > pfn)
max = tmp;
else {
min = tmp;
break;
}
}
if (to)
*to = min;
this_pfn = PFN(ca->array[min]);
if (this_pfn == pfn)
return min;
return -ENOKEY;
}
int Mips3MapHandler(uintptr_t nHandler, unsigned int nStart, unsigned int nEnd, int nType)
{
const int maxPages = (PFN(nEnd) - PFN(nStart)) + 1;
int page = PFN(nStart);
for (int i = 0; i < maxPages; i++, page++) {
if (nType & MAP_READ)
g_mmap->MemMap[page] = (UINT8*) nHandler;
if (nType & MAP_WRITE)
g_mmap->MemMap[PAGE_WADD + page] = (UINT8*) nHandler;
}
return 0;
}
int Mips3MapMemory(unsigned char* pMemory, unsigned int nStart, unsigned int nEnd, int nType)
{
const int maxPages = (PFN(nEnd) - PFN(nStart)) + 1;
int page = PFN(nStart);
for (int i = 0; i < maxPages; i++, page++) {
if (nType & MAP_READ)
g_mmap->MemMap[page] = pMemory + (PAGE_SIZE * i);
if (nType & MAP_WRITE)
g_mmap->MemMap[PAGE_WADD + page] = pMemory + (PAGE_SIZE * i);
}
return 0;
}
static int array_dump(struct seq_file *m, void *v)
{
struct ce_array *ca = &ce_arr;
u64 prev = 0;
int i;
mutex_lock(&ce_mutex);
seq_printf(m, "{ n: %d\n", ca->n);
for (i = 0; i < ca->n; i++) {
u64 this = PFN(ca->array[i]);
seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));
WARN_ON(prev > this);
prev = this;
}
seq_printf(m, "}\n");
seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n",
ca->ces_entered, ca->pfns_poisoned);
seq_printf(m, "Flags: 0x%x\n", ca->flags);
seq_printf(m, "Timer interval: %lld seconds\n", timer_interval);
seq_printf(m, "Decays: %lld\n", ca->decays_done);
seq_printf(m, "Action threshold: %d\n", count_threshold);
mutex_unlock(&ce_mutex);
return 0;
}
/// <summary>
/// EPT violation (#VE) handler
/// </summary>
/// <param name="GuestState">Guest VM state</param>
VOID VmExitEptViolation( IN PGUEST_STATE GuestState )
{
//PEPT_PTE_ENTRY pPTE = NULL;
PEPT_DATA pEPT = &GuestState->Vcpu->EPT;
ULONG64 pfn = PFN( GuestState->PhysicalAddress.QuadPart );
PEPT_VIOLATION_DATA pViolationData = (PEPT_VIOLATION_DATA)&GuestState->ExitQualification;
// Page is hooked
PPAGE_HOOK_ENTRY pHookEntry = PHGetHookEntryByPFN( pfn, DATA_PAGE );
if (pHookEntry)
{
/*DPRINT(
"HyperBone: CPU %d: %s: Hooked page %s, EIP 0x%p, Linear 0x%p, Physical 0x%p, Violation data 0x%x\n",
CPU_IDX, __FUNCTION__,
pViolationData->Fields.Execute ? "EXECUTE" : (pViolationData->Fields.Read ? "READ" : "WRITE"),
GuestState->GuestRip, GuestState->LinearAddress, GuestState->PhysicalAddress.QuadPart, pViolationData->All
);*/
// Set target host PFN
ULONG64 TargetPFN = pHookEntry->DataPagePFN;
// Executable page for writing
if (pHookEntry->Type == HOOK_SWAP && pViolationData->Fields.Write)
TargetPFN = pfn;
// Executable page for TLB splitting
else if (pHookEntry->Type == HOOK_SPLIT && pViolationData->Fields.Execute)
TargetPFN = pHookEntry->CodePagePFN;
// Impossible execute violation
if(pHookEntry->Type == HOOK_SWAP && pViolationData->Fields.Execute)
{
DPRINT(
"HyperBone: CPU %d: %s: Impossible page 0x%p access 0x%X\n", CPU_IDX, __FUNCTION__,
GuestState->PhysicalAddress.QuadPart, pViolationData->All
);
}
else
EptUpdateTableRecursive( pEPT, pEPT->PML4Ptr, EPT_TOP_LEVEL, pfn, EPT_ACCESS_ALL, TargetPFN, 1 );
// Do a MTF exit upon resume
ToggleMTF( TRUE );
GuestState->Vcpu->HookDispatch.pEntry = pHookEntry;
GuestState->Vcpu->HookDispatch.Rip = GuestState->GuestRip;
}
// Create new identity page map
else
{
/*DPRINT(
"HyperBone: CPU %d: %s: EPT violation, EIP 0x%p, Linear 0x%p, Physical 0x%p, Violation data 0x%X\n",
CPU_IDX, __FUNCTION__,
GuestState->GuestRip, GuestState->LinearAddress, GuestState->PhysicalAddress.QuadPart, pViolationData->All
);*/
EptUpdateTableRecursive( pEPT, pEPT->PML4Ptr, EPT_TOP_LEVEL, pfn, EPT_ACCESS_ALL, pfn, 1 );
}
}
uint64_t read_dword(addr_t address)
{
address &= 0xFFFFFFFF;
UINT8 *pr = g_mmap->MemMap[PFN(address)];
if ((uintptr_t)pr >= MIPS_MAXHANDLER) {
return BURN_ENDIAN_SWAP_INT64(fast_read<uint64_t>(pr, address));
}
return g_mmap->ReadDouble[(uintptr_t)pr](address);
}
uint16_t read_half(addr_t address)
{
address &= 0xFFFFFFFF;
UINT8 *pr = g_mmap->MemMap[PFN(address)];
if ((uintptr_t)pr >= MIPS_MAXHANDLER) {
return BURN_ENDIAN_SWAP_INT16(fast_read<uint16_t>(pr, address));
}
return g_mmap->ReadHalf[(uintptr_t)pr](address);
}
uint8_t read_byte(addr_t address)
{
address &= 0xFFFFFFFF;
UINT8 *pr = g_mmap->MemMap[PFN(address)];
if ((uintptr_t)pr >= MIPS_MAXHANDLER) {
return pr[address & PAGE_MASK];
}
return g_mmap->ReadByte[(uintptr_t)pr](address);
}
void write_dword(addr_t address, uint64_t value)
{
address &= 0xFFFFFFFF;
UINT8 *pr = g_mmap->MemMap[PAGE_WADD + PFN(address)];
if ((uintptr_t)pr >= MIPS_MAXHANDLER) {
fast_write<uint64_t>(pr, address, BURN_ENDIAN_SWAP_INT64(value));
return;
}
g_mmap->WriteDouble[(uintptr_t)pr](address, value);
}
void write_byte(addr_t address, uint8_t value)
{
address &= 0xFFFFFFFF;
UINT8 *pr = g_mmap->MemMap[PAGE_WADD + PFN(address)];
if ((uintptr_t)pr >= MIPS_MAXHANDLER) {
pr[address & PAGE_MASK] = value;
return;
}
g_mmap->WriteByte[(uintptr_t)pr](address, value);
}
/// <summary>
/// Update EPT entry
/// </summary>
/// <param name="pEPTData">CPU EPT data</param>
/// <param name="pTable">EPT table</param>
/// <param name="level">EPT table level</param>
/// <param name="pfn">Page frame number to update</param>
/// <param name="access">New PFN access</param>
/// <param name="hostPFN">New hot PFN</param>
/// <param name="count">Number of entries to update</param>
/// <returns>Status code</returns>
NTSTATUS EptUpdateTableRecursive(
IN PEPT_DATA pEPTData,
IN PEPT_MMPTE pTable,
IN EPT_TABLE_LEVEL level,
IN ULONG64 pfn,
IN UCHAR access,
IN ULONG64 hostPFN,
IN ULONG count
)
{
if (level == EPT_LEVEL_PTE)
{
ULONG64 first = EptpTableOffset( pfn, level );
ASSERT( first + count <= EPT_TABLE_ENTRIES );
PEPT_PTE_ENTRY pPTE = (PEPT_PTE_ENTRY)pTable;
for (ULONG64 i = first; i < first + count; i++, hostPFN++)
{
pPTE[i].Fields.Read = (access & EPT_ACCESS_READ) != 0;
pPTE[i].Fields.Write = (access & EPT_ACCESS_WRITE) != 0;
pPTE[i].Fields.Execute = (access & EPT_ACCESS_EXEC) != 0;
pPTE[i].Fields.MemoryType = VMX_MEM_TYPE_WRITEBACK;
pPTE[i].Fields.PhysAddr = hostPFN;
}
return STATUS_SUCCESS;
}
ULONG64 offset = EptpTableOffset( pfn, level );
PEPT_MMPTE pEPT = &pTable[offset];
PEPT_MMPTE pNewEPT = 0;
if (pEPT->Fields.PhysAddr == 0)
{
pNewEPT = (PEPT_MMPTE)EptpAllocatePage( pEPTData );
if (pNewEPT == NULL)
return STATUS_INSUFFICIENT_RESOURCES;
pEPT->Fields.Present = 1;
pEPT->Fields.Write = 1;
pEPT->Fields.Execute = 1;
pEPT->Fields.PhysAddr = PFN( MmGetPhysicalAddress( pNewEPT ).QuadPart );
}
else
{
PHYSICAL_ADDRESS phys = { 0 };
phys.QuadPart = pEPT->Fields.PhysAddr << 12;
pNewEPT = MmGetVirtualForPhysical( phys );
}
return EptUpdateTableRecursive( pEPTData, pNewEPT, level - 1, pfn, access, hostPFN, count );
}
static u64 del_lru_elem_unlocked(struct ce_array *ca)
{
unsigned int min = FULL_COUNT_MASK;
int i, min_idx = 0;
for (i = 0; i < ca->n; i++) {
unsigned int this = FULL_COUNT(ca->array[i]);
if (min > this) {
min = this;
min_idx = i;
}
}
del_elem(ca, min_idx);
return PFN(ca->array[min_idx]);
}
/// <summary>
/// Hook function
/// </summary>
/// <param name="pFunc">Function address</param>
/// <param name="pHook">Hook address</param>
/// /// <param name="Type">Hook type</param>
/// <returns>Status code</returns>
NTSTATUS PHHook( IN PVOID pFunc, IN PVOID pHook, IN HOOK_TYPE Type )
{
PUCHAR CodePage = NULL;
BOOLEAN Newpage = FALSE;
PHYSICAL_ADDRESS phys = { 0 };
phys.QuadPart = MAXULONG64;
// No CPU support
if (!g_Data->EPTSupported || !g_Data->EPTExecOnlySupported)
return STATUS_NOT_SUPPORTED;
// Check if page is already hooked
PPAGE_HOOK_ENTRY pEntry = PHGetHookEntryByPage( pFunc, DATA_PAGE );
if (pEntry != NULL)
{
CodePage = pEntry->CodePageVA;
}
else
{
CodePage = MmAllocateContiguousMemory( PAGE_SIZE, phys );
Newpage = TRUE;
}
if (CodePage == NULL)
return STATUS_INSUFFICIENT_RESOURCES;
PPAGE_HOOK_ENTRY pHookEntry = ExAllocatePoolWithTag( NonPagedPool, sizeof( PAGE_HOOK_ENTRY ), HB_POOL_TAG );
if (pHookEntry == NULL)
return STATUS_INSUFFICIENT_RESOURCES;
RtlZeroMemory( pHookEntry, sizeof( PAGE_HOOK_ENTRY ) );
RtlCopyMemory( CodePage, PAGE_ALIGN( pFunc ), PAGE_SIZE );
// Copy original code
NTSTATUS status = PHpCopyCode( pFunc, pHookEntry->OriginalData, &pHookEntry->OriginalSize );
if (!NT_SUCCESS( status ))
{
ExFreePoolWithTag( pHookEntry, HB_POOL_TAG );
return status;
}
ULONG_PTR page_offset = (ULONG_PTR)pFunc - (ULONG_PTR)PAGE_ALIGN( pFunc );
// TODO: atomic memory patching, i.e. with other CPUs spinlocked
JUMP_THUNK thunk = { 0 };
PHpInitJumpThunk( &thunk, (ULONG64)pHook );
memcpy( CodePage + page_offset, &thunk, sizeof( thunk ) );
pHookEntry->Type = Type;
pHookEntry->OriginalPtr = pFunc;
pHookEntry->DataPageVA = PAGE_ALIGN( pFunc );
pHookEntry->DataPagePFN = PFN( MmGetPhysicalAddress( pFunc ).QuadPart );
pHookEntry->CodePageVA = CodePage;
pHookEntry->CodePagePFN = PFN( MmGetPhysicalAddress( CodePage ).QuadPart );
// Add to list
if (g_PageList.Flink == NULL)
InitializeListHead( &g_PageList );
InsertTailList( &g_PageList, &pHookEntry->Link );
// Create EPT page translation
if (Newpage)
{
HOOK_CONTEXT ctx = { 0 };
ctx.Hook = TRUE;
ctx.DataPagePFN = pHookEntry->DataPagePFN;
ctx.CodePagePFN = pHookEntry->CodePagePFN;
ctx.Type = Type;
KeGenericCallDpc( PHpHookCallbackDPC, &ctx );
}
return STATUS_SUCCESS;
}
return *((T*) ((uint8_t*) ptr + (adr & PAGE_MASK)));
}
template<typename T>
inline void fast_write(uint8_t *xptr, unsigned adr, T value) {
T *ptr = ((T*) ((uint8_t*) xptr + (adr & PAGE_MASK)));
*ptr = value;
}
UINT16 adsp21xx_data_read_word_16le(UINT32 address)
{
// address &= 0xFFFF;
address >>= 1;
address &= 0x3FFF;
UINT8 *pr = pMemMap->DataMap[PFN(address)];
if ((uintptr_t)pr >= ADSP_MAXHANDLER) {
return BURN_ENDIAN_SWAP_INT16(fast_read<uint16_t>(pr, address));
}
return pMemMap->dataReadWord[(uintptr_t)pr](address);
}
void adsp21xx_data_write_word_16le(UINT32 address, UINT16 data)
{
// address &= 0xFFFF;
address >>= 1;
address &= 0x3FFF;
UINT8 *pr = pMemMap->DataMap[PAGE_WADD + PFN(address)];
if ((uintptr_t)pr >= ADSP_MAXHANDLER) {
fast_write<uint16_t>(pr, address, BURN_ENDIAN_SWAP_INT16(data));