// Modify or restore MSR
_Use_decl_annotations_ EXTERN_C static NTSTATUS MsrHookCallback(void* Context) {
UNREFERENCED_PARAMETER(Context);
auto oldmsr = &g_MSRs[KeGetCurrentProcessorNumber()];
if (*oldmsr == 0) {
// Modify
*oldmsr = __readmsr(IA32_LSTAR);
__writemsr(IA32_LSTAR, reinterpret_cast<ULONG_PTR>(g_Trampoline));
LOG_INFO("MSR(%08x) %p => %p", IA32_LSTAR, *oldmsr, g_Trampoline);
} else {
// Restore
__writemsr(IA32_LSTAR, *oldmsr);
LOG_INFO("MSR(%08x) %p => %p", IA32_LSTAR, g_Trampoline, *oldmsr);
}
return STATUS_SUCCESS;
}
/// <summary>
/// Check if VT-x is supported
/// </summary>
/// <returns>TRUE if supported</returns>
BOOLEAN VmxHardSupported()
{
CPUID data = { 0 };
// VMX bit
__cpuid( (int*)&data, 1 );
if ((data.ecx & (1 << 5)) == 0)
return FALSE;
IA32_FEATURE_CONTROL_MSR Control = { 0 };
Control.All = __readmsr( MSR_IA32_FEATURE_CONTROL );
// BIOS lock check
if (Control.Fields.Lock == 0)
{
Control.Fields.Lock = TRUE;
Control.Fields.EnableVmxon = TRUE;
__writemsr( MSR_IA32_FEATURE_CONTROL, Control.All );
}
else if (Control.Fields.EnableVmxon == FALSE)
{
DPRINT( "HyperBone: CPU %d: %s: VMX locked off in BIOS\n", CPU_IDX, __FUNCTION__ );
return FALSE;
}
return TRUE;
}
VOID
NTAPI
HalpInitializeClock(VOID)
{
//PKPRCB Prcb = KeGetCurrentPrcb();
ULONG Increment;
USHORT RollOver;
ULONG Flags = 0;
/* Get increment and rollover for the largest time clock ms possible */
Increment = HalpRolloverTable[HalpLargestClockMS - 1].HighPart;
RollOver = (USHORT)HalpRolloverTable[HalpLargestClockMS - 1].LowPart;
/* Set the maximum and minimum increment with the kernel */
HalpCurrentTimeIncrement = Increment;
KeSetTimeIncrement(Increment, HalpRolloverTable[0].HighPart);
/* Disable interrupts */
Flags = __readmsr();
_disable();
/* Set the rollover */
__outbyte(TIMER_CONTROL_PORT, TIMER_SC0 | TIMER_BOTH | TIMER_MD2);
__outbyte(TIMER_DATA_PORT0, RollOver & 0xFF);
__outbyte(TIMER_DATA_PORT0, RollOver >> 8);
/* Restore interrupts if they were previously enabled */
__writemsr(Flags);
/* Save rollover and return */
HalpCurrentRollOver = RollOver;
}
VOID
NTAPI
WRMSR(IN ULONG Register,
IN LONGLONG Value)
{
/* Write to the MSR */
__writemsr(Register, Value);
}
UINT64
EFIAPI
AsmWriteMsr64 (
IN UINT32 Index,
IN UINT64 Value
)
{
__writemsr (Index, Value);
return Value;
}
void apic_clearPerfmon()
{
if (x2APICMode)
{
__writemsr(MSR_IA32_X2APIC_LVT_PMI, (__readmsr(MSR_IA32_X2APIC_LVT_PMI) & (UINT64)0xff));
__writemsr(MSR_IA32_X2APIC_EOI, 0);
}
else
{
//DbgPrint("Clear perfmon (APIC_BASE at %llx)\n", APIC_BASE);
//DbgPrint("APIC_BASE->LVT_Performance_Monitor.a at %p\n", &APIC_BASE->LVT_Performance_Monitor.a);
//DbgPrint("APIC_BASE->EOI.a at %p\n", &APIC_BASE->EOI.a);
//DbgPrint("APIC_BASE->LVT_Performance_Monitor.a had value %x\n", APIC_BASE->LVT_Performance_Monitor.a);
//DbgPrint("APIC_BASE->LVT_Performance_Monitor.b had value %x\n", APIC_BASE->LVT_Performance_Monitor.b);
// DbgPrint("APIC_BASE->LVT_Performance_Monitor.c had value %x\n", APIC_BASE->LVT_Performance_Monitor.c);
// DbgPrint("APIC_BASE->LVT_Performance_Monitor.d had value %x\n", APIC_BASE->LVT_Performance_Monitor.d);
APIC_BASE->LVT_Performance_Monitor.a = APIC_BASE->LVT_Performance_Monitor.a & 0xff;
APIC_BASE->EOI.a = 0;
}
}
static NTSTATUS set_lock_bit(void)
{
uintptr_t feat_ctl = __readmsr(MSR_IA32_FEATURE_CONTROL);
if (feat_ctl & FEATURE_CONTROL_LOCKED)
return STATUS_SUCCESS;
__writemsr(MSR_IA32_FEATURE_CONTROL, feat_ctl | FEATURE_CONTROL_LOCKED);
feat_ctl = __readmsr(MSR_IA32_FEATURE_CONTROL);
if (feat_ctl & FEATURE_CONTROL_LOCKED)
return STATUS_SUCCESS;
return STATUS_HV_ACCESS_DENIED;
}
//----------------------------------------------------------------------------
// MemUFlushPattern:
//
// Flush a pattern of 72 bit times (per DQ) from cache. This procedure is used
// to ensure cache miss on the next read training.
//
// In: Address - Physical address to be flushed
// ClCount - number of cachelines to be flushed
//FUNC_ATTRIBUTE(noinline)
VOID
MemUFlushPattern (
IN UINT32 Address,
IN UINT16 ClCount
)
{
UINTN Index;
// ssd - theory: a tlb flush is needed to avoid problems with clflush
__writemsr (0x20F, __readmsr (0x20F));
for (Index = 0; Index < ClCount; Index++) {
// mfence prevents speculative execution of the clflush
_mm_mfence ();
_mm_clflush_fs ((void *) (size_t) (Address + Index * 64));
}
}
/*
* @implemented
*/
VOID
NTAPI
HalCalibratePerformanceCounter(IN volatile PLONG Count,
IN ULONGLONG NewCount)
{
ULONG Flags = 0;
/* Disable interrupts */
Flags = __readmsr();
_disable();
/* Do a decrement for this CPU */
_InterlockedDecrement(Count);
/* Wait for other CPUs */
while (*Count);
/* Restore interrupts if they were previously enabled */
__writemsr(Flags);
}
void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
{
u32 val;
#if CONFIG_HAVE_ACPI_RESUME
void *resume_backup_memory;
#endif
/*
* All cores: allow caching of flash chip code and data
* (there are no cache-as-ram reliability concerns with family 14h)
*/
__writemsr (0x20c, (0x0100000000ull - CONFIG_ROM_SIZE) | 5);
__writemsr (0x20d, (0x1000000000ull - CONFIG_ROM_SIZE) | 0x800);
/* All cores: set pstate 0 (1600 MHz) early to save a few ms of boot time */
__writemsr (0xc0010062, 0);
if (!cpu_init_detectedx && boot_cpu()) {
post_code(0x30);
sb_Poweron_Init();
post_code(0x31);
smscsuperio_enable_serial(SERIAL_DEV, CONFIG_TTYS0_BASE);
console_init();
}
/* Halt if there was a built in self test failure */
post_code(0x34);
report_bist_failure(bist);
/* Load MPB */
val = cpuid_eax(1);
printk(BIOS_DEBUG, "BSP Family_Model: %08x \n", val);
printk(BIOS_DEBUG, "cpu_init_detectedx = %08lx \n", cpu_init_detectedx);
post_code(0x35);
printk(BIOS_DEBUG, "agesawrapper_amdinitmmio ");
val = agesawrapper_amdinitmmio();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x37);
printk(BIOS_DEBUG, "agesawrapper_amdinitreset ");
val = agesawrapper_amdinitreset();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x39);
printk(BIOS_DEBUG, "agesawrapper_amdinitearly ");
val = agesawrapper_amdinitearly ();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
#if CONFIG_HAVE_ACPI_RESUME
if (!acpi_is_wakeup_early()) { /* Check for S3 resume */
#endif
post_code(0x40);
printk(BIOS_DEBUG, "agesawrapper_amdinitpost ");
val = agesawrapper_amdinitpost ();
/* Reboots with outb(3,0x92), outb(4,0xcf9) or triple-fault all
* hang, looks like DRAM re-init goes wrong, don't know why. */
if (val == 7) /* fatal, amdinitenv below is going to hang */
outb(0x06, 0x0cf9); /* reset system harder instead */
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x42);
printk(BIOS_DEBUG, "agesawrapper_amdinitenv ");
val = agesawrapper_amdinitenv ();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
#if CONFIG_HAVE_ACPI_RESUME
} else { /* S3 detect */
printk(BIOS_INFO, "S3 detected\n");
post_code(0x60);
printk(BIOS_DEBUG, "agesawrapper_amdinitresume ");
val = agesawrapper_amdinitresume();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
printk(BIOS_DEBUG, "agesawrapper_amds3laterestore ");
val = agesawrapper_amds3laterestore ();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x61);
printk(BIOS_DEBUG, "Find resume memory location\n");
resume_backup_memory = backup_resume();
post_code(0x62);
printk(BIOS_DEBUG, "Move CAR stack.\n");
move_stack_high_mem();
printk(BIOS_DEBUG, "stack moved to: 0x%x\n", (u32) (resume_backup_memory + HIGH_MEMORY_SAVE));
post_code(0x63);
disable_cache_as_ram();
printk(BIOS_DEBUG, "CAR disabled.\n");
set_resume_cache();
/*
* Copy the system memory that is in the ramstage area to the
* reserved area.
*/
if (resume_backup_memory)
memcpy(resume_backup_memory, (void *)(CONFIG_RAMBASE), HIGH_MEMORY_SAVE);
printk(BIOS_DEBUG, "System memory saved. OK to load ramstage.\n");
}
#endif
/* Initialize i8259 pic */
post_code(0x43);
setup_i8259 ();
/* Initialize i8254 timers */
post_code(0x44);
setup_i8254 ();
post_code(0x50);
copy_and_run(0);
printk(BIOS_ERR, "Error: copy_and_run() returned!\n");
post_code(0x54); /* Should never see this post code. */
}
NTSTATUS __fastcall DispatchIOCTL(IN PIOCTL_INFO RequestInfo, OUT PULONG ResponseLength) {
NTSTATUS Status = STATUS_SUCCESS;
#define INPUT(Type) ((Type)(RequestInfo->InputBuffer))
#define OUTPUT(Type) ((Type)(RequestInfo->OutputBuffer))
#define SET_RESPONSE_LENGTH(Length) if (ResponseLength != NULL) {*ResponseLength = (Length);}
switch (RequestInfo->ControlCode) {
// DriverFunctions:
case GET_HANDLES_COUNT:
*OUTPUT(PULONG) = GetHandlesCount();
SET_RESPONSE_LENGTH(sizeof(ULONG));
break;
// NativeFunctions:
case START_BEEPER:
__outbyte(0x61, __inbyte(0x61) | 3);
break;
case STOP_BEEPER:
__outbyte(0x61, __inbyte(0x61) & 252);
break;
case SET_BEEPER_REGIME:
__outbyte(0x43, 0xB6);
break;
case SET_BEEPER_OUT:
__outbyte(0x61, __inbyte(0x61) | 2);
break;
case SET_BEEPER_IN:
__outbyte(0x61, __inbyte(0x61) & 253);
break;
case SET_BEEPER_DIVIDER:
SetBeeperDivider(*INPUT(PWORD));
break;
case SET_BEEPER_FREQUENCY:
SetBeeperFrequency(*INPUT(PWORD));
break;
case READ_IO_PORT_BYTE:
*OUTPUT(PBYTE) = __inbyte(*INPUT(PWORD));
SET_RESPONSE_LENGTH(sizeof(BYTE));
break;
case READ_IO_PORT_WORD:
*OUTPUT(PWORD) = __inword(*INPUT(PWORD));
SET_RESPONSE_LENGTH(sizeof(WORD));
break;
case READ_IO_PORT_DWORD:
*OUTPUT(PDWORD32) = __indword(*INPUT(PWORD));
SET_RESPONSE_LENGTH(sizeof(DWORD));
break;
case WRITE_IO_PORT_BYTE:
__outbyte(
INPUT(PWRITE_IO_PORT_BYTE_INPUT)->PortNumber,
INPUT(PWRITE_IO_PORT_BYTE_INPUT)->Data
);
break;
case WRITE_IO_PORT_WORD:
__outword(
INPUT(PWRITE_IO_PORT_WORD_INPUT)->PortNumber,
INPUT(PWRITE_IO_PORT_WORD_INPUT)->Data
);
break;
case WRITE_IO_PORT_DWORD:
__outdword(
INPUT(PWRITE_IO_PORT_DWORD_INPUT)->PortNumber,
INPUT(PWRITE_IO_PORT_DWORD_INPUT)->Data
);
break;
case RDPMC:
*OUTPUT(PULONGLONG) = __readpmc(*INPUT(PULONG));
SET_RESPONSE_LENGTH(sizeof(ULONGLONG));
break;
case RDMSR:
*OUTPUT(PULONGLONG) = __readmsr(*INPUT(PULONG));
SET_RESPONSE_LENGTH(sizeof(ULONGLONG));
break;
case WRMSR:
__writemsr(INPUT(PWRMSR_INPUT)->Index, INPUT(PWRMSR_INPUT)->Data);
break;
// MemoryUtils:
case ALLOC_KERNEL_MEMORY:
*OUTPUT(PUINT64) = (SIZE_T)GetMem(*INPUT(PSIZE_T));
SET_RESPONSE_LENGTH(sizeof(UINT64));
break;
case FREE_KERNEL_MEMORY:
FreeMem((PVOID)*INPUT(PUINT64));
break;
case COPY_MEMORY:
RtlCopyMemory(
(PVOID)INPUT(PCOPY_MEMORY_INPUT)->Destination,
(PVOID)INPUT(PCOPY_MEMORY_INPUT)->Source,
(SIZE_T)INPUT(PCOPY_MEMORY_INPUT)->Size
);
break;
case FILL_MEMORY:
RtlFillMemory(
(PVOID)INPUT(PFILL_MEMORY_INPUT)->Destination,
(SIZE_T)INPUT(PFILL_MEMORY_INPUT)->Size,
(BYTE)INPUT(PFILL_MEMORY_INPUT)->FillingByte
);
break;
case ALLOC_PHYSICAL_MEMORY:
*OUTPUT(PUINT64) = (SIZE_T)AllocPhysicalMemory(
INPUT(PALLOC_PHYSICAL_MEMORY_INPUT)->PhysicalAddress,
(SIZE_T)INPUT(PALLOC_PHYSICAL_MEMORY_INPUT)->Size
);
SET_RESPONSE_LENGTH(sizeof(UINT64));
break;
case FREE_PHYSICAL_MEMORY:
FreePhysicalMemory((PVOID)*INPUT(PUINT64));
break;
case GET_PHYSICAL_ADDRESS:
*OUTPUT(PPHYSICAL_ADDRESS) = GetPhysicalAddressInProcess(
(HANDLE)INPUT(PGET_PHYSICAL_ADDRESS_INPUT)->ProcessID,
(PVOID)INPUT(PGET_PHYSICAL_ADDRESS_INPUT)->VirtualAddress
);
SET_RESPONSE_LENGTH(sizeof(PHYSICAL_ADDRESS));
break;
case READ_PHYSICAL_MEMORY:
*OUTPUT(PBOOL) = ReadPhysicalMemory(
INPUT(PREAD_PHYSICAL_MEMORY_INPUT)->PhysicalAddress,
(PVOID)INPUT(PREAD_PHYSICAL_MEMORY_INPUT)->Buffer,
INPUT(PREAD_PHYSICAL_MEMORY_INPUT)->BufferSize
);
SET_RESPONSE_LENGTH(sizeof(BOOL));
break;
case WRITE_PHYSICAL_MEMORY:
*OUTPUT(PBOOL) = WritePhysicalMemory(
INPUT(PWRITE_PHYSICAL_MEMORY_INPUT)->PhysicalAddress,
(PVOID)INPUT(PWRITE_PHYSICAL_MEMORY_INPUT)->Buffer,
INPUT(PWRITE_PHYSICAL_MEMORY_INPUT)->BufferSize
);
SET_RESPONSE_LENGTH(sizeof(BOOL));
break;
case READ_DMI_MEMORY:
ReadDmiMemory((PVOID)*INPUT(PUINT64), DMI_SIZE);
SET_RESPONSE_LENGTH(DMI_SIZE);
break;
// ShellCode:
case EXECUTE_SHELL_CODE:
*OUTPUT(PSHELL_STATUS) = ExecuteShell(
(PVOID)INPUT(PEXECUTE_SHELL_CODE_INPUT)->EntryPoint,
(PVOID)INPUT(PEXECUTE_SHELL_CODE_INPUT)->CodeBlock,
(PVOID)INPUT(PEXECUTE_SHELL_CODE_INPUT)->InputData,
(PVOID)INPUT(PEXECUTE_SHELL_CODE_INPUT)->OutputData,
(PVOID)INPUT(PEXECUTE_SHELL_CODE_INPUT)->Result
);
SET_RESPONSE_LENGTH(sizeof(SHELL_STATUS));
break;
// ProcessesUtils:
case ALLOC_VIRTUAL_MEMORY:
OUTPUT(PALLOC_VIRTUAL_MEMORY_OUTPUT)->Status = VirtualAllocInProcess(
(HANDLE)INPUT(PALLOC_VIRTUAL_MEMORY_INPUT)->ProcessId,
(SIZE_T)INPUT(PALLOC_VIRTUAL_MEMORY_INPUT)->Size,
(PVOID*)&OUTPUT(PALLOC_VIRTUAL_MEMORY_OUTPUT)->VirtualAddress
);
SET_RESPONSE_LENGTH(sizeof(ALLOC_VIRTUAL_MEMORY_OUTPUT));
break;
case FREE_VIRTUAL_MEMORY:
*OUTPUT(PNTSTATUS) = VirtualFreeInProcess(
(HANDLE)INPUT(PFREE_VIRTUAL_MEMORY_INPUT)->ProcessId,
(PVOID)INPUT(PFREE_VIRTUAL_MEMORY_INPUT)->VirtualAddress
);
SET_RESPONSE_LENGTH(sizeof(NTSTATUS));
break;
case MAP_VIRTUAL_MEMORY:
OUTPUT(PMAP_VIRTUAL_MEMORY_OUTPUT)->MappedMemory = MapVirtualMemory(
(HANDLE)INPUT(PMAP_VIRTUAL_MEMORY_INPUT)->ProcessId,
INPUT(PMAP_VIRTUAL_MEMORY_INPUT)->VirtualAddress,
INPUT(PMAP_VIRTUAL_MEMORY_INPUT)->MapToVirtualAddress,
INPUT(PMAP_VIRTUAL_MEMORY_INPUT)->Size,
UserMode,
(PMDL*)&(OUTPUT(PMAP_VIRTUAL_MEMORY_OUTPUT)->Mdl)
);
SET_RESPONSE_LENGTH(sizeof(MAP_VIRTUAL_MEMORY_OUTPUT));
break;
case UNMAP_VIRTUAL_MEMORY:
UnmapVirtualMemory(
INPUT(PUNMAP_VIRTUAL_MEMORY_INPUT)->Mdl,
INPUT(PUNMAP_VIRTUAL_MEMORY_INPUT)->MappedMemory
);
break;
case READ_PROCESS_MEMORY:
*OUTPUT(PBOOL) = ReadProcessMemory(
(HANDLE)INPUT(PREAD_PROCESS_MEMORY_INPUT)->ProcessId,
(PVOID)INPUT(PREAD_PROCESS_MEMORY_INPUT)->VirtualAddress,
(PVOID)INPUT(PREAD_PROCESS_MEMORY_INPUT)->Buffer,
INPUT(PREAD_PROCESS_MEMORY_INPUT)->BytesToRead,
TRUE,
(MEMORY_ACCESS_TYPE)INPUT(PREAD_PROCESS_MEMORY_INPUT)->AccessType
);
SET_RESPONSE_LENGTH(sizeof(BOOL));
break;
case WRITE_PROCESS_MEMORY:
*OUTPUT(PBOOL) = WriteProcessMemory(
(HANDLE)INPUT(PWRITE_PROCESS_MEMORY_INPUT)->ProcessId,
(PVOID)INPUT(PWRITE_PROCESS_MEMORY_INPUT)->VirtualAddress,
(PVOID)INPUT(PWRITE_PROCESS_MEMORY_INPUT)->Buffer,
INPUT(PWRITE_PROCESS_MEMORY_INPUT)->BytesToWrite,
TRUE,
(MEMORY_ACCESS_TYPE)INPUT(PWRITE_PROCESS_MEMORY_INPUT)->AccessType
);
SET_RESPONSE_LENGTH(sizeof(BOOL));
break;
case RAISE_IOPL_BY_TF:
#ifdef _AMD64_
RaiseIOPLByTrapFrame();
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
case RESET_IOPL_BY_TF:
#ifdef _AMD64_
ResetIOPLByTrapFrame();
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
case RAISE_IOPL_BY_TF_SCAN:
RaiseIOPLByTrapFrameScan();
break;
case RESET_IOPL_BY_TF_SCAN:
ResetIOPLByTrapFrameScan();
break;
case RAISE_IOPL_BY_TSS:
#ifdef _X86_
RaiseIOPLByTSS();
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
case RESET_IOPL_BY_TSS:
#ifdef _x86_
ResetIOPLByTSS();
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
case RAISE_IOPM:
#ifdef _X86_
*OUTPUT(PNTSTATUS) = RaiseIOPM((HANDLE)*INPUT(PUINT64));
SET_RESPONSE_LENGTH(sizeof(NTSTATUS));
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
case RESET_IOPM:
#ifdef _X86_
*OUTPUT(PNTSTATUS) = ResetIOPM((HANDLE)*INPUT(PUINT64));
SET_RESPONSE_LENGTH(sizeof(NTSTATUS));
#else
Status = STATUS_NOT_IMPLEMENTED;
#endif
break;
// PCI:
case READ_PCI_CONFIG:
OUTPUT(PREAD_PCI_CONFIG_OUTPUT)->Status = ReadPciConfig(
INPUT(PREAD_PCI_CONFIG_INPUT)->PciAddress,
INPUT(PREAD_PCI_CONFIG_INPUT)->PciOffset,
(PVOID)INPUT(PREAD_PCI_CONFIG_INPUT)->Buffer,
INPUT(PREAD_PCI_CONFIG_INPUT)->BufferSize,
&OUTPUT(PREAD_PCI_CONFIG_OUTPUT)->BytesRead
);
SET_RESPONSE_LENGTH(sizeof(READ_PCI_CONFIG_OUTPUT));
break;
case WRITE_PCI_CONFIG:
OUTPUT(PWRITE_PCI_CONFIG_OUTPUT)->Status = WritePciConfig(
INPUT(PWRITE_PCI_CONFIG_INPUT)->PciAddress,
INPUT(PWRITE_PCI_CONFIG_INPUT)->PciOffset,
(PVOID)INPUT(PWRITE_PCI_CONFIG_INPUT)->Buffer,
INPUT(PWRITE_PCI_CONFIG_INPUT)->BufferSize,
&OUTPUT(PWRITE_PCI_CONFIG_OUTPUT)->BytesWritten
);
SET_RESPONSE_LENGTH(sizeof(WRITE_PCI_CONFIG_OUTPUT));
break;
default:
Status = STATUS_NOT_IMPLEMENTED;
break;
}
return Status;
#undef INPUT
#undef OUTPUT
#undef SET_RESPONSE_LENGTH
}
VOID
NTAPI
ApicInitializeLocalApic(ULONG Cpu)
{
APIC_BASE_ADRESS_REGISTER BaseRegister;
APIC_SPURIOUS_INERRUPT_REGISTER SpIntRegister;
LVT_REGISTER LvtEntry;
/* Enable the APIC if it wasn't yet */
BaseRegister.Long = __readmsr(MSR_APIC_BASE);
BaseRegister.Enable = 1;
BaseRegister.BootStrapCPUCore = (Cpu == 0);
__writemsr(MSR_APIC_BASE, BaseRegister.Long);
/* Set spurious vector and SoftwareEnable to 1 */
SpIntRegister.Long = ApicRead(APIC_SIVR);
SpIntRegister.Vector = APIC_SPURIOUS_VECTOR;
SpIntRegister.SoftwareEnable = 1;
SpIntRegister.FocusCPUCoreChecking = 0;
ApicWrite(APIC_SIVR, SpIntRegister.Long);
/* Read the version and save it globally */
if (Cpu == 0) ApicVersion = ApicRead(APIC_VER);
/* Set the mode to flat (max 8 CPUs supported!) */
ApicWrite(APIC_DFR, APIC_DF_Flat);
/* Set logical apic ID */
ApicWrite(APIC_LDR, ApicLogicalId(Cpu) << 24);
/* Set the spurious ISR */
KeRegisterInterruptHandler(APIC_SPURIOUS_VECTOR, ApicSpuriousService);
/* Create a template LVT */
LvtEntry.Long = 0;
LvtEntry.Vector = 0xFF;
LvtEntry.MessageType = APIC_MT_Fixed;
LvtEntry.DeliveryStatus = 0;
LvtEntry.RemoteIRR = 0;
LvtEntry.TriggerMode = APIC_TGM_Edge;
LvtEntry.Mask = 1;
LvtEntry.TimerMode = 0;
/* Initialize and mask LVTs */
ApicWrite(APIC_TMRLVTR, LvtEntry.Long);
ApicWrite(APIC_THRMLVTR, LvtEntry.Long);
ApicWrite(APIC_PCLVTR, LvtEntry.Long);
ApicWrite(APIC_EXT0LVTR, LvtEntry.Long);
ApicWrite(APIC_EXT1LVTR, LvtEntry.Long);
ApicWrite(APIC_EXT2LVTR, LvtEntry.Long);
ApicWrite(APIC_EXT3LVTR, LvtEntry.Long);
/* LINT0 */
LvtEntry.Vector = APIC_SPURIOUS_VECTOR;
LvtEntry.MessageType = APIC_MT_ExtInt;
ApicWrite(APIC_LINT0, LvtEntry.Long);
/* Enable LINT1 (NMI) */
LvtEntry.Mask = 0;
LvtEntry.Vector = APIC_NMI_VECTOR;
LvtEntry.MessageType = APIC_MT_NMI;
LvtEntry.TriggerMode = APIC_TGM_Level;
ApicWrite(APIC_LINT1, LvtEntry.Long);
/* Enable error LVTR */
LvtEntry.Vector = APIC_ERROR_VECTOR;
LvtEntry.MessageType = APIC_MT_Fixed;
ApicWrite(APIC_ERRLVTR, LvtEntry.Long);
/* Set the IRQL from the PCR */
ApicSetIrql(KeGetPcr()->Irql);
#ifdef APIC_LAZY_IRQL
/* Save the new hard IRQL in the IRR field */
KeGetPcr()->IRR = KeGetPcr()->Irql;
#endif
}
void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
{
u32 val;
/* all cores: allow caching of flash chip code and data
* (there are no cache-as-ram reliability concerns with family 14h)
*/
__writemsr (0x20c, (0x0100000000ull - CACHE_ROM_SIZE) | 5);
__writemsr (0x20d, (0x1000000000ull - CACHE_ROM_SIZE) | 0x800);
/* all cores: set pstate 0 (1600 MHz) early to save a few ms of boot time
*/
__writemsr (0xc0010062, 0);
if (!cpu_init_detectedx && boot_cpu()) {
post_code(0x30);
sb_Poweron_Init();
post_code(0x31);
kbc1100_early_init(CONFIG_SIO_PORT);
console_init();
}
/* Halt if there was a built in self test failure */
post_code(0x34);
report_bist_failure(bist);
/* Load MPB */
val = cpuid_eax(1);
printk(BIOS_DEBUG, "BSP Family_Model: %08x \n", val);
printk(BIOS_DEBUG, "cpu_init_detectedx = %08lx \n", cpu_init_detectedx);
post_code(0x35);
printk(BIOS_DEBUG, "agesawrapper_amdinitmmio ");
val = agesawrapper_amdinitmmio();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x37);
printk(BIOS_DEBUG, "agesawrapper_amdinitreset ");
val = agesawrapper_amdinitreset();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x39);
printk(BIOS_DEBUG, "agesawrapper_amdinitearly ");
val = agesawrapper_amdinitearly();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x40);
printk(BIOS_DEBUG, "agesawrapper_amdinitpost ");
val = agesawrapper_amdinitpost();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x41);
printk(BIOS_DEBUG, "agesawrapper_amdinitenv ");
val = agesawrapper_amdinitenv();
if (val)
printk(BIOS_DEBUG, "error level: %x \n", val);
else
printk(BIOS_DEBUG, "passed.\n");
post_code(0x50);
copy_and_run();
printk(BIOS_ERR, "Error: copy_and_run() returned!\n");
post_code(0x54); /* Should never see this post code. */
}
void WriteMsrReg(uint reg, uint64 val)
{
__writemsr(static_cast<ulong>(reg), val);
}
NTSTATUS VTxSoftwareStatus()
{
//
// Check the feature control bit MSR
//
IA32_FEATURE_CONTROL_MSR msr;
TO_ULL(msr) = __readmsr(MSR_IA32_FEATURE_CONTROL);
if (msr.Lock == 1)
{
// If the MSR is locked, it can't be modified
// If 'EnableVmxon' is unset, virtualization is not possible
if (msr.EnableVmxon == 0)
{
DbgLog("VMX is disabled in bios: MSR_IA32_FEATURE_CONTROL is 0x%llx\n", msr);
return STATUS_NOT_SUPPORTED;
}
}
else
{
// Force the lock to be on and enable VMXON
msr.Lock = 1;
msr.VmxonInSmx = 1;
msr.EnableVmxon = 1;
__writemsr(MSR_IA32_FEATURE_CONTROL, TO_ULL(msr));
}
//
// Setup CR0 correctly (Protected mode and paging must be enabled)
//
CR0_REG cr0;
TO_ULL(cr0) = __readcr0();
if (cr0.PE == 0 || cr0.PG == 0)
{
DbgLog("Error: Protected mode or paging is not set in CR0\n");
return STATUS_NOT_SUPPORTED;
}
else
{
// Required by first processors that supported VMX
cr0.NE = 1;
}
__writecr0(TO_ULL(cr0));
//
// Virtual Machine eXtensions Enable in CR4
// BIT #13 VMXE
//
__try
{
__writecr4(__readcr4() | (1 << 13));
}
__except (EXCEPTION_EXECUTE_HANDLER)
{
// Possible 'Privileged Instruction Exception' with CR4 bits
return GetExceptionCode();
}
return STATUS_SUCCESS;
}
void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
{
u32 val;
u8 reg8;
// all cores: allow caching of flash chip code and data
// (there are no cache-as-ram reliability concerns with family 14h)
__writemsr (0x20c, (0x0100000000ull - CONFIG_ROM_SIZE) | 5);
__writemsr (0x20d, (0x1000000000ull - CONFIG_ROM_SIZE) | 0x800);
// all cores: set pstate 0 (1600 MHz) early to save a few ms of boot time
__writemsr (0xc0010062, 0);
if (boot_cpu())
{
u8 reg8;
// SB800: program AcpiMmioEn to enable MMIO access to MiscCntrl register
outb(0x24, 0xCD6);
reg8 = inb(0xCD7);
reg8 |= 1;
reg8 &= ~(1 << 1);
outb(reg8, 0xCD7);
// program SB800 MiscCntrl
*(volatile u32 *)(0xFED80000+0xE00+0x40) &= ~((1 << 0) | (1 << 2)); /* 48Mhz */
*(volatile u32 *)(0xFED80000+0xE00+0x40) |= 1 << 1; /* 48Mhz */
}
// early enable of PrefetchEnSPIFromHost
if (boot_cpu())
{
__outdword (0xcf8, 0x8000a3b8);
__outdword (0xcfc, __indword (0xcfc) | 1 << 24);
}
// early enable of SPI 33 MHz fast mode read
if (boot_cpu())
{
volatile u32 *spiBase = (void *) 0xa0000000;
u32 save;
__outdword (0xcf8, 0x8000a3a0);
save = __indword (0xcfc);
__outdword (0xcfc, (u32) spiBase | 2); // set temp MMIO base
spiBase [3] = (spiBase [3] & ~(3 << 14)) | (1 << 14);
spiBase [0] |= 1 << 18; // fast read enable
__outdword (0xcfc, save); // clear temp base
}
if (!cpu_init_detectedx && boot_cpu()) {
post_code(0x30);
sb_poweron_init();
post_code(0x31);
f81865f_enable_serial(SERIAL_DEV, CONFIG_TTYS0_BASE);
console_init();
}
/* Halt if there was a built in self test failure */
post_code(0x34);
report_bist_failure(bist);
// Load MPB
val = cpuid_eax(1);
printk(BIOS_DEBUG, "BSP Family_Model: %08x \n", val);
printk(BIOS_DEBUG, "cpu_init_detectedx = %08lx \n", cpu_init_detectedx);
post_code(0x35);
val = agesawrapper_amdinitmmio();
post_code(0x37);
val = agesawrapper_amdinitreset();
if(val) {
printk(BIOS_DEBUG, "agesawrapper_amdinitreset failed: %x \n", val);
}
post_code(0x38);
printk(BIOS_DEBUG, "Got past sb800_early_setup\n");
post_code(0x39);
val = agesawrapper_amdinitearly ();
if(val) {
printk(BIOS_DEBUG, "agesawrapper_amdinitearly failed: %x \n", val);
}
printk(BIOS_DEBUG, "Got past agesawrapper_amdinitearly\n");
post_code(0x40);
val = agesawrapper_amdinitpost ();
if(val) {
printk(BIOS_DEBUG, "agesawrapper_amdinitpost failed: %x \n", val);
}
printk(BIOS_DEBUG, "Got past agesawrapper_amdinitpost\n");
post_code(0x41);
val = agesawrapper_amdinitenv ();
if(val) {
printk(BIOS_DEBUG, "agesawrapper_amdinitenv failed: %x \n", val);
}
printk(BIOS_DEBUG, "Got past agesawrapper_amdinitenv\n");
/* Initialize i8259 pic */
post_code(0x41);
setup_i8259 ();
/* Initialize i8254 timers */
post_code(0x42);
setup_i8254 ();
post_code(0x50);
copy_and_run(0);
post_code(0x54); // Should never see this post code.
}
void cache_as_ram_main(unsigned long bist, unsigned long cpu_init_detectedx)
{
u32 val;
/*
* All cores: allow caching of flash chip code and data
* (there are no cache-as-ram reliability concerns with family 14h)
*/
__writemsr (0x20c, (0x0100000000ull - CACHE_ROM_SIZE) | 5);
__writemsr (0x20d, (0x1000000000ull - CACHE_ROM_SIZE) | 0x800);
/* All cores: set pstate 0 (1600 MHz) early to save a few ms of boot time */
__writemsr (0xc0010062, 0);
amd_initmmio();
if (!cpu_init_detectedx && boot_cpu()) {
post_code(0x30);
sb_Poweron_Init();
post_code(0x31);
fintek_enable_serial(SERIAL_DEV, CONFIG_TTYS0_BASE);
console_init();
}
/* Halt if there was a built in self test failure */
post_code(0x34);
report_bist_failure(bist);
/* Load MPB */
val = cpuid_eax(1);
printk(BIOS_DEBUG, "BSP Family_Model: %08x\n", val);
printk(BIOS_DEBUG, "cpu_init_detectedx = %08lx\n", cpu_init_detectedx);
post_code(0x37);
agesawrapper_amdinitreset();
post_code(0x39);
agesawrapper_amdinitearly();
int s3resume = acpi_is_wakeup_s3();
if (!s3resume) {
post_code(0x40);
agesawrapper_amdinitpost();
post_code(0x42);
agesawrapper_amdinitenv();
amd_initenv();
} else { /* S3 detect */
printk(BIOS_INFO, "S3 detected\n");
post_code(0x60);
agesawrapper_amdinitresume();
agesawrapper_amds3laterestore();
post_code(0x61);
prepare_for_resume();
}
post_code(0x50);
copy_and_run();
printk(BIOS_ERR, "Error: copy_and_run() returned!\n");
post_code(0x54); /* Should never see this post code. */
}
VOID
NTAPI
KiInitializeCpu(PKIPCR Pcr)
{
ULONG64 Pat;
ULONG FeatureBits;
/* Initialize gs */
KiInitializeSegments();
/* Set GS base */
__writemsr(MSR_GS_BASE, (ULONG64)Pcr);
__writemsr(MSR_GS_SWAP, (ULONG64)Pcr);
/* Detect and set the CPU Type */
KiSetProcessorType();
/* Get the processor features for this CPU */
FeatureBits = KiGetFeatureBits();
/* Check if we support all needed features */
if ((FeatureBits & REQUIRED_FEATURE_BITS) != REQUIRED_FEATURE_BITS)
{
/* If not, bugcheck system */
FrLdrDbgPrint("CPU doesn't have needed features! Has: 0x%x, required: 0x%x\n",
FeatureBits, REQUIRED_FEATURE_BITS);
KeBugCheck(0);
}
/* Set DEP to always on */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
FeatureBits |= KF_NX_ENABLED;
/* Save feature bits */
Pcr->Prcb.FeatureBits = FeatureBits;
/* Enable fx save restore support */
__writecr4(__readcr4() | CR4_FXSR);
/* Enable XMMI exceptions */
__writecr4(__readcr4() | CR4_XMMEXCPT);
/* Enable Write-Protection */
__writecr0(__readcr0() | CR0_WP);
/* Disable fpu monitoring */
__writecr0(__readcr0() & ~CR0_MP);
/* Disable x87 fpu exceptions */
__writecr0(__readcr0() & ~CR0_NE);
/* LDT is unused */
__lldt(0);
/* Set the systemcall entry points */
__writemsr(MSR_LSTAR, (ULONG64)KiSystemCallEntry64);
__writemsr(MSR_CSTAR, (ULONG64)KiSystemCallEntry32);
__writemsr(MSR_STAR, ((ULONG64)KGDT64_R0_CODE << 32) |
((ULONG64)(KGDT64_R3_CMCODE|RPL_MASK) << 48));
/* Set the flags to be cleared when doing a syscall */
__writemsr(MSR_SYSCALL_MASK, EFLAGS_IF_MASK | EFLAGS_TF | EFLAGS_DF);
/* Enable syscall instruction and no-execute support */
__writemsr(MSR_EFER, __readmsr(MSR_EFER) | MSR_SCE | MSR_NXE);
/* Initialize the PAT */
Pat = (PAT_WB << 0) | (PAT_WC << 8) | (PAT_UCM << 16) | (PAT_UC << 24) |
(PAT_WB << 32) | (PAT_WC << 40) | (PAT_UCM << 48) | (PAT_UC << 56);
__writemsr(MSR_PAT, Pat);
}
static FORCEINLINE NTSTATUS
__InitHypercallPage()
{
ULONG eax = 'DEAD';
ULONG ebx = 'DEAD';
ULONG ecx = 'DEAD';
ULONG edx = 'DEAD';
ULONG Index;
ULONG HypercallMsr;
NTSTATUS Status;
Status = STATUS_UNSUCCESSFUL;
for (;;) {
CHAR Signature[13] = { 0 };
CpuId(__BaseLeaf, &eax, &ebx, &ecx, &edx);
*((PULONG)(Signature + 0)) = ebx;
*((PULONG)(Signature + 4)) = ecx;
*((PULONG)(Signature + 8)) = edx;
if (strcmp(Signature, XEN_SIGNATURE) == 0 &&
eax >= __BaseLeaf + 2)
break;
__BaseLeaf += 0x100;
if (__BaseLeaf > 0x40000100)
goto fail1;
}
CpuId(__BaseLeaf + 1, &eax, NULL, NULL, NULL);
__MajorVersion = (USHORT)(eax >> 16);
__MinorVersion = (USHORT)(eax & 0xFFFF);
LogVerbose("XEN %d.%d\n", __MajorVersion, __MinorVersion);
LogVerbose("INTERFACE 0x%08x\n", __XEN_INTERFACE_VERSION__);
if ((ULONG_PTR)__HypercallSection & (PAGE_SIZE - 1))
Hypercall = (PVOID)(((ULONG_PTR)__HypercallSection + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1));
else
Hypercall = (PVOID)__HypercallSection;
ASSERT3U(((ULONG_PTR)Hypercall & (PAGE_SIZE - 1)), ==, 0);
for (Index = 0; Index < MAXIMUM_HYPERCALL_PFN_COUNT; Index++) {
PHYSICAL_ADDRESS PhysicalAddress;
PhysicalAddress = MmGetPhysicalAddress((PUCHAR)Hypercall + (Index << PAGE_SHIFT));
__Pfn[Index] = (PFN_NUMBER)(PhysicalAddress.QuadPart >> PAGE_SHIFT);
}
CpuId(__BaseLeaf + 2, &eax, &ebx, NULL, NULL);
__PfnCount = eax;
ASSERT(__PfnCount <= MAXIMUM_HYPERCALL_PFN_COUNT);
HypercallMsr = ebx;
for (Index = 0; Index < __PfnCount; Index++) {
LogVerbose("HypercallPfn[%d]: %p\n", Index, (PVOID)__Pfn[Index]);
__writemsr(HypercallMsr, (ULONG64)__Pfn[Index] << PAGE_SHIFT);
}
return STATUS_SUCCESS;
fail1:
LogError("fail1 (%08x)", Status);
return Status;
}
VOID
NTAPI
HalpInitializeTsc(VOID)
{
ULONG_PTR Flags;
KIDTENTRY OldIdtEntry, *IdtPointer;
PKPCR Pcr = KeGetPcr();
UCHAR RegisterA, RegisterB;
/* Check if the CPU supports RDTSC */
if (!(KeGetCurrentPrcb()->FeatureBits & KF_RDTSC))
{
KeBugCheck(HAL_INITIALIZATION_FAILED);
}
/* Save flags and disable interrupts */
Flags = __readeflags();
_disable();
/* Enable the periodic interrupt in the CMOS */
RegisterB = HalpReadCmos(RTC_REGISTER_B);
HalpWriteCmos(RTC_REGISTER_B, RegisterB | RTC_REG_B_PI);
/* Modify register A to RTC_MODE to get SAMPLE_FREQENCY */
RegisterA = HalpReadCmos(RTC_REGISTER_A);
RegisterA = (RegisterA & 0xF0) | RTC_MODE;
HalpWriteCmos(RTC_REGISTER_A, RegisterA);
/* Save old IDT entry */
IdtPointer = KiGetIdtEntry(Pcr, HalpRtcClockVector);
OldIdtEntry = *IdtPointer;
/* Set the calibration ISR */
KeRegisterInterruptHandler(HalpRtcClockVector, TscCalibrationISR);
/* Reset TSC value to 0 */
__writemsr(MSR_RDTSC, 0);
/* Enable the timer interupt */
HalEnableSystemInterrupt(HalpRtcClockVector, CLOCK_LEVEL, Latched);
/* Read register C, so that the next interrupt can happen */
HalpReadCmos(RTC_REGISTER_C);;
/* Wait for completion */
_enable();
while (TscCalibrationPhase < NUM_SAMPLES) _ReadWriteBarrier();
_disable();
/* Disable the periodic interrupt in the CMOS */
HalpWriteCmos(RTC_REGISTER_B, RegisterB & ~RTC_REG_B_PI);
/* Disable the timer interupt */
HalDisableSystemInterrupt(HalpRtcClockVector, CLOCK_LEVEL);
/* Restore old IDT entry */
*IdtPointer = OldIdtEntry;
/* Calculate an average, using simplified linear regression */
HalpCpuClockFrequency.QuadPart = DoLinearRegression(NUM_SAMPLES - 1,
TscCalibrationArray);
/* Restore flags */
__writeeflags(Flags);
}
