VOID
NTAPI
HalpReturnTss(VOID)
{
PKGDTENTRY TssGdt;
PKTSS TssBase;
//
// Get the original TSS
//
TssGdt = &((PKIPCR)KeGetPcr())->GDT[HalpSavedTss / sizeof(KGDTENTRY)];
TssBase = (PKTSS)(ULONG_PTR)(TssGdt->BaseLow |
TssGdt->HighWord.Bytes.BaseMid << 16 |
TssGdt->HighWord.Bytes.BaseHi << 24);
//
// Switch to it
//
KeGetPcr()->TSS = TssBase;
//
// Set it up
//
TssGdt->HighWord.Bits.Type = I386_TSS;
TssGdt->HighWord.Bits.Pres = 1;
TssGdt->HighWord.Bits.Dpl = 0;
//
// Load old TSS
//
Ke386SetTr(HalpSavedTss);
}
ULONG_PTR
NTAPI
HalSetProfileInterval(IN ULONG_PTR Interval)
{
ULONGLONG TimerInterval;
ULONGLONG FixedInterval;
FixedInterval = (ULONGLONG)Interval;
/* Check bounds */
if (FixedInterval < HalMinProfileInterval)
{
FixedInterval = HalMinProfileInterval;
}
else if (FixedInterval > HalMaxProfileInterval)
{
FixedInterval = HalMaxProfileInterval;
}
/* Remember interval */
HalCurProfileInterval = FixedInterval;
/* Recalculate interval for APIC */
TimerInterval = FixedInterval * KeGetPcr()->HalReserved[HAL_PROFILING_MULTIPLIER] / HalMaxProfileInterval;
/* Remember recalculated interval in PCR */
KeGetPcr()->HalReserved[HAL_PROFILING_INTERVAL] = (ULONG)TimerInterval;
/* And set it */
ApicWrite(APIC_TICR, (ULONG)TimerInterval);
return Interval;
}
VOID
NTAPI
HalInitializeProfiling(VOID)
{
KeGetPcr()->HalReserved[HAL_PROFILING_INTERVAL] = HalCurProfileInterval;
KeGetPcr()->HalReserved[HAL_PROFILING_MULTIPLIER] = 1; /* TODO: HACK */
}
ULONG_PTR
FASTCALL
KiExitV86Mode(IN PKTRAP_FRAME TrapFrame)
{
PKV8086_STACK_FRAME StackFrame;
PKTHREAD Thread;
PKTRAP_FRAME PmTrapFrame;
PKV86_FRAME V86Frame;
PFX_SAVE_AREA NpxFrame;
/* Get the stack frame back */
StackFrame = CONTAINING_RECORD(TrapFrame->Esi, KV8086_STACK_FRAME, V86Frame);
PmTrapFrame = &StackFrame->TrapFrame;
V86Frame = &StackFrame->V86Frame;
NpxFrame = &StackFrame->NpxArea;
/* Copy the FPU frame back */
Thread = KeGetCurrentThread();
RtlCopyMemory(KiGetThreadNpxArea(Thread), NpxFrame, sizeof(FX_SAVE_AREA));
/* Set initial stack back */
Thread->InitialStack = (PVOID)((ULONG_PTR)V86Frame->ThreadStack + sizeof(FX_SAVE_AREA));
/* Set ESP0 back in the KTSS */
KeGetPcr()->TSS->Esp0 = (ULONG_PTR)&PmTrapFrame->V86Es;
/* Restore TEB addresses */
Thread->Teb = V86Frame->ThreadTeb;
KiSetTebBase(KeGetPcr(), V86Frame->ThreadTeb);
/* Enable interrupts and return a pointer to the trap frame */
_enable();
return (ULONG)PmTrapFrame;
}
VOID
NTAPI
HalpSetupRealModeIoPermissionsAndTask(VOID)
{
//
// Switch to valid TSS
//
HalpBorrowTss();
//
// Save a copy of the I/O Map and delete it
//
HalpSavedIoMap = (PUSHORT)&(KeGetPcr()->TSS->IoMaps[0]);
HalpStoreAndClearIopm();
//
// Save the IOPM and switch to the real-mode one
//
HalpSavedIopmBase = KeGetPcr()->TSS->IoMapBase;
KeGetPcr()->TSS->IoMapBase = KiComputeIopmOffset(1);
//
// Save our stack pointer
//
HalpSavedEsp0 = KeGetPcr()->TSS->Esp0;
}
VOID
NTAPI
HalpBorrowTss(VOID)
{
USHORT Tss;
PKGDTENTRY TssGdt;
ULONG_PTR TssLimit;
PKTSS TssBase;
//
// Get the current TSS and its GDT entry
//
Tss = Ke386GetTr();
TssGdt = &((PKIPCR)KeGetPcr())->GDT[Tss / sizeof(KGDTENTRY)];
//
// Get the KTSS limit and check if it has IOPM space
//
TssLimit = TssGdt->LimitLow | TssGdt->HighWord.Bits.LimitHi << 16;
//
// If the KTSS doesn't have enough space this is probably an NMI or DF
//
if (TssLimit > IOPM_SIZE)
{
//
// We are good to go
//
HalpSavedTss = 0;
return;
}
//
// Get the "real" TSS
//
TssGdt = &((PKIPCR)KeGetPcr())->GDT[KGDT_TSS / sizeof(KGDTENTRY)];
TssBase = (PKTSS)(ULONG_PTR)(TssGdt->BaseLow |
TssGdt->HighWord.Bytes.BaseMid << 16 |
TssGdt->HighWord.Bytes.BaseHi << 24);
//
// Switch to it
//
KeGetPcr()->TSS = TssBase;
//
// Set it up
//
TssGdt->HighWord.Bits.Type = I386_TSS;
TssGdt->HighWord.Bits.Pres = 1;
TssGdt->HighWord.Bits.Dpl = 0;
//
// Load new TSS and return old one
//
Ke386SetTr(KGDT_TSS);
HalpSavedTss = Tss;
}
/**********************************************************************
* NAME EXPORTED
* KfLowerIrql
*
* DESCRIPTION
* Restores the irq level on the current processor
*
* ARGUMENTS
* NewIrql = Irql to lower to
*
* RETURN VALUE
* None
*
* NOTES
* Uses fastcall convention
*/
VOID FASTCALL
KfLowerIrql (KIRQL NewIrql)
{
DPRINT("KfLowerIrql(NewIrql %d)\n", NewIrql);
if (NewIrql > KeGetPcr()->Irql)
{
DbgPrint ("(%s:%d) NewIrql %x CurrentIrql %x\n",
__FILE__, __LINE__, NewIrql, KeGetPcr()->Irql);
KeBugCheck(IRQL_NOT_LESS_OR_EQUAL);
for(;;);
}
HalpLowerIrql(NewIrql);
}
VOID
NTAPI
KiSwapProcess(IN PKPROCESS NewProcess,
IN PKPROCESS OldProcess)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
#ifdef CONFIG_SMP
LONG SetMember;
/* Update active processor mask */
SetMember = (LONG)Pcr->SetMember;
InterlockedXor((PLONG)&NewProcess->ActiveProcessors, SetMember);
InterlockedXor((PLONG)&OldProcess->ActiveProcessors, SetMember);
#endif
/* Check for new LDT */
if (NewProcess->LdtDescriptor.LimitLow != OldProcess->LdtDescriptor.LimitLow)
{
/* Not handled yet */
UNIMPLEMENTED_DBGBREAK();
return;
}
/* Update CR3 */
__writecr3(NewProcess->DirectoryTableBase[0]);
/* Clear GS */
Ke386SetGs(0);
/* Update IOPM offset */
Pcr->TSS->IoMapBase = NewProcess->IopmOffset;
}
VOID
HalpInitializeClock(VOID)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
ULONG ClockInterval;
SP804_CONTROL_REGISTER ControlRegister;
/* Setup the clock and profile interrupt */
Pcr->InterruptRoutine[CLOCK2_LEVEL] = HalpStallInterrupt;
/*
* Configure the interval to 10ms
* (INTERVAL (10ms) * TIMCLKfreq (1MHz))
* --------------------------------------- == 10^4
* (TIMCLKENXdiv (1) * PRESCALEdiv (1))
*/
ClockInterval = 0x2710;
/* Configure the timer */
ControlRegister.AsUlong = 0;
ControlRegister.Wide = TRUE;
ControlRegister.Periodic = TRUE;
ControlRegister.Interrupt = TRUE;
ControlRegister.Enabled = TRUE;
/* Enable the timer */
WRITE_REGISTER_ULONG(TIMER0_LOAD, ClockInterval);
WRITE_REGISTER_ULONG(TIMER0_CONTROL, ControlRegister.AsUlong);
}
VOID
HalpExecuteIrqs(KIRQL NewIrql)
{
ULONG IrqLimit, i;
IrqLimit = min(PROFILE_LEVEL - NewIrql, NR_IRQS);
/*
* For each irq if there have been any deferred interrupts then now
* dispatch them.
*/
for (i = 0; i < IrqLimit; i++)
{
if (HalpPendingInterruptCount[i] > 0)
{
KeGetPcr()->Irql = (KIRQL)IRQ_TO_DIRQL(i);
while (HalpPendingInterruptCount[i] > 0)
{
/*
* For each deferred interrupt execute all the handlers at DIRQL.
*/
HalpPendingInterruptCount[i]--;
//HalpHardwareInt[i]();
}
//KeGetPcr()->Irql--;
//HalpEndSystemInterrupt(KeGetPcr()->Irql);
}
}
}
VOID NTAPI
HalInitializeProcessor(ULONG ProcessorNumber,
PLOADER_PARAMETER_BLOCK LoaderBlock)
{
DPRINT("HalInitializeProcessor(%lu %p)\n", ProcessorNumber, LoaderBlock);
KeGetPcr()->StallScaleFactor = INITIAL_STALL_COUNT;
}
VOID
FASTCALL
KiInitializeTss(IN PKTSS64 Tss,
IN UINT64 Stack)
{
PKGDTENTRY64 TssEntry;
/* Get pointer to the GDT entry */
TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
/* Initialize the GDT entry */
KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
/* Zero out the TSS */
RtlZeroMemory(Tss, sizeof(KTSS64));
/* FIXME: I/O Map? */
Tss->IoMapBase = 0x68;
/* Setup ring 0 stack pointer */
Tss->Rsp0 = Stack;
/* Setup a stack for Double Fault Traps */
Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
/* Setup a stack for CheckAbort Traps */
Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
/* Setup a stack for NMI Traps */
Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
/* Load the task register */
__ltr(KGDT64_SYS_TSS);
}
/*
* @implemented
*/
BOOLEAN
NTAPI
Ke386QueryIoAccessMap(IN ULONG MapNumber,
IN PKIO_ACCESS_MAP IopmBuffer)
{
ULONG i;
PVOID Map;
PUCHAR p;
if (MapNumber > IOPM_COUNT)
return FALSE;
if (MapNumber == IO_ACCESS_MAP_NONE)
{
// no access, simply return a map of all 1s
p = (PUCHAR)IopmBuffer;
for (i = 0; i < IOPM_SIZE; i++) {
p[i] = (UCHAR)-1;
}
}
else
{
// copy the bits
Map = (PVOID)&(KeGetPcr()->TSS->IoMaps[MapNumber-1].IoMap);
RtlMoveMemory((PVOID)IopmBuffer, Map, IOPM_SIZE);
}
return TRUE;
}
FORCEINLINE
VOID
ApicLowerIrql(KIRQL Irql)
{
__writefsbyte(FIELD_OFFSET(KPCR, Irql), Irql);
/* Is the new Irql lower than set in the TPR? */
if (Irql < KeGetPcr()->IRR)
{
/* Save the new hard IRQL in the IRR field */
KeGetPcr()->IRR = Irql;
/* Need to lower it back */
ApicWrite(APIC_TPR, IrqlToTpr(Irql));
}
}
VOID
FORCEINLINE
KiCommonExit(IN PKTRAP_FRAME TrapFrame, BOOLEAN SkipPreviousMode)
{
/* Disable interrupts until we return */
_disable();
/* Check for APC delivery */
KiCheckForApcDelivery(TrapFrame);
/* Restore the SEH handler chain */
KeGetPcr()->NtTib.ExceptionList = TrapFrame->ExceptionList;
/* Check if there are active debug registers */
if (__builtin_expect(TrapFrame->Dr7 & ~DR7_RESERVED_MASK, 0))
{
/* Check if the frame was from user mode or v86 mode */
if ((TrapFrame->SegCs & MODE_MASK) ||
(TrapFrame->EFlags & EFLAGS_V86_MASK))
{
/* Handle debug registers */
KiHandleDebugRegistersOnTrapExit(TrapFrame);
}
}
/* Debugging checks */
KiExitTrapDebugChecks(TrapFrame, SkipPreviousMode);
}
BOOLEAN
HalAllProcessorsStarted (
VOID
)
{
if (KeGetPcr()->Number == 0) {
if (HalpFeatureBits & HAL_PERF_EVENTS) {
//
// Enable local perf events on each processor
//
HalpGenericCall (
HalpEnablePerfInterupt,
(ULONG) NULL,
HalpActiveProcessors
);
}
if (HalpFeatureBits & HAL_NO_SPECULATION) {
//
// Processor doesn't perform speculative execeution,
// remove fences in critical code paths
//
HalpRemoveFences ();
}
}
return TRUE;
}
VOID
FASTCALL
KiSwapContextEntry(IN PKSWITCHFRAME SwitchFrame,
IN ULONG_PTR OldThreadAndApcFlag)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
PKTHREAD OldThread, NewThread;
/* Save APC bypass disable */
SwitchFrame->ApcBypassDisable = OldThreadAndApcFlag & 3;
SwitchFrame->ExceptionList = Pcr->NtTib.ExceptionList;
/* Increase context switch count and check if tracing is enabled */
Pcr->ContextSwitches++;
if (Pcr->PerfGlobalGroupMask)
{
/* We don't support this yet on x86 either */
DPRINT1("WMI Tracing not supported\n");
ASSERT(FALSE);
}
/* Get thread pointers */
OldThread = (PKTHREAD)(OldThreadAndApcFlag & ~3);
NewThread = Pcr->PrcbData.CurrentThread;
/* Get the old thread and set its kernel stack */
OldThread->KernelStack = SwitchFrame;
/* Do the switch */
KiSwitchThreads(OldThread, NewThread->KernelStack);
}
VOID
FORCEINLINE
KiCommonExit(IN PKTRAP_FRAME TrapFrame, const ULONG Flags)
{
/* Disable interrupts until we return */
_disable();
/* Check for APC delivery */
KiCheckForApcDelivery(TrapFrame);
/* Debugging checks */
KiExitTrapDebugChecks(TrapFrame, Flags);
/* Restore the SEH handler chain */
KeGetPcr()->NtTib.ExceptionList = TrapFrame->ExceptionList;
/* Check if there are active debug registers */
if (__builtin_expect(TrapFrame->Dr7 & ~DR7_RESERVED_MASK, 0))
{
/* Not handled yet */
DbgPrint("Need Hardware Breakpoint Support!\n");
DbgBreakPoint();
while (TRUE);
}
}
KIRQL NTAPI KeGetCurrentIrql (VOID)
/*
* PURPOSE: Returns the current irq level
* RETURNS: The current irq level
*/
{
return(KeGetPcr()->Irql);
}
VOID
HalpLowerIrql(KIRQL NewIrql)
{
if (NewIrql >= PROFILE_LEVEL)
{
KeGetPcr()->Irql = NewIrql;
return;
}
HalpExecuteIrqs(NewIrql);
if (NewIrql >= DISPATCH_LEVEL)
{
KeGetPcr()->Irql = NewIrql;
return;
}
KeGetPcr()->Irql = DISPATCH_LEVEL;
if (((PKIPCR)KeGetPcr())->HalReserved[HAL_DPC_REQUEST])
{
((PKIPCR)KeGetPcr())->HalReserved[HAL_DPC_REQUEST] = FALSE;
KiDispatchInterrupt();
}
KeGetPcr()->Irql = APC_LEVEL;
if (NewIrql == APC_LEVEL)
{
return;
}
if (KeGetCurrentThread() != NULL &&
KeGetCurrentThread()->ApcState.KernelApcPending)
{
KiDeliverApc(KernelMode, NULL, NULL);
}
KeGetPcr()->Irql = PASSIVE_LEVEL;
}
VOID FASTCALL
HalClearSoftwareInterrupt(
IN KIRQL Request)
{
switch (Request)
{
case APC_LEVEL:
((PKIPCR)KeGetPcr())->HalReserved[HAL_APC_REQUEST] = FALSE;
break;
case DISPATCH_LEVEL:
((PKIPCR)KeGetPcr())->HalReserved[HAL_DPC_REQUEST] = FALSE;
break;
default:
DbgBreakPoint();
}
}
VOID
NTAPI
HalpInitProcessor(
IN ULONG ProcessorNumber,
IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
/* Set default IDR */
KeGetPcr()->IDR = 0xFFFFFFFB;
}
VOID
NTAPI
HalpCalibrateStallExecution(VOID)
{
// Timer interrupt is now active
HalpInitializeTsc();
KeGetPcr()->StallScaleFactor = (ULONG)(HalpCpuClockFrequency.QuadPart / 1000000);
}
KIRQL FASTCALL
KfRaiseIrql (KIRQL NewIrql)
{
KIRQL OldIrql;
DPRINT("KfRaiseIrql(NewIrql %d)\n", NewIrql);
if (NewIrql < KeGetPcr()->Irql)
{
DbgPrint ("%s:%d CurrentIrql %x NewIrql %x\n",
__FILE__,__LINE__,KeGetPcr()->Irql,NewIrql);
KeBugCheck (IRQL_NOT_GREATER_OR_EQUAL);
for(;;);
}
OldIrql = KeGetPcr()->Irql;
KeGetPcr()->Irql = NewIrql;
return OldIrql;
}
BOOLEAN
FASTCALL
KiSwapContextExit(IN PKTHREAD OldThread,
IN PKSWITCHFRAME SwitchFrame)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
PKPROCESS OldProcess, NewProcess;
PKTHREAD NewThread;
ARM_TTB_REGISTER TtbRegister;
/* We are on the new thread stack now */
NewThread = Pcr->PrcbData.CurrentThread;
/* Now we are the new thread. Check if it's in a new process */
OldProcess = OldThread->ApcState.Process;
NewProcess = NewThread->ApcState.Process;
if (OldProcess != NewProcess)
{
TtbRegister.AsUlong = NewProcess->DirectoryTableBase[0];
ASSERT(TtbRegister.Reserved == 0);
KeArmTranslationTableRegisterSet(TtbRegister);
}
/* Increase thread context switches */
NewThread->ContextSwitches++;
/* Load data from switch frame */
Pcr->NtTib.ExceptionList = SwitchFrame->ExceptionList;
/* DPCs shouldn't be active */
if (Pcr->PrcbData.DpcRoutineActive)
{
/* Crash the machine */
KeBugCheckEx(ATTEMPTED_SWITCH_FROM_DPC,
(ULONG_PTR)OldThread,
(ULONG_PTR)NewThread,
(ULONG_PTR)OldThread->InitialStack,
0);
}
/* Kernel APCs may be pending */
if (NewThread->ApcState.KernelApcPending)
{
/* Are APCs enabled? */
if (!NewThread->SpecialApcDisable)
{
/* Request APC delivery */
if (SwitchFrame->ApcBypassDisable) HalRequestSoftwareInterrupt(APC_LEVEL);
return TRUE;
}
}
/* Return */
return FALSE;
}
VOID
NTAPI
HalpEnableInterruptHandler(IN UCHAR Flags,
IN ULONG BusVector,
IN ULONG SystemVector,
IN KIRQL Irql,
IN PVOID Handler,
IN KINTERRUPT_MODE Mode)
{
/* Register the routine */
((PKIPCR)KeGetPcr())->InterruptRoutine[Irql] = Handler;
}
/*
* @implemented
*/
BOOLEAN
NTAPI
Ke386SetIoAccessMap(IN ULONG MapNumber,
IN PKIO_ACCESS_MAP IopmBuffer)
{
PKPROCESS CurrentProcess;
PKPRCB Prcb;
PVOID pt;
if ((MapNumber > IOPM_COUNT) || (MapNumber == IO_ACCESS_MAP_NONE))
return FALSE;
Prcb = KeGetCurrentPrcb();
// Copy the IOP map and load the map for the current process.
pt = &(KeGetPcr()->TSS->IoMaps[MapNumber-1].IoMap);
RtlMoveMemory(pt, (PVOID)IopmBuffer, IOPM_SIZE);
CurrentProcess = Prcb->CurrentThread->ApcState.Process;
KeGetPcr()->TSS->IoMapBase = CurrentProcess->IopmOffset;
return TRUE;
}
BOOLEAN NTAPI
HalBeginSystemInterrupt (KIRQL Irql,
ULONG Vector,
PKIRQL OldIrql)
{
ULONG irq;
if (Vector < IRQ_BASE || Vector >= IRQ_BASE + NR_IRQS)
{
return(FALSE);
}
irq = Vector - IRQ_BASE;
pic_mask_intr.both |= ((1 << irq) & 0xfffe); // do not disable the timer interrupt
if (irq < 8)
{
WRITE_PORT_UCHAR((PUCHAR)0x21, (UCHAR)(pic_mask.master|pic_mask_intr.master));
WRITE_PORT_UCHAR((PUCHAR)0x20, 0x20);
}
else
{
WRITE_PORT_UCHAR((PUCHAR)0xa1, (UCHAR)(pic_mask.slave|pic_mask_intr.slave));
/* Send EOI to the PICs */
WRITE_PORT_UCHAR((PUCHAR)0x20,0x20);
WRITE_PORT_UCHAR((PUCHAR)0xa0,0x20);
}
#if 0
if (KeGetPcr()->Irql >= Irql)
{
HalpPendingInterruptCount[irq]++;
return(FALSE);
}
#endif
*OldIrql = KeGetPcr()->Irql;
KeGetPcr()->Irql = Irql;
return(TRUE);
}
VOID
NTAPI
HalpRestoreIoPermissionsAndTask(VOID)
{
//
// Restore the stack pointer
//
KeGetPcr()->TSS->Esp0 = HalpSavedEsp0;
//
// Restore the I/O Map
//
HalpRestoreIopm();
//
// Restore the IOPM
//
KeGetPcr()->TSS->IoMapBase = HalpSavedIopmBase;
//
// Restore the TSS
//
if (HalpSavedTss) HalpReturnTss();
}
VOID
NTAPI
KeStallExecutionProcessor(ULONG MicroSeconds)
{
ULONG64 StartTime, EndTime;
/* Get the initial time */
StartTime = __rdtsc();
/* Calculate the ending time */
EndTime = StartTime + KeGetPcr()->StallScaleFactor * MicroSeconds;
/* Loop until time is elapsed */
while (__rdtsc() < EndTime);
}
