/*
* @implemented
*/
BOOLEAN
NTAPI
KeSynchronizeExecution(IN OUT PKINTERRUPT Interrupt,
IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
IN PVOID SynchronizeContext OPTIONAL)
{
BOOLEAN Success;
KIRQL OldIrql;
/* Raise IRQL */
OldIrql = KfRaiseIrql(Interrupt->SynchronizeIrql);
/* Acquire interrupt spinlock */
KeAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
/* Call the routine */
Success = SynchronizeRoutine(SynchronizeContext);
/* Release lock */
KeReleaseSpinLockFromDpcLevel(Interrupt->ActualLock);
/* Lower IRQL */
KfLowerIrql(OldIrql);
/* Return status */
return Success;
}
/*
* @implemented
*/
KIRQL
FASTCALL
KfAcquireSpinLock(PKSPIN_LOCK SpinLock)
{
/* Simply raise to dispatch */
return KfRaiseIrql(DISPATCH_LEVEL);
}
/*
* @implemented
*/
KIRQL
FASTCALL
KeAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber)
{
/* Simply raise to dispatch */
return KfRaiseIrql(DISPATCH_LEVEL);
}
/*
* @implemented
*/
VOID
NTAPI
KeRaiseIrql(KIRQL NewIrql,
PKIRQL OldIrql)
{
/* Call the fastcall function */
*OldIrql = KfRaiseIrql(NewIrql);
}
/*
* @implemented
*/
VOID
FASTCALL
KeAcquireInStackQueuedSpinLockRaiseToSynch(IN PKSPIN_LOCK SpinLock,
IN PKLOCK_QUEUE_HANDLE LockHandle)
{
/* Simply raise to dispatch */
LockHandle->OldIrql = KfRaiseIrql(DISPATCH_LEVEL);
}
VOID
HalpPCIAcquireType2Lock (
PKSPIN_LOCK SpinLock,
PKIRQL Irql
)
{
if (!HalpDoingCrashDump) {
*Irql = KfRaiseIrql (HIGH_LEVEL);
KiAcquireSpinLock (SpinLock);
} else {
*Irql = HIGH_LEVEL;
}
}
VOID
FASTCALL
KiTimedChainedDispatch2ndLvl(
PKINTERRUPT Interrupt
)
/*++
Routine Description:
This function performs the same function as KiChainedDispatch2ndLvl
except that it is written in C instead of assembly code and includes
code for timing ISRs.
I'd be interested in seeing some benchmarks to show if the assembly
code is actually faster. The Acquire/Release spinlock could be
inlined fairly easily.
Arguments:
Interrupt - Supplies a pointer to a control object of type interrupt.
Return Value:
None.
--*/
{
BOOLEAN Handled = FALSE;
PVOID ListEnd = &Interrupt->InterruptListEntry.Flink;
//
//BEGINTIMING
PKPRCB Prcb = KeGetCurrentPrcb();
ULONGLONG StartTimeHigher;
ULONGLONG StartTime;
ULONGLONG TimeHigher;
ULONGLONG ElapsedTime;
//BEGINTIMINGend
//
// For each interrupt on this chain.
//
do {
//
// If the current IRQL (IRQL raised to by nature of taking this
// interrupt) is not equal to the Synchronization IRQL required
// for this interrupt, raise to the appropriate level.
//
if (Interrupt->Irql != Interrupt->SynchronizeIrql) {
KfRaiseIrql(Interrupt->SynchronizeIrql);
}
//BEGINTIMING
StartTimeHigher = Prcb->IsrTime;
StartTime = RDTSC();
//BEGINTIMINGend
//
// Acquire the interrupt lock.
//
KiAcquireSpinLock(Interrupt->ActualLock);
//
// Call the Interrupt Service Routine.
//
Handled |= Interrupt->ServiceRoutine(Interrupt,
Interrupt->ServiceContext);
//
// Release the interrupt lock.
//
KiReleaseSpinLock(Interrupt->ActualLock);
//ENDTIMING
//
// ElapsedTime is time since we started looking at this element
// on the chain. (ie the current interrupt object).
//
ElapsedTime = RDTSC() - StartTime;
//
// TimeHigher is the amount Prcb->IsrTime has increased since we
// begin servicing this interrupt object, ie the amount of time
// spent in higher level ISRs.
//
TimeHigher = Prcb->IsrTime - StartTimeHigher;
//
// Adjust ElapsedTime to time spent on this interrupt object, excluding
// higher level ISRs.
//
ElapsedTime -= TimeHigher;
if (ElapsedTime > KiIsrTscLimit) {
//
// If there is a debugger attached, breakin. Otherwise do nothing.
// N.B. bugchecking is another possibility.
//
if (KdDebuggerEnabled) {
DbgPrint("KE; ISR time limit exceeded (intobj %p)\n",
Interrupt);
DbgBreakPoint();
}
}
//
// Update time spent processing interrupts. This doesn't need
// to be atomic as it doesn't matter if it's a little bit lossy.
// (Though a simple atomic add would do, it's per processor and
// at IRQL > DISPATCH_LEVEL so it doesn't need to be locked).
//
Prcb->IsrTime += ElapsedTime;
//ENDTIMINGend
//
// If IRQL was raised, lower to the previous level.
//
if (Interrupt->Irql != Interrupt->SynchronizeIrql) {
KfLowerIrql(Interrupt->Irql);
}
if ((Handled != FALSE) &&
(Interrupt->Mode == LevelSensitive)) {
//
// The interrupt has been handled.
//
return;
}
//
// If this is the last entry on the chain, get out, otherwise
// advance to the next entry.
//
if (Interrupt->InterruptListEntry.Flink == ListEnd) {
ASSERT(Interrupt->Mode != LevelSensitive);
//
// We should only get to the end of the list if
// (a) interrupts are on this chain are level sensitive and
// no ISR handled the request. This is a system fatal
// condition, or,
// (b) the chain has edge triggered interrupts in which case
// we must run the chain repeatedly until no ISR services
// the request.
//
// Question: Do we actually have chained edge triggered
// interrupts anymore?
//
if (Handled == FALSE) {
break;
}
}
Interrupt = CONTAINING_RECORD(Interrupt->InterruptListEntry.Flink,
KINTERRUPT,
InterruptListEntry);
} while (TRUE);
}
VOID
FASTCALL
KiChainedDispatch(IN PKTRAP_FRAME TrapFrame,
IN PKINTERRUPT Interrupt)
{
KIRQL OldIrql, OldInterruptIrql = 0;
BOOLEAN Handled;
PLIST_ENTRY NextEntry, ListHead;
/* Increase interrupt count */
KeGetCurrentPrcb()->InterruptCount++;
/* Begin the interrupt, making sure it's not spurious */
if (HalBeginSystemInterrupt(Interrupt->Irql,
Interrupt->Vector,
&OldIrql))
{
/* Get list pointers */
ListHead = &Interrupt->InterruptListEntry;
NextEntry = ListHead; /* The head is an entry! */
while (TRUE)
{
/* Check if this interrupt's IRQL is higher than the current one */
if (Interrupt->SynchronizeIrql > Interrupt->Irql)
{
/* Raise to higher IRQL */
OldInterruptIrql = KfRaiseIrql(Interrupt->SynchronizeIrql);
}
/* Acquire interrupt lock */
KxAcquireSpinLock(Interrupt->ActualLock);
/* Call the ISR */
Handled = Interrupt->ServiceRoutine(Interrupt,
Interrupt->ServiceContext);
/* Release interrupt lock */
KxReleaseSpinLock(Interrupt->ActualLock);
/* Check if this interrupt's IRQL is higher than the current one */
if (Interrupt->SynchronizeIrql > Interrupt->Irql)
{
/* Lower the IRQL back */
ASSERT(OldInterruptIrql == Interrupt->Irql);
KfLowerIrql(OldInterruptIrql);
}
/* Check if the interrupt got handled and it's level */
if ((Handled) && (Interrupt->Mode == LevelSensitive)) break;
/* What's next? */
NextEntry = NextEntry->Flink;
/* Is this the last one? */
if (NextEntry == ListHead)
{
/* Level should not have gotten here */
if (Interrupt->Mode == LevelSensitive) break;
/* As for edge, we can only exit once nobody can handle the interrupt */
if (!Handled) break;
}
/* Get the interrupt object for the next pass */
Interrupt = CONTAINING_RECORD(NextEntry, KINTERRUPT, InterruptListEntry);
}
/* Now call the epilogue code */
KiExitInterrupt(TrapFrame, OldIrql, FALSE);
}
else
{
/* Now call the epilogue code */
KiExitInterrupt(TrapFrame, OldIrql, TRUE);
}
}
NTSTATUS
KiContinueEx (
IN PCONTEXT ContextRecord,
IN BOOLEAN TestAlert,
IN PKEXCEPTION_FRAME ExceptionFrame,
IN PKTRAP_FRAME TrapFrame
)
/*++
Routine Description:
This function is called to copy the specified context frame to the
specified exception and trap frames for the continue system service.
N.B. If the previous mode is user mode, alerts are being tested, a user
APC is available to execute, and the specified context record was
previously placed on the stack by the user APC initialization routine,
then bypass context record copy to the kernel frames and back again.
Arguments:
ContextRecord - Supplies a pointer to a context record.
TestAlert - Supplies a boolean value that determines whether a test alert
is to be performed.
ExceptionFrame - Supplies a pointer to an exception frame.
TrapFrame - Supplies a pointer to a trap frame.
Return Value:
STATUS_ACCESS_VIOLATION is returned if the context record is not readable
from user mode.
STATUS_DATATYPE_MISALIGNMENT is returned if the context record is not
properly aligned.
STATUS_SUCCESS + 1 is returned if the context frame is copied successfully
to the specified exception and trap frames.
STATUS_SUCCESS is returned if the context record copy was bypassed and
another user APC is ready to be delivered.
--*/
{
KIRQL OldIrql;
NTSTATUS Status;
PKTHREAD Thread;
ULONG64 UserStack;
//
// Synchronize with other context operations.
//
// If the current IRQL is less than APC_LEVEL, then raise IRQL to APC level.
//
OldIrql = KeGetCurrentIrql();
if (OldIrql < APC_LEVEL) {
KfRaiseIrql(APC_LEVEL);
}
//
// If the previous mode was not kernel mode, then attempt to bypass
// the copy of the context record. Otherwise, copy context to kernel
// frames directly.
//
Status = STATUS_SUCCESS + 1;
Thread = KeGetCurrentThread();
if (Thread->PreviousMode != KernelMode) {
try {
if (TestAlert != FALSE) {
ProbeForWriteSmallStructure(ContextRecord,
sizeof(CONTEXT),
CONTEXT_ALIGN);
KeTestAlertThread(UserMode);
UserStack = (ContextRecord->Rsp - sizeof(MACHINE_FRAME)) & ~STACK_ROUND;
if (((UserStack - sizeof(CONTEXT)) == (ULONG64)ContextRecord) &&
(Thread->ApcState.UserApcPending != FALSE)) {
//
// Save the context record and exception frame addresses
// in the trap frame and deliver the user APC bypassing
// the context copy.
//
TrapFrame->ContextRecord = (ULONG64)ContextRecord;
TrapFrame->ExceptionFrame = (ULONG64)ExceptionFrame;
KiDeliverApc(UserMode, NULL, TrapFrame);
Status = STATUS_SUCCESS;
leave;
}
} else {
ProbeForReadSmallStructure(ContextRecord,
sizeof(CONTEXT),
CONTEXT_ALIGN);
}
KiContinuePreviousModeUser(ContextRecord,
ExceptionFrame,
TrapFrame);
} except(EXCEPTION_EXECUTE_HANDLER) {
Status = GetExceptionCode();
}
} else {
DECLSPEC_NORETURN
VOID
FASTCALL
KiTrap06Handler(IN PKTRAP_FRAME TrapFrame)
{
PUCHAR Instruction;
ULONG i;
KIRQL OldIrql;
/* Check for V86 GPF */
if (__builtin_expect(KiV86Trap(TrapFrame), 1))
{
/* Enter V86 trap */
KiEnterV86Trap(TrapFrame);
/* Must be a VDM process */
if (__builtin_expect(!PsGetCurrentProcess()->VdmObjects, 0))
{
/* Enable interrupts */
_enable();
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Go to APC level */
OldIrql = KfRaiseIrql(APC_LEVEL);
_enable();
/* Check for BOP */
if (!VdmDispatchBop(TrapFrame))
{
/* Should only happen in VDM mode */
UNIMPLEMENTED;
while (TRUE);
}
/* Bring IRQL back */
KfLowerIrql(OldIrql);
_disable();
/* Do a quick V86 exit if possible */
KiExitV86Trap(TrapFrame);
}
/* Save trap frame */
KiEnterTrap(TrapFrame);
/* Enable interrupts */
Instruction = (PUCHAR)TrapFrame->Eip;
_enable();
/* Check for user trap */
if (KiUserTrap(TrapFrame))
{
/* FIXME: Use SEH */
/* Scan next 4 opcodes */
for (i = 0; i < 4; i++)
{
/* Check for LOCK instruction */
if (Instruction[i] == 0xF0)
{
/* Send invalid lock sequence exception */
KiDispatchException0Args(STATUS_INVALID_LOCK_SEQUENCE,
TrapFrame->Eip,
TrapFrame);
}
}
/* FIXME: SEH ends here */
}
/* Kernel-mode or user-mode fault (but not LOCK) */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/*
* @implemented
*/
BOOLEAN
NTAPI
HalInitSystem(IN ULONG BootPhase,
IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
PKPRCB Prcb = KeGetCurrentPrcb();
/* Check the boot phase */
if (!BootPhase)
{
/* Get command-line parameters */
HalpGetParameters(LoaderBlock);
/* Checked HAL requires checked kernel */
#if DBG
if (!(Prcb->BuildType & PRCB_BUILD_DEBUG))
{
/* No match, bugcheck */
KeBugCheckEx(MISMATCHED_HAL, 2, Prcb->BuildType, 1, 0);
}
#else
/* Release build requires release HAL */
if (Prcb->BuildType & PRCB_BUILD_DEBUG)
{
/* No match, bugcheck */
KeBugCheckEx(MISMATCHED_HAL, 2, Prcb->BuildType, 0, 0);
}
#endif
#ifdef CONFIG_SMP
/* SMP HAL requires SMP kernel */
if (Prcb->BuildType & PRCB_BUILD_UNIPROCESSOR)
{
/* No match, bugcheck */
KeBugCheckEx(MISMATCHED_HAL, 2, Prcb->BuildType, 0, 0);
}
#endif
/* Validate the PRCB */
if (Prcb->MajorVersion != PRCB_MAJOR_VERSION)
{
/* Validation failed, bugcheck */
KeBugCheckEx(MISMATCHED_HAL, 1, Prcb->MajorVersion, 1, 0);
}
/* Initialize interrupts */
HalpInitializeInterrupts();
/* Force initial PIC state */
KfRaiseIrql(KeGetCurrentIrql());
/* Fill out the dispatch tables */
//HalQuerySystemInformation = NULL; // FIXME: TODO;
//HalSetSystemInformation = NULL; // FIXME: TODO;
//HalInitPnpDriver = NULL; // FIXME: TODO
//HalGetDmaAdapter = NULL; // FIXME: TODO;
//HalGetInterruptTranslator = NULL; // FIXME: TODO
//HalResetDisplay = NULL; // FIXME: TODO;
//HalHaltSystem = NULL; // FIXME: TODO;
/* Setup I/O space */
//HalpDefaultIoSpace.Next = HalpAddressUsageList;
//HalpAddressUsageList = &HalpDefaultIoSpace;
/* Setup busy waiting */
//HalpCalibrateStallExecution();
/* Initialize the clock */
HalpInitializeClock();
/* Setup time increments to 10ms and 1ms */
HalpCurrentTimeIncrement = 100000;
HalpNextTimeIncrement = 100000;
HalpNextIntervalCount = 0;
KeSetTimeIncrement(100000, 10000);
/*
* We could be rebooting with a pending profile interrupt,
* so clear it here before interrupts are enabled
*/
HalStopProfileInterrupt(ProfileTime);
/* Do some HAL-specific initialization */
HalpInitPhase0(LoaderBlock);
}
else if (BootPhase == 1)
{
/* Enable timer interrupt */
HalpEnableInterruptHandler(IDT_DEVICE,
0,
PRIMARY_VECTOR_BASE,
CLOCK2_LEVEL,
HalpClockInterrupt,
Latched);
#if 0
/* Enable IRQ 8 */
HalpEnableInterruptHandler(IDT_DEVICE,
0,
PRIMARY_VECTOR_BASE + 8,
PROFILE_LEVEL,
HalpProfileInterrupt,
Latched);
#endif
/* Initialize DMA. NT does this in Phase 0 */
//HalpInitDma();
/* Do some HAL-specific initialization */
HalpInitPhase1();
}
/* All done, return */
return TRUE;
}
VOID
NTAPI
INIT_FUNCTION
KiInitializeKernel(IN PKPROCESS InitProcess,
IN PKTHREAD InitThread,
IN PVOID IdleStack,
IN PKPRCB Prcb,
IN CCHAR Number,
IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
BOOLEAN NpxPresent;
ULONG FeatureBits;
ULONG PageDirectory[2];
PVOID DpcStack;
ULONG Vendor[3];
/* Detect and set the CPU Type */
KiSetProcessorType();
/* Check if an FPU is present */
NpxPresent = KiIsNpxPresent();
/* Initialize the Power Management Support for this PRCB */
PoInitializePrcb(Prcb);
/* Bugcheck if this is a 386 CPU */
if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
/* Get the processor features for the CPU */
FeatureBits = KiGetFeatureBits();
/* Set the default NX policy (opt-in) */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
/* Check if NPX is always on */
if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
{
/* Set it always on */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
FeatureBits |= KF_NX_ENABLED;
}
else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
{
/* Set it in opt-out mode */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
FeatureBits |= KF_NX_ENABLED;
}
else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
(strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
{
/* Set the feature bits */
FeatureBits |= KF_NX_ENABLED;
}
else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
(strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
{
/* Set disabled mode */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
FeatureBits |= KF_NX_DISABLED;
}
/* Save feature bits */
Prcb->FeatureBits = FeatureBits;
/* Save CPU state */
KiSaveProcessorControlState(&Prcb->ProcessorState);
/* Get cache line information for this CPU */
KiGetCacheInformation();
/* Initialize spinlocks and DPC data */
KiInitSpinLocks(Prcb, Number);
/* Check if this is the Boot CPU */
if (!Number)
{
/* Set Node Data */
KeNodeBlock[0] = &KiNode0;
Prcb->ParentNode = KeNodeBlock[0];
KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
/* Set boot-level flags */
KeI386NpxPresent = NpxPresent;
KeI386CpuType = Prcb->CpuType;
KeI386CpuStep = Prcb->CpuStep;
KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
KeProcessorLevel = (USHORT)Prcb->CpuType;
if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
KeFeatureBits = FeatureBits;
KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
/* Detect 8-byte compare exchange support */
if (!(KeFeatureBits & KF_CMPXCHG8B))
{
/* Copy the vendor string */
RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
/* Bugcheck the system. Windows *requires* this */
KeBugCheckEx(UNSUPPORTED_PROCESSOR,
(1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
Vendor[0],
Vendor[1],
Vendor[2]);
}
/* Set the current MP Master KPRCB to the Boot PRCB */
Prcb->MultiThreadSetMaster = Prcb;
/* Lower to APC_LEVEL */
KeLowerIrql(APC_LEVEL);
/* Initialize some spinlocks */
KeInitializeSpinLock(&KiFreezeExecutionLock);
KeInitializeSpinLock(&Ki486CompatibilityLock);
/* Initialize portable parts of the OS */
KiInitSystem();
/* Initialize the Idle Process and the Process Listhead */
InitializeListHead(&KiProcessListHead);
PageDirectory[0] = 0;
PageDirectory[1] = 0;
KeInitializeProcess(InitProcess,
0,
0xFFFFFFFF,
PageDirectory,
FALSE);
InitProcess->QuantumReset = MAXCHAR;
}
else
{
/* FIXME */
DPRINT1("SMP Boot support not yet present\n");
}
/* Setup the Idle Thread */
KeInitializeThread(InitProcess,
InitThread,
NULL,
NULL,
NULL,
NULL,
NULL,
IdleStack);
InitThread->NextProcessor = Number;
InitThread->Priority = HIGH_PRIORITY;
InitThread->State = Running;
InitThread->Affinity = 1 << Number;
InitThread->WaitIrql = DISPATCH_LEVEL;
InitProcess->ActiveProcessors = 1 << Number;
/* HACK for MmUpdatePageDir */
((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
/* Set basic CPU Features that user mode can read */
SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
(KeFeatureBits & KF_MMX) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
(KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
(KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
(KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
/* Set up the thread-related fields in the PRCB */
Prcb->CurrentThread = InitThread;
Prcb->NextThread = NULL;
Prcb->IdleThread = InitThread;
/* Initialize the Kernel Executive */
ExpInitializeExecutive(Number, LoaderBlock);
/* Only do this on the boot CPU */
if (!Number)
{
/* Calculate the time reciprocal */
KiTimeIncrementReciprocal =
KiComputeReciprocal(KeMaximumIncrement,
&KiTimeIncrementShiftCount);
/* Update DPC Values in case they got updated by the executive */
Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
Prcb->MinimumDpcRate = KiMinimumDpcRate;
Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
/* Allocate the DPC Stack */
DpcStack = MmCreateKernelStack(FALSE, 0);
if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
Prcb->DpcStack = DpcStack;
/* Allocate the IOPM save area. */
Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
PAGE_SIZE * 2,
' eK');
if (!Ki386IopmSaveArea)
{
/* Bugcheck. We need this for V86/VDM support. */
KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
}
}
/* Raise to Dispatch */
KfRaiseIrql(DISPATCH_LEVEL);
/* Set the Idle Priority to 0. This will jump into Phase 1 */
KeSetPriorityThread(InitThread, 0);
/* If there's no thread scheduled, put this CPU in the Idle summary */
KiAcquirePrcbLock(Prcb);
if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
KiReleasePrcbLock(Prcb);
/* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
KfRaiseIrql(HIGH_LEVEL);
LoaderBlock->Prcb = 0;
}
BOOLEAN
HalInitSystem (
IN ULONG Phase,
IN PLOADER_PARAMETER_BLOCK LoaderBlock
)
/*++
Routine Description:
This function initializes the Hardware Architecture Layer (HAL) for an
x86 system.
Arguments:
None.
Return Value:
A value of TRUE is returned is the initialization was successfully
complete. Otherwise a value of FALSE is returend.
--*/
{
PMEMORY_ALLOCATION_DESCRIPTOR Descriptor;
PLIST_ENTRY NextMd;
KIRQL CurrentIrql;
extern VOID HalpAddMem(IN PLOADER_PARAMETER_BLOCK);
PKPRCB pPRCB;
ULONG BuildType;
pPRCB = KeGetCurrentPrcb();
if (Phase == 0) {
if (CbusGetParameters (LoaderBlock)) {
DbgBreakPoint();
}
HalpBusType = LoaderBlock->u.I386.MachineType & 0x00ff;
//
// Verify Prcb version and build flags conform to
// this image
//
BuildType = 0;
#if DBG
BuildType |= PRCB_BUILD_DEBUG;
#endif
#ifdef NT_UP
BuildType |= PRCB_BUILD_UNIPROCESSOR;
#endif
if (pPRCB->MajorVersion != PRCB_MAJOR_VERSION) {
KeBugCheckEx (MISMATCHED_HAL,
1, pPRCB->MajorVersion, PRCB_MAJOR_VERSION, 0);
}
if (pPRCB->BuildType != BuildType) {
KeBugCheckEx (MISMATCHED_HAL,
2, pPRCB->BuildType, BuildType, 0);
}
//
// Phase 0 initialization
// only called by P0
//
//
// Check to make sure the MCA HAL is not running on an ISA/EISA
// system, and vice-versa.
//
#if MCA
if (HalpBusType != MACHINE_TYPE_MCA) {
KeBugCheckEx (MISMATCHED_HAL,
3, HalpBusType, MACHINE_TYPE_MCA, 0);
}
#else
if (HalpBusType == MACHINE_TYPE_MCA) {
KeBugCheckEx (MISMATCHED_HAL,
3, HalpBusType, 0, 0);
}
#endif
//
// Most HALs initialize their PICs at this point - we set up
// the Cbus PICs in HalInitializeProcessor() long ago...
// Now that the PICs are initialized, we need to mask them to
// reflect the current Irql
//
CurrentIrql = KeGetCurrentIrql();
KfRaiseIrql(CurrentIrql);
//
// Fill in handlers for APIs which this hal supports
//
HalQuerySystemInformation = HaliQuerySystemInformation;
HalSetSystemInformation = HaliSetSystemInformation;
//
// Initialize CMOS
//
HalpInitializeCmos();
//
// Register the PC-compatible base IO space used by hal
//
HalpRegisterAddressUsage (&HalpDefaultPcIoSpace);
if (HalpBusType == MACHINE_TYPE_EISA) {
HalpRegisterAddressUsage (&HalpEisaIoSpace);
}
//
// Cbus1: stall uses the APIC to figure it out (needed in phase0).
// the clock uses the APIC (needed in phase0)
// the perfcounter uses RTC irq8 (not needed till all cpus boot)
//
// Cbus2: stall uses the RTC irq8 to figure it out (needed in phase0).
// the clock uses the irq0 (needed in phase0)
// the perfcounter uses RTC irq8 (not needed till all cpus boot)
//
//
// set up the stall execution and enable clock interrupts now.
// APC, DPC and IPI are already enabled.
//
(*CbusBackend->HalInitializeInterrupts)(0);
HalStopProfileInterrupt(0);
HalpInitializeDisplay();
//
// Initialize spinlock used by HalGetBusData hardware access routines
//
KeInitializeSpinLock(&HalpSystemHardwareLock);
//
// Any additional memory must be recovered BEFORE Phase0 ends
//
HalpAddMem(LoaderBlock);
//
// Determine if there is physical memory above 16 MB.
//
LessThan16Mb = TRUE;
NextMd = LoaderBlock->MemoryDescriptorListHead.Flink;
while (NextMd != &LoaderBlock->MemoryDescriptorListHead) {
Descriptor = CONTAINING_RECORD( NextMd,
MEMORY_ALLOCATION_DESCRIPTOR,
ListEntry );
if (Descriptor->BasePage + Descriptor->PageCount > 0x1000) {
LessThan16Mb = FALSE;
}
NextMd = Descriptor->ListEntry.Flink;
}
//
// Determine the size need for map buffers. If this system has
// memory with a physical address of greater than
// MAXIMUM_PHYSICAL_ADDRESS, then allocate a large chunk; otherwise,
// allocate a small chunk.
//
// This should probably create a memory descriptor which describes
// the DMA map buffers reserved by the HAL, and then add it back in
// to the LoaderBlock so the kernel can report the correct amount
// of memory in the machine.
//
if (LessThan16Mb) {
//
// Allocate a small set of map buffers. They are only need for
// slave DMA devices.
//
HalpMapBufferSize = INITIAL_MAP_BUFFER_SMALL_SIZE;
} else {
//
// Allocate a larger set of map buffers. These are used for
// slave DMA controllers and Isa cards.
//
HalpMapBufferSize = INITIAL_MAP_BUFFER_LARGE_SIZE;
}
//
// Allocate map buffers for the adapter objects
//
HalpMapBufferPhysicalAddress.LowPart =
HalpAllocPhysicalMemory (LoaderBlock, MAXIMUM_PHYSICAL_ADDRESS,
HalpMapBufferSize >> PAGE_SHIFT, TRUE);
HalpMapBufferPhysicalAddress.HighPart = 0;
if (!HalpMapBufferPhysicalAddress.LowPart) {
//
// There was not a satisfactory block. Clear the allocation.
//
HalpMapBufferSize = 0;
}
} else {
ULONG_PTR
KeIpiGenericCall (
IN PKIPI_BROADCAST_WORKER BroadcastFunction,
IN ULONG_PTR Context
)
/*++
Routine Description:
This function executes the specified function on every processor in
the host configuration in a synchronous manner, i.e., the function
is executed on each target in series with the execution of the source
processor.
Arguments:
BroadcastFunction - Supplies the address of function that is executed
on each of the target processors.
Context - Supplies the value of the context parameter that is passed
to each function.
Return Value:
The value returned by the specified function on the source processor
is returned as the function value.
--*/
{
volatile LONG Count;
KIRQL OldIrql;
ULONG_PTR Status;
#if !defined(NT_UP)
KAFFINITY TargetProcessors;
#endif
//
// Raise IRQL to synchronization level and acquire the reverse stall spin
// lock to synchronize with other reverse stall functions.
//
OldIrql = KeGetCurrentIrql();
if (OldIrql < SYNCH_LEVEL) {
KfRaiseIrql(SYNCH_LEVEL);
}
#if !defined(NT_UP)
Count = KeNumberProcessors;
TargetProcessors = KeActiveProcessors & ~KeGetCurrentPrcb()->SetMember;
#endif
KeAcquireSpinLockAtDpcLevel(&KiReverseStallIpiLock);
//
// Initialize the broadcast packet, compute the set of target processors,
// and sent the packet to the target processors for execution.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiIpiGenericCallTarget,
(PVOID)(ULONG_PTR)BroadcastFunction,
(PVOID)Context,
(PVOID)&Count);
}
//
// Wait until all processors have entered the target routine and are
// waiting.
//
while (Count != 1) {
KeYieldProcessor();
}
#endif
//
// Raise IRQL to IPI_LEVEL, signal all other processors to proceed, and
// call the specified function on the source processor.
//
KfRaiseIrql(IPI_LEVEL);
Count = 0;
Status = BroadcastFunction(Context);
//
// Wait until all of the target processors have finished capturing the
// function parameters.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
#endif
//
// Release reverse stall spin lock, lower IRQL to its previous level,
// and return the function execution status.
//
KeReleaseSpinLockFromDpcLevel(&KiReverseStallIpiLock);
KeLowerIrql(OldIrql);
return Status;
}
KIRQL
NTAPI
KeRaiseIrqlToDpcLevel(VOID)
{
return KfRaiseIrql(DISPATCH_LEVEL);
}
KIRQL
NTAPI
KeRaiseIrqlToSynchLevel(VOID)
{
return KfRaiseIrql(SYNCH_LEVEL);
}
VOID
NTAPI
INIT_FUNCTION
KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
ULONG Cpu;
PKTHREAD InitialThread;
ULONG InitialStack;
PKGDTENTRY Gdt;
PKIDTENTRY Idt;
KIDTENTRY NmiEntry, DoubleFaultEntry;
PKTSS Tss;
PKIPCR Pcr;
/* Boot cycles timestamp */
BootCycles = __rdtsc();
/* Save the loader block and get the current CPU */
KeLoaderBlock = LoaderBlock;
Cpu = KeNumberProcessors;
if (!Cpu)
{
/* If this is the boot CPU, set FS and the CPU Number*/
Ke386SetFs(KGDT_R0_PCR);
__writefsdword(KPCR_PROCESSOR_NUMBER, Cpu);
/* Set the initial stack and idle thread as well */
LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
}
/* Save the initial thread and stack */
InitialStack = LoaderBlock->KernelStack;
InitialThread = (PKTHREAD)LoaderBlock->Thread;
/* Clean the APC List Head */
InitializeListHead(&InitialThread->ApcState.ApcListHead[KernelMode]);
/* Initialize the machine type */
KiInitializeMachineType();
/* Skip initial setup if this isn't the Boot CPU */
if (Cpu) goto AppCpuInit;
/* Get GDT, IDT, PCR and TSS pointers */
KiGetMachineBootPointers(&Gdt, &Idt, &Pcr, &Tss);
/* Setup the TSS descriptors and entries */
Ki386InitializeTss(Tss, Idt, Gdt);
/* Initialize the PCR */
RtlZeroMemory(Pcr, PAGE_SIZE);
KiInitializePcr(Cpu,
Pcr,
Idt,
Gdt,
Tss,
InitialThread,
(PVOID)KiDoubleFaultStack);
/* Set us as the current process */
InitialThread->ApcState.Process = &KiInitialProcess.Pcb;
/* Clear DR6/7 to cleanup bootloader debugging */
__writefsdword(KPCR_TEB, 0);
__writefsdword(KPCR_DR6, 0);
__writefsdword(KPCR_DR7, 0);
/* Setup the IDT */
KeInitExceptions();
/* Load Ring 3 selectors for DS/ES */
Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
/* Save NMI and double fault traps */
RtlCopyMemory(&NmiEntry, &Idt[2], sizeof(KIDTENTRY));
RtlCopyMemory(&DoubleFaultEntry, &Idt[8], sizeof(KIDTENTRY));
/* Copy kernel's trap handlers */
RtlCopyMemory(Idt,
(PVOID)KiIdtDescriptor.Base,
KiIdtDescriptor.Limit + 1);
/* Restore NMI and double fault */
RtlCopyMemory(&Idt[2], &NmiEntry, sizeof(KIDTENTRY));
RtlCopyMemory(&Idt[8], &DoubleFaultEntry, sizeof(KIDTENTRY));
AppCpuInit:
/* Loop until we can release the freeze lock */
do
{
/* Loop until execution can continue */
while (*(volatile PKSPIN_LOCK*)&KiFreezeExecutionLock == (PVOID)1);
} while(InterlockedBitTestAndSet((PLONG)&KiFreezeExecutionLock, 0));
/* Setup CPU-related fields */
__writefsdword(KPCR_NUMBER, Cpu);
__writefsdword(KPCR_SET_MEMBER, 1 << Cpu);
__writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
__writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
/* Initialize the Processor with HAL */
HalInitializeProcessor(Cpu, KeLoaderBlock);
/* Set active processors */
KeActiveProcessors |= __readfsdword(KPCR_SET_MEMBER);
KeNumberProcessors++;
/* Check if this is the boot CPU */
if (!Cpu)
{
/* Initialize debugging system */
KdInitSystem(0, KeLoaderBlock);
/* Check for break-in */
if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
}
/* Raise to HIGH_LEVEL */
KfRaiseIrql(HIGH_LEVEL);
/* Switch to new kernel stack and start kernel bootstrapping */
KiSwitchToBootStack(InitialStack & ~3);
}
KIRQL NTAPI
KeRaiseIrqlToSynchLevel (VOID)
{
return KfRaiseIrql (CLOCK2_LEVEL);
}
VOID
NTAPI
KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
CCHAR Cpu;
PKTHREAD InitialThread;
ULONG64 InitialStack;
PKIPCR Pcr;
/* HACK */
FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
//FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
/* Save the loader block */
KeLoaderBlock = LoaderBlock;
/* Get the current CPU number */
Cpu = KeNumberProcessors++; // FIXME
/* LoaderBlock initialization for Cpu 0 */
if (Cpu == 0)
{
/* Set the initial stack, idle thread and process */
LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
}
/* Get Pcr from loader block */
Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
/* Set the PRCB for this Processor */
KiProcessorBlock[Cpu] = &Pcr->Prcb;
/* Align stack to 16 bytes */
LoaderBlock->KernelStack &= ~(16 - 1);
/* Save the initial thread and stack */
InitialStack = LoaderBlock->KernelStack; // Checkme
InitialThread = (PKTHREAD)LoaderBlock->Thread;
/* Set us as the current process */
InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
/* Initialize the PCR */
KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
/* Initialize the CPU features */
KiInitializeCpu(Pcr);
/* Initial setup for the boot CPU */
if (Cpu == 0)
{
/* Initialize the module list (ntos, hal, kdcom) */
KiInitModuleList(LoaderBlock);
/* Setup the TSS descriptors and entries */
KiInitializeTss(Pcr->TssBase, InitialStack);
/* Setup the IDT */
KeInitExceptions();
/* Initialize debugging system */
KdInitSystem(0, KeLoaderBlock);
/* Check for break-in */
if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
}
DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
/* Acquire lock */
while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
{
/* Loop until lock is free */
while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
}
/* Initialize the Processor with HAL */
HalInitializeProcessor(Cpu, KeLoaderBlock);
/* Set processor as active */
KeActiveProcessors |= 1ULL << Cpu;
/* Release lock */
InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
/* Raise to HIGH_LEVEL */
KfRaiseIrql(HIGH_LEVEL);
/* Machine specific kernel initialization */
if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
/* Switch to new kernel stack and start kernel bootstrapping */
KiSwitchToBootStack(InitialStack & ~3);
}
DECLSPEC_NORETURN
VOID
FASTCALL
KiTrap0DHandler(IN PKTRAP_FRAME TrapFrame)
{
ULONG i, j, Iopl;
BOOLEAN Privileged = FALSE;
PUCHAR Instructions;
UCHAR Instruction = 0;
KIRQL OldIrql;
/* Check for V86 GPF */
if (__builtin_expect(KiV86Trap(TrapFrame), 1))
{
/* Enter V86 trap */
KiEnterV86Trap(TrapFrame);
/* Must be a VDM process */
if (__builtin_expect(!PsGetCurrentProcess()->VdmObjects, 0))
{
/* Enable interrupts */
_enable();
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Go to APC level */
OldIrql = KfRaiseIrql(APC_LEVEL);
_enable();
/* Handle the V86 opcode */
if (__builtin_expect(Ki386HandleOpcodeV86(TrapFrame) == 0xFF, 0))
{
/* Should only happen in VDM mode */
UNIMPLEMENTED;
while (TRUE);
}
/* Bring IRQL back */
KfLowerIrql(OldIrql);
_disable();
/* Do a quick V86 exit if possible */
KiExitV86Trap(TrapFrame);
}
/* Save trap frame */
KiEnterTrap(TrapFrame);
/* Check for user-mode GPF */
if (KiUserTrap(TrapFrame))
{
/* Should not be VDM */
ASSERT(KiVdmTrap(TrapFrame) == FALSE);
/* Enable interrupts and check error code */
_enable();
if (!TrapFrame->ErrCode)
{
/* FIXME: Use SEH */
Instructions = (PUCHAR)TrapFrame->Eip;
/* Scan next 15 bytes */
for (i = 0; i < 15; i++)
{
/* Skip prefix instructions */
for (j = 0; j < sizeof(KiTrapPrefixTable); j++)
{
/* Is this a prefix instruction? */
if (Instructions[i] == KiTrapPrefixTable[j])
{
/* Stop looking */
break;
}
}
/* Is this NOT any prefix instruction? */
if (j == sizeof(KiTrapPrefixTable))
{
/* We can go ahead and handle the fault now */
Instruction = Instructions[i];
break;
}
}
/* If all we found was prefixes, then this instruction is too long */
if (i == 15)
{
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Check for privileged instructions */
if (Instruction == 0xF4) // HLT
{
/* HLT is privileged */
Privileged = TRUE;
}
else if (Instruction == 0x0F)
{
/* Test if it's any of the privileged two-byte opcodes */
if (((Instructions[i + 1] == 0x00) && // LLDT or LTR
(((Instructions[i + 2] & 0x38) == 0x10) || // LLDT
(Instructions[i + 2] == 0x18))) || // LTR
((Instructions[i + 1] == 0x01) && // LGDT or LIDT or LMSW
(((Instructions[i + 2] & 0x38) == 0x10) || // LGDT
(Instructions[i + 2] == 0x18) || // LIDT
(Instructions[i + 2] == 0x30))) || // LMSW
(Instructions[i + 1] == 0x08) || // INVD
(Instructions[i + 1] == 0x09) || // WBINVD
(Instructions[i + 1] == 0x35) || // SYSEXIT
(Instructions[i + 1] == 0x26) || // MOV DR, XXX
(Instructions[i + 1] == 0x06) || // CLTS
(Instructions[i + 1] == 0x20) || // MOV CR, XXX
(Instructions[i + 1] == 0x24) || // MOV YYY, DR
(Instructions[i + 1] == 0x30) || // WRMSR
(Instructions[i + 1] == 0x33)) // RDPMC
// INVLPG, INVLPGA, SYSRET
{
/* These are all privileged */
Privileged = TRUE;
}
}
else
{
/* Get the IOPL and compare with the RPL mask */
Iopl = (TrapFrame->EFlags & EFLAGS_IOPL) >> 12;
if ((TrapFrame->SegCs & RPL_MASK) > Iopl)
{
/* I/O privilege error -- check for known instructions */
if ((Instruction == 0xFA) || (Instruction == 0xFB)) // CLI or STI
{
/* These are privileged */
Privileged = TRUE;
}
else
{
/* Last hope: an IN/OUT instruction */
for (j = 0; j < sizeof(KiTrapIoTable); j++)
{
/* Is this an I/O instruction? */
if (Instruction == KiTrapIoTable[j])
{
/* Then it's privileged */
Privileged = TRUE;
break;
}
}
}
}
}
/* So now... was the instruction privileged or not? */
if (Privileged)
{
/* Whew! We have a privileged instruction, so dispatch the fault */
KiDispatchException0Args(STATUS_PRIVILEGED_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
}
/* If we got here, send an access violation */
KiDispatchException2Args(STATUS_ACCESS_VIOLATION,
TrapFrame->Eip,
0,
0xFFFFFFFF,
TrapFrame);
}
NTSTATUS
KiContinue (
IN PCONTEXT ContextRecord,
IN PKEXCEPTION_FRAME ExceptionFrame,
IN PKTRAP_FRAME TrapFrame
)
/*++
Routine Description:
This function is called to copy the specified context frame to the
specified exception and trap frames for the continue system service.
Arguments:
ContextRecord - Supplies a pointer to a context record.
ExceptionFrame - Supplies a pointer to an exception frame.
TrapFrame - Supplies a pointer to a trap frame.
Return Value:
STATUS_ACCESS_VIOLATION is returned if the context record is not readable
from user mode.
STATUS_DATATYPE_MISALIGNMENT is returned if the context record is not
properly aligned.
STATUS_SUCCESS is returned if the context frame is copied successfully
to the specified exception and trap frames.
--*/
{
KIRQL OldIrql;
NTSTATUS Status;
//
// Synchronize with other context operations.
//
// If the current IRQL is less than APC_LEVEL, then raise IRQL to APC level.
//
OldIrql = KeGetCurrentIrql();
if (OldIrql < APC_LEVEL) {
KfRaiseIrql(APC_LEVEL);
}
//
// If the previous mode was not kernel mode, then use wrapper function
// to copy context to kernel frames. Otherwise, copy context to kernel
// frames directly.
//
Status = STATUS_SUCCESS;
if (KeGetPreviousMode() != KernelMode) {
try {
KiContinuePreviousModeUser(ContextRecord,
ExceptionFrame,
TrapFrame);
} except(EXCEPTION_EXECUTE_HANDLER) {
Status = GetExceptionCode();
}
} else {