VOID
KeStartAllProcessors (
VOID
)
/*++
Routine Description:
This function is called during phase 1 initialization on the master boot
processor to start all of the other registered processors.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
ULONG AllocationSize;
PUCHAR Base;
PKPCR CurrentPcr = KeGetPcr();
PVOID DataBlock;
PVOID DpcStack;
PKGDTENTRY64 GdtBase;
ULONG GdtOffset;
ULONG IdtOffset;
UCHAR Index;
PVOID KernelStack;
ULONG LogicalProcessors;
ULONG MaximumProcessors;
PKNODE Node;
UCHAR NodeNumber = 0;
UCHAR Number;
KIRQL OldIrql;
PKNODE OldNode;
PKNODE ParentNode;
PKPCR PcrBase;
PKPRCB Prcb;
USHORT ProcessorId;
KPROCESSOR_STATE ProcessorState;
PKTSS64 SysTssBase;
PKGDTENTRY64 TebBase;
PETHREAD Thread;
//
// Ensure that prefetch instructions in the IPI path are patched out
// if necessary before starting other processors.
//
OldIrql = KeRaiseIrqlToSynchLevel();
KiIpiSendRequest(1, 0, 0, IPI_FLUSH_SINGLE);
KeLowerIrql(OldIrql);
//
// Do not start additional processors if the relocate physical loader
// switch has been specified.
//
if (KeLoaderBlock->LoadOptions != NULL) {
if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) {
return;
}
}
//
// If this a multinode system and processor zero is not on node zero,
// then move it to the appropriate node.
//
if (KeNumberNodes > 1) {
if (NT_SUCCESS(KiQueryProcessorNode(0, &ProcessorId, &NodeNumber))) {
if (NodeNumber != 0) {
KiNode0.ProcessorMask = 0;
KiNodeInit[0] = KiNode0;
KeNodeBlock[0] = &KiNodeInit[0];
KiNode0 = *KeNodeBlock[NodeNumber];
KeNodeBlock[NodeNumber] = &KiNode0;
KiNode0.ProcessorMask = 1;
}
} else {
goto StartFailure;
}
}
//
// Calculate the size of the per processor data structures.
//
// This includes:
//
// PCR (including the PRCB)
// System TSS
// Idle Thread Object
// Double Fault Stack
// Machine Check Stack
// NMI Stack
// Multinode structure
// GDT
// IDT
//
// A DPC and Idle stack are also allocated, but they are done separately.
//
AllocationSize = ROUNDUP64(sizeof(KPCR)) +
ROUNDUP64(sizeof(KTSS64)) +
ROUNDUP64(sizeof(ETHREAD)) +
ROUNDUP64(DOUBLE_FAULT_STACK_SIZE) +
ROUNDUP64(KERNEL_MCA_EXCEPTION_STACK_SIZE) +
ROUNDUP64(NMI_STACK_SIZE) +
ROUNDUP64(sizeof(KNODE));
//
// Save the offset of the GDT in the allocation structure and add in
// the size of the GDT.
//
GdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
//
// Save the offset of the IDT in the allocation structure and add in
// the size of the IDT.
//
IdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
//
// If the registered number of processors is greater than the maximum
// number of processors supported, then only allow the maximum number
// of supported processors.
//
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
KeRegisteredProcessors = MAXIMUM_PROCESSORS;
}
//
// Set barrier that will prevent any other processor from entering the
// idle loop until all processors have been started.
//
KiBarrierWait = 1;
//
// Initialize the fixed part of the processor state that will be used to
// start processors. Each processor starts in the system initialization
// code with address of the loader parameter block as an argument.
//
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock;
ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup;
ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE;
ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK;
ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegSs = KGDT64_R0_DATA;
//
// Check to determine if hyper-threading is really enabled. Intel chips
// claim to be hyper-threaded with the number of logical processors
// greater than one even when hyper-threading is disabled in the BIOS.
//
LogicalProcessors = KiLogicalProcessors;
if (HalIsHyperThreadingEnabled() == FALSE) {
LogicalProcessors = 1;
}
//
// If the total number of logical processors has not been set with
// the /NUMPROC loader option, then set the maximum number of logical
// processors to the number of registered processors times the number
// of logical processors per registered processor.
//
// N.B. The number of logical processors is never allowed to exceed
// the number of registered processors times the number of logical
// processors per physical processor.
//
MaximumProcessors = KeNumprocSpecified;
if (MaximumProcessors == 0) {
MaximumProcessors = KeRegisteredProcessors * LogicalProcessors;
}
//
// Loop trying to start a new processors until a new processor can't be
// started or an allocation failure occurs.
//
// N.B. The below processor start code relies on the fact a physical
// processor is started followed by all its logical processors.
// The HAL guarantees this by sorting the ACPI processor table
// by APIC id.
//
Index = 0;
Number = 0;
while ((Index < (MAXIMUM_PROCESSORS - 1)) &&
((ULONG)KeNumberProcessors < MaximumProcessors) &&
((ULONG)KeNumberProcessors / LogicalProcessors) < KeRegisteredProcessors) {
//
// If this is a multinode system and current processor does not
// exist on any node, then skip it.
//
Index += 1;
if (KeNumberNodes > 1) {
if (!NT_SUCCESS(KiQueryProcessorNode(Index, &ProcessorId, &NodeNumber))) {
continue;
}
}
//
// Increment the processor number.
//
Number += 1;
//
// Allocate memory for the new processor specific data. If the
// allocation fails, then stop starting processors.
//
DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber);
if (DataBlock == NULL) {
goto StartFailure;
}
//
// Allocate a pool tag table for the new processor.
//
if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) {
goto StartFailure;
}
//
// Zero the allocated memory.
//
Base = (PUCHAR)DataBlock;
RtlZeroMemory(DataBlock, AllocationSize);
//
// Copy and initialize the GDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr,
&ProcessorState.SpecialRegisters.Gdtr,
Base + GdtOffset);
GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
//
// Encode the processor number in the upper 6 bits of the compatibility
// mode TEB descriptor.
//
TebBase = (PKGDTENTRY64)((PCHAR)GdtBase + KGDT64_R3_CMTEB);
TebBase->Bits.LimitHigh = Number >> 2;
TebBase->LimitLow = ((Number & 0x3) << 14) | (TebBase->LimitLow & 0x3fff);
//
// Copy and initialize the IDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr,
&ProcessorState.SpecialRegisters.Idtr,
Base + IdtOffset);
//
// Set the PCR base address for the next processor, set the processor
// number, and set the processor speed.
//
// N.B. The PCR address is passed to the next processor by computing
// the containing address with respect to the PRCB.
//
PcrBase = (PKPCR)Base;
PcrBase->ObsoleteNumber = Number;
PcrBase->Prcb.Number = Number;
PcrBase->Prcb.MHz = KeGetCurrentPrcb()->MHz;
Base += ROUNDUP64(sizeof(KPCR));
//
// Set the system TSS descriptor base for the next processor.
//
SysTssBase = (PKTSS64)Base;
KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase);
Base += ROUNDUP64(sizeof(KTSS64));
//
// Initialize the panic stack address for double fault and NMI.
//
Base += DOUBLE_FAULT_STACK_SIZE;
SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
//
// Initialize the machine check stack address.
//
Base += KERNEL_MCA_EXCEPTION_STACK_SIZE;
SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
//
// Initialize the NMI stack address.
//
Base += NMI_STACK_SIZE;
SysTssBase->Ist[TSS_IST_NMI] = (ULONG64)Base;
//
// Idle Thread thread object.
//
Thread = (PETHREAD)Base;
Base += ROUNDUP64(sizeof(ETHREAD));
//
// Set other special registers in the processor state.
//
ProcessorState.SpecialRegisters.Cr0 = ReadCR0();
ProcessorState.SpecialRegisters.Cr3 = ReadCR3();
ProcessorState.ContextFrame.EFlags = 0;
ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS;
GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89;
ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
//
// Allocate a kernel stack and a DPC stack for the next processor.
//
KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
if (KernelStack == NULL) {
goto StartFailure;
}
DpcStack = MmCreateKernelStack(FALSE, NodeNumber);
if (DpcStack == NULL) {
goto StartFailure;
}
//
// Initialize the kernel stack for the system TSS.
//
// N.B. System startup must be called with a stack pointer that is
// 8 mod 16.
//
SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4;
ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack - 8;
//
// If this is the first processor on this node, then use the space
// already allocated for the node. Otherwise, the space allocated
// is not used.
//
Node = KeNodeBlock[NodeNumber];
OldNode = Node;
if (Node == &KiNodeInit[NodeNumber]) {
Node = (PKNODE)Base;
*Node = KiNodeInit[NodeNumber];
KeNodeBlock[NodeNumber] = Node;
}
Base += ROUNDUP64(sizeof(KNODE));
//
// Set the parent node address.
//
PcrBase->Prcb.ParentNode = Node;
//
// Adjust the loader block so it has the next processor state. Ensure
// that the kernel stack has space for home registers for up to four
// parameters.
//
KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4);
KeLoaderBlock->Thread = (ULONG64)Thread;
KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
//
// Attempt to start the next processor. If a processor cannot be
// started, then deallocate memory and stop starting processors.
//
if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) {
//
// Restore the old node address in the node address array before
// freeing the allocated data block (the node structure lies
// within the data block).
//
*OldNode = *Node;
KeNodeBlock[NodeNumber] = OldNode;
ExDeletePoolTagTable(Number);
MmFreeIndependentPages(DataBlock, AllocationSize);
MmDeleteKernelStack(KernelStack, FALSE);
MmDeleteKernelStack(DpcStack, FALSE);
break;
}
Node->ProcessorMask |= AFFINITY_MASK(Number);
//
// Wait for processor to initialize.
//
while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) {
KeYieldProcessor();
}
}
//
// All processors have been started. If this is a multinode system, then
// allocate any missing node structures.
//
if (KeNumberNodes > 1) {
for (Index = 0; Index < KeNumberNodes; Index += 1) {
if (KeNodeBlock[Index] == &KiNodeInit[Index]) {
Node = ExAllocatePoolWithTag(NonPagedPool, sizeof(KNODE), ' eK');
if (Node != NULL) {
*Node = KiNodeInit[Index];
KeNodeBlock[Index] = Node;
} else {
goto StartFailure;
}
}
}
} else if (KiNode0.ProcessorMask != KeActiveProcessors) {
goto StartFailure;
}
//
// Clear node structure address for nonexistent nodes.
//
for (Index = KeNumberNodes; Index < MAXIMUM_CCNUMA_NODES; Index += 1) {
KeNodeBlock[Index] = NULL;
}
//
// Copy the node color and shifted color to the PRCB of each processor.
//
for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) {
Prcb = KiProcessorBlock[Index];
ParentNode = Prcb->ParentNode;
Prcb->NodeColor = ParentNode->Color;
Prcb->NodeShiftedColor = ParentNode->MmShiftedColor;
Prcb->SecondaryColorMask = MmSecondaryColorMask;
}
//
// Reset the initialization bit in prefetch retry.
//
KiPrefetchRetry &= ~0x80;
//
// Reset and synchronize the performance counters of all processors, by
// applying a null adjustment to the interrupt time.
//
KeAdjustInterruptTime(0);
//
// Allow all processors that were started to enter the idle loop and
// begin execution.
//
KiBarrierWait = 0;
#endif //
return;
//
// The failure to allocate memory or a unsuccessful status was returned
// during the attempt to start processors. This is considered fatal since
// something is very wrong.
//
#if !defined(NT_UP)
StartFailure:
KeBugCheckEx(PHASE1_INITIALIZATION_FAILED, 0, 0, 20, 0);
#endif
}
VOID
NTAPI
INIT_FUNCTION
KiInitializeKernel(IN PKPROCESS InitProcess,
IN PKTHREAD InitThread,
IN PVOID IdleStack,
IN PKPRCB Prcb,
IN CCHAR Number,
IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
BOOLEAN NpxPresent;
ULONG FeatureBits;
ULONG PageDirectory[2];
PVOID DpcStack;
ULONG Vendor[3];
/* Detect and set the CPU Type */
KiSetProcessorType();
/* Check if an FPU is present */
NpxPresent = KiIsNpxPresent();
/* Initialize the Power Management Support for this PRCB */
PoInitializePrcb(Prcb);
/* Bugcheck if this is a 386 CPU */
if (Prcb->CpuType == 3) KeBugCheckEx(UNSUPPORTED_PROCESSOR, 0x386, 0, 0, 0);
/* Get the processor features for the CPU */
FeatureBits = KiGetFeatureBits();
/* Set the default NX policy (opt-in) */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTIN;
/* Check if NPX is always on */
if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSON"))
{
/* Set it always on */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSON;
FeatureBits |= KF_NX_ENABLED;
}
else if (strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTOUT"))
{
/* Set it in opt-out mode */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_OPTOUT;
FeatureBits |= KF_NX_ENABLED;
}
else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=OPTIN")) ||
(strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE")))
{
/* Set the feature bits */
FeatureBits |= KF_NX_ENABLED;
}
else if ((strstr(KeLoaderBlock->LoadOptions, "NOEXECUTE=ALWAYSOFF")) ||
(strstr(KeLoaderBlock->LoadOptions, "EXECUTE")))
{
/* Set disabled mode */
SharedUserData->NXSupportPolicy = NX_SUPPORT_POLICY_ALWAYSOFF;
FeatureBits |= KF_NX_DISABLED;
}
/* Save feature bits */
Prcb->FeatureBits = FeatureBits;
/* Save CPU state */
KiSaveProcessorControlState(&Prcb->ProcessorState);
/* Get cache line information for this CPU */
KiGetCacheInformation();
/* Initialize spinlocks and DPC data */
KiInitSpinLocks(Prcb, Number);
/* Check if this is the Boot CPU */
if (!Number)
{
/* Set Node Data */
KeNodeBlock[0] = &KiNode0;
Prcb->ParentNode = KeNodeBlock[0];
KeNodeBlock[0]->ProcessorMask = Prcb->SetMember;
/* Set boot-level flags */
KeI386NpxPresent = NpxPresent;
KeI386CpuType = Prcb->CpuType;
KeI386CpuStep = Prcb->CpuStep;
KeProcessorArchitecture = PROCESSOR_ARCHITECTURE_INTEL;
KeProcessorLevel = (USHORT)Prcb->CpuType;
if (Prcb->CpuID) KeProcessorRevision = Prcb->CpuStep;
KeFeatureBits = FeatureBits;
KeI386FxsrPresent = (KeFeatureBits & KF_FXSR) ? TRUE : FALSE;
KeI386XMMIPresent = (KeFeatureBits & KF_XMMI) ? TRUE : FALSE;
/* Detect 8-byte compare exchange support */
if (!(KeFeatureBits & KF_CMPXCHG8B))
{
/* Copy the vendor string */
RtlCopyMemory(Vendor, Prcb->VendorString, sizeof(Vendor));
/* Bugcheck the system. Windows *requires* this */
KeBugCheckEx(UNSUPPORTED_PROCESSOR,
(1 << 24 ) | (Prcb->CpuType << 16) | Prcb->CpuStep,
Vendor[0],
Vendor[1],
Vendor[2]);
}
/* Set the current MP Master KPRCB to the Boot PRCB */
Prcb->MultiThreadSetMaster = Prcb;
/* Lower to APC_LEVEL */
KeLowerIrql(APC_LEVEL);
/* Initialize some spinlocks */
KeInitializeSpinLock(&KiFreezeExecutionLock);
KeInitializeSpinLock(&Ki486CompatibilityLock);
/* Initialize portable parts of the OS */
KiInitSystem();
/* Initialize the Idle Process and the Process Listhead */
InitializeListHead(&KiProcessListHead);
PageDirectory[0] = 0;
PageDirectory[1] = 0;
KeInitializeProcess(InitProcess,
0,
0xFFFFFFFF,
PageDirectory,
FALSE);
InitProcess->QuantumReset = MAXCHAR;
}
else
{
/* FIXME */
DPRINT1("SMP Boot support not yet present\n");
}
/* Setup the Idle Thread */
KeInitializeThread(InitProcess,
InitThread,
NULL,
NULL,
NULL,
NULL,
NULL,
IdleStack);
InitThread->NextProcessor = Number;
InitThread->Priority = HIGH_PRIORITY;
InitThread->State = Running;
InitThread->Affinity = 1 << Number;
InitThread->WaitIrql = DISPATCH_LEVEL;
InitProcess->ActiveProcessors = 1 << Number;
/* HACK for MmUpdatePageDir */
((PETHREAD)InitThread)->ThreadsProcess = (PEPROCESS)InitProcess;
/* Set basic CPU Features that user mode can read */
SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
(KeFeatureBits & KF_MMX) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
(KeFeatureBits & KF_CMPXCHG8B) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI)) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
((KeFeatureBits & KF_FXSR) && (KeFeatureBits & KF_XMMI64)) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
(KeFeatureBits & KF_3DNOW) ? TRUE: FALSE;
SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
(KeFeatureBits & KF_RDTSC) ? TRUE: FALSE;
/* Set up the thread-related fields in the PRCB */
Prcb->CurrentThread = InitThread;
Prcb->NextThread = NULL;
Prcb->IdleThread = InitThread;
/* Initialize the Kernel Executive */
ExpInitializeExecutive(Number, LoaderBlock);
/* Only do this on the boot CPU */
if (!Number)
{
/* Calculate the time reciprocal */
KiTimeIncrementReciprocal =
KiComputeReciprocal(KeMaximumIncrement,
&KiTimeIncrementShiftCount);
/* Update DPC Values in case they got updated by the executive */
Prcb->MaximumDpcQueueDepth = KiMaximumDpcQueueDepth;
Prcb->MinimumDpcRate = KiMinimumDpcRate;
Prcb->AdjustDpcThreshold = KiAdjustDpcThreshold;
/* Allocate the DPC Stack */
DpcStack = MmCreateKernelStack(FALSE, 0);
if (!DpcStack) KeBugCheckEx(NO_PAGES_AVAILABLE, 1, 0, 0, 0);
Prcb->DpcStack = DpcStack;
/* Allocate the IOPM save area. */
Ki386IopmSaveArea = ExAllocatePoolWithTag(PagedPool,
PAGE_SIZE * 2,
' eK');
if (!Ki386IopmSaveArea)
{
/* Bugcheck. We need this for V86/VDM support. */
KeBugCheckEx(NO_PAGES_AVAILABLE, 2, PAGE_SIZE * 2, 0, 0);
}
}
/* Raise to Dispatch */
KfRaiseIrql(DISPATCH_LEVEL);
/* Set the Idle Priority to 0. This will jump into Phase 1 */
KeSetPriorityThread(InitThread, 0);
/* If there's no thread scheduled, put this CPU in the Idle summary */
KiAcquirePrcbLock(Prcb);
if (!Prcb->NextThread) KiIdleSummary |= 1 << Number;
KiReleasePrcbLock(Prcb);
/* Raise back to HIGH_LEVEL and clear the PRCB for the loader block */
KfRaiseIrql(HIGH_LEVEL);
LoaderBlock->Prcb = 0;
}
