KCONTINUE_STATUS
KeSwitchFrozenProcessor (
IN ULONG ProcessorNumber
)
{
#if !defined(NT_UP)
PKPRCB TargetPrcb, CurrentPrcb;
//
// If Processor number is out of range, reselect current processor
//
if (ProcessorNumber >= (ULONG) KeNumberProcessors) {
return ContinueProcessorReselected;
}
TargetPrcb = KiProcessorBlock[ProcessorNumber];
CurrentPrcb = KeGetCurrentPrcb();
//
// Move active flag to correct processor.
//
CurrentPrcb->IpiFrozen &= ~FREEZE_ACTIVE;
TargetPrcb->IpiFrozen |= FREEZE_ACTIVE;
//
// If this processor is frozen in KiFreezeTargetExecution, return to it
//
if (FrozenState(CurrentPrcb->IpiFrozen) == TARGET_FROZEN) {
return ContinueNextProcessor;
}
//
// This processor must be FREEZE_OWNER, wait to be reselected as the
// active processor
//
if (FrozenState(CurrentPrcb->IpiFrozen) != FREEZE_OWNER) {
return ContinueError;
}
while (!(CurrentPrcb->IpiFrozen & FREEZE_ACTIVE)) {
KeYieldProcessor();
}
#endif // !defined(NT_UP)
//
// Reselect this processor
//
return ContinueProcessorReselected;
}
VOID
KiPollFreezeExecution(
VOID
)
/*++
Routine Description:
This routine is called from code that is spinning with interrupts
disabled, waiting for something to happen, when there is some
(possibly extremely small) chance that that thing will not happen
because a system freeze has been initiated.
N.B. Interrupts are disabled.
Arguments:
None.
Return Value:
None.
--*/
{
//
// Check to see if a freeze is pending for this processor.
//
PKPRCB Prcb = KeGetCurrentPrcb();
if ((Prcb->RequestSummary & IPI_FREEZE) != 0) {
//
// Clear the freeze request and freeze this processor.
//
InterlockedExchangeAdd((PLONG)&Prcb->RequestSummary, -(IPI_FREEZE));
KiFreezeTargetExecution(NULL, NULL);
} else {
//
// No freeze pending, assume this processor is spinning.
//
KeYieldProcessor();
}
}
VOID
KeInitialize(VOID) {
HANDLE testScrollerProcess, testShell;
HalInitialize();
#ifdef HAVE_KD
KdInitialize();
#endif
MmInitialize();
IoInitialize();
//Initialize ps manager, create initial process
PsInitialize();
ASSERT(KeCurrentProcess);
//Initialize message passing
MessInitialize();
KdPrint("Say %s to %s!", "hello", "ARCOS kernel debugger");
PsCreateProcess(TestScrollerProcess, 10, &testScrollerProcess, NULL);
ObCloseHandle(testScrollerProcess);
CreateProcess("shell", 31, &testShell, NULL);
ObCloseHandle(testShell);
KeRestoreInterrupts(TRUE);
while (1) {
// wait for something interesting to happen
KeYieldProcessor();
}
}
NTKERNELAPI
NTSTATUS
FASTCALL
ExTimedWaitForUnblockPushLock (
__inout PEX_PUSH_LOCK PushLock,
__inout PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock,
__in_opt PLARGE_INTEGER Timeout
)
{
ULONG i;
NTSTATUS Status;
UNREFERENCED_PARAMETER (PushLock);
KeInitializeEvent (&WaitBlock->WakeEvent, SynchronizationEvent, FALSE);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock->Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
return STATUS_SUCCESS;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
Status = KeWaitForSingleObject (&WaitBlock->WakeEvent,
WrPushLock,
KernelMode,
FALSE,
Timeout);
if (Status != STATUS_SUCCESS) {
ExfUnblockPushLock (PushLock,
WaitBlock);
}
} else {
Status = STATUS_SUCCESS;
}
return Status;
}
VOID
KeStartAllProcessors (
VOID
)
/*++
Routine Description:
This function is called during phase 1 initialization on the master boot
processor to start all of the other registered processors.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
ULONG AllocationSize;
PUCHAR Base;
PKPCR CurrentPcr = KeGetPcr();
PVOID DataBlock;
PVOID DpcStack;
PKGDTENTRY64 GdtBase;
ULONG GdtOffset;
ULONG IdtOffset;
UCHAR Index;
PVOID KernelStack;
ULONG LogicalProcessors;
ULONG MaximumProcessors;
PKNODE Node;
UCHAR NodeNumber = 0;
UCHAR Number;
KIRQL OldIrql;
PKNODE OldNode;
PKNODE ParentNode;
PKPCR PcrBase;
PKPRCB Prcb;
USHORT ProcessorId;
KPROCESSOR_STATE ProcessorState;
PKTSS64 SysTssBase;
PKGDTENTRY64 TebBase;
PETHREAD Thread;
//
// Ensure that prefetch instructions in the IPI path are patched out
// if necessary before starting other processors.
//
OldIrql = KeRaiseIrqlToSynchLevel();
KiIpiSendRequest(1, 0, 0, IPI_FLUSH_SINGLE);
KeLowerIrql(OldIrql);
//
// Do not start additional processors if the relocate physical loader
// switch has been specified.
//
if (KeLoaderBlock->LoadOptions != NULL) {
if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) {
return;
}
}
//
// If this a multinode system and processor zero is not on node zero,
// then move it to the appropriate node.
//
if (KeNumberNodes > 1) {
if (NT_SUCCESS(KiQueryProcessorNode(0, &ProcessorId, &NodeNumber))) {
if (NodeNumber != 0) {
KiNode0.ProcessorMask = 0;
KiNodeInit[0] = KiNode0;
KeNodeBlock[0] = &KiNodeInit[0];
KiNode0 = *KeNodeBlock[NodeNumber];
KeNodeBlock[NodeNumber] = &KiNode0;
KiNode0.ProcessorMask = 1;
}
} else {
goto StartFailure;
}
}
//
// Calculate the size of the per processor data structures.
//
// This includes:
//
// PCR (including the PRCB)
// System TSS
// Idle Thread Object
// Double Fault Stack
// Machine Check Stack
// NMI Stack
// Multinode structure
// GDT
// IDT
//
// A DPC and Idle stack are also allocated, but they are done separately.
//
AllocationSize = ROUNDUP64(sizeof(KPCR)) +
ROUNDUP64(sizeof(KTSS64)) +
ROUNDUP64(sizeof(ETHREAD)) +
ROUNDUP64(DOUBLE_FAULT_STACK_SIZE) +
ROUNDUP64(KERNEL_MCA_EXCEPTION_STACK_SIZE) +
ROUNDUP64(NMI_STACK_SIZE) +
ROUNDUP64(sizeof(KNODE));
//
// Save the offset of the GDT in the allocation structure and add in
// the size of the GDT.
//
GdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
//
// Save the offset of the IDT in the allocation structure and add in
// the size of the IDT.
//
IdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
//
// If the registered number of processors is greater than the maximum
// number of processors supported, then only allow the maximum number
// of supported processors.
//
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
KeRegisteredProcessors = MAXIMUM_PROCESSORS;
}
//
// Set barrier that will prevent any other processor from entering the
// idle loop until all processors have been started.
//
KiBarrierWait = 1;
//
// Initialize the fixed part of the processor state that will be used to
// start processors. Each processor starts in the system initialization
// code with address of the loader parameter block as an argument.
//
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock;
ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup;
ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE;
ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK;
ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegSs = KGDT64_R0_DATA;
//
// Check to determine if hyper-threading is really enabled. Intel chips
// claim to be hyper-threaded with the number of logical processors
// greater than one even when hyper-threading is disabled in the BIOS.
//
LogicalProcessors = KiLogicalProcessors;
if (HalIsHyperThreadingEnabled() == FALSE) {
LogicalProcessors = 1;
}
//
// If the total number of logical processors has not been set with
// the /NUMPROC loader option, then set the maximum number of logical
// processors to the number of registered processors times the number
// of logical processors per registered processor.
//
// N.B. The number of logical processors is never allowed to exceed
// the number of registered processors times the number of logical
// processors per physical processor.
//
MaximumProcessors = KeNumprocSpecified;
if (MaximumProcessors == 0) {
MaximumProcessors = KeRegisteredProcessors * LogicalProcessors;
}
//
// Loop trying to start a new processors until a new processor can't be
// started or an allocation failure occurs.
//
// N.B. The below processor start code relies on the fact a physical
// processor is started followed by all its logical processors.
// The HAL guarantees this by sorting the ACPI processor table
// by APIC id.
//
Index = 0;
Number = 0;
while ((Index < (MAXIMUM_PROCESSORS - 1)) &&
((ULONG)KeNumberProcessors < MaximumProcessors) &&
((ULONG)KeNumberProcessors / LogicalProcessors) < KeRegisteredProcessors) {
//
// If this is a multinode system and current processor does not
// exist on any node, then skip it.
//
Index += 1;
if (KeNumberNodes > 1) {
if (!NT_SUCCESS(KiQueryProcessorNode(Index, &ProcessorId, &NodeNumber))) {
continue;
}
}
//
// Increment the processor number.
//
Number += 1;
//
// Allocate memory for the new processor specific data. If the
// allocation fails, then stop starting processors.
//
DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber);
if (DataBlock == NULL) {
goto StartFailure;
}
//
// Allocate a pool tag table for the new processor.
//
if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) {
goto StartFailure;
}
//
// Zero the allocated memory.
//
Base = (PUCHAR)DataBlock;
RtlZeroMemory(DataBlock, AllocationSize);
//
// Copy and initialize the GDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr,
&ProcessorState.SpecialRegisters.Gdtr,
Base + GdtOffset);
GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
//
// Encode the processor number in the upper 6 bits of the compatibility
// mode TEB descriptor.
//
TebBase = (PKGDTENTRY64)((PCHAR)GdtBase + KGDT64_R3_CMTEB);
TebBase->Bits.LimitHigh = Number >> 2;
TebBase->LimitLow = ((Number & 0x3) << 14) | (TebBase->LimitLow & 0x3fff);
//
// Copy and initialize the IDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr,
&ProcessorState.SpecialRegisters.Idtr,
Base + IdtOffset);
//
// Set the PCR base address for the next processor, set the processor
// number, and set the processor speed.
//
// N.B. The PCR address is passed to the next processor by computing
// the containing address with respect to the PRCB.
//
PcrBase = (PKPCR)Base;
PcrBase->ObsoleteNumber = Number;
PcrBase->Prcb.Number = Number;
PcrBase->Prcb.MHz = KeGetCurrentPrcb()->MHz;
Base += ROUNDUP64(sizeof(KPCR));
//
// Set the system TSS descriptor base for the next processor.
//
SysTssBase = (PKTSS64)Base;
KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase);
Base += ROUNDUP64(sizeof(KTSS64));
//
// Initialize the panic stack address for double fault and NMI.
//
Base += DOUBLE_FAULT_STACK_SIZE;
SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
//
// Initialize the machine check stack address.
//
Base += KERNEL_MCA_EXCEPTION_STACK_SIZE;
SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
//
// Initialize the NMI stack address.
//
Base += NMI_STACK_SIZE;
SysTssBase->Ist[TSS_IST_NMI] = (ULONG64)Base;
//
// Idle Thread thread object.
//
Thread = (PETHREAD)Base;
Base += ROUNDUP64(sizeof(ETHREAD));
//
// Set other special registers in the processor state.
//
ProcessorState.SpecialRegisters.Cr0 = ReadCR0();
ProcessorState.SpecialRegisters.Cr3 = ReadCR3();
ProcessorState.ContextFrame.EFlags = 0;
ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS;
GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89;
ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
//
// Allocate a kernel stack and a DPC stack for the next processor.
//
KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
if (KernelStack == NULL) {
goto StartFailure;
}
DpcStack = MmCreateKernelStack(FALSE, NodeNumber);
if (DpcStack == NULL) {
goto StartFailure;
}
//
// Initialize the kernel stack for the system TSS.
//
// N.B. System startup must be called with a stack pointer that is
// 8 mod 16.
//
SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4;
ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack - 8;
//
// If this is the first processor on this node, then use the space
// already allocated for the node. Otherwise, the space allocated
// is not used.
//
Node = KeNodeBlock[NodeNumber];
OldNode = Node;
if (Node == &KiNodeInit[NodeNumber]) {
Node = (PKNODE)Base;
*Node = KiNodeInit[NodeNumber];
KeNodeBlock[NodeNumber] = Node;
}
Base += ROUNDUP64(sizeof(KNODE));
//
// Set the parent node address.
//
PcrBase->Prcb.ParentNode = Node;
//
// Adjust the loader block so it has the next processor state. Ensure
// that the kernel stack has space for home registers for up to four
// parameters.
//
KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4);
KeLoaderBlock->Thread = (ULONG64)Thread;
KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
//
// Attempt to start the next processor. If a processor cannot be
// started, then deallocate memory and stop starting processors.
//
if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) {
//
// Restore the old node address in the node address array before
// freeing the allocated data block (the node structure lies
// within the data block).
//
*OldNode = *Node;
KeNodeBlock[NodeNumber] = OldNode;
ExDeletePoolTagTable(Number);
MmFreeIndependentPages(DataBlock, AllocationSize);
MmDeleteKernelStack(KernelStack, FALSE);
MmDeleteKernelStack(DpcStack, FALSE);
break;
}
Node->ProcessorMask |= AFFINITY_MASK(Number);
//
// Wait for processor to initialize.
//
while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) {
KeYieldProcessor();
}
}
//
// All processors have been started. If this is a multinode system, then
// allocate any missing node structures.
//
if (KeNumberNodes > 1) {
for (Index = 0; Index < KeNumberNodes; Index += 1) {
if (KeNodeBlock[Index] == &KiNodeInit[Index]) {
Node = ExAllocatePoolWithTag(NonPagedPool, sizeof(KNODE), ' eK');
if (Node != NULL) {
*Node = KiNodeInit[Index];
KeNodeBlock[Index] = Node;
} else {
goto StartFailure;
}
}
}
} else if (KiNode0.ProcessorMask != KeActiveProcessors) {
goto StartFailure;
}
//
// Clear node structure address for nonexistent nodes.
//
for (Index = KeNumberNodes; Index < MAXIMUM_CCNUMA_NODES; Index += 1) {
KeNodeBlock[Index] = NULL;
}
//
// Copy the node color and shifted color to the PRCB of each processor.
//
for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) {
Prcb = KiProcessorBlock[Index];
ParentNode = Prcb->ParentNode;
Prcb->NodeColor = ParentNode->Color;
Prcb->NodeShiftedColor = ParentNode->MmShiftedColor;
Prcb->SecondaryColorMask = MmSecondaryColorMask;
}
//
// Reset the initialization bit in prefetch retry.
//
KiPrefetchRetry &= ~0x80;
//
// Reset and synchronize the performance counters of all processors, by
// applying a null adjustment to the interrupt time.
//
KeAdjustInterruptTime(0);
//
// Allow all processors that were started to enter the idle loop and
// begin execution.
//
KiBarrierWait = 0;
#endif //
return;
//
// The failure to allocate memory or a unsuccessful status was returned
// during the attempt to start processors. This is considered fatal since
// something is very wrong.
//
#if !defined(NT_UP)
StartFailure:
KeBugCheckEx(PHASE1_INITIALIZATION_FAILED, 0, 0, 20, 0);
#endif
}
VOID
KiIpiStallOnPacketTargets (
KAFFINITY TargetSet
)
/*++
Routine Description:
This function waits until the specified set of processors have signaled
their completion of a requested function.
N.B. The exact protocol used between the source and the target of an
interprocessor request is not specified. Minimally the source
must construct an appropriate packet and send the packet to a set
of specified targets. Each target receives the address of the packet
address as an argument, and minimally must clear the packet address
when the mutually agreed upon protocol allows. The target has three
options:
1. Capture necessary information, release the source by clearing
the packet address, execute the request in parallel with the
source, and return from the interrupt.
2. Execute the request in series with the source, release the
source by clearing the packet address, and return from the
interrupt.
3. Execute the request in series with the source, release the
source, wait for a reply from the source based on a packet
parameter, and return from the interrupt.
This function is provided to enable the source to synchronize with the
target for cases 2 and 3 above.
N.B. There is no support for method 3 above.
Arguments:
TargetSet - Supplies the the target set of IPI processors.
Return Value:
None.
--*/
{
PKPRCB Prcb;
//
// Wait until the target set of processors is zero in the current
// processor's packet.
//
Prcb = KeGetCurrentPrcb();
while (Prcb->TargetSet != 0) {
KeYieldProcessor();
}
//
// If the target set is equal to the entire set of processors, then
// update the memory barrier time stamp.
//
#if defined(_ALPHA_)
if ((TargetSet | PCR->SetMember ) == KeActiveProcessors) {
InterlockedIncrement(&KiMbTimeStamp);
}
#endif
return;
}
BOOLEAN
KeFreezeExecution (
IN PKTRAP_FRAME TrapFrame,
IN PKEXCEPTION_FRAME ExceptionFrame
)
/*++
Routine Description:
This function freezes the execution of all other processors in the host
configuration and then returns to the caller.
Arguments:
TrapFrame - Supplies a pointer to a trap frame that describes the
trap.
ExceptionFrame - Supplies a pointer to an exception frame that
describes the trap.
Return Value:
Previous interrupt enable.
--*/
{
BOOLEAN Enable;
#if !defined(NT_UP)
BOOLEAN Flag;
PKPRCB Prcb;
ULONG TargetSet;
ULONG BitNumber;
KIRQL OldIrql;
#if IDBG
ULONG Count = 30000;
#endif
#endif
//
// Disable interrupts.
//
Enable = KiDisableInterrupts();
KiFreezeFlag = FREEZE_FROZEN;
#if !defined(NT_UP)
//
// Raise IRQL to HIGH_LEVEL.
//
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
if (FrozenState(KeGetCurrentPrcb()->IpiFrozen) == FREEZE_OWNER) {
//
// This processor already owns the freeze lock.
// Return without trying to re-acquire lock or without
// trying to IPI the other processors again
//
return Enable;
}
//
// Try to acquire the KiFreezeExecutionLock before sending the request.
// To prevent deadlock from occurring, we need to accept and process
// incoming FreexeExecution requests while we are waiting to acquire
// the FreezeExecutionFlag.
//
while (KiTryToAcquireSpinLock (&KiFreezeExecutionLock) == FALSE) {
//
// FreezeExecutionLock is busy. Another processor may be trying
// to IPI us - go service any IPI.
//
KiRestoreInterrupts(Enable);
Flag = KiIpiServiceRoutine((PVOID)TrapFrame, (PVOID)ExceptionFrame);
KiDisableInterrupts();
#if IDBG
if (Flag != FALSE) {
Count = 30000;
continue;
}
KeStallExecutionProcessor (100);
if (!Count--) {
Count = 30000;
if (KiTryToAcquireSpinLock (&KiFreezeLockBackup) == TRUE) {
KiFreezeFlag |= FREEZE_BACKUP;
break;
}
}
#endif
}
//
// After acquiring the lock flag, we send Freeze request to each processor
// in the system (other than us) and wait for it to become frozen.
//
Prcb = KeGetCurrentPrcb(); // Do this after spinlock is acquired.
TargetSet = KeActiveProcessors & ~(1 << Prcb->Number);
if (TargetSet) {
#if IDBG
Count = 400;
#endif
KiFreezeOwner = Prcb;
Prcb->IpiFrozen = FREEZE_OWNER | FREEZE_ACTIVE;
Prcb->SkipTick = TRUE;
KiIpiSend((KAFFINITY) TargetSet, IPI_FREEZE);
while (TargetSet != 0) {
BitNumber = KeFindFirstSetRightMember(TargetSet);
ClearMember(BitNumber, TargetSet);
Prcb = KiProcessorBlock[BitNumber];
#if IDBG
while (Prcb->IpiFrozen != TARGET_FROZEN) {
if (Count == 0) {
KiFreezeFlag |= FREEZE_SKIPPED_PROCESSOR;
break;
}
KeStallExecutionProcessor (10000);
Count--;
}
#else
while (Prcb->IpiFrozen != TARGET_FROZEN) {
KeYieldProcessor();
}
#endif
}
}
//
// Save the old IRQL and return whether interrupts were previous enabled.
//
KiOldIrql = OldIrql;
#endif // !defined(NT_UP)
return Enable;
}
VOID
KeThawExecution (
IN BOOLEAN Enable
)
/*++
Routine Description:
This function thaws the execution of all other processors in the host
configuration and then returns to the caller. It is intended for use by
the kernel debugger.
Arguments:
Enable - Supplies the previous interrupt enable that is to be restored
after having thawed the execution of all other processors.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
KIRQL OldIrql;
ULONG TargetSet;
ULONG BitNumber;
ULONG Flag;
PKPRCB Prcb;
//
// Before releasing FreezeExecutionLock clear any all targets IpiFrozen
// flag.
//
KeGetCurrentPrcb()->IpiFrozen = RUNNING;
TargetSet = KeActiveProcessors & ~(1 << KeGetCurrentPrcb()->Number);
while (TargetSet != 0) {
BitNumber = KeFindFirstSetRightMember(TargetSet);
ClearMember(BitNumber, TargetSet);
Prcb = KiProcessorBlock[BitNumber];
#if IDBG
//
// If the target processor was not forzen, then don't wait
// for target to unfreeze.
//
if (FrozenState(Prcb->IpiFrozen) != TARGET_FROZEN) {
Prcb->IpiFrozen = RUNNING;
continue;
}
#endif
Prcb->IpiFrozen = TARGET_THAW;
while (Prcb->IpiFrozen == TARGET_THAW) {
KeYieldProcessor();
}
}
//
// Capture the previous IRQL before releasing the freeze lock.
//
OldIrql = KiOldIrql;
#if IDBG
Flag = KiFreezeFlag;
KiFreezeFlag = 0;
if ((Flag & FREEZE_BACKUP) != 0) {
KiReleaseSpinLock(&KiFreezeLockBackup);
} else {
KiReleaseSpinLock(&KiFreezeExecutionLock);
}
#else
KiFreezeFlag = 0;
KiReleaseSpinLock(&KiFreezeExecutionLock);
#endif
#endif // !defined (NT_UP)
//
// Flush the current TB, instruction cache, and data cache.
//
KeFlushCurrentTb();
KeSweepCurrentIcache();
KeSweepCurrentDcache();
//
// Lower IRQL and restore interrupt enable
//
#if !defined(NT_UP)
KeLowerIrql(OldIrql);
#endif
KiRestoreInterrupts(Enable);
return;
}
VOID
KiFreezeTargetExecution (
IN PKTRAP_FRAME TrapFrame,
IN PKEXCEPTION_FRAME ExceptionFrame
)
/*++
Routine Description:
This function freezes the execution of the current running processor.
If a trapframe is supplied to current state is saved into the prcb
for the debugger.
Arguments:
TrapFrame - Supplies a pointer to the trap frame that describes the
trap.
ExceptionFrame - Supplies a pointer to the exception frame that
describes the trap.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
KIRQL OldIrql;
PKPRCB Prcb;
BOOLEAN Enable;
KCONTINUE_STATUS Status;
EXCEPTION_RECORD ExceptionRecord;
Enable = KiDisableInterrupts();
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
Prcb = KeGetCurrentPrcb();
Prcb->IpiFrozen = TARGET_FROZEN;
Prcb->SkipTick = TRUE;
if (TrapFrame != NULL) {
KiSaveProcessorState(TrapFrame, ExceptionFrame);
}
//
// Sweep the data cache in case this is a system crash and the bug
// check code is attempting to write a crash dump file.
//
KeSweepCurrentDcache();
//
// Wait for person requesting us to freeze to
// clear our frozen flag
//
while (FrozenState(Prcb->IpiFrozen) == TARGET_FROZEN) {
if (Prcb->IpiFrozen & FREEZE_ACTIVE) {
//
// This processor has been made the active processor
//
if (TrapFrame) {
RtlZeroMemory (&ExceptionRecord, sizeof ExceptionRecord);
ExceptionRecord.ExceptionCode = STATUS_WAKE_SYSTEM_DEBUGGER;
ExceptionRecord.ExceptionRecord = &ExceptionRecord;
ExceptionRecord.ExceptionAddress =
(PVOID)CONTEXT_TO_PROGRAM_COUNTER (&Prcb->ProcessorState.ContextFrame);
Status = (KiDebugSwitchRoutine) (
&ExceptionRecord,
&Prcb->ProcessorState.ContextFrame,
FALSE
);
} else {
Status = ContinueError;
}
//
// If status is anything other then, continue with next
// processor then reselect master
//
if (Status != ContinueNextProcessor) {
Prcb->IpiFrozen &= ~FREEZE_ACTIVE;
KiFreezeOwner->IpiFrozen |= FREEZE_ACTIVE;
}
}
KeYieldProcessor();
}
if (TrapFrame != NULL) {
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
}
Prcb->IpiFrozen = RUNNING;
KeFlushCurrentTb();
KeSweepCurrentIcache();
KeLowerIrql(OldIrql);
KiRestoreInterrupts(Enable);
#endif // !define(NT_UP)
return;
}
DECLSPEC_NOINLINE
PKAFFINITY
KiIpiSendRequest (
IN KAFFINITY TargetSet,
IN ULONG64 Parameter,
IN ULONG64 Count,
IN ULONG64 RequestType
)
/*++
Routine Description:
This routine executes the specified immediate request on the specified
set of processors.
N.B. This function MUST be called from a non-context switchable state.
Arguments:
TargetProcessors - Supplies the set of processors on which the specfied
operation is to be executed.
Parameter - Supplies the parameter data that will be packed into the
request summary.
Count - Supplies the count data that will be packed into the request summary.
RequestType - Supplies the type of immediate request.
Return Value:
The address of the appropriate request barrier is returned as the function
value.
--*/
{
#if !defined(NT_UP)
PKAFFINITY Barrier;
PKPRCB Destination;
ULONG Number;
KAFFINITY PacketTargetSet;
PKPRCB Prcb;
ULONG Processor;
PREQUEST_MAILBOX RequestMailbox;
KAFFINITY SetMember;
PVOID *Start;
KAFFINITY SummarySet;
KAFFINITY TargetMember;
REQUEST_SUMMARY Template;
PVOID *Virtual;
ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
//
// Initialize request template.
//
Prcb = KeGetCurrentPrcb();
Template.Summary = 0;
Template.IpiRequest = RequestType;
Template.Count = Count;
Template.Parameter = Parameter;
//
// If the target set contains one and only one processor, then use the
// target set for signal done synchronization. Otherwise, use packet
// barrier for signal done synchronization.
//
Prcb->TargetSet = TargetSet;
if ((TargetSet & (TargetSet - 1)) == 0) {
Template.IpiSynchType = TRUE;
Barrier = (PKAFFINITY)&Prcb->TargetSet;
} else {
Prcb->PacketBarrier = 1;
Barrier = (PKAFFINITY)&Prcb->PacketBarrier;
}
//
// Loop through the target set of processors and set the request summary.
// If a target processor is already processing a request, then remove
// that processor from the target set of processor that will be sent an
// interprocessor interrupt.
//
// N.B. It is guaranteed that there is at least one bit set in the target
// set.
//
Number = Prcb->Number;
SetMember = Prcb->SetMember;
SummarySet = TargetSet;
PacketTargetSet = TargetSet;
BitScanForward64(&Processor, SummarySet);
do {
Destination = KiProcessorBlock[Processor];
PrefetchForWrite(&Destination->SenderSummary);
RequestMailbox = &Destination->RequestMailbox[Number];
PrefetchForWrite(RequestMailbox);
TargetMember = AFFINITY_MASK(Processor);
//
// Make sure that processing of the last IPI is complete before sending
// another IPI to the same processor.
//
while ((Destination->SenderSummary & SetMember) != 0) {
KeYieldProcessor();
}
//
// If the request type is flush multiple and the flush entries will
// fit in the mailbox, then copy the virtual address array to the
// destination mailbox and change the request type to flush immediate.
//
// If the request type is packet ready, then copy the packet to the
// destination mailbox.
//
if (RequestType == IPI_FLUSH_MULTIPLE) {
Virtual = &RequestMailbox->Virtual[0];
Start = (PVOID *)Parameter;
switch (Count) {
//
// Copy of up to seven virtual addresses and a conversion of
// the request type to flush multiple immediate.
//
case 7:
Virtual[6] = Start[6];
case 6:
Virtual[5] = Start[5];
case 5:
Virtual[4] = Start[4];
case 4:
Virtual[3] = Start[3];
case 3:
Virtual[2] = Start[2];
case 2:
Virtual[1] = Start[1];
case 1:
Virtual[0] = Start[0];
Template.IpiRequest = IPI_FLUSH_MULTIPLE_IMMEDIATE;
break;
}
} else if (RequestType == IPI_PACKET_READY) {
RequestMailbox->RequestPacket = *(PKREQUEST_PACKET)Parameter;
}
RequestMailbox->RequestSummary = Template.Summary;
if (InterlockedExchangeAdd64((LONG64 volatile *)&Destination->SenderSummary,
SetMember) != 0) {
TargetSet ^= TargetMember;
}
SummarySet ^= TargetMember;
} while (BitScanForward64(&Processor, SummarySet) != FALSE);
//
// Request interprocessor interrupts on the remaining target set of
// processors.
//
//
// N.B. For packet sends, there exists a potential deadlock situation
// unless an IPI is sent to the original set of target processors.
// The deadlock arises from the fact that the targets will spin in
// their IPI routines.
//
if (RequestType == IPI_PACKET_READY) {
TargetSet = PacketTargetSet;
}
if (TargetSet != 0) {
HalRequestIpi(TargetSet);
}
return Barrier;
#else
UNREFERENCED_PARAMETER(TargetSet);
UNREFERENCED_PARAMETER(Parameter);
UNREFERENCED_PARAMETER(Count);
UNREFERENCED_PARAMETER(RequestType);
return NULL;
#endif
}
ULONG_PTR
KeIpiGenericCall (
IN PKIPI_BROADCAST_WORKER BroadcastFunction,
IN ULONG_PTR Context
)
/*++
Routine Description:
This function executes the specified function on every processor in
the host configuration in a synchronous manner, i.e., the function
is executed on each target in series with the execution of the source
processor.
Arguments:
BroadcastFunction - Supplies the address of function that is executed
on each of the target processors.
Context - Supplies the value of the context parameter that is passed
to each function.
Return Value:
The value returned by the specified function on the source processor
is returned as the function value.
--*/
{
volatile LONG Count;
KIRQL OldIrql;
ULONG_PTR Status;
#if !defined(NT_UP)
KAFFINITY TargetProcessors;
#endif
//
// Raise IRQL to synchronization level and acquire the reverse stall spin
// lock to synchronize with other reverse stall functions.
//
OldIrql = KeGetCurrentIrql();
if (OldIrql < SYNCH_LEVEL) {
KfRaiseIrql(SYNCH_LEVEL);
}
#if !defined(NT_UP)
Count = KeNumberProcessors;
TargetProcessors = KeActiveProcessors & ~KeGetCurrentPrcb()->SetMember;
#endif
KeAcquireSpinLockAtDpcLevel(&KiReverseStallIpiLock);
//
// Initialize the broadcast packet, compute the set of target processors,
// and sent the packet to the target processors for execution.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiIpiGenericCallTarget,
(PVOID)(ULONG_PTR)BroadcastFunction,
(PVOID)Context,
(PVOID)&Count);
}
//
// Wait until all processors have entered the target routine and are
// waiting.
//
while (Count != 1) {
KeYieldProcessor();
}
#endif
//
// Raise IRQL to IPI_LEVEL, signal all other processors to proceed, and
// call the specified function on the source processor.
//
KfRaiseIrql(IPI_LEVEL);
Count = 0;
Status = BroadcastFunction(Context);
//
// Wait until all of the target processors have finished capturing the
// function parameters.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
#endif
//
// Release reverse stall spin lock, lower IRQL to its previous level,
// and return the function execution status.
//
KeReleaseSpinLockFromDpcLevel(&KiReverseStallIpiLock);
KeLowerIrql(OldIrql);
return Status;
}
NTKERNELAPI
VOID
DECLSPEC_NOINLINE
FASTCALL
ExfAcquirePushLockShared (
__inout PEX_PUSH_LOCK PushLock
)
/*++
Routine Description:
Acquire a push lock shared
Arguments:
PushLock - Push lock to be acquired
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue, TopValue;
EX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
BOOLEAN Optimize;
#if defined (USE_EXP_BACKOFF)
RTL_BACKOFF Backoff = {0};
#endif
OldValue = ReadForWriteAccess (PushLock);
while (1) {
//
// If the lock is already held we need to wait if its not held shared
//
if (!OldValue.Locked || (!OldValue.Waiting && OldValue.Shared > 0)) {
if (OldValue.Waiting) {
NewValue.Value = OldValue.Value + EX_PUSH_LOCK_LOCK;
} else {
NewValue.Value = (OldValue.Value + EX_PUSH_LOCK_SHARE_INC) | EX_PUSH_LOCK_LOCK;
}
ASSERT (NewValue.Locked);
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
break;
}
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
} else {
WaitBlock.Flags = EX_PUSH_LOCK_FLAGS_SPINNING;
WaitBlock.ShareCount = 0;
Optimize = FALSE;
WaitBlock.Previous = NULL;
//
// Move the sharecount to our wait block if need be.
//
if (OldValue.Waiting) {
WaitBlock.Last = NULL;
WaitBlock.Next = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~EX_PUSH_LOCK_PTR_BITS);
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & (EX_PUSH_LOCK_LOCK | EX_PUSH_LOCK_MULTIPLE_SHARED)) |
EX_PUSH_LOCK_WAITING | EX_PUSH_LOCK_WAKING);
if (!OldValue.Waking) {
Optimize = TRUE;
}
} else {
WaitBlock.Last = &WaitBlock;
NewValue.Ptr = (PVOID)(((ULONG_PTR) &WaitBlock) |
(OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING);
}
ASSERT (NewValue.Waiting);
#if DBG
WaitBlock.Signaled = FALSE;
WaitBlock.OldValue = OldValue.Ptr;
WaitBlock.NewValue = NewValue.Ptr;
WaitBlock.PushLock = PushLock;
#endif
TopValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr == OldValue.Ptr) {
ULONG i;
//
// If we set the waiting bit then optimize the list
//
if (Optimize) {
ExpOptimizePushLockList (PushLock, TopValue);
}
//
// It is safe to initialize the gate here, as the interlocked operation below forces
// a gate signal to always follow gate initialization.
//
KeInitializeGate (&WaitBlock.WakeGate);
for (i = ExPushLockSpinCount; i > 0; i--) {
if (((*(volatile LONG *)&WaitBlock.Flags)&EX_PUSH_LOCK_FLAGS_SPINNING) == 0) {
break;
}
KeYieldProcessor ();
}
if (InterlockedBitTestAndReset ((LONG*)&WaitBlock.Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeWaitForGate (&WaitBlock.WakeGate, WrPushLock, KernelMode);
#if DBG
ASSERT (WaitBlock.Signaled);
#endif
}
} else {
#if defined (USE_EXP_BACKOFF)
//
// Use backof to limit memory bandwidth
//
RtlBackoff (&Backoff);
NewValue = *PushLock;
#endif
}
}
OldValue = NewValue;
}
}