VOID
KeStartAllProcessors (
VOID
)
/*++
Routine Description:
This function is called during phase 1 initialization on the master boot
processor to start all of the other registered processors.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
ULONG AllocationSize;
PUCHAR Base;
PKPCR CurrentPcr = KeGetPcr();
PVOID DataBlock;
PVOID DpcStack;
PKGDTENTRY64 GdtBase;
ULONG GdtOffset;
ULONG IdtOffset;
UCHAR Index;
PVOID KernelStack;
ULONG LogicalProcessors;
ULONG MaximumProcessors;
PKNODE Node;
UCHAR NodeNumber = 0;
UCHAR Number;
KIRQL OldIrql;
PKNODE OldNode;
PKNODE ParentNode;
PKPCR PcrBase;
PKPRCB Prcb;
USHORT ProcessorId;
KPROCESSOR_STATE ProcessorState;
PKTSS64 SysTssBase;
PKGDTENTRY64 TebBase;
PETHREAD Thread;
//
// Ensure that prefetch instructions in the IPI path are patched out
// if necessary before starting other processors.
//
OldIrql = KeRaiseIrqlToSynchLevel();
KiIpiSendRequest(1, 0, 0, IPI_FLUSH_SINGLE);
KeLowerIrql(OldIrql);
//
// Do not start additional processors if the relocate physical loader
// switch has been specified.
//
if (KeLoaderBlock->LoadOptions != NULL) {
if (strstr(KeLoaderBlock->LoadOptions, "RELOCATEPHYSICAL") != NULL) {
return;
}
}
//
// If this a multinode system and processor zero is not on node zero,
// then move it to the appropriate node.
//
if (KeNumberNodes > 1) {
if (NT_SUCCESS(KiQueryProcessorNode(0, &ProcessorId, &NodeNumber))) {
if (NodeNumber != 0) {
KiNode0.ProcessorMask = 0;
KiNodeInit[0] = KiNode0;
KeNodeBlock[0] = &KiNodeInit[0];
KiNode0 = *KeNodeBlock[NodeNumber];
KeNodeBlock[NodeNumber] = &KiNode0;
KiNode0.ProcessorMask = 1;
}
} else {
goto StartFailure;
}
}
//
// Calculate the size of the per processor data structures.
//
// This includes:
//
// PCR (including the PRCB)
// System TSS
// Idle Thread Object
// Double Fault Stack
// Machine Check Stack
// NMI Stack
// Multinode structure
// GDT
// IDT
//
// A DPC and Idle stack are also allocated, but they are done separately.
//
AllocationSize = ROUNDUP64(sizeof(KPCR)) +
ROUNDUP64(sizeof(KTSS64)) +
ROUNDUP64(sizeof(ETHREAD)) +
ROUNDUP64(DOUBLE_FAULT_STACK_SIZE) +
ROUNDUP64(KERNEL_MCA_EXCEPTION_STACK_SIZE) +
ROUNDUP64(NMI_STACK_SIZE) +
ROUNDUP64(sizeof(KNODE));
//
// Save the offset of the GDT in the allocation structure and add in
// the size of the GDT.
//
GdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr.Limit + 1;
//
// Save the offset of the IDT in the allocation structure and add in
// the size of the IDT.
//
IdtOffset = AllocationSize;
AllocationSize +=
CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr.Limit + 1;
//
// If the registered number of processors is greater than the maximum
// number of processors supported, then only allow the maximum number
// of supported processors.
//
if (KeRegisteredProcessors > MAXIMUM_PROCESSORS) {
KeRegisteredProcessors = MAXIMUM_PROCESSORS;
}
//
// Set barrier that will prevent any other processor from entering the
// idle loop until all processors have been started.
//
KiBarrierWait = 1;
//
// Initialize the fixed part of the processor state that will be used to
// start processors. Each processor starts in the system initialization
// code with address of the loader parameter block as an argument.
//
RtlZeroMemory(&ProcessorState, sizeof(KPROCESSOR_STATE));
ProcessorState.ContextFrame.Rcx = (ULONG64)KeLoaderBlock;
ProcessorState.ContextFrame.Rip = (ULONG64)KiSystemStartup;
ProcessorState.ContextFrame.SegCs = KGDT64_R0_CODE;
ProcessorState.ContextFrame.SegDs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegEs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegFs = KGDT64_R3_CMTEB | RPL_MASK;
ProcessorState.ContextFrame.SegGs = KGDT64_R3_DATA | RPL_MASK;
ProcessorState.ContextFrame.SegSs = KGDT64_R0_DATA;
//
// Check to determine if hyper-threading is really enabled. Intel chips
// claim to be hyper-threaded with the number of logical processors
// greater than one even when hyper-threading is disabled in the BIOS.
//
LogicalProcessors = KiLogicalProcessors;
if (HalIsHyperThreadingEnabled() == FALSE) {
LogicalProcessors = 1;
}
//
// If the total number of logical processors has not been set with
// the /NUMPROC loader option, then set the maximum number of logical
// processors to the number of registered processors times the number
// of logical processors per registered processor.
//
// N.B. The number of logical processors is never allowed to exceed
// the number of registered processors times the number of logical
// processors per physical processor.
//
MaximumProcessors = KeNumprocSpecified;
if (MaximumProcessors == 0) {
MaximumProcessors = KeRegisteredProcessors * LogicalProcessors;
}
//
// Loop trying to start a new processors until a new processor can't be
// started or an allocation failure occurs.
//
// N.B. The below processor start code relies on the fact a physical
// processor is started followed by all its logical processors.
// The HAL guarantees this by sorting the ACPI processor table
// by APIC id.
//
Index = 0;
Number = 0;
while ((Index < (MAXIMUM_PROCESSORS - 1)) &&
((ULONG)KeNumberProcessors < MaximumProcessors) &&
((ULONG)KeNumberProcessors / LogicalProcessors) < KeRegisteredProcessors) {
//
// If this is a multinode system and current processor does not
// exist on any node, then skip it.
//
Index += 1;
if (KeNumberNodes > 1) {
if (!NT_SUCCESS(KiQueryProcessorNode(Index, &ProcessorId, &NodeNumber))) {
continue;
}
}
//
// Increment the processor number.
//
Number += 1;
//
// Allocate memory for the new processor specific data. If the
// allocation fails, then stop starting processors.
//
DataBlock = MmAllocateIndependentPages(AllocationSize, NodeNumber);
if (DataBlock == NULL) {
goto StartFailure;
}
//
// Allocate a pool tag table for the new processor.
//
if (ExCreatePoolTagTable(Number, NodeNumber) == NULL) {
goto StartFailure;
}
//
// Zero the allocated memory.
//
Base = (PUCHAR)DataBlock;
RtlZeroMemory(DataBlock, AllocationSize);
//
// Copy and initialize the GDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Gdtr,
&ProcessorState.SpecialRegisters.Gdtr,
Base + GdtOffset);
GdtBase = (PKGDTENTRY64)ProcessorState.SpecialRegisters.Gdtr.Base;
//
// Encode the processor number in the upper 6 bits of the compatibility
// mode TEB descriptor.
//
TebBase = (PKGDTENTRY64)((PCHAR)GdtBase + KGDT64_R3_CMTEB);
TebBase->Bits.LimitHigh = Number >> 2;
TebBase->LimitLow = ((Number & 0x3) << 14) | (TebBase->LimitLow & 0x3fff);
//
// Copy and initialize the IDT for the next processor.
//
KiCopyDescriptorMemory(&CurrentPcr->Prcb.ProcessorState.SpecialRegisters.Idtr,
&ProcessorState.SpecialRegisters.Idtr,
Base + IdtOffset);
//
// Set the PCR base address for the next processor, set the processor
// number, and set the processor speed.
//
// N.B. The PCR address is passed to the next processor by computing
// the containing address with respect to the PRCB.
//
PcrBase = (PKPCR)Base;
PcrBase->ObsoleteNumber = Number;
PcrBase->Prcb.Number = Number;
PcrBase->Prcb.MHz = KeGetCurrentPrcb()->MHz;
Base += ROUNDUP64(sizeof(KPCR));
//
// Set the system TSS descriptor base for the next processor.
//
SysTssBase = (PKTSS64)Base;
KiSetDescriptorBase(KGDT64_SYS_TSS / 16, GdtBase, SysTssBase);
Base += ROUNDUP64(sizeof(KTSS64));
//
// Initialize the panic stack address for double fault and NMI.
//
Base += DOUBLE_FAULT_STACK_SIZE;
SysTssBase->Ist[TSS_IST_PANIC] = (ULONG64)Base;
//
// Initialize the machine check stack address.
//
Base += KERNEL_MCA_EXCEPTION_STACK_SIZE;
SysTssBase->Ist[TSS_IST_MCA] = (ULONG64)Base;
//
// Initialize the NMI stack address.
//
Base += NMI_STACK_SIZE;
SysTssBase->Ist[TSS_IST_NMI] = (ULONG64)Base;
//
// Idle Thread thread object.
//
Thread = (PETHREAD)Base;
Base += ROUNDUP64(sizeof(ETHREAD));
//
// Set other special registers in the processor state.
//
ProcessorState.SpecialRegisters.Cr0 = ReadCR0();
ProcessorState.SpecialRegisters.Cr3 = ReadCR3();
ProcessorState.ContextFrame.EFlags = 0;
ProcessorState.SpecialRegisters.Tr = KGDT64_SYS_TSS;
GdtBase[KGDT64_SYS_TSS / 16].Bytes.Flags1 = 0x89;
ProcessorState.SpecialRegisters.Cr4 = ReadCR4();
//
// Allocate a kernel stack and a DPC stack for the next processor.
//
KernelStack = MmCreateKernelStack(FALSE, NodeNumber);
if (KernelStack == NULL) {
goto StartFailure;
}
DpcStack = MmCreateKernelStack(FALSE, NodeNumber);
if (DpcStack == NULL) {
goto StartFailure;
}
//
// Initialize the kernel stack for the system TSS.
//
// N.B. System startup must be called with a stack pointer that is
// 8 mod 16.
//
SysTssBase->Rsp0 = (ULONG64)KernelStack - sizeof(PVOID) * 4;
ProcessorState.ContextFrame.Rsp = (ULONG64)KernelStack - 8;
//
// If this is the first processor on this node, then use the space
// already allocated for the node. Otherwise, the space allocated
// is not used.
//
Node = KeNodeBlock[NodeNumber];
OldNode = Node;
if (Node == &KiNodeInit[NodeNumber]) {
Node = (PKNODE)Base;
*Node = KiNodeInit[NodeNumber];
KeNodeBlock[NodeNumber] = Node;
}
Base += ROUNDUP64(sizeof(KNODE));
//
// Set the parent node address.
//
PcrBase->Prcb.ParentNode = Node;
//
// Adjust the loader block so it has the next processor state. Ensure
// that the kernel stack has space for home registers for up to four
// parameters.
//
KeLoaderBlock->KernelStack = (ULONG64)DpcStack - (sizeof(PVOID) * 4);
KeLoaderBlock->Thread = (ULONG64)Thread;
KeLoaderBlock->Prcb = (ULONG64)(&PcrBase->Prcb);
//
// Attempt to start the next processor. If a processor cannot be
// started, then deallocate memory and stop starting processors.
//
if (HalStartNextProcessor(KeLoaderBlock, &ProcessorState) == 0) {
//
// Restore the old node address in the node address array before
// freeing the allocated data block (the node structure lies
// within the data block).
//
*OldNode = *Node;
KeNodeBlock[NodeNumber] = OldNode;
ExDeletePoolTagTable(Number);
MmFreeIndependentPages(DataBlock, AllocationSize);
MmDeleteKernelStack(KernelStack, FALSE);
MmDeleteKernelStack(DpcStack, FALSE);
break;
}
Node->ProcessorMask |= AFFINITY_MASK(Number);
//
// Wait for processor to initialize.
//
while (*((volatile ULONG64 *)&KeLoaderBlock->Prcb) != 0) {
KeYieldProcessor();
}
}
//
// All processors have been started. If this is a multinode system, then
// allocate any missing node structures.
//
if (KeNumberNodes > 1) {
for (Index = 0; Index < KeNumberNodes; Index += 1) {
if (KeNodeBlock[Index] == &KiNodeInit[Index]) {
Node = ExAllocatePoolWithTag(NonPagedPool, sizeof(KNODE), ' eK');
if (Node != NULL) {
*Node = KiNodeInit[Index];
KeNodeBlock[Index] = Node;
} else {
goto StartFailure;
}
}
}
} else if (KiNode0.ProcessorMask != KeActiveProcessors) {
goto StartFailure;
}
//
// Clear node structure address for nonexistent nodes.
//
for (Index = KeNumberNodes; Index < MAXIMUM_CCNUMA_NODES; Index += 1) {
KeNodeBlock[Index] = NULL;
}
//
// Copy the node color and shifted color to the PRCB of each processor.
//
for (Index = 0; Index < (ULONG)KeNumberProcessors; Index += 1) {
Prcb = KiProcessorBlock[Index];
ParentNode = Prcb->ParentNode;
Prcb->NodeColor = ParentNode->Color;
Prcb->NodeShiftedColor = ParentNode->MmShiftedColor;
Prcb->SecondaryColorMask = MmSecondaryColorMask;
}
//
// Reset the initialization bit in prefetch retry.
//
KiPrefetchRetry &= ~0x80;
//
// Reset and synchronize the performance counters of all processors, by
// applying a null adjustment to the interrupt time.
//
KeAdjustInterruptTime(0);
//
// Allow all processors that were started to enter the idle loop and
// begin execution.
//
KiBarrierWait = 0;
#endif //
return;
//
// The failure to allocate memory or a unsuccessful status was returned
// during the attempt to start processors. This is considered fatal since
// something is very wrong.
//
#if !defined(NT_UP)
StartFailure:
KeBugCheckEx(PHASE1_INITIALIZATION_FAILED, 0, 0, 20, 0);
#endif
}
--*/
{
ULONG Index;
PKPRCB Prcb;
KAFFINITY TargetProcessors;
PWOW64_PROCESS Wow64Process;
KIRQL OldIrql;
BOOLEAN Flush2gb = FALSE;
#if 0
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
#endif
OldIrql = KeRaiseIrqlToSynchLevel();
Wow64Process = PsGetCurrentProcess()->Wow64Process;
#ifdef MI_ALTFLG_FLUSH2G
if ((Wow64Process != NULL) &&
((Wow64Process->AltFlags & MI_ALTFLG_FLUSH2G) != 0)) {
Flush2gb = TRUE;
}
#endif
//
// If a page table entry address array is specified, then set the
// specified page table entries to the specific value.
//
VOID
KeFlushEntireTb (
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all
processors that are currently running threads which are children
of the current process or flushes the entire translation buffer
on all processors in the host configuration.
Arguments:
Invalid - Supplies a boolean value that specifies the reason for
flushing the translation buffer.
AllProcessors - Supplies a boolean value that determines which
translation buffers are to be flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
PKTHREAD Thread;
BOOLEAN NeedTbFlush = FALSE;
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
OldIrql = KeRaiseIrqlToSynchLevel();
#if !defined(NT_UP)
TargetProcessors = KeActiveProcessors;
TargetProcessors &= PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushEntireTbTarget,
(PVOID)NeedTbFlush,
NULL,
NULL);
}
if (PsGetCurrentProcess()->Wow64Process != 0) {
KiInvalidateForwardProgressTbBuffer(TargetProcessors);
}
#endif
KeFlushCurrentTb();
//
// Wait until all target processors have finished.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
#endif
//
// Wait until all target processors have finished.
//
#if defined(NT_UP)
KeLowerIrql(OldIrql);
#else
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
KeLowerIrql(OldIrql);
#endif
return;
}
VOID
FASTCALL
KiRetireDpcList(IN PKPRCB Prcb)
{
PKDPC_DATA DpcData;
PLIST_ENTRY ListHead, DpcEntry;
PKDPC Dpc;
PKDEFERRED_ROUTINE DeferredRoutine;
PVOID DeferredContext, SystemArgument1, SystemArgument2;
ULONG_PTR TimerHand;
#ifdef CONFIG_SMP
KIRQL OldIrql;
#endif
/* Get data and list variables before starting anything else */
DpcData = &Prcb->DpcData[DPC_NORMAL];
ListHead = &DpcData->DpcListHead;
/* Main outer loop */
do
{
/* Set us as active */
Prcb->DpcRoutineActive = TRUE;
/* Check if this is a timer expiration request */
if (Prcb->TimerRequest)
{
/* It is, get the timer hand and disable timer request */
TimerHand = Prcb->TimerHand;
Prcb->TimerRequest = 0;
/* Expire timers with interrups enabled */
_enable();
KiTimerExpiration(NULL, NULL, (PVOID)TimerHand, NULL);
_disable();
}
/* Loop while we have entries in the queue */
while (DpcData->DpcQueueDepth != 0)
{
/* Lock the DPC data and get the DPC entry*/
KeAcquireSpinLockAtDpcLevel(&DpcData->DpcLock);
DpcEntry = ListHead->Flink;
/* Make sure we have an entry */
if (DpcEntry != ListHead)
{
/* Remove the DPC from the list */
RemoveEntryList(DpcEntry);
Dpc = CONTAINING_RECORD(DpcEntry, KDPC, DpcListEntry);
/* Clear its DPC data and save its parameters */
Dpc->DpcData = NULL;
DeferredRoutine = Dpc->DeferredRoutine;
DeferredContext = Dpc->DeferredContext;
SystemArgument1 = Dpc->SystemArgument1;
SystemArgument2 = Dpc->SystemArgument2;
/* Decrease the queue depth */
DpcData->DpcQueueDepth--;
/* Clear DPC Time */
Prcb->DebugDpcTime = 0;
/* Release the lock */
KeReleaseSpinLockFromDpcLevel(&DpcData->DpcLock);
/* Re-enable interrupts */
_enable();
/* Call the DPC */
DeferredRoutine(Dpc,
DeferredContext,
SystemArgument1,
SystemArgument2);
ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
/* Disable interrupts and keep looping */
_disable();
}
else
{
/* The queue should be flushed now */
ASSERT(DpcData->DpcQueueDepth == 0);
/* Release DPC Lock */
KeReleaseSpinLockFromDpcLevel(&DpcData->DpcLock);
}
}
/* Clear DPC Flags */
Prcb->DpcRoutineActive = FALSE;
Prcb->DpcInterruptRequested = FALSE;
#ifdef CONFIG_SMP
/* Check if we have deferred threads */
if (Prcb->DeferredReadyListHead.Next)
{
/* Re-enable interrupts and raise to synch */
_enable();
OldIrql = KeRaiseIrqlToSynchLevel();
/* Process deferred threads */
KiProcessDeferredReadyList(Prcb);
/* Lower IRQL back and disable interrupts */
KeLowerIrql(OldIrql);
_disable();
}
#endif
} while (DpcData->DpcQueueDepth != 0);
}
BOOLEAN
KeInvalidateAllCaches (
VOID
)
/*++
Routine Description:
This function writes back and invalidates the cache on all processors
in the host configuration.
Arguments:
None.
Return Value:
TRUE is returned as the function value.
--*/
{
#if !defined(NT_UP)
PKAFFINITY Barrier;
KIRQL OldIrql;
PKPRCB Prcb;
KAFFINITY TargetProcessors;
//
// Raise IRQL to SYNCH level.
//
// Send request to target processors, if any, invalidate the current cache,
// and wait for the IPI request barrier.
//
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors & ~Prcb->SetMember;
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, 0, 0, IPI_INVALIDATE_ALL);
WritebackInvalidate();
KiIpiWaitForRequestBarrier(Barrier);
} else {
WritebackInvalidate();
}
//
// Lower IRQL to its previous value.
//
KeLowerIrql(OldIrql);
#else
WritebackInvalidate();
#endif
return TRUE;
}
VOID
KeFlushEntireTb (
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all processors
that are currently running threads which are child of the current process
or flushes the entire translation buffer on all processors in the host
configuration.
Arguments:
Invalid - Supplies a boolean value that specifies the reason for flushing
the translation buffer.
AllProcessors - Supplies a boolean value that determines which translation
buffers are to be flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors, disable context switching,
// and send the flush entire parameters to the target processors,
// if any, for execution.
//
#if defined(NT_UP)
OldIrql = KeRaiseIrqlToSynchLevel();
#else
if (AllProcessors != FALSE) {
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors;
} else {
KiLockContextSwap(&OldIrql);
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
TargetProcessors &= ~Prcb->SetMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushTargetEntireTb,
NULL,
NULL,
NULL);
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushEntireTb);
}
#endif
//
// Flush TB on current processor.
//
KeFlushCurrentTb();
//
// Wait until all target processors have finished and complete packet.
//
#if defined(NT_UP)
KeLowerIrql(OldIrql);
#else
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
if (AllProcessors != FALSE) {
KeLowerIrql(OldIrql);
} else {
KiUnlockContextSwap(OldIrql);
}
#endif
return;
}
VOID
NTAPI
KiQuantumEnd(VOID)
{
PKPRCB Prcb = KeGetCurrentPrcb();
PKTHREAD NextThread, Thread = Prcb->CurrentThread;
/* Check if a DPC Event was requested to be signaled */
if (InterlockedExchange(&Prcb->DpcSetEventRequest, 0))
{
/* Signal it */
KeSetEvent(&Prcb->DpcEvent, 0, 0);
}
/* Raise to synchronization level and lock the PRCB and thread */
KeRaiseIrqlToSynchLevel();
KiAcquireThreadLock(Thread);
KiAcquirePrcbLock(Prcb);
/* Check if Quantum expired */
if (Thread->Quantum <= 0)
{
/* Check if we're real-time and with quantums disabled */
if ((Thread->Priority >= LOW_REALTIME_PRIORITY) &&
(Thread->ApcState.Process->DisableQuantum))
{
/* Otherwise, set maximum quantum */
Thread->Quantum = MAX_QUANTUM;
}
else
{
/* Reset the new Quantum */
Thread->Quantum = Thread->QuantumReset;
/* Calculate new priority */
Thread->Priority = KiComputeNewPriority(Thread, 1);
/* Check if a new thread is scheduled */
if (!Prcb->NextThread)
{
/* Get a new ready thread */
NextThread = KiSelectReadyThread(Thread->Priority, Prcb);
if (NextThread)
{
/* Found one, set it on standby */
NextThread->State = Standby;
Prcb->NextThread = NextThread;
}
}
else
{
/* Otherwise, make sure that this thread doesn't get preempted */
Thread->Preempted = FALSE;
}
}
}
/* Release the thread lock */
KiReleaseThreadLock(Thread);
/* Check if there's no thread scheduled */
if (!Prcb->NextThread)
{
/* Just leave now */
KiReleasePrcbLock(Prcb);
KeLowerIrql(DISPATCH_LEVEL);
return;
}
/* Get the next thread now */
NextThread = Prcb->NextThread;
/* Set current thread's swap busy to true */
KiSetThreadSwapBusy(Thread);
/* Switch threads in PRCB */
Prcb->NextThread = NULL;
Prcb->CurrentThread = NextThread;
/* Set thread to running and the switch reason to Quantum End */
NextThread->State = Running;
Thread->WaitReason = WrQuantumEnd;
/* Queue it on the ready lists */
KxQueueReadyThread(Thread, Prcb);
/* Set wait IRQL to APC_LEVEL */
Thread->WaitIrql = APC_LEVEL;
/* Swap threads */
KiSwapContext(APC_LEVEL, Thread);
/* Lower IRQL back to DISPATCH_LEVEL */
KeLowerIrql(DISPATCH_LEVEL);
}
VOID
KeFlushProcessTb (
VOID
)
/*++
Routine Description:
This function flushes the non-global translation buffer on all processors
that are currently running threads which are child of the current process.
Arguments:
None.
Return Value:
None.
--*/
{
PKAFFINITY Barrier;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors, disable context switching,
// and send the flush entire parameters to the target processors,
// if any, for execution.
//
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
TargetProcessors &= ~Prcb->SetMember;
//
// Send request to target processors, if any, flush the current process
// TB, and wait for the IPI request barrier.
//
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, 0, 0, IPI_FLUSH_PROCESS);
KiFlushProcessTb();
KiIpiWaitForRequestBarrier(Barrier);
} else {
KiFlushProcessTb();
}
//
// Lower IRQL to its previous value.
//
KeLowerIrql(OldIrql);
return;
}
/*
* @implemented
*/
NTSTATUS
NTAPI
NtYieldExecution(VOID)
{
NTSTATUS Status;
KIRQL OldIrql;
PKPRCB Prcb;
PKTHREAD Thread, NextThread;
/* NB: No instructions (other than entry code) should preceed this line */
/* Fail if there's no ready summary */
if (!KiGetCurrentReadySummary()) return STATUS_NO_YIELD_PERFORMED;
/* Now get the current thread, set the status... */
Status = STATUS_NO_YIELD_PERFORMED;
Thread = KeGetCurrentThread();
/* Raise IRQL to synch and get the KPRCB now */
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
/* Now check if there's still a ready summary */
if (Prcb->ReadySummary)
{
/* Acquire thread and PRCB lock */
KiAcquireThreadLock(Thread);
KiAcquirePrcbLock(Prcb);
/* Find a new thread to run if none was selected */
if (!Prcb->NextThread) Prcb->NextThread = KiSelectReadyThread(1, Prcb);
/* Make sure we still have a next thread to schedule */
NextThread = Prcb->NextThread;
if (NextThread)
{
/* Reset quantum and recalculate priority */
Thread->Quantum = Thread->QuantumReset;
Thread->Priority = KiComputeNewPriority(Thread, 1);
/* Release the thread lock */
KiReleaseThreadLock(Thread);
/* Set context swap busy */
KiSetThreadSwapBusy(Thread);
/* Set the new thread as running */
Prcb->NextThread = NULL;
Prcb->CurrentThread = NextThread;
NextThread->State = Running;
/* Setup a yield wait and queue the thread */
Thread->WaitReason = WrYieldExecution;
KxQueueReadyThread(Thread, Prcb);
/* Make it wait at APC_LEVEL */
Thread->WaitIrql = APC_LEVEL;
/* Sanity check */
ASSERT(OldIrql <= DISPATCH_LEVEL);
/* Swap to new thread */
KiSwapContext(APC_LEVEL, Thread);
Status = STATUS_SUCCESS;
}
else
{
/* Release the PRCB and thread lock */
KiReleasePrcbLock(Prcb);
KiReleaseThreadLock(Thread);
}
}
/* Lower IRQL and return */
KeLowerIrql(OldIrql);
return Status;
}
VOID
FASTCALL
KeFlushSingleTb (
IN PVOID Virtual,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes a single entry from translation buffer (TB)
on all processors that are currently running threads which are
children of the current process.
Arguments:
Virtual - Supplies a virtual address that is within the page whose
translation buffer entry is to be flushed.
AllProcessors - Supplies a boolean value that determines which
translation buffers are to be flushed.
Return Value:
The previous contents of the specified page table entry is returned
as the function value.
--*/
{
PKAFFINITY Barrier;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors and send the flush single
// parameters to the target processors, if any, for execution.
//
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
if (AllProcessors != FALSE) {
TargetProcessors = KeActiveProcessors;
} else {
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
//
// Send request to target processors, if any, flush the single entry from
// the current TB, and wait for the IPI request barrier.
//
TargetProcessors &= ~Prcb->SetMember;
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, (LONG64)Virtual, 0, IPI_FLUSH_SINGLE);
KiFlushSingleTb(Virtual);
KiIpiWaitForRequestBarrier(Barrier);
} else {
KiFlushSingleTb(Virtual);
}
//
// Lower IRQL to its previous value.
//
KeLowerIrql(OldIrql);
return;
}
VOID
KxFlushEntireTb (
VOID
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all
processors in the host configuration.
Arguments:
None.
Return Value:
None.
--*/
{
KIRQL OldIrql;
#if !defined(NT_UP)
PKAFFINITY Barrier;
PKPRCB Prcb;
KAFFINITY TargetProcessors;
#endif
//
// Raise IRQL to SYNCH level and set TB flush time stamp busy.
//
// Send request to target processors, if any, flush the current TB, and
// wait for the IPI request barrier.
//
OldIrql = KeRaiseIrqlToSynchLevel();
#if !defined(NT_UP)
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors & ~Prcb->SetMember;
KiSetTbFlushTimeStampBusy();
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, 0, 0, IPI_FLUSH_ALL);
KeFlushCurrentTb();
KiIpiWaitForRequestBarrier(Barrier);
} else {
KeFlushCurrentTb();
}
#else
KeFlushCurrentTb();
#endif
//
// Clear the TB flush time stamp busy and lower IRQL to its previous value.
//
KiClearTbFlushTimeStampBusy();
KeLowerIrql(OldIrql);
return;
}
VOID
KeFlushMultipleTb (
IN ULONG Number,
IN PVOID *Virtual,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes multiple entries from the translation buffer
on all processors that are currently running threads which are
children of the current process or flushes a multiple entries from
the translation buffer on all processors in the host configuration.
Arguments:
Number - Supplies the number of TB entries to flush.
Virtual - Supplies a pointer to an array of virtual addresses that
are within the pages whose translation buffer entries are to be
flushed.
AllProcessors - Supplies a boolean value that determines which
translation buffers are to be flushed.
Return Value:
The previous contents of the specified page table entry is returned
as the function value.
--*/
{
PKAFFINITY Barrier;
PVOID *End;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
ASSERT((Number != 0) && (Number <= FLUSH_MULTIPLE_MAXIMUM));
//
// Compute target set of processors.
//
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
if (AllProcessors != FALSE) {
TargetProcessors = KeActiveProcessors;
} else {
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
//
// Send request to target processors, if any, flush multiple entries in
// current TB, and wait for the IPI request barrier.
//
End = Virtual + Number;
TargetProcessors &= ~Prcb->SetMember;
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors,
(LONG64)Virtual,
Number,
IPI_FLUSH_MULTIPLE);
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
KiIpiWaitForRequestBarrier(Barrier);
} else {
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
}
//
// Lower IRQL to its previous value.
//
KeLowerIrql(OldIrql);
return;
}
NTSTATUS
NtYieldExecution (
VOID
)
/*++
Routine Description:
This function yields execution to any ready thread for up to one
quantum.
Arguments:
None.
Return Value:
None.
--*/
{
KIRQL OldIrql;
PKTHREAD NewThread;
PRKPRCB Prcb;
NTSTATUS Status;
PKTHREAD Thread;
//
// If no other threads are ready, then return immediately. Otherwise,
// attempt to yield execution.
//
// N.B. The test for ready threads is made outside any synchonization.
// Since this code cannot be perfectly synchronized under any
// conditions the lack of synchronization is of no consequence.
//
if (KiGetCurrentReadySummary() == 0) {
return STATUS_NO_YIELD_PERFORMED;
} else {
Status = STATUS_NO_YIELD_PERFORMED;
Thread = KeGetCurrentThread();
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
if (Prcb->ReadySummary != 0) {
//
// Acquire the thread lock and the PRCB lock.
//
// If a thread has not already been selected for execution, then
// attempt to select another thread for execution.
//
KiAcquireThreadLock(Thread);
KiAcquirePrcbLock(Prcb);
if (Prcb->NextThread == NULL) {
Prcb->NextThread = KiSelectReadyThread(1, Prcb);
}
//
// If a new thread has been selected for execution, then switch
// immediately to the selected thread.
//
if ((NewThread = Prcb->NextThread) != NULL) {
Thread->Quantum = Thread->QuantumReset;
//
// Compute the new thread priority.
//
// N.B. The new priority will never be greater than the previous
// priority.
//
Thread->Priority = KiComputeNewPriority(Thread, 1);
//
// Release the thread lock, set swap busy for the old thread,
// set the next thread to NULL, set the current thread to the
// new thread, set the new thread state to running, set the
// wait reason, queue the old running thread, and release the
// PRCB lock, and swp context to the new thread.
//
KiReleaseThreadLock(Thread);
KiSetContextSwapBusy(Thread);
Prcb->NextThread = NULL;
Prcb->CurrentThread = NewThread;
NewThread->State = Running;
Thread->WaitReason = WrYieldExecution;
KxQueueReadyThread(Thread, Prcb);
Thread->WaitIrql = APC_LEVEL;
ASSERT(OldIrql <= DISPATCH_LEVEL);
KiSwapContext(Thread, NewThread);
Status = STATUS_SUCCESS;
} else {
KiReleasePrcbLock(Prcb);
KiReleaseThreadLock(Thread);
}
}
//
// Lower IRQL to its previous level and return.
//
KeLowerIrql(OldIrql);
return Status;
}
}
VOID
FASTCALL
KeSignalGateBoostPriority (
__inout PKGATE Gate
)
/*++
Routine Description:
This function conditionally sets the signal state of a gate object and
attempts to unwait the first waiter.
Arguments:
Gate - Supplies a pointer to a dispatcher object of type gate.
Return Value:
None.
--*/
{
PKTHREAD CurrentThread;
PLIST_ENTRY Entry;
KIRQL OldIrql;
SCHAR Priority;
PKQUEUE Queue;
PKWAIT_BLOCK WaitBlock;
PKTHREAD WaitThread;
ASSERT_GATE(Gate);
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
//
// Raise IRQL to SYNCH_LEVEL and acquire the object lock.
//
// If the object is not already signaled, then attempt to wake a waiter.
//
CurrentThread = KeGetCurrentThread();
do {
OldIrql = KeRaiseIrqlToSynchLevel();
KiAcquireKobjectLock(Gate);
if (Gate->Header.SignalState == 0) {
//
// If there are any waiters, then remove and ready the first
// waiter. Otherwise, set the signal state.
//
if (IsListEmpty(&Gate->Header.WaitListHead) == FALSE) {
Entry = Gate->Header.WaitListHead.Flink;
WaitBlock = CONTAINING_RECORD(Entry, KWAIT_BLOCK, WaitListEntry);
WaitThread = WaitBlock->Thread;
//
// Try to acquire the thread lock.
//
// If the thread lock cannot be acquired, then release the
// object lock, lower IRQL to is previous value, and try to
// signal the gate again. Otherwise, remove the entry from
// the list, set the wait completion status to success, set
// the deferred processor number, set the thread state to
// deferred ready, release the thread lock, release the
// object lock, compute the new thread priority, ready the
// thread for execution, and exit the dispatcher.
//
if (KiTryToAcquireThreadLock(WaitThread)) {
RemoveEntryList(Entry);
WaitThread->WaitStatus = STATUS_SUCCESS;
WaitThread->State = DeferredReady;
WaitThread->DeferredProcessor = KeGetCurrentPrcb()->Number;
KiReleaseKobjectLock(Gate);
KiReleaseThreadLock(WaitThread);
Priority = CurrentThread->Priority;
if (Priority < LOW_REALTIME_PRIORITY) {
Priority = Priority - CurrentThread->PriorityDecrement;
if (Priority < CurrentThread->BasePriority) {
Priority = CurrentThread->BasePriority;
}
if (CurrentThread->PriorityDecrement != 0) {
CurrentThread->PriorityDecrement = 0;
CurrentThread->Quantum = CLOCK_QUANTUM_DECREMENT;
}
}
WaitThread->AdjustIncrement = Priority;
WaitThread->AdjustReason = (UCHAR)AdjustBoost;
//
// If the wait thread is associated with a queue, then
// increment the concurrency level.
//
// N.B. This can be done after dropping all other locks,
// but must be done holding the dispatcher lock
// since the concurrency count is not accessed with
// interlocked operations elsewhere.
//
if (WaitThread->Queue != NULL) {
KiLockDispatcherDatabaseAtSynchLevel();
if ((Queue = WaitThread->Queue) != NULL) {
Queue->CurrentCount += 1;
}
KiUnlockDispatcherDatabaseFromSynchLevel();
}
KiDeferredReadyThread(WaitThread);
KiExitDispatcher(OldIrql);
return;
} else {
KiReleaseKobjectLock(Gate);
KeLowerIrql(OldIrql);
continue;
}
} else {
Gate->Header.SignalState = 1;
break;
}
} else {
break;
}
} while (TRUE);
//
// Release the object lock and lower IRQL to its previous value.
//
KiReleaseKobjectLock(Gate);
KeLowerIrql(OldIrql);
return;
}
