VOID
NTAPI
KeThawExecution(IN BOOLEAN Enable)
{
/* Cleanup CPU caches */
KeFlushCurrentTb();
/* Re-enable interrupts */
if (Enable) _enable();
}
VOID
KiFlushEntireTbTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for flushing the entire TB.
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
//
// Flush the entire TB on the current processor.
//
KiIpiSignalPacketDone(SignalDone);
KeFlushCurrentTb();
#endif
return;
}
VOID
KeFlushEntireTb (
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all
processors that are currently running threads which are children
of the current process or flushes the entire translation buffer
on all processors in the host configuration.
Arguments:
Invalid - Supplies a boolean value that specifies the reason for
flushing the translation buffer.
AllProcessors - Supplies a boolean value that determines which
translation buffers are to be flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
KAFFINITY TargetProcessors;
PKTHREAD Thread;
BOOLEAN NeedTbFlush = FALSE;
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
OldIrql = KeRaiseIrqlToSynchLevel();
#if !defined(NT_UP)
TargetProcessors = KeActiveProcessors;
TargetProcessors &= PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushEntireTbTarget,
(PVOID)NeedTbFlush,
NULL,
NULL);
}
if (PsGetCurrentProcess()->Wow64Process != 0) {
KiInvalidateForwardProgressTbBuffer(TargetProcessors);
}
#endif
KeFlushCurrentTb();
//
// Wait until all target processors have finished.
//
#if !defined(NT_UP)
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
#endif
//
// Wait until all target processors have finished.
//
#if defined(NT_UP)
KeLowerIrql(OldIrql);
#else
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
KeLowerIrql(OldIrql);
#endif
return;
}
BOOLEAN
KdpTrap (
IN PKTRAP_FRAME TrapFrame,
IN PKEXCEPTION_FRAME ExceptionFrame,
IN PEXCEPTION_RECORD ExceptionRecord,
IN PCONTEXT ContextRecord,
IN KPROCESSOR_MODE PreviousMode,
IN BOOLEAN SecondChance
)
/*++
Routine Description:
This routine is called whenever a exception is dispatched and the kernel
debugger is active.
Arguments:
TrapFrame - Supplies a pointer to a trap frame that describes the
trap.
ExceptionFrame - Supplies a pointer to a exception frame that describes
the trap.
ExceptionRecord - Supplies a pointer to an exception record that
describes the exception.
ContextRecord - Supplies the context at the time of the exception.
PreviousMode - Supplies the previous processor mode.
SecondChance - Supplies a boolean value that determines whether this is
the second chance (TRUE) that the exception has been raised.
Return Value:
A value of TRUE is returned if the exception is handled. Otherwise a
value of FALSE is returned.
--*/
{
BOOLEAN Completion;
BOOLEAN Enable;
BOOLEAN UnloadSymbols = FALSE;
ULONG OldIar;
STRING Input;
STRING Output;
PKPRCB Prcb;
//
// Synchronize processor execution, save processor state, enter debugger,
// and flush the current TB.
//
re_enter_debugger:
Enable = KdEnterDebugger(TrapFrame, ExceptionFrame);
Prcb = KeGetCurrentPrcb();
KiSaveProcessorState(TrapFrame, ExceptionFrame);
KeFlushCurrentTb();
//
// If this is a breakpoint instruction, then check to determine if is
// an internal command.
//
if ((ExceptionRecord->ExceptionCode == STATUS_BREAKPOINT) &&
((ExceptionRecord->ExceptionInformation[0] & BREAKPOINT_CODE_MASK)
>= DEBUG_PRINT_BREAKPOINT)) {
//
// Switch on the breakpoint code.
//
switch (ExceptionRecord->ExceptionInformation[0] & BREAKPOINT_CODE_MASK) {
//
// Print a debug string.
//
// Arguments:
//
// a0 - Supplies a pointer to an output string buffer.
// a1 - Supplies the length of the output string buffer.
//
case DEBUG_PRINT_BREAKPOINT:
ContextRecord->Iar += 4;
Output.Buffer = (PCHAR)ContextRecord->Gpr3;
Output.Length = (USHORT)ContextRecord->Gpr4;
if (KdDebuggerNotPresent == FALSE) {
if (KdpPrintString(&Output)) {
ContextRecord->Gpr3 = (ULONG)STATUS_BREAKPOINT;
} else {
ContextRecord->Gpr3 = (ULONG)STATUS_SUCCESS;
}
} else {
ContextRecord->Gpr3 = (ULONG)STATUS_DEVICE_NOT_CONNECTED;
}
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
KdExitDebugger(Enable);
return TRUE;
//
// Print a debug prompt string, then input a string.
//
// r.3 - Supplies a pointer to an output string buffer.
// r.4 - Supplies the length of the output string buffer..
// r.5 - supplies a pointer to an input string buffer.
// r.6 - Supplies the length of the input string bufffer.
//
case DEBUG_PROMPT_BREAKPOINT:
ContextRecord->Iar += 4;
Output.Buffer = (PCHAR)ContextRecord->Gpr3;
Output.Length = (USHORT)ContextRecord->Gpr4;
Input.Buffer = (PCHAR)ContextRecord->Gpr5;
Input.MaximumLength = (USHORT)ContextRecord->Gpr6;
KdpPromptString(&Output, &Input);
ContextRecord->Gpr3 = Input.Length;
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
KdExitDebugger(Enable);
return TRUE;
//
// Load the symbolic information for an image.
//
// Arguments:
//
// r.3 - Supplies a pointer to an output string descriptor.
// r.4 - Supplies a the base address of the image.
//
case DEBUG_UNLOAD_SYMBOLS_BREAKPOINT:
UnloadSymbols = TRUE;
//
// Fall through
//
case DEBUG_LOAD_SYMBOLS_BREAKPOINT:
OldIar = ContextRecord->Iar;
if (KdDebuggerNotPresent == FALSE) {
KdpReportLoadSymbolsStateChange((PSTRING)ContextRecord->Gpr3,
(PKD_SYMBOLS_INFO) ContextRecord->Gpr4,
UnloadSymbols,
&Prcb->ProcessorState.ContextFrame);
}
RtlCopyMemory(ContextRecord,
&Prcb->ProcessorState.ContextFrame,
sizeof(CONTEXT));
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
KdExitDebugger(Enable);
//
// If the kernel debugger did not update the IAR, then increment
// past the breakpoint instruction.
//
if (ContextRecord->Iar == OldIar) {
ContextRecord->Iar += 4;
}
return TRUE;
//
// Unknown internal command.
//
default:
break;
}
}
//
// Report state change to kernel debugger on host machine.
//
Completion = KdpReportExceptionStateChange(
ExceptionRecord,
&Prcb->ProcessorState.ContextFrame,
SecondChance);
RtlCopyMemory(ContextRecord,
&Prcb->ProcessorState.ContextFrame,
sizeof(CONTEXT));
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
KdExitDebugger(Enable);
//
// check to see if the user of the remote debugger
// requested memory to be paged in
//
if (KdpPageInAddress) {
if (KeGetCurrentIrql() <= APC_LEVEL) {
//
// if the IQRL is below DPC level then cause
// the page fault to occur and then re-enter
// the debugger. this whole process is transparent
// to the user.
//
KdpPageInData( (PUCHAR)KdpPageInAddress );
KdpPageInAddress = 0;
KdpControlCPending = FALSE;
goto re_enter_debugger;
} else {
//
// we cannot take a page fault
// here so a worker item is queued to take the
// page fault. after the worker item takes the
// page fault it sets the contol-c flag so that
// the user re-enters the debugger just as if
// control-c was pressed.
//
if (KdpControlCPressed) {
ExInitializeWorkItem(
&KdpPageInWorkItem,
(PWORKER_THREAD_ROUTINE) KdpPageInData,
(PVOID) KdpPageInAddress
);
ExQueueWorkItem( &KdpPageInWorkItem, DelayedWorkQueue );
KdpPageInAddress = 0;
}
}
}
KdpControlCPressed = FALSE;
return Completion;
}
VOID
KiSyncNewRegionIdTarget (
IN PULONG SignalDone,
IN PVOID Parameter1,
IN PVOID Parameter2,
IN PVOID Parameter3
)
/*++
Routine Description:
This is the target function for synchronizing the region Ids
Arguments:
SignalDone Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Parameter1 - Parameter3 - Not used.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
PKTHREAD Thread;
PKPROCESS Process;
PREGION_MAP_INFO ProcessRegion;
PREGION_MAP_INFO MappedSession;
ULONG NewRid;
//
// Flush the entire TB on the current processor.
//
Thread = KeGetCurrentThread();
Process = Thread->ApcState.Process;
ProcessRegion = &Process->ProcessRegion;
MappedSession = Process->SessionMapInfo;
KiAcquireSpinLock(&KiMasterRidLock);
if (ProcessRegion->SequenceNumber != KiMasterSequence) {
KiMasterRid += 1;
ProcessRegion->RegionId = KiMasterRid;
ProcessRegion->SequenceNumber = KiMasterSequence;
KiSetRegionRegister(MM_LOWEST_USER_ADDRESS,
KiMakeValidRegionRegister(ProcessRegion->RegionId, PAGE_SHIFT));
}
if (MappedSession->SequenceNumber != KiMasterSequence) {
KiMasterRid += 1;
MappedSession->RegionId = KiMasterRid;
MappedSession->SequenceNumber = KiMasterSequence;
KiSetRegionRegister((PVOID)MM_SESSION_SPACE_DEFAULT,
KiMakeValidRegionRegister(MappedSession->RegionId, PAGE_SHIFT));
}
KiReleaseSpinLock(&KiMasterRidLock);
KiIpiSignalPacketDone(SignalDone);
KeFlushCurrentTb();
#endif
return;
}
VOID
KiSyncSessionTarget(
IN PULONG SignalDone,
IN PKPROCESS Process,
IN PVOID Parameter1,
IN PVOID Parameter2
)
/*++
Routine Description:
This is the target function for synchronizing the new session
region id. This routine is called when the session space is removed
and all the processors need to be notified.
Arguments:
SignalDone - Supplies a pointer to a variable that is cleared when the
requested operation has been performed.
Process - Supplies a KPROCESS pointer which needs to be sync'ed.
Return Value:
None.
Environment:
Kernel mode.
--*/
{
#if !defined(NT_UP)
PKTHREAD Thread;
ULONG NewRid;
//
// Flush the entire TB on the current processor.
//
Thread = KeGetCurrentThread();
//
// check to see if the current process is the process that needs to be
// sync'ed
//
if (Process == Thread->ApcState.Process) {
KiAcquireSpinLock(&KiMasterRidLock);
//
// disable the session region.
//
KiSetRegionRegister((PVOID)MM_SESSION_SPACE_DEFAULT,
KiMakeValidRegionRegister(Process->SessionMapInfo->RegionId, PAGE_SHIFT));
KiReleaseSpinLock(&KiMasterRidLock);
//
// flush the entire tb.
//
KeFlushCurrentTb();
}
KiIpiSignalPacketDone(SignalDone);
#endif
return;
}
VOID
KeThawExecution (
IN BOOLEAN Enable
)
/*++
Routine Description:
This function thaws the execution of all other processors in the host
configuration and then returns to the caller. It is intended for use by
the kernel debugger.
Arguments:
Enable - Supplies the previous interrupt enable that is to be restored
after having thawed the execution of all other processors.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
KIRQL OldIrql;
ULONG TargetSet;
ULONG BitNumber;
ULONG Flag;
PKPRCB Prcb;
//
// Before releasing FreezeExecutionLock clear any all targets IpiFrozen
// flag.
//
KeGetCurrentPrcb()->IpiFrozen = RUNNING;
TargetSet = KeActiveProcessors & ~(1 << KeGetCurrentPrcb()->Number);
while (TargetSet != 0) {
BitNumber = KeFindFirstSetRightMember(TargetSet);
ClearMember(BitNumber, TargetSet);
Prcb = KiProcessorBlock[BitNumber];
#if IDBG
//
// If the target processor was not forzen, then don't wait
// for target to unfreeze.
//
if (FrozenState(Prcb->IpiFrozen) != TARGET_FROZEN) {
Prcb->IpiFrozen = RUNNING;
continue;
}
#endif
Prcb->IpiFrozen = TARGET_THAW;
while (Prcb->IpiFrozen == TARGET_THAW) {
KeYieldProcessor();
}
}
//
// Capture the previous IRQL before releasing the freeze lock.
//
OldIrql = KiOldIrql;
#if IDBG
Flag = KiFreezeFlag;
KiFreezeFlag = 0;
if ((Flag & FREEZE_BACKUP) != 0) {
KiReleaseSpinLock(&KiFreezeLockBackup);
} else {
KiReleaseSpinLock(&KiFreezeExecutionLock);
}
#else
KiFreezeFlag = 0;
KiReleaseSpinLock(&KiFreezeExecutionLock);
#endif
#endif // !defined (NT_UP)
//
// Flush the current TB, instruction cache, and data cache.
//
KeFlushCurrentTb();
KeSweepCurrentIcache();
KeSweepCurrentDcache();
//
// Lower IRQL and restore interrupt enable
//
#if !defined(NT_UP)
KeLowerIrql(OldIrql);
#endif
KiRestoreInterrupts(Enable);
return;
}
BOOLEAN
KiSyncNewRegionId(
IN PREGION_MAP_INFO ProcessRegion,
IN PREGION_MAP_INFO SessionRegion
)
/*++
Routine Description:
Generate a new region id and synchronze the region Ids on all the processors
if necessary. If the region ids wrap then flush all processor TLB's.
Arguments:
ProcessRegion - Supplies a pointer to REGION_MAP_INFO for the user space.
SessionRegion - Supplies a pointer to REGION_MAP_INFO for the session space.
Return Value:
FALSE -- when region id has not been recycled.
TRUE -- when region id has been recycled.
Notes:
This routine called by KiSwapProcess, KeAttachSessionSpace and
KeCreateSessionSpace.
Environment:
Kernel mode.
KiLockDispaterLock or LockQueuedDispatcherLock is held
--*/
{
BOOLEAN RidRecycled = FALSE;
KAFFINITY TargetProcessors;
ULONG i;
//
// Invalidate the ForwardProgressTb buffer
//
for (i = 0; i < MAXIMUM_FWP_BUFFER_ENTRY; i += 1) {
PCR->ForwardProgressBuffer[(i*2)+1] = 0;
}
KiAcquireSpinLock(&KiMasterRidLock);
if ((ProcessRegion->SequenceNumber == KiMasterSequence) &&
(SessionRegion->SequenceNumber == KiMasterSequence)) {
goto not_recycled;
}
if (ProcessRegion->SequenceNumber != KiMasterSequence) {
if (KiMasterRid + 1 > KiMaximumRid) {
RidRecycled = TRUE;
} else {
KiMasterRid += 1;
ProcessRegion->RegionId = KiMasterRid;
ProcessRegion->SequenceNumber = KiMasterSequence;
}
}
if ((RidRecycled == FALSE) &&
(SessionRegion->SequenceNumber != KiMasterSequence)) {
if (KiMasterRid + 1 > KiMaximumRid) {
RidRecycled = TRUE;
} else {
KiMasterRid += 1;
SessionRegion->RegionId = KiMasterRid;
SessionRegion->SequenceNumber = KiMasterSequence;
}
}
if (RidRecycled == FALSE) {
goto not_recycled;
}
//
// Region Id must be recycled
//
KiMasterRid = START_PROCESS_RID;
//
// Since KiMasterSequence is 64-bit wide, it will not be recycled in your life time.
//
if (KiMasterSequence + 1 > MAXIMUM_SEQUENCE) {
KiMasterSequence = START_SEQUENCE;
} else {
KiMasterSequence += 1;
}
//
// update new process's ProcessRid and ProcessSequence
//
ProcessRegion->RegionId = KiMasterRid;
ProcessRegion->SequenceNumber = KiMasterSequence;
KiSetRegionRegister(MM_LOWEST_USER_ADDRESS,
KiMakeValidRegionRegister(ProcessRegion->RegionId, PAGE_SHIFT));
KiMasterRid += 1;
SessionRegion->RegionId = KiMasterRid;
SessionRegion->SequenceNumber = KiMasterSequence;
KiSetRegionRegister((PVOID)MM_SESSION_SPACE_DEFAULT,
KiMakeValidRegionRegister(SessionRegion->RegionId, PAGE_SHIFT));
//
// release mutex for master region id lock
//
KiReleaseSpinLock(&KiMasterRidLock);
#if !defined(NT_UP)
//
// broadcast Region Id sync
//
TargetProcessors = KeActiveProcessors;
TargetProcessors &= PCR->NotMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiSyncNewRegionIdTarget,
(PVOID)TRUE,
NULL,
NULL);
}
#endif
KeFlushCurrentTb();
#if !defined(NT_UP)
//
// Wait until all target processors have finished.
//
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
#endif
return TRUE;
not_recycled:
KiSetRegionRegister(MM_LOWEST_USER_ADDRESS,
KiMakeValidRegionRegister(ProcessRegion->RegionId, PAGE_SHIFT));
KiSetRegionRegister((PVOID)MM_SESSION_SPACE_DEFAULT,
KiMakeValidRegionRegister(SessionRegion->RegionId, PAGE_SHIFT));
//
// release mutex for master region id lock
//
KiReleaseSpinLock(&KiMasterRidLock);
return FALSE;
}
VOID
KiFreezeTargetExecution (
IN PKTRAP_FRAME TrapFrame,
IN PKEXCEPTION_FRAME ExceptionFrame
)
/*++
Routine Description:
This function freezes the execution of the current running processor.
If a trapframe is supplied to current state is saved into the prcb
for the debugger.
Arguments:
TrapFrame - Supplies a pointer to the trap frame that describes the
trap.
ExceptionFrame - Supplies a pointer to the exception frame that
describes the trap.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
KIRQL OldIrql;
PKPRCB Prcb;
BOOLEAN Enable;
KCONTINUE_STATUS Status;
EXCEPTION_RECORD ExceptionRecord;
Enable = KiDisableInterrupts();
KeRaiseIrql(HIGH_LEVEL, &OldIrql);
Prcb = KeGetCurrentPrcb();
Prcb->IpiFrozen = TARGET_FROZEN;
Prcb->SkipTick = TRUE;
if (TrapFrame != NULL) {
KiSaveProcessorState(TrapFrame, ExceptionFrame);
}
//
// Sweep the data cache in case this is a system crash and the bug
// check code is attempting to write a crash dump file.
//
KeSweepCurrentDcache();
//
// Wait for person requesting us to freeze to
// clear our frozen flag
//
while (FrozenState(Prcb->IpiFrozen) == TARGET_FROZEN) {
if (Prcb->IpiFrozen & FREEZE_ACTIVE) {
//
// This processor has been made the active processor
//
if (TrapFrame) {
RtlZeroMemory (&ExceptionRecord, sizeof ExceptionRecord);
ExceptionRecord.ExceptionCode = STATUS_WAKE_SYSTEM_DEBUGGER;
ExceptionRecord.ExceptionRecord = &ExceptionRecord;
ExceptionRecord.ExceptionAddress =
(PVOID)CONTEXT_TO_PROGRAM_COUNTER (&Prcb->ProcessorState.ContextFrame);
Status = (KiDebugSwitchRoutine) (
&ExceptionRecord,
&Prcb->ProcessorState.ContextFrame,
FALSE
);
} else {
Status = ContinueError;
}
//
// If status is anything other then, continue with next
// processor then reselect master
//
if (Status != ContinueNextProcessor) {
Prcb->IpiFrozen &= ~FREEZE_ACTIVE;
KiFreezeOwner->IpiFrozen |= FREEZE_ACTIVE;
}
}
KeYieldProcessor();
}
if (TrapFrame != NULL) {
KiRestoreProcessorState(TrapFrame, ExceptionFrame);
}
Prcb->IpiFrozen = RUNNING;
KeFlushCurrentTb();
KeSweepCurrentIcache();
KeLowerIrql(OldIrql);
KiRestoreInterrupts(Enable);
#endif // !define(NT_UP)
return;
}
VOID
HalpReleaseReservedVirtualSpace(
PVOID VirtualAddress,
ULONG LengthInPages
)
/*++
Routine Description:
This function will release the virtual address range previously
allocated with HalpAssignReservedVirtualSpace and should be
called with a virtual address previously allocated by a call to
HalpAssignReservedVirtualSpace the number of pages to release.
Arguments:
VirtualAddress of a previously allocated page in the HAL reserved
space.
Length (in pages) to be released.
Return Value:
None.
--*/
{
PHARDWARE_PTE PPte;
//
// Sanity Checks
//
if ( (LengthInPages > MAX_LENGTH) ||
(!LengthInPages) ||
((ULONG)VirtualAddress < MIN_VA) ||
((ULONG)VirtualAddress > MAX_VA) ) {
KeBugCheck(HAL_INITIALIZATION_FAILED);
return;
}
if ( !HalpReservedPtes ) {
KeBugCheck(HAL_INITIALIZATION_FAILED);
return;
}
PPte = HalpReservedPtes + PTE_INDEX(VirtualAddress);
do {
if ( !PPte->Valid ) {
break;
}
PPte->Valid = FALSE;
PPte++;
} while ( --LengthInPages );
if ( LengthInPages ) {
KeBugCheck(HAL_INITIALIZATION_FAILED);
}
KeFlushCurrentTb();
return;
}
VOID
KeFlushEntireTb (
IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all processors
that are currently running threads which are child of the current process
or flushes the entire translation buffer on all processors in the host
configuration.
Arguments:
Invalid - Supplies a boolean value that specifies the reason for flushing
the translation buffer.
AllProcessors - Supplies a boolean value that determines which translation
buffers are to be flushed.
Return Value:
None.
--*/
{
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors, disable context switching,
// and send the flush entire parameters to the target processors,
// if any, for execution.
//
#if defined(NT_UP)
OldIrql = KeRaiseIrqlToSynchLevel();
#else
if (AllProcessors != FALSE) {
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors;
} else {
KiLockContextSwap(&OldIrql);
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
}
TargetProcessors &= ~Prcb->SetMember;
if (TargetProcessors != 0) {
KiIpiSendPacket(TargetProcessors,
KiFlushTargetEntireTb,
NULL,
NULL,
NULL);
IPI_INSTRUMENT_COUNT (Prcb->Number, FlushEntireTb);
}
#endif
//
// Flush TB on current processor.
//
KeFlushCurrentTb();
//
// Wait until all target processors have finished and complete packet.
//
#if defined(NT_UP)
KeLowerIrql(OldIrql);
#else
if (TargetProcessors != 0) {
KiIpiStallOnPacketTargets(TargetProcessors);
}
if (AllProcessors != FALSE) {
KeLowerIrql(OldIrql);
} else {
KiUnlockContextSwap(OldIrql);
}
#endif
return;
}
DECLSPEC_NOINLINE
VOID
KiIpiProcessRequests (
VOID
)
/*++
Routine Description:
This routine processes interprocessor requests and returns a summary
of the requests that were processed.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
PVOID *End;
ULONG64 Number;
PKPRCB Packet;
PKPRCB Prcb;
ULONG Processor;
REQUEST_SUMMARY Request;
PREQUEST_MAILBOX RequestMailbox;
PKREQUEST_PACKET RequestPacket;
LONG64 SetMember;
PKPRCB Source;
KAFFINITY SummarySet;
KAFFINITY TargetSet;
PVOID *Virtual;
//
// Loop until the sender summary is zero.
//
Prcb = KeGetCurrentPrcb();
TargetSet = ReadForWriteAccess(&Prcb->SenderSummary);
SetMember = Prcb->SetMember;
while (TargetSet != 0) {
SummarySet = TargetSet;
BitScanForward64(&Processor, SummarySet);
do {
Source = KiProcessorBlock[Processor];
RequestMailbox = &Prcb->RequestMailbox[Processor];
Request.Summary = RequestMailbox->RequestSummary;
//
// If the request type is flush multiple immediate, flush process,
// flush single, or flush all, then packet done can be signaled
// before processing the request. Otherwise, the request type must
// be a packet request, a cache invalidate, or a flush multiple
//
if (Request.IpiRequest <= IPI_FLUSH_ALL) {
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE_IMMEDIATE) {
Number = Request.Count;
Virtual = &RequestMailbox->Virtual[0];
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_FLUSH_PROCESS) {
KiFlushProcessTb();
} else if (Request.IpiRequest == IPI_FLUSH_SINGLE) {
KiFlushSingleTb((PVOID)Request.Parameter);
} else {
ASSERT(Request.IpiRequest == IPI_FLUSH_ALL);
KeFlushCurrentTb();
}
} else {
//
// If the request type is packet ready, then call the worker
// function. Otherwise, the request must be either a flush
// multiple or a cache invalidate.
//
if (Request.IpiRequest == IPI_PACKET_READY) {
Packet = Source;
if (Request.IpiSynchType != 0) {
Packet = (PKPRCB)((ULONG64)Source + 1);
}
RequestPacket = (PKREQUEST_PACKET)&RequestMailbox->RequestPacket;
(RequestPacket->WorkerRoutine)((PKIPI_CONTEXT)Packet,
RequestPacket->CurrentPacket[0],
RequestPacket->CurrentPacket[1],
RequestPacket->CurrentPacket[2]);
} else {
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE) {
Number = Request.Count;
Virtual = (PVOID *)Request.Parameter;
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_INVALIDATE_ALL) {
WritebackInvalidate();
} else {
ASSERT(FALSE);
}
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
}
}
SummarySet ^= AFFINITY_MASK(Processor);
} while (BitScanForward64(&Processor, SummarySet) != FALSE);
//
// Clear target set in sender summary.
//
TargetSet =
InterlockedExchangeAdd64((LONG64 volatile *)&Prcb->SenderSummary,
-(LONG64)TargetSet) - TargetSet;
}
#endif
return;
}
VOID
KxFlushEntireTb (
VOID
)
/*++
Routine Description:
This function flushes the entire translation buffer (TB) on all
processors in the host configuration.
Arguments:
None.
Return Value:
None.
--*/
{
KIRQL OldIrql;
#if !defined(NT_UP)
PKAFFINITY Barrier;
PKPRCB Prcb;
KAFFINITY TargetProcessors;
#endif
//
// Raise IRQL to SYNCH level and set TB flush time stamp busy.
//
// Send request to target processors, if any, flush the current TB, and
// wait for the IPI request barrier.
//
OldIrql = KeRaiseIrqlToSynchLevel();
#if !defined(NT_UP)
Prcb = KeGetCurrentPrcb();
TargetProcessors = KeActiveProcessors & ~Prcb->SetMember;
KiSetTbFlushTimeStampBusy();
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, 0, 0, IPI_FLUSH_ALL);
KeFlushCurrentTb();
KiIpiWaitForRequestBarrier(Barrier);
} else {
KeFlushCurrentTb();
}
#else
KeFlushCurrentTb();
#endif
//
// Clear the TB flush time stamp busy and lower IRQL to its previous value.
//
KiClearTbFlushTimeStampBusy();
KeLowerIrql(OldIrql);
return;
}
