DECLSPEC_NOINLINE
VOID
KiIpiProcessRequests (
VOID
)
/*++
Routine Description:
This routine processes interprocessor requests and returns a summary
of the requests that were processed.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
PVOID *End;
ULONG64 Number;
PKPRCB Packet;
PKPRCB Prcb;
ULONG Processor;
REQUEST_SUMMARY Request;
PREQUEST_MAILBOX RequestMailbox;
PKREQUEST_PACKET RequestPacket;
LONG64 SetMember;
PKPRCB Source;
KAFFINITY SummarySet;
KAFFINITY TargetSet;
PVOID *Virtual;
//
// Loop until the sender summary is zero.
//
Prcb = KeGetCurrentPrcb();
TargetSet = ReadForWriteAccess(&Prcb->SenderSummary);
SetMember = Prcb->SetMember;
while (TargetSet != 0) {
SummarySet = TargetSet;
BitScanForward64(&Processor, SummarySet);
do {
Source = KiProcessorBlock[Processor];
RequestMailbox = &Prcb->RequestMailbox[Processor];
Request.Summary = RequestMailbox->RequestSummary;
//
// If the request type is flush multiple immediate, flush process,
// flush single, or flush all, then packet done can be signaled
// before processing the request. Otherwise, the request type must
// be a packet request, a cache invalidate, or a flush multiple
//
if (Request.IpiRequest <= IPI_FLUSH_ALL) {
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE_IMMEDIATE) {
Number = Request.Count;
Virtual = &RequestMailbox->Virtual[0];
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_FLUSH_PROCESS) {
KiFlushProcessTb();
} else if (Request.IpiRequest == IPI_FLUSH_SINGLE) {
KiFlushSingleTb((PVOID)Request.Parameter);
} else {
ASSERT(Request.IpiRequest == IPI_FLUSH_ALL);
KeFlushCurrentTb();
}
} else {
//
// If the request type is packet ready, then call the worker
// function. Otherwise, the request must be either a flush
// multiple or a cache invalidate.
//
if (Request.IpiRequest == IPI_PACKET_READY) {
Packet = Source;
if (Request.IpiSynchType != 0) {
Packet = (PKPRCB)((ULONG64)Source + 1);
}
RequestPacket = (PKREQUEST_PACKET)&RequestMailbox->RequestPacket;
(RequestPacket->WorkerRoutine)((PKIPI_CONTEXT)Packet,
RequestPacket->CurrentPacket[0],
RequestPacket->CurrentPacket[1],
RequestPacket->CurrentPacket[2]);
} else {
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE) {
Number = Request.Count;
Virtual = (PVOID *)Request.Parameter;
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_INVALIDATE_ALL) {
WritebackInvalidate();
} else {
ASSERT(FALSE);
}
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
}
}
SummarySet ^= AFFINITY_MASK(Processor);
} while (BitScanForward64(&Processor, SummarySet) != FALSE);
//
// Clear target set in sender summary.
//
TargetSet =
InterlockedExchangeAdd64((LONG64 volatile *)&Prcb->SenderSummary,
-(LONG64)TargetSet) - TargetSet;
}
#endif
return;
}
VOID
KeFlushProcessTb (
VOID
)
/*++
Routine Description:
This function flushes the non-global translation buffer on all processors
that are currently running threads which are child of the current process.
Arguments:
None.
Return Value:
None.
--*/
{
PKAFFINITY Barrier;
KIRQL OldIrql;
PKPRCB Prcb;
PKPROCESS Process;
KAFFINITY TargetProcessors;
//
// Compute the target set of processors, disable context switching,
// and send the flush entire parameters to the target processors,
// if any, for execution.
//
OldIrql = KeRaiseIrqlToSynchLevel();
Prcb = KeGetCurrentPrcb();
Process = Prcb->CurrentThread->ApcState.Process;
TargetProcessors = Process->ActiveProcessors;
TargetProcessors &= ~Prcb->SetMember;
//
// Send request to target processors, if any, flush the current process
// TB, and wait for the IPI request barrier.
//
if (TargetProcessors != 0) {
Barrier = KiIpiSendRequest(TargetProcessors, 0, 0, IPI_FLUSH_PROCESS);
KiFlushProcessTb();
KiIpiWaitForRequestBarrier(Barrier);
} else {
KiFlushProcessTb();
}
//
// Lower IRQL to its previous value.
//
KeLowerIrql(OldIrql);
return;
}
