gsize
(g_atomic_pointer_xor) (volatile void *atomic,
gsize val)
{
#if GLIB_SIZEOF_VOID_P == 8
return InterlockedXor64 (atomic, val);
#else
return InterlockedXor (atomic, val);
#endif
}
VOID
KiIpiSignalPacketDone (
IN PKIPI_CONTEXT SignalDone
)
/*++
Routine Description:
This routine signals that a processor has completed a packet by clearing
the calling processor's set member of the requesting processor's packet.
Arguments:
SignalDone - Supplies a pointer to the processor block of the sending
processor.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
LONG64 SetMember;
PKPRCB TargetPrcb;
//
// If the low bit of signal is set, then use target set to notify the
// sender that the operation is complete on the current processor.
// Otherwise, use packet barrier to notify the sender that the operation
// is complete on the current processor.
//
if (((ULONG64)SignalDone & 1) == 0) {
SetMember = KeGetCurrentPrcb()->SetMember;
TargetPrcb = (PKPRCB)SignalDone;
if (SetMember == InterlockedXor64((PLONG64)&TargetPrcb->TargetSet,
SetMember)) {
TargetPrcb->PacketBarrier = 0;
}
} else {
TargetPrcb = (PKPRCB)((ULONG64)SignalDone - 1);
TargetPrcb->TargetSet = 0;
}
#else
UNREFERENCED_PARAMETER(SignalDone);
#endif
return;
}
DECLSPEC_NOINLINE
VOID
KiIpiProcessRequests (
VOID
)
/*++
Routine Description:
This routine processes interprocessor requests and returns a summary
of the requests that were processed.
Arguments:
None.
Return Value:
None.
--*/
{
#if !defined(NT_UP)
PVOID *End;
ULONG64 Number;
PKPRCB Packet;
PKPRCB Prcb;
ULONG Processor;
REQUEST_SUMMARY Request;
PREQUEST_MAILBOX RequestMailbox;
PKREQUEST_PACKET RequestPacket;
LONG64 SetMember;
PKPRCB Source;
KAFFINITY SummarySet;
KAFFINITY TargetSet;
PVOID *Virtual;
//
// Loop until the sender summary is zero.
//
Prcb = KeGetCurrentPrcb();
TargetSet = ReadForWriteAccess(&Prcb->SenderSummary);
SetMember = Prcb->SetMember;
while (TargetSet != 0) {
SummarySet = TargetSet;
BitScanForward64(&Processor, SummarySet);
do {
Source = KiProcessorBlock[Processor];
RequestMailbox = &Prcb->RequestMailbox[Processor];
Request.Summary = RequestMailbox->RequestSummary;
//
// If the request type is flush multiple immediate, flush process,
// flush single, or flush all, then packet done can be signaled
// before processing the request. Otherwise, the request type must
// be a packet request, a cache invalidate, or a flush multiple
//
if (Request.IpiRequest <= IPI_FLUSH_ALL) {
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE_IMMEDIATE) {
Number = Request.Count;
Virtual = &RequestMailbox->Virtual[0];
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_FLUSH_PROCESS) {
KiFlushProcessTb();
} else if (Request.IpiRequest == IPI_FLUSH_SINGLE) {
KiFlushSingleTb((PVOID)Request.Parameter);
} else {
ASSERT(Request.IpiRequest == IPI_FLUSH_ALL);
KeFlushCurrentTb();
}
} else {
//
// If the request type is packet ready, then call the worker
// function. Otherwise, the request must be either a flush
// multiple or a cache invalidate.
//
if (Request.IpiRequest == IPI_PACKET_READY) {
Packet = Source;
if (Request.IpiSynchType != 0) {
Packet = (PKPRCB)((ULONG64)Source + 1);
}
RequestPacket = (PKREQUEST_PACKET)&RequestMailbox->RequestPacket;
(RequestPacket->WorkerRoutine)((PKIPI_CONTEXT)Packet,
RequestPacket->CurrentPacket[0],
RequestPacket->CurrentPacket[1],
RequestPacket->CurrentPacket[2]);
} else {
if (Request.IpiRequest == IPI_FLUSH_MULTIPLE) {
Number = Request.Count;
Virtual = (PVOID *)Request.Parameter;
End = Virtual + Number;
do {
KiFlushSingleTb(*Virtual);
Virtual += 1;
} while (Virtual < End);
} else if (Request.IpiRequest == IPI_INVALIDATE_ALL) {
WritebackInvalidate();
} else {
ASSERT(FALSE);
}
//
// If the synchronization type is target set, then the IPI was
// only between two processors and target set should be used
// for synchronization. Otherwise, packet barrier is used for
// synchronization.
//
if (Request.IpiSynchType == 0) {
if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet,
SetMember)) {
Source->PacketBarrier = 0;
}
} else {
Source->TargetSet = 0;
}
}
}
SummarySet ^= AFFINITY_MASK(Processor);
} while (BitScanForward64(&Processor, SummarySet) != FALSE);
//
// Clear target set in sender summary.
//
TargetSet =
InterlockedExchangeAdd64((LONG64 volatile *)&Prcb->SenderSummary,
-(LONG64)TargetSet) - TargetSet;
}
#endif
return;
}
