gsize (g_atomic_pointer_xor) (volatile void *atomic, gsize val) { #if GLIB_SIZEOF_VOID_P == 8 return InterlockedXor64 (atomic, val); #else return InterlockedXor (atomic, val); #endif }
VOID KiIpiSignalPacketDone ( IN PKIPI_CONTEXT SignalDone ) /*++ Routine Description: This routine signals that a processor has completed a packet by clearing the calling processor's set member of the requesting processor's packet. Arguments: SignalDone - Supplies a pointer to the processor block of the sending processor. Return Value: None. --*/ { #if !defined(NT_UP) LONG64 SetMember; PKPRCB TargetPrcb; // // If the low bit of signal is set, then use target set to notify the // sender that the operation is complete on the current processor. // Otherwise, use packet barrier to notify the sender that the operation // is complete on the current processor. // if (((ULONG64)SignalDone & 1) == 0) { SetMember = KeGetCurrentPrcb()->SetMember; TargetPrcb = (PKPRCB)SignalDone; if (SetMember == InterlockedXor64((PLONG64)&TargetPrcb->TargetSet, SetMember)) { TargetPrcb->PacketBarrier = 0; } } else { TargetPrcb = (PKPRCB)((ULONG64)SignalDone - 1); TargetPrcb->TargetSet = 0; } #else UNREFERENCED_PARAMETER(SignalDone); #endif return; }
DECLSPEC_NOINLINE VOID KiIpiProcessRequests ( VOID ) /*++ Routine Description: This routine processes interprocessor requests and returns a summary of the requests that were processed. Arguments: None. Return Value: None. --*/ { #if !defined(NT_UP) PVOID *End; ULONG64 Number; PKPRCB Packet; PKPRCB Prcb; ULONG Processor; REQUEST_SUMMARY Request; PREQUEST_MAILBOX RequestMailbox; PKREQUEST_PACKET RequestPacket; LONG64 SetMember; PKPRCB Source; KAFFINITY SummarySet; KAFFINITY TargetSet; PVOID *Virtual; // // Loop until the sender summary is zero. // Prcb = KeGetCurrentPrcb(); TargetSet = ReadForWriteAccess(&Prcb->SenderSummary); SetMember = Prcb->SetMember; while (TargetSet != 0) { SummarySet = TargetSet; BitScanForward64(&Processor, SummarySet); do { Source = KiProcessorBlock[Processor]; RequestMailbox = &Prcb->RequestMailbox[Processor]; Request.Summary = RequestMailbox->RequestSummary; // // If the request type is flush multiple immediate, flush process, // flush single, or flush all, then packet done can be signaled // before processing the request. Otherwise, the request type must // be a packet request, a cache invalidate, or a flush multiple // if (Request.IpiRequest <= IPI_FLUSH_ALL) { // // If the synchronization type is target set, then the IPI was // only between two processors and target set should be used // for synchronization. Otherwise, packet barrier is used for // synchronization. // if (Request.IpiSynchType == 0) { if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet, SetMember)) { Source->PacketBarrier = 0; } } else { Source->TargetSet = 0; } if (Request.IpiRequest == IPI_FLUSH_MULTIPLE_IMMEDIATE) { Number = Request.Count; Virtual = &RequestMailbox->Virtual[0]; End = Virtual + Number; do { KiFlushSingleTb(*Virtual); Virtual += 1; } while (Virtual < End); } else if (Request.IpiRequest == IPI_FLUSH_PROCESS) { KiFlushProcessTb(); } else if (Request.IpiRequest == IPI_FLUSH_SINGLE) { KiFlushSingleTb((PVOID)Request.Parameter); } else { ASSERT(Request.IpiRequest == IPI_FLUSH_ALL); KeFlushCurrentTb(); } } else { // // If the request type is packet ready, then call the worker // function. Otherwise, the request must be either a flush // multiple or a cache invalidate. // if (Request.IpiRequest == IPI_PACKET_READY) { Packet = Source; if (Request.IpiSynchType != 0) { Packet = (PKPRCB)((ULONG64)Source + 1); } RequestPacket = (PKREQUEST_PACKET)&RequestMailbox->RequestPacket; (RequestPacket->WorkerRoutine)((PKIPI_CONTEXT)Packet, RequestPacket->CurrentPacket[0], RequestPacket->CurrentPacket[1], RequestPacket->CurrentPacket[2]); } else { if (Request.IpiRequest == IPI_FLUSH_MULTIPLE) { Number = Request.Count; Virtual = (PVOID *)Request.Parameter; End = Virtual + Number; do { KiFlushSingleTb(*Virtual); Virtual += 1; } while (Virtual < End); } else if (Request.IpiRequest == IPI_INVALIDATE_ALL) { WritebackInvalidate(); } else { ASSERT(FALSE); } // // If the synchronization type is target set, then the IPI was // only between two processors and target set should be used // for synchronization. Otherwise, packet barrier is used for // synchronization. // if (Request.IpiSynchType == 0) { if (SetMember == InterlockedXor64((PLONG64)&Source->TargetSet, SetMember)) { Source->PacketBarrier = 0; } } else { Source->TargetSet = 0; } } } SummarySet ^= AFFINITY_MASK(Processor); } while (BitScanForward64(&Processor, SummarySet) != FALSE); // // Clear target set in sender summary. // TargetSet = InterlockedExchangeAdd64((LONG64 volatile *)&Prcb->SenderSummary, -(LONG64)TargetSet) - TargetSet; } #endif return; }