gsize (g_atomic_pointer_and) (volatile void *atomic, gsize val) { #if GLIB_SIZEOF_VOID_P == 8 return InterlockedAnd64 (atomic, val); #else return InterlockedAnd (atomic, val); #endif }
__host__ __device__ typename enable_if< sizeof(Integer64) == 8, Integer64 >::type atomic_fetch_and(Integer64 *x, Integer64 y) { #if defined(__CUDA_ARCH__) return atomicAnd(x, y); #elif defined(__GNUC__) return __atomic_fetch_and(x, y, __ATOMIC_SEQ_CST); #elif defined(_MSC_VER) return InterlockedAnd64(x, y); #elif defined(__clang__) return __c11_atomic_fetch_and(x, y) #else #error "No atomic_fetch_and implementation." #endif }
VOID NTAPI KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock) { CCHAR Cpu; PKTHREAD InitialThread; ULONG64 InitialStack; PKIPCR Pcr; /* HACK */ FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea; //FrLdrDbgPrint("Hello from KiSystemStartup!!!\n"); /* Save the loader block */ KeLoaderBlock = LoaderBlock; /* Get the current CPU number */ Cpu = KeNumberProcessors++; // FIXME /* LoaderBlock initialization for Cpu 0 */ if (Cpu == 0) { /* Set the initial stack, idle thread and process */ LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack; LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread; LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb; LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb; } /* Get Pcr from loader block */ Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb); /* Set the PRCB for this Processor */ KiProcessorBlock[Cpu] = &Pcr->Prcb; /* Align stack to 16 bytes */ LoaderBlock->KernelStack &= ~(16 - 1); /* Save the initial thread and stack */ InitialStack = LoaderBlock->KernelStack; // Checkme InitialThread = (PKTHREAD)LoaderBlock->Thread; /* Set us as the current process */ InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process; /* Initialize the PCR */ KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack); /* Initialize the CPU features */ KiInitializeCpu(Pcr); /* Initial setup for the boot CPU */ if (Cpu == 0) { /* Initialize the module list (ntos, hal, kdcom) */ KiInitModuleList(LoaderBlock); /* Setup the TSS descriptors and entries */ KiInitializeTss(Pcr->TssBase, InitialStack); /* Setup the IDT */ KeInitExceptions(); /* Initialize debugging system */ KdInitSystem(0, KeLoaderBlock); /* Check for break-in */ if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C); } DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n", Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase); /* Acquire lock */ while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0)) { /* Loop until lock is free */ while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1); } /* Initialize the Processor with HAL */ HalInitializeProcessor(Cpu, KeLoaderBlock); /* Set processor as active */ KeActiveProcessors |= 1ULL << Cpu; /* Release lock */ InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0); /* Raise to HIGH_LEVEL */ KfRaiseIrql(HIGH_LEVEL); /* Machine specific kernel initialization */ if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock); /* Switch to new kernel stack and start kernel bootstrapping */ KiSwitchToBootStack(InitialStack & ~3); }
inline ::LONGLONG and ( volatile ::LONGLONG& x, ::LONGLONG y ) { return InterlockedAnd64(&x, y); }
NTKERNELAPI VOID FASTCALL ExfWakePushLock ( IN PEX_PUSH_LOCK PushLock, IN EX_PUSH_LOCK TopValue ) /*++ Routine Description: Walks the pushlock waiting list and wakes waiters if the lock is still unacquired. Arguments: PushLock - Push lock to be walked TopValue - Start of the chain (*PushLock) Return Value: None --*/ { EX_PUSH_LOCK OldValue, NewValue; PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock, FirstWaitBlock, PreviousWaitBlock; KIRQL OldIrql; OldValue = TopValue; while (1) { // // Nobody should be walking the list while we manipulate it. // ASSERT (!OldValue.MultipleShared); // // No point waking somebody to find a locked lock. Just clear the waking bit // while (OldValue.Locked) { NewValue.Value = OldValue.Value - EX_PUSH_LOCK_WAKING; ASSERT (!NewValue.Waking); ASSERT (NewValue.Locked); ASSERT (NewValue.Waiting); if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr)) == OldValue.Ptr) { return; } OldValue = NewValue; } WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK) (OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS); FirstWaitBlock = WaitBlock; while (1) { NextWaitBlock = WaitBlock->Last; if (NextWaitBlock != NULL) { WaitBlock = NextWaitBlock; break; } PreviousWaitBlock = WaitBlock; WaitBlock = WaitBlock->Next; WaitBlock->Previous = PreviousWaitBlock; } if (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE && (PreviousWaitBlock = WaitBlock->Previous) != NULL) { FirstWaitBlock->Last = PreviousWaitBlock; WaitBlock->Previous = NULL; ASSERT (FirstWaitBlock != WaitBlock); ASSERT (PushLock->Waiting); #if defined (_WIN64) InterlockedAnd64 ((LONG64 *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING); #else InterlockedAnd ((LONG *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING); #endif break; } else { NewValue.Value = 0; ASSERT (!NewValue.Waking); if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr, NewValue.Ptr, OldValue.Ptr)) == OldValue.Ptr) { break; } OldValue = NewValue; } } // // If we are waking more than one thread then raise to DPC level to prevent us // getting rescheduled part way through the operation // OldIrql = DISPATCH_LEVEL; if (WaitBlock->Previous != NULL) { KeRaiseIrql (DISPATCH_LEVEL, &OldIrql); } while (1) { NextWaitBlock = WaitBlock->Previous; #if DBG ASSERT (!WaitBlock->Signaled); WaitBlock->Signaled = TRUE; #endif if (!InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) { KeSignalGateBoostPriority (&WaitBlock->WakeGate); } WaitBlock = NextWaitBlock; if (WaitBlock == NULL) { break; } } if (OldIrql != DISPATCH_LEVEL) { KeLowerIrql (OldIrql); } }