gsize
(g_atomic_pointer_and) (volatile void *atomic,
gsize val)
{
#if GLIB_SIZEOF_VOID_P == 8
return InterlockedAnd64 (atomic, val);
#else
return InterlockedAnd (atomic, val);
#endif
}
__host__ __device__
typename enable_if<
sizeof(Integer64) == 8,
Integer64
>::type
atomic_fetch_and(Integer64 *x, Integer64 y)
{
#if defined(__CUDA_ARCH__)
return atomicAnd(x, y);
#elif defined(__GNUC__)
return __atomic_fetch_and(x, y, __ATOMIC_SEQ_CST);
#elif defined(_MSC_VER)
return InterlockedAnd64(x, y);
#elif defined(__clang__)
return __c11_atomic_fetch_and(x, y)
#else
#error "No atomic_fetch_and implementation."
#endif
}
VOID
NTAPI
KiSystemStartup(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
{
CCHAR Cpu;
PKTHREAD InitialThread;
ULONG64 InitialStack;
PKIPCR Pcr;
/* HACK */
FrLdrDbgPrint = LoaderBlock->u.I386.CommonDataArea;
//FrLdrDbgPrint("Hello from KiSystemStartup!!!\n");
/* Save the loader block */
KeLoaderBlock = LoaderBlock;
/* Get the current CPU number */
Cpu = KeNumberProcessors++; // FIXME
/* LoaderBlock initialization for Cpu 0 */
if (Cpu == 0)
{
/* Set the initial stack, idle thread and process */
LoaderBlock->KernelStack = (ULONG_PTR)P0BootStack;
LoaderBlock->Thread = (ULONG_PTR)&KiInitialThread;
LoaderBlock->Process = (ULONG_PTR)&KiInitialProcess.Pcb;
LoaderBlock->Prcb = (ULONG_PTR)&KiInitialPcr.Prcb;
}
/* Get Pcr from loader block */
Pcr = CONTAINING_RECORD(LoaderBlock->Prcb, KIPCR, Prcb);
/* Set the PRCB for this Processor */
KiProcessorBlock[Cpu] = &Pcr->Prcb;
/* Align stack to 16 bytes */
LoaderBlock->KernelStack &= ~(16 - 1);
/* Save the initial thread and stack */
InitialStack = LoaderBlock->KernelStack; // Checkme
InitialThread = (PKTHREAD)LoaderBlock->Thread;
/* Set us as the current process */
InitialThread->ApcState.Process = (PVOID)LoaderBlock->Process;
/* Initialize the PCR */
KiInitializePcr(Pcr, Cpu, InitialThread, (PVOID)KiDoubleFaultStack);
/* Initialize the CPU features */
KiInitializeCpu(Pcr);
/* Initial setup for the boot CPU */
if (Cpu == 0)
{
/* Initialize the module list (ntos, hal, kdcom) */
KiInitModuleList(LoaderBlock);
/* Setup the TSS descriptors and entries */
KiInitializeTss(Pcr->TssBase, InitialStack);
/* Setup the IDT */
KeInitExceptions();
/* Initialize debugging system */
KdInitSystem(0, KeLoaderBlock);
/* Check for break-in */
if (KdPollBreakIn()) DbgBreakPointWithStatus(DBG_STATUS_CONTROL_C);
}
DPRINT1("Pcr = %p, Gdt = %p, Idt = %p, Tss = %p\n",
Pcr, Pcr->GdtBase, Pcr->IdtBase, Pcr->TssBase);
/* Acquire lock */
while (InterlockedBitTestAndSet64((PLONG64)&KiFreezeExecutionLock, 0))
{
/* Loop until lock is free */
while ((*(volatile KSPIN_LOCK*)&KiFreezeExecutionLock) & 1);
}
/* Initialize the Processor with HAL */
HalInitializeProcessor(Cpu, KeLoaderBlock);
/* Set processor as active */
KeActiveProcessors |= 1ULL << Cpu;
/* Release lock */
InterlockedAnd64((PLONG64)&KiFreezeExecutionLock, 0);
/* Raise to HIGH_LEVEL */
KfRaiseIrql(HIGH_LEVEL);
/* Machine specific kernel initialization */
if (Cpu == 0) KiInitializeKernelMachineDependent(&Pcr->Prcb, LoaderBlock);
/* Switch to new kernel stack and start kernel bootstrapping */
KiSwitchToBootStack(InitialStack & ~3);
}
inline ::LONGLONG and ( volatile ::LONGLONG& x, ::LONGLONG y )
{
return InterlockedAnd64(&x, y);
}
NTKERNELAPI
VOID
FASTCALL
ExfWakePushLock (
IN PEX_PUSH_LOCK PushLock,
IN EX_PUSH_LOCK TopValue
)
/*++
Routine Description:
Walks the pushlock waiting list and wakes waiters if the lock is still unacquired.
Arguments:
PushLock - Push lock to be walked
TopValue - Start of the chain (*PushLock)
Return Value:
None
--*/
{
EX_PUSH_LOCK OldValue, NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock, FirstWaitBlock, PreviousWaitBlock;
KIRQL OldIrql;
OldValue = TopValue;
while (1) {
//
// Nobody should be walking the list while we manipulate it.
//
ASSERT (!OldValue.MultipleShared);
//
// No point waking somebody to find a locked lock. Just clear the waking bit
//
while (OldValue.Locked) {
NewValue.Value = OldValue.Value - EX_PUSH_LOCK_WAKING;
ASSERT (!NewValue.Waking);
ASSERT (NewValue.Locked);
ASSERT (NewValue.Waiting);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
return;
}
OldValue = NewValue;
}
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)
(OldValue.Value & ~(ULONG_PTR)EX_PUSH_LOCK_PTR_BITS);
FirstWaitBlock = WaitBlock;
while (1) {
NextWaitBlock = WaitBlock->Last;
if (NextWaitBlock != NULL) {
WaitBlock = NextWaitBlock;
break;
}
PreviousWaitBlock = WaitBlock;
WaitBlock = WaitBlock->Next;
WaitBlock->Previous = PreviousWaitBlock;
}
if (WaitBlock->Flags&EX_PUSH_LOCK_FLAGS_EXCLUSIVE &&
(PreviousWaitBlock = WaitBlock->Previous) != NULL) {
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
ASSERT (FirstWaitBlock != WaitBlock);
ASSERT (PushLock->Waiting);
#if defined (_WIN64)
InterlockedAnd64 ((LONG64 *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#else
InterlockedAnd ((LONG *)&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
#endif
break;
} else {
NewValue.Value = 0;
ASSERT (!NewValue.Waking);
if ((NewValue.Ptr = InterlockedCompareExchangePointer (&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr)) == OldValue.Ptr) {
break;
}
OldValue = NewValue;
}
}
//
// If we are waking more than one thread then raise to DPC level to prevent us
// getting rescheduled part way through the operation
//
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous != NULL) {
KeRaiseIrql (DISPATCH_LEVEL, &OldIrql);
}
while (1) {
NextWaitBlock = WaitBlock->Previous;
#if DBG
ASSERT (!WaitBlock->Signaled);
WaitBlock->Signaled = TRUE;
#endif
if (!InterlockedBitTestAndReset (&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_SPINNING_V)) {
KeSignalGateBoostPriority (&WaitBlock->WakeGate);
}
WaitBlock = NextWaitBlock;
if (WaitBlock == NULL) {
break;
}
}
if (OldIrql != DISPATCH_LEVEL) {
KeLowerIrql (OldIrql);
}
}
