VOID KphpTestPushLockThreadStart(
__in PVOID Context
)
{
ULONG i, j;
for (i = 0; i < 400000; i++)
{
ExAcquirePushLockShared(&TestLock);
for (j = 0; j < 1000; j++)
YieldProcessor();
ExReleasePushLock(&TestLock);
ExAcquirePushLockExclusive(&TestLock);
for (j = 0; j < 9000; j++)
YieldProcessor();
ExReleasePushLock(&TestLock);
}
PsTerminateSystemThread(STATUS_SUCCESS);
}
void FastSpinlock::EnterWriteLock()
{
/// 락 순서 신경 안써도 되는 경우는 그냥 패스
if ( mLockOrder != LO_DONT_CARE)
LLockOrderChecker->Push(this);
while (true)
{
/// 다른놈이 writelock 풀어줄때까지 기다린다.
while (mLockFlag & LF_WRITE_MASK)
YieldProcessor();
if ((InterlockedAdd(&mLockFlag, LF_WRITE_FLAG) & LF_WRITE_MASK) == LF_WRITE_FLAG)
{
/// 다른놈이 readlock 풀어줄때까지 기다린다.
while (mLockFlag & LF_READ_MASK)
YieldProcessor();
return;
}
InterlockedAdd(&mLockFlag, -LF_WRITE_FLAG);
}
}
int _spin_lite_lock(spin_t *l)
{
CHECK_SPINLOCK_LITE(l);
int lscnt = 0;
_vol_spinlock v;
v.l = (LONG *)&l->l;
_spin_lite_lock_inc(bscnt);
while (InterlockedExchange(v.lv, EBUSY))
{
_spin_lite_lock_cnt(lscnt);
/* Don't lock the bus whilst waiting */
while (*v.lv)
{
_spin_lite_lock_cnt(lscnt);
YieldProcessor();
/* Compiler barrier. Prevent caching of *l */
_ReadWriteBarrier();
}
}
_spin_lite_lock_dec(bscnt);
_spin_lite_lock_stat(lscnt);
return 0;
}
bool CritSectEx::PerfLock(DWORD dwThreadID, DWORD dwTimeout)
{
#if DEBUG > 1
if( m_bIsLocked ){
fprintf( stderr, "Thread %lu attempting to lock mutex of thread %lu\n",
dwThreadID, m_nLocker
);
}
#endif
// Attempt spin-lock
for (DWORD dwSpin = 0; dwSpin < m_dwSpinMax; dwSpin++)
{
if (PerfLockImmediate(dwThreadID)){
return true;
}
YieldProcessor();
}
// Ensure we have the kernel event created
AllocateKernelSemaphore();
bool bval = PerfLockKernel(dwThreadID, dwTimeout);
WaiterMinus();
return bval;
}
unsigned int pthread_create_wrapper(void *args)
{
struct _pthread_v *tv = (struct _pthread_v*)args;
_pthread_once_raw(&_pthread_tls_once, pthread_tls_init);
TlsSetValue(_pthread_tls, tv);
if (!setjmp(tv->jb))
{
/* Call function and save return value */
tv->ret_arg = tv->func(tv->ret_arg);
/* Clean up destructors */
_pthread_cleanup_dest(tv);
}
/* If we exit too early, then we can race with create */
while (tv->h == (HANDLE) -1)
{
YieldProcessor();
_ReadWriteBarrier();
}
/* Make sure we free ourselves if we are detached */
if (!tv->h)
{
if(tv->keyval)
free(tv->keyval);
free(tv);
}
return 0;
}
int pthread_once(pthread_once_t *once, void (*func)(void))
{
long state = *once;
_ReadWriteBarrier();
while (state != 1)
{
if (!state)
{
if (!InterlockedCompareExchange(once, 2, 0))
{
func();
*once = 1;
return 0;
}
}
YieldProcessor();
_ReadWriteBarrier();
state = *once;
}
return 0;
}
int osd_work_item_wait(osd_work_item *item, osd_ticks_t timeout)
{
// if we're done already, just return
if (item->done)
return TRUE;
// if we don't have an event, create one
if (item->event == NULL)
item->event = CreateEvent(NULL, TRUE, FALSE, NULL); // manual reset, not signalled
else
ResetEvent(item->event);
// if we don't have an event, we need to spin (shouldn't ever really happen)
if (item->event != NULL)
{
osd_ticks_t stopspin = osd_ticks() + timeout;
while (!item->done && osd_ticks() < stopspin)
YieldProcessor();
}
// otherwise, block on the event until done
else if (!item->done)
WaitForSingleObject(item->event, timeout * 1000 / osd_ticks_per_second());
// return TRUE if the refcount actually hit 0
return item->done;
}
void FastSpinlock::EnterReadLock()
{
if (mLockOrder != LO_DONT_CARE)
LLockOrderChecker->Push(this);
while (true)
{
/// 다른놈이 writelock 풀어줄때까지 기다린다.
while (mLockFlag & LF_WRITE_MASK)
YieldProcessor();
//TODO: Readlock 진입 구현 (mLockFlag를 어떻게 처리하면 되는지?)
// if ( readlock을 얻으면 )
//return;
// else
// mLockFlag 원복
if ( ( InterlockedAdd( &mLockFlag, 1 ) & LF_WRITE_MASK ) != LF_WRITE_FLAG )
{
return;
}
else
{
InterlockedAdd( &mLockFlag, -1 );
}
}
}
int _pthread_once_raw(pthread_once_t *o, void (*func)(void))
{
long state = *o;
_ReadWriteBarrier();
while (state != 1)
{
if (!state)
{
if (!_InterlockedCompareExchange(o, 2, 0))
{
/* Success */
func();
/* Mark as done */
*o = 1;
return 0;
}
}
YieldProcessor();
_ReadWriteBarrier();
state = *o;
}
/* Done */
return 0;
}
static inline void pause(std::int32_t delay)
{
for (; delay > 0; --delay)
{
YieldProcessor();
}
}
int thread_once(thread_control_t *control, void (*callback)(void))
{
#ifdef ACE_WINDOWS
int state = (int)(*control);
_ReadWriteBarrier();
while (state != 1) {
if ((!state) && (!_InterlockedCompareExchange(control, 2, 0))) {
callback();
*control = 1;
return 0;
}
YieldProcessor();
_ReadWriteBarrier();
state = (int)(*control);
}
return 0;
#else
return pthread_once(control, callback);
#endif
}
/*++
* @name ExTimedWaitForUnblockPushLock
*
* The ExTimedWaitForUnblockPushLock routine waits for a pushlock
* to be unblocked, for a specified internal.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param WaitBlock
* Pointer to the pushlock's wait block.
*
* @param Timeout
* Amount of time to wait for this pushlock to be unblocked.
*
* @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
* code returned by KeWaitForSingleObject.
*
* @remarks If the wait fails, then a manual unblock is attempted.
*
*--*/
NTSTATUS
FASTCALL
ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
IN PVOID WaitBlock,
IN PLARGE_INTEGER Timeout)
{
NTSTATUS Status;
/* Initialize the wait event */
KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->WakeEvent,
SynchronizationEvent,
FALSE);
#ifdef CONFIG_SMP
/* Spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
/* Check if we got lucky and can leave early */
if (!(*(volatile LONG *)&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags & EX_PUSH_LOCK_WAITING))
return STATUS_SUCCESS;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags,
EX_PUSH_LOCK_FLAGS_WAIT_V))
{
/* Nobody removed it already, let's do a full wait */
Status = KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->
WakeEvent,
WrPushLock,
KernelMode,
FALSE,
Timeout);
/* Check if the wait was satisfied */
if (Status != STATUS_SUCCESS)
{
/* Try unblocking the pushlock if it was not */
ExfUnblockPushLock(PushLock, WaitBlock);
}
}
else
{
/* Someone beat us to it, no need to wait */
Status = STATUS_SUCCESS;
}
/* Return status */
return Status;
}
static unsigned __stdcall worker_thread_entry(void *param)
{
work_thread_info *thread = param;
osd_work_queue *queue = thread->queue;
// loop until we exit
for ( ;; )
{
// block waiting for work or exit
DWORD result = WAIT_OBJECT_0;
// bail on exit, and only wait if there are no pending items in queue
if (!queue->exiting && queue->list == NULL)
{
begin_timing(thread->waittime);
result = WaitForSingleObject(thread->wakeevent, INFINITE);
end_timing(thread->waittime);
}
if (queue->exiting)
break;
// indicate that we are live
interlocked_exchange32(&thread->active, TRUE);
interlocked_increment(&queue->livethreads);
// process work items
for ( ;; )
{
osd_ticks_t stopspin;
// process as much as we can
worker_thread_process(queue, thread);
// if we're a high frequency queue, spin for a while before giving up
if (queue->flags & WORK_QUEUE_FLAG_HIGH_FREQ)
{
// spin for a while looking for more work
begin_timing(thread->spintime);
stopspin = osd_ticks() + SPIN_LOOP_TIME;
while (queue->list == NULL && osd_ticks() < stopspin)
YieldProcessor();
end_timing(thread->spintime);
}
// if nothing more, release the processor
if (queue->list == NULL)
break;
add_to_stat(&queue->spinloops, 1);
}
// decrement the live thread count
interlocked_exchange32(&thread->active, FALSE);
interlocked_decrement(&queue->livethreads);
}
return 0;
}
void PALAPI Run_Thread (LPVOID lpParam)
{
int i = 0;
for(i=0; i < REPEAT_COUNT; i++ )
{
// No error code set nor does it have any return code
YieldProcessor();
}
}
/* //////////////////////////////////////////////////////////////////////////////////////
* implementation
*/
tb_bool_t tb_sched_yield()
{
// yield it in thread
#if defined(YieldProcessor)
YieldProcessor();
return tb_true;
#else
tb_usleep(1);
return tb_true;
#endif
}
void skylark::SpinLock::enterWriteLock()
{
thisThread->getLockOrderChecker()->push(this);
while (true)
{
while (mLockFlag & WRITE_MASK)
YieldProcessor();
if (((mLockFlag += WRITE_FLAG) & WRITE_MASK) == WRITE_FLAG)
{
while (mLockFlag & READ_MASK)
YieldProcessor();
return;
}
mLockFlag -= WRITE_FLAG;
}
}
void FastSpinlock::EnterWriteLock()
{
while (true)
{
/// wait a writelock
while (mLockFlag & LF_WRITE_MASK)
YieldProcessor();
if ((InterlockedAdd(&mLockFlag, LF_WRITE_FLAG) & LF_WRITE_MASK) == LF_WRITE_FLAG)
{
/// wait a readlock
while (mLockFlag & LF_READ_MASK)
YieldProcessor();
return;
}
InterlockedAdd(&mLockFlag, -LF_WRITE_FLAG);
}
}
void FastSpinlock::EnterReadLock()
{
while (true)
{
/// wait a writelock
while (mLockFlag & LF_WRITE_MASK)
YieldProcessor();
if ((InterlockedIncrement(&mLockFlag) & LF_WRITE_MASK) == 0)
return;
InterlockedDecrement(&mLockFlag);
}
}
/// <summary>
/// Sleeps in the microsecond.
/// </summary>
/// <param name="us">The time to sleep in us.</param>
VOID SleepMicrosecond(INT us)
{
UINT64 t1 = 0, t2 = 0, freq = 0;
UINT64 wait_tick;
QueryPerformanceFrequency((LARGE_INTEGER *) &freq);
wait_tick = freq * us / 1000000ULL;
QueryPerformanceCounter((LARGE_INTEGER *) &t1);
do
{
QueryPerformanceCounter((LARGE_INTEGER *) &t2);
YieldProcessor();
}
while ((t2-t1) < wait_tick);
}
VOID
NTAPI
KeQueryTickCount(IN PLARGE_INTEGER TickCount)
{
/* Loop until we get a perfect match */
for (;;)
{
/* Read the tick count value */
TickCount->HighPart = KeTickCount.High1Time;
TickCount->LowPart = KeTickCount.LowPart;
if (TickCount->HighPart == KeTickCount.High2Time) break;
YieldProcessor();
}
}
DWORD
WINAPI
HappyThread (
__in PVOID Argument
)
{
volatile PBOOL Stop = (PBOOL) Argument;
do {
YieldProcessor();
} while (!*Stop);
return 0;
}
/*
* @implemented
*/
VOID
NTAPI
KeQuerySystemTime(OUT PLARGE_INTEGER CurrentTime)
{
/* Loop until we get a perfect match */
for (;;)
{
/* Read the time value */
CurrentTime->HighPart = SharedUserData->SystemTime.High1Time;
CurrentTime->LowPart = SharedUserData->SystemTime.LowPart;
if (CurrentTime->HighPart ==
SharedUserData->SystemTime.High2Time) break;
YieldProcessor();
}
}
static unsigned __stdcall internal_win32_callback_wrapper(void *parg)
{
thread_t *t = (thread_t *)parg;
thread_once(&_tls_control, internal_tls_init);
TlsSetValue(_tls, t);
if (t) { t->callback(parg); }
while (t->thread == (HANDLE)-1) {
YieldProcessor();
_ReadWriteBarrier();
}
return 0;
}
void skylark::SpinLock::enterReadLock()
{
thisThread->getLockOrderChecker()->push(this);
while (true)
{
while (mLockFlag & WRITE_MASK)
YieldProcessor();
if ((++mLockFlag & WRITE_MASK) == 0)
{
return;
}
--mLockFlag;
}
}
int osd_work_queue_wait(osd_work_queue *queue, osd_ticks_t timeout)
{
// if no threads, no waiting
if (queue->threads == 0)
return TRUE;
// if no items, we're done
if (queue->items == 0)
return TRUE;
// if this is a multi queue, help out rather than doing nothing
if (queue->flags & WORK_QUEUE_FLAG_MULTI)
{
work_thread_info *thread = &queue->thread[queue->threads];
osd_ticks_t stopspin = osd_ticks() + timeout;
end_timing(thread->waittime);
// process what we can as a worker thread
worker_thread_process(queue, thread);
// if we're a high frequency queue, spin until done
if (queue->flags & WORK_QUEUE_FLAG_HIGH_FREQ)
{
// spin until we're done
begin_timing(thread->spintime);
while (queue->items != 0 && osd_ticks() < stopspin)
YieldProcessor();
end_timing(thread->spintime);
begin_timing(thread->waittime);
return (queue->items == 0);
}
begin_timing(thread->waittime);
}
// reset our done event and double-check the items before waiting
ResetEvent(queue->doneevent);
interlocked_exchange32(&queue->waiting, TRUE);
if (queue->items != 0)
WaitForSingleObject(queue->doneevent, timeout * 1000 / osd_ticks_per_second());
interlocked_exchange32(&queue->waiting, FALSE);
// return TRUE if we actually hit 0
return (queue->items == 0);
}
/**
* Waits for a wait block to be unblocked.
*
* \param WaitBlock A wait block.
* \param Spin TRUE to spin, FALSE to block immediately.
* \param Timeout A timeout value.
*/
__mayRaise FORCEINLINE NTSTATUS PhpBlockOnQueuedWaitBlock(
__inout PPH_QUEUED_WAIT_BLOCK WaitBlock,
__in BOOLEAN Spin,
__in_opt PLARGE_INTEGER Timeout
)
{
NTSTATUS status;
ULONG i;
if (Spin)
{
PHLIB_INC_STATISTIC(QlBlockSpins);
for (i = PhQueuedLockSpinCount; i != 0; i--)
{
if (!(*(volatile ULONG *)&WaitBlock->Flags & PH_QUEUED_WAITER_SPINNING))
return STATUS_SUCCESS;
YieldProcessor();
}
}
if (_interlockedbittestandreset((PLONG)&WaitBlock->Flags, PH_QUEUED_WAITER_SPINNING_SHIFT))
{
PHLIB_INC_STATISTIC(QlBlockWaits);
status = NtWaitForKeyedEvent(
PhQueuedLockKeyedEventHandle,
WaitBlock,
FALSE,
Timeout
);
// If an error occurred (timeout is not an error), raise an exception
// as it is nearly impossible to recover from this situation.
if (!NT_SUCCESS(status))
PhRaiseStatus(status);
}
else
{
status = STATUS_SUCCESS;
}
return status;
}
DWORD CALLBACK ThreadProcOSCS(void* p)
{
CRITICAL_SECTION* lock = (CRITICAL_SECTION*)p;
long value = InterlockedIncrement(&next);
EnterCriticalSection(lock);
for (uint32_t i = 0; i < _countof(buffer); ++i)
{
buffer[i] = value;
YieldProcessor();
}
LeaveCriticalSection(lock);
return 0;
}
DWORD CALLBACK ThreadProcCS(void* p)
{
UMCriticalSection* lock = (UMCriticalSection*)p;
long value = InterlockedIncrement(&next);
UMCriticalSectionAcquire(lock);
for (uint32_t i = 0; i < _countof(buffer); ++i)
{
buffer[i] = value;
YieldProcessor();
}
UMCriticalSectionRelease(lock);
return 0;
}
DWORD CALLBACK ThreadProcSpinLock(void* p)
{
UMSpinLock* lock = (UMSpinLock*)p;
long value = InterlockedIncrement(&next);
UMSpinLockAcquire(lock);
for (uint32_t i = 0; i < _countof(buffer); ++i)
{
buffer[i] = value;
YieldProcessor();
}
UMSpinLockRelease(lock);
return 0;
}
/*
* @implemented
*/
ULONGLONG
WINAPI
GetTickCount64(VOID)
{
ULARGE_INTEGER TickCount;
while (TRUE)
{
TickCount.HighPart = (ULONG)SharedUserData->TickCount.High1Time;
TickCount.LowPart = SharedUserData->TickCount.LowPart;
if (TickCount.HighPart == (ULONG)SharedUserData->TickCount.High2Time) break;
YieldProcessor();
}
return (UInt32x32To64(TickCount.LowPart, SharedUserData->TickCountMultiplier) >> 24) +
(UInt32x32To64(TickCount.HighPart, SharedUserData->TickCountMultiplier) << 8);
}