void
ngx_rwlock_wlock(ngx_atomic_t *lock)
{
ngx_uint_t i, n;
for ( ;; ) {
if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK)) {
return;
}
if (ngx_ncpu > 1) {
for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
for (i = 0; i < n; i++) {
ngx_cpu_pause();
}
if (*lock == 0
&& ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK))
{
return;
}
}
}
ngx_sched_yield();
}
}
void ngx_rwlock_reserve_read(ngx_rwlock_t *lock)
{
#if (NGX_HAVE_ATOMIC_OPS)
ngx_uint_t i, n;
#if (DISABLE_RWLOCK == 1)
rwl_lock_mutex(lock);
return;
#endif
for(;;) {
NGX_RWLOCK_MUTEX_COND(lock, (lock->lock != NGX_RWLOCK_WRITE), lock->lock++)
#if (DEBUG_NGX_RWLOCK == 1)
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0, "rwlock %p reserve read read (%i)", lock, lock->lock);
#endif
if(ngx_ncpu > 1) {
for(n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
for(i = 0; i < n; i++) {
ngx_cpu_pause();
}
#if (DEBUG_NGX_RWLOCK == 1)
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0, "rwlock %p read lock wait", lock);
#endif
NGX_RWLOCK_MUTEX_COND(lock, (lock->lock != NGX_RWLOCK_WRITE), lock->lock++)
}
}
ngx_sched_yield();
}
#else
#if (NGX_THREADS)
#error ngx_spinlock() or ngx_atomic_cmp_set() are not defined !
#endif
#endif
}
static void rwl_lock_mutex(ngx_rwlock_t *lock) {
#if (NGX_HAVE_ATOMIC_OPS)
ngx_atomic_t *mutex = &lock->mutex;
ngx_uint_t i, n;
for(;;) {
if(*mutex == 0 && ngx_atomic_cmp_set(mutex, 0, ngx_pid)) {
return;
}
if(ngx_ncpu > 1) {
for(n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
for(i = 0; i < n; i++) {
ngx_cpu_pause();
}
#if (DEBUG_NGX_RWLOCK)
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0, "rwlock %p mutex wait", lock);
#endif
if(*mutex == 0 && ngx_atomic_cmp_set(mutex, 0, ngx_pid)) {
return;
}
}
}
ngx_sched_yield();
}
#else
#if (NGX_THREADS)
#error ngx_spinlock() or ngx_atomic_cmp_set() are not defined !
#endif
#endif
}
void ngx_rwlock_reserve_write(ngx_rwlock_t * lock) {
#if (NGX_HAVE_ATOMIC_OPS)
ngx_uint_t i, n;
#if (DISABLE_RWLOCK == 1)
rwl_lock_mutex(lock);
return;
#endif
for(;;) {
if(ngx_rwlock_write_check(lock))
return;
if(ngx_ncpu > 1) {
for(n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
for(i = 0; i < n; i++) {
ngx_cpu_pause();
}
#if (DEBUG_NGX_RWLOCK == 1)
ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0, "rwlock %p write lock wait (reserved by %ui)", lock, lock->write_pid);
#endif
if(ngx_rwlock_write_check(lock))
return;
}
}
ngx_sched_yield();
}
#else
#if (NGX_THREADS)
#error ngx_spinlock() or ngx_atomic_cmp_set() are not defined !
#endif
#endif
}
void
ngx_spinlock(ngx_atomic_t *lock, ngx_atomic_int_t value, ngx_uint_t spin)
{
#if (NGX_HAVE_ATOMIC_OPS)
ngx_uint_t i, n;
for ( ;; ) {
if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, value)) {
return;
}
if (ngx_ncpu > 1) {
for (n = 1; n < spin; n <<= 1) {
for (i = 0; i < n; i++) {
ngx_cpu_pause();
}
if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, value)) {
return;
}
}
}
ngx_sched_yield();
}
#else
#if (NGX_THREADS)
#error ngx_spinlock() or ngx_atomic_cmp_set() are not defined !
#endif
#endif
}
void
ngx_rwlock_rlock(ngx_atomic_t *lock)
{
ngx_uint_t i, n;
ngx_atomic_uint_t readers;
for ( ;; ) {
readers = *lock;
if (readers != NGX_RWLOCK_WLOCK
&& ngx_atomic_cmp_set(lock, readers, readers + 1))
{
return;
}
if (ngx_ncpu > 1) {
for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) {
for (i = 0; i < n; i++) {
ngx_cpu_pause();
}
readers = *lock;
if (readers != NGX_RWLOCK_WLOCK
&& ngx_atomic_cmp_set(lock, readers, readers + 1))
{
return;
}
}
}
ngx_sched_yield();
}
}
void
ngx_shmtx_lock(ngx_shmtx_t *mtx)
{
ngx_uint_t i, n;
ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx lock");
for ( ;; ) {
if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) {
return;
}
if (ngx_ncpu > 1) {
for (n = 1; n < mtx->spin; n <<= 1) {
for (i = 0; i < n; i++) {
ngx_cpu_pause();
}
if (*mtx->lock == 0
&& ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid))
{
return;
}
}
}
#if (NGX_HAVE_POSIX_SEM)
if (mtx->semaphore) {
(void) ngx_atomic_fetch_add(mtx->wait, 1);
if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) {
(void) ngx_atomic_fetch_add(mtx->wait, -1);
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0,
"shmtx wait %uA", *mtx->wait);
while (sem_wait(&mtx->sem) == -1) {
ngx_err_t err;
err = ngx_errno;
if (err != NGX_EINTR) {
ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, err,
"sem_wait() failed while waiting on shmtx");
break;
}
}
ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0,
"shmtx awoke");
continue;
}
#endif
ngx_sched_yield();
}
}
