Exemple #1
0
/* CAS-less unlock, not quite as efficient and will make sure every vcore runs
 * (since we don't have a convenient way to make sure our qnode->next runs
 * yet, other than making sure everyone runs).
 *
 * To use this without ensuring all vcores run, you'll need the unlock code to
 * save pred to a specific field in the qnode and check both its initial pred
 * as well as its run time pred (who could be an usurper).  It's all possible,
 * but a little more difficult to follow.  Also, I'm adjusting this comment
 * months after writing it originally, so it is probably not sufficient, but
 * necessary. */
void __mcs_pdro_unlock_no_cas(struct mcs_pdro_lock *lock,
                             struct mcs_pdro_qnode *qnode)
{
	struct mcs_pdro_qnode *old_tail, *usurper;
	/* Check if someone is already waiting on us to unlock */
	if (qnode->next == 0) {
		cmb();	/* no need for CPU mbs, since there's an atomic_swap() */
		/* Unlock it */
		old_tail = atomic_swap_ptr((void**)&lock->lock, 0);
		/* no one else was already waiting, so we successfully unlocked and can
		 * return */
		if (old_tail == qnode)
			return;
		/* someone else was already waiting on the lock (last one on the list),
		 * and we accidentally took them off.  Try and put it back. */
		usurper = atomic_swap_ptr((void*)&lock->lock, old_tail);
		/* since someone else was waiting, they should have made themselves our
		 * next.  spin (very briefly!) til it happens. */
		while (qnode->next == 0) {
			/* make sure old_tail isn't preempted.  best we can do for now is
			 * to make sure all vcores run, and thereby get our next. */
			for (int i = 0; i < max_vcores(); i++)
				ensure_vcore_runs(i);
			cpu_relax();
		}
		if (usurper) {
			/* an usurper is someone who snuck in before we could put the old
			 * tail back.  They now have the lock.  Let's put whoever is
			 * supposed to be next as their next one. 
			 *
			 * First, we need to change our next's pred.  There's a slight race
			 * here, so our next will need to make sure both us and pred are
			 * done */
			/* I was trying to do something so we didn't need to ensure all
			 * vcores run, using more space in the qnode to figure out who our
			 * pred was a lock time (guessing actually, since there's a race,
			 * etc). */
			//qnode->next->pred = usurper;
			//wmb();
			usurper->next = qnode->next;
			/* could imagine another wmb() and a flag so our next knows to no
			 * longer check us too. */
		} else {
			/* No usurper meant we put things back correctly, so we should just
			 * pass the lock / unlock whoever is next */
			qnode->next->locked = 0;
		}
	} else {
		/* mb()s necessary since we didn't call an atomic_swap() */
		wmb();	/* need to make sure any previous writes don't pass unlocking */
		rwmb();	/* need to make sure any reads happen before the unlocking */
		/* simply unlock whoever is next */
		qnode->next->locked = 0;
	}
}
Exemple #2
0
/* Internal version of the locking function, doesn't care if notifs are
 * disabled.  While spinning, we'll check to see if other vcores involved in the
 * locking are running.  If we change to that vcore, we'll continue when our
 * vcore gets restarted.  If the change fails, it is because the vcore is
 * running, and we'll continue.
 *
 * It's worth noting that changing to another vcore won't hurt correctness.
 * Even if they are no longer the lockholder, they will be checking preemption
 * messages and will help break out of the deadlock.  So long as we don't
 * spin uncontrollably, we're okay. */
void __mcs_pdro_lock(struct mcs_pdro_lock *lock, struct mcs_pdro_qnode *qnode)
{
	struct mcs_pdro_qnode *predecessor;
	uint32_t pred_vcoreid;
	/* Now the actual lock */
	qnode->next = 0;
	cmb();	/* swap provides a CPU mb() */
	predecessor = atomic_swap_ptr((void**)&lock->lock, qnode);
	if (predecessor) {
		qnode->locked = 1;
		/* Read-in the vcoreid before releasing them.  We won't need to worry
		 * about their qnode memory being freed/reused (they can't til we fill
		 * in the 'next' slot), which is a bit of a performance win.  This also
		 * cuts down on cache-line contention when we ensure they run, which
		 * helps a lot too. */
		pred_vcoreid = ACCESS_ONCE(predecessor->vcoreid);
		wmb();	/* order the locked write before the next write */
		predecessor->next = qnode;
		/* no need for a wrmb(), since this will only get unlocked after they
		 * read our previous write */
		while (qnode->locked) {
			/* We don't know who the lock holder is (it hurts performance via
			 * 'true' sharing to track it)  Instead we'll make sure our pred is
			 * running, which trickles up to the lock holder. */
			ensure_vcore_runs(pred_vcoreid);
			cpu_relax();
		}
	}
}
static inline void
rw_swap(krwlock_t *rw, uintptr_t o, uintptr_t n)
{

	RW_INHERITDEBUG(n, o);
	n = (uintptr_t)atomic_swap_ptr((volatile void *)&rw->rw_owner,
	    (void *)n);
	RW_DASSERT(rw, n == o);
}
Exemple #4
0
void *wfl_remove(struct wfl *list)
{
    for (struct wfl_entry *p = list->head; p != NULL; p = p->next) {
        if (p->data != NULL) {
            void *data = atomic_swap_ptr(&p->data, 0);
            if (data != NULL)
                return data;
        }
    }
    return NULL;
}
Exemple #5
0
void
rump_unschedule_cpu1(struct lwp *l, void *interlock)
{
    struct rumpcpu *rcpu;
    struct cpu_info *ci;
    void *old;

    ci = l->l_cpu;
    ci->ci_curlwp = ci->ci_data.cpu_onproc = NULL;
    rcpu = cpuinfo_to_rumpcpu(ci);

    KASSERT(rcpu->rcpu_ci == ci);

    /*
     * Make sure all stores are seen before the CPU release.  This
     * is relevant only in the non-fastpath scheduling case, but
     * we don't know here if that's going to happen, so need to
     * expect the worst.
     *
     * If the scheduler interlock was requested by the caller, we
     * need to obtain it before we release the CPU.  Otherwise, we risk a
     * race condition where another thread is scheduled onto the
     * rump kernel CPU before our current thread can
     * grab the interlock.
     */
    if (interlock == rcpu->rcpu_mtx)
        rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
    else
        membar_exit();

    /* Release the CPU. */
    old = atomic_swap_ptr(&rcpu->rcpu_prevlwp, l);

    /* No waiters?  No problems.  We're outta here. */
    if (old == RCPULWP_BUSY) {
        return;
    }

    KASSERT(old == RCPULWP_WANTED);

    /*
     * Ok, things weren't so snappy.
     *
     * Snailpath: take lock and signal anyone waiting for this CPU.
     */

    if (interlock != rcpu->rcpu_mtx)
        rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
    if (rcpu->rcpu_wanted)
        rumpuser_cv_broadcast(rcpu->rcpu_cv);
    if (interlock != rcpu->rcpu_mtx)
        rumpuser_mutex_exit(rcpu->rcpu_mtx);
}
Exemple #6
0
static void
pefs_aesni_uninit(struct pefs_alg *pa)
{
	struct fpu_kern_ctx *fpu_ctx;
	u_int cpuid;

	CPU_FOREACH(cpuid) {
		fpu_ctx = (void *)atomic_swap_ptr(
		    (volatile void *)DPCPU_ID_PTR(cpuid, pefs_aesni_fpu),
		    (uintptr_t)NULL);
		if (fpu_ctx != NULL)
			fpu_kern_free_ctx(fpu_ctx);
	}
}
Exemple #7
0
void lt_rwlock_write_unlock(lt_rwlock_t * lock) 
{
	lt_thread_list_t * place_holder = NULL;

	/* As of this point, arriving readers should go into readers[1], the
	 * readers in readers[0] should go there too, and readers[1] should 
	 * be empty.
	 * Hence, we us place-holder lists for arriving readers, which we will
	 * add to readers[1] once we're sure the scheduler will take the proper
	 * list place_holder: empty list of threads;
	 *
	 * we assume readers[1] is empty */
	assert(lt_thread_list_empty(lock->readers[1]));
	/* so we put the place-holder at readers[0] */
	atomic_swap_ptr(lock->readers[0], place_holder);
	/* place_holder now contains readers[0]
	 * we now de-queue ourselves */
	assert(lt_thread_eq(lt_thread_queue_deq(lock->writers), lt_thread_self()) == 0);
	/* if the writers queue is empty, it will schedule into readers[1];
	 * if not, it will continue scheduling into readers[0], which is now the
	 * place-holder. */
	/* we add whatever we swapped out to readers[1] */
	lt_thread_list_move(lock->readers[1], place_holder);
	/* we swap again. As readers[0] only contains sleeping threads, we're sure
	 * all the threads in there are asleep and won't think they're in
	 * readers[1] */
	atomic_swap_ptr(place_holder, lock->readers[0]);
	/* we move whatever we got from readers[0] between the time put the 
	 * place-holder in place and the time we removed ourselves from the writers
	 * queue (which may have made it possible to schedule into readers[1]
	 * to readers[1]. If the scheduler is still scheduling into readers[0],
	 * these threads just got lucky */
	lt_thread_list_move(lock->readers[1], place_holder);
	/* now, we awake everyone in readers[1] */
	lt_thread_list_for_each(lock->readers[1], lt_thread_wake);
	/* and we're done */
}
Exemple #8
0
/* Similar to the original PDR lock, this tracks the lockholder for better
 * recovery from preemptions.  Under heavy contention, changing to the
 * lockholder instead of pred makes it more likely to have a vcore outside the
 * MCS chain handle the preemption.  If that never happens, performance will
 * suffer.
 *
 * Simply checking the lockholder causes a lot of unnecessary traffic, so we
 * first look for signs of preemption in read-mostly locations (by comparison,
 * the lockholder changes on every lock/unlock).
 *
 * We also use the "qnodes are in the lock" style, which is slightly slower than
 * using the stack in regular MCS/MCSPDR locks, but it speeds PDR up a bit by
 * not having to read other qnodes' memory to determine their vcoreid.  The
 * slowdown may be due to some weird caching/prefetch settings (like Adjacent
 * Cacheline Prefetch).
 *
 * Note that these locks, like all PDR locks, have opportunities to accidentally
 * ensure some vcore runs that isn't in the chain.  Whenever we read lockholder
 * or even pred, that particular vcore might subsequently unlock and then get
 * preempted (or change_to someone else) before we ensure they run.  If this
 * happens and there is another VC in the MCS chain, it will make sure the right
 * cores run.  If there are no other vcores in the chain, it is up to the rest
 * of the vcore/event handling system to deal with this, which should happen
 * when one of the other vcores handles the preemption message generated by our
 * change_to. */
void __mcs_pdr_lock(struct mcs_pdr_lock *lock, struct mcs_pdr_qnode *qnode)
{
	struct mcs_pdr_qnode *predecessor;
	uint32_t pred_vcoreid;
	struct mcs_pdr_qnode *qnode0 = qnode - vcore_id();
	seq_ctr_t seq;
	qnode->next = 0;
	cmb();	/* swap provides a CPU mb() */
	predecessor = atomic_swap_ptr((void**)&lock->lock, qnode);
	if (predecessor) {
		qnode->locked = 1;
		pred_vcoreid = predecessor - qnode0;	/* can compute this whenever */
		wmb();	/* order the locked write before the next write */
		predecessor->next = qnode;
		seq = ACCESS_ONCE(__procinfo.coremap_seqctr);
		/* no need for a wrmb(), since this will only get unlocked after they
		 * read our pred->next write */
		while (qnode->locked) {
			/* Check to see if anything is amiss.  If someone in the chain is
			 * preempted, then someone will notice.  Simply checking our pred
			 * isn't that great of an indicator of preemption.  The reason is
			 * that the offline vcore is most likely the lockholder (under heavy
			 * lock contention), and we want someone farther back in the chain
			 * to notice (someone that will stay preempted long enough for a
			 * vcore outside the chain to recover them).  Checking the seqctr
			 * will tell us of any preempts since we started, so if a storm
			 * starts while we're spinning, we can join in and try to save the
			 * lockholder before its successor gets it.
			 *
			 * Also, if we're the lockholder, then we need to let our pred run
			 * so they can hand us the lock. */
			if (vcore_is_preempted(pred_vcoreid) ||
			    seq != __procinfo.coremap_seqctr) {
				if (lock->lockholder_vcoreid == MCSPDR_NO_LOCKHOLDER ||
				    lock->lockholder_vcoreid == vcore_id())
					ensure_vcore_runs(pred_vcoreid);
				else
					ensure_vcore_runs(lock->lockholder_vcoreid);
			}
			cpu_relax();
		}
	} else {
		lock->lockholder_vcoreid = vcore_id();
	}
}
        //  Perform atomic 'exchange pointers' operation. Pointer is set
        //  to the 'val' value. Old value is returned.
        inline T *xchg (T *val_)
        {
#if defined ZMQ_ATOMIC_PTR_WINDOWS
            return (T*) InterlockedExchangePointer ((PVOID*) &ptr, val_);
#elif defined ZMQ_ATOMIC_PTR_INTRINSIC
            return (T*) __atomic_exchange_n (&ptr, val_, __ATOMIC_ACQ_REL);
#elif defined ZMQ_ATOMIC_PTR_CXX11
            return ptr.exchange(val_, std::memory_order_acq_rel);
#elif defined ZMQ_ATOMIC_PTR_ATOMIC_H
            return (T*) atomic_swap_ptr (&ptr, val_);
#elif defined ZMQ_ATOMIC_PTR_TILE
            return (T*) arch_atomic_exchange (&ptr, val_);
#elif defined ZMQ_ATOMIC_PTR_X86
            T *old;
            __asm__ volatile (
                "lock; xchg %0, %2"
                : "=r" (old), "=m" (ptr)
                : "m" (ptr), "0" (val_));
            return old;
#elif defined ZMQ_ATOMIC_PTR_ARM
            T* old;
            unsigned int flag;
            __asm__ volatile (
                "       dmb     sy\n\t"
                "1:     ldrex   %1, [%3]\n\t"
                "       strex   %0, %4, [%3]\n\t"
                "       teq     %0, #0\n\t"
                "       bne     1b\n\t"
                "       dmb     sy\n\t"
                : "=&r"(flag), "=&r"(old), "+Qo"(ptr)
                : "r"(&ptr), "r"(val_)
                : "cc");
            return old;
#elif defined ZMQ_ATOMIC_PTR_MUTEX
            sync.lock ();
            T *old = (T*) ptr;
            ptr = val_;
            sync.unlock ();
            return old;
#else
#error atomic_ptr is not implemented for this platform
#endif
        }
Exemple #10
0
static void
pefs_aesni_enter(struct pefs_session *xses)
{
	struct pefs_aesni_ses *ses = &xses->o.ps_aesni;

	if (is_fpu_kern_thread(0)) {
		ses->fpu_saved = 0;
		return;
	}

	critical_enter();
	ses->fpu_ctx = (void *)atomic_swap_ptr(
	    (volatile void *)DPCPU_PTR(pefs_aesni_fpu), (uintptr_t)NULL);
	if (ses->fpu_ctx != NULL) {
		ses->td = curthread;
		ses->fpu_cpuid = curcpu;
		fpu_kern_enter(ses->td, ses->fpu_ctx, FPU_KERN_NORMAL);
		ses->fpu_saved = 1;
	} else {
		ses->fpu_saved = -1;
	}
	critical_exit();
}
Exemple #11
0
        //  Perform atomic 'exchange pointers' operation. Pointer is set
        //  to the 'val' value. Old value is returned.
        inline T *xchg (T *val_)
        {
#if defined XS_ATOMIC_PTR_WINDOWS
            return (T*) InterlockedExchangePointer ((PVOID*) &ptr, val_);
#elif defined XS_ATOMIC_PTR_ATOMIC_H
            return (T*) atomic_swap_ptr (&ptr, val_);
#elif defined XS_ATOMIC_PTR_X86
            T *old;
            __asm__ volatile (
                "lock; xchg %0, %2"
                : "=r" (old), "=m" (ptr)
                : "m" (ptr), "0" (val_));
            return old;
#elif defined XS_ATOMIC_PTR_MUTEX
            sync.lock ();
            T *old = (T*) ptr;
            ptr = val_;
            sync.unlock ();
            return old;
#else
#error atomic_ptr is not implemented for this platform
#endif
        }
Exemple #12
0
/*
 * Acquire a lock waiting (spin or sleep) for it to become available.
 */
void
_lock_acquire(struct lock *lck, struct lockuser *lu, int prio)
{
	int i;
	int lval;

	/**
	 * XXX - We probably want to remove these checks to optimize
	 *       performance.  It is also a bug if any one of the 
	 *       checks fail, so it's probably better to just let it
	 *       SEGV and fix it.
	 */
#if 0
	if (lck == NULL || lu == NULL || lck->l_head == NULL)
		return;
#endif
	if ((lck->l_type & LCK_PRIORITY) != 0) {
		LCK_ASSERT(lu->lu_myreq->lr_locked == 1);
		LCK_ASSERT(lu->lu_myreq->lr_watcher == NULL);
		LCK_ASSERT(lu->lu_myreq->lr_owner == lu);
		LCK_ASSERT(lu->lu_watchreq == NULL);

		lu->lu_priority = prio;
	}
	/*
	 * Atomically swap the head of the lock request with
	 * this request.
	 */
	atomic_swap_ptr((void *)&lck->l_head, lu->lu_myreq,
	    (void *)&lu->lu_watchreq);

	if (lu->lu_watchreq->lr_locked != 0) {
		atomic_store_rel_ptr
		    ((volatile uintptr_t *)(void *)&lu->lu_watchreq->lr_watcher,
		    (uintptr_t)lu);
		if ((lck->l_wait == NULL) ||
		    ((lck->l_type & LCK_ADAPTIVE) == 0)) {
			while (lu->lu_watchreq->lr_locked != 0)
				;	/* spin, then yield? */
		} else {
			/*
			 * Spin for a bit before invoking the wait function.
			 *
			 * We should be a little smarter here.  If we're
			 * running on a single processor, then the lock
			 * owner got preempted and spinning will accomplish
			 * nothing but waste time.  If we're running on
			 * multiple processors, the owner could be running
			 * on another CPU and we might acquire the lock if
			 * we spin for a bit.
			 *
			 * The other thing to keep in mind is that threads
			 * acquiring these locks are considered to be in
			 * critical regions; they will not be preempted by
			 * the _UTS_ until they release the lock.  It is
			 * therefore safe to assume that if a lock can't
			 * be acquired, it is currently held by a thread
			 * running in another KSE.
			 */
			for (i = 0; i < MAX_SPINS; i++) {
				if (lu->lu_watchreq->lr_locked == 0)
					return;
				if (lu->lu_watchreq->lr_active == 0)
					break;
			}
			atomic_swap_int(&lu->lu_watchreq->lr_locked,
			    2, &lval);
			if (lval == 0)
				lu->lu_watchreq->lr_locked = 0;
			else
				lck->l_wait(lck, lu);

		}
	}
	lu->lu_myreq->lr_active = 1;
}
Exemple #13
0
static inline mcs_lock_qnode_t *mcs_qnode_swap(mcs_lock_qnode_t **addr,
                                               mcs_lock_qnode_t *val)
{
	return (mcs_lock_qnode_t*)atomic_swap_ptr((void**)addr, val);
}
Exemple #14
0
void* atomicExchange(void* volatile& val, void* exch)
{
    return atomic_swap_ptr(&val, exch);
}
Exemple #15
0
/*
 * Schedule a CPU.  This optimizes for the case where we schedule
 * the same thread often, and we have nCPU >= nFrequently-Running-Thread
 * (where CPU is virtual rump cpu, not host CPU).
 */
void
rump_schedule_cpu_interlock(struct lwp *l, void *interlock)
{
    struct rumpcpu *rcpu;
    struct cpu_info *ci;
    void *old;
    bool domigrate;
    bool bound = l->l_pflag & LP_BOUND;

    l->l_stat = LSRUN;

    /*
     * First, try fastpath: if we were the previous user of the
     * CPU, everything is in order cachewise and we can just
     * proceed to use it.
     *
     * If we are a different thread (i.e. CAS fails), we must go
     * through a memory barrier to ensure we get a truthful
     * view of the world.
     */

    KASSERT(l->l_target_cpu != NULL);
    rcpu = cpuinfo_to_rumpcpu(l->l_target_cpu);
    if (atomic_cas_ptr(&rcpu->rcpu_prevlwp, l, RCPULWP_BUSY) == l) {
        if (interlock == rcpu->rcpu_mtx)
            rumpuser_mutex_exit(rcpu->rcpu_mtx);
        SCHED_FASTPATH(rcpu);
        /* jones, you're the man */
        goto fastlane;
    }

    /*
     * Else, it's the slowpath for us.  First, determine if we
     * can migrate.
     */
    if (ncpu == 1)
        domigrate = false;
    else
        domigrate = true;

    /* Take lock.  This acts as a load barrier too. */
    if (interlock != rcpu->rcpu_mtx)
        rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);

    for (;;) {
        SCHED_SLOWPATH(rcpu);
        old = atomic_swap_ptr(&rcpu->rcpu_prevlwp, RCPULWP_WANTED);

        /* CPU is free? */
        if (old != RCPULWP_BUSY && old != RCPULWP_WANTED) {
            if (atomic_cas_ptr(&rcpu->rcpu_prevlwp,
                               RCPULWP_WANTED, RCPULWP_BUSY) == RCPULWP_WANTED) {
                break;
            }
        }

        /*
         * Do we want to migrate once?
         * This may need a slightly better algorithm, or we
         * might cache pingpong eternally for non-frequent
         * threads.
         */
        if (domigrate && !bound) {
            domigrate = false;
            SCHED_MIGRATED(rcpu);
            rumpuser_mutex_exit(rcpu->rcpu_mtx);
            rcpu = getnextcpu();
            rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
            continue;
        }

        /* Want CPU, wait until it's released an retry */
        rcpu->rcpu_wanted++;
        rumpuser_cv_wait_nowrap(rcpu->rcpu_cv, rcpu->rcpu_mtx);
        rcpu->rcpu_wanted--;
    }
    rumpuser_mutex_exit(rcpu->rcpu_mtx);

fastlane:
    ci = rcpu->rcpu_ci;
    l->l_cpu = l->l_target_cpu = ci;
    l->l_mutex = rcpu->rcpu_ci->ci_schedstate.spc_mutex;
    l->l_ncsw++;
    l->l_stat = LSONPROC;

    /*
     * No interrupts, so ci_curlwp === cpu_onproc.
     * Okay, we could make an attempt to not set cpu_onproc
     * in the case that an interrupt is scheduled immediately
     * after a user proc, but leave that for later.
     */
    ci->ci_curlwp = ci->ci_data.cpu_onproc = l;
}