/*
* Validate read set (check if all read addresses are still valid now).
*/
static inline int stm_validate(stm_tx_t *tx)
{
r_entry_t *r;
int i;
stm_word_t l;
PRINT_DEBUG("==> stm_validate(%p[%lu-%lu])\n", tx, (unsigned long)tx->start, (unsigned long)tx->end);
/* Validate reads */
r = tx->r_set.entries;
for (i = tx->r_set.nb_entries; i > 0; i--, r++) {
/* Read lock */
l = ATOMIC_LOAD(r->lock);
/* Unlocked and still the same version? */
if (LOCK_GET_OWNED(l)) {
/* Do we own the lock? */
w_entry_t *w = (w_entry_t *)LOCK_GET_ADDR(l);
/* Simply check if address falls inside our write set (avoids non-faulting load) */
if (!(tx->w_set.entries <= w && w < tx->w_set.entries + tx->w_set.nb_entries))
{
/* Locked by another transaction: cannot validate */
return 0;
}
/* We own the lock: OK */
} else {
if (LOCK_GET_TIMESTAMP(l) != r->version) {
/* Other version: cannot validate */
return 0;
}
/* Same version: OK */
}
}
return 1;
}
/*
* Called by the CURRENT thread to load a word-sized value in a unit transaction.
*/
stm_word_t stm_unit_load(volatile stm_word_t *addr, stm_word_t *timestamp)
{
volatile stm_word_t *lock;
stm_word_t l, l2, value;
PRINT_DEBUG2("==> stm_unit_load(a=%p)\n", addr);
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Read lock, value, lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
restart_no_load:
if (LOCK_GET_OWNED(l)) {
/* Locked: wait until lock is free */
sched_yield();
goto restart;
}
/* Not locked */
value = ATOMIC_LOAD_ACQ(addr);
l2 = ATOMIC_LOAD_ACQ(lock);
if (l != l2) {
l = l2;
goto restart_no_load;
}
if (timestamp != NULL)
*timestamp = LOCK_GET_TIMESTAMP(l);
return value;
}
/*
* Store a word-sized value in a unit transaction.
*/
static INLINE int
stm_unit_write(volatile stm_word_t *addr, stm_word_t value, stm_word_t mask, stm_word_t *timestamp)
{
#ifdef UNIT_TX
volatile stm_word_t *lock;
stm_word_t l;
PRINT_DEBUG2("==> stm_unit_write(a=%p,d=%p-%lu,m=0x%lx)\n",
addr, (void *)value, (unsigned long)value, (unsigned long)mask);
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Try to acquire lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
if (LOCK_GET_OWNED(l)) {
/* Locked: wait until lock is free */
#ifdef WAIT_YIELD
sched_yield();
#endif /* WAIT_YIELD */
goto restart;
}
/* Not locked */
if (timestamp != NULL && LOCK_GET_TIMESTAMP(l) > *timestamp) {
/* Return current timestamp */
*timestamp = LOCK_GET_TIMESTAMP(l);
return 0;
}
/* TODO: would need to store thread ID to be able to kill it (for wait freedom) */
if (ATOMIC_CAS_FULL(lock, l, LOCK_UNIT) == 0)
goto restart;
ATOMIC_STORE(addr, value);
/* Update timestamp with newer value (may exceed VERSION_MAX by up to MAX_THREADS) */
l = FETCH_INC_CLOCK + 1;
if (timestamp != NULL)
*timestamp = l;
/* Make sure that lock release becomes visible */
ATOMIC_STORE_REL(lock, LOCK_SET_TIMESTAMP(l));
if (unlikely(l >= VERSION_MAX)) {
/* Block all transactions and reset clock (current thread is not in active transaction) */
stm_quiesce_barrier(NULL, rollover_clock, NULL);
}
return 1;
#else /* ! UNIT_TX */
fprintf(stderr, "Unit transaction is not enabled\n");
exit(-1);
return 1;
#endif /* ! UNIT_TX */
}
/*
* Called by the CURRENT thread to load a word-sized value in a unit transaction.
*/
_CALLCONV stm_word_t
stm_unit_load(volatile stm_word_t *addr, stm_word_t *timestamp)
{
#ifdef UNIT_TX
volatile stm_word_t *lock;
stm_word_t l, l2, value;
PRINT_DEBUG2("==> stm_unit_load(a=%p)\n", addr);
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Read lock, value, lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
restart_no_load:
if (LOCK_GET_OWNED(l)) {
/* Locked: wait until lock is free */
#ifdef WAIT_YIELD
sched_yield();
#endif /* WAIT_YIELD */
goto restart;
}
/* Not locked */
value = ATOMIC_LOAD_ACQ(addr);
l2 = ATOMIC_LOAD_ACQ(lock);
if (l != l2) {
l = l2;
goto restart_no_load;
}
if (timestamp != NULL)
*timestamp = LOCK_GET_TIMESTAMP(l);
return value;
#else /* ! UNIT_TX */
fprintf(stderr, "Unit transaction is not enabled\n");
exit(-1);
return 1;
#endif /* ! UNIT_TX */
}
/*
* Store a word-sized value (return write set entry or NULL).
*/
static inline w_entry_t *stm_write(stm_tx_t *tx, volatile stm_word_t *addr, stm_word_t value, stm_word_t mask)
{
volatile stm_word_t *lock;
stm_word_t l, version;
w_entry_t *w;
w_entry_t *prev = NULL;
PRINT_DEBUG2("==> stm_write(t=%p[%lu-%lu],a=%p,d=%p-%lu,m=0x%lx)\n",
tx, (unsigned long)tx->start, (unsigned long)tx->end, addr, (void *)value, (unsigned long)value, (unsigned long)mask);
assert(IS_ACTIVE(tx->status));
if (tx->ro) {
/* Disable read-only and abort */
assert(tx->attr != NULL);
/* Update attributes to inform the caller */
tx->attr->read_only = 0;
tx->aborts_ro++;
stm_rollback(tx, STM_ABORT_RO_WRITE);
return NULL;
}
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Try to acquire lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
restart_no_load:
if (LOCK_GET_OWNED(l)) {
/* Locked */
if (l == LOCK_UNIT) {
/* Data modified by a unit store: should not last long => retry */
goto restart;
}
/* Do we own the lock? */
w = (w_entry_t *)LOCK_GET_ADDR(l);
/* Simply check if address falls inside our write set (avoids non-faulting load) */
if (tx->w_set.entries <= w && w < tx->w_set.entries + tx->w_set.nb_entries) {
/* Yes */
if (mask == 0) {
/* No need to insert new entry or modify existing one */
return w;
}
prev = w;
/* Did we previously write the same address? */
while (1) {
if (addr == prev->addr) {
/* No need to add to write set */
if (mask != ~(stm_word_t)0) {
if (prev->mask == 0)
prev->value = ATOMIC_LOAD(addr);
value = (prev->value & ~mask) | (value & mask);
}
prev->value = value;
prev->mask |= mask;
return prev;
}
if (prev->next == NULL) {
/* Remember last entry in linked list (for adding new entry) */
break;
}
prev = prev->next;
}
/* Get version from previous write set entry (all entries in linked list have same version) */
version = prev->version;
/* Must add to write set */
if (tx->w_set.nb_entries == tx->w_set.size)
stm_allocate_ws_entries(tx, 1);
w = &tx->w_set.entries[tx->w_set.nb_entries];
goto do_write;
}
/* Conflict: CM kicks in */
tx->c_lock = lock;
/* Abort */
tx->aborts_locked_write++;
stm_rollback(tx, STM_ABORT_WW_CONFLICT);
return NULL;
}
/* Not locked */
/* Handle write after reads (before CAS) */
version = LOCK_GET_TIMESTAMP(l);
acquire:
if (version > tx->end) {
/* We might have read an older version previously */
if (!tx->can_extend || stm_has_read(tx, lock) != NULL) {
/* Read version must be older (otherwise, tx->end >= version) */
/* Not much we can do: abort */
tx->aborts_validate_write++;
stm_rollback(tx, STM_ABORT_VAL_WRITE);
return NULL;
}
}
/* Acquire lock (ETL) */
if (tx->w_set.nb_entries == tx->w_set.size)
stm_allocate_ws_entries(tx, 1);
w = &tx->w_set.entries[tx->w_set.nb_entries];
if (ATOMIC_CAS_FULL(lock, l, LOCK_SET_ADDR_WRITE((stm_word_t)w)) == 0)
goto restart;
/* We own the lock here (ETL) */
do_write:
/* Add address to write set */
w->addr = addr;
w->mask = mask;
w->lock = lock;
if (mask == 0) {
/* Do not write anything */
#ifndef NDEBUG
w->value = 0;
#endif /* ! NDEBUG */
} else
{
/* Remember new value */
if (mask != ~(stm_word_t)0)
value = (ATOMIC_LOAD(addr) & ~mask) | (value & mask);
w->value = value;
}
w->version = version;
w->next = NULL;
if (prev != NULL) {
/* Link new entry in list */
prev->next = w;
}
tx->w_set.nb_entries++;
tx->w_set.has_writes++;
return w;
}
/*
* Rollback transaction.
*/
static inline void stm_rollback(stm_tx_t *tx, int reason)
{
w_entry_t *w;
int i;
PRINT_DEBUG("==> stm_rollback(%p[%lu-%lu])\n", tx, (unsigned long)tx->start, (unsigned long)tx->end);
assert(IS_ACTIVE(tx->status));
/* Drop locks */
i = tx->w_set.nb_entries;
if (i > 0) {
w = tx->w_set.entries;
for (; i > 0; i--, w++) {
if (w->next == NULL) {
/* Only drop lock for last covered address in write set */
ATOMIC_STORE(w->lock, LOCK_SET_TIMESTAMP(w->version));
}
}
/* Make sure that all lock releases become visible */
ATOMIC_MB_WRITE;
}
tx->retries++;
tx->aborts++;
if (tx->retries == 1)
tx->aborts_1++;
else if (tx->retries == 2)
tx->aborts_2++;
if (tx->max_retries < tx->retries)
tx->max_retries = tx->retries;
/* Callbacks */
if (nb_abort_cb != 0) {
int cb;
for (cb = 0; cb < nb_abort_cb; cb++)
abort_cb[cb].f(TXARGS abort_cb[cb].arg);
}
/* Set status to ABORTED */
SET_STATUS(tx->status, TX_ABORTED);
/* Reset nesting level */
tx->nesting = 1;
/* Wait until contented lock is free */
if (tx->c_lock != NULL) {
/* Busy waiting (yielding is expensive) */
while (LOCK_GET_OWNED(ATOMIC_LOAD(tx->c_lock))) {
sched_yield();
}
tx->c_lock = NULL;
}
/* Reset field to restart transaction */
stm_prepare(tx);
/* Jump back to transaction start */
if (tx->attr == NULL || !tx->attr->no_retry)
siglongjmp(tx->env, reason);
}