/* TODO: add noreturn attribute. */
void _ITM_CALL_CONVENTION _ITM_rollbackTransaction(TX_ARGS
const _ITM_srcLocation *__src)
{
/* TODO check exactly the purpose of this function */
TX_GET_ABI;
stm_rollback(tx, STM_ABORT_EXPLICIT);
}
/* TODO: add noreturn attribute. */
void _ITM_CALL_CONVENTION _ITM_abortTransaction(TX_ARGS
_ITM_abortReason __reason)
{
TX_GET_ABI;
if( __reason == userAbort) {
/* __tm_abort was invoked. */
__reason = STM_ABORT_NO_RETRY;
} else if(__reason == userRetry) {
/* __tm_retry was invoked. */
__reason = STM_ABORT_EXPLICIT;
}
stm_rollback(tx, __reason);
}
/*
* Catch signal (to emulate non-faulting load).
*/
static void signal_catcher(int sig)
{
stm_tx_t *tx = stm_get_tx();
/* A fault might only occur upon a load concurrent with a free (read-after-free) */
PRINT_DEBUG("Caught signal: %d\n", sig);
if (tx == NULL || (tx->attr != NULL && tx->attr->no_retry)) {
/* There is not much we can do: execution will restart at faulty load */
fprintf(stderr, "Error: invalid memory accessed and no longjmp destination\n");
exit(1);
}
tx->aborts_invalid_memory++;
/* Will cause a longjmp */
stm_rollback(tx, STM_ABORT_SIGNAL);
}
/*
* Catch signal (to emulate non-faulting load).
*/
static void
signal_catcher(int sig)
{
sigset_t block_signal;
stm_tx_t *tx = tls_get_tx();
/* A fault might only occur upon a load concurrent with a free (read-after-free) */
PRINT_DEBUG("Caught signal: %d\n", sig);
/* TODO: TX_KILLED should be also allowed */
if (tx == NULL || tx->attr.no_retry || GET_STATUS(tx->status) != TX_ACTIVE) {
/* There is not much we can do: execution will restart at faulty load */
fprintf(stderr, "Error: invalid memory accessed and no longjmp destination\n");
exit(1);
}
/* Unblock the signal since there is no return to signal handler */
sigemptyset(&block_signal);
sigaddset(&block_signal, sig);
pthread_sigmask(SIG_UNBLOCK, &block_signal, NULL);
/* Will cause a longjmp */
stm_rollback(tx, STM_ABORT_SIGNAL);
}
_CALLCONV void
stm_abort_tx(stm_tx_t *tx, int reason)
{
stm_rollback(tx, reason | STM_ABORT_EXPLICIT);
}
/*
* Called by the CURRENT thread to abort a transaction.
*/
_CALLCONV void
stm_abort(int reason)
{
TX_GET;
stm_rollback(tx, reason | STM_ABORT_EXPLICIT);
}
/*
* Set the CURRENT transaction as irrevocable.
*/
static INLINE int
int_stm_set_irrevocable(stm_tx_t *tx, int serial)
{
#ifdef IRREVOCABLE_ENABLED
# if CM == CM_MODULAR
stm_word_t t;
# endif /* CM == CM_MODULAR */
if (!IS_ACTIVE(tx->status) && serial != -1) {
/* Request irrevocability outside of a transaction or in abort handler (for next execution) */
tx->irrevocable = 1 + (serial ? 0x08 : 0);
return 0;
}
/* Are we already in irrevocable mode? */
if ((tx->irrevocable & 0x07) == 3) {
return 1;
}
if (tx->irrevocable == 0) {
/* Acquire irrevocability for the first time */
tx->irrevocable = 1 + (serial ? 0x08 : 0);
#ifdef HYBRID_ASF
/* TODO: we shouldn't use pthread_mutex/cond since it could use syscall. */
if (tx->software == 0) {
asf_abort(ASF_RETRY_IRREVOCABLE);
return 0;
}
#endif /* HYBRID_ASF */
/* Try acquiring global lock */
if (_tinystm.irrevocable == 1 || ATOMIC_CAS_FULL(&_tinystm.irrevocable, 0, 1) == 0) {
/* Transaction will acquire irrevocability after rollback */
stm_rollback(tx, STM_ABORT_IRREVOCABLE);
return 0;
}
/* Success: remember we have the lock */
tx->irrevocable++;
/* Try validating transaction */
#if DESIGN == WRITE_BACK_ETL
if (!stm_wbetl_validate(tx)) {
stm_rollback(tx, STM_ABORT_VALIDATE);
return 0;
}
#elif DESIGN == WRITE_BACK_CTL
if (!stm_wbctl_validate(tx)) {
stm_rollback(tx, STM_ABORT_VALIDATE);
return 0;
}
#elif DESIGN == WRITE_THROUGH
if (!stm_wt_validate(tx)) {
stm_rollback(tx, STM_ABORT_VALIDATE);
return 0;
}
#elif DESIGN == MODULAR
if ((tx->attr.id == WRITE_BACK_CTL && stm_wbctl_validate(tx))
|| (tx->attr.id == WRITE_THROUGH && stm_wt_validate(tx))
|| (tx->attr.id != WRITE_BACK_CTL && tx->attr.id != WRITE_THROUGH && stm_wbetl_validate(tx))) {
stm_rollback(tx, STM_ABORT_VALIDATE);
return 0;
}
#endif /* DESIGN == MODULAR */
# if CM == CM_MODULAR
/* We might still abort if we cannot set status (e.g., we are being killed) */
t = tx->status;
if (GET_STATUS(t) != TX_ACTIVE || ATOMIC_CAS_FULL(&tx->status, t, t + (TX_IRREVOCABLE - TX_ACTIVE)) == 0) {
stm_rollback(tx, STM_ABORT_KILLED);
return 0;
}
# endif /* CM == CM_MODULAR */
if (serial && tx->w_set.nb_entries != 0) {
/* TODO: or commit the transaction when we have the irrevocability. */
/* Don't mix transactional and direct accesses => restart with direct accesses */
stm_rollback(tx, STM_ABORT_IRREVOCABLE);
return 0;
}
} else if ((tx->irrevocable & 0x07) == 1) {
/* Acquire irrevocability after restart (no need to validate) */
while (_tinystm.irrevocable == 1 || ATOMIC_CAS_FULL(&_tinystm.irrevocable, 0, 1) == 0)
;
/* Success: remember we have the lock */
tx->irrevocable++;
}
assert((tx->irrevocable & 0x07) == 2);
/* Are we in serial irrevocable mode? */
if ((tx->irrevocable & 0x08) != 0) {
/* Stop all other threads */
if (stm_quiesce(tx, 1) != 0) {
/* Another thread is quiescing and we are active (trying to acquire irrevocability) */
assert(serial != -1);
stm_rollback(tx, STM_ABORT_IRREVOCABLE);
return 0;
}
}
/* We are in irrevocable mode */
tx->irrevocable++;
#else /* ! IRREVOCABLE_ENABLED */
fprintf(stderr, "Irrevocability is not supported in this configuration\n");
exit(-1);
#endif /* ! IRREVOCABLE_ENABLED */
return 1;
}
/*
* Store a word-sized value (return write set entry or NULL).
*/
static inline w_entry_t *stm_write(stm_tx_t *tx, volatile stm_word_t *addr, stm_word_t value, stm_word_t mask)
{
volatile stm_word_t *lock;
stm_word_t l, version;
w_entry_t *w;
w_entry_t *prev = NULL;
PRINT_DEBUG2("==> stm_write(t=%p[%lu-%lu],a=%p,d=%p-%lu,m=0x%lx)\n",
tx, (unsigned long)tx->start, (unsigned long)tx->end, addr, (void *)value, (unsigned long)value, (unsigned long)mask);
assert(IS_ACTIVE(tx->status));
if (tx->ro) {
/* Disable read-only and abort */
assert(tx->attr != NULL);
/* Update attributes to inform the caller */
tx->attr->read_only = 0;
tx->aborts_ro++;
stm_rollback(tx, STM_ABORT_RO_WRITE);
return NULL;
}
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Try to acquire lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
restart_no_load:
if (LOCK_GET_OWNED(l)) {
/* Locked */
if (l == LOCK_UNIT) {
/* Data modified by a unit store: should not last long => retry */
goto restart;
}
/* Do we own the lock? */
w = (w_entry_t *)LOCK_GET_ADDR(l);
/* Simply check if address falls inside our write set (avoids non-faulting load) */
if (tx->w_set.entries <= w && w < tx->w_set.entries + tx->w_set.nb_entries) {
/* Yes */
if (mask == 0) {
/* No need to insert new entry or modify existing one */
return w;
}
prev = w;
/* Did we previously write the same address? */
while (1) {
if (addr == prev->addr) {
/* No need to add to write set */
if (mask != ~(stm_word_t)0) {
if (prev->mask == 0)
prev->value = ATOMIC_LOAD(addr);
value = (prev->value & ~mask) | (value & mask);
}
prev->value = value;
prev->mask |= mask;
return prev;
}
if (prev->next == NULL) {
/* Remember last entry in linked list (for adding new entry) */
break;
}
prev = prev->next;
}
/* Get version from previous write set entry (all entries in linked list have same version) */
version = prev->version;
/* Must add to write set */
if (tx->w_set.nb_entries == tx->w_set.size)
stm_allocate_ws_entries(tx, 1);
w = &tx->w_set.entries[tx->w_set.nb_entries];
goto do_write;
}
/* Conflict: CM kicks in */
tx->c_lock = lock;
/* Abort */
tx->aborts_locked_write++;
stm_rollback(tx, STM_ABORT_WW_CONFLICT);
return NULL;
}
/* Not locked */
/* Handle write after reads (before CAS) */
version = LOCK_GET_TIMESTAMP(l);
acquire:
if (version > tx->end) {
/* We might have read an older version previously */
if (!tx->can_extend || stm_has_read(tx, lock) != NULL) {
/* Read version must be older (otherwise, tx->end >= version) */
/* Not much we can do: abort */
tx->aborts_validate_write++;
stm_rollback(tx, STM_ABORT_VAL_WRITE);
return NULL;
}
}
/* Acquire lock (ETL) */
if (tx->w_set.nb_entries == tx->w_set.size)
stm_allocate_ws_entries(tx, 1);
w = &tx->w_set.entries[tx->w_set.nb_entries];
if (ATOMIC_CAS_FULL(lock, l, LOCK_SET_ADDR_WRITE((stm_word_t)w)) == 0)
goto restart;
/* We own the lock here (ETL) */
do_write:
/* Add address to write set */
w->addr = addr;
w->mask = mask;
w->lock = lock;
if (mask == 0) {
/* Do not write anything */
#ifndef NDEBUG
w->value = 0;
#endif /* ! NDEBUG */
} else
{
/* Remember new value */
if (mask != ~(stm_word_t)0)
value = (ATOMIC_LOAD(addr) & ~mask) | (value & mask);
w->value = value;
}
w->version = version;
w->next = NULL;
if (prev != NULL) {
/* Link new entry in list */
prev->next = w;
}
tx->w_set.nb_entries++;
tx->w_set.has_writes++;
return w;
}
/*
* Load a word-sized value (invisible read).
*/
static inline stm_word_t stm_read_invisible(stm_tx_t *tx, volatile stm_word_t *addr)
{
volatile stm_word_t *lock;
stm_word_t l, l2, value, version;
r_entry_t *r;
w_entry_t *w;
PRINT_DEBUG2("==> stm_read_invisible(t=%p[%lu-%lu],a=%p)\n", tx, (unsigned long)tx->start, (unsigned long)tx->end, addr);
assert(IS_ACTIVE(tx->status));
/* Get reference to lock */
lock = GET_LOCK(addr);
/* Note: we could check for duplicate reads and get value from read set */
/* Read lock, value, lock */
restart:
l = ATOMIC_LOAD_ACQ(lock);
restart_no_load:
if (LOCK_GET_WRITE(l)) {
/* Locked */
if (l == LOCK_UNIT) {
/* Data modified by a unit store: should not last long => retry */
goto restart;
}
/* Do we own the lock? */
w = (w_entry_t *)LOCK_GET_ADDR(l);
/* Simply check if address falls inside our write set (avoids non-faulting load) */
if (tx->w_set.entries <= w && w < tx->w_set.entries + tx->w_set.nb_entries) {
/* Yes: did we previously write the same address? */
while (1) {
if (addr == w->addr) {
/* Yes: get value from write set (or from memory if mask was empty) */
value = (w->mask == 0 ? ATOMIC_LOAD(addr) : w->value);
break;
}
if (w->next == NULL) {
/* No: get value from memory */
value = ATOMIC_LOAD(addr);
break;
}
w = w->next;
}
/* No need to add to read set (will remain valid) */
return value;
}
/* Conflict: CM kicks in (we could also check for duplicate reads and get value from read set) */
tx->c_lock = lock;
/* Abort */
tx->aborts_locked_read++;
stm_rollback(tx, STM_ABORT_RW_CONFLICT);
return 0;
} else {
/* Not locked */
value = ATOMIC_LOAD_ACQ(addr);
l2 = ATOMIC_LOAD_ACQ(lock);
if (l != l2) {
l = l2;
goto restart_no_load;
}
/* Check timestamp */
version = LOCK_GET_TIMESTAMP(l);
/* Valid version? */
if (version > tx->end) {
/* No: try to extend first (except for read-only transactions: no read set) */
if (tx->ro || !tx->can_extend || !stm_extend(tx)) {
/* Not much we can do: abort */
tx->aborts_validate_read++;
stm_rollback(tx, STM_ABORT_VAL_READ);
return 0;
}
/* Verify that version has not been overwritten (read value has not
* yet been added to read set and may have not been checked during
* extend) */
l = ATOMIC_LOAD_ACQ(lock);
if (l != l2) {
l = l2;
goto restart_no_load;
}
/* Worked: we now have a good version (version <= tx->end) */
}
}
/* We have a good version: add to read set (update transactions) and return value */
if (!tx->ro) {
/* Add address and version to read set */
if (tx->r_set.nb_entries == tx->r_set.size)
stm_allocate_rs_entries(tx, 1);
r = &tx->r_set.entries[tx->r_set.nb_entries++];
r->version = version;
r->lock = lock;
}
return value;
}
/*
* Called by the CURRENT thread to abort a transaction.
*/
void stm_abort(TXPARAMS int reason)
{
TX_GET;
stm_rollback(tx, reason | STM_ABORT_EXPLICIT);
}
/*
* Called by the CURRENT thread to commit a transaction.
*/
int stm_commit(TXPARAM)
{
w_entry_t *w;
stm_word_t t;
int i;
TX_GET;
PRINT_DEBUG("==> stm_commit(%p[%lu-%lu])\n", tx, (unsigned long)tx->start, (unsigned long)tx->end);
/* Decrement nesting level */
if (--tx->nesting > 0)
return 1;
assert(IS_ACTIVE(tx->status));
/* A read-only transaction can commit immediately */
if (tx->w_set.nb_entries == 0)
goto end;
/* Update transaction */
/* Get commit timestamp (may exceed VERSION_MAX by up to MAX_THREADS) */
t = FETCH_INC_CLOCK + 1;
/* Try to validate (only if a concurrent transaction has committed since tx->start) */
if (tx->start != t - 1 && !stm_validate(tx)) {
/* Cannot commit */
tx->aborts_validate_commit++;
stm_rollback(tx, STM_ABORT_VALIDATE);
return 0;
}
/* Install new versions, drop locks and set new timestamp */
w = tx->w_set.entries;
for (i = tx->w_set.nb_entries; i > 0; i--, w++) {
if (w->mask != 0)
ATOMIC_STORE(w->addr, w->value);
/* Only drop lock for last covered address in write set */
if (w->next == NULL)
ATOMIC_STORE_REL(w->lock, LOCK_SET_TIMESTAMP(t));
}
end:
tx->retries = 0;
/* Callbacks */
if (nb_commit_cb != 0) {
int cb;
for (cb = 0; cb < nb_commit_cb; cb++)
commit_cb[cb].f(TXARGS commit_cb[cb].arg);
}
/* Set status to COMMITTED */
SET_STATUS(tx->status, TX_COMMITTED);
return 1;
}
