bool_t bst_help_relocate(operation_t* op, node_t* pred, operation_t* pred_op, node_t* curr, node_t* root){
CLEANUP_TRY();
int seen_state = op->relocate_op.state;
if (seen_state == STATE_OP_ONGOING) {
// VCAS in original implementation
operation_t* seen_op = CAS_PTR(&(op->relocate_op.dest->op), op->relocate_op.dest_op, FLAG(op, STATE_OP_RELOCATE));
if ((seen_op == op->relocate_op.dest_op) || (seen_op == (operation_t *)FLAG(op, STATE_OP_RELOCATE))){
CAS_U32(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_SUCCESSFUL);
seen_state = STATE_OP_SUCCESSFUL;
if (seen_op == op->relocate_op.dest_op) {
#if GC == 1
if (UNFLAG(seen_op)!=0) ssmem_free(alloc,(void*)UNFLAG(seen_op));
#endif
}
} else {
// VCAS in original implementation
seen_state = CAS_U32(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_FAILED);
}
}
if (seen_state == STATE_OP_SUCCESSFUL) {
skey_t UNUSED dummy0 = CAS_PTR(&(op->relocate_op.dest->key), op->relocate_op.remove_key, op->relocate_op.replace_key);
skey_t UNUSED dummy1 = CAS_PTR(&(op->relocate_op.dest->value), op->relocate_op.remove_value, op->relocate_op.replace_value);
void* UNUSED dummy2 = CAS_PTR(&(op->relocate_op.dest->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, STATE_OP_NONE));
}
bool_t bst_remove(bst_key_t k, node_t* root, int id){
//fprintf(stderr, "bst remove\n");
// node_t* pred;
// node_t* curr;
node_t* replace = NULL;
// operation_t* pred_op;
// operation_t* curr_op;
operation_t* replace_op = NULL;
operation_t* reloc_op = NULL;
bst_search_result_t* my_result;
while(TRUE) {
//root is now a global pointer to a node, not a node
my_result = bst_find(k, /*&pred, &pred_op, &curr, &curr_op,*/ root, root, id);
if (my_result->result != FOUND) {
return FALSE;
}
if (ISNULL(my_result->curr->right) || ISNULL(my_result->curr->left)) { // node has less than two children
if (CAS_PTR(&(my_result->curr->op), my_result->curr_op, FLAG(my_result->curr_op, STATE_OP_MARK)) == my_result->curr_op) {
bst_help_marked(my_result->pred, my_result->pred_op, my_result->curr/*, root*/);
return TRUE;
}
} else { // node has two children
node_t* curr = my_result->curr;
my_search_result[id]->pred = my_result->pred;
my_search_result[id]->pred_op = my_result->pred_op;
my_search_result[id]->curr = replace;
my_search_result[id]->curr_op = replace_op;
// my_result = bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root, id);
my_result = bst_find(k, curr, root, id);
if ((my_result->result == ABORT) || (my_result->curr->op != my_result->curr_op)) {
continue;
}
//allocate memory
//reloc_op = new RelocateOP(curr, curr_op, k, replace->key);
reloc_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
reloc_op->relocate_op.state = STATE_OP_ONGOING;
reloc_op->relocate_op.dest = my_result->curr;
reloc_op->relocate_op.dest_op = my_result->curr_op;
reloc_op->relocate_op.remove_key = k;
reloc_op->relocate_op.replace_key = replace->key;
// fprintf(stderr, "reloc_op address: %p, state address: %p, dest addr: %p, dest_op addr: %p, remove_key addr: %p, replace_key addr: %p \n", (unsigned long)reloc_op, &(reloc_op->relocate_op.state), &(reloc_op->relocate_op.dest), &(reloc_op->relocate_op.dest_op), &(reloc_op->relocate_op.remove_key), &(reloc_op->relocate_op.replace_key)
// );
if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) {
if (bst_help_relocate(reloc_op, my_result->pred, my_result->pred_op, replace/*, root*/)) {
return TRUE;
}
}
}
}
}
void bst_help_child_cas(operation_t* op, node_t* dest, node_t* root){
node_t** address = NULL;
if (op->child_cas_op.is_left) {
address = &(dest->left);
} else {
address = &(dest->right);
}
CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update);
CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE));
}
void bst_help_child_cas(operation_t* op, node_t* dest/*, node_t* root*/){
//fprintf(stderr, "bst help child cas\n");
node_t** address = NULL;
if (op->child_cas_op.is_left) {
address = &(dest->left);
} else {
address = &(dest->right);
}
CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update);
CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE));
}
int mcs_trylock(mcs_lock *L, mcs_qnode_ptr I) {
I->next=NULL;
#ifndef __tile__
if (CAS_PTR(L, NULL, I)==NULL) return 0;
return 1;
#else
MEM_BARRIER;
if (CAS_PTR( L, NULL, I)==NULL) return 0;
return 1;
#endif
}
void bst_help_marked(node_t* pred, operation_t* pred_op, node_t* curr/*, node_t* root*/){
//fprintf(stderr, "bst help marked\n");
node_t* new_ref;
if (ISNULL(curr->left)) {
if (ISNULL(curr->right)) {
new_ref = (node_t*)SETNULL(curr);
} else {
new_ref = curr->right;
}
} else {
new_ref = curr->left;
}
// allocate memory
// operation_t* cas_op = new child_cas_op(curr==pred->left, curr, new_ref);
operation_t* cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
cas_op->child_cas_op.is_left = (curr == pred->left);
cas_op->child_cas_op.expected = curr;
cas_op->child_cas_op.update = new_ref;
// fprintf(stderr, "cas_op address: %p, is_left address: %p, expected addr: %p, update addr: %p\n", (unsigned long)cas_op, &(cas_op->child_cas_op.is_left), &(cas_op->child_cas_op.expected), &(cas_op->child_cas_op.update)
// );
if (CAS_PTR(&(pred->op), pred_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == pred_op) {
bst_help_child_cas(cas_op, pred/*, root*/);
}
}
void bst_help_child_cas(operation_t* op, node_t* dest, node_t* root){
CLEANUP_TRY();
node_t** address = NULL;
if (op->child_cas_op.is_left) {
address = (node_t**) &(dest->left);
} else {
address = (node_t**) &(dest->right);
}
void* UNUSED dummy0 = CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update);
#ifdef __tile__
MEM_BARRIER;
#endif
void* UNUSED dummy1 = CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE));
}
bool_t bst_remove(bst_key_t k, node_t* root){
node_t* pred;
node_t* curr;
node_t* replace;
operation_t* pred_op;
operation_t* curr_op;
operation_t* replace_op;
operation_t* reloc_op;
while(TRUE) {
if (bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root) != FOUND) {
return FALSE;
}
if (ISNULL(curr->right) || ISNULL(curr->left)) { // node has less than two children
if (CAS_PTR(&(curr->op), curr_op, FLAG(curr_op, STATE_OP_MARK)) == curr_op) {
bst_help_marked(pred, pred_op, curr, root);
return TRUE;
}
} else { // node has two children
if ((bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root) == ABORT) || (curr->op != curr_op)) {
continue;
}
reloc_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
reloc_op->relocate_op.state = STATE_OP_ONGOING;
reloc_op->relocate_op.dest = curr;
reloc_op->relocate_op.dest_op = curr_op;
reloc_op->relocate_op.remove_key = k;
reloc_op->relocate_op.replace_key = replace->key;
#if defined(__tile__)
MEM_BARRIER;
#endif
if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) {
if (bst_help_relocate(reloc_op, pred, pred_op, replace, root)) {
return TRUE;
}
}
}
}
}
volatile qnode * hclh_acquire(local_queue *lq, global_queue *gq, qnode *my_qnode) {
volatile qnode* my_pred;
do
{
#if defined(OPTERON_OPTIMIZE)
PREFETCHW(lq);
#endif /* OPTERON_OPTIMIZE */
my_pred = *lq;
} while (CAS_PTR(lq, my_pred, my_qnode)!=my_pred);
if (my_pred != NULL)
{
uint16_t i_own_lock = wait_for_grant_or_cluster_master(my_pred, my_qnode->fields.cluster_id);
if (i_own_lock)
{
return my_pred;
}
}
PAUSE; PAUSE;
volatile qnode * local_tail;
do
{
#if defined(OPTERON_OPTIMIZE)
PREFETCHW(gq);
PREFETCHW(lq);
#endif /* OPTERON_OPTIMIZE */
my_pred = *gq;
local_tail = *lq;
PAUSE;
} while(CAS_PTR(gq, my_pred, local_tail)!=my_pred);
local_tail->fields.tail_when_spliced = 1;
#if defined(OPTERON_OPTIMIZE)
PREFETCHW(my_pred);
#endif /* OPTERON_OPTIMIZE */
while (my_pred->fields.successor_must_wait) {
PAUSE;
#if defined(OPTERON_OPTIMIZE)
pause_rep(23);
PREFETCHW(my_pred);
#endif /* OPTERON_OPTIMIZE */
}
return my_pred;
}
bool_t bst_add(skey_t k,sval_t v, node_t* root){
node_t* pred;
node_t* curr;
node_t* new_node = NULL;
operation_t* pred_op;
operation_t* curr_op;
operation_t* cas_op;
sval_t result;
while(TRUE) {
UPDATE_TRY();
result = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root);
if (result & val_mask) {
#if GC == 1
if (new_node!=NULL) {
ssmem_free(alloc,new_node);
}
#endif
return FALSE;
}
if (new_node == NULL) {
new_node = create_node(k,v,0);
}
bool_t is_left = (result == NOT_FOUND_L);
node_t* old;
if (is_left) {
old = (node_t*) curr->left;
} else {
old = (node_t*) curr->right;
}
cas_op = alloc_op();
cas_op->child_cas_op.is_left = is_left;
cas_op->child_cas_op.expected = old;
cas_op->child_cas_op.update = new_node;
#if defined(__tile__)
MEM_BARRIER;
#endif
if (CAS_PTR(&curr->op, curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == curr_op) {
bst_help_child_cas(cas_op, curr, root);
#if GC == 1
//if (UNFLAG(curr_op)!=0) ssmem_free(alloc,(void*)UNFLAG(curr_op));
#endif
return TRUE;
} else {
#if GC == 1
ssmem_free(alloc,cas_op);
#endif
}
}
}
bool_t bst_insert(skey_t key, sval_t val, node_t* node_r) {
node_t* new_internal = NULL;
node_t* new_node = NULL;
uint created = 0;
while (1) {
UPDATE_TRY();
bst_seek(key, node_r);
if (seek_record->leaf->key == key) {
#if GC == 1
if (created) {
ssmem_free(alloc, new_internal);
ssmem_free(alloc, new_node);
}
#endif
return FALSE;
}
node_t* parent = seek_record->parent;
node_t* leaf = seek_record->leaf;
node_t** child_addr;
if (key < parent->key) {
child_addr= (node_t**) &(parent->left);
} else {
child_addr= (node_t**) &(parent->right);
}
if (likely(created==0)) {
new_internal=create_node(max(key,leaf->key),0,0);
new_node = create_node(key,val,0);
created=1;
} else {
new_internal->key=max(key,leaf->key);
}
if ( key < leaf->key) {
new_internal->left = new_node;
new_internal->right = leaf;
} else {
new_internal->right = new_node;
new_internal->left = leaf;
}
#ifdef __tile__
MEM_BARRIER;
#endif
node_t* result = CAS_PTR(child_addr, ADDRESS(leaf), ADDRESS(new_internal));
if (result == ADDRESS(leaf)) {
return TRUE;
}
node_t* chld = *child_addr;
if ( (ADDRESS(chld)==leaf) && (GETFLAG(chld) || GETTAG(chld)) ) {
bst_cleanup(key);
}
}
}
bool_t bst_add(bst_key_t k, node_t* root, int id){
//fprintf(stderr, "bst add\n");
// node_t* pred;
// node_t* curr;
node_t* new_node;
// operation_t* pred_op;
// operation_t* curr_op;
operation_t* cas_op;
// search_res_t result;
bst_search_result_t* my_result;
while(TRUE) {
//root is now a global pointer to a node, not a node
my_result = bst_find(k, /*&pred, &pred_op, &curr, &curr_op, */root, root, id);
if (my_result->result == FOUND) {
return FALSE;
}
// allocate memory
// new_node = new Node(k);
new_node = (node_t*) ssalloc(sizeof(node_t));
new_node->key = k;
new_node->op = NULL;
new_node->left = NULL;
new_node->right = NULL;
// fprintf(stderr, "new_node address: %p, 64bit aligned: %d key address %p, left node addr: %p, right node addr: %p, op addr: %p\n", new_node, ((unsigned long)new_node & 7) == 0,&(new_node->key), &(new_node->left), &(new_node->right), &(new_node->op)
// );
bool_t is_left = (my_result->result == NOT_FOUND_L);
node_t* old;
if (is_left) {
old = my_result->curr->left;
} else {
old = my_result->curr->right;
}
// allocate memory
//cas_op = new child_cas_op_t(is_left, old, new_node)
cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
cas_op->child_cas_op.is_left = is_left;
cas_op->child_cas_op.expected = old;
cas_op->child_cas_op.update = new_node;
// fprintf(stderr, "cas_op address: %p, is_left address: %p, expected addr: %p, update addr: %p\n", (unsigned long)cas_op, &(cas_op->child_cas_op.is_left), &(cas_op->child_cas_op.expected), &(cas_op->child_cas_op.update)
// );
if (CAS_PTR(&(my_result->curr->op), my_result->curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == my_result->curr_op) {
// legit cast? YES!! verif
bst_help_child_cas(cas_op, my_result->curr/*, root*/);
return TRUE;
}
}
}
bool_t bst_help_relocate(operation_t* op, node_t* pred, operation_t* pred_op, node_t* curr/*, node_t* root*/){
//fprintf(stderr, "bst help relocate\n");
int seen_state = op->relocate_op.state;
if (seen_state == STATE_OP_ONGOING) {
//VCAS in original implementation
operation_t* seen_op = CAS_PTR(&(op->relocate_op.dest->op), op->relocate_op.dest_op, FLAG(op, STATE_OP_RELOCATE));
if ((seen_op == op->relocate_op.dest_op) || (seen_op == (operation_t *)FLAG(op, STATE_OP_RELOCATE))){
CAS_PTR(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_SUCCESSFUL);
seen_state = STATE_OP_SUCCESSFUL;
} else {
// VCAS
seen_state = CAS_PTR(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_FAILED);
}
}
if (seen_state == STATE_OP_SUCCESSFUL) {
// TODO not clear in the paper code
CAS_PTR(&(op->relocate_op.dest->key), op->relocate_op.remove_key, op->relocate_op.replace_key);
CAS_PTR(&(op->relocate_op.dest->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, STATE_OP_NONE));
}
bool_t result = (seen_state == STATE_OP_SUCCESSFUL);
if (op->relocate_op.dest == curr) {
return result;
}
CAS_PTR(&(curr->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, result ? STATE_OP_MARK : STATE_OP_NONE));
if (result) {
if (op->relocate_op.dest == pred) {
pred_op = (operation_t *)FLAG(op, STATE_OP_NONE);
}
bst_help_marked(pred, pred_op, curr/*, root*/);
}
return result;
}
volatile qnode * hclh_acquire(local_queue *lq, global_queue *gq, qnode *my_qnode) {
//splice my_qnode into local queue
volatile qnode* my_pred;
do
{
my_pred = *lq;
} while (CAS_PTR(lq, my_pred, my_qnode)!=my_pred);
if (my_pred != NULL)
{
uint16_t i_own_lock = wait_for_grant_or_cluster_master(my_pred, my_qnode->fields.cluster_id);
if (i_own_lock)
{
//I have the lock; return qnode just released by previous owner
return my_pred;
}
}
//at this point, I'm cluster master. Wait to allow time for other acquireres to show up.
PAUSE; PAUSE;
volatile qnode * local_tail;
//splice local queue into global queue
do
{
my_pred = *gq;
local_tail = *lq;
PAUSE;
} while(CAS_PTR(gq, my_pred, local_tail)!=my_pred);
//inform successor that it is the new master
local_tail->fields.tail_when_spliced = 1;
//wait for predecessor to release lock
while (my_pred->fields.successor_must_wait) {
PAUSE;
}
//I have the lock. return qnode just released by previous owner for next lock access
return my_pred;
}
sval_t bst_remove(skey_t key, node_t* node_r) {
bool_t injecting = TRUE;
node_t* leaf;
sval_t val = 0;
while (1) {
UPDATE_TRY();
bst_seek(key, node_r);
val = seek_record->leaf->value;
node_t* parent = seek_record->parent;
node_t** child_addr;
if (key < parent->key) {
child_addr = (node_t**) &(parent->left);
} else {
child_addr = (node_t**) &(parent->right);
}
if (injecting == TRUE) {
leaf = seek_record->leaf;
if (leaf->key != key) {
return 0;
}
node_t* lf = ADDRESS(leaf);
node_t* result = CAS_PTR(child_addr, lf, FLAG(lf));
if (result == ADDRESS(leaf)) {
injecting = FALSE;
bool_t done = bst_cleanup(key);
if (done == TRUE) {
return val;
}
} else {
node_t* chld = *child_addr;
if ( (ADDRESS(chld) == leaf) && (GETFLAG(chld) || GETTAG(chld)) ) {
bst_cleanup(key);
}
}
} else {
if (seek_record->leaf != leaf) {
return val;
} else {
bool_t done = bst_cleanup(key);
if (done == TRUE) {
return val;
}
}
}
}
}
bool_t bst_add(bst_key_t k, node_t* root){
node_t* pred;
node_t* curr;
node_t* new_node;
operation_t* pred_op;
operation_t* curr_op;
operation_t* cas_op;
search_res_t result;
while(TRUE) {
result = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root);
if (result == FOUND) {
return FALSE;
}
new_node = (node_t*) ssalloc(sizeof(node_t));
new_node->key = k;
new_node->op = NULL;
new_node->left = NULL;
new_node->right = NULL;
bool_t is_left = (result == NOT_FOUND_L);
node_t* old;
if (is_left) {
old = curr->left;
} else {
old = curr->right;
}
cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
cas_op->child_cas_op.is_left = is_left;
cas_op->child_cas_op.expected = old;
cas_op->child_cas_op.update = new_node;
#if defined(__tile__)
MEM_BARRIER;
#endif
if (CAS_PTR(&curr->op, curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == curr_op) {
bst_help_child_cas(cas_op, curr, root);
return TRUE;
}
}
}
void bst_help_marked(node_t* pred, operation_t* pred_op, node_t* curr, node_t* root){
node_t* new_ref;
if (ISNULL(curr->left)) {
if (ISNULL(curr->right)) {
new_ref = (node_t*)SETNULL(curr);
} else {
new_ref = curr->right;
}
} else {
new_ref = curr->left;
}
operation_t* cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t));
cas_op->child_cas_op.is_left = (curr == pred->left);
cas_op->child_cas_op.expected = curr;
cas_op->child_cas_op.update = new_ref;
if (CAS_PTR(&(pred->op), pred_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == pred_op) {
bst_help_child_cas(cas_op, pred, root);
}
}
void mcs_release(mcs_lock *L, mcs_qnode_ptr I)
{
#ifdef __tile__
MEM_BARRIER;
#endif
mcs_qnode_ptr succ;
#if defined(OPTERON_OPTIMIZE)
PREFETCHW(I);
#endif /* OPTERON_OPTIMIZE */
if (!(succ = I->next)) /* I seem to have no succ. */
{
/* try to fix global pointer */
if (CAS_PTR(L, I, NULL) == I)
return;
do {
succ = I->next;
PAUSE;
} while (!succ); // wait for successor
}
succ->waiting = 0;
}
sval_t bst_remove(skey_t k, node_t* root){
node_t* pred;
node_t* curr;
node_t* replace;
sval_t val;
operation_t* pred_op;
operation_t* curr_op;
operation_t* replace_op;
operation_t* reloc_op=NULL;
while(TRUE) {
UPDATE_TRY();
sval_t res = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root);
if (!(res & val_mask)) {
#if GC == 1
//if (reloc_op!=NULL) ssmem_free(alloc,reloc_op);
#endif
return 0;
}
if (ISNULL((node_t*) curr->right) || ISNULL((node_t*) curr->left)) { // node has less than two children
if (CAS_PTR(&(curr->op), curr_op, FLAG(curr_op, STATE_OP_MARK)) == curr_op) {
bst_help_marked(pred, pred_op, curr, root);
#if GC == 1
//if (reloc_op!=NULL) ssmem_free(alloc,reloc_op);
if (UNFLAG(curr->op)!=0) ssmem_free(alloc,(void*)UNFLAG(curr->op));
ssmem_free(alloc,curr);
#endif
return res;
}
} else { // node has two children
val = bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root);
if ((val == ABORT) || (curr->op != curr_op)) {
continue;
}
//if (reloc_op==NULL) {
reloc_op = alloc_op();
//}
reloc_op->relocate_op.state = STATE_OP_ONGOING;
reloc_op->relocate_op.dest = curr;
reloc_op->relocate_op.dest_op = curr_op;
reloc_op->relocate_op.remove_key = k;
reloc_op->relocate_op.remove_value = res;
reloc_op->relocate_op.replace_key = replace->key;
reloc_op->relocate_op.replace_value = replace->value;
#if defined(__tile__)
MEM_BARRIER;
#endif
if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) {
#if GC == 1
if (UNFLAG(replace_op)!=0) ssmem_free(alloc,(void*)UNFLAG(replace_op));
#endif
if (bst_help_relocate(reloc_op, pred, pred_op, replace, root)) {
//if (UNFLAG(replace->op)!=0) ssmem_free(alloc,(void*)UNFLAG(replace->op));
#if GC == 1
//ssmem_free(alloc,replace);
#endif
return res;
}
} else {
#if GC == 1
ssmem_free(alloc,reloc_op);
// reloc_op=NULL;
#endif
}
}
}
}
