bool_t bst_help_relocate(operation_t* op, node_t* pred, operation_t* pred_op, node_t* curr, node_t* root){ CLEANUP_TRY(); int seen_state = op->relocate_op.state; if (seen_state == STATE_OP_ONGOING) { // VCAS in original implementation operation_t* seen_op = CAS_PTR(&(op->relocate_op.dest->op), op->relocate_op.dest_op, FLAG(op, STATE_OP_RELOCATE)); if ((seen_op == op->relocate_op.dest_op) || (seen_op == (operation_t *)FLAG(op, STATE_OP_RELOCATE))){ CAS_U32(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_SUCCESSFUL); seen_state = STATE_OP_SUCCESSFUL; if (seen_op == op->relocate_op.dest_op) { #if GC == 1 if (UNFLAG(seen_op)!=0) ssmem_free(alloc,(void*)UNFLAG(seen_op)); #endif } } else { // VCAS in original implementation seen_state = CAS_U32(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_FAILED); } } if (seen_state == STATE_OP_SUCCESSFUL) { skey_t UNUSED dummy0 = CAS_PTR(&(op->relocate_op.dest->key), op->relocate_op.remove_key, op->relocate_op.replace_key); skey_t UNUSED dummy1 = CAS_PTR(&(op->relocate_op.dest->value), op->relocate_op.remove_value, op->relocate_op.replace_value); void* UNUSED dummy2 = CAS_PTR(&(op->relocate_op.dest->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, STATE_OP_NONE)); }
bool_t bst_remove(bst_key_t k, node_t* root, int id){ //fprintf(stderr, "bst remove\n"); // node_t* pred; // node_t* curr; node_t* replace = NULL; // operation_t* pred_op; // operation_t* curr_op; operation_t* replace_op = NULL; operation_t* reloc_op = NULL; bst_search_result_t* my_result; while(TRUE) { //root is now a global pointer to a node, not a node my_result = bst_find(k, /*&pred, &pred_op, &curr, &curr_op,*/ root, root, id); if (my_result->result != FOUND) { return FALSE; } if (ISNULL(my_result->curr->right) || ISNULL(my_result->curr->left)) { // node has less than two children if (CAS_PTR(&(my_result->curr->op), my_result->curr_op, FLAG(my_result->curr_op, STATE_OP_MARK)) == my_result->curr_op) { bst_help_marked(my_result->pred, my_result->pred_op, my_result->curr/*, root*/); return TRUE; } } else { // node has two children node_t* curr = my_result->curr; my_search_result[id]->pred = my_result->pred; my_search_result[id]->pred_op = my_result->pred_op; my_search_result[id]->curr = replace; my_search_result[id]->curr_op = replace_op; // my_result = bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root, id); my_result = bst_find(k, curr, root, id); if ((my_result->result == ABORT) || (my_result->curr->op != my_result->curr_op)) { continue; } //allocate memory //reloc_op = new RelocateOP(curr, curr_op, k, replace->key); reloc_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); reloc_op->relocate_op.state = STATE_OP_ONGOING; reloc_op->relocate_op.dest = my_result->curr; reloc_op->relocate_op.dest_op = my_result->curr_op; reloc_op->relocate_op.remove_key = k; reloc_op->relocate_op.replace_key = replace->key; // fprintf(stderr, "reloc_op address: %p, state address: %p, dest addr: %p, dest_op addr: %p, remove_key addr: %p, replace_key addr: %p \n", (unsigned long)reloc_op, &(reloc_op->relocate_op.state), &(reloc_op->relocate_op.dest), &(reloc_op->relocate_op.dest_op), &(reloc_op->relocate_op.remove_key), &(reloc_op->relocate_op.replace_key) // ); if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) { if (bst_help_relocate(reloc_op, my_result->pred, my_result->pred_op, replace/*, root*/)) { return TRUE; } } } } }
void bst_help_child_cas(operation_t* op, node_t* dest, node_t* root){ node_t** address = NULL; if (op->child_cas_op.is_left) { address = &(dest->left); } else { address = &(dest->right); } CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update); CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE)); }
void bst_help_child_cas(operation_t* op, node_t* dest/*, node_t* root*/){ //fprintf(stderr, "bst help child cas\n"); node_t** address = NULL; if (op->child_cas_op.is_left) { address = &(dest->left); } else { address = &(dest->right); } CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update); CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE)); }
int mcs_trylock(mcs_lock *L, mcs_qnode_ptr I) { I->next=NULL; #ifndef __tile__ if (CAS_PTR(L, NULL, I)==NULL) return 0; return 1; #else MEM_BARRIER; if (CAS_PTR( L, NULL, I)==NULL) return 0; return 1; #endif }
void bst_help_marked(node_t* pred, operation_t* pred_op, node_t* curr/*, node_t* root*/){ //fprintf(stderr, "bst help marked\n"); node_t* new_ref; if (ISNULL(curr->left)) { if (ISNULL(curr->right)) { new_ref = (node_t*)SETNULL(curr); } else { new_ref = curr->right; } } else { new_ref = curr->left; } // allocate memory // operation_t* cas_op = new child_cas_op(curr==pred->left, curr, new_ref); operation_t* cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); cas_op->child_cas_op.is_left = (curr == pred->left); cas_op->child_cas_op.expected = curr; cas_op->child_cas_op.update = new_ref; // fprintf(stderr, "cas_op address: %p, is_left address: %p, expected addr: %p, update addr: %p\n", (unsigned long)cas_op, &(cas_op->child_cas_op.is_left), &(cas_op->child_cas_op.expected), &(cas_op->child_cas_op.update) // ); if (CAS_PTR(&(pred->op), pred_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == pred_op) { bst_help_child_cas(cas_op, pred/*, root*/); } }
void bst_help_child_cas(operation_t* op, node_t* dest, node_t* root){ CLEANUP_TRY(); node_t** address = NULL; if (op->child_cas_op.is_left) { address = (node_t**) &(dest->left); } else { address = (node_t**) &(dest->right); } void* UNUSED dummy0 = CAS_PTR(address, op->child_cas_op.expected, op->child_cas_op.update); #ifdef __tile__ MEM_BARRIER; #endif void* UNUSED dummy1 = CAS_PTR(&(dest->op), FLAG(op, STATE_OP_CHILDCAS), FLAG(op, STATE_OP_NONE)); }
bool_t bst_remove(bst_key_t k, node_t* root){ node_t* pred; node_t* curr; node_t* replace; operation_t* pred_op; operation_t* curr_op; operation_t* replace_op; operation_t* reloc_op; while(TRUE) { if (bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root) != FOUND) { return FALSE; } if (ISNULL(curr->right) || ISNULL(curr->left)) { // node has less than two children if (CAS_PTR(&(curr->op), curr_op, FLAG(curr_op, STATE_OP_MARK)) == curr_op) { bst_help_marked(pred, pred_op, curr, root); return TRUE; } } else { // node has two children if ((bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root) == ABORT) || (curr->op != curr_op)) { continue; } reloc_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); reloc_op->relocate_op.state = STATE_OP_ONGOING; reloc_op->relocate_op.dest = curr; reloc_op->relocate_op.dest_op = curr_op; reloc_op->relocate_op.remove_key = k; reloc_op->relocate_op.replace_key = replace->key; #if defined(__tile__) MEM_BARRIER; #endif if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) { if (bst_help_relocate(reloc_op, pred, pred_op, replace, root)) { return TRUE; } } } } }
volatile qnode * hclh_acquire(local_queue *lq, global_queue *gq, qnode *my_qnode) { volatile qnode* my_pred; do { #if defined(OPTERON_OPTIMIZE) PREFETCHW(lq); #endif /* OPTERON_OPTIMIZE */ my_pred = *lq; } while (CAS_PTR(lq, my_pred, my_qnode)!=my_pred); if (my_pred != NULL) { uint16_t i_own_lock = wait_for_grant_or_cluster_master(my_pred, my_qnode->fields.cluster_id); if (i_own_lock) { return my_pred; } } PAUSE; PAUSE; volatile qnode * local_tail; do { #if defined(OPTERON_OPTIMIZE) PREFETCHW(gq); PREFETCHW(lq); #endif /* OPTERON_OPTIMIZE */ my_pred = *gq; local_tail = *lq; PAUSE; } while(CAS_PTR(gq, my_pred, local_tail)!=my_pred); local_tail->fields.tail_when_spliced = 1; #if defined(OPTERON_OPTIMIZE) PREFETCHW(my_pred); #endif /* OPTERON_OPTIMIZE */ while (my_pred->fields.successor_must_wait) { PAUSE; #if defined(OPTERON_OPTIMIZE) pause_rep(23); PREFETCHW(my_pred); #endif /* OPTERON_OPTIMIZE */ } return my_pred; }
bool_t bst_add(skey_t k,sval_t v, node_t* root){ node_t* pred; node_t* curr; node_t* new_node = NULL; operation_t* pred_op; operation_t* curr_op; operation_t* cas_op; sval_t result; while(TRUE) { UPDATE_TRY(); result = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root); if (result & val_mask) { #if GC == 1 if (new_node!=NULL) { ssmem_free(alloc,new_node); } #endif return FALSE; } if (new_node == NULL) { new_node = create_node(k,v,0); } bool_t is_left = (result == NOT_FOUND_L); node_t* old; if (is_left) { old = (node_t*) curr->left; } else { old = (node_t*) curr->right; } cas_op = alloc_op(); cas_op->child_cas_op.is_left = is_left; cas_op->child_cas_op.expected = old; cas_op->child_cas_op.update = new_node; #if defined(__tile__) MEM_BARRIER; #endif if (CAS_PTR(&curr->op, curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == curr_op) { bst_help_child_cas(cas_op, curr, root); #if GC == 1 //if (UNFLAG(curr_op)!=0) ssmem_free(alloc,(void*)UNFLAG(curr_op)); #endif return TRUE; } else { #if GC == 1 ssmem_free(alloc,cas_op); #endif } } }
bool_t bst_insert(skey_t key, sval_t val, node_t* node_r) { node_t* new_internal = NULL; node_t* new_node = NULL; uint created = 0; while (1) { UPDATE_TRY(); bst_seek(key, node_r); if (seek_record->leaf->key == key) { #if GC == 1 if (created) { ssmem_free(alloc, new_internal); ssmem_free(alloc, new_node); } #endif return FALSE; } node_t* parent = seek_record->parent; node_t* leaf = seek_record->leaf; node_t** child_addr; if (key < parent->key) { child_addr= (node_t**) &(parent->left); } else { child_addr= (node_t**) &(parent->right); } if (likely(created==0)) { new_internal=create_node(max(key,leaf->key),0,0); new_node = create_node(key,val,0); created=1; } else { new_internal->key=max(key,leaf->key); } if ( key < leaf->key) { new_internal->left = new_node; new_internal->right = leaf; } else { new_internal->right = new_node; new_internal->left = leaf; } #ifdef __tile__ MEM_BARRIER; #endif node_t* result = CAS_PTR(child_addr, ADDRESS(leaf), ADDRESS(new_internal)); if (result == ADDRESS(leaf)) { return TRUE; } node_t* chld = *child_addr; if ( (ADDRESS(chld)==leaf) && (GETFLAG(chld) || GETTAG(chld)) ) { bst_cleanup(key); } } }
bool_t bst_add(bst_key_t k, node_t* root, int id){ //fprintf(stderr, "bst add\n"); // node_t* pred; // node_t* curr; node_t* new_node; // operation_t* pred_op; // operation_t* curr_op; operation_t* cas_op; // search_res_t result; bst_search_result_t* my_result; while(TRUE) { //root is now a global pointer to a node, not a node my_result = bst_find(k, /*&pred, &pred_op, &curr, &curr_op, */root, root, id); if (my_result->result == FOUND) { return FALSE; } // allocate memory // new_node = new Node(k); new_node = (node_t*) ssalloc(sizeof(node_t)); new_node->key = k; new_node->op = NULL; new_node->left = NULL; new_node->right = NULL; // fprintf(stderr, "new_node address: %p, 64bit aligned: %d key address %p, left node addr: %p, right node addr: %p, op addr: %p\n", new_node, ((unsigned long)new_node & 7) == 0,&(new_node->key), &(new_node->left), &(new_node->right), &(new_node->op) // ); bool_t is_left = (my_result->result == NOT_FOUND_L); node_t* old; if (is_left) { old = my_result->curr->left; } else { old = my_result->curr->right; } // allocate memory //cas_op = new child_cas_op_t(is_left, old, new_node) cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); cas_op->child_cas_op.is_left = is_left; cas_op->child_cas_op.expected = old; cas_op->child_cas_op.update = new_node; // fprintf(stderr, "cas_op address: %p, is_left address: %p, expected addr: %p, update addr: %p\n", (unsigned long)cas_op, &(cas_op->child_cas_op.is_left), &(cas_op->child_cas_op.expected), &(cas_op->child_cas_op.update) // ); if (CAS_PTR(&(my_result->curr->op), my_result->curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == my_result->curr_op) { // legit cast? YES!! verif bst_help_child_cas(cas_op, my_result->curr/*, root*/); return TRUE; } } }
bool_t bst_help_relocate(operation_t* op, node_t* pred, operation_t* pred_op, node_t* curr/*, node_t* root*/){ //fprintf(stderr, "bst help relocate\n"); int seen_state = op->relocate_op.state; if (seen_state == STATE_OP_ONGOING) { //VCAS in original implementation operation_t* seen_op = CAS_PTR(&(op->relocate_op.dest->op), op->relocate_op.dest_op, FLAG(op, STATE_OP_RELOCATE)); if ((seen_op == op->relocate_op.dest_op) || (seen_op == (operation_t *)FLAG(op, STATE_OP_RELOCATE))){ CAS_PTR(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_SUCCESSFUL); seen_state = STATE_OP_SUCCESSFUL; } else { // VCAS seen_state = CAS_PTR(&(op->relocate_op.state), STATE_OP_ONGOING, STATE_OP_FAILED); } } if (seen_state == STATE_OP_SUCCESSFUL) { // TODO not clear in the paper code CAS_PTR(&(op->relocate_op.dest->key), op->relocate_op.remove_key, op->relocate_op.replace_key); CAS_PTR(&(op->relocate_op.dest->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, STATE_OP_NONE)); } bool_t result = (seen_state == STATE_OP_SUCCESSFUL); if (op->relocate_op.dest == curr) { return result; } CAS_PTR(&(curr->op), FLAG(op, STATE_OP_RELOCATE), FLAG(op, result ? STATE_OP_MARK : STATE_OP_NONE)); if (result) { if (op->relocate_op.dest == pred) { pred_op = (operation_t *)FLAG(op, STATE_OP_NONE); } bst_help_marked(pred, pred_op, curr/*, root*/); } return result; }
volatile qnode * hclh_acquire(local_queue *lq, global_queue *gq, qnode *my_qnode) { //splice my_qnode into local queue volatile qnode* my_pred; do { my_pred = *lq; } while (CAS_PTR(lq, my_pred, my_qnode)!=my_pred); if (my_pred != NULL) { uint16_t i_own_lock = wait_for_grant_or_cluster_master(my_pred, my_qnode->fields.cluster_id); if (i_own_lock) { //I have the lock; return qnode just released by previous owner return my_pred; } } //at this point, I'm cluster master. Wait to allow time for other acquireres to show up. PAUSE; PAUSE; volatile qnode * local_tail; //splice local queue into global queue do { my_pred = *gq; local_tail = *lq; PAUSE; } while(CAS_PTR(gq, my_pred, local_tail)!=my_pred); //inform successor that it is the new master local_tail->fields.tail_when_spliced = 1; //wait for predecessor to release lock while (my_pred->fields.successor_must_wait) { PAUSE; } //I have the lock. return qnode just released by previous owner for next lock access return my_pred; }
sval_t bst_remove(skey_t key, node_t* node_r) { bool_t injecting = TRUE; node_t* leaf; sval_t val = 0; while (1) { UPDATE_TRY(); bst_seek(key, node_r); val = seek_record->leaf->value; node_t* parent = seek_record->parent; node_t** child_addr; if (key < parent->key) { child_addr = (node_t**) &(parent->left); } else { child_addr = (node_t**) &(parent->right); } if (injecting == TRUE) { leaf = seek_record->leaf; if (leaf->key != key) { return 0; } node_t* lf = ADDRESS(leaf); node_t* result = CAS_PTR(child_addr, lf, FLAG(lf)); if (result == ADDRESS(leaf)) { injecting = FALSE; bool_t done = bst_cleanup(key); if (done == TRUE) { return val; } } else { node_t* chld = *child_addr; if ( (ADDRESS(chld) == leaf) && (GETFLAG(chld) || GETTAG(chld)) ) { bst_cleanup(key); } } } else { if (seek_record->leaf != leaf) { return val; } else { bool_t done = bst_cleanup(key); if (done == TRUE) { return val; } } } } }
bool_t bst_add(bst_key_t k, node_t* root){ node_t* pred; node_t* curr; node_t* new_node; operation_t* pred_op; operation_t* curr_op; operation_t* cas_op; search_res_t result; while(TRUE) { result = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root); if (result == FOUND) { return FALSE; } new_node = (node_t*) ssalloc(sizeof(node_t)); new_node->key = k; new_node->op = NULL; new_node->left = NULL; new_node->right = NULL; bool_t is_left = (result == NOT_FOUND_L); node_t* old; if (is_left) { old = curr->left; } else { old = curr->right; } cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); cas_op->child_cas_op.is_left = is_left; cas_op->child_cas_op.expected = old; cas_op->child_cas_op.update = new_node; #if defined(__tile__) MEM_BARRIER; #endif if (CAS_PTR(&curr->op, curr_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == curr_op) { bst_help_child_cas(cas_op, curr, root); return TRUE; } } }
void bst_help_marked(node_t* pred, operation_t* pred_op, node_t* curr, node_t* root){ node_t* new_ref; if (ISNULL(curr->left)) { if (ISNULL(curr->right)) { new_ref = (node_t*)SETNULL(curr); } else { new_ref = curr->right; } } else { new_ref = curr->left; } operation_t* cas_op = (operation_t*) ssalloc_alloc(1, sizeof(operation_t)); cas_op->child_cas_op.is_left = (curr == pred->left); cas_op->child_cas_op.expected = curr; cas_op->child_cas_op.update = new_ref; if (CAS_PTR(&(pred->op), pred_op, FLAG(cas_op, STATE_OP_CHILDCAS)) == pred_op) { bst_help_child_cas(cas_op, pred, root); } }
void mcs_release(mcs_lock *L, mcs_qnode_ptr I) { #ifdef __tile__ MEM_BARRIER; #endif mcs_qnode_ptr succ; #if defined(OPTERON_OPTIMIZE) PREFETCHW(I); #endif /* OPTERON_OPTIMIZE */ if (!(succ = I->next)) /* I seem to have no succ. */ { /* try to fix global pointer */ if (CAS_PTR(L, I, NULL) == I) return; do { succ = I->next; PAUSE; } while (!succ); // wait for successor } succ->waiting = 0; }
sval_t bst_remove(skey_t k, node_t* root){ node_t* pred; node_t* curr; node_t* replace; sval_t val; operation_t* pred_op; operation_t* curr_op; operation_t* replace_op; operation_t* reloc_op=NULL; while(TRUE) { UPDATE_TRY(); sval_t res = bst_find(k, &pred, &pred_op, &curr, &curr_op, root, root); if (!(res & val_mask)) { #if GC == 1 //if (reloc_op!=NULL) ssmem_free(alloc,reloc_op); #endif return 0; } if (ISNULL((node_t*) curr->right) || ISNULL((node_t*) curr->left)) { // node has less than two children if (CAS_PTR(&(curr->op), curr_op, FLAG(curr_op, STATE_OP_MARK)) == curr_op) { bst_help_marked(pred, pred_op, curr, root); #if GC == 1 //if (reloc_op!=NULL) ssmem_free(alloc,reloc_op); if (UNFLAG(curr->op)!=0) ssmem_free(alloc,(void*)UNFLAG(curr->op)); ssmem_free(alloc,curr); #endif return res; } } else { // node has two children val = bst_find(k, &pred, &pred_op, &replace, &replace_op, curr, root); if ((val == ABORT) || (curr->op != curr_op)) { continue; } //if (reloc_op==NULL) { reloc_op = alloc_op(); //} reloc_op->relocate_op.state = STATE_OP_ONGOING; reloc_op->relocate_op.dest = curr; reloc_op->relocate_op.dest_op = curr_op; reloc_op->relocate_op.remove_key = k; reloc_op->relocate_op.remove_value = res; reloc_op->relocate_op.replace_key = replace->key; reloc_op->relocate_op.replace_value = replace->value; #if defined(__tile__) MEM_BARRIER; #endif if (CAS_PTR(&(replace->op), replace_op, FLAG(reloc_op, STATE_OP_RELOCATE)) == replace_op) { #if GC == 1 if (UNFLAG(replace_op)!=0) ssmem_free(alloc,(void*)UNFLAG(replace_op)); #endif if (bst_help_relocate(reloc_op, pred, pred_op, replace, root)) { //if (UNFLAG(replace->op)!=0) ssmem_free(alloc,(void*)UNFLAG(replace->op)); #if GC == 1 //ssmem_free(alloc,replace); #endif return res; } } else { #if GC == 1 ssmem_free(alloc,reloc_op); // reloc_op=NULL; #endif } } } }