void *
work_fn(void *gcc_is_ass)
{
// Forever,
do {
// Pick a key to use. Look it up to see if anyone else is using it.
uint32_t key = rand_64() % g_config.n_keys;
uint32_t die = rand_64() & 0x03;
if (SHASH_OK == shash_put_unique(g_config.in_progress_hash, &key, 0) )
{
cl_rv rv;
// Make the key into a string
char key_s[g_config.key_len+1];
my_itoa(key_s, key, g_config.key_len);
// Make an cl_object that represents the key
cl_object key_o;
citrusleaf_object_init_str(&key_o, key_s);
cf_digest d;
citrusleaf_calculate_digest(g_config.set, &key_o, &d);
if (VALUE_UNINIT == g_config.values[key]) {
// simply set the value to something - can't really check anything because we don't know the state
if (0 != write_new_value(key, &key_o, &d)) {
if (g_config.strict) goto Fail;
}
atomic_int_add(g_config.key_counter, 1);
}
else if (VALUE_DELETED == g_config.values[key]) {
// Shouldn't exist
cl_bin *cl_v = 0;
int cl_v_len;
rv = citrusleaf_get_all(g_config.asc, g_config.ns, g_config.set, &key_o, &cl_v, &cl_v_len, g_config.timeout_ms, NULL);
if (rv != CITRUSLEAF_FAIL_NOTFOUND) {
fprintf(stderr, "Get after delete returned improper value when should be deleted %d key %s digest %"PRIx64"\n",rv,key_s, *(uint64_t *)&d);
if (g_config.strict) goto Fail;
}
if (cl_v) free(cl_v);
atomic_int_add(g_config.read_counter, 1); // did two ops here
// write a new value
if (die < 2) {
if (0 != write_new_value(key, &key_o, &d)) {
if (g_config.strict) goto Fail;
}
atomic_int_add(g_config.key_counter, 1);
}
}
// Value is well known. Check to see that it's still right.
else {
cl_bin values[1];
strcpy(values[0].bin_name, g_config.bin);
citrusleaf_object_init(&values[0].object);
// Make string version of old value for checking
char new_value_str[g_config.value_len+1];
my_itoa(new_value_str, g_config.values[key], g_config.value_len);
citrusleaf_object_init_str(&values[0].object, new_value_str);
rv = citrusleaf_verify(g_config.asc, g_config.ns, g_config.set, &key_o, values, 1, g_config.timeout_ms, NULL);
if (rv != 0) {
fprintf(stderr, "Get returned improper value %d when should be set : key %d digest %"PRIx64"\n",rv,key, *(uint64_t *)&d);
if (g_config.strict) goto Fail;
goto V1;
}
// test!
if (values[0].object.type != CL_STR) {
fprintf(stderr, "read value has wrong type: expect string (3) got %d\n",(int)values[0].object.type);
if (g_config.strict) return((void *)-1);
}
else if (strcmp(values[0].object.u.str, new_value_str) != 0) {
fprintf(stderr, "read value does not match set value.\n");
fprintf(stderr, " expecting: %s\n",new_value_str);
fprintf(stderr, " got: %s\n",values[0].object.u.str);
if (g_config.strict) goto Fail;
}
citrusleaf_object_free(&values[0].object);
atomic_int_add(g_config.read_counter, 1);
// Delete, write new value, what's your pleasure?
V1:
if (die < 2) {
if (0 != write_new_value(key, &key_o, &d)) {
if (g_config.strict) return((void *)-1);
}
}
// Delete!
else if (die == 2) {
rv = citrusleaf_delete_verify(g_config.asc, g_config.ns, g_config.set, &key_o, 0);
if (rv != 0) {
fprintf(stderr, "Delete returned improper value %d, fail: key %d digest %"PRIx64"\n",rv, key, *(uint64_t *)&d);
if (g_config.strict) goto Fail;
}
cl_bin values[1];
strcpy(values[0].bin_name, g_config.bin);
citrusleaf_object_init(&values[0].object);
rv = citrusleaf_get(g_config.asc, g_config.ns, g_config.set, &key_o, values, 1, g_config.timeout_ms, NULL);
if (rv != CITRUSLEAF_FAIL_NOTFOUND) {
fprintf(stderr, "Get after delete returned improper value %d digest %"PRIx64"\n",rv, *(uint64_t *)&d);
if (g_config.strict) goto Fail;
}
citrusleaf_object_free(&values[0].object);
g_config.values[key] = VALUE_DELETED;
atomic_int_add(g_config.read_counter, 1); // did two ops here
atomic_int_add(g_config.delete_counter, 1);
atomic_int_add(g_config.key_counter, -1);
}
}
// remove my lock on this key
shash_delete(g_config.in_progress_hash, &key);
}
} while (1);
Fail:
abort();
return((void *)-1);
}
static void*
async_receiver_fn(void *thdata)
{
int rv = -1;
bool network_error = false;
cl_async_work *workitem = NULL;
// cl_async_work *tmpworkitem = NULL;
as_msg msg;
cf_queue *q_to_use = NULL;
cl_cluster_node *thisnode = NULL;
uint8_t rd_stack_buf[STACK_BUF_SZ];
uint8_t *rd_buf = rd_stack_buf;
size_t rd_buf_sz = 0;
uint64_t acktrid;
// uint64_t starttime, endtime;
int progress_timeout_ms;
unsigned int thread_id = cf_atomic32_incr(&g_thread_count);
if (thdata == NULL) {
q_to_use = g_cl_async_q;
} else {
thisnode = (cl_cluster_node *)thdata;
q_to_use = thisnode->asyncwork_q;
}
//Infinite loop which keeps picking work items from the list and try to find the end result
while(1) {
network_error = false;
#if ONEASYNCFD
if(thisnode->dunned == true) {
do {
rv = cf_queue_pop(thisnode->asyncwork_q, &workitem, CF_QUEUE_NOWAIT);
if (rv == CF_QUEUE_OK) {
cl_cluster_node_put(thisnode);
free(workitem);
}
} while (rv == CF_QUEUE_OK);
//We want to delete all the workitems of this node
shash_reduce_delete(g_cl_async_hashtab, cl_del_node_asyncworkitems, thisnode);
break;
}
#endif
//This call will block if there is no element in the queue
cf_queue_pop(q_to_use, &workitem, CF_QUEUE_FOREVER);
//TODO: What if the node gets dunned while this pop call is blocked ?
#if ONEASYNCFD
//cf_debug("Elements remaining in this node's queue=%d, Hash table size=%d",
// cf_queue_sz(thisnode->asyncwork_q), shash_get_size(g_cl_async_hashtab));
#endif
// If we have no progress in 50ms, we should move to the next workitem
// and revisit this workitem at a later stage
progress_timeout_ms = DEFAULT_PROGRESS_TIMEOUT;
// Read into this fine cl_msg, which is the short header
rv = cf_socket_read_timeout(workitem->fd, (uint8_t *) &msg, sizeof(as_msg), workitem->deadline, progress_timeout_ms);
if (rv) {
#if DEBUG
cf_debug("Citrusleaf: error when reading header from server - rv %d fd %d", rv, workitem->fd);
#endif
if (rv != ETIMEDOUT) {
cf_error("Citrusleaf: error when reading header from server - rv %d fd %d",rv,workitem->fd);
network_error = true;
goto Error;
} else {
goto Retry;
}
}
#ifdef DEBUG_VERBOSE
dump_buf("read header from cluster", (uint8_t *) &msg, sizeof(cl_msg));
#endif
cl_proto_swap(&msg.proto);
cl_msg_swap_header(&msg.m);
// second read for the remainder of the message
rd_buf_sz = msg.proto.sz - msg.m.header_sz;
if (rd_buf_sz > 0) {
if (rd_buf_sz > sizeof(rd_stack_buf)) {
rd_buf = malloc(rd_buf_sz);
if (!rd_buf) {
cf_error("malloc fail: trying %zu",rd_buf_sz);
rv = -1;
goto Error;
}
}
rv = cf_socket_read_timeout(workitem->fd, rd_buf, rd_buf_sz, workitem->deadline, progress_timeout_ms);
if (rv) {
//We already read some part of the message before but failed to read the
//remaining data for whatever reason (network error or timeout). We cannot
//reread as we already read partial data. Declare this as error.
cf_error("Timeout after reading the header but before reading the body");
goto Error;
}
#ifdef DEBUG_VERBOSE
dump_buf("read body from cluster", rd_buf, rd_buf_sz);
#endif
}
rv = CITRUSLEAF_OK;
goto Ok;
Retry:
//We are trying to postpone the reading
if (workitem->deadline && workitem->deadline < cf_getms()) {
cf_error("async receiver: out of time : deadline %"PRIu64" now %"PRIu64,
workitem->deadline, cf_getms());
//cf_error("async receiver: Workitem missed the final deadline");
rv = CITRUSLEAF_FAIL_TIMEOUT;
goto Error;
} else {
//We have time. Push the element back to the queue to be considered later
cf_queue_push(q_to_use, &workitem);
}
//If we allocated memory in this loop, release it.
if (rd_buf && (rd_buf != rd_stack_buf)) {
free(rd_buf);
}
cf_atomic_int_incr(&g_async_stats.retries);
continue;
Error:
if (network_error == true) {
/*
* In case of Async work (for XDS), it may be extreme to
* dun a node in case of network error. We just cleanup
* things and retry to connect to the remote cluster.
* The network error may be a transient one.
*/
}
#if ONEASYNCFD
//Do not close FD
#else
//We do not know the state of FD. It may have pending data to be read.
//We cannot reuse the FD. So, close it to be on safe side.
cf_error("async receiver: Closing the fd %d because of error", workitem->fd);
cf_close(workitem->fd);
workitem->fd = -1;
#endif
cf_atomic_int_incr(&g_async_stats.dropouts);
//Continue down with what we do during an Ok
//Inform the caller that there is no response from the server for this workitem.
//No response does not mean that the work is not done. The work might be
//successfully completed on the server side, we just didnt get response for it.
if (g_fail_cb_fn) {
g_fail_cb_fn(workitem->udata, rv, workitem->starttime);
}
Ok:
//rd_buf may not be there during an error condition.
if (rd_buf && (rv == CITRUSLEAF_OK)) {
//As of now, async functionality is there only for put call.
//In put call, we do not get anything back other than the trid field.
//So, just pass variable to get back the trid and ignore others.
if (0 != cl_parse(&msg.m, rd_buf, rd_buf_sz, NULL, NULL, NULL, &acktrid, NULL)) {
rv = CITRUSLEAF_FAIL_UNKNOWN;
}
else {
rv = msg.m.result_code;
if (workitem->trid != acktrid) {
#if ONEASYNCFD
//It is likely that we may get response for a different trid.
//Just delete the correct one from the queue
//put back the current workitem back in the queue.
shash_get(g_cl_async_hashtab, &acktrid, &tmpworkitem);
cf_queue_delete(q_to_use, &tmpworkitem, true);
cf_queue_push(q_to_use, &workitem);
//From now on workitem will be the one for which we got ack
workitem = tmpworkitem;
#endif
#ifdef DEBUG
cf_debug("Got reply for a different trid. Expected=%"PRIu64" Got=%"PRIu64" FD=%d",
workitem->trid, acktrid, workitem->fd);
#endif
}
}
if (g_success_cb_fn) {
g_success_cb_fn(workitem->udata, rv, workitem->starttime);
}
}
//Remember to put back the FD into the pool, if it is re-usable.
if (workitem->fd != -1) {
cl_cluster_node_fd_put(workitem->node, workitem->fd, true);
}
//Also decrement the reference count for this node
cl_cluster_node_put(workitem->node);
#if ONEASYNCFD
//Delete the item from the global hashtable
if (shash_delete(g_cl_async_hashtab, &workitem->trid) != SHASH_OK)
{
#if DEBUG
cf_debug("Failure while trying to delete trid=%"PRIu64" from hashtable", workitem->trid);
#endif
}
#endif
//Push it back into the free pool. If the attempt fails, free it.
if (cf_queue_push(g_cl_workitems_freepool_q, &workitem) == -1) {
free(workitem);
}
//If we allocated memory in this loop, release it.
if (rd_buf && (rd_buf != rd_stack_buf)) {
free(rd_buf);
}
// Kick this thread out if its ID is greater than total
if (thread_id > cf_atomic32_get(g_async_num_threads)) {
cf_atomic32_decr(&g_thread_count);
return NULL;
}
}//The infnite loop
return NULL;
}
