int main(){
// thread that listens for the incoming connections
backend_thread();
printf("server thread says you bye bye!\n");
return 0;
}
int main(){
// thread that listens for the incoming connections
//pthread_t thread;
//pthread_create(&thread, NULL, &backend_thread, NULL);
//pthread_join(thread, NULL);
backend_thread();
printf("server thread says you bye bye!\n");
return 0;
}
esdm_status esdm_scheduler_enqueue_write(esdm_instance_t *esdm, io_request_status_t * status, esdm_dataset_t *dataset, void *buf, esdm_dataspace_t* space){
GError * error;
//Gather I/O recommendations
//esdm_performance_recommendation(esdm, NULL, NULL); // e.g., split, merge, replication?
//esdm_layout_recommendation(esdm, NULL, NULL); // e.g., merge, split, transform?
// choose the dimension to split
int split_dim = 0;
for(int i=0; i < space->dimensions; i++){
if (space->size[i] != 1){
split_dim = i;
break;
}
}
// how big is one sub-hypercube? we call it y axis for the easier reading
uint64_t one_y_size = 1;
for (int i = 0; i < space->dimensions; i++)
{
if(i != split_dim){
one_y_size *= space->size[i];
}
}
one_y_size *= esdm_sizeof(space->datatype);
if (one_y_size == 0){
return ESDM_SUCCESS;
}
uint64_t y_count = space->size[split_dim];
uint64_t per_backend[esdm->modules->backend_count];
memset(per_backend, 0, sizeof(per_backend));
while(y_count > 0){
for (int i = 0; i < esdm->modules->backend_count; i++) {
status->pending_ops++;
esdm_backend* b = esdm->modules->backends[i];
// how many of these fit into our buffer
uint64_t backend_y_per_buffer = b->config->max_fragment_size / one_y_size;
if (backend_y_per_buffer == 0){
backend_y_per_buffer = 1;
}
if (backend_y_per_buffer >= y_count){
per_backend[i] += y_count;
y_count = 0;
break;
}else{
per_backend[i] += backend_y_per_buffer;
y_count -= backend_y_per_buffer;
}
}
}
ESDM_DEBUG_FMT("Will submit %d operations and for backend0: %d y-blocks", status->pending_ops, per_backend[0]);
uint64_t offset_y = 0;
int64_t dim[space->dimensions];
int64_t offset[space->dimensions];
memcpy(offset, space->offset, space->dimensions * sizeof(int64_t));
memcpy(dim, space->size, space->dimensions * sizeof(int64_t));
for (int i = 0; i < esdm->modules->backend_count; i++) {
esdm_backend* b = esdm->modules->backends[i];
// how many of these fit into our buffer
uint64_t backend_y_per_buffer = b->config->max_fragment_size / one_y_size;
if (backend_y_per_buffer == 0){
backend_y_per_buffer = 1;
}
uint64_t y_total_access = per_backend[i];
while(y_total_access > 0){
uint64_t y_to_access = y_total_access > backend_y_per_buffer ? backend_y_per_buffer : y_total_access ;
y_total_access -= y_to_access;
dim[split_dim] = y_to_access;
offset[split_dim] = offset_y + space->offset[split_dim];
io_work_t * task = (io_work_t*) malloc(sizeof(io_work_t));
esdm_dataspace_t* subspace = esdm_dataspace_subspace(space, space->dimensions, dim, offset);
task->parent = status;
task->op = ESDM_OP_WRITE;
task->fragment = esdm_fragment_create(dataset, subspace, (char*) buf + offset_y * one_y_size);
task->fragment->backend = b;
task->callback = NULL;
if (b->threads == 0){
backend_thread(task, b);
}else{
g_thread_pool_push(b->threadPool, task, & error);
}
offset_y += y_to_access;
}
}
// now enqueue the operations
return ESDM_SUCCESS;
}
esdm_status esdm_scheduler_enqueue_read(esdm_instance_t *esdm, io_request_status_t * status, int frag_count, esdm_fragment_t** read_frag, void * buf, esdm_dataspace_t * buf_space){
GError * error;
status->pending_ops += frag_count;
int i, x;
for (i = 0; i < frag_count; i++){
esdm_fragment_t * f = read_frag[i];
json_t *root = load_json(f->metadata->json);
json_t * elem;
elem = json_object_get(root, "plugin");
//const char * plugin_type = json_string_value(elem);
elem = json_object_get(root, "id");
const char * plugin_id = json_string_value(elem);
elem = json_object_get(root, "offset");
//const char * offset_str = json_string_value(elem);
elem = json_object_get(root, "size");
//const char * size_str = json_string_value(elem);
if(!plugin_id){
printf("Backend ID needs to be given\n");
exit(1);
}
// find the backend for the fragment
esdm_backend* backend_to_use = NULL;
for (x = 0; x < esdm->modules->backend_count; x++){
esdm_backend* b_tmp = esdm->modules->backends[x];
if (strcmp(b_tmp->config->id, plugin_id) == 0){
DEBUG("Found plugin %s", plugin_id);
backend_to_use = b_tmp;
break;
}
}
if(backend_to_use == NULL){
printf("Error no backend found for ID: %s\n", plugin_id);
exit(1);
}
// esdm_fragment_print(read_frag[i]);
// printf("\n");
DEBUG("OFFSET/SIZE: %s %s\n", offset_str, size_str);
//for verification purposes, we could read back the metadata stored and compare it...
//esdm_dataspace_t * space = NULL;
//esdm_status ret = esdm_dataspace_overlap_str(parent_space, 'x', (char*)offset_str, (char*)size_str, & space);
//assert(ret == ESDM_SUCCESS);
uint64_t size = esdm_dataspace_size(f->dataspace);
//printf("SIZE: %ld\n", size);
f->backend = backend_to_use;
io_work_t * task = (io_work_t*) malloc(sizeof(io_work_t));
task->parent = status;
task->op = ESDM_OP_READ;
task->fragment = f;
if(f->in_place){
DEBUG("inplace!", "");
task->callback = NULL;
f->buf = buf;
}else{
f->buf = malloc(size);
task->callback = read_copy_callback;
task->data.mem_buf = buf;
task->data.buf_space = buf_space;
}
if (backend_to_use->threads == 0){
backend_thread(task, backend_to_use);
}else{
g_thread_pool_push(backend_to_use->threadPool, task, & error);
}
}
return ESDM_SUCCESS;
}
