/*---------------------------------------------------------------------------*/ static hg_return_t hg_bulk_complete(struct hg_bulk_op_id *hg_bulk_op_id) { hg_context_t *context = hg_bulk_op_id->context; hg_return_t ret = HG_SUCCESS; /* Mark operation as completed */ hg_atomic_incr32(&hg_bulk_op_id->completed); if (hg_bulk_op_id->hg_bulk_origin->eager_mode) { /* In the case of eager bulk transfer, directly trigger the operation * to avoid potential deadlocks */ ret = hg_bulk_trigger_entry(hg_bulk_op_id); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not trigger completion entry"); goto done; } } else { struct hg_completion_entry *hg_completion_entry = &hg_bulk_op_id->hg_completion_entry; hg_completion_entry->op_type = HG_BULK; hg_completion_entry->op_id.hg_bulk_op_id = hg_bulk_op_id; ret = hg_core_completion_add(context, hg_completion_entry, hg_bulk_op_id->is_self); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not add HG completion entry to completion queue"); goto done; } } done: return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_proc_set_buf_ptr(hg_proc_t proc, void *buf_ptr) { struct hg_proc *hg_proc = (struct hg_proc *) proc; ptrdiff_t new_pos, lim_pos; hg_return_t ret = HG_SUCCESS; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); ret = HG_INVALID_PARAM; goto done; } /* Work out new position */ new_pos = (char *) buf_ptr - (char *) hg_proc->current_buf->buf; lim_pos = (ptrdiff_t) hg_proc->current_buf->size; if (new_pos > lim_pos) { HG_LOG_ERROR("Out of memory"); ret = HG_SIZE_ERROR; return ret; } hg_proc->current_buf->buf_ptr = buf_ptr; hg_proc->current_buf->size_left = hg_proc->current_buf->size - (hg_size_t) new_pos; #ifdef HG_HAS_XDR xdr_setpos(&hg_proc->current_buf->xdr, new_pos); #endif done: return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t HG_Bulk_cancel(hg_op_id_t op_id) { struct hg_bulk_op_id *hg_bulk_op_id = (struct hg_bulk_op_id *) op_id; hg_return_t ret = HG_SUCCESS; if (!hg_bulk_op_id) { HG_LOG_ERROR("NULL HG bulk operation ID"); ret = HG_INVALID_PARAM; goto done; } if (HG_UTIL_TRUE != hg_atomic_cas32(&hg_bulk_op_id->completed, 1, 0)) { unsigned int i = 0; /* Cancel all NA operations issued */ for (i = 0; i < hg_bulk_op_id->op_count; i++) { na_return_t na_ret; /* Cancel NA operation */ na_ret = NA_Cancel(HG_Core_class_get_na(hg_bulk_op_id->hg_class), HG_Core_context_get_na(hg_bulk_op_id->context), hg_bulk_op_id->na_op_ids[i]); if (na_ret != NA_SUCCESS) { HG_LOG_ERROR("Could not cancel op id"); ret = HG_NA_ERROR; goto done; } } } done: return ret; }
/*---------------------------------------------------------------------------*/ static hg_return_t hg_test_drc_token_acquire(struct hg_test_info *hg_test_info) { hg_return_t ret = HG_SUCCESS; #ifndef HG_TEST_DRC_IGNORE int rc; #endif /* Acquire credential */ #ifndef HG_TEST_DRC_IGNORE if (!hg_test_info->credential) { drc_acquire_again: rc = drc_acquire(&hg_test_info->credential, 0); if (rc != DRC_SUCCESS) { /* failed to acquire credential */ if (rc == -DRC_EINVAL) { sleep(1); goto drc_acquire_again; } HG_LOG_ERROR("drc_acquire() failed (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } } #else hg_test_info->credential = 12345; #endif printf("# Acquired credential %u\n", hg_test_info->credential); fflush(stdout); /* Access credential */ #ifndef HG_TEST_DRC_IGNORE drc_access_again: rc = drc_access(hg_test_info->credential, 0, &hg_test_info->credential_info); if (rc != DRC_SUCCESS) { /* failed to access credential */ if (rc == -DRC_EINVAL) { sleep(1); goto drc_access_again; } HG_LOG_ERROR("drc_access() failed (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } #endif /* Set cookie for further use */ #ifndef HG_TEST_DRC_IGNORE hg_test_info->cookie = drc_get_first_cookie(hg_test_info->credential_info); #else hg_test_info->cookie = 123456789; #endif printf("# Cookie is %u\n", hg_test_info->cookie); fflush(stdout); #ifndef HG_TEST_DRC_IGNORE done: #endif return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t HG_Bulk_create(hg_class_t *hg_class, hg_uint32_t count, void **buf_ptrs, const hg_size_t *buf_sizes, hg_uint8_t flags, hg_bulk_t *handle) { struct hg_bulk *hg_bulk = NULL; hg_return_t ret = HG_SUCCESS; if (!hg_class) { HG_LOG_ERROR("NULL HG class"); ret = HG_INVALID_PARAM; goto done; } if (!count) { HG_LOG_ERROR("Invalid number of segments"); ret = HG_INVALID_PARAM; goto done; } if (!buf_sizes) { HG_LOG_ERROR("NULL segment pointer"); ret = HG_INVALID_PARAM; goto done; } switch (flags) { case HG_BULK_READWRITE: break; case HG_BULK_READ_ONLY: break; case HG_BULK_WRITE_ONLY: break; default: HG_LOG_ERROR("Unrecognized handle flag"); ret = HG_INVALID_PARAM; goto done; } ret = hg_bulk_create(hg_class, count, buf_ptrs, buf_sizes, flags, &hg_bulk); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not create bulk handle"); goto done; } *handle = (hg_bulk_t) hg_bulk; done: if (ret != HG_SUCCESS) { hg_bulk_free(hg_bulk); } return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_test_drc_release(struct hg_test_info *hg_test_info) { hg_return_t ret = HG_SUCCESS; #ifndef HG_TEST_DRC_IGNORE int rc; #endif /* Release the reference to the credential */ #ifndef HG_TEST_DRC_IGNORE if (hg_test_info->credential_info) { rc = drc_release_local(&hg_test_info->credential_info); if (rc != DRC_SUCCESS) { /* failed to release credential info */ HG_LOG_ERROR("Could not release credential info (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } free((void *) hg_test_info->credential_info); } if (hg_test_info->wlm_id && hg_test_info->credential) { rc = drc_revoke(hg_test_info->credential, hg_test_info->wlm_id, DRC_FLAGS_TARGET_WLM); if (rc != DRC_SUCCESS) { /* failed to release credential info */ HG_LOG_ERROR("Could not revoke access for %d (%d, %s)", hg_test_info->wlm_id, rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } } if (hg_test_info->credential) { printf("# Releasing credential %u\n", hg_test_info->credential); rc = drc_release(hg_test_info->credential, 0); if (rc != DRC_SUCCESS) { /* failed to release credential */ HG_LOG_ERROR("Could not release credential (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } } done: #else (void) hg_test_info; #endif return ret; }
/*---------------------------------------------------------------------------*/ void * hg_proc_buf_alloc(hg_size_t size) { hg_size_t alignment; void *mem_ptr = NULL; #ifdef _WIN32 SYSTEM_INFO system_info; GetSystemInfo (&system_info); alignment = system_info.dwPageSize; mem_ptr = _aligned_malloc(size, alignment); #else alignment = sysconf(_SC_PAGE_SIZE); if (posix_memalign(&mem_ptr, alignment, size) != 0) { HG_LOG_ERROR("posix_memalign failed"); return NULL; } #endif if (mem_ptr) { memset(mem_ptr, 0, size); } return mem_ptr; }
/*---------------------------------------------------------------------------*/ static int hg_bulk_transfer_cb(const struct na_cb_info *callback_info) { struct hg_bulk_op_id *hg_bulk_op_id = (struct hg_bulk_op_id *) callback_info->arg; na_return_t na_ret = NA_SUCCESS; int ret = 0; if (callback_info->ret == NA_CANCELED) { /* If canceled, mark handle as canceled */ hg_atomic_cas32(&hg_bulk_op_id->canceled, 0, 1); } else if (callback_info->ret != NA_SUCCESS) { HG_LOG_ERROR("Error in NA callback: %s", NA_Error_to_string(callback_info->ret)); na_ret = NA_PROTOCOL_ERROR; goto done; } /* When all NA transfers that correspond to bulk operation complete * add HG user callback to completion queue */ if ((unsigned int) hg_atomic_incr32(&hg_bulk_op_id->op_completed_count) == hg_bulk_op_id->op_count) { hg_bulk_complete(hg_bulk_op_id); ret++; } done: (void) na_ret; return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_proc_free(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_return_t ret = HG_SUCCESS; if (!hg_proc) goto done; #ifdef HG_HAS_CHECKSUMS if (hg_proc->proc_buf.checksum != MCHECKSUM_OBJECT_NULL) { int checksum_ret; checksum_ret = mchecksum_destroy(hg_proc->proc_buf.checksum); if (checksum_ret != MCHECKSUM_SUCCESS) { HG_LOG_ERROR("Could not destroy checksum"); ret = HG_CHECKSUM_ERROR; } } #endif /* Free extra proc buffer if needed */ if (hg_proc->extra_buf.buf && hg_proc->extra_buf.is_mine) { free (hg_proc->extra_buf.buf); hg_proc->extra_buf.buf = NULL; } /* Free proc */ free(hg_proc); hg_proc = NULL; done: return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t HG_Bulk_access(hg_bulk_t handle, hg_size_t offset, hg_size_t size, hg_uint8_t flags, hg_uint32_t max_count, void **buf_ptrs, hg_size_t *buf_sizes, hg_uint32_t *actual_count) { struct hg_bulk *hg_bulk = (struct hg_bulk *) handle; hg_uint32_t count = 0; hg_return_t ret = HG_SUCCESS; if (!hg_bulk) { HG_LOG_ERROR("NULL memory handle passed"); ret = HG_INVALID_PARAM; goto done; } if (!size || !max_count) goto done; hg_bulk_access(hg_bulk, offset, size, flags, max_count, buf_ptrs, buf_sizes, &count); done: if (ret == HG_SUCCESS) { if (actual_count) *actual_count = count; } return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_proc_memcpy(hg_proc_t proc, void *data, hg_size_t data_size) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_return_t ret = HG_SUCCESS; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); ret = HG_INVALID_PARAM; goto done; } if (hg_proc->op == HG_FREE) goto done; /* If not enough space allocate extra space if encoding or * just get extra buffer if decoding */ if (hg_proc->current_buf->size_left < data_size) { hg_proc_set_size(proc, hg_proc->proc_buf.size + hg_proc->extra_buf.size + data_size); } /* Process data */ hg_proc->current_buf->buf_ptr = hg_proc_buf_memcpy(hg_proc->current_buf->buf_ptr, data, data_size, hg_proc->op); hg_proc->current_buf->size_left -= data_size; #ifdef HG_HAS_CHECKSUMS /* Update checksum */ if (hg_proc->current_buf->update_checksum) { int checksum_ret; checksum_ret = mchecksum_update(hg_proc->current_buf->checksum, data, data_size); if (checksum_ret != MCHECKSUM_SUCCESS) { HG_LOG_ERROR("Could not update checksum"); ret = HG_CHECKSUM_ERROR; goto done; } } #endif done: return ret; }
/*---------------------------------------------------------------------------*/ static hg_return_t hg_test_bulk_transfer_cb(const struct hg_cb_info *hg_cb_info) { struct hg_test_bulk_args *bulk_args = (struct hg_test_bulk_args *) hg_cb_info->arg; hg_bulk_t local_bulk_handle = hg_cb_info->info.bulk.local_handle; hg_return_t ret = HG_SUCCESS; bulk_write_out_t out_struct; void *buf; size_t write_ret; if (hg_cb_info->ret == HG_CANCELED) { printf("HG_Bulk_transfer() was successfully canceled\n"); /* Fill output structure */ out_struct.ret = 0; } else if (hg_cb_info->ret != HG_SUCCESS) { HG_LOG_ERROR("Error in callback"); ret = HG_PROTOCOL_ERROR; goto done; } if (hg_cb_info->ret == HG_SUCCESS) { /* Call bulk_write */ HG_Bulk_access(local_bulk_handle, 0, bulk_args->nbytes, HG_BULK_READWRITE, 1, &buf, NULL, NULL); write_ret = bulk_write(bulk_args->fildes, buf, 0, bulk_args->nbytes, 1); /* Fill output structure */ out_struct.ret = write_ret; } /* Free block handle */ ret = HG_Bulk_free(local_bulk_handle); if (ret != HG_SUCCESS) { fprintf(stderr, "Could not free HG bulk handle\n"); return ret; } /* Send response back */ ret = HG_Respond(bulk_args->handle, NULL, NULL, &out_struct); if (ret != HG_SUCCESS) { fprintf(stderr, "Could not respond\n"); return ret; } done: HG_Destroy(bulk_args->handle); free(bulk_args); return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_bulk_trigger_entry(struct hg_bulk_op_id *hg_bulk_op_id) { hg_return_t ret = HG_SUCCESS; /* Execute callback */ if (hg_bulk_op_id->callback) { struct hg_cb_info hg_cb_info; hg_cb_info.arg = hg_bulk_op_id->arg; hg_cb_info.ret = hg_atomic_get32(&hg_bulk_op_id->canceled) ? HG_CANCELED : HG_SUCCESS; hg_cb_info.type = HG_CB_BULK; hg_cb_info.info.bulk.op = hg_bulk_op_id->op; hg_cb_info.info.bulk.origin_handle = (hg_bulk_t) hg_bulk_op_id->hg_bulk_origin; hg_cb_info.info.bulk.local_handle = (hg_bulk_t) hg_bulk_op_id->hg_bulk_local; hg_bulk_op_id->callback(&hg_cb_info); } /* Decrement ref_count */ ret = hg_bulk_free(hg_bulk_op_id->hg_bulk_origin); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not free bulk handle"); goto done; } ret = hg_bulk_free(hg_bulk_op_id->hg_bulk_local); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not free bulk handle"); goto done; } /* Free op */ free(hg_bulk_op_id->na_op_ids); free(hg_bulk_op_id); done: return ret; }
/*---------------------------------------------------------------------------*/ static HG_INLINE hg_return_t hg_proc_hg_test_drc_grant_out_t(hg_proc_t proc, void *data) { hg_test_drc_grant_out_t *struct_data = (hg_test_drc_grant_out_t *) data; hg_return_t ret = HG_SUCCESS; #ifdef HG_TEST_DRC_USE_TOKEN ret = hg_proc_hg_string_t(proc, &struct_data->token); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Proc error"); goto done; } #else ret = hg_proc_hg_uint32_t(proc, &struct_data->credential); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Proc error"); goto done; } #endif done: return ret; }
/*---------------------------------------------------------------------------*/ static HG_INLINE hg_return_t hg_proc_hg_test_drc_grant_in_t(hg_proc_t proc, void *data) { hg_test_drc_grant_in_t *struct_data = (hg_test_drc_grant_in_t *) data; hg_return_t ret = HG_SUCCESS; ret = hg_proc_hg_uint32_t(proc, &struct_data->wlm_id); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Proc error"); goto done; } done: return ret; }
/*---------------------------------------------------------------------------*/ static hg_return_t hg_test_drc_token_request_cb(const struct hg_cb_info *callback_info) { hg_request_t *request = (hg_request_t *) callback_info->arg; hg_return_t ret = HG_SUCCESS; if (callback_info->ret != HG_SUCCESS) { HG_LOG_ERROR("Return from callback info is not HG_SUCCESS"); goto done; } done: hg_request_complete(request); return ret; }
/*---------------------------------------------------------------------------*/ void * hg_proc_get_buf_ptr(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; void *ptr = NULL; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } ptr = hg_proc->current_buf->buf_ptr; done: return ptr; }
/*---------------------------------------------------------------------------*/ hg_size_t hg_proc_get_size_left(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_size_t size = 0; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } size = hg_proc->current_buf->size_left; done: return size; }
/*---------------------------------------------------------------------------*/ hg_size_t HG_Bulk_get_size(hg_bulk_t handle) { hg_size_t ret = 0; struct hg_bulk *hg_bulk = (struct hg_bulk *) handle; if (!hg_bulk) { HG_LOG_ERROR("NULL bulk handle"); goto done; } ret = hg_bulk->total_size; done: return ret; }
/*---------------------------------------------------------------------------*/ hg_size_t hg_proc_get_size(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_size_t size = 0; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } size = hg_proc->proc_buf.size + hg_proc->extra_buf.size; done: return size; }
/*---------------------------------------------------------------------------*/ hg_proc_op_t hg_proc_get_op(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_proc_op_t proc_op = HG_ENCODE; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } proc_op = hg_proc->op; done: return proc_op; }
/*---------------------------------------------------------------------------*/ hg_class_t * hg_proc_get_class(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_class_t *hg_class = NULL; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } hg_class = hg_proc->hg_class; done: return hg_class; }
/*---------------------------------------------------------------------------*/ hg_uint32_t HG_Bulk_get_segment_count(hg_bulk_t handle) { struct hg_bulk *hg_bulk = (struct hg_bulk *) handle; hg_uint32_t ret = 0; if (!hg_bulk) { HG_LOG_ERROR("NULL bulk handle"); goto done; } ret = hg_bulk->segment_count; done: return ret; }
/** * Deserialize memcpy */ static HG_INLINE hg_return_t hg_bulk_deserialize_memcpy(const char **src, ssize_t *src_left, void *dest, size_t n) { hg_return_t ret = HG_SUCCESS; if ((*src_left -= (ssize_t) n) < 0) { HG_LOG_ERROR("Buffer size too small"); ret = HG_SIZE_ERROR; goto done; } memcpy(dest, *src, n); *src += n; done: return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t HG_Bulk_ref_incr(hg_bulk_t handle) { struct hg_bulk *hg_bulk = (struct hg_bulk *) handle; hg_return_t ret = HG_SUCCESS; if (!hg_bulk) { HG_LOG_ERROR("NULL memory handle passed"); ret = HG_INVALID_PARAM; goto done; } /* Increment ref count */ hg_atomic_incr32(&hg_bulk->ref_count); done: return ret; }
/*---------------------------------------------------------------------------*/ hg_size_t hg_proc_get_size_used(hg_proc_t proc) { struct hg_proc *hg_proc = (struct hg_proc *) proc; hg_size_t size = 0; if (!hg_proc) { HG_LOG_ERROR("Proc is not initialized"); goto done; } if(hg_proc->extra_buf.size > 0) size = (hg_proc->proc_buf.size + hg_proc->extra_buf.size) - hg_proc->extra_buf.size_left; else size = hg_proc->proc_buf.size - hg_proc->proc_buf.size_left; done: return size; }
/*---------------------------------------------------------------------------*/ hg_size_t HG_Bulk_get_serialize_size(hg_bulk_t handle, hg_bool_t request_eager) { struct hg_bulk *hg_bulk = (struct hg_bulk *) handle; hg_size_t ret = 0; hg_uint32_t i; if (!hg_bulk) { HG_LOG_ERROR("NULL bulk handle"); goto done; } /* Permission flags */ ret = sizeof(hg_bulk->flags); /* Segments */ ret += sizeof(hg_bulk->total_size) + sizeof(hg_bulk->segment_count) + hg_bulk->segment_count * sizeof(*hg_bulk->segments); /* NA mem handles */ ret += sizeof(hg_bulk->na_mem_handle_count); for (i = 0; i < hg_bulk->na_mem_handle_count; i++) { na_size_t serialize_size = 0; if (hg_bulk->na_mem_handles[i]) { serialize_size = NA_Mem_handle_get_serialize_size( HG_Core_class_get_na(hg_bulk->hg_class), hg_bulk->na_mem_handles[i]); } ret += sizeof(serialize_size) + serialize_size; } /* Eager mode */ ret += sizeof(hg_bulk->eager_mode); if (request_eager && (hg_bulk->flags == HG_BULK_READ_ONLY)) ret += hg_bulk->total_size; done: return ret; }
/*---------------------------------------------------------------------------*/ hg_return_t hg_test_drc_acquire(int argc, char *argv[], struct hg_test_info *hg_test_info) { struct hg_test_info hg_test_drc_info = { 0 }; struct hg_init_info hg_test_drc_init_info = { 0 }; hg_return_t ret = HG_SUCCESS; if (!hg_test_info->credential) { /* Create an NA class with "tcp" protocol */ hg_test_drc_info.na_test_info.extern_init = NA_TRUE; hg_test_drc_info.na_test_info.protocol = strdup("tcp"); hg_test_drc_info.na_test_info.listen = hg_test_info->na_test_info.listen; if (NA_Test_init(argc, argv, &hg_test_drc_info.na_test_info) != NA_SUCCESS) { HG_LOG_ERROR("Could not initialize NA test layer"); ret = HG_NA_ERROR; goto done; } /* Assign NA class */ hg_test_drc_init_info.na_class = hg_test_drc_info.na_test_info.na_class; /* Init HG HL with init options */ ret = HG_Hl_init_opt(NULL, hg_test_drc_info.na_test_info.listen, &hg_test_drc_init_info); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not initialize HG HL"); goto done; } hg_test_drc_info.hg_class = HG_CLASS_DEFAULT; hg_test_drc_info.context = HG_CONTEXT_DEFAULT; hg_test_drc_info.request_class = HG_REQUEST_CLASS_DEFAULT; /* Attach test info to class */ HG_Class_set_data(hg_test_drc_info.hg_class, &hg_test_drc_info, NULL); /* Register routines */ hg_test_drc_register(hg_test_drc_info.hg_class); /* Acquire DRC token */ if (hg_test_drc_info.na_test_info.listen) { char addr_string[NA_TEST_MAX_ADDR_NAME]; na_size_t addr_string_len = NA_TEST_MAX_ADDR_NAME; hg_addr_t self_addr; ret = hg_test_drc_token_acquire(&hg_test_drc_info); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not acquire DRC token"); goto done; } /* TODO only rank 0 */ ret = HG_Addr_self(hg_test_drc_info.hg_class, &self_addr); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not get self addr"); goto done; } ret = HG_Addr_to_string(hg_test_drc_info.hg_class, addr_string, &addr_string_len, self_addr); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not convert addr to string"); goto done; } HG_Addr_free(hg_test_drc_info.hg_class, self_addr); na_test_set_config(addr_string); /* Used by CTest Test Driver to know when to launch clients */ MERCURY_TESTING_READY_MSG(); /* Progress */ do { unsigned int total_count = 0; unsigned int actual_count = 0; do { ret = HG_Trigger(hg_test_drc_info.context, 0, 1, &actual_count); total_count += actual_count; } while ((ret == HG_SUCCESS) && actual_count); /* Break as soon as something was triggered */ if (total_count) break; ret = HG_Progress(hg_test_drc_info.context, HG_MAX_IDLE_TIME); } while (ret == HG_SUCCESS || ret == HG_TIMEOUT); } else { char test_addr_name[NA_TEST_MAX_ADDR_NAME] = { '\0' }; if (hg_test_drc_info.na_test_info.mpi_comm_rank == 0) na_test_get_config(test_addr_name, NA_TEST_MAX_ADDR_NAME); /* Broadcast addr name */ NA_Test_bcast(test_addr_name, NA_TEST_MAX_ADDR_NAME, 0, &hg_test_drc_info.na_test_info); hg_test_drc_info.na_test_info.target_name = strdup(test_addr_name); printf("# Target name read: %s\n", hg_test_drc_info.na_test_info.target_name); ret = hg_test_drc_token_request(&hg_test_drc_info); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not request DRC token"); goto done; } } #ifdef MERCURY_HAS_PARALLEL_TESTING /* TODO bcast cookie when parallel mode */ #endif /* Finalize HG HL interface */ ret = HG_Hl_finalize(); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not finalize HG HL"); goto done; } /* Finalize NA test class interface */ #ifdef MERCURY_HAS_PARALLEL_TESTING hg_test_drc_info.na_test_info.mpi_no_finalize = NA_TRUE; #endif if (NA_Test_finalize(&hg_test_drc_info.na_test_info) != NA_SUCCESS) { HG_LOG_ERROR("Could not finalize NA test interface"); ret = HG_NA_ERROR; goto done; } hg_test_info->credential = hg_test_drc_info.credential; } else { hg_test_drc_info.credential = hg_test_info->credential; ret = hg_test_drc_token_acquire(&hg_test_drc_info); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not acquire DRC token"); goto done; } } /* Copy cookie/credential info */ hg_test_info->wlm_id = hg_test_drc_info.wlm_id; hg_test_info->credential_info = hg_test_drc_info.credential_info; hg_test_info->cookie = hg_test_drc_info.cookie; /* Sleep a few seconds to make sure listener is initialized */ if (!hg_test_drc_info.na_test_info.listen) { unsigned int sleep_sec = 5; printf("# Sleeping now for %d seconds...\n", sleep_sec); fflush(stdout); sleep(sleep_sec); } done: return ret; }
/*---------------------------------------------------------------------------*/ static hg_return_t hg_test_drc_token_request(struct hg_test_info *hg_test_info) { hg_request_t *request = NULL; hg_handle_t handle; #ifdef HG_TEST_DRC_USE_TOKEN hg_string_t token; #else hg_uint32_t credential; #endif hg_test_drc_grant_in_t in_struct; hg_test_drc_grant_out_t out_struct; hg_return_t ret = HG_SUCCESS; #ifndef HG_TEST_DRC_IGNORE int rc; #endif /* Look up target addr using target name info */ ret = HG_Hl_addr_lookup_wait(hg_test_info->context, hg_test_info->request_class, hg_test_info->na_test_info.target_name, &hg_test_info->target_addr, HG_MAX_IDLE_TIME); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not find addr for target %s", hg_test_info->na_test_info.target_name); goto done; } /* Create new request */ request = hg_request_create(hg_test_info->request_class); /* Create request with invalid RPC id */ ret = HG_Create(hg_test_info->context, hg_test_info->target_addr, hg_test_drc_grant_id_g, &handle); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not create handle"); goto done; } /* Get WLM ID and set input */ #ifndef HG_TEST_DRC_IGNORE in_struct.wlm_id = drc_get_wlm_id(); #else in_struct.wlm_id = 12340; #endif /* Forward call to target addr */ printf("# %u requesting access to remote...\n", in_struct.wlm_id); fflush(stdout); ret = HG_Forward(handle, hg_test_drc_token_request_cb, request, &in_struct); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not forward call with id=%d", hg_test_drc_grant_id_g); goto done; } /* Wait for completion */ hg_request_wait(request, HG_MAX_IDLE_TIME, NULL); /* Get output */ ret = HG_Get_output(handle, &out_struct); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not get output"); goto done; } #ifdef HG_TEST_DRC_USE_TOKEN /* Get token back */ token = out_struct.token; printf("# Received token %s\n", token); fflush(stdout); /* Translate token */ #ifndef HG_TEST_DRC_IGNORE rc = drc_access_with_token(token, 0, &hg_test_info->credential_info); if (rc != DRC_SUCCESS) {/* failed to grant access to the credential */ HG_LOG_ERROR("drc_access_with_token() failed (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } #endif #else /* Get credential back */ credential = out_struct.credential; printf("# Received credential %u\n", credential); fflush(stdout); /* Access credential */ #ifndef HG_TEST_DRC_IGNORE drc_access_again: rc = drc_access(credential, 0, &hg_test_info->credential_info); if (rc != DRC_SUCCESS) { /* failed to access credential */ if (rc == -DRC_EINVAL) { sleep(1); goto drc_access_again; } HG_LOG_ERROR("drc_access() failed (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } #endif #endif /* Set cookie for further use */ #ifndef HG_TEST_DRC_IGNORE hg_test_info->cookie = drc_get_first_cookie(hg_test_info->credential_info); #else hg_test_info->cookie = 123456789; #endif printf("# Cookie is %u\n", hg_test_info->cookie); fflush(stdout); /* Clean up resources */ ret = HG_Free_output(handle, &out_struct); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not free output"); goto done; } ret = HG_Destroy(handle); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not destroy handle"); goto done; } hg_request_destroy(request); /* Free target addr */ ret = HG_Addr_free(hg_test_info->hg_class, hg_test_info->target_addr); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not free addr"); goto done; } done: return ret; }
/*---------------------------------------------------------------------------*/ static hg_return_t hg_test_drc_grant_cb(hg_handle_t handle) { const struct hg_info *hg_info = NULL; struct hg_test_info *hg_test_info = NULL; hg_test_drc_grant_in_t in_struct; hg_test_drc_grant_out_t out_struct; hg_return_t ret = HG_SUCCESS; #ifdef HG_TEST_DRC_USE_TOKEN hg_string_t token; #endif #ifndef HG_TEST_DRC_IGNORE int rc; #endif /* Get info from handle */ hg_info = HG_Get_info(handle); /* Get test info */ hg_test_info = (struct hg_test_info *) HG_Class_get_data(hg_info->hg_class); /* Get input buffer */ ret = HG_Get_input(handle, &in_struct); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not get input"); goto done; } /* Get parameters */ hg_test_info->wlm_id = in_struct.wlm_id; /* Grant access to another job */ printf("# Granting access to wlm_id %u...\n", hg_test_info->wlm_id); fflush(stdout); #ifndef HG_TEST_DRC_IGNORE drc_grant_again: rc = drc_grant(hg_test_info->credential, hg_test_info->wlm_id, DRC_FLAGS_TARGET_WLM); if (rc != DRC_SUCCESS && rc != -DRC_ALREADY_GRANTED) { if (rc == -DRC_EINVAL) { sleep(1); goto drc_grant_again; } HG_LOG_ERROR("drc_grant() to %d failed (%d, %s)", hg_test_info->wlm_id, rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } #endif #ifdef HG_TEST_DRC_USE_TOKEN /* Get the token to pass around to processes in other job */ #ifndef HG_TEST_DRC_IGNORE rc = drc_get_credential_token(hg_test_info->credential, &token); if (rc != DRC_SUCCESS) { HG_LOG_ERROR("drc_get_credential_token() failed (%d, %s)", rc, drc_strerror(-rc)); ret = HG_PROTOCOL_ERROR; goto done; } #else token = "my_test_token"; #endif /* Fill output structure */ printf("# Access granted, token is %s\n", token); fflush(stdout); out_struct.token = token; #else out_struct.credential = hg_test_info->credential; #endif /* Free handle and send response back */ ret = HG_Respond(handle, NULL, NULL, &out_struct); if (ret != HG_SUCCESS) { HG_LOG_ERROR("Could not respond"); goto done; } HG_Free_input(handle, &in_struct); HG_Destroy(handle); done: return ret; }